From 67b9060d43165f49c55471734f4f2622c35d1f72 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Fri, 25 Oct 2024 16:07:49 +0200 Subject: [PATCH 01/12] Use lstcam_calib instead of lstchain for the calibration --- extra/history_files/sequence_LST1_04183.history | 4 ++-- src/osa/configs/sequencer.cfg | 4 ++-- src/osa/scripts/calibration_pipeline.py | 6 ++++-- src/osa/scripts/tests/test_osa_scripts.py | 4 ++-- src/osa/tests/test_jobs.py | 2 +- src/osa/workflow/stages.py | 4 ++-- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/extra/history_files/sequence_LST1_04183.history b/extra/history_files/sequence_LST1_04183.history index 12d437c6..037d9cae 100644 --- a/extra/history_files/sequence_LST1_04183.history +++ b/extra/history_files/sequence_LST1_04183.history @@ -1,2 +1,2 @@ -04179 onsite_create_drs4_pedestal_file v0.7.0 2021-12-19 22:11 None None 0 -04183 onsite_create_calibration_file v0.7.0 2021-12-19 22:11 None None 0 \ No newline at end of file +04179 lstcam_calib_onsite_create_drs4_pedestal_file v0.7.0 2021-12-19 22:11 None None 0 +04183 lstcam_calib_onsite_create_calibration_file v0.7.0 2021-12-19 22:11 None None 0 \ No newline at end of file diff --git a/src/osa/configs/sequencer.cfg b/src/osa/configs/sequencer.cfg index 2b65a317..c11ad8ff 100644 --- a/src/osa/configs/sequencer.cfg +++ b/src/osa/configs/sequencer.cfg @@ -41,8 +41,8 @@ DL2_PROD_ID: model2 [lstchain] # Calibration steps in calibration pipeline script -drs4_baseline: onsite_create_drs4_pedestal_file -charge_calibration: onsite_create_calibration_file +drs4_baseline: lstcam_calib_onsite_create_drs4_pedestal_file +charge_calibration: lstcam_calib_onsite_create_calibration_file use_ff_heuristic_id: False # Data processing steps in datasequence script diff --git a/src/osa/scripts/calibration_pipeline.py b/src/osa/scripts/calibration_pipeline.py index 51d43080..cec62a2a 100644 --- a/src/osa/scripts/calibration_pipeline.py +++ b/src/osa/scripts/calibration_pipeline.py @@ -44,8 +44,9 @@ def is_calibration_produced(drs4_pedestal_run_id: int, pedcal_run_id: int) -> bo def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list: """Build the create_drs4_pedestal command.""" base_dir = Path(cfg.get("LST1", "BASE")).resolve() + command = cfg.get("lstchain", "drs4_baseline") return [ - "onsite_create_drs4_pedestal_file", + command, f"--run_number={drs4_pedestal_run_id}", f"--base_dir={base_dir}", "--no-progress", @@ -55,8 +56,9 @@ def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list: def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> list: """Build the create_calibration_file command.""" base_dir = Path(cfg.get("LST1", "BASE")).resolve() + command = cfg.get("lstchain", "charge_calibration") cmd = [ - "onsite_create_calibration_file", + command, f"--pedestal_run={drs4_pedestal_run_id}", f"--run_number={pedcal_run_id}", f"--base_dir={base_dir}", diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index ea27e3a6..a79d8cc1 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -300,7 +300,7 @@ def test_drs4_pedestal_cmd(base_test_dir): cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804") expected_command = [ - "onsite_create_drs4_pedestal_file", + "lstcam_calib_onsite_create_drs4_pedestal_file", "--run_number=01804", f"--base_dir={base_test_dir}", "--no-progress", @@ -313,7 +313,7 @@ def test_calibration_file_cmd(base_test_dir): cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809") expected_command = [ - "onsite_create_calibration_file", + "lstcam_calib_onsite_create_calibration_file", "--pedestal_run=01804", "--run_number=01809", f"--base_dir={base_test_dir}", diff --git a/src/osa/tests/test_jobs.py b/src/osa/tests/test_jobs.py index 53fa03d3..a93c50e4 100644 --- a/src/osa/tests/test_jobs.py +++ b/src/osa/tests/test_jobs.py @@ -395,7 +395,7 @@ def test_set_cache_dirs(): def test_calibration_history_level(): from osa.job import check_history_level - levels = {"onsite_create_drs4_pedestal_file": 1, "onsite_create_calibration_file": 0} + levels = {"lstcam_calib_onsite_create_drs4_pedestal_file": 1, "lstcam_calib_onsite_create_calibration_file": 0} level, exit_status = check_history_level(calibration_history_file, levels) assert level == 0 assert exit_status == 0 diff --git a/src/osa/workflow/stages.py b/src/osa/workflow/stages.py index d1d7935e..92b49b4c 100644 --- a/src/osa/workflow/stages.py +++ b/src/osa/workflow/stages.py @@ -81,9 +81,9 @@ def _clean_up(self): self._remove_dl1b_output('dl1_LST-1.Run') elif self.command == "lstchain_check_dl1": self._remove_dl1b_output('datacheck_dl1_LST-1.Run') - elif self.command == "onsite_create_calibration_file": + elif self.command == cfg.get("lstchain", "charge_calibration"): self._remove_calibration() - elif self.command == "onsite_create_drs4_pedestal_file": + elif self.command == cfg.get("lstchain", "drs4_baseline"): self._remove_drs4_baseline() def _remove_drs4_baseline(self): From 6f46f5e260da1ed88f2f4fec914a5f2c9399fdda Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Mon, 28 Oct 2024 11:01:44 +0100 Subject: [PATCH 02/12] get the scripts names from the cfg also for the tests --- src/osa/configs/sequencer.cfg | 2 ++ src/osa/scripts/tests/test_osa_scripts.py | 5 +++-- src/osa/tests/test_jobs.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/osa/configs/sequencer.cfg b/src/osa/configs/sequencer.cfg index c11ad8ff..4658995b 100644 --- a/src/osa/configs/sequencer.cfg +++ b/src/osa/configs/sequencer.cfg @@ -41,6 +41,8 @@ DL2_PROD_ID: model2 [lstchain] # Calibration steps in calibration pipeline script +#drs4_baseline: onsite_create_drs4_pedestal_file +#charge_calibration: onsite_create_calibration_file drs4_baseline: lstcam_calib_onsite_create_drs4_pedestal_file charge_calibration: lstcam_calib_onsite_create_calibration_file use_ff_heuristic_id: False diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index a79d8cc1..32449699 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -8,6 +8,7 @@ import yaml from osa.configs import options +from osa.configs.config import cfg from osa.scripts.closer import is_sequencer_successful, is_finished_check ALL_SCRIPTS = [ @@ -300,7 +301,7 @@ def test_drs4_pedestal_cmd(base_test_dir): cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804") expected_command = [ - "lstcam_calib_onsite_create_drs4_pedestal_file", + cfg.get("lstchain", "drs4_baseline"), "--run_number=01804", f"--base_dir={base_test_dir}", "--no-progress", @@ -313,7 +314,7 @@ def test_calibration_file_cmd(base_test_dir): cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809") expected_command = [ - "lstcam_calib_onsite_create_calibration_file", + cfg.get("lstchain", "charge_calibration"), "--pedestal_run=01804", "--run_number=01809", f"--base_dir={base_test_dir}", diff --git a/src/osa/tests/test_jobs.py b/src/osa/tests/test_jobs.py index a93c50e4..e24c6b3a 100644 --- a/src/osa/tests/test_jobs.py +++ b/src/osa/tests/test_jobs.py @@ -395,7 +395,7 @@ def test_set_cache_dirs(): def test_calibration_history_level(): from osa.job import check_history_level - levels = {"lstcam_calib_onsite_create_drs4_pedestal_file": 1, "lstcam_calib_onsite_create_calibration_file": 0} + levels = {cfg.get("lstchain", "drs4_baseline"): 1, cfg.get("lstchain", "charge_calibration"): 0} level, exit_status = check_history_level(calibration_history_file, levels) assert level == 0 assert exit_status == 0 From ec222ebe1f6006ffae6ac7bedbc7101cad2aed42 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Mon, 28 Oct 2024 16:01:45 +0100 Subject: [PATCH 03/12] add lstcam_calib to the environment --- environment.yml | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index fd675889..6ac716f3 100644 --- a/environment.yml +++ b/environment.yml @@ -27,6 +27,7 @@ dependencies: - pyirf~=0.10 - lstchain>=0.10.7 - tenacity + - lstcam_calib # dev dependencies - pytest - pytest-cov diff --git a/pyproject.toml b/pyproject.toml index 684a5bc8..688921a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "pymongo", "gammapy~=1.1", "tenacity", + "lstcam_calib", ] dynamic = ["version"] From 2821b53e1232023a754fe424aca95d5b9f66c8e7 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Mon, 28 Oct 2024 16:11:03 +0100 Subject: [PATCH 04/12] lstcam_calib installable with pip --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 6ac716f3..8938cc66 100644 --- a/environment.yml +++ b/environment.yml @@ -27,7 +27,8 @@ dependencies: - pyirf~=0.10 - lstchain>=0.10.7 - tenacity - - lstcam_calib + - pip: + - lstcam_calib # dev dependencies - pytest - pytest-cov From 0ae3b2cab777ee2ce46e80426af0cb28d665ce1b Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Mon, 28 Oct 2024 16:27:07 +0100 Subject: [PATCH 05/12] require python>=3.10 --- .github/workflows/ci.yml | 2 -- pyproject.toml | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 347b4b56..9d066a26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,6 @@ jobs: strategy: matrix: include: - - python-version: "3.9" - - python-version: "3.10" - python-version: "3.11" diff --git a/pyproject.toml b/pyproject.toml index 688921a4..38bc098f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,12 +26,11 @@ classifiers = [ "Topic :: Scientific/Engineering :: Astronomy", "Topic :: Scientific/Engineering :: Physics", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "astropy~=5.0", "lstchain>=0.10.7", From 2de1289ab9fa82cc40df9de0ade9336ab913521e Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Mon, 11 Nov 2024 17:26:20 +0100 Subject: [PATCH 06/12] require lstchain>=0.10.13 to use lstcam_calib --- environment.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 8938cc66..f35534d2 100644 --- a/environment.yml +++ b/environment.yml @@ -25,7 +25,7 @@ dependencies: - ctapipe_io_lst=0.22 - ctaplot~=0.6.4 - pyirf~=0.10 - - lstchain>=0.10.7 + - lstchain>=0.10.13 - tenacity - pip: - lstcam_calib diff --git a/pyproject.toml b/pyproject.toml index 38bc098f..aec43f7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ requires-python = ">=3.10" dependencies = [ "astropy~=5.0", - "lstchain>=0.10.7", + "lstchain>=0.10.13", "matplotlib", "numpy", "pandas", From 28bb52a62af3da1ca792efdc08968cacf2dff903 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Wed, 13 Nov 2024 15:29:08 +0100 Subject: [PATCH 07/12] adapt the arguments to the lstcam_calib scripts --- src/osa/scripts/calibration_pipeline.py | 10 +++++----- src/osa/scripts/tests/test_osa_scripts.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/osa/scripts/calibration_pipeline.py b/src/osa/scripts/calibration_pipeline.py index cec62a2a..7e3a8034 100644 --- a/src/osa/scripts/calibration_pipeline.py +++ b/src/osa/scripts/calibration_pipeline.py @@ -47,8 +47,8 @@ def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list: command = cfg.get("lstchain", "drs4_baseline") return [ command, - f"--run_number={drs4_pedestal_run_id}", - f"--base_dir={base_dir}", + f"--run-number={drs4_pedestal_run_id}", + f"--base-dir={base_dir}", "--no-progress", ] @@ -59,9 +59,9 @@ def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> l command = cfg.get("lstchain", "charge_calibration") cmd = [ command, - f"--pedestal_run={drs4_pedestal_run_id}", - f"--run_number={pedcal_run_id}", - f"--base_dir={base_dir}", + f"--pedestal-run={drs4_pedestal_run_id}", + f"--run-number={pedcal_run_id}", + f"--base-dir={base_dir}", ] # In case of problems with trigger tagging: if cfg.getboolean("lstchain", "use_ff_heuristic_id"): diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index 32449699..ea63e204 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -302,8 +302,8 @@ def test_drs4_pedestal_cmd(base_test_dir): cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804") expected_command = [ cfg.get("lstchain", "drs4_baseline"), - "--run_number=01804", - f"--base_dir={base_test_dir}", + "--run-number=01804", + f"--base-dir={base_test_dir}", "--no-progress", ] assert cmd == expected_command @@ -315,9 +315,9 @@ def test_calibration_file_cmd(base_test_dir): cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809") expected_command = [ cfg.get("lstchain", "charge_calibration"), - "--pedestal_run=01804", - "--run_number=01809", - f"--base_dir={base_test_dir}", + "--pedestal-run=01804", + "--run-number=01809", + f"--base-dir={base_test_dir}", ] assert cmd == expected_command From cbc5a58c9f5d69e340ed72cd14d7dd57a5c6fcfb Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Wed, 13 Nov 2024 16:47:31 +0100 Subject: [PATCH 08/12] allow the calibration_pipeline to work also for the lstchain scripts --- src/osa/scripts/calibration_pipeline.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/osa/scripts/calibration_pipeline.py b/src/osa/scripts/calibration_pipeline.py index 7e3a8034..c18508a0 100644 --- a/src/osa/scripts/calibration_pipeline.py +++ b/src/osa/scripts/calibration_pipeline.py @@ -47,8 +47,8 @@ def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list: command = cfg.get("lstchain", "drs4_baseline") return [ command, - f"--run-number={drs4_pedestal_run_id}", - f"--base-dir={base_dir}", + "-r", drs4_pedestal_run_id, + "-b", base_dir, "--no-progress", ] @@ -59,9 +59,9 @@ def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> l command = cfg.get("lstchain", "charge_calibration") cmd = [ command, - f"--pedestal-run={drs4_pedestal_run_id}", - f"--run-number={pedcal_run_id}", - f"--base-dir={base_dir}", + "-p", drs4_pedestal_run_id, + "-r", pedcal_run_id, + "-b", base_dir, ] # In case of problems with trigger tagging: if cfg.getboolean("lstchain", "use_ff_heuristic_id"): From df125d72bffa3ddafc424c6c69b0ca53d0185ee2 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 26 Nov 2024 11:33:40 +0100 Subject: [PATCH 09/12] Update ctapipe_io_lst version in environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f35534d2..151e84b6 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - pymongo - scikit-learn=1.2 - protobuf=3.20 - - ctapipe_io_lst=0.22 + - ctapipe_io_lst=0.24 - ctaplot~=0.6.4 - pyirf~=0.10 - lstchain>=0.10.13 From c609be3b5149d94408995f7eb2e4e445ee654f28 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 26 Nov 2024 11:46:49 +0100 Subject: [PATCH 10/12] Remove unneeded protobuf pin in environment.yml --- environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment.yml b/environment.yml index 151e84b6..46e850f8 100644 --- a/environment.yml +++ b/environment.yml @@ -21,7 +21,6 @@ dependencies: - click - pymongo - scikit-learn=1.2 - - protobuf=3.20 - ctapipe_io_lst=0.24 - ctaplot~=0.6.4 - pyirf~=0.10 From 4c81add52e1d8fe2736d9eb5783a3695703a5f55 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Tue, 26 Nov 2024 16:55:51 +0100 Subject: [PATCH 11/12] adapt tests --- src/osa/scripts/tests/test_osa_scripts.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index ea63e204..da43f776 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -302,8 +302,10 @@ def test_drs4_pedestal_cmd(base_test_dir): cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804") expected_command = [ cfg.get("lstchain", "drs4_baseline"), - "--run-number=01804", - f"--base-dir={base_test_dir}", + "-r", + "01804", + "-b", + base_test_dir, "--no-progress", ] assert cmd == expected_command @@ -315,9 +317,12 @@ def test_calibration_file_cmd(base_test_dir): cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809") expected_command = [ cfg.get("lstchain", "charge_calibration"), - "--pedestal-run=01804", - "--run-number=01809", - f"--base-dir={base_test_dir}", + "-p", + "01804", + "-r", + "01809", + "-b", + base_test_dir, ] assert cmd == expected_command From f865da4776650d8c9486263405a701d8446f371e Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Tue, 26 Nov 2024 20:07:55 +0100 Subject: [PATCH 12/12] transform run numbers to string in cmd --- src/osa/scripts/calibration_pipeline.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/osa/scripts/calibration_pipeline.py b/src/osa/scripts/calibration_pipeline.py index c18508a0..893f7eab 100644 --- a/src/osa/scripts/calibration_pipeline.py +++ b/src/osa/scripts/calibration_pipeline.py @@ -47,20 +47,19 @@ def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list: command = cfg.get("lstchain", "drs4_baseline") return [ command, - "-r", drs4_pedestal_run_id, + "-r", str(drs4_pedestal_run_id), "-b", base_dir, "--no-progress", ] - def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> list: """Build the create_calibration_file command.""" base_dir = Path(cfg.get("LST1", "BASE")).resolve() command = cfg.get("lstchain", "charge_calibration") cmd = [ command, - "-p", drs4_pedestal_run_id, - "-r", pedcal_run_id, + "-p", str(drs4_pedestal_run_id), + "-r", str(pedcal_run_id), "-b", base_dir, ] # In case of problems with trigger tagging: