Merge pull request #266 from cta-observatory/prov-dl2

Do not create the DL2 directory if --no-dl2 option is given
cta-observatory · Jan 12, 2024 · 41e7313 · 41e7313
2 parents e2e2ad0 + 18c5637
commit 41e7313
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 21 deletions.
diff --git a/src/osa/scripts/provprocess.py b/src/osa/scripts/provprocess.py
@@ -81,8 +81,8 @@ def parse_lines_log(filter_cut, calib_runs, run_number):
         filter_cut = "all"
     cuts = {
         "calibration": ["drs4_pedestal", "calibrate_charge"],
-        "r0_to_dl1": ["r0_to_dl1", "dl1ab"],
-        "dl1_to_dl2": ["dl1_datacheck", "dl1_to_dl2"],
+        "r0_to_dl1": ["r0_to_dl1", "dl1ab", "dl1_datacheck"],
+        "dl1_to_dl2": ["dl1_to_dl2"],
     }
     cuts["all"] = cuts["calibration"] + cuts["r0_to_dl1"] + cuts["dl1_to_dl2"]
 
@@ -413,29 +413,31 @@ def produce_provenance(session_log_filename, base_filename):
             read_prov(filename=session_log_filename),
             str(paths_r0_dl1["out_path"]),
         )
-        dl1_lines = plines_r0 + plines_ab[1:]
+        plines_check = parse_lines_run(
+            "dl1_datacheck",
+            read_prov(filename=session_log_filename),
+            str(paths_r0_dl1["out_path"]),
+        )
+        dl1_lines = plines_r0 + plines_ab[1:] + plines_check[1:]
 
     # create r0_to_dl1 prov files only if filtering
     if options.filter == "r0_to_dl1":
-        produce_provenance_files(plines_r0 + plines_ab[1:], paths_r0_dl1)
+        produce_provenance_files(plines_r0 + plines_ab[1:] + plines_check[1:], paths_r0_dl1)
 
     if options.filter == "dl1_to_dl2" or not options.filter:
-        paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename)
-        plines_check = parse_lines_run(
-            "dl1_datacheck",
-            read_prov(filename=session_log_filename),
-            str(paths_dl1_dl2["out_path"]),
-        )
-        plines_dl2 = parse_lines_run(
-            "dl1_to_dl2",
-            read_prov(filename=session_log_filename),
-            str(paths_dl1_dl2["out_path"]),
-        )
-        dl1_dl2_lines = plines_check + plines_dl2[1:]
+        if not options.no_dl2:
+            paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename)
+            plines_dl2 = parse_lines_run(
+                "dl1_to_dl2",
+                read_prov(filename=session_log_filename),
+                str(paths_dl1_dl2["out_path"]),
+            )
+            dl1_dl2_lines = plines_dl2
 
     # create dl1_to_dl2 prov files only if filtering
     if options.filter == "dl1_to_dl2":
-        produce_provenance_files(plines_check + plines_dl2[1:], paths_dl1_dl2)
+        if not options.no_dl2:
+            produce_provenance_files(plines_dl2, paths_dl1_dl2)
 
     # create calibration_to_dl1 and calibration_to_dl2 prov files
     if not options.filter:

diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py
@@ -99,10 +99,10 @@ def test_simulate_processing(
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 16
-    assert len(dl1["activity"]) == 4
-    assert len(dl1["used"]) == 13
-    assert len(dl1["wasGeneratedBy"]) == 7
+    assert len(dl1["entity"]) == 19
+    assert len(dl1["activity"]) == 5
+    assert len(dl1["used"]) == 15
+    assert len(dl1["wasGeneratedBy"]) == 10
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)