From c05b37f2712ed03926acd885f1cb72fb73f98d50 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Tue, 4 Jan 2022 15:19:24 +0100 Subject: [PATCH 01/33] remove not used log level in handler --- osa/provenance/config/logger.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/osa/provenance/config/logger.yaml b/osa/provenance/config/logger.yaml index bf9d2a57..a39a26b7 100644 --- a/osa/provenance/config/logger.yaml +++ b/osa/provenance/config/logger.yaml @@ -7,7 +7,6 @@ formatters: handlers: provHandler: class: logging.handlers.WatchedFileHandler - level: INFO formatter: simple filename: prov.log loggers: From aaf4358e0088f7a37ea4370003eefa30fde804e9 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Tue, 4 Jan 2022 15:20:06 +0100 Subject: [PATCH 02/33] improve code comments --- osa/provenance/utils.py | 15 +++++++-------- osa/scripts/simulate_processing.py | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index 3a5c0220..aa5d4451 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -19,13 +19,12 @@ def parse_variables(class_instance): # datasequence.py # -c cfg/sequencer.cfg # -d 2020_02_18 - # -o /fefs/aswg/data/real/DL1/20200218/v0.4.3_v00/ # --prod-id v0.4.3_v00 - # /fefs/aswg/data/real/calibration/20200218/v00/calibration.Run02006.0000.hdf5 - # /fefs/aswg/data/real/calibration/20200218/v00/drs4_pedestal.Run02005.0000.fits - # /fefs/aswg/data/real/calibration/20191124/v00/time_calibration.Run01625.0000.hdf5 - # /fefs/home/lapp/DrivePositioning/drive_log_20_02_18.txt - # /fefs/aswg/data/real/monitoring/RunSummary/RunSummary_20200101.ecsv + # --pedcal-file ../calibration/20200218/v00/calibration.Run02006.0000.hdf5 + # --drs4-pedestal-file ../calibration/20200218/v00/drs4_pedestal.Run02005.0000.fits + # --time-calib-file ../calibration/20191124/v00/time_calibration.Run01625.0000.hdf5 + # --drive-file ../lapp/DrivePositioning/drive_log_20_02_18.txt + # --run-summary ../monitoring/RunSummary/RunSummary_20200101.ecsv # 02006.0000 # LST1 @@ -43,9 +42,9 @@ def parse_variables(class_instance): if class_instance.__name__ == "r0_to_dl1": # calibrationfile [0] .../20200218/v00/calibration.Run02006.0000.hdf5 # pedestalfile [1] .../20200218/v00/drs4_pedestal.Run02005.0000.fits - # time_calibration [2] .../20191124/v00/time_calibration.Run01625.0000.hdf5 + # timecalibfile [2] .../20191124/v00/time_calibration.Run01625.0000.hdf5 # drivefile [3] .../DrivePositioning/drive_log_20_02_18.txt - # runsummary_file [4] .../RunSummary/RunSummary_20200101.ecsv + # runsummaryfile [4] .../RunSummary/RunSummary_20200101.ecsv # run_str [5] 02006.0000 class_instance.ObservationRun = class_instance.args[5].split(".")[0] diff --git a/osa/scripts/simulate_processing.py b/osa/scripts/simulate_processing.py index 956373f8..214dbe92 100644 --- a/osa/scripts/simulate_processing.py +++ b/osa/scripts/simulate_processing.py @@ -1,4 +1,10 @@ -"""Simulate executions of data processing pipeline and produce provenance.""" +""" +Simulate executions of data processing pipeline and produce provenance. +If it is not executed by tests, please run pytest --basetemp=test_osa first. +It needs to have test_osa folder filled with test datasets. + +python osa/scripts/simulate_processing.py +""" import logging import multiprocessing as mp @@ -150,21 +156,17 @@ def simulate_processing(): run_list = extractruns(sub_run_list) sequence_list = extractsequences(run_list) - # skip drs4 and calibration + # simulate data calibration and reduction for sequence in sequence_list: processed = False for sub_list in sequence.subrun_list: if sub_list.runobj.type == "PEDCALIB": - args_cal = parse_template( - create_job_template(sequence, get_content=True), 0 - ) + args_cal = parse_template(create_job_template(sequence, get_content=True), 0) simulate_calibration(args_cal) elif sub_list.runobj.type == "DATA": with mp.Pool() as poolproc: args_proc = [ - parse_template( - create_job_template(sequence, get_content=True), subrun_idx - ) + parse_template(create_job_template(sequence, get_content=True), subrun_idx) for subrun_idx in range(sub_list.subrun) ] processed = poolproc.map(simulate_subrun_processing, args_proc) From 4e9a1e0aa99322ca6ae1ef1cc54221c1c9859c4b Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Thu, 2 Dec 2021 17:37:16 +0100 Subject: [PATCH 03/33] add drs4_pedestal definition in the model --- osa/provenance/config/definition.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/osa/provenance/config/definition.yaml b/osa/provenance/config/definition.yaml index 03f8d1bf..46ab4a07 100644 --- a/osa/provenance/config/definition.yaml +++ b/osa/provenance/config/definition.yaml @@ -52,6 +52,28 @@ # activities: + drs4_pedestal: + description: + "Create pedestal file" + parameters: + usage: + - role: "Subrun for pedestal" + description: "Raw observation file for pedestal" + entityName: RawObservationFile + value: RawObservationFilePedestal +# filepath: /fefs/aswg/data/real/R0/20210913/LST-1.1.Run06268.0000.fits.fz + generation: + - role: "Pedestal" + description: "Pedestal calibration file" + entityName: PedestalFile + value: PedestalFile +# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/drs4_pedestal.Run06268.0000.fits + - role: "Check plot for pedestal" + description: "Pedestal check plot" + entityName: PedestalCheckPlot + value: PedestalCheckPlot +# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/log/drs4_pedestal.Run02068.0000.pdf + r0_to_dl1: description: "Create DL1 files for an observation run and subrun" From fe48311dda315d7edd415888d6b673cb4d012f2c Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Tue, 4 Jan 2022 15:40:24 +0100 Subject: [PATCH 04/33] refactor utils.py --- osa/provenance/utils.py | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index aa5d4451..96931d8e 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -13,6 +13,8 @@ __all__ = ["parse_variables", "get_log_config", "store_conda_env_export"] +REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"] + def parse_variables(class_instance): """Parse variables needed in model""" @@ -35,9 +37,13 @@ def parse_variables(class_instance): dl1_dir = cfg.get("LST1", "DL1_DIR") dl2_dir = cfg.get("LST1", "DL2_DIR") nightdir = lstdate_to_dir(options.date) - muon_dir = Path(dl1_dir) / nightdir / options.prod_id - outdir_dl1 = Path(dl1_dir) / nightdir / options.prod_id / options.dl1_prod_id - outdir_dl2 = Path(dl2_dir) / nightdir / options.prod_id / options.dl2_prod_id + class_instance.SoftwareVersion = get_lstchain_version() + class_instance.ProcessingConfigFile = options.configfile + class_instance.ObservationDate = nightdir + if class_instance.__name__ in REDUCTION_TASKS: + muon_dir = Path(dl1_dir) / nightdir / options.prod_id + outdir_dl1 = Path(dl1_dir) / nightdir / options.prod_id / options.dl1_prod_id + outdir_dl2 = Path(dl2_dir) / nightdir / options.prod_id / options.dl2_prod_id if class_instance.__name__ == "r0_to_dl1": # calibrationfile [0] .../20200218/v00/calibration.Run02006.0000.hdf5 @@ -48,11 +54,7 @@ def parse_variables(class_instance): # run_str [5] 02006.0000 class_instance.ObservationRun = class_instance.args[5].split(".")[0] - class_instance.ObservationDate = nightdir - class_instance.SoftwareVersion = get_lstchain_version() - class_instance.session_name = class_instance.ObservationRun - class_instance.ProcessingConfigFile = options.configfile - # Use realpath to resolve symbolic links and return abspath + # use realpath to resolve symbolic links and return abspath calibration_file = os.path.realpath(class_instance.args[0]) pedestal_file = os.path.realpath(class_instance.args[1]) timecalibration_file = os.path.realpath(class_instance.args[2]) @@ -76,14 +78,8 @@ def parse_variables(class_instance): class_instance.Analysisconfigfile_dl1 = configfile_dl1 class_instance.ObservationRun = class_instance.args[0].split(".")[0] - class_instance.ObservationDate = nightdir - class_instance.SoftwareVersion = get_lstchain_version() - class_instance.session_name = class_instance.ObservationRun - class_instance.ProcessingConfigFile = options.configfile - class_instance.PedestalCleaning = "True" class_instance.StoreImage = cfg.getboolean("lstchain", "store_image_dl1ab") - class_instance.DL1SubrunDataset = ( f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5" ) @@ -92,11 +88,6 @@ def parse_variables(class_instance): # run_str [0] 02006.0000 class_instance.ObservationRun = class_instance.args[0].split(".")[0] - class_instance.ObservationDate = nightdir - class_instance.SoftwareVersion = get_lstchain_version() - class_instance.session_name = class_instance.ObservationRun - class_instance.ProcessingConfigFile = options.configfile - class_instance.DL1SubrunDataset = ( f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5" ) @@ -118,11 +109,6 @@ def parse_variables(class_instance): class_instance.Analysisconfigfile_dl2 = configfile_dl2 class_instance.ObservationRun = class_instance.args[0].split(".")[0] - class_instance.ObservationDate = nightdir - class_instance.SoftwareVersion = get_lstchain_version() - class_instance.session_name = class_instance.ObservationRun - class_instance.ProcessingConfigFile = options.configfile - class_instance.RFModelEnergyFile = str( Path(rf_models_directory) / "reg_energy.sav" ) @@ -145,6 +131,9 @@ def parse_variables(class_instance): class_instance.DL2MergedFile = ( f"{outdir_dl2}/dl2_LST-1.Run{class_instance.ObservationRun}.h5" ) + + if class_instance.__name__ in REDUCTION_TASKS: + class_instance.session_name = class_instance.ObservationRun return class_instance From 261ccfcc138f728a92adc71612551354278319f9 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Tue, 4 Jan 2022 16:03:04 +0100 Subject: [PATCH 05/33] add drs4_pedestal instance vars --- osa/provenance/utils.py | 21 +++++++++++++++++++++ osa/scripts/simulate_processing.py | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index 96931d8e..f232b0e8 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -18,6 +18,14 @@ def parse_variables(class_instance): """Parse variables needed in model""" + + # calibration_pipeline.py + # -c cfg/sequencer.cfg + # -d 2020_02_18 + # --drs4-pedestal-run 01804 + # --pedcal-run 01805 + # LST1 + # datasequence.py # -c cfg/sequencer.cfg # -d 2020_02_18 @@ -45,6 +53,19 @@ def parse_variables(class_instance): outdir_dl1 = Path(dl1_dir) / nightdir / options.prod_id / options.dl1_prod_id outdir_dl2 = Path(dl2_dir) / nightdir / options.prod_id / options.dl2_prod_id + if class_instance.__name__ == "drs4_pedestal": + # drs4-pedestal-run [0] 01804 + # history_file [1] .../20210913/v0.7.5/sequence_LST1_01805.0000.history + class_instance.RawObservationFilePedestal = ( + f"{rawdir}/{nightdir}/LST-1.1.Run{class_instance.args[0]}.fits.fz" + ) + class_instance.PedestalFile = ( + f"{options.directory}/drs4_pedestal.Run{class_instance.args[0]}.0000.h5" + ) + class_instance.PedestalCheckPlot = ( + f"{options.directory}/log/drs4_pedestal.Run{class_instance.args[0]}.0000.pdf" + ) + if class_instance.__name__ == "r0_to_dl1": # calibrationfile [0] .../20200218/v00/calibration.Run02006.0000.hdf5 # pedestalfile [1] .../20200218/v00/drs4_pedestal.Run02005.0000.fits diff --git a/osa/scripts/simulate_processing.py b/osa/scripts/simulate_processing.py index 214dbe92..9984834f 100644 --- a/osa/scripts/simulate_processing.py +++ b/osa/scripts/simulate_processing.py @@ -130,8 +130,8 @@ def parse_template(template, idx): args.append(line.strip()) if "subprocess.run" in line: keep = True - # Remove last three elements - return args[0:-3] + # remove last three elements + return args[:-3] def simulate_calibration(args): From 3a82cf1c3c2b014e8a65f2b84f829d42aa9c004e Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Thu, 2 Dec 2021 17:39:29 +0100 Subject: [PATCH 06/33] adapt provenance/capture.py to calibration --- osa/provenance/capture.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/osa/provenance/capture.py b/osa/provenance/capture.py index 2ff9383b..af9c2df4 100644 --- a/osa/provenance/capture.py +++ b/osa/provenance/capture.py @@ -52,6 +52,7 @@ PROV_PREFIX = provconfig["PREFIX"] SUPPORTED_HASH_METHOD = ["md5"] SUPPORTED_HASH_BUFFER = ["content", "path"] +REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"] # global variables sessions = set() @@ -99,8 +100,14 @@ def wrapper(*args, **kwargs): # variables parsing global session_name, session_tag class_instance = parse_variables(class_instance) - session_tag = f"{activity}:{class_instance.ObservationRun}" - session_name = f"{class_instance.ObservationRun}" + if class_instance.__name__ in REDUCTION_TASKS: + session_tag = f"{activity}:{class_instance.ObservationRun}" + session_name = f"{class_instance.ObservationRun}" + else: + session_tag = f"{activity}:{class_instance.PedestalRun}-{class_instance.CalibrationRun}" + session_name = f"{class_instance.PedestalRun}-{class_instance.CalibrationRun}" + # OSA specific + # variables parsing # provenance capture before execution derivation_records = get_derivation_records(class_instance, activity) @@ -357,15 +364,18 @@ def log_session(class_instance, start): "startTime": start, "system": system, # OSA specific - "software_version": class_instance.SoftwareVersion, "observation_date": class_instance.ObservationDate, - "observation_run": class_instance.ObservationRun, # a session is run-wise + "software_version": class_instance.SoftwareVersion, "config_file": class_instance.ProcessingConfigFile, - "config_file_hash": get_file_hash( - class_instance.ProcessingConfigFile, buffer="path" - ), + "config_file_hash": get_file_hash(class_instance.ProcessingConfigFile, buffer="path"), "config_file_hash_type": get_hash_method(), } + if class_instance.__name__ in REDUCTION_TASKS: + log_record["observation_run"] = class_instance.ObservationRun # a session is run-wise + else: + log_record["pedestal_run"] = class_instance.PedestalRun + log_record["calibration_run"] = class_instance.CalibrationRun + log_prov_info(log_record) return session_id From 9a91a3453f5b9604fef21a6e5a00d9aafcef52bc Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Tue, 4 Jan 2022 16:05:36 +0100 Subject: [PATCH 07/33] trace drs4_pedestal --- osa/scripts/calibration_pipeline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/osa/scripts/calibration_pipeline.py b/osa/scripts/calibration_pipeline.py index 2d2b869a..2789fd4c 100644 --- a/osa/scripts/calibration_pipeline.py +++ b/osa/scripts/calibration_pipeline.py @@ -16,6 +16,7 @@ from osa.configs.config import cfg from osa.job import historylevel from osa.job import run_program_with_history_logging +from osa.provenance.capture import trace from osa.utils.cliopts import calibration_pipeline_cliparsing from osa.utils.logging import myLogger @@ -90,6 +91,7 @@ def calibration_sequence(drs4_pedestal_run_id: str, pedcal_run_id: str) -> int: return rc +@trace def drs4_pedestal(drs4_pedestal_run_id: str, history_file: Path) -> int: """ Create a DRS4 pedestal file for baseline correction. From 5f7cefa4f69262cc6a33f1cf1beb1f88bd42d8fc Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Tue, 4 Jan 2022 16:08:06 +0100 Subject: [PATCH 08/33] add drs4_pedestal entities to model --- osa/provenance/config/definition.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/osa/provenance/config/definition.yaml b/osa/provenance/config/definition.yaml index 46ab4a07..ec487db2 100644 --- a/osa/provenance/config/definition.yaml +++ b/osa/provenance/config/definition.yaml @@ -302,6 +302,10 @@ entities: PythonObject: description: "Python variable in memory" type: PythonObject + RawObservationFile: + description: "Raw observation compressed FITS file" + type: File + contentType: application/fits R0SubrunDataset: description: "R0 subrun file in FITS format on the disk" type: File @@ -318,6 +322,10 @@ entities: description: "Pedestal file in FITS format on the disk" type: File contentType: application/fits + PedestalCheckPlot: + description: "Pedestal check plot PDF file" + type: File + contentType: application/pdf CoefficientsCalibrationFile: description: "Coefficients calibration file in HDF5 format on the disk" type: File From 8512671b096fdedacaaa3f6b835a028948059e61 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Tue, 4 Jan 2022 16:28:26 +0100 Subject: [PATCH 09/33] fix paths --- osa/provenance/utils.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index f232b0e8..74c3dad3 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -40,30 +40,34 @@ def parse_variables(class_instance): configfile_dl1 = cfg.get("lstchain", "dl1ab_config") configfile_dl2 = cfg.get("lstchain", "dl2_config") - rawdir = cfg.get("LST1", "R0_DIR") + raw_dir = cfg.get("LST1", "R0_DIR") rf_models_directory = cfg.get("lstchain", "RF_MODELS") dl1_dir = cfg.get("LST1", "DL1_DIR") dl2_dir = cfg.get("LST1", "DL2_DIR") - nightdir = lstdate_to_dir(options.date) + base_dir = Path(cfg.get("LST1", "BASE")).resolve() + calib_dir = base_dir / "monitoring" / "PixelCalibration" / "LevelA" + night_dir = lstdate_to_dir(options.date) class_instance.SoftwareVersion = get_lstchain_version() class_instance.ProcessingConfigFile = options.configfile - class_instance.ObservationDate = nightdir + class_instance.ObservationDate = night_dir if class_instance.__name__ in REDUCTION_TASKS: - muon_dir = Path(dl1_dir) / nightdir / options.prod_id - outdir_dl1 = Path(dl1_dir) / nightdir / options.prod_id / options.dl1_prod_id - outdir_dl2 = Path(dl2_dir) / nightdir / options.prod_id / options.dl2_prod_id + muon_dir = Path(dl1_dir) / night_dir / options.prod_id + outdir_dl1 = Path(dl1_dir) / night_dir / options.prod_id / options.dl1_prod_id + outdir_dl2 = Path(dl2_dir) / night_dir / options.prod_id / options.dl2_prod_id if class_instance.__name__ == "drs4_pedestal": # drs4-pedestal-run [0] 01804 # history_file [1] .../20210913/v0.7.5/sequence_LST1_01805.0000.history + pedestal_dir = calib_dir / "drs4_baseline" / night_dir / class_instance.SoftwareVersion + class_instance.RawObservationFilePedestal = ( - f"{rawdir}/{nightdir}/LST-1.1.Run{class_instance.args[0]}.fits.fz" + f"{raw_dir}/{night_dir}/LST-1.1.Run{class_instance.args[0]}.fits.fz" ) class_instance.PedestalFile = ( - f"{options.directory}/drs4_pedestal.Run{class_instance.args[0]}.0000.h5" + f"{pedestal_dir}/drs4_pedestal.Run{class_instance.args[0]}.0000.h5" ) class_instance.PedestalCheckPlot = ( - f"{options.directory}/log/drs4_pedestal.Run{class_instance.args[0]}.0000.pdf" + f"{pedestal_dir}/log/drs4_pedestal.Run{class_instance.args[0]}.0000.pdf" ) if class_instance.__name__ == "r0_to_dl1": @@ -80,7 +84,7 @@ def parse_variables(class_instance): pedestal_file = os.path.realpath(class_instance.args[1]) timecalibration_file = os.path.realpath(class_instance.args[2]) class_instance.R0SubrunDataset = ( - f"{rawdir}/{nightdir}/LST-1.1.Run{class_instance.args[5]}.fits.fz" + f"{raw_dir}/{night_dir}/LST-1.1.Run{class_instance.args[5]}.fits.fz" ) class_instance.CoefficientsCalibrationFile = calibration_file class_instance.PedestalFile = pedestal_file From 52f4fa778b5a66bbf650d33a0322319eb4a357ed Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Tue, 4 Jan 2022 16:36:07 +0100 Subject: [PATCH 10/33] add pedestal and calibration runs as session tag --- osa/provenance/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index 74c3dad3..7b6bb01d 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -59,7 +59,10 @@ def parse_variables(class_instance): # drs4-pedestal-run [0] 01804 # history_file [1] .../20210913/v0.7.5/sequence_LST1_01805.0000.history pedestal_dir = calib_dir / "drs4_baseline" / night_dir / class_instance.SoftwareVersion - + class_instance.PedestalRun = class_instance.args[0] + sequence_filename = os.path.basename(class_instance.args[1]) + tailname = sequence_filename.replace("sequence_LST1_", "") + class_instance.CalibrationRun = tailname.split(".")[0] class_instance.RawObservationFilePedestal = ( f"{raw_dir}/{night_dir}/LST-1.1.Run{class_instance.args[0]}.fits.fz" ) From 21ec79133733cc960271ed1eb27619ccd935a5f5 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Thu, 2 Dec 2021 19:11:39 +0100 Subject: [PATCH 11/33] fix provenance/capture.py modifs for calibration --- osa/provenance/capture.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/osa/provenance/capture.py b/osa/provenance/capture.py index af9c2df4..26abf443 100644 --- a/osa/provenance/capture.py +++ b/osa/provenance/capture.py @@ -212,7 +212,7 @@ def get_entity_id(value, item): entity_name = item["entityName"] entity_type = definition["entities"][entity_name]["type"] except Exception as ex: - logger.warning(f"{ex} in {item}") + logger.warning(f"Not found in model {ex} in {item}") entity_name = "" entity_type = "" @@ -227,7 +227,8 @@ def get_entity_id(value, item): # osa specific hash path # async calls does not allow for hash content return get_file_hash(value, buffer="path") - + # osa specific hash path + # async calls does not allow for hash content try: entity_id = abs(hash(value) + hash(str(value))) if hasattr(value, "entity_version"): @@ -338,7 +339,9 @@ def get_python_packages(): def log_prov_info(prov_dict): """Write a dictionary to the logger.""" - prov_dict["session_tag"] = session_tag # OSA specific session tag + # OSA specific session tag used in merging prov from parallel sessions + prov_dict["session_tag"] = session_tag + # record_date = datetime.datetime.now().isoformat() logger.info(f"{PROV_PREFIX}{record_date}{PROV_PREFIX}{prov_dict}") @@ -351,9 +354,16 @@ def log_session(class_instance, start): session_id = abs(hash(class_instance)) lines = read_prov(filename=LOG_FILENAME) for line in lines: - if line.get("observation_run", 0) == class_instance.ObservationRun: + if class_instance.__name__ in REDUCTION_TASKS: + if line.get("observation_run", 0) == class_instance.ObservationRun: + session_id = lines[0]["session_id"] + sessions.add(session_id) + elif line.get("pedestal_run", 0) == class_instance.PedestalRun: session_id = lines[0]["session_id"] sessions.add(session_id) + # OSA specific + # prov session is outside scripting and is run-wise + # we may have different sessions/runs in the same log file if session_id not in sessions: sessions.add(session_id) @@ -365,6 +375,7 @@ def log_session(class_instance, start): "system": system, # OSA specific "observation_date": class_instance.ObservationDate, + # OSA specific "software_version": class_instance.SoftwareVersion, "config_file": class_instance.ProcessingConfigFile, "config_file_hash": get_file_hash(class_instance.ProcessingConfigFile, buffer="path"), From 301c7a7b326983f52bf7f65c36907e74746dd395 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Tue, 4 Jan 2022 17:01:05 +0100 Subject: [PATCH 12/33] add calibrate_charge definition in the model --- osa/provenance/config/definition.yaml | 55 +++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/osa/provenance/config/definition.yaml b/osa/provenance/config/definition.yaml index ec487db2..f5017d4c 100644 --- a/osa/provenance/config/definition.yaml +++ b/osa/provenance/config/definition.yaml @@ -74,6 +74,53 @@ activities: value: PedestalCheckPlot # filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/log/drs4_pedestal.Run02068.0000.pdf + calibrate_charge: + description: + "Create charge calibration file" + parameters: + usage: + - role: "Subrun for calibration" + description: "Raw observation file for calibration" + entityName: RawObservationFile + value: RawObservationFileCalibration +# filepath: /fefs/aswg/data/real/R0/20210913/LST-1.1.Run06274.0000.fits.fz + - role: "Pedestal file" + description: "Pedestal file used" + entityName: PedestalFile + value: PedestalFile +# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/drs4_pedestal.Run06268.0000.fits + - role: "Run summary" + description: "Run summary configuration" + entityName: RunSummaryFile + value: RunSummaryFile +# filepath: /fefs/aswg/data/real/monitoring/RunSummary/RunSummary_20210913.ecsv + - role: "Configuration file" + description: "Configuration file for camera" + entityName: AnalysisConfigFile + value: CalibrationConfigurationFile +# filepath: /fefs/aswg/software/virtual_env/ctasoft/cta-lstchain/lstchain/data/onsite_camera_calibration_param.json + - role: "Systematics correction file" + description: "Systematics correction file" + entityName: SystematicsCorrectionFile + value: SystematicsCorrectionFile +# filepath: /path/to/ff_systematics_file.h5 + generation: + - role: "Coefficients calibration file" + description: "Coefficients calibration file" + entityName: CoefficientsCalibrationFile + value: CoefficientsCalibrationFile +# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/calibration.Run06274.0000.hdf5 + - role: "Check plot for calibration" + description: "Calibration check plot" + entityName: CalibrationCheckPlot + value: CalibrationCheckPlot +# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/log/calibration.Run06274.0000.pedestal.Run06268.0000.pdf + - role: "Time calibration file" + description: "Time calibration file" + entityName: TimeCalibrationFile + value: TimeCalibrationFile +# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/time_calibration.Run06274.0000.hdf5 + r0_to_dl1: description: "Create DL1 files for an observation run and subrun" @@ -330,10 +377,18 @@ entities: description: "Coefficients calibration file in HDF5 format on the disk" type: File contentType: application/x-hdf + SystematicsCorrectionFile: + description: "Systematics correction file in HDF5 format on the disk" + type: File + contentType: application/x-hdf TimeCalibrationFile: description: "Time calibration file in HDF5 format on the disk" type: File contentType: application/x-hdf + CalibrationCheckPlot: + description: "Calibration check plot PDF file" + type: File + contentType: application/pdf AnalysisConfigFile: description: "LSTChain analysis configuration file in JSON format on the disk" type: File From 6302436d9f098a2a3e8a01fa6bd510b21077edd1 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Fri, 7 Jan 2022 13:30:27 +0100 Subject: [PATCH 13/33] add drs4_pedestal_run_id, pedcal_run_id as params for both calib functions --- osa/provenance/utils.py | 30 ++++++++++++++--------------- osa/scripts/calibration_pipeline.py | 22 +++++++++++++-------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index 7b6bb01d..a1e76796 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -38,31 +38,30 @@ def parse_variables(class_instance): # 02006.0000 # LST1 + flat_date = lstdate_to_dir(options.date) configfile_dl1 = cfg.get("lstchain", "dl1ab_config") configfile_dl2 = cfg.get("lstchain", "dl2_config") raw_dir = cfg.get("LST1", "R0_DIR") rf_models_directory = cfg.get("lstchain", "RF_MODELS") dl1_dir = cfg.get("LST1", "DL1_DIR") dl2_dir = cfg.get("LST1", "DL2_DIR") - base_dir = Path(cfg.get("LST1", "BASE")).resolve() calib_dir = base_dir / "monitoring" / "PixelCalibration" / "LevelA" night_dir = lstdate_to_dir(options.date) class_instance.SoftwareVersion = get_lstchain_version() class_instance.ProcessingConfigFile = options.configfile - class_instance.ObservationDate = night_dir + class_instance.ObservationDate = flat_date if class_instance.__name__ in REDUCTION_TASKS: - muon_dir = Path(dl1_dir) / night_dir / options.prod_id - outdir_dl1 = Path(dl1_dir) / night_dir / options.prod_id / options.dl1_prod_id - outdir_dl2 = Path(dl2_dir) / night_dir / options.prod_id / options.dl2_prod_id - - if class_instance.__name__ == "drs4_pedestal": - # drs4-pedestal-run [0] 01804 - # history_file [1] .../20210913/v0.7.5/sequence_LST1_01805.0000.history - pedestal_dir = calib_dir / "drs4_baseline" / night_dir / class_instance.SoftwareVersion + muon_dir = Path(dl1_dir) / flat_date / options.prod_id + outdir_dl1 = Path(dl1_dir) / flat_date / options.prod_id / options.dl1_prod_id + outdir_dl2 = Path(dl2_dir) / flat_date / options.prod_id / options.dl2_prod_id + + if class_instance.__name__ in ["drs4_pedestal", "calibrate_charge"]: + # drs4_pedestal_run_id [0] 01804 + # pedcal_run_id [1] 01805 + # history_file [2] .../20210913/v0.7.5/sequence_LST1_01805.0000.history class_instance.PedestalRun = class_instance.args[0] - sequence_filename = os.path.basename(class_instance.args[1]) - tailname = sequence_filename.replace("sequence_LST1_", "") - class_instance.CalibrationRun = tailname.split(".")[0] + class_instance.CalibrationRun = class_instance.args[1] + class_instance.RawObservationFilePedestal = ( f"{raw_dir}/{night_dir}/LST-1.1.Run{class_instance.args[0]}.fits.fz" ) @@ -87,7 +86,7 @@ def parse_variables(class_instance): pedestal_file = os.path.realpath(class_instance.args[1]) timecalibration_file = os.path.realpath(class_instance.args[2]) class_instance.R0SubrunDataset = ( - f"{raw_dir}/{night_dir}/LST-1.1.Run{class_instance.args[5]}.fits.fz" + f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[5]}.fits.fz" ) class_instance.CoefficientsCalibrationFile = calibration_file class_instance.PedestalFile = pedestal_file @@ -149,7 +148,6 @@ def parse_variables(class_instance): class_instance.RFModelGammanessFile = str( Path(rf_models_directory) / "cls_gh.sav" ) - class_instance.DL1SubrunDataset = ( f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5" ) @@ -159,7 +157,7 @@ def parse_variables(class_instance): class_instance.DL2MergedFile = ( f"{outdir_dl2}/dl2_LST-1.Run{class_instance.ObservationRun}.h5" ) - + if class_instance.__name__ in REDUCTION_TASKS: class_instance.session_name = class_instance.ObservationRun diff --git a/osa/scripts/calibration_pipeline.py b/osa/scripts/calibration_pipeline.py index 2789fd4c..4bdab295 100644 --- a/osa/scripts/calibration_pipeline.py +++ b/osa/scripts/calibration_pipeline.py @@ -78,11 +78,11 @@ def calibration_sequence(drs4_pedestal_run_id: str, pedcal_run_id: str) -> int: log.info(f"Going to level {level}") if level == 2: - rc = drs4_pedestal(drs4_pedestal_run_id, history_file) + rc = drs4_pedestal(drs4_pedestal_run_id, pedcal_run_id, history_file) level -= 1 log.info(f"Going to level {level}") if level == 1: - rc = calibrate_charge(pedcal_run_id, history_file) + rc = calibrate_charge(drs4_pedestal_run_id, pedcal_run_id, history_file) level -= 1 log.info(f"Going to level {level}") if level == 0: @@ -92,7 +92,7 @@ def calibration_sequence(drs4_pedestal_run_id: str, pedcal_run_id: str) -> int: @trace -def drs4_pedestal(drs4_pedestal_run_id: str, history_file: Path) -> int: +def drs4_pedestal(drs4_pedestal_run_id: str, pedcal_run_id: str, history_file: Path) -> int: """ Create a DRS4 pedestal file for baseline correction. @@ -100,6 +100,8 @@ def drs4_pedestal(drs4_pedestal_run_id: str, history_file: Path) -> int: ---------- drs4_pedestal_run_id : str String with run number of the pedestal run + pedcal_run_id : str + String with run number of the pedcal run history_file : `pathlib.Path` Path to the history file @@ -122,14 +124,18 @@ def drs4_pedestal(drs4_pedestal_run_id: str, history_file: Path) -> int: ) -def calibrate_charge(calibration_run: str, history_file: Path) -> int: +def calibrate_charge(drs4_pedestal_run_id: str, pedcal_run_id: str, history_file: Path) -> int: """ Create the calibration file to transform from ADC counts to photo-electrons Parameters ---------- - calibration_run - history_file + drs4_pedestal_run_id : str + String with run number of the pedestal run + pedcal_run_id : str + String with run number of the pedcal run + history_file : `pathlib.Path` + Path to the history file Returns ------- @@ -139,12 +145,12 @@ def calibrate_charge(calibration_run: str, history_file: Path) -> int: if options.simulate: return 0 - cmd = calibration_file_command(pedcal_run_id=calibration_run) + cmd = calibration_file_command(pedcal_run_id=pedcal_run_id) return run_program_with_history_logging( command_args=cmd, history_file=history_file, - run=calibration_run, + run=pedcal_run_id, prod_id=options.calib_prod_id, command=cmd[0] ) From 1bd0d1b5115a47cc9e48cf0ac2580e664b69b82b Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Fri, 7 Jan 2022 13:31:34 +0100 Subject: [PATCH 14/33] comment entities not used in osa but in lstchain --- osa/provenance/config/definition.yaml | 52 +++++++++++++-------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/osa/provenance/config/definition.yaml b/osa/provenance/config/definition.yaml index f5017d4c..b6bcf8e6 100644 --- a/osa/provenance/config/definition.yaml +++ b/osa/provenance/config/definition.yaml @@ -89,20 +89,20 @@ activities: entityName: PedestalFile value: PedestalFile # filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/drs4_pedestal.Run06268.0000.fits - - role: "Run summary" - description: "Run summary configuration" - entityName: RunSummaryFile - value: RunSummaryFile +# - role: "Run summary" +# description: "Run summary configuration" +# entityName: RunSummaryFile +# value: RunSummaryFile # filepath: /fefs/aswg/data/real/monitoring/RunSummary/RunSummary_20210913.ecsv - - role: "Configuration file" - description: "Configuration file for camera" - entityName: AnalysisConfigFile - value: CalibrationConfigurationFile +# - role: "Configuration file" +# description: "Configuration file for camera" +# entityName: AnalysisConfigFile +# value: CalibrationConfigurationFile # filepath: /fefs/aswg/software/virtual_env/ctasoft/cta-lstchain/lstchain/data/onsite_camera_calibration_param.json - - role: "Systematics correction file" - description: "Systematics correction file" - entityName: SystematicsCorrectionFile - value: SystematicsCorrectionFile +# - role: "Systematics correction file" +# description: "Systematics correction file" +# entityName: SystematicsCorrectionFile +# value: SystematicsCorrectionFile # filepath: /path/to/ff_systematics_file.h5 generation: - role: "Coefficients calibration file" @@ -110,16 +110,16 @@ activities: entityName: CoefficientsCalibrationFile value: CoefficientsCalibrationFile # filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/calibration.Run06274.0000.hdf5 - - role: "Check plot for calibration" - description: "Calibration check plot" - entityName: CalibrationCheckPlot - value: CalibrationCheckPlot -# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/log/calibration.Run06274.0000.pedestal.Run06268.0000.pdf - - role: "Time calibration file" - description: "Time calibration file" - entityName: TimeCalibrationFile - value: TimeCalibrationFile -# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/time_calibration.Run06274.0000.hdf5 +# - role: "Check plot for calibration" +# description: "Calibration check plot" +# entityName: CalibrationCheckPlot +# value: CalibrationCheckPlot +## filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/log/calibration.Run06274.0000.pedestal.Run06268.0000.pdf +# - role: "Time calibration file" +# description: "Time calibration file" +# entityName: TimeCalibrationFile +# value: TimeCalibrationFile +## filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/time_calibration.Run06274.0000.hdf5 r0_to_dl1: description: @@ -377,10 +377,10 @@ entities: description: "Coefficients calibration file in HDF5 format on the disk" type: File contentType: application/x-hdf - SystematicsCorrectionFile: - description: "Systematics correction file in HDF5 format on the disk" - type: File - contentType: application/x-hdf +# SystematicsCorrectionFile: +# description: "Systematics correction file in HDF5 format on the disk" +# type: File +# contentType: application/x-hdf TimeCalibrationFile: description: "Time calibration file in HDF5 format on the disk" type: File From 6c959e633f529ef35c8cc02f7e351e43b5b8f6cf Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Fri, 7 Jan 2022 13:32:04 +0100 Subject: [PATCH 15/33] add calibrate_charge instance vars --- osa/provenance/utils.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index a1e76796..2591e2c7 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -9,6 +9,7 @@ from osa.configs import options from osa.configs.config import cfg +from osa.job import get_time_calibration_file from osa.utils.utils import get_lstchain_version, lstdate_to_dir __all__ = ["parse_variables", "get_log_config", "store_conda_env_export"] @@ -45,8 +46,12 @@ def parse_variables(class_instance): rf_models_directory = cfg.get("lstchain", "RF_MODELS") dl1_dir = cfg.get("LST1", "DL1_DIR") dl2_dir = cfg.get("LST1", "DL2_DIR") - calib_dir = base_dir / "monitoring" / "PixelCalibration" / "LevelA" - night_dir = lstdate_to_dir(options.date) + calib_dir = cfg.get("LST1", "CALIB_DIR") + pedestal_dir = cfg.get("LST1", "PEDESTAL_DIR") + time_calib_dir = cfg.get("LST1", "TIMECALIB_DIR") + # summary_dir = cfg.get("LST1", "RUN_SUMMARY_DIR") + # calib_base_dir = cfg.get("LST1", "CALIB_BASE_DIR") + # sys_dir = calib_base_dir / "ffactor_systematics" class_instance.SoftwareVersion = get_lstchain_version() class_instance.ProcessingConfigFile = options.configfile class_instance.ObservationDate = flat_date @@ -62,15 +67,28 @@ def parse_variables(class_instance): class_instance.PedestalRun = class_instance.args[0] class_instance.CalibrationRun = class_instance.args[1] + pro = "pro" + # TODO - massive reprocessing vs. next day processing + + # according to code in onsite scripts in lstchain + # class_instance.RawObservationFilePedestal = ( - f"{raw_dir}/{night_dir}/LST-1.1.Run{class_instance.args[0]}.fits.fz" - ) - class_instance.PedestalFile = ( - f"{pedestal_dir}/drs4_pedestal.Run{class_instance.args[0]}.0000.h5" - ) - class_instance.PedestalCheckPlot = ( - f"{pedestal_dir}/log/drs4_pedestal.Run{class_instance.args[0]}.0000.pdf" - ) + f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[0]}.fits.fz" + ) + class_instance.RawObservationFileCalibration = ( + f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[1]}.fits.fz" + ) + pedestal_plot = f"drs4_pedestal.Run{class_instance.args[0]}.0000.pdf" + class_instance.PedestalCheckPlot = Path(pedestal_dir) / flat_date / pro / "log" / pedestal_plot + calibration_plot = f"calibration_filters_52.Run{class_instance.args[1]}.0000.pdf" + class_instance.CalibrationCheckPlot = Path(calib_dir) / flat_date / pro / "log" / calibration_plot + # according to code in sequence_calibration_filenames from job.py + # + drs4_pedestal_file = f"drs4_pedestal.Run{class_instance.args[0]}.0000.h5" + class_instance.PedestalFile = Path(pedestal_dir) / flat_date / pro / drs4_pedestal_file + calibration_file = f"calibration_filters_52.Run{class_instance.args[1]}.0000.h5" + class_instance.CoefficientsCalibrationFile = Path(calib_dir) / flat_date / pro / calibration_file + class_instance.TimeCalibrationFile = get_time_calibration_file(class_instance.args[1]) if class_instance.__name__ == "r0_to_dl1": # calibrationfile [0] .../20200218/v00/calibration.Run02006.0000.hdf5 From 77c83703ade02200213784f63d20555179ab4168 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Thu, 2 Dec 2021 22:13:39 +0100 Subject: [PATCH 16/33] refactor provenance/capture.py --- osa/provenance/capture.py | 59 +++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/osa/provenance/capture.py b/osa/provenance/capture.py index 26abf443..afe29c84 100644 --- a/osa/provenance/capture.py +++ b/osa/provenance/capture.py @@ -55,7 +55,6 @@ REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"] # global variables -sessions = set() traced_entities = {} session_name = "" session_tag = "" @@ -351,43 +350,35 @@ def log_session(class_instance, start): # OSA specific # prov session is outside scripting and is run-wise # we may have different sessions/runs in the same log file - session_id = abs(hash(class_instance)) - lines = read_prov(filename=LOG_FILENAME) - for line in lines: - if class_instance.__name__ in REDUCTION_TASKS: - if line.get("observation_run", 0) == class_instance.ObservationRun: - session_id = lines[0]["session_id"] - sessions.add(session_id) - elif line.get("pedestal_run", 0) == class_instance.PedestalRun: - session_id = lines[0]["session_id"] - sessions.add(session_id) + # session_id = abs(hash(class_instance)) + if class_instance.__name__ in REDUCTION_TASKS: + session_id = f"{class_instance.ObservationDate}{class_instance.ObservationRun}" + else: + session_id = f"{class_instance.PedestalRun}{class_instance. CalibrationRun}" # OSA specific # prov session is outside scripting and is run-wise # we may have different sessions/runs in the same log file - if session_id not in sessions: - sessions.add(session_id) - system = get_system_provenance() - log_record = { - "session_id": session_id, - "name": session_name, - "startTime": start, - "system": system, - # OSA specific - "observation_date": class_instance.ObservationDate, - # OSA specific - "software_version": class_instance.SoftwareVersion, - "config_file": class_instance.ProcessingConfigFile, - "config_file_hash": get_file_hash(class_instance.ProcessingConfigFile, buffer="path"), - "config_file_hash_type": get_hash_method(), - } - if class_instance.__name__ in REDUCTION_TASKS: - log_record["observation_run"] = class_instance.ObservationRun # a session is run-wise - else: - log_record["pedestal_run"] = class_instance.PedestalRun - log_record["calibration_run"] = class_instance.CalibrationRun - - log_prov_info(log_record) + system = get_system_provenance() + log_record = { + "session_id": session_id, + "name": session_name, + "startTime": start, + "system": system, + # OSA specific + "observation_date": class_instance.ObservationDate, + # OSA specific + "software_version": class_instance.SoftwareVersion, + "config_file": class_instance.ProcessingConfigFile, + "config_file_hash": get_file_hash(class_instance.ProcessingConfigFile, buffer="path"), + "config_file_hash_type": get_hash_method(), + } + if class_instance.__name__ in REDUCTION_TASKS: + log_record["observation_run"] = class_instance.ObservationRun # a session is run-wise + else: + log_record["pedestal_run"] = class_instance.PedestalRun + log_record["calibration_run"] = class_instance.CalibrationRun + log_prov_info(log_record) return session_id From 39ad393d7e0955f364f65f21c03acec517ce6e89 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Fri, 7 Jan 2022 13:33:10 +0100 Subject: [PATCH 17/33] trace calibrate_charge --- osa/scripts/calibration_pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/osa/scripts/calibration_pipeline.py b/osa/scripts/calibration_pipeline.py index 4bdab295..8c3597fd 100644 --- a/osa/scripts/calibration_pipeline.py +++ b/osa/scripts/calibration_pipeline.py @@ -124,6 +124,7 @@ def drs4_pedestal(drs4_pedestal_run_id: str, pedcal_run_id: str, history_file: P ) +@trace def calibrate_charge(drs4_pedestal_run_id: str, pedcal_run_id: str, history_file: Path) -> int: """ Create the calibration file to transform from ADC counts to photo-electrons From 1869f64610ae38ca5d05fac332c94979d94699f1 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Thu, 2 Dec 2021 22:14:37 +0100 Subject: [PATCH 18/33] do not copy DL1Check files --- osa/provenance/capture.py | 1 - osa/scripts/provprocess.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/osa/provenance/capture.py b/osa/provenance/capture.py index afe29c84..1a9ff2dd 100644 --- a/osa/provenance/capture.py +++ b/osa/provenance/capture.py @@ -15,7 +15,6 @@ import psutil import yaml -from osa.provenance.io import read_prov from osa.provenance.utils import get_log_config, parse_variables # gammapy specific diff --git a/osa/scripts/provprocess.py b/osa/scripts/provprocess.py index ad2ddcb5..d53fc4e8 100644 --- a/osa/scripts/provprocess.py +++ b/osa/scripts/provprocess.py @@ -212,6 +212,7 @@ def parse_lines_run(filter_step, prov_lines, out): filepath and content_type != "application/x-spss-sav" and name != "DL2MergedFile" + and not name.startswith("DL1Check") and not remove ): copy_used_file(filepath, out) From 350bc993681f879e6b1eeff07f9975c4407c0be4 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Fri, 7 Jan 2022 13:35:31 +0100 Subject: [PATCH 19/33] fix int conversion in get_time_calibration_file call --- osa/provenance/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index 2591e2c7..4332b37e 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -48,7 +48,6 @@ def parse_variables(class_instance): dl2_dir = cfg.get("LST1", "DL2_DIR") calib_dir = cfg.get("LST1", "CALIB_DIR") pedestal_dir = cfg.get("LST1", "PEDESTAL_DIR") - time_calib_dir = cfg.get("LST1", "TIMECALIB_DIR") # summary_dir = cfg.get("LST1", "RUN_SUMMARY_DIR") # calib_base_dir = cfg.get("LST1", "CALIB_BASE_DIR") # sys_dir = calib_base_dir / "ffactor_systematics" @@ -88,7 +87,7 @@ def parse_variables(class_instance): class_instance.PedestalFile = Path(pedestal_dir) / flat_date / pro / drs4_pedestal_file calibration_file = f"calibration_filters_52.Run{class_instance.args[1]}.0000.h5" class_instance.CoefficientsCalibrationFile = Path(calib_dir) / flat_date / pro / calibration_file - class_instance.TimeCalibrationFile = get_time_calibration_file(class_instance.args[1]) + class_instance.TimeCalibrationFile = get_time_calibration_file(int(class_instance.args[1])) if class_instance.__name__ == "r0_to_dl1": # calibrationfile [0] .../20200218/v00/calibration.Run02006.0000.hdf5 From 088ff3d61380096e5f51b05d3efab785f7354a23 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Fri, 7 Jan 2022 16:00:57 +0100 Subject: [PATCH 20/33] fix CI tests --- osa/scripts/tests/test_osa_scripts.py | 84 +++++++++++++-------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/osa/scripts/tests/test_osa_scripts.py b/osa/scripts/tests/test_osa_scripts.py index d5efb4f8..1c2e982c 100644 --- a/osa/scripts/tests/test_osa_scripts.py +++ b/osa/scripts/tests/test_osa_scripts.py @@ -127,11 +127,11 @@ def test_autocloser(running_analysis_dir): result = run_program( "python", "osa/scripts/autocloser.py", - "-c", + "--config", "cfg/sequencer.cfg", - "-d", + "--date", "2020_01_17", - "-t", + "--test", "LST1", ) assert os.path.exists(running_analysis_dir) @@ -227,17 +227,13 @@ def test_calibration_pipeline(running_analysis_dir): output = run_program( "calibration_pipeline", - "-c", + "--config", "cfg/sequencer.cfg", - "-d", - "2020_01_17", - "-s", - "--prod-id", - prod_id, - "--drs4-pedestal-run", - drs4_run_number, - "--pedcal-run", - pedcal_run_number, + "--date=2020_01_17", + "--simulate", + f"--prod-id={prod_id}", + f"--drs4-pedestal-run={drs4_run_number}", + f"--pedcal-run={pedcal_run_number}", "LST1", ) assert output.returncode == 0 @@ -253,11 +249,11 @@ def test_drs4_pedestal_cmd(base_test_dir): from osa.scripts.calibration_pipeline import drs4_pedestal_command cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804") expected_command = [ - 'onsite_create_drs4_pedestal_file', - '--run_number=01804', - f'--base_dir={base_test_dir}', - '--no-progress', - '--yes' + "onsite_create_drs4_pedestal_file", + "--run_number=01804", + f"--base_dir={base_test_dir}", + "--no-progress", + "--yes" ] assert cmd == expected_command @@ -266,35 +262,37 @@ def test_calibration_file_cmd(base_test_dir): from osa.scripts.calibration_pipeline import calibration_file_command cmd = calibration_file_command(pedcal_run_id="01805") expected_command = [ - 'onsite_create_calibration_file', - '--run_number=01805', - f'--base_dir={base_test_dir}', - '--yes', - '--filters=52' + "onsite_create_calibration_file", + "--run_number=01805", + f"--base_dir={base_test_dir}", + "--yes", + "--filters=52" ] assert cmd == expected_command -def test_drs4_pedestal(running_analysis_dir): - from osa.scripts.calibration_pipeline import drs4_pedestal - history_file = running_analysis_dir / "calibration_sequence.history" - with pytest.raises(SystemExit): - rc = drs4_pedestal( - drs4_pedestal_run_id="01804", - history_file=history_file - ) - assert rc != 0 - - -def test_calibrate_charge(running_analysis_dir): - from osa.scripts.calibration_pipeline import calibrate_charge - history_file = running_analysis_dir / "calibration_sequence.history" - with pytest.raises(SystemExit): - rc = calibrate_charge( - calibration_run="01805", - history_file=history_file - ) - assert rc != 0 +# def test_drs4_pedestal(running_analysis_dir): +# from osa.scripts.calibration_pipeline import drs4_pedestal +# history_file = running_analysis_dir / "calibration_sequence.history" +# with pytest.raises(SystemExit): +# rc = drs4_pedestal( +# drs4_pedestal_run_id="01804", +# pedcal_run_id="01805", +# history_file=history_file +# ) +# assert rc != 0 +# +# +# def test_calibrate_charge(running_analysis_dir): +# from osa.scripts.calibration_pipeline import calibrate_charge +# history_file = running_analysis_dir / "calibration_sequence.history" +# with pytest.raises(SystemExit): +# rc = calibrate_charge( +# drs4_pedestal_run_id="01804", +# pedcal_run_id="01805", +# history_file=history_file +# ) +# assert rc != 0 def test_look_for_datacheck_files( From dc59182a310f4f8beab50824c704f20193e47e5e Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Sat, 8 Jan 2022 12:23:47 +0100 Subject: [PATCH 21/33] add pedestal/pedcalib run ids as arguments for provprocess.py --- osa/job.py | 8 ++++---- osa/scripts/closer.py | 5 ++++- osa/scripts/simulate_processing.py | 4 ++++ osa/utils/cliopts.py | 8 ++++++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/osa/job.py b/osa/job.py index 24d9682d..21860b50 100644 --- a/osa/job.py +++ b/osa/job.py @@ -289,13 +289,13 @@ def sequence_calibration_filenames(sequence_list): if not sequence.parent_list: drs4_pedestal_run_id = sequence.previousrun - calibration_run_id = sequence.run + pedcal_run_id = sequence.run else: drs4_pedestal_run_id = sequence.parent_list[0].previousrun - calibration_run_id = sequence.parent_list[0].run + pedcal_run_id = sequence.parent_list[0].run drs4_pedestal_file = f"drs4_pedestal.Run{drs4_pedestal_run_id:05d}.0000.h5" - calibration_file = f"calibration_filters_52.Run{calibration_run_id:05d}.0000.h5" + calibration_file = f"calibration_filters_52.Run{pedcal_run_id:05d}.0000.h5" # Assign the calibration and drive files to the sequence object sequence.drive = drive_file @@ -305,7 +305,7 @@ def sequence_calibration_filenames(sequence_list): sequence.calibration = ( Path(cfg.get("LST1", "CALIB_DIR")) / nightdir / "pro" / calibration_file ) - sequence.time_calibration = get_time_calibration_file(calibration_run_id) + sequence.time_calibration = get_time_calibration_file(pedcal_run_id) def plot_job_statistics(sacct_output: pd.DataFrame, directory: Path): diff --git a/osa/scripts/closer.py b/osa/scripts/closer.py index 8198ce5d..34b92ed1 100644 --- a/osa/scripts/closer.py +++ b/osa/scripts/closer.py @@ -95,7 +95,6 @@ def main(): sys.exit(-1) store_conda_env_export() - post_process(sequencer_tuple) @@ -340,6 +339,8 @@ def extract_provenance(seq_list): for sequence in seq_list: if sequence.type == "DATA": + drs4_pedestal_run_id = str(sequence.pedestal).split(".")[1].replace("Run", "") + pedcal_run_id = str(sequence.calibration).split(".")[1].replace("Run", "") cmd = [ "sbatch", "-D", @@ -349,6 +350,8 @@ def extract_provenance(seq_list): "provprocess", "-c", options.configfile, + drs4_pedestal_run_id, + pedcal_run_id, f"{sequence.run:05d}", nightdir, options.prod_id, diff --git a/osa/scripts/simulate_processing.py b/osa/scripts/simulate_processing.py index 9984834f..2d39a599 100644 --- a/osa/scripts/simulate_processing.py +++ b/osa/scripts/simulate_processing.py @@ -170,6 +170,8 @@ def simulate_processing(): for subrun_idx in range(sub_list.subrun) ] processed = poolproc.map(simulate_subrun_processing, args_proc) + drs4_pedestal_run_id = str(sequence.pedestal).split(".")[1].replace("Run", "") + pedcal_run_id = str(sequence.calibration).split(".")[1].replace("Run", "") # produce prov if overwrite prov arg if processed and options.provenance: @@ -178,6 +180,8 @@ def simulate_processing(): command, "-c", options.configfile, + drs4_pedestal_run_id, + pedcal_run_id, sequence.run_str, options.directory, options.prod_id, diff --git a/osa/utils/cliopts.py b/osa/utils/cliopts.py index b89125e5..30d0341a 100644 --- a/osa/utils/cliopts.py +++ b/osa/utils/cliopts.py @@ -631,6 +631,12 @@ def provprocess_argparser(): dest="quit", help="use this flag to reset session and remove log file", ) + parser.add_argument( + "drs4_pedestal_run_id", help="Number of the drs4_pedestal used in the calibration" + ) + parser.add_argument( + "pedcal_run_id", help="Number of the used pedcal used in the calibration" + ) parser.add_argument( "run", help="Number of the run whose provenance is to be extracted" ) @@ -652,6 +658,8 @@ def provprocessparsing(): log.error("incorrect value for --filter argument, type -h for help") # set global variables + options.drs4_pedestal_run_id = opts.drs4_pedestal_run_id + options.pedcal_run_id = opts.pedcal_run_id options.run = opts.run options.date = opts.date options.prod_id = get_prod_id() From 7244a20b9355ace844040570296f8025ddd1c112 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Sat, 8 Jan 2022 14:09:40 +0100 Subject: [PATCH 22/33] make whole session starts with calibration --- osa/scripts/provprocess.py | 35 +++++++++++++++++++++++++++-------- osa/utils/cliopts.py | 4 ++-- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/osa/scripts/provprocess.py b/osa/scripts/provprocess.py index d53fc4e8..30d48675 100644 --- a/osa/scripts/provprocess.py +++ b/osa/scripts/provprocess.py @@ -62,13 +62,14 @@ def copy_used_file(src, outdir): log.warning(f"could not copy {src} file into {destpath}: {ex}") -def parse_lines_log(filter_step, run_number): +def parse_lines_log(filter_cut, calib_runs, run_number): """ Filter content in log file to produce a run/process wise session log. Parameters ---------- - filter_step + filter_cut + calib_runs run_number Returns @@ -76,6 +77,15 @@ def parse_lines_log(filter_step, run_number): filtered """ filtered = [] + if not filter_cut: + filter_cut = "all" + cuts = { + "calibration": ["drs4_pedestal", "calibrate_charge"], + "r0_to_dl1": ["r0_to_dl1", "dl1ab"], + "dl1_to_dl2": ["dl1_datacheck", "dl1_to_dl2"] + } + cuts["all"] = cuts["calibration"] + cuts["r0_to_dl1"] + cuts["dl1_to_dl2"] + with open(LOG_FILENAME, "r") as f: for line in f.readlines(): ll = line.split(PROV_PREFIX) @@ -90,20 +100,28 @@ def parse_lines_log(filter_step, run_number): session_tag = prov_dict.get("session_tag", "0:0") session_id = prov_dict.get("session_id", False) tag_activity, tag_run = session_tag.split(":") - # filter by run - if tag_run == run_number: + + # filter by run and calib runs + if tag_run in [run_number, calib_runs]: keep = True # filter by activity - if filter_step not in ["", tag_activity]: + if tag_activity not in cuts[filter_cut]: keep = False - # always keep first session start - if session_id and tag_run == run_number: + # only keep first session start + if session_id and (tag_run in [run_number, calib_runs]): keep = True + # make session starts with calibration + if session_id and filter_cut == "all" and not filtered: + prov_dict["session_id"] = f"{options.date}{run_number}" + prov_dict["name"] = f"{options.date}{run_number}" + prov_dict["observation_run"] = run_number + line = f"{ll[0]}{PROV_PREFIX}{ll[1]}{PROV_PREFIX}{prov_dict}\n" # remove parallel sessions if session_id and filtered: keep = False if keep: filtered.append(line) + return filtered @@ -438,7 +456,8 @@ def main(): session_log_filename = f"{base_filename}.log" # parse LOG_FILENAME content for a specific run / process - parsed_content = parse_lines_log(options.filter, options.run) + calib_runs = f"{options.drs4_pedestal_run_id}-{options.pedcal_run_id}" + parsed_content = parse_lines_log(options.filter, calib_runs, options.run) # create temporal session log file with open(session_log_filename, "w") as f: diff --git a/osa/utils/cliopts.py b/osa/utils/cliopts.py index 30d0341a..78056a62 100644 --- a/osa/utils/cliopts.py +++ b/osa/utils/cliopts.py @@ -623,7 +623,7 @@ def provprocess_argparser(): action="store", dest="filter", default="", - help="filter by process granularity [r0_to_dl1 or dl1_to_dl2]", + help="filter by process granularity [calibration, r0_to_dl1 or dl1_to_dl2]", ) parser.add_argument( "-q", @@ -654,7 +654,7 @@ def provprocessparsing(): opts = provprocess_argparser().parse_args() # checking arguments - if opts.filter not in ["r0_to_dl1", "dl1_to_dl2", ""]: + if opts.filter not in ["calibration", "r0_to_dl1", "dl1_to_dl2", ""]: log.error("incorrect value for --filter argument, type -h for help") # set global variables From d590109b06507209a875950d1fca227487fed7db Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Sat, 8 Jan 2022 20:15:24 +0100 Subject: [PATCH 23/33] add calibrationcheckplot and timecalibrationfile to model --- osa/provenance/config/definition.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/osa/provenance/config/definition.yaml b/osa/provenance/config/definition.yaml index b6bcf8e6..d3a69093 100644 --- a/osa/provenance/config/definition.yaml +++ b/osa/provenance/config/definition.yaml @@ -110,16 +110,16 @@ activities: entityName: CoefficientsCalibrationFile value: CoefficientsCalibrationFile # filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/calibration.Run06274.0000.hdf5 -# - role: "Check plot for calibration" -# description: "Calibration check plot" -# entityName: CalibrationCheckPlot -# value: CalibrationCheckPlot -## filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/log/calibration.Run06274.0000.pedestal.Run06268.0000.pdf -# - role: "Time calibration file" -# description: "Time calibration file" -# entityName: TimeCalibrationFile -# value: TimeCalibrationFile -## filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/time_calibration.Run06274.0000.hdf5 + - role: "Check plot for calibration" + description: "Calibration check plot" + entityName: CalibrationCheckPlot + value: CalibrationCheckPlot +# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/log/calibration.Run06274.0000.pedestal.Run06268.0000.pdf + - role: "Time calibration file" + description: "Time calibration file" + entityName: TimeCalibrationFile + value: TimeCalibrationFile +# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/time_calibration.Run06274.0000.hdf5 r0_to_dl1: description: From 1eca05d7406fc83f4fb82693a15b07903872347d Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Sat, 8 Jan 2022 20:16:19 +0100 Subject: [PATCH 24/33] use os.path.realpath when recording filepaths --- osa/provenance/utils.py | 77 ++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index 4332b37e..46bd3a73 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -70,24 +70,29 @@ def parse_variables(class_instance): # TODO - massive reprocessing vs. next day processing # according to code in onsite scripts in lstchain - # - class_instance.RawObservationFilePedestal = ( + class_instance.RawObservationFilePedestal = os.path.realpath( f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[0]}.fits.fz" ) - class_instance.RawObservationFileCalibration = ( + class_instance.RawObservationFileCalibration = os.path.realpath( f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[1]}.fits.fz" ) - pedestal_plot = f"drs4_pedestal.Run{class_instance.args[0]}.0000.pdf" - class_instance.PedestalCheckPlot = Path(pedestal_dir) / flat_date / pro / "log" / pedestal_plot - calibration_plot = f"calibration_filters_52.Run{class_instance.args[1]}.0000.pdf" - class_instance.CalibrationCheckPlot = Path(calib_dir) / flat_date / pro / "log" / calibration_plot + class_instance.PedestalCheckPlot = os.path.realpath( + f"{pedestal_dir}/{flat_date}/{pro}/log/drs4_pedestal.Run{class_instance.args[0]}.0000.pdf" + ) + class_instance.CalibrationCheckPlot = os.path.realpath( + f"{calib_dir}/{flat_date}/{pro}/log/calibration_filters_52.Run{class_instance.args[1]}.0000.pdf" + ) + # according to code in sequence_calibration_filenames from job.py - # - drs4_pedestal_file = f"drs4_pedestal.Run{class_instance.args[0]}.0000.h5" - class_instance.PedestalFile = Path(pedestal_dir) / flat_date / pro / drs4_pedestal_file - calibration_file = f"calibration_filters_52.Run{class_instance.args[1]}.0000.h5" - class_instance.CoefficientsCalibrationFile = Path(calib_dir) / flat_date / pro / calibration_file - class_instance.TimeCalibrationFile = get_time_calibration_file(int(class_instance.args[1])) + class_instance.PedestalFile = os.path.realpath( + f"{pedestal_dir}/{flat_date}/{pro}/drs4_pedestal.Run{class_instance.args[0]}.0000.h5" + ) + class_instance.CoefficientsCalibrationFile = os.path.realpath( + f"{calib_dir}/{flat_date}/{pro}/calibration_filters_52.Run{class_instance.args[1]}.0000.h5" + ) + class_instance.TimeCalibrationFile = os.path.realpath( + get_time_calibration_file(int(class_instance.args[1])) + ) if class_instance.__name__ == "r0_to_dl1": # calibrationfile [0] .../20200218/v00/calibration.Run02006.0000.hdf5 @@ -102,29 +107,29 @@ def parse_variables(class_instance): calibration_file = os.path.realpath(class_instance.args[0]) pedestal_file = os.path.realpath(class_instance.args[1]) timecalibration_file = os.path.realpath(class_instance.args[2]) - class_instance.R0SubrunDataset = ( + class_instance.R0SubrunDataset = os.path.realpath( f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[5]}.fits.fz" ) class_instance.CoefficientsCalibrationFile = calibration_file class_instance.PedestalFile = pedestal_file class_instance.TimeCalibrationFile = timecalibration_file - class_instance.PointingFile = str(class_instance.args[3]) - class_instance.RunSummaryFile = str(class_instance.args[4]) - class_instance.DL1SubrunDataset = ( + class_instance.PointingFile = os.path.realpath(class_instance.args[3]) + class_instance.RunSummaryFile = os.path.realpath(class_instance.args[4]) + class_instance.DL1SubrunDataset = os.path.realpath( f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[5]}.h5" ) - class_instance.MuonsSubrunDataset = ( + class_instance.MuonsSubrunDataset = os.path.realpath( f"{muon_dir}/muons_LST-1.Run{class_instance.args[5]}.fits" ) if class_instance.__name__ == "dl1ab": # run_str [0] 02006.0000 - class_instance.Analysisconfigfile_dl1 = configfile_dl1 + class_instance.Analysisconfigfile_dl1 = os.path.realpath(configfile_dl1) class_instance.ObservationRun = class_instance.args[0].split(".")[0] class_instance.PedestalCleaning = "True" class_instance.StoreImage = cfg.getboolean("lstchain", "store_image_dl1ab") - class_instance.DL1SubrunDataset = ( + class_instance.DL1SubrunDataset = os.path.realpath( f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5" ) @@ -132,19 +137,19 @@ def parse_variables(class_instance): # run_str [0] 02006.0000 class_instance.ObservationRun = class_instance.args[0].split(".")[0] - class_instance.DL1SubrunDataset = ( + class_instance.DL1SubrunDataset = os.path.realpath( f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5" ) - class_instance.MuonsSubrunDataset = ( + class_instance.MuonsSubrunDataset = os.path.realpath( f"{muon_dir}/muons_LST-1.Run{class_instance.args[0]}.fits" ) - class_instance.DL1CheckSubrunDataset = ( + class_instance.DL1CheckSubrunDataset = os.path.realpath( f"{outdir_dl1}/datacheck_dl1_LST-1.Run{class_instance.args[0]}.h5" ) - class_instance.DL1CheckHDF5File = ( + class_instance.DL1CheckHDF5File = os.path.realpath( f"{outdir_dl1}/datacheck_dl1_LST-1.Run{class_instance.ObservationRun}.h5" ) - class_instance.DL1CheckPDFFile = ( + class_instance.DL1CheckPDFFile = os.path.realpath( f"{outdir_dl1}/datacheck_dl1_LST-1.Run{class_instance.ObservationRun}.pdf" ) @@ -153,25 +158,25 @@ def parse_variables(class_instance): class_instance.Analysisconfigfile_dl2 = configfile_dl2 class_instance.ObservationRun = class_instance.args[0].split(".")[0] - class_instance.RFModelEnergyFile = str( - Path(rf_models_directory) / "reg_energy.sav" + class_instance.RFModelEnergyFile = os.path.realpath( + f"{rf_models_directory}/reg_energy.sav" ) - class_instance.RFModelDispNormFile = str( - Path(rf_models_directory) / "reg_disp_norm.sav" + class_instance.RFModelDispNormFile = os.path.realpath( + f"{rf_models_directory}/reg_disp_norm.sav" ) - class_instance.RFModelDispSignFile = str( - Path(rf_models_directory) / "reg_disp_sign.sav" + class_instance.RFModelDispSignFile = os.path.realpath( + f"{rf_models_directory}/reg_disp_sign.sav" ) - class_instance.RFModelGammanessFile = str( - Path(rf_models_directory) / "cls_gh.sav" + class_instance.RFModelGammanessFile = os.path.realpath( + f"{rf_models_directory}/cls_gh.sav" ) - class_instance.DL1SubrunDataset = ( + class_instance.DL1SubrunDataset = os.path.realpath( f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5" ) - class_instance.DL2SubrunDataset = ( + class_instance.DL2SubrunDataset = os.path.realpath( f"{outdir_dl2}/dl2_LST-1.Run{class_instance.args[0]}.h5" ) - class_instance.DL2MergedFile = ( + class_instance.DL2MergedFile = os.path.realpath( f"{outdir_dl2}/dl2_LST-1.Run{class_instance.ObservationRun}.h5" ) From ae3d595d3cfd8666c6d5b44ebb7a8623b1c7b63c Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Sat, 8 Jan 2022 20:16:45 +0100 Subject: [PATCH 25/33] use run number as the session name --- osa/scripts/provprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osa/scripts/provprocess.py b/osa/scripts/provprocess.py index 30d48675..8186836d 100644 --- a/osa/scripts/provprocess.py +++ b/osa/scripts/provprocess.py @@ -113,7 +113,7 @@ def parse_lines_log(filter_cut, calib_runs, run_number): # make session starts with calibration if session_id and filter_cut == "all" and not filtered: prov_dict["session_id"] = f"{options.date}{run_number}" - prov_dict["name"] = f"{options.date}{run_number}" + prov_dict["name"] = run_number prov_dict["observation_run"] = run_number line = f"{ll[0]}{PROV_PREFIX}{ll[1]}{PROV_PREFIX}{prov_dict}\n" # remove parallel sessions From 3976461c6f74c3c5779802766e45f89fc52efb77 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Sat, 8 Jan 2022 20:17:45 +0100 Subject: [PATCH 26/33] process calibration info --- osa/scripts/provprocess.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/osa/scripts/provprocess.py b/osa/scripts/provprocess.py index 8186836d..53577713 100644 --- a/osa/scripts/provprocess.py +++ b/osa/scripts/provprocess.py @@ -382,6 +382,27 @@ def produce_provenance(session_log_filename, base_filename): and graphs according to granularity. """ + if options.filter == "calibration" or not options.filter: + paths_calibration = define_paths( + "calibration_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename + ) + plines_drs4 = parse_lines_run( + "drs4_pedestal", + read_prov(filename=session_log_filename), + str(paths_calibration["out_path"]), + ) + plines_calib = parse_lines_run( + "calibrate_charge", + read_prov(filename=session_log_filename), + str(paths_calibration["out_path"]), + ) + calibration_lines = plines_drs4 + plines_calib[1:] + + # TODO + # create calibration prov files only if filtering + if options.filter == "calibration": + pass + if options.filter == "r0_to_dl1" or not options.filter: paths_r0_dl1 = define_paths( "r0_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename @@ -391,14 +412,15 @@ def produce_provenance(session_log_filename, base_filename): read_prov(filename=session_log_filename), str(paths_r0_dl1["out_path"]), ) - lines_r0_dl1 = copy.deepcopy(plines_r0) plines_ab = parse_lines_run( "dl1ab", read_prov(filename=session_log_filename), str(paths_r0_dl1["out_path"]), ) - lines_dl1ab = copy.deepcopy(plines_ab) - dl1_lines = lines_r0_dl1 + lines_dl1ab[1:] + dl1_lines = plines_r0 + plines_ab[1:] + + # create r0_to_dl1 prov files only if filtering + if options.filter == "r0_to_dl1": produce_provenance_files(plines_r0 + plines_ab[1:], paths_r0_dl1) if options.filter == "dl1_to_dl2" or not options.filter: @@ -410,20 +432,18 @@ def produce_provenance(session_log_filename, base_filename): read_prov(filename=session_log_filename), str(paths_dl1_dl2["out_path"]), ) - lines_check = copy.deepcopy(plines_check) plines_dl2 = parse_lines_run( "dl1_to_dl2", read_prov(filename=session_log_filename), str(paths_dl1_dl2["out_path"]), ) - lines_dl2 = copy.deepcopy(plines_dl2) - dl1_dl2_lines = lines_check + lines_dl2[1:] + dl1_dl2_lines = plines_check + plines_dl2[1:] - # create last step products only if filtering + # create dl1_to_dl2 prov files only if filtering if options.filter == "dl1_to_dl2": produce_provenance_files(plines_check + plines_dl2[1:], paths_dl1_dl2) - # create all steps products in last step path + # create calibration_to_dl1 and calibration_to_dl2 prov files if not options.filter: all_lines = dl1_lines + dl1_dl2_lines[1:] paths_r0_dl2 = define_paths( From 2d9c319385805dbe538c51ca7d4328e44cb9f5e7 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Sat, 8 Jan 2022 20:18:36 +0100 Subject: [PATCH 27/33] create only calibration_to_dl1 and calibration_to_dl2 prov files --- osa/scripts/provprocess.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/osa/scripts/provprocess.py b/osa/scripts/provprocess.py index 53577713..7598e605 100644 --- a/osa/scripts/provprocess.py +++ b/osa/scripts/provprocess.py @@ -445,11 +445,18 @@ def produce_provenance(session_log_filename, base_filename): # create calibration_to_dl1 and calibration_to_dl2 prov files if not options.filter: - all_lines = dl1_lines + dl1_dl2_lines[1:] - paths_r0_dl2 = define_paths( - "r0_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename + calibration_to_dl1 = define_paths( + "calibration_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename + ) + calibration_to_dl2 = define_paths( + "calibration_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename ) - produce_provenance_files(all_lines, paths_r0_dl2) + calibration_to_dl1_lines = calibration_lines + dl1_lines[1:] + lines_dl1 = copy.deepcopy(calibration_to_dl1_lines) + calibration_to_dl2_lines = calibration_to_dl1_lines + dl1_dl2_lines[1:] + lines_dl2 = copy.deepcopy(calibration_to_dl2_lines) + produce_provenance_files(lines_dl1, calibration_to_dl1) + produce_provenance_files(lines_dl2, calibration_to_dl2) def main(): From 4554fef987c894f9237bf80070c434b984d10249 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Sat, 8 Jan 2022 20:19:15 +0100 Subject: [PATCH 28/33] remove hash and has_type as properties of entities in graph --- osa/provenance/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osa/provenance/io.py b/osa/provenance/io.py index 7b9491ef..7a3d4e7b 100644 --- a/osa/provenance/io.py +++ b/osa/provenance/io.py @@ -186,7 +186,7 @@ def provlist2provdoc(provlist): records[progen_id] = progen ent.wasDerivedFrom(progen) for k, v in provdict.items(): - if k != "session_tag": + if k not in["session_tag", "hash", "hash_type"]: ent.add_attributes({k: str(v)}) # agent return pdoc From b767a60d7e40ac551ebb478f55403f8d2bfed6fe Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Sat, 8 Jan 2022 20:23:00 +0100 Subject: [PATCH 29/33] black scripts --- osa/provenance/capture.py | 4 +--- osa/provenance/utils.py | 8 ++------ osa/scripts/provprocess.py | 24 +++++++++--------------- osa/scripts/simulate_processing.py | 8 ++------ 4 files changed, 14 insertions(+), 30 deletions(-) diff --git a/osa/provenance/capture.py b/osa/provenance/capture.py index 1a9ff2dd..c3e04cf5 100644 --- a/osa/provenance/capture.py +++ b/osa/provenance/capture.py @@ -412,9 +412,7 @@ def get_derivation_records(class_instance, activity): log_record = {"entity_id": new_id, "progenitor_id": entity_id} records.append(log_record) traced_entities[var] = (new_id, item) - logger.warning( - f"Derivation detected in {activity} for {var}. ID: {new_id}" - ) + logger.warning(f"Derivation detected in {activity} for {var}. ID: {new_id}") return records diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index 46bd3a73..1a7f37e6 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -158,18 +158,14 @@ def parse_variables(class_instance): class_instance.Analysisconfigfile_dl2 = configfile_dl2 class_instance.ObservationRun = class_instance.args[0].split(".")[0] - class_instance.RFModelEnergyFile = os.path.realpath( - f"{rf_models_directory}/reg_energy.sav" - ) + class_instance.RFModelEnergyFile = os.path.realpath(f"{rf_models_directory}/reg_energy.sav") class_instance.RFModelDispNormFile = os.path.realpath( f"{rf_models_directory}/reg_disp_norm.sav" ) class_instance.RFModelDispSignFile = os.path.realpath( f"{rf_models_directory}/reg_disp_sign.sav" ) - class_instance.RFModelGammanessFile = os.path.realpath( - f"{rf_models_directory}/cls_gh.sav" - ) + class_instance.RFModelGammanessFile = os.path.realpath(f"{rf_models_directory}/cls_gh.sav") class_instance.DL1SubrunDataset = os.path.realpath( f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5" ) diff --git a/osa/scripts/provprocess.py b/osa/scripts/provprocess.py index 7598e605..d4009b75 100644 --- a/osa/scripts/provprocess.py +++ b/osa/scripts/provprocess.py @@ -82,7 +82,7 @@ def parse_lines_log(filter_cut, calib_runs, run_number): cuts = { "calibration": ["drs4_pedestal", "calibrate_charge"], "r0_to_dl1": ["r0_to_dl1", "dl1ab"], - "dl1_to_dl2": ["dl1_datacheck", "dl1_to_dl2"] + "dl1_to_dl2": ["dl1_datacheck", "dl1_to_dl2"], } cuts["all"] = cuts["calibration"] + cuts["r0_to_dl1"] + cuts["dl1_to_dl2"] @@ -90,9 +90,7 @@ def parse_lines_log(filter_cut, calib_runs, run_number): for line in f.readlines(): ll = line.split(PROV_PREFIX) if len(ll) != 3: - log.warning( - f"format {PROV_PREFIX} mismatch in log file {LOG_FILENAME}\n{line}" - ) + log.warning(f"format {PROV_PREFIX} mismatch in log file {LOG_FILENAME}\n{line}") continue prov_str = ll.pop() prov_dict = yaml.safe_load(prov_str) @@ -227,11 +225,11 @@ def parse_lines_run(filter_step, prov_lines, out): # copy used files not subruns not RFs not mergedDL2 if ( - filepath - and content_type != "application/x-spss-sav" - and name != "DL2MergedFile" - and not name.startswith("DL1Check") - and not remove + filepath + and content_type != "application/x-spss-sav" + and name != "DL2MergedFile" + and not name.startswith("DL1Check") + and not remove ): copy_used_file(filepath, out) if session_id and osa_cfg and not osa_config_copied: @@ -404,9 +402,7 @@ def produce_provenance(session_log_filename, base_filename): pass if options.filter == "r0_to_dl1" or not options.filter: - paths_r0_dl1 = define_paths( - "r0_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename - ) + paths_r0_dl1 = define_paths("r0_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename) plines_r0 = parse_lines_run( "r0_to_dl1", read_prov(filename=session_log_filename), @@ -424,9 +420,7 @@ def produce_provenance(session_log_filename, base_filename): produce_provenance_files(plines_r0 + plines_ab[1:], paths_r0_dl1) if options.filter == "dl1_to_dl2" or not options.filter: - paths_dl1_dl2 = define_paths( - "dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename - ) + paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename) plines_check = parse_lines_run( "dl1_datacheck", read_prov(filename=session_log_filename), diff --git a/osa/scripts/simulate_processing.py b/osa/scripts/simulate_processing.py index 2d39a599..cee6a2b3 100644 --- a/osa/scripts/simulate_processing.py +++ b/osa/scripts/simulate_processing.py @@ -74,12 +74,8 @@ def do_setup(): CONFIG_FLAGS["TearSubDL2"] = ( False if path_dl2_sub.exists() or options.provenance else path_dl2_sub ) - CONFIG_FLAGS["TearDL1"] = ( - False if path_dl1.exists() or options.provenance else path_dl1 - ) - CONFIG_FLAGS["TearDL2"] = ( - False if path_dl2.exists() or options.provenance else path_dl2 - ) + CONFIG_FLAGS["TearDL1"] = False if path_dl1.exists() or options.provenance else path_dl1 + CONFIG_FLAGS["TearDL2"] = False if path_dl2.exists() or options.provenance else path_dl2 if options.provenance and not options.force: if path_sub_analysis.exists(): From 742a9ee2b865db9459641ff9685e3f12c841d3ae Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Sat, 8 Jan 2022 20:27:19 +0100 Subject: [PATCH 30/33] fix tests --- osa/scripts/tests/test_osa_scripts.py | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/osa/scripts/tests/test_osa_scripts.py b/osa/scripts/tests/test_osa_scripts.py index 1c2e982c..0e663069 100644 --- a/osa/scripts/tests/test_osa_scripts.py +++ b/osa/scripts/tests/test_osa_scripts.py @@ -65,12 +65,12 @@ def test_simulate_processing(drs4_time_calibration_files, run_summary_file): prov_dl1_path = Path("./test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/log") prov_dl2_path = Path("./test_osa/test_files0/DL2/20200117/v0.1.0/tailcut84_model1/log") - prov_file_dl1 = prov_dl1_path / "r0_to_dl1_01807_prov.log" - prov_file_dl2 = prov_dl2_path / "r0_to_dl2_01807_prov.log" - json_file_dl1 = prov_dl1_path / "r0_to_dl1_01807_prov.json" - json_file_dl2 = prov_dl2_path / "r0_to_dl2_01807_prov.json" - pdf_file_dl1 = prov_dl1_path / "r0_to_dl1_01807_prov.pdf" - pdf_file_dl2 = prov_dl2_path / "r0_to_dl2_01807_prov.pdf" + prov_file_dl1 = prov_dl1_path / "calibration_to_dl1_01807_prov.log" + prov_file_dl2 = prov_dl2_path / "calibration_to_dl2_01807_prov.log" + json_file_dl1 = prov_dl1_path / "calibration_to_dl1_01807_prov.json" + json_file_dl2 = prov_dl2_path / "calibration_to_dl2_01807_prov.json" + pdf_file_dl1 = prov_dl1_path / "calibration_to_dl1_01807_prov.pdf" + pdf_file_dl2 = prov_dl2_path / "calibration_to_dl2_01807_prov.pdf" assert prov_file_dl1.exists() assert prov_file_dl2.exists() @@ -79,17 +79,17 @@ def test_simulate_processing(drs4_time_calibration_files, run_summary_file): with open(json_file_dl1) as file: dl1 = yaml.safe_load(file) - assert len(dl1["entity"]) == 11 - assert len(dl1["activity"]) == 2 - assert len(dl1["used"]) == 9 - assert len(dl1["wasGeneratedBy"]) == 3 + assert len(dl1["entity"]) == 15 + assert len(dl1["activity"]) == 4 + assert len(dl1["used"]) == 12 + assert len(dl1["wasGeneratedBy"]) == 8 with open(json_file_dl2) as file: dl2 = yaml.safe_load(file) - assert len(dl2["entity"]) == 20 - assert len(dl2["activity"]) == 4 - assert len(dl2["used"]) == 17 - assert len(dl2["wasGeneratedBy"]) == 8 + assert len(dl2["entity"]) == 24 + assert len(dl2["activity"]) == 6 + assert len(dl2["used"]) == 20 + assert len(dl2["wasGeneratedBy"]) == 13 rc = run_program("simulate_processing", "-p") assert rc.returncode == 0 From 14a00380d0f6f2d93cbda416f47515bae50e31c2 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Mon, 10 Jan 2022 12:17:42 +0100 Subject: [PATCH 31/33] fix codacy issues --- osa/provenance/capture.py | 16 +++++++++++---- osa/provenance/utils.py | 8 ++++++-- osa/scripts/calibration_pipeline.py | 32 ++++++++++++++++------------- osa/scripts/provprocess.py | 12 ++++++++--- osa/scripts/simulate_processing.py | 23 +++++++++++++-------- 5 files changed, 60 insertions(+), 31 deletions(-) diff --git a/osa/provenance/capture.py b/osa/provenance/capture.py index c3e04cf5..f2a63f13 100644 --- a/osa/provenance/capture.py +++ b/osa/provenance/capture.py @@ -102,7 +102,9 @@ def wrapper(*args, **kwargs): session_tag = f"{activity}:{class_instance.ObservationRun}" session_name = f"{class_instance.ObservationRun}" else: - session_tag = f"{activity}:{class_instance.PedestalRun}-{class_instance.CalibrationRun}" + session_tag = ( + f"{activity}:{class_instance.PedestalRun}-{class_instance.CalibrationRun}" + ) session_name = f"{class_instance.PedestalRun}-{class_instance.CalibrationRun}" # OSA specific # variables parsing @@ -369,11 +371,15 @@ def log_session(class_instance, start): # OSA specific "software_version": class_instance.SoftwareVersion, "config_file": class_instance.ProcessingConfigFile, - "config_file_hash": get_file_hash(class_instance.ProcessingConfigFile, buffer="path"), + "config_file_hash": get_file_hash( + class_instance.ProcessingConfigFile, buffer="path" + ), "config_file_hash_type": get_hash_method(), } if class_instance.__name__ in REDUCTION_TASKS: - log_record["observation_run"] = class_instance.ObservationRun # a session is run-wise + log_record[ + "observation_run" + ] = class_instance.ObservationRun # a session is run-wise else: log_record["pedestal_run"] = class_instance.PedestalRun log_record["calibration_run"] = class_instance.CalibrationRun @@ -412,7 +418,9 @@ def get_derivation_records(class_instance, activity): log_record = {"entity_id": new_id, "progenitor_id": entity_id} records.append(log_record) traced_entities[var] = (new_id, item) - logger.warning(f"Derivation detected in {activity} for {var}. ID: {new_id}") + logger.warning( + f"Derivation detected in {activity} for {var}. ID: {new_id}" + ) return records diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index 1a7f37e6..46bd3a73 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -158,14 +158,18 @@ def parse_variables(class_instance): class_instance.Analysisconfigfile_dl2 = configfile_dl2 class_instance.ObservationRun = class_instance.args[0].split(".")[0] - class_instance.RFModelEnergyFile = os.path.realpath(f"{rf_models_directory}/reg_energy.sav") + class_instance.RFModelEnergyFile = os.path.realpath( + f"{rf_models_directory}/reg_energy.sav" + ) class_instance.RFModelDispNormFile = os.path.realpath( f"{rf_models_directory}/reg_disp_norm.sav" ) class_instance.RFModelDispSignFile = os.path.realpath( f"{rf_models_directory}/reg_disp_sign.sav" ) - class_instance.RFModelGammanessFile = os.path.realpath(f"{rf_models_directory}/cls_gh.sav") + class_instance.RFModelGammanessFile = os.path.realpath( + f"{rf_models_directory}/cls_gh.sav" + ) class_instance.DL1SubrunDataset = os.path.realpath( f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5" ) diff --git a/osa/scripts/calibration_pipeline.py b/osa/scripts/calibration_pipeline.py index 8c3597fd..dd14a7dc 100644 --- a/osa/scripts/calibration_pipeline.py +++ b/osa/scripts/calibration_pipeline.py @@ -35,11 +35,11 @@ def drs4_pedestal_command(drs4_pedestal_run_id: str) -> list: """Build the create_drs4_pedestal command.""" base_dir = Path(cfg.get("LST1", "BASE")).resolve() return [ - 'onsite_create_drs4_pedestal_file', - f'--run_number={drs4_pedestal_run_id}', - f'--base_dir={base_dir}', - '--no-progress', - '--yes' + "onsite_create_drs4_pedestal_file", + f"--run_number={drs4_pedestal_run_id}", + f"--base_dir={base_dir}", + "--no-progress", + "--yes", ] @@ -47,11 +47,11 @@ def calibration_file_command(pedcal_run_id: str) -> list: """Build the create_calibration_file command.""" base_dir = Path(cfg.get("LST1", "BASE")).resolve() return [ - 'onsite_create_calibration_file', - f'--run_number={pedcal_run_id}', - f'--base_dir={base_dir}', - '--yes', - '--filters=52' + "onsite_create_calibration_file", + f"--run_number={pedcal_run_id}", + f"--base_dir={base_dir}", + "--yes", + "--filters=52", ] @@ -92,7 +92,9 @@ def calibration_sequence(drs4_pedestal_run_id: str, pedcal_run_id: str) -> int: @trace -def drs4_pedestal(drs4_pedestal_run_id: str, pedcal_run_id: str, history_file: Path) -> int: +def drs4_pedestal( + drs4_pedestal_run_id: str, pedcal_run_id: str, history_file: Path +) -> int: """ Create a DRS4 pedestal file for baseline correction. @@ -120,12 +122,14 @@ def drs4_pedestal(drs4_pedestal_run_id: str, pedcal_run_id: str, history_file: P history_file=history_file, run=drs4_pedestal_run_id, prod_id=options.calib_prod_id, - command=cmd[0] + command=cmd[0], ) @trace -def calibrate_charge(drs4_pedestal_run_id: str, pedcal_run_id: str, history_file: Path) -> int: +def calibrate_charge( + drs4_pedestal_run_id: str, pedcal_run_id: str, history_file: Path +) -> int: """ Create the calibration file to transform from ADC counts to photo-electrons @@ -153,7 +157,7 @@ def calibrate_charge(drs4_pedestal_run_id: str, pedcal_run_id: str, history_file history_file=history_file, run=pedcal_run_id, prod_id=options.calib_prod_id, - command=cmd[0] + command=cmd[0], ) diff --git a/osa/scripts/provprocess.py b/osa/scripts/provprocess.py index d4009b75..f71b7347 100644 --- a/osa/scripts/provprocess.py +++ b/osa/scripts/provprocess.py @@ -90,7 +90,9 @@ def parse_lines_log(filter_cut, calib_runs, run_number): for line in f.readlines(): ll = line.split(PROV_PREFIX) if len(ll) != 3: - log.warning(f"format {PROV_PREFIX} mismatch in log file {LOG_FILENAME}\n{line}") + log.warning( + f"format {PROV_PREFIX} mismatch in log file {LOG_FILENAME}\n{line}" + ) continue prov_str = ll.pop() prov_dict = yaml.safe_load(prov_str) @@ -402,7 +404,9 @@ def produce_provenance(session_log_filename, base_filename): pass if options.filter == "r0_to_dl1" or not options.filter: - paths_r0_dl1 = define_paths("r0_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename) + paths_r0_dl1 = define_paths( + "r0_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename + ) plines_r0 = parse_lines_run( "r0_to_dl1", read_prov(filename=session_log_filename), @@ -420,7 +424,9 @@ def produce_provenance(session_log_filename, base_filename): produce_provenance_files(plines_r0 + plines_ab[1:], paths_r0_dl1) if options.filter == "dl1_to_dl2" or not options.filter: - paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename) + paths_dl1_dl2 = define_paths( + "dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename + ) plines_check = parse_lines_run( "dl1_datacheck", read_prov(filename=session_log_filename), diff --git a/osa/scripts/simulate_processing.py b/osa/scripts/simulate_processing.py index cee6a2b3..2e64985e 100644 --- a/osa/scripts/simulate_processing.py +++ b/osa/scripts/simulate_processing.py @@ -1,10 +1,9 @@ -""" -Simulate executions of data processing pipeline and produce provenance. +"""Simulate executions of data processing pipeline and produce provenance. + If it is not executed by tests, please run pytest --basetemp=test_osa first. It needs to have test_osa folder filled with test datasets. -python osa/scripts/simulate_processing.py -""" +python osa/scripts/simulate_processing.py""" import logging import multiprocessing as mp @@ -74,8 +73,12 @@ def do_setup(): CONFIG_FLAGS["TearSubDL2"] = ( False if path_dl2_sub.exists() or options.provenance else path_dl2_sub ) - CONFIG_FLAGS["TearDL1"] = False if path_dl1.exists() or options.provenance else path_dl1 - CONFIG_FLAGS["TearDL2"] = False if path_dl2.exists() or options.provenance else path_dl2 + CONFIG_FLAGS["TearDL1"] = ( + False if path_dl1.exists() or options.provenance else path_dl1 + ) + CONFIG_FLAGS["TearDL2"] = ( + False if path_dl2.exists() or options.provenance else path_dl2 + ) if options.provenance and not options.force: if path_sub_analysis.exists(): @@ -157,12 +160,16 @@ def simulate_processing(): processed = False for sub_list in sequence.subrun_list: if sub_list.runobj.type == "PEDCALIB": - args_cal = parse_template(create_job_template(sequence, get_content=True), 0) + args_cal = parse_template( + create_job_template(sequence, get_content=True), 0 + ) simulate_calibration(args_cal) elif sub_list.runobj.type == "DATA": with mp.Pool() as poolproc: args_proc = [ - parse_template(create_job_template(sequence, get_content=True), subrun_idx) + parse_template( + create_job_template(sequence, get_content=True), subrun_idx + ) for subrun_idx in range(sub_list.subrun) ] processed = poolproc.map(simulate_subrun_processing, args_proc) From 4f9dedfbf9d23cc907c7e43d7b870028c68adaf4 Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Tue, 11 Jan 2022 11:43:46 +0100 Subject: [PATCH 32/33] remove time_calib file since not produced in calibrate_charge --- osa/provenance/config/definition.yaml | 11 +++++------ osa/provenance/utils.py | 4 ---- osa/scripts/tests/test_osa_scripts.py | 4 ++-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/osa/provenance/config/definition.yaml b/osa/provenance/config/definition.yaml index d3a69093..f6d7ea3a 100644 --- a/osa/provenance/config/definition.yaml +++ b/osa/provenance/config/definition.yaml @@ -104,6 +104,11 @@ activities: # entityName: SystematicsCorrectionFile # value: SystematicsCorrectionFile # filepath: /path/to/ff_systematics_file.h5 +# - role: "Time calibration file" +# description: "Time calibration file" +# entityName: TimeCalibrationFile +# value: TimeCalibrationFile +# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/time_calibration.Run06274.0000.hdf5 generation: - role: "Coefficients calibration file" description: "Coefficients calibration file" @@ -115,12 +120,6 @@ activities: entityName: CalibrationCheckPlot value: CalibrationCheckPlot # filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/log/calibration.Run06274.0000.pedestal.Run06268.0000.pdf - - role: "Time calibration file" - description: "Time calibration file" - entityName: TimeCalibrationFile - value: TimeCalibrationFile -# filepath: /fefs/aswg/data/real/calibration/20210913/v0.7.5/time_calibration.Run06274.0000.hdf5 - r0_to_dl1: description: "Create DL1 files for an observation run and subrun" diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index 46bd3a73..16c8e17b 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -9,7 +9,6 @@ from osa.configs import options from osa.configs.config import cfg -from osa.job import get_time_calibration_file from osa.utils.utils import get_lstchain_version, lstdate_to_dir __all__ = ["parse_variables", "get_log_config", "store_conda_env_export"] @@ -90,9 +89,6 @@ def parse_variables(class_instance): class_instance.CoefficientsCalibrationFile = os.path.realpath( f"{calib_dir}/{flat_date}/{pro}/calibration_filters_52.Run{class_instance.args[1]}.0000.h5" ) - class_instance.TimeCalibrationFile = os.path.realpath( - get_time_calibration_file(int(class_instance.args[1])) - ) if class_instance.__name__ == "r0_to_dl1": # calibrationfile [0] .../20200218/v00/calibration.Run02006.0000.hdf5 diff --git a/osa/scripts/tests/test_osa_scripts.py b/osa/scripts/tests/test_osa_scripts.py index 0e663069..3009a927 100644 --- a/osa/scripts/tests/test_osa_scripts.py +++ b/osa/scripts/tests/test_osa_scripts.py @@ -82,14 +82,14 @@ def test_simulate_processing(drs4_time_calibration_files, run_summary_file): assert len(dl1["entity"]) == 15 assert len(dl1["activity"]) == 4 assert len(dl1["used"]) == 12 - assert len(dl1["wasGeneratedBy"]) == 8 + assert len(dl1["wasGeneratedBy"]) == 7 with open(json_file_dl2) as file: dl2 = yaml.safe_load(file) assert len(dl2["entity"]) == 24 assert len(dl2["activity"]) == 6 assert len(dl2["used"]) == 20 - assert len(dl2["wasGeneratedBy"]) == 13 + assert len(dl2["wasGeneratedBy"]) == 12 rc = run_program("simulate_processing", "-p") assert rc.returncode == 0 From 217fc1719bb40736a091e09e3ce1a45293e3c5dc Mon Sep 17 00:00:00 2001 From: Jose Enrique Ruiz Date: Wed, 12 Jan 2022 10:27:58 +0100 Subject: [PATCH 33/33] pedestal file in HDF5 format --- osa/provenance/config/definition.yaml | 4 ++-- osa/provenance/utils.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/osa/provenance/config/definition.yaml b/osa/provenance/config/definition.yaml index f6d7ea3a..10f0d886 100644 --- a/osa/provenance/config/definition.yaml +++ b/osa/provenance/config/definition.yaml @@ -365,9 +365,9 @@ entities: type: File contentType: text/plain PedestalFile: - description: "Pedestal file in FITS format on the disk" + description: "Pedestal file in HDF5 format on the disk" type: File - contentType: application/fits + contentType: application/x-hdf PedestalCheckPlot: description: "Pedestal check plot PDF file" type: File diff --git a/osa/provenance/utils.py b/osa/provenance/utils.py index 16c8e17b..dc65814a 100644 --- a/osa/provenance/utils.py +++ b/osa/provenance/utils.py @@ -30,11 +30,11 @@ def parse_variables(class_instance): # -c cfg/sequencer.cfg # -d 2020_02_18 # --prod-id v0.4.3_v00 - # --pedcal-file ../calibration/20200218/v00/calibration.Run02006.0000.hdf5 - # --drs4-pedestal-file ../calibration/20200218/v00/drs4_pedestal.Run02005.0000.fits - # --time-calib-file ../calibration/20191124/v00/time_calibration.Run01625.0000.hdf5 - # --drive-file ../lapp/DrivePositioning/drive_log_20_02_18.txt - # --run-summary ../monitoring/RunSummary/RunSummary_20200101.ecsv + # --pedcal-file .../20200218/pro/calibration_filters_52.Run02006.0000.h5 + # --drs4-pedestal-file .../20200218/pro/drs4_pedestal.Run02005.0000.h5 + # --time-calib-file .../20191124/pro/time_calibration.Run01625.0000.h5 + # --drive-file .../lapp/DrivePositioning/drive_log_20_02_18.txt + # --run-summary .../monitoring/RunSummary/RunSummary_20200101.ecsv # 02006.0000 # LST1 @@ -91,9 +91,9 @@ def parse_variables(class_instance): ) if class_instance.__name__ == "r0_to_dl1": - # calibrationfile [0] .../20200218/v00/calibration.Run02006.0000.hdf5 - # pedestalfile [1] .../20200218/v00/drs4_pedestal.Run02005.0000.fits - # timecalibfile [2] .../20191124/v00/time_calibration.Run01625.0000.hdf5 + # calibrationfile [0] .../20200218/pro/calibration_filters_52.Run02006.0000.h5 + # pedestalfile [1] .../20200218/pro/drs4_pedestal.Run02005.0000.h5 + # timecalibfile [2] .../20191124/pro/time_calibration.Run01625.0000.h5 # drivefile [3] .../DrivePositioning/drive_log_20_02_18.txt # runsummaryfile [4] .../RunSummary/RunSummary_20200101.ecsv # run_str [5] 02006.0000