From 213e94fd9407c03abf987e6e6d4cad057dd40f37 Mon Sep 17 00:00:00 2001 From: BastienLacave Date: Thu, 13 Jun 2024 13:59:22 +0200 Subject: [PATCH 01/23] Real data processing --- ctlearn/utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ctlearn/utils.py b/ctlearn/utils.py index dfab096a..6592c873 100644 --- a/ctlearn/utils.py +++ b/ctlearn/utils.py @@ -105,7 +105,8 @@ def setup_DL1DataReader(config, mode): "Data format is not implemented in the DL1DH reader. Available data formats are 'stage1' and 'dl1dh'." ) # Check weather the file is MC simulation or real observational data - if data_format == "dl1dh" and "source_name" in f.root._v_attrs: + if f.root._v_attrs['CTA PROCESS TYPE'] == 'Observation': + # if data_format == "dl1dh" and "source_name" in f.root._v_attrs: mc_file = False # Retrieve the name convention for the dl1b parameters if data_format == "dl1dh": @@ -175,9 +176,10 @@ def setup_DL1DataReader(config, mode): ): config["Data"]["parameter_settings"] = {"parameter_list": dl1bparameter_names} if "direction" in tasks or mode == "predict": - event_info.append("true_alt") - event_info.append("true_az") - transformations.append({"name": "DeltaAltAz_fix_subarray"}) + if mc_file: + event_info.append("true_alt") + event_info.append("true_az") + transformations.append({"name": "DeltaAltAz_fix_subarray"}) if "cherenkov_photons" in tasks: if "trigger_settings" in config["Data"]: config["Data"]["trigger_settings"]["reco_cherenkov_photons"] = True @@ -187,12 +189,13 @@ def setup_DL1DataReader(config, mode): ) if "type" in tasks or mode == "predict": - event_info.append("true_shower_primary_id") + if mc_file: + event_info.append("true_shower_primary_id") if "energy" in tasks or mode == "predict": if mc_file: event_info.append("true_energy") - transformations.append({"name": "MCEnergy"}) + transformations.append({"name": "MCEnergy"}) stack_telescope_images = config["Input"].get("stack_telescope_images", False) if config["Data"]["mode"] == "stereo" and not stack_telescope_images: From 3139f30ab562b67610cf0a32ca5b40317d66cfbb Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Thu, 13 Jun 2024 14:19:18 +0200 Subject: [PATCH 02/23] remove dl1dh reader --- .gitignore | 5 +- ctlearn/data_loader.py | 31 +++--------- ctlearn/output_handler.py | 64 ------------------------ ctlearn/run_model.py | 10 ++-- ctlearn/utils.py | 102 ++++++++++---------------------------- 5 files changed, 39 insertions(+), 173 deletions(-) diff --git a/.gitignore b/.gitignore index 9693f6b5..3f7d4240 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,9 @@ # Compiled Python files __pycache__/ *.py[cod] - +.DS_Store +ctlearn/.DS_Store ctlearn.egg-info/ # Sphinx documentation -docs/build/ \ No newline at end of file +docs/build/ diff --git a/ctlearn/data_loader.py b/ctlearn/data_loader.py index f5021d8f..410203f0 100644 --- a/ctlearn/data_loader.py +++ b/ctlearn/data_loader.py @@ -7,7 +7,7 @@ class KerasBatchGenerator(tf.keras.utils.Sequence): def __init__( self, - DL1DataReaderDL1DH, + DLDataReader, indices, batch_size=64, mode="train", @@ -16,7 +16,7 @@ def __init__( stack_telescope_images=False, ): "Initialization" - self.DL1DataReaderDL1DH = DL1DataReaderDL1DH + self.DLDataReader = DLDataReader self.batch_size = batch_size self.indices = indices self.mode = mode @@ -39,8 +39,6 @@ def __init__( # Additional info self.evt_pos, self.obs_pos = None, None self.event_list, self.obs_list = [], [] - self.mjd_pos, self.milli_pos, self.nano_pos = None, None, None - self.mjd_list, self.milli_list, self.nano_list = [], [], [] # Labels self.prt_pos, self.enr_pos, self.drc_pos = None, None, None self.prt_labels = [] @@ -49,7 +47,7 @@ def __init__( self.trgpatch_labels = [] self.energy_unit = None - for i, desc in enumerate(self.DL1DataReaderDL1DH.example_description): + for i, desc in enumerate(self.DLDataReader.example_description): if "HWtrigger" in desc["name"]: self.trg_pos = i self.trg_shape = desc["shape"] @@ -78,12 +76,6 @@ def __init__( self.evt_pos = i elif "obs_id" in desc["name"]: self.obs_pos = i - elif "mjd" in desc["name"]: - self.mjd_pos = i - elif "milli_sec" in desc["name"]: - self.milli_pos = i - elif "nano_sec" in desc["name"]: - self.nano_pos = i # Retrieve shape from a single image in stereo analysis if self.trg_pos is not None and self.img_pos is not None: @@ -139,7 +131,7 @@ def __data_generation(self, batch_indices): ) # Generate data for i, index in enumerate(batch_indices): - event = self.DL1DataReaderDL1DH[index] + event = self.DLDataReader[index] # Fill the features if self.trg_pos is not None: triggers[i] = event[self.trg_pos] @@ -155,7 +147,7 @@ def __data_generation(self, batch_indices): if self.prt_pos is not None: particletype[ i - ] = self.DL1DataReaderDL1DH.shower_primary_id_to_class[ + ] = self.DLDataReader.shower_primary_id_to_class[ int(event[self.prt_pos]) ] if self.enr_pos is not None: @@ -186,13 +178,6 @@ def __data_generation(self, batch_indices): self.event_list.append(np.float32(event[self.evt_pos])) if self.obs_pos is not None: self.obs_list.append(np.float32(event[self.obs_pos])) - # Save timestamp - if self.mjd_pos is not None: - self.mjd_list.append(np.float32(event[self.mjd_pos])) - if self.milli_pos is not None: - self.milli_list.append(np.float32(event[self.milli_pos])) - if self.nano_pos is not None: - self.nano_list.append(np.float32(event[self.nano_pos])) features = {} if self.trg_pos is not None: @@ -209,11 +194,11 @@ def __data_generation(self, batch_indices): if self.prt_pos is not None: labels["type"] = tf.keras.utils.to_categorical( particletype, - num_classes=self.DL1DataReaderDL1DH.num_classes, + num_classes=self.DLDataReader.num_classes, ) label = tf.keras.utils.to_categorical( particletype, - num_classes=self.DL1DataReaderDL1DH.num_classes, + num_classes=self.DLDataReader.num_classes, ) if self.enr_pos is not None: labels["energy"] = energy.reshape((-1, 1)) @@ -221,7 +206,7 @@ def __data_generation(self, batch_indices): if self.drc_pos is not None: labels["direction"] = direction label = direction - if self.trgpatch_pos is not None and self.DL1DataReaderDL1DH.reco_cherenkov_photons: + if self.trgpatch_pos is not None and self.DLDataReader.reco_cherenkov_photons: labels["cherenkov_photons"] = trigger_patches_true_image_sum label = trigger_patches_true_image_sum diff --git a/ctlearn/output_handler.py b/ctlearn/output_handler.py index a8d95649..75330e71 100644 --- a/ctlearn/output_handler.py +++ b/ctlearn/output_handler.py @@ -9,35 +9,6 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): if not os.path.exists(prediction_dir): os.makedirs(prediction_dir) - # Store the information for observational data - if reader.instrument_id == "MAGIC" and reader.process_type == "Observation": - # Dump the information of the run to hdf5 file - run_info = {} - run_info["run_number"] = reader._v_attrs["run_number"] - run_info["magic_number"] = reader._v_attrs["magic_number"] - run_info["num_events"] = reader._v_attrs["num_events"] - run_info["run_start_mjd"] = reader._v_attrs["run_start_mjd"] - run_info["run_start_ms"] = reader._v_attrs["run_start_ms"] - run_info["run_start_ns"] = reader._v_attrs["run_start_ns"] - run_info["run_stop_mjd"] = reader._v_attrs["run_stop_mjd"] - run_info["run_stop_ms"] = reader._v_attrs["run_stop_ms"] - run_info["run_stop_ns"] = reader._v_attrs["run_stop_ns"] - pd.DataFrame(data=run_info, index=[0]).to_hdf( - h5file, key=f"/info/run", mode="a" - ) - # Dump the information of the obsveration to hdf5 file - obs_info = {} - obs_info["source_name"] = reader._v_attrs["source_name"] - obs_info["project_name"] = reader._v_attrs["project_name"] - obs_info["observation_mode"] = reader._v_attrs["observation_mode"] - obs_info["source_dec"] = reader._v_attrs["source_dec"] - obs_info["source_ra"] = reader._v_attrs["source_ra"] - obs_info["telescope_dec"] = reader._v_attrs["telescope_dec"] - obs_info["telescope_ra"] = reader._v_attrs["telescope_ra"] - pd.DataFrame(data=obs_info, index=[0]).to_hdf( - h5file, key=f"/info/obs", mode="a" - ) - # Store dl2 data reco = {} if os.path.isfile(h5file): @@ -70,41 +41,6 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): ) reco["obs_id"] = obs_id - # Store the timestamp - if data.mjd_pos: - mjd = data.mjd_list[data.batch_size :] - if rest_data: - mjd = np.concatenate( - ( - mjd, - rest_data.mjd_list[rest_data.batch_size :], - ), - axis=0, - ) - reco["mjd"] = mjd - if data.milli_pos: - milli_sec = data.milli_list[data.batch_size :] - if rest_data: - milli_sec = np.concatenate( - ( - milli_sec, - rest_data.milli_list[rest_data.batch_size :], - ), - axis=0, - ) - reco["milli_sec"] = milli_sec - if data.nano_pos: - nano_sec = data.nano_list[data.batch_size :] - if rest_data: - nano_sec = np.concatenate( - ( - nano_sec, - rest_data.nano_list[rest_data.batch_size :], - ), - axis=0, - ) - reco["nano_sec"] = nano_sec - # Store pointings if data.pon_pos: pointing_alt = np.array(data.pointing)[data.batch_size :, 0] diff --git a/ctlearn/run_model.py b/ctlearn/run_model.py index eee46ca0..66167238 100644 --- a/ctlearn/run_model.py +++ b/ctlearn/run_model.py @@ -17,7 +17,7 @@ import tensorflow as tf from tensorflow.python import debug as tf_debug -from dl1_data_handler.reader import DL1DataReaderSTAGE1, DL1DataReaderDL1DH +from dl1_data_handler.reader import DLDataReader from ctlearn.data_loader import KerasBatchGenerator from ctlearn.output_handler import * from ctlearn.utils import * @@ -54,7 +54,7 @@ def run_model(config, mode="train", debug=False, log_to_file=False): class_names = [name[0] for name in class_names] # Set up the DL1DataReader - config["Data"], data_format = setup_DL1DataReader(config, mode) + config["Data"] = setup_DL1DataReader(config, mode) # Set up logging, saving the config and optionally logging to a file logger = setup_logging(config, model_dir, debug, log_to_file) @@ -72,11 +72,7 @@ def run_model(config, mode="train", debug=False, log_to_file=False): # Create data reader logger.info("Loading data:") logger.info(" For a large dataset, this may take a while...") - if data_format == "stage1": - reader = DL1DataReaderSTAGE1(**config["Data"]) - elif data_format == "dl1dh": - reader = DL1DataReaderDL1DH(**config["Data"]) - + reader = DLDataReader(**config["Data"]) logger.info(" Number of events loaded: {}".format(len(reader))) # Set up the KerasBatchGenerator diff --git a/ctlearn/utils.py b/ctlearn/utils.py index 6592c873..f2da80c5 100644 --- a/ctlearn/utils.py +++ b/ctlearn/utils.py @@ -95,32 +95,17 @@ def setup_DL1DataReader(config, mode): mc_file = True dl1bparameter_names = None with tables.open_file(config["Data"]["file_list"][0], mode="r") as f: - # Retrieve the data format of the hdf5 file - if "CTA PRODUCT DATA MODEL NAME" in f.root._v_attrs: - data_format = "stage1" - elif "dl1_data_handler_version" in f.root._v_attrs: - data_format = "dl1dh" - else: - raise ValueError( - "Data format is not implemented in the DL1DH reader. Available data formats are 'stage1' and 'dl1dh'." - ) + # Check weather the file is MC simulation or real observational data if f.root._v_attrs['CTA PROCESS TYPE'] == 'Observation': - # if data_format == "dl1dh" and "source_name" in f.root._v_attrs: mc_file = False # Retrieve the name convention for the dl1b parameters - if data_format == "dl1dh": - first_tablename = next(f.root.Parameters0._f_iter_nodes()).name - dl1bparameter_names = f.root.Parameters0._f_get_child( - f"{first_tablename}" - ).colnames - else: - first_tablename = next( - f.root.dl1.event.telescope.parameters._f_iter_nodes() - ).name - dl1bparameter_names = f.root.dl1.event.telescope.parameters._f_get_child( - f"{first_tablename}" - ).colnames + first_tablename = next( + f.root.dl1.event.telescope.parameters._f_iter_nodes() + ).name + dl1bparameter_names = f.root.dl1.event.telescope.parameters._f_get_child( + f"{first_tablename}" + ).colnames allow_overwrite = config["Data"].get("allow_overwrite", True) if "allow_overwrite" in config["Data"]: @@ -132,61 +117,24 @@ def setup_DL1DataReader(config, mode): tasks = config["Reco"] transformations = [] event_info = [] - if data_format == "dl1dh": - if ( - "parameter_settings" not in config["Data"] - and dl1bparameter_names is not None - and mode == "predict" - ): - config["Data"]["parameter_settings"] = {"parameter_list": dl1bparameter_names} - # Parse list of event selection filters - event_selection = {} - for s in config["Data"].get("event_selection", {}): - s = {"module": "dl1_data_handler.filters", **s} - filter_fn, filter_params = load_from_module(**s) - event_selection[filter_fn] = filter_params - config["Data"]["event_selection"] = event_selection - - # Parse list of image selection filters - image_selection = {} - for s in config["Data"].get("image_selection", {}): - s = {"module": "dl1_data_handler.filters", **s} - filter_fn, filter_params = load_from_module(**s) - image_selection[filter_fn] = filter_params - config["Data"]["image_selection"] = image_selection - - if "direction" in tasks: - event_info.append("src_pos_cam_x") - event_info.append("src_pos_cam_y") - transformations.append( - { - "name": "AltAz", - "args": { - "alt_col_name": "src_pos_cam_x", - "az_col_name": "src_pos_cam_y", - "deg2rad": False, - }, - } + if ( + "parameter_settings" not in config["Data"] + and dl1bparameter_names is not None + and mode == "predict" + ): + config["Data"]["parameter_settings"] = {"parameter_list": dl1bparameter_names} + if "direction" in tasks or mode == "predict": + if mc_file: + event_info.append("true_alt") + event_info.append("true_az") + transformations.append({"name": "DeltaAltAz_fix_subarray"}) + if "cherenkov_photons" in tasks: + if "trigger_settings" in config["Data"]: + config["Data"]["trigger_settings"]["reco_cherenkov_photons"] = True + else: + raise ValueError( + "Required trigger settings are not provided for the regression of Cherenkov photons." ) - else: - if ( - "parameter_settings" not in config["Data"] - and dl1bparameter_names is not None - and mode == "predict" - ): - config["Data"]["parameter_settings"] = {"parameter_list": dl1bparameter_names} - if "direction" in tasks or mode == "predict": - if mc_file: - event_info.append("true_alt") - event_info.append("true_az") - transformations.append({"name": "DeltaAltAz_fix_subarray"}) - if "cherenkov_photons" in tasks: - if "trigger_settings" in config["Data"]: - config["Data"]["trigger_settings"]["reco_cherenkov_photons"] = True - else: - raise ValueError( - "Required trigger settings are not provided for the regression of Cherenkov photons." - ) if "type" in tasks or mode == "predict": if mc_file: @@ -240,7 +188,7 @@ def setup_DL1DataReader(config, mode): if data_format == "dl1dh" and not mc_file: config["Data"]["event_info"].extend(["mjd", "milli_sec", "nano_sec"]) - return config["Data"], data_format + return config["Data"] def load_from_module(name, module, path=None, args=None): From 427b829a563c24edbbeb1011dd71f0591e75d9a9 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 14 Jun 2024 13:16:42 +0200 Subject: [PATCH 03/23] fix telescope pointings --- ctlearn/data_loader.py | 7 ------- ctlearn/output_handler.py | 35 +++++++---------------------------- ctlearn/utils.py | 2 +- 3 files changed, 8 insertions(+), 36 deletions(-) diff --git a/ctlearn/data_loader.py b/ctlearn/data_loader.py index 410203f0..b11188f6 100644 --- a/ctlearn/data_loader.py +++ b/ctlearn/data_loader.py @@ -30,8 +30,6 @@ def __init__( self.singleimg_shape = None self.trg_pos, self.trg_shape = None, None self.trgpatch_pos, self.trgpatch_shape = None, None - self.pon_pos = None - self.pointing = [] self.wvf_pos, self.wvf_shape = None, None self.img_pos, self.img_shape = None, None self.prm_pos, self.prm_shape = None, None @@ -51,8 +49,6 @@ def __init__( if "HWtrigger" in desc["name"]: self.trg_pos = i self.trg_shape = desc["shape"] - elif "pointing" in desc["name"]: - self.pon_pos = i elif "waveform" in desc["name"]: self.wvf_pos = i self.wvf_shape = desc["shape"] @@ -167,9 +163,6 @@ def __data_generation(self, batch_indices): self.az_labels.append(np.float32(event[self.drc_pos][1])) if self.trgpatch_pos is not None: self.trgpatch_labels.append(np.float32(event[self.trgpatch_pos])) - # Save pointing - if self.pon_pos is not None: - self.pointing.append(event[self.pon_pos]) # Save all parameters for the prediction phase if self.prm_pos is not None: self.parameter_list.append(event[self.prm_pos]) diff --git a/ctlearn/output_handler.py b/ctlearn/output_handler.py index 75330e71..7e80d1cf 100644 --- a/ctlearn/output_handler.py +++ b/ctlearn/output_handler.py @@ -41,32 +41,13 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): ) reco["obs_id"] = obs_id - # Store pointings - if data.pon_pos: - pointing_alt = np.array(data.pointing)[data.batch_size :, 0] - pointing_az = np.array(data.pointing)[data.batch_size :, 1] - if rest_data: - pointing_alt = np.concatenate( - ( - pointing_alt, - np.array(rest_data.pointing)[rest_data.batch_size :, 0], - ), - axis=0, - ) - pointing_az = np.concatenate( - ( - pointing_az, - np.array(rest_data.pointing)[rest_data.batch_size :, 1], - ), - axis=0, + # Store telescope pointings + if reader.telescope_pointings is not None: + for tel_id, pointing_table in enumerate(reader.telescope_pointings): + pd.DataFrame(data=pointing_table).to_hdf( + h5file, key=f"/configuration/telescope/pointing/{tel_id}", mode="a" ) - else: - pointing_alt = np.array([reader.pointing[0]] * len(reader)) - pointing_az = np.array([reader.pointing[1]] * len(reader)) - reco["pointing_alt"] = pointing_alt - reco["pointing_az"] = pointing_az - # Store predictions and simulation values # Gamma/hadron classification if data.prt_pos: @@ -125,7 +106,6 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): ), axis=0, ) - + pointing_alt ) az = ( np.concatenate( @@ -135,7 +115,6 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): ), axis=0, ) - + pointing_az ) if "corsika_version" not in reader._v_attrs: reco["source_alt"] = alt @@ -145,8 +124,8 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): reco["true_az"] = az if "direction" in tasks: - reco["reco_alt"] = np.array(predictions[:, 0]) + pointing_alt - reco["reco_az"] = np.array(predictions[:, 1]) + pointing_az + reco["reco_alt"] = np.array(predictions[:, 0]) + reco["reco_az"] = np.array(predictions[:, 1]) if data.trgpatch_pos: cherenkov_photons = data.trgpatch_labels[data.batch_size :] diff --git a/ctlearn/utils.py b/ctlearn/utils.py index f2da80c5..ab9e3cd8 100644 --- a/ctlearn/utils.py +++ b/ctlearn/utils.py @@ -127,7 +127,7 @@ def setup_DL1DataReader(config, mode): if mc_file: event_info.append("true_alt") event_info.append("true_az") - transformations.append({"name": "DeltaAltAz_fix_subarray"}) + transformations.append({"name": "DeltaAltAz"}) if "cherenkov_photons" in tasks: if "trigger_settings" in config["Data"]: config["Data"]["trigger_settings"]["reco_cherenkov_photons"] = True From 2c0286b34c7c398733a8358b3b2b33de875fdec2 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 14 Jun 2024 13:17:55 +0200 Subject: [PATCH 04/23] remove dl1dh relic --- ctlearn/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ctlearn/utils.py b/ctlearn/utils.py index ab9e3cd8..3c8e3628 100644 --- a/ctlearn/utils.py +++ b/ctlearn/utils.py @@ -185,8 +185,6 @@ def setup_DL1DataReader(config, mode): if "event_info" not in config["Data"]: config["Data"]["event_info"] = [] config["Data"]["event_info"].extend(["event_id", "obs_id"]) - if data_format == "dl1dh" and not mc_file: - config["Data"]["event_info"].extend(["mjd", "milli_sec", "nano_sec"]) return config["Data"] From 10e6146471c5f273764499202bef617e77dc279f Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 14 Jun 2024 14:08:59 +0200 Subject: [PATCH 05/23] store pointing properly --- ctlearn/output_handler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ctlearn/output_handler.py b/ctlearn/output_handler.py index 7e80d1cf..e630cb3d 100644 --- a/ctlearn/output_handler.py +++ b/ctlearn/output_handler.py @@ -2,6 +2,7 @@ import logging import numpy as np import pandas as pd +from astropy.table import Table def write_output(h5file, data, rest_data, reader, predictions, tasks): @@ -43,8 +44,8 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): # Store telescope pointings if reader.telescope_pointings is not None: - for tel_id, pointing_table in enumerate(reader.telescope_pointings): - pd.DataFrame(data=pointing_table).to_hdf( + for tel_id in reader.telescope_pointings: + pd.DataFrame(data=reader.telescope_pointings[tel_id].to_pandas()).to_hdf( h5file, key=f"/configuration/telescope/pointing/{tel_id}", mode="a" ) From 11f76da1df922f3bc4e56c40b9cc968ec53c85cf Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Mon, 17 Jun 2024 17:23:17 +0200 Subject: [PATCH 06/23] fix handling of the pointing in the output handler --- ctlearn/output_handler.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ctlearn/output_handler.py b/ctlearn/output_handler.py index e630cb3d..990c5c55 100644 --- a/ctlearn/output_handler.py +++ b/ctlearn/output_handler.py @@ -46,7 +46,7 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): if reader.telescope_pointings is not None: for tel_id in reader.telescope_pointings: pd.DataFrame(data=reader.telescope_pointings[tel_id].to_pandas()).to_hdf( - h5file, key=f"/configuration/telescope/pointing/{tel_id}", mode="a" + h5file, key=f"/monitoring/telescope/pointing/{tel_id}", mode="a" ) # Store predictions and simulation values @@ -117,16 +117,16 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): axis=0, ) ) - if "corsika_version" not in reader._v_attrs: - reco["source_alt"] = alt - reco["source_az"] = az - else: - reco["true_alt"] = alt - reco["true_az"] = az + reco["true_alt"] = alt if reader.fix_pointing_alt is None else alt + reader.fix_pointing_alt + reco["true_az"] = az if reader.fix_pointing_az is None else az + reader.fix_pointing_az if "direction" in tasks: - reco["reco_alt"] = np.array(predictions[:, 0]) - reco["reco_az"] = np.array(predictions[:, 1]) + reco["reco_alt"] = np.array(predictions[:, 0]) if reader.fix_pointing_alt is None else np.array(predictions[:, 0]) + reader.fix_pointing_alt + reco["reco_az"] = np.array(predictions[:, 1]) if reader.fix_pointing_az is None else np.array(predictions[:, 1]) + reader.fix_pointing_az + if reader.fix_pointing_alt is not None: + reco["pointing_alt"] = np.full(len(reco["reco_alt"]), reader.fix_pointing_alt) + if reader.fix_pointing_az is not None: + reco["pointing_az"] = np.full(len(reco["reco_az"]), reader.fix_pointing_az) if data.trgpatch_pos: cherenkov_photons = data.trgpatch_labels[data.batch_size :] From 48541875daf5ce1fa44d9536b677a59a9587ff68 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 2 Jul 2024 13:26:09 +0200 Subject: [PATCH 07/23] added NSB flag also for the training phase set NSB patches to off as default in the Trigger configs --- ctlearn/default_config_files/triggerSingleCNN.yml | 2 +- ctlearn/default_config_files/triggerTRN.yml | 2 +- ctlearn/run_model.py | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ctlearn/default_config_files/triggerSingleCNN.yml b/ctlearn/default_config_files/triggerSingleCNN.yml index d5e3cf25..27558aaf 100644 --- a/ctlearn/default_config_files/triggerSingleCNN.yml +++ b/ctlearn/default_config_files/triggerSingleCNN.yml @@ -3,7 +3,7 @@ Data: trigger_settings: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False - include_nsb_patches: 'auto' + include_nsb_patches: 'off' trigger_patch_from_simulation: True waveform_settings: waveform_type: 'raw' diff --git a/ctlearn/default_config_files/triggerTRN.yml b/ctlearn/default_config_files/triggerTRN.yml index e6cd74d1..51dfc834 100644 --- a/ctlearn/default_config_files/triggerTRN.yml +++ b/ctlearn/default_config_files/triggerTRN.yml @@ -3,7 +3,7 @@ Data: trigger_settings: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False - include_nsb_patches: 'auto' + include_nsb_patches: 'off' trigger_patch_from_simulation: True waveform_settings: waveform_type: 'raw' diff --git a/ctlearn/run_model.py b/ctlearn/run_model.py index 66167238..a2c75d95 100644 --- a/ctlearn/run_model.py +++ b/ctlearn/run_model.py @@ -483,7 +483,7 @@ def main(): "--nsb", default=False, action=argparse.BooleanOptionalAction, - help="Flag, if the network should predict on NSB trigger patches", + help="Flag, if the network should include NSB trigger patches at training phase. In prediction mode, the network should only predict on NSB trigger patches", ) parser.add_argument( "--pretrained_weights", "-w", help="Path to the pretrained weights" @@ -662,6 +662,13 @@ def main(): "example_identifiers_file" ] = f"{config['Logging']['model_directory']}/example_identifiers_file.h5" + # AI-Trigger settings weather to include NSB trigger patches at training phase or not + if "trigger_settings" in config["Data"]: + if args.nsb: + config["Data"]["trigger_settings"]["include_nsb_patches"] = "auto" + else: + config["Data"]["trigger_settings"]["include_nsb_patches"] = "off" + run_model(config, mode="train", debug=args.debug, log_to_file=args.log_to_file) if "predict" in args.mode: From 95f97e30639fae8f2369f7b49f97c1ba39c6e8a8 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 2 Jul 2024 16:33:26 +0200 Subject: [PATCH 08/23] change config parameter from the trigger patch default is finding from file --- ctlearn/default_config_files/triggerSingleCNN.yml | 2 +- ctlearn/default_config_files/triggerTRN.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ctlearn/default_config_files/triggerSingleCNN.yml b/ctlearn/default_config_files/triggerSingleCNN.yml index 27558aaf..fb94aabf 100644 --- a/ctlearn/default_config_files/triggerSingleCNN.yml +++ b/ctlearn/default_config_files/triggerSingleCNN.yml @@ -4,7 +4,7 @@ Data: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False include_nsb_patches: 'off' - trigger_patch_from_simulation: True + get_trigger_patch: 'file' waveform_settings: waveform_type: 'raw' waveform_sequence_length: 5 diff --git a/ctlearn/default_config_files/triggerTRN.yml b/ctlearn/default_config_files/triggerTRN.yml index 51dfc834..da77d8cb 100644 --- a/ctlearn/default_config_files/triggerTRN.yml +++ b/ctlearn/default_config_files/triggerTRN.yml @@ -4,7 +4,7 @@ Data: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False include_nsb_patches: 'off' - trigger_patch_from_simulation: True + get_trigger_patch: 'file' waveform_settings: waveform_type: 'raw' waveform_sequence_length: 5 From d4edbdcb283baab1542ef896effa5a957c69fddc Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 2 Jul 2024 19:34:50 +0200 Subject: [PATCH 09/23] added command line flag for getting trigger patches from file set default back to simulation --- ctlearn/default_config_files/triggerSingleCNN.yml | 2 +- ctlearn/default_config_files/triggerTRN.yml | 2 +- ctlearn/run_model.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ctlearn/default_config_files/triggerSingleCNN.yml b/ctlearn/default_config_files/triggerSingleCNN.yml index fb94aabf..39443f57 100644 --- a/ctlearn/default_config_files/triggerSingleCNN.yml +++ b/ctlearn/default_config_files/triggerSingleCNN.yml @@ -4,7 +4,7 @@ Data: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False include_nsb_patches: 'off' - get_trigger_patch: 'file' + get_trigger_patch: 'simulation' waveform_settings: waveform_type: 'raw' waveform_sequence_length: 5 diff --git a/ctlearn/default_config_files/triggerTRN.yml b/ctlearn/default_config_files/triggerTRN.yml index da77d8cb..d93a179d 100644 --- a/ctlearn/default_config_files/triggerTRN.yml +++ b/ctlearn/default_config_files/triggerTRN.yml @@ -4,7 +4,7 @@ Data: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False include_nsb_patches: 'off' - get_trigger_patch: 'file' + get_trigger_patch: 'simulation' waveform_settings: waveform_type: 'raw' waveform_sequence_length: 5 diff --git a/ctlearn/run_model.py b/ctlearn/run_model.py index a2c75d95..8f5ece87 100644 --- a/ctlearn/run_model.py +++ b/ctlearn/run_model.py @@ -485,6 +485,12 @@ def main(): action=argparse.BooleanOptionalAction, help="Flag, if the network should include NSB trigger patches at training phase. In prediction mode, the network should only predict on NSB trigger patches", ) + parser.add_argument( + "--trigger_patches_from_file", + default=False, + action=argparse.BooleanOptionalAction, + help="Flag, if the trigger patches should be retrieved from an external file", + ) parser.add_argument( "--pretrained_weights", "-w", help="Path to the pretrained weights" ) @@ -668,6 +674,8 @@ def main(): config["Data"]["trigger_settings"]["include_nsb_patches"] = "auto" else: config["Data"]["trigger_settings"]["include_nsb_patches"] = "off" + if args.trigger_patches_from_file: + config["Data"]["trigger_settings"]["get_trigger_patch"] = "file" run_model(config, mode="train", debug=args.debug, log_to_file=args.log_to_file) @@ -705,6 +713,8 @@ def main(): config["Data"]["trigger_settings"]["include_nsb_patches"] = "all" else: config["Data"]["trigger_settings"]["include_nsb_patches"] = "off" + if args.trigger_patches_from_file: + config["Data"]["trigger_settings"]["get_trigger_patch"] = "file" if args.tel_types: config["Data"]["selected_telescope_types"] = args.tel_types if args.allowed_tels: @@ -780,6 +790,8 @@ def main(): config["Data"]["trigger_settings"]["include_nsb_patches"] = "all" else: config["Data"]["trigger_settings"]["include_nsb_patches"] = "off" + if args.trigger_patches_from_file: + config["Data"]["trigger_settings"]["get_trigger_patch"] = "file" if args.tel_types: config["Data"]["selected_telescope_types"] = args.tel_types if args.allowed_tels: From 8cab507603e71e0f4c44a0fea90a124bc5124f22 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 3 Jul 2024 15:39:53 +0200 Subject: [PATCH 10/23] properly treat direction task --- ctlearn/data_loader.py | 11 +++++++---- ctlearn/default_models/head.py | 2 +- ctlearn/utils.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ctlearn/data_loader.py b/ctlearn/data_loader.py index b11188f6..0f211db3 100644 --- a/ctlearn/data_loader.py +++ b/ctlearn/data_loader.py @@ -39,9 +39,10 @@ def __init__( self.event_list, self.obs_list = [], [] # Labels self.prt_pos, self.enr_pos, self.drc_pos = None, None, None + self.drc_unit = None self.prt_labels = [] self.enr_labels = [] - self.alt_labels, self.az_labels = [], [] + self.az_labels, self.alt_labels, self.sep_labels = [], [], [] self.trgpatch_labels = [] self.energy_unit = None @@ -68,6 +69,7 @@ def __init__( self.energy_unit = desc["unit"] elif "direction" in desc["name"]: self.drc_pos = i + self.drc_unit = desc["unit"] elif "event_id" in desc["name"]: self.evt_pos = i elif "obs_id" in desc["name"]: @@ -120,7 +122,7 @@ def __data_generation(self, batch_indices): if self.enr_pos is not None: energy = np.empty((self.batch_size)) if self.drc_pos is not None: - direction = np.empty((self.batch_size, 2)) + direction = np.empty((self.batch_size, 3)) if self.trgpatch_pos is not None: trigger_patches_true_image_sum = np.empty( (self.batch_size, *self.trgpatch_shape) @@ -159,8 +161,9 @@ def __data_generation(self, batch_indices): if self.enr_pos is not None: self.enr_labels.append(np.float32(event[self.enr_pos][0])) if self.drc_pos is not None: - self.alt_labels.append(np.float32(event[self.drc_pos][0])) - self.az_labels.append(np.float32(event[self.drc_pos][1])) + self.az_labels.append(np.float32(event[self.drc_pos][0])) + self.alt_labels.append(np.float32(event[self.drc_pos][1])) + self.sep_labels.append(np.float32(event[self.drc_pos][2])) if self.trgpatch_pos is not None: self.trgpatch_labels.append(np.float32(event[self.trgpatch_pos])) # Save all parameters for the prediction phase diff --git a/ctlearn/default_models/head.py b/ctlearn/default_models/head.py index a3d22f2b..ab681fc6 100644 --- a/ctlearn/default_models/head.py +++ b/ctlearn/default_models/head.py @@ -43,7 +43,7 @@ def standard_head(inputs, tasks, params): logits["direction"] = fully_connect( inputs, standard_head_settings["direction"]["fc_head"], - expected_logits_dimension=2, + expected_logits_dimension=3, name="direction", ) losses["direction"] = tf.keras.losses.MeanAbsoluteError( diff --git a/ctlearn/utils.py b/ctlearn/utils.py index 3c8e3628..96ab77a8 100644 --- a/ctlearn/utils.py +++ b/ctlearn/utils.py @@ -127,7 +127,7 @@ def setup_DL1DataReader(config, mode): if mc_file: event_info.append("true_alt") event_info.append("true_az") - transformations.append({"name": "DeltaAltAz"}) + transformations.append({"name": "SkyOffsetSeparation"}) if "cherenkov_photons" in tasks: if "trigger_settings" in config["Data"]: config["Data"]["trigger_settings"]["reco_cherenkov_photons"] = True @@ -143,7 +143,7 @@ def setup_DL1DataReader(config, mode): if "energy" in tasks or mode == "predict": if mc_file: event_info.append("true_energy") - transformations.append({"name": "MCEnergy"}) + transformations.append({"name": "LogEnergy"}) stack_telescope_images = config["Input"].get("stack_telescope_images", False) if config["Data"]["mode"] == "stereo" and not stack_telescope_images: From c4d2472bf2ceb02449e866eb26e980383887f158 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Thu, 4 Jul 2024 19:12:09 +0200 Subject: [PATCH 11/23] transform back to coordinates when offsets are predicted using astropy function 'spherical_offsets_by' --- ctlearn/data_loader.py | 2 +- ctlearn/output_handler.py | 90 +++++++++++++++++++++++++++------------ ctlearn/utils.py | 2 +- 3 files changed, 64 insertions(+), 30 deletions(-) diff --git a/ctlearn/data_loader.py b/ctlearn/data_loader.py index 0f211db3..800da56e 100644 --- a/ctlearn/data_loader.py +++ b/ctlearn/data_loader.py @@ -159,7 +159,7 @@ def __data_generation(self, batch_indices): if self.prt_pos is not None: self.prt_labels.append(np.float32(event[self.prt_pos])) if self.enr_pos is not None: - self.enr_labels.append(np.float32(event[self.enr_pos][0])) + self.enr_labels.append(np.float32(event[self.enr_pos])) if self.drc_pos is not None: self.az_labels.append(np.float32(event[self.drc_pos][0])) self.alt_labels.append(np.float32(event[self.drc_pos][1])) diff --git a/ctlearn/output_handler.py b/ctlearn/output_handler.py index 990c5c55..a3ab0b38 100644 --- a/ctlearn/output_handler.py +++ b/ctlearn/output_handler.py @@ -3,6 +3,8 @@ import numpy as np import pandas as pd from astropy.table import Table +from astropy.coordinates import SkyCoord +import astropy.units as u def write_output(h5file, data, rest_data, reader, predictions, tasks): @@ -90,43 +92,75 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): ) reco["true_energy"] = true_energy if "energy" in tasks: - if data.energy_unit == "log(TeV)" or np.min(predictions) < 0.0: + if data.energy_unit == "log(TeV)": reco["reco_energy"] = np.power(10, predictions)[:, 0] else: reco["reco_energy"] = np.array(predictions)[:, 0] # Arrival direction regression if data.drc_pos: - alt = data.alt_labels[data.batch_size :] - az = data.az_labels[data.batch_size :] + true_az_offset = data.az_labels[data.batch_size :] + true_alt_offset = data.alt_labels[data.batch_size :] + true_sep = data.sep_labels[data.batch_size :] if rest_data: - alt = ( - np.concatenate( - ( - alt, - rest_data.alt_labels[rest_data.batch_size :], - ), - axis=0, - ) + true_az_offset = np.concatenate( + ( + true_az_offset, + rest_data.az_labels[rest_data.batch_size :], + ), + axis=0, ) - az = ( - np.concatenate( - ( - az, - rest_data.az_labels[rest_data.batch_size :], - ), - axis=0, - ) + true_alt_offset = np.concatenate( + ( + true_alt_offset, + rest_data.alt_labels[rest_data.batch_size :], + ), + axis=0, ) - reco["true_alt"] = alt if reader.fix_pointing_alt is None else alt + reader.fix_pointing_alt - reco["true_az"] = az if reader.fix_pointing_az is None else az + reader.fix_pointing_az - + true_sep = np.concatenate( + ( + true_sep, + rest_data.sep_labels[rest_data.batch_size :], + ), + axis=0, + ) + reco["true_sep"] = true_sep + if reader.fix_pointing is None: + reco["true_az"] = true_az_offset + reco["true_alt"] = true_alt_offset + else: + true_az, true_alt = [], [] + for az_off, alt_off in zip(true_az_offset, true_alt_offset): + true_direction = reader.fix_pointing.spherical_offsets_by( + u.Quantity(az_off, unit=data.drc_unit), + u.Quantity(alt_off, unit=data.drc_unit), + ) + true_az.append(true_direction.az.to_value(data.drc_unit)) + true_alt.append(true_direction.alt.to_value(data.drc_unit)) + reco["true_az"] = np.array(true_az) + reco["true_alt"] = np.array(true_alt) if "direction" in tasks: - reco["reco_alt"] = np.array(predictions[:, 0]) if reader.fix_pointing_alt is None else np.array(predictions[:, 0]) + reader.fix_pointing_alt - reco["reco_az"] = np.array(predictions[:, 1]) if reader.fix_pointing_az is None else np.array(predictions[:, 1]) + reader.fix_pointing_az - if reader.fix_pointing_alt is not None: - reco["pointing_alt"] = np.full(len(reco["reco_alt"]), reader.fix_pointing_alt) - if reader.fix_pointing_az is not None: - reco["pointing_az"] = np.full(len(reco["reco_az"]), reader.fix_pointing_az) + if reader.fix_pointing is None: + reco["reco_az"] = np.array(predictions[:, 0]) + reco["reco_alt"] = np.array(predictions[:, 1]) + reco["reco_sep"] = np.array(predictions[:, 2]) + else: + reco_az, reco_alt = [], [] + for az_off, alt_off in zip(predictions[:, 0], predictions[:, 1]): + reco_direction = reader.fix_pointing.spherical_offsets_by( + u.Quantity(az_off, unit=data.drc_unit), + u.Quantity(alt_off, unit=data.drc_unit), + ) + reco_az.append(reco_direction.az.to_value(data.drc_unit)) + reco_alt.append(reco_direction.alt.to_value(data.drc_unit)) + reco["reco_az"] = np.array(reco_az) + reco["reco_alt"] = np.array(reco_alt) + reco["reco_sep"] = np.array(predictions[:, 2]) + reco["pointing_az"] = np.full( + len(reco["reco_az"]), reader.fix_pointing.az.to_value(data.drc_unit) + ) + reco["pointing_alt"] = np.full( + len(reco["reco_alt"]), reader.fix_pointing.alt.to_value(data.drc_unit) + ) if data.trgpatch_pos: cherenkov_photons = data.trgpatch_labels[data.batch_size :] diff --git a/ctlearn/utils.py b/ctlearn/utils.py index 96ab77a8..ab4b1ef9 100644 --- a/ctlearn/utils.py +++ b/ctlearn/utils.py @@ -123,7 +123,7 @@ def setup_DL1DataReader(config, mode): and mode == "predict" ): config["Data"]["parameter_settings"] = {"parameter_list": dl1bparameter_names} - if "direction" in tasks or mode == "predict": + if "direction" in tasks: if mc_file: event_info.append("true_alt") event_info.append("true_az") From b32ce7ac5e60f48c6559783bf39c98bbdba75757 Mon Sep 17 00:00:00 2001 From: BastienLacave Date: Thu, 13 Jun 2024 13:59:22 +0200 Subject: [PATCH 12/23] Real data processing --- ctlearn/utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ctlearn/utils.py b/ctlearn/utils.py index dfab096a..6592c873 100644 --- a/ctlearn/utils.py +++ b/ctlearn/utils.py @@ -105,7 +105,8 @@ def setup_DL1DataReader(config, mode): "Data format is not implemented in the DL1DH reader. Available data formats are 'stage1' and 'dl1dh'." ) # Check weather the file is MC simulation or real observational data - if data_format == "dl1dh" and "source_name" in f.root._v_attrs: + if f.root._v_attrs['CTA PROCESS TYPE'] == 'Observation': + # if data_format == "dl1dh" and "source_name" in f.root._v_attrs: mc_file = False # Retrieve the name convention for the dl1b parameters if data_format == "dl1dh": @@ -175,9 +176,10 @@ def setup_DL1DataReader(config, mode): ): config["Data"]["parameter_settings"] = {"parameter_list": dl1bparameter_names} if "direction" in tasks or mode == "predict": - event_info.append("true_alt") - event_info.append("true_az") - transformations.append({"name": "DeltaAltAz_fix_subarray"}) + if mc_file: + event_info.append("true_alt") + event_info.append("true_az") + transformations.append({"name": "DeltaAltAz_fix_subarray"}) if "cherenkov_photons" in tasks: if "trigger_settings" in config["Data"]: config["Data"]["trigger_settings"]["reco_cherenkov_photons"] = True @@ -187,12 +189,13 @@ def setup_DL1DataReader(config, mode): ) if "type" in tasks or mode == "predict": - event_info.append("true_shower_primary_id") + if mc_file: + event_info.append("true_shower_primary_id") if "energy" in tasks or mode == "predict": if mc_file: event_info.append("true_energy") - transformations.append({"name": "MCEnergy"}) + transformations.append({"name": "MCEnergy"}) stack_telescope_images = config["Input"].get("stack_telescope_images", False) if config["Data"]["mode"] == "stereo" and not stack_telescope_images: From be0887d2b33d8a8be0e89e79c25ba76b830099cf Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Thu, 13 Jun 2024 14:19:18 +0200 Subject: [PATCH 13/23] remove dl1dh reader --- .gitignore | 5 +- ctlearn/data_loader.py | 31 +++--------- ctlearn/output_handler.py | 64 ------------------------ ctlearn/run_model.py | 10 ++-- ctlearn/utils.py | 102 ++++++++++---------------------------- 5 files changed, 39 insertions(+), 173 deletions(-) diff --git a/.gitignore b/.gitignore index 9693f6b5..3f7d4240 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,9 @@ # Compiled Python files __pycache__/ *.py[cod] - +.DS_Store +ctlearn/.DS_Store ctlearn.egg-info/ # Sphinx documentation -docs/build/ \ No newline at end of file +docs/build/ diff --git a/ctlearn/data_loader.py b/ctlearn/data_loader.py index f5021d8f..410203f0 100644 --- a/ctlearn/data_loader.py +++ b/ctlearn/data_loader.py @@ -7,7 +7,7 @@ class KerasBatchGenerator(tf.keras.utils.Sequence): def __init__( self, - DL1DataReaderDL1DH, + DLDataReader, indices, batch_size=64, mode="train", @@ -16,7 +16,7 @@ def __init__( stack_telescope_images=False, ): "Initialization" - self.DL1DataReaderDL1DH = DL1DataReaderDL1DH + self.DLDataReader = DLDataReader self.batch_size = batch_size self.indices = indices self.mode = mode @@ -39,8 +39,6 @@ def __init__( # Additional info self.evt_pos, self.obs_pos = None, None self.event_list, self.obs_list = [], [] - self.mjd_pos, self.milli_pos, self.nano_pos = None, None, None - self.mjd_list, self.milli_list, self.nano_list = [], [], [] # Labels self.prt_pos, self.enr_pos, self.drc_pos = None, None, None self.prt_labels = [] @@ -49,7 +47,7 @@ def __init__( self.trgpatch_labels = [] self.energy_unit = None - for i, desc in enumerate(self.DL1DataReaderDL1DH.example_description): + for i, desc in enumerate(self.DLDataReader.example_description): if "HWtrigger" in desc["name"]: self.trg_pos = i self.trg_shape = desc["shape"] @@ -78,12 +76,6 @@ def __init__( self.evt_pos = i elif "obs_id" in desc["name"]: self.obs_pos = i - elif "mjd" in desc["name"]: - self.mjd_pos = i - elif "milli_sec" in desc["name"]: - self.milli_pos = i - elif "nano_sec" in desc["name"]: - self.nano_pos = i # Retrieve shape from a single image in stereo analysis if self.trg_pos is not None and self.img_pos is not None: @@ -139,7 +131,7 @@ def __data_generation(self, batch_indices): ) # Generate data for i, index in enumerate(batch_indices): - event = self.DL1DataReaderDL1DH[index] + event = self.DLDataReader[index] # Fill the features if self.trg_pos is not None: triggers[i] = event[self.trg_pos] @@ -155,7 +147,7 @@ def __data_generation(self, batch_indices): if self.prt_pos is not None: particletype[ i - ] = self.DL1DataReaderDL1DH.shower_primary_id_to_class[ + ] = self.DLDataReader.shower_primary_id_to_class[ int(event[self.prt_pos]) ] if self.enr_pos is not None: @@ -186,13 +178,6 @@ def __data_generation(self, batch_indices): self.event_list.append(np.float32(event[self.evt_pos])) if self.obs_pos is not None: self.obs_list.append(np.float32(event[self.obs_pos])) - # Save timestamp - if self.mjd_pos is not None: - self.mjd_list.append(np.float32(event[self.mjd_pos])) - if self.milli_pos is not None: - self.milli_list.append(np.float32(event[self.milli_pos])) - if self.nano_pos is not None: - self.nano_list.append(np.float32(event[self.nano_pos])) features = {} if self.trg_pos is not None: @@ -209,11 +194,11 @@ def __data_generation(self, batch_indices): if self.prt_pos is not None: labels["type"] = tf.keras.utils.to_categorical( particletype, - num_classes=self.DL1DataReaderDL1DH.num_classes, + num_classes=self.DLDataReader.num_classes, ) label = tf.keras.utils.to_categorical( particletype, - num_classes=self.DL1DataReaderDL1DH.num_classes, + num_classes=self.DLDataReader.num_classes, ) if self.enr_pos is not None: labels["energy"] = energy.reshape((-1, 1)) @@ -221,7 +206,7 @@ def __data_generation(self, batch_indices): if self.drc_pos is not None: labels["direction"] = direction label = direction - if self.trgpatch_pos is not None and self.DL1DataReaderDL1DH.reco_cherenkov_photons: + if self.trgpatch_pos is not None and self.DLDataReader.reco_cherenkov_photons: labels["cherenkov_photons"] = trigger_patches_true_image_sum label = trigger_patches_true_image_sum diff --git a/ctlearn/output_handler.py b/ctlearn/output_handler.py index a8d95649..75330e71 100644 --- a/ctlearn/output_handler.py +++ b/ctlearn/output_handler.py @@ -9,35 +9,6 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): if not os.path.exists(prediction_dir): os.makedirs(prediction_dir) - # Store the information for observational data - if reader.instrument_id == "MAGIC" and reader.process_type == "Observation": - # Dump the information of the run to hdf5 file - run_info = {} - run_info["run_number"] = reader._v_attrs["run_number"] - run_info["magic_number"] = reader._v_attrs["magic_number"] - run_info["num_events"] = reader._v_attrs["num_events"] - run_info["run_start_mjd"] = reader._v_attrs["run_start_mjd"] - run_info["run_start_ms"] = reader._v_attrs["run_start_ms"] - run_info["run_start_ns"] = reader._v_attrs["run_start_ns"] - run_info["run_stop_mjd"] = reader._v_attrs["run_stop_mjd"] - run_info["run_stop_ms"] = reader._v_attrs["run_stop_ms"] - run_info["run_stop_ns"] = reader._v_attrs["run_stop_ns"] - pd.DataFrame(data=run_info, index=[0]).to_hdf( - h5file, key=f"/info/run", mode="a" - ) - # Dump the information of the obsveration to hdf5 file - obs_info = {} - obs_info["source_name"] = reader._v_attrs["source_name"] - obs_info["project_name"] = reader._v_attrs["project_name"] - obs_info["observation_mode"] = reader._v_attrs["observation_mode"] - obs_info["source_dec"] = reader._v_attrs["source_dec"] - obs_info["source_ra"] = reader._v_attrs["source_ra"] - obs_info["telescope_dec"] = reader._v_attrs["telescope_dec"] - obs_info["telescope_ra"] = reader._v_attrs["telescope_ra"] - pd.DataFrame(data=obs_info, index=[0]).to_hdf( - h5file, key=f"/info/obs", mode="a" - ) - # Store dl2 data reco = {} if os.path.isfile(h5file): @@ -70,41 +41,6 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): ) reco["obs_id"] = obs_id - # Store the timestamp - if data.mjd_pos: - mjd = data.mjd_list[data.batch_size :] - if rest_data: - mjd = np.concatenate( - ( - mjd, - rest_data.mjd_list[rest_data.batch_size :], - ), - axis=0, - ) - reco["mjd"] = mjd - if data.milli_pos: - milli_sec = data.milli_list[data.batch_size :] - if rest_data: - milli_sec = np.concatenate( - ( - milli_sec, - rest_data.milli_list[rest_data.batch_size :], - ), - axis=0, - ) - reco["milli_sec"] = milli_sec - if data.nano_pos: - nano_sec = data.nano_list[data.batch_size :] - if rest_data: - nano_sec = np.concatenate( - ( - nano_sec, - rest_data.nano_list[rest_data.batch_size :], - ), - axis=0, - ) - reco["nano_sec"] = nano_sec - # Store pointings if data.pon_pos: pointing_alt = np.array(data.pointing)[data.batch_size :, 0] diff --git a/ctlearn/run_model.py b/ctlearn/run_model.py index e831c8e4..b4cbf4f2 100644 --- a/ctlearn/run_model.py +++ b/ctlearn/run_model.py @@ -17,7 +17,7 @@ import tensorflow as tf from tensorflow.python import debug as tf_debug -from dl1_data_handler.reader import DL1DataReaderSTAGE1, DL1DataReaderDL1DH +from dl1_data_handler.reader import DLDataReader from ctlearn.data_loader import KerasBatchGenerator from ctlearn.output_handler import * from ctlearn.utils import * @@ -54,7 +54,7 @@ def run_model(config, mode="train", debug=False, log_to_file=False): class_names = [name[0] for name in class_names] # Set up the DL1DataReader - config["Data"], data_format = setup_DL1DataReader(config, mode) + config["Data"] = setup_DL1DataReader(config, mode) # Set up logging, saving the config and optionally logging to a file logger = setup_logging(config, model_dir, debug, log_to_file) @@ -72,11 +72,7 @@ def run_model(config, mode="train", debug=False, log_to_file=False): # Create data reader logger.info("Loading data:") logger.info(" For a large dataset, this may take a while...") - if data_format == "stage1": - reader = DL1DataReaderSTAGE1(**config["Data"]) - elif data_format == "dl1dh": - reader = DL1DataReaderDL1DH(**config["Data"]) - + reader = DLDataReader(**config["Data"]) logger.info(" Number of events loaded: {}".format(len(reader))) # Set up the KerasBatchGenerator diff --git a/ctlearn/utils.py b/ctlearn/utils.py index 6592c873..f2da80c5 100644 --- a/ctlearn/utils.py +++ b/ctlearn/utils.py @@ -95,32 +95,17 @@ def setup_DL1DataReader(config, mode): mc_file = True dl1bparameter_names = None with tables.open_file(config["Data"]["file_list"][0], mode="r") as f: - # Retrieve the data format of the hdf5 file - if "CTA PRODUCT DATA MODEL NAME" in f.root._v_attrs: - data_format = "stage1" - elif "dl1_data_handler_version" in f.root._v_attrs: - data_format = "dl1dh" - else: - raise ValueError( - "Data format is not implemented in the DL1DH reader. Available data formats are 'stage1' and 'dl1dh'." - ) + # Check weather the file is MC simulation or real observational data if f.root._v_attrs['CTA PROCESS TYPE'] == 'Observation': - # if data_format == "dl1dh" and "source_name" in f.root._v_attrs: mc_file = False # Retrieve the name convention for the dl1b parameters - if data_format == "dl1dh": - first_tablename = next(f.root.Parameters0._f_iter_nodes()).name - dl1bparameter_names = f.root.Parameters0._f_get_child( - f"{first_tablename}" - ).colnames - else: - first_tablename = next( - f.root.dl1.event.telescope.parameters._f_iter_nodes() - ).name - dl1bparameter_names = f.root.dl1.event.telescope.parameters._f_get_child( - f"{first_tablename}" - ).colnames + first_tablename = next( + f.root.dl1.event.telescope.parameters._f_iter_nodes() + ).name + dl1bparameter_names = f.root.dl1.event.telescope.parameters._f_get_child( + f"{first_tablename}" + ).colnames allow_overwrite = config["Data"].get("allow_overwrite", True) if "allow_overwrite" in config["Data"]: @@ -132,61 +117,24 @@ def setup_DL1DataReader(config, mode): tasks = config["Reco"] transformations = [] event_info = [] - if data_format == "dl1dh": - if ( - "parameter_settings" not in config["Data"] - and dl1bparameter_names is not None - and mode == "predict" - ): - config["Data"]["parameter_settings"] = {"parameter_list": dl1bparameter_names} - # Parse list of event selection filters - event_selection = {} - for s in config["Data"].get("event_selection", {}): - s = {"module": "dl1_data_handler.filters", **s} - filter_fn, filter_params = load_from_module(**s) - event_selection[filter_fn] = filter_params - config["Data"]["event_selection"] = event_selection - - # Parse list of image selection filters - image_selection = {} - for s in config["Data"].get("image_selection", {}): - s = {"module": "dl1_data_handler.filters", **s} - filter_fn, filter_params = load_from_module(**s) - image_selection[filter_fn] = filter_params - config["Data"]["image_selection"] = image_selection - - if "direction" in tasks: - event_info.append("src_pos_cam_x") - event_info.append("src_pos_cam_y") - transformations.append( - { - "name": "AltAz", - "args": { - "alt_col_name": "src_pos_cam_x", - "az_col_name": "src_pos_cam_y", - "deg2rad": False, - }, - } + if ( + "parameter_settings" not in config["Data"] + and dl1bparameter_names is not None + and mode == "predict" + ): + config["Data"]["parameter_settings"] = {"parameter_list": dl1bparameter_names} + if "direction" in tasks or mode == "predict": + if mc_file: + event_info.append("true_alt") + event_info.append("true_az") + transformations.append({"name": "DeltaAltAz_fix_subarray"}) + if "cherenkov_photons" in tasks: + if "trigger_settings" in config["Data"]: + config["Data"]["trigger_settings"]["reco_cherenkov_photons"] = True + else: + raise ValueError( + "Required trigger settings are not provided for the regression of Cherenkov photons." ) - else: - if ( - "parameter_settings" not in config["Data"] - and dl1bparameter_names is not None - and mode == "predict" - ): - config["Data"]["parameter_settings"] = {"parameter_list": dl1bparameter_names} - if "direction" in tasks or mode == "predict": - if mc_file: - event_info.append("true_alt") - event_info.append("true_az") - transformations.append({"name": "DeltaAltAz_fix_subarray"}) - if "cherenkov_photons" in tasks: - if "trigger_settings" in config["Data"]: - config["Data"]["trigger_settings"]["reco_cherenkov_photons"] = True - else: - raise ValueError( - "Required trigger settings are not provided for the regression of Cherenkov photons." - ) if "type" in tasks or mode == "predict": if mc_file: @@ -240,7 +188,7 @@ def setup_DL1DataReader(config, mode): if data_format == "dl1dh" and not mc_file: config["Data"]["event_info"].extend(["mjd", "milli_sec", "nano_sec"]) - return config["Data"], data_format + return config["Data"] def load_from_module(name, module, path=None, args=None): From 62047ae83ce5abba9513218cc3c0054aa36acdc8 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 14 Jun 2024 13:16:42 +0200 Subject: [PATCH 14/23] fix telescope pointings --- ctlearn/data_loader.py | 7 ------- ctlearn/output_handler.py | 35 +++++++---------------------------- ctlearn/utils.py | 2 +- 3 files changed, 8 insertions(+), 36 deletions(-) diff --git a/ctlearn/data_loader.py b/ctlearn/data_loader.py index 410203f0..b11188f6 100644 --- a/ctlearn/data_loader.py +++ b/ctlearn/data_loader.py @@ -30,8 +30,6 @@ def __init__( self.singleimg_shape = None self.trg_pos, self.trg_shape = None, None self.trgpatch_pos, self.trgpatch_shape = None, None - self.pon_pos = None - self.pointing = [] self.wvf_pos, self.wvf_shape = None, None self.img_pos, self.img_shape = None, None self.prm_pos, self.prm_shape = None, None @@ -51,8 +49,6 @@ def __init__( if "HWtrigger" in desc["name"]: self.trg_pos = i self.trg_shape = desc["shape"] - elif "pointing" in desc["name"]: - self.pon_pos = i elif "waveform" in desc["name"]: self.wvf_pos = i self.wvf_shape = desc["shape"] @@ -167,9 +163,6 @@ def __data_generation(self, batch_indices): self.az_labels.append(np.float32(event[self.drc_pos][1])) if self.trgpatch_pos is not None: self.trgpatch_labels.append(np.float32(event[self.trgpatch_pos])) - # Save pointing - if self.pon_pos is not None: - self.pointing.append(event[self.pon_pos]) # Save all parameters for the prediction phase if self.prm_pos is not None: self.parameter_list.append(event[self.prm_pos]) diff --git a/ctlearn/output_handler.py b/ctlearn/output_handler.py index 75330e71..7e80d1cf 100644 --- a/ctlearn/output_handler.py +++ b/ctlearn/output_handler.py @@ -41,32 +41,13 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): ) reco["obs_id"] = obs_id - # Store pointings - if data.pon_pos: - pointing_alt = np.array(data.pointing)[data.batch_size :, 0] - pointing_az = np.array(data.pointing)[data.batch_size :, 1] - if rest_data: - pointing_alt = np.concatenate( - ( - pointing_alt, - np.array(rest_data.pointing)[rest_data.batch_size :, 0], - ), - axis=0, - ) - pointing_az = np.concatenate( - ( - pointing_az, - np.array(rest_data.pointing)[rest_data.batch_size :, 1], - ), - axis=0, + # Store telescope pointings + if reader.telescope_pointings is not None: + for tel_id, pointing_table in enumerate(reader.telescope_pointings): + pd.DataFrame(data=pointing_table).to_hdf( + h5file, key=f"/configuration/telescope/pointing/{tel_id}", mode="a" ) - else: - pointing_alt = np.array([reader.pointing[0]] * len(reader)) - pointing_az = np.array([reader.pointing[1]] * len(reader)) - reco["pointing_alt"] = pointing_alt - reco["pointing_az"] = pointing_az - # Store predictions and simulation values # Gamma/hadron classification if data.prt_pos: @@ -125,7 +106,6 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): ), axis=0, ) - + pointing_alt ) az = ( np.concatenate( @@ -135,7 +115,6 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): ), axis=0, ) - + pointing_az ) if "corsika_version" not in reader._v_attrs: reco["source_alt"] = alt @@ -145,8 +124,8 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): reco["true_az"] = az if "direction" in tasks: - reco["reco_alt"] = np.array(predictions[:, 0]) + pointing_alt - reco["reco_az"] = np.array(predictions[:, 1]) + pointing_az + reco["reco_alt"] = np.array(predictions[:, 0]) + reco["reco_az"] = np.array(predictions[:, 1]) if data.trgpatch_pos: cherenkov_photons = data.trgpatch_labels[data.batch_size :] diff --git a/ctlearn/utils.py b/ctlearn/utils.py index f2da80c5..ab9e3cd8 100644 --- a/ctlearn/utils.py +++ b/ctlearn/utils.py @@ -127,7 +127,7 @@ def setup_DL1DataReader(config, mode): if mc_file: event_info.append("true_alt") event_info.append("true_az") - transformations.append({"name": "DeltaAltAz_fix_subarray"}) + transformations.append({"name": "DeltaAltAz"}) if "cherenkov_photons" in tasks: if "trigger_settings" in config["Data"]: config["Data"]["trigger_settings"]["reco_cherenkov_photons"] = True From a4a3343acc87e263356f9e137ef71b49b56ce95e Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 14 Jun 2024 13:17:55 +0200 Subject: [PATCH 15/23] remove dl1dh relic --- ctlearn/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ctlearn/utils.py b/ctlearn/utils.py index ab9e3cd8..3c8e3628 100644 --- a/ctlearn/utils.py +++ b/ctlearn/utils.py @@ -185,8 +185,6 @@ def setup_DL1DataReader(config, mode): if "event_info" not in config["Data"]: config["Data"]["event_info"] = [] config["Data"]["event_info"].extend(["event_id", "obs_id"]) - if data_format == "dl1dh" and not mc_file: - config["Data"]["event_info"].extend(["mjd", "milli_sec", "nano_sec"]) return config["Data"] From 736bcd199970cc3b51e81dea2983c92178c1a2f2 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 14 Jun 2024 14:08:59 +0200 Subject: [PATCH 16/23] store pointing properly --- ctlearn/output_handler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ctlearn/output_handler.py b/ctlearn/output_handler.py index 7e80d1cf..e630cb3d 100644 --- a/ctlearn/output_handler.py +++ b/ctlearn/output_handler.py @@ -2,6 +2,7 @@ import logging import numpy as np import pandas as pd +from astropy.table import Table def write_output(h5file, data, rest_data, reader, predictions, tasks): @@ -43,8 +44,8 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): # Store telescope pointings if reader.telescope_pointings is not None: - for tel_id, pointing_table in enumerate(reader.telescope_pointings): - pd.DataFrame(data=pointing_table).to_hdf( + for tel_id in reader.telescope_pointings: + pd.DataFrame(data=reader.telescope_pointings[tel_id].to_pandas()).to_hdf( h5file, key=f"/configuration/telescope/pointing/{tel_id}", mode="a" ) From ec6057438982baef8c9cf70b365010529dedcd57 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Mon, 17 Jun 2024 17:23:17 +0200 Subject: [PATCH 17/23] fix handling of the pointing in the output handler --- ctlearn/output_handler.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ctlearn/output_handler.py b/ctlearn/output_handler.py index e630cb3d..990c5c55 100644 --- a/ctlearn/output_handler.py +++ b/ctlearn/output_handler.py @@ -46,7 +46,7 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): if reader.telescope_pointings is not None: for tel_id in reader.telescope_pointings: pd.DataFrame(data=reader.telescope_pointings[tel_id].to_pandas()).to_hdf( - h5file, key=f"/configuration/telescope/pointing/{tel_id}", mode="a" + h5file, key=f"/monitoring/telescope/pointing/{tel_id}", mode="a" ) # Store predictions and simulation values @@ -117,16 +117,16 @@ def write_output(h5file, data, rest_data, reader, predictions, tasks): axis=0, ) ) - if "corsika_version" not in reader._v_attrs: - reco["source_alt"] = alt - reco["source_az"] = az - else: - reco["true_alt"] = alt - reco["true_az"] = az + reco["true_alt"] = alt if reader.fix_pointing_alt is None else alt + reader.fix_pointing_alt + reco["true_az"] = az if reader.fix_pointing_az is None else az + reader.fix_pointing_az if "direction" in tasks: - reco["reco_alt"] = np.array(predictions[:, 0]) - reco["reco_az"] = np.array(predictions[:, 1]) + reco["reco_alt"] = np.array(predictions[:, 0]) if reader.fix_pointing_alt is None else np.array(predictions[:, 0]) + reader.fix_pointing_alt + reco["reco_az"] = np.array(predictions[:, 1]) if reader.fix_pointing_az is None else np.array(predictions[:, 1]) + reader.fix_pointing_az + if reader.fix_pointing_alt is not None: + reco["pointing_alt"] = np.full(len(reco["reco_alt"]), reader.fix_pointing_alt) + if reader.fix_pointing_az is not None: + reco["pointing_az"] = np.full(len(reco["reco_az"]), reader.fix_pointing_az) if data.trgpatch_pos: cherenkov_photons = data.trgpatch_labels[data.batch_size :] From c30d162621040c536635b5af7ed3dd586b2cb214 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 2 Jul 2024 13:26:09 +0200 Subject: [PATCH 18/23] added NSB flag also for the training phase set NSB patches to off as default in the Trigger configs --- ctlearn/default_config_files/triggerSingleCNN.yml | 2 +- ctlearn/default_config_files/triggerTRN.yml | 2 +- ctlearn/run_model.py | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ctlearn/default_config_files/triggerSingleCNN.yml b/ctlearn/default_config_files/triggerSingleCNN.yml index d5e3cf25..27558aaf 100644 --- a/ctlearn/default_config_files/triggerSingleCNN.yml +++ b/ctlearn/default_config_files/triggerSingleCNN.yml @@ -3,7 +3,7 @@ Data: trigger_settings: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False - include_nsb_patches: 'auto' + include_nsb_patches: 'off' trigger_patch_from_simulation: True waveform_settings: waveform_type: 'raw' diff --git a/ctlearn/default_config_files/triggerTRN.yml b/ctlearn/default_config_files/triggerTRN.yml index e6cd74d1..51dfc834 100644 --- a/ctlearn/default_config_files/triggerTRN.yml +++ b/ctlearn/default_config_files/triggerTRN.yml @@ -3,7 +3,7 @@ Data: trigger_settings: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False - include_nsb_patches: 'auto' + include_nsb_patches: 'off' trigger_patch_from_simulation: True waveform_settings: waveform_type: 'raw' diff --git a/ctlearn/run_model.py b/ctlearn/run_model.py index b4cbf4f2..5d49b9db 100644 --- a/ctlearn/run_model.py +++ b/ctlearn/run_model.py @@ -483,7 +483,7 @@ def main(): "--nsb", default=False, action=argparse.BooleanOptionalAction, - help="Flag, if the network should predict on NSB trigger patches", + help="Flag, if the network should include NSB trigger patches at training phase. In prediction mode, the network should only predict on NSB trigger patches", ) parser.add_argument( "--pretrained_weights", "-w", help="Path to the pretrained weights" @@ -662,6 +662,13 @@ def main(): "example_identifiers_file" ] = f"{config['Logging']['model_directory']}/example_identifiers_file.h5" + # AI-Trigger settings weather to include NSB trigger patches at training phase or not + if "trigger_settings" in config["Data"]: + if args.nsb: + config["Data"]["trigger_settings"]["include_nsb_patches"] = "auto" + else: + config["Data"]["trigger_settings"]["include_nsb_patches"] = "off" + run_model(config, mode="train", debug=args.debug, log_to_file=args.log_to_file) if "predict" in args.mode: From dd70f5698b73ea654658469c4d8c0f3d5c968ffd Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 2 Jul 2024 16:33:26 +0200 Subject: [PATCH 19/23] change config parameter from the trigger patch default is finding from file --- ctlearn/default_config_files/triggerSingleCNN.yml | 2 +- ctlearn/default_config_files/triggerTRN.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ctlearn/default_config_files/triggerSingleCNN.yml b/ctlearn/default_config_files/triggerSingleCNN.yml index 27558aaf..fb94aabf 100644 --- a/ctlearn/default_config_files/triggerSingleCNN.yml +++ b/ctlearn/default_config_files/triggerSingleCNN.yml @@ -4,7 +4,7 @@ Data: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False include_nsb_patches: 'off' - trigger_patch_from_simulation: True + get_trigger_patch: 'file' waveform_settings: waveform_type: 'raw' waveform_sequence_length: 5 diff --git a/ctlearn/default_config_files/triggerTRN.yml b/ctlearn/default_config_files/triggerTRN.yml index 51dfc834..da77d8cb 100644 --- a/ctlearn/default_config_files/triggerTRN.yml +++ b/ctlearn/default_config_files/triggerTRN.yml @@ -4,7 +4,7 @@ Data: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False include_nsb_patches: 'off' - trigger_patch_from_simulation: True + get_trigger_patch: 'file' waveform_settings: waveform_type: 'raw' waveform_sequence_length: 5 From 99708ab449817c77384d625dd5ac0601169515a2 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 2 Jul 2024 19:34:50 +0200 Subject: [PATCH 20/23] added command line flag for getting trigger patches from file set default back to simulation --- ctlearn/default_config_files/triggerSingleCNN.yml | 2 +- ctlearn/default_config_files/triggerTRN.yml | 2 +- ctlearn/run_model.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ctlearn/default_config_files/triggerSingleCNN.yml b/ctlearn/default_config_files/triggerSingleCNN.yml index fb94aabf..39443f57 100644 --- a/ctlearn/default_config_files/triggerSingleCNN.yml +++ b/ctlearn/default_config_files/triggerSingleCNN.yml @@ -4,7 +4,7 @@ Data: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False include_nsb_patches: 'off' - get_trigger_patch: 'file' + get_trigger_patch: 'simulation' waveform_settings: waveform_type: 'raw' waveform_sequence_length: 5 diff --git a/ctlearn/default_config_files/triggerTRN.yml b/ctlearn/default_config_files/triggerTRN.yml index da77d8cb..d93a179d 100644 --- a/ctlearn/default_config_files/triggerTRN.yml +++ b/ctlearn/default_config_files/triggerTRN.yml @@ -4,7 +4,7 @@ Data: number_of_trigger_patches: [7, 7] reco_cherenkov_photons: False include_nsb_patches: 'off' - get_trigger_patch: 'file' + get_trigger_patch: 'simulation' waveform_settings: waveform_type: 'raw' waveform_sequence_length: 5 diff --git a/ctlearn/run_model.py b/ctlearn/run_model.py index 5d49b9db..e133897a 100644 --- a/ctlearn/run_model.py +++ b/ctlearn/run_model.py @@ -485,6 +485,12 @@ def main(): action=argparse.BooleanOptionalAction, help="Flag, if the network should include NSB trigger patches at training phase. In prediction mode, the network should only predict on NSB trigger patches", ) + parser.add_argument( + "--trigger_patches_from_file", + default=False, + action=argparse.BooleanOptionalAction, + help="Flag, if the trigger patches should be retrieved from an external file", + ) parser.add_argument( "--pretrained_weights", "-w", help="Path to the pretrained weights" ) @@ -668,6 +674,8 @@ def main(): config["Data"]["trigger_settings"]["include_nsb_patches"] = "auto" else: config["Data"]["trigger_settings"]["include_nsb_patches"] = "off" + if args.trigger_patches_from_file: + config["Data"]["trigger_settings"]["get_trigger_patch"] = "file" run_model(config, mode="train", debug=args.debug, log_to_file=args.log_to_file) @@ -705,6 +713,8 @@ def main(): config["Data"]["trigger_settings"]["include_nsb_patches"] = "all" else: config["Data"]["trigger_settings"]["include_nsb_patches"] = "off" + if args.trigger_patches_from_file: + config["Data"]["trigger_settings"]["get_trigger_patch"] = "file" if args.tel_types: config["Data"]["selected_telescope_types"] = args.tel_types if args.allowed_tels: @@ -780,6 +790,8 @@ def main(): config["Data"]["trigger_settings"]["include_nsb_patches"] = "all" else: config["Data"]["trigger_settings"]["include_nsb_patches"] = "off" + if args.trigger_patches_from_file: + config["Data"]["trigger_settings"]["get_trigger_patch"] = "file" if args.tel_types: config["Data"]["selected_telescope_types"] = args.tel_types if args.allowed_tels: From 28a4538cacfbed1337e634a13e8e97b0189359eb Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 12 Jul 2024 17:13:56 +0200 Subject: [PATCH 21/23] update version for zenodo --- .zenodo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zenodo.json b/.zenodo.json index bf5defcb..fb6c656f 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -62,5 +62,5 @@ ], "title": "CTLearn: Deep learning for imaging atmospheric Cherenkov telescopes event reconstruction", "upload_type": "software", - "version": "0.7.0" + "version": "0.8.0" } From e359b3b608de03b9e8fe2b8de1160f4bd8837216 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 12 Jul 2024 17:14:14 +0200 Subject: [PATCH 22/23] upgrade to dl1dh v0.12.0 --- environment-cluster.yml | 2 +- environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment-cluster.yml b/environment-cluster.yml index c0c4d08f..b4db2ea5 100644 --- a/environment-cluster.yml +++ b/environment-cluster.yml @@ -7,7 +7,7 @@ channels: - ctlearn-project dependencies: - python=3.10 - - dl1_data_handler==0.11.1 + - dl1_data_handler==0.12.0 - astropy - numpy - pip diff --git a/environment.yml b/environment.yml index 5df78b8c..bc9ce380 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,7 @@ channels: - ctlearn-project dependencies: - python=3.10 - - dl1_data_handler==0.11.1 + - dl1_data_handler==0.12.0 - astropy - matplotlib - numpy From 3b8d2cd354b6f734de7383f7390a5f0d7d86ac90 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 12 Jul 2024 17:27:35 +0200 Subject: [PATCH 23/23] install dl1dh via pypi --- environment-cluster.yml | 2 +- environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment-cluster.yml b/environment-cluster.yml index b4db2ea5..ce140fc5 100644 --- a/environment-cluster.yml +++ b/environment-cluster.yml @@ -7,7 +7,6 @@ channels: - ctlearn-project dependencies: - python=3.10 - - dl1_data_handler==0.12.0 - astropy - numpy - pip @@ -15,6 +14,7 @@ dependencies: - scikit-learn - pip: - ctaplot + - dl1_data_handler==0.12.0 - matplotlib==3.6.2 - numba>=0.56.2,<0.57 - numexpr==2.8.4 diff --git a/environment.yml b/environment.yml index bc9ce380..e40ea9cf 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,6 @@ channels: - ctlearn-project dependencies: - python=3.10 - - dl1_data_handler==0.12.0 - astropy - matplotlib - numpy @@ -19,5 +18,6 @@ dependencies: - ctaplot - numba>=0.56.2,<0.57 - tensorflow>=2.15,<2.16 + - dl1_data_handler==0.12.0 - pydot - pyirf