From 78f53c77a90930d08d064293dfcae5a492c0aae9 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Thu, 26 Oct 2023 17:55:04 +0200 Subject: [PATCH 01/14] Initial commit --- ctapipe/io/tableloader.py | 352 ++++++++++++++++++++++---- ctapipe/io/tests/test_table_loader.py | 182 ++++++------- 2 files changed, 398 insertions(+), 136 deletions(-) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index f7077969e8e..74d7ccea347 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -161,7 +161,7 @@ class TableLoader(Component): * `TableLoader.read_subarray_events` * `TableLoader.read_telescope_events` - * `TableLoader.read_telescope_events_by_type` retuns a dict with a table per + * `TableLoader.read_telescope_events_by_type` returns a dict with a table per telescope type, which is needed for e.g. DL1 image data that might have different shapes for each of the telescope types as tables do not support variable length columns. @@ -175,37 +175,33 @@ class TableLoader(Component): input_url = traits.Path( directory_ok=False, exists=True, allow_none=True, default_value=None ).tag(config=True) - - load_dl1_images = traits.Bool(False, help="load extracted images").tag(config=True) - load_dl1_parameters = traits.Bool( - True, help="load reconstructed image parameters" - ).tag(config=True) - load_dl1_muons = traits.Bool(False, help="load muon ring parameters").tag( + dl1_images = traits.Bool(False, help="load extracted images").tag(config=True) + dl1_parameters = traits.Bool(True, help="load reconstructed image parameters").tag( config=True ) + dl1_muons = traits.Bool(True, help="load muon ring parameters").tag(config=True) - load_dl2 = traits.Bool(False, help="load available dl2 stereo parameters").tag( + dl2 = traits.Bool(True, help="load available dl2 stereo parameters").tag( config=True ) - load_simulated = traits.Bool(False, help="load simulated shower information").tag( + simulated = traits.Bool(True, help="load simulated shower information").tag( config=True ) - load_true_images = traits.Bool(False, help="load simulated shower images").tag( + true_images = traits.Bool(True, help="load simulated shower images").tag( config=True ) - load_true_parameters = traits.Bool( - False, help="load image parameters obtained from true images" + true_parameters = traits.Bool( + True, help="load image parameters obtained from true images" ).tag(config=True) - load_instrument = traits.Bool( - False, help="join subarray instrument information to each event" + instrument = traits.Bool( + True, help="join subarray instrument information to each event" ).tag(config=True) - load_observation_info = traits.Bool( - False, help="join observation information to each event" + observation_info = traits.Bool( + True, help="join observation information to each event" ).tag(config=True) - focal_length_choice = traits.UseEnum( FocalLengthKind, default_value=FocalLengthKind.EFFECTIVE, @@ -247,15 +243,11 @@ def __init__(self, input_url=None, h5file=None, **kwargs): Provenance().add_input_file(self.input_url, role="Event data") - self.instrument_table = None - if self.load_instrument: - self.instrument_table = self.subarray.to_table("joined") - groups = { - "load_dl1_parameters": PARAMETERS_GROUP, - "load_dl1_images": IMAGES_GROUP, - "load_true_parameters": TRUE_PARAMETERS_GROUP, - "load_true_images": TRUE_IMAGES_GROUP, + "dl1_parameters": PARAMETERS_GROUP, + "dl1_images": IMAGES_GROUP, + "true_parameters": TRUE_PARAMETERS_GROUP, + "true_images": TRUE_IMAGES_GROUP, } for attr, group in groups.items(): if getattr(self, attr) and group not in self.h5file.root: @@ -369,23 +361,55 @@ def _join_observation_info(self, table): table.remove_column("__index__") return joint - def read_subarray_events(self, start=None, stop=None, keep_order=True): + def _fallback_to_traits(self, **kwargs): + return {k: v if v is not None else getattr(self, k) for k, v in kwargs.items()} + + def read_subarray_events( + self, + start=None, + stop=None, + dl2=None, + simulated=None, + observation_info=None, + keep_order=True, + ): """Read subarray-based event information. + Parameters + ---------- + + dl2: bool + load available dl2 stereo parameters + simulated: bool + load simulated shower information + observation_info: bool + join observation information to each event + start: int + First *subarray* event to read + stop: int + Last *subarray* event (non-inclusive) + Returns ------- table: astropy.io.Table Table with primary index columns "obs_id" and "event_id". """ + # Setting None flags to traits default values + locals().update( + self._fallback_to_traits( + dl2=dl2, simulated=simulated, observation_info=observation_info + ) + ) + table = read_table(self.h5file, TRIGGER_TABLE, start=start, stop=stop) if keep_order: self._add_index_if_needed(table) - if self.load_simulated and SHOWER_TABLE in self.h5file: + if simulated and SHOWER_TABLE in self.h5file: showers = read_table(self.h5file, SHOWER_TABLE, start=start, stop=stop) table = _merge_subarray_tables(table, showers) - if self.load_dl2: + if dl2: if DL2_SUBARRAY_GROUP in self.h5file: for group_name in self.h5file.root[DL2_SUBARRAY_GROUP]._v_children: group_path = f"{DL2_SUBARRAY_GROUP}/{group_name}" @@ -400,7 +424,7 @@ def read_subarray_events(self, start=None, stop=None, keep_order=True): ) table = _merge_subarray_tables(table, dl2) - if self.load_observation_info: + if observation_info: table = self._join_observation_info(table) if keep_order: @@ -424,7 +448,20 @@ def read_subarray_events_chunked(self, chunk_size, *args, **kwargs): kwargs=kwargs, ) - def _read_telescope_events_for_id(self, tel_id, start=None, stop=None): + def _read_telescope_events_for_id( + self, + tel_id, + dl1_images, + dl1_parameters, + dl1_muons, + dl2, + simulated, + true_images, + true_parameters, + instrument, + start=None, + stop=None, + ): """Read telescope-based event information for a single telescope. This is the most low-level function doing the actual reading. @@ -434,15 +471,44 @@ def _read_telescope_events_for_id(self, tel_id, start=None, stop=None): tel_id: int Telescope identification number. start: int - First subarray event index to read + First *subarray* event to read stop: int - Last subarray event index to read + Last *subarray* event (non-inclusive) + dl1_images: bool + load extracted images + dl1_parameters: bool + load reconstructed image parameters + dl1_muons: bool + load muon ring parameters + dl2: bool + load available dl2 stereo parameters + simulated: bool + load simulated shower information + true_images: bool + load simulated shower images + true_parameters: bool + load image parameters obtained from true images + instrument: bool + join subarray instrument information to each event Returns ------- table: astropy.io.Table Table with primary index columns "obs_id", "event_id" and "tel_id". """ + # Setting None flags to traits default values + locals().update( + self._fallback_to_traits( + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + ) + ) if tel_id is None: raise ValueError("Please, specify a telescope ID.") @@ -468,25 +534,25 @@ def _read_telescope_events_for_id(self, tel_id, start=None, stop=None): stop=trigger_stop, ) - if self.load_dl1_parameters: + if dl1_parameters: parameters = self._read_telescope_table( PARAMETERS_GROUP, tel_id, start=tel_start, stop=tel_stop ) table = _merge_telescope_tables(table, parameters) - if self.load_dl1_muons: + if dl1_muons: muon_parameters = self._read_telescope_table( MUON_GROUP, tel_id, start=tel_start, stop=tel_stop ) table = _merge_telescope_tables(table, muon_parameters) - if self.load_dl1_images: + if dl1_images: images = self._read_telescope_table( IMAGES_GROUP, tel_id, start=tel_start, stop=tel_stop ) table = _merge_telescope_tables(table, images) - if self.load_dl2: + if dl2: if DL2_TELESCOPE_GROUP in self.h5file: dl2_tel_group = self.h5file.root[DL2_TELESCOPE_GROUP] for group_name in dl2_tel_group._v_children: @@ -503,24 +569,25 @@ def _read_telescope_events_for_id(self, tel_id, start=None, stop=None): table = _merge_telescope_tables(table, dl2) - if self.load_true_images: + if true_images: true_images = self._read_telescope_table( TRUE_IMAGES_GROUP, tel_id, start=tel_start, stop=tel_stop ) table = _merge_telescope_tables(table, true_images) - if self.load_true_parameters: + if true_parameters: true_parameters = self._read_telescope_table( TRUE_PARAMETERS_GROUP, tel_id, start=tel_start, stop=tel_stop ) table = _join_telescope_events(table, true_parameters) - if self.load_instrument: + if instrument: + self.instrument_table = self.subarray.to_table("joined") table = join_allow_empty( table, self.instrument_table, keys=["tel_id"], join_type="left" ) - if self.load_simulated and TRUE_IMPACT_GROUP in self.h5file.root: + if simulated and TRUE_IMPACT_GROUP in self.h5file.root: impacts = self._read_telescope_table( TRUE_IMPACT_GROUP, tel_id, @@ -531,9 +598,34 @@ def _read_telescope_events_for_id(self, tel_id, start=None, stop=None): return table - def _read_telescope_events_for_ids(self, tel_ids, start=None, stop=None): + def _read_telescope_events_for_ids( + self, + tel_ids, + dl1_images, + dl1_parameters, + dl1_muons, + dl2, + simulated, + true_images, + true_parameters, + instrument, + start=None, + stop=None, + ): tables = [ - self._read_telescope_events_for_id(tel_id, start=start, stop=stop) + self._read_telescope_events_for_id( + tel_id, + start=start, + stop=stop, + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + ) for tel_id in tel_ids ] return vstack(tables) @@ -572,7 +664,20 @@ def _get_tel_start_stop(self, tel_id, start, stop): return tel_start, tel_stop - def read_telescope_events(self, telescopes=None, start=None, stop=None): + def read_telescope_events( + self, + telescopes=None, + start=None, + stop=None, + dl1_images=None, + dl1_parameters=None, + dl1_muons=None, + dl2=None, + simulated=None, + true_images=None, + true_parameters=None, + instrument=None, + ): """ Read telescope-based event information. @@ -594,19 +699,61 @@ def read_telescope_events(self, telescopes=None, start=None, stop=None): First *subarray* event to read stop: int Last *subarray* event (non-inclusive) + dl1_images: bool + load extracted images + dl1_parameters: bool + load reconstructed image parameters + dl1_muons: bool + load muon ring parameters + dl2: bool + load available dl2 stereo parameters + simulated: bool + load simulated shower information + true_images: bool + load simulated shower images + true_parameters: bool + load image parameters obtained from true images + instrument: bool + join subarray instrument information to each event Returns ------- events: astropy.io.Table Table with primary index columns "obs_id", "event_id" and "tel_id". """ + # Setting None flags to traits default values + locals().update( + self._fallback_to_traits( + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + ) + ) if telescopes is None: tel_ids = tuple(self.subarray.tel.keys()) else: tel_ids = self.subarray.get_tel_ids(telescopes) - table = self._read_telescope_events_for_ids(tel_ids, start, stop) + table = self._read_telescope_events_for_ids( + tel_ids, + start, + stop, + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + ) + table = self._join_subarray_info(table, start=start, stop=stop) # sort back to order in the file @@ -666,14 +813,41 @@ def _n_total_telescope_events(self): return self._n_telescope_events.sum(axis=1) def read_telescope_events_by_type( - self, telescopes=None, start=None, stop=None + self, + telescopes=None, + start=None, + stop=None, + dl1_images=None, + dl1_parameters=None, + dl1_muons=None, + dl2=None, + simulated=None, + true_images=None, + true_parameters=None, + instrument=None, ) -> Dict[str, Table]: - """Read telescope-based event information. + """Read subarray-based event information. Parameters ---------- telescopes: List[Union[int, str, TelescopeDescription]] Any list containing a combination of telescope IDs or telescope_descriptions. + dl1_images: bool + load extracted images + dl1_parameters: bool + load reconstructed image parameters + dl1_muons: bool + load muon ring parameters + dl2: bool + load available dl2 stereo parameters + simulated: bool + load simulated shower information + true_images: bool + load simulated shower images + true_parameters: bool + load image parameters obtained from true images + instrument: bool + join subarray instrument information to each event Returns ------- @@ -681,6 +855,19 @@ def read_telescope_events_by_type( Dictionary of tables organized by telescope types Table with primary index columns "obs_id", "event_id" and "tel_id". """ + # Setting None flags to traits default values + locals().update( + self._fallback_to_traits( + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + ) + ) if telescopes is None: tel_ids = tuple(self.subarray.tel.keys()) @@ -695,7 +882,20 @@ def read_telescope_events_by_type( by_type = defaultdict(list) for tel_id in tel_ids: key = str(self.subarray.tel[tel_id]) - table = self._read_telescope_events_for_id(tel_id, start=start, stop=stop) + table = self._read_telescope_events_for_id( + tel_id, + start=start, + stop=stop, + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + ) + if len(table) > 0: by_type[key].append(table) @@ -730,14 +930,41 @@ def read_telescope_events_by_type_chunked(self, chunk_size, *args, **kwargs): ) def read_telescope_events_by_id( - self, telescopes=None, start=None, stop=None + self, + telescopes=None, + start=None, + stop=None, + dl1_images=None, + dl1_parameters=None, + dl1_muons=None, + dl2=None, + simulated=None, + true_images=None, + true_parameters=None, + instrument=None, ) -> Dict[int, Table]: - """Read telescope-based event information. + """Read subarray-based event information. Parameters ---------- telescopes: List[Union[int, str, TelescopeDescription]] Any list containing a combination of telescope IDs or telescope_descriptions. + dl1_images: bool + load extracted images + dl1_parameters: bool + load reconstructed image parameters + dl1_muons: bool + load muon ring parameters + dl2: bool + load available dl2 stereo parameters + simulated: bool + load simulated shower information + true_images: bool + load simulated shower images + true_parameters: bool + load image parameters obtained from true images + instrument: bool + join subarray instrument information to each event Returns ------- @@ -745,6 +972,19 @@ def read_telescope_events_by_id( Dictionary of tables organized by telescope ids Table with primary index columns "obs_id", "event_id" and "tel_id". """ + # Setting None flags to traits default values + locals().update( + self._fallback_to_traits( + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + ) + ) if telescopes is None: tel_ids = tuple(self.subarray.tel.keys()) @@ -759,7 +999,19 @@ def read_telescope_events_by_id( by_id = {} for tel_id in tel_ids: # no events for this telescope in range start/stop - table = self._read_telescope_events_for_id(tel_id, start=start, stop=stop) + table = self._read_telescope_events_for_id( + tel_id, + start=start, + stop=stop, + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + ) if len(table) > 0: by_id[tel_id] = table diff --git a/ctapipe/io/tests/test_table_loader.py b/ctapipe/io/tests/test_table_loader.py index 864a53ba9f7..da22628526b 100644 --- a/ctapipe/io/tests/test_table_loader.py +++ b/ctapipe/io/tests/test_table_loader.py @@ -70,17 +70,17 @@ def test_telescope_events_for_tel_id(dl1_file): """Test loading data for a single telescope""" from ctapipe.io.tableloader import TableLoader - loader = TableLoader(dl1_file, load_dl1_parameters=True) + loader = TableLoader(dl1_file) with loader as table_loader: - table = table_loader.read_telescope_events([8]) + table = table_loader.read_telescope_events([8], dl1_parameters=True) assert "hillas_length" in table.colnames assert "time" in table.colnames assert "event_type" in table.colnames assert np.all(table["tel_id"] == 8) - with TableLoader(dl1_file, load_dl1_images=True) as table_loader: - table = table_loader.read_telescope_events([8]) + with TableLoader(dl1_file) as table_loader: + table = table_loader.read_telescope_events([8], dl1_images=True) assert "image" in table.colnames assert np.all(table["tel_id"] == 8) assert table["obs_id"].dtype == np.int32 @@ -94,11 +94,11 @@ def test_telescope_muon_events_for_tel_id(dl1_muon_output_file): with TableLoader( dl1_muon_output_file, - load_dl1_muons=True, - load_dl1_parameters=False, focal_length_choice="EQUIVALENT", ) as table_loader: - table = table_loader.read_telescope_events([1]) + table = table_loader.read_telescope_events( + [1], dl1_muons=True, dl1_parameters=False + ) assert "muonring_radius" in table.colnames assert "muonparameters_containment" in table.colnames assert "muonefficiency_optical_efficiency" in table.colnames @@ -107,27 +107,27 @@ def test_telescope_muon_events_for_tel_id(dl1_muon_output_file): assert not table_loader.h5file.isopen -def test_load_instrument(dl1_file): +def test_instrument(dl1_file): """Test joining instrument data onto telescope events""" from ctapipe.io.tableloader import TableLoader - with TableLoader(dl1_file, load_instrument=True) as table_loader: + with TableLoader(dl1_file) as table_loader: expected = table_loader.subarray.tel[8].optics.equivalent_focal_length - table = table_loader.read_telescope_events([8]) + table = table_loader.read_telescope_events([8], instrument=True) assert "equivalent_focal_length" in table.colnames assert np.all(table["equivalent_focal_length"] == expected) -def test_load_simulated(dl1_file): +def test_simulated(dl1_file): """Test joining simulation info onto telescope events""" from ctapipe.io.tableloader import TableLoader - with TableLoader(dl1_file, load_simulated=True) as table_loader: - table = table_loader.read_subarray_events() + with TableLoader(dl1_file) as table_loader: + table = table_loader.read_subarray_events(simulated=True) assert "true_energy" in table.colnames assert table["obs_id"].dtype == np.int32 - table = table_loader.read_telescope_events([8]) + table = table_loader.read_telescope_events([8], simulated=True) assert "true_energy" in table.colnames assert "true_impact_distance" in table.colnames @@ -136,10 +136,10 @@ def test_true_images(dl1_file): """Test joining true images onto telescope events""" from ctapipe.io.tableloader import TableLoader - with TableLoader( - dl1_file, load_dl1_parameters=False, load_true_images=True - ) as table_loader: - table = table_loader.read_telescope_events(["MST_MST_NectarCam"]) + with TableLoader(dl1_file) as table_loader: + table = table_loader.read_telescope_events( + ["MST_MST_NectarCam"], dl1_parameters=False, true_images=True + ) assert "true_image" in table.colnames @@ -147,10 +147,10 @@ def test_true_parameters(dl1_file): """Test joining true parameters onto telescope events""" from ctapipe.io.tableloader import TableLoader - with TableLoader( - dl1_file, load_dl1_parameters=False, load_true_parameters=True - ) as table_loader: - table = table_loader.read_telescope_events() + with TableLoader(dl1_file) as table_loader: + table = table_loader.read_telescope_events( + dl1_parameters=False, true_parameters=True + ) assert "true_hillas_intensity" in table.colnames @@ -158,8 +158,8 @@ def test_observation_info(dl1_file): """Test joining observation info onto telescope events""" from ctapipe.io.tableloader import TableLoader - with TableLoader(dl1_file, load_observation_info=True) as table_loader: - table = table_loader.read_telescope_events() + with TableLoader(dl1_file) as table_loader: + table = table_loader.read_telescope_events(observation_info=True) assert "subarray_pointing_lat" in table.colnames @@ -167,12 +167,8 @@ def test_read_subarray_events(dl2_shower_geometry_file): """Test reading subarray events""" from ctapipe.io.tableloader import TableLoader - with TableLoader( - dl2_shower_geometry_file, - load_dl2=True, - load_simulated=True, - ) as table_loader: - table = table_loader.read_subarray_events() + with TableLoader(dl2_shower_geometry_file) as table_loader: + table = table_loader.read_subarray_events(dl2=True, simulated=True) assert "HillasReconstructor_alt" in table.colnames assert "true_energy" in table.colnames assert "time" in table.colnames @@ -187,11 +183,9 @@ def test_table_loader_keeps_original_order(dl2_merged_file): # check we actually have unsorted input assert not np.all(np.diff(trigger["obs_id"]) >= 0) - with TableLoader( - dl2_merged_file, load_dl2=True, load_simulated=True - ) as table_loader: - events = table_loader.read_subarray_events() - tel_events = table_loader.read_telescope_events() + with TableLoader(dl2_merged_file) as table_loader: + events = table_loader.read_subarray_events(dl2=True, simulated=True) + tel_events = table_loader.read_telescope_events(dl2=True, simulated=True) check_equal_array_event_order(events, trigger) check_equal_array_event_order(events, tel_events) @@ -204,17 +198,17 @@ def test_read_telescope_events_type(dl2_shower_geometry_file): subarray = SubarrayDescription.from_hdf(dl2_shower_geometry_file) - with TableLoader( - dl2_shower_geometry_file, - load_dl1_images=False, - load_dl1_parameters=False, - load_dl2=True, - load_simulated=True, - load_true_images=True, - load_instrument=True, - ) as table_loader: + with TableLoader(dl2_shower_geometry_file) as table_loader: - table = table_loader.read_telescope_events(["MST_MST_FlashCam"]) + table = table_loader.read_telescope_events( + ["MST_MST_FlashCam"], + dl1_images=False, + dl1_parameters=False, + dl2=True, + simulated=True, + true_images=True, + instrument=True, + ) assert "HillasReconstructor_alt" in table.colnames assert "true_energy" in table.colnames @@ -233,17 +227,17 @@ def test_read_telescope_events_by_type(dl2_shower_geometry_file): subarray = SubarrayDescription.from_hdf(dl2_shower_geometry_file) - with TableLoader( - dl2_shower_geometry_file, - load_dl1_images=False, - load_dl1_parameters=False, - load_dl2=True, - load_simulated=True, - load_true_images=True, - load_instrument=True, - ) as table_loader: + with TableLoader(dl2_shower_geometry_file) as table_loader: - tables = table_loader.read_telescope_events_by_type([25, 130]) + tables = table_loader.read_telescope_events_by_type( + [25, 130], + dl1_images=False, + dl1_parameters=False, + dl2=True, + simulated=True, + true_images=True, + instrument=True, + ) for tel_type in ["MST_MST_NectarCam", "MST_MST_FlashCam"]: @@ -287,19 +281,40 @@ def test_chunked(dl2_shower_geometry_file): start = 0 stop = chunk_size - with TableLoader( - dl2_shower_geometry_file, - load_dl1_images=False, - load_true_images=False, - load_dl1_parameters=True, - load_dl2=True, - load_simulated=True, - ) as table_loader: + with TableLoader(dl2_shower_geometry_file) as table_loader: - tel_event_it = table_loader.read_telescope_events_chunked(chunk_size) - event_it = table_loader.read_subarray_events_chunked(chunk_size) - by_type_it = table_loader.read_telescope_events_by_type_chunked(chunk_size) - by_id_it = table_loader.read_telescope_events_by_id_chunked(chunk_size) + tel_event_it = table_loader.read_telescope_events_chunked( + chunk_size, + dl1_images=False, + true_images=False, + dl1_parameters=True, + dl2=True, + simulated=True, + ) + event_it = table_loader.read_subarray_events_chunked( + chunk_size, + dl1_images=False, + true_images=False, + dl1_parameters=True, + dl2=True, + simulated=True, + ) + by_type_it = table_loader.read_telescope_events_by_type_chunked( + chunk_size, + dl1_images=False, + true_images=False, + dl1_parameters=True, + dl2=True, + simulated=True, + ) + by_id_it = table_loader.read_telescope_events_by_id_chunked( + chunk_size, + dl1_images=False, + true_images=False, + dl1_parameters=True, + dl2=True, + simulated=True, + ) iters = (event_it, tel_event_it, by_type_it, by_id_it) @@ -386,26 +401,26 @@ def test_read_unavailable_telescope(dl2_shower_geometry_file): """Reading a telescope that is not part of the subarray of the file should fail.""" from ctapipe.io import TableLoader - with TableLoader( - dl2_shower_geometry_file, - load_dl1_parameters=False, - load_dl2=True, - ) as loader: + with TableLoader(dl2_shower_geometry_file) as loader: tel_id = max(loader.subarray.tel.keys()) + 1 with pytest.raises(ValueError): - loader.read_telescope_events([tel_id]) + loader.read_telescope_events( + [tel_id], + dl1_parameters=False, + dl2=True, + ) def test_read_empty_table(dl2_shower_geometry_file): """Reading an empty table should return an empty table.""" from ctapipe.io import TableLoader - with TableLoader( - dl2_shower_geometry_file, - load_dl1_parameters=False, - load_dl2=True, - ) as loader: - table = loader.read_telescope_events([6]) + with TableLoader(dl2_shower_geometry_file) as loader: + table = loader.read_telescope_events( + [6], + dl1_parameters=False, + dl2=True, + ) assert len(table) == 0 @@ -417,13 +432,8 @@ def test_order_merged(): trigger = read_table(path, "/dl1/event/subarray/trigger") tel_trigger = read_table(path, "/dl1/event/telescope/trigger") - with TableLoader( - path, - load_dl1_parameters=True, - load_dl2=True, - load_observation_info=True, - ) as loader: - events = loader.read_subarray_events() + with TableLoader(path) as loader: + events = loader.read_subarray_events(dl2=True, observation_info=True) check_equal_array_event_order(events, trigger) tables = loader.read_telescope_events_by_id() From 866f69ba0abcc3c909c5e626feb1b862e8c11a9f Mon Sep 17 00:00:00 2001 From: Christophe COSSOU Date: Fri, 27 Oct 2023 09:22:01 +0200 Subject: [PATCH 02/14] Fix call order for read_telescope_events --- ctapipe/io/tableloader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index 74d7ccea347..dd5e1c6d5fa 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -615,8 +615,6 @@ def _read_telescope_events_for_ids( tables = [ self._read_telescope_events_for_id( tel_id, - start=start, - stop=stop, dl1_images=dl1_images, dl1_parameters=dl1_parameters, dl1_muons=dl1_muons, @@ -625,6 +623,8 @@ def _read_telescope_events_for_ids( true_images=true_images, true_parameters=true_parameters, instrument=instrument, + start=start, + stop=stop, ) for tel_id in tel_ids ] @@ -742,8 +742,6 @@ def read_telescope_events( table = self._read_telescope_events_for_ids( tel_ids, - start, - stop, dl1_images=dl1_images, dl1_parameters=dl1_parameters, dl1_muons=dl1_muons, @@ -752,6 +750,8 @@ def read_telescope_events( true_images=true_images, true_parameters=true_parameters, instrument=instrument, + start=start, + stop=stop, ) table = self._join_subarray_info(table, start=start, stop=stop) From 31be16f435024175f3eef0a0ee9da472bba29b82 Mon Sep 17 00:00:00 2001 From: Christophe COSSOU Date: Fri, 27 Oct 2023 14:44:59 +0200 Subject: [PATCH 03/14] Fix all unit test but the chunks one. --- ctapipe/io/tableloader.py | 46 +++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index dd5e1c6d5fa..f113fd6a0c2 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -609,6 +609,7 @@ def _read_telescope_events_for_ids( true_images, true_parameters, instrument, + observation_info, start=None, stop=None, ): @@ -630,11 +631,24 @@ def _read_telescope_events_for_ids( ] return vstack(tables) - def _join_subarray_info(self, table, start=None, stop=None, subarray_events=None): + def _join_subarray_info( + self, + table, + start=None, + stop=None, + dl2=None, + simulated=None, + observation_info=None, + subarray_events=None, + ): + if subarray_events is None: subarray_events = self.read_subarray_events( start=start, stop=stop, + dl2=dl2, + simulated=simulated, + observation_info=observation_info, keep_order=False, ) table = join_allow_empty( @@ -677,6 +691,7 @@ def read_telescope_events( true_images=None, true_parameters=None, instrument=None, + observation_info=None, ): """ Read telescope-based event information. @@ -732,6 +747,7 @@ def read_telescope_events( true_images=true_images, true_parameters=true_parameters, instrument=instrument, + observation_info=observation_info, ) ) @@ -750,11 +766,19 @@ def read_telescope_events( true_images=true_images, true_parameters=true_parameters, instrument=instrument, + observation_info=observation_info, start=start, stop=stop, ) - table = self._join_subarray_info(table, start=start, stop=stop) + table = self._join_subarray_info( + table, + dl2=dl2, + simulated=simulated, + observation_info=observation_info, + start=start, + stop=stop, + ) # sort back to order in the file table = _join_subarray_events( @@ -824,6 +848,7 @@ def read_telescope_events_by_type( simulated=None, true_images=None, true_parameters=None, + observation_info=None, instrument=None, ) -> Dict[str, Table]: """Read subarray-based event information. @@ -866,6 +891,7 @@ def read_telescope_events_by_type( true_images=true_images, true_parameters=true_parameters, instrument=instrument, + observation_info=observation_info, ) ) @@ -875,7 +901,12 @@ def read_telescope_events_by_type( tel_ids = self.subarray.get_tel_ids(telescopes) subarray_events = self.read_subarray_events( - start=start, stop=stop, keep_order=False + start=start, + stop=stop, + dl2=dl2, + simulated=simulated, + observation_info=observation_info, + keep_order=False, ) self._add_index_if_needed(subarray_events) @@ -942,6 +973,7 @@ def read_telescope_events_by_id( true_images=None, true_parameters=None, instrument=None, + observation_info=None, ) -> Dict[int, Table]: """Read subarray-based event information. @@ -983,6 +1015,7 @@ def read_telescope_events_by_id( true_images=true_images, true_parameters=true_parameters, instrument=instrument, + observation_info=observation_info, ) ) @@ -992,7 +1025,12 @@ def read_telescope_events_by_id( tel_ids = self.subarray.get_tel_ids(telescopes) subarray_events = self.read_subarray_events( - start=start, stop=stop, keep_order=False + start=start, + stop=stop, + dl2=dl2, + simulated=simulated, + observation_info=observation_info, + keep_order=False, ) self._add_index_if_needed(subarray_events) From 4e1f21c135403f025a082e04dd0ecc2b66ffd0f4 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Fri, 27 Oct 2023 15:01:52 +0200 Subject: [PATCH 04/14] Fix chunked test - function call for reading subarray events passes wrong parameters --- ctapipe/io/tests/test_table_loader.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/ctapipe/io/tests/test_table_loader.py b/ctapipe/io/tests/test_table_loader.py index da22628526b..b801d1e3846 100644 --- a/ctapipe/io/tests/test_table_loader.py +++ b/ctapipe/io/tests/test_table_loader.py @@ -199,7 +199,6 @@ def test_read_telescope_events_type(dl2_shower_geometry_file): subarray = SubarrayDescription.from_hdf(dl2_shower_geometry_file) with TableLoader(dl2_shower_geometry_file) as table_loader: - table = table_loader.read_telescope_events( ["MST_MST_FlashCam"], dl1_images=False, @@ -228,7 +227,6 @@ def test_read_telescope_events_by_type(dl2_shower_geometry_file): subarray = SubarrayDescription.from_hdf(dl2_shower_geometry_file) with TableLoader(dl2_shower_geometry_file) as table_loader: - tables = table_loader.read_telescope_events_by_type( [25, 130], dl1_images=False, @@ -240,7 +238,6 @@ def test_read_telescope_events_by_type(dl2_shower_geometry_file): ) for tel_type in ["MST_MST_NectarCam", "MST_MST_FlashCam"]: - table = tables[tel_type] assert "HillasReconstructor_alt" in table.colnames @@ -282,44 +279,52 @@ def test_chunked(dl2_shower_geometry_file): stop = chunk_size with TableLoader(dl2_shower_geometry_file) as table_loader: - tel_event_it = table_loader.read_telescope_events_chunked( chunk_size, dl1_images=False, - true_images=False, dl1_parameters=True, + dl1_muons=False, dl2=True, simulated=True, + true_images=False, + true_parameters=False, + instrument=False, + observation_info=False, ) event_it = table_loader.read_subarray_events_chunked( chunk_size, - dl1_images=False, - true_images=False, - dl1_parameters=True, dl2=True, simulated=True, + observation_info=False, ) by_type_it = table_loader.read_telescope_events_by_type_chunked( chunk_size, dl1_images=False, - true_images=False, dl1_parameters=True, + dl1_muons=False, dl2=True, simulated=True, + true_images=False, + true_parameters=False, + instrument=False, + observation_info=False, ) by_id_it = table_loader.read_telescope_events_by_id_chunked( chunk_size, dl1_images=False, - true_images=False, dl1_parameters=True, + dl1_muons=False, dl2=True, simulated=True, + true_images=False, + true_parameters=False, + instrument=False, + observation_info=False, ) iters = (event_it, tel_event_it, by_type_it, by_id_it) for chunk, (events, tel_events, by_type, by_id) in enumerate(zip(*iters)): - expected_start = chunk * chunk_size expected_stop = min(n_events, (chunk + 1) * chunk_size) From ddcf4c0b27aad5bfcd5a9ed620aace411055d23f Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Wed, 15 Nov 2023 14:29:48 +0100 Subject: [PATCH 05/14] Adding args check function and adapting tests --- ctapipe/instrument/tests/test_trigger.py | 24 ++- ctapipe/io/tableloader.py | 203 ++++++++++++--------- ctapipe/tools/apply_models.py | 8 +- ctapipe/tools/tests/test_apply_models.py | 28 ++- ctapipe/tools/tests/test_merge.py | 9 +- ctapipe/tools/tests/test_process.py | 16 +- ctapipe/tools/train_disp_reconstructor.py | 12 +- ctapipe/tools/train_energy_regressor.py | 11 +- ctapipe/tools/train_particle_classifier.py | 16 +- examples/tutorials/ctapipe_overview.py | 13 +- 10 files changed, 196 insertions(+), 144 deletions(-) diff --git a/ctapipe/instrument/tests/test_trigger.py b/ctapipe/instrument/tests/test_trigger.py index fec25900e6e..0f7a375df0e 100644 --- a/ctapipe/instrument/tests/test_trigger.py +++ b/ctapipe/instrument/tests/test_trigger.py @@ -105,7 +105,6 @@ def test_software_trigger_simtel(allowed_tels): with EventSource( path, focal_length_choice="EQUIVALENT", allowed_tels=allowed_tels ) as source: - trigger = SoftwareTrigger( subarray=source.subarray, min_telescopes=2, @@ -152,7 +151,6 @@ def test_software_trigger_simtel_single_lsts(): with EventSource( path, focal_length_choice="EQUIVALENT", allowed_tels=allowed_tels ) as source: - trigger = SoftwareTrigger( subarray=source.subarray, min_telescopes=2, @@ -218,18 +216,28 @@ def test_software_trigger_simtel_process(tmp_path): with TableLoader( output_path, - load_simulated=True, - load_dl1_parameters=True, focal_length_choice="EQUIVALENT", ) as loader: - events_trigger = loader.read_telescope_events("LST_LST_LSTCam") + events_trigger = loader.read_telescope_events( + "LST_LST_LSTCam", + dl1_muons=False, + dl2=False, + true_parameters=False, + instrument=False, + observation_info=False, + ) with TableLoader( output_path_no_software_trigger, - load_simulated=True, - load_dl1_parameters=True, focal_length_choice="EQUIVALENT", ) as loader: - events_no_trigger = loader.read_telescope_events("LST_LST_LSTCam") + events_no_trigger = loader.read_telescope_events( + "LST_LST_LSTCam", + dl1_muons=False, + dl2=False, + true_parameters=False, + instrument=False, + observation_info=False, + ) assert len(events_no_trigger) > len(events_trigger) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index f113fd6a0c2..c69a8af31e8 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -82,7 +82,6 @@ def __len__(self): return self.n_chunks def __getitem__(self, chunk): - if chunk < 0: chunk = self.n_chunks - chunk @@ -188,7 +187,7 @@ class TableLoader(Component): simulated = traits.Bool(True, help="load simulated shower information").tag( config=True ) - true_images = traits.Bool(True, help="load simulated shower images").tag( + true_images = traits.Bool(False, help="load simulated shower images").tag( config=True ) true_parameters = traits.Bool( @@ -243,19 +242,6 @@ def __init__(self, input_url=None, h5file=None, **kwargs): Provenance().add_input_file(self.input_url, role="Event data") - groups = { - "dl1_parameters": PARAMETERS_GROUP, - "dl1_images": IMAGES_GROUP, - "true_parameters": TRUE_PARAMETERS_GROUP, - "true_images": TRUE_IMAGES_GROUP, - } - for attr, group in groups.items(): - if getattr(self, attr) and group not in self.h5file.root: - self.log.info( - "Setting %s to False, input file does not contain such data", attr - ) - setattr(self, attr, False) - def close(self): """Close the underlying hdf5 file""" if self._should_close: @@ -274,10 +260,40 @@ def __len__(self): """Number of subarray events in input file""" return self.h5file.root[TRIGGER_TABLE].shape[0] + def _check_args(self, **kwargs): + """Checking args: + 1) If None, set to default (trait) value + 2) If True but correlated group is not included in input file - set to False + returns a dict with new args""" + groups = { + "dl1_parameters": PARAMETERS_GROUP, + "dl1_images": IMAGES_GROUP, + "dl1_muons": MUON_GROUP, + "true_parameters": TRUE_PARAMETERS_GROUP, + "true_images": TRUE_IMAGES_GROUP, + "observation_info": OBSERVATION_TABLE, + } + updated_attributes = {} + for key, value in kwargs.items(): + updated_value = value + if updated_value is None: + updated_value = getattr(self, key) + if ( + updated_value + and key in groups.keys() + and groups[key] not in self.h5file.root + ): + self.log.info( + "Setting %s to False, input file does not contain such data", key + ) + updated_value = False + updated_attributes[key] = updated_value + return updated_attributes + def _read_telescope_table(self, group, tel_id, start=None, stop=None): key = f"{group}/tel_{tel_id:03d}" - if key in self.h5file: + if key in self.h5file.root: table = read_table(self.h5file, key, start=start, stop=stop) else: table = _empty_telescope_events_table() @@ -361,9 +377,6 @@ def _join_observation_info(self, table): table.remove_column("__index__") return joint - def _fallback_to_traits(self, **kwargs): - return {k: v if v is not None else getattr(self, k) for k, v in kwargs.items()} - def read_subarray_events( self, start=None, @@ -394,13 +407,14 @@ def read_subarray_events( table: astropy.io.Table Table with primary index columns "obs_id" and "event_id". """ - # Setting None flags to traits default values - locals().update( - self._fallback_to_traits( - dl2=dl2, simulated=simulated, observation_info=observation_info - ) + updated_attributes = self._check_args( + dl2=dl2, simulated=simulated, observation_info=observation_info ) + dl2 = updated_attributes["dl2"] + simulated = updated_attributes["simulated"] + observation_info = updated_attributes["observation_info"] + table = read_table(self.h5file, TRIGGER_TABLE, start=start, stop=stop) if keep_order: self._add_index_if_needed(table) @@ -496,19 +510,6 @@ def _read_telescope_events_for_id( table: astropy.io.Table Table with primary index columns "obs_id", "event_id" and "tel_id". """ - # Setting None flags to traits default values - locals().update( - self._fallback_to_traits( - dl1_images=dl1_images, - dl1_parameters=dl1_parameters, - dl1_muons=dl1_muons, - dl2=dl2, - simulated=simulated, - true_images=true_images, - true_parameters=true_parameters, - instrument=instrument, - ) - ) if tel_id is None: raise ValueError("Please, specify a telescope ID.") @@ -609,7 +610,6 @@ def _read_telescope_events_for_ids( true_images, true_parameters, instrument, - observation_info, start=None, stop=None, ): @@ -634,14 +634,13 @@ def _read_telescope_events_for_ids( def _join_subarray_info( self, table, + dl2, + simulated, + observation_info, start=None, stop=None, - dl2=None, - simulated=None, - observation_info=None, subarray_events=None, ): - if subarray_events is None: subarray_events = self.read_subarray_events( start=start, @@ -730,27 +729,36 @@ def read_telescope_events( load image parameters obtained from true images instrument: bool join subarray instrument information to each event + observation_info: bool + join observation information to each event Returns ------- events: astropy.io.Table Table with primary index columns "obs_id", "event_id" and "tel_id". """ - # Setting None flags to traits default values - locals().update( - self._fallback_to_traits( - dl1_images=dl1_images, - dl1_parameters=dl1_parameters, - dl1_muons=dl1_muons, - dl2=dl2, - simulated=simulated, - true_images=true_images, - true_parameters=true_parameters, - instrument=instrument, - observation_info=observation_info, - ) + updated_attributes = self._check_args( + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + observation_info=observation_info, ) + dl1_images = updated_attributes["dl1_images"] + dl1_parameters = updated_attributes["dl1_parameters"] + dl1_muons = updated_attributes["dl1_muons"] + dl2 = updated_attributes["dl2"] + simulated = updated_attributes["simulated"] + true_images = updated_attributes["true_images"] + true_parameters = updated_attributes["true_parameters"] + instrument = updated_attributes["instrument"] + observation_info = updated_attributes["observation_info"] + if telescopes is None: tel_ids = tuple(self.subarray.tel.keys()) else: @@ -766,7 +774,6 @@ def read_telescope_events( true_images=true_images, true_parameters=true_parameters, instrument=instrument, - observation_info=observation_info, start=start, stop=stop, ) @@ -848,8 +855,8 @@ def read_telescope_events_by_type( simulated=None, true_images=None, true_parameters=None, - observation_info=None, instrument=None, + observation_info=None, ) -> Dict[str, Table]: """Read subarray-based event information. @@ -873,6 +880,8 @@ def read_telescope_events_by_type( load image parameters obtained from true images instrument: bool join subarray instrument information to each event + observation_info: bool + join observation information to each event Returns ------- @@ -880,21 +889,28 @@ def read_telescope_events_by_type( Dictionary of tables organized by telescope types Table with primary index columns "obs_id", "event_id" and "tel_id". """ - # Setting None flags to traits default values - locals().update( - self._fallback_to_traits( - dl1_images=dl1_images, - dl1_parameters=dl1_parameters, - dl1_muons=dl1_muons, - dl2=dl2, - simulated=simulated, - true_images=true_images, - true_parameters=true_parameters, - instrument=instrument, - observation_info=observation_info, - ) + updated_attributes = self._check_args( + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + observation_info=observation_info, ) + dl1_images = updated_attributes["dl1_images"] + dl1_parameters = updated_attributes["dl1_parameters"] + dl1_muons = updated_attributes["dl1_muons"] + dl2 = updated_attributes["dl2"] + simulated = updated_attributes["simulated"] + true_images = updated_attributes["true_images"] + true_parameters = updated_attributes["true_parameters"] + instrument = updated_attributes["instrument"] + observation_info = updated_attributes["observation_info"] + if telescopes is None: tel_ids = tuple(self.subarray.tel.keys()) else: @@ -933,7 +949,11 @@ def read_telescope_events_by_type( by_type = {k: vstack(ts) for k, ts in by_type.items()} for key in by_type.keys(): by_type[key] = self._join_subarray_info( - by_type[key], subarray_events=subarray_events + by_type[key], + dl2=dl2, + simulated=simulated, + observation_info=observation_info, + subarray_events=subarray_events, ) self._sort_to_original_order(by_type[key], include_tel_id=True) @@ -997,6 +1017,8 @@ def read_telescope_events_by_id( load image parameters obtained from true images instrument: bool join subarray instrument information to each event + observation_info: bool + join observation information to each event Returns ------- @@ -1004,21 +1026,28 @@ def read_telescope_events_by_id( Dictionary of tables organized by telescope ids Table with primary index columns "obs_id", "event_id" and "tel_id". """ - # Setting None flags to traits default values - locals().update( - self._fallback_to_traits( - dl1_images=dl1_images, - dl1_parameters=dl1_parameters, - dl1_muons=dl1_muons, - dl2=dl2, - simulated=simulated, - true_images=true_images, - true_parameters=true_parameters, - instrument=instrument, - observation_info=observation_info, - ) + updated_attributes = self._check_args( + dl1_images=dl1_images, + dl1_parameters=dl1_parameters, + dl1_muons=dl1_muons, + dl2=dl2, + simulated=simulated, + true_images=true_images, + true_parameters=true_parameters, + instrument=instrument, + observation_info=observation_info, ) + dl1_images = updated_attributes["dl1_images"] + dl1_parameters = updated_attributes["dl1_parameters"] + dl1_muons = updated_attributes["dl1_muons"] + dl2 = updated_attributes["dl2"] + simulated = updated_attributes["simulated"] + true_images = updated_attributes["true_images"] + true_parameters = updated_attributes["true_parameters"] + instrument = updated_attributes["instrument"] + observation_info = updated_attributes["observation_info"] + if telescopes is None: tel_ids = tuple(self.subarray.tel.keys()) else: @@ -1055,7 +1084,11 @@ def read_telescope_events_by_id( for tel_id in by_id.keys(): by_id[tel_id] = self._join_subarray_info( - by_id[tel_id], subarray_events=subarray_events + by_id[tel_id], + dl2=dl2, + simulated=simulated, + observation_info=observation_info, + subarray_events=subarray_events, ) self._sort_to_original_order(by_id[tel_id], include_tel_id=False) diff --git a/ctapipe/tools/apply_models.py b/ctapipe/tools/apply_models.py index 2df691d8bf4..3d18521a6ad 100644 --- a/ctapipe/tools/apply_models.py +++ b/ctapipe/tools/apply_models.py @@ -137,12 +137,6 @@ def setup(self): TableLoader( self.input_url, parent=self, - load_dl1_parameters=True, - load_dl2=True, - load_instrument=True, - load_dl1_images=False, - load_simulated=False, - load_observation_info=True, ) ) @@ -154,7 +148,7 @@ def setup(self): def start(self): """Apply models to input tables""" chunk_iterator = self.loader.read_telescope_events_by_id_chunked( - self.chunk_size + self.chunk_size, dl1_muons=False, simulated=False, true_parameters=False ) bar = tqdm( chunk_iterator, diff --git a/ctapipe/tools/tests/test_apply_models.py b/ctapipe/tools/tests/test_apply_models.py index 13f8edb805a..dce78d0fb40 100644 --- a/ctapipe/tools/tests/test_apply_models.py +++ b/ctapipe/tools/tests/test_apply_models.py @@ -48,9 +48,11 @@ def test_apply_energy_regressor( assert colname in table.colnames assert table[colname].description == field.description - with TableLoader(output_path, load_dl2=True) as loader: - - events = loader.read_subarray_events() + with TableLoader(output_path) as loader: + events = loader.read_subarray_events( + simulated=False, + observation_info=False, + ) assert f"{prefix}_energy" in events.colnames assert f"{prefix}_energy_uncert" in events.colnames assert f"{prefix}_is_valid" in events.colnames @@ -60,7 +62,10 @@ def test_apply_energy_regressor( np.isfinite(events[f"{prefix}_energy"][events[f"{prefix}_is_valid"]]) ) - tel_events = loader.read_telescope_events() + tel_events = loader.read_telescope_events( + simulated=False, + observation_info=False, + ) assert f"{prefix}_energy" in tel_events.colnames assert f"{prefix}_energy_uncert" in tel_events.colnames assert f"{prefix}_is_valid" in tel_events.colnames @@ -181,8 +186,11 @@ def test_apply_all( table = read_table(output_path, key) check_equal_array_event_order(tel_trigger[tel_mask], table) - with TableLoader(output_path, load_dl2=True) as loader: - events = loader.read_subarray_events() + with TableLoader(output_path) as loader: + events = loader.read_subarray_events( + simulated=False, + observation_info=False, + ) assert f"{prefix_clf}_prediction" in events.colnames assert f"{prefix_clf}_telescopes" in events.colnames assert f"{prefix_clf}_is_valid" in events.colnames @@ -193,7 +201,13 @@ def test_apply_all( assert f"{prefix_disp}_is_valid" in events.colnames assert f"{prefix_disp}_goodness_of_fit" in events.colnames - tel_events = loader.read_telescope_events() + tel_events = loader.read_telescope_events( + dl1_muons=False, + simulated=False, + true_parameters=False, + instrument=False, + observation_info=False, + ) assert f"{prefix_clf}_prediction" in tel_events.colnames assert f"{prefix_clf}_telescopes" in tel_events.colnames assert f"{prefix_clf}_is_valid" in tel_events.colnames diff --git a/ctapipe/tools/tests/test_merge.py b/ctapipe/tools/tests/test_merge.py index 1e09ad044d3..e453de1b9ae 100644 --- a/ctapipe/tools/tests/test_merge.py +++ b/ctapipe/tools/tests/test_merge.py @@ -157,8 +157,13 @@ def test_dl2(tmp_path, dl2_shower_geometry_file, dl2_proton_geometry_file): assert len(sbs) == 2, "should have two SB entries" # regression test for #2048 - loader = TableLoader(output, load_dl2=True, load_simulated=True) - tel_events = loader.read_telescope_events() + loader = TableLoader(output) + tel_events = loader.read_telescope_events( + dl1_parameters=False, + dl1_muons=False, + true_parameters=False, + instrument=False, + ) assert "true_impact_distance" in tel_events.colnames # regression test for #2051 assert "HillasReconstructor_tel_impact_distance" in tel_events.colnames diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index 01039649265..9504830505e 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -401,11 +401,13 @@ def test_read_from_simtel_and_dl1(prod5_proton_simtel_path, tmp_path): ] + few_tels assert run_tool(ProcessorTool(), argv=argv, cwd=tmp_path) == 0 - args = dict(load_dl2=True, load_simulated=True) - with TableLoader(dl2_from_simtel, **args) as loader: - events_from_simtel = loader.read_subarray_events() - with TableLoader(dl2_from_dl1, **args) as loader: - events_from_dl1 = loader.read_subarray_events() + args = dict( + observation_info=False, + ) + with TableLoader(dl2_from_simtel) as loader: + events_from_simtel = loader.read_subarray_events(**args) + with TableLoader(dl2_from_dl1) as loader: + events_from_dl1 = loader.read_subarray_events(**args) # both files should contain identical data assert_array_equal(events_from_simtel["event_id"], events_from_dl1["event_id"]) @@ -495,8 +497,8 @@ def test_only_trigger_and_simulation(tmp_path): raises=True, ) - with TableLoader(output, load_simulated=True) as loader: - events = loader.read_subarray_events() + with TableLoader(output) as loader: + events = loader.read_subarray_events(dl2=False, observation_info=False) assert len(events) == 7 assert "tels_with_trigger" in events.colnames assert "true_energy" in events.colnames diff --git a/ctapipe/tools/train_disp_reconstructor.py b/ctapipe/tools/train_disp_reconstructor.py index 5953bda8038..cf2c020d2ed 100644 --- a/ctapipe/tools/train_disp_reconstructor.py +++ b/ctapipe/tools/train_disp_reconstructor.py @@ -77,12 +77,6 @@ def setup(self): self.loader = self.enter_context( TableLoader( parent=self, - load_dl1_images=False, - load_dl1_parameters=True, - load_dl2=True, - load_simulated=True, - load_instrument=True, - load_observation_info=True, ) ) self.n_events.attach_subarray(self.loader.subarray) @@ -112,7 +106,11 @@ def start(self): self.log.info("done") def _read_table(self, telescope_type): - table = self.loader.read_telescope_events([telescope_type]) + table = self.loader.read_telescope_events( + [telescope_type], + dl1_muons=False, + true_parameters=False, + ) self.log.info("Events read from input: %d", len(table)) if len(table) == 0: raise TooFewEvents( diff --git a/ctapipe/tools/train_energy_regressor.py b/ctapipe/tools/train_energy_regressor.py index 89647f6c39b..c4c0baa6e75 100644 --- a/ctapipe/tools/train_energy_regressor.py +++ b/ctapipe/tools/train_energy_regressor.py @@ -77,11 +77,6 @@ def setup(self): self.loader = self.enter_context( TableLoader( parent=self, - load_dl1_images=False, - load_dl1_parameters=True, - load_dl2=True, - load_simulated=True, - load_instrument=True, ) ) self.n_events.attach_subarray(self.loader.subarray) @@ -113,7 +108,11 @@ def start(self): self.log.info("done") def _read_table(self, telescope_type): - table = self.loader.read_telescope_events([telescope_type]) + table = self.loader.read_telescope_events( + [telescope_type], + dl1_muons=False, + true_parameters=False, + ) self.log.info("Events read from input: %d", len(table)) if len(table) == 0: raise TooFewEvents( diff --git a/ctapipe/tools/train_particle_classifier.py b/ctapipe/tools/train_particle_classifier.py index b8511c6fd1e..8ffe4dae347 100644 --- a/ctapipe/tools/train_particle_classifier.py +++ b/ctapipe/tools/train_particle_classifier.py @@ -107,11 +107,6 @@ def setup(self): TableLoader( parent=self, input_url=self.input_url_signal, - load_dl1_images=False, - load_dl1_parameters=True, - load_dl2=True, - load_simulated=True, - load_instrument=True, ) ) @@ -119,11 +114,6 @@ def setup(self): TableLoader( parent=self, input_url=self.input_url_background, - load_dl1_images=False, - load_dl1_parameters=True, - load_dl2=True, - load_simulated=True, - load_instrument=True, ) ) @@ -162,7 +152,11 @@ def start(self): self.log.info("done") def _read_table(self, telescope_type, loader, n_events=None): - table = loader.read_telescope_events([telescope_type]) + table = loader.read_telescope_events( + [telescope_type], + dl1_muons=False, + true_parameters=False, + ) self.log.info("Events read from input: %d", len(table)) if len(table) == 0: raise TooFewEvents( diff --git a/examples/tutorials/ctapipe_overview.py b/examples/tutorials/ctapipe_overview.py index e96cea70f45..0f2e722171f 100644 --- a/examples/tutorials/ctapipe_overview.py +++ b/examples/tutorials/ctapipe_overview.py @@ -373,7 +373,6 @@ with DataWriter( source, output_path=f.name, overwrite=True, write_showers=True ) as writer: - for event in source: energy = event.simulation.shower.energy n_telescopes_r1 = len(event.r1.tel) @@ -466,9 +465,16 @@ # ~~~~~~~~~~~~~~~~~~~~~~~~ # -loader = TableLoader(f.name, load_simulated=True, load_dl1_parameters=True) +loader = TableLoader(f.name) -dl1_table = loader.read_telescope_events(["LST_LST_LSTCam"]) +dl1_table = loader.read_telescope_events( + ["LST_LST_LSTCam"], + dl1_muons=False, + dl2=False, + true_parameters=False, + instrument=False, + observation_info=False, +) ###################################################################### plt.scatter( @@ -526,7 +532,6 @@ for i in range(9): - model = toymodel.Gaussian( x=np.random.uniform(-0.8, 0.8) * u.m, y=np.random.uniform(-0.8, 0.8) * u.m, From 8c09e16be2f67970e6d23f3e2ed47da7b4ebfad6 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Wed, 15 Nov 2023 17:07:06 +0100 Subject: [PATCH 06/14] Undo debugging adaption --- ctapipe/io/tableloader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index c69a8af31e8..266f5728c09 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -262,8 +262,8 @@ def __len__(self): def _check_args(self, **kwargs): """Checking args: - 1) If None, set to default (trait) value - 2) If True but correlated group is not included in input file - set to False + 1) If None, set to default (trait) value. + 2) If True but correlated group is not included in input file - set to False. returns a dict with new args""" groups = { "dl1_parameters": PARAMETERS_GROUP, @@ -293,7 +293,7 @@ def _check_args(self, **kwargs): def _read_telescope_table(self, group, tel_id, start=None, stop=None): key = f"{group}/tel_{tel_id:03d}" - if key in self.h5file.root: + if key in self.h5file: table = read_table(self.h5file, key, start=start, stop=stop) else: table = _empty_telescope_events_table() From 7d3abd7b1924ea0538b1e4022fdb2cc0b6c88735 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Sat, 25 Nov 2023 16:49:43 +0100 Subject: [PATCH 07/14] Add TableLoader tutorial from dpps meeting - not finished yet, have to check how the sphinx gallery format works --- examples/tutorials/ctapipe_overview.py | 4 +- .../tutorials/tableloader_and_eventsources.py | 296 ++++++++++++++++++ 2 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 examples/tutorials/tableloader_and_eventsources.py diff --git a/examples/tutorials/ctapipe_overview.py b/examples/tutorials/ctapipe_overview.py index 0f2e722171f..05374947ec7 100644 --- a/examples/tutorials/ctapipe_overview.py +++ b/examples/tutorials/ctapipe_overview.py @@ -400,9 +400,9 @@ ###################################################################### -loader = TableLoader(f.name, load_dl2=True, load_simulated=True) +loader = TableLoader(f.name) -events = loader.read_subarray_events() +events = loader.read_subarray_events(dl2=True, simulated=True) ###################################################################### theta = angular_separation( diff --git a/examples/tutorials/tableloader_and_eventsources.py b/examples/tutorials/tableloader_and_eventsources.py new file mode 100644 index 00000000000..73df30e177e --- /dev/null +++ b/examples/tutorials/tableloader_and_eventsources.py @@ -0,0 +1,296 @@ +""" +TableLoader and EventSources +============================ + +This hands-on was presented in October 2023 at the DPPS Meeting +at CEA Paris-Saclay(J. Hackfeld) + +""" + +###################################################################### +# `ctapipe` provides basically two different ways of accessing its data products: +# - event-wise (***EventSource***) +# - column-wise (***TableLoader***) +# +# **EventSource(s):** +# EventSources read input files and generate `ctapipe.containers.ArrayEventContainer` +# instances when iterated over. +# +# A new EventSource should be created for each type of event file read +# into ctapipe, e.g. simtel files are read by the `ctapipe.io.SimTelEventSource`. +# +# EventSource provides a common high-level interface for accessing event +# information from different data sources. Creating an EventSource for a new +# file format or other event source ensures that data can be accessed in a common way, +# irregardless of the file format or data origin. +# +# EventSource itself is an abstract class, but will create an +# appropriate subclass if a compatible source is found for the given +# `input_url`. +# +# **TableLoader:** +# Loads telescope-event or subarray-event data from ctapipe HDF5 files and returns `astropy.table`. +# See [Astropy docs](https://docs.astropy.org/en/stable/table/) or [this video from A.Donath at the ESCAPE Summer School 2021](https://www.youtube.com/watch?v=uzhQ6RIGHQA) +# +# This class provides high-level access to data stored in ctapipe HDF5 files, +# such as created by the ctapipe-process tool (`ctapipe.tools.process.ProcessorTool`). +# +# There are multiple `TableLoader` methods loading data from all relevant tables, +# depending on the options, and joins them into single tables: +# +# - `TableLoader.read_subarray_events` +# - `TableLoader.read_telescope_events` +# - `TableLoader.read_telescope_events_by_id` +# - `TableLoader.read_telescope_events_by_type` +# +# +# The last one returns a dict with a table per telescope type, which is needed for e.g. DL1 image data that might have +# different shapes for each of the telescope types as tables do not support variable length columns. +# + +from astropy.table import Table +from traitlets.config import Config + +from ctapipe import utils +from ctapipe.calib import CameraCalibrator +from ctapipe.io import DataWriter, EventSource, TableLoader + +simtel_path = utils.get_dataset_path("gamma_prod5.simtel.zst") + +###################################################################### +# EventSource: +# ------------ +# The already implemented EventSources are: + +! ctapipe-info --event-sources + +###################################################################### +# `EventSource` will create an appropriate subclass if a compatible source is found for the given `input_url`: +# +source = EventSource(input_url=simtel_path, max_events=5) +source + +###################################################################### +# You can now loop over the `ctapipe.containers.ArrayEventContainer` generated by the source. +# + +for event in source: + print(event.count) + +event + +###################################################################### +# Every time a new loop is started through the source, +# it tries to restart from the first event, which might not be supported +# by the event source. It is encouraged to use ``EventSource`` in a **context manager** to ensure +# the correct cleanups are performed when you are finished with the source: +# + +###################################################################### +# You can hand in the Ids of the telescopes to be included in the data +# with the `allowed_tels` attribute. If given, only this subset of telescopes +# will be present in the generated events. If None, all available telescopes are used. +# + +eventsource_config = Config( + { + "EventSource": { + "max_events": 5, + "allowed_tels" : [3, 4, 9] + } + } +) + +with EventSource(input_url=simtel_path, config=eventsource_config) as source: + for event in source: + print(f"Event Count: {event.count}, Tels with trigger: {event.trigger.tels_with_trigger}") + +###################################################################### +# E.g. calibrate your data in the event loop and write it to an .h5 file with the `DataWriter`: +# + +source = EventSource(input_url=simtel_path, max_events=50) +calibrate = CameraCalibrator(subarray=source.subarray) + +with DataWriter(event_source=source, output_path="events.dl1.h5", write_parameters=False, overwrite=True, write_images=True) as write_data: + for event in source: + calibrate(event) + write_data(event) + +###################################################################### +# Alternatively doing it with `ctapipe-process`: +# + +! ctapipe-process -i {simtel_path} -o events.dl1.h5 --overwrite --progress + +###################################################################### +# TableLoader: +# ------------ +# +# Create a TableLoader instance with the above created dl1 file: + +loader = TableLoader(input_url="events.dl1.h5") + +###################################################################### +# Alternatively using a config file: +# + +tableloader_config = Config( + { + "TableLoader": { + "input_url": "events.dl1.h5", + } + } +) + +loader = TableLoader(config=tableloader_config) + +###################################################################### +# Reading subarray-based event information: +# + +subarray_events = loader.read_subarray_events( + start=None, + stop=None, + dl2=False, + simulated=True, + observation_info=False, +) + +subarray_events + +###################################################################### +# Reading subarray-based event information in chunks: +# + +subarray_events_chunked = loader.read_subarray_events_chunked( + chunk_size=500, + start=None, + stop=None, + dl2=False, + simulated=True, + observation_info=False +) + +subarray_events_chunked + +for chunk in subarray_events_chunked: + print(chunk) + +###################################################################### +# Reading just LST events: +# + +lst_events = loader.read_telescope_events( + telescopes=[1, 2, 3, 4], + start=None, + stop=None, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=True, + true_parameters=False, + instrument=True, + observation_info=False +) + +lst_events + +###################################################################### +# Loading telescope events by type returns a dict with the different telescope types: +# + +telescope_events_by_type = loader.read_telescope_events_by_type( + telescopes=["LST_LST_LSTCam", "MST_MST_FlashCam"], + start=None, + stop=None, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=True, + true_parameters=False, + instrument=True, + observation_info=False +) + +telescope_events_by_type + + +###################################################################### +# Loading telescope events by ID returns a dict with the different telescope IDs: +# + +telescope_events_by_id = loader.read_telescope_events_by_id( + telescopes=[1, 2], + start=None, + stop=None, + dl1_images=True, + dl1_parameters=False, + dl1_muons=False, + dl2=False, + simulated=False, + true_images=True, + true_parameters=False, + instrument=True, + observation_info=False +) + +telescope_events_by_id + +###################################################################### +# `read_telescope_events_chunked`, `read_telescope_events_by_type_chunked` and `read_telescope_events_by_id_chunked` are also available. + +###################################################################### +# Reading simulation-configuration, shower-distribution or observation-information: +# + +simulation_configuration = loader.read_simulation_configuration() +shower_distribution = loader.read_shower_distribution() +observation_information = loader.read_observation_information() + +simulation_configuration + +###################################################################### +# Now you have `astropy.table`s including all the relevant data you need for your analyses. +# + +###################################################################### +# It is recommended to use the `TableLoader` when loading data into Tables, because its much +# faster than the EventSource: +# +# E.g. You want to analyze your brand new image cleaning algorithm on LST images: +# +# - with the`EventSource`: + +def ES(input_url): + table = Table() + images = [] + with EventSource(input_url=input_url, allowed_tels=[1, 2, 3, 4]) as source: + for event in source: + tel = event.dl1.tel + for tel_id in tel.keys(): + images.append(tel[tel_id].image) + table['image'] = images + return images + +###################################################################### +# - with the `TableLoader` +# + +def TL(input_url): + loader = TableLoader(input_url=input_url) + tel_events = loader.read_telescope_events( + telescopes=[1, 2, 3, 4], + dl1_images=True, + true_images=True, + ) + + return tel_events + +###################################################################### +%timeit ES("events.dl1.h5") +%timeit TL("events.dl1.h5") From cec371419dc25932bf2a2bfda9b3217f9d1e9022 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Wed, 13 Dec 2023 11:13:08 +0100 Subject: [PATCH 08/14] Update example Tutorial --- docs/sg_execution_times.rst | 85 ++++++ .../tutorials/tableloader_and_eventsources.py | 284 +++++++++--------- 2 files changed, 220 insertions(+), 149 deletions(-) create mode 100644 docs/sg_execution_times.rst diff --git a/docs/sg_execution_times.rst b/docs/sg_execution_times.rst new file mode 100644 index 00000000000..339c2ca8397 --- /dev/null +++ b/docs/sg_execution_times.rst @@ -0,0 +1,85 @@ + +:orphan: + +.. _sphx_glr_sg_execution_times: + + +Computation times +================= +**00:06.933** total execution time for 17 files **from all galleries**: + +.. container:: + + .. raw:: html + + + + + + + + .. list-table:: + :header-rows: 1 + :class: table table-striped sg-datatable + + * - Example + - Time + - Mem (MB) + * - :ref:`sphx_glr_auto_examples_tutorials_tableloader_and_eventsources.py` (``../examples/tutorials/tableloader_and_eventsources.py``) + - 00:06.933 + - 0.0 + * - :ref:`sphx_glr_auto_examples_algorithms_convert_images_to_2d.py` (``../examples/algorithms/convert_images_to_2d.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_algorithms_dilate_image.py` (``../examples/algorithms/dilate_image.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_algorithms_nd_interpolation.py` (``../examples/algorithms/nd_interpolation.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_core_InstrumentDescription.py` (``../examples/core/InstrumentDescription.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_core_command_line_tools.py` (``../examples/core/command_line_tools.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_core_containers.py` (``../examples/core/containers.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_core_provenance.py` (``../examples/core/provenance.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_core_table_writer_reader.py` (``../examples/core/table_writer_reader.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_tutorials_calibrated_data_exploration.py` (``../examples/tutorials/calibrated_data_exploration.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_tutorials_coordinates_example.py` (``../examples/tutorials/coordinates_example.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_tutorials_ctapipe_handson.py` (``../examples/tutorials/ctapipe_handson.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_tutorials_ctapipe_overview.py` (``../examples/tutorials/ctapipe_overview.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_tutorials_raw_data_exploration.py` (``../examples/tutorials/raw_data_exploration.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_tutorials_theta_square.py` (``../examples/tutorials/theta_square.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_visualization_array_display.py` (``../examples/visualization/array_display.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_auto_examples_visualization_camera_display.py` (``../examples/visualization/camera_display.py``) + - 00:00.000 + - 0.0 diff --git a/examples/tutorials/tableloader_and_eventsources.py b/examples/tutorials/tableloader_and_eventsources.py index 73df30e177e..a91bb124559 100644 --- a/examples/tutorials/tableloader_and_eventsources.py +++ b/examples/tutorials/tableloader_and_eventsources.py @@ -1,54 +1,67 @@ """ -TableLoader and EventSources -============================ +IO: TableLoader and EventSources +================================ This hands-on was presented in October 2023 at the DPPS Meeting -at CEA Paris-Saclay(J. Hackfeld) - +at CEA Paris-Saclay (J. Hackfeld). """ -###################################################################### -# `ctapipe` provides basically two different ways of accessing its data products: -# - event-wise (***EventSource***) -# - column-wise (***TableLoader***) -# +# %% +# Introduction +# ------------ +# ``ctapipe`` provides basically two different ways of accessing its data products: +# +# - event-wise (**EventSource**) +# - column-wise (**TableLoader**) +# # **EventSource(s):** -# EventSources read input files and generate `ctapipe.containers.ArrayEventContainer` +# +# EventSources read input files and generate ``ctapipe.containers.ArrayEventContainer`` # instances when iterated over. -# +# # A new EventSource should be created for each type of event file read -# into ctapipe, e.g. simtel files are read by the `ctapipe.io.SimTelEventSource`. -# +# into ctapipe, e.g. simtel files are read by the ``ctapipe.io.SimTelEventSource`` . +# # EventSource provides a common high-level interface for accessing event # information from different data sources. Creating an EventSource for a new # file format or other event source ensures that data can be accessed in a common way, # irregardless of the file format or data origin. -# +# # EventSource itself is an abstract class, but will create an # appropriate subclass if a compatible source is found for the given -# `input_url`. -# -# **TableLoader:** -# Loads telescope-event or subarray-event data from ctapipe HDF5 files and returns `astropy.table`. -# See [Astropy docs](https://docs.astropy.org/en/stable/table/) or [this video from A.Donath at the ESCAPE Summer School 2021](https://www.youtube.com/watch?v=uzhQ6RIGHQA) -# +# ``input_url`` . +# +# **TableLoader**: +# +# Loads telescope-event or subarray-event data from ctapipe HDF5 files and returns +# ``astropy.table`` . +# See `Astropy docs `__ +# or `this video from A.Donath at the ESCAPE Summer School +# 2021 `__. +# # This class provides high-level access to data stored in ctapipe HDF5 files, -# such as created by the ctapipe-process tool (`ctapipe.tools.process.ProcessorTool`). -# -# There are multiple `TableLoader` methods loading data from all relevant tables, -# depending on the options, and joins them into single tables: -# -# - `TableLoader.read_subarray_events` -# - `TableLoader.read_telescope_events` -# - `TableLoader.read_telescope_events_by_id` -# - `TableLoader.read_telescope_events_by_type` -# -# -# The last one returns a dict with a table per telescope type, which is needed for e.g. DL1 image data that might have -# different shapes for each of the telescope types as tables do not support variable length columns. -# - -from astropy.table import Table +# such as created by the ctapipe-process tool ( ``ctapipe.tools.process.ProcessorTool`` ). +# +# There are multiple ``TableLoader`` methods loading data from all relevant tables (depending +# on the options) and **joins** them into single tables: +# +# - ``TableLoader.read_subarray_events`` +# - ``TableLoader.read_telescope_events`` +# - ``TableLoader.read_telescope_events_by_id`` +# - ``TableLoader.read_telescope_events_by_type`` +# +# The last one returns a dict with a table per telescope type, which is needed for e.g. DL1 +# image data that might have different shapes for each of the telescope types as tables do +# not support variable length columns. +# +# **It is recommended to use the** ``TableLoader`` **when loading data into Tables, because +# its much faster than EventSources!** + +# %% +# Code Examples +# ------------- +# First import some classes/modules and get the example dataset path: + from traitlets.config import Config from ctapipe import utils @@ -57,97 +70,110 @@ simtel_path = utils.get_dataset_path("gamma_prod5.simtel.zst") -###################################################################### -# EventSource: -# ------------ +# %% +# EventSource(s) +# -------------- # The already implemented EventSources are: -! ctapipe-info --event-sources +sources = EventSource.non_abstract_subclasses() +maxlen = max(len(source) for source in sources) +for source in sources.values(): + print(f"{source.__name__:>{maxlen}s} -- {source.__module__}.{source.__qualname__}") + +# %% +# ``EventSource`` will create an appropriate subclass if a compatible source is found for +# the given ``input_url`` : -###################################################################### -# `EventSource` will create an appropriate subclass if a compatible source is found for the given `input_url`: -# source = EventSource(input_url=simtel_path, max_events=5) -source +print(source) -###################################################################### -# You can now loop over the `ctapipe.containers.ArrayEventContainer` generated by the source. -# +# %% +# You can now loop over the ``ctapipe.containers.ArrayEventContainer`` generated by the source. for event in source: print(event.count) -event +# %% -###################################################################### +print(repr(event)) + +# %% # Every time a new loop is started through the source, # it tries to restart from the first event, which might not be supported -# by the event source. It is encouraged to use ``EventSource`` in a **context manager** to ensure -# the correct cleanups are performed when you are finished with the source: -# +# by the event source. It is encouraged to use ``EventSource`` in a **context manager** +# to ensure the correct cleanups are performed when you are finished with the source: -###################################################################### -# You can hand in the Ids of the telescopes to be included in the data -# with the `allowed_tels` attribute. If given, only this subset of telescopes +with EventSource(input_url=simtel_path) as source: + for event in source: + print( + f"Event Count: {event.count}, Tels with trigger: {event.trigger.tels_with_trigger}" + ) + +# %% +# You can hand in the ID's of the telescopes to be included in the data +# with the ``allowed_tels`` attribute. If given, only this subset of telescopes # will be present in the generated events. If None, all available telescopes are used. -# eventsource_config = Config( - { - "EventSource": { - "max_events": 5, - "allowed_tels" : [3, 4, 9] - } - } + {"EventSource": {"max_events": 5, "allowed_tels": [3, 4, 9]}} ) with EventSource(input_url=simtel_path, config=eventsource_config) as source: for event in source: - print(f"Event Count: {event.count}, Tels with trigger: {event.trigger.tels_with_trigger}") + print( + f"Event Count: {event.count}, Tels with trigger: {event.trigger.tels_with_trigger}" + ) -###################################################################### -# E.g. calibrate your data in the event loop and write it to an .h5 file with the `DataWriter`: -# +# %% +# If you want to calibrate your data in the event loop and write it to an .h5 file with +# the ``DataWriter`` : +# source = EventSource(input_url=simtel_path, max_events=50) calibrate = CameraCalibrator(subarray=source.subarray) -with DataWriter(event_source=source, output_path="events.dl1.h5", write_parameters=False, overwrite=True, write_images=True) as write_data: +with DataWriter( + event_source=source, + output_path="events.dl1.h5", + write_parameters=False, + overwrite=True, + write_images=True, +) as write_data: for event in source: calibrate(event) write_data(event) -###################################################################### -# Alternatively doing it with `ctapipe-process`: -# - -! ctapipe-process -i {simtel_path} -o events.dl1.h5 --overwrite --progress +# %% +# Alternatively doing it with ``ctapipe-process`` would look like this: +# +# .. code-block:: bash +# +# ! ctapipe-process -i {simtel_path} -o events.dl1.h5 --overwrite --progress +# -###################################################################### -# TableLoader: -# ------------ +# %% +# TableLoader +# ----------- # # Create a TableLoader instance with the above created dl1 file: loader = TableLoader(input_url="events.dl1.h5") -###################################################################### +# %% # Alternatively using a config file: -# tableloader_config = Config( { "TableLoader": { "input_url": "events.dl1.h5", - } + } } ) loader = TableLoader(config=tableloader_config) -###################################################################### +# %% # Reading subarray-based event information: -# subarray_events = loader.read_subarray_events( start=None, @@ -157,29 +183,25 @@ observation_info=False, ) -subarray_events +print(subarray_events) -###################################################################### +# %% # Reading subarray-based event information in chunks: -# subarray_events_chunked = loader.read_subarray_events_chunked( - chunk_size=500, - start=None, - stop=None, + chunk_size=3, dl2=False, simulated=True, - observation_info=False + observation_info=False, ) -subarray_events_chunked +print(subarray_events_chunked) for chunk in subarray_events_chunked: - print(chunk) + print(" \n", chunk) -###################################################################### +# %% # Reading just LST events: -# lst_events = loader.read_telescope_events( telescopes=[1, 2, 3, 4], @@ -193,14 +215,13 @@ true_images=True, true_parameters=False, instrument=True, - observation_info=False + observation_info=False, ) -lst_events +print(lst_events) -###################################################################### +# %% # Loading telescope events by type returns a dict with the different telescope types: -# telescope_events_by_type = loader.read_telescope_events_by_type( telescopes=["LST_LST_LSTCam", "MST_MST_FlashCam"], @@ -214,18 +235,18 @@ true_images=True, true_parameters=False, instrument=True, - observation_info=False + observation_info=False, ) -telescope_events_by_type +for tel_type, table in telescope_events_by_type.items(): + print(f"Telescope Type: {tel_type} \n", table, "\n") -###################################################################### +# %% # Loading telescope events by ID returns a dict with the different telescope IDs: -# telescope_events_by_id = loader.read_telescope_events_by_id( - telescopes=[1, 2], + telescopes=[3, 14], start=None, stop=None, dl1_images=True, @@ -236,61 +257,26 @@ true_images=True, true_parameters=False, instrument=True, - observation_info=False + observation_info=False, ) +for id, table in telescope_events_by_id.items(): + print(f"Telescope ID: {id} \n", table, "\n") -telescope_events_by_id - -###################################################################### -# `read_telescope_events_chunked`, `read_telescope_events_by_type_chunked` and `read_telescope_events_by_id_chunked` are also available. +# %% +# - ``read_telescope_events_chunked`` +# - ``read_telescope_events_by_type_chunked`` +# - ``read_telescope_events_by_id_chunked`` +# +# are also available. -###################################################################### -# Reading simulation-configuration, shower-distribution or observation-information: -# +# %% +# Reading e.g. simulation- or observation-information: simulation_configuration = loader.read_simulation_configuration() -shower_distribution = loader.read_shower_distribution() observation_information = loader.read_observation_information() -simulation_configuration +print(simulation_configuration) -###################################################################### -# Now you have `astropy.table`s including all the relevant data you need for your analyses. -# - -###################################################################### -# It is recommended to use the `TableLoader` when loading data into Tables, because its much -# faster than the EventSource: -# -# E.g. You want to analyze your brand new image cleaning algorithm on LST images: +# %% +# Now you have ``astropy.table`` s including all the relevant data you need for your analyses. # -# - with the`EventSource`: - -def ES(input_url): - table = Table() - images = [] - with EventSource(input_url=input_url, allowed_tels=[1, 2, 3, 4]) as source: - for event in source: - tel = event.dl1.tel - for tel_id in tel.keys(): - images.append(tel[tel_id].image) - table['image'] = images - return images - -###################################################################### -# - with the `TableLoader` -# - -def TL(input_url): - loader = TableLoader(input_url=input_url) - tel_events = loader.read_telescope_events( - telescopes=[1, 2, 3, 4], - dl1_images=True, - true_images=True, - ) - - return tel_events - -###################################################################### -%timeit ES("events.dl1.h5") -%timeit TL("events.dl1.h5") From a718cfc7f0ce06831a0dfc1a1e62f1528d3b7f1e Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld <53918415+Hckjs@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:33:26 +0100 Subject: [PATCH 09/14] Delete docs/sg_execution_times.rst --- docs/sg_execution_times.rst | 85 ------------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 docs/sg_execution_times.rst diff --git a/docs/sg_execution_times.rst b/docs/sg_execution_times.rst deleted file mode 100644 index 339c2ca8397..00000000000 --- a/docs/sg_execution_times.rst +++ /dev/null @@ -1,85 +0,0 @@ - -:orphan: - -.. _sphx_glr_sg_execution_times: - - -Computation times -================= -**00:06.933** total execution time for 17 files **from all galleries**: - -.. container:: - - .. raw:: html - - - - - - - - .. list-table:: - :header-rows: 1 - :class: table table-striped sg-datatable - - * - Example - - Time - - Mem (MB) - * - :ref:`sphx_glr_auto_examples_tutorials_tableloader_and_eventsources.py` (``../examples/tutorials/tableloader_and_eventsources.py``) - - 00:06.933 - - 0.0 - * - :ref:`sphx_glr_auto_examples_algorithms_convert_images_to_2d.py` (``../examples/algorithms/convert_images_to_2d.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_algorithms_dilate_image.py` (``../examples/algorithms/dilate_image.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_algorithms_nd_interpolation.py` (``../examples/algorithms/nd_interpolation.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_core_InstrumentDescription.py` (``../examples/core/InstrumentDescription.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_core_command_line_tools.py` (``../examples/core/command_line_tools.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_core_containers.py` (``../examples/core/containers.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_core_provenance.py` (``../examples/core/provenance.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_core_table_writer_reader.py` (``../examples/core/table_writer_reader.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_tutorials_calibrated_data_exploration.py` (``../examples/tutorials/calibrated_data_exploration.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_tutorials_coordinates_example.py` (``../examples/tutorials/coordinates_example.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_tutorials_ctapipe_handson.py` (``../examples/tutorials/ctapipe_handson.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_tutorials_ctapipe_overview.py` (``../examples/tutorials/ctapipe_overview.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_tutorials_raw_data_exploration.py` (``../examples/tutorials/raw_data_exploration.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_tutorials_theta_square.py` (``../examples/tutorials/theta_square.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_visualization_array_display.py` (``../examples/visualization/array_display.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_auto_examples_visualization_camera_display.py` (``../examples/visualization/camera_display.py``) - - 00:00.000 - - 0.0 From a81fb43d9e5f09c02c5ed049f1bcc31a5d59febd Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Wed, 13 Dec 2023 11:53:29 +0100 Subject: [PATCH 10/14] Fixing some Codacy quality checks --- ctapipe/io/tableloader.py | 10 +++---- .../tutorials/tableloader_and_eventsources.py | 27 ++++++++++--------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index 266f5728c09..f2bf116f5c1 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -278,11 +278,7 @@ def _check_args(self, **kwargs): updated_value = value if updated_value is None: updated_value = getattr(self, key) - if ( - updated_value - and key in groups.keys() - and groups[key] not in self.h5file.root - ): + if updated_value and key in groups and groups[key] not in self.h5file.root: self.log.info( "Setting %s to False, input file does not contain such data", key ) @@ -583,9 +579,9 @@ def _read_telescope_events_for_id( table = _join_telescope_events(table, true_parameters) if instrument: - self.instrument_table = self.subarray.to_table("joined") + instrument_table = self.subarray.to_table("joined") table = join_allow_empty( - table, self.instrument_table, keys=["tel_id"], join_type="left" + table, instrument_table, keys=["tel_id"], join_type="left" ) if simulated and TRUE_IMPACT_GROUP in self.h5file.root: diff --git a/examples/tutorials/tableloader_and_eventsources.py b/examples/tutorials/tableloader_and_eventsources.py index a91bb124559..0072152281e 100644 --- a/examples/tutorials/tableloader_and_eventsources.py +++ b/examples/tutorials/tableloader_and_eventsources.py @@ -42,20 +42,20 @@ # This class provides high-level access to data stored in ctapipe HDF5 files, # such as created by the ctapipe-process tool ( ``ctapipe.tools.process.ProcessorTool`` ). # -# There are multiple ``TableLoader`` methods loading data from all relevant tables (depending -# on the options) and **joins** them into single tables: +# There are multiple ``TableLoader`` methods loading data from all relevant tables +# (depending on the options) and **joins** them into single tables: # # - ``TableLoader.read_subarray_events`` # - ``TableLoader.read_telescope_events`` # - ``TableLoader.read_telescope_events_by_id`` # - ``TableLoader.read_telescope_events_by_type`` # -# The last one returns a dict with a table per telescope type, which is needed for e.g. DL1 -# image data that might have different shapes for each of the telescope types as tables do -# not support variable length columns. +# The last one returns a dict with a table per telescope type, which is needed for +# e.g. DL1 image data that might have different shapes for each of the telescope +# types as tables do not support variable length columns. # -# **It is recommended to use the** ``TableLoader`` **when loading data into Tables, because -# its much faster than EventSources!** +# **It is recommended to use the** ``TableLoader`` **when loading data into Tables, +# because its much faster than EventSources!** # %% # Code Examples @@ -88,7 +88,8 @@ print(source) # %% -# You can now loop over the ``ctapipe.containers.ArrayEventContainer`` generated by the source. +# You can now loop over the ``ctapipe.containers.ArrayEventContainer`` generated by the +# source. for event in source: print(event.count) @@ -106,7 +107,8 @@ with EventSource(input_url=simtel_path) as source: for event in source: print( - f"Event Count: {event.count}, Tels with trigger: {event.trigger.tels_with_trigger}" + f"Event Count: {event.count}," + f"Tels with trigger: {event.trigger.tels_with_trigger}" ) # %% @@ -259,8 +261,8 @@ instrument=True, observation_info=False, ) -for id, table in telescope_events_by_id.items(): - print(f"Telescope ID: {id} \n", table, "\n") +for tel_id, table in telescope_events_by_id.items(): + print(f"Telescope ID: {tel_id} \n", table, "\n") # %% # - ``read_telescope_events_chunked`` @@ -278,5 +280,6 @@ print(simulation_configuration) # %% -# Now you have ``astropy.table`` s including all the relevant data you need for your analyses. +# Now you have ``astropy.table`` s including all the relevant data you need +# for your analyses. # From d61782110b351b49a22ceffcdb5b05a82f0d0f46 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Wed, 13 Dec 2023 13:31:20 +0100 Subject: [PATCH 11/14] Adding changelog --- docs/changes/2482.api.rst | 5 +++++ examples/tutorials/tableloader_and_eventsources.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 docs/changes/2482.api.rst diff --git a/docs/changes/2482.api.rst b/docs/changes/2482.api.rst new file mode 100644 index 00000000000..a878f2f4f31 --- /dev/null +++ b/docs/changes/2482.api.rst @@ -0,0 +1,5 @@ +Move the `TableLoader` options from being traitlets to +each `read_...` method allowing to load different data with the +same `TableLoader`-Instance. + +In addition the default values for the options have changed. diff --git a/examples/tutorials/tableloader_and_eventsources.py b/examples/tutorials/tableloader_and_eventsources.py index 0072152281e..a0d96bdf105 100644 --- a/examples/tutorials/tableloader_and_eventsources.py +++ b/examples/tutorials/tableloader_and_eventsources.py @@ -123,7 +123,8 @@ with EventSource(input_url=simtel_path, config=eventsource_config) as source: for event in source: print( - f"Event Count: {event.count}, Tels with trigger: {event.trigger.tels_with_trigger}" + f"Event Count: {event.count}," + f"Tels with trigger: {event.trigger.tels_with_trigger}" ) # %% From 8b9aa392374a018177e679b8e2052b3d464570b1 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Fri, 15 Dec 2023 16:38:11 +0100 Subject: [PATCH 12/14] Use some more html repr in the tutorials - Fix changelog --- docs/changes/2482.api.rst | 6 +++--- examples/tutorials/tableloader_and_eventsources.py | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/docs/changes/2482.api.rst b/docs/changes/2482.api.rst index a878f2f4f31..63ea13831e7 100644 --- a/docs/changes/2482.api.rst +++ b/docs/changes/2482.api.rst @@ -1,5 +1,5 @@ -Move the `TableLoader` options from being traitlets to -each `read_...` method allowing to load different data with the -same `TableLoader`-Instance. +Move the TableLoader options from being traitlets to +each read_... method allowing to load different data with the +same TableLoader-Instance. In addition the default values for the options have changed. diff --git a/examples/tutorials/tableloader_and_eventsources.py b/examples/tutorials/tableloader_and_eventsources.py index a0d96bdf105..36ea8982ef3 100644 --- a/examples/tutorials/tableloader_and_eventsources.py +++ b/examples/tutorials/tableloader_and_eventsources.py @@ -186,7 +186,7 @@ observation_info=False, ) -print(subarray_events) +subarray_events # %% # Reading subarray-based event information in chunks: @@ -198,8 +198,6 @@ observation_info=False, ) -print(subarray_events_chunked) - for chunk in subarray_events_chunked: print(" \n", chunk) @@ -221,7 +219,7 @@ observation_info=False, ) -print(lst_events) +lst_events # %% # Loading telescope events by type returns a dict with the different telescope types: @@ -244,7 +242,6 @@ for tel_type, table in telescope_events_by_type.items(): print(f"Telescope Type: {tel_type} \n", table, "\n") - # %% # Loading telescope events by ID returns a dict with the different telescope IDs: @@ -278,7 +275,7 @@ simulation_configuration = loader.read_simulation_configuration() observation_information = loader.read_observation_information() -print(simulation_configuration) +simulation_configuration # %% # Now you have ``astropy.table`` s including all the relevant data you need From f46e8a672d076149dd46ccd0cfc68446814d9b28 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Tue, 19 Dec 2023 12:05:09 +0100 Subject: [PATCH 13/14] Change default values of TableLoader --- ctapipe/instrument/tests/test_trigger.py | 6 --- ctapipe/io/tableloader.py | 6 +-- ctapipe/io/tests/test_table_loader.py | 50 ++++-------------------- ctapipe/tools/apply_models.py | 6 ++- ctapipe/tools/tests/test_apply_models.py | 6 --- ctapipe/tools/tests/test_merge.py | 2 - ctapipe/tools/tests/test_process.py | 9 ++--- ctapipe/tools/utils.py | 3 +- examples/tutorials/ctapipe_overview.py | 5 +-- 9 files changed, 21 insertions(+), 72 deletions(-) diff --git a/ctapipe/instrument/tests/test_trigger.py b/ctapipe/instrument/tests/test_trigger.py index 0f7a375df0e..40cabb997eb 100644 --- a/ctapipe/instrument/tests/test_trigger.py +++ b/ctapipe/instrument/tests/test_trigger.py @@ -220,11 +220,8 @@ def test_software_trigger_simtel_process(tmp_path): ) as loader: events_trigger = loader.read_telescope_events( "LST_LST_LSTCam", - dl1_muons=False, dl2=False, true_parameters=False, - instrument=False, - observation_info=False, ) with TableLoader( @@ -233,11 +230,8 @@ def test_software_trigger_simtel_process(tmp_path): ) as loader: events_no_trigger = loader.read_telescope_events( "LST_LST_LSTCam", - dl1_muons=False, dl2=False, true_parameters=False, - instrument=False, - observation_info=False, ) assert len(events_no_trigger) > len(events_trigger) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index f2bf116f5c1..267c2d18e08 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -178,7 +178,7 @@ class TableLoader(Component): dl1_parameters = traits.Bool(True, help="load reconstructed image parameters").tag( config=True ) - dl1_muons = traits.Bool(True, help="load muon ring parameters").tag(config=True) + dl1_muons = traits.Bool(False, help="load muon ring parameters").tag(config=True) dl2 = traits.Bool(True, help="load available dl2 stereo parameters").tag( config=True @@ -195,11 +195,11 @@ class TableLoader(Component): ).tag(config=True) instrument = traits.Bool( - True, help="join subarray instrument information to each event" + False, help="join subarray instrument information to each event" ).tag(config=True) observation_info = traits.Bool( - True, help="join observation information to each event" + False, help="join observation information to each event" ).tag(config=True) focal_length_choice = traits.UseEnum( FocalLengthKind, diff --git a/ctapipe/io/tests/test_table_loader.py b/ctapipe/io/tests/test_table_loader.py index b801d1e3846..2262dd4de53 100644 --- a/ctapipe/io/tests/test_table_loader.py +++ b/ctapipe/io/tests/test_table_loader.py @@ -73,7 +73,7 @@ def test_telescope_events_for_tel_id(dl1_file): loader = TableLoader(dl1_file) with loader as table_loader: - table = table_loader.read_telescope_events([8], dl1_parameters=True) + table = table_loader.read_telescope_events([8]) assert "hillas_length" in table.colnames assert "time" in table.colnames assert "event_type" in table.colnames @@ -123,11 +123,11 @@ def test_simulated(dl1_file): from ctapipe.io.tableloader import TableLoader with TableLoader(dl1_file) as table_loader: - table = table_loader.read_subarray_events(simulated=True) + table = table_loader.read_subarray_events() assert "true_energy" in table.colnames assert table["obs_id"].dtype == np.int32 - table = table_loader.read_telescope_events([8], simulated=True) + table = table_loader.read_telescope_events([8]) assert "true_energy" in table.colnames assert "true_impact_distance" in table.colnames @@ -148,9 +148,7 @@ def test_true_parameters(dl1_file): from ctapipe.io.tableloader import TableLoader with TableLoader(dl1_file) as table_loader: - table = table_loader.read_telescope_events( - dl1_parameters=False, true_parameters=True - ) + table = table_loader.read_telescope_events(dl1_parameters=False) assert "true_hillas_intensity" in table.colnames @@ -168,7 +166,7 @@ def test_read_subarray_events(dl2_shower_geometry_file): from ctapipe.io.tableloader import TableLoader with TableLoader(dl2_shower_geometry_file) as table_loader: - table = table_loader.read_subarray_events(dl2=True, simulated=True) + table = table_loader.read_subarray_events() assert "HillasReconstructor_alt" in table.colnames assert "true_energy" in table.colnames assert "time" in table.colnames @@ -184,8 +182,8 @@ def test_table_loader_keeps_original_order(dl2_merged_file): assert not np.all(np.diff(trigger["obs_id"]) >= 0) with TableLoader(dl2_merged_file) as table_loader: - events = table_loader.read_subarray_events(dl2=True, simulated=True) - tel_events = table_loader.read_telescope_events(dl2=True, simulated=True) + events = table_loader.read_subarray_events() + tel_events = table_loader.read_telescope_events() check_equal_array_event_order(events, trigger) check_equal_array_event_order(events, tel_events) @@ -201,10 +199,7 @@ def test_read_telescope_events_type(dl2_shower_geometry_file): with TableLoader(dl2_shower_geometry_file) as table_loader: table = table_loader.read_telescope_events( ["MST_MST_FlashCam"], - dl1_images=False, dl1_parameters=False, - dl2=True, - simulated=True, true_images=True, instrument=True, ) @@ -231,8 +226,6 @@ def test_read_telescope_events_by_type(dl2_shower_geometry_file): [25, 130], dl1_images=False, dl1_parameters=False, - dl2=True, - simulated=True, true_images=True, instrument=True, ) @@ -281,45 +274,18 @@ def test_chunked(dl2_shower_geometry_file): with TableLoader(dl2_shower_geometry_file) as table_loader: tel_event_it = table_loader.read_telescope_events_chunked( chunk_size, - dl1_images=False, - dl1_parameters=True, - dl1_muons=False, - dl2=True, - simulated=True, - true_images=False, true_parameters=False, - instrument=False, - observation_info=False, ) event_it = table_loader.read_subarray_events_chunked( chunk_size, - dl2=True, - simulated=True, - observation_info=False, ) by_type_it = table_loader.read_telescope_events_by_type_chunked( chunk_size, - dl1_images=False, - dl1_parameters=True, - dl1_muons=False, - dl2=True, - simulated=True, - true_images=False, true_parameters=False, - instrument=False, - observation_info=False, ) by_id_it = table_loader.read_telescope_events_by_id_chunked( chunk_size, - dl1_images=False, - dl1_parameters=True, - dl1_muons=False, - dl2=True, - simulated=True, - true_images=False, true_parameters=False, - instrument=False, - observation_info=False, ) iters = (event_it, tel_event_it, by_type_it, by_id_it) @@ -412,7 +378,6 @@ def test_read_unavailable_telescope(dl2_shower_geometry_file): loader.read_telescope_events( [tel_id], dl1_parameters=False, - dl2=True, ) @@ -424,7 +389,6 @@ def test_read_empty_table(dl2_shower_geometry_file): table = loader.read_telescope_events( [6], dl1_parameters=False, - dl2=True, ) assert len(table) == 0 diff --git a/ctapipe/tools/apply_models.py b/ctapipe/tools/apply_models.py index 1d40c2393fd..6ec237006e0 100644 --- a/ctapipe/tools/apply_models.py +++ b/ctapipe/tools/apply_models.py @@ -157,7 +157,11 @@ def setup(self): def start(self): """Apply models to input tables""" chunk_iterator = self.loader.read_telescope_events_by_id_chunked( - self.chunk_size, dl1_muons=False, simulated=False, true_parameters=False + self.chunk_size, + simulated=False, + true_parameters=False, + observation_info=True, + instrument=True, ) bar = tqdm( chunk_iterator, diff --git a/ctapipe/tools/tests/test_apply_models.py b/ctapipe/tools/tests/test_apply_models.py index 7a23ac1ad30..0aa61cef86f 100644 --- a/ctapipe/tools/tests/test_apply_models.py +++ b/ctapipe/tools/tests/test_apply_models.py @@ -51,7 +51,6 @@ def test_apply_energy_regressor( with TableLoader(output_path) as loader: events = loader.read_subarray_events( simulated=False, - observation_info=False, ) assert f"{prefix}_energy" in events.colnames assert f"{prefix}_energy_uncert" in events.colnames @@ -64,7 +63,6 @@ def test_apply_energy_regressor( tel_events = loader.read_telescope_events( simulated=False, - observation_info=False, ) assert f"{prefix}_energy" in tel_events.colnames assert f"{prefix}_energy_uncert" in tel_events.colnames @@ -188,7 +186,6 @@ def test_apply_all( with TableLoader(output_path) as loader: events = loader.read_subarray_events( simulated=False, - observation_info=False, ) assert f"{prefix_clf}_prediction" in events.colnames assert f"{prefix_clf}_telescopes" in events.colnames @@ -201,11 +198,8 @@ def test_apply_all( assert f"{prefix_disp}_goodness_of_fit" in events.colnames tel_events = loader.read_telescope_events( - dl1_muons=False, simulated=False, true_parameters=False, - instrument=False, - observation_info=False, ) assert f"{prefix_clf}_prediction" in tel_events.colnames assert f"{prefix_clf}_telescopes" in tel_events.colnames diff --git a/ctapipe/tools/tests/test_merge.py b/ctapipe/tools/tests/test_merge.py index e453de1b9ae..09b3bca217e 100644 --- a/ctapipe/tools/tests/test_merge.py +++ b/ctapipe/tools/tests/test_merge.py @@ -160,9 +160,7 @@ def test_dl2(tmp_path, dl2_shower_geometry_file, dl2_proton_geometry_file): loader = TableLoader(output) tel_events = loader.read_telescope_events( dl1_parameters=False, - dl1_muons=False, true_parameters=False, - instrument=False, ) assert "true_impact_distance" in tel_events.colnames # regression test for #2051 diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index 8fe99989472..95237160535 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -393,13 +393,10 @@ def test_read_from_simtel_and_dl1(prod5_proton_simtel_path, tmp_path): ] + few_tels assert run_tool(ProcessorTool(), argv=argv, cwd=tmp_path) == 0 - args = dict( - observation_info=False, - ) with TableLoader(dl2_from_simtel) as loader: - events_from_simtel = loader.read_subarray_events(**args) + events_from_simtel = loader.read_subarray_events() with TableLoader(dl2_from_dl1) as loader: - events_from_dl1 = loader.read_subarray_events(**args) + events_from_dl1 = loader.read_subarray_events() # both files should contain identical data assert_array_equal(events_from_simtel["event_id"], events_from_dl1["event_id"]) @@ -490,7 +487,7 @@ def test_only_trigger_and_simulation(tmp_path): ) with TableLoader(output) as loader: - events = loader.read_subarray_events(dl2=False, observation_info=False) + events = loader.read_subarray_events(dl2=False) assert len(events) == 7 assert "tels_with_trigger" in events.colnames assert "true_energy" in events.colnames diff --git a/ctapipe/tools/utils.py b/ctapipe/tools/utils.py index 1b41dd50105..2c6c3ff5bd9 100644 --- a/ctapipe/tools/utils.py +++ b/ctapipe/tools/utils.py @@ -101,8 +101,9 @@ def read_training_events( chunk_iterator = loader.read_telescope_events_chunked( chunk_size, telescopes=[telescope_type], - dl1_muons=False, true_parameters=False, + instrument=True, + observation_info=True, ) table = [] n_events_in_file = 0 diff --git a/examples/tutorials/ctapipe_overview.py b/examples/tutorials/ctapipe_overview.py index 9af4231e64e..767e125e3fe 100644 --- a/examples/tutorials/ctapipe_overview.py +++ b/examples/tutorials/ctapipe_overview.py @@ -401,7 +401,7 @@ ###################################################################### loader = TableLoader(f.name) -events = loader.read_subarray_events(dl2=True, simulated=True) +events = loader.read_subarray_events() ###################################################################### theta = angular_separation( @@ -468,11 +468,8 @@ dl1_table = loader.read_telescope_events( ["LST_LST_LSTCam"], - dl1_muons=False, dl2=False, true_parameters=False, - instrument=False, - observation_info=False, ) ###################################################################### From 592278bb3145b91ed7e0379bdd837899b8b264d9 Mon Sep 17 00:00:00 2001 From: Jonas Hackfeld Date: Tue, 19 Dec 2023 12:25:43 +0100 Subject: [PATCH 14/14] Fix changelog --- docs/changes/2482.api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changes/2482.api.rst b/docs/changes/2482.api.rst index 63ea13831e7..00400681c31 100644 --- a/docs/changes/2482.api.rst +++ b/docs/changes/2482.api.rst @@ -1,5 +1,5 @@ -Move the TableLoader options from being traitlets to -each read_... method allowing to load different data with the +Move the ``TableLoader`` options from being traitlets to +each ``read_...`` method allowing to load different data with the same TableLoader-Instance. In addition the default values for the options have changed.