diff --git a/ctapipe/conftest.py b/ctapipe/conftest.py index cec3dcce2a2..891d4658c90 100644 --- a/ctapipe/conftest.py +++ b/ctapipe/conftest.py @@ -44,7 +44,8 @@ def _global_example_event(): print("******************** LOAD TEST EVENT ***********************") - with SimTelEventSource(input_url=filename) as reader: + # FIXME: switch to prod5b+ file that contains effective focal length + with SimTelEventSource(input_url=filename, focal_length_choice='nominal') as reader: event = next(iter(reader)) return event @@ -59,7 +60,7 @@ def example_subarray(): print("******************** LOAD TEST EVENT ***********************") - with SimTelEventSource(input_url=filename) as reader: + with SimTelEventSource(input_url=filename, focal_length_choice='nominal') as reader: return reader.subarray @@ -84,7 +85,7 @@ def _subarray_and_event_gamma_off_axis_500_gev(): path = get_dataset_path("lst_prod3_calibration_and_mcphotons.simtel.zst") - with SimTelEventSource(path) as source: + with SimTelEventSource(path, focal_length_choice='nominal') as source: it = iter(source) # we want the second event, first event is a corner clipper next(it) @@ -353,6 +354,7 @@ def dl1_muon_file(dl1_tmp_path): "--write-images", "--DataWriter.write_parameters=False", "--DataWriter.Contact.name=αℓℓ the äüöß", + "--SimTelEventSource.focal_length_choice=nominal", ] assert run_tool(ProcessorTool(), argv=argv, cwd=dl1_tmp_path) == 0 return output diff --git a/ctapipe/io/eventseeker.py b/ctapipe/io/eventseeker.py index 3c58643db8a..bb77b069c12 100644 --- a/ctapipe/io/eventseeker.py +++ b/ctapipe/io/eventseeker.py @@ -28,7 +28,7 @@ class EventSeeker(Component): To obtain a particular event in a simtel file: >>> from ctapipe.io import SimTelEventSource - >>> event_source = SimTelEventSource(input_url="dataset://gamma_test_large.simtel.gz") + >>> event_source = SimTelEventSource(input_url="dataset://gamma_test_large.simtel.gz", focal_length_choice="nominal") >>> seeker = EventSeeker(event_source=event_source) >>> event = seeker.get_event_index(2) >>> print(event.count) @@ -37,7 +37,7 @@ class EventSeeker(Component): To obtain a particular event in a simtel file from its event_id: >>> from ctapipe.io import SimTelEventSource - >>> event_source = SimTelEventSource(input_url="dataset://gamma_test_large.simtel.gz", back_seekable=True) + >>> event_source = SimTelEventSource(input_url="dataset://gamma_test_large.simtel.gz", back_seekable=True, focal_length_choice="nominal") >>> seeker = EventSeeker(event_source=event_source) >>> event = seeker.get_event_id(31007) >>> print(event.count) diff --git a/ctapipe/io/eventsource.py b/ctapipe/io/eventsource.py index 4b46f33c4b5..61a9c50c058 100644 --- a/ctapipe/io/eventsource.py +++ b/ctapipe/io/eventsource.py @@ -36,7 +36,7 @@ class EventSource(Component): appropriate subclass if a compatible source is found for the given ``input_url``. - >>> EventSource(input_url="dataset://gamma_test_large.simtel.gz") + >>> EventSource(input_url="dataset://gamma_prod5.simtel.zst") An ``EventSource`` can also be created through the configuration system, @@ -45,7 +45,7 @@ class EventSource(Component): >>> self.source = EventSource(parent=self) # doctest: +SKIP To loop through the events in a file: - >>> source = EventSource(input_url="dataset://gamma_test_large.simtel.gz", max_events=2) + >>> source = EventSource(input_url="dataset://gamma_prod5.simtel.zst", max_events=2) >>> for event in source: ... print(event.count) 0 @@ -58,7 +58,7 @@ class EventSource(Component): It is encouraged to use ``EventSource`` in a context manager to ensure the correct cleanups are performed when you are finished with the source: - >>> with EventSource(input_url="dataset://gamma_test_large.simtel.gz", max_events=2) as source: + >>> with EventSource(input_url="dataset://gamma_prod5.simtel.zst", max_events=2) as source: ... for event in source: ... print(event.count) 0 diff --git a/ctapipe/io/simteleventsource.py b/ctapipe/io/simteleventsource.py index 853a47910b5..cdab4f590cb 100644 --- a/ctapipe/io/simteleventsource.py +++ b/ctapipe/io/simteleventsource.py @@ -180,7 +180,7 @@ class SimTelEventSource(EventSource): focal_length_choice = CaselessStrEnum( ["nominal", "effective"], - default_value="nominal", + default_value="effective", help=( "if both nominal and effective focal lengths are available in the " "SimTelArray file, which one to use when constructing the " @@ -306,26 +306,31 @@ def prepare_subarray_info(self, telescope_descriptions, header): pixel_settings = telescope_description["pixel_settings"] n_pixels = cam_settings["n_pixels"] - focal_length = u.Quantity(cam_settings["focal_length"], u.m) mirror_area = u.Quantity(cam_settings["mirror_area"], u.m**2) - if self.focal_length_choice == "effective": - try: - focal_length = u.Quantity( - cam_settings["effective_focal_length"], u.m - ) - except KeyError as err: - raise RuntimeError( - f"the SimTelEventSource option 'focal_length_choice' was set to " - f"{self.focal_length_choice}, but the effective focal length " - f"was not present in the file. ({err})" - ) + nominal_focal_length = u.Quantity(cam_settings["focal_length"], u.m) + effective_focal_length = u.Quantity( + cam_settings.get("effective_focal_length", np.nan), u.m + ) try: - telescope = guess_telescope(n_pixels, focal_length) + telescope = guess_telescope(n_pixels, nominal_focal_length) except ValueError: telescope = unknown_telescope(mirror_area, n_pixels) + if self.focal_length_choice == "effective": + if np.isnan(effective_focal_length): + raise RuntimeError( + f"`SimTelEventSource.focal_length_choice` was set to" + f" {self.focal_length_choice!r}, but the effective focal length" + f" was not present in the file. " + " Use nominal focal length or adapt your simulation configuration" + " to include the effective focal length" + ) + focal_length = effective_focal_length + else: + focal_length = nominal_focal_length + optics = OpticsDescription( name=telescope.name, num_mirrors=telescope.n_mirrors, diff --git a/ctapipe/io/tests/conftest.py b/ctapipe/io/tests/conftest.py index 27705d2c05a..8c818a23302 100644 --- a/ctapipe/io/tests/conftest.py +++ b/ctapipe/io/tests/conftest.py @@ -1,6 +1,5 @@ import pytest from ctapipe.io import EventSource, DataWriter -from ctapipe.utils import get_dataset_path @pytest.fixture(scope="session") @@ -9,11 +8,10 @@ def r1_path(tmp_path_factory): @pytest.fixture(scope="session") -def r1_hdf5_file(r1_path): +def r1_hdf5_file(prod5_proton_simtel_path, r1_path): source = EventSource( - get_dataset_path("gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz"), + prod5_proton_simtel_path, max_events=5, - allowed_tels=[1, 2, 3, 4], ) path = r1_path / "test_r1.h5" diff --git a/ctapipe/io/tests/test_datawriter.py b/ctapipe/io/tests/test_datawriter.py index 97feb96d752..ce5d0f24cb9 100644 --- a/ctapipe/io/tests/test_datawriter.py +++ b/ctapipe/io/tests/test_datawriter.py @@ -51,6 +51,7 @@ def test_write(tmpdir: Path): get_dataset_path("gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz"), max_events=20, allowed_tels=[1, 2, 3, 4], + focal_length_choice='nominal', ) calibrate = CameraCalibrator(subarray=source.subarray) @@ -131,6 +132,7 @@ def test_roundtrip(tmpdir: Path): get_dataset_path("gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz"), max_events=20, allowed_tels=[1, 2, 3, 4], + focal_length_choice='nominal', ) calibrate = CameraCalibrator(subarray=source.subarray) @@ -203,7 +205,7 @@ def test_dl1writer_no_events(tmpdir: Path): output_path = Path(tmpdir / "no_events.dl1.h5") dataset = "lst_prod3_calibration_and_mcphotons.simtel.zst" - with EventSource(get_dataset_path(dataset)) as source: + with EventSource(get_dataset_path(dataset), focal_length_choice='nominal') as source: # exhaust source for _ in source: pass @@ -233,7 +235,7 @@ def test_dl1writer_no_events(tmpdir: Path): def test_metadata(tmpdir: Path): output_path = Path(tmpdir / "metadata.dl1.h5") - dataset = "lst_prod3_calibration_and_mcphotons.simtel.zst" + dataset = "gamma_20deg_0deg_run2___cta-prod5-paranal_desert-2147m-Paranal-dark_cone10-100evts.simtel.zst" config = Config( { diff --git a/ctapipe/io/tests/test_event_source.py b/ctapipe/io/tests/test_event_source.py index a46f85061cf..9a839ff89dc 100644 --- a/ctapipe/io/tests/test_event_source.py +++ b/ctapipe/io/tests/test_event_source.py @@ -6,6 +6,8 @@ from ctapipe.core import Component +prod5_path = "gamma_20deg_0deg_run2___cta-prod5-paranal_desert-2147m-Paranal-dark_cone10-100evts.simtel.zst" + def test_construct(): # at least one of input_url / parent / config is required @@ -43,20 +45,20 @@ def datalevels(self): def test_can_be_implemented(): - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) test_reader = DummyReader(input_url=dataset) assert test_reader is not None def test_is_iterable(): - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) test_reader = DummyReader(input_url=dataset) for _ in test_reader: pass def test_function(): - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) reader = EventSource(input_url=dataset) assert isinstance(reader, SimTelEventSource) assert reader.input_url == dataset @@ -75,7 +77,7 @@ def test_function_nonexistant_file(): def test_from_config(): - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) config = Config({"EventSource": {"input_url": dataset}}) reader = EventSource(config=config) assert isinstance(reader, SimTelEventSource) @@ -83,7 +85,7 @@ def test_from_config(): def test_from_config_parent(): - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) class Parent(Component): def __init__(self, config=None, parent=None): @@ -109,7 +111,7 @@ def __init__(self, config=None, parent=None): def test_from_config_default(): old_default = EventSource.input_url.default_value - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) EventSource.input_url.default_value = dataset config = Config() reader = EventSource(config=config, parent=None) @@ -119,7 +121,7 @@ def test_from_config_default(): def test_from_config_invalid_type(): - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) EventSource.input_url.default_value = dataset config = Config({"EventSource": {"input_url": 124}}) with pytest.raises(TraitError): @@ -127,10 +129,10 @@ def test_from_config_invalid_type(): def test_event_source_input_url_config_override(): - dataset1 = get_dataset_path("gamma_test_large.simtel.gz") - dataset2 = get_dataset_path( + dataset1 = get_dataset_path( "gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz" ) + dataset2 = get_dataset_path(prod5_path) config = Config({"EventSource": {"input_url": dataset1}}) reader = EventSource(input_url=dataset2, config=config) @@ -141,13 +143,13 @@ def test_event_source_input_url_config_override(): def test_max_events(): max_events = 10 - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) reader = EventSource(input_url=dataset, max_events=max_events) assert reader.max_events == max_events def test_max_events_from_config(): - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) max_events = 10 config = Config({"EventSource": {"input_url": dataset, "max_events": max_events}}) reader = EventSource(config=config) @@ -155,15 +157,15 @@ def test_max_events_from_config(): def test_allowed_tels(): - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) reader = EventSource(input_url=dataset) assert reader.allowed_tels is None reader = EventSource(input_url=dataset, allowed_tels={1, 3}) - assert len(reader.allowed_tels) == 2 + assert reader.allowed_tels == {1, 3} def test_allowed_tels_from_config(): - dataset = get_dataset_path("gamma_test_large.simtel.gz") + dataset = get_dataset_path(prod5_path) config = Config({"EventSource": {"input_url": dataset, "allowed_tels": {1, 3}}}) reader = EventSource(config=config, parent=None) - assert len(reader.allowed_tels) == 2 + assert reader.allowed_tels == {1, 3} diff --git a/ctapipe/io/tests/test_eventseeker.py b/ctapipe/io/tests/test_eventseeker.py index fee3119c380..58ec2547afa 100644 --- a/ctapipe/io/tests/test_eventseeker.py +++ b/ctapipe/io/tests/test_eventseeker.py @@ -8,7 +8,11 @@ def test_eventseeker(): - with SimTelEventSource(input_url=dataset, back_seekable=True) as reader: + with SimTelEventSource( + input_url=dataset, + back_seekable=True, + focal_length_choice='nominal', + ) as reader: seeker = EventSeeker(event_source=reader) @@ -33,7 +37,8 @@ def test_eventseeker(): seeker.get_event_index(dict()) with SimTelEventSource( - input_url=dataset, max_events=5, back_seekable=True + input_url=dataset, max_events=5, back_seekable=True, + focal_length_choice='nominal', ) as reader: seeker = EventSeeker(event_source=reader) with pytest.raises(IndexError): @@ -42,7 +47,10 @@ def test_eventseeker(): def test_eventseeker_edit(): - with SimTelEventSource(input_url=dataset, back_seekable=True) as reader: + with SimTelEventSource( + input_url=dataset, back_seekable=True, + focal_length_choice='nominal', + ) as reader: seeker = EventSeeker(event_source=reader) event = seeker.get_event_index(1) assert event.count == 1 @@ -54,7 +62,10 @@ def test_eventseeker_edit(): def test_eventseeker_simtel(): # Ensure the EventSeeker can forward seek even if back-seeking is not possible - with SimTelEventSource(input_url=dataset, back_seekable=False) as reader: + with SimTelEventSource( + input_url=dataset, back_seekable=False, + focal_length_choice='nominal', + ) as reader: seeker = EventSeeker(event_source=reader) event = seeker.get_event_index(1) assert event.count == 1 diff --git a/ctapipe/io/tests/test_hdf5eventsource.py b/ctapipe/io/tests/test_hdf5eventsource.py index a8634e79985..eaab48b2b0f 100644 --- a/ctapipe/io/tests/test_hdf5eventsource.py +++ b/ctapipe/io/tests/test_hdf5eventsource.py @@ -136,4 +136,4 @@ def test_read_r1(r1_hdf5_file): pass assert e is not None - assert e.count == 4 + assert e.count == 3 diff --git a/ctapipe/io/tests/test_prod2.py b/ctapipe/io/tests/test_prod2.py index 200d2406014..98be8be0eb5 100644 --- a/ctapipe/io/tests/test_prod2.py +++ b/ctapipe/io/tests/test_prod2.py @@ -9,7 +9,10 @@ def test_eventio_prod2(): with pytest.warns(UnknownPixelShapeWarning): - with SimTelEventSource(input_url=dataset) as reader: + with SimTelEventSource( + input_url=dataset, + focal_length_choice='nominal', + ) as reader: for event in reader: if event.count == 2: break diff --git a/ctapipe/io/tests/test_simteleventsource.py b/ctapipe/io/tests/test_simteleventsource.py index d94a09c61b4..77eba9e8d12 100644 --- a/ctapipe/io/tests/test_simteleventsource.py +++ b/ctapipe/io/tests/test_simteleventsource.py @@ -2,8 +2,8 @@ from ctapipe.instrument.camera.geometry import UnknownPixelShapeWarning import numpy as np -from astropy.utils.data import download_file import astropy.units as u +from astropy.coordinates import Angle from itertools import zip_longest import pytest from astropy.time import Time @@ -20,18 +20,21 @@ gamma_test_large_path = get_dataset_path("gamma_test_large.simtel.gz") gamma_test_path = get_dataset_path("gamma_test.simtel.gz") calib_events_path = get_dataset_path("lst_prod3_calibration_and_mcphotons.simtel.zst") +prod5b_path = get_dataset_path("gamma_20deg_0deg_run2___cta-prod5-paranal_desert-2147m-Paranal-dark_cone10-100evts.simtel.zst") def test_positional_input(): - source = SimTelEventSource(gamma_test_large_path) - assert source.input_url == Path(gamma_test_large_path) + source = SimTelEventSource(prod5b_path) + assert source.input_url == Path(prod5b_path) def test_simtel_event_source_on_gamma_test_one_event(): + assert SimTelEventSource.is_compatible(gamma_test_large_path) + with SimTelEventSource( - input_url=gamma_test_large_path, back_seekable=True + input_url=gamma_test_large_path, back_seekable=True, + focal_length_choice='nominal', ) as reader: - assert reader.is_compatible(gamma_test_large_path) assert not reader.is_stream for event in reader: @@ -47,7 +50,7 @@ def test_simtel_event_source_on_gamma_test_one_event(): def test_that_event_is_not_modified_after_loop(): - dataset = gamma_test_large_path + dataset = prod5b_path with SimTelEventSource(input_url=dataset, max_events=2) as source: for event in source: last_event = copy.deepcopy(event) @@ -61,12 +64,11 @@ def test_that_event_is_not_modified_after_loop(): def test_additional_meta_data_from_simulation_config(): - with SimTelEventSource(input_url=gamma_test_large_path) as reader: - next(iter(reader)) - - # for expectation values - from astropy import units as u - from astropy.coordinates import Angle + with SimTelEventSource( + input_url=gamma_test_large_path, + focal_length_choice='nominal', + ) as reader: + pass # There should be only one observation assert len(reader.obs_ids) == 1 @@ -104,7 +106,10 @@ def test_additional_meta_data_from_simulation_config(): def test_properties(): - source = SimTelEventSource(input_url=gamma_test_large_path) + source = SimTelEventSource( + input_url=gamma_test_large_path, + focal_length_choice='nominal', + ) assert source.is_simulation assert source.datalevels == (DataLevel.R0, DataLevel.R1) @@ -112,11 +117,14 @@ def test_properties(): assert source.simulation_config[7514].corsika_version == 6990 -def test_gamma_file(): +def test_gamma_file_prod2(): dataset = gamma_test_path with pytest.warns(UnknownPixelShapeWarning): - with SimTelEventSource(input_url=dataset) as reader: + with SimTelEventSource( + input_url=dataset, + focal_length_choice='nominal', + ) as reader: assert reader.is_compatible(dataset) assert reader.is_stream # using gzip subprocess makes it a stream @@ -134,7 +142,9 @@ def test_gamma_file(): def test_max_events(): max_events = 5 with SimTelEventSource( - input_url=gamma_test_large_path, max_events=max_events + input_url=gamma_test_large_path, + max_events=max_events, + focal_length_choice='nominal', ) as reader: count = 0 for _ in reader: @@ -143,7 +153,11 @@ def test_max_events(): def test_pointing(): - with SimTelEventSource(input_url=gamma_test_large_path, max_events=3) as reader: + with SimTelEventSource( + input_url=gamma_test_large_path, + max_events=3, + focal_length_choice='nominal', + ) as reader: for e in reader: assert np.isclose(e.pointing.array_altitude.to_value(u.deg), 70) assert np.isclose(e.pointing.array_azimuth.to_value(u.deg), 0) @@ -160,7 +174,8 @@ def test_allowed_telescopes(): # test that the allowed_tels mask works: allowed_tels = {3, 4} with SimTelEventSource( - input_url=gamma_test_large_path, allowed_tels=allowed_tels, max_events=5 + input_url=gamma_test_large_path, allowed_tels=allowed_tels, max_events=5, + focal_length_choice='nominal', ) as reader: assert not allowed_tels.symmetric_difference(reader.subarray.tel_ids) for event in reader: @@ -188,7 +203,8 @@ def test_calibration_events(): EventType.SUBARRAY, ] with SimTelEventSource( - input_url=calib_events_path, skip_calibration_events=False + input_url=calib_events_path, skip_calibration_events=False, + focal_length_choice='nominal', ) as reader: for event, expected_type in zip_longest(reader, expected_types): @@ -197,7 +213,10 @@ def test_calibration_events(): def test_trigger_times(): - source = SimTelEventSource(input_url=calib_events_path) + source = SimTelEventSource( + input_url=calib_events_path, + focal_length_choice='nominal', + ) t0 = Time("2020-05-06T15:30:00") t1 = Time("2020-05-06T15:40:00") @@ -209,7 +228,10 @@ def test_trigger_times(): def test_true_image(): - with SimTelEventSource(input_url=calib_events_path) as reader: + with SimTelEventSource( + input_url=calib_events_path, + focal_length_choice='nominal', + ) as reader: for event in reader: for tel in event.simulation.tel.values(): @@ -218,7 +240,10 @@ def test_true_image(): def test_instrument(): """Test if same telescope types share a single instance of CameraGeometry""" - source = SimTelEventSource(input_url=gamma_test_large_path) + source = SimTelEventSource( + input_url=gamma_test_large_path, + focal_length_choice='nominal', + ) subarray = source.subarray assert subarray.tel[1].optics.num_mirrors == 1 @@ -278,39 +303,36 @@ def test_apply_simtel_r1_calibration_2_channel(): assert r1_waveforms[1, 0] == (r0_waveforms[0, 1, 0] - ped[0, 1]) * dc_to_pe[0, 1] -def test_effective_focal_length(): - test_file_url = ( - "https://github.com/cta-observatory/pyeventio/raw/master/tests" - "/resources/prod4_pixelsettings_v3.gz" - ) - test_file = download_file(test_file_url) +def test_focal_length_choice(): + # this file does not contain the effective focal length + with pytest.raises(RuntimeError): + SimTelEventSource(gamma_test_large_path) - focal_length_nominal = 0 - focal_length_effective = 0 + with pytest.raises(RuntimeError): + SimTelEventSource(gamma_test_large_path, focal_length_choice='effective') - with SimTelEventSource( - input_url=test_file, focal_length_choice="nominal" - ) as source: - subarray = source.subarray - focal_length_nominal = subarray.tel[1].optics.equivalent_focal_length + s = SimTelEventSource(gamma_test_large_path, focal_length_choice='nominal') + assert s.subarray.tel[1].optics.equivalent_focal_length == 28 * u.m - with SimTelEventSource( - input_url=test_file, focal_length_choice="effective" - ) as source: - subarray = source.subarray - focal_length_effective = subarray.tel[1].optics.equivalent_focal_length + # this file does + s = SimTelEventSource(prod5b_path, focal_length_choice='effective') + assert u.isclose(s.subarray.tel[1].optics.equivalent_focal_length, 29.3 * u.m, atol=0.05 * u.m) + # check guessing of the name is not affected by focal length choice + assert str(s.subarray.tel[1]) == 'LST_LST_LSTCam' + + s = SimTelEventSource(prod5b_path, focal_length_choice='nominal') + assert u.isclose(s.subarray.tel[1].optics.equivalent_focal_length, 28.0 * u.m, atol=0.05 * u.m) + # check guessing of the name is not affected by focal length choice + assert str(s.subarray.tel[1]) == 'LST_LST_LSTCam' - assert focal_length_nominal > 0 - assert focal_length_effective > 0 - assert focal_length_nominal != focal_length_effective def test_only_config(): config = Config() - config.SimTelEventSource.input_url = gamma_test_large_path + config.SimTelEventSource.input_url = prod5b_path s = SimTelEventSource(config=config) - assert s.input_url == Path(gamma_test_large_path).absolute() + assert s.input_url == Path(prod5b_path).absolute() def test_calibscale_and_calibshift(prod5_gamma_simtel_path): @@ -355,11 +377,17 @@ def test_calibscale_and_calibshift(prod5_gamma_simtel_path): def test_true_image_sum(): # this file does not contain true pe info - with SimTelEventSource(gamma_test_large_path) as s: + with SimTelEventSource( + gamma_test_large_path, + focal_length_choice='nominal', + ) as s: e = next(iter(s)) assert np.all(np.isnan(sim.true_image_sum) for sim in e.simulation.tel.values()) - with SimTelEventSource(calib_events_path) as s: + with SimTelEventSource( + calib_events_path, + focal_length_choice='nominal', + ) as s: e = next(iter(s)) true_image_sums = {} @@ -371,8 +399,10 @@ def test_true_image_sum(): # check it also works with allowed_tels, since the values # are stored in a flat array in simtel - with SimTelEventSource(calib_events_path, allowed_tels={2, 3}) as s: + with SimTelEventSource( + calib_events_path, allowed_tels={2, 3}, + focal_length_choice='nominal', + ) as s: e = next(iter(s)) - assert e.simulation.tel[2].true_image_sum == true_image_sums[2] assert e.simulation.tel[3].true_image_sum == true_image_sums[3] diff --git a/ctapipe/reco/tests/test_HillasReconstructor.py b/ctapipe/reco/tests/test_HillasReconstructor.py index 7ee4f1ad0c5..0b594cab952 100644 --- a/ctapipe/reco/tests/test_HillasReconstructor.py +++ b/ctapipe/reco/tests/test_HillasReconstructor.py @@ -197,11 +197,11 @@ def test_reconstruction_against_simulation(subarray_and_event_gamma_off_axis_500 ) def test_CameraFrame_against_TelescopeFrame(filename): - input_file = get_dataset_path( - "gamma_divergent_LaPalma_baseline_20Zd_180Az_prod3_test.simtel.gz" - ) + input_file = get_dataset_path(filename) + # "gamma_divergent_LaPalma_baseline_20Zd_180Az_prod3_test.simtel.gz" + # ) - source = SimTelEventSource(input_file, max_events=10) + source = SimTelEventSource(input_file, max_events=10, focal_length_choice="nominal") # too few events survive for this test with the defautl quality criteria, # use less restrictive ones diff --git a/ctapipe/reco/tests/test_reconstruction_methods.py b/ctapipe/reco/tests/test_reconstruction_methods.py index 1989e3962e8..ec99c7488bd 100644 --- a/ctapipe/reco/tests/test_reconstruction_methods.py +++ b/ctapipe/reco/tests/test_reconstruction_methods.py @@ -30,7 +30,7 @@ def test_reconstructors(reconstructors): "gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz" ) - source = EventSource(filename, max_events=10) + source = EventSource(filename, max_events=10, focal_length_choice="nominal") subarray = source.subarray calib = CameraCalibrator(source.subarray) image_processor = ImageProcessor(source.subarray) diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index d8f1a4b88ec..c9f4fbb5459 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -213,6 +213,7 @@ def test_stage_2_from_simtel(tmp_path): f"--output={output}", "--max-events=5", "--overwrite", + "--SimTelEventSource.focal_length_choice=nominal", ], cwd=tmp_path, ) @@ -288,6 +289,7 @@ def test_training_from_simtel(tmp_path): f"--output={output}", "--max-events=5", "--overwrite", + "--SimTelEventSource.focal_length_choice=nominal", ], cwd=tmp_path, ) diff --git a/ctapipe/tools/tests/test_tools.py b/ctapipe/tools/tests/test_tools.py index f906213b40d..ebf74a751a1 100644 --- a/ctapipe/tools/tests/test_tools.py +++ b/ctapipe/tools/tests/test_tools.py @@ -13,6 +13,7 @@ GAMMA_TEST_LARGE = get_dataset_path("gamma_test_large.simtel.gz") LST_MUONS = get_dataset_path("lst_muons.simtel.zst") +PROD5B_PATH = get_dataset_path("gamma_20deg_0deg_run2___cta-prod5-paranal_desert-2147m-Paranal-dark_cone10-100evts.simtel.zst") def test_muon_reconstruction_simtel(tmp_path): @@ -25,6 +26,7 @@ def test_muon_reconstruction_simtel(tmp_path): argv=[ f"--input={LST_MUONS}", f"--output={muon_simtel_output_file}", + "--SimTelEventSource.focal_length_choice=nominal", "--overwrite", ], cwd=tmp_path, @@ -71,7 +73,12 @@ def test_display_dl1(tmp_path, dl1_image_file, dl1_parameters_file): # test simtel assert ( run_tool( - DisplayDL1Calib(), argv=["--max-events=1", "--telescope=11"], cwd=tmp_path + DisplayDL1Calib(), + argv=[ + "--max-events=1", "--telescope=11", + "--SimTelEventSource.focal_length_choice=nominal", + ], + cwd=tmp_path ) == 0 ) @@ -125,7 +132,7 @@ def test_dump_triggers(tmp_path): sys.argv = ["dump_triggers"] outfile = tmp_path / "triggers.fits" - tool = DumpTriggersTool(infile=GAMMA_TEST_LARGE, outfile=str(outfile)) + tool = DumpTriggersTool(infile=PROD5B_PATH, outfile=str(outfile)) assert run_tool(tool, cwd=tmp_path) == 0 @@ -139,18 +146,18 @@ def test_dump_instrument(tmp_path): sys.argv = ["dump_instrument"] tool = DumpInstrumentTool() - assert run_tool(tool, [f"--input={GAMMA_TEST_LARGE}"], cwd=tmp_path) == 0 + assert run_tool(tool, [f"--input={PROD5B_PATH}"], cwd=tmp_path) == 0 assert (tmp_path / "FlashCam.camgeom.fits.gz").exists() assert ( - run_tool(tool, [f"--input={GAMMA_TEST_LARGE}", "--format=ecsv"], cwd=tmp_path) + run_tool(tool, [f"--input={PROD5B_PATH}", "--format=ecsv"], cwd=tmp_path) == 0 ) assert (tmp_path / "MonteCarloArray.optics.ecsv").exists() assert ( - run_tool(tool, [f"--input={GAMMA_TEST_LARGE}", "--format=hdf5"], cwd=tmp_path) + run_tool(tool, [f"--input={PROD5B_PATH}", "--format=hdf5"], cwd=tmp_path) == 0 ) assert (tmp_path / "subarray.h5").exists() diff --git a/ctapipe/utils/datasets.py b/ctapipe/utils/datasets.py index 42f187f2b82..71ba61485ed 100644 --- a/ctapipe/utils/datasets.py +++ b/ctapipe/utils/datasets.py @@ -28,7 +28,7 @@ __all__ = ["get_dataset_path", "find_in_path", "find_all_matching_datasets"] -DEFAULT_URL = "http://cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.3/" +DEFAULT_URL = "http://cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.4/" def get_searchpath_dirs(searchpath=os.getenv("CTAPIPE_SVC_PATH"), url=DEFAULT_URL): diff --git a/docs/examples/InstrumentDescription.ipynb b/docs/examples/InstrumentDescription.ipynb index f70a901ce53..dbefaa51c48 100644 --- a/docs/examples/InstrumentDescription.ipynb +++ b/docs/examples/InstrumentDescription.ipynb @@ -21,8 +21,7 @@ "from ctapipe.io import EventSource\n", "import numpy as np\n", "\n", - "#filename = get_dataset_path(\"gamma_test_large.simtel.gz\") # try this one as well\n", - "filename = get_dataset_path(\"gamma_test_large.simtel.gz\") \n", + "filename = get_dataset_path(\"gamma_prod5.simtel.zst\") \n", "\n", "with EventSource(filename, max_events=1) as source:\n", " subarray = source.subarray" @@ -359,7 +358,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -373,7 +372,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.8.13" } }, "nbformat": 4, diff --git a/docs/tutorials/calibrated_data_exploration.ipynb b/docs/tutorials/calibrated_data_exploration.ipynb index d2c5bddf28d..725d29d84a1 100644 --- a/docs/tutorials/calibrated_data_exploration.ipynb +++ b/docs/tutorials/calibrated_data_exploration.ipynb @@ -48,7 +48,7 @@ "metadata": {}, "outputs": [], "source": [ - "filename = get_dataset_path(\"gamma_test.simtel.gz\")\n", + "filename = get_dataset_path(\"gamma_prod5.simtel.zst\")\n", "source = EventSource(filename, max_events=2)\n", "\n", "for event in source:\n", @@ -85,7 +85,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [], "source": [ "print(event.r1)" @@ -351,11 +353,18 @@ "ad.values = hit_pattern\n", "ad.add_labels()\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -369,7 +378,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.8.13" } }, "nbformat": 4, diff --git a/docs/tutorials/coordinates_example.ipynb b/docs/tutorials/coordinates_example.ipynb index 6f6d2d47042..429efc7ed15 100644 --- a/docs/tutorials/coordinates_example.ipynb +++ b/docs/tutorials/coordinates_example.ipynb @@ -90,11 +90,11 @@ }, "outputs": [], "source": [ - "filename = get_dataset_path(\"gamma_test_large.simtel.gz\")\n", - "source = EventSource(filename, max_events=4)\n", + "filename = get_dataset_path(\"gamma_prod5.simtel.zst\")\n", + "source = EventSource(filename)\n", "\n", "events = [copy.deepcopy(event) for event in source]\n", - "event = events[3]\n", + "event = events[4]\n", "\n", "layout = set(source.subarray.tel_ids) " ] @@ -123,8 +123,8 @@ }, "outputs": [], "source": [ - "print(f'Telescope with data: {event.r0.tel.keys()}')\n", - "tel_id = 2" + "print(f'Telescope with data: {event.r1.tel.keys()}')\n", + "tel_id = 3" ] }, { @@ -620,7 +620,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -634,7 +634,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.8.13" } }, "nbformat": 4, diff --git a/docs/tutorials/ctapipe_handson.ipynb b/docs/tutorials/ctapipe_handson.ipynb index d9ce8fd3760..68115dfd9f5 100644 --- a/docs/tutorials/ctapipe_handson.ipynb +++ b/docs/tutorials/ctapipe_handson.ipynb @@ -35,7 +35,7 @@ "metadata": {}, "outputs": [], "source": [ - "path = utils.get_dataset_path(\"gamma_test_large.simtel.gz\")" + "path = utils.get_dataset_path(\"gamma_prod5.simtel.zst\")" ] }, { @@ -44,7 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "source = EventSource(path, max_events=4)\n", + "source = EventSource(path, max_events=5)\n", "\n", "for event in source:\n", " print(event.count, event.index.event_id, event.simulation.shower.energy)" @@ -65,7 +65,7 @@ "metadata": {}, "outputs": [], "source": [ - "event.r0" + "event.r1" ] }, { @@ -74,8 +74,8 @@ "metadata": {}, "outputs": [], "source": [ - "for event in EventSource(path, max_events=4):\n", - " print(event.count, event.r0.tel.keys())" + "for event in EventSource(path, max_events=5):\n", + " print(event.count, event.r1.tel.keys())" ] }, { @@ -84,7 +84,7 @@ "metadata": {}, "outputs": [], "source": [ - "event.r0.tel[2]" + "event.r0.tel[3]" ] }, { @@ -366,7 +366,7 @@ "metadata": {}, "outputs": [], "source": [ - "for event in EventSource(path, max_events=4):\n", + "for event in EventSource(path, max_events=5):\n", " calib(event) # fills in r1, dl0, and dl1\n", " print(event.dl1.tel.keys())" ] @@ -480,8 +480,8 @@ "disp = CameraDisplay(tel.camera.geometry, image=cleaned)\n", "disp.cmap = plt.cm.coolwarm\n", "disp.add_colorbar()\n", - "plt.xlim(-1.0,0)\n", - "plt.ylim(0,1.0)" + "plt.xlim(0.5, 1.0)\n", + "plt.ylim(-1.0, 0.0)" ] }, { @@ -503,8 +503,8 @@ "disp = CameraDisplay(tel.camera.geometry, image=cleaned)\n", "disp.cmap = plt.cm.coolwarm\n", "disp.add_colorbar()\n", - "plt.xlim(-1.0,0)\n", - "plt.ylim(0,1.0)\n", + "plt.xlim(0.5, 1.0)\n", + "plt.ylim(-1.0, 0.0)\n", "disp.overlay_moments(params, color='white', lw=2)" ] }, @@ -557,8 +557,8 @@ "metadata": {}, "outputs": [], "source": [ - "data = utils.get_dataset_path(\"gamma_test_large.simtel.gz\") \n", - "source = EventSource(data, allowed_tels=[1,2,3,4], max_events=10) # remove the max_events limit to get more stats" + "data = utils.get_dataset_path(\"gamma_prod5.simtel.zst\") \n", + "source = EventSource(data) # remove the max_events limit to get more stats" ] }, { @@ -573,7 +573,8 @@ " for tel_id, tel_data in event.dl1.tel.items():\n", " tel = source.subarray.tel[tel_id]\n", " mask = tailcuts_clean(tel.camera.geometry, tel_data.image)\n", - " params = hillas_parameters(tel.camera.geometry[mask], tel_data.image[mask])" + " if np.count_nonzero(mask) > 0:\n", + " params = hillas_parameters(tel.camera.geometry[mask], tel_data.image[mask])" ] }, { @@ -645,20 +646,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If you do this yourself, loop over more events to get better statistics" + "If you do this yourself, chose a larger file to loop over more events to get better statistics" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -672,7 +666,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.8" + "version": "3.8.13" } }, "nbformat": 4, diff --git a/docs/tutorials/ctapipe_overview.ipynb b/docs/tutorials/ctapipe_overview.ipynb index 0c4b98f18e2..64ae83a4cdd 100644 --- a/docs/tutorials/ctapipe_overview.ipynb +++ b/docs/tutorials/ctapipe_overview.ipynb @@ -222,11 +222,11 @@ "from ctapipe.io import EventSource\n", "from ctapipe.utils.datasets import get_dataset_path\n", "\n", - "input_url = get_dataset_path('gamma_test_large.simtel.gz')\n", + "input_url = get_dataset_path('gamma_prod5.simtel.zst')\n", "\n", "# EventSource() automatically detects what kind of file we are giving it,\n", "# if already supported by ctapipe\n", - "source = EventSource(input_url, max_events=49)\n", + "source = EventSource(input_url, max_events=5)\n", "\n", "print(type(source))" ] @@ -351,7 +351,7 @@ }, "outputs": [], "source": [ - "tel_id = 4" + "tel_id = 130" ] }, { @@ -425,12 +425,12 @@ }, "outputs": [], "source": [ - "# unoptimized cleaning levels, copied from \n", - "# https://github.com/tudo-astroparticlephysics/cta_preprocessing\n", + "# unoptimized cleaning levels\n", "cleaning_level = {\n", - " 'ASTRICam': (5, 7, 2), # (5, 10)?\n", - " 'LSTCam': (3.5, 7.5, 2), # ?? (3, 6) for Abelardo...\n", - " 'FlashCam': (4, 8, 2), # there is some scaling missing?\n", + " 'CHEC': (2, 4, 2),\n", + " 'LSTCam': (3.5, 7, 2),\n", + " 'FlashCam': (3.5, 7, 2), \n", + " 'NectarCam': (4, 8, 2), \n", "}" ] }, @@ -675,19 +675,21 @@ " \"picture_threshold_pe\": [\n", " (\"type\", \"LST_LST_LSTCam\", 7.5),\n", " (\"type\", \"MST_MST_FlashCam\", 8),\n", - " (\"type\", \"SST_ASTRI_ASTRICam\", 7),\n", + " (\"type\", \"MST_MST_NectarCam\", 8),\n", + " (\"type\", \"SST_ASTRI_CHEC\", 7),\n", " ],\n", " \"boundary_threshold_pe\": [\n", " (\"type\", \"LST_LST_LSTCam\", 5),\n", " (\"type\", \"MST_MST_FlashCam\", 4),\n", - " (\"type\", \"SST_ASTRI_ASTRICam\", 4),\n", + " (\"type\", \"MST_MST_NectarCam\", 4),\n", + " (\"type\", \"SST_ASTRI_CHEC\", 4),\n", " ]\n", " \n", " }\n", " }\n", "})\n", "\n", - "input_url = get_dataset_path('gamma_test_large.simtel.gz')\n", + "input_url = get_dataset_path('gamma_prod5.simtel.zst')\n", "source = EventSource(input_url)\n", "\n", "calibrator = CameraCalibrator(subarray=source.subarray)\n", @@ -809,8 +811,8 @@ ")\n", "\n", "plt.legend()\n", - "plt.xlim(-400, 400)\n", - "plt.ylim(-400, 400)" + "# plt.xlim(-400, 400)\n", + "# plt.ylim(-400, 400)" ] }, { diff --git a/docs/tutorials/raw_data_exploration.ipynb b/docs/tutorials/raw_data_exploration.ipynb index 6c2d63207f8..884fd025696 100644 --- a/docs/tutorials/raw_data_exploration.ipynb +++ b/docs/tutorials/raw_data_exploration.ipynb @@ -24,11 +24,12 @@ "outputs": [], "source": [ "from ctapipe.utils import get_dataset_path\n", - "from ctapipe.io import EventSource, EventSeeker\n", + "from ctapipe.io import EventSource\n", "from ctapipe.visualization import CameraDisplay\n", "from ctapipe.instrument import CameraGeometry\n", "from matplotlib import pyplot as plt\n", "from astropy import units as u\n", + "\n", "%matplotlib inline" ] }, @@ -47,8 +48,7 @@ "metadata": {}, "outputs": [], "source": [ - "source = EventSource(get_dataset_path(\"gamma_test_large.simtel.gz\"), max_events=100, back_seekable=True)\n", - "seeker = EventSeeker(source)" + "source = EventSource(get_dataset_path(\"gamma_prod5.simtel.zst\"), max_events=5)" ] }, { @@ -66,8 +66,10 @@ "metadata": {}, "outputs": [], "source": [ - "event = seeker.get_event_index(3) # get 3rd event\n", - "event" + "# so we can advance through events one-by-one\n", + "event_iterator = iter(source)\n", + "\n", + "event = next(event_iterator)" ] }, { @@ -83,7 +85,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(repr(event.r0))" + "event.r0" ] }, { @@ -115,7 +117,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "note that the event has 2 telescopes in it: 38,40... Let's try the next one:" + "note that the event has 3 telescopes in it: Let's try the next one:" ] }, { @@ -124,7 +126,7 @@ "metadata": {}, "outputs": [], "source": [ - "event = seeker.get_event_index(5) # get the next event\n", + "event = next(event_iterator)\n", "print(event.r0.tel.keys())" ] }, @@ -132,7 +134,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "now, we have a larger event with many telescopes... Let's look at the data from **CT7**:" + "now, we have a larger event with many telescopes... Let's look at one of them:" ] }, { @@ -141,7 +143,7 @@ "metadata": {}, "outputs": [], "source": [ - "teldata = event.r0.tel[15]\n", + "teldata = event.r0.tel[26]\n", "print(teldata)\n", "teldata" ] @@ -419,7 +421,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [], "source": [ "for tel in event.r0.tel.keys():\n", @@ -488,7 +492,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -502,7 +506,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.8" + "version": "3.8.13" } }, "nbformat": 4, diff --git a/docs/tutorials/theta_square.ipynb b/docs/tutorials/theta_square.ipynb index b141c58626e..91191ae275a 100644 --- a/docs/tutorials/theta_square.ipynb +++ b/docs/tutorials/theta_square.ipynb @@ -73,7 +73,10 @@ }, "outputs": [], "source": [ - "source = EventSource(\"dataset://gamma_test_large.simtel.gz\", allowed_tels={1, 2, 3, 4})\n", + "source = EventSource(\n", + " \"dataset://gamma_prod5.simtel.zst\",\n", + "# allowed_tels={1, 2, 3, 4},\n", + ")\n", "\n", "subarray = source.subarray\n", "\n",