From b312680d6f8b1352b48120934da52d0282f8f8fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Thu, 21 Feb 2019 17:26:21 +0100 Subject: [PATCH 01/23] Only guess telescope names --- ctapipe/instrument/__init__.py | 2 + ctapipe/instrument/camera.py | 85 +++------------------- ctapipe/instrument/guess.py | 53 ++++++++++++++ ctapipe/instrument/optics.py | 74 ++----------------- ctapipe/instrument/telescope.py | 29 ++------ ctapipe/instrument/tests/test_camera.py | 7 -- ctapipe/instrument/tests/test_guessing.py | 25 +++++++ ctapipe/instrument/tests/test_optics.py | 33 ++++----- ctapipe/instrument/tests/test_telescope.py | 17 ----- 9 files changed, 117 insertions(+), 208 deletions(-) create mode 100644 ctapipe/instrument/guess.py create mode 100644 ctapipe/instrument/tests/test_guessing.py delete mode 100644 ctapipe/instrument/tests/test_telescope.py diff --git a/ctapipe/instrument/__init__.py b/ctapipe/instrument/__init__.py index d4e4d10638d..715fe6ca41c 100644 --- a/ctapipe/instrument/__init__.py +++ b/ctapipe/instrument/__init__.py @@ -3,6 +3,7 @@ from .telescope import TelescopeDescription from .optics import OpticsDescription from .subarray import SubarrayDescription +from .guess import guess_telescope __all__ = [ @@ -11,4 +12,5 @@ 'TelescopeDescription', 'OpticsDescription', 'SubarrayDescription', + 'guess_telescope', ] diff --git a/ctapipe/instrument/camera.py b/ctapipe/instrument/camera.py index 1be691b45bd..5e6ee1f3cec 100644 --- a/ctapipe/instrument/camera.py +++ b/ctapipe/instrument/camera.py @@ -20,28 +20,18 @@ logger = logging.getLogger(__name__) -# dictionary to convert number of pixels to camera + the focal length of the -# telescope into a camera type for use in `CameraGeometry.guess()` -# Key = (num_pix, focal_length_in_meters) -# Value = (type, subtype, pixtype, pixrotation, camrotation) -_CAMERA_GEOMETRY_TABLE = { - (2048, 2.3): ('SST', 'CHEC', 'rectangular', 0 * u.degree, 0 * u.degree), - (2048, 2.2): ('SST', 'CHEC', 'rectangular', 0 * u.degree, 0 * u.degree), - (2048, 36.0): ('LST', 'HESS-II', 'hexagonal', 0 * u.degree, - 0 * u.degree), - (960, None): ('MST', 'HESS-I', 'hexagonal', 0 * u.degree, - 0 * u.degree), - (1855, 16.0): ('MST', 'NectarCam', 'hexagonal', - 0 * u.degree, -100.893 * u.degree), - (1855, 28.0): ('LST', 'LSTCam', 'hexagonal', - 0. * u.degree, -100.893 * u.degree), - (1296, None): ('SST', 'DigiCam', 'hexagonal', 30 * u.degree, 0 * u.degree), - (1764, None): ('MST', 'FlashCam', 'hexagonal', 30 * u.degree, 0 * u.degree), - (2368, None): ('SST', 'ASTRICam', 'rectangular', 0 * u.degree, - 0 * u.degree), - (11328, None): ('SCT', 'SCTCam', 'rectangular', 0 * u.degree, 0 * u.degree), -} +def simtel_shape_to_type(pixel_shape): + if pixel_shape == 1: + return 'hexagonal', Angle(0, u.deg) + + if pixel_shape == 2: + return 'rectangular', Angle(0, u.deg) + + if pixel_shape == 3: + return 'hexagonal', Angle(30, u.deg) + + raise ValueError(f'Unknown pixel_shape {pixel_shape}') class CameraGeometry: @@ -143,47 +133,6 @@ def __getitem__(self, slice_): apply_derotation=False, ) - @classmethod - @u.quantity_input - def guess(cls, pix_x: u.m, pix_y: u.m, optical_foclen: u.m, - apply_derotation=True): - """ - Construct a `CameraGeometry` by guessing the appropriate quantities - from a list of pixel positions and the focal length. - """ - # only construct a new one if it has never been constructed before, - # to speed up access. Otherwise return the already constructed instance - # the identifier uses the values of pix_x (which are converted to a - # string to make them hashable) and the optical_foclen. So far, - # that is enough to uniquely identify a geometry. - identifier = (pix_x.value.tostring(), optical_foclen) - if identifier in CameraGeometry._geometry_cache: - return CameraGeometry._geometry_cache[identifier] - - # now try to determine the camera type using the map defined at the - # top of this file. - - tel_type, cam_id, pix_type, pix_rotation, cam_rotation = \ - _guess_camera_type(len(pix_x), optical_foclen) - - area = cls._calc_pixel_area(pix_x, pix_y, pix_type) - - instance = cls( - cam_id=cam_id, - pix_id=np.arange(len(pix_x)), - pix_x=pix_x, - pix_y=pix_y, - pix_area=np.ones(pix_x.shape) * area, - neighbors=None, - pix_type=pix_type, - pix_rotation=Angle(pix_rotation), - cam_rotation=Angle(cam_rotation), - apply_derotation=apply_derotation - ) - instance.cam_rotation = Angle(cam_rotation) - CameraGeometry._geometry_cache[identifier] = instance - return instance - @staticmethod def _calc_pixel_area(pix_x, pix_y, pix_type): """ recalculate pixel area based on the pixel type and layout @@ -663,18 +612,6 @@ def _find_neighbor_pixels(pix_x, pix_y, rad): return neighbors -def _guess_camera_type(npix, optical_foclen): - global _CAMERA_GEOMETRY_TABLE - - try: - return _CAMERA_GEOMETRY_TABLE[(npix, None)] - except KeyError: - return _CAMERA_GEOMETRY_TABLE.get( - (npix, round(optical_foclen.value, 1)), - ('unknown', 'unknown', 'hexagonal', - 0 * u.degree, 0 * u.degree) - ) - def _neighbor_list_to_matrix(neighbors): """ diff --git a/ctapipe/instrument/guess.py b/ctapipe/instrument/guess.py new file mode 100644 index 00000000000..d9b9a0d9447 --- /dev/null +++ b/ctapipe/instrument/guess.py @@ -0,0 +1,53 @@ +from collections import namedtuple +import astropy.units as u + + +GuessingKey = namedtuple('GuessingKey', ['n_pixels', 'focal_length']) +GuessingResult = namedtuple( + 'GuessingResult', ['type', 'telescope_name', 'camera_name', 'mirror_type'] +) + + +TELESCOPE_NAMES = { + GuessingKey(2048, 2.28): GuessingResult('SST', 'GCT', 'CHEC', 'SC'), + GuessingKey(2368, 2.15): GuessingResult('SST', 'ASTRI', 'ASTRICam', 'SC'), + GuessingKey(1296, 5.60): GuessingResult('SST', '1M', 'DigiCam', 'DC'), + GuessingKey(1764, 16.0): GuessingResult('MST', 'MST', 'FlashCam', 'DC'), + GuessingKey(1855, 16.0): GuessingResult('MST', 'MST', 'NectarCam', 'DC'), + GuessingKey(1855, 28.0): GuessingResult('LST', 'LST', 'LSTCam', 'DC'), + GuessingKey(11328, 5.59): GuessingResult('MST', 'SCT', 'SCTCam', 'SC'), + + # None CTA Telescopes + GuessingKey(960, 15.0): GuessingResult('MST', 'HESS-I', 'HESS-I', 'DC'), + GuessingKey(2048, 36.0): GuessingResult('LST', 'HESS-II', 'HESS-II', 'DC'), + GuessingKey(1440, 4.998): GuessingResult('SST', 'FACT', 'FACT', 'DC'), +} + + +def guess_telescope(n_pixels, focal_length): + ''' + From n_pixels of the camera and the focal_length, + guess which telescope we are dealing with. + This is mainly needed to add human readable names + to telescopes read from simtel array. + + Parameters + ---------- + n_pixels: int + number of pixels of the telescope's camera + focal_length: float or u.Quantity[length] + Focal length, either in m or as astropy quantity + + Returns + ------- + result: GuessingResult + A namedtuple having type, telescope_name, camera_name and mirror_type fields + ''' + + # allow unit input + focal_length = u.Quantity(focal_length, u.m).to_value(u.m) + + try: + return TELESCOPE_NAMES[(n_pixels, round(focal_length, 2))] + except KeyError: + raise ValueError(f'Unknown telescope: n_pixel={n_pixels}, f={focal_length}') diff --git a/ctapipe/instrument/optics.py b/ctapipe/instrument/optics.py index a7555313edf..f9f69bbf05a 100644 --- a/ctapipe/instrument/optics.py +++ b/ctapipe/instrument/optics.py @@ -9,20 +9,6 @@ logger = logging.getLogger(__name__) -_FOCLEN_TO_TEL_INFO = { - # foclen: tel_type, tel_subtype, mirror_type - 28.0: ('LST', '', 'DC'), - 16.0: ('MST', '', 'DC'), - 2.28: ('SST', 'GCT', 'SC'), - 2.15: ('SST', 'ASTRI', 'SC'), - 5.6: ('SST', '1M', 'DC'), - 5.58: ('MST', 'SCT', 'SC'), - 5.59: ('MST', 'SCT', 'SC'), - 15.0: ('MST', 'HESS', 'DC'), - 14.98: ('MST', 'HESS', 'DC'), - 36.0: ('LST', 'HESS', 'DC') -} - class OpticsDescription: """ @@ -77,32 +63,6 @@ def __init__(self, mirror_type, tel_type, self.num_mirror_tiles = num_mirror_tiles - @classmethod - @u.quantity_input - def guess(cls, equivalent_focal_length: u.m): - """ - Construct an OpticsDescription by guessing from metadata (e.g. when - using a simulation where the exact type is not known) - - Parameters - ---------- - equivalent_focal_length: Quantity('m') - effective optical focal-length in meters - - Raises - ------ - KeyError: - on unknown focal length - """ - - tel_type, tel_subtype, mir_type = \ - telescope_info_from_metadata(equivalent_focal_length) - - return cls(mirror_type=mir_type, - tel_type=tel_type, - tel_subtype=tel_subtype, - equivalent_focal_length=equivalent_focal_length) - @classmethod def from_name(cls, name, optics_table='optics'): """ @@ -123,8 +83,7 @@ def from_name(cls, name, optics_table='optics'): OpticsDescription """ - table = get_table_dataset(optics_table, - role='dl0.tel.svc.optics') + table = get_table_dataset(optics_table, role='dl0.tel.svc.optics') mask = table['tel_description'] == name if 'equivalent_focal_length' in table.colnames: @@ -134,11 +93,14 @@ def from_name(cls, name, optics_table='optics'): logger.warning("Optics table format out of date: " "'effective_focal_length' " "should be 'equivalent_focal_length'") + subtype = str(table['tel_subtype'][mask][0]) + if subtype == '--': + subtype = '' optics = cls( mirror_type=table['mirror_type'][mask][0], - tel_type=table['tel_type'][mask][0], - tel_subtype=table['tel_subtype'][mask][0], + tel_type=str(table['tel_type'][mask][0]), + tel_subtype=subtype, equivalent_focal_length=flen, mirror_area=table['mirror_area'][mask].quantity[0], num_mirror_tiles=table['num_mirror_tiles'][mask][0], @@ -171,27 +133,3 @@ def __str__(self): return f"{self.tel_type}-{self.tel_subtype}" else: return self.tel_type - - -def telescope_info_from_metadata(focal_length): - """ - helper func to return telescope and mirror info based on metadata - - Parameters - ---------- - focal_length: float - effective focal length - - Returns - ------- - str,str,str: - tel_type ('LST', 'MST' or 'SST'), - tel_subtype (model), - mirror_type ('SC' or 'DC') - - Raises: - ------- - KeyError: - if unable to find optics type - """ - return _FOCLEN_TO_TEL_INFO[round(focal_length.to('m').value, 2)] diff --git a/ctapipe/instrument/telescope.py b/ctapipe/instrument/telescope.py index 41041e1098a..5e763498d78 100644 --- a/ctapipe/instrument/telescope.py +++ b/ctapipe/instrument/telescope.py @@ -13,7 +13,6 @@ telescope :-)) """ - from .optics import OpticsDescription from .camera import CameraGeometry @@ -37,7 +36,6 @@ class TelescopeDescription: the camera associated with this telescope """ - def __init__(self, optics: OpticsDescription, camera: CameraGeometry): @@ -55,25 +53,6 @@ def camera(self): """ CameraGeometry for this telescope""" return self._camera - @classmethod - def guess(cls, pix_x, pix_y, equivalent_focal_length): - """ - Construct a TelescopeDescription from metadata, filling in the - missing information using a lookup table. - - Parameters - ---------- - pix_x: array - array of pixel x-positions with units - pix_y: array - array of pixel y-positions with units - equivalent_focal_length: float - effective focal length of telescope with units (m) - """ - camera = CameraGeometry.guess(pix_x, pix_y, equivalent_focal_length) - optics = OpticsDescription.guess(equivalent_focal_length) - return cls(optics=optics, camera=camera) - @classmethod def from_name(cls, optics_name, camera_name): """ @@ -101,6 +80,8 @@ def __str__(self): return str(self.optics) + ":" + str(self.camera) def __repr__(self): - return "{}(optics={}, camera={})".format(self.__class__.__name__, - str(self.optics), - str(self.camera)) + return "{}(optics={}, camera={})".format( + self.__class__.__name__, + str(self.optics), + str(self.camera), + ) diff --git a/ctapipe/instrument/tests/test_camera.py b/ctapipe/instrument/tests/test_camera.py index 806b1eea2aa..8b18030d7da 100644 --- a/ctapipe/instrument/tests/test_camera.py +++ b/ctapipe/instrument/tests/test_camera.py @@ -47,13 +47,6 @@ def test_load_hess_camera(): assert len(geom.pix_x) == 1855 -def test_guess_camera(): - px = np.linspace(-10, 10, 11328) * u.m - py = np.linspace(-10, 10, 11328) * u.m - geom = CameraGeometry.guess(px, py, 0 * u.m) - assert geom.pix_type.startswith('rect') - - def test_position_to_pix_index(): geom = CameraGeometry.from_name("LSTCam") x, y = 0.80 * u.m, 0.79 * u.m, diff --git a/ctapipe/instrument/tests/test_guessing.py b/ctapipe/instrument/tests/test_guessing.py new file mode 100644 index 00000000000..011f3787505 --- /dev/null +++ b/ctapipe/instrument/tests/test_guessing.py @@ -0,0 +1,25 @@ +from ctapipe.instrument import TelescopeDescription +from astropy import units as u +from pytest import raises + + +def test_guessing(): + from ctapipe.instrument import guess_telescope + + guess = guess_telescope(2048, 2.28) + assert guess.type == 'SST' + assert guess.telescope_name == 'GCT' + + guess = guess_telescope(2048, 2.28 * u.m) + assert guess.type == 'SST' + assert guess.telescope_name == 'GCT' + + with raises(ValueError): + guess = guess_telescope(100, 2.28 * u.m) + + foclen = 16 * u.m + n_pixels = 1764 + guess = guess_telescope(n_pixels, foclen) + + assert guess.camera_name == 'FlashCam' + assert guess.type == 'MST' diff --git a/ctapipe/instrument/tests/test_optics.py b/ctapipe/instrument/tests/test_optics.py index c1792127186..889af4376d7 100644 --- a/ctapipe/instrument/tests/test_optics.py +++ b/ctapipe/instrument/tests/test_optics.py @@ -4,32 +4,29 @@ def test_guess_optics(): - od = OpticsDescription.guess(28.0 * u.m) + from ctapipe.instrument import guess_telescope + answer = guess_telescope(1855, 28.0 * u.m) + od = OpticsDescription.from_name(answer.telescope_name) od.info() assert od.tel_type == 'LST' assert od.tel_subtype == '' assert od.mirror_type == 'DC' - with pytest.raises(KeyError): - OpticsDescription.guess(0 * u.m) # unknown tel - def test_construct_optics(): with pytest.raises(ValueError): - OpticsDescription(mirror_type="DC", - tel_type="bad", # bad value - tel_subtype="1M", - equivalent_focal_length=10 * u.m) + OpticsDescription( + mirror_type="DC", + tel_type="bad", # bad value + tel_subtype="1M", + equivalent_focal_length=10 * u.m, + ) with pytest.raises(ValueError): - OpticsDescription(mirror_type="bad", # bad value - tel_type="MST", - tel_subtype="1M", - equivalent_focal_length=10 * u.m) - - with pytest.raises(u.UnitsError): - OpticsDescription.guess(28.0 * u.kg) # bad unit - - with pytest.raises(TypeError): - OpticsDescription.guess(28.0) # not a unit quantity + OpticsDescription( + mirror_type="bad", # bad value + tel_type="MST", + tel_subtype="1M", + equivalent_focal_length=10 * u.m, + ) diff --git a/ctapipe/instrument/tests/test_telescope.py b/ctapipe/instrument/tests/test_telescope.py deleted file mode 100644 index 06e9ac48f10..00000000000 --- a/ctapipe/instrument/tests/test_telescope.py +++ /dev/null @@ -1,17 +0,0 @@ -from ctapipe.instrument import TelescopeDescription -from astropy import units as u -import numpy as np - - -def test_telescope_description(): - - # setup a dummy telescope that look like an MST with FlashCam - foclen = 16 * u.m - pix_x = np.arange(1764, dtype=np.float) * u.m - pix_y = np.arange(1764, dtype=np.float) * u.m - - tel = TelescopeDescription.guess(pix_x, pix_y, foclen) - - assert tel.camera.cam_id == 'FlashCam' - assert tel.optics.tel_type == 'MST' - assert str(tel) == 'MST:FlashCam' From e54685167ad03238538942318d88acc7b9faad30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Thu, 21 Feb 2019 17:49:27 +0100 Subject: [PATCH 02/23] Build telescope description in SimTelEventSource --- ctapipe/conftest.py | 6 +-- ctapipe/instrument/camera.py | 26 +++++------ ctapipe/instrument/guess.py | 4 +- ctapipe/instrument/tests/test_guessing.py | 5 +- ctapipe/io/simteleventsource.py | 56 +++++++++++++++++++---- 5 files changed, 66 insertions(+), 31 deletions(-) diff --git a/ctapipe/conftest.py b/ctapipe/conftest.py index c82015995a2..4e3d8b6dfce 100644 --- a/ctapipe/conftest.py +++ b/ctapipe/conftest.py @@ -6,7 +6,6 @@ from copy import deepcopy -from ctapipe.io.eventseeker import EventSeeker from ctapipe.io import SimTelEventSource from ctapipe.utils import get_dataset_path @@ -17,13 +16,12 @@ def _global_example_event(): helper to get a single event from a MC file. Don't use this fixture directly, rather use `test_event` """ - filename = get_dataset_path('gamma_test.simtel.gz') + filename = get_dataset_path('gamma_test_large.simtel.gz') print("******************** LOAD TEST EVENT ***********************") with SimTelEventSource(input_url=filename) as reader: - seeker = EventSeeker(reader) - event = seeker['409'] + event = next(iter(reader)) return event diff --git a/ctapipe/instrument/camera.py b/ctapipe/instrument/camera.py index 5e6ee1f3cec..875bbbf683e 100644 --- a/ctapipe/instrument/camera.py +++ b/ctapipe/instrument/camera.py @@ -21,19 +21,6 @@ logger = logging.getLogger(__name__) -def simtel_shape_to_type(pixel_shape): - if pixel_shape == 1: - return 'hexagonal', Angle(0, u.deg) - - if pixel_shape == 2: - return 'rectangular', Angle(0, u.deg) - - if pixel_shape == 3: - return 'hexagonal', Angle(30, u.deg) - - raise ValueError(f'Unknown pixel_shape {pixel_shape}') - - class CameraGeometry: """`CameraGeometry` is a class that stores information about a Cherenkov Camera that us useful for imaging algorithms and @@ -552,6 +539,19 @@ def position_to_pix_index(self, x, y): return pix_indices if len(pix_indices) > 1 else pix_indices[0] + @staticmethod + def simtel_shape_to_type(pixel_shape): + if pixel_shape == 1: + return 'hexagonal', Angle(0, u.deg) + + if pixel_shape == 2: + return 'rectangular', Angle(0, u.deg) + + if pixel_shape == 3: + return 'hexagonal', Angle(30, u.deg) + + raise ValueError(f'Unknown pixel_shape {pixel_shape}') + # ====================================================================== # utility functions: diff --git a/ctapipe/instrument/guess.py b/ctapipe/instrument/guess.py index d9b9a0d9447..4b551781478 100644 --- a/ctapipe/instrument/guess.py +++ b/ctapipe/instrument/guess.py @@ -4,7 +4,7 @@ GuessingKey = namedtuple('GuessingKey', ['n_pixels', 'focal_length']) GuessingResult = namedtuple( - 'GuessingResult', ['type', 'telescope_name', 'camera_name', 'mirror_type'] + 'GuessingResult', ['type', 'name', 'camera_name', 'mirror_type'] ) @@ -23,6 +23,8 @@ GuessingKey(1440, 4.998): GuessingResult('SST', 'FACT', 'FACT', 'DC'), } +UNKNOWN_TELESCOPE = GuessingResult('UNKNOWN', 'UNKNOWN', 'UNKNOWN', 'UNKNOWN') + def guess_telescope(n_pixels, focal_length): ''' diff --git a/ctapipe/instrument/tests/test_guessing.py b/ctapipe/instrument/tests/test_guessing.py index 011f3787505..231983cf89c 100644 --- a/ctapipe/instrument/tests/test_guessing.py +++ b/ctapipe/instrument/tests/test_guessing.py @@ -1,4 +1,3 @@ -from ctapipe.instrument import TelescopeDescription from astropy import units as u from pytest import raises @@ -8,11 +7,11 @@ def test_guessing(): guess = guess_telescope(2048, 2.28) assert guess.type == 'SST' - assert guess.telescope_name == 'GCT' + assert guess.name == 'GCT' guess = guess_telescope(2048, 2.28 * u.m) assert guess.type == 'SST' - assert guess.telescope_name == 'GCT' + assert guess.name == 'GCT' with raises(ValueError): guess = guess_telescope(100, 2.28 * u.m) diff --git a/ctapipe/io/simteleventsource.py b/ctapipe/io/simteleventsource.py index af328f39357..b74ad5efc97 100644 --- a/ctapipe/io/simteleventsource.py +++ b/ctapipe/io/simteleventsource.py @@ -5,7 +5,13 @@ from astropy import units as u from astropy.coordinates import Angle from astropy.time import Time -from ctapipe.instrument import TelescopeDescription, SubarrayDescription +from ctapipe.instrument import ( + TelescopeDescription, + SubarrayDescription, + CameraGeometry, + OpticsDescription, +) +from ctapipe.instrument.guess import guess_telescope, UNKNOWN_TELESCOPE import struct import gzip @@ -50,18 +56,48 @@ def prepare_subarray_info(telescope_descriptions, header): for tel_id, telescope_description in telescope_descriptions.items(): cam_settings = telescope_description['camera_settings'] - tel_description = TelescopeDescription.guess( - cam_settings['pixel_x'] * u.m, - cam_settings['pixel_y'] * u.m, - equivalent_focal_length=cam_settings['focal_length'] * u.m + + n_pixels = cam_settings['n_pixels'] + focal_length = u.Quantity(cam_settings['focal_length'], u.m) + + try: + telescope = guess_telescope(n_pixels, focal_length) + except ValueError: + telescope = UNKNOWN_TELESCOPE + + pixel_shape = cam_settings['pixel_shape'][0] + pix_type, pix_rotation = CameraGeometry.simtel_shape_to_type(pixel_shape) + camera = CameraGeometry( + telescope.camera_name, + pix_id=np.arange(n_pixels), + pix_x=u.Quantity(cam_settings['pixel_x'], u.m), + pix_y=u.Quantity(cam_settings['pixel_y'], u.m), + pix_area=u.Quantity(cam_settings['pixel_area'], u.m**2), + pix_type=pix_type, + pix_rotation=pix_rotation, + cam_rotation=-Angle(cam_settings['cam_rot'], u.rad), + apply_derotation=True, + ) - tel_description.optics.mirror_area = ( - cam_settings['mirror_area'] * u.m ** 2 + + if telescope.name != telescope.type: + subtype = telescope.name + else: + subtype = '' + + optics = OpticsDescription( + mirror_type=telescope.mirror_type, + tel_type=telescope.type, + tel_subtype=subtype, + equivalent_focal_length=focal_length, + mirror_area=u.Quantity(cam_settings['mirror_area'], u.m**2), + num_mirror_tiles=cam_settings['n_mirrors'], ) - tel_description.optics.num_mirror_tiles = ( - cam_settings['n_mirrors'] + + tel_descriptions[tel_id] = TelescopeDescription( + camera=camera, + optics=optics, ) - tel_descriptions[tel_id] = tel_description tel_idx = np.where(header['tel_id'] == tel_id)[0][0] tel_positions[tel_id] = header['tel_pos'][tel_idx] * u.m From 744fb96e4b4c1ed6741016e97539f6ea6cb9cbc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Fri, 22 Feb 2019 09:59:48 +0100 Subject: [PATCH 03/23] Adapt hessio eventsource --- ctapipe/io/hessioeventsource.py | 69 ++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/ctapipe/io/hessioeventsource.py b/ctapipe/io/hessioeventsource.py index 554ba58a30d..df56c2f36a8 100644 --- a/ctapipe/io/hessioeventsource.py +++ b/ctapipe/io/hessioeventsource.py @@ -3,7 +3,14 @@ from astropy.time import Time from ctapipe.io.eventsource import EventSource from ctapipe.io.containers import DataContainer -from ctapipe.instrument import TelescopeDescription, SubarrayDescription +from ctapipe.instrument import ( + TelescopeDescription, + SubarrayDescription, + OpticsDescription, + CameraGeometry, +) +from ctapipe.instrument.guess import guess_telescope, UNKNOWN_TELESCOPE +import numpy as np __all__ = ['HESSIOEventSource'] @@ -182,21 +189,57 @@ def _build_subarray_info(self, file): for tel_id in telescope_ids: try: - - pix_pos = file.get_pixel_position(tel_id) * u.m - foclen = file.get_optical_foclen(tel_id) * u.m - mirror_area = file.get_mirror_area(tel_id) * u.m ** 2 - num_tiles = file.get_mirror_number(tel_id) - tel_pos = file.get_telescope_position(tel_id) * u.m - - tel = TelescopeDescription.guess(*pix_pos, - equivalent_focal_length=foclen) - tel.optics.mirror_area = mirror_area - tel.optics.num_mirror_tiles = num_tiles + tel = self._build_telescope_description(file, tel_id) + tel_pos = u.Quantity(file.get_telescope_position(tel_id), u.m) subarray.tels[tel_id] = tel subarray.positions[tel_id] = tel_pos - except self.pyhessio.HessioGeneralError: pass return subarray + + def _build_telescope_description(self, file, tel_id): + pix_x, pix_y = u.Quantity(file.get_pixel_position(tel_id), u.m) + focal_length = u.Quantity(file.get_optical_foclen(tel_id), u.m) + n_pixels = len(pix_x) + + try: + telescope = guess_telescope(n_pixels, focal_length) + except ValueError: + telescope = UNKNOWN_TELESCOPE + + pixel_shape = file.get_pixel_shape(tel_id)[0] + pix_type, pix_rot = CameraGeometry.simtel_shape_to_type(pixel_shape) + pix_area = u.Quantity(file.get_pixel_area(tel_id), u.m**2) + + mirror_area = u.Quantity(file.get_mirror_area(tel_id), u.m**2) + num_tiles = file.get_mirror_number(tel_id) + cam_rot = file.get_camera_rotation_angle(tel_id) + + camera = CameraGeometry( + telescope.camera_name, + pix_id=np.arange(n_pixels), + pix_x=pix_x, + pix_y=pix_y, + pix_area=pix_area, + pix_type=pix_type, + pix_rotation=pix_rot, + cam_rotation=-Angle(cam_rot, u.rad), + apply_derotation=True, + ) + + if telescope.name != telescope.type: + subtype = telescope.name + else: + subtype = '' + + optics = OpticsDescription( + mirror_type=telescope.mirror_type, + tel_type=telescope.type, + tel_subtype=subtype, + equivalent_focal_length=focal_length, + mirror_area=mirror_area, + num_mirror_tiles=num_tiles, + ) + + return TelescopeDescription(camera=camera, optics=optics) From 4fb9af883ba1dbeee6bc93252f4f3c8896f3a975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Fri, 22 Feb 2019 10:00:13 +0100 Subject: [PATCH 04/23] Adapt tests to work with gamma_test_large --- ctapipe/io/tests/test_simteleventsource.py | 56 ++++++++++------------ 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/ctapipe/io/tests/test_simteleventsource.py b/ctapipe/io/tests/test_simteleventsource.py index c8618761111..d836360648f 100644 --- a/ctapipe/io/tests/test_simteleventsource.py +++ b/ctapipe/io/tests/test_simteleventsource.py @@ -7,7 +7,6 @@ from itertools import zip_longest gamma_test_large_path = get_dataset_path("gamma_test_large.simtel.gz") -gamma_test_path = get_dataset_path("gamma_test.simtel.gz") def compare_sources(input_url): @@ -66,23 +65,18 @@ def compare_sources(input_url): def test_compare_event_hessio_and_simtel(): - for input_url in (gamma_test_path, gamma_test_large_path): - compare_sources(input_url) + compare_sources(gamma_test_large_path) def test_simtel_event_source_on_gamma_test_one_event(): - with SimTelEventSource(input_url=gamma_test_path) as reader: - assert reader.is_compatible(gamma_test_path) + with SimTelEventSource(input_url=gamma_test_large_path) as reader: + assert reader.is_compatible(gamma_test_large_path) assert not reader.is_stream for event in reader: - if event.count == 0: - assert event.r0.tels_with_data == {38, 47} - elif event.count == 1: - assert event.r0.tels_with_data == {11, 21, 24, 26, 61, 63, 118, - 119} - else: + if event.count > 1: break + for event in reader: # Check generator has restarted from beginning assert event.count == 0 @@ -90,25 +84,24 @@ def test_simtel_event_source_on_gamma_test_one_event(): # test that max_events works: max_events = 5 - with SimTelEventSource(input_url=gamma_test_path, max_events=max_events) as reader: + with SimTelEventSource(input_url=gamma_test_large_path, max_events=max_events) as reader: count = 0 for _ in reader: count += 1 assert count == max_events # test that the allowed_tels mask works: - with pytest.warns(UserWarning): - with SimTelEventSource( - input_url=gamma_test_path, - allowed_tels={3, 4} - ) as reader: - for event in reader: - assert event.r0.tels_with_data.issubset(reader.allowed_tels) + with SimTelEventSource( + input_url=gamma_test_large_path, + allowed_tels={3, 4} + ) as reader: + for event in reader: + assert event.r0.tels_with_data.issubset(reader.allowed_tels) def test_that_event_is_not_modified_after_loop(): - dataset = gamma_test_path + dataset = gamma_test_large_path with SimTelEventSource(input_url=dataset, max_events=2) as source: for event in source: last_event = copy.deepcopy(event) @@ -122,7 +115,7 @@ def test_that_event_is_not_modified_after_loop(): def test_additional_meta_data_from_mc_header(): - with SimTelEventSource(input_url=gamma_test_path) as reader: + with SimTelEventSource(input_url=gamma_test_large_path) as reader: data = next(iter(reader)) # for expectation values @@ -130,26 +123,25 @@ def test_additional_meta_data_from_mc_header(): from astropy.coordinates import Angle assert data.mcheader.corsika_version == 6990 - assert data.mcheader.simtel_version == 1404919891 assert data.mcheader.spectral_index == -2.0 - assert data.mcheader.shower_prog_start == 1408536000 - assert data.mcheader.shower_reuse == 10 + assert data.mcheader.shower_reuse == 20 assert data.mcheader.core_pos_mode == 1 - assert data.mcheader.diffuse == 0 - assert data.mcheader.atmosphere == 24 + assert data.mcheader.diffuse == 1 + assert data.mcheader.atmosphere == 26 + # value read by hand from input card name_expectation = { 'energy_range_min': u.Quantity(3.0e-03, u.TeV), 'energy_range_max': u.Quantity(3.3e+02, u.TeV), - 'prod_site_B_total': u.Quantity(27.181243896484375, u.uT), + 'prod_site_B_total': u.Quantity(23.11772346496582, u.uT), 'prod_site_B_declination': Angle(0.0 * u.rad), - 'prod_site_B_inclination': Angle(-1.1581752300262451 * u.rad), - 'prod_site_alt': 1640.0 * u.m, - 'max_scatter_range': 2500.0 * u.m, + 'prod_site_B_inclination': Angle(-0.39641156792640686 * u.rad), + 'prod_site_alt': 2150.0 * u.m, + 'max_scatter_range': 3000.0 * u.m, 'min_az': 0.0 * u.rad, 'min_alt': 1.2217305 * u.rad, - 'max_viewcone_radius': 0.0 * u.deg, - 'corsika_wlen_min': 250 * u.nm, + 'max_viewcone_radius': 10.0 * u.deg, + 'corsika_wlen_min': 240 * u.nm, } From 664e84c9aab1f11a6208804403bdf3720b22808b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Fri, 22 Feb 2019 16:42:18 +0100 Subject: [PATCH 05/23] Start refactoring instrument classes --- ctapipe/calib/camera/tests/test_calibrator.py | 4 +- ctapipe/instrument/camera.py | 23 ++- ctapipe/instrument/optics.py | 94 ++++++------ ctapipe/instrument/subarray.py | 136 ++++++++---------- ctapipe/instrument/telescope.py | 45 ++++-- ctapipe/instrument/tests/test_optics.py | 34 ++--- ctapipe/instrument/tests/test_subarray.py | 27 ++-- ctapipe/instrument/tests/test_telescope.py | 5 + ctapipe/io/hessioeventsource.py | 16 +-- ctapipe/io/simteleventsource.py | 12 +- .../reco/tests/test_HillasReconstructor.py | 4 +- 11 files changed, 201 insertions(+), 199 deletions(-) create mode 100644 ctapipe/instrument/tests/test_telescope.py diff --git a/ctapipe/calib/camera/tests/test_calibrator.py b/ctapipe/calib/camera/tests/test_calibrator.py index 7379a72defd..9b96e6bec92 100644 --- a/ctapipe/calib/camera/tests/test_calibrator.py +++ b/ctapipe/calib/camera/tests/test_calibrator.py @@ -31,14 +31,14 @@ def test_manual_extractor(): def test_eventsource_r1(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") eventsource = SimTelEventSource(input_url=dataset) calibrator = CameraCalibrator(eventsource=eventsource) assert isinstance(calibrator.r1, HESSIOR1Calibrator) def test_eventsource_override_r1(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") eventsource = SimTelEventSource(input_url=dataset) calibrator = CameraCalibrator( eventsource=eventsource, diff --git a/ctapipe/instrument/camera.py b/ctapipe/instrument/camera.py index 875bbbf683e..543f0844c0a 100644 --- a/ctapipe/instrument/camera.py +++ b/ctapipe/instrument/camera.py @@ -95,13 +95,22 @@ def __init__(self, cam_id, pix_id, pix_x, pix_y, pix_area, pix_type, self.border_cache = {} def __eq__(self, other): - return ((self.cam_id == other.cam_id) - and (self.pix_x == other.pix_x).all() - and (self.pix_y == other.pix_y).all() - and (self.pix_type == other.pix_type) - and (self.pix_rotation == other.pix_rotation) - and (self.pix_type == other.pix_type) - ) + return all( + self.cam_id == other.cam_id, + (self.pix_x == other.pix_x).all(), + (self.pix_y == other.pix_y).all(), + self.pix_type == other.pix_type, + self.pix_rotation == other.pix_rotation, + ) + + def __hash__(self): + return hash(( + self.cam_id, + self.pix_x[0].to_value(u.m), + self.pix_y[0].to_value(u.m), + self.pix_type, + self.pix_rotation.deg, + )) def __len__(self): return self.n_pixels diff --git a/ctapipe/instrument/optics.py b/ctapipe/instrument/optics.py index f9f69bbf05a..b83f5405d49 100644 --- a/ctapipe/instrument/optics.py +++ b/ctapipe/instrument/optics.py @@ -23,12 +23,8 @@ class OpticsDescription: Parameters ---------- - mirror_type: str - 'SC' or 'DC' - tel_type: str - 'SST', 'MST','LST' - tel_subtype: str - subtype of telescope, e.g. '1M' or 'ASTRI' + num_mirrors: int + Number of mirrors, i. e. 2 for Schwarzschild-Couder else 1 equivalent_focal_length: Quantity(float) effective focal-length of telescope, independent of which type of optics (as in the Monte-Carlo) @@ -45,24 +41,35 @@ class OpticsDescription: if the units of one of the inputs are missing or incompatible """ - def __init__(self, mirror_type, tel_type, - tel_subtype, equivalent_focal_length, - mirror_area=None, num_mirror_tiles=None): - - if tel_type not in ['LST', 'MST', 'SST']: - raise ValueError("Unknown tel_type %s", tel_type) - - if mirror_type not in ['SC', 'DC']: - raise ValueError("Unknown mirror_type: %s", mirror_type) - - self.mirror_type = mirror_type - self.tel_type = tel_type - self.tel_subtype = tel_subtype + @u.quantity_input(mirror_area=u.m**2, equivalent_focal_length=u.m) + def __init__( + self, + name, + num_mirrors, + equivalent_focal_length, + mirror_area=None, + num_mirror_tiles=None + ): + + self.name = name self.equivalent_focal_length = equivalent_focal_length.to(u.m) self.mirror_area = mirror_area - + self.num_mirrors = num_mirrors self.num_mirror_tiles = num_mirror_tiles + def __hash__(self): + '''Make this hashable, so it can be used as dict keys or in sets''' + return hash(( + self.equivalent_focal_length.to_value(u.m), + self.mirror_area, + self.num_mirrors, + self.num_mirror_tiles, + )) + + def __eq__(self, other): + '''Make this hashable, so it can be used as dict keys or in sets''' + return hash(self) == hash(other) + @classmethod def from_name(cls, name, optics_table='optics'): """ @@ -86,21 +93,12 @@ def from_name(cls, name, optics_table='optics'): table = get_table_dataset(optics_table, role='dl0.tel.svc.optics') mask = table['tel_description'] == name - if 'equivalent_focal_length' in table.colnames: - flen = table['equivalent_focal_length'][mask].quantity[0] - else: - flen = table['effective_focal_length'][mask].quantity[0] - logger.warning("Optics table format out of date: " - "'effective_focal_length' " - "should be 'equivalent_focal_length'") - subtype = str(table['tel_subtype'][mask][0]) - if subtype == '--': - subtype = '' + flen = table['equivalent_focal_length'][mask].quantity[0] + num_mirrors = 1 if table['mirror_type'][mask][0] == 'DC' else 2 optics = cls( - mirror_type=table['mirror_type'][mask][0], - tel_type=str(table['tel_type'][mask][0]), - tel_subtype=subtype, + name=name, + num_mirrors=num_mirrors, equivalent_focal_length=flen, mirror_area=table['mirror_area'][mask].quantity[0], num_mirror_tiles=table['num_mirror_tiles'][mask][0], @@ -112,24 +110,18 @@ def get_known_optics_names(cls, optics_table='optics'): table = get_table_dataset(optics_table, 'get_known_optics') return np.array(table['tel_description']) - @property - def identifier(self): - """ returns a tuple of (tel_type, tel_subtype). Use str(optics) to - get a text-based identifier.""" - return self.tel_type, self.tel_subtype - - def info(self, printer=print): - printer(f'OpticsDescription: "{self}"') - printer(f' - mirror_type: {self.mirror_type}') - printer(f' - num_mirror_tiles: {self.num_mirror_tiles}') - printer(f' - mirror_area: {self.mirror_area}') - def __repr__(self): - return "{}(tel_type='{}', tel_subtype='{}')".format( - str(self.__class__.__name__), self.tel_type, self.tel_subtype) + return ( + f"{self.__class__.__name__}" + f"(name={self.name}" + f", equivalent_focal_length={self.equivalent_focal_length:.2f}" + f", num_mirros={self.num_mirrors}" + f", mirror_area={self.mirror_area:.2f}" + ")" + ) def __str__(self): - if self.tel_subtype != '': - return f"{self.tel_type}-{self.tel_subtype}" - else: - return self.tel_type + return self.name + + def __hash__(self): + return hash((self.equivalent_focal_length, self.num_mirrors, self.mirror_area)) diff --git a/ctapipe/instrument/subarray.py b/ctapipe/instrument/subarray.py index 79084ce6ca9..5a4be41e24b 100644 --- a/ctapipe/instrument/subarray.py +++ b/ctapipe/instrument/subarray.py @@ -43,16 +43,15 @@ class SubarrayDescription: array of tel_ids tel_indices: dict dict mapping tel_id to index in array attributes - """ def __init__(self, name, tel_positions=None, tel_descriptions=None): - - self.name = name #: name of telescope + self.name = name self.positions = tel_positions or dict() self.tels = tel_descriptions or dict() - assert set(self.positions.keys()) == set(self.tels.keys()) + if self.positions.keys() != self.tels.keys(): + raise ValueError('Telescope ids in positions and descriptions do not match') def __str__(self): return self.name @@ -61,7 +60,8 @@ def __repr__(self): return "{}(name='{}', num_tels={})".format( self.__class__.__name__, self.name, - self.num_tels) + self.num_tels, + ) @property def tel(self): @@ -72,6 +72,9 @@ def tel(self): def num_tels(self): return len(self.tels) + def __len__(self): + return len(self.tels) + def info(self, printer=print): """ print descriptive info about subarray @@ -88,11 +91,11 @@ def info(self, printer=print): printer("") printer(" TYPE Num IDmin IDmax") printer("=====================================") + for teltype, tels in teltypes.items(): - printer("{:>20s} {:4d} {:4d} ..{:4d}".format(teltype, - len(tels), - min(tels), - max(tels))) + printer("{:>20s} {:4d} {:4d} ..{:4d}".format( + teltype, len(tels), min(tels), max(tels) + )) @property def tel_coords(self): @@ -112,27 +115,6 @@ def tel_coords(self): frame=GroundFrame() ) - @property - def pos_x(self): - """ telescope x position as array """ - warnings.warn("SubarrayDescription.pos_x is deprecated. Use " - "tel_coords.x") - return self.tel_coords.x - - @property - def pos_y(self): - """ telescope y positions as an array""" - warnings.warn("SubarrayDescription.pos_y is deprecated. Use " - "tel_coords.y") - return self.tel_coords.y - - @property - def pos_z(self): - """ telescope y positions as an array""" - warnings.warn("SubarrayDescription.pos_z is deprecated. Use " - "tel_coords.z") - return self.tel_coords.z - @property def tel_ids(self): """ telescope IDs as an array""" @@ -170,44 +152,40 @@ def to_table(self, kind="subarray"): if kind == 'subarray': - ids = [x for x in self.tels] - descs = [str(x) for x in self.tels.values()] - mirror_types = [x.optics.mirror_type for x in self.tels.values()] - tel_types = [x.optics.tel_type for x in self.tels.values()] - tel_subtypes = [x.optics.tel_subtype for x in self.tels.values()] - cam_types = [x.camera.cam_id for x in self.tels.values()] + ids = list(self.tels.keys()) + descs = [str(t) for t in self.tels.values()] + num_mirrors = [t.optics.num_mirrors for t in self.tels.values()] + tel_names = [t.name for t in self.tels.values()] + tel_types = [t.type for t in self.tels.values()] + cam_types = [t.camera.cam_id for t in self.tels.values()] tel_coords = self.tel_coords - tab = Table(dict(tel_id=np.array(ids, dtype=np.short), - tel_pos_x=tel_coords.x, - tel_pos_y=tel_coords.y, - tel_pos_z=tel_coords.z, - tel_type=tel_types, - tel_subtype=tel_subtypes, - mirror_type=mirror_types, - camera_type=cam_types, - tel_description=descs)) + tab = Table(dict( + id=np.array(ids, dtype=np.short), + pos_x=tel_coords.x, + pos_y=tel_coords.y, + pos_z=tel_coords.z, + name=tel_names, + type=tel_types, + num_mirrors=num_mirrors, + camera_type=cam_types, + tel_description=descs, + )) elif kind == 'optics': - - optics_ids = {x.optics.identifier for x in self.tels.values()} - optics_list = [] - - # get one example of each OpticsDescription - for oid in optics_ids: - optics_list.append(next(x.optics for x in self.tels.values() if - x.optics.identifier == oid)) + unique_types = set(self.tels.values()) cols = { - 'tel_description': [str(x) for x in optics_list], - 'tel_type': [x.tel_type for x in optics_list], - 'tel_subtype': [x.tel_subtype for x in optics_list], - 'mirror_area': np.array([x.mirror_area.to('m2').value for x - in optics_list]) * u.m ** 2, - 'mirror_type': [x.mirror_type for x in optics_list], - 'num_mirror_tiles': [x.num_mirror_tiles for x in optics_list], - 'equivalent_focal_length': [x.equivalent_focal_length.to('m') - for x in optics_list] * u.m, + 'description': [str(t) for t in unique_types], + 'name': [t.name for t in unique_types], + 'type': [t.type for t in unique_types], + 'mirror_area': [t.mirror_area.to(u.m**2) for t in unique_types], + 'num_mirrors': [t.mirror_area for t in unique_types], + 'num_mirror_tiles': [t.num_mirror_tiles for t in unique_types], + 'equivalent_focal_length': [ + t.equivalent_focal_length.to(u.m) + for t in unique_types + ], } tab = Table(cols) @@ -248,21 +226,28 @@ def peek(self): from matplotlib import pyplot as plt from astropy.visualization import quantity_support - types = {str(tel) for tel in self.tels.values()} + types = set(self.tels.values()) tab = self.to_table() plt.figure(figsize=(8, 8)) with quantity_support(): - for teltype in types: - tels = tab[tab['tel_description'] == teltype]['tel_id'] - sub = self.select_subarray(teltype, tels) + for tel_type in types: + tels = tab[tab['tel_description'] == str(tel_type)]['tel_id'] + sub = self.select_subarray(tel_type, tels) tel_coords = sub.tel_coords - radius = np.array([np.sqrt(tel.optics.mirror_area / np.pi).value - for tel in sub.tels.values()]) - - plt.scatter(tel_coords.x, tel_coords.y, s=radius * 8, alpha=0.5, - label=teltype) + radius = np.array([ + np.sqrt(tel.optics.mirror_area / np.pi).value + for tel in sub.tels.values() + ]) + + plt.scatter( + tel_coords.x, + tel_coords.y, + s=radius * 8, + alpha=0.5, + label=tel_type, + ) plt.legend(loc='best') plt.title(self.name) @@ -271,20 +256,17 @@ def peek(self): @property def telescope_types(self): """ list of telescope types in the array""" - tel_types = {str(tt) for tt in self.tel.values()} - return list(tel_types) + return [t.type + ':' + t.camera.cam_id for t in set(self.tel.values())] @property def camera_types(self): """ list of camera types in the array """ - cam_types = {str(tt.camera) for tt in self.tel.values()} - return list(cam_types) + return [t.camera.cam_id for t in set(self.tel.values())] @property def optics_types(self): """ list of optics types in the array """ - cam_types = {str(tt.optics) for tt in self.tel.values()} - return list(cam_types) + return [t.optics for t in set(self.tel.values())] def get_tel_ids_for_type(self, tel_type): """ diff --git a/ctapipe/instrument/telescope.py b/ctapipe/instrument/telescope.py index 5e763498d78..27945528f2d 100644 --- a/ctapipe/instrument/telescope.py +++ b/ctapipe/instrument/telescope.py @@ -15,6 +15,7 @@ """ from .optics import OpticsDescription from .camera import CameraGeometry +from .guess import guess_telescope class TelescopeDescription: @@ -30,28 +31,36 @@ class TelescopeDescription: Parameters ---------- + name: str + Telescope name + type: str + Telescope type optics: OpticsDescription the optics associated with this telescope camera: CameraGeometry the camera associated with this telescope """ - def __init__(self, - optics: OpticsDescription, - camera: CameraGeometry): + def __init__( + self, + name, + type, + optics: OpticsDescription, + camera: CameraGeometry + ): - self._optics = optics - self._camera = camera + self.name = name + self.type = type + self.optics = optics + self.camera = camera - @property - def optics(self): - """ OpticsDescription for this telescope """ - return self._optics + def __hash__(self): + '''Make this hashable, so it can be used as dict keys or in sets''' + return hash((self.optics, self.camera)) - @property - def camera(self): - """ CameraGeometry for this telescope""" - return self._camera + def __eq__(self, other): + '''Make this hashable, so it can be used as dict keys or in sets''' + return hash(self) == hash(other) @classmethod def from_name(cls, optics_name, camera_name): @@ -72,15 +81,21 @@ def from_name(cls, optics_name, camera_name): TelescopeDescription """ + camera = CameraGeometry.from_name(camera_name) optics = OpticsDescription.from_name(optics_name) - return cls(optics=optics, camera=camera) + + t = guess_telescope(camera.n_pixels, optics.equivalent_focal_length) + + return cls(name=t.name, type=t.type, optics=optics, camera=camera) def __str__(self): return str(self.optics) + ":" + str(self.camera) def __repr__(self): - return "{}(optics={}, camera={})".format( + return "{}({}, type={}, optics={}, camera={})".format( + self.name, + self.type, self.__class__.__name__, str(self.optics), str(self.camera), diff --git a/ctapipe/instrument/tests/test_optics.py b/ctapipe/instrument/tests/test_optics.py index 889af4376d7..d752dc2b457 100644 --- a/ctapipe/instrument/tests/test_optics.py +++ b/ctapipe/instrument/tests/test_optics.py @@ -6,27 +6,27 @@ def test_guess_optics(): from ctapipe.instrument import guess_telescope answer = guess_telescope(1855, 28.0 * u.m) - od = OpticsDescription.from_name(answer.telescope_name) - od.info() - assert od.tel_type == 'LST' - assert od.tel_subtype == '' - assert od.mirror_type == 'DC' + od = OpticsDescription.from_name(answer.name) + + assert od.equivalent_focal_length.to_value(u.m) == 28 + assert od.num_mirrors == 1 def test_construct_optics(): - with pytest.raises(ValueError): - OpticsDescription( - mirror_type="DC", - tel_type="bad", # bad value - tel_subtype="1M", - equivalent_focal_length=10 * u.m, - ) + OpticsDescription( + name='test', + num_mirrors=1, + num_mirror_tiles=100, + mirror_area=u.Quantity(550, u.m**2), + equivalent_focal_length=u.Quantity(10, u.m), + ) - with pytest.raises(ValueError): + with pytest.raises(TypeError): OpticsDescription( - mirror_type="bad", # bad value - tel_type="MST", - tel_subtype="1M", - equivalent_focal_length=10 * u.m, + name='test', + num_mirrors=1, + num_mirror_tiles=100, + mirror_area=550, + equivalent_focal_length=10, ) diff --git a/ctapipe/instrument/tests/test_subarray.py b/ctapipe/instrument/tests/test_subarray.py index 6a4bb81d5c6..ebc46e44153 100644 --- a/ctapipe/instrument/tests/test_subarray.py +++ b/ctapipe/instrument/tests/test_subarray.py @@ -13,16 +13,20 @@ def test_subarray_description(): tel = {} n_tels = 10 - for ii in range(n_tels): - tel[ii + 1] = TelescopeDescription.from_name(optics_name="MST", - camera_name="NectarCam") - pos[ii + 1] = np.random.uniform(-100, 100, size=3) * u.m + for tel_id in range(1, n_tels + 1): + tel[tel_id] = TelescopeDescription.from_name( + optics_name="MST", + camera_name="NectarCam", + ) + pos[tel_id] = np.random.uniform(-100, 100, size=3) * u.m - sub = SubarrayDescription("test array", - tel_positions=pos, - tel_descriptions=tel) + sub = SubarrayDescription( + "test array", + tel_positions=pos, + tel_descriptions=tel + ) - sub.info() + assert len(sub.telescope_types) == 1 assert str(sub) == "test array" assert sub.num_tels == n_tels @@ -31,9 +35,9 @@ def test_subarray_description(): assert sub.tel[1].camera is not None assert 0 not in sub.tel # check that there is no tel 0 (1 is first above) assert len(sub.to_table()) == n_tels - assert len(set(sub.camera_types)) == 1 # only 1 camera type + assert len(sub.camera_types) == 1 # only 1 camera type assert sub.camera_types[0] == 'NectarCam' - assert sub.optics_types[0] == 'MST' + assert sub.optics_types[0].equivalent_focal_length.to_value(u.m) == 16.0 assert sub.telescope_types[0] == 'MST:NectarCam' assert sub.tel_coords assert isinstance(sub.tel_coords, SkyCoord) @@ -46,4 +50,5 @@ def test_subarray_description(): assert subsub.tel_ids[3] == 6 - +if __name__ == '__main__': + test_subarray_description() diff --git a/ctapipe/instrument/tests/test_telescope.py b/ctapipe/instrument/tests/test_telescope.py new file mode 100644 index 00000000000..26667495f57 --- /dev/null +++ b/ctapipe/instrument/tests/test_telescope.py @@ -0,0 +1,5 @@ +def test_hash(): + + from ctapipe.instrument.telescope import TelescopeDescription + from ctapipe.instrument.optics import OpticsDescription + from ctapipe.instrument.camera import CameraGeometry diff --git a/ctapipe/io/hessioeventsource.py b/ctapipe/io/hessioeventsource.py index df56c2f36a8..962bdbdae79 100644 --- a/ctapipe/io/hessioeventsource.py +++ b/ctapipe/io/hessioeventsource.py @@ -215,6 +215,7 @@ def _build_telescope_description(self, file, tel_id): mirror_area = u.Quantity(file.get_mirror_area(tel_id), u.m**2) num_tiles = file.get_mirror_number(tel_id) cam_rot = file.get_camera_rotation_angle(tel_id) + num_mirrors = file.get_mirror_number(tel_id) camera = CameraGeometry( telescope.camera_name, @@ -228,18 +229,15 @@ def _build_telescope_description(self, file, tel_id): apply_derotation=True, ) - if telescope.name != telescope.type: - subtype = telescope.name - else: - subtype = '' - optics = OpticsDescription( - mirror_type=telescope.mirror_type, - tel_type=telescope.type, - tel_subtype=subtype, + name=telescope.name, + num_mirrors=num_mirrors, equivalent_focal_length=focal_length, mirror_area=mirror_area, num_mirror_tiles=num_tiles, ) - return TelescopeDescription(camera=camera, optics=optics) + return TelescopeDescription( + name=telescope.name, type=telescope.type, + camera=camera, optics=optics, + ) diff --git a/ctapipe/io/simteleventsource.py b/ctapipe/io/simteleventsource.py index b74ad5efc97..d65f2fa32c9 100644 --- a/ctapipe/io/simteleventsource.py +++ b/ctapipe/io/simteleventsource.py @@ -80,21 +80,17 @@ def prepare_subarray_info(telescope_descriptions, header): ) - if telescope.name != telescope.type: - subtype = telescope.name - else: - subtype = '' - optics = OpticsDescription( - mirror_type=telescope.mirror_type, - tel_type=telescope.type, - tel_subtype=subtype, + name=telescope.name, + num_mirrors=cam_settings['n_mirrors'], equivalent_focal_length=focal_length, mirror_area=u.Quantity(cam_settings['mirror_area'], u.m**2), num_mirror_tiles=cam_settings['n_mirrors'], ) tel_descriptions[tel_id] = TelescopeDescription( + name=telescope.name, + type=telescope.type, camera=camera, optics=optics, ) diff --git a/ctapipe/reco/tests/test_HillasReconstructor.py b/ctapipe/reco/tests/test_HillasReconstructor.py index 4dc1828c13a..2bffb4f4f00 100644 --- a/ctapipe/reco/tests/test_HillasReconstructor.py +++ b/ctapipe/reco/tests/test_HillasReconstructor.py @@ -91,14 +91,14 @@ def test_reconstruction(): in the end, proper units in the output are asserted """ - filename = get_dataset_path("gamma_test.simtel.gz") + filename = get_dataset_path("gamma_test_large.simtel.gz") fit = HillasReconstructor() tel_azimuth = {} tel_altitude = {} - source = event_source(filename) + source = event_source(filename, max_events=10) for event in source: From e9f81f301d85817f5e7fd35152757c539202da05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 16:32:29 +0100 Subject: [PATCH 06/23] Add test for isntrument hashing --- ctapipe/instrument/tests/test_telescope.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ctapipe/instrument/tests/test_telescope.py b/ctapipe/instrument/tests/test_telescope.py index 26667495f57..c3d519d62e6 100644 --- a/ctapipe/instrument/tests/test_telescope.py +++ b/ctapipe/instrument/tests/test_telescope.py @@ -1,5 +1,22 @@ def test_hash(): - from ctapipe.instrument.telescope import TelescopeDescription from ctapipe.instrument.optics import OpticsDescription from ctapipe.instrument.camera import CameraGeometry + + types = ['LST', 'MST', 'SST'] + names = ['LST', 'MST', 'SST-1M'] + cameras = ['LSTCam', 'FlashCam', 'DigiCam'] + + telescopes = [] + for name, type, camera in zip(names, types, cameras): + for i in range(3): + + telescopes.append(TelescopeDescription( + name=name, + type=type, + optics=OpticsDescription.from_name(name), + camera=CameraGeometry.from_name(camera) + )) + + assert len(telescopes) == 9 + assert len(set(telescopes)) == 3 From 7ef533042ecf7af1aa3c8e2caefb441f08bcc949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 16:32:57 +0100 Subject: [PATCH 07/23] Better error for optics.from_name if no match is found --- ctapipe/instrument/optics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ctapipe/instrument/optics.py b/ctapipe/instrument/optics.py index b83f5405d49..7273437bb5d 100644 --- a/ctapipe/instrument/optics.py +++ b/ctapipe/instrument/optics.py @@ -92,6 +92,8 @@ def from_name(cls, name, optics_table='optics'): """ table = get_table_dataset(optics_table, role='dl0.tel.svc.optics') mask = table['tel_description'] == name + if mask.sum() == 0: + raise ValueError(f'Unknown telescope name {name}') flen = table['equivalent_focal_length'][mask].quantity[0] From 4057a07cef54fefaa874b24c36e89720aa8964c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 16:34:33 +0100 Subject: [PATCH 08/23] Fix optics table --- ctapipe/instrument/subarray.py | 21 ++++++++++++--------- ctapipe/instrument/tests/test_subarray.py | 2 ++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/ctapipe/instrument/subarray.py b/ctapipe/instrument/subarray.py index 5a4be41e24b..7708f715a60 100644 --- a/ctapipe/instrument/subarray.py +++ b/ctapipe/instrument/subarray.py @@ -4,7 +4,6 @@ __all__ = ['SubarrayDescription'] -import warnings from collections import defaultdict import numpy as np @@ -175,19 +174,23 @@ def to_table(self, kind="subarray"): elif kind == 'optics': unique_types = set(self.tels.values()) + mirror_area = u.Quantity( + [t.optics.mirror_area.to_value(u.m**2) for t in unique_types], + u.m**2, + ) + focal_length = u.Quantity( + [t.optics.equivalent_focal_length.to_value(u.m) for t in unique_types], + u.m, + ) cols = { 'description': [str(t) for t in unique_types], 'name': [t.name for t in unique_types], 'type': [t.type for t in unique_types], - 'mirror_area': [t.mirror_area.to(u.m**2) for t in unique_types], - 'num_mirrors': [t.mirror_area for t in unique_types], - 'num_mirror_tiles': [t.num_mirror_tiles for t in unique_types], - 'equivalent_focal_length': [ - t.equivalent_focal_length.to(u.m) - for t in unique_types - ], + 'mirror_area': mirror_area, + 'num_mirrors': [t.optics.num_mirrors for t in unique_types], + 'num_mirror_tiles': [t.optics.num_mirror_tiles for t in unique_types], + 'equivalent_focal_length': focal_length, } - tab = Table(cols) else: diff --git a/ctapipe/instrument/tests/test_subarray.py b/ctapipe/instrument/tests/test_subarray.py index ebc46e44153..b1eac5b6ed4 100644 --- a/ctapipe/instrument/tests/test_subarray.py +++ b/ctapipe/instrument/tests/test_subarray.py @@ -49,6 +49,8 @@ def test_subarray_description(): assert subsub.tel_indices[6] == 3 assert subsub.tel_ids[3] == 6 + assert len(sub.to_table(kind='optics')) == 1 + if __name__ == '__main__': test_subarray_description() From c16ea5ed306a8162e117a36369df439de452cdce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 16:35:16 +0100 Subject: [PATCH 09/23] More places to use gamma_test_large --- ctapipe/io/tests/test_event_source_factory.py | 14 +++++++------- ctapipe/io/tests/test_eventseeker.py | 2 +- ctapipe/tools/bokeh/file_viewer.py | 2 +- ctapipe/tools/tests/test_tools.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ctapipe/io/tests/test_event_source_factory.py b/ctapipe/io/tests/test_event_source_factory.py index 1def67c6cdb..d68015033de 100644 --- a/ctapipe/io/tests/test_event_source_factory.py +++ b/ctapipe/io/tests/test_event_source_factory.py @@ -7,7 +7,7 @@ def test_factory(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") reader = event_source(input_url=dataset) assert isinstance(reader, SimTelEventSource) assert reader.input_url == dataset @@ -60,7 +60,7 @@ def test_from_config_invalid_type(): def test_event_source_config(): - dataset1 = get_dataset_path("gamma_test.simtel.gz") + dataset1 = get_dataset_path("gamma_test_large.simtel.gz") dataset2 = get_dataset_path("gamma_test_large.simtel.gz") config = Config({'EventSource': {'input_url': dataset1}}) reader = event_source(dataset2, config=config) @@ -69,7 +69,7 @@ def test_event_source_config(): def test_event_source_input_url_config_override(): - dataset1 = get_dataset_path("gamma_test.simtel.gz") + dataset1 = get_dataset_path("gamma_test_large.simtel.gz") dataset2 = get_dataset_path("gamma_test_large.simtel.gz") config = Config({'EventSource': {'input_url': dataset1}}) reader = event_source(input_url=dataset2, config=config) @@ -79,13 +79,13 @@ def test_event_source_input_url_config_override(): def test_factory_max_events(): max_events = 10 - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") reader = event_source(input_url=dataset, max_events=max_events) assert reader.max_events == max_events def test_factory_max_events_from_config(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") max_events = 10 config = Config({'EventSource': { 'input_url': dataset, @@ -96,7 +96,7 @@ def test_factory_max_events_from_config(): def test_factory_allowed_tels(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") reader = event_source(input_url=dataset) assert len(reader.allowed_tels) == 0 reader = event_source(input_url=dataset, allowed_tels={1, 3}) @@ -104,7 +104,7 @@ def test_factory_allowed_tels(): def test_factory_allowed_tels_from_config(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") config = Config({'EventSource': { 'input_url': dataset, 'allowed_tels': {1, 3} diff --git a/ctapipe/io/tests/test_eventseeker.py b/ctapipe/io/tests/test_eventseeker.py index a1b0fa772ba..785baeeec93 100644 --- a/ctapipe/io/tests/test_eventseeker.py +++ b/ctapipe/io/tests/test_eventseeker.py @@ -3,7 +3,7 @@ from ctapipe.io.eventseeker import EventSeeker import pytest -dataset = get_dataset_path("gamma_test.simtel.gz") +dataset = get_dataset_path("gamma_test_large.simtel.gz") def test_eventseeker(): diff --git a/ctapipe/tools/bokeh/file_viewer.py b/ctapipe/tools/bokeh/file_viewer.py index 0bb47965c30..ba3fc086c89 100644 --- a/ctapipe/tools/bokeh/file_viewer.py +++ b/ctapipe/tools/bokeh/file_viewer.py @@ -27,7 +27,7 @@ class BokehFileViewer(Tool): disable_server = Bool(False, help="Do not start the bokeh server " "(useful for testing)").tag(config=True) - default_url = get_dataset_path("gamma_test.simtel.gz") + default_url = get_dataset_path("gamma_test_large.simtel.gz") EventSource.input_url.default_value = default_url cleaner_product = tool_utils.enum_trait( diff --git a/ctapipe/tools/tests/test_tools.py b/ctapipe/tools/tests/test_tools.py index 5af9f3c96e0..1d9516c8cb1 100644 --- a/ctapipe/tools/tests/test_tools.py +++ b/ctapipe/tools/tests/test_tools.py @@ -59,7 +59,7 @@ def test_bokeh_file_viewer(): tool = BokehFileViewer(disable_server=True) tool.run() - assert tool.reader.input_url == get_dataset_path("gamma_test.simtel.gz") + assert tool.reader.input_url == get_dataset_path("gamma_test_large.simtel.gz") def test_extract_charge_resolution(tmpdir): From 142a5b2ce869c0aa9c7fb2a5d08a8e39b1490201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 16:39:26 +0100 Subject: [PATCH 10/23] Add comment --- ctapipe/calib/camera/tests/test_calibrator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ctapipe/calib/camera/tests/test_calibrator.py b/ctapipe/calib/camera/tests/test_calibrator.py index 9b96e6bec92..229f8016bf5 100644 --- a/ctapipe/calib/camera/tests/test_calibrator.py +++ b/ctapipe/calib/camera/tests/test_calibrator.py @@ -11,12 +11,13 @@ def test_camera_calibrator(example_event): - telid = 11 + telid = next(iter(example_event.r0.tel.keys())) calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator") calibrator.calibrate(example_event) image = example_event.dl1.tel[telid].image + # why do we test for this specific value? assert_allclose(image[0, 0], -2.216, 1e-3) From b1b5229d630746e82ad7ebd064ba2cdc8088f026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 16:56:01 +0100 Subject: [PATCH 11/23] More fixes --- ctapipe/calib/camera/tests/test_r1.py | 9 +++++---- ctapipe/instrument/optics.py | 3 --- ctapipe/instrument/subarray.py | 2 +- ctapipe/io/tests/test_event_source.py | 4 ++-- ctapipe/io/tests/test_hessio_event_source.py | 18 +++++++++--------- ctapipe/io/tests/test_simtel_event_source.py | 16 +++++++--------- ctapipe/io/tests/test_targetio_event_source.py | 2 +- ctapipe/plotting/tests/test_camera.py | 2 +- docs/tutorials/ctapipe_handson.ipynb | 12 +++++++----- examples/array_display.py | 10 ++++++---- examples/calc_pedestals.py | 2 +- examples/instrument_info.py | 6 +++--- 12 files changed, 43 insertions(+), 43 deletions(-) diff --git a/ctapipe/calib/camera/tests/test_r1.py b/ctapipe/calib/camera/tests/test_r1.py index 846455f934c..dbfe7dcf314 100644 --- a/ctapipe/calib/camera/tests/test_r1.py +++ b/ctapipe/calib/camera/tests/test_r1.py @@ -14,8 +14,11 @@ from ctapipe.utils import get_dataset_path +dataset = get_dataset_path("gamma_test_large.simtel.gz") + + def test_hessio_r1_calibrator(example_event): - telid = 11 + telid = list(example_event.r0.tel.keys())[0] calibrator = HESSIOR1Calibrator() calibrator.calibrate(example_event) @@ -24,7 +27,7 @@ def test_hessio_r1_calibrator(example_event): def test_null_r1_calibrator(example_event): - telid = 11 + telid = list(example_event.r0.tel.keys())[0] calibrator = NullR1Calibrator() calibrator.calibrate(example_event) @@ -84,7 +87,6 @@ def test_factory_from_product(): def test_factory_for_eventsource(): - dataset = get_dataset_path("gamma_test.simtel.gz") eventsource = SimTelEventSource(input_url=dataset) calibrator = CameraR1Calibrator.from_eventsource(eventsource=eventsource) assert isinstance(calibrator, HESSIOR1Calibrator) @@ -104,7 +106,6 @@ def is_compatible(file_path): def test_factory_from_unknown_eventsource(): - dataset = get_dataset_path("gamma_test.simtel.gz") eventsource = UnknownEventSource(input_url=dataset) calibrator = CameraR1Calibrator.from_eventsource(eventsource=eventsource) assert isinstance(calibrator, NullR1Calibrator) diff --git a/ctapipe/instrument/optics.py b/ctapipe/instrument/optics.py index 7273437bb5d..7f84a9474fe 100644 --- a/ctapipe/instrument/optics.py +++ b/ctapipe/instrument/optics.py @@ -124,6 +124,3 @@ def __repr__(self): def __str__(self): return self.name - - def __hash__(self): - return hash((self.equivalent_focal_length, self.num_mirrors, self.mirror_area)) diff --git a/ctapipe/instrument/subarray.py b/ctapipe/instrument/subarray.py index 7708f715a60..3fad814b045 100644 --- a/ctapipe/instrument/subarray.py +++ b/ctapipe/instrument/subarray.py @@ -236,7 +236,7 @@ def peek(self): with quantity_support(): for tel_type in types: - tels = tab[tab['tel_description'] == str(tel_type)]['tel_id'] + tels = tab[tab['tel_description'] == str(tel_type)]['id'] sub = self.select_subarray(tel_type, tels) tel_coords = sub.tel_coords radius = np.array([ diff --git a/ctapipe/io/tests/test_event_source.py b/ctapipe/io/tests/test_event_source.py index 08afaa776ad..b27fa94e205 100644 --- a/ctapipe/io/tests/test_event_source.py +++ b/ctapipe/io/tests/test_event_source.py @@ -22,13 +22,13 @@ def is_compatible(file_path): def test_can_be_implemented(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") test_reader = DummyReader(input_url=dataset) assert test_reader is not None def test_is_iterable(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") test_reader = DummyReader(input_url=dataset) for _ in test_reader: pass diff --git a/ctapipe/io/tests/test_hessio_event_source.py b/ctapipe/io/tests/test_hessio_event_source.py index e1575909c79..5f5ea24a4f7 100644 --- a/ctapipe/io/tests/test_hessio_event_source.py +++ b/ctapipe/io/tests/test_hessio_event_source.py @@ -3,7 +3,7 @@ from ctapipe.utils import get_dataset_path from ctapipe.io.hessioeventsource import HESSIOEventSource -dataset = get_dataset_path("gamma_test.simtel.gz") +dataset = get_dataset_path("gamma_test_large.simtel.gz") pytest.importorskip('pyhessio') @@ -12,13 +12,9 @@ def test_hessio_file_reader(): with HESSIOEventSource(input_url=dataset) as reader: assert not reader.is_stream for event in reader: - if event.count == 0: - assert event.r0.tels_with_data == {38, 47} - elif event.count == 1: - assert event.r0.tels_with_data == {11, 21, 24, 26, 61, 63, 118, - 119} - else: + if event.count == 1: break + for event in reader: # Check generator has restarted from beginning assert event.count == 0 @@ -33,14 +29,18 @@ def test_hessio_file_reader(): assert count == max_events # test that the allowed_tels mask works: - with HESSIOEventSource(input_url=dataset, allowed_tels={3, 4}) as reader: + with HESSIOEventSource( + input_url=dataset, + allowed_tels={3, 4}, + max_events=max_events, + ) as reader: for event in reader: assert event.r0.tels_with_data.issubset(reader.allowed_tels) def test_that_event_is_not_modified_after_loop(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") with HESSIOEventSource(input_url=dataset, max_events=2) as source: for event in source: last_event = copy.deepcopy(event) diff --git a/ctapipe/io/tests/test_simtel_event_source.py b/ctapipe/io/tests/test_simtel_event_source.py index bca7e6d588b..03fb799e5f9 100644 --- a/ctapipe/io/tests/test_simtel_event_source.py +++ b/ctapipe/io/tests/test_simtel_event_source.py @@ -2,20 +2,18 @@ from ctapipe.utils import get_dataset_path from ctapipe.io.simteleventsource import SimTelEventSource -dataset = get_dataset_path("gamma_test.simtel.gz") +dataset = get_dataset_path("gamma_test_large.simtel.gz") + def test_hessio_file_reader(): with SimTelEventSource(input_url=dataset) as reader: assert reader.is_compatible(dataset) assert not reader.is_stream + for event in reader: - if event.count == 0: - assert event.r0.tels_with_data == {38, 47} - elif event.count == 1: - assert event.r0.tels_with_data == {11, 21, 24, 26, 61, 63, 118, - 119} - else: + if event.count == 1: break + for event in reader: # Check generator has restarted from beginning assert event.count == 0 @@ -30,14 +28,14 @@ def test_hessio_file_reader(): assert count == max_events # test that the allowed_tels mask works: - with SimTelEventSource(input_url=dataset, allowed_tels={3, 4}) as reader: + with SimTelEventSource(input_url=dataset, allowed_tels={3, 4}, max_events=10) as reader: for event in reader: assert event.r0.tels_with_data.issubset(reader.allowed_tels) def test_that_event_is_not_modified_after_loop(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") with SimTelEventSource(input_url=dataset, max_events=2) as source: for event in source: last_event = copy.deepcopy(event) diff --git a/ctapipe/io/tests/test_targetio_event_source.py b/ctapipe/io/tests/test_targetio_event_source.py index 0b8604dd782..f6fd46fabaf 100644 --- a/ctapipe/io/tests/test_targetio_event_source.py +++ b/ctapipe/io/tests/test_targetio_event_source.py @@ -58,7 +58,7 @@ def test_compatible(): dataset = get_dataset_path("chec_r1.tio") assert TargetIOEventSource.is_compatible(dataset) - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") assert not TargetIOEventSource.is_compatible(dataset) diff --git a/ctapipe/plotting/tests/test_camera.py b/ctapipe/plotting/tests/test_camera.py index 10f4133e08e..eeda403f98c 100644 --- a/ctapipe/plotting/tests/test_camera.py +++ b/ctapipe/plotting/tests/test_camera.py @@ -5,7 +5,7 @@ def test_eventplotter(): - dataset = get_dataset_path("gamma_test.simtel.gz") + dataset = get_dataset_path("gamma_test_large.simtel.gz") with event_source(dataset, max_events=1) as source: event = next(iter(source)) diff --git a/docs/tutorials/ctapipe_handson.ipynb b/docs/tutorials/ctapipe_handson.ipynb index ad1e22edf2b..d860d9b6299 100644 --- a/docs/tutorials/ctapipe_handson.ipynb +++ b/docs/tutorials/ctapipe_handson.ipynb @@ -34,8 +34,8 @@ "metadata": {}, "outputs": [], "source": [ - "path = utils.get_dataset_path(\"gamma_test.simtel.gz\")\n", - "source = event_source(path, max_events=5)" + "path = utils.get_dataset_path(\"gamma_test_large.simtel.gz\")\n", + "source = event_source(path, max_events=4)" ] }, { @@ -245,16 +245,18 @@ "metadata": {}, "outputs": [], "source": [ - "tel.camera.info()" + "tel.camera" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ - "tel.optics.info()" + "tel.optics" ] }, { diff --git a/examples/array_display.py b/examples/array_display.py index 80efc09086f..80744e398e6 100755 --- a/examples/array_display.py +++ b/examples/array_display.py @@ -13,10 +13,12 @@ plt.figure(figsize=(9.5, 8.5)) # load up a single event, so we can get the subarray info: - source = event_source(datasets.get_dataset_path("gamma_test.simtel.gz"), - max_events=1) - for event in source: - pass + source = event_source( + datasets.get_dataset_path("gamma_test_large.simtel.gz"), + max_events=1, + ) + + event = next(iter(source)) # display the array subarray = event.inst.subarray diff --git a/examples/calc_pedestals.py b/examples/calc_pedestals.py index f777a2e17ff..a5a251ee2b5 100755 --- a/examples/calc_pedestals.py +++ b/examples/calc_pedestals.py @@ -28,7 +28,7 @@ def plot_peds(peds, pedvars): if len(sys.argv) > 1: filename = sys.argv.pop(1) else: - filename = get_dataset_path("gamma_test.simtel.gz") + filename = get_dataset_path("gamma_test_large.simtel.gz") # set a fixed window (for now from samples 20 to the end), which may not # be appropriate for all telescopes (this should eventually be diff --git a/examples/instrument_info.py b/examples/instrument_info.py index 9ff1f66ff51..b9276b6ed7d 100644 --- a/examples/instrument_info.py +++ b/examples/instrument_info.py @@ -8,10 +8,10 @@ if __name__ == '__main__': # load up one event so that we get the instrument info - infile = get_dataset_path("gamma_test.simtel.gz") + infile = get_dataset_path("gamma_test_large.simtel.gz") + with event_source(infile) as source: - gsource = (x for x in source) - event = next(gsource) + event = next(iter(source)) print("------ Input: ", infile) From 09eb4bfc1b313ae2095aee99ffd84de706185efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 17:00:06 +0100 Subject: [PATCH 12/23] More fixes for simtel file --- ctapipe/calib/camera/tests/test_dl0.py | 6 ++++-- ctapipe/io/tests/test_hessio.py | 14 -------------- 2 files changed, 4 insertions(+), 16 deletions(-) delete mode 100644 ctapipe/io/tests/test_hessio.py diff --git a/ctapipe/calib/camera/tests/test_dl0.py b/ctapipe/calib/camera/tests/test_dl0.py index 3442c5beca6..b985978b4d1 100644 --- a/ctapipe/calib/camera/tests/test_dl0.py +++ b/ctapipe/calib/camera/tests/test_dl0.py @@ -11,7 +11,8 @@ def previous_calibration(event): def test_camera_dl0_reducer(example_event): previous_calibration(example_event) - telid = 11 + + telid = list(example_event.r0.keys())[0] reducer = CameraDL0Reducer() reducer.reduce(example_event) @@ -20,7 +21,8 @@ def test_camera_dl0_reducer(example_event): def test_check_r1_exists(example_event): - telid = 11 + telid = list(example_event.r0.keys())[0] + previous_calibration(example_event) reducer = CameraDL0Reducer() assert (reducer.check_r1_exists(example_event, telid) is True) diff --git a/ctapipe/io/tests/test_hessio.py b/ctapipe/io/tests/test_hessio.py deleted file mode 100644 index 7a3aaced742..00000000000 --- a/ctapipe/io/tests/test_hessio.py +++ /dev/null @@ -1,14 +0,0 @@ -import pytest -from ctapipe.utils import get_dataset_path -from ctapipe.io.hessioeventsource import HESSIOEventSource - -pytest.importorskip('pyhessio') - - -def test_hessio_event_source(): - filename = get_dataset_path("gamma_test.simtel.gz") - - with HESSIOEventSource(input_url=filename) as source: - event = next(iter(source)) - tels = event.dl0.tels_with_data - assert tels == {38, 47} From 01b702c38da7b1733810a3cc1517693ab227e591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 17:04:38 +0100 Subject: [PATCH 13/23] Replace more hard coded telids --- ctapipe/calib/camera/tests/test_dl0.py | 4 ++-- ctapipe/calib/camera/tests/test_dl1.py | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ctapipe/calib/camera/tests/test_dl0.py b/ctapipe/calib/camera/tests/test_dl0.py index b985978b4d1..bbd6ed28dd8 100644 --- a/ctapipe/calib/camera/tests/test_dl0.py +++ b/ctapipe/calib/camera/tests/test_dl0.py @@ -12,7 +12,7 @@ def previous_calibration(event): def test_camera_dl0_reducer(example_event): previous_calibration(example_event) - telid = list(example_event.r0.keys())[0] + telid = list(example_event.r0.tel.keys())[0] reducer = CameraDL0Reducer() reducer.reduce(example_event) @@ -21,7 +21,7 @@ def test_camera_dl0_reducer(example_event): def test_check_r1_exists(example_event): - telid = list(example_event.r0.keys())[0] + telid = list(example_event.r0.tel.keys())[0] previous_calibration(example_event) reducer = CameraDL0Reducer() diff --git a/ctapipe/calib/camera/tests/test_dl1.py b/ctapipe/calib/camera/tests/test_dl1.py index 7c155d0656c..f88985aadd0 100644 --- a/ctapipe/calib/camera/tests/test_dl1.py +++ b/ctapipe/calib/camera/tests/test_dl1.py @@ -14,7 +14,8 @@ def previous_calibration(event): def test_integration_correction(example_event): - telid = 11 + telid = list(example_event.r0.tel.keys())[0] + width = 7 shift = 3 shape = example_event.mc.tel[telid].reference_pulse_shape @@ -36,8 +37,9 @@ def test_integration_correction_no_ref_pulse(example_event): def test_camera_dl1_calibrator(example_event): + telid = list(example_event.r0.tel.keys())[0] + previous_calibration(example_event) - telid = 11 calibrator = CameraDL1Calibrator() @@ -50,7 +52,8 @@ def test_camera_dl1_calibrator(example_event): def test_check_dl0_exists(example_event): - telid = 11 + telid = list(example_event.r0.tel.keys())[0] + previous_calibration(example_event) calibrator = CameraDL1Calibrator() assert(calibrator.check_dl0_exists(example_event, telid) is True) From 7a5280859cacede60f22739815c76dee41e2bc96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 17:30:53 +0100 Subject: [PATCH 14/23] More telid fixes --- ctapipe/calib/camera/tests/test_calibrator.py | 2 +- ctapipe/calib/camera/tests/test_dl0.py | 4 ++-- ctapipe/calib/camera/tests/test_dl1.py | 8 ++++---- ctapipe/calib/camera/tests/test_r1.py | 6 +++--- ctapipe/image/muon/muon_reco_functions.py | 2 +- ctapipe/image/tests/test_charge_extraction.py | 14 +++++++------- ctapipe/image/tests/test_waveform_cleaning.py | 6 +++--- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ctapipe/calib/camera/tests/test_calibrator.py b/ctapipe/calib/camera/tests/test_calibrator.py index 229f8016bf5..489ca275f2b 100644 --- a/ctapipe/calib/camera/tests/test_calibrator.py +++ b/ctapipe/calib/camera/tests/test_calibrator.py @@ -11,7 +11,7 @@ def test_camera_calibrator(example_event): - telid = next(iter(example_event.r0.tel.keys())) + telid = list(example_event.r0.tel)[0] calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator") diff --git a/ctapipe/calib/camera/tests/test_dl0.py b/ctapipe/calib/camera/tests/test_dl0.py index bbd6ed28dd8..c6af5db31e5 100644 --- a/ctapipe/calib/camera/tests/test_dl0.py +++ b/ctapipe/calib/camera/tests/test_dl0.py @@ -12,7 +12,7 @@ def previous_calibration(event): def test_camera_dl0_reducer(example_event): previous_calibration(example_event) - telid = list(example_event.r0.tel.keys())[0] + telid = list(example_event.r0.tel)[0] reducer = CameraDL0Reducer() reducer.reduce(example_event) @@ -21,7 +21,7 @@ def test_camera_dl0_reducer(example_event): def test_check_r1_exists(example_event): - telid = list(example_event.r0.tel.keys())[0] + telid = list(example_event.r0.tel)[0] previous_calibration(example_event) reducer = CameraDL0Reducer() diff --git a/ctapipe/calib/camera/tests/test_dl1.py b/ctapipe/calib/camera/tests/test_dl1.py index f88985aadd0..aef4e80af9b 100644 --- a/ctapipe/calib/camera/tests/test_dl1.py +++ b/ctapipe/calib/camera/tests/test_dl1.py @@ -14,7 +14,7 @@ def previous_calibration(event): def test_integration_correction(example_event): - telid = list(example_event.r0.tel.keys())[0] + telid = list(example_event.r0.tel)[0] width = 7 shift = 3 @@ -29,7 +29,7 @@ def test_integration_correction(example_event): def test_integration_correction_no_ref_pulse(example_event): previous_calibration(example_event) - telid = list(example_event.dl0.tel.keys())[0] + telid = list(example_event.dl0.tel)[0] delattr(example_event, 'mc') calibrator = CameraDL1Calibrator() correction = calibrator.get_correction(example_event, telid) @@ -37,7 +37,7 @@ def test_integration_correction_no_ref_pulse(example_event): def test_camera_dl1_calibrator(example_event): - telid = list(example_event.r0.tel.keys())[0] + telid = list(example_event.r0.tel)[0] previous_calibration(example_event) @@ -52,7 +52,7 @@ def test_camera_dl1_calibrator(example_event): def test_check_dl0_exists(example_event): - telid = list(example_event.r0.tel.keys())[0] + telid = list(example_event.r0.tel)[0] previous_calibration(example_event) calibrator = CameraDL1Calibrator() diff --git a/ctapipe/calib/camera/tests/test_r1.py b/ctapipe/calib/camera/tests/test_r1.py index dbfe7dcf314..8c650f04e9b 100644 --- a/ctapipe/calib/camera/tests/test_r1.py +++ b/ctapipe/calib/camera/tests/test_r1.py @@ -18,7 +18,7 @@ def test_hessio_r1_calibrator(example_event): - telid = list(example_event.r0.tel.keys())[0] + telid = list(example_event.r0.tel)[0] calibrator = HESSIOR1Calibrator() calibrator.calibrate(example_event) @@ -27,7 +27,7 @@ def test_hessio_r1_calibrator(example_event): def test_null_r1_calibrator(example_event): - telid = list(example_event.r0.tel.keys())[0] + telid = list(example_event.r0.tel)[0] calibrator = NullR1Calibrator() calibrator.calibrate(example_event) @@ -71,7 +71,7 @@ def test_targetio_calibrator_wrong_file(example_event): def test_check_r0_exists(example_event): - telid = 11 + telid = list(example_event.r0.tel)[0] calibrator = HESSIOR1Calibrator() assert (calibrator.check_r0_exists(example_event, telid) is True) diff --git a/ctapipe/image/muon/muon_reco_functions.py b/ctapipe/image/muon/muon_reco_functions.py index 2716df51d08..d4a48fff9d0 100644 --- a/ctapipe/image/muon/muon_reco_functions.py +++ b/ctapipe/image/muon/muon_reco_functions.py @@ -36,7 +36,7 @@ def analyze_muon_event(event): """ names = ['LST:LSTCam', 'MST:NectarCam', 'MST:FlashCam', 'MST-SCT:SCTCam', - 'SST-1M:DigiCam', 'SST-GCT:CHEC', 'SST-ASTRI:ASTRICam', 'SST-ASTRI:CHEC'] + '1M:DigiCam', 'GCT:CHEC', 'ASTRI:ASTRICam', 'ASTRI:CHEC'] tail_cuts = [(5, 7), (5, 7), (10, 12), (5, 7), (5, 7), (5, 7), (5, 7), (5, 7)] # 10, 12? impact = [(0.2, 0.9), (0.1, 0.95), (0.2, 0.9), (0.2, 0.9), diff --git a/ctapipe/image/tests/test_charge_extraction.py b/ctapipe/image/tests/test_charge_extraction.py index 3cb2055c346..71cff76faad 100644 --- a/ctapipe/image/tests/test_charge_extraction.py +++ b/ctapipe/image/tests/test_charge_extraction.py @@ -13,7 +13,7 @@ def test_full_integration(example_event): - telid = 11 + telid = list(example_event.r0.tel)[0] data = example_event.r0.tel[telid].waveform nsamples = data.shape[2] ped = example_event.mc.tel[telid].pedestal @@ -30,7 +30,7 @@ def test_full_integration(example_event): def test_simple_integration(example_event): - telid = 11 + telid = list(example_event.r0.tel)[0] data = example_event.r0.tel[telid].waveform nsamples = data.shape[2] @@ -48,7 +48,7 @@ def test_simple_integration(example_event): def test_global_peak_integration(example_event): - telid = 11 + telid = list(example_event.r0.tel)[0] data = example_event.r0.tel[telid].waveform nsamples = data.shape[2] ped = example_event.mc.tel[telid].pedestal @@ -65,7 +65,7 @@ def test_global_peak_integration(example_event): def test_local_peak_integration(example_event): - telid = 11 + telid = list(example_event.r0.tel)[0] data = example_event.r0.tel[telid].waveform nsamples = data.shape[2] ped = example_event.mc.tel[telid].pedestal @@ -82,7 +82,7 @@ def test_local_peak_integration(example_event): def test_nb_peak_integration(example_event): - telid = 11 + telid = list(example_event.r0.tel)[0] data = example_event.r0.tel[telid].waveform nsamples = data.shape[2] ped = example_event.mc.tel[telid].pedestal @@ -102,7 +102,7 @@ def test_nb_peak_integration(example_event): def test_averagewf_peak_integration(example_event): - telid = 11 + telid = list(example_event.r0.tel)[0] data = example_event.r0.tel[telid].waveform nsamples = data.shape[2] ped = example_event.mc.tel[telid].pedestal @@ -121,7 +121,7 @@ def test_averagewf_peak_integration(example_event): def test_charge_extractor_factory(example_event): extractor = ChargeExtractor.from_name('LocalPeakIntegrator') - telid = 11 + telid = list(example_event.r0.tel)[0] data = example_event.r0.tel[telid].waveform nsamples = data.shape[2] ped = example_event.mc.tel[telid].pedestal diff --git a/ctapipe/image/tests/test_waveform_cleaning.py b/ctapipe/image/tests/test_waveform_cleaning.py index 3db045a7552..50e06240ec1 100644 --- a/ctapipe/image/tests/test_waveform_cleaning.py +++ b/ctapipe/image/tests/test_waveform_cleaning.py @@ -7,7 +7,7 @@ def test_null_cleaner(example_event): - telid = 11 + telid = list(example_event.r0.tel)[0] data = example_event.r0.tel[telid].waveform nsamples = data.shape[2] ped = example_event.mc.tel[telid].pedestal @@ -21,7 +21,7 @@ def test_null_cleaner(example_event): def test_checm_cleaner_average(example_event): - telid = 11 + telid = list(example_event.r0.tel)[0] data = example_event.r0.tel[telid].waveform nsamples = data.shape[2] ped = example_event.mc.tel[telid].pedestal @@ -36,7 +36,7 @@ def test_checm_cleaner_average(example_event): def test_checm_cleaner_local(example_event): - telid = 11 + telid = list(example_event.r0.tel)[0] data = example_event.r0.tel[telid].waveform nsamples = data.shape[2] ped = example_event.mc.tel[telid].pedestal From 7d5f22f9e21262e8d07e9722a0e25f0f549eb91e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 17:31:09 +0100 Subject: [PATCH 15/23] Test event seeker only for its features --- ctapipe/io/tests/test_eventseeker.py | 34 ++++++++++++---------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/ctapipe/io/tests/test_eventseeker.py b/ctapipe/io/tests/test_eventseeker.py index 785baeeec93..0566ee97d6f 100644 --- a/ctapipe/io/tests/test_eventseeker.py +++ b/ctapipe/io/tests/test_eventseeker.py @@ -13,38 +13,36 @@ def test_eventseeker(): seeker = EventSeeker(reader=reader) event = seeker[1] - assert event.r0.tels_with_data == {11, 21, 24, 26, 61, 63, 118, 119} + assert event.count == 1 event = seeker[0] - assert event.r0.tels_with_data == {38, 47} + assert event.count == 0 - event = seeker['409'] - assert event.r0.event_id == 409 + event = seeker['31007'] + assert event.r0.event_id == 31007 - assert event.r0.tels_with_data == {11, 21, 24, 26, 61, 63, 118, 119} - tel_list = [{38, 47}, {11, 21, 24, 26, 61, 63, 118, 119}] events = seeker[0:2] - events_tels = [e.r0.tels_with_data for e in events] - assert events_tels == tel_list + + for i, event in enumerate(events): + assert event.count == i + events = seeker[[0, 1]] - events_tels = [e.r0.tels_with_data for e in events] - assert events_tels == tel_list + for i, event in enumerate(events): + assert event.count == i - events = seeker[['408', '409']] - events_tels = [e.r0.tels_with_data for e in events] - assert events_tels == tel_list - assert len(seeker) == 9 + ids = ['23703', '31007'] + events = seeker[ids] + + for i, event in zip(ids, events): + assert event.r0.event_id == int(i) with pytest.raises(IndexError): event = seeker[200] - assert event is not None with pytest.raises(ValueError): event = seeker['t'] - assert event is not None with pytest.raises(TypeError): event = seeker[dict()] - assert event is not None with SimTelEventSource(input_url=dataset, max_events=5) as reader: seeker = EventSeeker(reader=reader) @@ -53,11 +51,9 @@ def test_eventseeker(): assert event is not None class StreamFileReader(SimTelEventSource): - def is_stream(self): return True with StreamFileReader(input_url=dataset) as reader: with pytest.raises(IOError): seeker = EventSeeker(reader=reader) - assert seeker is not None From f1474f8c62d13beecc827ac2e60a20582a96acab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 17:42:39 +0100 Subject: [PATCH 16/23] Store n_mirrors in guessing result --- ctapipe/instrument/guess.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ctapipe/instrument/guess.py b/ctapipe/instrument/guess.py index 4b551781478..72cdaaee076 100644 --- a/ctapipe/instrument/guess.py +++ b/ctapipe/instrument/guess.py @@ -4,26 +4,26 @@ GuessingKey = namedtuple('GuessingKey', ['n_pixels', 'focal_length']) GuessingResult = namedtuple( - 'GuessingResult', ['type', 'name', 'camera_name', 'mirror_type'] + 'GuessingResult', ['type', 'name', 'camera_name', 'n_mirrors'] ) TELESCOPE_NAMES = { - GuessingKey(2048, 2.28): GuessingResult('SST', 'GCT', 'CHEC', 'SC'), - GuessingKey(2368, 2.15): GuessingResult('SST', 'ASTRI', 'ASTRICam', 'SC'), - GuessingKey(1296, 5.60): GuessingResult('SST', '1M', 'DigiCam', 'DC'), - GuessingKey(1764, 16.0): GuessingResult('MST', 'MST', 'FlashCam', 'DC'), - GuessingKey(1855, 16.0): GuessingResult('MST', 'MST', 'NectarCam', 'DC'), - GuessingKey(1855, 28.0): GuessingResult('LST', 'LST', 'LSTCam', 'DC'), - GuessingKey(11328, 5.59): GuessingResult('MST', 'SCT', 'SCTCam', 'SC'), + GuessingKey(2048, 2.28): GuessingResult('SST', 'GCT', 'CHEC', 2), + GuessingKey(2368, 2.15): GuessingResult('SST', 'ASTRI', 'ASTRICam', 2), + GuessingKey(1296, 5.60): GuessingResult('SST', '1M', 'DigiCam', 1), + GuessingKey(1764, 16.0): GuessingResult('MST', 'MST', 'FlashCam', 1), + GuessingKey(1855, 16.0): GuessingResult('MST', 'MST', 'NectarCam', 1), + GuessingKey(1855, 28.0): GuessingResult('LST', 'LST', 'LSTCam', 1), + GuessingKey(11328, 5.59): GuessingResult('MST', 'SCT', 'SCTCam', 1), # None CTA Telescopes - GuessingKey(960, 15.0): GuessingResult('MST', 'HESS-I', 'HESS-I', 'DC'), - GuessingKey(2048, 36.0): GuessingResult('LST', 'HESS-II', 'HESS-II', 'DC'), - GuessingKey(1440, 4.998): GuessingResult('SST', 'FACT', 'FACT', 'DC'), + GuessingKey(960, 15.0): GuessingResult('MST', 'HESS-I', 'HESS-I', 1), + GuessingKey(2048, 36.0): GuessingResult('LST', 'HESS-II', 'HESS-II', 1), + GuessingKey(1440, 4.998): GuessingResult('SST', 'FACT', 'FACT', 1), } -UNKNOWN_TELESCOPE = GuessingResult('UNKNOWN', 'UNKNOWN', 'UNKNOWN', 'UNKNOWN') +UNKNOWN_TELESCOPE = GuessingResult('UNKNOWN', 'UNKNOWN', 'UNKNOWN', -1) def guess_telescope(n_pixels, focal_length): @@ -43,7 +43,7 @@ def guess_telescope(n_pixels, focal_length): Returns ------- result: GuessingResult - A namedtuple having type, telescope_name, camera_name and mirror_type fields + A namedtuple having type, telescope_name, camera_name and n_mirrors fields ''' # allow unit input From 0b2230ad50371ce19d680c7858de3619d4b774b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 18:02:28 +0100 Subject: [PATCH 17/23] Add tests and fix CameraGeometry.equals --- ctapipe/instrument/camera.py | 19 ++++++++++++++----- ctapipe/instrument/tests/test_camera.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/ctapipe/instrument/camera.py b/ctapipe/instrument/camera.py index 543f0844c0a..02b38030660 100644 --- a/ctapipe/instrument/camera.py +++ b/ctapipe/instrument/camera.py @@ -95,13 +95,22 @@ def __init__(self, cam_id, pix_id, pix_x, pix_y, pix_area, pix_type, self.border_cache = {} def __eq__(self, other): - return all( - self.cam_id == other.cam_id, + if self.cam_id != other.cam_id: + return False + + if self.n_pixels != other.n_pixels: + return False + + if self.pix_type != other.pix_type: + return False + + if self.pix_rotation != other.pix_rotation: + return False + + return all([ (self.pix_x == other.pix_x).all(), (self.pix_y == other.pix_y).all(), - self.pix_type == other.pix_type, - self.pix_rotation == other.pix_rotation, - ) + ]) def __hash__(self): return hash(( diff --git a/ctapipe/instrument/tests/test_camera.py b/ctapipe/instrument/tests/test_camera.py index 8b18030d7da..162b5b0c600 100644 --- a/ctapipe/instrument/tests/test_camera.py +++ b/ctapipe/instrument/tests/test_camera.py @@ -177,3 +177,21 @@ def test_border_pixels(): assert cam.get_border_pixel_mask(1)[0] assert cam.get_border_pixel_mask(1)[2351] assert not cam.get_border_pixel_mask(1)[521] + + +def test_equals(): + cam1 = CameraGeometry.from_name("LSTCam") + cam2 = CameraGeometry.from_name("LSTCam") + cam3 = CameraGeometry.from_name("ASTRICam") + + assert cam1 is not cam2 + assert cam1 == cam2 + assert cam1 != cam3 + + +def test_hashing(): + cam1 = CameraGeometry.from_name("LSTCam") + cam2 = CameraGeometry.from_name("LSTCam") + cam3 = CameraGeometry.from_name("ASTRICam") + + len(set([cam1, cam2, cam3])) == 2 From abfb686933b48df7b2eaafe8791e027ed2990bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Mon, 25 Feb 2019 18:16:52 +0100 Subject: [PATCH 18/23] Fix notebook --- docs/examples/InstrumentDescription.ipynb | 31 +++++++++---- docs/examples/convert_hex_to_square.ipynb | 5 +- .../calibrated_data_exploration.ipynb | 46 +++++++++---------- docs/tutorials/raw_data_exploration.ipynb | 10 ++-- examples/calibration_pipeline.py | 2 +- 5 files changed, 52 insertions(+), 42 deletions(-) diff --git a/docs/examples/InstrumentDescription.ipynb b/docs/examples/InstrumentDescription.ipynb index 6311a2fdb62..27d5a841a32 100644 --- a/docs/examples/InstrumentDescription.ipynb +++ b/docs/examples/InstrumentDescription.ipynb @@ -27,12 +27,11 @@ "import numpy as np\n", "\n", "#filename = get_dataset_path(\"gamma_test_large.simtel.gz\") # try this one as well\n", - "filename = get_dataset_path(\"gamma_test.simtel.gz\") \n", + "filename = get_dataset_path(\"gamma_test_large.simtel.gz\") \n", "\n", "# just get the first event in the file:\n", "with event_source(filename, max_events=1) as source:\n", - " for event in source:\n", - " pass \n" + " event = next(iter(source))\n" ] }, { @@ -92,7 +91,7 @@ "outputs": [], "source": [ "tab = subarray.to_table()\n", - "sc_tels = tab[tab['mirror_type']=='SC']['tel_id'] # select tel_id of entries where the mirror type is SC\n", + "sc_tels = tab[tab['num_mirrors'] == 2]['id'] # select tel_id of entries where the mirror type is SC\n", "newsub = subarray.select_subarray(\"SCTels\", sc_tels)\n", "newsub.info()" ] @@ -110,9 +109,9 @@ "metadata": {}, "outputs": [], "source": [ - "gtab = tab.group_by('mirror_type')\n", - "sc = gtab.groups[1]\n", - "newsub = subarray.select_subarray(\"SCTels\", sc['tel_id'])\n", + "gtab = tab.group_by('num_mirrors')\n", + "sc = gtab.groups[0]\n", + "newsub = subarray.select_subarray(\"SCTels\", sc['id'])\n", "newsub.info()" ] }, @@ -242,7 +241,7 @@ "outputs": [], "source": [ "df = subarray.to_table().to_pandas()\n", - "df.set_index('tel_id')" + "df.set_index('id')" ] }, { @@ -279,11 +278,25 @@ "metadata": {}, "outputs": [], "source": [ - "lsts = subarray.select_subarray(\"LSTs\", df.loc[g.groups['LST:LSTCam']]['tel_id'])\n", + "lsts = subarray.select_subarray(\"LSTs\", df.loc[g.groups['LST:LSTCam']]['id'])\n", "lsts.info()\n", "lsts.peek()\n", "lsts.footprint" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/docs/examples/convert_hex_to_square.ipynb b/docs/examples/convert_hex_to_square.ipynb index 13354519ccd..462a26ace79 100644 --- a/docs/examples/convert_hex_to_square.ipynb +++ b/docs/examples/convert_hex_to_square.ipynb @@ -34,7 +34,7 @@ "metadata": {}, "outputs": [], "source": [ - "filename = get_dataset_path('gamma_test.simtel.gz')" + "filename = get_dataset_path('gamma_test_large.simtel.gz')" ] }, { @@ -43,8 +43,7 @@ "metadata": {}, "outputs": [], "source": [ - "source = event_source(filename)\n", - "source.allowed_tels = {1}" + "source = event_source(filename, max_events=10, allowed_tels={1})" ] }, { diff --git a/docs/tutorials/calibrated_data_exploration.ipynb b/docs/tutorials/calibrated_data_exploration.ipynb index fd37b9b3d02..1e2076240f8 100644 --- a/docs/tutorials/calibrated_data_exploration.ipynb +++ b/docs/tutorials/calibrated_data_exploration.ipynb @@ -10,9 +10,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import ctapipe\n", @@ -50,7 +48,7 @@ "metadata": {}, "outputs": [], "source": [ - "filename = get_dataset_path(\"gamma_test.simtel.gz\")\n", + "filename = get_dataset_path(\"gamma_test_large.simtel.gz\")\n", "source = event_source(filename, max_events=2)\n", "\n", "for event in source:\n", @@ -110,9 +108,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from ctapipe.calib import CameraCalibrator\n", @@ -157,13 +153,11 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from ctapipe.visualization import CameraDisplay\n", - "tel_id = list(event.r1.tels_with_data)[0]\n", + "tel_id = sorted(event.r1.tels_with_data)[1]\n", "sub = event.inst.subarray\n", "camera = sub.tel[tel_id].camera\n", "image = event.dl1.tel[tel_id].image[0] " @@ -181,9 +175,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from ctapipe.image import tailcuts_clean, hillas_parameters" @@ -195,7 +187,7 @@ "metadata": {}, "outputs": [], "source": [ - "mask = tailcuts_clean(camera, image, picture_thresh=10, boundary_thresh=5)\n", + "mask = tailcuts_clean(camera, image, picture_thresh=10, boundary_thresh=5, min_number_picture_neighbors=2)\n", "cleaned = image.copy()\n", "cleaned[~mask] = 0\n", "disp = CameraDisplay(camera, image=cleaned)\n" @@ -225,8 +217,9 @@ "disp.add_colorbar()\n", "disp.overlay_moments(params, color='red', lw=3)\n", "disp.highlight_pixels(mask, color='white', alpha=0.3, linewidth=2)\n", - "plt.xlim(-0.7,0.2)\n", - "plt.ylim(-0.5,0.2)" + "\n", + "plt.xlim(params.x.to_value(u.m) - 0.5, params.x.to_value(u.m) + 0.5)\n", + "plt.ylim(params.y.to_value(u.m) - 0.5, params.y.to_value(u.m) + 0.5)" ] }, { @@ -266,12 +259,10 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "cam_ids = set(sub.get_tel_ids_for_type(\"MST:NectarCam\"))" + "cam_ids = set(sub.get_tel_ids_for_type(\"MST:FlashCam\"))" ] }, { @@ -335,9 +326,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "from ctapipe.visualization import ArrayDisplay" @@ -349,7 +338,7 @@ "metadata": {}, "outputs": [], "source": [ - "nectarcam_subarray = sub.select_subarray(\"NectarCams\", cam_ids)\n", + "nectarcam_subarray = sub.select_subarray(\"FlashCam\", cam_ids)\n", "\n", "hit_pattern = np.zeros(shape=nectarcam_subarray.num_tels)\n", "hit_pattern[[nectarcam_subarray.tel_indices[x] for x in cams_in_event ]] = 100\n", @@ -361,6 +350,13 @@ "ad.values = hit_pattern\n", "ad.add_labels()\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/docs/tutorials/raw_data_exploration.ipynb b/docs/tutorials/raw_data_exploration.ipynb index 27b6e6fc740..b857f8de93f 100644 --- a/docs/tutorials/raw_data_exploration.ipynb +++ b/docs/tutorials/raw_data_exploration.ipynb @@ -47,7 +47,7 @@ "metadata": {}, "outputs": [], "source": [ - "source = event_source(get_dataset_path(\"gamma_test.simtel.gz\"), max_events=100)\n", + "source = event_source(get_dataset_path(\"gamma_test_large.simtel.gz\"), max_events=100)\n", "seeker = EventSeeker(source)" ] }, @@ -132,7 +132,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "now, we have a larger event with many telescopes... Let's look at the data from **CT24**:" + "now, we have a larger event with many telescopes... Let's look at the data from **CT7**:" ] }, { @@ -141,7 +141,7 @@ "metadata": {}, "outputs": [], "source": [ - "teldata = event.r0.tel[24]\n", + "teldata = event.r0.tel[7]\n", "print(teldata)\n", "teldata" ] @@ -424,7 +424,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [], "source": [ "for tel in event.r0.tels_with_data:\n", diff --git a/examples/calibration_pipeline.py b/examples/calibration_pipeline.py index 5f837425615..3211d20a553 100644 --- a/examples/calibration_pipeline.py +++ b/examples/calibration_pipeline.py @@ -188,7 +188,7 @@ def setup(self): kwargs = dict(config=self.config, tool=self) self.eventsource = event_source( - get_dataset_path("gamma_test.simtel.gz"), + get_dataset_path("gamma_test_large.simtel.gz"), **kwargs ) From 2b26b31698883ca1fe928cc041274550f6bef901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Tue, 26 Feb 2019 10:42:28 +0100 Subject: [PATCH 19/23] Reduce repetition --- ctapipe/io/simteleventsource.py | 49 +++++++++++++++++---------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/ctapipe/io/simteleventsource.py b/ctapipe/io/simteleventsource.py index 58bbef0c4ee..24ccb5385f7 100644 --- a/ctapipe/io/simteleventsource.py +++ b/ctapipe/io/simteleventsource.py @@ -12,8 +12,6 @@ OpticsDescription, ) from ctapipe.instrument.guess import guess_telescope, UNKNOWN_TELESCOPE -import struct -import gzip from traitlets import Bool from eventio.simtel.simtelfile import SimTelFile @@ -191,39 +189,43 @@ def __generator(self): tel_index = self.file_.header['tel_id'].tolist().index(tel_id) telescope_description = self.file_.telescope_descriptions[tel_id] - data.mc.tel[tel_id].dc_to_pe = array_event['laser_calibrations'][tel_id]['calib'] - data.mc.tel[tel_id].pedestal = array_event['camera_monitorings'][tel_id]['pedestal'] adc_samples = telescope_event.get('adc_samples') if adc_samples is None: adc_samples = telescope_event['adc_sums'][:, :, np.newaxis] - data.r0.tel[tel_id].waveform = adc_samples - data.r0.tel[tel_id].num_samples = adc_samples.shape[-1] + + r0 = data.r0.tel[tel_id] + r0.waveform = adc_samples + r0.num_samples = adc_samples.shape[-1] # We should not calculate stuff in an event source # if this is not needed, we calculate it for nothing - data.r0.tel[tel_id].image = adc_samples.sum(axis=-1) + r0.image = adc_samples.sum(axis=-1) pixel_lists = telescope_event['pixel_lists'] - data.r0.tel[tel_id].num_trig_pix = pixel_lists.get(0, {'pixels': 0})['pixels'] - if data.r0.tel[tel_id].num_trig_pix > 0: - data.r0.tel[tel_id].trig_pix_id = pixel_lists[0]['pixel_list'] + r0.num_trig_pix = pixel_lists.get(0, {'pixels': 0})['pixels'] + if r0.num_trig_pix > 0: + r0.trig_pix_id = pixel_lists[0]['pixel_list'] pixel_settings = telescope_description['pixel_settings'] - data.mc.tel[tel_id].reference_pulse_shape = pixel_settings['ref_shape'].astype('float64') - data.mc.tel[tel_id].meta['refstep'] = float(pixel_settings['ref_step']) - data.mc.tel[tel_id].time_slice = float(pixel_settings['time_slice']) - - n_pixel = data.r0.tel[tel_id].waveform.shape[-2] - data.mc.tel[tel_id].photo_electron_image = ( - array_event.get('photoelectrons', {}) - .get(tel_index, {}) - .get('photoelectrons', np.zeros(n_pixel, dtype='float32')) + n_pixel = r0.waveform.shape[-2] + + mc = data.mc.tel[tel_id] + mc.dc_to_pe = array_event['laser_calibrations'][tel_id]['calib'] + mc.pedestal = array_event['camera_monitorings'][tel_id]['pedestal'] + mc.reference_pulse_shape = pixel_settings['ref_shape'].astype('float64') + mc.meta['refstep'] = float(pixel_settings['ref_step']) + mc.time_slice = float(pixel_settings['time_slice']) + mc.photo_electron_image = ( + array_event + .get('photoelectrons', {}) + .get(tel_index, {}) + .get('photoelectrons', np.zeros(n_pixel, dtype='float32')) ) tracking_position = tracking_positions[tel_id] - data.mc.tel[tel_id].azimuth_raw = tracking_position['azimuth_raw'] - data.mc.tel[tel_id].altitude_raw = tracking_position['altitude_raw'] - data.mc.tel[tel_id].azimuth_cor = tracking_position.get('azimuth_cor', 0) - data.mc.tel[tel_id].altitude_cor = tracking_position.get('altitude_cor', 0) + mc.azimuth_raw = tracking_position['azimuth_raw'] + mc.altitude_raw = tracking_position['altitude_raw'] + mc.azimuth_cor = tracking_position.get('azimuth_cor', 0) + mc.altitude_cor = tracking_position.get('altitude_cor', 0) yield data def fill_mc_information(self, data, array_event): @@ -282,4 +284,3 @@ def fill_mc_information(self, data, array_event): data.mcheader.corsika_wlen_max = mc_run_head['corsika_wlen_max'] * u.nm data.mcheader.corsika_low_E_detail = mc_run_head['corsika_low_E_detail'] data.mcheader.corsika_high_E_detail = mc_run_head['corsika_high_E_detail'] - From 6bf773f979048d0225bd79dc192a921e0fe53c0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Tue, 26 Feb 2019 11:24:46 +0100 Subject: [PATCH 20/23] Make prod2 readable again but emit warning --- ctapipe/instrument/camera.py | 5 ++++- ctapipe/io/hessioeventsource.py | 13 ++++++++++++- ctapipe/io/simteleventsource.py | 12 +++++++++++- ctapipe/io/tests/test_prod2.py | 25 +++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 ctapipe/io/tests/test_prod2.py diff --git a/ctapipe/instrument/camera.py b/ctapipe/instrument/camera.py index 02b38030660..b50595a6e8f 100644 --- a/ctapipe/instrument/camera.py +++ b/ctapipe/instrument/camera.py @@ -630,7 +630,6 @@ def _find_neighbor_pixels(pix_x, pix_y, rad): return neighbors - def _neighbor_list_to_matrix(neighbors): """ convert a neighbor adjacency list (list of list of neighbors) to a 2D @@ -645,3 +644,7 @@ def _neighbor_list_to_matrix(neighbors): neigh2d[ipix, neighbor] = True return neigh2d + + +class UnknownPixelShapeWarning(UserWarning): + pass diff --git a/ctapipe/io/hessioeventsource.py b/ctapipe/io/hessioeventsource.py index 962bdbdae79..cf4b565f98a 100644 --- a/ctapipe/io/hessioeventsource.py +++ b/ctapipe/io/hessioeventsource.py @@ -9,8 +9,10 @@ OpticsDescription, CameraGeometry, ) +from ctapipe.instrument.camera import UnknownPixelShapeWarning from ctapipe.instrument.guess import guess_telescope, UNKNOWN_TELESCOPE import numpy as np +import warnings __all__ = ['HESSIOEventSource'] @@ -209,7 +211,16 @@ def _build_telescope_description(self, file, tel_id): telescope = UNKNOWN_TELESCOPE pixel_shape = file.get_pixel_shape(tel_id)[0] - pix_type, pix_rot = CameraGeometry.simtel_shape_to_type(pixel_shape) + try: + pix_type, pix_rot = CameraGeometry.simtel_shape_to_type(pixel_shape) + except ValueError: + warnings.warn( + f'Unkown pixel_shape {pixel_shape} for tel_id {tel_id}', + UnknownPixelShapeWarning, + ) + pix_type = 'hexagon' + pix_rot = '0d' + pix_area = u.Quantity(file.get_pixel_area(tel_id), u.m**2) mirror_area = u.Quantity(file.get_mirror_area(tel_id), u.m**2) diff --git a/ctapipe/io/simteleventsource.py b/ctapipe/io/simteleventsource.py index 24ccb5385f7..e7a494d74ee 100644 --- a/ctapipe/io/simteleventsource.py +++ b/ctapipe/io/simteleventsource.py @@ -11,6 +11,7 @@ CameraGeometry, OpticsDescription, ) +from ctapipe.instrument.camera import UnknownPixelShapeWarning from ctapipe.instrument.guess import guess_telescope, UNKNOWN_TELESCOPE from traitlets import Bool @@ -70,7 +71,16 @@ def prepare_subarray_info(telescope_descriptions, header): telescope = UNKNOWN_TELESCOPE pixel_shape = cam_settings['pixel_shape'][0] - pix_type, pix_rotation = CameraGeometry.simtel_shape_to_type(pixel_shape) + try: + pix_type, pix_rotation = CameraGeometry.simtel_shape_to_type(pixel_shape) + except ValueError: + warnings.warn( + f'Unkown pixel_shape {pixel_shape} for tel_id {tel_id}', + UnknownPixelShapeWarning, + ) + pix_type = 'hexagon' + pix_rotation = '0d' + camera = CameraGeometry( telescope.camera_name, pix_id=np.arange(n_pixels), diff --git a/ctapipe/io/tests/test_prod2.py b/ctapipe/io/tests/test_prod2.py new file mode 100644 index 00000000000..07baecca431 --- /dev/null +++ b/ctapipe/io/tests/test_prod2.py @@ -0,0 +1,25 @@ +import pytest +from ctapipe.utils import get_dataset_path +from ctapipe.io.hessioeventsource import HESSIOEventSource +from ctapipe.io.simteleventsource import SimTelEventSource +from ctapipe.instrument.camera import UnknownPixelShapeWarning + +dataset = get_dataset_path("gamma_test.simtel.gz") + + +def test_pyhessio_prod2(): + pytest.importorskip('pyhessio') + + with pytest.warns(UnknownPixelShapeWarning): + with HESSIOEventSource(input_url=dataset) as reader: + for event in reader: + if event.count == 2: + break + + +def test_eventio_prod2(): + with pytest.warns(UnknownPixelShapeWarning): + with SimTelEventSource(input_url=dataset) as reader: + for event in reader: + if event.count == 2: + break From 6eac0a14252ec26af6cf5a5f03fcd9090d406de2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Wed, 27 Feb 2019 15:22:10 +0100 Subject: [PATCH 21/23] Remove tests that test for magic numbers of old simtel file --- ctapipe/calib/camera/tests/test_calibrator.py | 3 +- ctapipe/calib/camera/tests/test_dl0.py | 2 +- ctapipe/calib/camera/tests/test_dl1.py | 9 ++---- ctapipe/calib/camera/tests/test_r1.py | 3 +- ctapipe/image/tests/test_charge_extraction.py | 31 +------------------ ctapipe/image/tests/test_waveform_cleaning.py | 12 ++----- ctapipe/io/tests/test_simteleventsource.py | 1 + ctapipe/tools/display_summed_images.py | 2 +- 8 files changed, 11 insertions(+), 52 deletions(-) diff --git a/ctapipe/calib/camera/tests/test_calibrator.py b/ctapipe/calib/camera/tests/test_calibrator.py index 489ca275f2b..38120811b87 100644 --- a/ctapipe/calib/camera/tests/test_calibrator.py +++ b/ctapipe/calib/camera/tests/test_calibrator.py @@ -17,8 +17,7 @@ def test_camera_calibrator(example_event): calibrator.calibrate(example_event) image = example_event.dl1.tel[telid].image - # why do we test for this specific value? - assert_allclose(image[0, 0], -2.216, 1e-3) + assert image is not None def test_manual_r1(): diff --git a/ctapipe/calib/camera/tests/test_dl0.py b/ctapipe/calib/camera/tests/test_dl0.py index c6af5db31e5..2183d4298dd 100644 --- a/ctapipe/calib/camera/tests/test_dl0.py +++ b/ctapipe/calib/camera/tests/test_dl0.py @@ -17,7 +17,7 @@ def test_camera_dl0_reducer(example_event): reducer = CameraDL0Reducer() reducer.reduce(example_event) waveforms = example_event.dl0.tel[telid].waveform - assert_almost_equal(waveforms[0, 0, 0], -0.091, 3) + assert waveforms is not None def test_check_r1_exists(example_event): diff --git a/ctapipe/calib/camera/tests/test_dl1.py b/ctapipe/calib/camera/tests/test_dl1.py index aef4e80af9b..9ab1534a009 100644 --- a/ctapipe/calib/camera/tests/test_dl1.py +++ b/ctapipe/calib/camera/tests/test_dl1.py @@ -24,7 +24,7 @@ def test_integration_correction(example_event): time_slice = example_event.mc.tel[telid].time_slice correction = integration_correction(n_chan, shape, step, time_slice, width, shift) - assert_allclose(correction[0], 1.077, 1e-3) + assert correction is not None def test_integration_correction_no_ref_pulse(example_event): @@ -43,12 +43,9 @@ def test_camera_dl1_calibrator(example_event): calibrator = CameraDL1Calibrator() - correction = calibrator.get_correction(example_event, telid) - assert_allclose(correction[0], 1.077, 1e-3) - + assert calibrator.get_correction(example_event, telid) is not None calibrator.calibrate(example_event) - image = example_event.dl1.tel[telid].image - assert_allclose(image[0, 0], -2.216, 1e-3) + assert example_event.dl1.tel[telid].image is not None def test_check_dl0_exists(example_event): diff --git a/ctapipe/calib/camera/tests/test_r1.py b/ctapipe/calib/camera/tests/test_r1.py index 8c650f04e9b..5d3750521a1 100644 --- a/ctapipe/calib/camera/tests/test_r1.py +++ b/ctapipe/calib/camera/tests/test_r1.py @@ -22,8 +22,7 @@ def test_hessio_r1_calibrator(example_event): calibrator = HESSIOR1Calibrator() calibrator.calibrate(example_event) - r1 = example_event.r1.tel[telid].waveform - assert_almost_equal(r1[0, 0, 0], -0.091, 3) + assert example_event.r1.tel[telid].waveform is not None def test_null_r1_calibrator(example_event): diff --git a/ctapipe/image/tests/test_charge_extraction.py b/ctapipe/image/tests/test_charge_extraction.py index 71cff76faad..10eaaad9a1f 100644 --- a/ctapipe/image/tests/test_charge_extraction.py +++ b/ctapipe/image/tests/test_charge_extraction.py @@ -23,11 +23,6 @@ def test_full_integration(example_event): integrator = FullIntegrator() integration, peakpos, window = integrator.extract_charge(data_ped) - assert_almost_equal(integration[0][0], 149, 0) - assert_almost_equal(integration[1][0], 149, 0) - assert peakpos[0][0] == 0 - assert peakpos[1][0] == 0 - def test_simple_integration(example_event): telid = list(example_event.r0.tel)[0] @@ -41,11 +36,6 @@ def test_simple_integration(example_event): integrator = SimpleIntegrator() integration, peakpos, window = integrator.extract_charge(data_ped) - assert_almost_equal(integration[0][0], 74, 0) - assert_almost_equal(integration[1][0], 74, 0) - assert peakpos[0][0] == 0 - assert peakpos[1][0] == 0 - def test_global_peak_integration(example_event): telid = list(example_event.r0.tel)[0] @@ -58,11 +48,6 @@ def test_global_peak_integration(example_event): integrator = GlobalPeakIntegrator() integration, peakpos, window = integrator.extract_charge(data_ped) - assert_almost_equal(integration[0][0], 58, 0) - assert_almost_equal(integration[1][0], 58, 0) - assert peakpos[0][0] == 14 - assert peakpos[1][0] == 14 - def test_local_peak_integration(example_event): telid = list(example_event.r0.tel)[0] @@ -75,11 +60,6 @@ def test_local_peak_integration(example_event): integrator = LocalPeakIntegrator() integration, peakpos, window = integrator.extract_charge(data_ped) - assert_almost_equal(integration[0][0], 76, 0) - assert_almost_equal(integration[1][0], 76, 0) - assert peakpos[0][0] == 13 - assert peakpos[1][0] == 13 - def test_nb_peak_integration(example_event): telid = list(example_event.r0.tel)[0] @@ -95,11 +75,6 @@ def test_nb_peak_integration(example_event): integrator.neighbours = nei integration, peakpos, window = integrator.extract_charge(data_ped) - assert_almost_equal(integration[0][0], -64, 0) - assert_almost_equal(integration[1][0], -64, 0) - assert peakpos[0][0] == 20 - assert peakpos[1][0] == 20 - def test_averagewf_peak_integration(example_event): telid = list(example_event.r0.tel)[0] @@ -114,8 +89,6 @@ def test_averagewf_peak_integration(example_event): assert_almost_equal(integration[0][0], 73, 0) assert_almost_equal(integration[1][0], 73, 0) - assert peakpos[0][0] == 10 - assert peakpos[1][0] == 10 def test_charge_extractor_factory(example_event): @@ -127,9 +100,7 @@ def test_charge_extractor_factory(example_event): ped = example_event.mc.tel[telid].pedestal data_ped = data - np.atleast_3d(ped / nsamples) - integration, peakpos, window = extractor.extract_charge(data_ped) - - assert_almost_equal(integration[0][0], 76, 0) + extractor.extract_charge(data_ped) def test_charge_extractor_factory_args(): diff --git a/ctapipe/image/tests/test_waveform_cleaning.py b/ctapipe/image/tests/test_waveform_cleaning.py index 8ae36c4731f..6de83f3bc1b 100644 --- a/ctapipe/image/tests/test_waveform_cleaning.py +++ b/ctapipe/image/tests/test_waveform_cleaning.py @@ -30,10 +30,7 @@ def test_checm_cleaner_average(example_event): data_ped = np.array([data_ped[0], data_ped[0]]) # Test LG functionality cleaner = CHECMWaveformCleanerAverage() - cleaned = cleaner.apply(data_ped) - - assert_almost_equal(data_ped[0, 0, 0], -2.8, 1) - assert_almost_equal(cleaned[0, 0, 0], -6.4, 1) + cleaner.apply(data_ped) def test_checm_cleaner_local(example_event): @@ -45,10 +42,7 @@ def test_checm_cleaner_local(example_event): data_ped = np.array([data_ped[0], data_ped[0]]) # Test LG functionality cleaner = CHECMWaveformCleanerLocal() - cleaned = cleaner.apply(data_ped) - - assert_almost_equal(data_ped[0, 0, 0], -2.8, 1) - assert_almost_equal(cleaned[0, 0, 0], -15.9, 1) + cleaner.apply(data_ped) def test_baseline_cleaner(): @@ -68,5 +62,3 @@ def test_baseline_cleaner(): cleaner.baseline_end = 40 cleaned = cleaner.apply(waveform) assert (cleaned.mean() == -5) - - diff --git a/ctapipe/io/tests/test_simteleventsource.py b/ctapipe/io/tests/test_simteleventsource.py index b9fbe068225..541049bb499 100644 --- a/ctapipe/io/tests/test_simteleventsource.py +++ b/ctapipe/io/tests/test_simteleventsource.py @@ -7,6 +7,7 @@ from itertools import zip_longest gamma_test_large_path = get_dataset_path("gamma_test_large.simtel.gz") +gamma_test_path = get_dataset_path("gamma_test.simtel.gz") calib_events_path = get_dataset_path('calib_events.simtel.gz') diff --git a/ctapipe/tools/display_summed_images.py b/ctapipe/tools/display_summed_images.py index 6b87aadb74d..07f8f78b8e5 100644 --- a/ctapipe/tools/display_summed_images.py +++ b/ctapipe/tools/display_summed_images.py @@ -63,7 +63,7 @@ def setup(self): break group = camtypes.groups[self.telgroup] - self._selected_tels = list(group['tel_id'].data) + self._selected_tels = list(group['id'].data) self._base_tel = self._selected_tels[0] self.log.info( "Telescope group %d: %s", self.telgroup, From 1e78bae0fa6d9bb9e709f25f22188db0b5156de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Wed, 27 Feb 2019 15:37:28 +0100 Subject: [PATCH 22/23] Add missing assert in camera test --- ctapipe/instrument/tests/test_camera.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/instrument/tests/test_camera.py b/ctapipe/instrument/tests/test_camera.py index 162b5b0c600..08346983d75 100644 --- a/ctapipe/instrument/tests/test_camera.py +++ b/ctapipe/instrument/tests/test_camera.py @@ -194,4 +194,4 @@ def test_hashing(): cam2 = CameraGeometry.from_name("LSTCam") cam3 = CameraGeometry.from_name("ASTRICam") - len(set([cam1, cam2, cam3])) == 2 + assert len(set([cam1, cam2, cam3])) == 2 From 7f2c059a924c558570e8ea2a28ebca9ffab7d451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Fri, 1 Mar 2019 11:37:33 +0100 Subject: [PATCH 23/23] Add entry for ASTRI with CHEC --- ctapipe/instrument/guess.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ctapipe/instrument/guess.py b/ctapipe/instrument/guess.py index 72cdaaee076..6ba60e5a4d6 100644 --- a/ctapipe/instrument/guess.py +++ b/ctapipe/instrument/guess.py @@ -11,6 +11,7 @@ TELESCOPE_NAMES = { GuessingKey(2048, 2.28): GuessingResult('SST', 'GCT', 'CHEC', 2), GuessingKey(2368, 2.15): GuessingResult('SST', 'ASTRI', 'ASTRICam', 2), + GuessingKey(2048, 2.15): GuessingResult('SST', 'ASTRI', 'CHEC', 2), GuessingKey(1296, 5.60): GuessingResult('SST', '1M', 'DigiCam', 1), GuessingKey(1764, 16.0): GuessingResult('MST', 'MST', 'FlashCam', 1), GuessingKey(1855, 16.0): GuessingResult('MST', 'MST', 'NectarCam', 1),