diff --git a/.travis.yml b/.travis.yml index 95bc16e96eb..1e36fbbea87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ install: - pip install codecov # ----- SST1M: - conda install protobuf - - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v0.44.5.tar.gz + - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.0.2.tar.gz # ----- end of SST1M # ----- target_software - conda install -c conda-forge cfitsio diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 192b3898c8b..36aa27efb85 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -23,6 +23,10 @@ 'TargetIOCameraContainer', 'SST1MContainer', 'SST1MCameraContainer', + 'LSTContainer', + 'LSTCameraContainer', + 'NectarCAMContainer', + 'NectarCAMCameraContainer', 'MCEventContainer', 'MCHeaderContainer', 'MCCameraEventContainer', @@ -440,6 +444,82 @@ class NectarCAMDataContainer(DataContainer): nectarcam = Field(NectarCAMContainer(), "NectarCAM Specific Information") +class LSTServiceContainer(Container): + """ + Container for Fields that are specific to each LST camera configuration + """ + + # Data from the CameraConfig table + telescope_id = Field(-1, "telescope id") + cs_serial = Field(None, "serial number of the camera server") + configuration_id = Field(None, "id of the CameraConfiguration") + date = Field(None, "NTP start of run date") + num_pixels = Field(-1, "number of pixels") + num_samples = Field(-1, "num samples") + pixel_ids = Field([], "id of the pixels in the waveform array") + data_model_version = Field(None, "data model version") + + idaq_version = Field(0o0, "idaq version") + cdhs_version = Field(0o0, "cdhs version") + algorithms = Field(None, "algorithms") + pre_proc_algorithms = Field(None, "pre processing algorithms") + module_ids = Field([], "module ids") + num_modules = Field(-1, "number of modules") + + +class LSTEventContainer(Container): + """ + Container for Fields that are specific to each LST event + """ + + # Data from the CameraEvent table + configuration_id = Field(None, "id of the CameraConfiguration") + event_id = Field(None, "local id of the event") + tel_event_id = Field(None, "global id of the event") + pixel_status = Field([], "status of the pixels") + ped_id = Field(None, "tel_event_id of the event used for pedestal substraction") + module_status = Field([], "status of the modules") + extdevices_presence = Field(None, "presence of data for external devices") + tib_data = Field([], "TIB data array") + cdts_data = Field([], "CDTS data array") + swat_data = Field([], "SWAT data array") + counters = Field([], "counters") + chips_flags = Field([], "chips flags") + first_capacitor_id = Field([], "first capacitor id") + drs_tag_status = Field([], "DRS tag status") + drs_tag = Field([], "DRS tag") + + +class LSTCameraContainer(Container): + """ + Container for Fields that are specific to each LST camera + """ + evt = Field(LSTEventContainer(), "LST specific event Information") + svc = Field(LSTServiceContainer(), "LST specific camera_config Information") + + + + +class LSTContainer(Container): + """ + Storage for the LSTCameraContainer for each telescope + """ + tels_with_data = Field([], "list of telescopes with data") + + # create the camera container + tel = Field( + Map(LSTCameraContainer), + "map of tel_id to LSTTelContainer") + + + +class LSTDataContainer(DataContainer): + """ + Data container including LST information + """ + lst = Field(LSTContainer(), "LST specific Information") + + class TargetIOCameraContainer(Container): """ Container for Fields that are specific to cameras that use TARGET diff --git a/ctapipe/io/eventsourcefactory.py b/ctapipe/io/eventsourcefactory.py index 26dd033ce6b..b1ae3ac90a8 100644 --- a/ctapipe/io/eventsourcefactory.py +++ b/ctapipe/io/eventsourcefactory.py @@ -6,6 +6,7 @@ import ctapipe.io.hessioeventsource from . import sst1meventsource from . import nectarcameventsource +from . import lsteventsource import ctapipe.io.targetioeventsource diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py new file mode 100644 index 00000000000..b71306e9dee --- /dev/null +++ b/ctapipe/io/lsteventsource.py @@ -0,0 +1,141 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +EventSource for LSTCam protobuf-fits.fz-files. + +Needs protozfits v1.02.0 from github.com/cta-sst-1m/protozfitsreader +""" + +import numpy as np +from .eventsource import EventSource +from .containers import LSTDataContainer + +__all__ = ['LSTEventSource'] + + +class LSTEventSource(EventSource): + + def __init__(self, config=None, tool=None, **kwargs): + super().__init__(config=config, tool=tool, **kwargs) + from protozfits import File + self.file = File(self.input_url) + self.camera_config = next(self.file.CameraConfig) + + + def _generator(self): + + # container for LST data + data = LSTDataContainer() + data.meta['input_url'] = self.input_url + + # fill LST data from the CameraConfig table + self.fill_lst_service_container_from_zfile(data.lst, self.camera_config) + + for count, event in enumerate(self.file.Events): + + + data.count = count + + # fill specific LST event data + self.fill_lst_event_container_from_zfile(data.lst, event) + + # fill general R0 data + self.fill_r0_container_from_zfile(data.r0, event) + yield data + + + @staticmethod + def is_compatible(file_path): + from astropy.io import fits + try: + # The file contains two tables: + # 1: CameraConfig + # 2: Events + h = fits.open(file_path)[2].header + ttypes = [ + h[x] for x in h.keys() if 'TTYPE' in x + ] + except OSError: + # not even a fits file + return False + + except IndexError: + # A fits file of a different format + return False + + is_protobuf_zfits_file = ( + (h['XTENSION'] == 'BINTABLE') and + (h['EXTNAME'] == 'Events') and + (h['ZTABLE'] is True) and + (h['ORIGIN'] == 'CTA') and + (h['PBFHEAD'] == 'R1.CameraEvent') + ) + + is_lst_file = 'lstcam_counters' in ttypes + return is_protobuf_zfits_file & is_lst_file + + def fill_lst_service_container_from_zfile(self, container, camera_config): + + container.tels_with_data = [camera_config.telescope_id, ] + + svc_container = container.tel[camera_config.telescope_id].svc + + svc_container.telescope_id = camera_config.telescope_id + svc_container.cs_serial = camera_config.cs_serial + svc_container.configuration_id = camera_config.configuration_id + svc_container.date = camera_config.date + svc_container.num_pixels = camera_config.num_pixels + svc_container.num_samples = camera_config.num_samples + svc_container.pixel_ids = camera_config.expected_pixels_id + svc_container.data_model_version = camera_config.data_model_version + + svc_container.num_modules = camera_config.lstcam.num_modules + svc_container.module_ids = camera_config.lstcam.expected_modules_id + svc_container.idaq_version = camera_config.lstcam.idaq_version + svc_container.cdhs_version = camera_config.lstcam.cdhs_version + svc_container.algorithms = camera_config.lstcam.algorithms + svc_container.pre_proc_algorithms = camera_config.lstcam.pre_proc_algorithms + + + + def fill_lst_event_container_from_zfile(self, container, event): + + event_container = container.tel[self.camera_config.telescope_id].evt + + event_container.configuration_id = event.configuration_id + event_container.event_id = event.event_id + event_container.tel_event_id = event.tel_event_id + event_container.pixel_status = event.pixel_status + event_container.ped_id = event.ped_id + event_container.module_status = event.lstcam.module_status + event_container.extdevices_presence = event.lstcam.extdevices_presence + event_container.tib_data = event.lstcam.tib_data + event_container.cdts_data = event.lstcam.cdts_data + event_container.swat_data = event.lstcam.swat_data + event_container.counters = event.lstcam.counters + event_container.chips_flags = event.lstcam.chips_flags + event_container.first_capacitor_id = event.lstcam.first_capacitor_id + event_container.drs_tag_status = event.lstcam.drs_tag_status + event_container.drs_tag = event.lstcam.drs_tag + + def fill_r0_camera_container_from_zfile(self, container, event): + + container.num_samples = self.camera_config.num_samples + container.trigger_time = event.trigger_time_s + container.trigger_type = event.trigger_type + + container.waveform = np.array( + ( + event.waveform + ).reshape(2, self.camera_config.num_pixels, container.num_samples)) + + + def fill_r0_container_from_zfile(self, container, event): + container.obs_id = -1 + container.event_id = event.event_id + + container.tels_with_data = [self.camera_config.telescope_id, ] + r0_camera_container = container.tel[self.camera_config.telescope_id] + self.fill_r0_camera_container_from_zfile( + r0_camera_container, + event + ) diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index f64f5bac90a..06bf44d3029 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -2,7 +2,7 @@ """ EventSource for NectarCam protobuf-fits.fz-files. -Needs protozfits v0.44.5 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.02.0 from github.com/cta-sst-1m/protozfitsreader """ import numpy as np @@ -16,8 +16,8 @@ class NectarCAMEventSource(EventSource): def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) - from protozfits import SimpleFile - self.file = SimpleFile(self.input_url) + from protozfits import File + self.file = File(self.input_url) self.header = next(self.file.RunHeader) diff --git a/ctapipe/io/sst1meventsource.py b/ctapipe/io/sst1meventsource.py index ea4c80d67d7..35c8a7a99de 100644 --- a/ctapipe/io/sst1meventsource.py +++ b/ctapipe/io/sst1meventsource.py @@ -2,7 +2,7 @@ """ EventSource for SST1M/digicam protobuf-fits.fz-files. -Needs protozfits v0.44.3 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.0.2 from github.com/cta-sst-1m/protozfitsreader """ import numpy as np from .eventsource import EventSource @@ -16,8 +16,8 @@ class SST1MEventSource(EventSource): def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) - from protozfits import SimpleFile - self.file = SimpleFile(self.input_url) + from protozfits import File + self.file = File(self.input_url) # TODO: Correct pixel ordering self._tel_desc = TelescopeDescription.from_name( optics_name='SST-1M', diff --git a/ctapipe/io/tests/test_lsteventsource.py b/ctapipe/io/tests/test_lsteventsource.py new file mode 100644 index 00000000000..51b68da56da --- /dev/null +++ b/ctapipe/io/tests/test_lsteventsource.py @@ -0,0 +1,55 @@ +from pkg_resources import resource_filename +import os + +import pytest +pytest.importorskip("protozfits", minversion="1.0.2") + +example_file_path = resource_filename( + 'protozfits', + os.path.join( + 'tests', + 'resources', + 'example_LST_R1_10_evts.fits.fz' + ) +) + +FIRST_EVENT_NUMBER_IN_FILE = 1 +# ADC_SAMPLES_SHAPE = (2, 14, 40) + + +def test_loop_over_events(): + from ctapipe.io.lsteventsource import LSTEventSource + + n_events = 10 + inputfile_reader = LSTEventSource( + input_url=example_file_path, + max_events=n_events + ) + + for i, event in enumerate(inputfile_reader): + assert event.r0.tels_with_data == [0] + for telid in event.r0.tels_with_data: + assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i + n_gain = 2 + num_pixels = event.lst.tel[telid].svc.num_pixels + num_samples = event.lst.tel[telid].svc.num_samples + waveform_shape = (n_gain, num_pixels, num_samples) + assert event.r0.tel[telid].waveform.shape == waveform_shape + + # make sure max_events works + assert i == n_events - 1 + + +def test_is_compatible(): + from ctapipe.io.lsteventsource import LSTEventSource + + assert LSTEventSource.is_compatible(example_file_path) + + +def test_factory_for_lst_file(): + from ctapipe.io.eventsourcefactory import EventSourceFactory + from ctapipe.io.lsteventsource import LSTEventSource + + reader = EventSourceFactory.produce(input_url=example_file_path) + assert isinstance(reader, LSTEventSource) + assert reader.input_url == example_file_path