Lst reader (#749)

* LST data reader code

* Add LST reader files

* Complete LST reader

* Improved container organisation: introduced EVT and SVT concept for LSTCameraContainer (Fields: LSTEventContainer and LSTServiceContainer)

* Updated request for protozfitsreader library from release 0.44.5 to 1.0.2

* Change of import from module SimpleFile to module File to be compatible with version 1.0.2 of the protozfit library

.travis.yml

@@ -54,7 +54,7 @@ install:
     - pip install codecov
     # ----- SST1M:
     - conda install protobuf
-    - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v0.44.5.tar.gz
+    - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.0.2.tar.gz
     # ----- end of SST1M
     # ----- target_software
     - conda install -c conda-forge cfitsio

ctapipe/io/containers.py

@@ -23,6 +23,10 @@
            'TargetIOCameraContainer',
            'SST1MContainer',
            'SST1MCameraContainer',
+           'LSTContainer',
+           'LSTCameraContainer',
+           'NectarCAMContainer',
+           'NectarCAMCameraContainer',
            'MCEventContainer',
            'MCHeaderContainer',
            'MCCameraEventContainer',
@@ -440,6 +444,82 @@ class NectarCAMDataContainer(DataContainer):
     nectarcam = Field(NectarCAMContainer(), "NectarCAM Specific Information")
 
 
+class LSTServiceContainer(Container):
+    """
+    Container for Fields that are specific to each LST camera configuration
+    """
+
+    # Data from the CameraConfig table
+    telescope_id = Field(-1, "telescope id")
+    cs_serial = Field(None, "serial number of the camera server")
+    configuration_id = Field(None, "id of the CameraConfiguration")
+    date = Field(None, "NTP start of run date")
+    num_pixels = Field(-1, "number of pixels")
+    num_samples = Field(-1, "num samples")
+    pixel_ids = Field([], "id of the pixels in the waveform array")
+    data_model_version = Field(None, "data model version")
+
+    idaq_version = Field(0o0, "idaq version")
+    cdhs_version = Field(0o0, "cdhs version")
+    algorithms = Field(None, "algorithms")
+    pre_proc_algorithms = Field(None, "pre processing algorithms")
+    module_ids = Field([], "module ids")
+    num_modules = Field(-1, "number of modules")
+
+
+class LSTEventContainer(Container):
+    """
+    Container for Fields that are specific to each LST event
+    """
+
+    # Data from the CameraEvent table
+    configuration_id = Field(None, "id of the CameraConfiguration")
+    event_id = Field(None, "local id of the event")
+    tel_event_id = Field(None, "global id of the event")
+    pixel_status = Field([], "status of the pixels")
+    ped_id = Field(None, "tel_event_id of the event used for pedestal substraction")
+    module_status = Field([], "status of the modules")
+    extdevices_presence = Field(None, "presence of data for external devices")
+    tib_data = Field([], "TIB data array")
+    cdts_data = Field([], "CDTS data array")
+    swat_data = Field([], "SWAT data array")
+    counters = Field([], "counters")
+    chips_flags = Field([], "chips flags")
+    first_capacitor_id = Field([], "first capacitor id")
+    drs_tag_status = Field([], "DRS tag status")
+    drs_tag = Field([], "DRS tag")
+
+
+class LSTCameraContainer(Container):
+    """
+    Container for Fields that are specific to each LST camera
+    """
+    evt = Field(LSTEventContainer(), "LST specific event Information")
+    svc = Field(LSTServiceContainer(), "LST specific camera_config Information")
+
+
+
+
+class LSTContainer(Container):
+    """
+    Storage for the LSTCameraContainer for each telescope
+    """
+    tels_with_data = Field([], "list of telescopes with data")
+
+    # create the camera container
+    tel = Field(
+        Map(LSTCameraContainer),
+        "map of tel_id to LSTTelContainer")
+
+
+
+class LSTDataContainer(DataContainer):
+    """
+    Data container including LST information
+    """
+    lst = Field(LSTContainer(), "LST specific Information")
+
+
 class TargetIOCameraContainer(Container):
     """
     Container for Fields that are specific to cameras that use TARGET

ctapipe/io/eventsourcefactory.py

@@ -6,6 +6,7 @@
 import ctapipe.io.hessioeventsource
 from . import sst1meventsource
 from . import nectarcameventsource
+from . import lsteventsource
 import ctapipe.io.targetioeventsource
 
 

ctapipe/io/lsteventsource.py

@@ -0,0 +1,141 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+"""
+EventSource for LSTCam protobuf-fits.fz-files.
+
+Needs protozfits v1.02.0 from github.com/cta-sst-1m/protozfitsreader
+"""
+
+import numpy as np
+from .eventsource import EventSource
+from .containers import LSTDataContainer
+
+__all__ = ['LSTEventSource']
+
+
+class LSTEventSource(EventSource):
+
+    def __init__(self, config=None, tool=None, **kwargs):
+        super().__init__(config=config, tool=tool, **kwargs)
+        from protozfits import File
+        self.file = File(self.input_url)
+        self.camera_config = next(self.file.CameraConfig)
+
+
+    def _generator(self):
+
+        # container for LST data
+        data = LSTDataContainer()
+        data.meta['input_url'] = self.input_url
+
+        # fill LST data from the CameraConfig table
+        self.fill_lst_service_container_from_zfile(data.lst, self.camera_config)
+
+        for count, event in enumerate(self.file.Events):
+
+
+            data.count = count
+
+            # fill specific LST event data
+            self.fill_lst_event_container_from_zfile(data.lst, event)
+
+            # fill general R0 data
+            self.fill_r0_container_from_zfile(data.r0, event)
+            yield data
+
+
+    @staticmethod
+    def is_compatible(file_path):
+        from astropy.io import fits
+        try:
+            # The file contains two tables:
+            #  1: CameraConfig
+            #  2: Events
+            h = fits.open(file_path)[2].header
+            ttypes = [
+                h[x] for x in h.keys() if 'TTYPE' in x
+            ]
+        except OSError:
+            # not even a fits file
+            return False
+
+        except IndexError:
+            # A fits file of a different format
+            return False
+
+        is_protobuf_zfits_file = (
+            (h['XTENSION'] == 'BINTABLE') and
+            (h['EXTNAME'] == 'Events') and
+            (h['ZTABLE'] is True) and
+            (h['ORIGIN'] == 'CTA') and
+            (h['PBFHEAD'] == 'R1.CameraEvent')
+        )
+
+        is_lst_file = 'lstcam_counters' in ttypes
+        return is_protobuf_zfits_file & is_lst_file
+
+    def fill_lst_service_container_from_zfile(self, container, camera_config):
+
+        container.tels_with_data = [camera_config.telescope_id, ]
+
+        svc_container = container.tel[camera_config.telescope_id].svc
+
+        svc_container.telescope_id = camera_config.telescope_id
+        svc_container.cs_serial = camera_config.cs_serial
+        svc_container.configuration_id = camera_config.configuration_id
+        svc_container.date = camera_config.date
+        svc_container.num_pixels = camera_config.num_pixels
+        svc_container.num_samples = camera_config.num_samples
+        svc_container.pixel_ids = camera_config.expected_pixels_id
+        svc_container.data_model_version = camera_config.data_model_version
+
+        svc_container.num_modules = camera_config.lstcam.num_modules
+        svc_container.module_ids = camera_config.lstcam.expected_modules_id
+        svc_container.idaq_version = camera_config.lstcam.idaq_version
+        svc_container.cdhs_version = camera_config.lstcam.cdhs_version
+        svc_container.algorithms = camera_config.lstcam.algorithms
+        svc_container.pre_proc_algorithms = camera_config.lstcam.pre_proc_algorithms
+
+
+
+    def fill_lst_event_container_from_zfile(self, container, event):
+
+        event_container = container.tel[self.camera_config.telescope_id].evt
+
+        event_container.configuration_id = event.configuration_id
+        event_container.event_id = event.event_id
+        event_container.tel_event_id = event.tel_event_id
+        event_container.pixel_status = event.pixel_status
+        event_container.ped_id = event.ped_id
+        event_container.module_status = event.lstcam.module_status
+        event_container.extdevices_presence = event.lstcam.extdevices_presence
+        event_container.tib_data = event.lstcam.tib_data
+        event_container.cdts_data = event.lstcam.cdts_data
+        event_container.swat_data = event.lstcam.swat_data
+        event_container.counters = event.lstcam.counters
+        event_container.chips_flags = event.lstcam.chips_flags
+        event_container.first_capacitor_id = event.lstcam.first_capacitor_id
+        event_container.drs_tag_status = event.lstcam.drs_tag_status
+        event_container.drs_tag = event.lstcam.drs_tag
+
+    def fill_r0_camera_container_from_zfile(self, container, event):
+
+        container.num_samples = self.camera_config.num_samples
+        container.trigger_time = event.trigger_time_s
+        container.trigger_type = event.trigger_type
+
+        container.waveform = np.array(
+            (
+                event.waveform
+            ).reshape(2, self.camera_config.num_pixels, container.num_samples))
+
+
+    def fill_r0_container_from_zfile(self, container, event):
+        container.obs_id = -1
+        container.event_id = event.event_id
+
+        container.tels_with_data = [self.camera_config.telescope_id, ]
+        r0_camera_container = container.tel[self.camera_config.telescope_id]
+        self.fill_r0_camera_container_from_zfile(
+            r0_camera_container,
+            event
+        )

ctapipe/io/nectarcameventsource.py

@@ -2,7 +2,7 @@
 """
 EventSource for NectarCam protobuf-fits.fz-files.
 
-Needs protozfits v0.44.5 from github.com/cta-sst-1m/protozfitsreader
+Needs protozfits v1.02.0 from github.com/cta-sst-1m/protozfitsreader
 """
 
 import numpy as np
@@ -16,8 +16,8 @@ class NectarCAMEventSource(EventSource):
 
     def __init__(self, config=None, tool=None, **kwargs):
         super().__init__(config=config, tool=tool, **kwargs)
-        from protozfits import SimpleFile
-        self.file = SimpleFile(self.input_url)
+        from protozfits import File
+        self.file = File(self.input_url)
         self.header = next(self.file.RunHeader)
 
 

ctapipe/io/sst1meventsource.py

@@ -2,7 +2,7 @@
 """
 EventSource for SST1M/digicam protobuf-fits.fz-files.
 
-Needs protozfits v0.44.3 from github.com/cta-sst-1m/protozfitsreader
+Needs protozfits v1.0.2 from github.com/cta-sst-1m/protozfitsreader
 """
 import numpy as np
 from .eventsource import EventSource
@@ -16,8 +16,8 @@ class SST1MEventSource(EventSource):
 
     def __init__(self, config=None, tool=None, **kwargs):
         super().__init__(config=config, tool=tool, **kwargs)
-        from protozfits import SimpleFile
-        self.file = SimpleFile(self.input_url)
+        from protozfits import File
+        self.file = File(self.input_url)
         # TODO: Correct pixel ordering
         self._tel_desc = TelescopeDescription.from_name(
             optics_name='SST-1M',

ctapipe/io/tests/test_lsteventsource.py

@@ -0,0 +1,55 @@
+from pkg_resources import resource_filename
+import os
+
+import pytest
+pytest.importorskip("protozfits", minversion="1.0.2")
+
+example_file_path = resource_filename(
+    'protozfits',
+    os.path.join(
+        'tests',
+        'resources',
+        'example_LST_R1_10_evts.fits.fz'
+    )
+)
+
+FIRST_EVENT_NUMBER_IN_FILE = 1
+# ADC_SAMPLES_SHAPE = (2, 14, 40)
+
+
+def test_loop_over_events():
+    from ctapipe.io.lsteventsource import LSTEventSource
+
+    n_events = 10
+    inputfile_reader = LSTEventSource(
+        input_url=example_file_path,
+        max_events=n_events
+    )
+
+    for i, event in enumerate(inputfile_reader):
+        assert event.r0.tels_with_data == [0]
+        for telid in event.r0.tels_with_data:
+            assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i
+            n_gain = 2
+            num_pixels = event.lst.tel[telid].svc.num_pixels
+            num_samples = event.lst.tel[telid].svc.num_samples
+            waveform_shape = (n_gain, num_pixels, num_samples)
+            assert event.r0.tel[telid].waveform.shape == waveform_shape
+
+    # make sure max_events works
+    assert i == n_events - 1
+
+
+def test_is_compatible():
+    from ctapipe.io.lsteventsource import LSTEventSource
+
+    assert LSTEventSource.is_compatible(example_file_path)
+
+
+def test_factory_for_lst_file():
+    from ctapipe.io.eventsourcefactory import EventSourceFactory
+    from ctapipe.io.lsteventsource import LSTEventSource
+
+    reader = EventSourceFactory.produce(input_url=example_file_path)
+    assert isinstance(reader, LSTEventSource)
+    assert reader.input_url == example_file_path