From f1b80967162544f188f940b95f44f07139d7a44f Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 28 Sep 2017 12:47:28 +0200 Subject: [PATCH 01/82] New directory tree for the atmosphere calibration code --- ctapipe/atmosphere/__init__.py | 0 ctapipe/atmosphere/fram/__init__.py | 0 ctapipe/atmosphere/gdas/__init__.py | 0 ctapipe/atmosphere/lidar/__init__.py | 0 docs/atmosphere/index.rst | 47 ++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+) create mode 100644 ctapipe/atmosphere/__init__.py create mode 100644 ctapipe/atmosphere/fram/__init__.py create mode 100644 ctapipe/atmosphere/gdas/__init__.py create mode 100644 ctapipe/atmosphere/lidar/__init__.py create mode 100644 docs/atmosphere/index.rst diff --git a/ctapipe/atmosphere/__init__.py b/ctapipe/atmosphere/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ctapipe/atmosphere/fram/__init__.py b/ctapipe/atmosphere/fram/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ctapipe/atmosphere/gdas/__init__.py b/ctapipe/atmosphere/gdas/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ctapipe/atmosphere/lidar/__init__.py b/ctapipe/atmosphere/lidar/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/atmosphere/index.rst b/docs/atmosphere/index.rst new file mode 100644 index 00000000000..776cacbca37 --- /dev/null +++ b/docs/atmosphere/index.rst @@ -0,0 +1,47 @@ +.. _atmosphere: + +====================== + Atmosphere (`atmosphere`) +====================== + +.. currentmodule:: ctapipe.atmosphere + + +Introduction +============ + +This module include all the functions and classes needed for the monitoring +of the atmosphere above the CTA sites. + +It consists in three main sub-modules: + +* Lidar Calibration + +* Fram Calibration + +* GDAS Calibration + +For more information on where you should implement your code, please have a look to the README.rst files inside each directory. + + +Getting Started +=============== + +TODO: add examples. + +Submodules +========== + +.. toctree:: + :maxdepth: 1 + :glob: + + index_* + + + +Reference/API +============= + +.. automodapi:: ctapipe.atmosphere + :no-inheritance-diagram: From 66346d14ef26fefa1b7e7dd3bfe29d10178190f4 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 20 Oct 2017 17:51:48 +0200 Subject: [PATCH 02/82] Fix doc error --- docs/atmosphere/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/atmosphere/index.rst b/docs/atmosphere/index.rst index 776cacbca37..69b1e98e943 100644 --- a/docs/atmosphere/index.rst +++ b/docs/atmosphere/index.rst @@ -1,8 +1,8 @@ .. _atmosphere: -====================== +========================== Atmosphere (`atmosphere`) -====================== +========================== .. currentmodule:: ctapipe.atmosphere From 90f77fb732434c0c0ad36fd3d531a33590c0093f Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 11 May 2018 16:42:12 +0200 Subject: [PATCH 03/82] LST data reader code --- ctapipe/io/containers.py | 56 ++++++++++++++++++++++++++++++++ ctapipe/io/eventsourcefactory.py | 1 + 2 files changed, 57 insertions(+) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 192b3898c8b..13b464c481b 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -439,6 +439,62 @@ class NectarCAMDataContainer(DataContainer): """ nectarcam = Field(NectarCAMContainer(), "NectarCAM Specific Information") +class LSTCameraContainer(Container): + """ + Container for Fields that are specific to camera that use zfit + """ + camera_event_type = Field(int, "camera event type") + + + integrals = Field(None, ( + "numpy array containing waveform integrals" + "(n_channels x n_pixels)" + )) + + + def fill_from_zfile_event(self, event, num_samples): + self.camera_event_type = event.eventType + + self.integrals = np.array([ + event.hiGain.integrals.gains, + event.loGain.integrals.gains, + ]) + + + +class LSTContainer(Container): + """ + Storage for the LSTCameraContainer for each telescope + """ + tels_with_data = Field([], "list of telescopes with data") + tel = Field( + Map(LSTCameraContainer), + "map of tel_id to LSTContainer") + + def fill_from_zfile_event(self, event, num_samples): + self.tels_with_data = [event.telescope_id, ] + lst_cam_container = self.tel[event.telescope_id] + lst_cam_container.fill_from_zfile_event( + event, + num_samples, + ) + + +class LSTDataContainer(DataContainer): + """ + Data container including LST information + """ + lst = Field(LSTContainer(), "LST Specific Information") + + + +class TargetIOCameraContainer(Container): + """ + Container for Fields that are specific to cameras that use TARGET + """ + first_cell_ids = Field(None, ("numpy array of the first_cell_id of each" + "waveform in the camera image (n_pixels)")) + class TargetIOCameraContainer(Container): """ diff --git a/ctapipe/io/eventsourcefactory.py b/ctapipe/io/eventsourcefactory.py index 26dd033ce6b..ebd4cac9c87 100644 --- a/ctapipe/io/eventsourcefactory.py +++ b/ctapipe/io/eventsourcefactory.py @@ -6,6 +6,7 @@ import ctapipe.io.hessioeventsource from . import sst1meventsource from . import nectarcameventsource +from . import lstcameventsource import ctapipe.io.targetioeventsource From 599b72ed89bf365a5b31500a61c1b7ed1ca7fd8a Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 11 May 2018 16:45:59 +0200 Subject: [PATCH 04/82] Add LST reader files --- ctapipe/io/lsteventsource.py | 94 +++++++++++++++++++++++++ ctapipe/io/tests/test_lsteventsource.py | 51 ++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 ctapipe/io/lsteventsource.py create mode 100644 ctapipe/io/tests/test_lsteventsource.py diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py new file mode 100644 index 00000000000..35fe1ace2f2 --- /dev/null +++ b/ctapipe/io/lsteventsource.py @@ -0,0 +1,94 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +EventSource for NectarCam protobuf-fits.fz-files. + +Needs protozfits v0.44.5 from github.com/cta-sst-1m/protozfitsreader +""" + +import numpy as np +from .eventsource import EventSource +from .containers import LSTDataContainer + +__all__ = ['LSTEventSource'] + + +class LSTEventSource(EventSource): + + def __init__(self, config=None, tool=None, **kwargs): + super().__init__(config=config, tool=tool, **kwargs) + from protozfits import SimpleFile + self.file = SimpleFile(self.input_url) + self.header = next(self.file.CameraConfig) + + + def _generator(self): + + self._pixel_sort_ids = None + + for count, event in enumerate(self.file.Events): + data = LSTDataContainer() + data.count = count + # fill specific LST data + data.lst.fill_from_zfile_event(event, self.header.num_samples) + # fill general R0 data + self.fill_R0Container_from_zfile_event(data.r0, event) + yield data + + + @staticmethod + def is_compatible(file_path): + from astropy.io import fits + try: + # The file contains two tables: + # 1: CameraConfig + # 2: Events <--- this is what we need to look at + h = fits.open(file_path)[2].header + ttypes = [ + h[x] for x in h.keys() if 'TTYPE' in x + ] + except OSError: + # not even a fits file + return False + + except IndexError: + # A fits file of a different format + return False + + is_protobuf_zfits_file = ( + (h['XTENSION'] == 'BINTABLE') and + (h['EXTNAME'] == 'Events') and + (h['ZTABLE'] is True) and + (h['ORIGIN'] == 'CTA') and + (h['PBFHEAD'] == 'DataModel.CameraEvent') + ) + + is_lst_file = 'lstcam_idaq_version' in ttypes + return is_protobuf_zfits_file & is_lst_file + + + def fill_R0CameraContainer_from_zfile_event(self, container, event): + container.trigger_time = ( + event.local_time_sec * 1E9 + event.local_time_nanosec) + container.trigger_type = event.event_type + + container.waveform = np.array([ + ( + event.hiGain.waveforms.samples + ).reshape(-1, self.header.numTraces), + ( + event.loGain.waveforms.samples + ).reshape(-1, self.header.numTraces) + ]) + + container.num_samples = container.waveform.shape[1] + + def fill_R0Container_from_zfile_event(self, container, event): + container.obs_id = -1 + container.event_id = event.event_id + + container.tels_with_data = [self.header.telescope_id, ] + r0_cam_container = container.tel[self.header.telescope_id] + self.fill_R0CameraContainer_from_zfile_event( + r0_cam_container, + event + ) diff --git a/ctapipe/io/tests/test_lsteventsource.py b/ctapipe/io/tests/test_lsteventsource.py new file mode 100644 index 00000000000..f40aecfc8fa --- /dev/null +++ b/ctapipe/io/tests/test_lsteventsource.py @@ -0,0 +1,51 @@ +from pkg_resources import resource_filename +import os + +import pytest +pytest.importorskip("protozfits", minversion="0.44.5") + +example_file_path = resource_filename( + 'protozfits', + os.path.join( + 'tests', + 'resources', + 'example_9evts_NectarCAM.fits.fz' + ) +) + +FIRST_EVENT_NUMBER_IN_FILE = 1 +ADC_SAMPLES_SHAPE = (2, 84, 60) + + +def test_loop_over_events(): + from ctapipe.io.lsteventsource import LSTEventSource + + N_EVENTS = 3 + inputfile_reader = LSTEventSource( + input_url=example_file_path, + max_events=N_EVENTS + ) + + for i, event in enumerate(inputfile_reader): + assert event.r0.tels_with_data == [0] + for telid in event.r0.tels_with_data: + assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i + assert event.r0.tel[telid].waveform.shape == ADC_SAMPLES_SHAPE + + # make sure max_events works + assert i == N_EVENTS - 1 + + +def test_is_compatible(): + from ctapipe.io.lasteventsource import LSTEventSource + + assert LSTEventSource.is_compatible(example_file_path) + + +def test_factory_for_nectarcam_file(): + from ctapipe.io.eventsourcefactory import EventSourceFactory + from ctapipe.io.lsteventsource import LSTEventSource + + reader = EventSourceFactory.produce(input_url=example_file_path) + assert isinstance(reader, LSTEventSource) + assert reader.input_url == example_file_path From 6fc6a8d3daf4212c4e6a6553beae22e534b9b27a Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 24 May 2018 13:36:04 +0200 Subject: [PATCH 05/82] Complete LST reader --- ctapipe/io/containers.py | 100 +++++++++++++++++------- ctapipe/io/eventsourcefactory.py | 2 +- ctapipe/io/lsteventsource.py | 58 +++++++------- ctapipe/io/nectarcameventsource.py | 6 +- ctapipe/io/tests/test_lsteventsource.py | 12 +-- 5 files changed, 114 insertions(+), 64 deletions(-) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 13b464c481b..a3e79cabf6a 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -23,6 +23,10 @@ 'TargetIOCameraContainer', 'SST1MContainer', 'SST1MCameraContainer', + 'LSTContainer', + 'LSTCameraContainer', + 'NectarCAMContainer', + 'NectarCAMCameraContainer', 'MCEventContainer', 'MCHeaderContainer', 'MCCameraEventContainer', @@ -441,25 +445,68 @@ class NectarCAMDataContainer(DataContainer): class LSTCameraContainer(Container): """ - Container for Fields that are specific to camera that use zfit - """ - camera_event_type = Field(int, "camera event type") - - - integrals = Field(None, ( - "numpy array containing waveform integrals" - "(n_channels x n_pixels)" - )) - - - def fill_from_zfile_event(self, event, num_samples): - self.camera_event_type = event.eventType - - self.integrals = np.array([ - event.hiGain.integrals.gains, - event.loGain.integrals.gains, - ]) - + Container for Fields that are specific to each LST camera + """ + + # Data from the CameraConfig table + telescope_id = Field(-1, "telescope id") + cs_serial=Field(None, "serial of the camera server") + configuration_id = Field(None, "?") + date= Field(None, "NTP start of run date") + num_pixels = Field(-1, "number of pixels") + num_samples= Field(-1, "num samples") + pixel_ids=Field([],"pixels id") + data_model_version=Field(None,"data model version") + + idaq_version = Field(0o0, "idaq version") + cdhs_version = Field(0o0, "cdhs version") + algorithms = Field(None, "algorithms") + pre_proc_algorithms = Field(None, "pre processing algorithms") + module_ids = Field([], "module ids") + num_modules= Field(-1, "number of modules") + + # Data from the CameraEvent table + pixel_status = Field([],"status of the pixels") + module_status = Field([], "status of the modules") + extdevices_presence = Field(None, "presence of data for external devices") + tib_data = Field([], "TIB data array") + cdts_data = Field([], "CDTS data array") + swat_data = Field([], "SWAT data array") + counters = Field([], "counters") + chips_flags = Field([], "chips flags") + first_capacitor_id = Field([], "first capacitor id") + drs_tag_status = Field([], "DRS tag status") + drs_tag = Field([], "DRS tag") + + # Fill data LST specifics + def fill_from_zfile(self, header, event): + self.telescope_id=header.telescope_id + self.cs_serial=header.cs_serial + self.configuration_id=header.configuration_id + self.date=header.date + self.num_pixels = header.num_pixels + self.num_samples=header.num_samples + self.pixel_ids=header.expected_pixels_id + self.data_model_version= header.data_model_version + + self.num_modules = header.lstcam.num_modules + self.module_ids = header.lstcam.expected_modules_id + self.idaq_version = header.lstcam.idaq_version + self.cdhs_version = header.lstcam.cdhs_version + self.algorithms = header.lstcam.algorithms + self.pre_proc_algorithms = header.lstcam.pre_proc_algorithms + + self.pixel_status = event.pixel_status + self.module_status = event.lstcam.module_status + self.extdevices_presence = event.lstcam.extdevices_presence + self.tib_data=event.lstcam.tib_data + self.cdts_data=event.lstcam.cdts_data + self.swat_data=event.lstcam.swat_data + self.counters=event.lstcam.counters + self.chips_flags = event.lstcam.chips_flags + self.first_capacitor_id = event.lstcam.first_capacitor_id + self.drs_tag_status = event.lstcam.drs_tag_status + self.drs_tag = event.lstcam.drs_tag class LSTContainer(Container): @@ -467,17 +514,16 @@ class LSTContainer(Container): Storage for the LSTCameraContainer for each telescope """ tels_with_data = Field([], "list of telescopes with data") + + # create the camera container tel = Field( Map(LSTCameraContainer), - "map of tel_id to LSTContainer") + "map of tel_id to LSTCameraContainer") - def fill_from_zfile_event(self, event, num_samples): - self.tels_with_data = [event.telescope_id, ] - lst_cam_container = self.tel[event.telescope_id] - lst_cam_container.fill_from_zfile_event( - event, - num_samples, - ) + def fill_from_zfile(self, header,event): + self.tels_with_data = [header.telescope_id, ] + lst_camera = self.tel[header.telescope_id] + lst_camera.fill_from_zfile(header, event) class LSTDataContainer(DataContainer): diff --git a/ctapipe/io/eventsourcefactory.py b/ctapipe/io/eventsourcefactory.py index ebd4cac9c87..b1ae3ac90a8 100644 --- a/ctapipe/io/eventsourcefactory.py +++ b/ctapipe/io/eventsourcefactory.py @@ -6,7 +6,7 @@ import ctapipe.io.hessioeventsource from . import sst1meventsource from . import nectarcameventsource -from . import lstcameventsource +from . import lsteventsource import ctapipe.io.targetioeventsource diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 35fe1ace2f2..c5b4c892b89 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -1,8 +1,8 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ -EventSource for NectarCam protobuf-fits.fz-files. +EventSource for LSTCam protobuf-fits.fz-files. -Needs protozfits v0.44.5 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.02.0 from github.com/cta-sst-1m/protozfitsreader """ import numpy as np @@ -16,22 +16,27 @@ class LSTEventSource(EventSource): def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) - from protozfits import SimpleFile - self.file = SimpleFile(self.input_url) + from protozfits import File + self.file = File(self.input_url) self.header = next(self.file.CameraConfig) def _generator(self): - self._pixel_sort_ids = None + # container for LST data + data = LSTDataContainer() + data.meta['input_url'] = self.input_url for count, event in enumerate(self.file.Events): - data = LSTDataContainer() + + data.count = count + # fill specific LST data - data.lst.fill_from_zfile_event(event, self.header.num_samples) + data.lst.fill_from_zfile(self.header,event) + # fill general R0 data - self.fill_R0Container_from_zfile_event(data.r0, event) + self.fill_R0Container_from_zfile(data.r0, event) yield data @@ -40,7 +45,7 @@ def is_compatible(file_path): from astropy.io import fits try: # The file contains two tables: - # 1: CameraConfig + # 1: CameraConfiguration # 2: Events <--- this is what we need to look at h = fits.open(file_path)[2].header ttypes = [ @@ -59,36 +64,35 @@ def is_compatible(file_path): (h['EXTNAME'] == 'Events') and (h['ZTABLE'] is True) and (h['ORIGIN'] == 'CTA') and - (h['PBFHEAD'] == 'DataModel.CameraEvent') + (h['PBFHEAD'] == 'R1.CameraEvent') ) - is_lst_file = 'lstcam_idaq_version' in ttypes + is_lst_file = 'lstcam_counters' in ttypes return is_protobuf_zfits_file & is_lst_file - def fill_R0CameraContainer_from_zfile_event(self, container, event): - container.trigger_time = ( - event.local_time_sec * 1E9 + event.local_time_nanosec) - container.trigger_type = event.event_type + def fill_R0CameraContainer_from_zfile(self, container, event): - container.waveform = np.array([ - ( - event.hiGain.waveforms.samples - ).reshape(-1, self.header.numTraces), + + container.num_samples = self.header.num_samples + container.trigger_time = event.trigger_time_s + container.trigger_type = event.trigger_type + + container.waveform = np.array( ( - event.loGain.waveforms.samples - ).reshape(-1, self.header.numTraces) - ]) + event.waveform + ).reshape(2, self.header.num_pixels, container.num_samples)) + + - container.num_samples = container.waveform.shape[1] - def fill_R0Container_from_zfile_event(self, container, event): + def fill_R0Container_from_zfile(self, container, event): container.obs_id = -1 container.event_id = event.event_id container.tels_with_data = [self.header.telescope_id, ] - r0_cam_container = container.tel[self.header.telescope_id] - self.fill_R0CameraContainer_from_zfile_event( - r0_cam_container, + r0_camera_container = container.tel[self.header.telescope_id] + self.fill_R0CameraContainer_from_zfile( + r0_camera_container, event ) diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index f64f5bac90a..06bf44d3029 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -2,7 +2,7 @@ """ EventSource for NectarCam protobuf-fits.fz-files. -Needs protozfits v0.44.5 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.02.0 from github.com/cta-sst-1m/protozfitsreader """ import numpy as np @@ -16,8 +16,8 @@ class NectarCAMEventSource(EventSource): def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) - from protozfits import SimpleFile - self.file = SimpleFile(self.input_url) + from protozfits import File + self.file = File(self.input_url) self.header = next(self.file.RunHeader) diff --git a/ctapipe/io/tests/test_lsteventsource.py b/ctapipe/io/tests/test_lsteventsource.py index f40aecfc8fa..1680b74dd4d 100644 --- a/ctapipe/io/tests/test_lsteventsource.py +++ b/ctapipe/io/tests/test_lsteventsource.py @@ -2,25 +2,25 @@ import os import pytest -pytest.importorskip("protozfits", minversion="0.44.5") +pytest.importorskip("protozfits", minversion="1.0.2") example_file_path = resource_filename( 'protozfits', os.path.join( 'tests', 'resources', - 'example_9evts_NectarCAM.fits.fz' + 'example_LST_R1_10_evts.fits.fz' ) ) FIRST_EVENT_NUMBER_IN_FILE = 1 -ADC_SAMPLES_SHAPE = (2, 84, 60) +ADC_SAMPLES_SHAPE = (2, 14, 40) def test_loop_over_events(): from ctapipe.io.lsteventsource import LSTEventSource - N_EVENTS = 3 + N_EVENTS = 10 inputfile_reader = LSTEventSource( input_url=example_file_path, max_events=N_EVENTS @@ -37,12 +37,12 @@ def test_loop_over_events(): def test_is_compatible(): - from ctapipe.io.lasteventsource import LSTEventSource + from ctapipe.io.lsteventsource import LSTEventSource assert LSTEventSource.is_compatible(example_file_path) -def test_factory_for_nectarcam_file(): +def test_factory_for_lst_file(): from ctapipe.io.eventsourcefactory import EventSourceFactory from ctapipe.io.lsteventsource import LSTEventSource From d4386fecbfc442c92ca9e3089999686d6a203e72 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 25 May 2018 17:10:49 +0200 Subject: [PATCH 06/82] Improved container organisation: introduced EVT and SVT concept for LSTCameraContainer (Fields: LSTEventContainer and LSTServiceContainer) --- ctapipe/io/containers.py | 67 +++++++++++++++--------------------- ctapipe/io/lsteventsource.py | 67 +++++++++++++++++++++++++++++------- 2 files changed, 82 insertions(+), 52 deletions(-) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index a3e79cabf6a..e78edd0449d 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -443,19 +443,20 @@ class NectarCAMDataContainer(DataContainer): """ nectarcam = Field(NectarCAMContainer(), "NectarCAM Specific Information") -class LSTCameraContainer(Container): + +class LSTServiceContainer(Container): """ - Container for Fields that are specific to each LST camera + Container for Fields that are specific to each LST camera configuration """ # Data from the CameraConfig table telescope_id = Field(-1, "telescope id") - cs_serial=Field(None, "serial of the camera server") - configuration_id = Field(None, "?") + cs_serial=Field(None, "serial number of the camera server") + configuration_id = Field(None, "id of the CameraConfiguration") date= Field(None, "NTP start of run date") num_pixels = Field(-1, "number of pixels") num_samples= Field(-1, "num samples") - pixel_ids=Field([],"pixels id") + pixel_ids=Field([],"id of the pixels in the waveform array") data_model_version=Field(None,"data model version") idaq_version = Field(0o0, "idaq version") @@ -465,8 +466,18 @@ class LSTCameraContainer(Container): module_ids = Field([], "module ids") num_modules= Field(-1, "number of modules") + +class LSTEventContainer(Container): + """ + Container for Fields that are specific to each LST event + """ + # Data from the CameraEvent table + configuration_id = Field(None,"id of the CameraConfiguration") + event_id=Field(None,"local id of the event") + tel_event_id = Field(None, "global id of the event") pixel_status = Field([],"status of the pixels") + ped_id = Field(None, "tel_event_id of the event used for pedestal substraction") module_status = Field([], "status of the modules") extdevices_presence = Field(None, "presence of data for external devices") tib_data = Field([], "TIB data array") @@ -478,35 +489,15 @@ class LSTCameraContainer(Container): drs_tag_status = Field([], "DRS tag status") drs_tag = Field([], "DRS tag") - # Fill data LST specifics - def fill_from_zfile(self, header, event): - self.telescope_id=header.telescope_id - self.cs_serial=header.cs_serial - self.configuration_id=header.configuration_id - self.date=header.date - self.num_pixels = header.num_pixels - self.num_samples=header.num_samples - self.pixel_ids=header.expected_pixels_id - self.data_model_version= header.data_model_version - - self.num_modules = header.lstcam.num_modules - self.module_ids = header.lstcam.expected_modules_id - self.idaq_version = header.lstcam.idaq_version - self.cdhs_version = header.lstcam.cdhs_version - self.algorithms = header.lstcam.algorithms - self.pre_proc_algorithms = header.lstcam.pre_proc_algorithms - - self.pixel_status = event.pixel_status - self.module_status = event.lstcam.module_status - self.extdevices_presence = event.lstcam.extdevices_presence - self.tib_data=event.lstcam.tib_data - self.cdts_data=event.lstcam.cdts_data - self.swat_data=event.lstcam.swat_data - self.counters=event.lstcam.counters - self.chips_flags = event.lstcam.chips_flags - self.first_capacitor_id = event.lstcam.first_capacitor_id - self.drs_tag_status = event.lstcam.drs_tag_status - self.drs_tag = event.lstcam.drs_tag + +class LSTCameraContainer(Container): + """ + Container for Fields that are specific to each LST camera + """ + evt = Field(LSTEventContainer(), "LST specific event Information") + svc = Field(LSTServiceContainer(), "LST specific camera_config Information") + + class LSTContainer(Container): @@ -518,19 +509,15 @@ class LSTContainer(Container): # create the camera container tel = Field( Map(LSTCameraContainer), - "map of tel_id to LSTCameraContainer") + "map of tel_id to LSTTelContainer") - def fill_from_zfile(self, header,event): - self.tels_with_data = [header.telescope_id, ] - lst_camera = self.tel[header.telescope_id] - lst_camera.fill_from_zfile(header, event) class LSTDataContainer(DataContainer): """ Data container including LST information """ - lst = Field(LSTContainer(), "LST Specific Information") + lst = Field(LSTContainer(), "LST specific Information") diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index c5b4c892b89..958b18293b3 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -18,7 +18,7 @@ def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) from protozfits import File self.file = File(self.input_url) - self.header = next(self.file.CameraConfig) + self.camera_config = next(self.file.CameraConfig) def _generator(self): @@ -27,13 +27,16 @@ def _generator(self): data = LSTDataContainer() data.meta['input_url'] = self.input_url + # fill LST data from the CameraConfig table + self.fill_LSTServiceContainer_from_zfile(data.lst, self.camera_config) + for count, event in enumerate(self.file.Events): data.count = count - # fill specific LST data - data.lst.fill_from_zfile(self.header,event) + # fill specific LST event data + self.fill_LSTEventContainer_from_zfile( data.lst, event) # fill general R0 data self.fill_R0Container_from_zfile(data.r0, event) @@ -45,8 +48,8 @@ def is_compatible(file_path): from astropy.io import fits try: # The file contains two tables: - # 1: CameraConfiguration - # 2: Events <--- this is what we need to look at + # 1: CameraConfig + # 2: Events h = fits.open(file_path)[2].header ttypes = [ h[x] for x in h.keys() if 'TTYPE' in x @@ -70,28 +73,68 @@ def is_compatible(file_path): is_lst_file = 'lstcam_counters' in ttypes return is_protobuf_zfits_file & is_lst_file + def fill_LSTServiceContainer_from_zfile(self, container,camera_config): + + container.tels_with_data = [camera_config.telescope_id, ] + + svc_container = container.tel[camera_config.telescope_id].svc + + svc_container.telescope_id=camera_config.telescope_id + svc_container.cs_serial=camera_config.cs_serial + svc_container.configuration_id=camera_config.configuration_id + svc_container.date=camera_config.date + svc_container.num_pixels = camera_config.num_pixels + svc_container.num_samples=camera_config.num_samples + svc_container.pixel_ids=camera_config.expected_pixels_id + svc_container.data_model_version= camera_config.data_model_version + + svc_container.num_modules = camera_config.lstcam.num_modules + svc_container.module_ids = camera_config.lstcam.expected_modules_id + svc_container.idaq_version = camera_config.lstcam.idaq_version + svc_container.cdhs_version = camera_config.lstcam.cdhs_version + svc_container.algorithms = camera_config.lstcam.algorithms + svc_container.pre_proc_algorithms = camera_config.lstcam.pre_proc_algorithms + - def fill_R0CameraContainer_from_zfile(self, container, event): + def fill_LSTEventContainer_from_zfile(self,container, event): - container.num_samples = self.header.num_samples + event_container = container.tel[self.camera_config.telescope_id].evt + + event_container.configuration_id=event.configuration_id + event_container.event_id=event.event_id + event_container.tel_event_id=event.tel_event_id + event_container.pixel_status = event.pixel_status + event_container.ped_id = event.ped_id + event_container.module_status = event.lstcam.module_status + event_container.extdevices_presence = event.lstcam.extdevices_presence + event_container.tib_data=event.lstcam.tib_data + event_container.cdts_data=event.lstcam.cdts_data + event_container.swat_data=event.lstcam.swat_data + event_container.counters=event.lstcam.counters + event_container.chips_flags = event.lstcam.chips_flags + event_container.first_capacitor_id = event.lstcam.first_capacitor_id + event_container.drs_tag_status = event.lstcam.drs_tag_status + event_container.drs_tag = event.lstcam.drs_tag + + def fill_R0CameraContainer_from_zfile(self, container, event): + + container.num_samples = self.camera_config.num_samples container.trigger_time = event.trigger_time_s container.trigger_type = event.trigger_type container.waveform = np.array( ( event.waveform - ).reshape(2, self.header.num_pixels, container.num_samples)) - - + ).reshape(2, self.camera_config.num_pixels, container.num_samples)) def fill_R0Container_from_zfile(self, container, event): container.obs_id = -1 container.event_id = event.event_id - container.tels_with_data = [self.header.telescope_id, ] - r0_camera_container = container.tel[self.header.telescope_id] + container.tels_with_data = [self.camera_config.telescope_id, ] + r0_camera_container = container.tel[self.camera_config.telescope_id] self.fill_R0CameraContainer_from_zfile( r0_camera_container, event From 31d35e1bc87a6d8f618d3fad144c2ba818d0fa67 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Wed, 30 May 2018 15:10:40 +0200 Subject: [PATCH 07/82] Minor corrections --- ctapipe/io/containers.py | 16 ++++++++-------- ctapipe/io/lsteventsource.py | 20 ++++++++++---------- ctapipe/io/tests/test_lsteventsource.py | 14 +++++++++----- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index e78edd0449d..cfe2c47ecde 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -451,13 +451,13 @@ class LSTServiceContainer(Container): # Data from the CameraConfig table telescope_id = Field(-1, "telescope id") - cs_serial=Field(None, "serial number of the camera server") + cs_serial = Field(None, "serial number of the camera server") configuration_id = Field(None, "id of the CameraConfiguration") - date= Field(None, "NTP start of run date") + date = Field(None, "NTP start of run date") num_pixels = Field(-1, "number of pixels") - num_samples= Field(-1, "num samples") - pixel_ids=Field([],"id of the pixels in the waveform array") - data_model_version=Field(None,"data model version") + num_samples = Field(-1, "num samples") + pixel_ids = Field([],"id of the pixels in the waveform array") + data_model_version = Field(None,"data model version") idaq_version = Field(0o0, "idaq version") cdhs_version = Field(0o0, "cdhs version") @@ -473,10 +473,10 @@ class LSTEventContainer(Container): """ # Data from the CameraEvent table - configuration_id = Field(None,"id of the CameraConfiguration") - event_id=Field(None,"local id of the event") + configuration_id = Field(None, "id of the CameraConfiguration") + event_id = Field(None, "local id of the event") tel_event_id = Field(None, "global id of the event") - pixel_status = Field([],"status of the pixels") + pixel_status = Field([], "status of the pixels") ped_id = Field(None, "tel_event_id of the event used for pedestal substraction") module_status = Field([], "status of the modules") extdevices_presence = Field(None, "presence of data for external devices") diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 958b18293b3..2bd6e65dcd1 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -36,7 +36,7 @@ def _generator(self): data.count = count # fill specific LST event data - self.fill_LSTEventContainer_from_zfile( data.lst, event) + self.fill_LSTEventContainer_from_zfile(data.lst, event) # fill general R0 data self.fill_R0Container_from_zfile(data.r0, event) @@ -73,7 +73,7 @@ def is_compatible(file_path): is_lst_file = 'lstcam_counters' in ttypes return is_protobuf_zfits_file & is_lst_file - def fill_LSTServiceContainer_from_zfile(self, container,camera_config): + def fill_LSTServiceContainer_from_zfile(self, container, camera_config): container.tels_with_data = [camera_config.telescope_id, ] @@ -97,21 +97,21 @@ def fill_LSTServiceContainer_from_zfile(self, container,camera_config): - def fill_LSTEventContainer_from_zfile(self,container, event): + def fill_LSTEventContainer_from_zfile(self, container, event): event_container = container.tel[self.camera_config.telescope_id].evt - event_container.configuration_id=event.configuration_id - event_container.event_id=event.event_id - event_container.tel_event_id=event.tel_event_id + event_container.configuration_id = event.configuration_id + event_container.event_id = event.event_id + event_container.tel_event_id = event.tel_event_id event_container.pixel_status = event.pixel_status event_container.ped_id = event.ped_id event_container.module_status = event.lstcam.module_status event_container.extdevices_presence = event.lstcam.extdevices_presence - event_container.tib_data=event.lstcam.tib_data - event_container.cdts_data=event.lstcam.cdts_data - event_container.swat_data=event.lstcam.swat_data - event_container.counters=event.lstcam.counters + event_container.tib_data = event.lstcam.tib_data + event_container.cdts_data = event.lstcam.cdts_data + event_container.swat_data = event.lstcam.swat_data + event_container.counters = event.lstcam.counters event_container.chips_flags = event.lstcam.chips_flags event_container.first_capacitor_id = event.lstcam.first_capacitor_id event_container.drs_tag_status = event.lstcam.drs_tag_status diff --git a/ctapipe/io/tests/test_lsteventsource.py b/ctapipe/io/tests/test_lsteventsource.py index 1680b74dd4d..13fe0862b9b 100644 --- a/ctapipe/io/tests/test_lsteventsource.py +++ b/ctapipe/io/tests/test_lsteventsource.py @@ -14,26 +14,30 @@ ) FIRST_EVENT_NUMBER_IN_FILE = 1 -ADC_SAMPLES_SHAPE = (2, 14, 40) +#ADC_SAMPLES_SHAPE = (2, 14, 40) def test_loop_over_events(): from ctapipe.io.lsteventsource import LSTEventSource - N_EVENTS = 10 + n_events = 10 inputfile_reader = LSTEventSource( input_url=example_file_path, - max_events=N_EVENTS + max_events=n_events ) for i, event in enumerate(inputfile_reader): assert event.r0.tels_with_data == [0] for telid in event.r0.tels_with_data: assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i - assert event.r0.tel[telid].waveform.shape == ADC_SAMPLES_SHAPE + n_gain=2 + num_pixels=event.lst.tel[telid].svc.num_pixels + num_samples=event.lst.tel[telid].svc.num_samples + waveform_shape = (n_gain, num_pixels, num_samples) + assert event.r0.tel[telid].waveform.shape == waveform_shape # make sure max_events works - assert i == N_EVENTS - 1 + assert i == n_events - 1 def test_is_compatible(): From fcc2f12a7849b5004d797339b04995ec4f4cb0d4 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 31 May 2018 11:00:39 +0200 Subject: [PATCH 08/82] Style changes --- ctapipe/io/containers.py | 6 +++--- ctapipe/io/lsteventsource.py | 10 +++++----- ctapipe/io/tests/test_lsteventsource.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index cfe2c47ecde..3a062c7fab2 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -456,15 +456,15 @@ class LSTServiceContainer(Container): date = Field(None, "NTP start of run date") num_pixels = Field(-1, "number of pixels") num_samples = Field(-1, "num samples") - pixel_ids = Field([],"id of the pixels in the waveform array") - data_model_version = Field(None,"data model version") + pixel_ids = Field([], "id of the pixels in the waveform array") + data_model_version = Field(None, "data model version") idaq_version = Field(0o0, "idaq version") cdhs_version = Field(0o0, "cdhs version") algorithms = Field(None, "algorithms") pre_proc_algorithms = Field(None, "pre processing algorithms") module_ids = Field([], "module ids") - num_modules= Field(-1, "number of modules") + num_modules = Field(-1, "number of modules") class LSTEventContainer(Container): diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 2bd6e65dcd1..f2cf9758ad4 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -79,13 +79,13 @@ def fill_LSTServiceContainer_from_zfile(self, container, camera_config): svc_container = container.tel[camera_config.telescope_id].svc - svc_container.telescope_id=camera_config.telescope_id + svc_container.telescope_id = camera_config.telescope_id svc_container.cs_serial=camera_config.cs_serial - svc_container.configuration_id=camera_config.configuration_id - svc_container.date=camera_config.date + svc_container.configuration_id = camera_config.configuration_id + svc_container.date = camera_config.date svc_container.num_pixels = camera_config.num_pixels - svc_container.num_samples=camera_config.num_samples - svc_container.pixel_ids=camera_config.expected_pixels_id + svc_container.num_samples = camera_config.num_samples + svc_container.pixel_ids = camera_config.expected_pixels_id svc_container.data_model_version= camera_config.data_model_version svc_container.num_modules = camera_config.lstcam.num_modules diff --git a/ctapipe/io/tests/test_lsteventsource.py b/ctapipe/io/tests/test_lsteventsource.py index 13fe0862b9b..51b68da56da 100644 --- a/ctapipe/io/tests/test_lsteventsource.py +++ b/ctapipe/io/tests/test_lsteventsource.py @@ -14,7 +14,7 @@ ) FIRST_EVENT_NUMBER_IN_FILE = 1 -#ADC_SAMPLES_SHAPE = (2, 14, 40) +# ADC_SAMPLES_SHAPE = (2, 14, 40) def test_loop_over_events(): @@ -30,9 +30,9 @@ def test_loop_over_events(): assert event.r0.tels_with_data == [0] for telid in event.r0.tels_with_data: assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i - n_gain=2 - num_pixels=event.lst.tel[telid].svc.num_pixels - num_samples=event.lst.tel[telid].svc.num_samples + n_gain = 2 + num_pixels = event.lst.tel[telid].svc.num_pixels + num_samples = event.lst.tel[telid].svc.num_samples waveform_shape = (n_gain, num_pixels, num_samples) assert event.r0.tel[telid].waveform.shape == waveform_shape From 679fbf9b465d4dca6a158a0e8dd2d6a4391569e8 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 31 May 2018 11:30:14 +0200 Subject: [PATCH 09/82] Style changes --- ctapipe/io/containers.py | 9 --------- ctapipe/io/lsteventsource.py | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 3a062c7fab2..36aa27efb85 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -520,15 +520,6 @@ class LSTDataContainer(DataContainer): lst = Field(LSTContainer(), "LST specific Information") - -class TargetIOCameraContainer(Container): - """ - Container for Fields that are specific to cameras that use TARGET - """ - first_cell_ids = Field(None, ("numpy array of the first_cell_id of each" - "waveform in the camera image (n_pixels)")) - - class TargetIOCameraContainer(Container): """ Container for Fields that are specific to cameras that use TARGET diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index f2cf9758ad4..b71306e9dee 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -28,7 +28,7 @@ def _generator(self): data.meta['input_url'] = self.input_url # fill LST data from the CameraConfig table - self.fill_LSTServiceContainer_from_zfile(data.lst, self.camera_config) + self.fill_lst_service_container_from_zfile(data.lst, self.camera_config) for count, event in enumerate(self.file.Events): @@ -36,10 +36,10 @@ def _generator(self): data.count = count # fill specific LST event data - self.fill_LSTEventContainer_from_zfile(data.lst, event) + self.fill_lst_event_container_from_zfile(data.lst, event) # fill general R0 data - self.fill_R0Container_from_zfile(data.r0, event) + self.fill_r0_container_from_zfile(data.r0, event) yield data @@ -73,20 +73,20 @@ def is_compatible(file_path): is_lst_file = 'lstcam_counters' in ttypes return is_protobuf_zfits_file & is_lst_file - def fill_LSTServiceContainer_from_zfile(self, container, camera_config): + def fill_lst_service_container_from_zfile(self, container, camera_config): container.tels_with_data = [camera_config.telescope_id, ] svc_container = container.tel[camera_config.telescope_id].svc svc_container.telescope_id = camera_config.telescope_id - svc_container.cs_serial=camera_config.cs_serial + svc_container.cs_serial = camera_config.cs_serial svc_container.configuration_id = camera_config.configuration_id svc_container.date = camera_config.date svc_container.num_pixels = camera_config.num_pixels svc_container.num_samples = camera_config.num_samples svc_container.pixel_ids = camera_config.expected_pixels_id - svc_container.data_model_version= camera_config.data_model_version + svc_container.data_model_version = camera_config.data_model_version svc_container.num_modules = camera_config.lstcam.num_modules svc_container.module_ids = camera_config.lstcam.expected_modules_id @@ -97,7 +97,7 @@ def fill_LSTServiceContainer_from_zfile(self, container, camera_config): - def fill_LSTEventContainer_from_zfile(self, container, event): + def fill_lst_event_container_from_zfile(self, container, event): event_container = container.tel[self.camera_config.telescope_id].evt @@ -117,7 +117,7 @@ def fill_LSTEventContainer_from_zfile(self, container, event): event_container.drs_tag_status = event.lstcam.drs_tag_status event_container.drs_tag = event.lstcam.drs_tag - def fill_R0CameraContainer_from_zfile(self, container, event): + def fill_r0_camera_container_from_zfile(self, container, event): container.num_samples = self.camera_config.num_samples container.trigger_time = event.trigger_time_s @@ -129,13 +129,13 @@ def fill_R0CameraContainer_from_zfile(self, container, event): ).reshape(2, self.camera_config.num_pixels, container.num_samples)) - def fill_R0Container_from_zfile(self, container, event): + def fill_r0_container_from_zfile(self, container, event): container.obs_id = -1 container.event_id = event.event_id container.tels_with_data = [self.camera_config.telescope_id, ] r0_camera_container = container.tel[self.camera_config.telescope_id] - self.fill_R0CameraContainer_from_zfile( + self.fill_r0_camera_container_from_zfile( r0_camera_container, event ) From 136ae4307a02cfea511bcca56203e9f569c49e04 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 31 May 2018 11:57:48 +0200 Subject: [PATCH 10/82] Updated request for protozfitsreader library from release 0.44.5 to 1.0.2 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 95bc16e96eb..1e36fbbea87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ install: - pip install codecov # ----- SST1M: - conda install protobuf - - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v0.44.5.tar.gz + - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.0.2.tar.gz # ----- end of SST1M # ----- target_software - conda install -c conda-forge cfitsio From e118351f37791bb85610a36fb48ef7be3a715a59 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 31 May 2018 14:57:01 +0200 Subject: [PATCH 11/82] Change of import from module SimpleFile to module File to be compatible with version 1.0.2 of the protozfit library --- ctapipe/io/sst1meventsource.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ctapipe/io/sst1meventsource.py b/ctapipe/io/sst1meventsource.py index ea4c80d67d7..35c8a7a99de 100644 --- a/ctapipe/io/sst1meventsource.py +++ b/ctapipe/io/sst1meventsource.py @@ -2,7 +2,7 @@ """ EventSource for SST1M/digicam protobuf-fits.fz-files. -Needs protozfits v0.44.3 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.0.2 from github.com/cta-sst-1m/protozfitsreader """ import numpy as np from .eventsource import EventSource @@ -16,8 +16,8 @@ class SST1MEventSource(EventSource): def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) - from protozfits import SimpleFile - self.file = SimpleFile(self.input_url) + from protozfits import File + self.file = File(self.input_url) # TODO: Correct pixel ordering self._tel_desc = TelescopeDescription.from_name( optics_name='SST-1M', From b95fbcbea1c9aa7fddca786eaabcc8d11c4f5d11 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 1 Jun 2018 14:49:46 +0200 Subject: [PATCH 12/82] remove ctapipe/atmosphere and doc/atmosphere --- ctapipe/atmosphere/__init__.py | 0 ctapipe/atmosphere/fram/__init__.py | 0 ctapipe/atmosphere/gdas/__init__.py | 0 ctapipe/atmosphere/lidar/__init__.py | 0 docs/atmosphere/index.rst | 47 ---------------------------- 5 files changed, 47 deletions(-) delete mode 100644 ctapipe/atmosphere/__init__.py delete mode 100644 ctapipe/atmosphere/fram/__init__.py delete mode 100644 ctapipe/atmosphere/gdas/__init__.py delete mode 100644 ctapipe/atmosphere/lidar/__init__.py delete mode 100644 docs/atmosphere/index.rst diff --git a/ctapipe/atmosphere/__init__.py b/ctapipe/atmosphere/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/ctapipe/atmosphere/fram/__init__.py b/ctapipe/atmosphere/fram/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/ctapipe/atmosphere/gdas/__init__.py b/ctapipe/atmosphere/gdas/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/ctapipe/atmosphere/lidar/__init__.py b/ctapipe/atmosphere/lidar/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/atmosphere/index.rst b/docs/atmosphere/index.rst deleted file mode 100644 index 69b1e98e943..00000000000 --- a/docs/atmosphere/index.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. _atmosphere: - -========================== - Atmosphere (`atmosphere`) -========================== - -.. currentmodule:: ctapipe.atmosphere - - -Introduction -============ - -This module include all the functions and classes needed for the monitoring -of the atmosphere above the CTA sites. - -It consists in three main sub-modules: - -* Lidar Calibration - -* Fram Calibration - -* GDAS Calibration - -For more information on where you should implement your code, please have a look to the README.rst files inside each directory. - - -Getting Started -=============== - -TODO: add examples. - -Submodules -========== - -.. toctree:: - :maxdepth: 1 - :glob: - - index_* - - - -Reference/API -============= - -.. automodapi:: ctapipe.atmosphere - :no-inheritance-diagram: From 902a88bebc44c8ac17661f87a70a772b6894bd52 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 26 Jul 2018 10:18:25 +0200 Subject: [PATCH 13/82] Added a check on the number of gains in the LST event.waveform --- ctapipe/io/lsteventsource.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index b71306e9dee..29a5f37eb88 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -97,6 +97,7 @@ def fill_lst_service_container_from_zfile(self, container, camera_config): + def fill_lst_event_container_from_zfile(self, container, event): event_container = container.tel[self.camera_config.telescope_id].evt @@ -123,10 +124,19 @@ def fill_r0_camera_container_from_zfile(self, container, event): container.trigger_time = event.trigger_time_s container.trigger_type = event.trigger_type + # verify the number of gains + if event.waveform.shape[0]==(self.camera_config.num_pixels*container.num_samples): + n_gains=1 + elif event.waveform.shape[0]==(self.camera_config.num_pixels*container.num_samples*2): + n_gains=2 + else: + raise ValueError("Waveform matrix dimension not supported: N_chan x N_pix x N_samples= '{}'".format(event.waveform.shape[0])) + + container.waveform = np.array( ( event.waveform - ).reshape(2, self.camera_config.num_pixels, container.num_samples)) + ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples)) def fill_r0_container_from_zfile(self, container, event): From a1cb9d54048d5c5708f66ff574e2b3751378de69 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 31 Aug 2018 17:02:17 +0200 Subject: [PATCH 14/82] Added class MultiFiles in order to read events belonging to the same run but spread in multiple fits files --- .travis.yml | 2 +- ctapipe/io/lsteventsource.py | 112 +++++++++++++++++++++++++++++++++-- 2 files changed, 107 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1e36fbbea87..61dc9f03488 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ install: - pip install codecov # ----- SST1M: - conda install protobuf - - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.0.2.tar.gz + - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.4.0.tar.gz # ----- end of SST1M # ----- target_software - conda install -c conda-forge cfitsio diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 29a5f37eb88..83c00899a3c 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -2,10 +2,12 @@ """ EventSource for LSTCam protobuf-fits.fz-files. -Needs protozfits v1.02.0 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.0.2 from github.com/cta-sst-1m/protozfitsreader """ import numpy as np +from os.path import exists +from ctapipe.core import Provenance from .eventsource import EventSource from .containers import LSTDataContainer @@ -16,9 +18,11 @@ class LSTEventSource(EventSource): def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) - from protozfits import File - self.file = File(self.input_url) - self.camera_config = next(self.file.CameraConfig) + + self.multi_file = MultiFiles(self.input_url) + self.camera_config = self.multi_file.camera_config + + self.log.info("Read {} input files".format(self.multi_file.num_inputs())) def _generator(self): @@ -30,8 +34,8 @@ def _generator(self): # fill LST data from the CameraConfig table self.fill_lst_service_container_from_zfile(data.lst, self.camera_config) - for count, event in enumerate(self.file.Events): - + # loop on events + for count, event in enumerate(self.multi_file): data.count = count @@ -149,3 +153,99 @@ def fill_r0_container_from_zfile(self, container, event): r0_camera_container, event ) + + +class MultiFiles: + ''' + In LST they have multiple file writers, which save the incoming events + into different files, so in case one has 10 events and 4 files, + it might look like this: + f1 = [0, 4] + f2 = [1, 5, 8] + f3 = [2, 6, 9] + f4 = [3, 7] + The task of MultiZFitsFiles is to open these 4 files simultaneously + and return the events in the correct order, so the user does not really + have to know about these existence of 4 files. + ''' + + def __init__(self, input_url): + + self._file = {} + self._events = {} + self._events_table = {} + self._camera_config = {} + + paths = [input_url, ] + + # test how many files are there + if '000.fits.fz' in input_url: + i = 0 + while True: + input_url = input_url.replace(str(i).zfill(3) + '.fits.fz', str(i + 1).zfill(3) + '.fits.fz') + if exists(input_url): + paths.append(input_url) + # keep track of all input files + Provenance().add_input_file(input_url, role='dl0.sub.evt') + i = i + 1 + else: + break + + # open the files and get the first fits Tables + from protozfits import File + + for path in paths: + self._file[path] = File(path) + self._events_table[path] = File(path).Events + try: + + self._events[path] = next(self._file[path].Events) + self._camera_config[path] = next(self._file[path].CameraConfig) + + except StopIteration: + pass + + # verify that the configuration_id of all files are the same in the CameraConfig table + for path in paths: + assert self._camera_config[path].configuration_id == self._camera_config[paths[0]].configuration_id + + # keep the cameraConfig of the first file + self.camera_config = self._camera_config[paths[0]] + + def camera_config(self): + return self.camera_config + + def __iter__(self): + return self + + def __next__(self): + return self.next_event() + + def next_event(self): + # check for the minimal event id + if not self._events: + raise StopIteration + + min_path = min( + self._events.items(), + key=lambda item: item[1].event_id, + )[0] + + # return the minimal event id + next_event = self._events[min_path] + try: + self._events[min_path] = next(self._file[min_path].Events) + except StopIteration: + del self._events[min_path] + + return next_event + + def __len__(self): + total_length = sum( + len(table) + for table in self._events_table.values() + ) + return total_length + + def num_inputs(self): + return len(self._file) From 937d54237ba8f652e423a4159e173026ab9b9f7a Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Tue, 4 Sep 2018 12:39:08 +0200 Subject: [PATCH 15/82] Correct the protozfits version (for the moment the reader need only 1.0.2) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 61dc9f03488..1e36fbbea87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ install: - pip install codecov # ----- SST1M: - conda install protobuf - - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.4.0.tar.gz + - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.0.2.tar.gz # ----- end of SST1M # ----- target_software - conda install -c conda-forge cfitsio From e507ebb82079fdd881f725ef2b33b808d6bcbc4c Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Tue, 4 Sep 2018 12:59:32 +0200 Subject: [PATCH 16/82] Added comments to the MultiFiles class --- ctapipe/io/lsteventsource.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 83c00899a3c..57fc328660c 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -167,6 +167,11 @@ class MultiFiles: The task of MultiZFitsFiles is to open these 4 files simultaneously and return the events in the correct order, so the user does not really have to know about these existence of 4 files. + + In case of multiple input files the name of the files must finish with suffix + *000.fits.fz, *001.fits.fz, etc... and the user must give as input_url the name + of the first file (*000.fits.fz). The program will search for the other files. + In the case of only one input file the input_url can have any form. ''' def __init__(self, input_url): From 3326fb2635acdede95f99a90b6666d56d10a5d6e Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Wed, 5 Sep 2018 17:18:04 +0200 Subject: [PATCH 17/82] Ask for version protozfit 1.40 and minor code corrections --- .travis.yml | 2 +- ctapipe/io/lsteventsource.py | 28 ++++++++++++------------- ctapipe/io/tests/test_lsteventsource.py | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1e36fbbea87..61dc9f03488 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ install: - pip install codecov # ----- SST1M: - conda install protobuf - - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.0.2.tar.gz + - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.4.0.tar.gz # ----- end of SST1M # ----- target_software - conda install -c conda-forge cfitsio diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 57fc328660c..f8a0553a3aa 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -2,7 +2,7 @@ """ EventSource for LSTCam protobuf-fits.fz-files. -Needs protozfits v1.0.2 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.4.0 from github.com/cta-sst-1m/protozfitsreader """ import numpy as np @@ -80,7 +80,6 @@ def is_compatible(file_path): def fill_lst_service_container_from_zfile(self, container, camera_config): container.tels_with_data = [camera_config.telescope_id, ] - svc_container = container.tel[camera_config.telescope_id].svc svc_container.telescope_id = camera_config.telescope_id @@ -129,12 +128,13 @@ def fill_r0_camera_container_from_zfile(self, container, event): container.trigger_type = event.trigger_type # verify the number of gains - if event.waveform.shape[0]==(self.camera_config.num_pixels*container.num_samples): - n_gains=1 - elif event.waveform.shape[0]==(self.camera_config.num_pixels*container.num_samples*2): - n_gains=2 + if event.waveform.shape[0] == (self.camera_config.num_pixels*container.num_samples): + n_gains = 1 + elif event.waveform.shape[0] == (self.camera_config.num_pixels*container.num_samples*2): + n_gains = 2 else: - raise ValueError("Waveform matrix dimension not supported: N_chan x N_pix x N_samples= '{}'".format(event.waveform.shape[0])) + raise ValueError("Waveform matrix dimension not supported: " + "N_chan x N_pix x N_samples= '{}'".format(event.waveform.shape[0])) container.waveform = np.array( @@ -170,7 +170,7 @@ class MultiFiles: In case of multiple input files the name of the files must finish with suffix *000.fits.fz, *001.fits.fz, etc... and the user must give as input_url the name - of the first file (*000.fits.fz). The program will search for the other files. + of the first file (*000.fits.fz). The program will search for the other files. In the case of only one input file the input_url can have any form. ''' @@ -187,7 +187,8 @@ def __init__(self, input_url): if '000.fits.fz' in input_url: i = 0 while True: - input_url = input_url.replace(str(i).zfill(3) + '.fits.fz', str(i + 1).zfill(3) + '.fits.fz') + input_url = input_url.replace(str(i).zfill(3) + + '.fits.fz', str(i + 1).zfill(3) + '.fits.fz') if exists(input_url): paths.append(input_url) # keep track of all input files @@ -210,16 +211,15 @@ def __init__(self, input_url): except StopIteration: pass - # verify that the configuration_id of all files are the same in the CameraConfig table + # verify that the configuration_id of all files are the same + # in the CameraConfig table for path in paths: - assert self._camera_config[path].configuration_id == self._camera_config[paths[0]].configuration_id + assert self._camera_config[path].configuration_id == \ + self._camera_config[paths[0]].configuration_id # keep the cameraConfig of the first file self.camera_config = self._camera_config[paths[0]] - def camera_config(self): - return self.camera_config - def __iter__(self): return self diff --git a/ctapipe/io/tests/test_lsteventsource.py b/ctapipe/io/tests/test_lsteventsource.py index 51b68da56da..8ff3a5a9bf3 100644 --- a/ctapipe/io/tests/test_lsteventsource.py +++ b/ctapipe/io/tests/test_lsteventsource.py @@ -2,7 +2,7 @@ import os import pytest -pytest.importorskip("protozfits", minversion="1.0.2") +pytest.importorskip("protozfits", minversion="1.4.0") example_file_path = resource_filename( 'protozfits', From dbaa23c37a9d07c913ef5aa671b4ce919628fe57 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 6 Sep 2018 09:44:26 +0200 Subject: [PATCH 18/82] Minor changes for better coding standard --- ctapipe/io/lsteventsource.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index f8a0553a3aa..e09434f456d 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -128,13 +128,16 @@ def fill_r0_camera_container_from_zfile(self, container, event): container.trigger_type = event.trigger_type # verify the number of gains - if event.waveform.shape[0] == (self.camera_config.num_pixels*container.num_samples): + if event.waveform.shape[0] == (self.camera_config.num_pixels * + container.num_samples): n_gains = 1 - elif event.waveform.shape[0] == (self.camera_config.num_pixels*container.num_samples*2): + elif event.waveform.shape[0] == (self.camera_config.num_pixels * + container.num_samples * 2): n_gains = 2 else: raise ValueError("Waveform matrix dimension not supported: " - "N_chan x N_pix x N_samples= '{}'".format(event.waveform.shape[0])) + "N_chan x N_pix x N_samples= '{}'" + .format(event.waveform.shape[0])) container.waveform = np.array( @@ -188,7 +191,8 @@ def __init__(self, input_url): i = 0 while True: input_url = input_url.replace(str(i).zfill(3) + - '.fits.fz', str(i + 1).zfill(3) + '.fits.fz') + '.fits.fz', str(i + 1) + .zfill(3) + '.fits.fz') if exists(input_url): paths.append(input_url) # keep track of all input files From 10a23a3737f7afd3401aecf973c0d00babe31da7 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 6 Sep 2018 11:00:54 +0200 Subject: [PATCH 19/82] Minor style changes --- ctapipe/io/lsteventsource.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index e09434f456d..6133ca26b29 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -218,8 +218,8 @@ def __init__(self, input_url): # verify that the configuration_id of all files are the same # in the CameraConfig table for path in paths: - assert self._camera_config[path].configuration_id == \ - self._camera_config[paths[0]].configuration_id + assert ( self._camera_config[path].configuration_id + == self._camera_config[paths[0]].configuration_id ) # keep the cameraConfig of the first file self.camera_config = self._camera_config[paths[0]] From 3f783397c390ec0901c5fef985daec8c75c6b676 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 13 Sep 2018 12:13:24 +0200 Subject: [PATCH 20/82] Minor changes --- ctapipe/io/lsteventsource.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 6133ca26b29..d03d5ca2c48 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -218,8 +218,8 @@ def __init__(self, input_url): # verify that the configuration_id of all files are the same # in the CameraConfig table for path in paths: - assert ( self._camera_config[path].configuration_id - == self._camera_config[paths[0]].configuration_id ) + assert (self._camera_config[path].configuration_id + == self._camera_config[paths[0]].configuration_id) # keep the cameraConfig of the first file self.camera_config = self._camera_config[paths[0]] From c8308669bbb886e8e2d9d0b5a6af9b21ef11aede Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 17 Sep 2018 16:52:08 +0200 Subject: [PATCH 21/82] Added NectarCAM reader for reading events in the new R1 format. The old reader and containers are kept under the name extention "_OldFormat", (for the moment tey are necessary in order to read the old test data) --- ctapipe/io/containers.py | 88 +++++++++++- ctapipe/io/nectarcameventsource.py | 130 +++++++++++++----- ctapipe/io/tests/test_nectarcameventsource.py | 21 +-- 3 files changed, 192 insertions(+), 47 deletions(-) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 71361348844..d27642b80bc 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -25,8 +25,8 @@ 'SST1MCameraContainer', 'LSTContainer', 'LSTCameraContainer', - 'NectarCAMContainer', - 'NectarCAMCameraContainer', + 'NectarCAMContainerOldFormat', + 'NectarCAMCameraContainerOldFormat', 'MCEventContainer', 'MCHeaderContainer', 'MCCameraEventContainer', @@ -398,7 +398,7 @@ class SST1MDataContainer(DataContainer): sst1m = Field(SST1MContainer(), "optional SST1M Specific Information") -class NectarCAMCameraContainer(Container): +class NectarCAMCameraContainerOldFormat(Container): """ Container for Fields that are specific to camera that use zfit """ @@ -421,13 +421,13 @@ def fill_from_zfile_event(self, event, numTraces): -class NectarCAMContainer(Container): +class NectarCAMContainerOldFormat(Container): """ Storage for the NectarCAMCameraContainer for each telescope """ tels_with_data = Field([], "list of telescopes with data") tel = Field( - Map(NectarCAMCameraContainer), + Map(NectarCAMCameraContainerOldFormat), "map of tel_id to NectarCameraContainer") def fill_from_zfile_event(self, event, numTraces): @@ -439,11 +439,87 @@ def fill_from_zfile_event(self, event, numTraces): ) +class NectarCAMDataContainerOldFormat(DataContainer): + """ + Data container including NectarCAM information + """ + nectarcam = Field(NectarCAMContainerOldFormat(), "NectarCAM Specific Information") + + + +class NectarCAMServiceContainer(Container): + """ + Container for Fields that are specific to each NectarCAM camera configuration + """ + + # Data from the CameraConfig table + telescope_id = Field(-1, "telescope id") + cs_serial = Field(None, "serial number of the camera server") + configuration_id = Field(None, "id of the CameraConfiguration") + date = Field(None, "NTP start of run date") + num_pixels = Field(-1, "number of pixels") + num_samples = Field(-1, "num samples") + pixel_ids = Field([], "id of the pixels in the waveform array") + data_model_version = Field(None, "data model version") + + idaq_version = Field(0o0, "idaq version") + cdhs_version = Field(0o0, "cdhs version") + #acquisition_mode = Field(-1, "acquisition mode") + algorithms = Field(None, "algorithms") + pre_proc_algorithms = Field(None, "pre processing algorithms") + module_ids = Field([], "module ids") + num_modules = Field(-1, "number of modules") + + +class NectarCAMEventContainer(Container): + """ + Container for Fields that are specific to each NectarCAM event + """ + + # Data from the CameraEvent table + configuration_id = Field(None, "id of the CameraConfiguration") + event_id = Field(None, "local id of the event") + tel_event_id = Field(None, "global id of the event") + pixel_status = Field([], "status of the pixels") + ped_id = Field(None, "tel_event_id of the event used for pedestal substraction") + module_status = Field([], "status of the modules") + extdevices_presence = Field(None, "presence of data for external devices") + tib_data = Field([], "TIB data array") + cdts_data = Field([], "CDTS data array") + swat_data = Field([], "SWAT data array") + counters = Field([], "counters") + + + +class NectarCAMCameraContainer(Container): + """ + Container for Fields that are specific to each NectarCAM camera + """ + evt = Field(NectarCAMEventContainer(), "NectarCAM specific event Information") + svc = Field(NectarCAMServiceContainer(), "NectarCAM specific camera_config Information") + + + + +class NectarCAMContainer(Container): + """ + Storage for the NectarCAMCameraContainer for each telescope + """ + tels_with_data = Field([], "list of telescopes with data") + + # create the camera container + tel = Field( + Map(NectarCAMCameraContainer), + "map of tel_id to NectarCAMCameraContainer") + + + class NectarCAMDataContainer(DataContainer): """ Data container including NectarCAM information """ - nectarcam = Field(NectarCAMContainer(), "NectarCAM Specific Information") + nectarcam = Field(NectarCAMContainer(), "NectarCAM specific Information") + class LSTServiceContainer(Container): diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index 06bf44d3029..66665cc89ec 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -1,12 +1,15 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ -EventSource for NectarCam protobuf-fits.fz-files. +EventSource for LSTCam protobuf-fits.fz-files. -Needs protozfits v1.02.0 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.4.0 from github.com/cta-sst-1m/protozfitsreader """ import numpy as np +from os.path import exists +from ctapipe.core import Provenance from .eventsource import EventSource +from .lsteventsource import MultiFiles from .containers import NectarCAMDataContainer __all__ = ['NectarCAMEventSource'] @@ -16,22 +19,32 @@ class NectarCAMEventSource(EventSource): def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) - from protozfits import File - self.file = File(self.input_url) - self.header = next(self.file.RunHeader) + + self.multi_file = MultiFiles(self.input_url) + self.camera_config = self.multi_file.camera_config + + self.log.info("Read {} input files".format(self.multi_file.num_inputs())) def _generator(self): - self._pixel_sort_ids = None + # container for LST data + data = NectarCAMDataContainer() + data.meta['input_url'] = self.input_url + + # fill data from the CameraConfig table + self.fill_lst_service_container_from_zfile(data.nectarcam, self.camera_config) + + # loop on events + for count, event in enumerate(self.multi_file): - for count, event in enumerate(self.file.Events): - data = NectarCAMDataContainer() data.count = count - # fill specific NectarCAM data - data.nectarcam.fill_from_zfile_event(event, self.header.numTraces) + + # fill specific NectarCAM event data + self.fill_lst_event_container_from_zfile(data.nectarcam, event) + # fill general R0 data - self.fill_R0Container_from_zfile_event(data.r0, event) + self.fill_r0_container_from_zfile(data.r0, event) yield data @@ -40,8 +53,8 @@ def is_compatible(file_path): from astropy.io import fits try: # The file contains two tables: - # 1: RunHeader - # 2: Events <--- this is what we need to look at + # 1: CameraConfig + # 2: Events h = fits.open(file_path)[2].header ttypes = [ h[x] for x in h.keys() if 'TTYPE' in x @@ -59,36 +72,87 @@ def is_compatible(file_path): (h['EXTNAME'] == 'Events') and (h['ZTABLE'] is True) and (h['ORIGIN'] == 'CTA') and - (h['PBFHEAD'] == 'DataModel.CameraEvent') + (h['PBFHEAD'] == 'R1.CameraEvent') ) - is_nectarcam_file = 'hiGain_integrals_gains' in ttypes + is_nectarcam_file = 'nectarcam_counters' in ttypes return is_protobuf_zfits_file & is_nectarcam_file + def fill_lst_service_container_from_zfile(self, container, camera_config): - def fill_R0CameraContainer_from_zfile_event(self, container, event): - container.trigger_time = ( - event.local_time_sec * 1E9 + event.local_time_nanosec) - container.trigger_type = event.event_type + container.tels_with_data = [camera_config.telescope_id, ] + svc_container = container.tel[camera_config.telescope_id].svc - container.waveform = np.array([ - ( - event.hiGain.waveforms.samples - ).reshape(-1, self.header.numTraces), + svc_container.telescope_id = camera_config.telescope_id + svc_container.cs_serial = camera_config.cs_serial + svc_container.configuration_id = camera_config.configuration_id + #svc_container.acquisition_mode = camera_config.acquisition_mode + svc_container.date = camera_config.date + svc_container.num_pixels = camera_config.num_pixels + svc_container.num_samples = camera_config.num_samples + svc_container.pixel_ids = camera_config.expected_pixels_id + svc_container.data_model_version = camera_config.data_model_version + + svc_container.num_modules = camera_config.lstcam.num_modules + svc_container.module_ids = camera_config.lstcam.expected_modules_id + svc_container.idaq_version = camera_config.lstcam.idaq_version + svc_container.cdhs_version = camera_config.lstcam.cdhs_version + svc_container.algorithms = camera_config.lstcam.algorithms + svc_container.pre_proc_algorithms = camera_config.lstcam.pre_proc_algorithms + + + + + def fill_lst_event_container_from_zfile(self, container, event): + + event_container = container.tel[self.camera_config.telescope_id].evt + + event_container.configuration_id = event.configuration_id + event_container.event_id = event.event_id + event_container.tel_event_id = event.tel_event_id + event_container.pixel_status = event.pixel_status + event_container.ped_id = event.ped_id + event_container.module_status = event.lstcam.module_status + event_container.extdevices_presence = event.lstcam.extdevices_presence + event_container.tib_data = event.lstcam.tib_data + event_container.cdts_data = event.lstcam.cdts_data + event_container.swat_data = event.lstcam.swat_data + event_container.counters = event.lstcam.counters + + + def fill_r0_camera_container_from_zfile(self, container, event): + + container.num_samples = self.camera_config.num_samples + container.trigger_time = event.trigger_time_s + container.trigger_type = event.trigger_type + + # verify the number of gains + if event.waveform.shape[0] == (self.camera_config.num_pixels * + container.num_samples): + n_gains = 1 + elif event.waveform.shape[0] == (self.camera_config.num_pixels * + container.num_samples * 2): + n_gains = 2 + else: + raise ValueError("Waveform matrix dimension not supported: " + "N_chan x N_pix x N_samples= '{}'" + .format(event.waveform.shape[0])) + + + container.waveform = np.array( ( - event.loGain.waveforms.samples - ).reshape(-1, self.header.numTraces) - ]) + event.waveform + ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples)) - container.num_samples = container.waveform.shape[1] - def fill_R0Container_from_zfile_event(self, container, event): + def fill_r0_container_from_zfile(self, container, event): container.obs_id = -1 - container.event_id = event.eventNumber + container.event_id = event.event_id - container.tels_with_data = [self.header.telescopeID, ] - r0_cam_container = container.tel[self.header.telescopeID] - self.fill_R0CameraContainer_from_zfile_event( - r0_cam_container, + container.tels_with_data = [self.camera_config.telescope_id, ] + r0_camera_container = container.tel[self.camera_config.telescope_id] + self.fill_r0_camera_container_from_zfile( + r0_camera_container, event ) + diff --git a/ctapipe/io/tests/test_nectarcameventsource.py b/ctapipe/io/tests/test_nectarcameventsource.py index ac1f37d4f17..7982e74627e 100644 --- a/ctapipe/io/tests/test_nectarcameventsource.py +++ b/ctapipe/io/tests/test_nectarcameventsource.py @@ -2,38 +2,43 @@ import os import pytest -pytest.importorskip("protozfits", minversion="0.44.5") +pytest.importorskip("protozfits", minversion="1.4.0") example_file_path = resource_filename( 'protozfits', os.path.join( 'tests', 'resources', - 'example_9evts_NectarCAM.fits.fz' + '' ) ) FIRST_EVENT_NUMBER_IN_FILE = 1 -ADC_SAMPLES_SHAPE = (2, 84, 60) + +example_file_path="/ctadata/NectarCAM/2018/20180809/NectarCAM.Run0889.0000.fits.fz" def test_loop_over_events(): from ctapipe.io.nectarcameventsource import NectarCAMEventSource - N_EVENTS = 3 + n_events = 10 inputfile_reader = NectarCAMEventSource( input_url=example_file_path, - max_events=N_EVENTS + max_events=n_events ) for i, event in enumerate(inputfile_reader): assert event.r0.tels_with_data == [0] for telid in event.r0.tels_with_data: assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i - assert event.r0.tel[telid].waveform.shape == ADC_SAMPLES_SHAPE + n_gain = 2 + num_pixels = event.nectarcam.tel[telid].svc.num_pixels + num_samples = event.nectarcam.tel[telid].svc.num_samples + waveform_shape = (n_gain, num_pixels, num_samples) + assert event.r0.tel[telid].waveform.shape == waveform_shape # make sure max_events works - assert i == N_EVENTS - 1 + assert i == n_events - 1 def test_is_compatible(): @@ -44,7 +49,7 @@ def test_is_compatible(): def test_factory_for_nectarcam_file(): from ctapipe.io.eventsourcefactory import EventSourceFactory - from ctapipe.io.nectarcameventsource import NectarCAMEventSource + from ctapipe.io.lsteventsource import NectarCAMEventSource reader = EventSourceFactory.produce(input_url=example_file_path) assert isinstance(reader, NectarCAMEventSource) From 26050bcfb537776248a8fbd9ba40dc1439ad64c5 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 21 Sep 2018 11:23:00 +0200 Subject: [PATCH 22/82] Corrected bug in reading the table CameraConfig --- ctapipe/io/containers.py | 4 +-- ctapipe/io/lsteventsource.py | 43 ++++++++++++++++++++++++++++++ ctapipe/io/nectarcameventsource.py | 26 +++++++++--------- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index d27642b80bc..e93179f0f1b 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -464,9 +464,9 @@ class NectarCAMServiceContainer(Container): idaq_version = Field(0o0, "idaq version") cdhs_version = Field(0o0, "cdhs version") - #acquisition_mode = Field(-1, "acquisition mode") + acquisition_mode = Field(-1, "acquisition mode") algorithms = Field(None, "algorithms") - pre_proc_algorithms = Field(None, "pre processing algorithms") + #pre_proc_algorithms = Field(None, "pre processing algorithms") module_ids = Field([], "module ids") num_modules = Field(-1, "number of modules") diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index d03d5ca2c48..586208c2143 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -158,6 +158,47 @@ def fill_r0_container_from_zfile(self, container, event): ) + def _build_subarray_info(self, file): + """ + constructs a SubarrayDescription object from the info in an + EventIO/HESSSIO file + + Parameters + ---------- + file: HessioFile + The open pyhessio file + + Returns + ------- + SubarrayDescription : + instrumental information + """ + telescope_ids = list(file.get_telescope_ids()) + subarray = SubarrayDescription("LST Data") + + for tel_id in telescope_ids: + try: + + pix_pos = file.get_pixel_position(tel_id) * u.m + foclen = file.get_optical_foclen(tel_id) * u.m + mirror_area = file.get_mirror_area(tel_id) * u.m ** 2 + num_tiles = file.get_mirror_number(tel_id) + tel_pos = file.get_telescope_position(tel_id) * u.m + + tel = TelescopeDescription.guess(*pix_pos, + equivalent_focal_length=foclen) + tel.optics.mirror_area = mirror_area + tel.optics.num_mirror_tiles = num_tiles + subarray.tels[tel_id] = tel + subarray.positions[tel_id] = tel_pos + + except self.pyhessio.HessioGeneralError: + pass + + return subarray + + + class MultiFiles: ''' In LST they have multiple file writers, which save the incoming events @@ -258,3 +299,5 @@ def __len__(self): def num_inputs(self): return len(self._file) + + diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index 66665cc89ec..a561d928091 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -86,19 +86,19 @@ def fill_lst_service_container_from_zfile(self, container, camera_config): svc_container.telescope_id = camera_config.telescope_id svc_container.cs_serial = camera_config.cs_serial svc_container.configuration_id = camera_config.configuration_id - #svc_container.acquisition_mode = camera_config.acquisition_mode + svc_container.acquisition_mode = camera_config.nectarcam.acquisition_mode svc_container.date = camera_config.date svc_container.num_pixels = camera_config.num_pixels svc_container.num_samples = camera_config.num_samples svc_container.pixel_ids = camera_config.expected_pixels_id svc_container.data_model_version = camera_config.data_model_version - svc_container.num_modules = camera_config.lstcam.num_modules - svc_container.module_ids = camera_config.lstcam.expected_modules_id - svc_container.idaq_version = camera_config.lstcam.idaq_version - svc_container.cdhs_version = camera_config.lstcam.cdhs_version - svc_container.algorithms = camera_config.lstcam.algorithms - svc_container.pre_proc_algorithms = camera_config.lstcam.pre_proc_algorithms + svc_container.num_modules = camera_config.nectarcam.num_modules + svc_container.module_ids = camera_config.nectarcam.expected_modules_id + svc_container.idaq_version = camera_config.nectarcam.idaq_version + svc_container.cdhs_version = camera_config.nectarcam.cdhs_version + svc_container.algorithms = camera_config.nectarcam.algorithms + #vc_container.pre_proc_algorithms = camera_config.nectarcam.pre_proc_algorithms @@ -112,12 +112,12 @@ def fill_lst_event_container_from_zfile(self, container, event): event_container.tel_event_id = event.tel_event_id event_container.pixel_status = event.pixel_status event_container.ped_id = event.ped_id - event_container.module_status = event.lstcam.module_status - event_container.extdevices_presence = event.lstcam.extdevices_presence - event_container.tib_data = event.lstcam.tib_data - event_container.cdts_data = event.lstcam.cdts_data - event_container.swat_data = event.lstcam.swat_data - event_container.counters = event.lstcam.counters + event_container.module_status = event.nectarcam.module_status + event_container.extdevices_presence = event.nectarcam.extdevices_presence + event_container.tib_data = event.nectarcam.tib_data + event_container.cdts_data = event.nectarcam.cdts_data + event_container.swat_data = event.nectarcam.swat_data + event_container.counters = event.nectarcam.counters def fill_r0_camera_container_from_zfile(self, container, event): From 052bace1cf4f8aad98052e52f6d830e931601800 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 8 Oct 2018 18:31:57 +0200 Subject: [PATCH 23/82] Added LST1 TelescopeDescription with a call to TelescopeDescription.from_name("LST","LST1Cam"). The description is given by the LST usual optics file (from MC) and a specifically created camera file, called LST1Cam.camgeom.fits.gz For the moment, the position of the telescope is taken from MC: (50 m, 50 m, 16 m) --- ctapipe/io/lsteventsource.py | 65 ++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index d03d5ca2c48..d73373d93ec 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -6,11 +6,16 @@ """ import numpy as np -from os.path import exists + +from astropy import units as u +from os import listdir +from os import getcwd from ctapipe.core import Provenance +from ctapipe.instrument import TelescopeDescription, SubarrayDescription from .eventsource import EventSource from .containers import LSTDataContainer + __all__ = ['LSTEventSource'] @@ -21,19 +26,36 @@ def __init__(self, config=None, tool=None, **kwargs): self.multi_file = MultiFiles(self.input_url) self.camera_config = self.multi_file.camera_config - self.log.info("Read {} input files".format(self.multi_file.num_inputs())) + def _generator(self): # container for LST data data = LSTDataContainer() data.meta['input_url'] = self.input_url + data.meta['max_events'] = self.max_events + # fill LST data from the CameraConfig table self.fill_lst_service_container_from_zfile(data.lst, self.camera_config) + # Instrument information + # LSTs telescope position taken from MC from the moment + tel_pos = np.array([50., 50., 16.])* u.m + + for tel_id in data.lst.tels_with_data: + + assert (tel_id == 0) # only LST1 for the moment (id = 0) + subarray = SubarrayDescription("LST1 subarray") + + # camera info from file LST1Cam.camgeom.fits.gz + subarray.tels[tel_id] = TelescopeDescription.from_name("LST","LST1Cam") + subarray.positions[tel_id] = tel_pos + + data.inst.subarray=subarray + # loop on events for count, event in enumerate(self.multi_file): @@ -184,22 +206,27 @@ def __init__(self, input_url): self._events_table = {} self._camera_config = {} - paths = [input_url, ] - - # test how many files are there - if '000.fits.fz' in input_url: - i = 0 - while True: - input_url = input_url.replace(str(i).zfill(3) + - '.fits.fz', str(i + 1) - .zfill(3) + '.fits.fz') - if exists(input_url): - paths.append(input_url) - # keep track of all input files - Provenance().add_input_file(input_url, role='dl0.sub.evt') - i = i + 1 - else: - break + + # test how many streams are there: + # file name must be [stream name].[all the rest] + # All the files with the same [all the rest] are opened + + if ('/' in input_url): + dir, name = input_url.rsplit('/', 1) + else: + dir = getcwd() + name = input_url + + ls = listdir(dir) + stream, run = name.split('.', 1) + paths = [] + for file_name in ls: + if run in file_name: + full_name=dir + '/' + file_name + paths.append(full_name) + Provenance().add_input_file(full_name, role='dl0.sub.evt') + + # open the files and get the first fits Tables from protozfits import File @@ -258,3 +285,5 @@ def __len__(self): def num_inputs(self): return len(self._file) + + From d85b241f12018f36aa1543220ab05ceea3a11ad9 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Tue, 9 Oct 2018 17:56:09 +0200 Subject: [PATCH 24/82] Correction of wrong naming --- ctapipe/io/nectarcameventsource.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index a561d928091..a28f45edd0a 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -6,8 +6,8 @@ """ import numpy as np -from os.path import exists -from ctapipe.core import Provenance +from astropy import units as u +from ctapipe.instrument import TelescopeDescription, SubarrayDescription from .eventsource import EventSource from .lsteventsource import MultiFiles from .containers import NectarCAMDataContainer @@ -28,12 +28,26 @@ def __init__(self, config=None, tool=None, **kwargs): def _generator(self): - # container for LST data + # container for NectarCAM data data = NectarCAMDataContainer() data.meta['input_url'] = self.input_url # fill data from the CameraConfig table - self.fill_lst_service_container_from_zfile(data.nectarcam, self.camera_config) + self.fill_nectarcam_service_container_from_zfile(data.nectarcam, self.camera_config) + + # Instrument information + # NectarCam telescope position taken from MC from the moment + tel_pos = np.array([50., 50., 16.]) * u.m + + for tel_id in data.nectarcam.tels_with_data: + assert (tel_id == 0) # only one telescope for the moment (id = 0) + subarray = SubarrayDescription("NectarCAM subarray") + + # camera info from file + subarray.tels[tel_id] = TelescopeDescription.from_name("MST", "ProtoNectarCAM") + subarray.positions[tel_id] = tel_pos + + data.inst.subarray = subarray # loop on events for count, event in enumerate(self.multi_file): @@ -41,7 +55,7 @@ def _generator(self): data.count = count # fill specific NectarCAM event data - self.fill_lst_event_container_from_zfile(data.nectarcam, event) + self.fill_nectarcam_event_container_from_zfile(data.nectarcam, event) # fill general R0 data self.fill_r0_container_from_zfile(data.r0, event) @@ -78,7 +92,7 @@ def is_compatible(file_path): is_nectarcam_file = 'nectarcam_counters' in ttypes return is_protobuf_zfits_file & is_nectarcam_file - def fill_lst_service_container_from_zfile(self, container, camera_config): + def fill_nectarcam_service_container_from_zfile(self, container, camera_config): container.tels_with_data = [camera_config.telescope_id, ] svc_container = container.tel[camera_config.telescope_id].svc @@ -103,7 +117,7 @@ def fill_lst_service_container_from_zfile(self, container, camera_config): - def fill_lst_event_container_from_zfile(self, container, event): + def fill_nectarcam_event_container_from_zfile(self, container, event): event_container = container.tel[self.camera_config.telescope_id].evt From f7854b13cdb77e6d3ecee38a9b7107805c874a8f Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Wed, 10 Oct 2018 16:22:31 +0200 Subject: [PATCH 25/82] Change in hte convention of multiple file reading: Expects file names like: [stream name]Run[all the rest] All the files with the same [all the rest] are opened Set optics.tel_subtype=' ' --- ctapipe/io/lsteventsource.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index d73373d93ec..944b10cf269 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -42,19 +42,25 @@ def _generator(self): self.fill_lst_service_container_from_zfile(data.lst, self.camera_config) # Instrument information - # LSTs telescope position taken from MC from the moment - tel_pos = np.array([50., 50., 16.])* u.m for tel_id in data.lst.tels_with_data: assert (tel_id == 0) # only LST1 for the moment (id = 0) - subarray = SubarrayDescription("LST1 subarray") # camera info from file LST1Cam.camgeom.fits.gz - subarray.tels[tel_id] = TelescopeDescription.from_name("LST","LST1Cam") - subarray.positions[tel_id] = tel_pos + tel_descr = TelescopeDescription.from_name("LST", "LST1Cam") + tel_descr.optics.tel_subtype = '' # to correct bug in reading + tels = {tel_id: tel_descr} - data.inst.subarray=subarray + # LSTs telescope position taken from MC from the moment + tel_pos = {tel_id: [50., 50., 16] * u.m} + + + subarray = SubarrayDescription("LST1 subarray") + subarray.tels = tels + subarray.positions = tel_pos + + data.inst.subarray = subarray # loop on events for count, event in enumerate(self.multi_file): @@ -208,7 +214,7 @@ def __init__(self, input_url): # test how many streams are there: - # file name must be [stream name].[all the rest] + # file name must be [stream name]Run[all the rest] # All the files with the same [all the rest] are opened if ('/' in input_url): @@ -217,9 +223,16 @@ def __init__(self, input_url): dir = getcwd() name = input_url + + if ('Run' in name) : + stream, run = name.split('Run', 1) + else : + run = name + + ls = listdir(dir) - stream, run = name.split('.', 1) paths = [] + for file_name in ls: if run in file_name: full_name=dir + '/' + file_name From d6b6d57d414bd921237b4568a3be077f9542b583 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 15 Oct 2018 13:59:43 +0200 Subject: [PATCH 26/82] Added to the NectarCAM reader the geometrical information of the NectarCAM prototype (file ProtoNectarCAM.camgeom.fits.gz). Changed the way in which the waveform array is filled in order to take into accout the case of missing pixels --- ctapipe/io/containers.py | 2 + ctapipe/io/lsteventsource.py | 1 - ctapipe/io/nectarcameventsource.py | 101 +++++++++++++++++------------ 3 files changed, 61 insertions(+), 43 deletions(-) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 6b722249b51..231236cf0c2 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -26,6 +26,8 @@ 'SST1MCameraContainer', 'LSTContainer', 'LSTCameraContainer', + 'NectarCAMContainer', + 'NectarCAMCameraContainer', 'MCEventContainer', 'MCHeaderContainer', 'MCCameraEventContainer', diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 944b10cf269..b7714b23bbe 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -42,7 +42,6 @@ def _generator(self): self.fill_lst_service_container_from_zfile(data.lst, self.camera_config) # Instrument information - for tel_id in data.lst.tels_with_data: assert (tel_id == 0) # only LST1 for the moment (id = 0) diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index a28f45edd0a..3ef23cf9a1d 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -29,37 +29,46 @@ def __init__(self, config=None, tool=None, **kwargs): def _generator(self): # container for NectarCAM data - data = NectarCAMDataContainer() - data.meta['input_url'] = self.input_url + self.data = NectarCAMDataContainer() + self.data.meta['input_url'] = self.input_url # fill data from the CameraConfig table - self.fill_nectarcam_service_container_from_zfile(data.nectarcam, self.camera_config) + self.fill_nectarcam_service_container_from_zfile() - # Instrument information - # NectarCam telescope position taken from MC from the moment - tel_pos = np.array([50., 50., 16.]) * u.m - for tel_id in data.nectarcam.tels_with_data: + # Instrument information + for tel_id in self.data.nectarcam.tels_with_data: assert (tel_id == 0) # only one telescope for the moment (id = 0) - subarray = SubarrayDescription("NectarCAM subarray") - # camera info from file - subarray.tels[tel_id] = TelescopeDescription.from_name("MST", "ProtoNectarCAM") - subarray.positions[tel_id] = tel_pos + # camera info from file LST1Cam.camgeom.fits.gz + tel_descr = TelescopeDescription.from_name("MST", "ProtoNectarCAM") + tel_descr.optics.tel_subtype = '' # to correct bug in reading + self.n_camera_pixels=tel_descr.camera.n_pixels + tels = {tel_id: tel_descr} + + # LSTs telescope position + tel_pos = {tel_id: [0., 0., 0] * u.m} + + + self.subarray = SubarrayDescription("MST prototype subarray") + self.subarray.tels = tels + self.subarray.positions = tel_pos + + self.data.inst.subarray = self.subarray + - data.inst.subarray = subarray # loop on events for count, event in enumerate(self.multi_file): - data.count = count + self.data.count = count # fill specific NectarCAM event data - self.fill_nectarcam_event_container_from_zfile(data.nectarcam, event) + self.fill_nectarcam_event_container_from_zfile(event) # fill general R0 data - self.fill_r0_container_from_zfile(data.r0, event) - yield data + self.fill_r0_container_from_zfile(event) + yield self.data @staticmethod @@ -92,34 +101,38 @@ def is_compatible(file_path): is_nectarcam_file = 'nectarcam_counters' in ttypes return is_protobuf_zfits_file & is_nectarcam_file - def fill_nectarcam_service_container_from_zfile(self, container, camera_config): - - container.tels_with_data = [camera_config.telescope_id, ] - svc_container = container.tel[camera_config.telescope_id].svc - - svc_container.telescope_id = camera_config.telescope_id - svc_container.cs_serial = camera_config.cs_serial - svc_container.configuration_id = camera_config.configuration_id - svc_container.acquisition_mode = camera_config.nectarcam.acquisition_mode - svc_container.date = camera_config.date - svc_container.num_pixels = camera_config.num_pixels - svc_container.num_samples = camera_config.num_samples - svc_container.pixel_ids = camera_config.expected_pixels_id - svc_container.data_model_version = camera_config.data_model_version - - svc_container.num_modules = camera_config.nectarcam.num_modules - svc_container.module_ids = camera_config.nectarcam.expected_modules_id - svc_container.idaq_version = camera_config.nectarcam.idaq_version - svc_container.cdhs_version = camera_config.nectarcam.cdhs_version - svc_container.algorithms = camera_config.nectarcam.algorithms + def fill_nectarcam_service_container_from_zfile(self): + + + #container.tels_with_data = [camera_config.telescope_id, ] + #svc_container = container.tel[self.camera_config.telescope_id].svc + self.data.nectarcam.tels_with_data = [self.camera_config.telescope_id, ] + svc_container = self.data.nectarcam.tel[self.camera_config.telescope_id].svc + + svc_container.telescope_id = self.camera_config.telescope_id + svc_container.cs_serial = self.camera_config.cs_serial + svc_container.configuration_id = self.camera_config.configuration_id + svc_container.acquisition_mode = self.camera_config.nectarcam.acquisition_mode + svc_container.date = self.camera_config.date + svc_container.num_pixels = self.camera_config.num_pixels + svc_container.num_samples = self.camera_config.num_samples + svc_container.pixel_ids = self.camera_config.expected_pixels_id + svc_container.data_model_version = self.camera_config.data_model_version + + svc_container.num_modules = self.camera_config.nectarcam.num_modules + svc_container.module_ids = self.camera_config.nectarcam.expected_modules_id + svc_container.idaq_version = self.camera_config.nectarcam.idaq_version + svc_container.cdhs_version = self.camera_config.nectarcam.cdhs_version + svc_container.algorithms = self.camera_config.nectarcam.algorithms #vc_container.pre_proc_algorithms = camera_config.nectarcam.pre_proc_algorithms - def fill_nectarcam_event_container_from_zfile(self, container, event): + def fill_nectarcam_event_container_from_zfile(self, event): - event_container = container.tel[self.camera_config.telescope_id].evt + # event_container = container.tel[self.camera_config.telescope_id].evt + event_container = self.data.nectarcam.tel[self.camera_config.telescope_id].evt event_container.configuration_id = event.configuration_id event_container.event_id = event.event_id @@ -136,6 +149,7 @@ def fill_nectarcam_event_container_from_zfile(self, container, event): def fill_r0_camera_container_from_zfile(self, container, event): + container.num_samples = self.camera_config.num_samples container.trigger_time = event.trigger_time_s container.trigger_type = event.trigger_type @@ -152,19 +166,22 @@ def fill_r0_camera_container_from_zfile(self, container, event): "N_chan x N_pix x N_samples= '{}'" .format(event.waveform.shape[0])) + container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) - container.waveform = np.array( - ( + reshaped_waveform = np.array( event.waveform - ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples)) + ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) + container.waveform[:, self.camera_config.expected_pixels_id,:] = reshaped_waveform - def fill_r0_container_from_zfile(self, container, event): + def fill_r0_container_from_zfile(self, event): + container=self.data.r0 container.obs_id = -1 container.event_id = event.event_id container.tels_with_data = [self.camera_config.telescope_id, ] r0_camera_container = container.tel[self.camera_config.telescope_id] + self.fill_r0_camera_container_from_zfile( r0_camera_container, event From 7d4c033e8b085f1ee614d0a4560bb819b6e0ac8a Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 15 Oct 2018 14:12:54 +0200 Subject: [PATCH 27/82] Changed the name of the camera geometry file to PrototypeNectarCAM.camgeom.fits.gz --- ctapipe/io/nectarcameventsource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index 3ef23cf9a1d..d1758840791 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -41,7 +41,7 @@ def _generator(self): assert (tel_id == 0) # only one telescope for the moment (id = 0) # camera info from file LST1Cam.camgeom.fits.gz - tel_descr = TelescopeDescription.from_name("MST", "ProtoNectarCAM") + tel_descr = TelescopeDescription.from_name("MST", "PrototypeNectarCAM") tel_descr.optics.tel_subtype = '' # to correct bug in reading self.n_camera_pixels=tel_descr.camera.n_pixels tels = {tel_id: tel_descr} From 60bde0f9b8cbe6043ff5fa304ee221e6b02b88b7 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 15 Oct 2018 15:38:54 +0200 Subject: [PATCH 28/82] Prepared code for a variable number of modules present in the data --- ctapipe/io/lsteventsource.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 944b10cf269..0b1757744ae 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -50,6 +50,7 @@ def _generator(self): # camera info from file LST1Cam.camgeom.fits.gz tel_descr = TelescopeDescription.from_name("LST", "LST1Cam") tel_descr.optics.tel_subtype = '' # to correct bug in reading + self.n_camera_pixels = tel_descr.camera.n_pixels tels = {tel_id: tel_descr} # LSTs telescope position taken from MC from the moment @@ -167,6 +168,13 @@ def fill_r0_camera_container_from_zfile(self, container, event): "N_chan x N_pix x N_samples= '{}'" .format(event.waveform.shape[0])) + # container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) + + # reshaped_waveform = np.array( + # event.waveform +# ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) + # container.waveform[:, self.camera_config.expected_pixels_id,:] = reshaped_waveform + container.waveform = np.array( ( From 1c043d3c6bc9c24003d516eaafc28421757459f9 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 22 Oct 2018 13:20:18 +0200 Subject: [PATCH 29/82] Moved to protozfits libray version 1.4.2 Added Instrument information to NectarCAM reader from file PrototypeNectarCAM.camgeom.fits.gz --- .travis.yml | 2 +- ctapipe/io/lsteventsource.py | 7 ++----- ctapipe/io/nectarcameventsource.py | 3 ++- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 61dc9f03488..5272b19a313 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ install: - pip install codecov # ----- SST1M: - conda install protobuf - - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.4.0.tar.gz + - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.4.2.tar.gz # ----- end of SST1M # ----- target_software - conda install -c conda-forge cfitsio diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index b7714b23bbe..b981a4667ec 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -2,9 +2,8 @@ """ EventSource for LSTCam protobuf-fits.fz-files. -Needs protozfits v1.4.0 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.4.2 from github.com/cta-sst-1m/protozfitsreader """ - import numpy as np from astropy import units as u @@ -296,6 +295,4 @@ def __len__(self): return total_length def num_inputs(self): - return len(self._file) - - + return len(self._file) \ No newline at end of file diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index d1758840791..05150f11731 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -2,7 +2,7 @@ """ EventSource for LSTCam protobuf-fits.fz-files. -Needs protozfits v1.4.0 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.4.2 from github.com/cta-sst-1m/protozfitsreader """ import numpy as np @@ -43,6 +43,7 @@ def _generator(self): # camera info from file LST1Cam.camgeom.fits.gz tel_descr = TelescopeDescription.from_name("MST", "PrototypeNectarCAM") tel_descr.optics.tel_subtype = '' # to correct bug in reading + tel_descr.camera.rotate(10.3*u.deg) self.n_camera_pixels=tel_descr.camera.n_pixels tels = {tel_id: tel_descr} From 3bc4eb9f5b406736c0ff155c3564d538c272a2f0 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 22 Oct 2018 13:46:44 +0200 Subject: [PATCH 30/82] Moved to version 1.4.2 of the protozfits library --- .travis.yml | 2 +- ctapipe/io/lsteventsource.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 61dc9f03488..5272b19a313 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ install: - pip install codecov # ----- SST1M: - conda install protobuf - - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.4.0.tar.gz + - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.4.2.tar.gz # ----- end of SST1M # ----- target_software - conda install -c conda-forge cfitsio diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 0b1757744ae..a7fb8ab0f3a 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -2,7 +2,7 @@ """ EventSource for LSTCam protobuf-fits.fz-files. -Needs protozfits v1.4.0 from github.com/cta-sst-1m/protozfitsreader +Needs protozfits v1.4.2 from github.com/cta-sst-1m/protozfitsreader """ import numpy as np @@ -168,18 +168,18 @@ def fill_r0_camera_container_from_zfile(self, container, event): "N_chan x N_pix x N_samples= '{}'" .format(event.waveform.shape[0])) - # container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) + container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) - # reshaped_waveform = np.array( - # event.waveform -# ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) - # container.waveform[:, self.camera_config.expected_pixels_id,:] = reshaped_waveform + reshaped_waveform = np.array( + event.waveform + ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) + container.waveform[:, self.camera_config.expected_pixels_id,:] = reshaped_waveform - container.waveform = np.array( - ( - event.waveform - ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples)) + # container.waveform = np.array( + # ( + # event.waveform + # ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples)) def fill_r0_container_from_zfile(self, container, event): From 984143159f655c5bc7a50e63240d737b876452d2 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 22 Oct 2018 15:14:37 +0200 Subject: [PATCH 31/82] Minor change --- ctapipe/io/lsteventsource.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index a7fb8ab0f3a..da674769db9 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -173,6 +173,7 @@ def fill_r0_camera_container_from_zfile(self, container, event): reshaped_waveform = np.array( event.waveform ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) + container.waveform[:, self.camera_config.expected_pixels_id,:] = reshaped_waveform From 5eb005bf1579f366620b76c42c06521cdb01065c Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 25 Oct 2018 14:12:22 +0200 Subject: [PATCH 32/82] Minor changes --- ctapipe/io/lsteventsource.py | 12 +++---- ctapipe/io/nectarcameventsource.py | 6 +++- ctapipe/io/tests/test_lsteventsource.py | 7 ++-- ctapipe/io/tests/test_nectarcameventsource.py | 32 ++++++++++--------- 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 2ae2d1a2f0e..66bb1f7fb0d 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -116,9 +116,6 @@ def fill_lst_service_container_from_zfile(self): self.data.lst.tels_with_data = [self.camera_config.telescope_id, ] svc_container = self.data.lst.tel[self.camera_config.telescope_id].svc - #container.tels_with_data = [camera_config.telescope_id, ] - #svc_container = container.tel[camera_config.telescope_id].svc - svc_container.telescope_id = self.camera_config.telescope_id svc_container.cs_serial = self.camera_config.cs_serial svc_container.configuration_id = self.camera_config.configuration_id @@ -140,7 +137,7 @@ def fill_lst_service_container_from_zfile(self): def fill_lst_event_container_from_zfile(self,event): - #event_container = container.tel[self.camera_config.telescope_id].evt + event_container = self.data.lst.tel[self.camera_config.telescope_id].evt event_container.configuration_id = event.configuration_id @@ -177,13 +174,16 @@ def fill_r0_camera_container_from_zfile(self,container, event): "N_chan x N_pix x N_samples= '{}'" .format(event.waveform.shape[0])) - container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) reshaped_waveform = np.array( event.waveform ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) - container.waveform[:, self.camera_config.expected_pixels_id,:] = reshaped_waveform + # initialize the waveform container to zero + container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) + + # re-order the waveform following the expected_pixels_id values (rank = pixel id) + container.waveform[:, self.camera_config.expected_pixels_id, :] = reshaped_waveform def fill_r0_container_from_zfile(self, event): diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index 9596ef926ae..50b4555b126 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -172,11 +172,15 @@ def fill_r0_camera_container_from_zfile(self, container, event): "N_chan x N_pix x N_samples= '{}'" .format(event.waveform.shape[0])) - container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) reshaped_waveform = np.array( event.waveform ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) + + # initialize the waveform container to zero + container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) + + # re-order the waveform following the expected_pixels_id values (rank = pixel id) container.waveform[:, self.camera_config.expected_pixels_id,:] = reshaped_waveform diff --git a/ctapipe/io/tests/test_lsteventsource.py b/ctapipe/io/tests/test_lsteventsource.py index 8ff3a5a9bf3..d00e843099f 100644 --- a/ctapipe/io/tests/test_lsteventsource.py +++ b/ctapipe/io/tests/test_lsteventsource.py @@ -2,7 +2,7 @@ import os import pytest -pytest.importorskip("protozfits", minversion="1.4.0") +pytest.importorskip("protozfits", minversion="1.4.2") example_file_path = resource_filename( 'protozfits', @@ -31,9 +31,10 @@ def test_loop_over_events(): for telid in event.r0.tels_with_data: assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i n_gain = 2 - num_pixels = event.lst.tel[telid].svc.num_pixels + n_camera_pixels = event.inst.subarray.tels[telid].camera.n_pixels num_samples = event.lst.tel[telid].svc.num_samples - waveform_shape = (n_gain, num_pixels, num_samples) + waveform_shape = (n_gain, n_camera_pixels, num_samples) + assert event.r0.tel[telid].waveform.shape == waveform_shape # make sure max_events works diff --git a/ctapipe/io/tests/test_nectarcameventsource.py b/ctapipe/io/tests/test_nectarcameventsource.py index 7982e74627e..cc183bb89a1 100644 --- a/ctapipe/io/tests/test_nectarcameventsource.py +++ b/ctapipe/io/tests/test_nectarcameventsource.py @@ -2,21 +2,22 @@ import os import pytest -pytest.importorskip("protozfits", minversion="1.4.0") - -example_file_path = resource_filename( - 'protozfits', - os.path.join( - 'tests', - 'resources', - '' - ) -) +from ctapipe.utils import get_dataset_path +pytest.importorskip("protozfits", minversion="1.4.2") + +#example_file_path = resource_filename( +# 'protozfits', +# os.path.join( +# 'tests', +# 'resources', +# 'NectarCAM.Run0890.10events.fits.fz' +# ) +#) FIRST_EVENT_NUMBER_IN_FILE = 1 -example_file_path="/ctadata/NectarCAM/2018/20180809/NectarCAM.Run0889.0000.fits.fz" - +#example_file_path="NectarCam.Run0890.10events.fits.fz" +example_file_path = get_dataset_path("NectarCam.Run0890.10events.fits.fz") def test_loop_over_events(): from ctapipe.io.nectarcameventsource import NectarCAMEventSource @@ -32,9 +33,10 @@ def test_loop_over_events(): for telid in event.r0.tels_with_data: assert event.r0.event_id == FIRST_EVENT_NUMBER_IN_FILE + i n_gain = 2 - num_pixels = event.nectarcam.tel[telid].svc.num_pixels + n_camera_pixels = event.inst.subarray.tels[telid].camera.n_pixels + num_samples = event.nectarcam.tel[telid].svc.num_samples - waveform_shape = (n_gain, num_pixels, num_samples) + waveform_shape = (n_gain, n_camera_pixels, num_samples) assert event.r0.tel[telid].waveform.shape == waveform_shape # make sure max_events works @@ -49,7 +51,7 @@ def test_is_compatible(): def test_factory_for_nectarcam_file(): from ctapipe.io.eventsourcefactory import EventSourceFactory - from ctapipe.io.lsteventsource import NectarCAMEventSource + from ctapipe.io.nectarcameventsource import NectarCAMEventSource reader = EventSourceFactory.produce(input_url=example_file_path) assert isinstance(reader, NectarCAMEventSource) From 3b8de9ac28fd29c3ff2ab119e40e024d3d3ed748 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Tue, 30 Oct 2018 10:41:50 +0100 Subject: [PATCH 33/82] - Changed the way data files are opened: Now all files of a give Run are opened togehter in order to read event in the proper order (probably this will be changed as soon the zfit writer working criteria will be better defined). - Minor cosmetics changes --- ctapipe/io/containers.py | 5 +- ctapipe/io/lsteventsource.py | 101 ++++++++---------- ctapipe/io/nectarcameventsource.py | 24 +++-- ctapipe/io/tests/test_nectarcameventsource.py | 11 +- 4 files changed, 63 insertions(+), 78 deletions(-) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 50685eb7d23..c4b18e4d89a 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -424,7 +424,7 @@ class NectarCAMServiceContainer(Container): cdhs_version = Field(0o0, "cdhs version") acquisition_mode = Field(-1, "acquisition mode") algorithms = Field(None, "algorithms") - #pre_proc_algorithms = Field(None, "pre processing algorithms") + # pre_proc_algorithms = Field(None, "pre processing algorithms") module_ids = Field([], "module ids") num_modules = Field(-1, "number of modules") @@ -454,7 +454,8 @@ class NectarCAMCameraContainer(Container): Container for Fields that are specific to each NectarCAM camera """ evt = Field(NectarCAMEventContainer(), "NectarCAM specific event Information") - svc = Field(NectarCAMServiceContainer(), "NectarCAM specific camera_config Information") + svc = Field(NectarCAMServiceContainer(), "NectarCAM specific camera_config " + "Information") diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 66bb1f7fb0d..feab065128a 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -7,10 +7,12 @@ import numpy as np from astropy import units as u -from os import listdir +#from os import listdir +import glob from os import getcwd from ctapipe.core import Provenance -from ctapipe.instrument import TelescopeDescription, SubarrayDescription, CameraGeometry, OpticsDescription +from ctapipe.instrument import TelescopeDescription, SubarrayDescription, \ + CameraGeometry, OpticsDescription from .eventsource import EventSource from .containers import LSTDataContainer @@ -43,14 +45,14 @@ def _generator(self): # Instrument information for tel_id in self.data.lst.tels_with_data: - assert (tel_id == 0) # only LST1 for the moment (id = 0) + assert (tel_id == 0) # only LST1 for the moment (id = 0) # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("LST") - optics.tel_subtype = '' # to correct bug in reading + optics.tel_subtype = '' # to correct bug in reading # camera info from LSTCam-[geometry_version].camgeom.fits.gz file - geometry_version=1 + geometry_version = 1 camera = CameraGeometry.from_name("LSTCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) @@ -133,9 +135,7 @@ def fill_lst_service_container_from_zfile(self): svc_container.pre_proc_algorithms = self.camera_config.lstcam.pre_proc_algorithms - - - def fill_lst_event_container_from_zfile(self,event): + def fill_lst_event_container_from_zfile(self, event): event_container = self.data.lst.tel[self.camera_config.telescope_id].evt @@ -156,7 +156,7 @@ def fill_lst_event_container_from_zfile(self,event): event_container.drs_tag_status = event.lstcam.drs_tag_status event_container.drs_tag = event.lstcam.drs_tag - def fill_r0_camera_container_from_zfile(self,container, event): + def fill_r0_camera_container_from_zfile(self, container, event): container.num_samples = self.camera_config.num_samples container.trigger_time = event.trigger_time_s @@ -180,14 +180,18 @@ def fill_r0_camera_container_from_zfile(self,container, event): ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) # initialize the waveform container to zero - container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) + container.waveform = np.zeros([n_gains, self.n_camera_pixels, + container.num_samples]) # re-order the waveform following the expected_pixels_id values (rank = pixel id) - container.waveform[:, self.camera_config.expected_pixels_id, :] = reshaped_waveform + container.waveform[:, self.camera_config.expected_pixels_id, :] =\ + reshaped_waveform def fill_r0_container_from_zfile(self, event): + container = self.data.r0 + container.obs_id = -1 container.event_id = event.event_id @@ -200,23 +204,12 @@ def fill_r0_container_from_zfile(self, event): class MultiFiles: - ''' - In LST they have multiple file writers, which save the incoming events - into different files, so in case one has 10 events and 4 files, - it might look like this: - f1 = [0, 4] - f2 = [1, 5, 8] - f3 = [2, 6, 9] - f4 = [3, 7] - The task of MultiZFitsFiles is to open these 4 files simultaneously - and return the events in the correct order, so the user does not really - have to know about these existence of 4 files. - - In case of multiple input files the name of the files must finish with suffix - *000.fits.fz, *001.fits.fz, etc... and the user must give as input_url the name - of the first file (*000.fits.fz). The program will search for the other files. - In the case of only one input file the input_url can have any form. - ''' + + """ + This class open all the files related to a given run xxxx if the file name + contains Runxxxx and all the files are in the same directory + (this will probably be changed in the future) + """ def __init__(self, input_url): @@ -224,35 +217,29 @@ def __init__(self, input_url): self._events = {} self._events_table = {} self._camera_config = {} + self.camera_config = None - - # test how many streams are there: - # file name must be [stream name]Run[all the rest] - # All the files with the same [all the rest] are opened - + # In the input_url contain Runxxxx, test how many files with Runxxxx in the name + # are in the directory and open all of them + # If not "Run" tag is present in the name it opens only the input_url file if ('/' in input_url): - dir, name = input_url.rsplit('/', 1) + dir_name, name = input_url.rsplit('/', 1) else: - dir = getcwd() + dir_name = getcwd() name = input_url - - if ('Run' in name) : - stream, run = name.split('Run', 1) - else : + if ('Run' in name): + idx = name.find('Run') + run = name[idx:idx + 7] + else: run = name - - ls = listdir(dir) + ls = glob.glob(dir_name + "/*.fits.fz") paths = [] - for file_name in ls: if run in file_name: - full_name=dir + '/' + file_name - paths.append(full_name) - Provenance().add_input_file(full_name, role='dl0.sub.evt') - - + paths.append(file_name) + Provenance().add_input_file(file_name, role='dl0.sub.evt') # open the files and get the first fits Tables from protozfits import File @@ -263,19 +250,21 @@ def __init__(self, input_url): try: self._events[path] = next(self._file[path].Events) - self._camera_config[path] = next(self._file[path].CameraConfig) + + # verify where the CameraConfig is present + if 'CameraConfig' in self._file[path].__dict__.keys(): + self._camera_config[path] = next(self._file[path].CameraConfig) + + # for the moment it takes the first CameraConfig it finds (to be changed) + if(self.camera_config == None): + self.camera_config = self._camera_config[path] + except StopIteration: pass - # verify that the configuration_id of all files are the same - # in the CameraConfig table - for path in paths: - assert (self._camera_config[path].configuration_id - == self._camera_config[paths[0]].configuration_id) - - # keep the cameraConfig of the first file - self.camera_config = self._camera_config[paths[0]] + # verify that soemwhere the CameraConfing is present + assert (self.camera_config) def __iter__(self): return self diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index 50b4555b126..e6cc538182d 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -7,7 +7,8 @@ import numpy as np from astropy import units as u -from ctapipe.instrument import TelescopeDescription, SubarrayDescription, CameraGeometry, OpticsDescription +from ctapipe.instrument import TelescopeDescription, SubarrayDescription, \ + CameraGeometry, OpticsDescription from .eventsource import EventSource from .lsteventsource import MultiFiles from .containers import NectarCAMDataContainer @@ -42,17 +43,17 @@ def _generator(self): # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("MST") - optics.tel_subtype = '' # to correct bug in reading + optics.tel_subtype = '' # to correct bug in reading # camera info from NectarCam-[geometry_version].camgeom.fits.gz file geometry_version = 1 - camera = CameraGeometry.from_name("NectarCam",geometry_version) + camera = CameraGeometry.from_name("NectarCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) tel_descr.optics.tel_subtype = '' # to correct bug in reading - tel_descr.camera.rotate(10.3*u.deg) - self.n_camera_pixels=tel_descr.camera.n_pixels + tel_descr.camera.rotate(10.3 * u.deg) + self.n_camera_pixels = tel_descr.camera.n_pixels tels = {tel_id: tel_descr} # LSTs telescope position @@ -130,7 +131,7 @@ def fill_nectarcam_service_container_from_zfile(self): svc_container.idaq_version = self.camera_config.nectarcam.idaq_version svc_container.cdhs_version = self.camera_config.nectarcam.cdhs_version svc_container.algorithms = self.camera_config.nectarcam.algorithms - #svc_container.pre_proc_algorithms = camera_config.nectarcam.pre_proc_algorithms + # svc_container.pre_proc_algorithms = camera_config.nectarcam.pre_proc_algorithms @@ -174,18 +175,21 @@ def fill_r0_camera_container_from_zfile(self, container, event): reshaped_waveform = np.array( - event.waveform + event.waveform ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) # initialize the waveform container to zero - container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) + container.waveform = np.zeros([n_gains, + self.n_camera_pixels, + container.num_samples]) # re-order the waveform following the expected_pixels_id values (rank = pixel id) - container.waveform[:, self.camera_config.expected_pixels_id,:] = reshaped_waveform + container.waveform[:, self.camera_config.expected_pixels_id, :] \ + = reshaped_waveform def fill_r0_container_from_zfile(self, event): - container=self.data.r0 + container = self.data.r0 container.obs_id = -1 container.event_id = event.event_id diff --git a/ctapipe/io/tests/test_nectarcameventsource.py b/ctapipe/io/tests/test_nectarcameventsource.py index cc183bb89a1..7351d577afe 100644 --- a/ctapipe/io/tests/test_nectarcameventsource.py +++ b/ctapipe/io/tests/test_nectarcameventsource.py @@ -5,18 +5,9 @@ from ctapipe.utils import get_dataset_path pytest.importorskip("protozfits", minversion="1.4.2") -#example_file_path = resource_filename( -# 'protozfits', -# os.path.join( -# 'tests', -# 'resources', -# 'NectarCAM.Run0890.10events.fits.fz' -# ) -#) - FIRST_EVENT_NUMBER_IN_FILE = 1 -#example_file_path="NectarCam.Run0890.10events.fits.fz" +# example_file_path="NectarCam.Run0890.10events.fits.fz" example_file_path = get_dataset_path("NectarCam.Run0890.10events.fits.fz") def test_loop_over_events(): From 070c2a8223469f17e87ff89bd8bb8df2fd7c9451 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Tue, 30 Oct 2018 11:57:59 +0100 Subject: [PATCH 34/82] Change name of test data file --- ctapipe/io/tests/test_nectarcameventsource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/io/tests/test_nectarcameventsource.py b/ctapipe/io/tests/test_nectarcameventsource.py index 7351d577afe..ed56b9ea604 100644 --- a/ctapipe/io/tests/test_nectarcameventsource.py +++ b/ctapipe/io/tests/test_nectarcameventsource.py @@ -8,7 +8,7 @@ FIRST_EVENT_NUMBER_IN_FILE = 1 # example_file_path="NectarCam.Run0890.10events.fits.fz" -example_file_path = get_dataset_path("NectarCam.Run0890.10events.fits.fz") +example_file_path = get_dataset_path("NectarCAM.Run0890.10events.fits.fz") def test_loop_over_events(): from ctapipe.io.nectarcameventsource import NectarCAMEventSource From eb45471be16872354d9950a9e03a5a49b91b1bc4 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 9 Nov 2018 11:00:29 +0100 Subject: [PATCH 35/82] Added a root container for Monitor Data: MonDataContainer and sub-containers forthe flat-field coefficients --- ctapipe/io/containers.py | 69 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index c4b18e4d89a..825fc4fe7c1 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -44,6 +44,12 @@ 'LeakageContainer', 'ConcentrationContainer', 'TimingParametersContainer', + 'StatInfoContainer', + 'FlatFieldCameraContainer', + 'PedestalCameraContainer', + 'FlatFieldContainer', + 'PedestalContainer', + 'MonitorDataContainer' ] @@ -758,3 +764,66 @@ class TimingParametersContainer(Container): """ slope = Field(nan, 'Slope of arrival times along main shower axis') intercept = Field(nan, 'intercept of arrival times along main shower axis') + + +class FlatFieldCameraContainer(Container): + """ + Container for relative flat-field coefficients + per camera + """ + + mean_time_s = Field(0, 'Mean time, seconds since reference') + range_time_s = Field([], 'Range of time') + + n_events = Field(0,'Number of events used for statistics') + + relative_gain_mean = Field(None, + "np array of the relative flat-field coefficient mean (n_chan X N_pix)" + ) + relative_gain_median = Field(None, + "np array of the relative flat-field coefficient median (n_chan X N_pix)" + ) + + relative_gain_rms = Field(None, + "np array of the relative flat-field coefficient rms (n_chan X N_pix)" + ) + + + +class PedestalCameraContainer(Container): + """ + Container for pedestals per camera + """ + # still to be filled + + + +class FlatFieldContainer(Container): + """ + Container for relative flat field coefficients + """ + tels_with_data = Field([], "list of telescopes with data") + tel = Field( + Map(FlatFieldCameraContainer), + "map of tel_id to FlatFiledCameraContainer") + + + +class PedestalContainer(Container): + """ + Container for pedestal data + """ + tels_with_data = Field([], "list of telescopes with data") + tel = Field( + Map(PedestalCameraContainer), + "map of tel_id to PedestalCameraContainer") + +class MonDataContainer(Container): + """ + Root container for MON data + """ + flatfield = Field(FlatFieldContainer(), "Relative flat field data") + pedestal = Field(PedestalContainer(), "Pedestal data") + + + From 06d202660c7a05512d8b032e37bd10f3a9d903c2 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 19 Nov 2018 10:28:33 +0100 Subject: [PATCH 36/82] Added code for flatfield calibration --- ctapipe/calib/camera/__init__.py | 1 + ctapipe/calib/camera/flatfield.py | 174 +++++++++++++++++++ ctapipe/calib/camera/tests/test_flatfield.py | 20 +++ ctapipe/io/containers.py | 18 +- ctapipe/io/lsteventsource.py | 12 +- ctapipe/io/nectarcameventsource.py | 5 +- examples/calc_flatfield.py | 100 +++++++++++ 7 files changed, 316 insertions(+), 14 deletions(-) create mode 100644 ctapipe/calib/camera/flatfield.py create mode 100644 ctapipe/calib/camera/tests/test_flatfield.py create mode 100644 examples/calc_flatfield.py diff --git a/ctapipe/calib/camera/__init__.py b/ctapipe/calib/camera/__init__.py index 385056a67a3..9a68433e4c7 100644 --- a/ctapipe/calib/camera/__init__.py +++ b/ctapipe/calib/camera/__init__.py @@ -7,3 +7,4 @@ from .dl0 import * from .dl1 import * from .calibrator import * +from .flatfield import * \ No newline at end of file diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py new file mode 100644 index 00000000000..7bdb1ab22a3 --- /dev/null +++ b/ctapipe/calib/camera/flatfield.py @@ -0,0 +1,174 @@ +""" +Factory for the estimation of the flat field coefficients +""" +from abc import abstractmethod +import numpy as np + +from ctapipe.core import Component, Factory +from ctapipe.core.traits import Int +from ctapipe.io.containers import FlatFieldCameraContainer + +__all__ = [ + 'FlatFieldCalculator', + 'FlasherFlatFieldCalculator', + 'FlatFieldFactory' +] + + +class FlatFieldCalculator(Component): + """ + Parent class for the flat field calculators. Fills the MON.flatfield container. + + """ + max_time_range_s = Int(60, help='Define the maximum time interval per' + ' coefficient flat-filed calculation').tag(config=True) + max_events = Int(1000, help='Define the maximum number of events per ' + ' coefficient flat-filed calculation').tag(config=True) + n_channels = Int(2, help='Define the number of channels to be ' + 'treated ').tag(config=True) + + + def __init__(self, config=None, tool=None, **kwargs): + """ + Parent class for the flat field calculators. Fills the MON.flatfield container. + + Parameters + ---------- + config : traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + Set to None if no configuration to pass. + tool : ctapipe.core.Tool + Tool executable that is calling this component. + Passes the correct logger to the component. + Set to None if no Tool to pass. + + kwargs + + """ + super().__init__(config=config, parent=tool, **kwargs) + self.container = FlatFieldCameraContainer() + + + + @abstractmethod + def calculate_relative_gain(self,event): + """ + Parameters + ---------- + event + + """ + + + +class FlasherFlatFieldCalculator(FlatFieldCalculator): + """ + Class for calculating flat field coefficients witht the + flasher data. Fills the MON.flatfield container. + """ + + def __init__(self, config=None, tool=None, **kwargs): + """ + Parent class for the flat field calculators. Fills the MON.flatfield container. + + Parameters + ---------- + config : traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + Set to None if no configuration to pass. + tool : ctapipe.core.Tool + Tool executable that is calling this component. + Passes the correct logger to the component. + Set to None if no Tool to pass. + eventsource : ctapipe.io.eventsource.EventSource + EventSource that is being used to read the events. The EventSource + contains information (such as metadata or inst) which indicates + the appropriate R1Calibrator to use. + kwargs + + """ + super().__init__(config=config, tool=tool, **kwargs) + + self.log.info("Used events statistics : %d", self.max_events) + self.count = 0 + + + + + def calculate_relative_gain(self, event): + """ + calculate the relative flat filed coefficients + + Parameters + ---------- + event : specific camera + + """ + # initialize the np array at each cycle + + if (self.count == 0): + self.time_start = event.trigger_time + + self.event_median = np.zeros((self.max_events, self.n_channels)) + + n_pix = event.waveform.shape[1] + self.trace_integral = np.zeros((self.max_events, self.n_channels, n_pix)) + + + + trace = event.waveform[:,:,:] + + + # here if necessary subtract pedestals + # e.g. with calc_pedestals_from_traces() + # ... + + # extract the signal (for the moment very rough integral): x(i,j) + # for the moment no check on the values + self.trace_integral[self.count,:,:] = trace[:,:,:].sum(axis=2) + + # extract the median on all the camera per event: (i) + self.event_median[self.count,:] = \ + np.median(self.trace_integral[self.count,:,:], axis=1) + time = event.trigger_time + + # increment the internal counter + self.count = self.count+1 + + # check if to create a calibration event + if ((time - self.time_start) > self.max_time_range_s or self.count == self.max_events): + + + # extract for each pixel : x(i,j)/(i) = g(i,j) + rel_gain_event = self.trace_integral/self.event_median[:,:, np.newaxis] + + # extract the median, mean and std over all the events j and + # fill the container and return it + self.container.mean_time_s = (time - self.time_start)/2 + self.container.range_time_s = [self.time_start, time] + self.container.n_events = self.count + 1 + self.container.relative_gain_median = np.median(rel_gain_event, axis=0) + self.container.relative_gain_mean = np.mean(rel_gain_event, axis=0) + self.container.relative_gain_rms = np.std(rel_gain_event, axis=0) + + # re-initialize the event count + self.count = 0 + + return self.container + + return None + + + + +class FlatFieldFactory(Factory): + """ + Factory to obtain flat-field coefficients + """ + base = FlatFieldCalculator + default = 'FlasherFlatFieldCalculator' + custom_product_help = ('') + + \ No newline at end of file diff --git a/ctapipe/calib/camera/tests/test_flatfield.py b/ctapipe/calib/camera/tests/test_flatfield.py new file mode 100644 index 00000000000..26e5669f4dd --- /dev/null +++ b/ctapipe/calib/camera/tests/test_flatfield.py @@ -0,0 +1,20 @@ +import numpy as np +import pytest + +from ctapipe.calib.camera.flatfield import FlasherFlatFieldCalculator + + + + +def test_FlasherFlatFieldCalculator(): + ff_calculator = FlasherFlatFieldCalculator() + + for chan in [0, 1]: + gs.channel = chan + + waveforms_2g = np.random.normal(size=(2, 1000, 30)) + waveforms_1g, mask = gs.select_gains("NectarCam", waveforms_2g) + + assert waveforms_1g.shape == (1000, 30) + assert (waveforms_1g == waveforms_2g[chan]).all() + assert mask.shape == (1000,) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 825fc4fe7c1..980f8e1365b 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -788,15 +788,7 @@ class FlatFieldCameraContainer(Container): "np array of the relative flat-field coefficient rms (n_chan X N_pix)" ) - - -class PedestalCameraContainer(Container): - """ - Container for pedestals per camera - """ - # still to be filled - - + class FlatFieldContainer(Container): """ @@ -807,7 +799,13 @@ class FlatFieldContainer(Container): Map(FlatFieldCameraContainer), "map of tel_id to FlatFiledCameraContainer") - + +class PedestalCameraContainer(Container): + """ + Container for pedestals per camera + """ + # still to be filled + class PedestalContainer(Container): """ diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index feab065128a..59f6cf617aa 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -29,7 +29,9 @@ def __init__(self, config=None, tool=None, **kwargs): self.camera_config = self.multi_file.camera_config self.log.info("Read {} input files".format(self.multi_file.num_inputs())) - + def rewind(self): + self.multi_file.rewind() + def _generator(self): @@ -52,7 +54,7 @@ def _generator(self): optics.tel_subtype = '' # to correct bug in reading # camera info from LSTCam-[geometry_version].camgeom.fits.gz file - geometry_version = 1 + geometry_version = 2 camera = CameraGeometry.from_name("LSTCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) @@ -297,6 +299,12 @@ def __len__(self): for table in self._events_table.values() ) return total_length + + def rewind(self): + for name, file in self._file.items(): + file.Events.protobuf_i_fits.rewind() + + def num_inputs(self): return len(self._file) diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index e6cc538182d..d4459e47b2c 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -46,13 +46,14 @@ def _generator(self): optics.tel_subtype = '' # to correct bug in reading # camera info from NectarCam-[geometry_version].camgeom.fits.gz file - geometry_version = 1 + geometry_version = 2 camera = CameraGeometry.from_name("NectarCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) tel_descr.optics.tel_subtype = '' # to correct bug in reading - tel_descr.camera.rotate(10.3 * u.deg) + #Stel_descr.camera.rotate(10.3 * u.deg) + self.n_camera_pixels = tel_descr.camera.n_pixels tels = {tel_id: tel_descr} diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py new file mode 100644 index 00000000000..247d4386e63 --- /dev/null +++ b/examples/calc_flatfield.py @@ -0,0 +1,100 @@ +""" +Extract flat field coefficients from flasher data files. +""" + +import os + +import numpy as np +from tqdm import tqdm +from traitlets import Dict, List, Int, Unicode + + +from ctapipe.core import Provenance +from ctapipe.io import HDF5TableWriter +from ctapipe.core import Tool +from ctapipe.io import EventSourceFactory +from ctapipe.calib.camera.flatfield import FlatFieldFactory +from ctapipe.io.containers import MonDataContainer + + +class FlatFieldGenerator(Tool): + name = "FlatFieldGenerator" + description = "Generate a HDF5 file with flat field coefficients" + + + output_file = Unicode('flatfield.hdf5', + help='Name of the output flat field file ' \ + 'file').tag(config=True) + + aliases = Dict(dict(input_file='EventSourceFactory.input_url', + max_events='EventSourceFactory.max_events', + allowed_tels= 'EventSourceFactory.allowed_tels', + generator='FlatFieldFactory.product', + max_time_range_s='FlatFieldFactory.max_time_range_s', + ff_events='FlatFieldFactory.max_events', + n_channels='FlatFieldFactory.n_channels' + )) + + classes = List([EventSourceFactory, + FlatFieldFactory, + MonDataContainer, + HDF5TableWriter + ]) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.eventsource = None + self.flatfield = None + self.container = None + self.writer = None + + def setup(self): + self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" + kwargs = dict(config=self.config, tool=self) + + # open an extractor per camera + self.eventsource = EventSourceFactory.produce(**kwargs) + self.flatfield = FlatFieldFactory.produce(**kwargs) + self.container = MonDataContainer() + self.writer = HDF5TableWriter( + filename=self.output_file, group_name='flatfield', overwrite=True + ) + + + def start(self): + desc = "Flat field coefficient calculator" + + for count, event in enumerate(self.eventsource): + + for tel_id in event.r0.tels_with_data: + # initialize the flat filed containers + self.container.flatfield.tels_with_data.append(tel_id) + #SSself.container.flatfield.tel[tel_id].tel_id = tel_id + + ff_data = self.flatfield.calculate_relative_gain(event.r0.tel[tel_id]) + + if ff_data: + self.container.flatfield.tel[tel_id] = ff_data + table_name='tel_'+str(tel_id) + + self.log.info("write event in table: /flatfield/%s", table_name) + self.writer.write(table_name,ff_data) + + + + + + def finish(self): + + Provenance().add_output_file(self.output_file, + role='mon.tel.flatfield') + self.writer.close() + + +def main(): + exe = FlatFieldGenerator() + exe.run() + + +if __name__ == '__main__': + main() From 8e672466816768b5860a36ffc67ca7c75cc0e6c4 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 19 Nov 2018 10:39:25 +0100 Subject: [PATCH 37/82] Changed geometry version from 1 (with bugs) to 2 --- ctapipe/io/lsteventsource.py | 2 +- ctapipe/io/nectarcameventsource.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index feab065128a..cae68d85cb4 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -52,7 +52,7 @@ def _generator(self): optics.tel_subtype = '' # to correct bug in reading # camera info from LSTCam-[geometry_version].camgeom.fits.gz file - geometry_version = 1 + geometry_version = 2 camera = CameraGeometry.from_name("LSTCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index e6cc538182d..cbfd5a94890 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -46,13 +46,13 @@ def _generator(self): optics.tel_subtype = '' # to correct bug in reading # camera info from NectarCam-[geometry_version].camgeom.fits.gz file - geometry_version = 1 + geometry_version = 2 camera = CameraGeometry.from_name("NectarCam", geometry_version) tel_descr = TelescopeDescription(optics, camera) tel_descr.optics.tel_subtype = '' # to correct bug in reading - tel_descr.camera.rotate(10.3 * u.deg) + self.n_camera_pixels = tel_descr.camera.n_pixels tels = {tel_id: tel_descr} From 4e31c2e4bbca2e5002472f12d5118872c5112f3f Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 19 Nov 2018 11:12:33 +0100 Subject: [PATCH 38/82] Minor changes --- ctapipe/io/lsteventsource.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index cae68d85cb4..a07164e878e 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -207,8 +207,9 @@ class MultiFiles: """ This class open all the files related to a given run xxxx if the file name - contains Runxxxx and all the files are in the same directory - (this will probably be changed in the future) + contains Runxxxx and all the files are in the same directory. If you want + to open only some files, link them from an empty directory + (this will probably be changed in the near future) """ def __init__(self, input_url): From b54060dfe3f5f835918ac7d3c47c10925b2bb78a Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 19 Nov 2018 16:05:56 +0100 Subject: [PATCH 39/82] Flat-field code working with the r0.waveform as input --- ctapipe/calib/camera/flatfield.py | 11 +++---- ctapipe/calib/camera/tests/test_flatfield.py | 34 +++++++++++++------- examples/calc_flatfield.py | 10 +++--- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index 7bdb1ab22a3..62922968a0e 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -82,10 +82,7 @@ def __init__(self, config=None, tool=None, **kwargs): Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. - eventsource : ctapipe.io.eventsource.EventSource - EventSource that is being used to read the events. The EventSource - contains information (such as metadata or inst) which indicates - the appropriate R1Calibrator to use. + kwargs """ @@ -157,8 +154,10 @@ def calculate_relative_gain(self, event): self.count = 0 return self.container - - return None + + else: + + return None diff --git a/ctapipe/calib/camera/tests/test_flatfield.py b/ctapipe/calib/camera/tests/test_flatfield.py index 26e5669f4dd..5e9d62cf94e 100644 --- a/ctapipe/calib/camera/tests/test_flatfield.py +++ b/ctapipe/calib/camera/tests/test_flatfield.py @@ -1,20 +1,30 @@ import numpy as np import pytest +from ctapipe.utils import get_dataset_path +from ctapipe.io.nectarcameventsource import NectarCAMEventSource from ctapipe.calib.camera.flatfield import FlasherFlatFieldCalculator - - def test_FlasherFlatFieldCalculator(): - ff_calculator = FlasherFlatFieldCalculator() + + example_file_path = get_dataset_path("NectarCAM.Run0890.10events.fits.fz") + inputfile_reader = NectarCAMEventSource( + input_url=example_file_path, + max_events=10 + ) - for chan in [0, 1]: - gs.channel = chan - - waveforms_2g = np.random.normal(size=(2, 1000, 30)) - waveforms_1g, mask = gs.select_gains("NectarCam", waveforms_2g) - - assert waveforms_1g.shape == (1000, 30) - assert (waveforms_1g == waveforms_2g[chan]).all() - assert mask.shape == (1000,) + ff_calculator = FlasherFlatFieldCalculator() + ff_calculator.max_events=3 + + for i,event in enumerate(inputfile_reader): + for tel_id in event.r0.tels_with_data: + + + ff_data = ff_calculator.calculate_relative_gain(event.r0.tel[tel_id]) + + if ff_calculator.count == ff_calculator.max_events: + assert(ff_data) + + + diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index 247d4386e63..f64c8d6951a 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -2,11 +2,8 @@ Extract flat field coefficients from flasher data files. """ -import os - -import numpy as np -from tqdm import tqdm -from traitlets import Dict, List, Int, Unicode +#from tqdm import tqdm +from traitlets import Dict, List, Unicode from ctapipe.core import Provenance @@ -67,9 +64,10 @@ def start(self): for count, event in enumerate(self.eventsource): for tel_id in event.r0.tels_with_data: + # initialize the flat filed containers self.container.flatfield.tels_with_data.append(tel_id) - #SSself.container.flatfield.tel[tel_id].tel_id = tel_id + ff_data = self.flatfield.calculate_relative_gain(event.r0.tel[tel_id]) From ffae0ac8637f98c907c689fa3ebdf5377fbd818c Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 23 Nov 2018 09:14:32 +0100 Subject: [PATCH 40/82] First version of a tool for the estimation of the flat field coefficients - Created a FlatFieldGenerator (for the moment in the ctapipe/examples directory) - Created a FlatfieldFactory which calculates the FF coefficients after cleaning and integration of the waveform - Moved pedestals.py module to ctapipe/calib/camera from ctapipe/calib - Added r0.tel[id].pixel_status array to R0CameraContainer in order to mask bad pixels - --- ctapipe/calib/camera/__init__.py | 3 +- ctapipe/calib/camera/flatfield.py | 126 ++++++++++++++++++------ ctapipe/calib/{ => camera}/pedestals.py | 0 ctapipe/image/waveform_cleaning.py | 16 ++- ctapipe/io/containers.py | 2 +- ctapipe/io/lsteventsource.py | 3 + ctapipe/io/nectarcameventsource.py | 6 ++ examples/calc_flatfield.py | 34 +++++-- 8 files changed, 146 insertions(+), 44 deletions(-) rename ctapipe/calib/{ => camera}/pedestals.py (100%) diff --git a/ctapipe/calib/camera/__init__.py b/ctapipe/calib/camera/__init__.py index 9a68433e4c7..91245e2df0d 100644 --- a/ctapipe/calib/camera/__init__.py +++ b/ctapipe/calib/camera/__init__.py @@ -7,4 +7,5 @@ from .dl0 import * from .dl1 import * from .calibrator import * -from .flatfield import * \ No newline at end of file +from .flatfield import * +from .pedestals import * \ No newline at end of file diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index 62922968a0e..dc49e4ef3ce 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -5,6 +5,8 @@ import numpy as np from ctapipe.core import Component, Factory + +from ctapipe.image import ChargeExtractorFactory, WaveformCleanerFactory from ctapipe.core.traits import Int from ctapipe.io.containers import FlatFieldCameraContainer @@ -22,13 +24,13 @@ class FlatFieldCalculator(Component): """ max_time_range_s = Int(60, help='Define the maximum time interval per' ' coefficient flat-filed calculation').tag(config=True) - max_events = Int(1000, help='Define the maximum number of events per ' + max_events = Int(10000, help='Define the maximum number of events per ' ' coefficient flat-filed calculation').tag(config=True) n_channels = Int(2, help='Define the number of channels to be ' 'treated ').tag(config=True) - def __init__(self, config=None, tool=None, **kwargs): + def __init__(self, config=None, tool=None, extractor_product=None,cleaner_product=None,**kwargs): """ Parent class for the flat field calculators. Fills the MON.flatfield container. @@ -47,8 +49,36 @@ def __init__(self, config=None, tool=None, **kwargs): """ super().__init__(config=config, parent=tool, **kwargs) + + # initialize the output self.container = FlatFieldCameraContainer() + # load the waveform charge extractor and cleaner + kwargs_ = dict() + if extractor_product: + kwargs_['product'] = extractor_product + self.extractor = ChargeExtractorFactory.produce( + config=config, + tool=tool, + **kwargs_ + ) + self.log.info(f"extractor {self.extractor}") + + kwargs_ = dict() + if cleaner_product: + kwargs_['product'] = cleaner_product + self.cleaner = WaveformCleanerFactory.produce( + config=config, + tool=tool, + **kwargs_ + ) + self.log.info(f"cleaner {self.cleaner}") + + #self.extractor = ChargeExtractorFactory.produce(config=config, tool=tool,**kwargs) + #self.log.info(f"extractor {self.extractor}") + + #self.cleaner = WaveformCleanerFactory.produce(config=config, tool=tool, **kwargs) + #self.log.info(f"cleaner {self.cleaner}") @abstractmethod @@ -68,7 +98,7 @@ class FlasherFlatFieldCalculator(FlatFieldCalculator): flasher data. Fills the MON.flatfield container. """ - def __init__(self, config=None, tool=None, **kwargs): + def __init__(self, config=None, tool=None, **kwargs): """ Parent class for the flat field calculators. Fills the MON.flatfield container. @@ -86,15 +116,41 @@ def __init__(self, config=None, tool=None, **kwargs): kwargs """ - super().__init__(config=config, tool=tool, **kwargs) - + super().__init__(config=config, tool=tool, **kwargs) + + #self.extractor = config.extractor + #self.cleaner = config.cleaner self.log.info("Used events statistics : %d", self.max_events) self.count = 0 - + def _extract_charge(self, event,tel_id): + + waveforms = event.r0.tel[tel_id].waveform + + + # Clean waveforms + if(self.cleaner): + cleaned = self.cleaner.apply(waveforms) + else: # do nothing + cleaned=waveforms + + # Extract charge + if(self.extractor): + if self.extractor.requires_neighbours(): + g = event.inst.subarray.tel[tel_id].camera + self.extractor.neighbours = g.neighbor_matrix_where + + charge, peakpos, window = self.extractor.extract_charge(cleaned) + + else: # sum all the samples + charge = cleaned.sum(axis=2) + peakpos = np.argmax(cleaned,axis=2) + + + return charge, peakpos - def calculate_relative_gain(self, event): + def calculate_relative_gain(self, event, tel_id): """ calculate the relative flat filed coefficients @@ -104,47 +160,55 @@ def calculate_relative_gain(self, event): """ # initialize the np array at each cycle + waveform = event.r0.tel[tel_id].waveform + trigger_time = event.r0.tel[tel_id].trigger_time + pixel_status = event.r0.tel[tel_id].pixel_status if (self.count == 0): - self.time_start = event.trigger_time - + self.time_start = trigger_time + + if(waveform.shape[0] < self.n_channels): + self.n_channels = waveform.shape[0] + self.event_median = np.zeros((self.max_events, self.n_channels)) - n_pix = event.waveform.shape[1] + n_pix = waveform.shape[1] self.trace_integral = np.zeros((self.max_events, self.n_channels, n_pix)) + self.trace_mask = np.zeros((self.max_events, self.n_channels, n_pix)) - - trace = event.waveform[:,:,:] - - - # here if necessary subtract pedestals - # e.g. with calc_pedestals_from_traces() - # ... - - # extract the signal (for the moment very rough integral): x(i,j) - # for the moment no check on the values - self.trace_integral[self.count,:,:] = trace[:,:,:].sum(axis=2) + # extract the charge of the event and the peak position + integral, peakpos =self._extract_charge(event,tel_id) + + # remember the charge + self.trace_integral[self.count]= integral + + # keep the mask of not working pixels (to be improved) + self.trace_mask[self.count]=[pixel_status==0,pixel_status==0] + - # extract the median on all the camera per event: (i) - self.event_median[self.count,:] = \ - np.median(self.trace_integral[self.count,:,:], axis=1) - time = event.trigger_time + # extract the median on all the camera per event: (i) (for not masked pixels) + masked_integral=np.ma.array(integral,mask=self.trace_mask[self.count]) + self.event_median[self.count,:] = np.ma.median(masked_integral,axis=1) + # increment the internal counter self.count = self.count+1 # check if to create a calibration event - if ((time - self.time_start) > self.max_time_range_s or self.count == self.max_events): + if ((trigger_time - self.time_start) > self.max_time_range_s or self.count == self.max_events): - - # extract for each pixel : x(i,j)/(i) = g(i,j) - rel_gain_event = self.trace_integral/self.event_median[:,:, np.newaxis] + #consider only not masked data + masked_trace_integral = np.ma.array(self.trace_integral, mask=self.trace_mask) + + # extract for each pixel and each event : x(i,j)/(i) = g(i,j) + masked_rel_gain_event = masked_trace_integral/self.event_median[:,:, np.newaxis] + rel_gain_event = np.ma.getdata(masked_rel_gain_event) # extract the median, mean and std over all the events j and # fill the container and return it - self.container.mean_time_s = (time - self.time_start)/2 - self.container.range_time_s = [self.time_start, time] + self.container.mean_time_s = (trigger_time - self.time_start)/2 + self.container.range_time_s = [self.time_start, trigger_time] self.container.n_events = self.count + 1 self.container.relative_gain_median = np.median(rel_gain_event, axis=0) self.container.relative_gain_mean = np.mean(rel_gain_event, axis=0) diff --git a/ctapipe/calib/pedestals.py b/ctapipe/calib/camera/pedestals.py similarity index 100% rename from ctapipe/calib/pedestals.py rename to ctapipe/calib/camera/pedestals.py diff --git a/ctapipe/image/waveform_cleaning.py b/ctapipe/image/waveform_cleaning.py index 41491ecf196..d6c86dbd2c6 100644 --- a/ctapipe/image/waveform_cleaning.py +++ b/ctapipe/image/waveform_cleaning.py @@ -13,7 +13,7 @@ LocalPeakIntegrator) __all__ = ['WaveformCleanerFactory', 'CHECMWaveformCleanerAverage', - 'CHECMWaveformCleanerLocal', + 'CHECMWaveformCleanerLocal','BaselineWaveformCleaner', 'NullWaveformCleaner'] @@ -66,6 +66,20 @@ class NullWaveformCleaner(WaveformCleaner): def apply(self, waveforms): return waveforms +class BaselineWaveformCleaner(WaveformCleaner): + """ + Basic waveform cleaner that simply returns the waveform subtracted + from the baseline + """ + baseline_width = Int(10, help='Define then number of samples for estimating the ' + 'baseline').tag(config=True) + def apply(self, waveforms): + #self.log.debug(f"calculate baseline on first {self.baseline_width} samples") + # Subtract initial baseline + baseline_sub = waveforms - np.mean(waveforms[:, :self.baseline_width], axis=1)[:, None] + + return baseline_sub + class CHECMWaveformCleaner(WaveformCleaner): """ diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 980f8e1365b..60e4d56bfd2 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -151,7 +151,7 @@ class R0CameraContainer(Container): "(n_channels x n_pixels, n_samples)" )) num_samples = Field(None, "number of time samples for telescope") - + pixel_status = Field(0o0, "status of the pixels") class R0Container(Container): """ diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 59f6cf617aa..81c26d1e25f 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -176,6 +176,9 @@ def fill_r0_camera_container_from_zfile(self, container, event): "N_chan x N_pix x N_samples= '{}'" .format(event.waveform.shape[0])) + container.pixel_status=np.zeros([self.n_camera_pixels]) + container.pixel_status[self.camera_config.expected_pixels_id]= \ + event.pixel_status reshaped_waveform = np.array( event.waveform diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index d4459e47b2c..ada94e24142 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -174,6 +174,9 @@ def fill_r0_camera_container_from_zfile(self, container, event): "N_chan x N_pix x N_samples= '{}'" .format(event.waveform.shape[0])) + container.pixel_status=np.zeros([self.n_camera_pixels]) + container.pixel_status[self.camera_config.expected_pixels_id]= \ + event.pixel_status reshaped_waveform = np.array( event.waveform @@ -183,10 +186,13 @@ def fill_r0_camera_container_from_zfile(self, container, event): container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) + # re-order the waveform following the expected_pixels_id values (rank = pixel id) container.waveform[:, self.camera_config.expected_pixels_id, :] \ = reshaped_waveform + + def fill_r0_container_from_zfile(self, event): diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index f64c8d6951a..28f1067f555 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -10,6 +10,8 @@ from ctapipe.io import HDF5TableWriter from ctapipe.core import Tool from ctapipe.io import EventSourceFactory + +from ctapipe.image import ChargeExtractorFactory, WaveformCleanerFactory from ctapipe.calib.camera.flatfield import FlatFieldFactory from ctapipe.io.containers import MonDataContainer @@ -25,14 +27,25 @@ class FlatFieldGenerator(Tool): aliases = Dict(dict(input_file='EventSourceFactory.input_url', max_events='EventSourceFactory.max_events', - allowed_tels= 'EventSourceFactory.allowed_tels', + allowed_tels= 'EventSourceFactory.allowed_tels', + charge_extractor='ChargeExtractorFactory.product', + window_width='ChargeExtractorFactory.window_width', + t0='ChargeExtractorFactory.t0', + window_shift='ChargeExtractorFactory.window_shift', + sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG', + sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG', + lwt='ChargeExtractorFactory.lwt', + cleaner='WaveformCleanerFactory.product', + cleaner_width='WaveformCleanerFactory.baseline_width', generator='FlatFieldFactory.product', max_time_range_s='FlatFieldFactory.max_time_range_s', ff_events='FlatFieldFactory.max_events', - n_channels='FlatFieldFactory.n_channels' + n_channels='FlatFieldFactory.n_channels', )) classes = List([EventSourceFactory, + ChargeExtractorFactory, + WaveformCleanerFactory, FlatFieldFactory, MonDataContainer, HDF5TableWriter @@ -46,17 +59,16 @@ def __init__(self, **kwargs): self.writer = None def setup(self): - self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" + # self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" kwargs = dict(config=self.config, tool=self) - - # open an extractor per camera self.eventsource = EventSourceFactory.produce(**kwargs) self.flatfield = FlatFieldFactory.produce(**kwargs) + self.container = MonDataContainer() self.writer = HDF5TableWriter( filename=self.output_file, group_name='flatfield', overwrite=True ) - + def start(self): desc = "Flat field coefficient calculator" @@ -65,11 +77,13 @@ def start(self): for tel_id in event.r0.tels_with_data: - # initialize the flat filed containers - self.container.flatfield.tels_with_data.append(tel_id) - - ff_data = self.flatfield.calculate_relative_gain(event.r0.tel[tel_id]) + # initialize the flat fieLd containers + if (count == 0): + self.container.flatfield.tels_with_data.append(tel_id) + + + ff_data = self.flatfield.calculate_relative_gain(event, tel_id) if ff_data: self.container.flatfield.tel[tel_id] = ff_data From 0d5a6aefe5164a855f6682edd4480d7d56166a63 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 23 Nov 2018 09:47:33 +0100 Subject: [PATCH 41/82] Minor style changes --- ctapipe/io/lsteventsource.py | 7 ++++--- ctapipe/io/nectarcameventsource.py | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index a07164e878e..cd67b06d653 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -7,7 +7,6 @@ import numpy as np from astropy import units as u -#from os import listdir import glob from os import getcwd from ctapipe.core import Provenance @@ -177,7 +176,9 @@ def fill_r0_camera_container_from_zfile(self, container, event): reshaped_waveform = np.array( event.waveform - ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) + ).reshape(n_gains, + self.camera_config.num_pixels, + container.num_samples) # initialize the waveform container to zero container.waveform = np.zeros([n_gains, self.n_camera_pixels, @@ -257,7 +258,7 @@ def __init__(self, input_url): self._camera_config[path] = next(self._file[path].CameraConfig) # for the moment it takes the first CameraConfig it finds (to be changed) - if(self.camera_config == None): + if(self.camera_config is None): self.camera_config = self._camera_config[path] diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index cbfd5a94890..9f8bd70e6f8 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -176,7 +176,9 @@ def fill_r0_camera_container_from_zfile(self, container, event): reshaped_waveform = np.array( event.waveform - ).reshape(n_gains, self.camera_config.num_pixels, container.num_samples) + ).reshape(n_gains, + self.camera_config.num_pixels, + container.num_samples) # initialize the waveform container to zero container.waveform = np.zeros([n_gains, From b8ad8b9ced9137a723b0be9164428a4df43f940d Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 23 Nov 2018 12:42:14 +0100 Subject: [PATCH 42/82] Moved to last version of ctapipe-extra: 0.2.15 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 2b962cc03f4..20d5952baa2 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ channels: - conda-forge dependencies: - zeromq=4.2.2 - - ctapipe-extra=0.2.14 + - ctapipe-extra=0.2.15 - astropy - bokeh - cython From bffaf5b20012ae4e79a5c6e38dfa5a9f40626383 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 23 Nov 2018 12:49:13 +0100 Subject: [PATCH 43/82] Moved to last version of ctapipe-extra: 0.2.15 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 2b962cc03f4..20d5952baa2 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ channels: - conda-forge dependencies: - zeromq=4.2.2 - - ctapipe-extra=0.2.14 + - ctapipe-extra=0.2.15 - astropy - bokeh - cython From 7e829959daeb3cfb023e93a53fde43a6629f4ee3 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 3 Dec 2018 16:56:54 +0100 Subject: [PATCH 44/82] move back to ctapipe-extra version: 0.2.14 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 20d5952baa2..2b962cc03f4 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ channels: - conda-forge dependencies: - zeromq=4.2.2 - - ctapipe-extra=0.2.15 + - ctapipe-extra=0.2.14 - astropy - bokeh - cython From fda32590b307447ef9e1ecdcccf32fa01a33c4da Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Tue, 4 Dec 2018 11:13:15 +0100 Subject: [PATCH 45/82] Go back to ctapipe-extra release 0.2.15 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 2b962cc03f4..20d5952baa2 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ channels: - conda-forge dependencies: - zeromq=4.2.2 - - ctapipe-extra=0.2.14 + - ctapipe-extra=0.2.15 - astropy - bokeh - cython From ebd77b57e519b08340639927d8cf8a93ae11dbb2 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Tue, 4 Dec 2018 16:58:33 +0100 Subject: [PATCH 46/82] Added the possibility to use wild chards in the file name as in MAGICEventSourceROOT --- ctapipe/io/lsteventsource.py | 81 +++++++++++++++++++----------- ctapipe/io/nectarcameventsource.py | 36 ++++++++++++- 2 files changed, 86 insertions(+), 31 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index cd67b06d653..adb9b8b8fea 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -21,10 +21,49 @@ class LSTEventSource(EventSource): + """ + EventSource for LST r0 data. + """ + + + def __init__(self, config=None, tool=None, **kwargs): + + """ + Constructor + Parameters + ---------- + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + Set to None if no configuration to pass. + tool: ctapipe.core.Tool + Tool executable that is calling this component. + Passes the correct logger to the component. + Set to None if no Tool to pass. + kwargs: dict + Additional parameters to be passed. + NOTE: The file mask of the data to read can be passed with + the 'input_url' parameter. + """ + + + # EventSource can not handle file wild cards as input_url + # To overcome this we substitute the input_url with first file matching + # the specified file mask (copied from MAGICEventSourceROOT). + + + self.file_list = glob.glob(kwargs['input_url']) + self.file_list.sort() + + + kwargs['input_url']=self.file_list[0] + super().__init__(config=config, tool=tool, **kwargs) + - self.multi_file = MultiFiles(self.input_url) + self.multi_file = MultiFiles(self.file_list) + self.camera_config = self.multi_file.camera_config self.log.info("Read {} input files".format(self.multi_file.num_inputs())) @@ -207,13 +246,11 @@ def fill_r0_container_from_zfile(self, event): class MultiFiles: """ - This class open all the files related to a given run xxxx if the file name - contains Runxxxx and all the files are in the same directory. If you want - to open only some files, link them from an empty directory - (this will probably be changed in the near future) + This class open all the files in file_list and read the events following + the event_id order """ - def __init__(self, input_url): + def __init__(self, file_list): self._file = {} self._events = {} @@ -221,36 +258,20 @@ def __init__(self, input_url): self._camera_config = {} self.camera_config = None - # In the input_url contain Runxxxx, test how many files with Runxxxx in the name - # are in the directory and open all of them - # If not "Run" tag is present in the name it opens only the input_url file - if ('/' in input_url): - dir_name, name = input_url.rsplit('/', 1) - else: - dir_name = getcwd() - name = input_url - - if ('Run' in name): - idx = name.find('Run') - run = name[idx:idx + 7] - else: - run = name - - ls = glob.glob(dir_name + "/*.fits.fz") + paths = [] - for file_name in ls: - if run in file_name: - paths.append(file_name) - Provenance().add_input_file(file_name, role='dl0.sub.evt') + for file_name in file_list: + paths.append(file_name) + Provenance().add_input_file(file_name, role='r0.sub.evt') # open the files and get the first fits Tables from protozfits import File for path in paths: - self._file[path] = File(path) - self._events_table[path] = File(path).Events - try: + try: + self._file[path] = File(path) + self._events_table[path] = File(path).Events self._events[path] = next(self._file[path].Events) # verify where the CameraConfig is present @@ -265,7 +286,7 @@ def __init__(self, input_url): except StopIteration: pass - # verify that soemwhere the CameraConfing is present + # verify that somewhere the CameraConfing is present assert (self.camera_config) def __iter__(self): diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index 9f8bd70e6f8..3d3220d09e8 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -6,6 +6,7 @@ """ import numpy as np +import glob from astropy import units as u from ctapipe.instrument import TelescopeDescription, SubarrayDescription, \ CameraGeometry, OpticsDescription @@ -17,11 +18,44 @@ class NectarCAMEventSource(EventSource): + """ + EventSource for NectarCam r0 data. + """ def __init__(self, config=None, tool=None, **kwargs): + + + """ + Constructor + Parameters + ---------- + config: traitlets.loader.Config + Configuration specified by config file or cmdline arguments. + Used to set traitlet values. + Set to None if no configuration to pass. + tool: ctapipe.core.Tool + Tool executable that is calling this component. + Passes the correct logger to the component. + Set to None if no Tool to pass. + kwargs: dict + Additional parameters to be passed. + NOTE: The file mask of the data to read can be passed with + the 'input_url' parameter. + """ + # EventSource can not handle file wild cards as input_url + # To overcome this we substitute the input_url with first file matching + # the specified file mask (copied from MAGICEventSourceROOT). + + + self.file_list = glob.glob(kwargs['input_url']) + self.file_list.sort() + + kwargs['input_url']=self.file_list[0] + super().__init__(config=config, tool=tool, **kwargs) + - self.multi_file = MultiFiles(self.input_url) + self.multi_file = MultiFiles(self.file_list) self.camera_config = self.multi_file.camera_config self.log.info("Read {} input files".format(self.multi_file.num_inputs())) From 03519c1e4ade9caa483ebd8288ea76e79861e783 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Wed, 5 Dec 2018 16:53:12 +0100 Subject: [PATCH 47/82] Corrected code related to wildchards in input file name: for the moment they don't work with the use of a config dictionary but only with the direct set of the "input_url" argument. --- ctapipe/io/lsteventsource.py | 18 +++++++++--------- ctapipe/io/nectarcameventsource.py | 17 +++++++++-------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index adb9b8b8fea..a02e2f42b05 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -52,15 +52,15 @@ def __init__(self, config=None, tool=None, **kwargs): # To overcome this we substitute the input_url with first file matching # the specified file mask (copied from MAGICEventSourceROOT). - - self.file_list = glob.glob(kwargs['input_url']) - self.file_list.sort() - - - kwargs['input_url']=self.file_list[0] - - super().__init__(config=config, tool=tool, **kwargs) - + if 'input_url' in kwargs.keys(): + self.file_list = glob.glob(kwargs['input_url']) + self.file_list.sort() + kwargs['input_url'] = self.file_list[0] + super().__init__(config=config, tool=tool, **kwargs) + else: + super().__init__(config=config, tool=tool, **kwargs) + self.file_list = [self.input_url] + self.multi_file = MultiFiles(self.file_list) diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index 3d3220d09e8..fea98edf4a4 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -46,14 +46,15 @@ def __init__(self, config=None, tool=None, **kwargs): # To overcome this we substitute the input_url with first file matching # the specified file mask (copied from MAGICEventSourceROOT). - - self.file_list = glob.glob(kwargs['input_url']) - self.file_list.sort() - - kwargs['input_url']=self.file_list[0] - - super().__init__(config=config, tool=tool, **kwargs) - + + if 'input_url' in kwargs.keys(): + self.file_list = glob.glob(kwargs['input_url']) + self.file_list.sort() + kwargs['input_url'] = self.file_list[0] + super().__init__(config=config, tool=tool, **kwargs) + else: + super().__init__(config=config, tool=tool, **kwargs) + self.file_list = [self.input_url] self.multi_file = MultiFiles(self.file_list) self.camera_config = self.multi_file.camera_config From 8b9b3a6ae262c539811f1632860fc668a1d99404 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 7 Dec 2018 15:23:55 +0100 Subject: [PATCH 48/82] Added time to flat-field container --- ctapipe/calib/camera/flatfield.py | 149 ++++++++++--------- ctapipe/calib/camera/tests/test_flatfield.py | 6 +- ctapipe/io/containers.py | 20 ++- ctapipe/io/lsteventsource.py | 6 - examples/calc_flatfield.py | 6 +- 5 files changed, 98 insertions(+), 89 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index dc49e4ef3ce..bda263e4584 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -3,7 +3,7 @@ """ from abc import abstractmethod import numpy as np - +from astropy import units as u from ctapipe.core import Component, Factory from ctapipe.image import ChargeExtractorFactory, WaveformCleanerFactory @@ -29,8 +29,7 @@ class FlatFieldCalculator(Component): n_channels = Int(2, help='Define the number of channels to be ' 'treated ').tag(config=True) - - def __init__(self, config=None, tool=None, extractor_product=None,cleaner_product=None,**kwargs): + def __init__(self, config=None, tool=None, extractor_product=None, cleaner_product=None, **kwargs): """ Parent class for the flat field calculators. Fills the MON.flatfield container. @@ -44,7 +43,10 @@ def __init__(self, config=None, tool=None, extractor_product=None,cleaner_produc Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. - + extractor_product : str + The ChargeExtractor to use. + cleaner_product : str + The WaveformCleaner to use. kwargs """ @@ -73,14 +75,7 @@ def __init__(self, config=None, tool=None, extractor_product=None,cleaner_produc **kwargs_ ) self.log.info(f"cleaner {self.cleaner}") - - #self.extractor = ChargeExtractorFactory.produce(config=config, tool=tool,**kwargs) - #self.log.info(f"extractor {self.extractor}") - - #self.cleaner = WaveformCleanerFactory.produce(config=config, tool=tool, **kwargs) - #self.log.info(f"cleaner {self.cleaner}") - - + @abstractmethod def calculate_relative_gain(self,event): """ @@ -89,9 +84,8 @@ def calculate_relative_gain(self,event): event """ - - - + + class FlasherFlatFieldCalculator(FlatFieldCalculator): """ Class for calculating flat field coefficients witht the @@ -100,7 +94,7 @@ class FlasherFlatFieldCalculator(FlatFieldCalculator): def __init__(self, config=None, tool=None, **kwargs): """ - Parent class for the flat field calculators. Fills the MON.flatfield container. + Parent class for the flat-field calculators. Fills the MON.flatfield container. Parameters ---------- @@ -112,108 +106,127 @@ def __init__(self, config=None, tool=None, **kwargs): Tool executable that is calling this component. Passes the correct logger to the component. Set to None if no Tool to pass. - kwargs """ super().__init__(config=config, tool=tool, **kwargs) - - #self.extractor = config.extractor - #self.cleaner = config.cleaner + self.log.info("Used events statistics : %d", self.max_events) self.count = 0 - - def _extract_charge(self, event,tel_id): + def _extract_charge(self, event, tel_id): + """ + Extract the charge and the time from a calibration event + + Parameters + ---------- + event : general event container + + tel_id : telescope id + """ waveforms = event.r0.tel[tel_id].waveform - - - # Clean waveforms - if(self.cleaner): + + # Clean waveforms + if self.cleaner: cleaned = self.cleaner.apply(waveforms) - else: # do nothing - cleaned=waveforms + # do nothing + else: + cleaned = waveforms - # Extract charge - if(self.extractor): + # Extract charge and time + if self.extractor: if self.extractor.requires_neighbours(): g = event.inst.subarray.tel[tel_id].camera self.extractor.neighbours = g.neighbor_matrix_where - charge, peakpos, window = self.extractor.extract_charge(cleaned) - - else: # sum all the samples + charge, peak_pos, window = self.extractor.extract_charge(cleaned) + + # sum all the samples + else: charge = cleaned.sum(axis=2) - peakpos = np.argmax(cleaned,axis=2) - - - return charge, peakpos + peak_pos = np.argmax(cleaned,axis=2) + + return charge, peak_pos def calculate_relative_gain(self, event, tel_id): """ - calculate the relative flat filed coefficients + calculate the relative flat field coefficients Parameters ---------- - event : specific camera + event : general event container + tel_id : telescope id for which we calculate the gain """ + # initialize the np array at each cycle waveform = event.r0.tel[tel_id].waveform trigger_time = event.r0.tel[tel_id].trigger_time pixel_status = event.r0.tel[tel_id].pixel_status - if (self.count == 0): + if self.count == 0: self.time_start = trigger_time - if(waveform.shape[0] < self.n_channels): + if waveform.shape[0] < self.n_channels: self.n_channels = waveform.shape[0] self.event_median = np.zeros((self.max_events, self.n_channels)) n_pix = waveform.shape[1] self.trace_integral = np.zeros((self.max_events, self.n_channels, n_pix)) + self.trace_time = np.zeros((self.max_events, self.n_channels, n_pix)) self.trace_mask = np.zeros((self.max_events, self.n_channels, n_pix)) - - # extract the charge of the event and the peak position + # extract the charge of the event and the peak position (assumed as time for the moment) integral, peakpos =self._extract_charge(event,tel_id) # remember the charge - self.trace_integral[self.count]= integral - + self.trace_integral[self.count] = integral + + # remember the time + self.trace_time[self.count] = peakpos + # keep the mask of not working pixels (to be improved) - self.trace_mask[self.count]=[pixel_status==0,pixel_status==0] - - + self.trace_mask[self.count] = [pixel_status == 0, pixel_status == 0] + # extract the median on all the camera per event: (i) (for not masked pixels) - masked_integral=np.ma.array(integral,mask=self.trace_mask[self.count]) - self.event_median[self.count,:] = np.ma.median(masked_integral,axis=1) - - + masked_integral = np.ma.array(integral, mask=self.trace_mask[self.count]) + self.event_median[self.count, :] = np.ma.median(masked_integral, axis=1) + # increment the internal counter self.count = self.count+1 # check if to create a calibration event - if ((trigger_time - self.time_start) > self.max_time_range_s or self.count == self.max_events): + if (trigger_time - self.time_start) > self.max_time_range_s or self.count == self.max_events: - #consider only not masked data - masked_trace_integral = np.ma.array(self.trace_integral, mask=self.trace_mask) + # consider only not masked data + masked_trace_integral = np.ma.array(self.trace_integral, mask=self.trace_mask) + masked_trace_time = np.ma.array(self.trace_time, mask=self.trace_mask) - # extract for each pixel and each event : x(i,j)/(i) = g(i,j) - masked_rel_gain_event = masked_trace_integral/self.event_median[:,:, np.newaxis] - rel_gain_event = np.ma.getdata(masked_rel_gain_event) - + # extract for each pixel and each event : x(i,j)/(i) = g(i,j) + masked_relative_gain_event = masked_trace_integral/self.event_median[:,:, np.newaxis] + relative_gain_event = np.ma.getdata(masked_relative_gain_event) + # extract the median, mean and std over all the events j and # fill the container and return it - self.container.mean_time_s = (trigger_time - self.time_start)/2 - self.container.range_time_s = [self.time_start, trigger_time] - self.container.n_events = self.count + 1 - self.container.relative_gain_median = np.median(rel_gain_event, axis=0) - self.container.relative_gain_mean = np.mean(rel_gain_event, axis=0) - self.container.relative_gain_rms = np.std(rel_gain_event, axis=0) - + self.container.time_mean = (trigger_time - self.time_start)/2 * u.s + self.container.time_range = [self.time_start, trigger_time] * u.s + self.container.n_events = self.count + self.container.relative_gain_median = np.median(relative_gain_event, axis=0) + self.container.relative_gain_mean = np.mean(relative_gain_event, axis=0) + self.container.relative_gain_rms = np.std(relative_gain_event, axis=0) + + # extract the average time over the camera and the events + camera_time_median = np.ma.median(masked_trace_time) + camera_time_mean = np.ma.mean(masked_trace_time) + pixel_time_median = np.ma.median(masked_trace_time, axis=0) + pixel_time_mean = np.ma.mean(masked_trace_time, axis=0) + + # fill the container + self.container.relative_time_median = np.ma.getdata(pixel_time_median-camera_time_median) + self.container.relative_time_mean = np.ma.getdata(pixel_time_mean-camera_time_mean) + # re-initialize the event count self.count = 0 @@ -222,8 +235,6 @@ def calculate_relative_gain(self, event, tel_id): else: return None - - class FlatFieldFactory(Factory): @@ -232,6 +243,6 @@ class FlatFieldFactory(Factory): """ base = FlatFieldCalculator default = 'FlasherFlatFieldCalculator' - custom_product_help = ('') + custom_product_help = ('Flat-flield method to use') \ No newline at end of file diff --git a/ctapipe/calib/camera/tests/test_flatfield.py b/ctapipe/calib/camera/tests/test_flatfield.py index 5e9d62cf94e..6ca5e8f3270 100644 --- a/ctapipe/calib/camera/tests/test_flatfield.py +++ b/ctapipe/calib/camera/tests/test_flatfield.py @@ -5,10 +5,10 @@ from ctapipe.io.nectarcameventsource import NectarCAMEventSource from ctapipe.calib.camera.flatfield import FlasherFlatFieldCalculator - def test_FlasherFlatFieldCalculator(): - example_file_path = get_dataset_path("NectarCAM.Run0890.10events.fits.fz") + example_file_path = get_dataset_path("NectarCAM.Run0890.10events.fits.fz") + inputfile_reader = NectarCAMEventSource( input_url=example_file_path, max_events=10 @@ -21,7 +21,7 @@ def test_FlasherFlatFieldCalculator(): for tel_id in event.r0.tels_with_data: - ff_data = ff_calculator.calculate_relative_gain(event.r0.tel[tel_id]) + ff_data = ff_calculator.calculate_relative_gain(event, tel_id) if ff_calculator.count == ff_calculator.max_events: assert(ff_data) diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 60e4d56bfd2..b4f9b3ec560 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -44,7 +44,6 @@ 'LeakageContainer', 'ConcentrationContainer', 'TimingParametersContainer', - 'StatInfoContainer', 'FlatFieldCameraContainer', 'PedestalCameraContainer', 'FlatFieldContainer', @@ -768,12 +767,12 @@ class TimingParametersContainer(Container): class FlatFieldCameraContainer(Container): """ - Container for relative flat-field coefficients - per camera + Container for relative camera flat-field coefficients + """ - mean_time_s = Field(0, 'Mean time, seconds since reference') - range_time_s = Field([], 'Range of time') + time_mean = Field(0, 'Mean time, seconds since reference', unit=u.s) + time_range = Field([], 'Range of time of the calibration data [t_min, t_max]', unit=u.s) n_events = Field(0,'Number of events used for statistics') @@ -787,8 +786,13 @@ class FlatFieldCameraContainer(Container): relative_gain_rms = Field(None, "np array of the relative flat-field coefficient rms (n_chan X N_pix)" ) - - + + relative_time_mean = Field(None, + "np array of the relative time mean (n_chan X N_pix)" + , unit=u.ns) + relative_time_median = Field(None, + "np array of the relative time median (n_chan X N_pix)" + , unit=u.ns) class FlatFieldContainer(Container): """ @@ -816,7 +820,7 @@ class PedestalContainer(Container): Map(PedestalCameraContainer), "map of tel_id to PedestalCameraContainer") -class MonDataContainer(Container): +class MonitorDataContainer(Container): """ Root container for MON data """ diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 3d106a62f23..99e90db51ec 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -61,7 +61,6 @@ def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) self.file_list = [self.input_url] - self.multi_file = MultiFiles(self.file_list) self.camera_config = self.multi_file.camera_config @@ -69,7 +68,6 @@ def __init__(self, config=None, tool=None, **kwargs): def rewind(self): self.multi_file.rewind() - def _generator(self): @@ -78,7 +76,6 @@ def _generator(self): self.data.meta['input_url'] = self.input_url self.data.meta['max_events'] = self.max_events - # fill LST data from the CameraConfig table self.fill_lst_service_container_from_zfile() @@ -177,7 +174,6 @@ def fill_lst_service_container_from_zfile(self): def fill_lst_event_container_from_zfile(self, event): - event_container = self.data.lst.tel[self.camera_config.telescope_id].evt event_container.configuration_id = event.configuration_id @@ -329,8 +325,6 @@ def __len__(self): def rewind(self): for name, file in self._file.items(): file.Events.protobuf_i_fits.rewind() - - def num_inputs(self): return len(self._file) diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index 28f1067f555..7c55064f211 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -13,7 +13,7 @@ from ctapipe.image import ChargeExtractorFactory, WaveformCleanerFactory from ctapipe.calib.camera.flatfield import FlatFieldFactory -from ctapipe.io.containers import MonDataContainer +from ctapipe.io.containers import MonitorDataContainer class FlatFieldGenerator(Tool): @@ -47,7 +47,7 @@ class FlatFieldGenerator(Tool): ChargeExtractorFactory, WaveformCleanerFactory, FlatFieldFactory, - MonDataContainer, + MonitorDataContainer, HDF5TableWriter ]) @@ -64,7 +64,7 @@ def setup(self): self.eventsource = EventSourceFactory.produce(**kwargs) self.flatfield = FlatFieldFactory.produce(**kwargs) - self.container = MonDataContainer() + self.container = MonitorDataContainer() self.writer = HDF5TableWriter( filename=self.output_file, group_name='flatfield', overwrite=True ) From e1f69ec9906b467a8978af81d9ce10c6873b9095 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 17 Dec 2018 17:29:55 +0100 Subject: [PATCH 49/82] Take away ctapipe-extra==0.2.16 --- environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment.yml b/environment.yml index c77981b86eb..087142e26b8 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,6 @@ channels: - cta-observatory - conda-forge dependencies: - - ctapipe-extra=0.2.16 - zeromq - astropy - bokeh From 3c163526cbd5e64e6ae1d26de42eef67f5085779 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 18 Dec 2018 14:09:49 +0100 Subject: [PATCH 50/82] style: white-spaces; desc --> doc-string; remove indentation --- examples/calc_flatfield.py | 94 ++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index 7c55064f211..d26cd257c3d 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -1,11 +1,8 @@ """ Extract flat field coefficients from flasher data files. """ - -#from tqdm import tqdm from traitlets import Dict, List, Unicode - from ctapipe.core import Provenance from ctapipe.io import HDF5TableWriter from ctapipe.core import Tool @@ -18,30 +15,31 @@ class FlatFieldGenerator(Tool): name = "FlatFieldGenerator" - description = "Generate a HDF5 file with flat field coefficients" - - - output_file = Unicode('flatfield.hdf5', - help='Name of the output flat field file ' \ - 'file').tag(config=True) - - aliases = Dict(dict(input_file='EventSourceFactory.input_url', - max_events='EventSourceFactory.max_events', - allowed_tels= 'EventSourceFactory.allowed_tels', - charge_extractor='ChargeExtractorFactory.product', - window_width='ChargeExtractorFactory.window_width', - t0='ChargeExtractorFactory.t0', - window_shift='ChargeExtractorFactory.window_shift', - sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG', - sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG', - lwt='ChargeExtractorFactory.lwt', - cleaner='WaveformCleanerFactory.product', - cleaner_width='WaveformCleanerFactory.baseline_width', - generator='FlatFieldFactory.product', - max_time_range_s='FlatFieldFactory.max_time_range_s', - ff_events='FlatFieldFactory.max_events', - n_channels='FlatFieldFactory.n_channels', - )) + description = "Generate a HDF5 file with flat field coefficients" + + output_file = Unicode( + 'flatfield.hdf5', + help='Name of the output flat field file file' + ).tag(config=True) + + aliases = Dict(dict( + input_file='EventSourceFactory.input_url', + max_events='EventSourceFactory.max_events', + allowed_tels='EventSourceFactory.allowed_tels', + charge_extractor='ChargeExtractorFactory.product', + window_width='ChargeExtractorFactory.window_width', + t0='ChargeExtractorFactory.t0', + window_shift='ChargeExtractorFactory.window_shift', + sig_amp_cut_HG='ChargeExtractorFactory.sig_amp_cut_HG', + sig_amp_cut_LG='ChargeExtractorFactory.sig_amp_cut_LG', + lwt='ChargeExtractorFactory.lwt', + cleaner='WaveformCleanerFactory.product', + cleaner_width='WaveformCleanerFactory.baseline_width', + generator='FlatFieldFactory.product', + max_time_range_s='FlatFieldFactory.max_time_range_s', + ff_events='FlatFieldFactory.max_events', + n_channels='FlatFieldFactory.n_channels', + )) classes = List([EventSourceFactory, ChargeExtractorFactory, @@ -59,47 +57,43 @@ def __init__(self, **kwargs): self.writer = None def setup(self): - # self.log_format = "%(levelname)s: %(message)s [%(name)s.%(funcName)s]" kwargs = dict(config=self.config, tool=self) self.eventsource = EventSourceFactory.produce(**kwargs) self.flatfield = FlatFieldFactory.produce(**kwargs) - + self.container = MonitorDataContainer() self.writer = HDF5TableWriter( filename=self.output_file, group_name='flatfield', overwrite=True ) - def start(self): - desc = "Flat field coefficient calculator" - + '''Flat field coefficient calculator''' + for count, event in enumerate(self.eventsource): - + for tel_id in event.r0.tels_with_data: - - + # initialize the flat fieLd containers if (count == 0): self.container.flatfield.tels_with_data.append(tel_id) - - + ff_data = self.flatfield.calculate_relative_gain(event, tel_id) - - if ff_data: + + if ff_data: self.container.flatfield.tel[tel_id] = ff_data - table_name='tel_'+str(tel_id) - - self.log.info("write event in table: /flatfield/%s", table_name) - self.writer.write(table_name,ff_data) - - - - + table_name = 'tel_' + str(tel_id) + + self.log.info( + "write event in table: /flatfield/%s", + table_name + ) + self.writer.write(table_name, ff_data) def finish(self): - - Provenance().add_output_file(self.output_file, - role='mon.tel.flatfield') + Provenance().add_output_file( + self.output_file, + role='mon.tel.flatfield' + ) self.writer.close() From 31fbb7fd51ec1a93dc1d99cf3e402eb13aaffba1 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 18 Dec 2018 14:23:01 +0100 Subject: [PATCH 51/82] style: white-spaces --- ctapipe/io/nectarcameventsource.py | 39 +++++++++++++----------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/ctapipe/io/nectarcameventsource.py b/ctapipe/io/nectarcameventsource.py index b541c533e3e..9cc7b5a7b4d 100644 --- a/ctapipe/io/nectarcameventsource.py +++ b/ctapipe/io/nectarcameventsource.py @@ -5,11 +5,15 @@ Needs protozfits v1.4.2 from github.com/cta-sst-1m/protozfitsreader """ -import numpy as np import glob +import numpy as np from astropy import units as u -from ctapipe.instrument import TelescopeDescription, SubarrayDescription, \ - CameraGeometry, OpticsDescription +from ctapipe.instrument import ( + TelescopeDescription, + SubarrayDescription, + CameraGeometry, + OpticsDescription, +) from .eventsource import EventSource from .lsteventsource import MultiFiles from .containers import NectarCAMDataContainer @@ -23,8 +27,6 @@ class NectarCAMEventSource(EventSource): """ def __init__(self, config=None, tool=None, **kwargs): - - """ Constructor Parameters @@ -46,7 +48,6 @@ def __init__(self, config=None, tool=None, **kwargs): # To overcome this we substitute the input_url with first file matching # the specified file mask (copied from MAGICEventSourceROOT). - if 'input_url' in kwargs.keys(): self.file_list = glob.glob(kwargs['input_url']) self.file_list.sort() @@ -59,8 +60,9 @@ def __init__(self, config=None, tool=None, **kwargs): self.multi_file = MultiFiles(self.file_list) self.camera_config = self.multi_file.camera_config - self.log.info("Read {} input files".format(self.multi_file.num_inputs())) - + self.log.info("Read {} input files".format( + self.multi_file.num_inputs() + )) def _generator(self): @@ -71,7 +73,6 @@ def _generator(self): # fill data from the CameraConfig table self.fill_nectarcam_service_container_from_zfile() - # Instrument information for tel_id in self.data.nectarcam.tels_with_data: assert (tel_id == 0) # only one telescope for the moment (id = 0) @@ -94,15 +95,12 @@ def _generator(self): # LSTs telescope position tel_pos = {tel_id: [0., 0., 0] * u.m} - self.subarray = SubarrayDescription("MST prototype subarray") self.subarray.tels = tels self.subarray.positions = tel_pos self.data.inst.subarray = self.subarray - - # loop on events for count, event in enumerate(self.multi_file): @@ -115,7 +113,6 @@ def _generator(self): self.fill_r0_container_from_zfile(event) yield self.data - @staticmethod def is_compatible(file_path): from astropy.io import fits @@ -167,7 +164,6 @@ def fill_nectarcam_service_container_from_zfile(self): svc_container.idaq_version = self.camera_config.nectarcam.idaq_version svc_container.cdhs_version = self.camera_config.nectarcam.cdhs_version svc_container.algorithms = self.camera_config.nectarcam.algorithms - # svc_container.pre_proc_algorithms = camera_config.nectarcam.pre_proc_algorithms def fill_nectarcam_event_container_from_zfile(self, event): @@ -206,15 +202,17 @@ def fill_r0_camera_container_from_zfile(self, container, event): .format(event.waveform.shape[0])) - container.pixel_status=np.zeros([self.n_camera_pixels]) - container.pixel_status[self.camera_config.expected_pixels_id]= \ + container.pixel_status = np.zeros([self.n_camera_pixels]) + container.pixel_status[self.camera_config.expected_pixels_id] = \ event.pixel_status reshaped_waveform = np.array( event.waveform - ).reshape(n_gains, - self.camera_config.num_pixels, - container.num_samples) + ).reshape( + n_gains, + self.camera_config.num_pixels, + container.num_samples + ) # initialize the waveform container to zero container.waveform = np.zeros([n_gains, @@ -225,8 +223,6 @@ def fill_r0_camera_container_from_zfile(self, container, event): container.waveform[:, self.camera_config.expected_pixels_id, :] \ = reshaped_waveform - - def fill_r0_container_from_zfile(self, event): container = self.data.r0 container.obs_id = -1 @@ -239,4 +235,3 @@ def fill_r0_container_from_zfile(self, event): r0_camera_container, event ) - From a28f92f321fd5b07f2658141e63b8748ced7480c Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 18 Dec 2018 14:32:38 +0100 Subject: [PATCH 52/82] more style: white-spaces; import order; assert is not a function --- ctapipe/calib/camera/__init__.py | 2 +- ctapipe/io/lsteventsource.py | 55 +++++++++++++------------------- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/ctapipe/calib/camera/__init__.py b/ctapipe/calib/camera/__init__.py index 91245e2df0d..1acd56c3a14 100644 --- a/ctapipe/calib/camera/__init__.py +++ b/ctapipe/calib/camera/__init__.py @@ -8,4 +8,4 @@ from .dl1 import * from .calibrator import * from .flatfield import * -from .pedestals import * \ No newline at end of file +from .pedestals import * diff --git a/ctapipe/io/lsteventsource.py b/ctapipe/io/lsteventsource.py index 77123d3fbd0..a3639d8e36c 100644 --- a/ctapipe/io/lsteventsource.py +++ b/ctapipe/io/lsteventsource.py @@ -4,11 +4,9 @@ Needs protozfits v1.4.2 from github.com/cta-sst-1m/protozfitsreader """ +import glob import numpy as np - from astropy import units as u -import glob -from os import getcwd from ctapipe.core import Provenance from ctapipe.instrument import TelescopeDescription, SubarrayDescription, \ CameraGeometry, OpticsDescription @@ -20,15 +18,9 @@ class LSTEventSource(EventSource): - - """ - EventSource for LST r0 data. - """ - - + """EventSource for LST r0 data.""" def __init__(self, config=None, tool=None, **kwargs): - """ Constructor Parameters @@ -47,7 +39,6 @@ def __init__(self, config=None, tool=None, **kwargs): the 'input_url' parameter. """ - # EventSource can not handle file wild cards as input_url # To overcome this we substitute the input_url with first file matching # the specified file mask (copied from MAGICEventSourceROOT). @@ -62,14 +53,17 @@ def __init__(self, config=None, tool=None, **kwargs): self.file_list = [self.input_url] self.multi_file = MultiFiles(self.file_list) - + self.camera_config = self.multi_file.camera_config - self.log.info("Read {} input files".format(self.multi_file.num_inputs())) + self.log.info( + "Read {} input files".format( + self.multi_file.num_inputs() + ) + ) def rewind(self): self.multi_file.rewind() - def _generator(self): # container for LST data @@ -83,7 +77,7 @@ def _generator(self): # Instrument information for tel_id in self.data.lst.tels_with_data: - assert (tel_id == 0) # only LST1 for the moment (id = 0) + assert tel_id == 0 # only LST1 for the moment (id = 0) # optics info from standard optics.fits.gz file optics = OpticsDescription.from_name("LST") @@ -101,7 +95,6 @@ def _generator(self): # LSTs telescope position taken from MC from the moment tel_pos = {tel_id: [50., 50., 16] * u.m} - subarray = SubarrayDescription("LST1 subarray") subarray.tels = tels subarray.positions = tel_pos @@ -120,7 +113,6 @@ def _generator(self): self.fill_r0_container_from_zfile(event) yield self.data - @staticmethod def is_compatible(file_path): from astropy.io import fits @@ -212,27 +204,28 @@ def fill_r0_camera_container_from_zfile(self, container, event): "N_chan x N_pix x N_samples= '{}'" .format(event.waveform.shape[0])) - container.pixel_status=np.zeros([self.n_camera_pixels]) - container.pixel_status[self.camera_config.expected_pixels_id]= \ + container.pixel_status = np.zeros([self.n_camera_pixels]) + container.pixel_status[self.camera_config.expected_pixels_id] = \ event.pixel_status reshaped_waveform = np.array( - event.waveform - ).reshape(n_gains, - self.camera_config.num_pixels, - container.num_samples) + event.waveform + ).reshape( + n_gains, + self.camera_config.num_pixels, + container.num_samples + ) # initialize the waveform container to zero container.waveform = np.zeros([n_gains, self.n_camera_pixels, container.num_samples]) - # re-order the waveform following the expected_pixels_id values (rank = pixel id) + # re-order the waveform following the expected_pixels_id values + # (rank = pixel id) container.waveform[:, self.camera_config.expected_pixels_id, :] =\ reshaped_waveform def fill_r0_container_from_zfile(self, event): - - container = self.data.r0 container.obs_id = -1 @@ -250,7 +243,7 @@ class MultiFiles: """ This class open all the files in file_list and read the events following - the event_id order + the event_id order """ def __init__(self, file_list): @@ -261,9 +254,8 @@ def __init__(self, file_list): self._camera_config = {} self.camera_config = None - paths = [] - for file_name in file_list: + for file_name in file_list: paths.append(file_name) Provenance().add_input_file(file_name, role='r0.sub.evt') @@ -285,12 +277,11 @@ def __init__(self, file_list): if(self.camera_config is None): self.camera_config = self._camera_config[path] - except StopIteration: pass # verify that somewhere the CameraConfing is present - assert (self.camera_config) + assert self.camera_config def __iter__(self): return self @@ -323,7 +314,7 @@ def __len__(self): for table in self._events_table.values() ) return total_length - + def rewind(self): for name, file in self._file.items(): file.Events.protobuf_i_fits.rewind() From 37498b410c1a1c619873f4b7fbed9665c591d98c Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 18 Dec 2018 14:36:13 +0100 Subject: [PATCH 53/82] style: white spaces; unused imports --- ctapipe/calib/camera/tests/test_flatfield.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/ctapipe/calib/camera/tests/test_flatfield.py b/ctapipe/calib/camera/tests/test_flatfield.py index 6ca5e8f3270..a33f256f769 100644 --- a/ctapipe/calib/camera/tests/test_flatfield.py +++ b/ctapipe/calib/camera/tests/test_flatfield.py @@ -1,12 +1,10 @@ -import numpy as np -import pytest from ctapipe.utils import get_dataset_path - from ctapipe.io.nectarcameventsource import NectarCAMEventSource from ctapipe.calib.camera.flatfield import FlasherFlatFieldCalculator + def test_FlasherFlatFieldCalculator(): - + example_file_path = get_dataset_path("NectarCAM.Run0890.10events.fits.fz") inputfile_reader = NectarCAMEventSource( @@ -15,16 +13,12 @@ def test_FlasherFlatFieldCalculator(): ) ff_calculator = FlasherFlatFieldCalculator() - ff_calculator.max_events=3 + ff_calculator.max_events = 3 - for i,event in enumerate(inputfile_reader): + for event in inputfile_reader: for tel_id in event.r0.tels_with_data: - - + ff_data = ff_calculator.calculate_relative_gain(event, tel_id) - - if ff_calculator.count == ff_calculator.max_events: - assert(ff_data) - - + if ff_calculator.count == ff_calculator.max_events: + assert ff_data From 472425db93003d735c9f24d47492ca73fb446547 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 18 Dec 2018 14:38:16 +0100 Subject: [PATCH 54/82] white spaces --- ctapipe/calib/camera/flatfield.py | 49 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index bda263e4584..22277b761d0 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -51,10 +51,10 @@ def __init__(self, config=None, tool=None, extractor_product=None, cleaner_produ """ super().__init__(config=config, parent=tool, **kwargs) - + # initialize the output self.container = FlatFieldCameraContainer() - + # load the waveform charge extractor and cleaner kwargs_ = dict() if extractor_product: @@ -65,7 +65,7 @@ def __init__(self, config=None, tool=None, extractor_product=None, cleaner_produ **kwargs_ ) self.log.info(f"extractor {self.extractor}") - + kwargs_ = dict() if cleaner_product: kwargs_['product'] = cleaner_product @@ -82,7 +82,7 @@ def calculate_relative_gain(self,event): Parameters ---------- event - + """ @@ -92,9 +92,10 @@ class FlasherFlatFieldCalculator(FlatFieldCalculator): flasher data. Fills the MON.flatfield container. """ - def __init__(self, config=None, tool=None, **kwargs): + def __init__(self, config=None, tool=None, **kwargs): """ - Parent class for the flat-field calculators. Fills the MON.flatfield container. + Parent class for the flat-field calculators. + Fills the MON.flatfield container. Parameters ---------- @@ -109,7 +110,7 @@ def __init__(self, config=None, tool=None, **kwargs): kwargs """ - super().__init__(config=config, tool=tool, **kwargs) + super().__init__(config=config, tool=tool, **kwargs) self.log.info("Used events statistics : %d", self.max_events) self.count = 0 @@ -124,7 +125,7 @@ def _extract_charge(self, event, tel_id): tel_id : telescope id """ - + waveforms = event.r0.tel[tel_id].waveform # Clean waveforms @@ -145,10 +146,10 @@ def _extract_charge(self, event, tel_id): # sum all the samples else: charge = cleaned.sum(axis=2) - peak_pos = np.argmax(cleaned,axis=2) + peak_pos = np.argmax(cleaned, axis=2) return charge, peak_pos - + def calculate_relative_gain(self, event, tel_id): """ calculate the relative flat field coefficients @@ -164,22 +165,22 @@ def calculate_relative_gain(self, event, tel_id): waveform = event.r0.tel[tel_id].waveform trigger_time = event.r0.tel[tel_id].trigger_time pixel_status = event.r0.tel[tel_id].pixel_status - + if self.count == 0: self.time_start = trigger_time - + if waveform.shape[0] < self.n_channels: self.n_channels = waveform.shape[0] - + self.event_median = np.zeros((self.max_events, self.n_channels)) - + n_pix = waveform.shape[1] self.trace_integral = np.zeros((self.max_events, self.n_channels, n_pix)) self.trace_time = np.zeros((self.max_events, self.n_channels, n_pix)) self.trace_mask = np.zeros((self.max_events, self.n_channels, n_pix)) # extract the charge of the event and the peak position (assumed as time for the moment) - integral, peakpos =self._extract_charge(event,tel_id) + integral, peakpos = self._extract_charge(event, tel_id) # remember the charge self.trace_integral[self.count] = integral @@ -196,14 +197,14 @@ def calculate_relative_gain(self, event, tel_id): # increment the internal counter self.count = self.count+1 - + # check if to create a calibration event if (trigger_time - self.time_start) > self.max_time_range_s or self.count == self.max_events: - + # consider only not masked data masked_trace_integral = np.ma.array(self.trace_integral, mask=self.trace_mask) masked_trace_time = np.ma.array(self.trace_time, mask=self.trace_mask) - + # extract for each pixel and each event : x(i,j)/(i) = g(i,j) masked_relative_gain_event = masked_trace_integral/self.event_median[:,:, np.newaxis] relative_gain_event = np.ma.getdata(masked_relative_gain_event) @@ -229,12 +230,12 @@ def calculate_relative_gain(self, event, tel_id): # re-initialize the event count self.count = 0 - + return self.container - - else: - - return None + + else: + + return None class FlatFieldFactory(Factory): @@ -244,5 +245,3 @@ class FlatFieldFactory(Factory): base = FlatFieldCalculator default = 'FlasherFlatFieldCalculator' custom_product_help = ('Flat-flield method to use') - - \ No newline at end of file From ab0b60a680f740f15bfb27364d8fe34c1554d548 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 18 Dec 2018 15:43:36 +0100 Subject: [PATCH 55/82] more white spaces --- ctapipe/calib/camera/flatfield.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index 22277b761d0..98854bafb39 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -77,7 +77,7 @@ def __init__(self, config=None, tool=None, extractor_product=None, cleaner_produ self.log.info(f"cleaner {self.cleaner}") @abstractmethod - def calculate_relative_gain(self,event): + def calculate_relative_gain(self, event): """ Parameters ---------- @@ -196,10 +196,13 @@ def calculate_relative_gain(self, event, tel_id): self.event_median[self.count, :] = np.ma.median(masked_integral, axis=1) # increment the internal counter - self.count = self.count+1 + self.count = self.count + 1 # check if to create a calibration event - if (trigger_time - self.time_start) > self.max_time_range_s or self.count == self.max_events: + if ( + (trigger_time - self.time_start) > self.max_time_range_s + or self.count == self.max_events + ): # consider only not masked data masked_trace_integral = np.ma.array(self.trace_integral, mask=self.trace_mask) From a86f2f5e0247ecfa5867b33381e1f983a43f1119 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 18 Dec 2018 15:45:13 +0100 Subject: [PATCH 56/82] rename `count` to `num_events_seen`; count does not say what it counts --- ctapipe/calib/camera/flatfield.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index 98854bafb39..68fedfd3f3b 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -113,7 +113,7 @@ def __init__(self, config=None, tool=None, **kwargs): super().__init__(config=config, tool=tool, **kwargs) self.log.info("Used events statistics : %d", self.max_events) - self.count = 0 + self.num_events_seen = 0 def _extract_charge(self, event, tel_id): """ @@ -166,7 +166,7 @@ def calculate_relative_gain(self, event, tel_id): trigger_time = event.r0.tel[tel_id].trigger_time pixel_status = event.r0.tel[tel_id].pixel_status - if self.count == 0: + if self.num_events_seen == 0: self.time_start = trigger_time if waveform.shape[0] < self.n_channels: @@ -183,25 +183,24 @@ def calculate_relative_gain(self, event, tel_id): integral, peakpos = self._extract_charge(event, tel_id) # remember the charge - self.trace_integral[self.count] = integral + self.trace_integral[self.num_events_seen] = integral # remember the time - self.trace_time[self.count] = peakpos + self.trace_time[self.num_events_seen] = peakpos # keep the mask of not working pixels (to be improved) - self.trace_mask[self.count] = [pixel_status == 0, pixel_status == 0] + self.trace_mask[self.num_events_seen] = [pixel_status == 0, pixel_status == 0] # extract the median on all the camera per event: (i) (for not masked pixels) - masked_integral = np.ma.array(integral, mask=self.trace_mask[self.count]) - self.event_median[self.count, :] = np.ma.median(masked_integral, axis=1) + masked_integral = np.ma.array(integral, mask=self.trace_mask[self.num_events_seen]) + self.event_median[self.num_events_seen, :] = np.ma.median(masked_integral, axis=1) - # increment the internal counter - self.count = self.count + 1 + self.num_events_seen += 1 # check if to create a calibration event if ( (trigger_time - self.time_start) > self.max_time_range_s - or self.count == self.max_events + or self.num_events_seen == self.max_events ): # consider only not masked data @@ -216,7 +215,7 @@ def calculate_relative_gain(self, event, tel_id): # fill the container and return it self.container.time_mean = (trigger_time - self.time_start)/2 * u.s self.container.time_range = [self.time_start, trigger_time] * u.s - self.container.n_events = self.count + self.container.n_events = self.num_events_seen self.container.relative_gain_median = np.median(relative_gain_event, axis=0) self.container.relative_gain_mean = np.mean(relative_gain_event, axis=0) self.container.relative_gain_rms = np.std(relative_gain_event, axis=0) @@ -232,7 +231,7 @@ def calculate_relative_gain(self, event, tel_id): self.container.relative_time_mean = np.ma.getdata(pixel_time_mean-camera_time_mean) # re-initialize the event count - self.count = 0 + self.num_events_seen = 0 return self.container From 55f74d95f5d8f3cab25f783ba4850acbeaf47163 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 18 Dec 2018 16:27:53 +0100 Subject: [PATCH 57/82] adjust test to renamed member --- ctapipe/calib/camera/tests/test_flatfield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/calib/camera/tests/test_flatfield.py b/ctapipe/calib/camera/tests/test_flatfield.py index a33f256f769..22f355d5684 100644 --- a/ctapipe/calib/camera/tests/test_flatfield.py +++ b/ctapipe/calib/camera/tests/test_flatfield.py @@ -20,5 +20,5 @@ def test_FlasherFlatFieldCalculator(): ff_data = ff_calculator.calculate_relative_gain(event, tel_id) - if ff_calculator.count == ff_calculator.max_events: + if ff_calculator.num_events_seen == ff_calculator.max_events: assert ff_data From c9d57a848b19e625a10923dd8063ed9decaf2703 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 18 Dec 2018 17:02:13 +0100 Subject: [PATCH 58/82] some renaming; white spaces; comments --- ctapipe/calib/camera/flatfield.py | 64 ++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index 68fedfd3f3b..902dba2130a 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -115,6 +115,13 @@ def __init__(self, config=None, tool=None, **kwargs): self.log.info("Used events statistics : %d", self.max_events) self.num_events_seen = 0 + # members to keep state in calculate_relative_gain() + self.time_start = None # trigger time of first event in sample + self.event_median = None # med. charge in camera per event in sample + self.trace_integral = None # charge per event in sample + self.trace_time = None # arrival time per event in sample + self.bad_pixels_of_sample = None # bad pixels per event in sample + def _extract_charge(self, event, tel_id): """ Extract the charge and the time from a calibration event @@ -175,11 +182,13 @@ def calculate_relative_gain(self, event, tel_id): self.event_median = np.zeros((self.max_events, self.n_channels)) n_pix = waveform.shape[1] - self.trace_integral = np.zeros((self.max_events, self.n_channels, n_pix)) - self.trace_time = np.zeros((self.max_events, self.n_channels, n_pix)) - self.trace_mask = np.zeros((self.max_events, self.n_channels, n_pix)) + shape = (self.max_events, self.n_channels, n_pix) + self.trace_integral = np.zeros(shape) + self.trace_time = np.zeros(shape) + self.bad_pixels_of_sample = np.zeros(shape) - # extract the charge of the event and the peak position (assumed as time for the moment) + # extract the charge of the event and + # the peak position (assumed as time for the moment) integral, peakpos = self._extract_charge(event, tel_id) # remember the charge @@ -189,36 +198,51 @@ def calculate_relative_gain(self, event, tel_id): self.trace_time[self.num_events_seen] = peakpos # keep the mask of not working pixels (to be improved) - self.trace_mask[self.num_events_seen] = [pixel_status == 0, pixel_status == 0] + bad_pixels = np.array([pixel_status == 0, pixel_status == 0]) + self.bad_pixels_of_sample[self.num_events_seen, :] = bad_pixels - # extract the median on all the camera per event: (i) (for not masked pixels) - masked_integral = np.ma.array(integral, mask=self.trace_mask[self.num_events_seen]) - self.event_median[self.num_events_seen, :] = np.ma.median(masked_integral, axis=1) + # extract the median on all the camera per event: (i) + # (for not masked pixels) + masked_integral = np.ma.array(integral, mask=bad_pixels) + self.event_median[self.num_events_seen, :] = np.ma.median( + masked_integral, axis=1) self.num_events_seen += 1 + sample_age = trigger_time - self.time_start # check if to create a calibration event if ( - (trigger_time - self.time_start) > self.max_time_range_s + sample_age > self.max_time_range_s or self.num_events_seen == self.max_events ): # consider only not masked data - masked_trace_integral = np.ma.array(self.trace_integral, mask=self.trace_mask) - masked_trace_time = np.ma.array(self.trace_time, mask=self.trace_mask) + masked_trace_integral = np.ma.array( + self.trace_integral, + mask=self.bad_pixels_of_sample + ) + masked_trace_time = np.ma.array( + self.trace_time, + mask=self.bad_pixels_of_sample + ) # extract for each pixel and each event : x(i,j)/(i) = g(i,j) - masked_relative_gain_event = masked_trace_integral/self.event_median[:,:, np.newaxis] + masked_relative_gain_event = ( + masked_trace_integral / self.event_median[:, :, np.newaxis]) relative_gain_event = np.ma.getdata(masked_relative_gain_event) # extract the median, mean and std over all the events j and # fill the container and return it - self.container.time_mean = (trigger_time - self.time_start)/2 * u.s + self.container.time_mean = ( + (trigger_time - self.time_start) / 2 * u.s) self.container.time_range = [self.time_start, trigger_time] * u.s self.container.n_events = self.num_events_seen - self.container.relative_gain_median = np.median(relative_gain_event, axis=0) - self.container.relative_gain_mean = np.mean(relative_gain_event, axis=0) - self.container.relative_gain_rms = np.std(relative_gain_event, axis=0) + self.container.relative_gain_median = np.median( + relative_gain_event, axis=0) + self.container.relative_gain_mean = np.mean( + relative_gain_event, axis=0) + self.container.relative_gain_rms = np.std( + relative_gain_event, axis=0) # extract the average time over the camera and the events camera_time_median = np.ma.median(masked_trace_time) @@ -227,8 +251,12 @@ def calculate_relative_gain(self, event, tel_id): pixel_time_mean = np.ma.mean(masked_trace_time, axis=0) # fill the container - self.container.relative_time_median = np.ma.getdata(pixel_time_median-camera_time_median) - self.container.relative_time_mean = np.ma.getdata(pixel_time_mean-camera_time_mean) + self.container.relative_time_median = np.ma.getdata( + pixel_time_median - camera_time_median + ) + self.container.relative_time_mean = np.ma.getdata( + pixel_time_mean - camera_time_mean + ) # re-initialize the event count self.num_events_seen = 0 From df393d91c754174bf814d8c67b1ce2091a92d804 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 18 Dec 2018 17:26:28 +0100 Subject: [PATCH 59/82] break out two functions, which could be tested outside the framework --- ctapipe/calib/camera/flatfield.py | 100 ++++++++++++++++++------------ 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index 902dba2130a..5465b55eb62 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -215,52 +215,27 @@ def calculate_relative_gain(self, event, tel_id): sample_age > self.max_time_range_s or self.num_events_seen == self.max_events ): - - # consider only not masked data - masked_trace_integral = np.ma.array( + relative_gain_results = calculate_relative_gain_results( + self.event_median, self.trace_integral, - mask=self.bad_pixels_of_sample + self.bad_pixels_of_sample, ) - masked_trace_time = np.ma.array( + time_results = calculate_time_results( self.trace_time, - mask=self.bad_pixels_of_sample + self.bad_pixels_of_sample, + self.time_start, + trigger_time, ) - # extract for each pixel and each event : x(i,j)/(i) = g(i,j) - masked_relative_gain_event = ( - masked_trace_integral / self.event_median[:, :, np.newaxis]) - relative_gain_event = np.ma.getdata(masked_relative_gain_event) - - # extract the median, mean and std over all the events j and - # fill the container and return it - self.container.time_mean = ( - (trigger_time - self.time_start) / 2 * u.s) - self.container.time_range = [self.time_start, trigger_time] * u.s - self.container.n_events = self.num_events_seen - self.container.relative_gain_median = np.median( - relative_gain_event, axis=0) - self.container.relative_gain_mean = np.mean( - relative_gain_event, axis=0) - self.container.relative_gain_rms = np.std( - relative_gain_event, axis=0) - - # extract the average time over the camera and the events - camera_time_median = np.ma.median(masked_trace_time) - camera_time_mean = np.ma.mean(masked_trace_time) - pixel_time_median = np.ma.median(masked_trace_time, axis=0) - pixel_time_mean = np.ma.mean(masked_trace_time, axis=0) - - # fill the container - self.container.relative_time_median = np.ma.getdata( - pixel_time_median - camera_time_median - ) - self.container.relative_time_mean = np.ma.getdata( - pixel_time_mean - camera_time_mean - ) + result = { + 'n_events': self.num_events_seen, + **relative_gain_results, + **time_results, + } + for key, value in result.items(): + setattr(self.container, key, value) - # re-initialize the event count self.num_events_seen = 0 - return self.container else: @@ -268,6 +243,53 @@ def calculate_relative_gain(self, event, tel_id): return None +def calculate_time_results( + trace_time, + bad_pixels_of_sample, + time_start, + trigger_time, +): + masked_trace_time = np.ma.array( + trace_time, + mask=bad_pixels_of_sample + ) + + # extract the average time over the camera and the events + camera_time_median = np.ma.median(masked_trace_time) + camera_time_mean = np.ma.mean(masked_trace_time) + pixel_time_median = np.ma.median(masked_trace_time, axis=0) + pixel_time_mean = np.ma.mean(masked_trace_time, axis=0) + + return { + 'time_mean': (trigger_time - time_start) / 2 * u.s, + 'time_range': [time_start, trigger_time] * u.s, + 'relative_time_median': np.ma.getdata( + pixel_time_median - camera_time_median), + 'relative_time_mean': np.ma.getdata( + pixel_time_mean - camera_time_mean), + } + + +def calculate_relative_gain_results( + event_median, + trace_integral, + bad_pixels_of_sample, +): + masked_trace_integral = np.ma.array( + trace_integral, + mask=bad_pixels_of_sample + ) + relative_gain_event = np.ma.getdata( + masked_trace_integral / event_median[:, :, np.newaxis] + ) + + return { + 'relative_gain_median': np.median(relative_gain_event, axis=0), + 'relative_gain_mean': np.mean(relative_gain_event, axis=0), + 'relative_gain_rms': np.std(relative_gain_event, axis=0), + } + + class FlatFieldFactory(Factory): """ Factory to obtain flat-field coefficients From 443c7572c3105c66e07409c118b609b3b2dfa3c3 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 19 Dec 2018 12:30:12 +0100 Subject: [PATCH 60/82] no c-style backets needed in if-clause --- examples/calc_flatfield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index d26cd257c3d..46738b6c862 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -74,7 +74,7 @@ def start(self): for tel_id in event.r0.tels_with_data: # initialize the flat fieLd containers - if (count == 0): + if count == 0: self.container.flatfield.tels_with_data.append(tel_id) ff_data = self.flatfield.calculate_relative_gain(event, tel_id) From 277cf186d8085bd4d8787efe96fca9201d494198 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 19 Dec 2018 12:34:56 +0100 Subject: [PATCH 61/82] white-spaces and shorten help text --- ctapipe/calib/camera/flatfield.py | 36 +++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index 5465b55eb62..d0be26d5150 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -19,19 +19,33 @@ class FlatFieldCalculator(Component): """ - Parent class for the flat field calculators. Fills the MON.flatfield container. - + Parent class for the flat field calculators. + Fills the MON.flatfield container. """ - max_time_range_s = Int(60, help='Define the maximum time interval per' - ' coefficient flat-filed calculation').tag(config=True) - max_events = Int(10000, help='Define the maximum number of events per ' - ' coefficient flat-filed calculation').tag(config=True) - n_channels = Int(2, help='Define the number of channels to be ' - 'treated ').tag(config=True) - - def __init__(self, config=None, tool=None, extractor_product=None, cleaner_product=None, **kwargs): + max_time_range_s = Int( + 60, + help='sample duration in seconds' + ).tag(config=True) + max_events = Int( + 10000, + help='sample size' + ).tag(config=True) + n_channels = Int( + 2, + help='number of channels to be treated' + ).tag(config=True) + + def __init__( + self, + config=None, + tool=None, + extractor_product=None, + cleaner_product=None, + **kwargs + ): """ - Parent class for the flat field calculators. Fills the MON.flatfield container. + Parent class for the flat field calculators. + Fills the MON.flatfield container. Parameters ---------- From 7b04c71b559c9fa2cd69441e075f51fc64584f3d Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 19 Dec 2018 12:35:48 +0100 Subject: [PATCH 62/82] rename some fields --- ctapipe/calib/camera/flatfield.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index d0be26d5150..52f1776fc7f 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -22,11 +22,11 @@ class FlatFieldCalculator(Component): Parent class for the flat field calculators. Fills the MON.flatfield container. """ - max_time_range_s = Int( + sample_duration = Int( 60, help='sample duration in seconds' ).tag(config=True) - max_events = Int( + sample_size = Int( 10000, help='sample size' ).tag(config=True) @@ -126,7 +126,7 @@ def __init__(self, config=None, tool=None, **kwargs): """ super().__init__(config=config, tool=tool, **kwargs) - self.log.info("Used events statistics : %d", self.max_events) + self.log.info("Used events statistics : %d", self.sample_size) self.num_events_seen = 0 # members to keep state in calculate_relative_gain() @@ -193,10 +193,10 @@ def calculate_relative_gain(self, event, tel_id): if waveform.shape[0] < self.n_channels: self.n_channels = waveform.shape[0] - self.event_median = np.zeros((self.max_events, self.n_channels)) + self.event_median = np.zeros((self.sample_size, self.n_channels)) n_pix = waveform.shape[1] - shape = (self.max_events, self.n_channels, n_pix) + shape = (self.sample_size, self.n_channels, n_pix) self.trace_integral = np.zeros(shape) self.trace_time = np.zeros(shape) self.bad_pixels_of_sample = np.zeros(shape) @@ -226,8 +226,8 @@ def calculate_relative_gain(self, event, tel_id): sample_age = trigger_time - self.time_start # check if to create a calibration event if ( - sample_age > self.max_time_range_s - or self.num_events_seen == self.max_events + sample_age > self.sample_duration + or self.num_events_seen == self.sample_size ): relative_gain_results = calculate_relative_gain_results( self.event_median, From 5707c263e9bb7fc7a966e6156d44926c18f4a854 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 19 Dec 2018 12:46:59 +0100 Subject: [PATCH 63/82] add docu in doc-strings; clean up some doc-strings --- ctapipe/calib/camera/flatfield.py | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index 52f1776fc7f..b3a005f0f8a 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -92,44 +92,33 @@ def __init__( @abstractmethod def calculate_relative_gain(self, event): - """ + """calculate relative gain from event Parameters ---------- - event + event: DataContainer + + Returns: FlatFieldCameraContainer or None + None is returned if no new flat field coefficients were calculated + e.g. due to insufficient statistics. """ class FlasherFlatFieldCalculator(FlatFieldCalculator): - """ - Class for calculating flat field coefficients witht the - flasher data. Fills the MON.flatfield container. - """ def __init__(self, config=None, tool=None, **kwargs): - """ - Parent class for the flat-field calculators. - Fills the MON.flatfield container. + """Calculates flat field coefficients from flasher data - Parameters - ---------- - config : traitlets.loader.Config - Configuration specified by config file or cmdline arguments. - Used to set traitlet values. - Set to None if no configuration to pass. - tool : ctapipe.core.Tool - Tool executable that is calling this component. - Passes the correct logger to the component. - Set to None if no Tool to pass. - kwargs + based on the best algorithm described by S. Fegan in MST-CAM-TN-0060 + Parameters: see base class FlatFieldCalculator """ super().__init__(config=config, tool=tool, **kwargs) self.log.info("Used events statistics : %d", self.sample_size) - self.num_events_seen = 0 # members to keep state in calculate_relative_gain() + self.num_events_seen = 0 self.time_start = None # trigger time of first event in sample self.event_median = None # med. charge in camera per event in sample self.trace_integral = None # charge per event in sample From d52612da1902472268b9649d63b4d1216e2a6216 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 19 Dec 2018 13:51:55 +0100 Subject: [PATCH 64/82] adjust test to renamed fields --- ctapipe/calib/camera/tests/test_flatfield.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ctapipe/calib/camera/tests/test_flatfield.py b/ctapipe/calib/camera/tests/test_flatfield.py index 22f355d5684..24b1fe12da1 100644 --- a/ctapipe/calib/camera/tests/test_flatfield.py +++ b/ctapipe/calib/camera/tests/test_flatfield.py @@ -12,13 +12,12 @@ def test_FlasherFlatFieldCalculator(): max_events=10 ) - ff_calculator = FlasherFlatFieldCalculator() - ff_calculator.max_events = 3 + ff_calculator = FlasherFlatFieldCalculator(sample_size=3) for event in inputfile_reader: for tel_id in event.r0.tels_with_data: ff_data = ff_calculator.calculate_relative_gain(event, tel_id) - if ff_calculator.num_events_seen == ff_calculator.max_events: + if ff_calculator.num_events_seen == ff_calculator.sample_size: assert ff_data From a9063a47cf48f3cf2dd958215af6bbe3bef099b9 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 19 Dec 2018 14:43:36 +0100 Subject: [PATCH 65/82] rename some variables; break out more functions --- ctapipe/calib/camera/flatfield.py | 77 +++++++++++++++---------------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index b3a005f0f8a..f17385e8ffd 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -120,10 +120,10 @@ def __init__(self, config=None, tool=None, **kwargs): # members to keep state in calculate_relative_gain() self.num_events_seen = 0 self.time_start = None # trigger time of first event in sample - self.event_median = None # med. charge in camera per event in sample - self.trace_integral = None # charge per event in sample - self.trace_time = None # arrival time per event in sample - self.bad_pixels_of_sample = None # bad pixels per event in sample + self.charge_medians = None # med. charge in camera per event in sample + self.charges = None # charge per event in sample + self.arrival_times = None # arrival time per event in sample + self.sample_bad_pixels = None # bad pixels per event in sample def _extract_charge(self, event, tel_id): """ @@ -178,39 +178,12 @@ def calculate_relative_gain(self, event, tel_id): if self.num_events_seen == 0: self.time_start = trigger_time - - if waveform.shape[0] < self.n_channels: - self.n_channels = waveform.shape[0] - - self.event_median = np.zeros((self.sample_size, self.n_channels)) - - n_pix = waveform.shape[1] - shape = (self.sample_size, self.n_channels, n_pix) - self.trace_integral = np.zeros(shape) - self.trace_time = np.zeros(shape) - self.bad_pixels_of_sample = np.zeros(shape) + self.setup_sample_buffers(waveform, self.sample_size) # extract the charge of the event and # the peak position (assumed as time for the moment) - integral, peakpos = self._extract_charge(event, tel_id) - - # remember the charge - self.trace_integral[self.num_events_seen] = integral - - # remember the time - self.trace_time[self.num_events_seen] = peakpos - - # keep the mask of not working pixels (to be improved) - bad_pixels = np.array([pixel_status == 0, pixel_status == 0]) - self.bad_pixels_of_sample[self.num_events_seen, :] = bad_pixels - - # extract the median on all the camera per event: (i) - # (for not masked pixels) - masked_integral = np.ma.array(integral, mask=bad_pixels) - self.event_median[self.num_events_seen, :] = np.ma.median( - masked_integral, axis=1) - - self.num_events_seen += 1 + charge, arrival_time = self._extract_charge(event, tel_id) + self.collect_sample(charge, pixel_status, arrival_time) sample_age = trigger_time - self.time_start # check if to create a calibration event @@ -219,13 +192,13 @@ def calculate_relative_gain(self, event, tel_id): or self.num_events_seen == self.sample_size ): relative_gain_results = calculate_relative_gain_results( - self.event_median, - self.trace_integral, - self.bad_pixels_of_sample, + self.charge_medians, + self.charges, + self.sample_bad_pixels, ) time_results = calculate_time_results( - self.trace_time, - self.bad_pixels_of_sample, + self.arrival_times, + self.sample_bad_pixels, self.time_start, trigger_time, ) @@ -245,6 +218,32 @@ def calculate_relative_gain(self, event, tel_id): return None + def setup_sample_buffers(self, waveform, sample_size): + n_channels = waveform.shape[0] + n_pix = waveform.shape[1] + shape = (sample_size, n_channels, n_pix) + + self.charge_medians = np.zeros((sample_size, n_channels)) + self.charges = np.zeros(shape) + self.arrival_times = np.zeros(shape) + self.sample_bad_pixels = np.zeros(shape) + + def collect_sample(self, charge, pixel_status, arrival_time): + + # extract the charge of the event and + # the peak position (assumed as time for the moment) + bad_pixels = np.zeros(charge.shape, dtype=np.bool) + bad_pixels[:] = pixel_status == 0 + + good_charge = np.ma.array(charge, mask=bad_pixels) + charge_median = np.ma.median(good_charge, axis=1) + + self.charges[self.num_events_seen] = charge + self.arrival_times[self.num_events_seen] = arrival_time + self.sample_bad_pixels[self.num_events_seen] = bad_pixels + self.charge_medians[self.num_events_seen] = charge_median + self.num_events_seen += 1 + def calculate_time_results( trace_time, From 944fc6291eb014d771ef0a10651892f89a4e03a1 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 7 Jan 2019 14:45:15 +0100 Subject: [PATCH 66/82] Added tel_id as input argument of FlatFieldCalculator and removed looped over telescopes in calc_flatfield.py --- ctapipe/calib/camera/flatfield.py | 23 +++++++++-------- ctapipe/calib/camera/tests/test_flatfield.py | 9 +++---- examples/calc_flatfield.py | 27 +++++++++----------- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index f17385e8ffd..f754e1f4364 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -22,6 +22,11 @@ class FlatFieldCalculator(Component): Parent class for the flat field calculators. Fills the MON.flatfield container. """ + + tel_id = Int( + 0, + help='telescope id' + ).tag(config=True) sample_duration = Int( 60, help='sample duration in seconds' @@ -125,7 +130,7 @@ def __init__(self, config=None, tool=None, **kwargs): self.arrival_times = None # arrival time per event in sample self.sample_bad_pixels = None # bad pixels per event in sample - def _extract_charge(self, event, tel_id): + def _extract_charge(self, event): """ Extract the charge and the time from a calibration event @@ -133,10 +138,9 @@ def _extract_charge(self, event, tel_id): ---------- event : general event container - tel_id : telescope id """ - waveforms = event.r0.tel[tel_id].waveform + waveforms = event.r0.tel[self.tel_id].waveform # Clean waveforms if self.cleaner: @@ -148,7 +152,7 @@ def _extract_charge(self, event, tel_id): # Extract charge and time if self.extractor: if self.extractor.requires_neighbours(): - g = event.inst.subarray.tel[tel_id].camera + g = event.inst.subarray.tel[self.tel_id].camera self.extractor.neighbours = g.neighbor_matrix_where charge, peak_pos, window = self.extractor.extract_charge(cleaned) @@ -160,7 +164,7 @@ def _extract_charge(self, event, tel_id): return charge, peak_pos - def calculate_relative_gain(self, event, tel_id): + def calculate_relative_gain(self, event): """ calculate the relative flat field coefficients @@ -168,13 +172,12 @@ def calculate_relative_gain(self, event, tel_id): ---------- event : general event container - tel_id : telescope id for which we calculate the gain """ # initialize the np array at each cycle - waveform = event.r0.tel[tel_id].waveform - trigger_time = event.r0.tel[tel_id].trigger_time - pixel_status = event.r0.tel[tel_id].pixel_status + waveform = event.r0.tel[self.tel_id].waveform + trigger_time = event.r0.tel[self.tel_id].trigger_time + pixel_status = event.r0.tel[self.tel_id].pixel_status if self.num_events_seen == 0: self.time_start = trigger_time @@ -182,7 +185,7 @@ def calculate_relative_gain(self, event, tel_id): # extract the charge of the event and # the peak position (assumed as time for the moment) - charge, arrival_time = self._extract_charge(event, tel_id) + charge, arrival_time = self._extract_charge(event) self.collect_sample(charge, pixel_status, arrival_time) sample_age = trigger_time - self.time_start diff --git a/ctapipe/calib/camera/tests/test_flatfield.py b/ctapipe/calib/camera/tests/test_flatfield.py index 24b1fe12da1..51c1dec428b 100644 --- a/ctapipe/calib/camera/tests/test_flatfield.py +++ b/ctapipe/calib/camera/tests/test_flatfield.py @@ -12,12 +12,11 @@ def test_FlasherFlatFieldCalculator(): max_events=10 ) - ff_calculator = FlasherFlatFieldCalculator(sample_size=3) + ff_calculator = FlasherFlatFieldCalculator(sample_size=3, tel_id=0) for event in inputfile_reader: - for tel_id in event.r0.tels_with_data: - ff_data = ff_calculator.calculate_relative_gain(event, tel_id) + ff_data = ff_calculator.calculate_relative_gain(event) - if ff_calculator.num_events_seen == ff_calculator.sample_size: - assert ff_data + if ff_calculator.num_events_seen == ff_calculator.sample_size: + assert ff_data diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index 46738b6c862..e7d40133309 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -36,6 +36,7 @@ class FlatFieldGenerator(Tool): cleaner='WaveformCleanerFactory.product', cleaner_width='WaveformCleanerFactory.baseline_width', generator='FlatFieldFactory.product', + tel_id='FlatFieldFactory.tel_id', max_time_range_s='FlatFieldFactory.max_time_range_s', ff_events='FlatFieldFactory.max_events', n_channels='FlatFieldFactory.n_channels', @@ -69,25 +70,21 @@ def setup(self): def start(self): '''Flat field coefficient calculator''' - for count, event in enumerate(self.eventsource): - - for tel_id in event.r0.tels_with_data: + # initialize the flat fieLd containers + self.container.flatfield.tels_with_data.append(self.flatfield.tel_id) - # initialize the flat fieLd containers - if count == 0: - self.container.flatfield.tels_with_data.append(tel_id) + for count, event in enumerate(self.eventsource): - ff_data = self.flatfield.calculate_relative_gain(event, tel_id) + ff_data = self.flatfield.calculate_relative_gain(event) - if ff_data: - self.container.flatfield.tel[tel_id] = ff_data - table_name = 'tel_' + str(tel_id) + if ff_data: + self.container.flatfield.tel[self.flatfield.tel_id] = ff_data + table_name = 'tel_' + str(self.flatfield.tel_id) - self.log.info( - "write event in table: /flatfield/%s", - table_name - ) - self.writer.write(table_name, ff_data) + self.log.info( + "write event in table: /flatfield/%s", table_name + ) + self.writer.write(table_name, ff_data) def finish(self): Provenance().add_output_file( From ae87b2e131b61e20097b393a86f025cf74c0456e Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 7 Jan 2019 16:08:04 +0100 Subject: [PATCH 67/82] Changed name of FlatFieldCalculator arguments --- examples/calc_flatfield.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index e7d40133309..e0422064c25 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -37,8 +37,8 @@ class FlatFieldGenerator(Tool): cleaner_width='WaveformCleanerFactory.baseline_width', generator='FlatFieldFactory.product', tel_id='FlatFieldFactory.tel_id', - max_time_range_s='FlatFieldFactory.max_time_range_s', - ff_events='FlatFieldFactory.max_events', + sample_duration='FlatFieldFactory.sample_duration', + sample_size='FlatFieldFactory.sample_size', n_channels='FlatFieldFactory.n_channels', )) From faddfa005843c5659f2a1ee89a41a064ef138d03 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 7 Jan 2019 16:15:53 +0100 Subject: [PATCH 68/82] Minor changes --- ctapipe/calib/camera/flatfield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index f754e1f4364..d2d6371caf9 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -25,7 +25,7 @@ class FlatFieldCalculator(Component): tel_id = Int( 0, - help='telescope id' + help='id of the telescope to calculate the flat-field coefficients' ).tag(config=True) sample_duration = Int( 60, From e3925ecc43951bc24cf31447e17d4bbdc823b963 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Wed, 9 Jan 2019 14:48:53 +0100 Subject: [PATCH 69/82] Changed name of class FlatFieldGenaretor to FlatFieldHDF5Writer for consistency with python naming convention --- examples/calc_flatfield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index e0422064c25..09d073e6664 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -13,7 +13,7 @@ from ctapipe.io.containers import MonitorDataContainer -class FlatFieldGenerator(Tool): +class FlatFieldHDF5Writer(Tool): name = "FlatFieldGenerator" description = "Generate a HDF5 file with flat field coefficients" From 36b91372e831df8fe95099d6147cfd9d0b3d911e Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Wed, 9 Jan 2019 15:02:19 +0100 Subject: [PATCH 70/82] Change forgotten names --- examples/calc_flatfield.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index 09d073e6664..0a7a61a7cc2 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -14,7 +14,7 @@ class FlatFieldHDF5Writer(Tool): - name = "FlatFieldGenerator" + name = "FlatFieldHDF5Writer" description = "Generate a HDF5 file with flat field coefficients" output_file = Unicode( @@ -95,7 +95,7 @@ def finish(self): def main(): - exe = FlatFieldGenerator() + exe = FlatFieldHDF5Writer() exe.run() From 4d56184f06eab1ba8032e9fee95143014bfd4049 Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Sun, 13 Jan 2019 12:17:34 +0100 Subject: [PATCH 71/82] Minor style changes --- ctapipe/image/waveform_cleaning.py | 11 +++++---- ctapipe/io/containers.py | 36 ++++++++++++++++++------------ 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/ctapipe/image/waveform_cleaning.py b/ctapipe/image/waveform_cleaning.py index d6c86dbd2c6..b4d4ed42fe0 100644 --- a/ctapipe/image/waveform_cleaning.py +++ b/ctapipe/image/waveform_cleaning.py @@ -13,7 +13,7 @@ LocalPeakIntegrator) __all__ = ['WaveformCleanerFactory', 'CHECMWaveformCleanerAverage', - 'CHECMWaveformCleanerLocal','BaselineWaveformCleaner', + 'CHECMWaveformCleanerLocal', 'BaselineWaveformCleaner', 'NullWaveformCleaner'] @@ -66,17 +66,20 @@ class NullWaveformCleaner(WaveformCleaner): def apply(self, waveforms): return waveforms + class BaselineWaveformCleaner(WaveformCleaner): """ Basic waveform cleaner that simply returns the waveform subtracted from the baseline """ baseline_width = Int(10, help='Define then number of samples for estimating the ' - 'baseline').tag(config=True) + 'baseline').tag(config=True) + def apply(self, waveforms): - #self.log.debug(f"calculate baseline on first {self.baseline_width} samples") + # self.log.debug(f"calculate baseline on first {self.baseline_width} samples") # Subtract initial baseline - baseline_sub = waveforms - np.mean(waveforms[:, :self.baseline_width], axis=1)[:, None] + baseline_sub = waveforms - \ + np.mean(waveforms[:, :self.baseline_width], axis=1)[:, None] return baseline_sub diff --git a/ctapipe/io/containers.py b/ctapipe/io/containers.py index 7ca330130a7..a4a0a67e603 100644 --- a/ctapipe/io/containers.py +++ b/ctapipe/io/containers.py @@ -783,27 +783,34 @@ class FlatFieldCameraContainer(Container): """ time_mean = Field(0, 'Mean time, seconds since reference', unit=u.s) - time_range = Field([], 'Range of time of the calibration data [t_min, t_max]', unit=u.s) - - n_events = Field(0,'Number of events used for statistics') - - relative_gain_mean = Field(None, + time_range = Field( + [], + 'Range of time of the calibration data [t_min, t_max]', + unit=u.s + ) + n_events = Field(0, 'Number of events used for statistics') + relative_gain_mean = Field( + None, "np array of the relative flat-field coefficient mean (n_chan X N_pix)" ) - relative_gain_median = Field(None, + relative_gain_median = Field( + None, "np array of the relative flat-field coefficient median (n_chan X N_pix)" ) - - relative_gain_rms = Field(None, + relative_gain_rms = Field( + None, "np array of the relative flat-field coefficient rms (n_chan X N_pix)" ) + relative_time_mean = Field( + None, + "np array of the relative time mean (n_chan X N_pix)", + unit=u.ns + ) + relative_time_median = Field( + None, + "np array of the relative time median (n_chan X N_pix)", + unit=u.ns) - relative_time_mean = Field(None, - "np array of the relative time mean (n_chan X N_pix)" - , unit=u.ns) - relative_time_median = Field(None, - "np array of the relative time median (n_chan X N_pix)" - , unit=u.ns) class FlatFieldContainer(Container): """ @@ -831,6 +838,7 @@ class PedestalContainer(Container): Map(PedestalCameraContainer), "map of tel_id to PedestalCameraContainer") + class MonitorDataContainer(Container): """ Root container for MON data From 0f5f1437fde10c05c8169c6b8a019fc06d9b71df Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Mon, 14 Jan 2019 09:54:39 +0100 Subject: [PATCH 72/82] Minor style changes --- ctapipe/calib/camera/__init__.py | 4 ++-- ctapipe/calib/camera/tests/test_flatfield.py | 2 +- ctapipe/image/waveform_cleaning.py | 4 ++-- examples/calc_flatfield.py | 5 ++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ctapipe/calib/camera/__init__.py b/ctapipe/calib/camera/__init__.py index 1acd56c3a14..9d8467b184e 100644 --- a/ctapipe/calib/camera/__init__.py +++ b/ctapipe/calib/camera/__init__.py @@ -7,5 +7,5 @@ from .dl0 import * from .dl1 import * from .calibrator import * -from .flatfield import * -from .pedestals import * + + diff --git a/ctapipe/calib/camera/tests/test_flatfield.py b/ctapipe/calib/camera/tests/test_flatfield.py index 51c1dec428b..491b9da04df 100644 --- a/ctapipe/calib/camera/tests/test_flatfield.py +++ b/ctapipe/calib/camera/tests/test_flatfield.py @@ -3,7 +3,7 @@ from ctapipe.calib.camera.flatfield import FlasherFlatFieldCalculator -def test_FlasherFlatFieldCalculator(): +def test_flasherflatfieldcalculator(): example_file_path = get_dataset_path("NectarCAM.Run0890.10events.fits.fz") diff --git a/ctapipe/image/waveform_cleaning.py b/ctapipe/image/waveform_cleaning.py index b4d4ed42fe0..67f0e526939 100644 --- a/ctapipe/image/waveform_cleaning.py +++ b/ctapipe/image/waveform_cleaning.py @@ -78,8 +78,8 @@ class BaselineWaveformCleaner(WaveformCleaner): def apply(self, waveforms): # self.log.debug(f"calculate baseline on first {self.baseline_width} samples") # Subtract initial baseline - baseline_sub = waveforms - \ - np.mean(waveforms[:, :self.baseline_width], axis=1)[:, None] + baseline_sub = waveforms - np.mean(waveforms[:, :self.baseline_width], + axis=1)[:, None] return baseline_sub diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index 0a7a61a7cc2..c06c0000102 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -81,9 +81,8 @@ def start(self): self.container.flatfield.tel[self.flatfield.tel_id] = ff_data table_name = 'tel_' + str(self.flatfield.tel_id) - self.log.info( - "write event in table: /flatfield/%s", table_name - ) + self.log.info("write event in table: /flatfield/%s", + table_name) self.writer.write(table_name, ff_data) def finish(self): From 67b0e11a0870b89dc770bab10e7bb03c1b5629fa Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 17 Jan 2019 15:34:11 +0100 Subject: [PATCH 73/82] Corrected bug in BaselineWaveformCleaner (wrong shape in the baseline_sub caliculation) Patched setting of event trigger time for MC events FlatFieldCalculator.calculate_relative_gain --- ctapipe/calib/camera/flatfield.py | 8 ++++++++ ctapipe/image/waveform_cleaning.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index d2d6371caf9..7d35bc592b6 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -1,6 +1,7 @@ """ Factory for the estimation of the flat field coefficients """ + from abc import abstractmethod import numpy as np from astropy import units as u @@ -177,6 +178,11 @@ def calculate_relative_gain(self, event): # initialize the np array at each cycle waveform = event.r0.tel[self.tel_id].waveform trigger_time = event.r0.tel[self.tel_id].trigger_time + + # patch for MC data which no trigger time defined + if (trigger_time == None): + trigger_time = event.trig.gps_time.unix + pixel_status = event.r0.tel[self.tel_id].pixel_status if self.num_events_seen == 0: @@ -189,6 +195,8 @@ def calculate_relative_gain(self, event): self.collect_sample(charge, pixel_status, arrival_time) sample_age = trigger_time - self.time_start + + # check if to create a calibration event if ( sample_age > self.sample_duration diff --git a/ctapipe/image/waveform_cleaning.py b/ctapipe/image/waveform_cleaning.py index 67f0e526939..be6ed1c6485 100644 --- a/ctapipe/image/waveform_cleaning.py +++ b/ctapipe/image/waveform_cleaning.py @@ -78,8 +78,8 @@ class BaselineWaveformCleaner(WaveformCleaner): def apply(self, waveforms): # self.log.debug(f"calculate baseline on first {self.baseline_width} samples") # Subtract initial baseline - baseline_sub = waveforms - np.mean(waveforms[:, :self.baseline_width], - axis=1)[:, None] + baseline_sub = waveforms - np.mean(waveforms[:,:, :self.baseline_width], + axis=2)[:,:, None] return baseline_sub From 340d87de7522aa7f233857811b28f65d6f6fa66a Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Thu, 17 Jan 2019 17:33:04 +0100 Subject: [PATCH 74/82] Corrected case of MC events which do not have a pixel_status defined --- ctapipe/calib/camera/flatfield.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index 7d35bc592b6..f4de088bca2 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -177,13 +177,16 @@ def calculate_relative_gain(self, event): # initialize the np array at each cycle waveform = event.r0.tel[self.tel_id].waveform - trigger_time = event.r0.tel[self.tel_id].trigger_time - # patch for MC data which no trigger time defined - if (trigger_time == None): + + # patches for MC data + if not event.mcheader.simtel_version: + trigger_time = event.r0.tel[self.tel_id].trigger_time + pixel_status = event.r0.tel[self.tel_id].pixel_status + else: trigger_time = event.trig.gps_time.unix + pixel_status = np.ones(waveform.shape[1]) - pixel_status = event.r0.tel[self.tel_id].pixel_status if self.num_events_seen == 0: self.time_start = trigger_time From 4d52dc9b0b262a6707f25065c78243ca1b388f98 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 17 Jan 2019 18:48:56 +0100 Subject: [PATCH 75/82] remove optional deps in order to fix tests --- .travis.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82f87bce8a8..09468ea8ff1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,11 +11,6 @@ os: - linux # - osx # currently osx python projects are not supported in Travis -env: - global: - - TARGET_SOFTWARE=$HOME/Software/target_software - - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/Software/TargetDriver/install/lib:$HOME/Software/TargetIO/install/lib:$HOME/Software/TargetCalib/install/lib - - TARGETCALIBPATH=$HOME/Software/TargetCalib/install before_install: @@ -52,20 +47,6 @@ install: - source activate cta-dev - ulimit -s 16000 # increase stack size limit, for libhessio - pip install travis-sphinx - - pip install codecov - # ----- SST1M: - - pip install https://github.com/cta-sst-1m/protozfitsreader/archive/v1.4.2.tar.gz - # ----- end of SST1M - # ----- target_software - - conda install -c conda-forge cfitsio - - conda install swig - - mkdir -p $HOME/Software - - cd $HOME/Software - - git clone https://github.com/watsonjj/target_software.git - - cd $TARGET_SOFTWARE - - ./install.sh - - cd $TRAVIS_BUILD_DIR - # ----- end of target_software - python setup.py develop script: From c90caf26b4aa62de0654fefb86ee9202b7763f34 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Thu, 17 Jan 2019 19:16:41 +0100 Subject: [PATCH 76/82] remove tests for py3.5 since it does actually test for 3.6 only --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 09468ea8ff1..fa0b17ee016 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ sudo: required python: # not used directly, but this sets TRAVIS_PYTHON_VERSION so we can use it # in anaconda as well (we don't support python less than 3.5) - - 3.5 - 3.6 os: From ebc63ce688dd8551e47d505312b366bea0b3c96c Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Fri, 18 Jan 2019 09:15:03 +0100 Subject: [PATCH 77/82] add py3.7 and osx to test matrix --- .travis.yml | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index fa0b17ee016..352cde6f756 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,23 @@ -language: python -sudo: required +language: generic -python: - # not used directly, but this sets TRAVIS_PYTHON_VERSION so we can use it - # in anaconda as well (we don't support python less than 3.5) - - 3.6 - -os: - - linux - # - osx # currently osx python projects are not supported in Travis +matrix: + include: + - os: linux + python: 3.6 + env: + - PYTHON_VERSION=3.6 + - os: linux + python: 3.7 + env: + - PYTHON_VERSION=3.7 + - os: osx + python: 3.6 + env: + - PYTHON_VERSION=3.6 + - os: osx + python: 3.7 + env: + - PYTHON_VERSION=3.7 before_install: @@ -41,7 +50,7 @@ before_install: - git fetch --tags install: - - conda create --name cta-dev python=$TRAVIS_PYTHON_VERSION + - conda create --name cta-dev python=$PYTHON_VERSION - conda env update -n cta-dev --file environment.yml - source activate cta-dev - ulimit -s 16000 # increase stack size limit, for libhessio From 77524847bccd426bd1592085246c21164adad32f Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Fri, 18 Jan 2019 09:20:39 +0100 Subject: [PATCH 78/82] xvfb does not work on osx --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 352cde6f756..55a3571adaa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: # against future changes - export PYTHONIOENCODING=UTF8 - - export MPLBACKEND=agg + - export MPLBACKEND=Agg # Install miniconda following instructions at # http://conda.pydata.org/docs/travis.html @@ -43,10 +43,6 @@ before_install: - conda update -q conda # get latest conda version # Useful for debugging any issues with conda - conda info -a - - # Make sure that interactive matplotlib backends work - - export DISPLAY=:99.0 - - sh -e /etc/init.d/xvfb start - git fetch --tags install: From d1bd8457eb007f688ec58162c85769247d87a985 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Fri, 18 Jan 2019 09:35:07 +0100 Subject: [PATCH 79/82] allow this to take 30min --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 55a3571adaa..a9acba1cc44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ before_install: install: - conda create --name cta-dev python=$PYTHON_VERSION - - conda env update -n cta-dev --file environment.yml + - travis_timeout 30 conda env update -n cta-dev --file environment.yml - source activate cta-dev - ulimit -s 16000 # increase stack size limit, for libhessio - pip install travis-sphinx From 8fcdc8febc786793358fc5ff0083cbb91043d412 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Fri, 18 Jan 2019 09:42:06 +0100 Subject: [PATCH 80/82] fix typo its travis_wait .. not what I wrote before --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a9acba1cc44..22ab26de179 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ before_install: install: - conda create --name cta-dev python=$PYTHON_VERSION - - travis_timeout 30 conda env update -n cta-dev --file environment.yml + - travis_wait 20 conda env update -n cta-dev --file environment.yml - source activate cta-dev - ulimit -s 16000 # increase stack size limit, for libhessio - pip install travis-sphinx From 360a8a2991fe7fdd72add078efd23f132e28d41d Mon Sep 17 00:00:00 2001 From: Franca Cassol Date: Fri, 18 Jan 2019 11:01:27 +0100 Subject: [PATCH 81/82] Moved test_pedestal.py to ctapipe/calib/camera/tests Added a a window_shift argument to BaselineWaveformCleaner --- ctapipe/calib/{ => camera}/tests/test_pedestals.py | 0 ctapipe/image/waveform_cleaning.py | 4 +++- examples/calc_flatfield.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) rename ctapipe/calib/{ => camera}/tests/test_pedestals.py (100%) diff --git a/ctapipe/calib/tests/test_pedestals.py b/ctapipe/calib/camera/tests/test_pedestals.py similarity index 100% rename from ctapipe/calib/tests/test_pedestals.py rename to ctapipe/calib/camera/tests/test_pedestals.py diff --git a/ctapipe/image/waveform_cleaning.py b/ctapipe/image/waveform_cleaning.py index be6ed1c6485..5ad6ebf6ba5 100644 --- a/ctapipe/image/waveform_cleaning.py +++ b/ctapipe/image/waveform_cleaning.py @@ -75,10 +75,12 @@ class BaselineWaveformCleaner(WaveformCleaner): baseline_width = Int(10, help='Define then number of samples for estimating the ' 'baseline').tag(config=True) + window_shift = Int(0, help='Define the shift of the window on which caluclate ' + 'the baseline.').tag(config=True) def apply(self, waveforms): # self.log.debug(f"calculate baseline on first {self.baseline_width} samples") # Subtract initial baseline - baseline_sub = waveforms - np.mean(waveforms[:,:, :self.baseline_width], + baseline_sub = waveforms - np.mean(waveforms[:,:, self.window_shift:self.baseline_width], axis=2)[:,:, None] return baseline_sub diff --git a/examples/calc_flatfield.py b/examples/calc_flatfield.py index c06c0000102..8cfca81d7e3 100644 --- a/examples/calc_flatfield.py +++ b/examples/calc_flatfield.py @@ -35,6 +35,7 @@ class FlatFieldHDF5Writer(Tool): lwt='ChargeExtractorFactory.lwt', cleaner='WaveformCleanerFactory.product', cleaner_width='WaveformCleanerFactory.baseline_width', + cleaner_shift='WaveformCleanerFactory.window_shift', generator='FlatFieldFactory.product', tel_id='FlatFieldFactory.tel_id', sample_duration='FlatFieldFactory.sample_duration', From 14524490a84a69bcc8ef18772c41561da2979039 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 22 Jan 2019 13:08:57 +0100 Subject: [PATCH 82/82] add pytest.importorskip where needed --- ctapipe/calib/camera/tests/test_flatfield.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ctapipe/calib/camera/tests/test_flatfield.py b/ctapipe/calib/camera/tests/test_flatfield.py index 491b9da04df..1308471c1de 100644 --- a/ctapipe/calib/camera/tests/test_flatfield.py +++ b/ctapipe/calib/camera/tests/test_flatfield.py @@ -1,7 +1,14 @@ +import pytest from ctapipe.utils import get_dataset_path from ctapipe.io.nectarcameventsource import NectarCAMEventSource from ctapipe.calib.camera.flatfield import FlasherFlatFieldCalculator +# this test cannot run if protozfits is not installed +# since NectarCAMEventSource needs lsteventsource.MultiFiles which +# in turn needs protozfits, which is an optional dependency, so +# in case this is not installed the tests in this file cannot succeed +pytest.importorskip("protozfits", minversion="0.44.3") + def test_flasherflatfieldcalculator():