diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21e0d6a7..13f2b422 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: CI # by not building all branches on push, we avoid the duplicated builds in PRs on: + workflow_dispatch: push: # TODO # Remove comment: just a test to force CI on push diff --git a/docs/user-guide/env-vars.rst b/docs/user-guide/env-vars.rst index 2e9ffa00..1d2c344b 100644 --- a/docs/user-guide/env-vars.rst +++ b/docs/user-guide/env-vars.rst @@ -9,7 +9,7 @@ However, some environment variables are needed for ``nectarchain`` to work prope Mandatory --------- -:``NECTARCAMDATA``: path to local NectarCAM data. It can contain ``fits.fz`` run files, `~nectarchain.data.container.waveformsContainer.WaveformsContainer` or `~nectarchain.data.container.chargesContainer.ChargesContainer` HDF5 files. This is also where the `~nectarchain.data.management.DataManagement.findrun` method will automatically store NectarCAM run files when fetched from DIRAC. +:``NECTARCAMDATA``: path to local NectarCAM data. It can contain ``fits.fz`` run files, `~nectarchain.data.container.waveforms_container.WaveformsContainer` or `~nectarchain.data.container.charges_container.ChargesContainer` HDF5 files. This is also where the `~nectarchain.data.management.DataManagement.findrun` method will automatically store NectarCAM run files when fetched from DIRAC. Optional -------- diff --git a/docs/user-guide/howto-pedestal.rst b/docs/user-guide/howto-pedestal.rst index d0c44285..e1eaa50d 100644 --- a/docs/user-guide/howto-pedestal.rst +++ b/docs/user-guide/howto-pedestal.rst @@ -25,7 +25,7 @@ processed in slices with a fixed number of events set by the The pedestal estimation tool inherits the configurable parameters of the -`~nectarchain.makers.component.PedestalComponent.PedestalEstimationComponent`. +`~nectarchain.makers.component.pedestal_component.PedestalEstimationComponent`. The data can be filtered based on time using the ``ucts_tmin`` and ``ucts_tmax`` parameters and to eliminate outlier waveforms using the ``filter_method`` parameter. Two different methods to exclude outlier waveforms are implemented: @@ -46,11 +46,11 @@ To run the example script: Inspect the results ========================= The results are stored in a -`~nectarchain.data.container.pedestalContainer.NectarCAMPedestalContainer`. The +`~nectarchain.data.container.pedestal_container.NectarCAMPedestalContainer`. The results include information on pixels that were flagged as having an abnormal behavior during the computation of the pedestals. The flags are defined in in -`~nectarchain.data.container.pedestalContainer.PedestalFlagBits`. The +`~nectarchain.data.container.pedestal_container.PedestalFlagBits`. The results are accessible on the fly if the tool is run interactively (as in the example above) and stored in a `.h5` file. The user script `nectarchain/user_scripts/ltibaldo/show_pedestal_output.py` provides an example of how to access the results from disk and produce some plots: diff --git a/pyproject.toml b/pyproject.toml index a6fe588d..93ae88c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ test = [ dev = [ "setuptools_scm", ] - docs = [ "sphinx", "sphinx-autodoc-typehints", @@ -55,15 +54,30 @@ docs = [ "numpydoc", "tomli; python_version < '3.11'" ] - # we can use self-references to simplify all all = [ "nectarchain[test,docs,dev]", ] + +[tool.pytest.ini_options] +addopts = "--ignore=src/nectarchain/user_scripts" + + + [tool.setuptools.packages.find] where = ["src"] exclude = ["nectarchain._dev_version"] [tool.setuptools_scm] write_to = "src/nectarchain/_version.py" + +[tool.ruff.lint] +select = ["E", "F", "W"] # This ensures it mimics Flake8's error codes. +exclude = ["build/", "dist/", "*.pyc"] + +[tool.ruff] +line-length = 88 + +[tool.black] +line-length = 88 \ No newline at end of file diff --git a/src/nectarchain/data/__init__.py b/src/nectarchain/data/__init__.py index 1506a100..bc094f86 100644 --- a/src/nectarchain/data/__init__.py +++ b/src/nectarchain/data/__init__.py @@ -1,2 +1,34 @@ -from .container import * -from .management import * +"""Description: This file is used to import all the classes and functions +from the data module.""" + +from .container import ( + ArrayDataContainer, + ChargesContainer, + ChargesContainers, + GainContainer, + NectarCAMContainer, + NectarCAMPedestalContainer, + SPEfitContainer, + TriggerMapContainer, + WaveformsContainer, + WaveformsContainers, + get_array_keys, + merge_map_ArrayDataContainer, +) +from .management import DataManagement + +__all__ = [ + "ArrayDataContainer", + "NectarCAMContainer", + "TriggerMapContainer", + "get_array_keys", + "merge_map_ArrayDataContainer", + "ChargesContainer", + "ChargesContainers", + "WaveformsContainer", + "WaveformsContainers", + "GainContainer", + "SPEfitContainer", + "NectarCAMPedestalContainer", + "DataManagement", +] diff --git a/src/nectarchain/data/container/__init__.py b/src/nectarchain/data/container/__init__.py index 5b525634..574b5ed1 100644 --- a/src/nectarchain/data/container/__init__.py +++ b/src/nectarchain/data/container/__init__.py @@ -1,4 +1,7 @@ -from .chargesContainer import * +"""This file is used to import all the containerclasses in the data/container +folder.""" + +from .charges_container import ChargesContainer, ChargesContainers from .core import ( ArrayDataContainer, NectarCAMContainer, @@ -6,7 +9,27 @@ get_array_keys, merge_map_ArrayDataContainer, ) -from .eventSource import * -from .gainContainer import * -from .waveformsContainer import * -from .pedestalContainer import * +from .gain_container import GainContainer, SPEfitContainer +from .pedestal_container import ( + NectarCAMPedestalContainer, + NectarCAMPedestalContainers, + PedestalFlagBits, +) +from .waveforms_container import WaveformsContainer, WaveformsContainers + +__all__ = [ + "ArrayDataContainer", + "NectarCAMContainer", + "TriggerMapContainer", + "get_array_keys", + "merge_map_ArrayDataContainer", + "ChargesContainer", + "ChargesContainers", + "WaveformsContainer", + "WaveformsContainers", + "GainContainer", + "SPEfitContainer", + "NectarCAMPedestalContainer", + "NectarCAMPedestalContainers", + "PedestalFlagBits", +] diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/charges_container.py similarity index 83% rename from src/nectarchain/data/container/chargesContainer.py rename to src/nectarchain/data/container/charges_container.py index 02ef5603..23a3bd5a 100644 --- a/src/nectarchain/data/container/chargesContainer.py +++ b/src/nectarchain/data/container/charges_container.py @@ -1,20 +1,20 @@ import logging -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import numpy as np from ctapipe.containers import Field, Map, partial from .core import ArrayDataContainer, TriggerMapContainer +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["ChargesContainer", "ChargesContainers"] class ChargesContainer(ArrayDataContainer): - """ - A container that holds information about charges from a specific run. + """A container that holds information about charges from a specific run. Fields: charges_hg (np.ndarray): An array of high gain charges. @@ -40,13 +40,13 @@ class ChargesContainer(ArrayDataContainer): class ChargesContainers(TriggerMapContainer): - """ - Class representing a ChargesContainers. - - This class inherits from the `TriggerMapContainer` class and is used to store trigger or slices of data mappings of `ChargesContainer`. + """Class representing a ChargesContainers.This class inherits from the + `TriggerMapContainer` class and is used to store trigger or slices of data + mappings of `ChargesContainer`. Attributes: - containers (Field): A field representing the trigger or slices of data mapping of `ChargesContainer`. + containers (Field): A field representing the trigger or slices + of data mapping of `ChargesContainer`. """ containers = Field( diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index 0f7aa4e7..32384536 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -1,11 +1,6 @@ -import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import copy import importlib +import logging from pathlib import Path import numpy as np @@ -14,6 +9,11 @@ from ctapipe.io import HDF5TableReader from tables.exceptions import NoSuchNodeError +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = [ "ArrayDataContainer", "TriggerMapContainer", @@ -23,8 +23,8 @@ def get_array_keys(container: Container): - """ - Return a list of keys corresponding to fields which are array type in the given container. + """Return a list of keys corresponding to fields which are array type in the given + container. Parameters: container (Container): The container object to search for array fields. @@ -48,15 +48,14 @@ def get_array_keys(container: Container): class NectarCAMContainer(Container): - """ - Base class for the NectarCAM containers. This container cannot be recursive, - to be directly written with a HDF5TableWriter. + """Base class for the NectarCAM containers. + + This container cannot be recursive, to be directly written with a HDF5TableWriter. """ @staticmethod - def _container_from_hdf5(path, container_class): - """ - Static method to read a container from an HDF5 file. + def _container_from_hdf5(path, container_class, index_component=0): + """Static method to read a container from an HDF5 file. Parameters: path (str or Path): The path to the HDF5 file. @@ -66,7 +65,8 @@ def _container_from_hdf5(path, container_class): Container: The container from the data in the HDF5 file. Example: - >>> container = NectarCAMContainer._container_from_hdf5('path_to_file.h5', MyContainerClass) + >>> container = NectarCAMContainer._container_from_hdf5('path_to_file.h5', + MyContainerClass) """ if isinstance(path, str): path = Path(path) @@ -74,7 +74,7 @@ def _container_from_hdf5(path, container_class): container = container_class() with HDF5TableReader(path) as reader: tableReader = reader.read( - table_name=f"/data/{container_class.__name__}", + table_name=f"/data/{container_class.__name__}_{index_component}", containers=container_class, ) container = next(tableReader) @@ -82,9 +82,8 @@ def _container_from_hdf5(path, container_class): yield container @classmethod - def from_hdf5(cls, path): - """ - Reads a container from an HDF5 file. + def from_hdf5(cls, path, index_component=0): + """Reads a container from an HDF5 file. Parameters: path (str or Path): The path to the HDF5 file. @@ -99,12 +98,13 @@ def from_hdf5(cls, path): >>> container = NectarCAMContainer.from_hdf5('path_to_file.h5') """ - return cls._container_from_hdf5(path, container_class=cls) + return cls._container_from_hdf5( + path, container_class=cls, index_component=index_component + ) class ArrayDataContainer(NectarCAMContainer): - """ - A container that holds information about waveforms from a specific run. + """A container that holds information about waveforms from a specific run. Attributes: run_number (int): The run number associated with the waveforms. @@ -172,17 +172,18 @@ class ArrayDataContainer(NectarCAMContainer): class TriggerMapContainer(Container): - """ - Class representing a TriggerMapContainer. + """Class representing a TriggerMapContainer. - This class inherits from the `Container` class and is used to store trigger mappings of containers. + This class inherits from the `Container` class and is used to store trigger mappings + of containers. Attributes: containers (Field): A field representing the trigger mapping of containers. Methods: is_empty(): Checks if the TriggerMapContainer is empty. - validate(): Validates the TriggerMapContainer by checking if all the containers mapped are filled by correct type. + validate(): Validates the TriggerMapContainer by checking if all the containers + mapped are filled by correct type. Example: >>> container = TriggerMapContainer() @@ -198,15 +199,14 @@ class TriggerMapContainer(Container): ) @classmethod - def from_hdf5(cls, path, slice_index=None): - """ - Reads a container from an HDF5 file. + def from_hdf5(cls, path, slice_index=None, index_component=0): + """Reads a container from an HDF5 file. Parameters: path (str or Path): The path to the HDF5 file. - slice_index (int, optional): The index of the slice of data within the hdf5 file to read. Default is None. - - This method will call the _container_from_hdf5 method with the container argument associated to its own class (ArrayDataContainer) + slice_index (int, optional): The index of the slice of data within the hdf5 file + to read. Default is None.This method will call the _container_from_hdf5 method + with the container argument associated to its own class (ArrayDataContainer) Yields: Container: The container generator linked to the HDF5 file. @@ -216,25 +216,37 @@ def from_hdf5(cls, path, slice_index=None): """ return cls._container_from_hdf5( - path, slice_index=slice_index, container_class=cls + path, + slice_index=slice_index, + container_class=cls, + index_component=index_component, ) @staticmethod - def _container_from_hdf5(path, container_class, slice_index=None): - """ - Reads a container from an HDF5 file. + def _container_from_hdf5( + path, container_class, slice_index=None, index_component=0 + ): + # The way this method is coded is bad, there are confliuct behavior bettween + # containers inherited from TriggerMapContainer to truly be mapped with trigger, + # and those mapped with slices + """Reads a container from an HDF5 file. Parameters: path (str or Path): The path to the HDF5 file. container_class (Container): The class of the container to be read. - slice_index (int, optional): The index of the slice of data within the hdf5 file to read. Default is None. + slice_index (int, optional): The index of the slice of data within the hdf5 file + to read. Default is None. - This method first checks if the path is a string and converts it to a Path object if it is. - It then imports the module of the container class and creates an instance of the container class. + This method first checks if the path is a string and converts it to a Path + object + if it is. + It then imports the module of the container class and creates an instance of the + container class. If the HDF5 file contains more than one slice and no slice index is provided, it reads all slices and yields a generator of containers. - If a slice index is provided, it reads only the specified slice and returns the container instance. + If a slice index is provided, it reads only the specified slice and returns the + container instance. Yields: Container: The container associated to the HDF5 file. @@ -244,77 +256,107 @@ def _container_from_hdf5(path, container_class, slice_index=None): Exception: If any other error occurs. Example: - >>> container = ArrayDataContainer._container_from_hdf5('path_to_file.h5', MyContainerClass) + >>> container = ArrayDataContainer._container_from_hdf5('path_to_file.h5', + MyContainerClass) """ if isinstance(path, str): path = Path(path) - module = importlib.import_module(f"{container_class.__module__}") + module = importlib.import_module(f"{container_class.__module__}") # noqa :F841 container = eval(f"module.{container_class.__name__}")() with HDF5TableReader(path) as reader: if len(reader._h5file.root.__members__) > 1 and slice_index is None: log.info( - f"reading {container_class.__name__} containing {len(reader._h5file.root.__members__)} slices, will return a generator" + f"reading {container_class.__name__} containing" + f"{len(reader._h5file.root.__members__)}" + f"slices, will return a generator" ) - for data in reader._h5file.root.__members__: - # container.containers[data] = eval(f"module.{container_class.__name__}s")() - for key, trigger in EventType.__members__.items(): - try: - waveforms_data = eval( - f"reader._h5file.root.{data}.__members__" + for data in np.sort(reader._h5file.root.__members__): + # container.containers[data] = + # eval(f"module.{container_class.__name__}s")() + + _container = eval( + f"module." + f"{container.fields['containers'].default_factory.args[0].__name__}" # noqa + ) + waveforms_data = eval(f"reader._h5file.root.{data}.__members__") + _mask = [_container.__name__ in _word for _word in waveforms_data] + _waveforms_data = np.array(waveforms_data)[_mask] + if len(_waveforms_data) == 1: + if issubclass(_container, TriggerMapContainer) or issubclass( + _container, ArrayDataContainer + ): + for key, trigger in EventType.__members__.items(): + try: + tableReader = reader.read( + table_name=f"/{data}/{_waveforms_data[0]}" + f"/{trigger.name}", + containers=_container, + ) + container.containers[trigger] = next(tableReader) + except NoSuchNodeError as err: + log.warning(err) + except Exception as err: + log.error(err, exc_info=True) + raise err + else: + tableReader = reader.read( + table_name=f"/{data}/{_waveforms_data[0]}", + containers=_container, ) - _mask = [ - container_class.__name__ in _word - for _word in waveforms_data - ] - _waveforms_data = np.array(waveforms_data)[_mask] - if len(_waveforms_data) == 1: - tableReader = reader.read( - table_name=f"/{data}/{_waveforms_data[0]}/{trigger.name}", - containers=container_class, - ) - # container.containers[data].containers[trigger] = next(tableReader) - container.containers[trigger] = next(tableReader) - - else: - log.info( - f"there is {len(_waveforms_data)} entry corresponding to a {container_class} table save, unable to load" - ) - except NoSuchNodeError as err: - log.warning(err) - except Exception as err: - log.error(err, exc_info=True) - raise err - yield container + container.containers[data] = next(tableReader) + else: + log.info( + f"there is {len(_waveforms_data)} entry" + f"corresponding to a {container_class}" + f"table save, unable to load" + ) + + yield container else: if slice_index is None: log.info( - f"reading {container_class.__name__} containing a single slice, will return the {container_class.__name__} instance" + f"reading {container_class.__name__} containing" + f"a single slice," + f"will return the {container_class.__name__} instance" ) data = "data" else: log.info( - f"reading slice {slice_index} of {container_class.__name__}, will return the {container_class.__name__} instance" + f"reading slice {slice_index} of {container_class.__name__}," + f"will return the {container_class.__name__} instance" ) data = f"data_{slice_index}" - for key, trigger in EventType.__members__.items(): - try: - tableReader = reader.read( - table_name=f"/{data}/{trigger.name}", - containers=eval( - f"module.{container.fields['containers'].default_factory.args[0].__name__}" - ), - ) - container.containers[trigger] = next(tableReader) - except NoSuchNodeError as err: - log.warning(err) - except Exception as err: - log.error(err, exc_info=True) - raise err + _container = eval( + f"module.{container.fields['containers'].default_factory.args[0].__name__}" # noqa + ) + if issubclass(_container, TriggerMapContainer) or issubclass( + _container, ArrayDataContainer + ): + for key, trigger in EventType.__members__.items(): + try: + tableReader = reader.read( + table_name=f"/{data}/{_container.__name__}_" + f"{index_component}/{trigger.name}", + containers=_container, + ) + container.containers[trigger] = next(tableReader) + except NoSuchNodeError as err: + log.warning(err) + except Exception as err: + log.error(err, exc_info=True) + raise err + else: + tableReader = reader.read( + table_name=f"/{data}/{_container.__name__}_" + f"{index_component}", + containers=_container, + ) + container.containers[data] = next(tableReader) yield container def is_empty(self): - """This method check if the container is empty + """This method check if the container is empty. Returns: bool: True if the container is empty, False otherwise. @@ -322,7 +364,8 @@ def is_empty(self): return len(self.containers.keys()) == 0 def validate(self): - """apply the validate method recursively to all the containers that are mapped within the TriggerMapContainer + """Apply the validate method recursively to all the containers that are mapped + within the TriggerMapContainer. Raises: FieldValidationError: if one container is not valid. @@ -334,18 +377,21 @@ def validate(self): else: if not (isinstance(container, container_type)): raise FieldValidationError( - "all the containers mapped must have the same type to be merged " + "all the containers mapped must have the same type to be merged" ) def merge_map_ArrayDataContainer(triggerMapContainer: TriggerMapContainer): - """ - Merge and map ArrayDataContainer + """Merge and map ArrayDataContainer. - This function takes a TriggerMapContainer as input and merges the array fields of the containers mapped within the TriggerMapContainer. The merged array fields are concatenated along the 0th axis. The function also updates the 'nevents' field of the output container by summing the 'nevents' field of all the mapped containers. + This function takes a TriggerMapContainer as input and merges the array fields of + the containers mapped within the TriggerMapContainer. The merged array fields are + concatenated along the 0th axis. The function also updates the 'nevents' field of + the output container by summing the 'nevents' field of all the mapped containers. Parameters: - triggerMapContainer (TriggerMapContainer): The TriggerMapContainer object containing the containers to be merged and mapped. + triggerMapContainer (TriggerMapContainer): The TriggerMapContainer object + containing the containers to be merged and mapped. Returns: ArrayDataContainer: The merged and mapped ArrayDataContainer object. @@ -374,7 +420,8 @@ def merge_map_ArrayDataContainer(triggerMapContainer: TriggerMapContainer): """ triggerMapContainer.validate() log.warning( - "TAKE CARE TO MERGE CONTAINERS ONLY IF PIXELS ID, RUN_NUMBER (OR ANY FIELD THAT ARE NOT ARRAY) ARE THE SAME" + "TAKE CARE TO MERGE CONTAINERS ONLY IF PIXELS ID, RUN_NUMBER (OR ANY FIELD THAT\ + ARE NOT ARRAY) ARE THE SAME" ) keys = list(triggerMapContainer.containers.keys()) output_container = copy.deepcopy(triggerMapContainer.containers[keys[0]]) diff --git a/src/nectarchain/data/container/eventSource.py b/src/nectarchain/data/container/eventSource.py deleted file mode 100644 index bf55c8e5..00000000 --- a/src/nectarchain/data/container/eventSource.py +++ /dev/null @@ -1,261 +0,0 @@ -import logging -import struct - -import numpy as np -from ctapipe.containers import EventType -from ctapipe_io_nectarcam import ( - NectarCAMDataContainer, - NectarCAMEventSource, - TriggerBits, - time_from_unix_tai_ns, -) -from ctapipe_io_nectarcam.anyarray_dtypes import ( - CDTS_AFTER_37201_DTYPE, - CDTS_BEFORE_37201_DTYPE, - TIB_DTYPE, -) -from ctapipe_io_nectarcam.constants import N_PIXELS -from ctapipe_io_nectarcam.containers import NectarCAMEventContainer - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - - -__all__ = ["LightNectarCAMEventSource"] - - -def fill_nectarcam_event_container_from_zfile(self, array_event, event): - """ - Fill the NectarCAM event container from the zfile event data. - - Parameters: - - array_event: The NectarCAMDataContainer object to fill with event data. - - event: The event data from the zfile. - - Returns: - - None - - This function fills the NectarCAM event container in the NectarCAMDataContainer - object with the event data from the zfile. It unpacks the necessary data from the - event and assigns it to the corresponding fields in the event container. - - The function performs the following steps: - 1. Assigns the tel_id to the local variable. - 2. Creates a new NectarCAMEventContainer object and assigns it to the - event_container field of the NectarCAMDataContainer object. - 3. Assigns the extdevices_presence field of the event to the extdevices_presence - field of the event_container. - 4. Assigns the counters field of the event to the counters field of the - event_container. - 5. Unpacks the TIB data from the event and assigns it to the corresponding fields in - the event_container. - 6. Unpacks the CDTS data from the event and assigns it to the corresponding fields - in the event_container. - 7. Calls the unpack_feb_data function to unpack the FEB counters and trigger pattern - from the event and assign them to the corresponding fields in the event_container. - - """ - - tel_id = self.tel_id - event_container = NectarCAMEventContainer() - array_event.nectarcam.tel[tel_id].evt = event_container - event_container.extdevices_presence = event.nectarcam.extdevices_presence - event_container.counters = event.nectarcam.counters - # unpack TIB data - unpacked_tib = event.nectarcam.tib_data.view(TIB_DTYPE)[0] - event_container.tib_masked_trigger = unpacked_tib[4] - # unpack CDTS data - is_old_cdts = len(event.nectarcam.cdts_data) < 36 - if is_old_cdts: - unpacked_cdts = event.nectarcam.cdts_data.view(CDTS_BEFORE_37201_DTYPE)[0] - event_container.ucts_event_counter = unpacked_cdts[0] - event_container.ucts_timestamp = unpacked_cdts[3] - event_container.ucts_trigger_type = unpacked_cdts[5] - else: - unpacked_cdts = event.nectarcam.cdts_data.view(CDTS_AFTER_37201_DTYPE)[0] - event_container.ucts_timestamp = unpacked_cdts[0] - event_container.ucts_event_counter = unpacked_cdts[2] - event_container.ucts_busy_counter = unpacked_cdts[3] - event_container.ucts_trigger_type = unpacked_cdts[6] - # Unpack FEB counters and trigger pattern - self.unpack_feb_data(event_container, event) - - -def unpack_feb_data(self, event_container, event): - """Unpack FEB counters and trigger pattern""" - # Deduce data format version - bytes_per_module = ( - len(event.nectarcam.counters) // self.nectarcam_service.num_modules - ) - # Remain compatible with data before addition of trigger pattern - module_fmt = "IHHIBBBBBBBB" if bytes_per_module > 16 else "IHHIBBBB" - n_fields = len(module_fmt) - rec_fmt = "=" + module_fmt * self.nectarcam_service.num_modules - # Unpack - unpacked_feb = struct.unpack(rec_fmt, event.nectarcam.counters) - # Initialize field containers - if bytes_per_module > 16: - n_patterns = 4 - event_container.trigger_pattern = np.zeros( - shape=(n_patterns, N_PIXELS), dtype=bool - ) - - for module_idx, module_id in enumerate(self.nectarcam_service.module_ids): - offset = module_id * 7 - if bytes_per_module > 16: - field_id = 8 - # Decode trigger pattern - for pattern_id in range(n_patterns): - value = unpacked_feb[n_fields * module_idx + field_id + pattern_id] - module_pattern = [ - int(digit) for digit in reversed(bin(value)[2:].zfill(7)) - ] - event_container.trigger_pattern[ - pattern_id, offset : offset + 7 - ] = module_pattern - - -def fill_trigger_info(self, array_event): - """ - Fill the trigger information for a given event. - - Parameters: - array_event (NectarCAMEventContainer): The NectarCAMEventContainer object to - fill with trigger information. - - Returns: - None - - Raises: - None - """ - - tel_id = self.tel_id - nectarcam = array_event.nectarcam.tel[tel_id] - tib_available = nectarcam.evt.extdevices_presence & 1 - ucts_available = nectarcam.evt.extdevices_presence & 2 - # fill trigger time using UCTS timestamp - trigger = array_event.trigger - trigger_time = nectarcam.evt.ucts_timestamp - trigger_time = time_from_unix_tai_ns(trigger_time) - trigger.time = trigger_time - trigger.tels_with_trigger = [tel_id] - trigger.tel[tel_id].time = trigger.time - # decide which source to use, if both are available, - # the option decides, if not, fallback to the avilable source - # if no source available, warn and do not fill trigger info - if tib_available and ucts_available: - if self.default_trigger_type == "ucts": - trigger_bits = nectarcam.evt.ucts_trigger_type - else: - trigger_bits = nectarcam.evt.tib_masked_trigger - elif tib_available: - trigger_bits = nectarcam.evt.tib_masked_trigger - elif ucts_available: - trigger_bits = nectarcam.evt.ucts_trigger_type - else: - self.log.warning("No trigger info available.") - trigger.event_type = EventType.UNKNOWN - return - if ( - ucts_available - and nectarcam.evt.ucts_trigger_type == 42 # TODO check if it's correct - and self.default_trigger_type == "ucts" - ): - self.log.warning( - "Event with UCTS trigger_type 42 found." - " Probably means unreliable or shifted UCTS data." - ' Consider switching to TIB using `default_trigger_type="tib"`' - ) - # first bit mono trigger, second stereo. - # If *only* those two are set, we assume it's a physics event - # for all other we only check if the flag is present - if (trigger_bits & TriggerBits.PHYSICS) and not (trigger_bits & TriggerBits.OTHER): - trigger.event_type = EventType.SUBARRAY - elif trigger_bits & TriggerBits.CALIBRATION: - trigger.event_type = EventType.FLATFIELD - elif trigger_bits & TriggerBits.PEDESTAL: - trigger.event_type = EventType.SKY_PEDESTAL - elif trigger_bits & TriggerBits.SINGLE_PE: - trigger.event_type = EventType.SINGLE_PE - else: - self.log.warning( - f"Event {array_event.index.event_id} has unknown event type, trigger: " - f"{trigger_bits:08b}" - ) - trigger.event_type = EventType.UNKNOWN - - -class LightNectarCAMEventSource(NectarCAMEventSource): - """ - LightNectarCAMEventSource is a subclass of NectarCAMEventSource that provides a - generator for iterating over NectarCAM events. - - This implementation of the NectarCAMEventSource is much lighter than the one within - ctapipe_io_nectarcam, only the fields interesting for nectarchain are kept. - - Parameters - ---------- - input_url : str - The input URL of the data source. - max_events : int - The maximum number of events to process. - tel_id : int - The telescope ID. - nectarcam_service : NectarCAMService - The service container for NectarCAM. - trigger_information : bool - Flag indicating whether to fill trigger information in the event container. - obs_ids : list - The list of observation IDs. - multi_file : MultiFileReader - The multi-file reader for reading the data source. - r0_r1_calibrator : R0R1Calibrator - The calibrator for R0 to R1 conversion. - calibrate_flatfields_and_pedestals : bool - Flag indicating whether to calibrate flatfield and pedestal events. - """ - - def _generator(self): - """ - The generator function that yields NectarCAMDataContainer objects representing - each event. - - Yields - ------ - NectarCAMDataContainer : - The NectarCAMDataContainer object representing each event. - - Raises - ------ - None - - """ - # container for NectarCAM data - array_event = NectarCAMDataContainer() - array_event.meta["input_url"] = self.input_url - array_event.meta["max_events"] = self.max_events - array_event.meta["origin"] = "NectarCAM" - - # also add service container to the event section - array_event.nectarcam.tel[self.tel_id].svc = self.nectarcam_service - - # initialize general monitoring container - self.initialize_mon_container(array_event) - - # loop on events - for count, event in enumerate(self.multi_file): - array_event.count = count - array_event.index.event_id = event.event_id - array_event.index.obs_id = self.obs_ids[0] - - # fill R0/R1 data - self.fill_r0r1_container(array_event, event) - # fill specific NectarCAM event data - self.fill_nectarcam_event_container_from_zfile(array_event, event) - - if self.trigger_information: - self.fill_trigger_info(array_event) - - yield array_event diff --git a/src/nectarchain/data/container/gainContainer.py b/src/nectarchain/data/container/gain_container.py similarity index 92% rename from src/nectarchain/data/container/gainContainer.py rename to src/nectarchain/data/container/gain_container.py index d1950cf3..7cbffbf8 100644 --- a/src/nectarchain/data/container/gainContainer.py +++ b/src/nectarchain/data/container/gain_container.py @@ -1,25 +1,27 @@ import logging -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import numpy as np from ctapipe.containers import Field from .core import NectarCAMContainer +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["GainContainer", "SPEfitContainer"] class GainContainer(NectarCAMContainer): - """ - Class representing a GainContainer. + """Class representing a GainContainer. - This class is a subclass of NectarCAMContainer and provides additional fields and methods specific to gain calibration data. + This class is a subclass of NectarCAMContainer and provides additional fields and + methods specific to gain calibration data. Attributes: - is_valid (np.ndarray): Array of booleans indicating the validity of each gain value. + is_valid (np.ndarray): Array of booleans indicating the validity of each gain + value. high_gain (np.ndarray): Array of high gain values. low_gain (np.ndarray): Array of low gain values. pixels_id (np.ndarray): Array of pixel IDs. @@ -54,10 +56,10 @@ def from_hdf5(cls, path): class SPEfitContainer(GainContainer): - """ - Class representing a SPEfitContainer. + """Class representing a SPEfitContainer. - This class is a subclass of GainContainer and provides additional fields specific to single photoelectron (SPE) fit data. + This class is a subclass of GainContainer and provides additional fields specific to + single photoelectron (SPE) fit data. Attributes: likelihood (np.ndarray): Array of likelihood values. diff --git a/src/nectarchain/data/container/pedestalContainer.py b/src/nectarchain/data/container/pedestal_container.py similarity index 77% rename from src/nectarchain/data/container/pedestalContainer.py rename to src/nectarchain/data/container/pedestal_container.py index 77959b95..b0c11d72 100644 --- a/src/nectarchain/data/container/pedestalContainer.py +++ b/src/nectarchain/data/container/pedestal_container.py @@ -2,19 +2,24 @@ from enum import IntFlag, auto, unique import numpy as np -from ctapipe.containers import Field +from ctapipe.containers import Field, Map, partial -from .core import NectarCAMContainer +from .core import NectarCAMContainer, TriggerMapContainer logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers +__all__ = [ + "NectarCAMPedestalContainer", + "PedestalFlagBits", + "NectarCAMPedestalContainers", +] + @unique class PedestalFlagBits(IntFlag): - """ - Define the bits corresponding to pedestal-related flags + """Define the bits corresponding to pedestal-related flags. NEVENTS: Number of events below the acceptable minimum value MEAN_PEDESTAL: Mean pedestal outside acceptable range @@ -31,8 +36,7 @@ class PedestalFlagBits(IntFlag): class NectarCAMPedestalContainer(NectarCAMContainer): - """ - A container that holds estimated pedestals + """A container that holds estimated pedestals. Fields: nsamples (int): The number of samples in the waveforms. @@ -105,3 +109,19 @@ class NectarCAMPedestalContainer(NectarCAMContainer): The meaning of the mask bits is defined in the class\ ~nectarchain.data.container.PedestalFlagBits", ) + + +class NectarCAMPedestalContainers(TriggerMapContainer): + """Class representing a NectarCAMPedestalContainers.This class inherits from the + `TriggerMapContainer` class and is used to store slices of data + mappings of `NectarCAMPedestalContainer`. + + Attributes: + containers (Field): A field representing the slices + of data mapping of `NectarCAMPedestalContainer`. + """ + + containers = Field( + default_factory=partial(Map, NectarCAMPedestalContainer), + description="slices of data mapping of NectarCAMPedestalContainer", + ) diff --git a/src/nectarchain/data/container/tests/test_charge.py b/src/nectarchain/data/container/tests/test_charge.py index 3d146b7b..750b2e0c 100644 --- a/src/nectarchain/data/container/tests/test_charge.py +++ b/src/nectarchain/data/container/tests/test_charge.py @@ -1,7 +1,4 @@ -import glob - import numpy as np -import pytest from ctapipe.containers import EventType from ctapipe.io import HDF5TableWriter @@ -131,7 +128,7 @@ def test_from_hdf5(self, tmp_path="/tmp"): mode="w", group_name="data", ) - writer.write(table_name="ChargesContainer", containers=charge_container) + writer.write(table_name="ChargesContainer_0", containers=charge_container) writer.close() loaded_charge_container = next(ChargesContainer.from_hdf5(tmp_path)) diff --git a/src/nectarchain/data/container/tests/test_core.py b/src/nectarchain/data/container/tests/test_core_container.py similarity index 96% rename from src/nectarchain/data/container/tests/test_core.py rename to src/nectarchain/data/container/tests/test_core_container.py index 10513a04..8c12edcc 100644 --- a/src/nectarchain/data/container/tests/test_core.py +++ b/src/nectarchain/data/container/tests/test_core_container.py @@ -1,6 +1,5 @@ import numpy as np -import pytest -from ctapipe.containers import Container, EventType, Field +from ctapipe.containers import EventType, Field from ctapipe.io import HDF5TableWriter from nectarchain.data.container import ( @@ -120,9 +119,9 @@ def test_from_hdf5(self, tmp_path="/tmp"): mode="w", group_name="data", ) - writer.write(table_name="ArrayDataContainer", containers=arrayDataContainer) + writer.write(table_name="ArrayDataContainer_0", containers=arrayDataContainer) writer.close() - loaded_arrayDataContainer = next(arrayDataContainer.from_hdf5(tmp_path)) + loaded_arrayDataContainer = next(ArrayDataContainer.from_hdf5(tmp_path)) assert isinstance(loaded_arrayDataContainer, ArrayDataContainer) assert loaded_arrayDataContainer.run_number == arrayDataContainer.run_number assert ( diff --git a/src/nectarchain/data/container/tests/test_gain.py b/src/nectarchain/data/container/tests/test_gain.py index e6a10b62..e356e6d7 100644 --- a/src/nectarchain/data/container/tests/test_gain.py +++ b/src/nectarchain/data/container/tests/test_gain.py @@ -1,8 +1,4 @@ -import glob - import numpy as np -import pytest -from ctapipe.containers import EventType from ctapipe.io import HDF5TableWriter from nectarchain.data.container import GainContainer, SPEfitContainer @@ -83,7 +79,7 @@ def test_from_hdf5(self, tmp_path="/tmp"): mode="w", group_name="data", ) - writer.write(table_name="GainContainer", containers=gain_container) + writer.write(table_name="GainContainer_0", containers=gain_container) writer.close() loaded_gain_container = next(GainContainer.from_hdf5(tmp_path)) @@ -172,7 +168,7 @@ def test_from_hdf5(self, tmp_path="/tmp"): mode="w", group_name="data", ) - writer.write(table_name="SPEfitContainer", containers=SPEfit_container) + writer.write(table_name="SPEfitContainer_0", containers=SPEfit_container) writer.close() loaded_SPEfit_container = next(SPEfitContainer.from_hdf5(tmp_path)) diff --git a/src/nectarchain/data/container/tests/test_pedestal.py b/src/nectarchain/data/container/tests/test_pedestal.py index e2c5e144..07ba89b6 100644 --- a/src/nectarchain/data/container/tests/test_pedestal.py +++ b/src/nectarchain/data/container/tests/test_pedestal.py @@ -1,8 +1,8 @@ import numpy as np -import tempfile -from ctapipe.io import HDF5TableWriter + from nectarchain.data.container import NectarCAMPedestalContainer + def generate_mock_pedestal_container(): # fixed values nchannels = 2 @@ -19,7 +19,7 @@ def generate_mock_pedestal_container(): pedestal_mean_lg = np.float64(np.random.uniform(240, 260, size=(npixels, nsamples))) pedestal_std_hg = np.float64(np.random.normal(size=(npixels, nsamples))) pedestal_std_lg = np.float64(np.random.normal(size=(npixels, nsamples))) - pixel_mask = np.int8(np.random.randint(0,2,size=(nchannels,npixels))) + pixel_mask = np.int8(np.random.randint(0, 2, size=(nchannels, npixels))) # create pedestal container pedestal_container = NectarCAMPedestalContainer( @@ -32,42 +32,47 @@ def generate_mock_pedestal_container(): pedestal_mean_lg=pedestal_mean_lg, pedestal_std_hg=pedestal_std_hg, pedestal_std_lg=pedestal_std_lg, - pixel_mask = pixel_mask + pixel_mask=pixel_mask, ) pedestal_container.validate() # create dictionary that duplicates content - dict = {'nsamples': nsamples, - 'nevents': nevents, - 'pixels_id': pixels_id, - 'ucts_timestamp_min': ucts_timestamp_min, - 'ucts_timestamp_max': ucts_timestamp_max, - 'pedestal_mean_hg': pedestal_mean_hg, - 'pedestal_mean_lg': pedestal_mean_lg, - 'pedestal_std_hg': pedestal_std_hg, - 'pedestal_std_lg': pedestal_std_lg, - 'pixel_mask': pixel_mask - } + _dict = { + "nsamples": nsamples, + "nevents": nevents, + "pixels_id": pixels_id, + "ucts_timestamp_min": ucts_timestamp_min, + "ucts_timestamp_max": ucts_timestamp_max, + "pedestal_mean_hg": pedestal_mean_hg, + "pedestal_mean_lg": pedestal_mean_lg, + "pedestal_std_hg": pedestal_std_hg, + "pedestal_std_lg": pedestal_std_lg, + "pixel_mask": pixel_mask, + } # return both container and input content - return pedestal_container, dict + return pedestal_container, _dict -class TestNectarCAMPedestalContainer: +class TestNectarCAMPedestalContainer: def test_create_pedestal_container_mem(self): # create mock pedestal container pedestal_container, dict = generate_mock_pedestal_container() # check that all fields are filled correctly with input values - assert pedestal_container.nsamples == dict['nsamples'] - assert pedestal_container.nevents.tolist() == dict['nevents'].tolist() - assert pedestal_container.ucts_timestamp_min == dict['ucts_timestamp_min'] - assert pedestal_container.ucts_timestamp_max == dict['ucts_timestamp_max'] - assert np.allclose(pedestal_container.pedestal_mean_hg, dict['pedestal_mean_hg']) - assert np.allclose(pedestal_container.pedestal_mean_lg, dict['pedestal_mean_lg']) - assert np.allclose(pedestal_container.pedestal_std_hg, dict['pedestal_std_hg']) - assert np.allclose(pedestal_container.pedestal_std_lg, dict['pedestal_std_lg']) - assert np.allclose(pedestal_container.pixel_mask, dict['pixel_mask']) + assert pedestal_container.nsamples == dict["nsamples"] + assert pedestal_container.nevents.tolist() == dict["nevents"].tolist() + assert pedestal_container.ucts_timestamp_min == dict["ucts_timestamp_min"] + assert pedestal_container.ucts_timestamp_max == dict["ucts_timestamp_max"] + assert np.allclose( + pedestal_container.pedestal_mean_hg, dict["pedestal_mean_hg"] + ) + assert np.allclose( + pedestal_container.pedestal_mean_lg, dict["pedestal_mean_lg"] + ) + assert np.allclose(pedestal_container.pedestal_std_hg, dict["pedestal_std_hg"]) + assert np.allclose(pedestal_container.pedestal_std_lg, dict["pedestal_std_lg"]) + assert np.allclose(pedestal_container.pixel_mask, dict["pixel_mask"]) # FIXME # Guillaume is working on generalizing the fromhdf5 method to all containers @@ -95,10 +100,15 @@ def test_create_pedestal_container_mem(self): # assert pedestal_container.nevents.tolist() == dict['nevents'].tolist() # assert pedestal_container.ucts_timestamp_min == dict['ucts_timestamp_min'] # assert pedestal_container.ucts_timestamp_max == dict['ucts_timestamp_max'] - # assert np.allclose(pedestal_container.pedestal_mean_hg, dict['pedestal_mean_hg']) - # assert np.allclose(pedestal_container.pedestal_mean_lg, dict['pedestal_mean_lg']) - # assert np.allclose(pedestal_container.pedestal_std_hg, dict['pedestal_std_hg']) - # assert np.allclose(pedestal_container.pedestal_std_lg, dict['pedestal_std_lg']) + # assert np.allclose(pedestal_container.pedestal_mean_hg, + # dict['pedestal_mean_hg']) + # assert np.allclose(pedestal_container.pedestal_mean_lg, + # dict['pedestal_mean_lg']) + # assert np.allclose(pedestal_container.pedestal_std_hg, + # dict['pedestal_std_hg']) + # assert np.allclose(pedestal_container.pedestal_std_lg, + # dict['pedestal_std_lg']) + if __name__ == "__main__": pass diff --git a/src/nectarchain/data/container/tests/test_waveform.py b/src/nectarchain/data/container/tests/test_waveform.py index 54851e36..01d62a9d 100644 --- a/src/nectarchain/data/container/tests/test_waveform.py +++ b/src/nectarchain/data/container/tests/test_waveform.py @@ -1,7 +1,4 @@ -import glob - import numpy as np -import pytest from ctapipe.containers import EventType from ctapipe.io import HDF5TableWriter @@ -112,7 +109,7 @@ def test_from_hdf5(self, tmp_path="/tmp"): mode="w", group_name="data", ) - writer.write(table_name="WaveformsContainer", containers=waveform_container) + writer.write(table_name="WaveformsContainer_0", containers=waveform_container) writer.close() loaded_waveform_container = next(WaveformsContainer.from_hdf5(tmp_path)) diff --git a/src/nectarchain/data/container/waveformsContainer.py b/src/nectarchain/data/container/waveforms_container.py similarity index 74% rename from src/nectarchain/data/container/waveformsContainer.py rename to src/nectarchain/data/container/waveforms_container.py index 05eb0fd7..f430b951 100644 --- a/src/nectarchain/data/container/waveformsContainer.py +++ b/src/nectarchain/data/container/waveforms_container.py @@ -1,18 +1,19 @@ import logging -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import numpy as np from ctapipe.containers import Field, Map, partial from .core import ArrayDataContainer, TriggerMapContainer +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + +__all__ = ["WaveformsContainer", "WaveformsContainers"] + class WaveformsContainer(ArrayDataContainer): - """ - A container that holds information about waveforms from a specific run. + """A container that holds information about waveforms from a specific run. Fields: nsamples (int): The number of samples in the waveforms. @@ -25,7 +26,8 @@ class WaveformsContainer(ArrayDataContainer): type=np.uint8, description="number of samples in the waveforms", ) - # subarray = Field(type=SubarrayDescription, description="The subarray description") + # subarray = Field(type=SubarrayDescription, + # description="The subarray description") wfs_hg = Field( type=np.ndarray, dtype=np.uint16, ndim=3, description="high gain waveforms" ) @@ -35,13 +37,14 @@ class WaveformsContainer(ArrayDataContainer): class WaveformsContainers(TriggerMapContainer): - """ - Class representing a container for waveforms from specific runs. + """Class representing a container for waveforms from specific runs. - This class inherits from the `TriggerMapContainer` class and is used to store trigger or slices of data mappings of `WaveformsContainer` instances. + This class inherits from the `TriggerMapContainer` class and is used to store + trigger or slices of data mappings of `WaveformsContainer` instances. Attributes: - containers (Field): A field representing the trigger or slices of data mapping of `WaveformsContainer` instances. + containers (Field): A field representing the trigger or slices of data mapping + of `WaveformsContainer` instances. """ containers = Field( diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index 0782192f..6b970437 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -24,17 +24,15 @@ DIRAC.initialize() except ImportError: log.warning("DIRAC probably not installed") - pass except Exception as e: log.warning(f"DIRAC could not be properly initialized: {e}") - pass class DataManagement: @staticmethod def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: - """method to find in NECTARCAMDATA the list of ``*.fits.fz`` files associated to - run_number + """Method to find in NECTARCAMDATA the list of ``*.fits.fz`` files + associated to run_number. Parameters ---------- @@ -45,7 +43,6 @@ def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: ------- (PosixPath,list): the path list of ``*.fits.fz`` files - """ basepath = f"{os.environ['NECTARCAMDATA']}/runs/" list = glob.glob( @@ -82,7 +79,7 @@ def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: @staticmethod def getRunFromDIRAC(lfns: list): - """Method to get run files from the EGI grid from input lfns + """Method to get run files from the EGI grid from input lfns. Parameters ---------- @@ -115,8 +112,7 @@ def get_GRID_location( username=None, password=None, ): - """ - Method to get run location on GRID from Elog (work in progress!) + """Method to get run location on GRID from Elog (work in progress!) Parameters ---------- @@ -139,7 +135,6 @@ def get_GRID_location( Returns ------- __get_GRID_location_ELog or __get_GRID_location_DIRAC - """ if fromElog: return __class__.__get_GRID_location_ELog( @@ -263,11 +258,13 @@ def __get_GRID_location_ELog( else: return url_data + @staticmethod def find_waveforms(run_number, max_events=None): return __class__.__find_computed_data( run_number=run_number, max_events=max_events, data_type="waveforms" ) + @staticmethod def find_charges( run_number, method="FullWaveformSum", str_extractor_kwargs="", max_events=None ): @@ -278,14 +275,69 @@ def find_charges( data_type="charges", ) - def find_SPE_HHV(run_number, method="FullWaveformSum", str_extractor_kwargs=""): + @staticmethod + def find_photostat( + FF_run_number, + ped_run_number, + FF_method="FullWaveformSum", + ped_method="FullWaveformSum", + str_extractor_kwargs="", + ): + full_file = glob.glob( + pathlib.Path( + f"{os.environ.get('NECTARCAMDATA','/tmp')}/PhotoStat/" + f"PhotoStatisticNectarCAM_FFrun{FF_run_number}_{FF_method}" + f"_{str_extractor_kwargs}_Pedrun{ped_run_number}_{ped_method}.h5" + ).__str__() + ) + log.debug("for now it does not check if there are files with max events") + if len(full_file) != 1: + raise Exception(f"the files is {full_file}") + return full_file + + @staticmethod + def find_SPE_combined( + run_number, method="FullWaveformSum", str_extractor_kwargs="" + ): + return __class__.find_SPE_HHV( + run_number=run_number, + method=method, + str_extractor_kwargs=str_extractor_kwargs, + keyword="FlatFieldCombined", + ) + + @staticmethod + def find_SPE_nominal( + run_number, method="FullWaveformSum", str_extractor_kwargs="", free_pp_n=False + ): + return __class__.find_SPE_HHV( + run_number=run_number, + method=method, + str_extractor_kwargs=str_extractor_kwargs, + free_pp_n=free_pp_n, + keyword="FlatFieldSPENominal", + ) + + @staticmethod + def find_SPE_HHV( + run_number, + method="FullWaveformSum", + str_extractor_kwargs="", + free_pp_n=False, + **kwargs, + ): + keyword = kwargs.get("keyword", "FlatFieldSPEHHV") + std_key = "" if free_pp_n else "Std" full_file = glob.glob( pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/SPEfit/" - f"FlatFieldSPEHHVStdNectarCAM_run{run_number}_{method}" + f"{keyword}{std_key}NectarCAM_run{run_number}*_{method}" f"_{str_extractor_kwargs}.h5" ).__str__() ) + # need to improve the files search !! + # -> unstable behavior with SPE results computed + # with maxevents not to None if len(full_file) != 1: all_files = glob.glob( pathlib.Path( @@ -308,6 +360,7 @@ def find_SPE_HHV(run_number, method="FullWaveformSum", str_extractor_kwargs=""): else: return full_file + @staticmethod def __find_computed_data( run_number, max_events=None, ext=".h5", data_type="waveforms" ): diff --git a/src/nectarchain/data/tests/test_management.py b/src/nectarchain/data/tests/test_management.py index 5871ed8e..f77393f6 100644 --- a/src/nectarchain/data/tests/test_management.py +++ b/src/nectarchain/data/tests/test_management.py @@ -1 +1 @@ -import pytest +# To DO diff --git a/src/nectarchain/display/__init__.py b/src/nectarchain/display/__init__.py index 4941e7d3..af897f07 100644 --- a/src/nectarchain/display/__init__.py +++ b/src/nectarchain/display/__init__.py @@ -1 +1,6 @@ -from .display import * +""" This module contains the display class for the container. +""" + +from .display import ContainerDisplay + +__all__ = ["ContainerDisplay"] diff --git a/src/nectarchain/display/display.py b/src/nectarchain/display/display.py index 9df8a759..11c85737 100644 --- a/src/nectarchain/display/display.py +++ b/src/nectarchain/display/display.py @@ -1,6 +1,7 @@ import logging from abc import ABC +import numpy as np from ctapipe.visualization import CameraDisplay from matplotlib import pyplot as plt @@ -10,7 +11,7 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -import numpy as np +__all__ = ["ContainerDisplay"] class ContainerDisplay(ABC): @@ -29,14 +30,16 @@ def display(container: ArrayDataContainer, evt, geometry, cmap="gnuplot2"): image = container.charges_hg pixels_id = container.pixels_id elif isinstance(container, WaveformsContainer): - image = container.wfs_hg.sum(axis=2) + image = container.wfs_hg.mean(axis=2) pixels_id = container.pixels_id else: log.error( - "container can't be displayed, must be a ChargesContainer or a WaveformsContainer" + "container can't be displayed, must be a ChargesContainer or a\ + WaveformsContainer" ) raise Exception( - "container can't be displayed, must be a ChargesContainer or a WaveformsContainer" + "container can't be displayed, must be a ChargesContainer or a\ + WaveformsContainer" ) highlighten_pixels = np.array([], dtype=int) @@ -55,8 +58,8 @@ def display(container: ArrayDataContainer, evt, geometry, cmap="gnuplot2"): disp = CameraDisplay(geometry=geometry, image=image[evt], cmap=cmap) disp.highlight_pixels(highlighten_pixels, color="r", linewidth=2) - disp.add_colorbar() - return disp + disp.add_colorbar(label="ADC") + return {"disp": disp, "highlighten_pixels": highlighten_pixels} @staticmethod def plot_waveform(waveformsContainer: WaveformsContainer, evt, **kwargs): diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index 80f7517b..6018536f 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -3,11 +3,11 @@ from ctapipe.coordinates import EngineeringCameraFrame from ctapipe.image.extractor import FixedWindowSum # noqa: F401 from ctapipe.image.extractor import FullWaveformSum # noqa: F401 +from ctapipe.image.extractor import GlobalPeakWindowSum # noqa: F401 from ctapipe.image.extractor import LocalPeakWindowSum # noqa: F401 from ctapipe.image.extractor import NeighborPeakWindowSum # noqa: F401 from ctapipe.image.extractor import SlidingWindowMaxSum # noqa: F401 from ctapipe.image.extractor import TwoPassWindowSum # noqa: F401 -from ctapipe.image.extractor import GlobalPeakWindowSum from ctapipe.visualization import CameraDisplay from ctapipe_io_nectarcam import constants from matplotlib import pyplot as plt diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/__init__.py index d690d0f6..dcca88f6 100644 --- a/src/nectarchain/makers/__init__.py +++ b/src/nectarchain/makers/__init__.py @@ -1,4 +1,17 @@ -# from .chargesMakers import * -from .chargesMakers import * -from .core import * -from .waveformsMakers import * +""" Description: This file is used to import all the classes from the different files in +the makers folder. +""" + +from .charges_makers import ChargesNectarCAMCalibrationTool +from .core import ( + DelimiterLoopNectarCAMCalibrationTool, + EventsLoopNectarCAMCalibrationTool, +) +from .waveforms_makers import WaveformsNectarCAMCalibrationTool + +__all__ = [ + "ChargesNectarCAMCalibrationTool", + "DelimiterLoopNectarCAMCalibrationTool", + "EventsLoopNectarCAMCalibrationTool", + "WaveformsNectarCAMCalibrationTool", +] diff --git a/src/nectarchain/makers/calibration/__init__.py b/src/nectarchain/makers/calibration/__init__.py index 0ded83f5..abc33bd6 100644 --- a/src/nectarchain/makers/calibration/__init__.py +++ b/src/nectarchain/makers/calibration/__init__.py @@ -1,3 +1,21 @@ -from .flatfieldMakers import * -from .gain import * -from .pedestalMakers import * +from .flatfield_makers import FlatfieldNectarCAMCalibrationTool +from .gain import ( + FlatFieldSPECombinedStdNectarCAMCalibrationTool, + FlatFieldSPEHHVNectarCAMCalibrationTool, + FlatFieldSPEHHVStdNectarCAMCalibrationTool, + FlatFieldSPENominalNectarCAMCalibrationTool, + FlatFieldSPENominalStdNectarCAMCalibrationTool, + PhotoStatisticNectarCAMCalibrationTool, +) +from .pedestal_makers import PedestalNectarCAMCalibrationTool + +__all__ = [ + "FlatfieldNectarCAMCalibrationTool", + "FlatFieldSPECombinedStdNectarCAMCalibrationTool", + "FlatFieldSPEHHVNectarCAMCalibrationTool", + "FlatFieldSPEHHVStdNectarCAMCalibrationTool", + "FlatFieldSPENominalNectarCAMCalibrationTool", + "FlatFieldSPENominalStdNectarCAMCalibrationTool", + "PedestalNectarCAMCalibrationTool", + "PhotoStatisticNectarCAMCalibrationTool", +] diff --git a/src/nectarchain/makers/calibration/core.py b/src/nectarchain/makers/calibration/core.py index c6c46e5f..05c8d2ca 100644 --- a/src/nectarchain/makers/calibration/core.py +++ b/src/nectarchain/makers/calibration/core.py @@ -1,14 +1,15 @@ import logging +from ctapipe.core.traits import List + +from ..core import EventsLoopNectarCAMCalibrationTool + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from ctapipe.core.traits import List -from ..core import EventsLoopNectarCAMCalibrationTool - -__all__ = [""] +__all__ = None class NectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): @@ -21,62 +22,3 @@ class NectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): help="the list of pixel id to apply the components", allow_none=True, ).tag(config=True) - - -''' - def setup(self) -> None: - super().setup() - self.__results = QTable() - self.__results.add_column( - Column( - self.pixels_id, - self.pixels_id.name, - unit=u.dimensionless_unscaled, - ) - ) - self.__results.meta[__class__.NP_PIXELS] = self.npixels - self.__results.meta[ - "comments" - ] = f'Produced with NectarChain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' - - def finish(self, path, **kwargs): - """ - Saves the results to a file in the specified path. - - Args: - path (str): The path to save the results. - **kwargs: Additional keyword arguments. - - Keyword Args: - overwrite (bool): Whether to overwrite an existing file. Defaults to False. - """ - path = Path(path) - path.mkdir(parents=True, exist_ok=True) - log.info(f"data saved in {path}") - self._results.write( - f"{path}/results_{self._reduced_name}.ecsv", - format="ascii.ecsv", - overwrite=kwargs.get("overwrite", False), - ) - - - @property - def _results(self): - """ - Get the result table. - - Returns: - QTable: The result table. - """ - return self.__results - - @property - def results(self): - """ - Get a copy of the result table. - - Returns: - QTable: A copy of the result table. - """ - return copy(self.__results) -''' diff --git a/src/nectarchain/makers/calibration/flatfieldMakers.py b/src/nectarchain/makers/calibration/flatfield_makers.py similarity index 87% rename from src/nectarchain/makers/calibration/flatfieldMakers.py rename to src/nectarchain/makers/calibration/flatfield_makers.py index 01da9958..046ce2de 100644 --- a/src/nectarchain/makers/calibration/flatfieldMakers.py +++ b/src/nectarchain/makers/calibration/flatfield_makers.py @@ -1,10 +1,11 @@ import logging +from .core import NectarCAMCalibrationTool + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from .core import NectarCAMCalibrationTool __all__ = ["FlatfieldNectarCAMCalibrationTool"] @@ -12,5 +13,6 @@ class FlatfieldNectarCAMCalibrationTool(NectarCAMCalibrationTool): def start(self): raise NotImplementedError( - "The computation of the flatfield calibration is not yet implemented, feel free to contribute !:)" + "The computation of the flatfield calibration is not yet implemented, \ + feel free to contribute !:)" ) diff --git a/src/nectarchain/makers/calibration/gain/__init__.py b/src/nectarchain/makers/calibration/gain/__init__.py index 518e6584..eaa6cc9e 100644 --- a/src/nectarchain/makers/calibration/gain/__init__.py +++ b/src/nectarchain/makers/calibration/gain/__init__.py @@ -1,5 +1,19 @@ -from .FlatFieldSPEMakers import * -from .photostat_makers import * +from .flatfield_spe_makers import ( + FlatFieldSPECombinedStdNectarCAMCalibrationTool, + FlatFieldSPEHHVNectarCAMCalibrationTool, + FlatFieldSPEHHVStdNectarCAMCalibrationTool, + FlatFieldSPENominalNectarCAMCalibrationTool, + FlatFieldSPENominalStdNectarCAMCalibrationTool, +) +from .photostat_makers import PhotoStatisticNectarCAMCalibrationTool -# from .WhiteTargetSPEMakers import * -# from .PhotoStatisticMakers import * +# from .white_target_spe_makers import * + +__all__ = [ + "FlatFieldSPENominalNectarCAMCalibrationTool", + "FlatFieldSPENominalStdNectarCAMCalibrationTool", + "FlatFieldSPEHHVNectarCAMCalibrationTool", + "FlatFieldSPEHHVStdNectarCAMCalibrationTool", + "FlatFieldSPECombinedStdNectarCAMCalibrationTool", + "PhotoStatisticNectarCAMCalibrationTool", +] diff --git a/src/nectarchain/makers/calibration/gain/core.py b/src/nectarchain/makers/calibration/gain/core.py index 095fc32c..02723e57 100644 --- a/src/nectarchain/makers/calibration/gain/core.py +++ b/src/nectarchain/makers/calibration/gain/core.py @@ -1,12 +1,13 @@ import logging +from ctapipe.core.traits import Bool + +from ..core import NectarCAMCalibrationTool + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from ctapipe.core.traits import Bool - -from ..core import NectarCAMCalibrationTool __all__ = ["GainNectarCAMCalibrationTool"] diff --git a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/flatfield_spe_makers.py similarity index 71% rename from src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py rename to src/nectarchain/makers/calibration/gain/flatfield_spe_makers.py index 8f6948e9..07854746 100644 --- a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/flatfield_spe_makers.py @@ -1,9 +1,4 @@ import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import os import pathlib @@ -18,6 +13,11 @@ from ...extractor.utils import CtapipeExtractor from .core import GainNectarCAMCalibrationTool +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = [ "FlatFieldSPENominalNectarCAMCalibrationTool", "FlatFieldSPENominalStdNectarCAMCalibrationTool", @@ -57,7 +57,8 @@ def __init__(self, *args, **kwargs): ) if len(files) == 1: log.warning( - "You asked events_per_slice but you don't want to reload events and a charges file is on disk, then events_per_slice is set to None" + "You asked events_per_slice but you don't want to reload events and\ + a charges file is on disk, then events_per_slice is set to None" ) self.events_per_slice = None @@ -70,9 +71,15 @@ def _init_output_path(self): else: ext = f"_sliced{self.events_per_slice}.h5" if self.max_events is None: - filename = f"{self.name}_run{self.run_number}_{self.method}_{str_extractor_kwargs}{ext}" + filename = ( + f"{self.name}_run{self.run_number}_" + f"{self.method}_{str_extractor_kwargs}{ext}" + ) else: - filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}_{self.method}_{str_extractor_kwargs}{ext}" + filename = ( + f"{self.name}_run{self.run_number}_maxevents" + f"{self.max_events}_{self.method}_{str_extractor_kwargs}{ext}" + ) self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/SPEfit/{filename}" @@ -98,7 +105,10 @@ def start( if self.reload_events or len(files) != 1: if len(files) != 1: self.log.info( - f"{len(files)} computed charges files found with max_events > {self.max_events} for run {self.run_number} with extraction method {self.method} and {str_extractor_kwargs},\n reload charges from event loop" + f"{len(files)} computed charges files found with max_events >" + f"{self.max_events} for run {self.run_number} with extraction" + f"method {self.method} and {str_extractor_kwargs},\n reload" + f"charges from event loop" ) super().start( n_events=n_events, @@ -108,29 +118,43 @@ def start( ) else: self.log.info(f"reading computed charge from files {files[0]}") - chargesContainers = ChargesContainer.from_hdf5(files[0]) + chargesContainers = ChargesContainers.from_hdf5(files[0]) if isinstance(chargesContainers, ChargesContainer): self.components[0]._chargesContainers = chargesContainers - elif isinstance(chargesContainers, ChargesContainers): - self.log.debug("merging along TriggerType") - self.components[0]._chargesContainers = merge_map_ArrayDataContainer( - chargesContainers - ) else: - self.log.debug("merging along slices") - chargesContaienrs_merdes_along_slices = ( - ArrayDataComponent.merge_along_slices( - containers_generator=chargesContainers + n_slices = 0 + try: + while True: + next(chargesContainers) + n_slices += 1 + except StopIteration: + pass + chargesContainers = ChargesContainers.from_hdf5(files[0]) + if n_slices == 1: + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._chargesContainers = merge_map_ArrayDataContainer( + chargesContainers + ) + else: + self.log.info("merging along slices") + chargesContaienrs_merdes_along_slices = ( + ArrayDataComponent.merge_along_slices( + containers_generator=chargesContainers + ) + ) + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._chargesContainers = merge_map_ArrayDataContainer( + chargesContaienrs_merdes_along_slices ) - ) - self.log.debug("merging along TriggerType") - self.components[0]._chargesContainers = merge_map_ArrayDataContainer( - chargesContaienrs_merdes_along_slices - ) def _write_container(self, container: Container, index_component: int = 0) -> None: # if isinstance(container,SPEfitContainer) : - # self.writer.write(table_name = f"{self.method}_{CtapipeExtractor.get_extractor_kwargs_str(self.extractor_kwargs)}", + # self.writer.write(table_name = f"{self.method}_ + # {CtapipeExtractor.get_extractor_kwargs_str(self.extractor_kwargs)}", # containers = container, # ) # else : @@ -173,7 +197,7 @@ class FlatFieldSPENominalStdNectarCAMCalibrationTool( class FlatFieldSPECombinedStdNectarCAMCalibrationTool( FlatFieldSPENominalNectarCAMCalibrationTool ): - name = "FlatFieldCombinedStddNectarCAM" + name = "FlatFieldCombinedStdNectarCAM" componentsList = ComponentNameList( NectarCAMComponent, default_value=["FlatFieldCombinedSPEStdNectarCAMComponent"], @@ -188,9 +212,15 @@ def _init_output_path(self): self.extractor_kwargs ) if self.max_events is None: - filename = f"{self.name}_run{self.run_number}_HHV{HHVrun}_{self.method}_{str_extractor_kwargs}.h5" + filename = ( + f"{self.name}_run{self.run_number}_HHV{HHVrun}_{self.method}_" + f"{str_extractor_kwargs}.h5" + ) else: - filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}_HHV{HHVrun}_{self.method}_{str_extractor_kwargs}.h5" + filename = ( + f"{self.name}_run{self.run_number}_maxevents{self.max_events}_" + f"HHV{HHVrun}_{self.method}_{str_extractor_kwargs}.h5" + ) self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/SPEfit/{filename}" diff --git a/src/nectarchain/makers/calibration/gain/photostat_makers.py b/src/nectarchain/makers/calibration/gain/photostat_makers.py index 302fac51..83d35b3b 100644 --- a/src/nectarchain/makers/calibration/gain/photostat_makers.py +++ b/src/nectarchain/makers/calibration/gain/photostat_makers.py @@ -1,28 +1,29 @@ import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import os import pathlib import numpy as np -from ctapipe.containers import Container, EventType +from ctapipe.containers import Container from ctapipe.core.traits import ComponentNameList, Integer, Path from ....data.container import ChargesContainer, ChargesContainers -from ....data.container.core import NectarCAMContainer, merge_map_ArrayDataContainer +from ....data.container.core import merge_map_ArrayDataContainer from ....data.management import DataManagement from ...component import ArrayDataComponent, NectarCAMComponent from ...extractor.utils import CtapipeExtractor from .core import GainNectarCAMCalibrationTool +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["PhotoStatisticNectarCAMCalibrationTool"] class PhotoStatisticNectarCAMCalibrationTool(GainNectarCAMCalibrationTool): - ###TO DO : IMPLEMENT a MOTHER PHOTOSTAT CLASS WITH ONLY 1 RUN WITH FF AND PEDESTAL INTERLEAVED. + # TO DO : IMPLEMENT a MOTHER PHOTOSTAT CLASS WITH ONLY 1 RUN WITH FF AND PEDESTAL + # INTERLEAVED. name = "PhotoStatisticNectarCAM" componentsList = ComponentNameList( @@ -38,14 +39,16 @@ class PhotoStatisticNectarCAMCalibrationTool(GainNectarCAMCalibrationTool): ).tag(config=True) run_file = Path( - help="desactivated for PhotoStatistic maker with FF and pedestal runs separated", + help="desactivated for PhotoStatistic maker\ + with FF and pedestal runs separated", default_value=None, allow_none=True, read_only=True, ).tag(config=False) events_per_slice = Integer( - help="desactivated for PhotoStatistic maker with FF and pedestal runs separated", + help="desactivated for PhotoStatistic maker\ + with FF and pedestal runs separated", default_value=None, allow_none=True, read_only=True, @@ -59,9 +62,17 @@ def _init_output_path(self): self.extractor_kwargs ) if self.max_events is None: - filename = f"{self.name}_FFrun{self.run_number}_{self.method}_{str_extractor_kwargs}_Pedrun{self.Ped_run_number}_FullWaveformSum.h5" + filename = ( + f"{self.name}_FFrun{self.run_number}_{self.method}" + f"_{str_extractor_kwargs}_Pedrun{self.Ped_run_number}" + f"_FullWaveformSum.h5" + ) else: - filename = f"{self.name}_FFrun{self.run_number}_{self.method}_{str_extractor_kwargs}_Pedrun{self.Ped_run_number}_FullWaveformSum_maxevents{self.max_events}.h5" + filename = ( + f"{self.name}_FFrun{self.run_number}_{self.method}" + f"_{str_extractor_kwargs}_Pedrun{self.Ped_run_number}_" + f"FullWaveformSum_maxevents{self.max_events}.h5" + ) self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/PhotoStat/{filename}" ) @@ -100,10 +111,19 @@ def start( if self.reload_events or len(FF_files) != 1 or len(Ped_files) != 1: if len(FF_files) != 1 or len(Ped_files) != 1: self.log.info( - f"{len(FF_files)} computed charges FF files found with max_events > {self.max_events} for run {self.run_number} with extraction method {self.method} and {str_extractor_kwargs},\n reload charges from event loop" + f"{len(FF_files)} computed charges FF files found" + f"with max_events >" + f"{self.max_events} for run {self.run_number}" + f"with extraction method" + f"{self.method} and {str_extractor_kwargs},\n reload charges" + f"from event loop" ) self.log.info( - f"{len(Ped_files)} computed charges FF files found with max_events > {self.max_events} for run {self.Ped_run_number} with extraction method FullWaveformSum,\n reload charges from event loop" + f"{len(Ped_files)} computed charges FF files found" + f"with max_events >" + f"{self.max_events} for run {self.Ped_run_number}" + f"with extraction" + f"method FullWaveformSum,\n reload charges from event loop" ) super().start( @@ -115,50 +135,77 @@ def start( ) else: self.log.info(f"reading computed charge from FF file {FF_files[0]}") - chargesContainers = ChargesContainer.from_hdf5(FF_files[0]) + chargesContainers = ChargesContainers.from_hdf5(FF_files[0]) if isinstance(chargesContainers, ChargesContainer): self.components[0]._FF_chargesContainers = chargesContainers - elif isinstance(chargesContainers, ChargesContainers): - self.log.debug("merging along TriggerType") - self.components[0]._FF_chargesContainers = merge_map_ArrayDataContainer( - chargesContainers - ) else: - self.log.debug("merging along slices") - chargesContaienrs_merdes_along_slices = ( - ArrayDataComponent.merge_along_slices(chargesContainers) - ) - self.log.debug("merging along TriggerType") - self.components[0]._FF_chargesContainers = merge_map_ArrayDataContainer( - chargesContaienrs_merdes_along_slices - ) + n_slices = 0 + try: + while True: + next(chargesContainers) + n_slices += 1 + except StopIteration: + pass + chargesContainers = ChargesContainers.from_hdf5(FF_files[0]) + if n_slices == 1: + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._FF_chargesContainers = merge_map_ArrayDataContainer( + chargesContainers + ) + else: + self.log.info("merging along slices") + chargesContaienrs_merdes_along_slices = ( + ArrayDataComponent.merge_along_slices( + containers_generator=chargesContainers + ) + ) + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._FF_chargesContainers = merge_map_ArrayDataContainer( + chargesContaienrs_merdes_along_slices + ) self.log.info(f"reading computed charge from Ped file {Ped_files[0]}") - chargesContainers = ChargesContainer.from_hdf5(Ped_files[0]) + chargesContainers = ChargesContainers.from_hdf5(Ped_files[0]) if isinstance(chargesContainers, ChargesContainer): self.components[0]._Ped_chargesContainers = chargesContainers - elif isinstance(chargesContainers, ChargesContainers): - self.log.debug("merging along TriggerType") - self.components[ - 0 - ]._Ped_chargesContainers = merge_map_ArrayDataContainer( - chargesContainers - ) else: - self.log.debug("merging along slices") - chargesContaienrs_merdes_along_slices = ( - ArrayDataComponent.merge_along_slices(chargesContainers) - ) - self.log.debug("merging along TriggerType") - self.components[ - 0 - ]._Ped_chargesContainers = merge_map_ArrayDataContainer( - chargesContaienrs_merdes_along_slices - ) + n_slices = 0 + try: + while True: + next(chargesContainers) + n_slices += 1 + except StopIteration: + pass + chargesContainers = ChargesContainers.from_hdf5(Ped_files[0]) + if n_slices == 1: + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._Ped_chargesContainers = merge_map_ArrayDataContainer( + chargesContainers + ) + else: + self.log.info("merging along slices") + chargesContaienrs_merdes_along_slices = ( + ArrayDataComponent.merge_along_slices( + containers_generator=chargesContainers + ) + ) + self.log.info("merging along TriggerType") + self.components[ + 0 + ]._Ped_chargesContainers = merge_map_ArrayDataContainer( + chargesContaienrs_merdes_along_slices + ) def _write_container(self, container: Container, index_component: int = 0) -> None: # if isinstance(container,SPEfitContainer) : - # self.writer.write(table_name = f"{self.method}_{CtapipeExtractor.get_extractor_kwargs_str(self.extractor_kwargs)}", + # self.writer.write(table_name = f"{self.method}_{CtapipeExtractor.get_extrac + # tor_kwargs_str(self.extractor_kwargs)}", # containers = container, # ) # else : diff --git a/src/nectarchain/makers/calibration/gain/WhiteTargetSPEMakers.py b/src/nectarchain/makers/calibration/gain/white_target_spe_makers.py similarity index 100% rename from src/nectarchain/makers/calibration/gain/WhiteTargetSPEMakers.py rename to src/nectarchain/makers/calibration/gain/white_target_spe_makers.py diff --git a/src/nectarchain/makers/calibration/pedestalMakers.py b/src/nectarchain/makers/calibration/pedestalMakers.py deleted file mode 100644 index a3f2c1ab..00000000 --- a/src/nectarchain/makers/calibration/pedestalMakers.py +++ /dev/null @@ -1,179 +0,0 @@ -import logging -import os - -import numpy as np - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - -import pathlib -import tables - -from ctapipe.core.traits import ComponentNameList - -from ctapipe_io_nectarcam.constants import N_GAINS, HIGH_GAIN, LOW_GAIN - -from .core import NectarCAMCalibrationTool -from ..component import NectarCAMComponent -from ...data.container import NectarCAMPedestalContainer - -__all__ = ["PedestalNectarCAMCalibrationTool"] - - -class PedestalNectarCAMCalibrationTool(NectarCAMCalibrationTool): - name = "PedestalNectarCAMCalibrationTool" - - componentsList = ComponentNameList(NectarCAMComponent, - default_value=["PedestalEstimationComponent"], - help="List of Component names to be applied, the order will be respected", ).tag( - config=True) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def _init_output_path(self): - """ - Initialize output path - """ - - if self.events_per_slice is None: - ext = ".h5" - else: - ext = f"_sliced{self.events_per_slice}.h5" - if self.max_events is None: - filename = f"{self.name}_run{self.run_number}{ext}" - else: - filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}{ext}" - - self.output_path = pathlib.Path( - f"{os.environ.get('NECTARCAMDATA', '/tmp')}/PedestalEstimation/{filename}") - - def _combine_results(self): - """ - Method that combines sliced results to reduce memory load - Can only be called after the file with the sliced results has been saved to disk - """ - - # re-open results - # TODO: update to use nectarchain generators when available - with tables.open_file(self.output_path) as h5file: - # Loop over sliced results to fill the combined results - self.log.info("Combine sliced results") - for i, result in enumerate(h5file.root.__members__): - if result == 'data_combined': - log.error('Trying to combine results that already contain combined data') - table = h5file.root[result][NectarCAMPedestalContainer.__name__][0] - if i == 0: - # initialize fields for the combined results based on first slice - nsamples = table['nsamples'] - nevents = np.zeros(len(table['nevents'])) - pixels_id = table['pixels_id'] - ucts_timestamp_min = table['ucts_timestamp_min'] - ucts_timestamp_max = table['ucts_timestamp_max'] - pedestal_mean_hg = np.zeros(np.shape(table['pedestal_mean_hg'])) - pedestal_mean_lg = np.zeros(np.shape(table['pedestal_mean_lg'])) - pedestal_std_hg = np.zeros(np.shape(table['pedestal_std_hg'])) - pedestal_std_lg = np.zeros(np.shape(table['pedestal_std_lg'])) - else: - # otherwise consider the overall time interval - ucts_timestamp_min = np.minimum(ucts_timestamp_min, - table['ucts_timestamp_min']) - ucts_timestamp_max = np.maximum(ucts_timestamp_max, - table['ucts_timestamp_max']) - # for all slices - # derive from pixel mask a mask that sets usable pixels - # accept only pixels for which no flags were raised - usable_pixels = table['pixel_mask'] == 0 - # use a pixel only if it has no flag on either channel - usable_pixels = np.logical_and(usable_pixels[0], - usable_pixels[1]) - - # cumulated number of events - nevents += table['nevents'] * usable_pixels - - # add mean, std sum elements - pedestal_mean_hg += table['pedestal_mean_hg'] \ - * table['nevents'][:, np.newaxis] \ - * usable_pixels[:, np.newaxis] - pedestal_mean_lg += table['pedestal_mean_lg'] \ - * table['nevents'][:, np.newaxis] \ - * usable_pixels[:, np.newaxis] - pedestal_std_hg += table['pedestal_std_hg'] ** 2 \ - * table['nevents'][:, np.newaxis] \ - * usable_pixels[:, np.newaxis] - pedestal_std_lg += table['pedestal_std_lg'] ** 2 \ - * table['nevents'][:, np.newaxis] \ - * usable_pixels[:, np.newaxis] - - # calculate final values of mean and std - pedestal_mean_hg /= nevents[:, np.newaxis] - pedestal_mean_lg /= nevents[:, np.newaxis] - pedestal_std_hg /= nevents[:, np.newaxis] - pedestal_std_hg = np.sqrt(pedestal_std_hg) - pedestal_std_lg /= nevents[:, np.newaxis] - pedestal_std_lg = np.sqrt(pedestal_std_lg) - - # flag bad pixels in overall results based on same criteria as for individual slides - # reconstitute dictionary with cumulated results consistently with - # PedestalComponent - ped_stats = {} - array_shape = np.append([N_GAINS], np.shape(pedestal_mean_hg)) - for statistic in ['mean', 'std']: - ped_stat = np.zeros(array_shape) - if statistic == 'mean': - ped_stat[HIGH_GAIN] = pedestal_mean_hg - ped_stat[LOW_GAIN] = pedestal_mean_lg - elif statistic == 'std': - ped_stat[HIGH_GAIN] = pedestal_std_hg - ped_stat[LOW_GAIN] = pedestal_std_lg - # Store the result in the dictionary - ped_stats[statistic] = ped_stat - # use flagging method from PedestalComponent - pixel_mask = self.components[0].flag_bad_pixels(ped_stats, nevents) - - output = NectarCAMPedestalContainer( - nsamples=nsamples, - nevents=nevents, - pixels_id=pixels_id, - ucts_timestamp_min=ucts_timestamp_min, - ucts_timestamp_max=ucts_timestamp_max, - pedestal_mean_hg=pedestal_mean_hg, - pedestal_mean_lg=pedestal_mean_lg, - pedestal_std_hg=pedestal_std_hg, - pedestal_std_lg=pedestal_std_lg, - pixel_mask=pixel_mask, - ) - - return output - - def finish(self, return_output_component=False, *args, **kwargs): - """ - Redefines finish method to combine sliced results - """ - - self.log.info("finishing Tool") - - # finish components - output = self._finish_components(*args, **kwargs) - - # close writer - self.writer.close() - - # Check if there are slices - if self.events_per_slice is None: - # If not nothing to do - pass - else: - # combine results - output = self._combine_results() - # add combined results to output - # re-initialise writer to store combined results - self._init_writer(sliced=True, group_name='data_combined') - # add combined results to writer - self._write_container(output) - self.writer.close() - - self.log.info("Shutting down.") - if return_output_component: - return output diff --git a/src/nectarchain/makers/calibration/pedestal_makers.py b/src/nectarchain/makers/calibration/pedestal_makers.py new file mode 100644 index 00000000..149111fe --- /dev/null +++ b/src/nectarchain/makers/calibration/pedestal_makers.py @@ -0,0 +1,192 @@ +import logging +import os +import pathlib + +import numpy as np +from ctapipe.core.traits import ComponentNameList +from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN, N_GAINS + +from ...data.container import NectarCAMPedestalContainer, NectarCAMPedestalContainers +from ..component import NectarCAMComponent +from .core import NectarCAMCalibrationTool + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + +__all__ = ["PedestalNectarCAMCalibrationTool"] + + +class PedestalNectarCAMCalibrationTool(NectarCAMCalibrationTool): + name = "PedestalNectarCAMCalibrationTool" + + componentsList = ComponentNameList( + NectarCAMComponent, + default_value=["PedestalEstimationComponent"], + help="List of Component names to be applied, the order will be respected", + ).tag(config=True) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def _init_output_path(self): + """ + Initialize output path + """ + + if self.events_per_slice is None: + ext = ".h5" + else: + ext = f"_sliced{self.events_per_slice}.h5" + if self.max_events is None: + filename = f"{self.name}_run{self.run_number}{ext}" + else: + filename = ( + f"{self.name}_run{self.run_number}_maxevents{self.max_events}{ext}" + ) + + self.output_path = pathlib.Path( + f"{os.environ.get('NECTARCAMDATA', '/tmp')}/PedestalEstimation/{filename}" + ) + + def _combine_results(self): + """ + Method that combines sliced results to reduce memory load + Can only be called after the file with the sliced results has been saved to disk + """ + + # re-open results + pedestalContainers = next( + NectarCAMPedestalContainers.from_hdf5(self.output_path) + ) + # Loop over sliced results to fill the combined results + if "data_combined" in pedestalContainers.containers.keys(): + log.error("Trying to combine results that already contain combined data") + self.log.info("Combine sliced results") + for i, (_, pedestalContainer) in enumerate( + pedestalContainers.containers.items() + ): + if i == 0: + # initialize fields for the combined results based on first slice + nsamples = pedestalContainer.nsamples + nevents = np.zeros(len(pedestalContainer.nevents)) + pixels_id = pedestalContainer.pixels_id + ucts_timestamp_min = pedestalContainer.ucts_timestamp_min + ucts_timestamp_max = pedestalContainer.ucts_timestamp_max + pedestal_mean_hg = np.zeros( + np.shape(pedestalContainer.pedestal_mean_hg) + ) + pedestal_mean_lg = np.zeros( + np.shape(pedestalContainer.pedestal_mean_lg) + ) + pedestal_std_hg = np.zeros(np.shape(pedestalContainer.pedestal_std_hg)) + pedestal_std_lg = np.zeros(np.shape(pedestalContainer.pedestal_std_lg)) + else: + # otherwise consider the overall time interval + ucts_timestamp_min = np.minimum( + ucts_timestamp_min, pedestalContainer.ucts_timestamp_min + ) + ucts_timestamp_max = np.maximum( + ucts_timestamp_max, pedestalContainer.ucts_timestamp_max + ) + # for all slices + # derive from pixel mask a mask that sets usable pixels + # accept only pixels for which no flags were raised + usable_pixels = pedestalContainer.pixel_mask == 0 + # use a pixel only if it has no flag on either channel + usable_pixels = np.logical_and(usable_pixels[0], usable_pixels[1]) + # cumulated number of events + nevents += pedestalContainer.nevents * usable_pixels + # add mean, std sum elements + pedestal_mean_hg += ( + pedestalContainer.pedestal_mean_hg + * pedestalContainer.nevents[:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + pedestal_mean_lg += ( + pedestalContainer.pedestal_mean_lg + * pedestalContainer.nevents[:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + pedestal_std_hg += ( + pedestalContainer.pedestal_std_hg**2 + * pedestalContainer.nevents[:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + pedestal_std_lg += ( + pedestalContainer.pedestal_std_lg**2 + * pedestalContainer.nevents[:, np.newaxis] + * usable_pixels[:, np.newaxis] + ) + # calculate final values of mean and std + pedestal_mean_hg /= nevents[:, np.newaxis] + pedestal_mean_lg /= nevents[:, np.newaxis] + pedestal_std_hg /= nevents[:, np.newaxis] + pedestal_std_hg = np.sqrt(pedestal_std_hg) + pedestal_std_lg /= nevents[:, np.newaxis] + pedestal_std_lg = np.sqrt(pedestal_std_lg) + # flag bad pixels in overall results based on same criteria as for individual + # slides + # reconstitute dictionary with cumulated results consistently with + # PedestalComponent + ped_stats = {} + array_shape = np.append([N_GAINS], np.shape(pedestal_mean_hg)) + for statistic in ["mean", "std"]: + ped_stat = np.zeros(array_shape) + if statistic == "mean": + ped_stat[HIGH_GAIN] = pedestal_mean_hg + ped_stat[LOW_GAIN] = pedestal_mean_lg + elif statistic == "std": + ped_stat[HIGH_GAIN] = pedestal_std_hg + ped_stat[LOW_GAIN] = pedestal_std_lg + # Store the result in the dictionary + ped_stats[statistic] = ped_stat + # use flagging method from PedestalComponent + pixel_mask = self.components[0].flag_bad_pixels(ped_stats, nevents) + + output = NectarCAMPedestalContainer( + nsamples=nsamples, + nevents=nevents, + pixels_id=pixels_id, + ucts_timestamp_min=ucts_timestamp_min, + ucts_timestamp_max=ucts_timestamp_max, + pedestal_mean_hg=pedestal_mean_hg, + pedestal_mean_lg=pedestal_mean_lg, + pedestal_std_hg=pedestal_std_hg, + pedestal_std_lg=pedestal_std_lg, + pixel_mask=pixel_mask, + ) + + return output + + def finish(self, return_output_component=False, *args, **kwargs): + """ + Redefines finish method to combine sliced results + """ + + self.log.info("finishing Tool") + + # finish components + output = self._finish_components(*args, **kwargs) + + # close writer + self.writer.close() + + # Check if there are slices + if self.events_per_slice is None: + # If not nothing to do + pass + else: + # combine results + output = self._combine_results() + # add combined results to output + # re-initialise writer to store combined results + self._init_writer(sliced=True, group_name="data_combined") + # add combined results to writer + self._write_container(output) + self.writer.close() + + self.log.info("Shutting down.") + if return_output_component: + return output diff --git a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py index d5f382dc..af72a269 100644 --- a/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py +++ b/src/nectarchain/makers/calibration/tests/test_pedestal_tool.py @@ -1,11 +1,10 @@ import tempfile import numpy as np -import tables from ctapipe.utils import get_dataset_path from ctapipe_io_nectarcam.constants import N_SAMPLES -from nectarchain.data.container import NectarCAMPedestalContainer, PedestalFlagBits +from nectarchain.data.container import NectarCAMPedestalContainers, PedestalFlagBits from nectarchain.makers.calibration import PedestalNectarCAMCalibrationTool runs = { @@ -32,8 +31,7 @@ def test_base(self): expected_ucts_timestamp_min = [1674462932637854793, 1715007113924900896] expected_ucts_timestamp_max = [1674462932695877994, 1715007123524920096] - for i, run in enumerate(runs["Run number"]): - run_number = runs["Run number"][i] + for i, run_number in enumerate(runs["Run number"]): run_file = runs["Run file"][i] n_pixels = runs["N pixels"][i] with tempfile.TemporaryDirectory() as tmpdirname: @@ -52,7 +50,7 @@ def test_base(self): pixel_mask_nevents_min=1, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -80,57 +78,75 @@ def test_base(self): # Check output on disk # FIXME: use tables for the moment, update when h5 reader in nectarchain # is working - with tables.open_file(outfile) as h5file: - for s in range(n_slices[i]): - # Check individual groups - group_name = "data_{}".format(s + 1) - assert group_name in h5file.root.__members__ - table = h5file.root[group_name][ - NectarCAMPedestalContainer.__name__ - ][0] - assert table["nsamples"] == N_SAMPLES - assert np.allclose(table["nevents"], events_per_slice, atol=7) - assert np.shape(table["pixels_id"]) == (n_pixels,) - assert np.shape(table["pedestal_mean_hg"]) == ( - n_pixels, - N_SAMPLES, - ) - assert np.shape(table["pedestal_mean_lg"]) == ( - n_pixels, - N_SAMPLES, - ) - assert np.shape(table["pedestal_std_hg"]) == ( - n_pixels, - N_SAMPLES, - ) - assert np.shape(table["pedestal_std_lg"]) == ( - n_pixels, - N_SAMPLES, - ) - # Check combined results - group_name = "data_combined" - table = h5file.root[group_name][ - NectarCAMPedestalContainer.__name__ - ][0] - assert table["nsamples"] == N_SAMPLES - assert np.all(table["nevents"] == max_events[i]) - assert np.shape(table["pixels_id"]) == (n_pixels,) - assert table["ucts_timestamp_min"] == np.uint64( - expected_ucts_timestamp_min[i] + pedestalContainers = next( + NectarCAMPedestalContainers.from_hdf5(outfile) + ) + j = 0 + for key, pedestalContainer in pedestalContainers.containers.items(): + if "combined" in key: + continue + # Check individual groups + group_name = "data_{}".format(i + 1) + assert group_name in pedestalContainers.containers.keys() + assert pedestalContainer.nsamples == N_SAMPLES + assert np.allclose( + pedestalContainer.nevents, events_per_slice, atol=7 ) - assert table["ucts_timestamp_max"] == np.uint64( - expected_ucts_timestamp_max[i] + assert np.shape(pedestalContainer.pixels_id) == (n_pixels,) + assert np.shape(pedestalContainer.pedestal_mean_hg) == ( + n_pixels, + N_SAMPLES, ) - assert np.shape(table["pedestal_mean_hg"]) == (n_pixels, N_SAMPLES) - assert np.shape(table["pedestal_mean_lg"]) == (n_pixels, N_SAMPLES) - assert np.shape(table["pedestal_std_hg"]) == (n_pixels, N_SAMPLES) - assert np.shape(table["pedestal_std_lg"]) == (n_pixels, N_SAMPLES) - assert np.allclose(table["pedestal_mean_hg"], 245.0, atol=20.0) - assert np.allclose(table["pedestal_mean_lg"], 245.0, atol=20.0) - assert np.allclose(table["pedestal_std_hg"], 10, atol=10) - assert np.allclose( - table["pedestal_std_lg"], 2.5, atol=2.0 if i == 0 else 2.3 + assert np.shape(pedestalContainer.pedestal_mean_lg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(pedestalContainer.pedestal_std_hg) == ( + n_pixels, + N_SAMPLES, ) + assert np.shape(pedestalContainer.pedestal_std_lg) == ( + n_pixels, + N_SAMPLES, + ) + j += 1 + # Check combined results + pedestalContainers = next( + NectarCAMPedestalContainers.from_hdf5(outfile) + ) + group_name = "data_combined" + pedestalContainer = pedestalContainers.containers[group_name] + assert pedestalContainer.nsamples == N_SAMPLES + assert np.all(pedestalContainer.nevents == max_events[i]) + assert np.shape(pedestalContainer.pixels_id) == (n_pixels,) + assert pedestalContainer.ucts_timestamp_min == np.uint64( + expected_ucts_timestamp_min[i] + ) + assert pedestalContainer.ucts_timestamp_max == np.uint64( + expected_ucts_timestamp_max[i] + ) + assert np.shape(pedestalContainer.pedestal_mean_hg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(pedestalContainer.pedestal_mean_lg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(pedestalContainer.pedestal_std_hg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.shape(pedestalContainer.pedestal_std_lg) == ( + n_pixels, + N_SAMPLES, + ) + assert np.allclose(pedestalContainer.pedestal_mean_hg, 245.0, atol=20.0) + assert np.allclose(pedestalContainer.pedestal_mean_lg, 245.0, atol=20.0) + assert np.allclose(pedestalContainer.pedestal_std_hg, 10, atol=10) + assert np.allclose( + pedestalContainer.pedestal_std_lg, 2.5, atol=2.0 if i == 0 else 2.3 + ) def test_timesel(self): """ @@ -143,7 +159,7 @@ def test_timesel(self): max_events = [n_slices[0] * events_per_slice, 13] tmin = [1674462932637860000, 1715007113924900000] tmax = [1674462932695700000, 1715007123524921000] - for i, run in enumerate(runs["Run number"]): + for i, _ in enumerate(runs["Run number"]): run_number = runs["Run number"][i] run_file = runs["Run file"][i] n_pixels = runs["N pixels"][i] @@ -166,7 +182,7 @@ def test_timesel(self): pixel_mask_nevents_min=1, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -198,7 +214,7 @@ def test_WaveformsStdFilter(self): n_slices = [3, 2] events_per_slice = 10 max_events = [n_slices[0] * events_per_slice, 13] - for i, run in enumerate(runs["Run number"]): + for i, _ in enumerate(runs["Run number"]): run_number = runs["Run number"][i] run_file = runs["Run file"][i] n_pixels = runs["N pixels"][i] @@ -219,7 +235,7 @@ def test_WaveformsStdFilter(self): pixel_mask_nevents_min=1, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -252,7 +268,7 @@ def test_ChargeDistributionFilter(self): n_slices = [2, 1] events_per_slice = 10 max_events = [n_slices[0] * events_per_slice - 1, 12] - for i, run in enumerate(runs["Run number"]): + for i, _ in enumerate(runs["Run number"]): run_number = runs["Run number"][i] run_file = runs["Run file"][i] n_pixels = runs["N pixels"][i] @@ -274,7 +290,7 @@ def test_ChargeDistributionFilter(self): pixel_mask_nevents_min=1, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -322,7 +338,7 @@ def test_pixel_mask(self): filter_method=None, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -332,13 +348,16 @@ def test_pixel_mask(self): flag_bit = PedestalFlagBits.NEVENTS assert np.all(output.pixel_mask & flag_bit == flag_bit) # Check that other flags were not raised - flag_bits = [PedestalFlagBits.MEAN_PEDESTAL, - PedestalFlagBits.STD_SAMPLE, - PedestalFlagBits.STD_PIXEL] + flag_bits = [ + PedestalFlagBits.MEAN_PEDESTAL, + PedestalFlagBits.STD_SAMPLE, + PedestalFlagBits.STD_PIXEL, + ] for flag_bit in flag_bits: assert np.all(output.pixel_mask & flag_bit == 0) - # For all the following tests we set the acceptable values to a range out of what + # For all the following tests we set the acceptable values to a range + # out of what # is normal. Since our test run is good we expect to flag all pixels # Condition on mean pedestal value @@ -352,11 +371,11 @@ def test_pixel_mask(self): overwrite=True, filter_method=None, pixel_mask_nevents_min=1, - pixel_mask_mean_min=1000., - pixel_mask_mean_max=1100., + pixel_mask_mean_min=1000.0, + pixel_mask_mean_max=1100.0, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -366,9 +385,11 @@ def test_pixel_mask(self): flag_bit = PedestalFlagBits.MEAN_PEDESTAL assert np.all(output.pixel_mask & flag_bit == flag_bit) # Check that other flags were not raised - flag_bits = [PedestalFlagBits.NEVENTS, - PedestalFlagBits.STD_SAMPLE, - PedestalFlagBits.STD_PIXEL] + flag_bits = [ + PedestalFlagBits.NEVENTS, + PedestalFlagBits.STD_SAMPLE, + PedestalFlagBits.STD_PIXEL, + ] for flag_bit in flag_bits: assert np.all(output.pixel_mask & flag_bit == 0) @@ -383,22 +404,25 @@ def test_pixel_mask(self): overwrite=True, filter_method=None, pixel_mask_nevents_min=1, - pixel_mask_std_sample_min=100. + pixel_mask_std_sample_min=100.0, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() output = tool.finish(return_output_component=True)[0] - # Check that all pixels were flagged as having a small sample std + # Check that all pixels were flagged as + # having a small sample std flag_bit = PedestalFlagBits.STD_SAMPLE assert np.all(output.pixel_mask & flag_bit == flag_bit) # Check that other flags were not raised - flag_bits = [PedestalFlagBits.NEVENTS, - PedestalFlagBits.MEAN_PEDESTAL, - PedestalFlagBits.STD_PIXEL] + flag_bits = [ + PedestalFlagBits.NEVENTS, + PedestalFlagBits.MEAN_PEDESTAL, + PedestalFlagBits.STD_PIXEL, + ] for flag_bit in flag_bits: assert np.all(output.pixel_mask & flag_bit == 0) @@ -413,10 +437,10 @@ def test_pixel_mask(self): overwrite=True, filter_method=None, pixel_mask_nevents_min=1, - pixel_mask_std_pixel_max=0.01 + pixel_mask_std_pixel_max=0.01, ) - tool.initialize() + # tool.initialize() tool.setup() tool.start() @@ -426,8 +450,10 @@ def test_pixel_mask(self): flag_bit = PedestalFlagBits.STD_PIXEL assert np.all(output.pixel_mask & flag_bit == flag_bit) # Check that other flags were not raised - flag_bits = [PedestalFlagBits.NEVENTS, - PedestalFlagBits.MEAN_PEDESTAL, - PedestalFlagBits.STD_SAMPLE] + flag_bits = [ + PedestalFlagBits.NEVENTS, + PedestalFlagBits.MEAN_PEDESTAL, + PedestalFlagBits.STD_SAMPLE, + ] for flag_bit in flag_bits: assert np.all(output.pixel_mask & flag_bit == 0) diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/charges_makers.py similarity index 72% rename from src/nectarchain/makers/chargesMakers.py rename to src/nectarchain/makers/charges_makers.py index 97ff9cb2..3998ea75 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/charges_makers.py @@ -1,31 +1,31 @@ import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import os import pathlib import numpy as np from ctapipe.core.traits import Bool, ComponentNameList -from ctapipe.image.extractor import ( +from ctapipe.image.extractor import FixedWindowSum # noqa: F401 +from ctapipe.image.extractor import FullWaveformSum # noqa: F401 +from ctapipe.image.extractor import GlobalPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import LocalPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import NeighborPeakWindowSum # noqa: F401 +from ctapipe.image.extractor import SlidingWindowMaxSum # noqa: F401 +from ctapipe.image.extractor import TwoPassWindowSum # noqa: F401 +from ctapipe.image.extractor import ( # noqa: F401 BaselineSubtractedNeighborPeakWindowSum, - FixedWindowSum, - FullWaveformSum, - GlobalPeakWindowSum, - LocalPeakWindowSum, - NeighborPeakWindowSum, - SlidingWindowMaxSum, - TwoPassWindowSum, ) -from ..data.container import ChargesContainers, WaveformsContainer, WaveformsContainers +from ..data.container import WaveformsContainer, WaveformsContainers from ..data.management import DataManagement from .component import ChargesComponent, NectarCAMComponent from .core import EventsLoopNectarCAMCalibrationTool from .extractor.utils import CtapipeExtractor +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["ChargesNectarCAMCalibrationTool"] @@ -53,9 +53,11 @@ def _init_output_path(self): self.extractor_kwargs ) if self.max_events is None: - filename = f"{self.name}_run{self.run_number}_{self.method}_{str_extractor_kwargs}.h5" + filename = f"{self.name}_run{self.run_number}_{self.method}" + f"{str_extractor_kwargs}.h5" else: - filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}_{self.method}_{str_extractor_kwargs}.h5" + filename = f"{self.name}_run{self.run_number}_maxevents{self.max_events}_" + f"{self.method}_{str_extractor_kwargs}.h5" self.output_path = pathlib.Path( f"{os.environ.get('NECTARCAMDATA','/tmp')}/runs/charges/{filename}" @@ -69,13 +71,16 @@ def start( *args, **kwargs, ): + # cette implémentation est complétement nulle if self.from_computed_waveforms: files = DataManagement.find_waveforms( run_number=self.run_number, max_events=self.max_events ) if len(files) != 1: self.log.info( - f"{len(files)} computed wavforms files found with max_events >= {self.max_events} for run {self.run_number}, reload waveforms from event loop" + f"{len(files)} computed wavforms files found with max_events >=" + f"{self.max_events} for run {self.run_number}, reload waveforms" + f"from event loop" ) super().start( n_events=n_events, @@ -85,19 +90,25 @@ def start( ) else: self.log.info( - f"{files[0]} is the computed wavforms files found with max_events >= {self.max_events} for run {self.run_number}" + f"{files[0]} is the computed wavforms files found" + f"with max_events >=" + f"{self.max_events} for run {self.run_number}" ) - waveformsContainers = WaveformsContainer.from_hdf5(files[0]) + waveformsContainers = WaveformsContainers.from_hdf5(files[0]) if not (isinstance(waveformsContainers, WaveformsContainer)): - chargesContainers = ChargesContainers() - if isinstance(waveformsContainers, WaveformsContainers): - self.log.debug( - "WaveformsContainer file container multiple trigger type" - ) + n_slices = 0 + try: + while True: + next(waveformsContainers) + n_slices += 1 + except StopIteration: + pass + waveformsContainers = WaveformsContainers.from_hdf5(files[0]) + if n_slices == 1: self._init_writer(sliced=False) chargesContainers = ( ChargesComponent._create_from_waveforms_looping_eventType( - waveformsContainers=waveformsContainers, + waveformsContainers=next(waveformsContainers), subarray=self.event_source.subarray, method=self.method, **self.extractor_kwargs, @@ -106,13 +117,14 @@ def start( self._write_container(container=chargesContainers) else: self.log.debug( - "WaveformsContainer file container multiple slices of the run events" + f"WaveformsContainer file contains {n_slices} slices of the" + f"run events" ) for slice_index, _waveformsContainers in enumerate( waveformsContainers ): self._init_writer(sliced=True, slice_index=slice_index) - chargesContainers = ChargesComponent._create_from_waveforms_looping_eventType( + chargesContainers = ChargesComponent._create_from_waveforms_looping_eventType( # noqa waveformsContainers=_waveformsContainers, subarray=self.event_source.subarray, method=self.method, @@ -121,7 +133,8 @@ def start( self._write_container(container=chargesContainers) else: self.log.debug( - "WaveformsContainer file container is a simple WaveformsContainer (not mapped)" + "WaveformsContainer file container is a simple \ + WaveformsContainer (not mapped)" ) self._init_writer(sliced=False) chargesContainers = ChargesComponent.create_from_waveforms( diff --git a/src/nectarchain/makers/component/__init__.py b/src/nectarchain/makers/component/__init__.py index f7b85ec9..62aae577 100644 --- a/src/nectarchain/makers/component/__init__.py +++ b/src/nectarchain/makers/component/__init__.py @@ -1,12 +1,18 @@ -from .chargesComponent import * -from .core import * -from .FlatFieldSPEComponent import * -from .PedestalComponent import * -from .gainComponent import * -from .photostatistic_algorithm import * -from .photostatistic_component import * -from .spe import * -from .waveformsComponent import * +from .charges_component import ChargesComponent +from .core import ArrayDataComponent, NectarCAMComponent, get_valid_component +from .flatfield_spe_component import ( + FlatFieldCombinedSPEStdNectarCAMComponent, + FlatFieldSingleHHVSPENectarCAMComponent, + FlatFieldSingleHHVSPEStdNectarCAMComponent, + FlatFieldSingleNominalSPENectarCAMComponent, + FlatFieldSingleNominalSPEStdNectarCAMComponent, +) +from .gain_component import GainNectarCAMComponent +from .pedestal_component import PedestalEstimationComponent +from .photostatistic_algorithm import PhotoStatisticAlgorithm +from .photostatistic_component import PhotoStatisticNectarCAMComponent +from .spe import SPECombinedalgorithm, SPEHHValgorithm, SPEHHVStdalgorithm +from .waveforms_component import WaveformsComponent __all__ = [ "ArrayDataComponent", @@ -19,9 +25,11 @@ "FlatFieldSingleNominalSPENectarCAMComponent", "FlatFieldSingleNominalSPEStdNectarCAMComponent", "FlatFieldCombinedSPEStdNectarCAMComponent", + "get_valid_component", "ChargesComponent", "WaveformsComponent", "PedestalEstimationComponent", "PhotoStatisticNectarCAMComponent", "PhotoStatisticAlgorithm", + "GainNectarCAMComponent", ] diff --git a/src/nectarchain/makers/component/chargesComponent.py b/src/nectarchain/makers/component/charges_component.py similarity index 94% rename from src/nectarchain/makers/component/chargesComponent.py rename to src/nectarchain/makers/component/charges_component.py index 0b997499..d93d7309 100644 --- a/src/nectarchain/makers/component/chargesComponent.py +++ b/src/nectarchain/makers/component/charges_component.py @@ -61,8 +61,7 @@ cache=True, ) def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): - """ - Compute histogram of charge with numba + """Compute histogram of charge with numba. Parameters ---------- @@ -71,7 +70,6 @@ def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): mask_broken_pix (np.ndarray(pixels)): mask on broxen pixels _mask (np.ndarray(pixels,nbins)): mask hist_ma_data (np.ndarray(pixels,nbins)): histogram - """ # print(f"charge.shape = {charge.shape[0]}") # print(f"_mask.shape = {_mask.shape[0]}") @@ -138,8 +136,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__peak_lg = {} def _init_trigger_type(self, trigger_type: EventType, **kwargs): - """ - Initializes the ChargesMaker based on the trigger type. + """Initializes the ChargesMaker based on the trigger type. Parameters ---------- @@ -149,7 +146,6 @@ def _init_trigger_type(self, trigger_type: EventType, **kwargs): Returns ------- None - """ super()._init_trigger_type(trigger_type, **kwargs) name = __class__._get_name_trigger(trigger_type) @@ -212,8 +208,7 @@ def _get_extractor_kwargs_from_method_and_kwargs(method: str, kwargs: dict): @staticmethod def _get_imageExtractor(method: str, subarray: SubarrayDescription, **kwargs): - """ - Create an instance of a charge extraction method based on the provided method + """Create an instance of a charge extraction method based on the provided method name and subarray description. Parameters @@ -250,8 +245,7 @@ def _get_imageExtractor(method: str, subarray: SubarrayDescription, **kwargs): return imageExtractor def finish(self, *args, **kwargs): - """ - Create an output container for the specified trigger type and method. + """Create an output container for the specified trigger type and method. Parameters ---------- @@ -263,7 +257,6 @@ def finish(self, *args, **kwargs): Returns ------- list: A list of ChargesContainer objects. - """ output = ChargesContainers() for i, trigger in enumerate(self.trigger_list): @@ -296,8 +289,7 @@ def finish(self, *args, **kwargs): @staticmethod def sort(chargesContainer: ChargesContainer, method: str = "event_id"): - """ - Sorts the charges in a ChargesContainer object based on the specified method. + """Sorts the charges in a ChargesContainer object based on the specified method. Parameters ---------- @@ -346,9 +338,8 @@ def sort(chargesContainer: ChargesContainer, method: str = "event_id"): @staticmethod def select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): - """ - Selects the charges from the ChargesContainer object for the given pixel_id and - returns the result transposed. + """Selects the charges from the ChargesContainer object for the given pixel_id + and returns the result transposed. Parameters ---------- @@ -371,9 +362,8 @@ def select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): @staticmethod def select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): - """ - Selects the charges from the ChargesContainer object for the given pixel_id and - returns the result transposed. + """Selects the charges from the ChargesContainer object for the given pixel_id + and returns the result transposed. Parameters ---------- @@ -395,8 +385,7 @@ def select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): return res def charges_hg(self, trigger: EventType): - """ - Returns the charges for a specific trigger type as a NumPy array of unsigned + """Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Parameters @@ -415,8 +404,7 @@ def charges_hg(self, trigger: EventType): ) def charges_lg(self, trigger: EventType): - """ - Returns the charges for a specific trigger type as a NumPy array of unsigned + """Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Parameters @@ -435,8 +423,7 @@ def charges_lg(self, trigger: EventType): ) def peak_hg(self, trigger: EventType): - """ - Returns the peak charges for a specific trigger type as a NumPy array of + """Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Parameters @@ -455,8 +442,7 @@ def peak_hg(self, trigger: EventType): ) def peak_lg(self, trigger: EventType): - """ - Returns the peak charges for a specific trigger type as a NumPy array of + """Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Parameters @@ -492,8 +478,7 @@ def create_from_waveforms( method: str = "FullWaveformSum", **kwargs, ) -> ChargesContainer: - """ - Create a ChargesContainer object from waveforms using the specified charge + """Create a ChargesContainer object from waveforms using the specified charge extraction method. Parameters @@ -546,8 +531,7 @@ def compute_charges( tel_id: int = None, **kwargs, ): - """ - Compute charge from waveforms. + """Compute charge from waveforms. Parameters ---------- @@ -626,8 +610,7 @@ def compute_charges( def histo_hg( chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True ) -> ma.masked_array: - """ - Computes histogram of high gain charges from a ChargesContainer object. + """Computes histogram of high gain charges from a ChargesContainer object. Parameters ---------- @@ -657,8 +640,7 @@ def histo_hg( def histo_lg( chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True ) -> ma.masked_array: - """ - Computes histogram of low gain charges from a ChargesContainer object. + """Computes histogram of low gain charges from a ChargesContainer object. Parameters ---------- @@ -691,9 +673,8 @@ def _histo( n_bins: int = 1000, autoscale: bool = True, ) -> ma.masked_array: - """ - Computes histogram of charges for a given field from a ChargesContainer object. - Numba is used to compute histograms in a vectorized way. + """Computes histogram of charges for a given field from a ChargesContainer + object. Numba is used to compute histograms in a vectorized way. Parameters ---------- diff --git a/src/nectarchain/makers/component/core.py b/src/nectarchain/makers/component/core.py index 5e36e71f..d5f88f16 100644 --- a/src/nectarchain/makers/component/core.py +++ b/src/nectarchain/makers/component/core.py @@ -9,6 +9,7 @@ from ctapipe.core.traits import ComponentNameList, Integer, Unicode from ctapipe.instrument import CameraGeometry from ctapipe_io_nectarcam import constants +from ctapipe_io_nectarcam.constants import N_PIXELS from ctapipe_io_nectarcam.containers import NectarCAMDataContainer from ...data.container.core import ArrayDataContainer @@ -29,7 +30,7 @@ def get_valid_component(): class NectarCAMComponent(TelescopeComponent): - """The base class for NectarCAM components""" + """The base class for NectarCAM components.""" SubComponents = ComponentNameList( Component, @@ -117,8 +118,8 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): self.__broken_pixels_lg = {} def _init_trigger_type(self, trigger: EventType, **kwargs): - """ - Initializes empty lists for different trigger types in the ArrayDataMaker class. + """Initializes empty lists for different trigger types in the ArrayDataMaker + class. Args: trigger (EventType): The trigger type for which the lists are being @@ -140,8 +141,8 @@ def _init_trigger_type(self, trigger: EventType, **kwargs): @staticmethod def _compute_broken_pixels(wfs_hg, wfs_lg, **kwargs): - """ - Computes broken pixels for high and low gain waveforms. + """Computes broken pixels for high and low gain waveforms. + Args: wfs_hg (ndarray): High gain waveforms. wfs_lg (ndarray): Low gain waveforms. @@ -159,8 +160,8 @@ def _compute_broken_pixels(wfs_hg, wfs_lg, **kwargs): def _compute_broken_pixels_event( event: NectarCAMDataContainer, pixels_id: np.ndarray, **kwargs ): - """ - Computes broken pixels for a specific event and pixel IDs. + """Computes broken pixels for a specific event and pixel IDs. + Args: event (NectarCAMDataContainer): An event. pixels_id (list or np.ndarray): IDs of pixels. @@ -175,8 +176,8 @@ def _compute_broken_pixels_event( @staticmethod def _get_name_trigger(trigger: EventType): - """ - Gets the name of a trigger event. + """Gets the name of a trigger event. + Args: trigger (EventType): A trigger event. Returns: @@ -189,8 +190,7 @@ def _get_name_trigger(trigger: EventType): return name def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): - """ - Method to extract data from the event. + """Method to extract data from the event. Parameters ---------- @@ -225,9 +225,17 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.__ucts_event_counter[f"{name}"].append( event.nectarcam.tel[__class__.TEL_ID.default_value].evt.ucts_event_counter ) - self.__trig_patter_all[f"{name}"].append( - event.nectarcam.tel[__class__.TEL_ID.default_value].evt.trigger_pattern.T - ) + if ( + event.nectarcam.tel[__class__.TEL_ID.default_value].evt.trigger_pattern + is None + ): + self.__trig_patter_all[f"{name}"].append(np.empty((4, N_PIXELS)).T) + else: + self.__trig_patter_all[f"{name}"].append( + event.nectarcam.tel[ + __class__.TEL_ID.default_value + ].evt.trigger_pattern.T + ) if kwargs.get("return_wfs", False): get_wfs_hg = event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id] @@ -242,9 +250,8 @@ def finish(self): def select_container_array_field( container: ArrayDataContainer, pixel_id: np.ndarray, field: str ) -> np.ndarray: - """ - Selects specific fields from an ArrayDataContainer object based on a given list - of pixel IDs. + """Selects specific fields from an ArrayDataContainer object based on a given + list of pixel IDs. Args: container (ArrayDataContainer): An object of type ArrayDataContainer that @@ -304,7 +311,7 @@ def merge_along_slices( def merge( container_a: ArrayDataContainer, container_b: ArrayDataContainer ) -> ArrayDataContainer: - """method to merge 2 ArrayDataContainer into one single ArrayDataContainer + """Method to merge 2 ArrayDataContainer into one single ArrayDataContainer. Returns: ArrayDataContainer: the merged object @@ -343,8 +350,7 @@ def merge( @property def nsamples(self): - """ - Returns a deep copy of the nsamples attribute. + """Returns a deep copy of the nsamples attribute. Returns: np.ndarray: A deep copy of the nsamples attribute. @@ -353,8 +359,7 @@ def nsamples(self): @property def _nsamples(self): - """ - Returns the nsamples attribute. + """Returns the nsamples attribute. Returns: np.ndarray: The nsamples attribute. @@ -362,8 +367,7 @@ def _nsamples(self): return self.__nsamples def nevents(self, trigger: EventType): - """ - Returns the number of events for the specified trigger type. + """Returns the number of events for the specified trigger type. Args: trigger (EventType): The trigger type for which the number of events is @@ -378,8 +382,7 @@ def nevents(self, trigger: EventType): @property def _broken_pixels_hg(self): - """ - Returns the broken_pixels_hg attribute. + """Returns the broken_pixels_hg attribute. Returns: np.ndarray: The broken_pixels_hg attribute. @@ -387,8 +390,8 @@ def _broken_pixels_hg(self): return self.__broken_pixels_hg def broken_pixels_hg(self, trigger: EventType): - """ - Returns an array of broken pixels for high gain for the specified trigger type. + """Returns an array of broken pixels for high gain for the specified trigger + type. Args: trigger (EventType): The trigger type for which the broken pixels for high @@ -405,8 +408,7 @@ def broken_pixels_hg(self, trigger: EventType): @property def _broken_pixels_lg(self): - """ - Returns the broken_pixels_lg attribute. + """Returns the broken_pixels_lg attribute. Returns: np.ndarray: The broken_pixels_lg attribute. @@ -414,8 +416,8 @@ def _broken_pixels_lg(self): return self.__broken_pixels_lg def broken_pixels_lg(self, trigger: EventType): - """ - Returns an array of broken pixels for low gain for the specified trigger type. + """Returns an array of broken pixels for low gain for the specified trigger + type. Args: trigger (EventType): The trigger type for which the broken pixels for low @@ -431,8 +433,7 @@ def broken_pixels_lg(self, trigger: EventType): ) def ucts_timestamp(self, trigger: EventType): - """ - Returns an array of UCTS timestamps for the specified trigger type. + """Returns an array of UCTS timestamps for the specified trigger type. Args: trigger (EventType): The trigger type for which the UCTS timestamps are @@ -447,8 +448,7 @@ def ucts_timestamp(self, trigger: EventType): ) def ucts_busy_counter(self, trigger: EventType): - """ - Returns an array of UCTS busy counters for the specified trigger type. + """Returns an array of UCTS busy counters for the specified trigger type. Args: trigger (EventType): The trigger type for which the UCTS busy counters are @@ -463,8 +463,7 @@ def ucts_busy_counter(self, trigger: EventType): ) def ucts_event_counter(self, trigger: EventType): - """ - Returns an array of UCTS event counters for the specified trigger type. + """Returns an array of UCTS event counters for the specified trigger type. Args: trigger (EventType): The trigger type for which the UCTS event counters are @@ -479,8 +478,7 @@ def ucts_event_counter(self, trigger: EventType): ) def event_type(self, trigger: EventType): - """ - Returns an array of event types for the specified trigger type. + """Returns an array of event types for the specified trigger type. Args: trigger (EventType): The trigger type for which the event types are @@ -495,8 +493,7 @@ def event_type(self, trigger: EventType): ) def event_id(self, trigger: EventType): - """ - Returns an array of event IDs for the specified trigger type. + """Returns an array of event IDs for the specified trigger type. Args: trigger (EventType): The trigger type for which the event IDs are requested. @@ -510,8 +507,7 @@ def event_id(self, trigger: EventType): ) def multiplicity(self, trigger: EventType): - """ - Returns an array of multiplicities for the specified trigger type. + """Returns an array of multiplicities for the specified trigger type. Args: trigger (EventType): The trigger type for which the multiplicities are @@ -529,8 +525,7 @@ def multiplicity(self, trigger: EventType): ) def trig_pattern(self, trigger: EventType): - """ - Returns an array of trigger patterns for the specified trigger type. + """Returns an array of trigger patterns for the specified trigger type. Args: trigger (EventType): The trigger type for which the trigger patterns are @@ -546,8 +541,7 @@ def trig_pattern(self, trigger: EventType): return tmp.any(axis=2) def trig_pattern_all(self, trigger: EventType): - """ - Returns an array of trigger patterns for all events for the specified trigger + """Returns an array of trigger patterns for all events for the specified trigger type. Args: diff --git a/src/nectarchain/makers/component/FlatFieldSPEComponent.py b/src/nectarchain/makers/component/flatfield_spe_component.py similarity index 92% rename from src/nectarchain/makers/component/FlatFieldSPEComponent.py rename to src/nectarchain/makers/component/flatfield_spe_component.py index 4c72452f..8a09ee42 100644 --- a/src/nectarchain/makers/component/FlatFieldSPEComponent.py +++ b/src/nectarchain/makers/component/flatfield_spe_component.py @@ -1,10 +1,5 @@ -import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import copy +import logging import numpy as np from ctapipe.core.traits import List, Unicode @@ -12,15 +7,17 @@ from ...data.container import merge_map_ArrayDataContainer from ...utils import ComponentUtils -from .chargesComponent import ChargesComponent -from .gainComponent import GainNectarCAMComponent -from .spe import ( - SPECombinedalgorithm, - SPEHHValgorithm, - SPEHHVStdalgorithm, - SPEnominalalgorithm, - SPEnominalStdalgorithm, -) +from .charges_component import ChargesComponent +from .gain_component import GainNectarCAMComponent +from .spe import SPECombinedalgorithm # noqa: F401 +from .spe import SPEHHValgorithm # noqa: F401 +from .spe import SPEHHVStdalgorithm # noqa: F401 +from .spe import SPEnominalalgorithm # noqa: F401 +from .spe import SPEnominalStdalgorithm # noqa: F401 + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers __all__ = [ "FlatFieldSingleNominalSPEStdNectarCAMComponent", @@ -47,12 +44,14 @@ class FlatFieldSingleNominalSPENectarCAMComponent(GainNectarCAMComponent): # Windows_lenght = Integer(40, # read_only = True, - # help = "The windows leght used for the savgol filter algorithm", + # help = "The windows leght used for the savgol + # filter algorithm", # ).tag(config = True) # # Order = Integer(2, # read_only = True, - # help = "The order of the polynome used in the savgol filter algorithm", + # help = "The order of the polynome used in the savgol + # filter algorithm", # ).tag(config = True) asked_pixels_id = List( @@ -79,7 +78,8 @@ class FlatFieldSingleNominalSPENectarCAMComponent(GainNectarCAMComponent): # ).tag(config = True) # # extractor_kwargs = Dict(default_value = {}, - # help = "The kwargs to be pass to the charge extractor method", + # help = "The kwargs to be pass to the charge extractor + # method", # ).tag(config = True) # constructor @@ -133,7 +133,7 @@ def finish(self, *args, **kwargs): spe_fit = eval(self.SPEfitalgorithm).create_from_chargesContainer( self._chargesContainers, parent=self, **self._SPEfitalgorithm_kwargs ) - fit_output = spe_fit.run(pixels_id=self.asked_pixels_id, *args, **kwargs) + _ = spe_fit.run(pixels_id=self.asked_pixels_id, *args, **kwargs) n_asked_pix = ( len(self._chargesContainers.pixels_id) if self.asked_pixels_id is None diff --git a/src/nectarchain/makers/component/gainComponent.py b/src/nectarchain/makers/component/gain_component.py similarity index 100% rename from src/nectarchain/makers/component/gainComponent.py rename to src/nectarchain/makers/component/gain_component.py index 60a7c1fd..f62537b7 100644 --- a/src/nectarchain/makers/component/gainComponent.py +++ b/src/nectarchain/makers/component/gain_component.py @@ -1,12 +1,12 @@ import logging +from abc import abstractmethod + +from .core import NectarCAMComponent logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from abc import abstractmethod - -from .core import NectarCAMComponent __all__ = ["GainNectarCAMComponent"] diff --git a/src/nectarchain/makers/component/PedestalComponent.py b/src/nectarchain/makers/component/pedestal_component.py similarity index 94% rename from src/nectarchain/makers/component/PedestalComponent.py rename to src/nectarchain/makers/component/pedestal_component.py index 7489342d..6900cae3 100644 --- a/src/nectarchain/makers/component/PedestalComponent.py +++ b/src/nectarchain/makers/component/pedestal_component.py @@ -10,9 +10,9 @@ from ...data.container import NectarCAMPedestalContainer, PedestalFlagBits from ...utils import ComponentUtils -from .chargesComponent import ChargesComponent +from .charges_component import ChargesComponent from .core import NectarCAMComponent -from .waveformsComponent import WaveformsComponent +from .waveforms_component import WaveformsComponent logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) @@ -24,12 +24,10 @@ class PedestalEstimationComponent(NectarCAMComponent): - """ - Component that computes calibration pedestal coefficients from raw data. - Waveforms can be filtered based on time, standard deviation of the waveforms - or charge distribution within the sample. - Use the ``events_per_slice`` parameter of ``NectarCAMComponent`` to reduce - memory load. + """Component that computes calibration pedestal coefficients from raw data. + Waveforms can be filtered based on time, standard deviation of the waveforms or + charge distribution within the sample. Use the ``events_per_slice`` parameter of + ``NectarCAMComponent`` to reduce memory load. Parameters ---------- @@ -62,7 +60,6 @@ class PedestalEstimationComponent(NectarCAMComponent): pixel_mask_std_pixel_max : float Maximum value of pedestal standard deviation in a pixel above which the pixel is flagged as bad - """ ucts_tmin = Integer( @@ -152,12 +149,10 @@ class PedestalEstimationComponent(NectarCAMComponent): SubComponents.read_only = True def __init__(self, subarray, config=None, parent=None, *args, **kwargs): - """ - Component that computes calibration pedestal coefficients from raw data. - Waveforms can be filtered based on time, standard deviation of the waveforms - or charge distribution within the sample. - Use the ``events_per_slice`` parameter of ``NectarCAMComponent`` to - reduce memory load. + """Component that computes calibration pedestal coefficients from raw data. + Waveforms can be filtered based on time, standard deviation of the waveforms or + charge distribution within the sample. Use the ``events_per_slice`` parameter of + ``NectarCAMComponent`` to reduce memory load. Parameters ---------- @@ -208,8 +203,7 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs): @staticmethod def calculate_stats(waveformsContainers, wfs_mask, statistics): - """ - Calculate statistics for the pedestals from a waveforms container. + """Calculate statistics for the pedestals from a waveforms container. Parameters ---------- @@ -250,8 +244,7 @@ def calculate_stats(waveformsContainers, wfs_mask, statistics): return ped_stats def flag_bad_pixels(self, ped_stats, nevents): - """ - Flag bad pixels based on pedestal properties + """Flag bad pixels based on pedestal properties. Parameters ---------- @@ -326,9 +319,7 @@ def flag_bad_pixels(self, ped_stats, nevents): return pixel_mask def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): - """ - Fill the waveform container looping over the events of type SKY_PEDESTAL. - """ + """Fill the waveform container looping over the events of type SKY_PEDESTAL.""" if event.trigger.event_type == EventType.SKY_PEDESTAL: self.waveformsComponent(event=event, *args, **kwargs) @@ -336,8 +327,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): pass def timestamp_mask(self, tmin, tmax): - """ - Generates a mask to filter waveforms outside the required time interval + """Generates a mask to filter waveforms outside the required time interval. Parameters ---------- @@ -383,10 +373,8 @@ def timestamp_mask(self, tmin, tmax): return new_mask def waveformsStdFilter_mask(self, threshold): - """ - Generates a mask to filter waveforms that have a standard deviation above - a threshold. - This option is effective for dark room verification data. + """Generates a mask to filter waveforms that have a standard deviation above a + threshold. This option is effective for dark room verification data. Parameters ---------- @@ -428,10 +416,8 @@ def waveformsStdFilter_mask(self, threshold): return new_mask def chargeDistributionFilter_mask(self, sigma_low, sigma_high): - """ - Generates a mask to filter waveforms that have a charge in the tails of the - distribution. - This option is useful for data with NSB. + """Generates a mask to filter waveforms that have a charge in the tails of the + distribution. This option is useful for data with NSB. Parameters ---------- @@ -501,10 +487,8 @@ def chargeDistributionFilter_mask(self, sigma_low, sigma_high): return new_mask def finish(self, *args, **kwargs): - """ - Finish the component by filtering the waveforms and calculating the pedestal - quantities. - """ + """Finish the component by filtering the waveforms and calculating the pedestal + quantities.""" # Use only pedestal type events waveformsContainers = self.waveformsComponent.finish() diff --git a/src/nectarchain/makers/component/photostatistic_algorithm.py b/src/nectarchain/makers/component/photostatistic_algorithm.py index 2b622aa1..8a9c991a 100644 --- a/src/nectarchain/makers/component/photostatistic_algorithm.py +++ b/src/nectarchain/makers/component/photostatistic_algorithm.py @@ -1,15 +1,7 @@ -import logging -import sys - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - - import copy +import logging import os -import matplotlib import numpy as np from astropy.visualization import quantity_support from ctapipe.core import Component @@ -19,6 +11,11 @@ from ...data.container import ChargesContainer, GainContainer, SPEfitContainer from ..component import ChargesComponent +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["PhotoStatisticAlgorithm"] @@ -118,9 +115,8 @@ def __get_charges_FF_Ped_reshaped( return out def __check_shape(self) -> None: - """ - Checks the shape of certain attributes and raises an exception if the shape is not as expected. - """ + """Checks the shape of certain attributes and raises an exception if the shape + is not as expected.""" try: self.__FFcharge_hg[0] * self.__FFcharge_lg[0] * self.__Pedcharge_hg[ 0 @@ -159,17 +155,18 @@ def run(self, pixels_id: np.ndarray = None, **kwargs) -> None: def plot_correlation( photoStat_gain: np.ndarray, SPE_gain: np.ndarray ) -> plt.Figure: - """ - Plot the correlation between the photo statistic gain and the single photoelectron (SPE) gain. + """Plot the correlation between the photo statistic gain and the single + photoelectron (SPE) gain. Args: photoStat_gain (np.ndarray): Array of photo statistic gain values. SPE_gain (np.ndarray): Array of SPE gain values. Returns: - fig (plt.Figure): The figure object containing the scatter plot and the linear fit line. + fig (plt.Figure): The figure object containing the scatter plot + and the linear fit line. """ - matplotlib.use("TkAgg") + # matplotlib.use("TkAgg") # Create a mask to filter the data points based on certain criteria mask = (photoStat_gain > 20) * (SPE_gain > 0) * (photoStat_gain < 80) @@ -185,7 +182,8 @@ def plot_correlation( x = np.linspace(photoStat_gain[mask].min(), photoStat_gain[mask].max(), 1000) # Define a lambda function for the linear fit line - y = lambda x: a * x + b + def y(x): + return a * x + b with quantity_support(): # Create a scatter plot of the filtered data points @@ -197,7 +195,8 @@ def plot_correlation( x, y(x), color="red", - label=f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n p_value = {p_value:.2e},\n std_err = {std_err:.2e}", + label=f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n\ + p_value = {p_value:.2e},\n std_err = {std_err:.2e}", ) # Plot the line y = x @@ -211,8 +210,7 @@ def plot_correlation( @property def SPE_resolution(self) -> float: - """ - Returns a deep copy of the SPE resolution. + """Returns a deep copy of the SPE resolution. Returns: float: The SPE resolution. @@ -221,8 +219,8 @@ def SPE_resolution(self) -> float: @property def sigmaPedHG(self) -> float: - """ - Calculates and returns the standard deviation of Pedcharge_hg multiplied by the square root of coefCharge_FF_Ped. + """Calculates and returns the standard deviation of Pedcharge_hg multiplied by + the square root of coefCharge_FF_Ped. Returns: float: The standard deviation of Pedcharge_hg. @@ -231,8 +229,7 @@ def sigmaPedHG(self) -> float: @property def sigmaChargeHG(self) -> float: - """ - Calculates and returns the standard deviation of FFcharge_hg minus meanPedHG. + """Calculates and returns the standard deviation of FFcharge_hg minus meanPedHG. Returns: float: The standard deviation of FFcharge_hg minus meanPedHG. @@ -241,8 +238,8 @@ def sigmaChargeHG(self) -> float: @property def meanPedHG(self) -> float: - """ - Calculates and returns the mean of Pedcharge_hg multiplied by coefCharge_FF_Ped. + """Calculates and returns the mean of Pedcharge_hg multiplied by + coefCharge_FF_Ped. Returns: float: The mean of Pedcharge_hg. @@ -251,8 +248,7 @@ def meanPedHG(self) -> float: @property def meanChargeHG(self) -> float: - """ - Calculates and returns the mean of FFcharge_hg minus meanPedHG. + """Calculates and returns the mean of FFcharge_hg minus meanPedHG. Returns: float: The mean of FFcharge_hg minus meanPedHG. @@ -261,8 +257,7 @@ def meanChargeHG(self) -> float: @property def BHG(self) -> float: - """ - Calculates and returns the BHG value. + """Calculates and returns the BHG value. Returns: float: The BHG value. @@ -282,8 +277,7 @@ def BHG(self) -> float: @property def gainHG(self) -> float: - """ - Calculates and returns the gain for high gain charge data. + """Calculates and returns the gain for high gain charge data. Returns: float: The gain for high gain charge data. @@ -296,8 +290,8 @@ def gainHG(self) -> float: @property def sigmaPedLG(self) -> float: - """ - Calculates and returns the standard deviation of Pedcharge_lg multiplied by the square root of coefCharge_FF_Ped. + """Calculates and returns the standard deviation of Pedcharge_lg multiplied by + the square root of coefCharge_FF_Ped. Returns: float: The standard deviation of Pedcharge_lg. @@ -306,8 +300,7 @@ def sigmaPedLG(self) -> float: @property def sigmaChargeLG(self) -> float: - """ - Calculates and returns the standard deviation of FFcharge_lg minus meanPedLG. + """Calculates and returns the standard deviation of FFcharge_lg minus meanPedLG. Returns: float: The standard deviation of FFcharge_lg minus meanPedLG. @@ -316,8 +309,8 @@ def sigmaChargeLG(self) -> float: @property def meanPedLG(self) -> float: - """ - Calculates and returns the mean of Pedcharge_lg multiplied by coefCharge_FF_Ped. + """Calculates and returns the mean of Pedcharge_lg multiplied by + coefCharge_FF_Ped. Returns: float: The mean of Pedcharge_lg. @@ -326,8 +319,7 @@ def meanPedLG(self) -> float: @property def meanChargeLG(self) -> float: - """ - Calculates and returns the mean of FFcharge_lg minus meanPedLG. + """Calculates and returns the mean of FFcharge_lg minus meanPedLG. Returns: float: The mean of FFcharge_lg minus meanPedLG. @@ -336,8 +328,7 @@ def meanChargeLG(self) -> float: @property def BLG(self) -> float: - """ - Calculates and returns the BLG value. + """Calculates and returns the BLG value. Returns: float: The BLG value. @@ -357,8 +348,7 @@ def BLG(self) -> float: @property def gainLG(self) -> float: - """ - Calculates and returns the gain for low gain charge data. + """Calculates and returns the gain for low gain charge data. Returns: float: The gain for low gain charge data. diff --git a/src/nectarchain/makers/component/photostatistic_component.py b/src/nectarchain/makers/component/photostatistic_component.py index 0b73e38d..5674fcca 100644 --- a/src/nectarchain/makers/component/photostatistic_component.py +++ b/src/nectarchain/makers/component/photostatistic_component.py @@ -1,10 +1,5 @@ -import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import copy +import logging from ctapipe.containers import EventType from ctapipe.core.traits import List, Path, Unicode @@ -13,16 +8,22 @@ from ...data.container import SPEfitContainer, merge_map_ArrayDataContainer from ...utils import ComponentUtils -from .chargesComponent import ChargesComponent -from .gainComponent import GainNectarCAMComponent -from .photostatistic_algorithm import PhotoStatisticAlgorithm +from .charges_component import ChargesComponent +from .gain_component import GainNectarCAMComponent +from .photostatistic_algorithm import PhotoStatisticAlgorithm # noqa: F401 + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + __all__ = ["PhotoStatisticNectarCAMComponent"] class PhotoStatisticNectarCAMComponent(GainNectarCAMComponent): SPE_result = Path( - help="the path of the SPE result container computed with very high voltage data", + help="the path of the SPE result container computed with very\ + high voltage data", ).tag(config=True) PhotoStatAlgorithm = Unicode( "PhotoStatisticAlgorithm", @@ -103,7 +104,8 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): self.Ped_chargesComponent(event=event, *args, **kwargs) else: self.log.warning( - f"event {event.index.event_id} is event type {event.trigger.event_type} which is not used here" + f"event {event.index.event_id} is event type {event.trigger.event_type}" + f"which is not used here" ) def finish(self, *args, **kwargs): @@ -130,5 +132,5 @@ def finish(self, *args, **kwargs): parent=self, **self._PhotoStatAlgorithm_kwargs, ) - fit_output = photo_stat.run(pixels_id=self.asked_pixels_id, *args, **kwargs) + _ = photo_stat.run(pixels_id=self.asked_pixels_id, *args, **kwargs) return photo_stat.results diff --git a/src/nectarchain/makers/component/spe/__init__.py b/src/nectarchain/makers/component/spe/__init__.py index 4933f073..479a9e51 100644 --- a/src/nectarchain/makers/component/spe/__init__.py +++ b/src/nectarchain/makers/component/spe/__init__.py @@ -1,2 +1,15 @@ -# from .parameters import * -from .spe_algorithm import * +from .spe_algorithm import ( + SPECombinedalgorithm, + SPEHHValgorithm, + SPEHHVStdalgorithm, + SPEnominalalgorithm, + SPEnominalStdalgorithm, +) + +__all__ = [ + "SPEHHValgorithm", + "SPEHHVStdalgorithm", + "SPEnominalStdalgorithm", + "SPEnominalalgorithm", + "SPECombinedalgorithm", +] diff --git a/src/nectarchain/makers/component/spe/parameters.py b/src/nectarchain/makers/component/spe/parameters.py index 28d384a3..65280b3a 100644 --- a/src/nectarchain/makers/component/spe/parameters.py +++ b/src/nectarchain/makers/component/spe/parameters.py @@ -1,12 +1,12 @@ +import copy import logging +import astropy.units as u +import numpy as np + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -import copy - -import astropy.units as u -import numpy as np __all__ = ["Parameter", "Parameters"] @@ -42,7 +42,11 @@ def from_instance(cls, parameter): ) def __str__(self): - return f"name : {self.__name}, value : {self.__value}, error : {self.__error}, unit : {self.__unit}, min : {self.__min}, max : {self.__max},frozen : {self.__frozen}" + return ( + f"name : {self.__name}, value : {self.__value}, error : {self.__error}," + f"unit : {self.__unit}, min : {self.__min}, max : {self.__max}," + f"frozen : {self.__frozen}" + ) @property def name(self): diff --git a/src/nectarchain/makers/component/spe/parameters_SPECombined_fromHHVFit.yaml b/src/nectarchain/makers/component/spe/parameters_SPECombined_fromHHVFit.yaml index 62515304..aeb19cd6 100644 --- a/src/nectarchain/makers/component/spe/parameters_SPECombined_fromHHVFit.yaml +++ b/src/nectarchain/makers/component/spe/parameters_SPECombined_fromHHVFit.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : .NAN, max : .NAN }, @@ -20,7 +20,7 @@ max : .NAN }, n : { - value : 0.697, + value : 0.713, min : .NAN, max : .NAN }, diff --git a/src/nectarchain/makers/component/spe/parameters_SPEHHV.yaml b/src/nectarchain/makers/component/spe/parameters_SPEHHV.yaml index 89ece153..5ece2735 100644 --- a/src/nectarchain/makers/component/spe/parameters_SPEHHV.yaml +++ b/src/nectarchain/makers/component/spe/parameters_SPEHHV.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : 0.2, max : 0.8 }, @@ -20,7 +20,7 @@ max : 0.7 }, n : { - value : 0.697, + value : 0.713, min : 0.5, max : 0.9 }, diff --git a/src/nectarchain/makers/component/spe/parameters_SPEHHVStd.yaml b/src/nectarchain/makers/component/spe/parameters_SPEHHVStd.yaml index 12746d7f..c9989f1f 100644 --- a/src/nectarchain/makers/component/spe/parameters_SPEHHVStd.yaml +++ b/src/nectarchain/makers/component/spe/parameters_SPEHHVStd.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : .NAN, max : .NAN }, @@ -20,7 +20,7 @@ max : 0.7 }, n : { - value : 0.697, + value : 0.713, min : .NAN, max : .NAN }, diff --git a/src/nectarchain/makers/component/spe/parameters_SPEnominal.yaml b/src/nectarchain/makers/component/spe/parameters_SPEnominal.yaml index 18b6ee31..914a8b15 100644 --- a/src/nectarchain/makers/component/spe/parameters_SPEnominal.yaml +++ b/src/nectarchain/makers/component/spe/parameters_SPEnominal.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : 0.2, max : 0.8 }, @@ -20,7 +20,7 @@ max : 0.7 }, n : { - value : 0.697, + value : 0.713, min : 0.5, max : 0.9 }, diff --git a/src/nectarchain/makers/component/spe/parameters_SPEnominalStd.yaml b/src/nectarchain/makers/component/spe/parameters_SPEnominalStd.yaml index 3be2f77f..6c87ecae 100644 --- a/src/nectarchain/makers/component/spe/parameters_SPEnominalStd.yaml +++ b/src/nectarchain/makers/component/spe/parameters_SPEnominalStd.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : .NAN, max : .NAN }, @@ -20,7 +20,7 @@ max : 0.7 }, n : { - value : 0.697, + value : 0.713, min : .NAN, max : .NAN }, diff --git a/src/nectarchain/makers/component/spe/parameters_signal_combined.yaml b/src/nectarchain/makers/component/spe/parameters_signal_combined.yaml index a04fc05a..080bb852 100644 --- a/src/nectarchain/makers/component/spe/parameters_signal_combined.yaml +++ b/src/nectarchain/makers/component/spe/parameters_signal_combined.yaml @@ -15,7 +15,7 @@ max : 5.0 }, pp : { - value : 0.45, + value : 0.454, min : .NAN, max : .NAN }, @@ -25,7 +25,7 @@ max : 0.7 }, n : { - value : 0.697, + value : 0.713, min : .NAN, max : .NAN }, diff --git a/src/nectarchain/makers/component/spe/spe_algorithm.py b/src/nectarchain/makers/component/spe/spe_algorithm.py index 0be6345f..03eda6a8 100644 --- a/src/nectarchain/makers/component/spe/spe_algorithm.py +++ b/src/nectarchain/makers/component/spe/spe_algorithm.py @@ -25,7 +25,7 @@ from ....data.container import ChargesContainer, SPEfitContainer from ....utils import MPE2, MeanValueError, Statistics, UtilsMinuit, weight_gaussian -from ..chargesComponent import ChargesComponent +from ..charges_component import ChargesComponent from .parameters import Parameter, Parameters mplstyle.use("fast") @@ -71,9 +71,7 @@ def __exit__(self, type, value, traceback): def init_processes( _class, minuitParameters_array: np.ndarray, charge: np.ndarray, counts: np.ndarray ): - """ - Initialize each process in the process pool with global variable fit_array. - """ + """Initialize each process in the process pool with global variable fit_array.""" global _minuitParameters_array global _charge global _counts @@ -168,8 +166,7 @@ def npixels(self): # methods def read_param_from_yaml(self, parameters_file, only_update=False) -> None: - """ - Reads parameters from a YAML file and updates the internal parameters of the + """Reads parameters from a YAML file and updates the internal parameters of the FlatFieldSPEMaker class. Parameters @@ -210,9 +207,8 @@ def read_param_from_yaml(self, parameters_file, only_update=False) -> None: def _update_parameters( parameters: Parameters, charge: np.ndarray, counts: np.ndarray, **kwargs ) -> Parameters: - """ - Update the parameters of the FlatFieldSPEMaker class based on the input charge - and counts data. + """Update the parameters of the FlatFieldSPEMaker class based on the input + charge and counts data. Parameters ---------- @@ -269,8 +265,7 @@ def _update_parameters( def _get_mean_gaussian_fit( charge: np.ndarray, counts: np.ndarray, pixel_id=None, **kwargs ) -> Tuple[np.ndarray, np.ndarray]: - """ - Perform a Gaussian fit on the data to determine the pedestal and mean values. + """Perform a Gaussian fit on the data to determine the pedestal and mean values. Parameters ---------- @@ -465,8 +460,7 @@ def __init__( parent=None, **kwargs, ) -> None: - """ - Initializes the FlatFieldSingleHHVSPEMaker object. + """Initializes the FlatFieldSingleHHVSPEMaker object. Parameters ---------- @@ -495,9 +489,8 @@ def __init__( def create_from_chargesContainer( cls, signal: ChargesContainer, config=None, parent=None, **kwargs ): - """ - Creates an instance of FlatFieldSingleHHVSPEMaker using charge and counts data - from a ChargesContainer object. + """Creates an instance of FlatFieldSingleHHVSPEMaker using charge and counts + data from a ChargesContainer object. Parameters ---------- @@ -524,39 +517,30 @@ def create_from_chargesContainer( # getters and setters @property def charge(self): - """ - Returns a deep copy of the ``__charge`` attribute. - """ + """Returns a deep copy of the ``__charge`` attribute.""" return copy.deepcopy(self.__charge) @property def _charge(self): - """ - Returns the ``__charge`` attribute. - """ + """Returns the ``__charge`` attribute.""" return self.__charge @property def counts(self): - """ - Returns a deep copy of the ``__counts`` attribute. - """ + """Returns a deep copy of the ``__counts`` attribute.""" return copy.deepcopy(self.__counts) @property def _counts(self): - """ - Returns the ``__counts`` attribute. - """ + """Returns the ``__counts`` attribute.""" return self.__counts # methods def _fill_results_table_from_dict( self, dico: dict, pixels_id: np.ndarray, return_fit_array: bool = True ) -> None: - """ - Populates the results table with fit values and errors for each pixel based on - the dictionary provided as input. + """Populates the results table with fit values and errors for each pixel based + on the dictionary provided as input. Parameters ---------- @@ -627,9 +611,8 @@ def _NG_Likelihood_Chi2( counts: np.ndarray, **kwargs, ): - """ - Calculates the chi-square value using the MPE2 function. - The different parameters are explained in `Caroff et al. (2019) `_. + """Calculates the chi-square value using the MPE2 function. The different + parameters are explained in `Caroff et al. (2019) `_. .. _CAROFF: https://ui.adsabs.harvard.edu/abs/2019SPIE11119E..1WC @@ -681,8 +664,7 @@ def _NG_Likelihood_Chi2( def _make_minuitParameters_array_from_parameters( self, pixels_id: np.ndarray = None, **kwargs ) -> np.ndarray: - """ - Create an array of Minuit fit instances based on the parameters and data for + """Create an array of Minuit fit instances based on the parameters and data for each pixel. Parameters @@ -705,7 +687,7 @@ def _make_minuitParameters_array_from_parameters( for i, _id in enumerate(pixels_id): index = np.where(self.pixels_id == _id)[0][0] - parameters = __class__._update_parameters( + parameters = self._update_parameters( self.parameters, self._charge[index].data[~self._charge[index].mask], self._counts[index].data[~self._charge[index].mask], @@ -726,8 +708,7 @@ def _make_minuitParameters_array_from_parameters( @staticmethod def run_fit(i: int, tol: float) -> dict: - """ - Perform a fit on a specific pixel using the Minuit package. + """Perform a fit on a specific pixel using the Minuit package. Parameters ---------- @@ -964,10 +945,10 @@ def plot_single_matplotlib( pedestalWidth: float, luminosity: float, likelihood: float, + **kwargs, ) -> tuple: - """ - Generate a plot of the data and a model fit for a specific pixel. - The different parameters are explained in `Caroff et al. (2019) `_. + """Generate a plot of the data and a model fit for a specific pixel. The + different parameters are explained in `Caroff et al. (2019) `_. .. _CAROFF: https://ui.adsabs.harvard.edu/abs/2019SPIE11119E..1WC @@ -1002,9 +983,12 @@ def plot_single_matplotlib( ------- : tuple A tuple containing the generated plot figure and the axes of the plot. - """ - fig, ax = plt.subplots(1, 1, figsize=(8, 8)) + if kwargs.get("ax", False) and kwargs.get("fig", False): + fig = kwargs.get("fig") + ax = kwargs.get("ax") + else: + fig, ax = plt.subplots(1, 1, figsize=(8, 8)) ax.errorbar(charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data") ax.plot( charge, @@ -1032,8 +1016,7 @@ def plot_single_matplotlib( return fig, ax def display(self, pixels_id: np.ndarray, package="pyqtgraph", **kwargs) -> None: - """ - Display and save the plot for each specified pixel ID. + """Display and save the plot for each specified pixel ID. Parameters ---------- @@ -1109,9 +1092,7 @@ def display(self, pixels_id: np.ndarray, package="pyqtgraph", **kwargs) -> None: class SPEHHValgorithm(SPEnominalalgorithm): - """ - Class to perform fit of the SPE HHV signal with ``n`` and ``pp`` free. - """ + """Class to perform fit of the SPE HHV signal with ``n`` and ``pp`` free.""" parameters_file = Unicode( "parameters_SPEHHV.yaml", @@ -1119,14 +1100,14 @@ class SPEHHValgorithm(SPEnominalalgorithm): help="The name of the SPE fit parameters file", ).tag(config=True) tol = Float( - 1e40, + 1e5, help="The tolerance used for minuit", read_only=True, ).tag(config=True) class SPEnominalStdalgorithm(SPEnominalalgorithm): - """Class to perform fit of the SPE signal with ``n`` and ``pp`` fixed""" + """Class to perform fit of the SPE signal with ``n`` and ``pp`` fixed.""" parameters_file = Unicode( "parameters_SPEnominalStd.yaml", @@ -1143,8 +1124,7 @@ def __init__( parent=None, **kwargs, ) -> None: - """ - Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. + """Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. Parameters ---------- @@ -1168,10 +1148,8 @@ def __init__( self.__fix_parameters() def __fix_parameters(self) -> None: - """ - Fixes the values of the ``n`` and ``pp`` parameters by setting their frozen - attribute to True. - """ + """Fixes the values of the ``n`` and ``pp`` parameters by setting their frozen + attribute to True.""" self.log.info("updating parameters by fixing pp and n") pp = self._parameters["pp"] pp.frozen = True @@ -1186,7 +1164,7 @@ class SPEHHVStdalgorithm(SPEnominalStdalgorithm): help="The name of the SPE fit parameters file", ).tag(config=True) tol = Float( - 1e40, + 1e5, help="The tolerance used for minuit", read_only=True, ).tag(config=True) @@ -1200,7 +1178,7 @@ class SPECombinedalgorithm(SPEnominalalgorithm): ).tag(config=True) tol = Float( - 1e5, + 1e-1, help="The tolerance used for minuit", read_only=True, ).tag(config=True) @@ -1224,8 +1202,7 @@ def __init__( parent=None, **kwargs, ) -> None: - """ - Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. + """Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. Parameters ---------- @@ -1248,7 +1225,7 @@ def __init__( ) self.__fix_parameters() - self._nectarGainSPEresult = SPEfitContainer.from_hdf5(self.SPE_result) + self._nectarGainSPEresult = next(SPEfitContainer.from_hdf5(self.SPE_result)) if ( len( pixels_id[ @@ -1268,8 +1245,7 @@ def __init__( ) def __fix_parameters(self) -> None: - """ - Fixes the parameters ``n``, ``pp``, ``res``, and possibly ``luminosity``. + """Fixes the parameters ``n``, ``pp``, ``res``, and possibly ``luminosity``. Parameters ---------- @@ -1288,10 +1264,11 @@ def __fix_parameters(self) -> None: luminosity = self._parameters["luminosity"] luminosity.frozen = True - def _make_fit_array_from_parameters(self, pixels_id=None, **kwargs): - """ - Generates the fit array from the fixed parameters and the fitted data obtained - from a 1400V run. + def _make_minuitParameters_array_from_parameters( + self, pixels_id: np.ndarray = None, **kwargs + ) -> np.ndarray: + """Generates the fit array from the fixed parameters and the fitted data + obtained from a 1400V run. Parameters ---------- @@ -1305,7 +1282,7 @@ def _make_fit_array_from_parameters(self, pixels_id=None, **kwargs): : array-like The fit array. """ - return super()._make_fit_array_from_parameters( + return super()._make_minuitParameters_array_from_parameters( pixels_id=pixels_id, nectarGainSPEresult=self._nectarGainSPEresult, **kwargs, @@ -1320,9 +1297,8 @@ def _update_parameters( nectarGainSPEresult: QTable, **kwargs, ): - """ - Updates the parameters with the fixed values from the fitted data obtained from - a 1400V run. + """Updates the parameters with the fixed values from the fitted data obtained + from a 1400V run. Parameters ---------- @@ -1354,9 +1330,9 @@ def _update_parameters( index = np.where(pixel_id == nectarGainSPEresult.pixels_id)[0][0] - resolution.value = nectarGainSPEresult.resolution[index] - pp.value = nectarGainSPEresult.pp[index] - n.value = nectarGainSPEresult.n[index]["n"] + resolution.value = nectarGainSPEresult.resolution[index][0] + pp.value = nectarGainSPEresult.pp[index][0] + n.value = nectarGainSPEresult.n[index][0] if luminosity.frozen: luminosity.value = nectarGainSPEresult.luminosity[index].value diff --git a/src/nectarchain/makers/component/waveformsComponent.py b/src/nectarchain/makers/component/waveforms_component.py similarity index 92% rename from src/nectarchain/makers/component/waveformsComponent.py rename to src/nectarchain/makers/component/waveforms_component.py index 74e13237..0127b5ab 100644 --- a/src/nectarchain/makers/component/waveformsComponent.py +++ b/src/nectarchain/makers/component/waveforms_component.py @@ -1,11 +1,5 @@ -import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - - import copy +import logging from argparse import ArgumentError import numpy as np @@ -18,6 +12,11 @@ from ...data.container import WaveformsContainer, WaveformsContainers from .core import ArrayDataComponent +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["WaveformsComponent"] @@ -47,7 +46,8 @@ def create_from_events_list( """Create a container for the extracted waveforms from a list of events. Args: - events_list (list[NectarCAMDataContainer]): A list of events to extract waveforms from. + events_list (list[NectarCAMDataContainer]): A list of events to extract + waveforms from. run_number (int): The ID of the run to be loaded. npixels (int): The number of pixels in the waveforms. nsamples (int): The number of samples in the waveforms. @@ -55,7 +55,8 @@ def create_from_events_list( pixels_id (int): The ID of the pixels to extract waveforms from. Returns: - WaveformsContainer: A container object that contains the extracted waveforms and other relevant information. + WaveformsContainer: A container object that contains the extracted waveforms + and other relevant information. """ if tel_id is None: tel_id = __class__.TEL_ID.default_value @@ -116,7 +117,6 @@ def _init_trigger_type(self, trigger_type: EventType, **kwargs): Args: trigger_type: The type of trigger. - """ super()._init_trigger_type(trigger_type, **kwargs) name = __class__._get_name_trigger(trigger_type) @@ -130,9 +130,9 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs): """Process an event and extract waveforms. Args: - event (NectarCAMDataContainer): The event to process and extract waveforms from. + event (NectarCAMDataContainer): The event to process and extract waveforms + from. trigger (EventType): The type of trigger for the event. - """ wfs_hg_tmp = np.zeros((self.npixels, self.nsamples), dtype=np.uint16) wfs_lg_tmp = np.zeros((self.npixels, self.nsamples), dtype=np.uint16) @@ -158,7 +158,8 @@ def finish(self, *args, **kwargs): trigger_type (EventType): The selected trigger types. Returns: - list[WaveformsContainer]: A list of output containers for the selected trigger types. + list[WaveformsContainer]: A list of output containers for the selected + trigger types. """ output = WaveformsContainers() for i, trigger in enumerate(self.trigger_list): @@ -195,7 +196,8 @@ def sort(waveformsContainer: WaveformsContainer, method: str = "event_id"): """Sort the waveformsContainer based on a specified method. Args: - waveformsContainer (WaveformsContainer): The waveformsContainer to be sorted. + waveformsContainer (WaveformsContainer): The waveformsContainer + to be sorted. method (str, optional): The sorting method. Defaults to 'event_id'. Returns: @@ -230,13 +232,16 @@ def sort(waveformsContainer: WaveformsContainer, method: str = "event_id"): @staticmethod def select_waveforms_hg( - waveformsContainer: WaveformsContainer, pixel_id: np.ndarray + waveformsContainer: WaveformsContainer, + pixel_id: np.ndarray, ): """Select HIGH GAIN waveforms from the container. Args: - waveformsContainer (WaveformsContainer): The container object that contains the waveforms. - pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms from the container. + waveformsContainer (WaveformsContainer): The container object that contains + the waveforms. + pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms + from the container. Returns: np.ndarray: An array of selected waveforms from the container. @@ -254,8 +259,10 @@ def select_waveforms_lg( """Select LOW GAIN waveforms from the container. Args: - waveformsContainer (WaveformsContainer): The container object that contains the waveforms. - pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms from the container. + waveformsContainer (WaveformsContainer): The container object that contains + the waveforms. + pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms + from the container. Returns: np.ndarray: An array of selected waveforms from the container. @@ -268,8 +275,7 @@ def select_waveforms_lg( @property def _geometry(self): - """ - Returns the private __geometry attribute of the WaveformsMaker class. + """Returns the private __geometry attribute of the WaveformsMaker class. :return: The value of the private __geometry attribute. """ @@ -277,8 +283,7 @@ def _geometry(self): @property def geometry(self): - """ - Returns a deep copy of the geometry attribute. + """Returns a deep copy of the geometry attribute. Returns: A deep copy of the geometry attribute. @@ -286,11 +291,11 @@ def geometry(self): return copy.deepcopy(self.__geometry) def wfs_hg(self, trigger: EventType): - """ - Returns the waveform data for the specified trigger type. + """Returns the waveform data for the specified trigger type. Args: - trigger (EventType): The type of trigger for which the waveform data is requested. + trigger (EventType): The type of trigger for which the waveform data is + requested. Returns: An array of waveform data for the specified trigger type. @@ -301,14 +306,16 @@ def wfs_hg(self, trigger: EventType): ) def wfs_lg(self, trigger: EventType): - """ - Returns the waveform data for the specified trigger type in the low gain channel. + """Returns the waveform data for the specified trigger type in the low gain + channel. Args: - trigger (EventType): The type of trigger for which the waveform data is requested. + trigger (EventType): The type of trigger for which the waveform data is + requested. Returns: - An array of waveform data for the specified trigger type in the low gain channel. + An array of waveform data for the specified trigger type in the low gain + channel. """ return np.array( self.__wfs_lg[__class__._get_name_trigger(trigger)], diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index cdcb02f9..af45c8cc 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -18,12 +18,13 @@ ) from ctapipe.io import HDF5TableWriter from ctapipe.io.datawriter import DATA_MODEL_VERSION +from ctapipe_io_nectarcam import LightNectarCAMEventSource +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer from tables.exceptions import HDF5ExtError from tqdm.auto import tqdm from traitlets import default from ..data import DataManagement -from ..data.container import LightNectarCAMEventSource from ..data.container.core import NectarCAMContainer, TriggerMapContainer from ..utils import ComponentUtils from .component import NectarCAMComponent, get_valid_component @@ -433,7 +434,9 @@ def start( self._setup_components() n_events_in_slice = 0 - def split_run(self, n_events_in_slice, event): + def split_run( + self, n_events_in_slice: int = None, event: NectarCAMDataContainer = None + ): """Method to decide if criteria to end a run slice are met""" condition = ( self.events_per_slice is not None @@ -468,7 +471,7 @@ def _write_container(self, container: Container, index_component: int = 0) -> No container.validate() if isinstance(container, NectarCAMContainer): self.writer.write( - table_name=str(container.__class__.__name__), + table_name=f"{container.__class__.__name__}_{index_component}", containers=container, ) elif isinstance(container, TriggerMapContainer): @@ -487,7 +490,8 @@ def _write_container(self, container: Container, index_component: int = 0) -> No log.warning(e, exc_info=True) self.log.warning("the container has not been written") except Exception as e: - self.log.error(e, exc_info=True) + log.error(e, exc_info=True) + self.log.error(e.args[0], exc_info=True) raise e @property @@ -550,7 +554,9 @@ class DelimiterLoopNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def split_run(self, n_events_in_slice, event): + def split_run( + self, n_events_in_slice: int = None, event: NectarCAMDataContainer = None + ): """Method to decide if criteria to end a run slice is met""" condition = event.trigger.event_type == EventType.UNKNOWN return condition diff --git a/src/nectarchain/makers/extractor/charge_extractor.py b/src/nectarchain/makers/extractor/charge_extractor.py index 670234c5..e34bfdef 100644 --- a/src/nectarchain/makers/extractor/charge_extractor.py +++ b/src/nectarchain/makers/extractor/charge_extractor.py @@ -28,12 +28,12 @@ def __call__(self, waveforms, telid, selected_gain_channel, substract_ped=False) shape[0] * shape[1], 1 ) - y = waveforms - (ped_mean) @ ( + _ = waveforms - (ped_mean) @ ( np.ones(1, shape[2]) ) # waveforms without pedestal else: log.info("do not substract pedestal") - y = waveforms + _ = waveforms waveforms.reshape(shape[0], shape[1], shape[2]) @@ -80,15 +80,17 @@ def extract_charge(y, height_peak, fixed_window): xi[peaks[max_peak_index]] < 40 ): # Search the adaptive integration window # calculate total gradients (not used, only for plot) - yi_grad_tot = np.gradient(yi, 1) + _ = np.gradient(yi, 1) maxposition = peaks[max_peak_index] - # calcualte grandients starting from the max peak and going to the left to find the left margin of the window + # calcualte grandients starting from the max peak and going to the left to + # find the left margin of the window yi_left = yi[:maxposition] yi_grad_left = np.gradient(yi_left[::-1], 0.9) change_grad_pos_left = ( np.where(yi_grad_left[:-1] * yi_grad_left[1:] < 0)[0] + 1 )[0] - # calcualte grandients starting from the max peak and going to the right to find the right margin of the window + # calcualte grandients starting from the max peak and going to the right to + # find the right margin of the window yi_right = yi[maxposition:] yi_grad_right = np.gradient(yi_right, 0.5) change_grad_pos_right = ( diff --git a/src/nectarchain/makers/extractor/utils.py b/src/nectarchain/makers/extractor/utils.py index 0fc1804a..4682d216 100644 --- a/src/nectarchain/makers/extractor/utils.py +++ b/src/nectarchain/makers/extractor/utils.py @@ -1,11 +1,11 @@ import logging +from ctapipe.containers import DL1CameraContainer + logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) log.handlers = logging.getLogger("__main__").handlers -from ctapipe.containers import DL1CameraContainer - class CtapipeExtractor: """ @@ -17,7 +17,8 @@ def get_image_peak_time(cameraContainer: DL1CameraContainer): Extracts the image and peak time from a DL1CameraContainer object. Parameters: - cameraContainer (DL1CameraContainer): The DL1CameraContainer object to extract the image and peak time from. + cameraContainer (DL1CameraContainer): The DL1CameraContainer object to extract + the image and peak time from. Returns: tuple: A tuple containing the image and peak time values from the container. diff --git a/src/nectarchain/makers/tests/test_charges_makers.py b/src/nectarchain/makers/tests/test_charges_makers.py index 88aa8c82..2a68b2a0 100644 --- a/src/nectarchain/makers/tests/test_charges_makers.py +++ b/src/nectarchain/makers/tests/test_charges_makers.py @@ -1,198 +1,221 @@ -# import logging -# -# import numpy as np -# from ctapipe.containers import EventType -# -# from nectarchain.data.container import ChargesContainer, ChargesContainerIO -# from nectarchain.makers import ChargesMaker, WaveformsMaker -# -# logging.basicConfig( -# format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG -# ) -# log = logging.getLogger(__name__) -# log.handlers = logging.getLogger("__main__").handlers -# -# -# class TestChargesMaker: -# run_number = 3938 -# max_events = 100 -# -# def test_instance(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# assert isinstance(chargesMaker, ChargesMaker) -# -# def test_shape_valid(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer = chargesMaker.make()[0] -# -# assert chargesContainer.nevents <= TestChargesMaker.max_events -# assert chargesContainer.run_number == TestChargesMaker.run_number -# assert chargesContainer.ucts_timestamp.shape == (chargesContainer.nevents,) -# assert chargesContainer.ucts_busy_counter.shape == (chargesContainer.nevents,) -# assert (chargesContainer.ucts_event_counter.shape == -# (chargesContainer.nevents,)) -# assert chargesContainer.event_type.shape == (chargesContainer.nevents,) -# assert chargesContainer.event_id.shape == (chargesContainer.nevents,) -# assert chargesContainer.trig_pattern_all.shape[0] == chargesContainer.nevents -# assert chargesContainer.trig_pattern_all.shape[2] == 4 -# -# assert chargesContainer.trig_pattern.shape[0] == chargesContainer.nevents -# assert chargesContainer.multiplicity.shape == (chargesContainer.nevents,) -# -# assert chargesContainer.charges_hg.mean() != 0 -# assert chargesContainer.charges_lg.mean() != 0 -# assert chargesContainer.peak_hg.mean() != 0 -# assert chargesContainer.peak_lg.mean() != 0 -# assert chargesContainer.charges_hg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# assert chargesContainer.charges_lg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# assert chargesContainer.peak_hg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# assert chargesContainer.peak_lg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# -# assert chargesContainer.broken_pixels_hg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# assert chargesContainer.broken_pixels_lg.shape == ( -# chargesContainer.nevents, -# chargesContainer.npixels, -# ) -# -# def test_make_restart_eventsource(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make(restart_from_begining=True) -# assert isinstance(chargesContainer_list[0], ChargesContainer) -# -# def test_make_LocalPeakWindowSum(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make( -# method="LocalPeakWindowSum", window_shift=-4, window_length=16 -# ) -# assert isinstance(chargesContainer_list[0], ChargesContainer) -# -# def test_all_multiple_trigger(self): -# trigger1 = EventType.FLATFIELD -# trigger2 = EventType.SKY_PEDESTAL -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make(trigger_type=[trigger1, trigger2]) -# for chargesContainer in chargesContainer_list: -# assert isinstance(chargesContainer, ChargesContainer) -# -# def test_all_trigger_None(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make() -# assert isinstance(chargesContainer_list[0], ChargesContainer) -# -# def test_create_from_waveforms(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make() -# chargesContainer = ChargesMaker.create_from_waveforms( -# waveformsContainer_list[0], -# method="LocalPeakWindowSum", -# window_shift=-4, -# window_length=16, -# ) -# assert isinstance(chargesContainer, ChargesContainer) -# -# def test_select_charges(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make() -# pixel_id = np.array([3, 67, 87]) -# assert isinstance( -# ChargesMaker.select_charges_hg(chargesContainer_list[0], pixel_id), -# np.ndarray, -# ) -# assert isinstance( -# ChargesMaker.select_charges_lg(chargesContainer_list[0], pixel_id), -# np.ndarray, -# ) -# -# def test_histo(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make() -# histo = ChargesMaker.histo_hg(chargesContainer_list[0]) -# assert isinstance(histo, np.ndarray) -# assert histo.mean() != 0 -# assert histo.shape[0] == 2 -# assert histo.shape[1] == chargesContainer_list[0].npixels -# histo = ChargesMaker.histo_lg(chargesContainer_list[0]) -# assert isinstance(histo, np.ndarray) -# assert histo.mean() != 0 -# assert histo.shape[0] == 2 -# assert histo.shape[1] == chargesContainer_list[0].npixels -# -# def test_sort_ChargesContainer(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make() -# sortWfs = ChargesMaker.sort(chargesContainer_list[0], method="event_id") -# assert np.array_equal( -# sortWfs.event_id, np.sort(chargesContainer_list[0].event_id) -# ) -# -# def test_write_load_container(self): -# chargesMaker = ChargesMaker( -# run_number=TestChargesMaker.run_number, -# max_events=TestChargesMaker.max_events, -# ) -# chargesContainer_list = chargesMaker.make() -# ChargesContainerIO.write( -# "/tmp/test_charge_container/", chargesContainer_list[0], overwrite=True -# ) -# loaded_charge = ChargesContainerIO.load( -# "/tmp/test_charge_container", run_number=TestChargesMaker.run_number -# ) -# assert np.array_equal( -# chargesContainer_list[0].charges_hg, loaded_charge.charges_hg -# ) -# -# -# if __name__ == "__main__": -# import logging -# -# logging.basicConfig( -# format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG -# ) -# log = logging.getLogger(__name__) -# log.handlers = logging.getLogger("__main__").handlers -# TestChargesMaker().test_write_load_container() +import tempfile +from pathlib import Path + +import numpy as np +from ctapipe.containers import EventType +from ctapipe.utils import get_dataset_path + +from nectarchain.data.container import ChargesContainer, ChargesContainers +from nectarchain.makers import ChargesNectarCAMCalibrationTool + +# This test file test the overall workflow of the ChargesNectarCAMCalibrationTool, +# adapted from the test of the WaveformsNectarCAMCalibrationTool. +# There are lot of code duplication but I don't care for now. Do you ? + + +class TestChargesNectarCAMCalibrationTool: + RUNS = { + "Run number": [3938, 5288], + "Run file": [ + get_dataset_path("NectarCAM.Run3938.30events.fits.fz"), + get_dataset_path("NectarCAM.Run5288.0001.fits.fz"), + ], + "nevents": [30, 13], + "N pixels": [1834, 1848], + "eventType": EventType.SKY_PEDESTAL, + "charges_lg_min": [957, 973], + "charges_lg_max": [2059, 2113], + "charges_lg_mean": [19067, 19306], + "charges_lg_std": [2273, 2376], + "charges_hg_min": [963, 972], + "charges_hg_max": [2057, 2109], + "charges_hg_mean": [19106, 19275], + "charges_hg_std": [2139, 2357], + "expected_ucts_timestamp_min": [1674462932637854793, 1715007113924900896], + "expected_ucts_timestamp_max": [1674462932695877994, 1715007123524920096], + } + OVERWRITE = True + METHOD = "LocalPeakWindowSum" + EXTRACTOR_KWARGS = {"window_shift": 4, "window_width": 8} + + def general_structure_testing( + self, output: ChargesContainer, nevents: int, n_pixels: int, run_number: int + ): + assert isinstance(output.pixels_id, np.ndarray) + assert output.pixels_id.dtype == np.uint16 + assert np.shape(output.pixels_id) == (n_pixels,) + assert output.run_number == run_number + assert output.camera == "NectarCam-003" + assert output.npixels == n_pixels + assert isinstance(output.ucts_busy_counter, np.ndarray) + assert output.ucts_busy_counter.dtype == np.uint32 + assert isinstance(output.ucts_event_counter, np.ndarray) + assert output.ucts_event_counter.dtype == np.uint32 + assert isinstance(output.event_type, np.ndarray) + assert output.event_type.dtype == np.uint8 + assert np.all(output.event_type == self.RUNS["eventType"].value) + assert isinstance(output.trig_pattern, np.ndarray) + assert output.trig_pattern.dtype == bool + assert isinstance(output.trig_pattern_all, np.ndarray) + assert output.trig_pattern_all.dtype == bool + assert isinstance(output.multiplicity, np.ndarray) + assert output.multiplicity.dtype == np.uint16 + assert isinstance(output.ucts_timestamp, np.ndarray) + assert output.ucts_timestamp.dtype == np.uint64 + + assert isinstance(output.event_id, np.ndarray) + assert output.event_id.dtype == np.uint32 + assert isinstance(output.broken_pixels_hg, np.ndarray) + assert output.broken_pixels_hg.dtype == bool + assert output.broken_pixels_hg.shape == (nevents, n_pixels) + assert isinstance(output.broken_pixels_lg, np.ndarray) + assert output.broken_pixels_lg.dtype == bool + assert output.broken_pixels_lg.shape == (nevents, n_pixels) + assert output.charges_hg.shape == (nevents, n_pixels) + assert output.charges_lg.shape == (nevents, n_pixels) + assert isinstance(output.charges_hg, np.ndarray) + assert isinstance(output.charges_lg, np.ndarray) + assert output.charges_hg.dtype == np.uint16 + assert output.charges_lg.dtype == np.uint16 + assert output.peak_hg.shape == (nevents, n_pixels) + assert output.peak_lg.shape == (nevents, n_pixels) + assert isinstance(output.peak_hg, np.ndarray) + assert isinstance(output.peak_lg, np.ndarray) + assert output.peak_hg.dtype == np.uint16 + assert output.peak_lg.dtype == np.uint16 + assert isinstance(output.method, str) + assert output.method == self.METHOD + + def test_base(self): + """ + Test basic functionality, including IO on disk + """ + + events_per_slice = [None, None, 10, 11, 8] + max_events = [None, 10, None, None, 10] + + for _max_events, _events_per_slice in zip(max_events, events_per_slice): + for i, run_number in enumerate(self.RUNS["Run number"]): + run_file = self.RUNS["Run file"][i] + n_pixels = self.RUNS["N pixels"][i] + with tempfile.TemporaryDirectory() as tmpdirname: + outfile = tmpdirname + "/charges.h5" + + # run tool + tool = ChargesNectarCAMCalibrationTool( + run_number=run_number, + run_file=run_file, + max_events=_max_events, + events_per_slice=_events_per_slice, + log_level=0, + output_path=outfile, + overwrite=self.OVERWRITE, + method=self.METHOD, + extractor_kwargs=self.EXTRACTOR_KWARGS, + ) + + tool.setup() + nevents = len(tool.event_source) + assert ( + nevents == self.RUNS["nevents"][i] + if _max_events is None + else _max_events + ) + tool.start() + output_containers = tool.finish(return_output_component=True)[0] + assert isinstance(output_containers, ChargesContainers) + output = output_containers.containers[self.RUNS["eventType"]] + assert isinstance(output, ChargesContainer) + # Check output in memory + if ( + _events_per_slice is not None + and nevents % _events_per_slice == 0 + ): + assert output.nevents is None + else: + if _events_per_slice is None: + assert ( + output.nevents == nevents + ) # nevents has been validated before + else: + assert output.nevents == nevents % _events_per_slice + + self.general_structure_testing( + output, + ( + nevents + if _events_per_slice is None + else nevents % _events_per_slice + ), + n_pixels, + run_number, + ) + + if _events_per_slice is None and _max_events is None: + # content only checked for the full run + assert np.min(output.ucts_timestamp) == np.uint64( + self.RUNS["expected_ucts_timestamp_min"][i] + ) + assert np.max(output.ucts_timestamp) == np.uint64( + self.RUNS["expected_ucts_timestamp_max"][i] + ) + assert ( + output.charges_lg.min() + == self.RUNS["charges_lg_min"][i] + ) + assert ( + output.charges_lg.max() + == self.RUNS["charges_lg_max"][i] + ) + assert ( + int(10 * output.charges_lg.mean()) + == self.RUNS["charges_lg_mean"][i] + ) + assert ( + int(10 * output.charges_lg.std()) + == self.RUNS["charges_lg_std"][i] + ) + assert ( + output.charges_hg.min() + == self.RUNS["charges_hg_min"][i] + ) + assert ( + output.charges_hg.max() + == self.RUNS["charges_hg_max"][i] + ) + assert ( + int(10 * output.charges_hg.mean()) + == self.RUNS["charges_hg_mean"][i] + ) + assert ( + int(10 * output.charges_hg.std()) + == self.RUNS["charges_hg_std"][i] + ) + + # Check output on disk + assert Path(outfile).exists() + + chargesContainers = ChargesContainers.from_hdf5(outfile) + ncontainers = 0 + for container in chargesContainers: + ncontainers += 1 + assert isinstance(container, ChargesContainers) + output = container.containers[self.RUNS["eventType"]] + if _events_per_slice is None: + expected_nevents = nevents + else: + if nevents % _events_per_slice == 0: + expected_nevents = _events_per_slice + else: + if ncontainers == 1: + expected_nevents = nevents % _events_per_slice + else: + expected_nevents = _events_per_slice + self.general_structure_testing( + output, expected_nevents, n_pixels, run_number + ) + assert ( + ncontainers == 1 + if _events_per_slice is None + else round(nevents / _events_per_slice) + ) diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py deleted file mode 100644 index 341c45f8..00000000 --- a/src/nectarchain/makers/tests/test_core.py +++ /dev/null @@ -1,43 +0,0 @@ -# import logging -# -# from nectarchain.data.container import ChargesContainer -# from nectarchain.makers import ArrayDataMaker, ChargesMaker, WaveformsMaker -# -# logging.basicConfig( -# format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG -# ) -# log = logging.getLogger(__name__) -# log.handlers = logging.getLogger("__main__").handlers -# -# -# class TestArrayDataMaker: -# run_number = 3938 -# max_events = 100 -# -# def test_merge(self): -# chargesMaker = ChargesMaker( -# run_number=TestArrayDataMaker.run_number, -# max_events=TestArrayDataMaker.max_events, -# ) -# charges_1 = chargesMaker.make() -# chargesMaker_2 = ChargesMaker( -# run_number=TestArrayDataMaker.run_number, -# max_events=TestArrayDataMaker.max_events, -# ) -# charges_2 = chargesMaker_2.make() -# -# merged = ArrayDataMaker.merge(charges_1, charges_2) -# assert isinstance(merged, ChargesContainer) -# -# def test_merge_different_container(self): -# chargesMaker = ChargesMaker( -# run_number=TestArrayDataMaker.run_number, -# max_events=TestArrayDataMaker.max_events, -# ) -# charges_1 = chargesMaker.make() -# wfsMaker_2 = WaveformsMaker( -# run_number=TestArrayDataMaker.run_number, -# max_events=TestArrayDataMaker.max_events, -# ) -# wfs_2 = wfsMaker_2.make() -# merged = ArrayDataMaker.merge(charges_1, wfs_2) diff --git a/src/nectarchain/makers/tests/test_core_makers.py b/src/nectarchain/makers/tests/test_core_makers.py new file mode 100644 index 00000000..d447ba8c --- /dev/null +++ b/src/nectarchain/makers/tests/test_core_makers.py @@ -0,0 +1,470 @@ +import logging +import os +import pathlib +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest +import traitlets +from ctapipe.containers import EventType, TriggerContainer +from ctapipe.core.container import Container +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam import LightNectarCAMEventSource +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer + +from nectarchain.data.container.core import NectarCAMContainer, TriggerMapContainer +from nectarchain.makers.component import NectarCAMComponent +from nectarchain.makers.core import ( + BaseNectarCAMCalibrationTool, + DelimiterLoopNectarCAMCalibrationTool, + EventsLoopNectarCAMCalibrationTool, +) + +logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG +) +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + +class TestBaseNectarCAMCalibrationTool: + RUN_NUMBER = 3938 + RUN_FILE = get_dataset_path("NectarCAM.Run3938.30events.fits.fz") + + def test_load_run(self): + eventsource = BaseNectarCAMCalibrationTool.load_run( + run_number=self.RUN_NUMBER, max_events=1, run_file=self.RUN_FILE + ) + assert isinstance(eventsource, LightNectarCAMEventSource) + + +class MockComponent(NectarCAMComponent): + def __init__(self, *args, **kwargs): + pass + + def __call__(self, event, *args, **kwargs): + pass + + def start(self): + pass + + def finish(self): + return [NectarCAMContainer()] + + +class TestEventsLoopNectarCAMCalibrationTool(TestBaseNectarCAMCalibrationTool): + MAX_EVENTS = 10 + EVENTS_PER_SLICE = 8 + + @pytest.fixture + def tool_instance(self): + return EventsLoopNectarCAMCalibrationTool(run_number=self.RUN_NUMBER) + + @pytest.fixture + def tool_instance_run_file(self): + return EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, + run_file=self.RUN_FILE, + output_path=pathlib.Path(f"/tmp/{np.random.random()}test_output.h5") + # to avoid I/O conflicts between tests + ) + + def test_init_output_path(self, tool_instance): + expected_path = pathlib.Path( + f"{os.environ.get('NECTARCAMDATA', '/tmp')}/runs" + f"/EventsLoopNectarCAMCalibration_run{self.RUN_NUMBER}.h5" + ) + assert tool_instance.output_path == expected_path + assert tool_instance.run_number == self.RUN_NUMBER + assert tool_instance.max_events is None + assert tool_instance.run_file is None + assert tool_instance.name == "EventsLoopNectarCAMCalibration" + assert tool_instance.events_per_slice is None + + def test_init_with_output_path(self): + custom_path = pathlib.Path("/custom/path/output.h5") + tool_instance = EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, output_path=custom_path + ) + assert tool_instance.output_path == custom_path + + def test_init_with_max_events(self): + tool_instance = EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, max_events=10 + ) + assert tool_instance.max_events == self.MAX_EVENTS + + def test_init_with_events_per_slice(self): + tool_instance = EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, events_per_slice=self.EVENTS_PER_SLICE + ) + assert tool_instance.events_per_slice == self.EVENTS_PER_SLICE + + def test_init_with_run_file(self): + tool_instance = EventsLoopNectarCAMCalibrationTool( + run_number=self.RUN_NUMBER, run_file=self.RUN_FILE + ) + assert tool_instance.run_file == self.RUN_FILE + + def test_load_eventsource(self, tool_instance_run_file): + tool_instance_run_file._load_eventsource() + assert isinstance( + tool_instance_run_file.event_source, LightNectarCAMEventSource + ) + assert tool_instance_run_file.event_source.input_url == self.RUN_FILE + + def test_load_eventsource_max_events(self, tool_instance_run_file): + tool_instance_run_file.max_events = self.MAX_EVENTS + tool_instance_run_file._load_eventsource() + assert isinstance( + tool_instance_run_file.event_source, LightNectarCAMEventSource + ) + assert tool_instance_run_file.event_source.input_url == self.RUN_FILE + assert tool_instance_run_file.event_source.max_events == self.MAX_EVENTS + + @patch("nectarchain.makers.core.HDF5TableWriter") + @patch("nectarchain.makers.core.os.remove") + @patch("nectarchain.makers.core.os.makedirs") + def test_init_writer_full_mode( + self, mock_makedirs, mock_remove, mock_writer, tool_instance + ): + tool_instance.overwrite = True + tool_instance._init_writer(sliced=False) + mock_remove.assert_called_once_with(tool_instance.output_path) + mock_makedirs.assert_called_once_with( + tool_instance.output_path.parent, exist_ok=True + ) + mock_writer.assert_called_once_with( + filename=tool_instance.output_path, + parent=tool_instance, + mode="w", + group_name="data", + ) + + @patch("nectarchain.makers.core.HDF5TableWriter") + @patch("nectarchain.makers.core.os.remove") + @patch("nectarchain.makers.core.os.makedirs") + def test_init_writer_sliced_mode( + self, mock_makedirs, mock_remove, mock_writer, tool_instance + ): + tool_instance.overwrite = True + tool_instance._init_writer(sliced=True, slice_index=1) + mock_remove.assert_not_called() + mock_makedirs.assert_called_once_with( + tool_instance.output_path.parent, exist_ok=True + ) + mock_writer.assert_called_once_with( + filename=tool_instance.output_path, + parent=tool_instance, + mode="a", + group_name="data_1", + ) + + @patch("nectarchain.makers.core.HDF5TableWriter") + @patch("nectarchain.makers.core.os.remove") + @patch("nectarchain.makers.core.os.makedirs") + def test_init_writer_overwrite_false( + self, mock_makedirs, mock_remove, mock_writer, tool_instance + ): + tool_instance.overwrite = False + with patch("nectarchain.makers.core.os.path.exists", return_value=True): + with pytest.raises(Exception): + tool_instance._init_writer(sliced=False) + mock_remove.assert_not_called() + mock_makedirs.assert_not_called() + mock_writer.assert_not_called() + + def test_setup_eventsource(self, tool_instance_run_file): + tool_instance_run_file._setup_eventsource() + + assert ( + tool_instance_run_file._npixels + == tool_instance_run_file.event_source.nectarcam_service.num_pixels + ) + assert np.all( + tool_instance_run_file._pixels_id + == tool_instance_run_file.event_source.nectarcam_service.pixel_ids + ) + assert isinstance( + tool_instance_run_file.event_source, LightNectarCAMEventSource + ) + + @patch( + "nectarchain.makers.core.ComponentUtils.get_class_name_from_ComponentName", + return_value="ValidComponentClass", + ) + @patch( + "nectarchain.makers.core.ComponentUtils.get_configurable_traits", + return_value={"trait1": "value1"}, + ) + def test_get_provided_component_kwargs( + self, mock_get_class_name, mock_get_valid_component, tool_instance + ): + tool_instance.trait1 = "value1" + output_component_kwargs = tool_instance._get_provided_component_kwargs( + "componentName" + ) + assert output_component_kwargs == {"trait1": "value1"} + + @patch("nectarchain.makers.core.Component") + @patch( + "nectarchain.makers.core.get_valid_component", + return_value=["WaveformsComponent"], + ) + @patch( + "nectarchain.makers.core.ComponentUtils.get_class_name_from_ComponentName", + return_value="WaveformsComponentClass", + ) + @patch( + "nectarchain.makers.core.ComponentUtils.get_configurable_traits", + return_value={"trait1": "value1"}, + ) + def test_setup_components( + self, + mock_get_configurable_traits, + mock_get_class_name, + mock_get_valid_component, + mock_component, + tool_instance_run_file, + ): + with pytest.raises(traitlets.traitlets.TraitError): + tool_instance_run_file.componentsList = ["ValidComponent"] + tool_instance_run_file.componentsList = ["WaveformsComponent"] + tool_instance_run_file.trait1 = "value1" + tool_instance_run_file._setup_eventsource() + tool_instance_run_file._setup_components() + mock_get_valid_component.assert_called_once() + mock_get_class_name.assert_called_once_with("WaveformsComponent") + mock_get_configurable_traits.assert_called_once_with("WaveformsComponentClass") + mock_component.from_name.assert_called_once_with( + "WaveformsComponent", + subarray=tool_instance_run_file.event_source.subarray, + parent=tool_instance_run_file, + trait1="value1", + ) + assert len(tool_instance_run_file.components) == 1 + assert ( + tool_instance_run_file.components[0] + == mock_component.from_name.return_value + ) + + @patch("nectarchain.makers.core.os.remove") + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_eventsource" + ) + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_components" + ) + @patch("nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._init_writer") + def test_setup( + self, + mock_init_writer, + mock_setup_components, + mock_setup_eventsource, + mock_remove, + tool_instance_run_file, + ): + tool_instance_run_file.overwrite = True + tool_instance_run_file.output_path = pathlib.Path("/tmp/test_output.h5") + + with patch("nectarchain.makers.core.pathlib.Path.exists", return_value=True): + tool_instance_run_file.setup() + + mock_setup_eventsource.assert_called_once() + mock_setup_components.assert_called_once() + mock_remove.assert_called_once_with(tool_instance_run_file.output_path) + mock_init_writer.assert_called_once_with(sliced=False, slice_index=1) + assert tool_instance_run_file._n_traited_events == 0 + + def test_setup_run_number_not_set(self, tool_instance): + tool_instance.run_number = -1 + with pytest.raises(Exception, match="run_number need to be set up"): + tool_instance.setup() + + def test_split_run(self, tool_instance): + event = NectarCAMDataContainer() + assert not (tool_instance.split_run(n_events_in_slice=6, event=event)) + tool_instance.events_per_slice = 4 + assert tool_instance.split_run(n_events_in_slice=6, event=event) + assert not (tool_instance.split_run(n_events_in_slice=2, event=event)) + + @patch("nectarchain.makers.core.Component") + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + def test_start(self, mock_finish_component, mock_component, tool_instance_run_file): + tool_instance_run_file.overwrite = True + tool_instance_run_file.setup() + n_events = len(tool_instance_run_file.event_source) + tool_instance_run_file.components = [mock_component.from_name.return_value] + tool_instance_run_file.start() + tool_instance_run_file.finish() + assert tool_instance_run_file._n_traited_events == n_events + + @patch("nectarchain.makers.core.Component") + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + def test_start_n_events( + self, mock_finish_component, mock_component, tool_instance_run_file + ): + tool_instance_run_file.overwrite = True + tool_instance_run_file.setup() + tool_instance_run_file.components = [mock_component.from_name.return_value] + tool_instance_run_file.start(n_events=10) + tool_instance_run_file.finish() + assert tool_instance_run_file._n_traited_events == 10 + + @patch("nectarchain.makers.core.Component") + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._setup_components" + ) + def test_start_sliced( + self, + mock_setup_components, + mock_finish_components, + mock_component, + tool_instance_run_file, + ): + tool_instance_run_file.overwrite = True + tool_instance_run_file.events_per_slice = self.EVENTS_PER_SLICE + tool_instance_run_file.setup() + n_events = len(tool_instance_run_file.event_source) + tool_instance_run_file.components = [MockComponent()] + tool_instance_run_file.start() + tool_instance_run_file.finish() + assert ( + mock_finish_components.call_count == n_events // self.EVENTS_PER_SLICE + 1 + ) + assert mock_setup_components.call_count == n_events // self.EVENTS_PER_SLICE + 1 + + @patch("nectarchain.makers.core.Component") + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + def test_finish( + self, mock_finish_components, mock_component, tool_instance_run_file + ): + tool_instance_run_file.overwrite = True + tool_instance_run_file.setup() + tool_instance_run_file.components = [mock_component.from_name.return_value] + mock_finish_components.return_value = ["output"] + + output = tool_instance_run_file.finish() + + mock_finish_components.assert_called_once() + assert output is None + assert tool_instance_run_file.writer.h5file.isopen == 0 + + @patch("nectarchain.makers.core.Component") + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + def test_finish_with_output( + self, mock_finish_components, mock_component, tool_instance_run_file + ): + tool_instance_run_file.overwrite = True + tool_instance_run_file.setup() + tool_instance_run_file.components = [mock_component.from_name.return_value] + + output = tool_instance_run_file.finish(return_output_component=True) + + mock_finish_components.assert_called_once() + assert output is not None + assert tool_instance_run_file.writer.h5file.isopen == 0 + + @patch( + "nectarchain.makers.core.EventsLoopNectarCAMCalibrationTool._finish_components" + ) + def test_finish_components(self, mock_finish_components, tool_instance_run_file): + tool_instance_run_file.overwrite = True + tool_instance_run_file.setup() + tool_instance_run_file.components = [MockComponent()] + + _ = tool_instance_run_file._finish_components() + + assert mock_finish_components.called_with([MockComponent().finish()], 0) + + @patch("nectarchain.makers.core.HDF5TableWriter") + def test_write_container_with_nectarcam_container( + self, mock_writer, tool_instance_run_file + ): + tool_instance_run_file.writer = mock_writer + container = MagicMock(spec=NectarCAMContainer) + container.validate = MagicMock() + tool_instance_run_file._write_container(container, index_component=0) + container.validate.assert_called_once() + mock_writer.write.assert_called_once_with( + table_name=f"{container.__class__.__name__}_0", + containers=container, + ) + + @patch("nectarchain.makers.core.HDF5TableWriter") + def test_write_container_with_triggermap_container( + self, mock_writer, tool_instance_run_file + ): + tool_instance_run_file.writer = mock_writer + container = MagicMock(spec=TriggerMapContainer) + container.validate = MagicMock() + container.containers = { + EventType.FLATFIELD: MagicMock(spec=Container), + EventType.UNKNOWN: MagicMock(spec=Container), + } + tool_instance_run_file._write_container(container, index_component=0) + container.validate.assert_called_once() + mock_writer.write.assert_any_call( + table_name=f"{container.containers[EventType.FLATFIELD].__class__.__name__}" + f"_0/{EventType.FLATFIELD.name}", + containers=container.containers[EventType.FLATFIELD], + ) + mock_writer.write.assert_any_call( + table_name=f"{container.containers[EventType.UNKNOWN].__class__.__name__}" + f"_0/{EventType.UNKNOWN.name}", + containers=container.containers[EventType.UNKNOWN], + ) + + @patch("nectarchain.makers.core.HDF5TableWriter") + def test_write_container_with_invalid_container( + self, mock_writer, tool_instance_run_file + ): + tool_instance_run_file.writer = mock_writer + container = MagicMock(spec=Container) + container.validate = MagicMock() + with pytest.raises( + TypeError, + match="component output must be an instance " + "of TriggerMapContainer or NectarCAMContainer", + ): + tool_instance_run_file._write_container(container, index_component=0) + container.validate.assert_called_once() + mock_writer.write.assert_not_called() + + @patch("nectarchain.makers.core.HDF5TableWriter") + def test_write_container_with_generic_exception( + self, mock_writer, tool_instance_run_file + ): + tool_instance_run_file.writer = mock_writer + container = MagicMock(spec=NectarCAMContainer) + container.validate = MagicMock(side_effect=Exception("Generic error")) + with patch.object(tool_instance_run_file.log, "error") as mock_log_error: + with pytest.raises(Exception, match="Generic error"): + tool_instance_run_file._write_container(container, index_component=0) + container.validate.assert_called_once() + mock_writer.write.assert_not_called() + mock_log_error.assert_called_with("Generic error", exc_info=True) + + +class TestDelimiterLoopNectarCAMCalibrationTool: + def test_split_run(self): + tool = DelimiterLoopNectarCAMCalibrationTool() + event = NectarCAMDataContainer( + trigger=TriggerContainer(event_type=EventType.FLATFIELD) + ) + assert not (tool.split_run(event=event)) + event = NectarCAMDataContainer( + trigger=TriggerContainer(event_type=EventType.UNKNOWN) + ) + assert tool.split_run(event=event) diff --git a/src/nectarchain/makers/tests/test_waveforms_makers.py b/src/nectarchain/makers/tests/test_waveforms_makers.py index 4ac32b38..553db7c7 100644 --- a/src/nectarchain/makers/tests/test_waveforms_makers.py +++ b/src/nectarchain/makers/tests/test_waveforms_makers.py @@ -1,166 +1,202 @@ -# import logging -# -# import numpy as np -# import pytest -# from ctapipe.containers import EventType -# -# from nectarchain.data.container import WaveformsContainer, WaveformsContainerIO -# from nectarchain.makers.waveforms_makers import WaveformsMaker -# -# logging.basicConfig( -# format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG -# ) -# log = logging.getLogger(__name__) -# log.handlers = logging.getLogger("__main__").handlers -# -# -# @pytest.disable() -# class TestWaveformsMaker: -# run_number = 3938 -# max_events = 100 -# -# def test_instance(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# assert isinstance(waveformsMaker, WaveformsMaker) -# -# def test_shape_valid(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer = waveformsMaker.make()[0] -# -# assert waveformsContainer.nevents <= TestWaveformsMaker.max_events -# assert waveformsContainer.run_number == TestWaveformsMaker.run_number -# assert (waveformsContainer.ucts_timestamp.shape == -# (waveformsContainer.nevents,)) -# assert waveformsContainer.ucts_busy_counter.shape == ( -# waveformsContainer.nevents, -# ) -# assert waveformsContainer.ucts_event_counter.shape == ( -# waveformsContainer.nevents, -# ) -# assert waveformsContainer.event_type.shape == (waveformsContainer.nevents,) -# assert waveformsContainer.event_id.shape == (waveformsContainer.nevents,) -# assert ( -# waveformsContainer.trig_pattern_all.shape[0] == waveformsContainer.nevents -# ) -# assert waveformsContainer.trig_pattern_all.shape[2] == 4 -# -# assert waveformsContainer.trig_pattern.shape[0] == waveformsContainer.nevents -# assert waveformsContainer.multiplicity.shape == (waveformsContainer.nevents,) -# -# assert waveformsContainer.wfs_hg.mean() != 0 -# assert waveformsContainer.wfs_lg.mean() != 0 -# assert waveformsContainer.wfs_hg.shape == ( -# waveformsContainer.nevents, -# waveformsContainer.npixels, -# waveformsContainer.nsamples, -# ) -# assert waveformsContainer.wfs_lg.shape == ( -# waveformsContainer.nevents, -# waveformsContainer.npixels, -# waveformsContainer.nsamples, -# ) -# assert waveformsContainer.broken_pixels_hg.shape == ( -# waveformsContainer.nevents, -# waveformsContainer.npixels, -# ) -# assert waveformsContainer.broken_pixels_lg.shape == ( -# waveformsContainer.nevents, -# waveformsContainer.npixels, -# ) -# -# def test_all_multiple_trigger(self): -# trigger1 = EventType.FLATFIELD -# trigger2 = EventType.SKY_PEDESTAL -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make( -# trigger_type=[trigger1, trigger2], restart_from_begining=True -# ) -# for waveformsContainer in waveformsContainer_list: -# assert isinstance(waveformsContainer, WaveformsContainer) -# assert waveformsContainer.wfs_hg.mean() != 0 -# -# def test_all_trigger_None(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make() -# assert isinstance(waveformsContainer_list[0], WaveformsContainer) -# -# def test_select_waveforms_hg(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make() -# pixel_id = np.array([3, 67, 87]) -# assert isinstance( -# WaveformsMaker.select_waveforms_hg(waveformsContainer_list[0], pixel_id), -# np.ndarray, -# ) -# assert isinstance( -# WaveformsMaker.select_waveforms_lg(waveformsContainer_list[0], pixel_id), -# np.ndarray, -# ) -# -# def test_sort_WaveformsContainer(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make() -# sortWfs = WaveformsMaker.sort(waveformsContainer_list[0], method="event_id") -# assert np.array_equal( -# sortWfs.event_id, np.sort(waveformsContainer_list[0].event_id) -# ) -# -# def test_write_load_container(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# waveformsContainer_list = waveformsMaker.make() -# WaveformsContainerIO.write( -# "/tmp/test_wfs_container/", waveformsContainer_list[0], overwrite=True -# ) -# loaded_wfs = WaveformsContainerIO.load( -# "/tmp/test_wfs_container", TestWaveformsMaker.run_number -# ) -# assert np.array_equal(waveformsContainer_list[0].wfs_hg, loaded_wfs.wfs_hg) -# -# def test_create_from_events_list(self): -# waveformsMaker = WaveformsMaker( -# run_number=TestWaveformsMaker.run_number, -# max_events=TestWaveformsMaker.max_events, -# ) -# events_list = [] -# for i, event in enumerate(waveformsMaker._reader): -# events_list.append(event) -# waveformsContainer = WaveformsMaker.create_from_events_list( -# events_list, -# waveformsMaker.run_number, -# waveformsMaker.npixels, -# waveformsMaker.nsamples, -# waveformsMaker.subarray, -# waveformsMaker.pixels_id, -# ) -# assert isinstance(waveformsContainer, WaveformsContainer) -# -# -# if __name__ == "__main__": -# import logging -# -# logging.basicConfig( -# format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG -# ) -# log = logging.getLogger(__name__) -# log.handlers = logging.getLogger("__main__").handlers +import tempfile +from pathlib import Path + +import numpy as np +from ctapipe.containers import EventType +from ctapipe.utils import get_dataset_path +from ctapipe_io_nectarcam.constants import N_SAMPLES + +from nectarchain.data.container import WaveformsContainer, WaveformsContainers +from nectarchain.makers import WaveformsNectarCAMCalibrationTool + +# This test file test the overall workflow of the WaveformsNectarCAMCalibrationTool, +# covering also the EventsLoopNectarCAMCalibrationTool, which cannot be tested on +# data because the ArrayDataComponent within the WaveformsNectarCAMCalibrationTool +# is an abstract Component. However the EventsLoopNectarCAMCalibrationTool is still +# covered by unit test using pytest patches to mock the Component behavior. + + +class TestWaveformsNectarCAMCalibrationTool: + RUNS = { + "Run number": [3938, 5288], + "Run file": [ + get_dataset_path("NectarCAM.Run3938.30events.fits.fz"), + get_dataset_path("NectarCAM.Run5288.0001.fits.fz"), + ], + "nevents": [30, 13], + "N pixels": [1834, 1848], + "eventType": EventType.SKY_PEDESTAL, + "wfs_lg_min": [230, 233], + "wfs_lg_max": [264, 269], + "wfs_lg_mean": [2468, 2509], + "wfs_lg_std": [33, 36], + "wfs_hg_min": [178, 195], + "wfs_hg_max": [288, 298], + "wfs_hg_mean": [2467, 2508], + "wfs_hg_std": [36, 38], + "expected_ucts_timestamp_min": [1674462932637854793, 1715007113924900896], + "expected_ucts_timestamp_max": [1674462932695877994, 1715007123524920096], + } + OVERWRITE = True + + def general_structure_testing( + self, output: WaveformsContainer, nevents: int, n_pixels: int, run_number: int + ): + assert isinstance(output.pixels_id, np.ndarray) + assert output.pixels_id.dtype == np.uint16 + assert np.shape(output.pixels_id) == (n_pixels,) + assert output.run_number == run_number + assert output.camera == "NectarCam-003" + assert output.npixels == n_pixels + assert isinstance(output.ucts_busy_counter, np.ndarray) + assert output.ucts_busy_counter.dtype == np.uint32 + assert isinstance(output.ucts_event_counter, np.ndarray) + assert output.ucts_event_counter.dtype == np.uint32 + assert isinstance(output.event_type, np.ndarray) + assert output.event_type.dtype == np.uint8 + assert np.all(output.event_type == self.RUNS["eventType"].value) + assert isinstance(output.trig_pattern, np.ndarray) + assert output.trig_pattern.dtype == bool + assert isinstance(output.trig_pattern_all, np.ndarray) + assert output.trig_pattern_all.dtype == bool + assert isinstance(output.multiplicity, np.ndarray) + assert output.multiplicity.dtype == np.uint16 + assert isinstance(output.ucts_timestamp, np.ndarray) + assert output.ucts_timestamp.dtype == np.uint64 + + assert isinstance(output.event_id, np.ndarray) + assert output.event_id.dtype == np.uint32 + assert isinstance(output.broken_pixels_hg, np.ndarray) + assert output.broken_pixels_hg.dtype == bool + assert output.broken_pixels_hg.shape == (nevents, n_pixels) + assert isinstance(output.broken_pixels_lg, np.ndarray) + assert output.broken_pixels_lg.dtype == bool + assert output.broken_pixels_lg.shape == (nevents, n_pixels) + assert output.wfs_hg.shape == (nevents, n_pixels, N_SAMPLES) + assert output.wfs_lg.shape == (nevents, n_pixels, N_SAMPLES) + assert isinstance(output.wfs_hg, np.ndarray) + assert isinstance(output.wfs_lg, np.ndarray) + assert output.wfs_hg.dtype == np.uint16 + assert output.wfs_lg.dtype == np.uint16 + + def test_base(self): + """ + Test basic functionality, including IO on disk + """ + + events_per_slice = [None, None, 10, 11, 8] + max_events = [None, 10, None, None, 10] + + for _max_events, _events_per_slice in zip(max_events, events_per_slice): + for i, run_number in enumerate(self.RUNS["Run number"]): + run_file = self.RUNS["Run file"][i] + n_pixels = self.RUNS["N pixels"][i] + with tempfile.TemporaryDirectory() as tmpdirname: + outfile = tmpdirname + "/waveforms.h5" + + # run tool + tool = WaveformsNectarCAMCalibrationTool( + run_number=run_number, + run_file=run_file, + max_events=_max_events, + events_per_slice=_events_per_slice, + log_level=0, + output_path=outfile, + overwrite=self.OVERWRITE, + ) + + tool.setup() + nevents = len(tool.event_source) + assert ( + nevents == self.RUNS["nevents"][i] + if _max_events is None + else _max_events + ) + tool.start() + output_containers = tool.finish(return_output_component=True)[0] + assert isinstance(output_containers, WaveformsContainers) + output = output_containers.containers[self.RUNS["eventType"]] + assert isinstance(output, WaveformsContainer) + # Check output in memory + if ( + _events_per_slice is not None + and nevents % _events_per_slice == 0 + ): + assert output.nevents is None + else: + assert output.nsamples == N_SAMPLES + if _events_per_slice is None: + assert ( + output.nevents == nevents + ) # nevents has been validated before + else: + assert output.nevents == nevents % _events_per_slice + + self.general_structure_testing( + output, + ( + nevents + if _events_per_slice is None + else nevents % _events_per_slice + ), + n_pixels, + run_number, + ) + + if _events_per_slice is None and _max_events is None: + # content only checked for the full run + assert np.min(output.ucts_timestamp) == np.uint64( + self.RUNS["expected_ucts_timestamp_min"][i] + ) + assert np.max(output.ucts_timestamp) == np.uint64( + self.RUNS["expected_ucts_timestamp_max"][i] + ) + assert output.wfs_lg.min() == self.RUNS["wfs_lg_min"][i] + assert output.wfs_lg.max() == self.RUNS["wfs_lg_max"][i] + assert ( + int(10 * output.wfs_lg.mean()) + == self.RUNS["wfs_lg_mean"][i] + ) + assert ( + int(10 * output.wfs_lg.std()) + == self.RUNS["wfs_lg_std"][i] + ) + assert output.wfs_hg.min() == self.RUNS["wfs_hg_min"][i] + assert output.wfs_hg.max() == self.RUNS["wfs_hg_max"][i] + assert ( + int(10 * output.wfs_hg.mean()) + == self.RUNS["wfs_hg_mean"][i] + ) + assert ( + int(10 * output.wfs_hg.std()) + == self.RUNS["wfs_hg_std"][i] + ) + + # Check output on disk + assert Path(outfile).exists() + + waveformsContainers = WaveformsContainers.from_hdf5(outfile) + ncontainers = 0 + for container in waveformsContainers: + ncontainers += 1 + assert isinstance(container, WaveformsContainers) + output = container.containers[self.RUNS["eventType"]] + if _events_per_slice is None: + expected_nevents = nevents + else: + if nevents % _events_per_slice == 0: + expected_nevents = _events_per_slice + else: + if ncontainers == 1: + expected_nevents = nevents % _events_per_slice + else: + expected_nevents = _events_per_slice + self.general_structure_testing( + output, expected_nevents, n_pixels, run_number + ) + + assert ( + ncontainers == 1 + if _events_per_slice is None + else round(nevents / _events_per_slice) + ) diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveforms_makers.py similarity index 100% rename from src/nectarchain/makers/waveformsMakers.py rename to src/nectarchain/makers/waveforms_makers.py index fb0f94cb..96cc6597 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveforms_makers.py @@ -1,9 +1,4 @@ import logging - -logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) -log.handlers = logging.getLogger("__main__").handlers - import os import pathlib @@ -12,6 +7,11 @@ from .component import NectarCAMComponent from .core import EventsLoopNectarCAMCalibrationTool +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + __all__ = ["WaveformsNectarCAMCalibrationTool"] diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/README.md b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/README.md similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/README.md rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/README.md diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/gui.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/gui.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/gui.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/gui.py diff --git a/src/nectarchain/makers/tests/__init__.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/__init__.py similarity index 100% rename from src/nectarchain/makers/tests/__init__.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/__init__.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/deadtime_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py similarity index 99% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/deadtime_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py index f554cd78..f475a588 100644 --- a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/deadtime_test.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/deadtime.py @@ -10,8 +10,9 @@ import numpy as np from astropy import units as u from iminuit import Minuit -from test_tools_components import DeadtimeTestTool -from utils import ( + +from .tools_components import DeadtimeTestTool +from .utils import ( ExponentialFitter, deadtime_and_expo_fit, deadtime_labels, diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/linearity_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py similarity index 99% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/linearity_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py index 7c6b8d52..32a0ef84 100644 --- a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/linearity_test.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/linearity.py @@ -9,8 +9,9 @@ import matplotlib.pyplot as plt import numpy as np from lmfit.models import Model -from test_tools_components import LinearityTestTool -from utils import ( + +from .tools_components import LinearityTestTool +from .utils import ( adc_to_pe, err_ratio, err_sum, diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pedestal_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py similarity index 98% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pedestal_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py index 36ab8c33..e942d5a0 100644 --- a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pedestal_test.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pedestal.py @@ -10,11 +10,12 @@ import pandas as pd from ctapipe.containers import EventType from ctapipe_io_nectarcam.containers import NectarCAMDataContainer -from test_tools_components import PedestalTool -from utils import adc_to_pe, pe2photons, photons2pe from nectarchain.makers.calibration import PedestalNectarCAMCalibrationTool +from .tools_components import PedestalTool +from .utils import adc_to_pe, pe2photons, photons2pe + def get_args(): """ diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pix_couple_tim_uncertainty_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py similarity index 99% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pix_couple_tim_uncertainty_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py index 637d5de2..6b4393ce 100644 --- a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pix_couple_tim_uncertainty_test.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_couple_tim_uncertainty.py @@ -5,7 +5,8 @@ import matplotlib.pyplot as plt import numpy as np -from test_tools_components import ToMPairsTool + +from .tools_components import ToMPairsTool def get_args(): diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pix_tim_uncertainty_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py similarity index 99% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pix_tim_uncertainty_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py index 7c32b623..1c7e57eb 100644 --- a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/pix_tim_uncertainty_test.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/pix_tim_uncertainty.py @@ -7,8 +7,9 @@ import matplotlib.pyplot as plt import numpy as np -from test_tools_components import TimingResolutionTestTool -from utils import pe2photons, photons2pe + +from .tools_components import TimingResolutionTestTool +from .utils import pe2photons, photons2pe def get_args(): diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/test_tools_components.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py similarity index 99% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/test_tools_components.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py index e22f575e..df87d7de 100644 --- a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/test_tools_components.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/tools_components.py @@ -16,13 +16,14 @@ from lmfit.models import Model from scipy.interpolate import InterpolatedUnivariateSpline from scipy.signal import find_peaks -from utils import adc_to_pe, argmedian from nectarchain.data.container import NectarCAMContainer from nectarchain.makers import EventsLoopNectarCAMCalibrationTool from nectarchain.makers.component import NectarCAMComponent from nectarchain.utils import ComponentUtils +from .utils import adc_to_pe, argmedian + # overriding so we can have maxevents in the path def _init_output_path(self): diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/trigger_timing_test.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py similarity index 98% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/trigger_timing_test.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py index 819402ff..cf0e7222 100644 --- a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/trigger_timing_test.py +++ b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/trigger_timing.py @@ -8,8 +8,9 @@ import matplotlib.pyplot as plt import numpy as np -from test_tools_components import TriggerTimingTestTool -from utils import pe2photons + +from .tools_components import TriggerTimingTestTool +from .utils import pe2photons def get_args(): diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/utils.py b/src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/utils.py similarity index 100% rename from src/nectarchain/user_scripts/dmousadi/test_scripts/tests/utils.py rename to src/nectarchain/user_scripts/dmousadi/TRR_scripts/src/utils.py diff --git a/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/__init__.py b/src/nectarchain/user_scripts/dmousadi/test_scripts/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py index f3991be0..4261f903 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py @@ -1,11 +1,11 @@ +import argparse +import copy +import glob import json import logging import os import sys from pathlib import Path -import argparse -import copy -import glob from nectarchain.data.management import DataManagement from nectarchain.makers.calibration import PhotoStatisticNectarCAMCalibrationTool @@ -13,7 +13,8 @@ parser = argparse.ArgumentParser( prog="gain_SPEfit_computation.py", - description=f"compute high and low gain with the Photo-statistic method, output data are saved in $NECTARCAMDATA/../PhotoStat/", + description=f"compute high and low gain with the Photo-statistic\ + method, output data are saved in $NECTARCAMDATA/../PhotoStat/", ) # run numbers parser.add_argument( @@ -60,13 +61,13 @@ "SlidingWindowMaxSum", "TwoPassWindowSum", ], - default="LocalPeakWindowSum", + default="GlobalPeakWindowSum", help="charge extractor method", type=str, ) parser.add_argument( "--extractor_kwargs", - default={"window_width": 16, "window_shift": 4}, + default={"window_width": 8, "window_shift": 4}, help="charge extractor kwargs", type=json.loads, ) @@ -124,10 +125,20 @@ def main( str_extractor_kwargs = CtapipeExtractor.get_extractor_kwargs_str( args.extractor_kwargs ) - path = DataManagement.find_SPE_HHV( + # path = DataManagement.find_SPE_HHV( + # run_number=args.HHV_run_number, + # method=args.method, + # str_extractor_kwargs=str_extractor_kwargs, + # ) + # path = DataManagement.find_SPE_nominal( + # run_number=args.HHV_run_number, + # method=args.method, + # str_extractor_kwargs=str_extractor_kwargs, + # ) + path = DataManagement.find_SPE_nominal( run_number=args.HHV_run_number, - method=args.method, - str_extractor_kwargs=str_extractor_kwargs, + method="GlobalPeakWindowSum", + str_extractor_kwargs=f"window_width_8_window_shift_4", ) if len(path) == 1: log.info( @@ -196,5 +207,10 @@ def main( kwargs.pop("figpath") kwargs.pop("HHV_run_number") + kwargs["FF_run_number"] = [3937] + kwargs["Ped_run_number"] = [3938] + kwargs["overwrite"] = True + args.HHV_run_number = 3936 + log.info(f"arguments passed to main are : {kwargs}") main(log=log, **kwargs) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py index 6cf83469..e965c9fa 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py @@ -1,12 +1,12 @@ +import argparse +import copy +import glob import json import logging import os import sys import time from pathlib import Path -import argparse -import copy -import glob from nectarchain.data.management import DataManagement from nectarchain.makers.calibration import ( @@ -71,7 +71,7 @@ ) parser.add_argument( "--extractor_kwargs", - default={"window_width": 16, "window_shift": 4}, + default={"window_width": 10, "window_shift": 4}, help="charge extractor kwargs", type=json.loads, ) @@ -218,6 +218,14 @@ def main( kwargs.pop("display") kwargs.pop("HHV_run_number") + # args.HHV_run_number = 3942 + # kwargs['run_number'] = [3936] + # kwargs['overwrite'] = True + # kwargs['asked_pixels_id'] = [45,600,800] + # kwargs['multiproc'] = False + # args.display = True + # args.figpath = "/home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + log.info(f"arguments passed to main are : {kwargs}") main(log=log, **kwargs) log.info(f"time for execution is {time.time() - t:.2e} sec") diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py index a24a5ce3..4bcb6010 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py @@ -72,7 +72,7 @@ ) parser.add_argument( "--extractor_kwargs", - default={"window_width": 16, "window_shift": 4}, + default={"window_width": 8, "window_shift": 4}, help="charge extractor kwargs", type=json.loads, ) @@ -216,6 +216,14 @@ def main( kwargs.pop("HHV") kwargs.pop("free_pp_n") + # kwargs['run_number'] = [3942] + # kwargs['overwrite'] = True + # kwargs['asked_pixels_id'] = [45,600,800] + # args.HHV = True + # kwargs['multiproc'] = True + # args.display = True + # args.figpath = "/home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + log.info(f"arguments passed to main are : {kwargs}") main(log=log, **kwargs) log.info(f"time for execution is {time.time() - t:.2e} sec") diff --git a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py index 7da5dd92..d435cad2 100644 --- a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py +++ b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py @@ -39,6 +39,12 @@ type=int, ) +parser.add_argument( + "--only_wfs", + action="store_true", + default=False, + help="to only reload wfs", +) # boolean arguments parser.add_argument( "--reload_wfs", @@ -124,7 +130,7 @@ def main( for _run_number, _max_events in zip(run_number, max_events): try: - if kwargs.get("reload_wfs", False): + if kwargs.get("only_wfs", False) or kwargs.get("reload_wfs", False): log.info("reloading waveforms") tool = WaveformsNectarCAMCalibrationTool( progress_bar=True, @@ -136,16 +142,17 @@ def main( tool.start() tool.finish() - tool = ChargesNectarCAMCalibrationTool( - progress_bar=True, - run_number=_run_number, - max_events=_max_events, - from_computed_waveforms=True, - **charges_kwargs, - ) - tool.setup() - tool.start() - tool.finish() + if not (kwargs.get("only_wfs", False)): + tool = ChargesNectarCAMCalibrationTool( + progress_bar=True, + run_number=_run_number, + max_events=_max_events, + from_computed_waveforms=True, + **charges_kwargs, + ) + tool.setup() + tool.start() + tool.finish() else: log.info("trying to compute charges from waveforms yet extracted") tool = ChargesNectarCAMCalibrationTool( @@ -203,5 +210,8 @@ def main( kwargs.pop("verbosity") log.info(f"arguments passed to main are : {kwargs}") - + # kwargs['reload_wfs'] = True + # kwargs['run_number'] = [3784]#[5436] + # kwargs['overwrite'] = True + # kwargs['events_per_slice'] = 2000 main(log=log, **kwargs) diff --git a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.sh b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.sh index 0f050238..31150040 100644 --- a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.sh +++ b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.sh @@ -16,4 +16,5 @@ python load_wfs_compute_charge.py -s 2633 2634 3784 -f 2608 -p 2609 --spe_nevents 49227 49148 -1 --method LocalPeakWindowSum --extractor_kwargs '{"window_width":16,"window_shift":4}' -python load_wfs_compute_charge.py -r 3942 --max_events 100 --method GlobalPeakWindowSum --extractor_kwargs '{"window_width":16,"window_shift":4}' --overwrite -v debug \ No newline at end of file +python load_wfs_compute_charge.py -r 3942 --max_events 100 --method GlobalPeakWindowSum --extractor_kwargs '{"window_width":16,"window_shift":4}' --overwrite -v debug + diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_charge_extraction.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_charge_extraction.sh new file mode 100755 index 00000000..9c7ea100 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_charge_extraction.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# 3936 3937 3938 3942 +#"FullWaveformSum" "LocalPeakWindowSum" "GlobalPeakWindowSum" +for METHOD in "FullWaveformSum" "LocalPeakWindowSum" "GlobalPeakWindowSum" +do + if [ $METHOD = "FullWaveformSum" ]; + then + export cmd="python load_wfs_compute_charge.py -r 3938 --events_per_slice 2000 --method $METHOD --overwrite -v INFO --reload_wfs" + echo $cmd + eval $cmd + else + for WIDTH in 8 9 10 11 12 16 + do + export cmd="python load_wfs_compute_charge.py -r 3938 --events_per_slice 2000 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO" + echo $cmd + eval $cmd + done + fi + #python load_wfs_compute_charge.py -r 3936 3942 --method LocalPeakWindowSum --extractor_kwargs "{'window_width':$WIDTH,'window_shift':4}" --overwrite -v INFO +done diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV.sh new file mode 100755 index 00000000..89343444 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV.sh @@ -0,0 +1,20 @@ +#!/bin/bash +export _QT_QPA_PLATFORM=$QT_QPA_PLATFORM +export QT_QPA_PLATFORM=offscreen +for METHOD in "GlobalPeakWindowSum" +do + for WIDTH in 8 9 10 11 12 16 + do + if [ $WIDTH -eq 8 ]; then + export cmd="python gain_SPEfit_computation.py -r 3942 --HHV --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO --display --figpath /home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + echo $cmd + eval $cmd + else + export cmd="python gain_SPEfit_computation.py -r 3942 --HHV --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO" + echo $cmd + eval $cmd + fi + done +done +export QT_QPA_PLATFORM=$_QT_QPA_PLATFORM +unset _QT_QPA_PLATFORM diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV_free.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV_free.sh new file mode 100755 index 00000000..0029c258 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_HHV_free.sh @@ -0,0 +1,20 @@ +#!/bin/bash +export _QT_QPA_PLATFORM=$QT_QPA_PLATFORM +export QT_QPA_PLATFORM=offscreen +for METHOD in "GlobalPeakWindowSum" #"LocalPeakWindowSum" +do + for WIDTH in 8 10 12 + do + if [ $WIDTH -eq 8 ]; then + export cmd="python gain_SPEfit_computation.py -r 3942 --HHV --free_pp_n --multiproc --nproc 8 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO --display --figpath /home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + echo $cmd + eval $cmd + else + export cmd="python gain_SPEfit_computation.py -r 3942 --HHV --free_pp_n --multiproc --nproc 8 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO" + echo $cmd + eval $cmd + fi + done +done +export QT_QPA_PLATFORM=$_QT_QPA_PLATFORM +unset _QT_QPA_PLATFORM \ No newline at end of file diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal.sh new file mode 100755 index 00000000..60ed23de --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal.sh @@ -0,0 +1,20 @@ +#!/bin/bash +export _QT_QPA_PLATFORM=$QT_QPA_PLATFORM +export QT_QPA_PLATFORM=offscreen +for METHOD in "LocalPeakWindowSum" "GlobalPeakWindowSum" +do + for WIDTH in 8 9 10 12 + do + if [ $WIDTH -eq 8 ]; then + export cmd="python gain_SPEfit_computation.py -r 3936 --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO --display --figpath /home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + echo $cmd + eval $cmd + else + export cmd="python gain_SPEfit_computation.py -r 3936 --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO" + echo $cmd + eval $cmd + fi + done +done +export QT_QPA_PLATFORM=$_QT_QPA_PLATFORM +unset _QT_QPA_PLATFORM diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal_from_HHV.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal_from_HHV.sh new file mode 100755 index 00000000..f153d8f9 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_nominal_from_HHV.sh @@ -0,0 +1,21 @@ +#!/bin/bash +export _QT_QPA_PLATFORM=$QT_QPA_PLATFORM +export QT_QPA_PLATFORM=offscreen +for METHOD in "LocalPeakWindowSum" "GlobalPeakWindowSum" +do + for WIDTH in 9 11 16 + do + if [ $WIDTH -eq 8 ]; then + export cmd="python gain_SPEfit_combined_computation.py -r 3936 --HHV_run_number 3942 --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO --display --figpath /home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + echo $cmd + eval $cmd + else + export cmd="python gain_SPEfit_combined_computation.py -r 3936 --HHV_run_number 3942 --multiproc --nproc 12 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO" + echo $cmd + eval $cmd + + fi + done +done +export QT_QPA_PLATFORM=$_QT_QPA_PLATFORM +unset _QT_QPA_PLATFORM \ No newline at end of file diff --git a/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_pstat.sh b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_pstat.sh new file mode 100755 index 00000000..e6d23cf6 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/script_loop_gain_pstat.sh @@ -0,0 +1,14 @@ +#!/bin/bash +export _QT_QPA_PLATFORM=$QT_QPA_PLATFORM +export QT_QPA_PLATFORM=offscreen +for METHOD in "LocalPeakWindowSum" "GlobalPeakWindowSum" +do + for WIDTH in 8 9 10 11 12 16 + do + export cmd="python gain_PhotoStat_computation.py --FF_run_number 3937 --Ped_run_number 3938 --HHV_run_number 3936 --method $METHOD --extractor_kwargs '{\"window_width\":$WIDTH,\"window_shift\":4}' --overwrite -v INFO --figpath /home/ggroller/projects/nectarchain/src/nectarchain/user_scripts/ggrolleron/local/figures" + echo $cmd + eval $cmd + done +done +export QT_QPA_PLATFORM=$_QT_QPA_PLATFORM +unset _QT_QPA_PLATFORM