From 5ac37566ebe8fdc5528dc887b2796b6ed9de0828 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 15:55:46 +0100 Subject: [PATCH 01/39] support loading config files from a YAML file --- ctapipe/core/tool.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index 8c42930df43..1bfb3db0c7f 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -7,8 +7,16 @@ import os import re +try: + import yaml + + HAS_YAML = True +except ImportError: + HAS_YAML = False + pass # no support for YAML + from traitlets import default -from traitlets.config import Application, Configurable +from traitlets.config import Application, Configurable, Config from .. import __version__ as version from .traits import Path, Enum, Bool, Dict @@ -189,11 +197,13 @@ def initialize(self, argv=None): self.update_logging_config() if self.config_file is not None: - self.log.debug(f"Loading config from '{self.config_file}'") + self.log.info(f"Loading config from '{self.config_file}'") try: - self.load_config_file(self.config_file) + self._load_tool_config_file(self.config_file) except Exception as err: - raise ToolConfigurationError(f"Couldn't read config file: {err}") + raise ToolConfigurationError( + f"Couldn't read config file: {err} {type(err)}" + ) # ensure command-line takes precedence over config file options: self.update_config(self.cli_config) @@ -201,6 +211,17 @@ def initialize(self, argv=None): self.log.info(f"ctapipe version {self.version_string}") + def _load_tool_config_file(self, path: pathlib.Path): + + if path.suffix in [".yaml", ".yml"] and HAS_YAML: + # do our own YAML loading + with open(path, "r") as infile: + config = Config(yaml.safe_load(infile)) + self.update_config(config) + else: + # fall back to traitlets.config.Application's implementation + self.load_config_file(str(path)) + def update_logging_config(self): """Update the configuration of loggers.""" cfg = create_logging_config( @@ -334,7 +355,7 @@ def version_string(self): return f"{version}" def get_current_config(self): - """ return the current configuration as a dict (e.g. the values + """return the current configuration as a dict (e.g. the values of all traits, even if they were not set during configuration) """ conf = { From ea3cb4b92f23a0337b118fc836c64bfc7c0f1501 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 15:56:07 +0100 Subject: [PATCH 02/39] a sample YAML version of the config --- .../tools/tests/resources/stage1_config.yaml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 ctapipe/tools/tests/resources/stage1_config.yaml diff --git a/ctapipe/tools/tests/resources/stage1_config.yaml b/ctapipe/tools/tests/resources/stage1_config.yaml new file mode 100644 index 00000000000..cb324542da9 --- /dev/null +++ b/ctapipe/tools/tests/resources/stage1_config.yaml @@ -0,0 +1,53 @@ +DataWriter: + Contact: + # please fill in your contact information here. It will be stored in the + # output files as provenance information + name: YOUR-NAME-HERE + email: YOUREMAIL@EXAMPLE.ORG + organization: YOUR-ORGANIZATION + + # options that control what is stored in the output file + overwrite: false # do not overwrite existing files + write_images: true # store DL1 images + write_parameters: true # store DL1 parameters + write_stereo_shower: false # store DL2 stereo geometry + +CameraCalibrator: + image_extractor_type: NeighborPeakWindowSum + +ImageProcessor: + image_cleaner_type: TailcutsImageCleaner + + # make sure you include a configuration for the image cleaner you selected + # above here. The section named by the image_cleaner_type will be used to + # configure it. + TailcutsImageCleaner: + # the thresholds for this image cleaner must be optimized for the data set + # you are analyzing. The defaults may not be correct. + picture_threshold_pe: + - [type, "*", 10.0] + - [type, LST_LST_LSTCam, 5.0] + - [type, MST_MST_NectarCam, 5.0] + - [type, SST_ASTRI_CHEC, 3.0] + boundary_threshold_pe: + - [type, "*", 5.0] + - [type, LST_LST_LSTCam, 2.5] + - [type, MST_MST_NectarCam, 2.5] + - [type, SST_ASTRI_CHEC, 1.5] + min_picture_neighbors: + - [type, "*", 2] + + # These specify which images should be parameterized: + ImageQualityQuery: + quality_criteria: + - [enough_pixels, "lambda im: np.count_nonzero(im) > 2"] + - [enough_charge, "lambda im: im.sum() > 50"] + +ShowerProcessor: + # These specify criteria for telescopes that should be included in stereo + # reconstruction: + ShowerQualityQuery: + quality_criteria: + - [enough intensity, "lambda p: p.hillas.intensity > 50"] + - [Positive width, "lambda p: p.hillas.width.value > 0"] + - [enough pixels, "lambda p: p.morphology.num_pixels > 3"] From f23996a45649c05001c15213c8ff58ebcf6b2a60 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 16:06:15 +0100 Subject: [PATCH 03/39] Modify one test to use YAML --- ctapipe/tools/tests/test_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index 38529f593b5..15895d924b7 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -24,7 +24,7 @@ def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file): """ check simtel to DL1 conversion """ - config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json") + config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.yaml") # DL1A file as input dl1b_from_dl1a_file = tmp_path / "dl1b_fromdl1a.dl1.h5" assert ( From c9e18c128b6df17c15de81a369887edb3c3af878 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 16:06:25 +0100 Subject: [PATCH 04/39] add yaml example to quickstart --- ctapipe/tools/quickstart.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ctapipe/tools/quickstart.py b/ctapipe/tools/quickstart.py index 72190dd65a4..f6c5150195f 100644 --- a/ctapipe/tools/quickstart.py +++ b/ctapipe/tools/quickstart.py @@ -14,7 +14,12 @@ __all__ = ["QuickStartTool"] -CONFIGS_TO_WRITE = ["stage1_config.json", "stage2_config.json", "training_config.json"] +CONFIGS_TO_WRITE = [ + "stage1_config.yaml", + "stage1_config.json", + "stage2_config.json", + "training_config.json", +] README_TEXT = f""" ctapipe working directory From b5862674553dffab5ac98e1a500b314ff1e2db50 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 16:17:45 +0100 Subject: [PATCH 05/39] just overload the original load_config() - to avoid confusion - cleaned up imports as well --- ctapipe/core/tool.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index 1bfb3db0c7f..8a4c0417863 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -1,11 +1,12 @@ """Classes to handle configurable command-line user interfaces.""" import logging import logging.config -import textwrap -from abc import abstractmethod -import pathlib import os +import pathlib import re +import textwrap +from abc import abstractmethod +from typing import Union try: import yaml @@ -16,14 +17,13 @@ pass # no support for YAML from traitlets import default -from traitlets.config import Application, Configurable, Config +from traitlets.config import Application, Config, Configurable from .. import __version__ as version -from .traits import Path, Enum, Bool, Dict from . import Provenance from .component import Component -from .logging import create_logging_config, ColoredFormatter, DEFAULT_LOGGING - +from .logging import DEFAULT_LOGGING, ColoredFormatter, create_logging_config +from .traits import Bool, Dict, Enum, Path __all__ = ["Tool", "ToolConfigurationError"] @@ -199,7 +199,7 @@ def initialize(self, argv=None): if self.config_file is not None: self.log.info(f"Loading config from '{self.config_file}'") try: - self._load_tool_config_file(self.config_file) + self.load_config_file(self.config_file) except Exception as err: raise ToolConfigurationError( f"Couldn't read config file: {err} {type(err)}" @@ -211,7 +211,9 @@ def initialize(self, argv=None): self.log.info(f"ctapipe version {self.version_string}") - def _load_tool_config_file(self, path: pathlib.Path): + def load_config_file(self, path: Union[std, pathlib.Path]): + + path = pathlib.Path(path) if path.suffix in [".yaml", ".yml"] and HAS_YAML: # do our own YAML loading @@ -220,7 +222,7 @@ def _load_tool_config_file(self, path: pathlib.Path): self.update_config(config) else: # fall back to traitlets.config.Application's implementation - self.load_config_file(str(path)) + super(self).load_config_file(str(path)) def update_logging_config(self): """Update the configuration of loggers.""" From 96404eee3923f0238fb9fda98bee7cb83774d00f Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 16:26:36 +0100 Subject: [PATCH 06/39] fixed typo --- ctapipe/core/tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index 8a4c0417863..0494a8b1f3e 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -211,7 +211,7 @@ def initialize(self, argv=None): self.log.info(f"ctapipe version {self.version_string}") - def load_config_file(self, path: Union[std, pathlib.Path]): + def load_config_file(self, path: Union[str, pathlib.Path]): path = pathlib.Path(path) From 00837746c8211e9248a0216a062b86cdc10b5ea5 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 16:28:56 +0100 Subject: [PATCH 07/39] add TOML support --- ctapipe/core/tool.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index 0494a8b1f3e..ee547c92109 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -16,6 +16,13 @@ HAS_YAML = False pass # no support for YAML +try: + import toml + + HAS_TOML = True +except ImportError: + HAS_TOML = False + from traitlets import default from traitlets.config import Application, Config, Configurable @@ -220,6 +227,10 @@ def load_config_file(self, path: Union[str, pathlib.Path]): with open(path, "r") as infile: config = Config(yaml.safe_load(infile)) self.update_config(config) + elif path.suffix == ".toml" and HAS_TOML: + with open(path, "r") as infile: + config = Config(toml.load(infile)) + self.update_config(config) else: # fall back to traitlets.config.Application's implementation super(self).load_config_file(str(path)) From 635594671ca91a137cfcb86e5bb3b284d6645862 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 16:39:06 +0100 Subject: [PATCH 08/39] try to import tomli first and fall back to toml --- ctapipe/core/tool.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index ee547c92109..eb963591b0c 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -17,11 +17,16 @@ pass # no support for YAML try: - import toml + import tomli HAS_TOML = True except ImportError: - HAS_TOML = False + try: + import toml + + HAS_TOML = True + except ImportError: + HAS_TOML = False from traitlets import default from traitlets.config import Application, Config, Configurable From 458e90a72b3df12bdc9b2d65f41c1d4248460646 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 16:42:01 +0100 Subject: [PATCH 09/39] remove self from super --- ctapipe/core/tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index eb963591b0c..b355acf18f6 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -238,7 +238,7 @@ def load_config_file(self, path: Union[str, pathlib.Path]): self.update_config(config) else: # fall back to traitlets.config.Application's implementation - super(self).load_config_file(str(path)) + super().load_config_file(str(path)) def update_logging_config(self): """Update the configuration of loggers.""" From fd500ed5a1c7e6add77a478d5709c0907e3132ef Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 16:56:26 +0100 Subject: [PATCH 10/39] fix import --- ctapipe/core/tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index b355acf18f6..efe39c4039b 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -17,7 +17,7 @@ pass # no support for YAML try: - import tomli + import tomli as toml HAS_TOML = True except ImportError: From bae4ba06c01b9db3303616eb76167e52fc17df93 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 16:56:49 +0100 Subject: [PATCH 11/39] added example TOML file and test --- ctapipe/tools/quickstart.py | 1 + .../tools/tests/resources/stage1_config.toml | 55 +++++++++++++++++++ ctapipe/tools/tests/test_process.py | 14 ++++- 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 ctapipe/tools/tests/resources/stage1_config.toml diff --git a/ctapipe/tools/quickstart.py b/ctapipe/tools/quickstart.py index f6c5150195f..a84a3653187 100644 --- a/ctapipe/tools/quickstart.py +++ b/ctapipe/tools/quickstart.py @@ -16,6 +16,7 @@ CONFIGS_TO_WRITE = [ "stage1_config.yaml", + "stage1_config.toml", "stage1_config.json", "stage2_config.json", "training_config.json", diff --git a/ctapipe/tools/tests/resources/stage1_config.toml b/ctapipe/tools/tests/resources/stage1_config.toml new file mode 100644 index 00000000000..4a2b1740233 --- /dev/null +++ b/ctapipe/tools/tests/resources/stage1_config.toml @@ -0,0 +1,55 @@ +[DataWriter.Contact] +# please fill in your contact information here. It will be stored in the +# output files as provenance information +name = "YOUR-NAME-HERE" +email = "YOUREMAIL@EXAMPLE.ORG" +organization = "YOUR-ORGANIZATION" + +[DataWriter] +# options that control what is stored in the output file +overwrite = false +write_images = true +write_parameters = true +write_stereo_shower = false +write_mono_shower = false +transform_image = true +transform_peak_time = true + +[CameraCalibrator] +image_extractor_type = "NeighborPeakWindowSum" + +[ImageProcessor] +image_cleaner_type = "TailcutsImageCleaner" + + + +[ImageProcessor.TailcutsImageCleaner] +picture_threshold_pe = [ + [ "type", "*", 10.0,], + [ "type", "LST_LST_LSTCam", 5.0,], + [ "type", "MST_MST_NectarCam", 5.0,], + [ "type", "SST_ASTRI_CHEC", 3.0,], +] +boundary_threshold_pe = [ + [ "type", "*", 5.0,], + [ "type", "LST_LST_LSTCam", 2.5,], + [ "type", "MST_MST_NectarCam", 2.5,], + [ "type", "SST_ASTRI_CHEC", 1.5,], +] +min_picture_neighbors = [ [ "type", "*", 2,],] + +[ImageProcessor.ImageQualityQuery] +# These specify which images should be parameterized: +quality_criteria = [ + [ "enough_pixels", "lambda im: np.count_nonzero(im) > 2",], + [ "enough_charge", "lambda im: im.sum() > 50",] +] + +[ShowerProcessor.ShowerQualityQuery] +# These specify criteria for telescopes that should be included in stereo +# reconstruction: +quality_criteria = [ + [ "enough intensity", "lambda p: p.hillas.intensity > 50",], + [ "Positive width", "lambda p: p.hillas.width.value > 0",], + [ "enough pixels", "lambda p: p.morphology.num_pixels > 3",], +] diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index 15895d924b7..45c46d67999 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -21,6 +21,18 @@ GAMMA_TEST_LARGE = get_dataset_path("gamma_test_large.simtel.gz") +def test_read_yaml_toml_config(dl1_image_file): + tool = ProcessorTool() + + for config_base in ["stage1_config.yaml", "stage1_config.toml"]: + config = files("ctapipe.tools.tests.resources").joinpath(config_base) + tool.load_config_file(config) + + tool.EventSource.input_url = dl1_image_file + tool.setup() + assert tool.config.DataWriter.name == "YOUR-NAME-HERE" + + def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file): """ check simtel to DL1 conversion """ @@ -131,7 +143,7 @@ def _generator(self): infile.write(b"dummy") infile.flush() - config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json") + config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.toml") tool = ProcessorTool() assert ( From 754d3a56285cb025d8bdd15afa01c524308cf67b Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 17:18:59 +0100 Subject: [PATCH 12/39] reverted to simpler TOML --- .../tools/tests/resources/stage1_config.toml | 44 +++++-------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/ctapipe/tools/tests/resources/stage1_config.toml b/ctapipe/tools/tests/resources/stage1_config.toml index 4a2b1740233..865e786df25 100644 --- a/ctapipe/tools/tests/resources/stage1_config.toml +++ b/ctapipe/tools/tests/resources/stage1_config.toml @@ -1,12 +1,4 @@ -[DataWriter.Contact] -# please fill in your contact information here. It will be stored in the -# output files as provenance information -name = "YOUR-NAME-HERE" -email = "YOUREMAIL@EXAMPLE.ORG" -organization = "YOUR-ORGANIZATION" - [DataWriter] -# options that control what is stored in the output file overwrite = false write_images = true write_parameters = true @@ -15,41 +7,25 @@ write_mono_shower = false transform_image = true transform_peak_time = true +[DataWriter.Contact] +name = "YOUR-NAME-HERE" +email = "YOUREMAIL@EXAMPLE.ORG" +organization = "YOUR-ORGANIZATION" + + [CameraCalibrator] image_extractor_type = "NeighborPeakWindowSum" [ImageProcessor] image_cleaner_type = "TailcutsImageCleaner" - - [ImageProcessor.TailcutsImageCleaner] -picture_threshold_pe = [ - [ "type", "*", 10.0,], - [ "type", "LST_LST_LSTCam", 5.0,], - [ "type", "MST_MST_NectarCam", 5.0,], - [ "type", "SST_ASTRI_CHEC", 3.0,], -] -boundary_threshold_pe = [ - [ "type", "*", 5.0,], - [ "type", "LST_LST_LSTCam", 2.5,], - [ "type", "MST_MST_NectarCam", 2.5,], - [ "type", "SST_ASTRI_CHEC", 1.5,], -] +picture_threshold_pe = [ [ "type", "*", 10.0,], [ "type", "LST_LST_LSTCam", 5.0,], [ "type", "MST_MST_NectarCam", 5.0,], [ "type", "SST_ASTRI_CHEC", 3.0,],] +boundary_threshold_pe = [ [ "type", "*", 5.0,], [ "type", "LST_LST_LSTCam", 2.5,], [ "type", "MST_MST_NectarCam", 2.5,], [ "type", "SST_ASTRI_CHEC", 1.5,],] min_picture_neighbors = [ [ "type", "*", 2,],] [ImageProcessor.ImageQualityQuery] -# These specify which images should be parameterized: -quality_criteria = [ - [ "enough_pixels", "lambda im: np.count_nonzero(im) > 2",], - [ "enough_charge", "lambda im: im.sum() > 50",] -] +quality_criteria = [ [ "enough_pixels", "lambda im: np.count_nonzero(im) > 2",], [ "enough_charge", "lambda im: im.sum() > 50",],] [ShowerProcessor.ShowerQualityQuery] -# These specify criteria for telescopes that should be included in stereo -# reconstruction: -quality_criteria = [ - [ "enough intensity", "lambda p: p.hillas.intensity > 50",], - [ "Positive width", "lambda p: p.hillas.width.value > 0",], - [ "enough pixels", "lambda p: p.morphology.num_pixels > 3",], -] +quality_criteria = [ [ "enough intensity", "lambda p: p.hillas.intensity > 50",], [ "Positive width", "lambda p: p.hillas.width.value > 0",], [ "enough pixels", "lambda p: p.morphology.num_pixels > 3",],] From 356235c20912057829ffce53a0773dacb1a05f48 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 17:58:46 +0100 Subject: [PATCH 13/39] fix typo in test --- ctapipe/tools/tests/resources/stage1_config.toml | 6 +++--- ctapipe/tools/tests/test_process.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ctapipe/tools/tests/resources/stage1_config.toml b/ctapipe/tools/tests/resources/stage1_config.toml index 865e786df25..3ee690a0e19 100644 --- a/ctapipe/tools/tests/resources/stage1_config.toml +++ b/ctapipe/tools/tests/resources/stage1_config.toml @@ -20,9 +20,9 @@ image_extractor_type = "NeighborPeakWindowSum" image_cleaner_type = "TailcutsImageCleaner" [ImageProcessor.TailcutsImageCleaner] -picture_threshold_pe = [ [ "type", "*", 10.0,], [ "type", "LST_LST_LSTCam", 5.0,], [ "type", "MST_MST_NectarCam", 5.0,], [ "type", "SST_ASTRI_CHEC", 3.0,],] -boundary_threshold_pe = [ [ "type", "*", 5.0,], [ "type", "LST_LST_LSTCam", 2.5,], [ "type", "MST_MST_NectarCam", 2.5,], [ "type", "SST_ASTRI_CHEC", 1.5,],] -min_picture_neighbors = [ [ "type", "*", 2,],] +picture_threshold_pe = [ [ "type", "*", "10.0",], [ "type", "LST_LST_LSTCam", "5.0",], [ "type", "MST_MST_NectarCam", "5.0",], [ "type", "SST_ASTRI_CHEC", "3.0",],] +boundary_threshold_pe = [ [ "type", "*", "5.0",], [ "type", "LST_LST_LSTCam", "2.5",], [ "type", "MST_MST_NectarCam", "2.5",], [ "type", "SST_ASTRI_CHEC", "1.5",],] +min_picture_neighbors = [ [ "type", "*", "2",],] [ImageProcessor.ImageQualityQuery] quality_criteria = [ [ "enough_pixels", "lambda im: np.count_nonzero(im) > 2",], [ "enough_charge", "lambda im: im.sum() > 50",],] diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index 45c46d67999..01edf6ff917 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -28,7 +28,7 @@ def test_read_yaml_toml_config(dl1_image_file): config = files("ctapipe.tools.tests.resources").joinpath(config_base) tool.load_config_file(config) - tool.EventSource.input_url = dl1_image_file + tool.config.EventSource.input_url = dl1_image_file tool.setup() assert tool.config.DataWriter.name == "YOUR-NAME-HERE" From 52c7afb09a083d687dfbf7942f16737c90a65be6 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 18:12:02 +0100 Subject: [PATCH 14/39] use tomli, fix test and example file --- ctapipe/core/tool.py | 9 +--- .../tools/tests/resources/stage1_config.toml | 46 ++++++++++++++----- ctapipe/tools/tests/test_process.py | 6 ++- environment.yml | 1 + setup.py | 1 + 5 files changed, 44 insertions(+), 19 deletions(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index efe39c4039b..8ede00cc80e 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -21,12 +21,7 @@ HAS_TOML = True except ImportError: - try: - import toml - - HAS_TOML = True - except ImportError: - HAS_TOML = False + HAS_TOML = False from traitlets import default from traitlets.config import Application, Config, Configurable @@ -233,7 +228,7 @@ def load_config_file(self, path: Union[str, pathlib.Path]): config = Config(yaml.safe_load(infile)) self.update_config(config) elif path.suffix == ".toml" and HAS_TOML: - with open(path, "r") as infile: + with open(path, "rb") as infile: config = Config(toml.load(infile)) self.update_config(config) else: diff --git a/ctapipe/tools/tests/resources/stage1_config.toml b/ctapipe/tools/tests/resources/stage1_config.toml index 3ee690a0e19..4a2b1740233 100644 --- a/ctapipe/tools/tests/resources/stage1_config.toml +++ b/ctapipe/tools/tests/resources/stage1_config.toml @@ -1,4 +1,12 @@ +[DataWriter.Contact] +# please fill in your contact information here. It will be stored in the +# output files as provenance information +name = "YOUR-NAME-HERE" +email = "YOUREMAIL@EXAMPLE.ORG" +organization = "YOUR-ORGANIZATION" + [DataWriter] +# options that control what is stored in the output file overwrite = false write_images = true write_parameters = true @@ -7,25 +15,41 @@ write_mono_shower = false transform_image = true transform_peak_time = true -[DataWriter.Contact] -name = "YOUR-NAME-HERE" -email = "YOUREMAIL@EXAMPLE.ORG" -organization = "YOUR-ORGANIZATION" - - [CameraCalibrator] image_extractor_type = "NeighborPeakWindowSum" [ImageProcessor] image_cleaner_type = "TailcutsImageCleaner" + + [ImageProcessor.TailcutsImageCleaner] -picture_threshold_pe = [ [ "type", "*", "10.0",], [ "type", "LST_LST_LSTCam", "5.0",], [ "type", "MST_MST_NectarCam", "5.0",], [ "type", "SST_ASTRI_CHEC", "3.0",],] -boundary_threshold_pe = [ [ "type", "*", "5.0",], [ "type", "LST_LST_LSTCam", "2.5",], [ "type", "MST_MST_NectarCam", "2.5",], [ "type", "SST_ASTRI_CHEC", "1.5",],] -min_picture_neighbors = [ [ "type", "*", "2",],] +picture_threshold_pe = [ + [ "type", "*", 10.0,], + [ "type", "LST_LST_LSTCam", 5.0,], + [ "type", "MST_MST_NectarCam", 5.0,], + [ "type", "SST_ASTRI_CHEC", 3.0,], +] +boundary_threshold_pe = [ + [ "type", "*", 5.0,], + [ "type", "LST_LST_LSTCam", 2.5,], + [ "type", "MST_MST_NectarCam", 2.5,], + [ "type", "SST_ASTRI_CHEC", 1.5,], +] +min_picture_neighbors = [ [ "type", "*", 2,],] [ImageProcessor.ImageQualityQuery] -quality_criteria = [ [ "enough_pixels", "lambda im: np.count_nonzero(im) > 2",], [ "enough_charge", "lambda im: im.sum() > 50",],] +# These specify which images should be parameterized: +quality_criteria = [ + [ "enough_pixels", "lambda im: np.count_nonzero(im) > 2",], + [ "enough_charge", "lambda im: im.sum() > 50",] +] [ShowerProcessor.ShowerQualityQuery] -quality_criteria = [ [ "enough intensity", "lambda p: p.hillas.intensity > 50",], [ "Positive width", "lambda p: p.hillas.width.value > 0",], [ "enough pixels", "lambda p: p.morphology.num_pixels > 3",],] +# These specify criteria for telescopes that should be included in stereo +# reconstruction: +quality_criteria = [ + [ "enough intensity", "lambda p: p.hillas.intensity > 50",], + [ "Positive width", "lambda p: p.hillas.width.value > 0",], + [ "enough pixels", "lambda p: p.morphology.num_pixels > 3",], +] diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index 01edf6ff917..0437f3c1f11 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -29,8 +29,12 @@ def test_read_yaml_toml_config(dl1_image_file): tool.load_config_file(config) tool.config.EventSource.input_url = dl1_image_file + tool.config.DataWriter.overwrite = True tool.setup() - assert tool.config.DataWriter.name == "YOUR-NAME-HERE" + assert ( + tool.get_current_config()["ProcessorTool"]["DataWriter"]["contact_info"].name + == "YOUR-NAME-HERE" + ) def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file): diff --git a/environment.yml b/environment.yml index e87cd2f9132..94590f7fa65 100644 --- a/environment.yml +++ b/environment.yml @@ -36,6 +36,7 @@ dependencies: - sphinx=3.5 - sphinx-automodapi - sphinx_rtd_theme + - tomli - tqdm - traitlets - vitables diff --git a/setup.py b/setup.py index 4613a1ea4b3..1e532b621dd 100755 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ "notebook", "graphviz", "pandas", + "tomli", ] setup( From fe74d713c76775ba33c6c731831898685406f37f Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 18:41:35 +0100 Subject: [PATCH 15/39] allow multiple config files, fixes #1732 --- ctapipe/core/tool.py | 27 ++++---- .../tests/resources/training_config.yaml | 62 +++++++++++++++++++ 2 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 ctapipe/tools/tests/resources/training_config.yaml diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index 8ede00cc80e..0f0b92c2724 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -23,7 +23,7 @@ except ImportError: HAS_TOML = False -from traitlets import default +from traitlets import default, List from traitlets.config import Application, Config, Configurable from .. import __version__ as version @@ -134,16 +134,18 @@ def main(): """ - config_file = Path( - exists=True, - directory_ok=False, - allow_none=True, - default_value=None, - help=( - "name of a configuration file with " - "parameters to load in addition to " - "command-line parameters" - ), + config_file = List( + trait=Path( + exists=True, + directory_ok=False, + allow_none=True, + default_value=None, + help=( + "name of a configuration file with " + "parameters to load in addition to " + "command-line parameters" + ), + ) ).tag(config=True) log_config = Dict(default_value=DEFAULT_LOGGING).tag(config=True) @@ -206,7 +208,8 @@ def initialize(self, argv=None): if self.config_file is not None: self.log.info(f"Loading config from '{self.config_file}'") try: - self.load_config_file(self.config_file) + for config_file in self.config_file: + self.load_config_file(config_file) except Exception as err: raise ToolConfigurationError( f"Couldn't read config file: {err} {type(err)}" diff --git a/ctapipe/tools/tests/resources/training_config.yaml b/ctapipe/tools/tests/resources/training_config.yaml new file mode 100644 index 00000000000..d1ca9c46bbc --- /dev/null +++ b/ctapipe/tools/tests/resources/training_config.yaml @@ -0,0 +1,62 @@ +DataWriter: + Contact: + name: YOUR-NAME-HERE + email: YOUREMAIL@EXAMPLE.ORG + organization: YOUR-ORGANIZATION + overwrite: false + write_images: false + write_parameters: true + write_stereo_shower: true + write_mono_shower: true + transform_image: true + transform_peak_time: true +CameraCalibrator: + image_extractor_type: NeighborPeakWindowSum +ImageProcessor: + image_cleaner_type: TailcutsImageCleaner + TailcutsImageCleaner: + picture_threshold_pe: + - - type + - '*' + - 10.0 + - - type + - LST_LST_LSTCam + - 5.0 + - - type + - MST_MST_NectarCam + - 5.0 + - - type + - SST_ASTRI_CHEC + - 3.0 + boundary_threshold_pe: + - - type + - '*' + - 5.0 + - - type + - LST_LST_LSTCam + - 2.5 + - - type + - MST_MST_NectarCam + - 2.5 + - - type + - SST_ASTRI_CHEC + - 1.5 + min_picture_neighbors: + - - type + - '*' + - 2 + ImageQualityQuery: + quality_criteria: + - - enough_pixels + - 'lambda im: np.count_nonzero(im) > 2' + - - enough_charge + - 'lambda im: im.sum() > 50' +ShowerProcessor: + ShowerQualityQuery: + quality_criteria: + - - enough intensity + - 'lambda p: p.hillas.intensity > 50' + - - Positive width + - 'lambda p: p.hillas.width.value > 0' + - - enough pixels + - 'lambda p: p.morphology.num_pixels > 3' From bd95bb6f5f0d4c211218ab1e418ac29a72d522f2 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 22 Mar 2022 20:33:09 +0100 Subject: [PATCH 16/39] removed unreachable code --- ctapipe/core/tool.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index 0f0b92c2724..e19b20c7a71 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -212,7 +212,7 @@ def initialize(self, argv=None): self.load_config_file(config_file) except Exception as err: raise ToolConfigurationError( - f"Couldn't read config file: {err} {type(err)}" + f"Couldn't read config file: {err} ({type(err)})" ) # ensure command-line takes precedence over config file options: @@ -512,7 +512,6 @@ def run_tool(tool: Tool, argv=None, cwd=None): # switch to cwd for running and back after os.chdir(cwd) tool.run(argv or []) - return 0 except SystemExit as e: return e.code finally: From 6c3d004824c3dd8b1c526ed7235842f1c772e17b Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 23 Mar 2022 10:45:54 +0100 Subject: [PATCH 17/39] add config files to provenance tracker --- ctapipe/core/tool.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index e19b20c7a71..ce5a9288b27 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -238,6 +238,8 @@ def load_config_file(self, path: Union[str, pathlib.Path]): # fall back to traitlets.config.Application's implementation super().load_config_file(str(path)) + Provenance().add_input_file(path, role="Tool Configuration") + def update_logging_config(self): """Update the configuration of loggers.""" cfg = create_logging_config( From 7f6cee95b9102288f84f513a4301e2a9e66317f7 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 23 Mar 2022 10:49:54 +0100 Subject: [PATCH 18/39] updated help string for config_file --- ctapipe/core/tool.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index ce5a9288b27..eea3d40d5ec 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -141,9 +141,11 @@ def main(): allow_none=True, default_value=None, help=( - "name of a configuration file with " - "parameters to load in addition to " - "command-line parameters" + "List of configuration files with parameters to load " + "in addition to command-line parameters. " + "The order listed is the order of precendence (later config parameters " + "overwrite earlier ones), however parameters specified on the " + "command line always have the highest precendence." ), ) ).tag(config=True) From 0693b05233b1ba0e3891cc6ef73a8bdd7e5d049b Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 23 Mar 2022 10:51:02 +0100 Subject: [PATCH 19/39] mention format in config help --- ctapipe/core/tool.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index eea3d40d5ec..ffcbc61f0cc 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -145,7 +145,8 @@ def main(): "in addition to command-line parameters. " "The order listed is the order of precendence (later config parameters " "overwrite earlier ones), however parameters specified on the " - "command line always have the highest precendence." + "command line always have the highest precendence. " + "Config files may be in JSON, YAML, TOML, or Python format" ), ) ).tag(config=True) From 90e5e9d3970ce4f16e1680e71a8ef0fcb0c0b8ba Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 23 Mar 2022 10:54:12 +0100 Subject: [PATCH 20/39] removed accidental commit --- .../tests/resources/training_config.yaml | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 ctapipe/tools/tests/resources/training_config.yaml diff --git a/ctapipe/tools/tests/resources/training_config.yaml b/ctapipe/tools/tests/resources/training_config.yaml deleted file mode 100644 index d1ca9c46bbc..00000000000 --- a/ctapipe/tools/tests/resources/training_config.yaml +++ /dev/null @@ -1,62 +0,0 @@ -DataWriter: - Contact: - name: YOUR-NAME-HERE - email: YOUREMAIL@EXAMPLE.ORG - organization: YOUR-ORGANIZATION - overwrite: false - write_images: false - write_parameters: true - write_stereo_shower: true - write_mono_shower: true - transform_image: true - transform_peak_time: true -CameraCalibrator: - image_extractor_type: NeighborPeakWindowSum -ImageProcessor: - image_cleaner_type: TailcutsImageCleaner - TailcutsImageCleaner: - picture_threshold_pe: - - - type - - '*' - - 10.0 - - - type - - LST_LST_LSTCam - - 5.0 - - - type - - MST_MST_NectarCam - - 5.0 - - - type - - SST_ASTRI_CHEC - - 3.0 - boundary_threshold_pe: - - - type - - '*' - - 5.0 - - - type - - LST_LST_LSTCam - - 2.5 - - - type - - MST_MST_NectarCam - - 2.5 - - - type - - SST_ASTRI_CHEC - - 1.5 - min_picture_neighbors: - - - type - - '*' - - 2 - ImageQualityQuery: - quality_criteria: - - - enough_pixels - - 'lambda im: np.count_nonzero(im) > 2' - - - enough_charge - - 'lambda im: im.sum() > 50' -ShowerProcessor: - ShowerQualityQuery: - quality_criteria: - - - enough intensity - - 'lambda p: p.hillas.intensity > 50' - - - Positive width - - 'lambda p: p.hillas.width.value > 0' - - - enough pixels - - 'lambda p: p.morphology.num_pixels > 3' From 3cbe0350c8281dce77c4212c2d4c940a378158c4 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 11 Apr 2022 22:00:34 +0200 Subject: [PATCH 21/39] a better config file with lots of explanation --- .../tools/tests/resources/stage1_config.yaml | 81 ++++++++++++++----- 1 file changed, 63 insertions(+), 18 deletions(-) diff --git a/ctapipe/tools/tests/resources/stage1_config.yaml b/ctapipe/tools/tests/resources/stage1_config.yaml index cb324542da9..642cff10cb4 100644 --- a/ctapipe/tools/tests/resources/stage1_config.yaml +++ b/ctapipe/tools/tests/resources/stage1_config.yaml @@ -1,3 +1,13 @@ +# ========================================================================== +# ctapipe-process configuration file. +# +# This configuration contains a subset of options needed for a basic analysis. +# Not all possible options are shown. To get a complete list, run: +# +# `ctapipe-process --help-all` +# +# ========================================================================== + DataWriter: Contact: # please fill in your contact information here. It will be stored in the @@ -12,10 +22,27 @@ DataWriter: write_parameters: true # store DL1 parameters write_stereo_shower: false # store DL2 stereo geometry +# The CameraCalibrator takes data from R1 or DL0 to DL1a level, applying finer +# calibration and turning waveforms into images. It is run only if DL1a images +# do not already exist in the input file. CameraCalibrator: + # Choose an extractor type from the following possibilities: + #'FullWaveformSum', 'FixedWindowSum', 'GlobalPeakWindowSum', + #'LocalPeakWindowSum', 'SlidingWindowMaxSum', 'NeighborPeakWindowSum', + #'TwoPassWindowSum', 'BaselineSubtractedNeighborPeakWindowSum' + # + #Note this is a telescope-wise parameter, so can be specified per telescope if + #necessary (see below for an example) image_extractor_type: NeighborPeakWindowSum +# The ImageProcessor performs the DL1a-> DL1b (image parameters) transition. It +# is run only if the parameters `DataWriter.write_image_parameters=True` and the +# parameters don't already exist in the input file (or if the user forces them +# to be re-computed using DataWriter.recompute_dl1=True) ImageProcessor: + # The image cleaner selects pixels which have signal in them and rejects those + # without. Options are: 'TailcutsImageCleaner', 'MARSImageCleaner', + # 'FACTImageCleaner' image_cleaner_type: TailcutsImageCleaner # make sure you include a configuration for the image cleaner you selected @@ -23,30 +50,48 @@ ImageProcessor: # configure it. TailcutsImageCleaner: # the thresholds for this image cleaner must be optimized for the data set - # you are analyzing. The defaults may not be correct. - picture_threshold_pe: - - [type, "*", 10.0] - - [type, LST_LST_LSTCam, 5.0] - - [type, MST_MST_NectarCam, 5.0] - - [type, SST_ASTRI_CHEC, 3.0] - boundary_threshold_pe: - - [type, "*", 5.0] - - [type, LST_LST_LSTCam, 2.5] - - [type, MST_MST_NectarCam, 2.5] - - [type, SST_ASTRI_CHEC, 1.5] - min_picture_neighbors: - - [type, "*", 2] + # you are analyzing. The defaults may not be correct and should be optimized + # for the given use case. + # + # These are telescope-wise parameters, where the options are patterns + # specified in a list in order of precedence, with later options overwriting + # earlier ones. Each pattern is a triplet of [scope, key, value], where the + # scope can be "type" (matching to the telescope type name) or "id" + # (matching a specific telescope ID number). In the case of "type", the key + # should be either a telescope type string, or part of one with "*" used as + # a wildcard match (e.g. "LST*" would match all LST telescopes). You can + # specify a universal default value using "*" as a key. Note that specifying + # a single value is equivalent to specifying a one-item list containing + # [type, '*', value] . + picture_threshold_pe: # top-level threshold in photoelectrons + - [type, "LST*", 6.0] + - [type, "MST*NectarCam", 8.0] + - [type, "SST_ASTRI_CHEC", 4.0] + boundary_threshold_pe: # second-level threshold in photoelectrons + - [type, "LST*", 3.0] + - [type, "MST*NectarCam", 4.0] + - [type, "SST_ASTRI_CHEC", 2.0] + keep_isolated_pixels: False # If False, pixels with < min_picture_neighbors are removed. + min_picture_neighbors: 2 # Minimum number of neighbors above threshold to consider - # These specify which images should be parameterized: + # Choose which images should be parameterized: ImageQualityQuery: + # quality critieria should be a list of pairs of [name, + # filter_function_string] The filter function should take a single value + # which is the image itself, a 1D np.array. quality_criteria: - - [enough_pixels, "lambda im: np.count_nonzero(im) > 2"] - - [enough_charge, "lambda im: im.sum() > 50"] + - ["enough_pixels", "lambda im: np.count_nonzero(im) > 2"] + - ["enough_charge", "lambda im: im.sum() > 50"] +# The ShowerProcessor performs the DL1 to DL2a (reconstructed shower geometry) +# transition. It is run only if the parameter DataWriter.write_stereo_shower=True. ShowerProcessor: - # These specify criteria for telescopes that should be included in stereo - # reconstruction: + # Choose which telescope events should be included in the reconstruction. ShowerQualityQuery: + # the quality criteria here should again be a list of [name, + # filter_function_string] pairs, with filter functions that take the set of + # image parameters, p (a `ctapipe.containers.ImageParametersContainer`), as + # an argument. quality_criteria: - [enough intensity, "lambda p: p.hillas.intensity > 50"] - [Positive width, "lambda p: p.hillas.width.value > 0"] From 84303381d6003278140a0ab9d73b425661ebb0fb Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 11 Apr 2022 22:09:06 +0200 Subject: [PATCH 22/39] updated config --- .../{stage1_config.yaml => base_config.yaml} | 1 + .../tools/tests/resources/stage2_config.json | 53 ------------------- .../tests/resources/training_config.json | 52 ------------------ 3 files changed, 1 insertion(+), 105 deletions(-) rename ctapipe/tools/tests/resources/{stage1_config.yaml => base_config.yaml} (98%) delete mode 100644 ctapipe/tools/tests/resources/stage2_config.json delete mode 100644 ctapipe/tools/tests/resources/training_config.json diff --git a/ctapipe/tools/tests/resources/stage1_config.yaml b/ctapipe/tools/tests/resources/base_config.yaml similarity index 98% rename from ctapipe/tools/tests/resources/stage1_config.yaml rename to ctapipe/tools/tests/resources/base_config.yaml index 642cff10cb4..bb8659078c6 100644 --- a/ctapipe/tools/tests/resources/stage1_config.yaml +++ b/ctapipe/tools/tests/resources/base_config.yaml @@ -96,3 +96,4 @@ ShowerProcessor: - [enough intensity, "lambda p: p.hillas.intensity > 50"] - [Positive width, "lambda p: p.hillas.width.value > 0"] - [enough pixels, "lambda p: p.morphology.num_pixels > 3"] + - [not clipped, "lambda p: p.leakage.intensity_width_2 < 0.8"] diff --git a/ctapipe/tools/tests/resources/stage2_config.json b/ctapipe/tools/tests/resources/stage2_config.json deleted file mode 100644 index adece5cf678..00000000000 --- a/ctapipe/tools/tests/resources/stage2_config.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "DataWriter": { - "Contact": { - "name": "YOUR-NAME-HERE", - "email": "YOUREMAIL@EXAMPLE.ORG", - "organization": "YOUR-ORGANIZATION" - }, - "overwrite": false, - "write_images": false, - "write_parameters": false, - "write_stereo_shower": true, - "write_mono_shower": true, - "transform_image": true, - "transform_peak_time": true - }, - "CameraCalibrator": { - "image_extractor_type": "NeighborPeakWindowSum" - }, - "ImageProcessor": { - "image_cleaner_type": "TailcutsImageCleaner", - "TailcutsImageCleaner": { - "picture_threshold_pe": [ - ["type", "*", 10.0], - ["type", "LST_LST_LSTCam", 5.0], - ["type", "MST_MST_NectarCam", 5.0], - ["type", "SST_ASTRI_CHEC", 3.0] - ], - "boundary_threshold_pe": [ - ["type", "*", 5.0], - ["type", "LST_LST_LSTCam", 2.5], - ["type", "MST_MST_NectarCam", 2.5], - ["type", "SST_ASTRI_CHEC", 1.5] - ], - "min_picture_neighbors": [["type", "*", 2]] - }, - "ImageQualityQuery": { - "quality_criteria": [ - ["enough_pixels", "lambda im: np.count_nonzero(im) > 2"], - ["enough_charge", "lambda im: im.sum() > 50"] - ] - } - }, - "ShowerProcessor": { - "ShowerQualityQuery": { - "quality_criteria": [ - ["enough intensity", "lambda p: p.hillas.intensity > 50"], - ["positive width", "lambda p: p.hillas.width.value > 0"], - ["enough pixels", "lambda p: p.morphology.num_pixels > 3"], - ["not clipped", "lambda p: p.leakage.intensity_width_2 < 2"] - ] - } - } -} diff --git a/ctapipe/tools/tests/resources/training_config.json b/ctapipe/tools/tests/resources/training_config.json deleted file mode 100644 index 0c3decb4e52..00000000000 --- a/ctapipe/tools/tests/resources/training_config.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "DataWriter": { - "Contact": { - "name": "YOUR-NAME-HERE", - "email": "YOUREMAIL@EXAMPLE.ORG", - "organization": "YOUR-ORGANIZATION" - }, - "overwrite": false, - "write_images": false, - "write_parameters": true, - "write_stereo_shower": true, - "write_mono_shower": true, - "transform_image": true, - "transform_peak_time": true - }, - "CameraCalibrator": { - "image_extractor_type": "NeighborPeakWindowSum" - }, - "ImageProcessor": { - "image_cleaner_type": "TailcutsImageCleaner", - "TailcutsImageCleaner": { - "picture_threshold_pe": [ - ["type", "*", 10.0], - ["type", "LST_LST_LSTCam", 5.0], - ["type", "MST_MST_NectarCam", 5.0], - ["type", "SST_ASTRI_CHEC", 3.0] - ], - "boundary_threshold_pe": [ - ["type", "*", 5.0], - ["type", "LST_LST_LSTCam", 2.5], - ["type", "MST_MST_NectarCam", 2.5], - ["type", "SST_ASTRI_CHEC", 1.5] - ], - "min_picture_neighbors": [["type", "*", 2]] - }, - "ImageQualityQuery": { - "quality_criteria": [ - ["enough_pixels", "lambda im: np.count_nonzero(im) > 2"], - ["enough_charge", "lambda im: im.sum() > 50"] - ] - } - }, - "ShowerProcessor": { - "ShowerQualityQuery": { - "quality_criteria": [ - ["enough intensity", "lambda p: p.hillas.intensity > 50"], - ["Positive width", "lambda p: p.hillas.width.value > 0"], - ["enough pixels", "lambda p: p.morphology.num_pixels > 3"] - ] - } - } -} From 14cb0fdc3399cf153b75e68c54fdf68455d51521 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 11 Apr 2022 22:24:56 +0200 Subject: [PATCH 23/39] updated configurations - include version - separate BASE and STAGE configs --- ctapipe/tools/quickstart.py | 29 +++++----- .../tools/tests/resources/base_config.yaml | 1 + .../tools/tests/resources/stage1_config.yaml | 15 ++++++ .../tools/tests/resources/stage2_config.json | 53 +++++++++++++++++++ .../tools/tests/resources/stage2_config.yaml | 16 ++++++ .../tests/resources/training_config.json | 52 ++++++++++++++++++ .../tests/resources/training_config.yaml | 17 ++++++ 7 files changed, 170 insertions(+), 13 deletions(-) create mode 100644 ctapipe/tools/tests/resources/stage1_config.yaml create mode 100644 ctapipe/tools/tests/resources/stage2_config.json create mode 100644 ctapipe/tools/tests/resources/stage2_config.yaml create mode 100644 ctapipe/tools/tests/resources/training_config.json create mode 100644 ctapipe/tools/tests/resources/training_config.yaml diff --git a/ctapipe/tools/quickstart.py b/ctapipe/tools/quickstart.py index 687c5e7a38d..a91dd781a15 100644 --- a/ctapipe/tools/quickstart.py +++ b/ctapipe/tools/quickstart.py @@ -15,11 +15,10 @@ __all__ = ["QuickStartTool"] CONFIGS_TO_WRITE = [ + "base_config.yaml", "stage1_config.yaml", - "stage1_config.toml", - "stage1_config.json", - "stage2_config.json", - "training_config.json", + "stage2_config.yaml", + "training_config.yaml", ] README_TEXT = f""" @@ -29,19 +28,22 @@ This working directory contains some example configuration files that are useful for processing data with `ctapipe-process`. These include: -- stage1_config.json: generate DL1 data from lower data levels -- stage2_config.json: generate DL2 shower geometry from DL1 or lower levels -- training_config.json: generate both DL1 parameter and DL2 shower geometry data +- base_config.yaml: stanard configuration options, to be included always +- stage1_config.yaml: generate DL1 data from lower data levels +- stage2_config.yaml: generate DL2 shower geometry from DL1 or lower levels +- training_config.yaml: generate both DL1 parameter and DL2 shower geometry + data, useful for training ML algorithms -You can modify these to change the output, and run ctapipe using: +You can modify these to change the output, and run ctapipe by including both the +base config plus one additional configuration using: ``` -ctapipe-process --config --input --output +ctapipe-process --config base_config.yaml --config --input --output ``` -Where is any ctapipe-readable event file at a lower or equal data -level to the one requested to be produced, and is one of the -configuration files generated by `ctapipe-quickstart`. +Where is one of the non-base configs above, is any +ctapipe-readable event file at a lower or equal data level to the one requested +to be produced. Details about all configuration options can be found by running: @@ -133,6 +135,7 @@ def setup(self): "YOUR-NAME-HERE": self.contact_name, "YOUREMAIL@EXAMPLE.ORG": self.contact_email, "YOUR-ORGANIZATION": self.contact_organization, + "VERSION": VERSION, } def start(self): @@ -162,7 +165,7 @@ def finish(self): def main(): - """ run the tool""" + """run the tool""" tool = QuickStartTool() tool.run() diff --git a/ctapipe/tools/tests/resources/base_config.yaml b/ctapipe/tools/tests/resources/base_config.yaml index bb8659078c6..2f439395b39 100644 --- a/ctapipe/tools/tests/resources/base_config.yaml +++ b/ctapipe/tools/tests/resources/base_config.yaml @@ -1,5 +1,6 @@ # ========================================================================== # ctapipe-process configuration file. +# version: VERSION # # This configuration contains a subset of options needed for a basic analysis. # Not all possible options are shown. To get a complete list, run: diff --git a/ctapipe/tools/tests/resources/stage1_config.yaml b/ctapipe/tools/tests/resources/stage1_config.yaml new file mode 100644 index 00000000000..77bac36c3c7 --- /dev/null +++ b/ctapipe/tools/tests/resources/stage1_config.yaml @@ -0,0 +1,15 @@ +# ====================================================================== +# ctapipe-process configuration file. +# version: VERSION +# +# Perform Stage 1 Processing +# +# This configuration enables options needed for Rx to DL1b (image parameters) +# ====================================================================== +# +# Make sure you first include `--config base_config.yaml` before including this file + +DataWriter: + write_images: true + write_parameters: true + write_stereo_shower: false diff --git a/ctapipe/tools/tests/resources/stage2_config.json b/ctapipe/tools/tests/resources/stage2_config.json new file mode 100644 index 00000000000..adece5cf678 --- /dev/null +++ b/ctapipe/tools/tests/resources/stage2_config.json @@ -0,0 +1,53 @@ +{ + "DataWriter": { + "Contact": { + "name": "YOUR-NAME-HERE", + "email": "YOUREMAIL@EXAMPLE.ORG", + "organization": "YOUR-ORGANIZATION" + }, + "overwrite": false, + "write_images": false, + "write_parameters": false, + "write_stereo_shower": true, + "write_mono_shower": true, + "transform_image": true, + "transform_peak_time": true + }, + "CameraCalibrator": { + "image_extractor_type": "NeighborPeakWindowSum" + }, + "ImageProcessor": { + "image_cleaner_type": "TailcutsImageCleaner", + "TailcutsImageCleaner": { + "picture_threshold_pe": [ + ["type", "*", 10.0], + ["type", "LST_LST_LSTCam", 5.0], + ["type", "MST_MST_NectarCam", 5.0], + ["type", "SST_ASTRI_CHEC", 3.0] + ], + "boundary_threshold_pe": [ + ["type", "*", 5.0], + ["type", "LST_LST_LSTCam", 2.5], + ["type", "MST_MST_NectarCam", 2.5], + ["type", "SST_ASTRI_CHEC", 1.5] + ], + "min_picture_neighbors": [["type", "*", 2]] + }, + "ImageQualityQuery": { + "quality_criteria": [ + ["enough_pixels", "lambda im: np.count_nonzero(im) > 2"], + ["enough_charge", "lambda im: im.sum() > 50"] + ] + } + }, + "ShowerProcessor": { + "ShowerQualityQuery": { + "quality_criteria": [ + ["enough intensity", "lambda p: p.hillas.intensity > 50"], + ["positive width", "lambda p: p.hillas.width.value > 0"], + ["enough pixels", "lambda p: p.morphology.num_pixels > 3"], + ["not clipped", "lambda p: p.leakage.intensity_width_2 < 2"] + ] + } + } +} diff --git a/ctapipe/tools/tests/resources/stage2_config.yaml b/ctapipe/tools/tests/resources/stage2_config.yaml new file mode 100644 index 00000000000..00ae9725b94 --- /dev/null +++ b/ctapipe/tools/tests/resources/stage2_config.yaml @@ -0,0 +1,16 @@ +# ====================================================================== +# ctapipe-process configuration file. +# version: VERSION +# +# Perform Stage 2 Processing +# +# This configuration enables options needed for Rx to DL2a (shower +# reconstruction) But not including any image parameters +# ====================================================================== +# +# Make sure you first include `--config base_config.yaml` before including this file + +DataWriter: + write_images: false + write_parameters: false + write_stereo_shower: true diff --git a/ctapipe/tools/tests/resources/training_config.json b/ctapipe/tools/tests/resources/training_config.json new file mode 100644 index 00000000000..0c3decb4e52 --- /dev/null +++ b/ctapipe/tools/tests/resources/training_config.json @@ -0,0 +1,52 @@ +{ + "DataWriter": { + "Contact": { + "name": "YOUR-NAME-HERE", + "email": "YOUREMAIL@EXAMPLE.ORG", + "organization": "YOUR-ORGANIZATION" + }, + "overwrite": false, + "write_images": false, + "write_parameters": true, + "write_stereo_shower": true, + "write_mono_shower": true, + "transform_image": true, + "transform_peak_time": true + }, + "CameraCalibrator": { + "image_extractor_type": "NeighborPeakWindowSum" + }, + "ImageProcessor": { + "image_cleaner_type": "TailcutsImageCleaner", + "TailcutsImageCleaner": { + "picture_threshold_pe": [ + ["type", "*", 10.0], + ["type", "LST_LST_LSTCam", 5.0], + ["type", "MST_MST_NectarCam", 5.0], + ["type", "SST_ASTRI_CHEC", 3.0] + ], + "boundary_threshold_pe": [ + ["type", "*", 5.0], + ["type", "LST_LST_LSTCam", 2.5], + ["type", "MST_MST_NectarCam", 2.5], + ["type", "SST_ASTRI_CHEC", 1.5] + ], + "min_picture_neighbors": [["type", "*", 2]] + }, + "ImageQualityQuery": { + "quality_criteria": [ + ["enough_pixels", "lambda im: np.count_nonzero(im) > 2"], + ["enough_charge", "lambda im: im.sum() > 50"] + ] + } + }, + "ShowerProcessor": { + "ShowerQualityQuery": { + "quality_criteria": [ + ["enough intensity", "lambda p: p.hillas.intensity > 50"], + ["Positive width", "lambda p: p.hillas.width.value > 0"], + ["enough pixels", "lambda p: p.morphology.num_pixels > 3"] + ] + } + } +} diff --git a/ctapipe/tools/tests/resources/training_config.yaml b/ctapipe/tools/tests/resources/training_config.yaml new file mode 100644 index 00000000000..3258342407f --- /dev/null +++ b/ctapipe/tools/tests/resources/training_config.yaml @@ -0,0 +1,17 @@ +# ====================================================================== +# ctapipe-process configuration file. +# version: VERSION +# +# Perform Stage 1 and Stage 2a processing to generate features needed for +# training machine learning algorithms. +# +# Note that here write_images is disabled by default (to avoid very large +# files), however for training deep learning algorithms, it should be turned on. +# ====================================================================== +# +# Make sure you first include `--config base_config.yaml` before including this file + +DataWriter: + write_images: false + write_parameters: true + write_stereo_shower: true From b51be12574a80a72cd0b03294125e3b12c783e61 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 11 Apr 2022 22:35:43 +0200 Subject: [PATCH 24/39] better test for YAML and TOML files --- ctapipe/tools/tests/test_process.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index e6b56366ac8..e9387422970 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -25,10 +25,18 @@ def resource_file(filename): return files("ctapipe.tools.tests").joinpath("resources", filename) -def test_read_yaml_toml_config(dl1_image_file): +@pytest.mark.parametrize( + "config_files", + [ + ("base_config.yaml", "stage1_config.yaml"), + ("stage1_config.toml",), + ("stage1_config.json",), + ], +) +def test_read_yaml_toml_config(dl1_image_file, config_files): tool = ProcessorTool() - for config_base in ["stage1_config.yaml", "stage1_config.toml"]: + for config_base in config_files: config = resource_file(config_base) tool.load_config_file(config) @@ -42,7 +50,7 @@ def test_read_yaml_toml_config(dl1_image_file): def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file): - """ check simtel to DL1 conversion """ + """check simtel to DL1 conversion""" config = resource_file("stage1_config.json") # DL1A file as input @@ -118,7 +126,7 @@ def test_stage1_datalevels(tmp_path): """test the dl1 tool on a file not providing r1, dl0 or dl1a""" class DummyEventSource(EventSource): - """ for testing """ + """for testing""" @staticmethod def is_compatible(file_path): @@ -173,7 +181,7 @@ def _generator(self): def test_stage_2_from_simtel(tmp_path): - """ check we can go to DL2 geometry from simtel file """ + """check we can go to DL2 geometry from simtel file""" config = resource_file("stage2_config.json") output = tmp_path / "test_stage2_from_simtel.DL2.h5" @@ -198,7 +206,7 @@ def test_stage_2_from_simtel(tmp_path): def test_stage_2_from_dl1_images(tmp_path, dl1_image_file): - """ check we can go to DL2 geometry from DL1 images """ + """check we can go to DL2 geometry from DL1 images""" config = resource_file("stage2_config.json") output = tmp_path / "test_stage2_from_dl1image.DL2.h5" @@ -222,7 +230,7 @@ def test_stage_2_from_dl1_images(tmp_path, dl1_image_file): def test_stage_2_from_dl1_params(tmp_path, dl1_parameters_file): - """ check we can go to DL2 geometry from DL1 parameters """ + """check we can go to DL2 geometry from DL1 parameters""" config = resource_file("stage2_config.json") output = tmp_path / "test_stage2_from_dl1param.DL2.h5" @@ -247,7 +255,7 @@ def test_stage_2_from_dl1_params(tmp_path, dl1_parameters_file): def test_training_from_simtel(tmp_path): - """ check we can write both dl1 and dl2 info (e.g. for training input) """ + """check we can write both dl1 and dl2 info (e.g. for training input)""" config = resource_file("training_config.json") output = tmp_path / "test_training.DL1DL2.h5" @@ -306,7 +314,7 @@ def test_image_modifications(tmp_path, dl1_image_file): @pytest.mark.parametrize("filename", CONFIGS_TO_WRITE) def test_quickstart_templates(filename): - """ ensure template configs have an appropriate placeholder for the contact info """ + """ensure template configs have an appropriate placeholder for the contact info""" config = resource_file(filename) text = config.read_text() @@ -316,7 +324,7 @@ def test_quickstart_templates(filename): def test_quickstart(tmp_path): - """ ensure quickstart tool generates expected output """ + """ensure quickstart tool generates expected output""" tool = QuickStartTool() run_tool( From 0bfa2fc4299b4e2f8e9c52e80cabf38927a2e245 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 11 Apr 2022 22:35:58 +0200 Subject: [PATCH 25/39] reformatting --- ctapipe/tools/tests/resources/base_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe/tools/tests/resources/base_config.yaml b/ctapipe/tools/tests/resources/base_config.yaml index 2f439395b39..e31387d4974 100644 --- a/ctapipe/tools/tests/resources/base_config.yaml +++ b/ctapipe/tools/tests/resources/base_config.yaml @@ -32,8 +32,8 @@ CameraCalibrator: #'LocalPeakWindowSum', 'SlidingWindowMaxSum', 'NeighborPeakWindowSum', #'TwoPassWindowSum', 'BaselineSubtractedNeighborPeakWindowSum' # - #Note this is a telescope-wise parameter, so can be specified per telescope if - #necessary (see below for an example) + # Note this is a telescope-wise parameter, so can be specified per telescope + # if necessary (see below for an example) image_extractor_type: NeighborPeakWindowSum # The ImageProcessor performs the DL1a-> DL1b (image parameters) transition. It From cd51134edacd72c25082f89335a81b6f4e9cf262 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 11 Apr 2022 22:44:35 +0200 Subject: [PATCH 26/39] add test for multiple configs --- ctapipe/tools/tests/test_process.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index e9387422970..0d0c7b756fd 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -49,6 +49,23 @@ def test_read_yaml_toml_config(dl1_image_file, config_files): ) +def test_multiple_configs(dl1_image_file): + tool = ProcessorTool() + + tool.load_config_file(resource_file("base_config.yaml")) + tool.load_config_file(resource_file("stage2_config.yaml")) + + tool.config.EventSource.input_url = dl1_image_file + tool.config.DataWriter.overwrite = True + tool.setup() + + # ensure the overwriting works (base config has this option disabled) + assert ( + tool.get_current_config()["ProcessorTool"]["DataWriter"]["write_stereo_shower"] + == True + ) + + def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file): """check simtel to DL1 conversion""" config = resource_file("stage1_config.json") From 7933565eabd038fed7d10bc060cef54f0a749b35 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 11 Apr 2022 22:46:17 +0200 Subject: [PATCH 27/39] clean up tests --- ctapipe/tools/tests/test_process.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index 0d0c7b756fd..247e468441a 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -33,7 +33,8 @@ def resource_file(filename): ("stage1_config.json",), ], ) -def test_read_yaml_toml_config(dl1_image_file, config_files): +def test_read_yaml_toml_json_config(dl1_image_file, config_files): + """check that we can read multiple formats of config file""" tool = ProcessorTool() for config_base in config_files: @@ -50,6 +51,7 @@ def test_read_yaml_toml_config(dl1_image_file, config_files): def test_multiple_configs(dl1_image_file): + """ensure a config file loaded later overwrites keys from an earlier one""" tool = ProcessorTool() tool.load_config_file(resource_file("base_config.yaml")) From 47c6ee2b21f8cd6745f2231f0a65021c7523e190 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 11 Apr 2022 22:49:21 +0200 Subject: [PATCH 28/39] fix typo and add detail --- ctapipe/tools/quickstart.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ctapipe/tools/quickstart.py b/ctapipe/tools/quickstart.py index a91dd781a15..298d6755c87 100644 --- a/ctapipe/tools/quickstart.py +++ b/ctapipe/tools/quickstart.py @@ -28,7 +28,10 @@ This working directory contains some example configuration files that are useful for processing data with `ctapipe-process`. These include: -- base_config.yaml: stanard configuration options, to be included always +- base_config.yaml: standard configuration options, to be included always + +In addition several sub-configurations to be included after base_config.yaml + - stage1_config.yaml: generate DL1 data from lower data levels - stage2_config.yaml: generate DL2 shower geometry from DL1 or lower levels - training_config.yaml: generate both DL1 parameter and DL2 shower geometry From 9db64f4537a7a2d7e7e64f2580756862705ed6a6 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 11 Apr 2022 23:00:37 +0200 Subject: [PATCH 29/39] fix some codacy warnings --- ctapipe/core/tool.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index ffcbc61f0cc..b59adf209a4 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -204,7 +204,7 @@ def __init__(self, **kwargs): self.update_logging_config() def initialize(self, argv=None): - """ handle config and any other low-level setup """ + """handle config and any other low-level setup""" self.parse_command_line(argv) self.update_logging_config() @@ -216,7 +216,7 @@ def initialize(self, argv=None): except Exception as err: raise ToolConfigurationError( f"Couldn't read config file: {err} ({type(err)})" - ) + ) from err # ensure command-line takes precedence over config file options: self.update_config(self.cli_config) @@ -224,7 +224,16 @@ def initialize(self, argv=None): self.log.info(f"ctapipe version {self.version_string}") - def load_config_file(self, path: Union[str, pathlib.Path]): + def load_config_file(self, path: Union[str, pathlib.Path]) -> None: + """ + Load a configuration file in one of the supported formats, and merge it with + the current config if it exists. + + Parameters + ---------- + path: Union[str, pathlib.Path] + config file to load. [yaml, toml, json, py] formats are supported + """ path = pathlib.Path(path) @@ -372,7 +381,7 @@ def write_provenance(self): @property def version_string(self): - """ a formatted version string with version, release, and git hash""" + """a formatted version string with version, release, and git hash""" return f"{version}" def get_current_config(self): @@ -392,7 +401,7 @@ def get_current_config(self): return conf def _repr_html_(self): - """ nice HTML rep, with blue for non-default values""" + """nice HTML rep, with blue for non-default values""" traits = self.traits() name = self.__class__.__name__ lines = [ From d8cf6b3d7632cc5a1f0807863a39761e54445067 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Mon, 11 Apr 2022 23:07:50 +0200 Subject: [PATCH 30/39] change Tool.config_file -> config_files since list --- ctapipe/core/tool.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index b59adf209a4..818ed63e342 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -14,7 +14,6 @@ HAS_YAML = True except ImportError: HAS_YAML = False - pass # no support for YAML try: import tomli as toml @@ -134,7 +133,7 @@ def main(): """ - config_file = List( + config_files = List( trait=Path( exists=True, directory_ok=False, @@ -178,7 +177,7 @@ def __init__(self, **kwargs): # make sure there are some default aliases in all Tools: super().__init__(**kwargs) aliases = { - ("c", "config"): "Tool.config_file", + ("c", "config"): "Tool.config_files", "log-level": "Tool.log_level", ("l", "log-file"): "Tool.log_file", "log-file-level": "Tool.log_file_level", @@ -208,10 +207,10 @@ def initialize(self, argv=None): self.parse_command_line(argv) self.update_logging_config() - if self.config_file is not None: - self.log.info(f"Loading config from '{self.config_file}'") + if self.config_files is not None: + self.log.info(f"Loading config from '%s'", self.config_files) try: - for config_file in self.config_file: + for config_file in self.config_files: self.load_config_file(config_file) except Exception as err: raise ToolConfigurationError( From 5a052e017a9bda98540e1738200ed21226a60194 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Tue, 12 Apr 2022 09:58:20 +0200 Subject: [PATCH 31/39] fix incorrect f-string (now a log statement) --- ctapipe/core/tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index 818ed63e342..36408ae90dd 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -208,7 +208,7 @@ def initialize(self, argv=None): self.update_logging_config() if self.config_files is not None: - self.log.info(f"Loading config from '%s'", self.config_files) + self.log.info("Loading config from '%s'", self.config_files) try: for config_file in self.config_files: self.load_config_file(config_file) From 94014009ff63465d3be7843f3c3f52fb252d468e Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 13 Apr 2022 18:17:33 +0200 Subject: [PATCH 32/39] update to reflect the new ShowerProcessor --- .../tools/tests/resources/base_config.yaml | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/ctapipe/tools/tests/resources/base_config.yaml b/ctapipe/tools/tests/resources/base_config.yaml index e31387d4974..e14e0d94b05 100644 --- a/ctapipe/tools/tests/resources/base_config.yaml +++ b/ctapipe/tools/tests/resources/base_config.yaml @@ -87,14 +87,19 @@ ImageProcessor: # The ShowerProcessor performs the DL1 to DL2a (reconstructed shower geometry) # transition. It is run only if the parameter DataWriter.write_stereo_shower=True. ShowerProcessor: - # Choose which telescope events should be included in the reconstruction. - ShowerQualityQuery: - # the quality criteria here should again be a list of [name, - # filter_function_string] pairs, with filter functions that take the set of - # image parameters, p (a `ctapipe.containers.ImageParametersContainer`), as - # an argument. - quality_criteria: - - [enough intensity, "lambda p: p.hillas.intensity > 50"] - - [Positive width, "lambda p: p.hillas.width.value > 0"] - - [enough pixels, "lambda p: p.morphology.num_pixels > 3"] - - [not clipped, "lambda p: p.leakage.intensity_width_2 < 0.8"] + # choose between HillasReconstructor and HillasIntersection (two + # implementations of the standard stereo line-intersection method) + reconstructor_type: HillasReconstructor + + HillasReconstructor: + # Choose which telescope events should be included in the reconstruction. + StereoQualityQuery: + # the quality criteria here should again be a list of [name, + # filter_function_string] pairs, with filter functions that take the set of + # image parameters, `p` (a `ctapipe.containers.ImageParametersContainer`), as + # an argument. + quality_criteria: + - [enough intensity, "lambda p: p.hillas.intensity > 50"] + - [Positive width, "lambda p: p.hillas.width.value > 0"] + - [enough pixels, "lambda p: p.morphology.num_pixels > 3"] + - [not clipped, "lambda p: p.leakage.intensity_width_2 < 0.8"] From 9383ff0d5dd6172b0ce1ce304d6d2efd40387afe Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 13 Apr 2022 18:21:57 +0200 Subject: [PATCH 33/39] added FlashCam config So that we don't crash if Prod5b files are processed with all telescopes --- ctapipe/tools/tests/resources/base_config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ctapipe/tools/tests/resources/base_config.yaml b/ctapipe/tools/tests/resources/base_config.yaml index e14e0d94b05..980a8d3f626 100644 --- a/ctapipe/tools/tests/resources/base_config.yaml +++ b/ctapipe/tools/tests/resources/base_config.yaml @@ -67,10 +67,12 @@ ImageProcessor: picture_threshold_pe: # top-level threshold in photoelectrons - [type, "LST*", 6.0] - [type, "MST*NectarCam", 8.0] + - [type, "MST*FlashCam", 10000] # disabled for now - [type, "SST_ASTRI_CHEC", 4.0] boundary_threshold_pe: # second-level threshold in photoelectrons - [type, "LST*", 3.0] - [type, "MST*NectarCam", 4.0] + - [type, "MST*FlashCam", 10000] # disabled for now - [type, "SST_ASTRI_CHEC", 2.0] keep_isolated_pixels: False # If False, pixels with < min_picture_neighbors are removed. min_picture_neighbors: 2 # Minimum number of neighbors above threshold to consider From 9ff865124835b123a7c169ad9e14c081485f3b17 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 13 Apr 2022 18:37:50 +0200 Subject: [PATCH 34/39] turn off all output by default in base config --- ctapipe/tools/tests/resources/base_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe/tools/tests/resources/base_config.yaml b/ctapipe/tools/tests/resources/base_config.yaml index 980a8d3f626..c62e8ef0b35 100644 --- a/ctapipe/tools/tests/resources/base_config.yaml +++ b/ctapipe/tools/tests/resources/base_config.yaml @@ -19,8 +19,8 @@ DataWriter: # options that control what is stored in the output file overwrite: false # do not overwrite existing files - write_images: true # store DL1 images - write_parameters: true # store DL1 parameters + write_images: false # store DL1 images + write_parameters: false # store DL1 parameters write_stereo_shower: false # store DL2 stereo geometry # The CameraCalibrator takes data from R1 or DL0 to DL1a level, applying finer From f4c89b455d57fe1500b9ec44992d7df7fee24e60 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 13 Apr 2022 18:50:18 +0200 Subject: [PATCH 35/39] fixed test --- ctapipe/tools/tests/test_process.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index 247e468441a..91fa8349c83 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -331,7 +331,9 @@ def test_image_modifications(tmp_path, dl1_image_file): assert modified_images["image"].sum() / unmodified_images["image"].sum() > 1.5 -@pytest.mark.parametrize("filename", CONFIGS_TO_WRITE) +@pytest.mark.parametrize( + "filename", ["base_config.yaml", "stage1_config.json", "stage1_config.toml"] +) def test_quickstart_templates(filename): """ensure template configs have an appropriate placeholder for the contact info""" config = resource_file(filename) From 376be7acb0f5750673c2987c67148f58817f4f70 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 13 Apr 2022 18:50:25 +0200 Subject: [PATCH 36/39] added other write options, just for discoverability --- ctapipe/tools/tests/resources/base_config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ctapipe/tools/tests/resources/base_config.yaml b/ctapipe/tools/tests/resources/base_config.yaml index c62e8ef0b35..737b0a93cff 100644 --- a/ctapipe/tools/tests/resources/base_config.yaml +++ b/ctapipe/tools/tests/resources/base_config.yaml @@ -18,10 +18,13 @@ DataWriter: organization: YOUR-ORGANIZATION # options that control what is stored in the output file + # by default here we write nothing (overide these in overwrite: false # do not overwrite existing files write_images: false # store DL1 images write_parameters: false # store DL1 parameters write_stereo_shower: false # store DL2 stereo geometry + write_raw_waveforms: false # write R0 waveforms + write_waveforms: false # write R1 waveforms # The CameraCalibrator takes data from R1 or DL0 to DL1a level, applying finer # calibration and turning waveforms into images. It is run only if DL1a images From 1ff1b9d9d8ddfaafdefabd14f41959b1b1e0855f Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 13 Apr 2022 18:57:37 +0200 Subject: [PATCH 37/39] move ctapipe.tools.test.resources to ctapipe.resources --- ctapipe/{tools/tests => }/resources/base_config.yaml | 0 .../{tools/tests => }/resources/image_modification_config.json | 0 ctapipe/{tools/tests => }/resources/stage1_config.json | 0 ctapipe/{tools/tests => }/resources/stage1_config.toml | 0 ctapipe/{tools/tests => }/resources/stage1_config.yaml | 0 ctapipe/{tools/tests => }/resources/stage2_config.json | 0 ctapipe/{tools/tests => }/resources/stage2_config.yaml | 0 ctapipe/{tools/tests => }/resources/training_config.json | 0 ctapipe/{tools/tests => }/resources/training_config.yaml | 0 ctapipe/tools/quickstart.py | 2 +- ctapipe/tools/tests/test_merge.py | 2 +- ctapipe/tools/tests/test_process.py | 2 +- 12 files changed, 3 insertions(+), 3 deletions(-) rename ctapipe/{tools/tests => }/resources/base_config.yaml (100%) rename ctapipe/{tools/tests => }/resources/image_modification_config.json (100%) rename ctapipe/{tools/tests => }/resources/stage1_config.json (100%) rename ctapipe/{tools/tests => }/resources/stage1_config.toml (100%) rename ctapipe/{tools/tests => }/resources/stage1_config.yaml (100%) rename ctapipe/{tools/tests => }/resources/stage2_config.json (100%) rename ctapipe/{tools/tests => }/resources/stage2_config.yaml (100%) rename ctapipe/{tools/tests => }/resources/training_config.json (100%) rename ctapipe/{tools/tests => }/resources/training_config.yaml (100%) diff --git a/ctapipe/tools/tests/resources/base_config.yaml b/ctapipe/resources/base_config.yaml similarity index 100% rename from ctapipe/tools/tests/resources/base_config.yaml rename to ctapipe/resources/base_config.yaml diff --git a/ctapipe/tools/tests/resources/image_modification_config.json b/ctapipe/resources/image_modification_config.json similarity index 100% rename from ctapipe/tools/tests/resources/image_modification_config.json rename to ctapipe/resources/image_modification_config.json diff --git a/ctapipe/tools/tests/resources/stage1_config.json b/ctapipe/resources/stage1_config.json similarity index 100% rename from ctapipe/tools/tests/resources/stage1_config.json rename to ctapipe/resources/stage1_config.json diff --git a/ctapipe/tools/tests/resources/stage1_config.toml b/ctapipe/resources/stage1_config.toml similarity index 100% rename from ctapipe/tools/tests/resources/stage1_config.toml rename to ctapipe/resources/stage1_config.toml diff --git a/ctapipe/tools/tests/resources/stage1_config.yaml b/ctapipe/resources/stage1_config.yaml similarity index 100% rename from ctapipe/tools/tests/resources/stage1_config.yaml rename to ctapipe/resources/stage1_config.yaml diff --git a/ctapipe/tools/tests/resources/stage2_config.json b/ctapipe/resources/stage2_config.json similarity index 100% rename from ctapipe/tools/tests/resources/stage2_config.json rename to ctapipe/resources/stage2_config.json diff --git a/ctapipe/tools/tests/resources/stage2_config.yaml b/ctapipe/resources/stage2_config.yaml similarity index 100% rename from ctapipe/tools/tests/resources/stage2_config.yaml rename to ctapipe/resources/stage2_config.yaml diff --git a/ctapipe/tools/tests/resources/training_config.json b/ctapipe/resources/training_config.json similarity index 100% rename from ctapipe/tools/tests/resources/training_config.json rename to ctapipe/resources/training_config.json diff --git a/ctapipe/tools/tests/resources/training_config.yaml b/ctapipe/resources/training_config.yaml similarity index 100% rename from ctapipe/tools/tests/resources/training_config.yaml rename to ctapipe/resources/training_config.yaml diff --git a/ctapipe/tools/quickstart.py b/ctapipe/tools/quickstart.py index 298d6755c87..a4c2446ec67 100644 --- a/ctapipe/tools/quickstart.py +++ b/ctapipe/tools/quickstart.py @@ -144,7 +144,7 @@ def setup(self): def start(self): for filename in CONFIGS_TO_WRITE: - config = files("ctapipe.tools.tests").joinpath("resources", filename) + config = files("ctapipe").joinpath("resources", filename) destination = self.workdir / filename if destination.exists(): diff --git a/ctapipe/tools/tests/test_merge.py b/ctapipe/tools/tests/test_merge.py index c1d2408006c..446bff21afe 100644 --- a/ctapipe/tools/tests/test_merge.py +++ b/ctapipe/tools/tests/test_merge.py @@ -18,7 +18,7 @@ def run_stage1(input_path, cwd, output_path=None): - config = files("ctapipe.tools.tests").joinpath("resources", "stage1_config.json") + config = files("ctapipe").joinpath("resources", "stage1_config.json") if output_path is None: output_path = Path( diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index 91fa8349c83..d8f1a4b88ec 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -22,7 +22,7 @@ def resource_file(filename): - return files("ctapipe.tools.tests").joinpath("resources", filename) + return files("ctapipe").joinpath("resources", filename) @pytest.mark.parametrize( From cf84524d9add46343f87a67ac902d9b6df2e01d2 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 13 Apr 2022 19:15:31 +0200 Subject: [PATCH 38/39] require pyyaml and remove HAS_YAML --- ctapipe/core/tool.py | 10 ++-------- setup.py | 9 +++++++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ctapipe/core/tool.py b/ctapipe/core/tool.py index 36408ae90dd..71c8bbd0066 100644 --- a/ctapipe/core/tool.py +++ b/ctapipe/core/tool.py @@ -7,13 +7,7 @@ import textwrap from abc import abstractmethod from typing import Union - -try: - import yaml - - HAS_YAML = True -except ImportError: - HAS_YAML = False +import yaml try: import tomli as toml @@ -236,7 +230,7 @@ def load_config_file(self, path: Union[str, pathlib.Path]) -> None: path = pathlib.Path(path) - if path.suffix in [".yaml", ".yml"] and HAS_YAML: + if path.suffix in [".yaml", ".yml"]: # do our own YAML loading with open(path, "r") as infile: config = Config(yaml.safe_load(infile)) diff --git a/setup.py b/setup.py index 47b3ce12a30..202d55827b2 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,12 @@ "ctapipe-fileinfo = ctapipe.tools.fileinfo:main", "ctapipe-quickstart = ctapipe.tools.quickstart:main", ] -tests_require = ["pytest", "pandas>=0.24.0", "importlib_resources;python_version<'3.9'"] +tests_require = [ + "pytest", + "pandas>=0.24.0", + "importlib_resources;python_version<'3.9'", + "tomli", +] docs_require = [ "sphinx_rtd_theme", "sphinx_automodapi", @@ -31,7 +36,6 @@ "notebook", "graphviz", "pandas", - "tomli", ] setup( @@ -58,6 +62,7 @@ "setuptools_scm>=3.4", "importlib_resources;python_version<'3.9'", "jinja2~=3.0.2", # for sphinx 3.5, update when moving to 4.x + "pyyaml>=5.1", ], # here are optional dependencies (as "tag" : "dependency spec") extras_require={ From ca66dbaced5f6731426a907c282591a518afe94b Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 13 Apr 2022 19:18:16 +0200 Subject: [PATCH 39/39] fixed sentence that ended too soon --- ctapipe/resources/base_config.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ctapipe/resources/base_config.yaml b/ctapipe/resources/base_config.yaml index 737b0a93cff..713bf491c74 100644 --- a/ctapipe/resources/base_config.yaml +++ b/ctapipe/resources/base_config.yaml @@ -17,8 +17,9 @@ DataWriter: email: YOUREMAIL@EXAMPLE.ORG organization: YOUR-ORGANIZATION - # options that control what is stored in the output file - # by default here we write nothing (overide these in + # options that control what is stored in the output file by default here we + # write nothing (can be overridden on the command-line or in subsequent config + # files) overwrite: false # do not overwrite existing files write_images: false # store DL1 images write_parameters: false # store DL1 parameters