From 54fa6790aeb9c8174f7a91a98a700334a9928fce Mon Sep 17 00:00:00 2001 From: Noah Biederbeck Date: Tue, 3 Aug 2021 18:13:41 +0200 Subject: [PATCH] include stage1_config.json in package (#1752) * Include stage1_config.json in package. * Use included stage1_config.json. * Add importlib_resources to tests_require. * Move stage1_config.json to tool test resources. * Require importlib_resources only for python < 3.9 * Remove examples/*.json from package_data * Add the resources to package_data. * Test the inclusion of resources in CI. * Use importlib resources with ProcessorTool --- .github/workflows/ci.yml | 2 +- .../tools/tests/resources}/stage1_config.json | 0 .../tools/tests/resources}/stage2_config.json | 0 .../tests/resources}/training_config.json | 0 ctapipe/tools/tests/test_merge.py | 7 ++++++- ctapipe/tools/tests/test_process.py | 19 +++++++++++-------- setup.py | 6 ++++-- 7 files changed, 22 insertions(+), 12 deletions(-) rename {examples => ctapipe/tools/tests/resources}/stage1_config.json (100%) rename {examples => ctapipe/tools/tests/resources}/stage2_config.json (100%) rename {examples => ctapipe/tools/tests/resources}/training_config.json (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07a471f1ea7..872e5a0a02b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: source .github/install.sh python --version pip install codecov pytest-cov pyflakes pytest-xdist - pip install -e .[all] + pip install .[all] pip freeze - name: Static codechecks diff --git a/examples/stage1_config.json b/ctapipe/tools/tests/resources/stage1_config.json similarity index 100% rename from examples/stage1_config.json rename to ctapipe/tools/tests/resources/stage1_config.json diff --git a/examples/stage2_config.json b/ctapipe/tools/tests/resources/stage2_config.json similarity index 100% rename from examples/stage2_config.json rename to ctapipe/tools/tests/resources/stage2_config.json diff --git a/examples/training_config.json b/ctapipe/tools/tests/resources/training_config.json similarity index 100% rename from examples/training_config.json rename to ctapipe/tools/tests/resources/training_config.json diff --git a/ctapipe/tools/tests/test_merge.py b/ctapipe/tools/tests/test_merge.py index 6ad291046b0..0a6d444973c 100644 --- a/ctapipe/tools/tests/test_merge.py +++ b/ctapipe/tools/tests/test_merge.py @@ -7,9 +7,14 @@ from ctapipe.tools.process import ProcessorTool +try: + from importlib.resources import files +except ImportError: + from importlib_resources import files + def run_stage1(input_path, cwd, output_path=None): - config = Path("./examples/stage1_config.json").absolute() + config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json") if output_path is None: output_path = Path( diff --git a/ctapipe/tools/tests/test_process.py b/ctapipe/tools/tests/test_process.py index a2925531663..2cea4de3dff 100644 --- a/ctapipe/tools/tests/test_process.py +++ b/ctapipe/tools/tests/test_process.py @@ -4,8 +4,6 @@ Test ctapipe-process on a few different use cases """ -from pathlib import Path - import pandas as pd import tables from ctapipe.core import run_tool @@ -13,13 +11,18 @@ from ctapipe.utils import get_dataset_path from ctapipe.io import DataLevel, EventSource +try: + from importlib.resources import files +except ImportError: + from importlib_resources import files + GAMMA_TEST_LARGE = get_dataset_path("gamma_test_large.simtel.gz") def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file): """ check simtel to DL1 conversion """ - config = Path("./examples/stage1_config.json").absolute() + config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json") # DL1A file as input dl1b_from_dl1a_file = tmp_path / "dl1b_fromdl1a.dl1.h5" assert ( @@ -124,7 +127,7 @@ def _generator(self): infile.write(b"dummy") infile.flush() - config = Path("./examples/stage1_config.json").absolute() + config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json") tool = ProcessorTool() assert ( @@ -147,7 +150,7 @@ def _generator(self): def test_stage_2_from_simtel(tmp_path): """ check we can go to DL2 geometry from simtel file """ - config = Path("./examples/stage2_config.json").absolute() + config = files("ctapipe.tools.tests.resources").joinpath("stage2_config.json") output = tmp_path / "test_stage2_from_simtel.DL2.h5" assert ( @@ -172,7 +175,7 @@ def test_stage_2_from_simtel(tmp_path): def test_stage_2_from_dl1_images(tmp_path, dl1_image_file): """ check we can go to DL2 geometry from DL1 images """ - config = Path("./examples/stage2_config.json").absolute() + config = files("ctapipe.tools.tests.resources").joinpath("stage2_config.json") output = tmp_path / "test_stage2_from_dl1image.DL2.h5" assert ( @@ -197,7 +200,7 @@ def test_stage_2_from_dl1_images(tmp_path, dl1_image_file): def test_stage_2_from_dl1_params(tmp_path, dl1_parameters_file): """ check we can go to DL2 geometry from DL1 parameters """ - config = Path("./examples/stage2_config.json").absolute() + config = files("ctapipe.tools.tests.resources").joinpath("stage2_config.json") output = tmp_path / "test_stage2_from_dl1param.DL2.h5" assert ( @@ -222,7 +225,7 @@ def test_stage_2_from_dl1_params(tmp_path, dl1_parameters_file): def test_training_from_simtel(tmp_path): """ check we can write both dl1 and dl2 info (e.g. for training input) """ - config = Path("./examples/training_config.json").absolute() + config = files("ctapipe.tools.tests.resources").joinpath("training_config.json") output = tmp_path / "test_training.DL1DL2.h5" assert ( diff --git a/setup.py b/setup.py index 9b1f5c603c4..8ab772a5cb9 100755 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "ctapipe-merge = ctapipe.tools.dl1_merge:main", "ctapipe-fileinfo = ctapipe.tools.fileinfo:main", ] -tests_require = ["pytest", "pandas>=0.24.0"] +tests_require = ["pytest", "pandas>=0.24.0", "importlib_resources;python_version<'3.9'"] docs_require = [ "sphinx_rtd_theme", "sphinx_automodapi", @@ -82,5 +82,7 @@ ], zip_safe=False, entry_points=entry_points, - package_data={"": ["tools/bokeh/*.yaml", "tools/bokeh/templates/*.html"]}, + package_data={ + "": ["tools/bokeh/*.yaml", "tools/bokeh/templates/*.html", "resources/*"] + }, )