include stage1_config.json in package (#1752)

* Include stage1_config.json in package.

* Use included stage1_config.json.

* Add importlib_resources to tests_require.

* Move stage1_config.json to tool test resources.

* Require importlib_resources only for python < 3.9

* Remove examples/*.json from package_data

* Add the resources to package_data.

* Test the inclusion of resources in CI.

* Use importlib resources with ProcessorTool

.github/workflows/ci.yml

@@ -40,7 +40,7 @@ jobs:
           source .github/install.sh
           python --version
           pip install codecov pytest-cov pyflakes pytest-xdist
-          pip install -e .[all]
+          pip install .[all]
           pip freeze
 
       - name: Static codechecks

ctapipe/tools/tests/test_merge.py

@@ -7,9 +7,14 @@
 
 from ctapipe.tools.process import ProcessorTool
 
+try:
+    from importlib.resources import files
+except ImportError:
+    from importlib_resources import files
+
 
 def run_stage1(input_path, cwd, output_path=None):
-    config = Path("./examples/stage1_config.json").absolute()
+    config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json")
 
     if output_path is None:
         output_path = Path(

ctapipe/tools/tests/test_process.py

@@ -4,22 +4,25 @@
 Test ctapipe-process on a few different use cases
 """
 
-from pathlib import Path
-
 import pandas as pd
 import tables
 from ctapipe.core import run_tool
 from ctapipe.tools.process import ProcessorTool
 from ctapipe.utils import get_dataset_path
 from ctapipe.io import DataLevel, EventSource
 
+try:
+    from importlib.resources import files
+except ImportError:
+    from importlib_resources import files
+
 GAMMA_TEST_LARGE = get_dataset_path("gamma_test_large.simtel.gz")
 
 
 def test_stage_1_dl1(tmp_path, dl1_image_file, dl1_parameters_file):
     """  check simtel to DL1 conversion """
 
-    config = Path("./examples/stage1_config.json").absolute()
+    config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json")
     # DL1A file as input
     dl1b_from_dl1a_file = tmp_path / "dl1b_fromdl1a.dl1.h5"
     assert (
@@ -124,7 +127,7 @@ def _generator(self):
         infile.write(b"dummy")
         infile.flush()
 
-    config = Path("./examples/stage1_config.json").absolute()
+    config = files("ctapipe.tools.tests.resources").joinpath("stage1_config.json")
     tool = ProcessorTool()
 
     assert (
@@ -147,7 +150,7 @@ def _generator(self):
 
 def test_stage_2_from_simtel(tmp_path):
     """ check we can go to DL2 geometry from simtel file """
-    config = Path("./examples/stage2_config.json").absolute()
+    config = files("ctapipe.tools.tests.resources").joinpath("stage2_config.json")
     output = tmp_path / "test_stage2_from_simtel.DL2.h5"
 
     assert (
@@ -172,7 +175,7 @@ def test_stage_2_from_simtel(tmp_path):
 
 def test_stage_2_from_dl1_images(tmp_path, dl1_image_file):
     """ check we can go to DL2 geometry from DL1 images """
-    config = Path("./examples/stage2_config.json").absolute()
+    config = files("ctapipe.tools.tests.resources").joinpath("stage2_config.json")
     output = tmp_path / "test_stage2_from_dl1image.DL2.h5"
 
     assert (
@@ -197,7 +200,7 @@ def test_stage_2_from_dl1_images(tmp_path, dl1_image_file):
 def test_stage_2_from_dl1_params(tmp_path, dl1_parameters_file):
     """ check we can go to DL2 geometry from DL1 parameters """
 
-    config = Path("./examples/stage2_config.json").absolute()
+    config = files("ctapipe.tools.tests.resources").joinpath("stage2_config.json")
     output = tmp_path / "test_stage2_from_dl1param.DL2.h5"
 
     assert (
@@ -222,7 +225,7 @@ def test_stage_2_from_dl1_params(tmp_path, dl1_parameters_file):
 def test_training_from_simtel(tmp_path):
     """ check we can write both dl1 and dl2 info (e.g. for training input) """
 
-    config = Path("./examples/training_config.json").absolute()
+    config = files("ctapipe.tools.tests.resources").joinpath("training_config.json")
     output = tmp_path / "test_training.DL1DL2.h5"
 
     assert (

setup.py

@@ -24,7 +24,7 @@
     "ctapipe-merge = ctapipe.tools.dl1_merge:main",
     "ctapipe-fileinfo = ctapipe.tools.fileinfo:main",
 ]
-tests_require = ["pytest", "pandas>=0.24.0"]
+tests_require = ["pytest", "pandas>=0.24.0", "importlib_resources;python_version<'3.9'"]
 docs_require = [
     "sphinx_rtd_theme",
     "sphinx_automodapi",
@@ -82,5 +82,7 @@
     ],
     zip_safe=False,
     entry_points=entry_points,
-    package_data={"": ["tools/bokeh/*.yaml", "tools/bokeh/templates/*.html"]},
+    package_data={
+        "": ["tools/bokeh/*.yaml", "tools/bokeh/templates/*.html", "resources/*"]
+    },
 )