Merge branch 'main' of github.com:cta-observatory/lstosa into cfg_pac…

…kage

.github/workflows/ci.yml

@@ -1,71 +1,81 @@
 name: CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - '**'
+  pull_request:
 
 env:
   MPLBACKEND: Agg
   PYTEST_ADDOPTS: --color=yes
 
 jobs:
+  pyflakes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: pyflakes
+        run: |
+          pip install pyflakes
+          pyflakes osa
+
   tests:
+    needs: pyflakes
     runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
     strategy:
       matrix:
-        python-version: [ 3.7, 3.8 ]
+        python-version: [3.7, 3.8]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v2.1.1
         with:
-          python-version: ${{ matrix.python-version }}
+          mamba-version: "*"
+          auto-update-conda: true
+          activate-environment: "osa-ci"
+          environment-file: environment.yml
 
       - name: Install dependencies
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
 
         run: |
-          . $CONDA/etc/profile.d/conda.sh
-          conda config --set always_yes yes --set changeps1 no
-          sed -i -e "s/- python=.*/- python=$PYTHON_VERSION/g" environment.yml
-          echo "Creating conda env"
-          conda install -c conda-forge mamba
-          mamba env create -n ci -f environment.yml
-          conda activate ci
+          python --version
           echo "Installing additional pip packages"
-          pip install pyflakes
           echo "pip install ."
           pip install .
 
-      - name: Run pyflakes
-        # need to use a login shell for the conda setup to work
-        run: |
-          . $CONDA/etc/profile.d/conda.sh
-          conda activate ci
-          pyflakes osa
-
       - name: Tests
         env:
           COVERAGE_PROCESS_START: .coveragerc
           PYTHONPATH: .
 
         run: |
-          # github actions starts a new shell for each "step", so we need to
-          # activate our env again
-          source $CONDA/etc/profile.d/conda.sh
-          conda activate ci
-
           # to measure coverage when using subprocess for testing scripts we create sitecustomize.py
           # as specified in https://coverage.readthedocs.io/en/latest/subprocess.html
           # to initialize the coverage start up.
           echo "create sitecustomize.py"
           echo "import coverage; coverage.process_startup()" > $PYTHONPATH/sitecustomize.py
-
           pytest --basetemp=test_osa -v --cov --cov-report=xml osa
           cat coverage.xml
 
       - name: Codecov
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3

.gitignore

@@ -92,3 +92,6 @@ target
 NightSummary/
 testweb.sh
 misc/
+
+# Ignore generated _version.py
+_version.py

README.md

@@ -0,0 +1,63 @@
+# lstosa
+
+  [![ci](https://github.com/cta-observatory/lstosa/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/cta-observatory/lstosa/actions/workflows/ci.yml)
+  [![Documentation Status](https://readthedocs.org/projects/lstosa/badge/?version=latest)](https://lstosa.readthedocs.io/en/latest/?badge=latest)
+  [![coverage](https://codecov.io/gh/cta-observatory/lstosa/branch/main/graph/badge.svg?token=Zjk1U1ytaG)](https://codecov.io/gh/cta-observatory/lstosa)
+  [![quality](https://app.codacy.com/project/badge/Grade/a8743a706e7c45fc989d5ebc4d61d54f)](https://www.codacy.com/gh/cta-observatory/lstosa/dashboard?utm_source=github.com&utm_medium=referral&utm_content=cta-observatory/lstosa&utm_campaign=Badge_Grade)
+
+
+On-Site processing pipeline for the the Large Size Telescope prototype
+of [CTA](https://www.cta-observatory.org/) (Cherenkov Telescope Array).
+
+This is a prototype data processing framework under development based on
+[cta-lstchain](https://github.com/cta-observatory/cta-lstchain) .
+
+-   Code: <https://github.com/cta-observatory/lstosa>
+-   Docs: <https://lstosa.readthedocs.io/>
+-   License:
+    [BSD-3-Clause](https://github.com/cta-observatory/lstosa/blob/main/LICENSE)
+
+# Install
+
+-   Install miniconda first.
+-   Create and activate the conda environment including
+    [cta-lstchain](https://github.com/cta-observatory/cta-lstchain) and
+    [ctapipe_io_lst](https://github.com/cta-observatory/ctapipe_io_lst):
+
+```bash
+git clone https://github.com/cta-observatory/lstosa.git
+cd lstosa
+conda env create -n osa -f environment.yml
+conda activate osa
+```
+
+In case you want to install the lstchain master version instead of a fixed tag you can run inside the `osa` environment:
+
+```bash
+pip install git+https://github.com/cta-observatory/cta-lstchain
+```
+
+- To update the environment (provided dependencies get updated) use:
+
+```bash
+conda env update -n osa -f environment.yml
+```
+
+- Install `lstosa`:
+
+```bash
+pip install .
+```
+
+# Data workflow
+
+```mermaid
+graph TD
+    A[R0] -->|r0_to_dl1| B(DL1a) & C(muons)
+    B --> |dl1ab| D(DL1ab)
+    D --> |dl1_to_dl2| E[DL2]
+    D & C--> |check_dl1| F[DL1 datacheck]
+    E --> |merge_hdf5_files| G(DL2 merged)
+    G & J--> |create_dl3_file| H(DL3)
+    I(DL2 gamma MC) --> |create_irf_files| J(IRF)
+```

environment.yml

@@ -36,9 +36,9 @@ dependencies:
   - sphinx-argparse
   - sphinx-autoapi
   - pip:
-      - lstchain~=0.9.4
+      - lstchain~=0.9.6
       - pytest_runner
       - pytest-ordering
-      - ctapipe_io_lst~=0.16.1
+      - ctapipe_io_lst~=0.18.1
       - ctaplot~=0.5.5
       - pyirf~=0.6.0

osa/_dev_version/__init__.py

@@ -0,0 +1,9 @@
+# Try to use setuptools_scm to get the current version; this is only used
+# in development installations from the git repository.
+# see lstosa/version.py for details. Taken from ctapipe project.
+try:
+    from setuptools_scm import get_version
+
+    version = get_version(root="../..", relative_to=__file__)
+except Exception as e:
+    raise ImportError(f"setuptools_scm broken or not installed: {e}")

osa/configs/sequencer.cfg

@@ -37,7 +37,7 @@ CALIB_PROD_ID: v01
 # Change this to produce a different DL1b or DL2 sub-productions.
 # Otherwise, keep it empty to use the common PROD-ID
 DL1_PROD_ID: tailcut84
-DL2_PROD_ID: tailcut84_model1
+DL2_PROD_ID: model2
 
 [lstchain]
 # Calibration steps in calibration pipeline script

osa/conftest.py

@@ -390,4 +390,3 @@ def run_catalog(run_catalog_dir):
     catalog_file.touch()
     catalog_file.write_text(source_information)
     return catalog_file
-

osa/provenance/io.py

@@ -186,7 +186,7 @@ def provlist2provdoc(provlist):
                     records[progen_id] = progen
                 ent.wasDerivedFrom(progen)
             for k, v in provdict.items():
-                if k not in["session_tag", "hash", "hash_type"]:
+                if k not in ["session_tag", "hash", "hash_type"]:
                     ent.add_attributes({k: str(v)})
         # agent
     return pdoc

osa/provenance/utils.py

@@ -3,7 +3,6 @@
 import logging
 import os
 import re
-import subprocess
 import sys
 from pathlib import Path
 
@@ -12,7 +11,7 @@
 from osa.utils.logging import myLogger
 from osa.utils.utils import get_lstchain_version, lstdate_to_dir
 
-__all__ = ["parse_variables", "get_log_config", "store_conda_env_export"]
+__all__ = ["parse_variables", "get_log_config"]
 
 REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
@@ -99,7 +98,7 @@ def parse_variables(class_instance):
         # calibration_file   [0] .../20200218/pro/calibration_filters_52.Run02006.0000.h5
         # drs4_pedestal_file [1] .../20200218/pro/drs4_pedestal.Run02005.0000.h5
         # time_calib_file    [2] .../20191124/pro/time_calibration.Run01625.0000.h5
-        # systematic_corr    [3] .../20200101/pro/no_sys_corrected_calibration_scan_fit_20210514.0000.h5
+        # systematic_corr    [3] .../20200101/pro/no_sys_corrected_calib_20210514.0000.h5
         # drive_file         [4] .../DrivePositioning/drive_log_20_02_18.txt
         # run_summary_file   [5] .../RunSummary/RunSummary_20200101.ecsv
         # pedestal_ids_file  [6] .../path/to/interleaved/pedestal/events.h5
@@ -231,11 +230,3 @@ def get_log_config():
         log_config = std_logger_file.read_text()
 
     return log_config
-
-
-def store_conda_env_export():
-    """Store file with `conda env export` output to log the packages versions used."""
-    analysis_log_dir = Path(options.directory) / "log"
-    analysis_log_dir.mkdir(parents=True, exist_ok=True)
-    conda_env_file = analysis_log_dir / "conda_env.yml"
-    subprocess.run(["conda", "env", "export", "--file", str(conda_env_file)], check=True)

osa/scripts/autocloser.py

@@ -369,7 +369,8 @@ def has_all_subruns(self):
         if self.dictSequence["Type"] == "PEDCALIB":
             log.debug("Cannot check for missing subruns in the middle for CALIBRATION")
             return True
-        search_str = f"{analysis_path(self.dictSequence['Tel'])}/dl1*{int(self.dictSequence['Run']):05d}*.h5"
+        search_str = f"{analysis_path(self.dictSequence['Tel'])}/" \
+                     f"dl1*{int(self.dictSequence['Run']):05d}*.h5"
         subrun_nrs = sorted(
             [int(os.path.basename(f).split(".")[2]) for f in glob.glob(search_str)]
         )
@@ -593,7 +594,7 @@ def understand_sequence(tel, seq):
 
     if args.onlyIncidences and args.force:
         log.error(
-            "The command line arguments 'onlyIncidences' and 'force' are incompatible with each other"
+            "The command line arguments 'onlyIncidences' and 'force' are incompatible"
         )
         sys.exit(1)
 

osa/scripts/closer.py

@@ -23,7 +23,6 @@
 )
 from osa.nightsummary.nightsummary import run_summary_table
 from osa.paths import destination_dir
-from osa.provenance.utils import store_conda_env_export
 from osa.raw import is_raw_data_available
 from osa.report import start
 from osa.utils.cliopts import closercliparsing
@@ -99,7 +98,6 @@ def main():
             log.error("Never thought about this possibility, please check the code")
             sys.exit(-1)
 
-        store_conda_env_export()
         save_job_information()
         post_process(sequencer_tuple)