ci: Physics performance monitoring (#1193)

This PR introduces a set of "physics" monitoring CI jobs based on histogram comparisons. 
Currently this runs

- CKF truth estimated
- CKF truth smeared
- CKF seeded
- truth tracking

I intend to run this for a bit and see how it behaves. If it works well, we should extend it to cover other areas as well.

.github/workflows/builds.yml

@@ -27,17 +27,25 @@ jobs:
       SETUP: true
       SETUP_LCG: source /opt/lcg_view/setup.sh
       INSTALL_DIR: ${{ github.workspace }}/install
+      ACTS_LOG_FAILURE_THRESHOLD: WARNING
     steps:
-      - uses: actions/checkout@v2
+      - name: Install git lfs
+        if: contains(matrix.image, 'ubuntu')
+        run: apt install -y git-lfs
+
+      - uses: actions/checkout@v3
+        if: contains(matrix.image, 'ubuntu')
         with:
           submodules: true
-        if: "!contains(matrix.image, 'lcg')"
-      - uses: actions/checkout@v2
-        if: contains(matrix.image, 'lcg')
+          lfs: true
+
+      - uses: actions/checkout@v3
+        if: "!contains(matrix.image, 'ubuntu')"
 
       - name: Define setup script
         if: contains(matrix.image, 'lcg')
         run: echo "SETUP=${SETUP_LCG}" >> $GITHUB_ENV
+
       - name: Configure
         # setting CMAKE_CXX_STANDARD=17 is a workaround for a bug in the
         # dd4hep CMake configuration that gets triggered on recent CMake
@@ -52,7 +60,6 @@ jobs:
           -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
           -DACTS_BUILD_EVERYTHING=ON
           -DACTS_BUILD_EXAMPLES_PYTHON_BINDINGS=ON
-          -DACTS_LOG_FAILURE_THRESHOLD=WARNING
           -DACTS_FORCE_ASSERTIONS=ON
       - name: Build
         run: ${SETUP} && cmake --build build --
@@ -68,13 +75,31 @@ jobs:
         shell: bash
         run: >
           ${SETUP}
+          && /usr/local/bin/download_geant4_data.sh
           && source /usr/local/bin/thisroot.sh
           && source /usr/local/bin/thisdd4hep_only.sh
-          && /usr/local/bin/download_geant4_data.sh
           && source /usr/local/bin/geant4.sh
           && source build/python/setup.sh
           && pip3 install -r Examples/Python/tests/requirements.txt
           && pytest -rFs
+      - name: Physics performance checks
+        if: contains(matrix.image, 'ubuntu')
+        shell: bash
+        run: >
+          ${SETUP}
+          && echo "::group::Dependencies"
+          && pip3 install histcmp==0.3.0
+          && source /usr/local/bin/thisroot.sh
+          && source /usr/local/bin/thisdd4hep_only.sh
+          && source /usr/local/bin/geant4.sh
+          && source build/python/setup.sh
+          && echo "::endgroup::"
+          && CI/physmon/phys_perf_mon.sh physmon
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: physmon
+          path: physmon
       - name: Install
         run: ${SETUP} && cmake --build build -- install
       - uses: actions/upload-artifact@v2

CI/physmon/ckf_truth_smeared.yml

@@ -0,0 +1,26 @@
+checks:
+  "*":
+    Chi2Test:
+      threshold: 0.01
+    KolmogorovTest:
+      threshold: 0.68
+    RatioCheck:
+      threshold: 3
+    ResidualCheck:
+      threshold: 3
+    IntegralCheck:
+      threshold: 3
+
+
+
+
+  nHoles_vs_eta:
+    KolmogorovTest:
+      threshold: 0.25
+  nOutliers_vs_pT:
+    KolmogorovTest:
+      threshold: 0.59
+
+  nStates_vs_eta:
+    KolmogorovTest: null
+    IntegralCheck: null

CI/physmon/phys_perf_mon.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -e
+
+outdir=$1
+mkdir -p $outdir
+
+refdir=CI/physmon/reference
+refcommit=$(cat $refdir/commit)
+commit=$(git rev-parse --short HEAD)
+
+echo "::group::Generate validation dataset"
+CI/physmon/physmon.py $outdir 2>&1 > $outdir/run.log
+echo "::endgroup::"
+
+set +e
+
+ec=0
+
+function run() {
+    a=$1
+    b=$2
+
+    echo "::group::Comparing $a vs. $b"
+
+    histcmp \
+        --label-reference=$refcommit \
+        --label-monitored=$commit \
+        "$@"
+
+    ec=$(($ec | $?))
+
+    echo "::endgroup::"
+}
+
+
+run \
+    physmon/performance_ckf_tracks_truth_smeared.root \
+    $refdir/performance_ckf_tracks_truth_smeared.root \
+    --title "CKF truth smeared" \
+    -c CI/physmon/ckf_truth_smeared.yml \
+    -o $outdir/ckf_truth_smeared.html \
+
+run \
+    physmon/performance_ckf_tracks_truth_estimated.root \
+    $refdir/performance_ckf_tracks_truth_estimated.root \
+    --title "CKF truth estimated" \
+    -o $outdir/ckf_truth_estimated.html \
+
+run \
+    physmon/performance_ckf_tracks_seeded.root \
+    $refdir/performance_ckf_tracks_seeded.root \
+    --title "CKF seeded" \
+    -o $outdir/ckf_seeded.html \
+
+run \
+    physmon/performance_truth_tracking.root \
+    $refdir/performance_truth_tracking.root \
+    --title "Truth tracking" \
+    -c CI/physmon/truth_tracking.yml \
+    -o $outdir/truth_tracking.html \
+
+
+
+exit $ec

CI/physmon/physmon.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+from pathlib import Path
+import argparse
+import tempfile
+import shutil
+import os
+import sys
+
+sys.path += [
+    str(Path(__file__).parent.parent.parent / "Examples/Scripts/Python/"),
+]
+
+# this has to happen before we import the ACTS module
+os.environ["ACTS_LOG_FAILURE_THRESHOLD"] = "FATAL"
+import acts.examples
+
+from truth_tracking import runTruthTracking
+from ckf_tracks import runCKFTracks
+from common import getOpenDataDetector
+
+parser = argparse.ArgumentParser()
+parser.add_argument("outdir")
+parser.add_argument("--events", type=int, default=10000)
+parser.add_argument("--skip", type=int, default=0)
+
+args = parser.parse_args()
+
+outdir = Path(args.outdir)
+outdir.mkdir(exist_ok=True)
+
+
+srcdir = Path(__file__).resolve().parent.parent.parent
+
+
+u = acts.UnitConstants
+
+matDeco = acts.IMaterialDecorator.fromFile(
+    srcdir / "thirdparty/OpenDataDetector/data/odd-material-maps.root",
+    level=acts.logging.INFO,
+)
+detector, trackingGeometry, decorators = getOpenDataDetector(matDeco)
+digiConfig = srcdir / "thirdparty/OpenDataDetector/config/odd-digi-smearing-config.json"
+geoSel = srcdir / "thirdparty/OpenDataDetector/config/odd-seeding-config.json"
+
+
+field = acts.ConstantBField(acts.Vector3(0, 0, 2 * u.T))
+
+s = acts.examples.Sequencer(
+    events=args.events, numThreads=-1, logLevel=acts.logging.INFO, skip=0
+)
+
+with tempfile.TemporaryDirectory() as temp:
+    tp = Path(temp)
+    runTruthTracking(
+        trackingGeometry,
+        field,
+        digiConfigFile=digiConfig,
+        outputDir=tp,
+        s=s,
+    )
+
+    s.run()
+    del s
+
+    perf_file = tp / "performance_track_fitter.root"
+    assert perf_file.exists(), "Performance file not found"
+    shutil.copy(perf_file, outdir / "performance_truth_tracking.root")
+
+
+for truthSmeared, truthEstimated, label in [
+    (True, False, "truth_smeared"),  # if first is true, second is ignored
+    (False, True, "truth_estimated"),
+    (False, False, "seeded"),
+]:
+    s = acts.examples.Sequencer(
+        events=args.events, numThreads=1, logLevel=acts.logging.INFO, skip=args.skip
+    )
+
+    with tempfile.TemporaryDirectory() as temp:
+        tp = Path(temp)
+        runCKFTracks(
+            trackingGeometry,
+            decorators=decorators,
+            field=field,
+            digiConfigFile=digiConfig,
+            geometrySelection=geoSel,
+            outputDir=tp,
+            outputCsv=False,
+            truthSmearedSeeded=truthSmeared,
+            truthEstimatedSeeded=truthEstimated,
+            s=s,
+        )
+
+        s.run()
+        del s
+
+        perf_file = tp / "performance_ckf.root"
+        assert perf_file.exists(), "Performance file not found"
+        shutil.copy(perf_file, outdir / f"performance_ckf_tracks_{label}.root")

CI/physmon/reference/commit

@@ -0,0 +1 @@
+45c23efea

CI/physmon/truth_tracking.yml

@@ -0,0 +1,51 @@
+checks:
+  "*":
+    Chi2Test:
+      threshold: 0.01
+    KolmogorovTest:
+      threshold: 0.68
+    RatioCheck:
+      threshold: 3
+    ResidualCheck:
+      threshold: 3
+    IntegralCheck:
+      threshold: 3
+
+
+
+
+  "nStates_*":
+    KolmogorovTest: null
+  "nMeasurements_*":
+    KolmogorovTest: null
+
+  pullmean_phi_vs_eta:
+    IntegralCheck:
+      threshold: 4
+  pullmean_theta_vs_eta:
+    IntegralCheck: null
+  pullmean_z0_vs_eta:
+    IntegralCheck: null
+
+  pullwidth_phi_vs_eta:
+    IntegralCheck:
+      threshold: 4.3
+  reswidth_d0_vs_eta:
+    IntegralCheck:
+      threshold: 4
+  resmean_d0_vs_eta:
+    IntegralCheck:
+      threshold: 4
+
+  pullwidth_*:
+    Chi2Test: null
+    KolmogorovTest: null
+  pullmean_*:
+    Chi2Test: null
+    KolmogorovTest: null
+  reswidth_*:
+    Chi2Test: null
+    KolmogorovTest: null
+  resmean_*:
+    Chi2Test: null
+    KolmogorovTest: null