From 2eb549cf98b8ea49a4eac891c1b779f4891598ba Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Wed, 11 May 2022 15:37:41 -0600 Subject: [PATCH] Improve example checking script (#295) * Skip problematic examples * Introduce variable controlling problem size * Introduce variable controlling problem size * Bug fix * Trivial change * Introduce variable controlling problem size * Trivial change * Remove redundant script run * Trivial change * Include ray tune examples * Update readme * Improve example checking script * Add examples test action * Fix action name * Add missing dependency list * Improve example test script * Enable brief error display * Change Xvfb display number * Add timeout on script run * Debug script hanging * Debug script hanging * Debug script hanging * Avoid excessive resource requests in ray[tune] examples * Remove debugging code * Update contributing docs --- .github/workflows/test_examples.yml | 80 +++++++++++++++++++ docs/source/contributing.rst | 5 +- examples/README.rst | 2 +- examples/scriptcheck.sh | 71 ++++++++++++---- examples/scripts/ct_abel_tv_admm.py | 4 +- examples/scripts/ct_abel_tv_admm_tune.py | 3 +- examples/scripts/deconv_circ_tv_admm.py | 3 +- examples/scripts/deconv_microscopy_tv_admm.py | 1 - examples/scripts/deconv_ppp_bm3d_admm.py | 3 +- examples/scripts/deconv_ppp_bm3d_pgm.py | 3 +- examples/scripts/deconv_ppp_bm4d_admm.py | 5 +- examples/scripts/deconv_ppp_dncnn_admm.py | 3 +- examples/scripts/deconv_tv_admm.py | 3 +- examples/scripts/deconv_tv_admm_tune.py | 3 +- examples/scripts/denoise_tv_iso_multi.py | 3 +- examples/scripts/denoise_tv_iso_pgm.py | 2 +- 16 files changed, 164 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/test_examples.yml diff --git a/.github/workflows/test_examples.yml b/.github/workflows/test_examples.yml new file mode 100644 index 000000000..6c447b89f --- /dev/null +++ b/.github/workflows/test_examples.yml @@ -0,0 +1,80 @@ +# Install scico requirements and run pytest + +name: test examples + +# Control when the workflow will run +on: + # Trigger the workflow on push or pull request events but only for the main branch + push: + branches: [ main ] + pull_request: + branches: [ main ] + + # Allow this workflow to be run manually from the Actions tab + workflow_dispatch: + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + label: linux-64 + prefix: /usr/share/miniconda3/envs/test-env + name: ${{ matrix.label }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Check-out the repository under $GITHUB_WORKSPACE + - uses: actions/checkout@v2 + with: + submodules: recursive + # Set up conda/mamba environment + - name: Set up mambaforge + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: test-env + use-mamba: true + python-version: 3.9 + # Configure conda environment cache + - name: Set up conda environment cache + uses: actions/cache@v2 + with: + path: ${{ env.CONDA }}/envs + key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev_requirements.txt') }}-${{ hashFiles('examples/examples_requirements.txt') }}-${{ env.CACHE_NUMBER }} + env: + CACHE_NUMBER: 0 # Increase this value to force cache reset + id: cache + # Display environment details + - name: Display environment details + run: | + conda info + printenv | sort + # Install required system package + - name: Install required system package + if: matrix.os == 'ubuntu-latest' + run: sudo apt-get install -y libopenblas-dev + # Install dependencies in conda environment + - name: Install dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: | + mamba install -c conda-forge pytest pytest-cov + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r dev_requirements.txt + mamba install -c astra-toolbox astra-toolbox + mamba install -c conda-forge pyyaml + pip install -r examples/examples_requirements.txt + # Install package to be tested + - name: Install package to be tested + run: pip install -e . + # Run example test + - name: Run example test + run: | + ${GITHUB_WORKSPACE}/examples/scriptcheck.sh -e -d diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 2897b8d98..081d9805d 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -358,12 +358,15 @@ particular: 3. Citations are included using the standard `Sphinx `__ ``:cite:`cite-key``` syntax, where ``cite-key`` is the key of an entry in ``docs/source/references.bib``. -4. Cross-references to other components of the documentation are included using the syntax described in the `nbsphinx documentation `__. +4. Cross-references to other components of the documentation are included using the syntax described in the `nbsphinx documentation `__. 5. External links are included using Markdown syntax ``[link text](url)``. +6. When constructing a synthetic image/volume for use in the example, define a global variable `N` that controls the size of the problem, and where relevant, define a global variable `maxiter` that controls the number of iterations of optimization algorithms such as ADMM. Adhering to this convention allows the ``examples/scriptcheck.sh`` utility to automatically construct less computationally expensive versions of the example scripts for testing that they run without any errors. + + Adding new examples ^^^^^^^^^^^^^^^^^^^ diff --git a/examples/README.rst b/examples/README.rst index e7c36865e..071842d9a 100644 --- a/examples/README.rst +++ b/examples/README.rst @@ -112,4 +112,4 @@ A number of files in this directory assist in the mangement of the usage example Auto-generate the docs example index ``docs/source/examples.rst`` from the example scripts index ``scripts/index.rst``. `scriptcheck.sh `_ - Run all example scripts with a reduced number of iterations as a rapid check that they are functioning correctly. + Run all example scripts with smaller problems and a reduced number of iterations as a rapid check that they are functioning correctly. diff --git a/examples/scriptcheck.sh b/examples/scriptcheck.sh index 63d2df2b1..71789ddb7 100755 --- a/examples/scriptcheck.sh +++ b/examples/scriptcheck.sh @@ -1,23 +1,55 @@ -#! /bin/bash +#!/usr/bin/env bash # Basic test of example script functionality by running them all with # optimization algorithms configured to use only a small number of iterations. # Currently only supported under Linux. +SCRIPT=$(basename $0) +SCRIPTPATH=$(realpath $(dirname $0)) +USAGE=$(cat <<-EOF +Usage: $SCRIPT [-h] [-d] + [-h] Display usage information + [-e] Display excerpt of error message on failure + [-d] Skip tests involving additional data downloads +EOF +) + +OPTIND=1 +DISPLAY_ERROR=0 +SKIP_DOWNLOAD=0 +while getopts ":hed" opt; do + case $opt in + h) echo "$USAGE"; exit 0;; + e) DISPLAY_ERROR=1;; + d) SKIP_DOWNLOAD=1;; + \?) echo "Error: invalid option -$OPTARG" >&2 + echo "$USAGE" >&2 + exit 1 + ;; + esac +done + +shift $((OPTIND-1)) +if [ ! $# -eq 0 ] ; then + echo "Error: no positional arguments" >&2 + echo "$USAGE" >&2 + exit 2 +fi + # Check for presence of Xvfb tool which is used to avoid plots being displayed. if [ ! "$(which Xvfb 2>/dev/null)" ]; then msg="Warning: required tool Xvfb not found: functionality will be degraded" echo $msg >&2 pid=0 else - Xvfb :11 -screen 0 800x600x16 > /dev/null 2>&1 & + Xvfb :20 -screen 0 800x600x16 > /dev/null 2>&1 & pid=$! export DISPLAY=:10.0 fi # Set environment variables and paths. This script is assumed to be run # from its root directory. -export PYTHONPATH=$(cd .. && pwd) +export PYTHONPATH=$SCRIPTPATH/.. export PYTHONIOENCODING=utf-8 d='/tmp/scriptcheck_'$$ mkdir -p $d @@ -32,29 +64,40 @@ function cleanupexit { trap cleanupexit SIGINT # Define regex strings. -re1="s/'maxiter' ?: ?[0-9]+/'maxiter': 3/g; " -re2="s/maxiter ?= ?[0-9]+/maxiter = 3/g; " -re3="s/input\(/#input\(/g; " -re4="s/fig.show\(/#fig.show\(/g" +re1="s/'maxiter' ?: ?[0-9]+/'maxiter': 2/g; " +re2="s/^maxiter ?= ?[0-9]+/maxiter = 2/g; " +re3="s/^N ?= ?[0-9]+/N = 32/g; " +re4="s/num_samples= ?[0-9]+/num_samples = 2/g; " +re5='s/\"cpu\": ?[0-9]+/\"cpu\": 1/g; ' +re6="s/^downsampling_rate ?= ?[0-9]+/downsampling_rate = 12/g; " +re7="s/input\(/#input\(/g; " +re8="s/fig.show\(/#fig.show\(/g" # Iterate over all scripts. -for f in scripts/*.py; do - printf "%-50s " $f +for f in $SCRIPTPATH/scripts/*.py; do + + printf "%-50s " $(basename $f) + + # Skip problem cases. + if [ $SKIP_DOWNLOAD -eq 1 ] && grep -q '_microscopy' <<< $f; then + printf "%s\n" skipped + continue + fi # Create temporary copy of script with all algorithm maxiter values set # to small number and final input statements commented out. g=$d/$(basename $f) - sed -E -e "$re1$re2$re3$re4" $f > $g - - # Run temporary script. - python $g > /dev/null 2>&1 + sed -E -e "$re1$re2$re3$re4$re5$re6$re7$re8" $f > $g # Run temporary script and print status message. - if python $g > /dev/null 2>&1; then + if output=$(timeout 60s python $g 2>&1); then printf "%s\n" succeeded else printf "%s\n" FAILED retval=1 + if [ $DISPLAY_ERROR -eq 1 ]; then + echo "$output" | tail -8 | sed -e 's/^/ /' + fi fi # Remove temporary script. diff --git a/examples/scripts/ct_abel_tv_admm.py b/examples/scripts/ct_abel_tv_admm.py index e715d10b6..fa373e8f7 100644 --- a/examples/scripts/ct_abel_tv_admm.py +++ b/examples/scripts/ct_abel_tv_admm.py @@ -24,7 +24,9 @@ """ Create a ground truth image. """ -x_gt = create_circular_phantom((256, 254), [100, 50, 25], [1, 0, 0.5]) +N = 256 # phantom size +x_gt = create_circular_phantom((N, N), [0.4 * N, 0.2 * N, 0.1 * N], [1, 0, 0.5]) + """ Set up the forward operator and create a test measurement diff --git a/examples/scripts/ct_abel_tv_admm_tune.py b/examples/scripts/ct_abel_tv_admm_tune.py index 2d132ffa5..268592f4a 100644 --- a/examples/scripts/ct_abel_tv_admm_tune.py +++ b/examples/scripts/ct_abel_tv_admm_tune.py @@ -27,7 +27,8 @@ """ Create a ground truth image. """ -x_gt = create_circular_phantom((256, 256), [100, 50, 25], [1, 0, 0.5]) +N = 256 # phantom size +x_gt = create_circular_phantom((N, N), [0.4 * N, 0.2 * N, 0.1 * N], [1, 0, 0.5]) """ diff --git a/examples/scripts/deconv_circ_tv_admm.py b/examples/scripts/deconv_circ_tv_admm.py index aa96e683f..f71f2b485 100644 --- a/examples/scripts/deconv_circ_tv_admm.py +++ b/examples/scripts/deconv_circ_tv_admm.py @@ -36,7 +36,8 @@ Create a ground truth image. """ phantom = SiemensStar(32) -x_gt = snp.pad(discrete_phantom(phantom, 240), 8) +N = 256 # image size +x_gt = snp.pad(discrete_phantom(phantom, N - 16), 8) x_gt = jax.device_put(x_gt) # convert to jax type, push to GPU diff --git a/examples/scripts/deconv_microscopy_tv_admm.py b/examples/scripts/deconv_microscopy_tv_admm.py index 67b775b78..02085b328 100644 --- a/examples/scripts/deconv_microscopy_tv_admm.py +++ b/examples/scripts/deconv_microscopy_tv_admm.py @@ -115,7 +115,6 @@ """ Show the recovered image. """ - fig, ax = plot.subplots(nrows=1, ncols=2, figsize=(14, 7)) plot.imview(tile_volume_slices(y), title="Blurred measurements", fig=fig, ax=ax[0]) plot.imview(tile_volume_slices(x), title="Deconvolved image", fig=fig, ax=ax[1]) diff --git a/examples/scripts/deconv_ppp_bm3d_admm.py b/examples/scripts/deconv_ppp_bm3d_admm.py index 4e9ac8370..4c23bc8c1 100644 --- a/examples/scripts/deconv_ppp_bm3d_admm.py +++ b/examples/scripts/deconv_ppp_bm3d_admm.py @@ -30,7 +30,8 @@ Create a ground truth image. """ np.random.seed(1234) -x_gt = discrete_phantom(Foam(size_range=[0.075, 0.0025], gap=1e-3, porosity=1), size=512) +N = 512 # image size +x_gt = discrete_phantom(Foam(size_range=[0.075, 0.0025], gap=1e-3, porosity=1), size=N) x_gt = jax.device_put(x_gt) # convert to jax array, push to GPU diff --git a/examples/scripts/deconv_ppp_bm3d_pgm.py b/examples/scripts/deconv_ppp_bm3d_pgm.py index e080b3d75..199709163 100644 --- a/examples/scripts/deconv_ppp_bm3d_pgm.py +++ b/examples/scripts/deconv_ppp_bm3d_pgm.py @@ -32,7 +32,8 @@ Create a ground truth image. """ np.random.seed(1234) -x_gt = discrete_phantom(Foam(size_range=[0.075, 0.0025], gap=1e-3, porosity=1), size=512) +N = 512 # image size +x_gt = discrete_phantom(Foam(size_range=[0.075, 0.0025], gap=1e-3, porosity=1), size=N) x_gt = jax.device_put(x_gt) # convert to jax type, push to GPU diff --git a/examples/scripts/deconv_ppp_bm4d_admm.py b/examples/scripts/deconv_ppp_bm4d_admm.py index 9bb191574..27f82c5f4 100644 --- a/examples/scripts/deconv_ppp_bm4d_admm.py +++ b/examples/scripts/deconv_ppp_bm4d_admm.py @@ -32,9 +32,8 @@ Create a ground truth image. """ np.random.seed(1234) -Nx = 128 -Ny = 128 -Nz = 128 +N = 128 # phantom size +Nx, Ny, Nz = N, N, N // 4 upsamp = 2 x_gt_hires = create_3D_foam_phantom((upsamp * Nz, upsamp * Ny, upsamp * Nx), N_sphere=100) x_gt = downsample_volume(x_gt_hires, upsamp) diff --git a/examples/scripts/deconv_ppp_dncnn_admm.py b/examples/scripts/deconv_ppp_dncnn_admm.py index f4bc18840..0d9036f28 100644 --- a/examples/scripts/deconv_ppp_dncnn_admm.py +++ b/examples/scripts/deconv_ppp_dncnn_admm.py @@ -31,7 +31,8 @@ Create a ground truth image. """ np.random.seed(1234) -x_gt = discrete_phantom(Foam(size_range=[0.075, 0.0025], gap=1e-3, porosity=1), size=512) +N = 512 # image size +x_gt = discrete_phantom(Foam(size_range=[0.075, 0.0025], gap=1e-3, porosity=1), size=N) x_gt = jax.device_put(x_gt) # convert to jax array, push to GPU diff --git a/examples/scripts/deconv_tv_admm.py b/examples/scripts/deconv_tv_admm.py index 5cecf5c5e..8dbdbb0ab 100644 --- a/examples/scripts/deconv_tv_admm.py +++ b/examples/scripts/deconv_tv_admm.py @@ -35,7 +35,8 @@ Create a ground truth image. """ phantom = SiemensStar(32) -x_gt = snp.pad(discrete_phantom(phantom, 240), 8) +N = 256 # image size +x_gt = snp.pad(discrete_phantom(phantom, N - 16), 8) x_gt = jax.device_put(x_gt) # convert to jax type, push to GPU diff --git a/examples/scripts/deconv_tv_admm_tune.py b/examples/scripts/deconv_tv_admm_tune.py index 99235f606..c1bcfd506 100644 --- a/examples/scripts/deconv_tv_admm_tune.py +++ b/examples/scripts/deconv_tv_admm_tune.py @@ -31,7 +31,8 @@ Create a ground truth image. """ phantom = SiemensStar(32) -x_gt = snp.pad(discrete_phantom(phantom, 240), 8) +N = 256 # image size +x_gt = snp.pad(discrete_phantom(phantom, N - 16), 8) """ diff --git a/examples/scripts/denoise_tv_iso_multi.py b/examples/scripts/denoise_tv_iso_multi.py index 238a9411a..6f9f4aa66 100644 --- a/examples/scripts/denoise_tv_iso_multi.py +++ b/examples/scripts/denoise_tv_iso_multi.py @@ -33,7 +33,8 @@ Create a ground truth image. """ phantom = SiemensStar(32) -x_gt = snp.pad(discrete_phantom(phantom, 240), 8) +N = 256 # image size +x_gt = snp.pad(discrete_phantom(phantom, N - 16), 8) x_gt = jax.device_put(x_gt) # convert to jax type, push to GPU diff --git a/examples/scripts/denoise_tv_iso_pgm.py b/examples/scripts/denoise_tv_iso_pgm.py index cb8ab58ec..ce89f4e6a 100644 --- a/examples/scripts/denoise_tv_iso_pgm.py +++ b/examples/scripts/denoise_tv_iso_pgm.py @@ -50,7 +50,7 @@ """ N = 256 # image size phantom = SiemensStar(16) -x_gt = snp.pad(discrete_phantom(phantom, 240), 8) +x_gt = snp.pad(discrete_phantom(phantom, N - 16), 8) x_gt = jax.device_put(x_gt) # convert to jax type, push to GPU x_gt = x_gt / x_gt.max()