Reduce github pytest workflow runtime using pytest-split

.github/isbin.sh

@@ -15,7 +15,8 @@ for f in $@; do
         Darwin) size=$(stat -f "%z" $f);;
         *)      echo "Error: unsupported operating system $OS" >&2; exit 1;;
     esac
-    if [ $size -gt $size_threshold ]; then
+    # Exception on maximum size for pytest-split .test_durations file
+    if [ $size -gt $size_threshold ] && [ "$(basename $f)" != ".test_durations" ]; then
         echo "file exceeds maximum allowable size of $size_threshold bytes"
         echo "raw data and ipynb files should go in scico-data"
         exit 2

.github/workflows/check_files.yml

@@ -2,23 +2,16 @@
 
 name: check files
 
-# Controls when the action will run.
 on: [push, pull_request]
 
-#
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  # This workflow contains a single job called "build"
   checkfiles:
-    # The type of runner that the job will run on
     runs-on: ubuntu-latest
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - id: files
-      uses: Ana06/get-changed-files@v1.2
+      uses: Ana06/get-changed-files@v2.2.0
       continue-on-error: true
     - run: |
        for f in ${{ steps.files.outputs.added }}; do

.github/workflows/lint.yml

@@ -2,33 +2,27 @@
 
 name: lint
 
-# Controls when the action will run.
 on:
     push:
         branches:
           - main
     pull_request:
 
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  # This workflow contains a single job called "build"
   lint:
-    # The type of runner that the job will run on
     runs-on: ubuntu-latest
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
-
+          python-version: "3.9"
       - name: Black code formatter
         uses: psf/black@stable
         with:
           version: "22.3"
-
       - name: Isort import sorter
-        uses: isort/[email protected]
+        uses: isort/isort-action@v1
+      - name: Pylint code analysis
+        run: |
+          pip install pylint
+          pylint --disable=all --enable=missing-docstring scico

.github/workflows/mypy.yml

@@ -2,32 +2,25 @@
 
 name: mypy
 
-# Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the main branch
   push:
     branches: [ main ]
   pull_request:
     branches: [ main ]
 
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 jobs:
   mypy:
-    # The type of runner that the job will run on
     runs-on: ubuntu-latest
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: recursive
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.9"
       - name: Install dependencies
         run: |
           pip install mypy

.github/workflows/pypi_upload.yml

@@ -1,6 +1,6 @@
 # When a tag is pushed, build packages and upload to PyPI
 
-name: pypi
+name: pypi upload
 
 # Trigger when tags are pushed
 on:
@@ -15,13 +15,13 @@ jobs:
     name: Upload package to PyPI
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: recursive
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.9"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/pytest_latest.yml

@@ -1,6 +1,6 @@
 # Install scico requirements and run pytest with latest jax version
 
-name: unit tests -- latest jax
+name: unit tests (latest jax)
 
 # Controls when the workflow will run
 on:
@@ -13,19 +13,15 @@ on:
 
 jobs:
   pytest-latest-jax:
-    # The type of runner that the job will run on
     runs-on: ubuntu-latest
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: recursive
       - name: Install Python 3
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.9"
       - name: Install lastversion
         run: |
           python -m pip install --upgrade pip

.github/workflows/pytest_macos.yml

@@ -0,0 +1,78 @@
+# Install scico requirements and run pytest
+
+name: unit tests (macos)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+  workflow_dispatch:
+
+jobs:
+
+  test:
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1, 2, 3, 4, 5]
+    name: pytest split ${{ matrix.group }} (macos)
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      # Check-out the repository under $GITHUB_WORKSPACE
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      # Set up conda/mamba environment
+      - name: Set up mambaforge
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            activate-environment: test-env
+            use-mamba: true
+            python-version: "3.9"
+      # Configure conda environment cache
+      - name: Set up conda environment cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.CONDA }}/envs
+          key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev_requirements.txt') }}-${{ env.CACHE_NUMBER }}
+        env:
+          CACHE_NUMBER: 1  # Increase this value to force cache reset
+        id: cache
+      # Display environment details
+      - name: Display environment details
+        run: |
+          conda info
+          printenv | sort
+      # Install dependencies in conda environment
+      - name: Install dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          mamba install -c conda-forge pytest pytest-cov
+          python -m pip install --upgrade pip
+          pip install pytest-split
+          pip install -r requirements.txt
+          pip install -r dev_requirements.txt
+          mamba install -c astra-toolbox astra-toolbox
+          mamba install -c conda-forge pyyaml
+          pip install --upgrade --force-reinstall scipy>=1.6.0  # Temporary fix for GLIBCXX_3.4.30 not found in conda forge version
+          pip install bm4d>=4.0.0
+          pip install "ray[tune]>=2.0.0"
+          pip install hyperopt
+      # Install package to be tested
+      - name: Install package to be tested
+        run: pip install -e .
+      # Run unit tests
+      - name: Run main unit tests
+        run: pytest -x --splits 5 --group ${{ matrix.group }}
+      # Run doc tests
+      - name: Run doc tests
+        if: matrix.group == 1
+        run: |
+          pytest --ignore-glob="*test_*.py" --doctest-modules scico