Initial notebook testing framework (#480)

* Initial notebook testing framework

* Update single notebook workflow

* Add new test notebook

* Fix typos in pip install

* Checking for changes is done by GA

* Remove tmate debugging session

* Only run on non-draft PRs

* Rename workflow files

* Enable trigger for going off draft mode on PRs

* Update regular tests to run on alibi directory only

* Rename job

* Remove test notebook

* Disable regular cronjob for now

* Add module docstring for testing example script

.github/workflows/ci.yml

@@ -43,8 +43,8 @@ jobs:
           mypy alibi
       - name: Test with pytest
         run: |
-          pytest -m tf1
-          pytest -m "not tf1"
+          pytest -m tf1 alibi
+          pytest -m "not tf1" alibi
       - name: Upload coverage to Codecov
         if: ${{ success() }}
         run: |

.github/workflows/test_all_notebooks.yml

@@ -0,0 +1,42 @@
+# This workflows executes all example notebooks to ensure they are up-to-date.
+
+name: test_all_notebooks
+
+on:
+  # Trigger the workflow on manual dispatch and once a week
+  workflow_dispatch
+  # schedule:
+  #  - cron: '0 0 * * 0'
+
+jobs:
+  test_notebooks:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.6', '3.7', '3.8', '3.9' ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip setuptools
+          pip install --upgrade --upgrade-strategy eager -r requirements/dev.txt
+          pip install --upgrade --upgrade-strategy eager -e .
+          pip install --upgrade --upgrade-strategy eager -e .[examples]
+          pip install --upgrade --upgrade-strategy eager -e .[ray]
+          pip install --upgrade --upgrade-strategy eager -e .[shap]
+          python -m spacy download en_core_web_md
+          pip freeze
+
+      - name: Run notebooks
+        run: |
+          pytest --nbmake examples/**/*.ipynb
+#          pytest -rA --durations=0 -vv testing/test_notebooks.py

.github/workflows/test_changed_notebooks.yml

@@ -0,0 +1,59 @@
+# This workflows executes new or modified example notebooks.
+
+name: test_changed_notebooks
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'examples/**/*.ipynb'
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'examples/**/*.ipynb'
+    types: [opened, synchronize, reopened, ready_for_review]
+
+jobs:
+  test_changed_notebooks:
+    if: github.event.pull_request.draft == false
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.6', '3.7', '3.8', '3.9' ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Check for new or changed .ipynb files
+        id: changed-ipynb
+        uses: tj-actions/[email protected]
+        with:
+          files: |
+            examples/*.ipynb
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip setuptools
+          pip install --upgrade --upgrade-strategy eager -r requirements/dev.txt
+          pip install --upgrade --upgrade-strategy eager -e .
+          pip install --upgrade --upgrade-strategy eager -e .[examples]
+          pip install --upgrade --upgrade-strategy eager -e .[ray]
+          pip install --upgrade --upgrade-strategy eager -e .[shap]
+          python -m spacy download en_core_web_md
+          pip freeze
+      - name: Run notebooks
+        run: |
+          pytest --nbmake `echo ${{ steps.changed-ipynb.outputs.all_modified_files }}`
+#          for file in "${{ steps.changed-ipynb.outputs.all_modified_files }}"; do
+#            python testing/test_notebooks.py $file
+#          done

requirements/dev.txt

@@ -5,8 +5,11 @@ mypy>=0.670, <0.900
 # testing
 pytest>=5.3.5, <7.0.0
 pytest-cov>=2.6.1, <3.0.0
-pytest-xdist>=1.28.0, <3.0.0 # for distributed testing, currently unused (see setup.cfg)s
+pytest-xdist>=1.28.0, <3.0.0 # for distributed testing, currently unused (see setup.cfg)
 pytest-lazy-fixture>=0.6.3, <0.7.0
+nbmake>=0.7, <1.0.0 # for notebook tests
+#pytest-timeout>=1.4.2, <2.0.0 # for notebook tests
+#jupytext>=1.12.0, <2.0.0 # for notebook tests
 codecov>=2.0.15, <3.0.0
 catboost>=0.23.0, <0.27.0
 alibi-testing @ git+git://github.com/SeldonIO/alibi-testing@master#egg=alibi-testing # pre-trained models for testing

testing/test_notebooks.py

@@ -0,0 +1,27 @@
+"""
+This script is an example of using `jupytext` to execute notebooks for testing instead of relying on `nbmake`
+plugin. This approach may be more flexible if our requirements change in the future.
+"""
+
+import argparse
+import glob
+import pytest
+from jupytext.cli import jupytext
+
+# list of all example notebooks
+notebooks = glob.glob('examples/*.ipynb')
+
+
+@pytest.mark.timeout(300)
+@pytest.mark.parametrize("notebook", notebooks)
+def test_notebook_execution(notebook):
+    jupytext(args=[notebook, "--execute"])
+
+
+if __name__ == '__main__':
+    # When run as a script, execute the notebook passed on the command line
+    parser = argparse.ArgumentParser()
+    parser.add_argument('notebook', type=str)
+    args = parser.parse_args()
+
+    test_notebook_execution(args.notebook)