Notebook testing with nbmake (#340)

.github/workflows/test_all_notebooks.yml

@@ -0,0 +1,40 @@
+# This workflows executes all example notebooks to ensure they are up-to-date.
+
+name: test_all_notebooks
+
+on:
+  # Trigger the workflow on manual dispatch and once a week
+  workflow_dispatch
+  # schedule:
+  #  - cron: '0 0 * * 0'
+
+jobs:
+  test_notebooks:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.6', '3.7', '3.8', '3.9' ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get install build-essential
+          pip install --upgrade pip
+          pip install -r requirements/dev.txt
+          pip install -e .[prophet]
+          pip install -e .[torch]
+          pip freeze
+
+      - name: Run notebooks
+        run: |
+          pytest --nbmake examples/*.ipynb
+#          pytest -rA --durations=0 -vv testing/test_notebooks.py

.github/workflows/test_changed_notebooks.yml

@@ -0,0 +1,57 @@
+# This workflows executes new or modified example notebooks.
+
+name: test_changed_notebooks
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'examples/**/*.ipynb'
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'examples/**/*.ipynb'
+    types: [opened, synchronize, reopened, ready_for_review]
+
+jobs:
+  test_changed_notebooks:
+    if: github.event.pull_request.draft == false
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.6', '3.7', '3.8', '3.9' ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Check for new or changed .ipynb files
+        id: changed-ipynb
+        uses: tj-actions/[email protected]
+        with:
+          files: |
+            examples/*.ipynb
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          sudo apt-get install build-essential
+          pip install --upgrade pip
+          pip install -r requirements/dev.txt
+          pip install -e .[prophet]
+          pip install -e .[torch]
+          pip freeze
+      - name: Run notebooks
+        run: |
+          pytest --nbmake `echo ${{ steps.changed-ipynb.outputs.all_modified_files }}`
+#          for file in "${{ steps.changed-ipynb.outputs.all_modified_files }}"; do
+#            python testing/test_notebooks.py $file
+#          done

requirements/dev.txt

@@ -5,7 +5,8 @@ mypy>=0.670, <0.900
 # testing
 pytest>=5.3.5, <7.0.0
 pytest-cov>=2.6.1, <3.0.0
-pytest-xdist>=1.28.0, <3.0.0 # for distributed testing, currently unused (see setup.cfg)s
+pytest-xdist>=1.28.0, <3.0.0 # for distributed testing, currently unused (see setup.cfg)
+nbmake>=0.7, <1.0.0 # for notebook tests
 # other
 pre-commit>=1.20.0, <3.0.0
 tox>=3.21.0, <4.0.0 # used to generate licence info via `make licenses`