Merge pull request #913 from NeurodataWithoutBorders/e2e

Split base tests from end to end, regenerate screenshots
NeurodataWithoutBorders · Aug 15, 2024 · 6356e1f · 6356e1f
2 parents baa7e68 + e698337
commit 6356e1f
Show file tree

Hide file tree

Showing 47 changed files with 232 additions and 56 deletions.
diff --git a/.github/workflows/daily_tests.yml b/.github/workflows/daily_tests.yml
@@ -20,6 +20,11 @@ jobs:
     uses: ./.github/workflows/testing_dev_with_live_services.yml
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  E2ELiveServices:
+    uses: ./.github/workflows/testing_dev_e2e_with_live_services.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       DANDI_STAGING_API_KEY: ${{ secrets.DANDI_STAGING_API_KEY }}
 
   BuildTests:

diff --git a/.github/workflows/deploy_tests_on_pull_request.yml b/.github/workflows/deploy_tests_on_pull_request.yml
@@ -18,6 +18,11 @@ jobs:
     uses: ./.github/workflows/testing_dev_with_live_services.yml
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  E2ELiveServices:
+    uses: ./.github/workflows/testing_dev_e2e_with_live_services.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       DANDI_STAGING_API_KEY: ${{ secrets.DANDI_STAGING_API_KEY }}
 
   BuildTests:

diff --git a/.github/workflows/testing_dev.yml b/.github/workflows/testing_dev.yml
@@ -29,8 +29,8 @@ jobs:
           - os: macos-13  # Mac x64 runner
             label: environments/environment-MAC-intel.yml
 
-#          - os: windows-latest
-#            label: environments/environment-Windows.yml
+          - os: windows-latest
+            label: environments/environment-Windows.yml
 
     steps:
       - uses: actions/checkout@v4
@@ -83,16 +83,6 @@ jobs:
         name: Run tests with xvfb
         run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test:coverage
 
-      - name: Archive E2E Test Screenshots
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-screenshots-${{ matrix.os }}
-          path: docs/assets/tutorials
-          retention-days: 1
-          overwrite: true
-
-
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4
         env:

diff --git a/.github/workflows/testing_dev_e2e_with_live_services.yml b/.github/workflows/testing_dev_e2e_with_live_services.yml
@@ -0,0 +1,106 @@
+name: End-to-end tests with live services
+on:
+  workflow_call:
+    secrets:
+      CODECOV_TOKEN:
+        required: true
+      DANDI_STAGING_API_KEY:
+        required: true
+
+jobs:
+
+  run:
+    # Will read on PR dashboard as 'Deploy / E2ELiveServices / {os}'
+    # Action dashboard identified by 'End-to-end tests with live services'
+    # Requirement settings identified as 'E2ELiveServices / {os}'
+    name: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            label: environments/environment-Linux.yml
+
+          - os: macos-latest  # Mac arm64 runner
+            label: environments/environment-MAC-apple-silicon.yml
+
+          - os: macos-13  # Mac x64 runner
+            label: environments/environment-MAC-intel.yml
+
+          - os: windows-latest
+            label: environments/environment-Windows.yml
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: git fetch --prune --unshallow --tags
+
+      # see https://github.com/conda-incubator/setup-miniconda#caching-environments
+      - name: Setup Mambaforge
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          activate-environment: nwb-guide
+          use-mamba: true
+
+      - name: Set cache date
+        id: get-date
+        run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Cache Conda env
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.CONDA }}/envs
+          key: conda-${{ runner.os }}-${{ runner.arch }}-${{steps.get-date.outputs.today }}-${{ hashFiles(matrix.label) }}-${{ env.CACHE_NUMBER }}
+        id: cache
+
+      - if: steps.cache.outputs.cache-hit != 'true'
+        name: Create and activate environment
+        run: mamba env update -n nwb-guide -f ${{ matrix.label }}
+
+      - name: Use Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install GUIDE
+        run: npm ci --verbose
+
+      - name: Install testing dependencies
+        run: pip install pytest pytest-cov
+
+      - name: Manually remove matplotlib
+        run: pip uninstall matplotlib --yes
+
+      - name: Create env file
+        run: |
+          touch .env
+          echo DANDI_STAGING_API_KEY=${{ secrets.DANDI_STAGING_API_KEY }} >> .env
+
+      - if: matrix.os != 'ubuntu-latest'
+        name: Run tests
+        run: npm run coverage:tutorial
+
+      - if: matrix.os == 'ubuntu-latest'
+        name: Run tests with xvfb
+        run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run coverage:tutorial
+
+      - name: Archive E2E Test Screenshots
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-screenshots-${{ matrix.os }}
+          path: docs/assets/tutorials
+          retention-days: 1
+          overwrite: true
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          fail_ci_if_error: true
diff --git a/.github/workflows/testing_dev_with_live_services.yml b/.github/workflows/testing_dev_with_live_services.yml
@@ -4,8 +4,6 @@ on:
     secrets:
       CODECOV_TOKEN:
         required: true
-      DANDI_STAGING_API_KEY:
-        required: true
 
 jobs:
 
@@ -31,8 +29,8 @@ jobs:
           - os: macos-13  # Mac x64 runner
             label: environments/environment-MAC-intel.yml
 
-#          - os: windows-latest
-#            label: environments/environment-Windows.yml
+          - os: windows-latest
+            label: environments/environment-Windows.yml
 
     steps:
       - uses: actions/checkout@v4
@@ -77,11 +75,6 @@ jobs:
       - name: Manually remove matplotlib
         run: pip uninstall matplotlib --yes
 
-      - name: Create env file
-        run: |
-          touch .env
-          echo DANDI_STAGING_API_KEY=${{ secrets.DANDI_STAGING_API_KEY }} >> .env
-
       - if: matrix.os != 'ubuntu-latest'
         name: Run tests
         run: npm run coverage:app

diff --git a/docs/assets/tutorials/dandi/create-dandiset.png b/docs/assets/tutorials/dandi/create-dandiset.png
diff --git a/docs/assets/tutorials/dandi/dandiset-id.png b/docs/assets/tutorials/dandi/dandiset-id.png
diff --git a/docs/assets/tutorials/dandi/review-page.png b/docs/assets/tutorials/dandi/review-page.png
diff --git a/docs/assets/tutorials/dataset-created.png b/docs/assets/tutorials/dataset-created.png
diff --git a/docs/assets/tutorials/dataset-creation.png b/docs/assets/tutorials/dataset-creation.png
diff --git a/docs/assets/tutorials/home-page.png b/docs/assets/tutorials/home-page.png
diff --git a/docs/assets/tutorials/multiple/conversion-results-page.png b/docs/assets/tutorials/multiple/conversion-results-page.png
diff --git a/docs/assets/tutorials/multiple/fail-name.png b/docs/assets/tutorials/multiple/fail-name.png
diff --git a/docs/assets/tutorials/multiple/home-page-complete.png b/docs/assets/tutorials/multiple/home-page-complete.png
diff --git a/docs/assets/tutorials/multiple/inspect-page.png b/docs/assets/tutorials/multiple/inspect-page.png
diff --git a/docs/assets/tutorials/multiple/metadata-ecephys.png b/docs/assets/tutorials/multiple/metadata-ecephys.png
diff --git a/docs/assets/tutorials/multiple/metadata-nwbfile.png b/docs/assets/tutorials/multiple/metadata-nwbfile.png
diff --git a/docs/assets/tutorials/multiple/pathexpansion-autocomplete-filled.png b/docs/assets/tutorials/multiple/pathexpansion-autocomplete-filled.png
diff --git a/docs/assets/tutorials/multiple/pathexpansion-autocomplete-open.png b/docs/assets/tutorials/multiple/pathexpansion-autocomplete-open.png
diff --git a/docs/assets/tutorials/multiple/pathexpansion-autocomplete-submitted.png b/docs/assets/tutorials/multiple/pathexpansion-autocomplete-submitted.png
diff --git a/docs/assets/tutorials/multiple/pathexpansion-completed.png b/docs/assets/tutorials/multiple/pathexpansion-completed.png
diff --git a/docs/assets/tutorials/multiple/pathexpansion-page.png b/docs/assets/tutorials/multiple/pathexpansion-page.png
diff --git a/docs/assets/tutorials/multiple/preview-page.png b/docs/assets/tutorials/multiple/preview-page.png
diff --git a/docs/assets/tutorials/multiple/subject-complete.png b/docs/assets/tutorials/multiple/subject-complete.png
diff --git a/docs/assets/tutorials/multiple/subject-error.png b/docs/assets/tutorials/multiple/subject-error.png
diff --git a/docs/assets/tutorials/multiple/subject-invalid.png b/docs/assets/tutorials/multiple/subject-invalid.png
diff --git a/docs/assets/tutorials/multiple/subject-page.png b/docs/assets/tutorials/multiple/subject-page.png
diff --git a/docs/assets/tutorials/multiple/workflow-page.png b/docs/assets/tutorials/multiple/workflow-page.png
diff --git a/docs/assets/tutorials/single/conversion-results-page.png b/docs/assets/tutorials/single/conversion-results-page.png
diff --git a/docs/assets/tutorials/single/home-page-complete.png b/docs/assets/tutorials/single/home-page-complete.png
diff --git a/docs/assets/tutorials/single/inspect-page.png b/docs/assets/tutorials/single/inspect-page.png
diff --git a/docs/assets/tutorials/single/metadata-ecephys.png b/docs/assets/tutorials/single/metadata-ecephys.png
diff --git a/docs/assets/tutorials/single/metadata-nwbfile.png b/docs/assets/tutorials/single/metadata-nwbfile.png
diff --git a/docs/assets/tutorials/single/preview-page.png b/docs/assets/tutorials/single/preview-page.png
diff --git a/docs/assets/tutorials/single/sourcedata-page-specified.png b/docs/assets/tutorials/single/sourcedata-page-specified.png
diff --git a/docs/developer_guide.rst b/docs/developer_guide.rst
@@ -253,8 +253,9 @@ Updating Tutorial Screenshots
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Before a release, you'll want to update the tutorial screenshots to reflect the latest changes in the application.
 
-#. To regenerate the dataset, you'll need to change ``regenerateTestData`` in the ``tests/e2e/config.ts`` to ``true``.
-#. Run the End-to-End Tests locally using ``npm test:app``.
+#. To regenerate the dataset, you'll need to change ``regenerateTestData`` in the ``tests/e2e/config.ts`` to ``true`` or delete the test dataset directory ``rm -rf ~/NWB_GUIDE/.test``.
+#. Create a ``.env`` file with the following content: ``DANDI_STAGING_API_KEY={your_dandi_staging_api_key}`` where ``{your_dandi_staging_api_key}`` is your DANDI staging API key from https://gui-staging.dandiarchive.org.
+#. Run the End-to-End Tests locally using ``npm test:tutorial``.
     - This will generate new screenshots in the ``docs/assets/tutorials`` directory.
 #. Review the new screenshots to ensure they are accurate.
 #. If the screenshots are accurate, commit them to the repository. Their paths should be consistent across runs—allowing the new versions to show up on the tutorial.