Add UI tests

.github/workflows/ui-tests.yml

@@ -10,6 +10,7 @@ jobs:
       matrix:
         python-version: ['3.10']
         node-version: [18.x]
+        progressive_rendering: [false, true]
       fail-fast: false
 
     steps:
@@ -31,7 +32,7 @@ jobs:
       - name: Launch Voila
         run: |
           # Mount a volume to overwrite the server configuration
-          jlpm start 2>&1 > /tmp/voila_server.log &
+          jlpm start --progressive_rendering=${{ matrix.progressive_rendering }} 2>&1 > /tmp/voila_server.log &
         working-directory: ui-tests
 
       - name: Install browser
@@ -45,6 +46,8 @@ jobs:
           timeout: 360000
 
       - name: Test
+        env:
+          PROGRESSIVE_RENDERING: ${{ matrix.progressive_rendering }}
         run: jlpm run test
         working-directory: ui-tests
 

.github/workflows/update_galata_references.yaml

@@ -15,29 +15,71 @@ defaults:
 jobs:
   update-reference-screenshots:
     name: Update Galata References
-    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'update galata references') }}
+    if: >
+      (
+        github.event.comment.author_association == 'OWNER' ||
+        github.event.comment.author_association == 'COLLABORATOR' ||
+        github.event.comment.author_association == 'MEMBER'
+      ) && github.event.issue.pull_request && contains(github.event.comment.body, 'please update snapshots')
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
-        node-version: [16.x]
+        python-version: [3.10]
+        node-version: [18.x]
 
     steps:
-      - name: Base Setup
-        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: React to the triggering comment
+        run: |
+          gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions --raw-field 'content=+1'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Configure git to use https
-        run: git config --global hub.protocol https
+      - name: Get PR Info
+        id: pr
+        env:
+          PR_NUMBER: ${{ github.event.issue.number }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          COMMENT_AT: ${{ github.event.comment.created_at }}
+        run: |
+          pr="$(gh api /repos/${GH_REPO}/pulls/${PR_NUMBER})"
+          head_sha="$(echo "$pr" | jq -r .head.sha)"
+          pushed_at="$(echo "$pr" | jq -r .pushed_at)"
+
+          if [[ $(date -d "$pushed_at" +%s) -gt $(date -d "$COMMENT_AT" +%s) ]]; then
+              echo "Updating is not allowed because the PR was pushed to (at $pushed_at) after the triggering comment was issued (at $COMMENT_AT)"
+              exit 1
+          fi
+
+          echo "head_sha=$head_sha" >> $GITHUB_OUTPUT
 
       - name: Checkout the branch from the PR that triggered the job
-        run: hub pr checkout ${{ github.event.issue.number }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh pr checkout ${{ github.event.issue.number }}
+
+      - name: Validate the fetched branch HEAD revision
+        env:
+          EXPECTED_SHA: ${{ steps.pr.outputs.head_sha }}
+        run: |
+          actual_sha="$(git rev-parse HEAD)"
+
+          if [[ "$actual_sha" != "$EXPECTED_SHA" ]]; then
+              echo "The HEAD of the checked out branch ($actual_sha) differs from the HEAD commit available at the time when trigger comment was submitted ($EXPECTED_SHA)"
+              exit 1
+          fi
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Configure git to use https
+        run: git config --global hub.protocol https
 
       - name: Install dependencies
         run: |
@@ -54,3 +96,11 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           server_url: http-get://localhost:8866/
           test_folder: ui-tests
+
+      - uses: jupyterlab/maintainer-tools/.github/actions/update-snapshots@main
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          server_url: http-get://localhost:8866/
+          test_folder: ui-tests
+          start_server_script : start:progressive_rendering
+          update_script: test:update:progressive_rendering

requirements-visual-test.txt

@@ -1,4 +1,5 @@
 bokeh
+bokeh_sampledata
 bqplot
 ipympl==0.9.2
 jupyterlab~=4.0

ui-tests/package.json

@@ -5,11 +5,13 @@
   "private": true,
   "scripts": {
     "start": "voila ../notebooks --no-browser --show_tracebacks True --Voila.mathjax_url=https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/MathJax.js",
+    "start:progressive_rendering": "voila ../notebooks --no-browser --show_tracebacks True --Voila.mathjax_url=https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/MathJax.js --progressive_rendering=true",
     "start:detached": "yarn run start&",
     "test": "npx playwright test",
     "test:debug": "PWDEBUG=1 playwright test",
     "test:report": "http-server ./playwright-report -a localhost -o",
-    "test:update": "npx playwright test --update-snapshots"
+    "test:update": "npx playwright test --update-snapshots",
+    "test:update:progressive_rendering": "PROGRESSIVE_RENDERING=true && npx playwright test --update-snapshots"
   },
   "author": "Project Jupyter",
   "license": "BSD-3-Clause",

ui-tests/tests/voila.test.ts

@@ -4,6 +4,8 @@
 import { expect, test } from '@playwright/test';
 import { addBenchmarkToTest } from './utils';
 
+const PREFIX =
+  process.env.PROGRESSIVE_RENDERING === 'true' ? 'progressive_rendering_' : '';
 test.describe('Voila performance Tests', () => {
   test.beforeEach(({ page }) => {
     page.setDefaultTimeout(120000);
@@ -24,7 +26,8 @@ test.describe('Voila performance Tests', () => {
       browserName
     );
 
-    expect(await page.screenshot()).toMatchSnapshot('voila-tree-classic.png');
+    // await expect(page).toHaveScreenshot('voila-tree-classic.png');
+    await expect(page).toHaveScreenshot(`${PREFIX}voila-tree-classic.png`);
   });
 
   test('Render tree light theme', async ({ page, browserName }, testInfo) => {
@@ -40,7 +43,7 @@ test.describe('Voila performance Tests', () => {
       browserName
     );
 
-    expect(await page.screenshot()).toMatchSnapshot('voila-tree-light.png');
+    await expect(page).toHaveScreenshot(`${PREFIX}voila-tree-light.png`);
   });
 
   test('Render tree dark theme', async ({ page, browserName }, testInfo) => {
@@ -56,7 +59,7 @@ test.describe('Voila performance Tests', () => {
       browserName
     );
 
-    expect(await page.screenshot()).toMatchSnapshot('voila-tree-dark.png');
+    await expect(page).toHaveScreenshot(`${PREFIX}voila-tree-dark.png`);
   });
 
   test('Render tree miami theme', async ({ page, browserName }, testInfo) => {
@@ -72,7 +75,7 @@ test.describe('Voila performance Tests', () => {
       browserName
     );
 
-    expect(await page.screenshot()).toMatchSnapshot('voila-tree-miami.png');
+    await expect(page).toHaveScreenshot(`${PREFIX}voila-tree-miami.png`);
   });
 
   test('Render and benchmark basics.ipynb with classic template', async ({
@@ -90,9 +93,7 @@ test.describe('Voila performance Tests', () => {
     // wait for the final MathJax message to be hidden
     await page.$('text=Typesetting math: 100%');
     await page.waitForSelector('#MathJax_Message', { state: 'hidden' });
-    expect(await page.screenshot()).toMatchSnapshot(
-      `${notebookName}-classic.png`
-    );
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}-classic.png`);
   });
 
   test('Render and benchmark basics.ipynb', async ({
@@ -117,7 +118,7 @@ test.describe('Voila performance Tests', () => {
     // wait for the final MathJax message to be hidden
     await page.$('text=Typesetting math: 100%');
     await page.waitForSelector('#MathJax_Message', { state: 'hidden' });
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 
   test('Render basics.ipynb with dark theme', async ({
@@ -135,7 +136,7 @@ test.describe('Voila performance Tests', () => {
     // wait for the final MathJax message to be hidden
     await page.$('text=Typesetting math: 100%');
     await page.waitForSelector('#MathJax_Message', { state: 'hidden' });
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}-dark.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}-dark.png`);
   });
 
   test('Render basics.ipynb with miami theme', async ({
@@ -155,30 +156,28 @@ test.describe('Voila performance Tests', () => {
     // wait for the final MathJax message to be hidden
     await page.$('text=Typesetting math: 100%');
     await page.waitForSelector('#MathJax_Message', { state: 'hidden' });
-    expect(await page.screenshot()).toMatchSnapshot(
-      `${notebookName}-miami.png`
-    );
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}-miami.png`);
   });
 
   test('Render 404 error', async ({ page }) => {
     await page.goto('/voila/render/unknown.ipynb');
     await page.waitForSelector('.voila-error');
 
-    expect(await page.screenshot()).toMatchSnapshot('404.png');
+    await expect(page).toHaveScreenshot('404.png');
   });
 
   test('Render 404 error with classic template', async ({ page }) => {
     await page.goto('/voila/render/unknown.ipynb?template=classic');
     await page.waitForSelector('.voila-error');
 
-    expect(await page.screenshot()).toMatchSnapshot('404-classic.png');
+    await expect(page).toHaveScreenshot('404-classic.png');
   });
 
   test('Render 404 error with dark theme', async ({ page }) => {
     await page.goto('/voila/render/unknown.ipynb?theme=dark');
     await page.waitForSelector('.voila-error');
 
-    expect(await page.screenshot()).toMatchSnapshot('404-dark.png');
+    await expect(page).toHaveScreenshot('404-dark.png');
   });
 
   test('Render and benchmark bqplot.ipynb', async ({
@@ -191,7 +190,7 @@ test.describe('Voila performance Tests', () => {
       await page.waitForSelector('svg.svg-figure');
     };
     await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName);
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 
   test('Render and benchmark dashboard.ipynb', async ({
@@ -222,7 +221,7 @@ test.describe('Voila performance Tests', () => {
       );
     };
     await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName);
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 
   test('Render and benchmark interactive.ipynb', async ({
@@ -240,7 +239,7 @@ test.describe('Voila performance Tests', () => {
       await page.mouse.click(0, 0);
     };
     await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName);
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 
   test('Render and benchmark ipympl.ipynb', async ({
@@ -253,7 +252,7 @@ test.describe('Voila performance Tests', () => {
       await page.waitForSelector('div.jupyter-matplotlib-figure');
     };
     await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName);
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 
   test('Render and benchmark mimerenderers.ipynb', async ({
@@ -268,7 +267,7 @@ test.describe('Voila performance Tests', () => {
       await page.waitForTimeout(2000);
     };
     await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName);
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 
   test('Render and benchmark bokeh.ipynb', async ({
@@ -281,7 +280,7 @@ test.describe('Voila performance Tests', () => {
       await page.waitForSelector('.bk-Canvas');
     };
     await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName);
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 
   test('Benchmark the multiple widgets notebook', async ({
@@ -301,7 +300,7 @@ test.describe('Voila performance Tests', () => {
       testInfo,
       browserName
     );
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 
   test('Render and benchmark query-strings.ipynb', async ({
@@ -311,20 +310,26 @@ test.describe('Voila performance Tests', () => {
     const notebookName = 'query-strings';
     const testFunction = async () => {
       await page.goto(`/voila/render/${notebookName}.ipynb`);
+      await page.waitForSelector(
+        'div.lm-Widget[data-mime-type="application/vnd.jupyter.stdout"]'
+      );
       const userName = await page.$$(
         'div.jp-RenderedText.jp-OutputArea-output > pre'
       );
       expect(await userName[1].innerHTML()).toContain('Hi Kim');
     };
     await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName);
     await page.goto(`/voila/render/${notebookName}.ipynb?username=Riley`);
+    await page.waitForSelector(
+      'div.lm-Widget[data-mime-type="application/vnd.jupyter.stdout"]'
+    );
     const userName = await page.$$(
       'div.jp-RenderedText.jp-OutputArea-output > pre'
     );
     expect(await userName[1].innerHTML()).toContain('Hi Riley');
     await page.$('text=Typesetting math: 100%');
     await page.waitForSelector('#MathJax_Message', { state: 'hidden' });
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 
   test('Render and benchmark reveal.ipynb', async ({
@@ -337,13 +342,13 @@ test.describe('Voila performance Tests', () => {
       await page.waitForSelector('span[role="presentation"] >> text=x');
     };
     await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName);
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 
   test('Render yaml.ipynb', async ({ page, browserName }, testInfo) => {
     const notebookName = 'yaml';
     await page.goto(`/voila/render/${notebookName}.ipynb`);
     await page.waitForSelector('span >> text=hey');
-    expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`);
+    await expect(page).toHaveScreenshot(`${PREFIX}${notebookName}.png`);
   });
 });