Merge pull request #143 from krassowski/feat/integrate-profiler

Integrate ui-profiler, improve dispatch, separate jlpm/yarn

.github/workflows/benchmark.yml

@@ -10,23 +10,48 @@ on:
         description: "Git repository reference to the challenger branch"
         required: true
       challenger_project:
-        description: "Playwright project to execute on the challenger version"
+        description: "Playwright project to execute (windowingMode JupyterLab 4; renderCellOnIdle: JupyterLab 2.3 or 3.x)"
         required: false
         default: "jupyterlab"
-        type: "string"
+        type: choice
+        options:
+          - jupyterlab
+          - jupyterlab-renderCellOnIdle-on
+          - jupyterlab-renderCellOnIdle-off
+          - jupyterlab-windowingMode-full
+          - jupyterlab-windowingMode-defer
+          - jupyterlab-windowingMode-none
+          - jupyterlab-1-2
+          - retrolab
+          - testing
       reference_branch:
         description: "Reference branch on the JupyterLab repository (default: master)"
         required: false
         default: "master"
       reference_project:
         description: "Playwright project to execute on the reference version"
         required: false
-        default: "jupyterlab"
-        type: "string"
+        default: ""
+        type: choice
+        options:
+          - jupyterlab
+          - jupyterlab-renderCellOnIdle-on
+          - jupyterlab-renderCellOnIdle-off
+          - jupyterlab-windowingMode-full
+          - jupyterlab-windowingMode-defer
+          - jupyterlab-windowingMode-none
+          - jupyterlab-1-2
+          - retrolab
+          - testing
       browser:
-        description: "Which browser to use (one of 'chromium' [default], 'firefox', 'webkit')"
+        description: "Which browser to use (default 'chromium')"
         required: false
         default: "chromium"
+        type: choice
+        options:
+          - chromium
+          - firefox
+          - webkit
       samples:
         description: "Number of samples to compute"
         required: false
@@ -49,7 +74,8 @@ permissions:
 
 jobs:
   test:
-    uses: jupyterlab/benchmarks/.github/workflows/run-benchmark.yml@master
+    # uses: jupyterlab/benchmarks/.github/workflows/run-benchmark.yml@master
+    uses: ./.github/workflows/run-benchmark.yml
     with:
       event: ${{ github.event_name }}
       # Repository to clone for scheduled benchmark

.github/workflows/profiler.yml

@@ -0,0 +1,90 @@
+name: JupyterLab UI Profiler
+
+on:
+  workflow_dispatch:
+    inputs:
+      challenger:
+        description: "JupyterLab Git repository with the challenger version (format {owner}/{repo})"
+        required: true
+        default: "jupyterlab/jupyterlab"
+      challenger_branch:
+        description: "Git repository reference to the challenger branch"
+        required: true
+        default: "master"
+      challenger_project:
+        description: "Playwright project to execute (windowingMode JupyterLab 4; renderCellOnIdle: JupyterLab 2.3 or 3.x)"
+        required: false
+        default: "jupyterlab"
+        type: choice
+        options:
+          - jupyterlab
+          - jupyterlab-renderCellOnIdle-on
+          - jupyterlab-renderCellOnIdle-off
+          - jupyterlab-windowingMode-full
+          - jupyterlab-windowingMode-defer
+          - jupyterlab-windowingMode-none
+          - jupyterlab-1-2
+          - retrolab
+          - testing
+      reference_branch:
+        description: "Reference branch on the JupyterLab repository (default: master)"
+        required: false
+        default: "master"
+      reference_project:
+        description: "Playwright project to execute on the reference version"
+        required: false
+        default: ""
+        type: choice
+        options:
+          - jupyterlab
+          - jupyterlab-renderCellOnIdle-on
+          - jupyterlab-renderCellOnIdle-off
+          - jupyterlab-windowingMode-full
+          - jupyterlab-windowingMode-defer
+          - jupyterlab-windowingMode-none
+          - jupyterlab-1-2
+          - retrolab
+          - testing
+      browser:
+        description: "Which browser to use (default 'chromium')"
+        required: false
+        default: "chromium"
+        type: choice
+        options:
+          - chromium
+          - firefox
+          - webkit
+      samples:
+        description: "Number of samples to compute (individual tests multiply this number by their a test-speicific factor)"
+        required: false
+        default: "25"
+        type: string
+      grep:
+        description: "Tests to include (grep expression)"
+        required: false
+        default: ""
+        type: string
+      grep_invert:
+        description: "Tests to skip (grep expression). Benchmarking individual rules requires a lot of time (reduce number of samples!)"
+        required: false
+        default: "rules"
+        type: string
+
+permissions:
+  issues: write
+
+jobs:
+  test:
+    # uses: jupyterlab/benchmarks/.github/workflows/run-profiler.yml@master
+    uses: ./.github/workflows/run-profiler.yml
+    with:
+      challenger: ${{ github.event.inputs.challenger || 'jupyterlab/jupyterlab' }}
+      challenger_branch: ${{ github.event.inputs.challenger_branch || 'master' }}
+      challenger_project: ${{ github.event.inputs.challenger_project || 'jupyterlab' }}
+      reference_branch: ${{ github.event.inputs.reference_branch || 'master' }}
+      reference_project: ${{ github.event.inputs.reference_project || 'jupyterlab' }}
+      browser: ${{ github.event.inputs.browser || 'chromium' }}
+      samples: ${{ github.event.inputs.samples || '25' }}
+      grep: ${{ github.event.inputs.grep || '' }}
+      grep_invert: ${{ github.event.inputs.grep_invert || '' }}
+      artifacts_name: 'profiler-report'

.github/workflows/run-benchmark.yml

@@ -63,6 +63,14 @@ on:
         required: false
         default: "benchmark-assets"
         type: string
+      grep_invert:
+        description: "Tests to skip"
+        required: false
+        default: "@slow"
+        type: string
+
+env:
+  YARN_ENABLE_IMMUTABLE_INSTALLS: 0
 
 permissions:
   issues: write
@@ -154,7 +162,7 @@ jobs:
           git checkout ${OLD_REF_SHA}
         working-directory: reference
 
-      - name: Install dependencies
+      - name: Install reference
         run: |
           set -ex
           echo "OLD_REF_SHA=$(git log -n1 --format='%H')" >> $GITHUB_ENV
@@ -169,24 +177,31 @@ jobs:
           set -ex
           pip install ipywidgets plotly
           pip install -v extensions/fixed-data-table
+          pip install jupyterlab-ui-profiler
+          jupyter labextension disable @jupyterlab/ui-profiler:user-interface
 
           jupyter server extension list
           jupyter labextension list
         working-directory: benchmarks
 
-      - name: Launch JupyterLab
+      - name: Install testing dependencies
+        shell: bash
+        run: |
+          yarn install
+        working-directory: benchmarks/tests
+
+      - name: Launch JupyterLab (reference)
         shell: bash
         run: |
-          jlpm install
           # Mount a volume to overwrite the server configuration
-          jlpm start-jlab 2>&1 > /tmp/jupyterlab_server_old.log &
+          yarn start-jlab 2>&1 > /tmp/jupyterlab_server_old.log &
         working-directory: benchmarks/tests
 
       - name: Install browser
         run: |
           set -ex
           # Install only Chromium browser
-          jlpm playwright install chromium
+          yarn playwright install chromium
         working-directory: benchmarks/tests
 
       - name: Wait for JupyterLab
@@ -195,16 +210,16 @@ jobs:
           resource: http-get://localhost:9999/lab
           timeout: 360000
 
-      - name: Execute benchmark tests
+      - name: Execute benchmark tests (reference)
         continue-on-error: true
         working-directory: benchmarks/tests
         run: |
           set -ex
           # Update test screenshots
-          BENCHMARK_NUMBER_SAMPLES=1 PW_VIDEO=1 jlpm run test --project ${{ inputs.reference_project }} -u
+          BENCHMARK_NUMBER_SAMPLES=1 PW_VIDEO=1 yarn run test --project ${{ inputs.reference_project }} -u --grep-invert '${{ inputs.grep_invert }}'
 
           # -u is needed to generate the benchmark expected file
-          jlpm run test --project ${{ inputs.reference_project }} -u
+          yarn run test --project ${{ inputs.reference_project }} -u --grep-invert '${{ inputs.grep_invert }}'
           cp tests-out/lab-benchmark-expected.json /tmp/
 
       - name: Kill the server
@@ -224,7 +239,7 @@ jobs:
           ref: ${{ env.CHALLENGER_REF }}
           path: challenger
 
-      - name: Install dependencies
+      - name: Install challenger
         run: |
           set -ex
           # Fix for installing JupyterLab 1
@@ -237,11 +252,10 @@ jobs:
           cp /tmp/lab-benchmark-expected.json ./tests-out
         working-directory: challenger
 
-      - name: Launch JupyterLab
+      - name: Launch JupyterLab (challenger)
         shell: bash
         run: |
-          # Mount a volume to overwrite the server configuration
-          jlpm start-jlab 2>&1 > /tmp/jupyterlab_server_new.log &
+          yarn start-jlab 2>&1 > /tmp/jupyterlab_server_new.log &
         working-directory: benchmarks/tests
 
       - name: Wait for JupyterLab
@@ -250,7 +264,7 @@ jobs:
           resource: http-get://localhost:9999/lab
           timeout: 360000
 
-      - name: Execute benchmark tests
+      - name: Execute benchmark tests (challenger)
         continue-on-error: true
         shell: bash
         env:
@@ -259,13 +273,13 @@ jobs:
         run: |
           set -ex
           # Update test screenshots
-          BENCHMARK_NUMBER_SAMPLES=1 PW_VIDEO=1 jlpm run test --project ${{ inputs.challenger_project }} -u
+          BENCHMARK_NUMBER_SAMPLES=1 PW_VIDEO=1 yarn run test --project ${{ inputs.challenger_project }} -u --grep-invert '${{ inputs.grep_invert }}'
 
           # Copy reference here otherwise it will use the value from the update screenshots
           # command called just before
           cp /tmp/lab-benchmark-expected.json ./tests-out
 
-          jlpm run test --project ${{ inputs.challenger_project }}
+          yarn run test --project ${{ inputs.challenger_project }} --grep-invert '${{ inputs.grep_invert }}'
         working-directory: benchmarks/tests
 
       - name: Generate the report
@@ -301,6 +315,12 @@ jobs:
           # Copy the reference data to upload it as artifact
           cp ./tests-out/lab-benchmark-expected.json ./benchmark-results/
 
+      - name: Set job summary
+        if: always()
+        shell: bash
+        run: |
+          cat ./benchmarks/tests/benchmark-results/lab-benchmark.md >> $GITHUB_STEP_SUMMARY
+
       - name: Publish the report
         if: ${{ inputs.event == 'schedule' }}
         uses: actions/github-script@v6

.github/workflows/run-memory-leak.yml

@@ -17,6 +17,9 @@ on:
         default: "7"
         type: string
 
+env:
+  YARN_ENABLE_IMMUTABLE_INSTALLS: 0
+
 jobs:
   memory-leak-test:
     runs-on: ubuntu-22.04
@@ -79,8 +82,7 @@ jobs:
       - name: Launch JupyterLab
         shell: bash
         run: |
-          # Mount a volume to overwrite the server configuration
-          jlpm start-jlab 2>&1 > /tmp/jupyterlab_server.log &
+          yarn start-jlab 2>&1 > /tmp/jupyterlab_server.log &
         working-directory: benchmarks/memory-leaks
 
       - name: Execute memory leaks test