Average and best of 10 values (#20)

WIth this change we run the performance tests 10 times, and record the average and the best of 10 in the performance report. This will hopefully lead to more stable results. Contributed on behalf of STMicroelectronics.
eclipse-theia · Oct 10, 2023 · 5764ecb · 5764ecb
1 parent 8082472
commit 5764ecb
Show file tree

Hide file tree

Showing 3 changed files with 233 additions and 103 deletions.
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -2,35 +2,32 @@ name: Performance
 
 on:
   workflow_dispatch:
+  pull_request: # FIXME JUST FOR TESTING
+    branches:
+      - main
   schedule:
     - cron: "0 5 * * *" # Runs every day at 5am: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events-schedule
 
 jobs:
-  build:
+  prepare:
     name: Performance benchmark on ubuntu-latest with Node.js 16.x
-
     runs-on: ubuntu-latest
-    timeout-minutes: 60
-
+    timeout-minutes: 20
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-
       - name: Use Node.js "16.x"
         uses: actions/setup-node@v3
         with:
           node-version: "16.x"
           registry-url: "https://registry.npmjs.org"
-
       - name: Use Python 3.x
         uses: actions/setup-python@v4
         with:
           python-version: "3.x"
-
       - name: Build
         shell: bash
         run: yarn
-
       - name: Checkout Theia
         uses: actions/checkout@v3
         with:
@@ -57,29 +54,87 @@ jobs:
         env:
           NODE_OPTIONS: --max_old_space_size=4096
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9
-      - name: Run Theia
+      - name: Cache Build Result
+        uses: actions/cache@v3
+        id: build-result
+        with:
+          path: ./*
+          key: ${{ github.run_number }}
+
+  run-test:
+    name: Performance benchmark on ubuntu-latest with Node.js 16.x
+    runs-on: ubuntu-latest
+    needs: [prepare]
+    strategy:
+      matrix:
+        run: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+    timeout-minutes: 40
+    steps:
+      - name: Restore Build Result (#${{ matrix.run }})
+        uses: actions/cache/restore@v3
+        id: build-result
+        with:
+          path: ./*
+          key: ${{ github.run_number }}
+          fail-on-cache-miss: true
+      - name: Use Node.js "16.x"
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16.x"
+          registry-url: "https://registry.npmjs.org"
+      - name: Use Python 3.x
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - name: Get History
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+          path: gh-pages
+      - name: Run Theia (#${{ matrix.run }})
         shell: bash
         working-directory: ./theia
         run: yarn browser start:debug &
-
-      - name: Run Performance Measurement
+      - name: Run Performance Measurement (#${{ matrix.run }})
         uses: GabrielBB/xvfb-action@v1
+        env:
+          RUN_NO: ${{ matrix.run }}
         with:
           run: yarn performance
+      - name: Upload performance measurement (#${{ matrix.run }})
+        uses: actions/upload-artifact@v3
+        with:
+          name: performance-measurement-${{ github.run_number }}
+          path: performance-metrics/*
 
+  generate-report:
+    name: Performance benchmark on ubuntu-latest with Node.js 16.x
+    runs-on: ubuntu-latest
+    needs: [run-test]
+    steps:
+      - name: Restore Build Result
+        uses: actions/cache/restore@v3
+        id: build-result
+        with:
+          path: ./*
+          key: ${{ github.run_number }}
+          fail-on-cache-miss: true
       - name: Get History
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         if: always() && github.ref == 'refs/heads/main'
         continue-on-error: true
         with:
           ref: gh-pages
           path: gh-pages
-
+      - name: Download Performance Measurements
+        uses: actions/download-artifact@v3
+        with:
+          name: performance-measurement-${{ github.run_number }}
+          path: performance-metrics
       - name: Prepare Report
         if: always() && github.ref == 'refs/heads/main'
         shell: bash
         run: yarn performance-report
-
       - name: Publish Report
         if: always() && github.ref == 'refs/heads/main'
         uses: peaceiris/actions-gh-pages@v2

diff --git a/scripts/fetch-metrics.ts b/scripts/fetch-metrics.ts
@@ -42,7 +42,8 @@ export async function fetchPerformanceMetrics({
     const dateString = `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}`;
     const timeString = `${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`;
     const timestamp = `${dateString}T${timeString}`;
-    const fileName = outputFileNamePrefix + timestamp + outputFileNamePostfix;
+    const runNo = process.env.RUN_NO ? `_${process.env.RUN_NO}` : '';
+    const fileName = outputFileNamePrefix + timestamp + runNo + outputFileNamePostfix;
     const targetFile = `${outputFilePath}/${fileName}`;
 
     fs.ensureDirSync(outputFilePath);