Merge pull request #9 from eclipsesource/add-performance-measurement-…

…task

Add performance measurement task

.github/workflows/performance.yml

@@ -0,0 +1,81 @@
+name: Performance
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+  schedule:
+    - cron: "0 5 * * *" # Runs every day at 5am: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events-schedule
+
+jobs:
+  build:
+    name: Performance benchmark on ubuntu-latest with Node.js 16.x
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Use Node.js "16.x"
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16.x"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Use Python 3.x
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+
+      - name: Build
+        shell: bash
+        run: yarn
+
+      - name: Checkout Theia
+        uses: actions/checkout@v3
+        with:
+          repository: eclipse-theia/theia
+          path: ./theia
+
+      - name: Build Theia
+        shell: bash
+        working-directory: ./theia
+        run: |
+          yarn --skip-integrity-check --network-timeout 100000
+          yarn browser build
+        env:
+          NODE_OPTIONS: --max_old_space_size=4096
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9
+
+      - name: Run Theia
+        shell: bash
+        working-directory: ./theia
+        run: yarn browser start &
+
+      - name: Run Performance Measurement
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: yarn performance
+
+      - name: Get History
+        uses: actions/checkout@v2
+        if: always() && github.ref == 'refs/heads/main'
+        continue-on-error: true
+        with:
+          ref: gh-pages
+          path: gh-pages
+
+      - name: Prepare Report
+        if: always() && github.ref == 'refs/heads/main'
+        shell: bash
+        run: yarn performance-report
+
+      - name: Publish Report
+        if: always() && github.ref == 'refs/heads/main'
+        uses: peaceiris/actions-gh-pages@v3
+        env:
+          PERSONAL_TOKEN: ${{ secrets.TOKEN }}
+          PUBLISH_DIR: ./public

.gitignore

@@ -12,3 +12,6 @@
 node_modules
 allure-results
 test-results
+performance-metrics
+public
+gh-pages

configs/performance.config.ts

@@ -0,0 +1,46 @@
+// *****************************************************************************
+// Copyright (C) 2023 STMicroelectronics and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { PlaywrightTestConfig } from '@playwright/test';
+import { MetricsFetchConfig } from '../scripts/fetch-metrics';
+
+const performanceConfig: PlaywrightTestConfig = {
+    testDir: '../lib/tests',
+    testMatch: ['**/*.performance.js'],
+    globalTeardown: require.resolve('../scripts/fetch-metrics.ts'),
+    workers: 1,
+    timeout: 60 * 1000,
+    use: {
+        baseURL: 'http://localhost:3000',
+        browserName: 'chromium',
+        viewport: { width: 1920, height: 1080 }
+    },
+    metadata: {
+        // custom metadata to configure /scripts/ci-global-tear-down.ts
+        performanceMetrics: <MetricsFetchConfig>{
+            metricsEndpoint: 'http://localhost:3000/metrics',
+            outputFileNamePrefix: '',
+            outputFileNamePostfix: '.txt',
+            outputFilePath: 'performance-metrics'
+        }
+    },
+    preserveOutput: 'failures-only',
+    reporter: [
+        ['list']
+    ],
+};
+
+export default performanceConfig;

configs/playwright.config.ts

@@ -18,7 +18,7 @@ import { PlaywrightTestConfig } from '@playwright/test';
 
 const config: PlaywrightTestConfig = {
     testDir: '../lib/tests',
-    testMatch: ['**/*.js'],
+    testMatch: ['**/*.test.js'],
     workers: 2,
     // Timeout for each test in milliseconds.
     timeout: 60 * 1000,

package.json

@@ -11,7 +11,9 @@
     "ui-tests-ci": "yarn && playwright test --config=./configs/playwright.ci.config.ts",
     "ui-tests-headful": "yarn && playwright test --config=./configs/playwright.headful.config.ts",
     "ui-tests-report-generate": "allure generate ./allure-results --clean -o allure-results/allure-report",
-    "ui-tests-report": "yarn ui-tests-report-generate && allure open allure-results/allure-report"
+    "ui-tests-report": "yarn ui-tests-report-generate && allure open allure-results/allure-report",
+    "performance": "yarn && playwright test --config=./configs/performance.config.ts",
+    "performance-report": "yarn && node lib/scripts/performance-report.js"
   },
   "dependencies": {
     "@playwright/test": "^1.35.1",
@@ -21,6 +23,12 @@
     "allure-commandline": "^2.23.0",
     "allure-playwright": "^2.4.0",
     "rimraf": "^3.0.0",
+    "node-fetch": "^2.6.7",
+    "@types/node-fetch": "^2.6.4",
+    "fs-extra": "^11.1.1",
+    "@types/fs-extra": "^11.0.1",
+    "yargs": "^17.7.2",
+    "@types/yargs": "^17.0.24",
     "typescript": "~4.5.5"
   }
 }

scripts/fetch-metrics.ts

@@ -0,0 +1,69 @@
+// *****************************************************************************
+// Copyright (C) 2023 STMicroelectronics and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { type FullConfig } from '@playwright/test';
+import * as fs from 'fs-extra';
+import fetch from 'node-fetch';
+
+async function fetchMetrics(config: FullConfig) {
+    await fetchPerformanceMetrics(config.metadata.performanceMetrics, config.metadata.totalTime);
+}
+
+export default fetchMetrics;
+
+export interface MetricsFetchConfig {
+    metricsEndpoint: string;
+    outputFileNamePrefix: string;
+    outputFileNamePostfix: string;
+    outputFilePath: string;
+}
+
+export async function fetchPerformanceMetrics({
+    metricsEndpoint,
+    outputFileNamePrefix,
+    outputFileNamePostfix,
+    outputFilePath }: MetricsFetchConfig,
+    totalTime?: string
+) {
+    const now = new Date();
+    const dateString = `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}`;
+    const timeString = `${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`;
+    const timestamp = `${dateString}T${timeString}`;
+    const fileName = outputFileNamePrefix + timestamp + outputFileNamePostfix;
+    const targetFile = `${outputFilePath}/${fileName}`;
+
+    fs.ensureDirSync(outputFilePath);
+    if (fs.existsSync(targetFile)) {
+        console.warn('Output file already exists, deleting it...');
+        fs.rmSync(targetFile);
+    }
+
+    try {
+        const response = await fetch(metricsEndpoint);
+        let data = await response.text();
+        if (totalTime) {
+            data += `
+
+playwright_total_time ${totalTime}`;
+        }
+
+        fs.writeFileSync(targetFile, data);
+    } catch (error) {
+        console.error('Error while fetching metrics', error);
+        process.exit(-1);
+    }
+}
+