Electron startup performance measurement script

- refactor a reusable module out of the startup measurement script - create a new script to measure Electron start-up performance - rename the measure-performance.js script to indicate that it measures the browser app specifically - add an --app/-a option to the extension-impact script to drive the Electron app as an alternative to the Browser app. Includes a bit of refactoring of this script and the app package template - incidentally fix the extension-impact.js stepping over itself trying to run the app while building it (was missing an await) Contributed on behalf of STMicroelectronics. Co-authored-by: Stefan Dirix <[email protected]> Signed-off-by: Christian W. Damus <[email protected]>
eclipse-theia · Nov 19, 2021 · 4ff4f95 · 4ff4f95
1 parent f09fc1a
commit 4ff4f95
Show file tree

Hide file tree

Showing 11 changed files with 607 additions and 120 deletions.
diff --git a/.gitignore b/.gitignore
@@ -30,3 +30,4 @@ dev-packages/electron/compile_commands.json
 .eslintcache
 scripts/download
 license-check-summary.txt*
+*-trace.json
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,7 @@
 # Change Log
 
+- [scripts] added Electron frontend start-up performance measurement script [#10442](https://github.com/eclipse-theia/theia/pull/10442) - Contributed on behalf of STMicroelectronics
+
 ## v1.19.0 - 10/28/2021
 
 [1.19.0 Milestone](https://github.com/eclipse-theia/theia/milestone/25)

diff --git a/package.json b/package.json
@@ -79,7 +79,9 @@
     "test:theia": "lerna run --scope \"@theia/!(example-)*\" test --stream --concurrency=1",
     "watch": "concurrently --kill-others -n tsc,browser,electron -c red,yellow,blue \"tsc -b -w --preserveWatchOutput\" \"yarn --cwd examples/browser watch:bundle\" \"yarn --cwd examples/electron watch:bundle\"",
     "watch:compile": "concurrently --kill-others -n cleanup,tsc -c magenta,red \"ts-clean dev-packages/* packages/* -w\" \"tsc -b -w --preserveWatchOutput\"",
-    "performance:startup": "concurrently --success first -k -r \"cd scripts/performance && node measure-performance.js --name Startup --folder startup --runs 10\" \"yarn --cwd examples/browser start\""
+    "performance:startup": "yarn performance:startup:browser && yarn performance:startup:electron",
+    "performance:startup:browser": "concurrently --success first -k -r \"cd scripts/performance && node browser-performance.js --name 'Browser Frontend Startup' --folder browser --runs 10\" \"yarn --cwd examples/browser start\"",
+    "performance:startup:electron": "yarn electron rebuild && cd scripts/performance && node electron-performance.js --name 'Electron Frontend Startup' --folder electron --runs 10"
   },
   "workspaces": [
     "dev-packages/*",

diff --git a/scripts/performance/.gitignore b/scripts/performance/.gitignore
@@ -3,3 +3,4 @@ workspace
 *.csv
 *.json
 !base-package.json
+!electron-trace-config.json
diff --git a/scripts/performance/README.md b/scripts/performance/README.md
@@ -1,13 +1,14 @@
 # Performance measurements
 
-This directory contains a script that measures the performance of Theia.
-Currently the support is limited to measuring the `browser-app`'s startup time using the `Largest contentful paint (LCP)` value.
+This directory contains scripts that measure the start-up performance of the Theia frontend in both the browser and the Electron examples.
 
-## Running the script
+The frontend's start-up time is measured using the timestamp of the last recorded `Largest contentful paint (LCP)` candidate metric.
+
+## Running the browser start-up script
 
 ### Quick Start
 
-Execute `yarn run performance:startup` in the root directory to startup the backend and execute the script.
+Execute `yarn run performance:startup:browser` in the root directory to startup the backend and execute the script.
 
 ### Prerequisites
 
@@ -16,21 +17,49 @@ This can either be done with the `Launch Browser Backend` launch config or by ru
 
 ### Executing the script
 
-The script can be executed using `node measure-performance.js` in this directory.
+The script can be executed using `node browser-performance.js` in this directory.
 
 The script accepts the following optional parameters:
 
--   `--name`: Specify a name for the current measurement (default: `StartupPerformance`)
+-   `--name`: Specify a name for the current measurement (default: `Browser Frontend Startup`)
 -   `--url`: Point Theia to a url for example for specifying a specifc workspace (default: `http://localhost:3000/#/<pathToMeasurementScript>/workspace`)
--   `--folder`: Folder name for the generated tracing files in the `profiles` folder (default: `profile`)
+-   `--folder`: Folder name for the generated tracing files in the `profiles` folder (default: `browser`)
 -   `--runs`: Number of runs for the measurement (default: `10`)
 -   `--headless`: Boolean, if the tests should be run in headless mode (default: `true`)
 
 _**Note**: When multiple runs are specified the script will calculate the mean and the standard deviation of all values._
 
+## Running the Electron start-up script
+
+### Quick Start
+
+Execute `yarn run performance:startup:electron` in the root directory to execute the script.
+
+### Prerequisites
+
+To run the script the Theia Electron example needs to be built. In the root directory:
+
+```console
+$ yarn
+$ yarn electron build
+```
+
+### Executing the script
+
+The script can be executed using `node electron-performance.js` in this directory.
+
+The script accepts the following optional parameters:
+
+-   `--name`: Specify a name for the current measurement (default: `Electron Frontend Startup`)
+-   `--folder`: Folder name for the generated tracing files in the `profiles` folder (default: `electron`)
+-   `--workspace`: Absolute path to a Theia workspace to open (default: an empty workspace folder)
+-   `--runs`: Number of runs for the measurement (default: `10`)
+
+_**Note**: When multiple runs are specified the script will calculate the mean and the standard deviation of all values, except for any runs that failed to capture a measurement due to an exception._
+
 ## Measure impact on startup performance of extensions
 
-To measure the startup performance impact that extensions have on the application, another script is avaiable, which uses the measurements from the `measure-performance.js` script.
+To measure the startup performance impact that extensions have on the application, another script is available, which uses the measurements from the `browser-performance.js` or `electron-performance.js` script.
 The `extension-impact.js` script runs the measurement for a defined base application (`base-package.json` in this directory) and then measures the startup time when one of the defined extensions is added to the base application.
 The script will then print a table (in CSV format) to the console (and store it in a file) which contains the mean, standard deviation (Std Dev) and coefficient of variation (CV) for each extensions run.
 Additionally, each extensions entry will contain the difference to the base application time.
@@ -40,14 +69,15 @@ Example Table:
 | Extension Name    | Mean (10 runs) (in s) | Std Dev (in s) | CV (%) | Delta (in s) |
 | ----------------- | --------------------- | -------------- | ------ | ------------ |
 | Base Theia        | 2.027                 | 0.084          | 4.144  | -            |
-| @theia/git:1.17.0 | 2.103                 | 0.041          | 1.950  | 0.076        |
+| @theia/git:1.19.0 | 2.103                 | 0.041          | 1.950  | 0.076        |
 
 ### Script usage
 
 The script can be executed by running `node extension-impact.js` in this directory.
 
 The following parameters are available:
 
+-   `--app`: The example app in which to measure performance, either `browser` or `electron` (default: `browser`)
 -   `--runs`: Specify the number of measurements for each extension (default: `10`)
 -   `--base-time`: Provide an existing measurement (mean) for the base Theia application. If none is provided it will be measured.
 -   `--extensions`: Provide a list of extensions (need to be locally installed) that shall be tested (default: all extensions in packages folder)
@@ -58,10 +88,11 @@ The following parameters are available:
     -   _not contain whitespaces_
     -   _and be separated by whitespaces_
 
-    _For example: `--extensions @theia/git:1.18.0 @theia/keymaps:1.18.0`_
+    _For example: `--extensions @theia/git:1.19.0 @theia/keymaps:1.19.0`_
 
 -   `--yarn`: Flag to trigger a full yarn at script startup (e.g. to build changes to extensions)
--   `--url`: Specify a URL that Theia should be launched with (can also be used to specify the workspace to be opened) (default: `http://localhost:3000/#/<pathToMeasurementScript>/workspace`)
--   `--file`: Relative path to the output file (default: `./extensions.csv`)
+-   `--url`: Specify a URL that Theia should be launched with (can be used to specify the workspace to be opened). _Applies only to the `browser` app_ (default: `http://localhost:3000/#/<GIT_ROOT>/scripts/performance/workspace`)
+-   `--workspace`: Specify a workspace on which to launch Theia. _Applies only to the `electron` app_ (default: `/<GIT_ROOT>/scripts/performance/workspace`)
+-   `--file`: Relative path to the output file (default: `./script.csv`)
 
 _**Note**: If no extensions are provided all extensions from the `packages` folder will be measured._
diff --git a/scripts/performance/base-package.json b/scripts/performance/base-package.json
@@ -1,31 +1,37 @@
 {
   "private": true,
-  "name": "@theia/example-browser",
-  "version": "1.18.0",
+  "name": "@theia/example-{{app}}",
+  "version": "{{version}}",
   "license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0",
   "theia": {
+    "target": "{{app}}",
     "frontend": {
       "config": {
-        "applicationName": "Theia Browser Example",
+        "applicationName": "Theia {{app}} Example",
         "preferences": {
           "files.enableTrash": false
         }
       }
+    },
+    "backend": {
+      "config": {
+        "resolveSystemPlugins": false
+      }
     }
   },
   "dependencies": {
-    "@theia/core": "1.18.0",
-    "@theia/plugin-ext": "1.18.0"
+    "@theia/core": "{{version}}",
+    "@theia/plugin-ext": "{{version}}"
   },
   "scripts": {
     "clean": "theia clean",
     "build": "yarn compile && yarn bundle",
     "bundle": "theia build --mode development",
     "compile": "tsc -b",
-    "rebuild": "theia rebuild:browser --cacheRoot ../..",
+    "rebuild": "theia rebuild:{{app}} --cacheRoot ../..",
     "start": "yarn rebuild && THEIA_CONFIG_DIR=./theia-config-dir theia start --plugins=local-dir:../../noPlugins --log-level=fatal"
   },
   "devDependencies": {
-    "@theia/cli": "1.18.0"
+    "@theia/cli": "{{version}}"
   }
 }
diff --git a/scripts/performance/measure-performance.js → scripts/performance/browser-performance.js b/scripts/performance/measure-performance.js → scripts/performance/browser-performance.js
@@ -17,23 +17,47 @@
 const puppeteer = require('puppeteer');
 const fs = require('fs');
 const fsExtra = require('fs-extra');
-const resolve = require('path').resolve;
+const { resolve } = require('path');
+const { delay, lcp, isLCP, measure } = require('./common-performance');
+
 const workspacePath = resolve('./workspace');
 const profilesPath = './profiles/';
 
-const lcp = 'Largest Contentful Paint (LCP)';
-const performanceTag = braceText('Performance');
-
-let name = 'StartupPerformance';
+let name = 'Browser Frontend Startup';
 let url = 'http://localhost:3000/#' + workspacePath;
-let folder = 'profile';
+let folder = 'browser';
 let headless = true;
 let runs = 10;
 
 (async () => {
     let defaultUrl = true;
+    const yargs = require('yargs');
+    const args = yargs(process.argv.slice(2)).option('name', {
+        alias: 'n',
+        desc: 'A name for the test suite',
+        type: 'string',
+        default: name
+    }).option('folder', {
+        alias: 'f',
+        desc: 'Name of a folder within the "profiles" folder in which to collect trace logs',
+        type: 'string',
+        default: folder
+    }).option('runs', {
+        alias: 'r',
+        desc: 'The number of times to run the test',
+        type: 'number',
+        default: runs
+    }).option('url', {
+        alias: 'u',
+        desc: 'URL on which to open Theia in the browser (e.g., to specify a workspace)',
+        type: 'string',
+        default: url
+    }).option('headless', {
+        desc: 'Run in headless mode (do not open a browser)',
+        type: 'boolean',
+        default: headless
+    }).wrap(Math.min(120, yargs.terminalWidth())).argv;
 
-    const args = require('yargs/yargs')(process.argv.slice(2)).argv;
     if (args.name) {
         name = args.name.toString();
     }
@@ -47,10 +71,15 @@ let runs = 10;
     if (args.runs) {
         runs = parseInt(args.runs.toString());
     }
-    if (args.headless) {
-        if (args.headless.toString() === 'false') {
-            headless = false;
-        }
+    if (args.headless !== undefined && args.headless.toString() === 'false') {
+        headless = false;
+    }
+
+    // Verify that the application exists
+    const mainJS = resolve(__dirname, '../../examples/browser/src-gen/frontend/index.html');
+    if (!fs.existsSync(mainJS)) {
+        console.error('Browser example app does not exist. Please build it before running this script.');
+        process.exit(1);
     }
 
     if (defaultUrl) { fsExtra.ensureDirSync(workspacePath); }
@@ -67,9 +96,9 @@ let runs = 10;
 })();
 
 async function measurePerformance(name, url, folder, headless, runs) {
-    const durations = [];
-    for (let i = 0; i < runs; i++) {
-        const runNr = i + 1;
+
+    /** @type import('./common-performance').TestFunction */
+    const testScenario = async (runNr) => {
         const browser = await puppeteer.launch({ headless: headless });
         const page = await browser.newPage();
 
@@ -85,82 +114,16 @@ async function measurePerformance(name, url, folder, headless, runs) {
 
         await browser.close();
 
-        const time = await analyzeStartup(file)
-        durations.push(time);
-        logDuration(name, runNr, lcp, time.toFixed(3), runs > 1);
-    }
-
-    if (runs > 1) {
-        const mean = calculateMean(durations);
-        logDuration(name, 'MEAN', lcp, mean);
-        logDuration(name, 'STDEV', lcp, calculateStandardDeviation(mean, durations));
-    }
-}
+        return file;
+    };
 
-async function analyzeStartup(profilePath) {
-    let startEvent;
-    const tracing = JSON.parse(fs.readFileSync('./' + profilePath, 'utf8'));
-    const lcpEvents = tracing.traceEvents.filter(x => {
-        if (isStart(x)) {
-            startEvent = x;
-            return false;
-        }
-        return isLCP(x);
-    });
-
-    if (startEvent !== undefined) {
-        return duration(lcpEvents[lcpEvents.length - 1], startEvent);
-    }
-    throw new Error('Could not analyze startup');
-}
-
-function isLCP(x) {
-    return x.name === 'largestContentfulPaint::Candidate';
+    measure(name, lcp, runs, testScenario, isStart, isLCP);
 }
 
 function isStart(x) {
     return x.name === 'TracingStartedInBrowser';
 }
 
-function duration(event, startEvent) {
-    return (event.ts - startEvent.ts) / 1000000;
-}
-
-function logDuration(name, run, metric, duration, multipleRuns = true) {
-    let runText = '';
-    if (multipleRuns) {
-        runText = braceText(run);
-    }
-    console.log(performanceTag + braceText(name) + runText + ' ' + metric + ': ' + duration + ' seconds');
-}
-
-function calculateMean(array) {
-    let sum = 0;
-    array.forEach(x => {
-        sum += x;
-    });
-    return (sum / array.length).toFixed(3);
-};
-
-function calculateStandardDeviation(mean, array) {
-    let sumOfDiffsSquared = 0;
-    array.forEach(time => {
-        sumOfDiffsSquared += Math.pow((time - mean), 2)
-    });
-    const variance = sumOfDiffsSquared / array.length;
-    return Math.sqrt(variance).toFixed(3);
-}
-
-function braceText(text) {
-    return '[' + text + ']';
-}
-
-function delay(time) {
-    return new Promise(function (resolve) {
-        setTimeout(resolve, time)
-    });
-}
-
 async function waitForDeployed(url, maxTries, ms) {
     let deployed = true;
     const browser = await puppeteer.launch({ headless: true });
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,3 +3,4 @@ workspace @@
     *.csv
     *.json
     !base-package.json
+    !electron-trace-config.json