diff --git a/.gitignore b/.gitignore index 517855f6f5672..dcb4c0a780269 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ dev-packages/electron/compile_commands.json .eslintcache scripts/download license-check-summary.txt* +*-trace.json diff --git a/CHANGELOG.md b/CHANGELOG.md index a157cc2521af6..a2e445d61a32d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Change Log +- [scripts] added Electron frontend start-up performance measurement script [#10442](https://github.com/eclipse-theia/theia/pull/10442) - Contributed on behalf of STMicroelectronics + ## v1.19.0 - 10/28/2021 [1.19.0 Milestone](https://github.com/eclipse-theia/theia/milestone/25) diff --git a/package.json b/package.json index f4973f58986af..2e92ca5666456 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,9 @@ "test:theia": "lerna run --scope \"@theia/!(example-)*\" test --stream --concurrency=1", "watch": "concurrently --kill-others -n tsc,browser,electron -c red,yellow,blue \"tsc -b -w --preserveWatchOutput\" \"yarn --cwd examples/browser watch:bundle\" \"yarn --cwd examples/electron watch:bundle\"", "watch:compile": "concurrently --kill-others -n cleanup,tsc -c magenta,red \"ts-clean dev-packages/* packages/* -w\" \"tsc -b -w --preserveWatchOutput\"", - "performance:startup": "concurrently --success first -k -r \"cd scripts/performance && node measure-performance.js --name Startup --folder startup --runs 10\" \"yarn --cwd examples/browser start\"" + "performance:startup": "yarn performance:startup:browser && yarn performance:startup:electron", + "performance:startup:browser": "concurrently --success first -k -r \"cd scripts/performance && node browser-performance.js --name 'Browser Frontend Startup' --folder browser --runs 10\" \"yarn --cwd examples/browser start\"", + "performance:startup:electron": "yarn electron rebuild && cd scripts/performance && node electron-performance.js --name 'Electron Frontend Startup' --folder electron --runs 10" }, "workspaces": [ "dev-packages/*", diff --git a/scripts/performance/.gitignore b/scripts/performance/.gitignore index fe4e404f4f80b..3c00e950fefa1 100644 --- a/scripts/performance/.gitignore +++ b/scripts/performance/.gitignore @@ -3,3 +3,4 @@ workspace *.csv *.json !base-package.json +!electron-trace-config.json diff --git a/scripts/performance/README.md b/scripts/performance/README.md index 3fa692dc7aef3..f6fd47940d58c 100644 --- a/scripts/performance/README.md +++ b/scripts/performance/README.md @@ -1,13 +1,14 @@ # Performance measurements -This directory contains a script that measures the performance of Theia. -Currently the support is limited to measuring the `browser-app`'s startup time using the `Largest contentful paint (LCP)` value. +This directory contains scripts that measure the start-up performance of the Theia frontend in both the browser and the Electron examples. -## Running the script +The frontend's start-up time is measured using the timestamp of the last recorded `Largest contentful paint (LCP)` candidate metric. + +## Running the browser start-up script ### Quick Start -Execute `yarn run performance:startup` in the root directory to startup the backend and execute the script. +Execute `yarn run performance:startup:browser` in the root directory to startup the backend and execute the script. ### Prerequisites @@ -16,21 +17,49 @@ This can either be done with the `Launch Browser Backend` launch config or by ru ### Executing the script -The script can be executed using `node measure-performance.js` in this directory. +The script can be executed using `node browser-performance.js` in this directory. The script accepts the following optional parameters: -- `--name`: Specify a name for the current measurement (default: `StartupPerformance`) +- `--name`: Specify a name for the current measurement (default: `Browser Frontend Startup`) - `--url`: Point Theia to a url for example for specifying a specifc workspace (default: `http://localhost:3000/#//workspace`) -- `--folder`: Folder name for the generated tracing files in the `profiles` folder (default: `profile`) +- `--folder`: Folder name for the generated tracing files in the `profiles` folder (default: `browser`) - `--runs`: Number of runs for the measurement (default: `10`) - `--headless`: Boolean, if the tests should be run in headless mode (default: `true`) _**Note**: When multiple runs are specified the script will calculate the mean and the standard deviation of all values._ +## Running the Electron start-up script + +### Quick Start + +Execute `yarn run performance:startup:electron` in the root directory to execute the script. + +### Prerequisites + +To run the script the Theia Electron example needs to be built. In the root directory: + +```console +$ yarn +$ yarn electron build +``` + +### Executing the script + +The script can be executed using `node electron-performance.js` in this directory. + +The script accepts the following optional parameters: + +- `--name`: Specify a name for the current measurement (default: `Electron Frontend Startup`) +- `--folder`: Folder name for the generated tracing files in the `profiles` folder (default: `electron`) +- `--workspace`: Absolute path to a Theia workspace to open (default: an empty workspace folder) +- `--runs`: Number of runs for the measurement (default: `10`) + +_**Note**: When multiple runs are specified the script will calculate the mean and the standard deviation of all values, except for any runs that failed to capture a measurement due to an exception._ + ## Measure impact on startup performance of extensions -To measure the startup performance impact that extensions have on the application, another script is avaiable, which uses the measurements from the `measure-performance.js` script. +To measure the startup performance impact that extensions have on the application, another script is available, which uses the measurements from the `browser-performance.js` or `electron-performance.js` script. The `extension-impact.js` script runs the measurement for a defined base application (`base-package.json` in this directory) and then measures the startup time when one of the defined extensions is added to the base application. The script will then print a table (in CSV format) to the console (and store it in a file) which contains the mean, standard deviation (Std Dev) and coefficient of variation (CV) for each extensions run. Additionally, each extensions entry will contain the difference to the base application time. @@ -40,7 +69,7 @@ Example Table: | Extension Name | Mean (10 runs) (in s) | Std Dev (in s) | CV (%) | Delta (in s) | | ----------------- | --------------------- | -------------- | ------ | ------------ | | Base Theia | 2.027 | 0.084 | 4.144 | - | -| @theia/git:1.17.0 | 2.103 | 0.041 | 1.950 | 0.076 | +| @theia/git:1.19.0 | 2.103 | 0.041 | 1.950 | 0.076 | ### Script usage @@ -48,6 +77,7 @@ The script can be executed by running `node extension-impact.js` in this directo The following parameters are available: +- `--app`: The example app in which to measure performance, either `browser` or `electron` (default: `browser`) - `--runs`: Specify the number of measurements for each extension (default: `10`) - `--base-time`: Provide an existing measurement (mean) for the base Theia application. If none is provided it will be measured. - `--extensions`: Provide a list of extensions (need to be locally installed) that shall be tested (default: all extensions in packages folder) @@ -58,10 +88,11 @@ The following parameters are available: - _not contain whitespaces_ - _and be separated by whitespaces_ - _For example: `--extensions @theia/git:1.18.0 @theia/keymaps:1.18.0`_ + _For example: `--extensions @theia/git:1.19.0 @theia/keymaps:1.19.0`_ - `--yarn`: Flag to trigger a full yarn at script startup (e.g. to build changes to extensions) -- `--url`: Specify a URL that Theia should be launched with (can also be used to specify the workspace to be opened) (default: `http://localhost:3000/#//workspace`) -- `--file`: Relative path to the output file (default: `./extensions.csv`) +- `--url`: Specify a URL that Theia should be launched with (can be used to specify the workspace to be opened). _Applies only to the `browser` app_ (default: `http://localhost:3000/#//scripts/performance/workspace`) +- `--workspace`: Specify a workspace on which to launch Theia. _Applies only to the `electron` app_ (default: `//scripts/performance/workspace`) +- `--file`: Relative path to the output file (default: `./script.csv`) _**Note**: If no extensions are provided all extensions from the `packages` folder will be measured._ diff --git a/scripts/performance/base-package.json b/scripts/performance/base-package.json index 8f98f575230cc..ee133ef164ff5 100644 --- a/scripts/performance/base-package.json +++ b/scripts/performance/base-package.json @@ -1,31 +1,37 @@ { "private": true, - "name": "@theia/example-browser", - "version": "1.18.0", + "name": "@theia/example-{{app}}", + "version": "{{version}}", "license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0", "theia": { + "target": "{{app}}", "frontend": { "config": { - "applicationName": "Theia Browser Example", + "applicationName": "Theia {{app}} Example", "preferences": { "files.enableTrash": false } } + }, + "backend": { + "config": { + "resolveSystemPlugins": false + } } }, "dependencies": { - "@theia/core": "1.18.0", - "@theia/plugin-ext": "1.18.0" + "@theia/core": "{{version}}", + "@theia/plugin-ext": "{{version}}" }, "scripts": { "clean": "theia clean", "build": "yarn compile && yarn bundle", "bundle": "theia build --mode development", "compile": "tsc -b", - "rebuild": "theia rebuild:browser --cacheRoot ../..", + "rebuild": "theia rebuild:{{app}} --cacheRoot ../..", "start": "yarn rebuild && THEIA_CONFIG_DIR=./theia-config-dir theia start --plugins=local-dir:../../noPlugins --log-level=fatal" }, "devDependencies": { - "@theia/cli": "1.18.0" + "@theia/cli": "{{version}}" } } diff --git a/scripts/performance/measure-performance.js b/scripts/performance/browser-performance.js similarity index 57% rename from scripts/performance/measure-performance.js rename to scripts/performance/browser-performance.js index 07254b47f3536..20ea6b381ed7f 100644 --- a/scripts/performance/measure-performance.js +++ b/scripts/performance/browser-performance.js @@ -17,23 +17,47 @@ const puppeteer = require('puppeteer'); const fs = require('fs'); const fsExtra = require('fs-extra'); -const resolve = require('path').resolve; +const { resolve } = require('path'); +const { delay, lcp, isLCP, measure } = require('./common-performance'); + const workspacePath = resolve('./workspace'); const profilesPath = './profiles/'; -const lcp = 'Largest Contentful Paint (LCP)'; -const performanceTag = braceText('Performance'); - -let name = 'StartupPerformance'; +let name = 'Browser Frontend Startup'; let url = 'http://localhost:3000/#' + workspacePath; -let folder = 'profile'; +let folder = 'browser'; let headless = true; let runs = 10; (async () => { let defaultUrl = true; + const yargs = require('yargs'); + const args = yargs(process.argv.slice(2)).option('name', { + alias: 'n', + desc: 'A name for the test suite', + type: 'string', + default: name + }).option('folder', { + alias: 'f', + desc: 'Name of a folder within the "profiles" folder in which to collect trace logs', + type: 'string', + default: folder + }).option('runs', { + alias: 'r', + desc: 'The number of times to run the test', + type: 'number', + default: runs + }).option('url', { + alias: 'u', + desc: 'URL on which to open Theia in the browser (e.g., to specify a workspace)', + type: 'string', + default: url + }).option('headless', { + desc: 'Run in headless mode (do not open a browser)', + type: 'boolean', + default: headless + }).wrap(Math.min(120, yargs.terminalWidth())).argv; - const args = require('yargs/yargs')(process.argv.slice(2)).argv; if (args.name) { name = args.name.toString(); } @@ -47,10 +71,15 @@ let runs = 10; if (args.runs) { runs = parseInt(args.runs.toString()); } - if (args.headless) { - if (args.headless.toString() === 'false') { - headless = false; - } + if (args.headless !== undefined && args.headless.toString() === 'false') { + headless = false; + } + + // Verify that the application exists + const mainJS = resolve(__dirname, '../../examples/browser/src-gen/frontend/index.html'); + if (!fs.existsSync(mainJS)) { + console.error('Browser example app does not exist. Please build it before running this script.'); + process.exit(1); } if (defaultUrl) { fsExtra.ensureDirSync(workspacePath); } @@ -67,9 +96,9 @@ let runs = 10; })(); async function measurePerformance(name, url, folder, headless, runs) { - const durations = []; - for (let i = 0; i < runs; i++) { - const runNr = i + 1; + + /** @type import('./common-performance').TestFunction */ + const testScenario = async (runNr) => { const browser = await puppeteer.launch({ headless: headless }); const page = await browser.newPage(); @@ -85,82 +114,16 @@ async function measurePerformance(name, url, folder, headless, runs) { await browser.close(); - const time = await analyzeStartup(file) - durations.push(time); - logDuration(name, runNr, lcp, time.toFixed(3), runs > 1); - } - - if (runs > 1) { - const mean = calculateMean(durations); - logDuration(name, 'MEAN', lcp, mean); - logDuration(name, 'STDEV', lcp, calculateStandardDeviation(mean, durations)); - } -} + return file; + }; -async function analyzeStartup(profilePath) { - let startEvent; - const tracing = JSON.parse(fs.readFileSync('./' + profilePath, 'utf8')); - const lcpEvents = tracing.traceEvents.filter(x => { - if (isStart(x)) { - startEvent = x; - return false; - } - return isLCP(x); - }); - - if (startEvent !== undefined) { - return duration(lcpEvents[lcpEvents.length - 1], startEvent); - } - throw new Error('Could not analyze startup'); -} - -function isLCP(x) { - return x.name === 'largestContentfulPaint::Candidate'; + measure(name, lcp, runs, testScenario, isStart, isLCP); } function isStart(x) { return x.name === 'TracingStartedInBrowser'; } -function duration(event, startEvent) { - return (event.ts - startEvent.ts) / 1000000; -} - -function logDuration(name, run, metric, duration, multipleRuns = true) { - let runText = ''; - if (multipleRuns) { - runText = braceText(run); - } - console.log(performanceTag + braceText(name) + runText + ' ' + metric + ': ' + duration + ' seconds'); -} - -function calculateMean(array) { - let sum = 0; - array.forEach(x => { - sum += x; - }); - return (sum / array.length).toFixed(3); -}; - -function calculateStandardDeviation(mean, array) { - let sumOfDiffsSquared = 0; - array.forEach(time => { - sumOfDiffsSquared += Math.pow((time - mean), 2) - }); - const variance = sumOfDiffsSquared / array.length; - return Math.sqrt(variance).toFixed(3); -} - -function braceText(text) { - return '[' + text + ']'; -} - -function delay(time) { - return new Promise(function (resolve) { - setTimeout(resolve, time) - }); -} - async function waitForDeployed(url, maxTries, ms) { let deployed = true; const browser = await puppeteer.launch({ headless: true }); diff --git a/scripts/performance/common-performance.js b/scripts/performance/common-performance.js new file mode 100644 index 0000000000000..570fd24aad169 --- /dev/null +++ b/scripts/performance/common-performance.js @@ -0,0 +1,235 @@ +/******************************************************************************** + * Copyright (C) 2021 STMicroelectronics and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ +// @ts-check + +/** + * An event in the performance trace (from the Chrome performance API). + * @typedef TraceEvent + * @property {string} name the event name + * @property {number} ts the timestamp, in microseconds since some time after host system start + */ + +/** + * A call-back that selects an event from the performance trace. + * + * @callback EventPredicate + * @param {TraceEvent} event an event to test + * @returns {boolean} whether the predicate selects the `event` + */ + +/** + * A call-back that runs the test scenario to be analyzed. + * + * @async + * @callback TestFunction + * @param {number} runNr the current run index of the multiple runs being executed + * @returns {PromiseLike} the path to the recorded performance profiling trace file + */ + +const fs = require('fs'); + +const performanceTag = braceText('Performance'); +const lcp = 'Largest Contentful Paint (LCP)'; + +/** + * Measure the performance of a `test` function implementing some `scenario` of interest. + * + * @param {string} name the application name to measure + * @param {string} scenario a label for the scenario being measured + * @param {number} runs the number of times to run the `test` scenario + * @param {TestFunction} test a function that executes the `scenario` to be measured, returning the file + * that records the performance profile trace + * @param {EventPredicate} isStartEvent a predicate matching the trace event that marks the start of the measured scenario + * @param {EventPredicate} isEndEvent a predicate matching the trace event that marks the end of the measured scenario + */ +async function measure(name, scenario, runs, test, isStartEvent, isEndEvent) { + const durations = []; + for (let i = 0; i < runs; i++) { + const runNr = i + 1; + + const file = await test(runNr); + let time; + + try { + time = await analyzeTrace(file, isStartEvent, isEndEvent); + + durations.push(time); + logDuration(name, runNr, scenario, time, runs > 1); + } catch (e) { + logException(name, runNr, scenario, e, runs > 1); + } + } + + logSummary(name, scenario, durations); +} + + +/** + * Log a summary of the given measured `durations`. + * + * @param {string} name the performance script name + * @param {string} scenario the scenario that was measured + * @param {number[]} durations the measurements captured for the `scenario` + */ +function logSummary(name, scenario, durations) { + if (durations.length > 1) { + const mean = calculateMean(durations); + const stdev = calculateStandardDeviation(mean, durations); + logDuration(name, 'MEAN', scenario, mean); + logDuration(name, 'STDEV', scenario, stdev); + } +} + +/** + * Analyze a performance trace file. + * + * @param {string} profilePath the profiling trace file path + * @param {EventPredicate} isStartEvent a predicate matching the trace event that marks the start of the measured scenario + * @param {EventPredicate} isEndEvent a predicate matching the trace event that marks the end of the measured scenario + */ +async function analyzeTrace(profilePath, isStartEvent, isEndEvent) { + let startEvent; + const tracing = JSON.parse(fs.readFileSync(profilePath, 'utf8')); + const endEvents = tracing.traceEvents.filter(e => { + if (startEvent === undefined && isStartEvent(e)) { + startEvent = e; + return false; + } + return isEndEvent(e); + }); + + if (startEvent !== undefined && endEvents.length > 0) { + return duration(endEvents[endEvents.length - 1], startEvent); + } + + throw new Error('Could not analyze performance trace'); +} + +/** + * Query whether a trace `event` is a candidate for the Largest Contentful Paint. + * + * @param {TraceEvent} event an event in the performance trace + * @returns whether the `event` is an LCP candidate + */ +function isLCP(event) { + return event.name === 'largestContentfulPaint::Candidate'; +} + +/** + * Compute the duration, in seconds, to an `event` from a start event. + * + * @param {TraceEvent} event the duration end event + * @param {TraceEvent} startEvent the duration start event + * @returns the duration, in seconds + */ +function duration(event, startEvent) { + return (event.ts - startEvent.ts) / 1_000_000; +} + +/** + * Log a `duration` measured for some scenario. + * + * @param {string} name the performance script name + * @param {number|string} run the run index number, or some kind of aggregate like 'Total' or 'Avg' + * @param {string} metric the scenario that was measured + * @param {number} duration the duration, in seconds, of the measured scenario + * @param {boolean=} multipleRuns whether the `run` logged is one of many being logged + */ +function logDuration(name, run, metric, duration, multipleRuns = true) { + let runText = ''; + if (multipleRuns) { + runText = braceText(run); + } + console.log(performanceTag + braceText(name) + runText + ' ' + metric + ': ' + duration.toFixed(3) + ' seconds'); +} + +/** + * Log an `exception` in measurement of some scenario. + * + * @param {string} name the performance script name + * @param {number|string} run the run index number, or some kind of aggregate like 'Total' or 'Avg' + * @param {string} metric the scenario that was measured + * @param {Error} exception the duration, in seconds, of the measured scenario + * @param {boolean=} multipleRuns whether the `run` logged is one of many being logged + */ +function logException(name, run, metric, exception, multipleRuns = true) { + let runText = ''; + if (multipleRuns) { + runText = braceText(run); + } + console.log(performanceTag + braceText(name) + runText + ' ' + metric + ' failed to obtain a measurement: ' + exception.message); + console.error(`Failed to obtain a measurement. The most likely cause is that the performance trace file was incomplete because the script did not wait long enough for "${metric}".`); + console.error(exception); +} + +/** + * Compute the arithmetic mean of an `array` of numbers. + * + * @param {number[]} array an array of numbers to average + * @returns the average of the `array` + */ +function calculateMean(array) { + let sum = 0; + array.forEach(x => { + sum += x; + }); + return (sum / array.length); +}; + +/** + * Compute the standard deviation from the mean of an `array` of numbers. + * + * @param {number[]} array an array of numbers + * @returns the standard deviation of the `array` from its mean + */ +function calculateStandardDeviation(mean, array) { + let sumOfDiffsSquared = 0; + array.forEach(time => { + sumOfDiffsSquared += Math.pow((time - mean), 2) + }); + const variance = sumOfDiffsSquared / array.length; + return Math.sqrt(variance); +} + +/** + * Surround a string of `text` in square braces. + * + * @param {string|number} text a string of text or a number that can be rendered as text + * @returns the `text` in braces + */ +function braceText(text) { + return '[' + text + ']'; +} + +/** + * Obtain a promise that resolves after some delay. + * + * @param {number} time a delay, in milliseconds + * @returns a promise that will resolve after the given number of milliseconds + */ +function delay(time) { + return new Promise(function (resolve) { + setTimeout(resolve, time) + }); +} + +module.exports = { + measure, analyzeTrace, + calculateMean, calculateStandardDeviation, + duration, logDuration, logSummary, + braceText, delay, + lcp, isLCP +}; diff --git a/scripts/performance/electron-performance.js b/scripts/performance/electron-performance.js new file mode 100644 index 0000000000000..4ba3a4e9f3fb3 --- /dev/null +++ b/scripts/performance/electron-performance.js @@ -0,0 +1,169 @@ +/******************************************************************************** + * Copyright (C) 2021 STMicroelectronics and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ +// @ts-check +const fs = require('fs'); +const fsx = require('fs-extra'); +const { resolve } = require('path'); +const { spawn, ChildProcess } = require('child_process'); +const { delay, lcp, isLCP, measure } = require('./common-performance'); +const traceConfigTemplate = require('./electron-trace-config.json'); +const { exit } = require('process'); + +const basePath = resolve(__dirname, '../..'); +const profilesPath = resolve(__dirname, './profiles/'); +const electronExample = resolve(basePath, 'examples/electron'); +const theia = resolve(electronExample, 'node_modules/.bin/theia'); + +let name = 'Electron Frontend Startup'; +let folder = 'electron'; +let runs = 10; +let workspace = resolve('./workspace'); + +(async () => { + let defaultWorkspace = true; + + const yargs = require('yargs'); + const args = yargs(process.argv.slice(2)).option('name', { + alias: 'n', + desc: 'A name for the test suite', + type: 'string', + default: name + }).option('folder', { + alias: 'f', + desc: 'Name of a folder within the "profiles" folder in which to collect trace logs', + type: 'string', + default: folder + }).option('runs', { + alias: 'r', + desc: 'The number of times to run the test', + type: 'number', + default: runs + }).option('workspace', { + alias: 'w', + desc: 'Path to a Theia workspace to open', + type: 'string', + default: workspace + }).wrap(Math.min(120, yargs.terminalWidth())).argv; + + if (args.name) { + name = args.name.toString(); + } + if (args.folder) { + folder = args.folder.toString(); + } + if (args.workspace) { + workspace = args.workspace.toString(); + if (resolve(workspace) !== workspace) { + console.log('Workspace path must be an absolute path:', workspace); + exit(1); + } + defaultWorkspace = false; + } + if (args.runs) { + runs = parseInt(args.runs.toString()); + } + + // Verify that the application exists + const mainJS = resolve(electronExample, 'src-gen/frontend/index.html'); + if (!fs.existsSync(mainJS)) { + console.error('Electron example app does not exist. Please build it before running this script.'); + process.exit(1); + } + + if (defaultWorkspace) { + // Ensure that it exists + fsx.ensureDirSync(workspace); + } + + await measurePerformance(); +})(); + +async function measurePerformance() { + fsx.emptyDirSync(resolve(profilesPath, folder)); + const traceConfigPath = resolve(profilesPath, folder, 'trace-config.json'); + + /** + * Generate trace config from the template. + * @param {number} runNr + * @returns {string} the output trace file path + */ + const traceConfigGenerator = (runNr) => { + const traceConfig = { ...traceConfigTemplate }; + const traceFilePath = resolve(profilesPath, folder, `${runNr}.json`); + traceConfig.result_file = traceFilePath + fs.writeFileSync(traceConfigPath, JSON.stringify(traceConfig, undefined, 2), 'utf-8'); + return traceFilePath; + }; + + const exitHandler = (andExit = false) => { + return () => { + if (electron && !electron.killed) { + process.kill(-electron.pid, 'SIGINT'); + } + if (andExit) { + process.exit(); + } + } + }; + + // Be sure not to leave a detached Electron child process + process.on('exit', exitHandler()); + process.on('SIGINT', exitHandler(true)); + process.on('SIGTERM', exitHandler(true)); + + let electron; + + /** @type import('./common-performance').TestFunction */ + const testScenario = async (runNr) => { + const traceFile = traceConfigGenerator(runNr); + electron = await launchElectron(traceConfigPath); + + // Uncomment this to see why the child process terminates early when driven by extension-impact.js + // electron.stderr.on('data', data => console.error(data.toString())); + + // Wait long enough to be sure that tracing has finished. Kill the process group + // because the 'theia' child process was detached + await delay(traceConfigTemplate.startup_duration * 1_000 * 3 / 2) + .then(() => electron.exitCode !== null || process.kill(-electron.pid, 'SIGINT')); + electron = undefined; + return traceFile; + }; + + measure(name, lcp, runs, testScenario, hasNonzeroTimestamp, isLCP); +} + +/** + * Launch the Electron app as a detached child process with tracing configured to start + * immediately upon launch. The child process is detached because otherwise the attempt + * to signal it to terminate when the test run is complete will not terminate the entire + * process tree but only the root `theia` process, leaving the electron app instance + * running until eventually this script itself exits. + * + * @param {string} traceConfigPath the path to the tracing configuration file with which to initiate tracing + * @returns {Promise} the Electron child process, if successfully launched + */ +async function launchElectron(traceConfigPath) { + const args = ['start', workspace, '--plugins=local-dir:../../plugins', `--trace-config-file=${traceConfigPath}`]; + if (process.platform === 'linux') { + args.push('--headless'); + } + return spawn(theia, args, { cwd: electronExample, detached: true }); +} + +function hasNonzeroTimestamp(traceEvent) { + return traceEvent.hasOwnProperty('ts') // The traces don't have explicit nulls or undefineds + && traceEvent.ts > 0; +} diff --git a/scripts/performance/electron-trace-config.json b/scripts/performance/electron-trace-config.json new file mode 100644 index 0000000000000..75731ca41afd9 --- /dev/null +++ b/scripts/performance/electron-trace-config.json @@ -0,0 +1,16 @@ +{ + "startup_duration": 10, + "result_file": "{{placeholder}}.json", + "trace_config": { + "enable_argument_filter": false, + "enable_systrace": false, + "included_categories": [ + "blink", + "loading", + "disabled-by-default-devtools.timeline" + ], + "excluded_categories": [ + "*" + ] + } +} diff --git a/scripts/performance/extension-impact.js b/scripts/performance/extension-impact.js index 0b5b8f702a1ef..f7eb24b7d1dc0 100644 --- a/scripts/performance/extension-impact.js +++ b/scripts/performance/extension-impact.js @@ -22,13 +22,16 @@ const mkdirp = require('mkdirp'); const path = require('path'); const env = Object.assign({}, process.env); env.PATH = path.resolve("../../node_modules/.bin") + path.delimiter + env.PATH; -const basePackage = require(`./base-package.json`); +let basePackage; +const { exit } = require('process'); let runs = 10; let baseTime; let extensions = []; let yarn = false; let url; +let workspace; let file = path.resolve('./script.csv'); +let hostApp = 'browser'; async function sigintHandler() { process.exit(); @@ -42,7 +45,9 @@ async function exitHandler() { (async () => { process.on('SIGINT', sigintHandler); process.on('exit', exitHandler); - const args = require('yargs/yargs')(process.argv.slice(2)) + + const yargs = require('yargs'); + const args = yargs(process.argv.slice(2)) .option('base-time', { alias: 'b', desc: 'Pass an existing mean of the base application', @@ -51,7 +56,8 @@ async function exitHandler() { .option('runs', { alias: 'r', desc: 'The number of runs to measure', - type: 'number' + type: 'number', + default: 10 }) .option('extensions', { alias: 'e', @@ -64,16 +70,28 @@ async function exitHandler() { .option('yarn', { alias: 'y', desc: 'Build all typescript sources on script start', - type: 'boolean' + type: 'boolean', + default: false }).option('url', { alias: 'u', - desc: 'Specify a custom URL at which to launch Theia (e.g. with a specific workspace)', + desc: 'Specify a custom URL at which to launch Theia in the browser (e.g. with a specific workspace)', + type: 'string' + }).option('workspace', { + alias: 'w', + desc: 'Specify an absolute path to a workspace on which to launch Theia in Electron', type: 'string' }).option('file', { alias: 'f', - desc: 'Specify the relative path to a CSV file which stores the result (default: ./extensions.csv)', - type: 'string' - }).argv; + desc: 'Specify the relative path to a CSV file which stores the result', + type: 'string', + default: file + }).option('app', { + alias: 'a', + desc: 'Specify in which application to run the tests', + type: 'string', + choices: ['browser', 'electron'], + default: 'browser' + }).wrap(Math.min(120, yargs.terminalWidth())).argv; if (args.baseTime) { baseTime = parseFloat(args.baseTime.toString()).toFixed(3); } @@ -93,6 +111,9 @@ async function exitHandler() { if (args.url) { url = args.url; } + if (args.workspace) { + workspace = args.workspace; + } if (args.file) { file = path.resolve(args.file); if (!file.endsWith('.csv')) { @@ -100,6 +121,11 @@ async function exitHandler() { return; } } + if (args.app) { + hostApp = args.app; + } + + preparePackageTemplate(); prepareWorkspace(); if (yarn) { execSync('yarn build', { cwd: '../../', stdio: 'pipe' }); @@ -110,7 +136,7 @@ async function exitHandler() { async function extensionImpact(extensions) { logToFile(`Extension Name, Mean (${runs} runs) (in s), Std Dev (in s), CV (%), Delta (in s)`); if (baseTime === undefined) { - calculateExtension(undefined); + await calculateExtension(undefined); } else { log(`Base Theia (provided), ${baseTime}, -, -, -`); } @@ -124,8 +150,21 @@ async function extensionImpact(extensions) { } } +function preparePackageTemplate() { + const core = require('../../packages/core/package.json'); + const version = core.version; + const content = readFileSync(path.resolve(__dirname, './base-package.json'), 'utf-8') + .replace(/\{\{app\}\}/g, hostApp) + .replace(/\{\{version\}\}/g, version); + basePackage = JSON.parse(content); + if (hostApp === 'electron') { + basePackage.dependencies['@theia/electron'] = version; + } + return basePackage; +} + function prepareWorkspace() { - copyFileSync('../../examples/browser/package.json', './backup-package.json'); + copyFileSync(`../../examples/${hostApp}/package.json`, './backup-package.json'); mkdirp('../../noPlugins', function (err) { if (err) { console.error(err); @@ -141,7 +180,7 @@ function prepareWorkspace() { } function cleanWorkspace() { - copyFileSync('./backup-package.json', '../../examples/browser/package.json'); + copyFileSync('./backup-package.json', `../../examples/${hostApp}/package.json`); unlinkSync('./backup-package.json'); rmdirSync('../../noPlugins'); rmdirSync('./theia-config-dir'); @@ -170,19 +209,41 @@ async function calculateExtension(extensionQualifier) { } else { extensionQualifier = "Base Theia"; } - logToConsole(`Building the browser example with ${extensionQualifier}.`); - writeFileSync(`../../examples/browser/package.json`, JSON.stringify(basePackageCopy, null, 2)); + logToConsole(`Building the ${hostApp} example with ${extensionQualifier}.`); + writeFileSync(`../../examples/${hostApp}/package.json`, JSON.stringify(basePackageCopy, null, 2)); try { - execSync('yarn browser build', { cwd: '../../', stdio: 'pipe' }); + execSync(`yarn ${hostApp} build`, { cwd: '../../', stdio: 'pipe' }); + + // Rebuild native modules if necessary + execSync(`yarn ${hostApp} rebuild`, { cwd: '../../', stdio: 'pipe' }); } catch (error) { log(`${extensionQualifier}, Error while building the package.json, -, -, -`); return; } logToConsole(`Measuring the startup time with ${extensionQualifier} ${runs} times. This may take a while.`); - const output = await execCommand( - `concurrently --success first -k -r "cd scripts/performance && node measure-performance.js --name Startup --folder script --runs ${runs}${url ? ' --url ' + url : ''}" ` - + `"yarn --cwd examples/browser start | grep -v '.*'"`, { env: env, cwd: '../../', shell: true }); + const appCommand = (app) => { + let command; + let cwd; + switch (app) { + case 'browser': + command = `concurrently --success first -k -r "cd scripts/performance && node browser-performance.js --name Browser --folder browser --runs ${runs}${url ? ' --url ' + url : ''}" ` + + `"yarn --cwd examples/browser start | grep -v '.*'"` + cwd = path.resolve(__dirname, '../../'); + break; + case 'electron': + command = `node electron-performance.js --name Electron --folder electron --runs ${runs}${workspace ? ' --workspace "' + workspace + '"' : ''}` + cwd = __dirname; + break; + default: + console.log('Unknown host app:', hostApp); + exit(1); + break; // Unreachable + } + return [command, cwd]; + }; + const [command, cwd] = appCommand(hostApp); + const output = await execCommand(command, { env: env, cwd: cwd, shell: true }); const mean = parseFloat(getMeasurement(output, '[MEAN] Largest Contentful Paint (LCP):')); const stdev = parseFloat(getMeasurement(output, '[STDEV] Largest Contentful Paint (LCP):'));