diff --git a/packages/utilities/fast-benchmarks/README.md b/packages/utilities/fast-benchmarks/README.md index a16cc3f5b04..b8094e92135 100644 --- a/packages/utilities/fast-benchmarks/README.md +++ b/packages/utilities/fast-benchmarks/README.md @@ -71,12 +71,16 @@ class XApp extends FASTElement { } ``` +#### Option to customize utils + There is a utility folder called `./utils`, where you have access to the `data` variable, this is an array of RandomItem objects. For more details on what the data encompasses, reference below in [utils.ts](#utils.ts). By default, there are 10k RandomItems generated, you can change this value in the [utils.ts](#utils.ts) as well. +If you are ok with these default values, you can ignore this section. + #### utils.ts ```ts @@ -103,6 +107,7 @@ To run a benchmark, supply all the options listed below in [Arguments](#argument | `--versions/-v` | `--versions 1.9.0 master local` `--versions=1.9.0` | `master` and `local` are special keywords that can be used. Or supply versions of the library, check available [versions](#library-versions). Multiple options have to be **delimited by spaces**, to include the `local` version, check details in [Running local version](#running-local-version). | Yes | | `--localBenchFile/-lb` | `--localBenchFile=index2` | Name of the local benchmark file **don't include the extension** This option is only turned on if you've supplied 'local' as one of the versions AND you want to add different implementation for the same benchmark test, check details in [Running local version](#running-local-version). | No | | `--operations/-o` | `--operations=create10k` `--operations create10k update10th` | Defaults to run all possible operations if this argument is not supplied. Name of operations to run benchmarks against, **don't include the extension**. Delimited by spaces | No | +| `--memory/-m` | `-memory` | Display only memory performance results | No | > Note: Running all possible operations will take an extremely long time. During local development, it is recommend to run one operation at a time to get faster results. @@ -116,6 +121,8 @@ _examples_: `yarn run benchmark --library=fast-foundation --benchmark=form-associated -v 2.34.0 2.42.1` +`yarn run benchmark --library=fast-foundation --benchmark=form-associated -v 2.34.0 2.42.1 --operations=createDelete5x -m` + #### Library versions - [fast-element](https://www.npmjs.com/package/@microsoft/fast-element) @@ -144,6 +151,8 @@ If you want to test your local implementation against master or existing version ... ``` +> There is an [open issue](https://github.com/Polymer/tachometer/issues/215) around local dependencies and when they are resolved. For now, you would have to symlink locally. + 2. [Create local implementation file](#create-local-implementation-file) 3. Run the benchmark diff --git a/packages/utilities/fast-benchmarks/benchmarks/fast-foundation/form-associated/index.ts b/packages/utilities/fast-benchmarks/benchmarks/fast-foundation/form-associated/index.ts index aeb49412097..c62fb940261 100644 --- a/packages/utilities/fast-benchmarks/benchmarks/fast-foundation/form-associated/index.ts +++ b/packages/utilities/fast-benchmarks/benchmarks/fast-foundation/form-associated/index.ts @@ -6,7 +6,7 @@ import { observable, repeat, } from "@microsoft/fast-element"; -import { FormAssociated, FoundationElement } from "@microsoft/fast-foundation"; +import { FormAssociated } from "@microsoft/fast-foundation"; import { data, RandomItem } from "../../../utils/index.js"; /* eslint-disable @typescript-eslint/naming-convention */ diff --git a/packages/utilities/fast-benchmarks/scripts/index.js b/packages/utilities/fast-benchmarks/scripts/index.js index e3d88f5ae09..286a48a672b 100644 --- a/packages/utilities/fast-benchmarks/scripts/index.js +++ b/packages/utilities/fast-benchmarks/scripts/index.js @@ -8,7 +8,7 @@ const errMessage = chalk.hex("#ffb638"); program .option("-l, --library ", "run benchmarks in library") .option("-b, --benchmark ", "run the benchmark: ") - // .option("-m, --memory", "check memory metrics") + .option("-m, --memory", "only display memory consumption results") .option( "-v, --versions [versions...]", "specify available versions, you can also use 'local' or 'master' that would point to github branches" @@ -74,10 +74,10 @@ async function buildBenchmark(configPath) { * @returns {Promise} */ async function runBenchmark(configPaths) { - // can be run synchronously - configPaths.forEach(configPath => { - // for (let configPath of configPaths) { - return new Promise((resolve, reject) => { + const promises = []; + for (let i = 0; i < configPaths.length; i++) { + const configPath = configPaths[i]; + const res = new Promise((resolve, reject) => { const args = ["tach", "--config", configPath]; const child = spawn("npx", args, { stdio: "inherit" }); child.on("close", code => { @@ -100,7 +100,9 @@ async function runBenchmark(configPaths) { return error; } }); - }); + promises.push(res); + } + return await Promise.all(promises); } const run = async () => { diff --git a/packages/utilities/fast-benchmarks/scripts/template.js b/packages/utilities/fast-benchmarks/scripts/template.js index 76026693ed0..9f8d6c4301a 100644 --- a/packages/utilities/fast-benchmarks/scripts/template.js +++ b/packages/utilities/fast-benchmarks/scripts/template.js @@ -170,7 +170,7 @@ const libraryDependencies = { }, }; async function generateBenchmarks( - { library, benchmark, versions }, + { library, benchmark, versions, memory }, operationProps, localProps ) { @@ -257,11 +257,11 @@ async function generateBenchmarks( ]; memoryBenchmarks.push(memoryBench); - benchmarks.push(bench); + if (!memory) benchmarks.push(bench); }); tachoData[`${operation}-memory`] = memoryBenchmarks; - tachoData[operation] = benchmarks; + if (!memory) tachoData[operation] = benchmarks; }); return tachoData; @@ -280,9 +280,10 @@ async function generateConfig(fileName, benchmarksHash) { const defaultBenchOptions = { // Tachometer default is 50, but locally let's only do 10 - sampleSize: 20, + sampleSize: 30, // Tachometer default is 3 minutes, but let's shrink it to 1 here to save some - timeout: 0, + timeout: 1, + autoSampleConditions: ["0%", "10%"], }; const pathsPromises = [];