update readme, add memory argument back

packages/utilities/fast-benchmarks/README.md

@@ -71,12 +71,16 @@ class XApp extends FASTElement {
 }
 ```
 
+#### Option to customize utils
+
 There is a utility folder called `./utils`, where you have access to the `data` variable, this is an array of RandomItem objects.
 
 For more details on what the data encompasses, reference below in [utils.ts](#utils.ts).
 
 By default, there are 10k RandomItems generated, you can change this value in the [utils.ts](#utils.ts) as well.
 
+If you are ok with these default values, you can ignore this section.
+
 #### utils.ts
 
 ```ts
@@ -103,6 +107,7 @@ To run a benchmark, supply all the options listed below in [Arguments](#argument
 | `--versions/-v`        | `--versions 1.9.0 master local` `--versions=1.9.0`           | `master` and `local` are special keywords that can be used. Or supply versions of the library, check available [versions](#library-versions). Multiple options have to be **delimited by spaces**, to include the `local` version, check details in [Running local version](#running-local-version). | Yes      |
 | `--localBenchFile/-lb` | `--localBenchFile=index2`                                    | Name of the local benchmark file **don't include the extension** This option is only turned on if you've supplied 'local' as one of the versions AND you want to add different implementation for the same benchmark test, check details in [Running local version](#running-local-version).         | No       |
 | `--operations/-o`      | `--operations=create10k` `--operations create10k update10th` | Defaults to run all possible operations if this argument is not supplied. Name of operations to run benchmarks against, **don't include the extension**. Delimited by spaces                                                                                                                         | No       |
+| `--memory/-m`          | `-memory`                                                    | Display only memory performance results                                                                                                                                                                                                                                                              | No       |
 
 > Note: Running all possible operations will take an extremely long time. During local development, it is recommend to run one operation at a time to get faster results.
 
@@ -116,6 +121,8 @@ _examples_:
 
 `yarn run benchmark --library=fast-foundation --benchmark=form-associated -v 2.34.0 2.42.1`
 
+`yarn run benchmark --library=fast-foundation --benchmark=form-associated -v 2.34.0 2.42.1 --operations=createDelete5x -m`
+
 #### Library versions
 
 -   [fast-element](https://www.npmjs.com/package/@microsoft/fast-element)
@@ -144,6 +151,8 @@ If you want to test your local implementation against master or existing version
 ...
 ```
 
+> There is an [open issue](https://github.com/Polymer/tachometer/issues/215) around local dependencies and when they are resolved. For now, you would have to symlink locally.
+
 2. [Create local implementation file](#create-local-implementation-file)
 
 3. Run the benchmark

packages/utilities/fast-benchmarks/benchmarks/fast-foundation/form-associated/index.ts

@@ -6,7 +6,7 @@ import {
     observable,
     repeat,
 } from "@microsoft/fast-element";
-import { FormAssociated, FoundationElement } from "@microsoft/fast-foundation";
+import { FormAssociated } from "@microsoft/fast-foundation";
 import { data, RandomItem } from "../../../utils/index.js";
 
 /* eslint-disable @typescript-eslint/naming-convention */

packages/utilities/fast-benchmarks/scripts/index.js

@@ -8,7 +8,7 @@ const errMessage = chalk.hex("#ffb638");
 program
     .option("-l, --library <name>", "run benchmarks in <name> library")
     .option("-b, --benchmark <name>", "run the benchmark: <name>")
-    // .option("-m, --memory", "check memory metrics")
+    .option("-m, --memory", "only display memory consumption results")
     .option(
         "-v, --versions [versions...]",
         "specify available versions, you can also use 'local' or 'master' that would point to github branches"
@@ -74,10 +74,10 @@ async function buildBenchmark(configPath) {
  * @returns {Promise}
  */
 async function runBenchmark(configPaths) {
-    // can be run synchronously
-    configPaths.forEach(configPath => {
-        // for (let configPath of configPaths) {
-        return new Promise((resolve, reject) => {
+    const promises = [];
+    for (let i = 0; i < configPaths.length; i++) {
+        const configPath = configPaths[i];
+        const res = new Promise((resolve, reject) => {
             const args = ["tach", "--config", configPath];
             const child = spawn("npx", args, { stdio: "inherit" });
             child.on("close", code => {
@@ -100,7 +100,9 @@ async function runBenchmark(configPaths) {
                 return error;
             }
         });
-    });
+        promises.push(res);
+    }
+    return await Promise.all(promises);
 }
 
 const run = async () => {

packages/utilities/fast-benchmarks/scripts/template.js

@@ -170,7 +170,7 @@ const libraryDependencies = {
     },
 };
 async function generateBenchmarks(
-    { library, benchmark, versions },
+    { library, benchmark, versions, memory },
     operationProps,
     localProps
 ) {
@@ -257,11 +257,11 @@ async function generateBenchmarks(
             ];
             memoryBenchmarks.push(memoryBench);
 
-            benchmarks.push(bench);
+            if (!memory) benchmarks.push(bench);
         });
 
         tachoData[`${operation}-memory`] = memoryBenchmarks;
-        tachoData[operation] = benchmarks;
+        if (!memory) tachoData[operation] = benchmarks;
     });
 
     return tachoData;
@@ -280,9 +280,10 @@ async function generateConfig(fileName, benchmarksHash) {
 
         const defaultBenchOptions = {
             // Tachometer default is 50, but locally let's only do 10
-            sampleSize: 20,
+            sampleSize: 30,
             // Tachometer default is 3 minutes, but let's shrink it to 1 here to save some
-            timeout: 0,
+            timeout: 1,
+            autoSampleConditions: ["0%", "10%"],
         };
 
         const pathsPromises = [];