dev: support filtering tests

streetsidesoftware · Apr 21, 2024 · 371b03b · 371b03b
1 parent 66f3375
commit 371b03b
Show file tree

Hide file tree

Showing 7 changed files with 108 additions and 44 deletions.
diff --git a/src/app.mts b/src/app.mts
@@ -3,39 +3,50 @@ import { fileURLToPath } from 'node:url';
 
 import chalk from 'chalk';
 import { Argument, Command, program as defaultCommand } from 'commander';
-import { globby } from 'globby';
+
+import { findFiles } from './findFiles.mjs';
 
 interface AppOptions {
     repeat?: number;
     timeout?: number;
     all?: boolean;
+    suite?: string[];
+    test?: string[];
 }
 
 const urlRunnerCli = new URL('./runBenchmarkCli.mjs', import.meta.url).toString();
 const pathToRunnerCliModule = fileURLToPath(urlRunnerCli);
 
-console.log('args: %o', process.argv);
-
 export async function app(program = defaultCommand): Promise<Command> {
-    const argument = new Argument('[suite...]', 'list of test suites to run');
+    const argument = new Argument('[filter...]', 'perf file filter.');
     argument.variadic = true;
 
     program
         .name('perf runner')
         .addArgument(argument)
         .description('Run performance tests.')
         .option('-a, --all', 'run all tests', false)
+        .option('-t, --timeout <timeout>', 'override the timeout for each test', (v) => Number(v))
+        .option('-s, --suite <suite...>', 'run matching suites', (v, a: string[] | undefined) => (a || []).concat(v))
+        .option('-T, --test <test...>', 'run matching test found in suites', (v, a: string[] | undefined) =>
+            (a || []).concat(v),
+        )
         .option('--repeat <count>', 'repeat the tests', (v) => Number(v), 1)
-        .option('-t, --timeout <timeout>', 'timeout for each test', (v) => Number(v), 1000)
-        .action(async (suiteNamesToRun: string[], options: AppOptions) => {
-            const found = await globby(['**/*.perf.{js,mjs,cjs}', '!**/node_modules/**']);
+        .action(async (suiteNamesToRun: string[], options: AppOptions, command: Command) => {
+            if (!suiteNamesToRun.length && !options.all) {
+                console.error(chalk.red('No tests to run.'));
+                console.error(chalk.yellow('Use --all to run all tests.\n'));
+                command.help();
+            }
+
+            // console.log('%o', options);
+
+            const found = await findFiles(['**/*.perf.{js,mjs,cjs}', '!**/node_modules/**']);
 
             const files = found.filter(
                 (file) => !suiteNamesToRun.length || suiteNamesToRun.some((name) => file.includes(name)),
             );
 
-            console.log('%o', { files, found });
-
             await spawnRunners(files, options);
 
             console.log(chalk.green('done.'));
@@ -58,6 +69,14 @@ async function spawnRunners(files: string[], options: AppOptions): Promise<void>
         cliOptions.push('--timeout', options.timeout.toString());
     }
 
+    if (options.suite?.length) {
+        cliOptions.push(...options.suite.flatMap((s) => ['--suite', s]));
+    }
+
+    if (options.test?.length) {
+        cliOptions.push(...options.test.flatMap((t) => ['--test', t]));
+    }
+
     for (const file of files) {
         try {
             const code = await spawnRunner([file, ...cliOptions]);

diff --git a/src/findFiles.mts b/src/findFiles.mts
@@ -7,11 +7,11 @@ export interface FindFileOptions {
     cwd?: string;
 }
 
-export async function findFiles(globs: string[], options: FindFileOptions) {
+export async function findFiles(globs: string[], options?: FindFileOptions) {
     const globOptions: GlobbyOptions = {
         ignore: excludes,
-        onlyFiles: options.onlyFiles ?? false,
-        cwd: options.cwd || process.cwd(),
+        onlyFiles: options?.onlyFiles ?? true,
+        cwd: options?.cwd || process.cwd(),
     };
     const files = await globby(
         globs.map((a) => a.trim()).filter((a) => !!a),

diff --git a/src/perf-suites/measureSearchLarge.perf.mts b/src/perf-suites/measureSearchLarge.perf.mts
@@ -1,8 +1,8 @@
 import { loremIpsum } from 'lorem-ipsum';
 
-import { suite } from '../perfSuite.mjs';
 import { SimpleTrie } from '../lib/SimpleTrie.mjs';
 import { Trie } from '../lib/Trie.mjs';
+import { suite } from '../perfSuite.mjs';
 
 const numWords = 10000;
 

diff --git a/src/perf-suites/trie.perf.mts b/src/perf-suites/trie.perf.mts
@@ -76,7 +76,7 @@ suite('trie-insert', 'Insert words into a trie', async (test, { beforeAll }) =>
         assert(trie.has('hello'));
         return trie;
     });
-});
+}).setTimeout(2000);
 
 const numberOfSearchWords = 1000;
 
@@ -119,7 +119,7 @@ suite('trie-search', 'Search for words in a trie', async (_test, { prepare }) =>
     }).test('TrieBuilder.build(true)', (trie) => {
         return searchWords.map((word) => trie.has(word));
     });
-});
+}).setTimeout(1000);
 
 async function loadWords() {
     if (words) return words;

diff --git a/src/perfSuite.mts b/src/perfSuite.mts
@@ -127,16 +127,25 @@ export function getActiveSuites(): PerfSuite[] {
     return Array.from(activeSuites);
 }
 
+export interface PerfSuiteRunTestsOptions {
+    /**
+     * Filter for the tests to run.
+     * Only run tests that contain the filter string.
+     * Empty array will run all tests.
+     */
+    tests?: string[] | undefined;
+}
+
 export interface PerfSuite {
     readonly name: string;
     readonly description?: string | undefined;
-    readonly runTests: () => Promise<RunnerResult>;
+    readonly runTests: (options: PerfSuiteRunTestsOptions) => Promise<RunnerResult>;
     /**
      * Sets the default timeout for all tests in the suite.
      * @param timeout - time in milliseconds.
      * @returns PerfSuite
      */
-    readonly setTimeout: (timeout: number) => this;
+    readonly setTimeout: (timeout: number | undefined) => this;
 }
 
 export function suite(name: string, suiteFn: SuiteFn): PerfSuite;
@@ -155,42 +164,42 @@ export function runSuite(name: string, description: string | undefined, suiteFn:
 export function runSuite(name: string, suiteFn: SuiteFn): Promise<RunnerResult>;
 export function runSuite(suiteOrName: PerfSuite | string, p2?: string | SuiteFn, p3?: SuiteFn): Promise<RunnerResult> {
     if (typeof suiteOrName === 'object') {
-        return suiteOrName.runTests();
+        return suiteOrName.runTests({});
     }
 
     if (typeof p2 === 'function') {
-        return suite(suiteOrName, p2).runTests();
+        return suite(suiteOrName, p2).runTests({});
     }
 
     assert(typeof p3 === 'function', 'suiteFn must be a function');
-    return suite(suiteOrName, p2, p3).runTests();
+    return suite(suiteOrName, p2, p3).runTests({});
 }
 
 function toError(e: unknown): Error {
     return e instanceof Error ? e : new Error(String(e));
 }
 
 function formatResult(result: TestResult, nameWidth: number): string {
-    const { name, duration, iterations, sd } = result;
+    const { name, duration, iterations } = result;
 
-    const min = sd.min;
-    const max = sd.max;
-    const p95 = sd.ok ? sd.p95 : NaN;
-    const mean = sd.ok ? sd.mean : NaN;
+    // const min = sd.min;
+    // const max = sd.max;
+    // const p95 = sd.ok ? sd.p95 : NaN;
+    // const mean = sd.ok ? sd.mean : NaN;
     const ops = (iterations * 1000) / duration;
 
     if (result.error) {
         return `${name.padEnd(nameWidth)}: ${duration.toFixed(2)}ms\n` + `Error: ${result.error}`;
     }
 
     const msg =
-        `${name.padEnd(nameWidth)}: ` +
-        `ops: ${ops.toFixed(2).padStart(8)} ` +
-        `cnt: ${iterations.toFixed(0).padStart(6)} ` +
-        `mean: ${mean.toPrecision(5).padStart(8)} ` +
-        `p95: ${p95.toPrecision(5).padStart(8)} ` +
-        `min/max: ${min.toPrecision(5).padStart(8)}/${max.toPrecision(5).padStart(8)} ` +
-        `${duration.toFixed(2).padStart(7)}ms `;
+        `${name.padEnd(nameWidth)}` +
+        ` ${ops.toFixed(2).padStart(8)} ops/sec` +
+        ` ${iterations.toFixed(0).padStart(6)} iterations` +
+        // ` mean: ${mean.toPrecision(5).padStart(8)} ` +
+        // ` p95: ${p95.toPrecision(5).padStart(8)} ` +
+        // ` min/max: ${min.toPrecision(5).padStart(8)}/${max.toPrecision(5).padStart(8)} ` +
+        ` ${duration.toFixed(2).padStart(7)}ms time`;
 
     return msg;
 }
@@ -210,17 +219,21 @@ class PerfSuiteImpl implements PerfSuite {
         registerSuite(this);
     }
 
-    runTests(): Promise<RunnerResult> {
-        return runTests(this);
+    runTests(options?: PerfSuiteRunTestsOptions): Promise<RunnerResult> {
+        return runTests(this, options);
     }
 
-    setTimeout(timeout: number): this {
-        this.timeout = timeout;
+    setTimeout(timeout: number | undefined): this {
+        this.timeout = timeout ?? this.timeout;
         return this;
     }
 }
 
-async function runTests(suite: PerfSuiteImpl, progress?: ProgressReporting): Promise<RunnerResult> {
+async function runTests(
+    suite: PerfSuiteImpl,
+    options: PerfSuiteRunTestsOptions | undefined,
+    progress?: ProgressReporting,
+): Promise<RunnerResult> {
     const stdout = progress?.stdout || process.stdout;
     const spinner = progress?.spinner || ora({ stream: stdout, discardStdin: false, hideCursor: false });
     const log = (msg: string) => stdout.write(msg + '\n');
@@ -279,8 +292,20 @@ async function runTests(suite: PerfSuiteImpl, progress?: ProgressReporting): Pro
     const beforeAllFns: UserFn[] = [];
     const afterAllFns: UserFn[] = [];
 
+    function filterTest(name: string): boolean {
+        if (!options?.tests) {
+            return true;
+        }
+
+        return options.tests.some((test) => name.toLowerCase().includes(test.toLowerCase()));
+    }
+
+    function addTest(test: TestDefinition) {
+        filterTest(test.name) && tests.push(test);
+    }
+
     function test(name: string, method: () => void, timeout?: number): void {
-        tests.push({ name, prepare: () => method, timeout });
+        addTest({ name, prepare: () => method, timeout });
     }
 
     function prepare<T>(prepareFn: () => T | Promise<T>): Prepared<Awaited<T>> {
@@ -297,7 +322,7 @@ async function runTests(suite: PerfSuiteImpl, progress?: ProgressReporting): Pro
 
                     return () => method(data);
                 };
-                tests.push({
+                addTest({
                     name,
                     prepare: fn,
                     timeout,

diff --git a/src/run.mts b/src/run.mts
@@ -7,6 +7,8 @@ import { getActiveSuites } from './perfSuite.mjs';
 export interface RunOptions {
     repeat?: number | undefined;
     timeout?: number | undefined;
+    suites?: string[] | undefined;
+    tests?: string[] | undefined;
 }
 
 /**
@@ -50,15 +52,16 @@ async function runTestSuites(
     suitesToRun: (string | PerfSuite)[],
     options: RunOptions,
 ): Promise<number> {
-    const timeout = options.timeout || 1000;
+    const timeout = options.timeout || undefined;
     const suitesRun = new Set<PerfSuite>();
 
     async function _runSuite(suites: PerfSuite[]) {
         for (const suite of suites) {
             if (suitesRun.has(suite)) continue;
+            if (!filterSuite(suite)) continue;
             suitesRun.add(suite);
             console.log(chalk.green(`Running Perf Suite: ${suite.name}`));
-            await suite.setTimeout(timeout).runTests();
+            await suite.setTimeout(timeout).runTests({ tests: options.tests });
         }
     }
 
@@ -84,4 +87,10 @@ async function runTestSuites(
     }
 
     return suitesRun.size;
+
+    function filterSuite(suite: PerfSuite): boolean {
+        const { suites } = options;
+        if (!suites?.length) return true;
+        return !!suites.find((name) => suite.name.toLowerCase().includes(name.toLowerCase()));
+    }
 }
diff --git a/src/runBenchmarkCli.mts b/src/runBenchmarkCli.mts
@@ -11,20 +11,31 @@ import { runBenchmarkSuites } from './run.mjs';
 const cwdUrl = pathToFileURL(process.cwd() + '/');
 
 async function run(args: string[]) {
-    const parseConfig: ParseArgsConfig = {
+    interface ParsedOptions {
+        repeat?: string;
+        timeout?: string;
+        test?: string[];
+        suite?: string[];
+    }
+
+    const parseConfig = {
         args,
         strict: true,
         allowPositionals: true,
         options: {
             repeat: { type: 'string', short: 'r' },
             timeout: { type: 'string', short: 't' },
+            test: { type: 'string', short: 'T', multiple: true },
+            suite: { type: 'string', short: 'S', multiple: true },
         },
-    };
+    } as const satisfies ParseArgsConfig;
 
     const parsed = parseArgs(parseConfig);
 
     const repeat = Number(parsed.values['repeat'] || '0') || undefined;
     const timeout = Number(parsed.values['timeout'] || '0') || undefined;
+    const tests = parsed.values['test'];
+    const suites = parsed.values['suite'];
 
     const errors: Error[] = [];
 
@@ -47,7 +58,7 @@ async function run(args: string[]) {
         return;
     }
 
-    await runBenchmarkSuites(undefined, { repeat, timeout });
+    await runBenchmarkSuites(undefined, { repeat, timeout, tests, suites });
 }
 
 run(process.argv.slice(2));