cli, stdlib: move bench to cli (#240)

Aligns the test runner and bench runner.
Making benchmarking more intuitive to use

packages/cli/index.d.ts

@@ -126,6 +126,48 @@ interface TestAssertion {
   message?: string;
 }
 
+/**
+ * Run the registered benchmarks
+ */
+export function mainBenchFn(meta: ImportMeta): void;
+
+/**
+ * Top level bench mark registering function
+ */
+export function bench(name: string, callback: BenchCallback): void;
+
+/**
+ * Argument to bench mark functions
+ */
+export interface BenchRunner {
+  /**
+   * Amount of iterations this call should do
+   */
+  N: number;
+
+  /**
+   * Reset the start time
+   * This can be used if a benchmark needs some setup
+   */
+  resetTime(): void;
+}
+
+/**
+ * Callback function executed while benchmarking
+ */
+export type BenchCallback = (BenchRunner) => void | Promise<void>;
+
+/**
+ * @private
+ */
+interface BenchState {
+  name: string;
+  N: number;
+  operationTimeNs: string;
+  callback: BenchCallback;
+  caughtException?: Error;
+}
+
 /**
  * Represents either a file in the `scripts` directory or a script from the package.json
  * Depending on the type contains either script or path

packages/cli/index.js

@@ -1,2 +1,3 @@
 export { collectScripts } from "./src/utils.js";
 export { test, mainTestFn } from "./src/testing/index.js";
+export { bench, mainBenchFn } from "./src/benchmarking/index.js";

packages/cli/scripts/bench.js

@@ -1,31 +1,26 @@
 import {
   filenameForModule,
-  logBenchResults,
   mainFn,
   processDirectoryRecursive,
 } from "@lbu/stdlib";
+import { mainBenchFn } from "../index.js";
 
 const __filename = filenameForModule(import.meta);
 
-const contentHandler = async (logger, file) => {
+const contentHandler = async (file) => {
   // Skip this index file
   if (file === __filename) {
     return;
   }
   if (!file.endsWith(".bench.js")) {
     return;
   }
-  const imported = await import(file);
-  if (imported && imported.runBench) {
-    await imported.runBench(logger);
-  }
+  await import(file);
 };
 
 mainFn(import.meta, main);
 
-async function main(logger) {
-  await processDirectoryRecursive(process.cwd(), (file) =>
-    contentHandler(logger, file),
-  );
-  logBenchResults(logger);
+async function main() {
+  await processDirectoryRecursive(process.cwd(), contentHandler);
+  mainBenchFn(import.meta);
 }

packages/cli/src/benchmarking/index.js

@@ -0,0 +1,2 @@
+export { bench } from "./runner.js";
+export { mainBenchFn } from "./utils.js";

packages/cli/src/benchmarking/printer.js

@@ -0,0 +1,117 @@
+import { AppError, isNil } from "@lbu/stdlib";
+import { benchLogger, state } from "./state.js";
+
+export function printBenchResults() {
+  const total = state.length;
+  let success = 0;
+
+  for (const bench of state) {
+    if (!bench.caughtException) {
+      success += 1;
+    }
+  }
+
+  benchLogger.info(state);
+
+  const result = [];
+
+  result.push("");
+  result.push(`Total benchmarks: ${total}`);
+  result.push(`          Passed: ${success}`);
+  result.push(`          Failed: ${total - success}`);
+  result.push(`-----------`);
+
+  printSuccessResults(
+    result,
+    state.filter((it) => isNil(it.caughtException)),
+  );
+  printErrorResults(
+    result,
+    state.filter((it) => !isNil(it.caughtException)),
+  );
+
+  let exitCode = 0;
+  let logFn = benchLogger.info;
+
+  if (total !== success) {
+    exitCode = 1;
+    logFn = benchLogger.error;
+  }
+
+  logFn(result.join("\n"));
+
+  return exitCode;
+}
+
+/**
+ * @param {string[]} result
+ * @param {BenchState[]} state
+ */
+function printSuccessResults(result, state) {
+  if (state.length === 0) {
+    return;
+  }
+
+  let longestName = 0;
+  let longestOperationTimeBeforeDot = 0;
+
+  for (const bench of state) {
+    if (bench.name.length > longestName) {
+      longestName = bench.name.length;
+    }
+
+    // We also line out on the '.'
+    // This results in easier to interpret results
+    const operationTimeSplit = bench.operationTimeNs.split(".");
+    bench.operationTimeBeforeDot = operationTimeSplit[0];
+
+    if (bench.operationTimeBeforeDot.length > longestOperationTimeBeforeDot) {
+      longestOperationTimeBeforeDot = bench.operationTimeBeforeDot.length;
+    }
+  }
+
+  for (const bench of state) {
+    result.push(
+      `${bench.name.padEnd(longestName, " ")}   ${String(bench.N).padStart(
+        10,
+        " ",
+      )}  iterations   ${bench.operationTimeBeforeDot.padStart(
+        longestOperationTimeBeforeDot,
+        " ",
+      )}  ns/op`,
+    );
+  }
+}
+
+/**
+ * @param {string[]} result
+ * @param {BenchState[]} state
+ */
+function printErrorResults(result, state) {
+  if (state.length === 0) {
+    return;
+  }
+
+  for (const bench of state) {
+    result.push(bench.name);
+
+    const indent = "  ";
+    const stack = bench.caughtException.stack
+      .split("\n")
+      .map((it, idx) => indent + (idx !== 0 ? "  " : "") + it.trim());
+
+    if (AppError.instanceOf(bench.caughtException)) {
+      result.push(
+        `${indent}AppError: ${bench.caughtException.key} - ${bench.caughtException.status}`,
+      );
+    } else {
+      result.push(
+        `${indent}${bench.caughtException.name} - ${bench.caughtException.message}`,
+      );
+    }
+
+    for (const item of stack) {
+      result.push(item);
+    }
+  }
+}

packages/cli/src/benchmarking/runner.js

@@ -0,0 +1,117 @@
+/**
+ * @param {BenchState[]} state
+ * @returns {Promise<void>}
+ */
+import { isNil } from "@lbu/stdlib";
+import { state } from "./state.js";
+
+export async function runBenchmarks(state) {
+  let i = 0;
+
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    if (isNil(state[i])) {
+      // Give a chance for async imports to run
+      await new Promise((r) => {
+        setTimeout(r, 2);
+      });
+
+      if (isNil(state[i])) {
+        break;
+      }
+    }
+
+    try {
+      const b = new InternalRunner(state[i]);
+      await b.exec();
+    } catch (e) {
+      state[i].caughtException = e;
+    }
+    i++;
+  }
+}
+
+/**
+ * @param {string} name
+ * @param {BenchCallback} callback
+ */
+export function bench(name, callback) {
+  state.push({ name, callback });
+}
+
+class InternalRunner {
+  static iterationBase = [1, 2, 5];
+
+  /**
+   * All iterations we can try to execute
+   */
+  static iterations = Array.from(
+    { length: InternalRunner.iterationBase.length * 9 },
+    (_, idx) => {
+      const base =
+        InternalRunner.iterationBase[idx % InternalRunner.iterationBase.length];
+      const times = Math.max(
+        1,
+        Math.pow(10, Math.floor(idx / InternalRunner.iterationBase.length)),
+      );
+
+      return base * times;
+    },
+  );
+
+  N = 0;
+  start = BigInt(0);
+
+  constructor(state) {
+    /**
+     * @type {BenchState}
+     */
+    this.state = state;
+  }
+
+  async exec() {
+    let i = 0;
+    while (i < InternalRunner.iterations.length) {
+      this.start = process.hrtime.bigint();
+      this.N = InternalRunner.iterations[i];
+
+      const res = this.state.callback(createBenchRunner(this));
+      if (res && typeof res.then === "function") {
+        await res;
+      }
+
+      const diff = process.hrtime.bigint() - this.start;
+      if (diff >= 1_000_000_000 || i === InternalRunner.iterations.length - 1) {
+        this.state.N = this.N;
+        this.state.operationTimeNs = (Number(diff) / this.N).toFixed(0);
+        break;
+      }
+
+      if (diff < 50_00_000) {
+        i = Math.min(i + 5, InternalRunner.iterations.length - 1);
+      } else if (diff < 100_000_000) {
+        i = Math.min(i + 4, InternalRunner.iterations.length - 1);
+      } else if (diff < 200_000_000) {
+        i = Math.min(i + 3, InternalRunner.iterations.length - 1);
+      } else if (diff < 300_000_000) {
+        i = Math.min(i + 2, InternalRunner.iterations.length - 1);
+      } else {
+        i++;
+      }
+    }
+  }
+}
+
+/**
+ *
+ * @param {InternalRunner} runner
+ * @returns {BenchRunner}
+ */
+function createBenchRunner(runner) {
+  return {
+    N: runner.N,
+    resetTime: () => {
+      runner.start = process.hrtime.bigint();
+    },
+  };
+}

packages/cli/src/benchmarking/state.js

@@ -0,0 +1,30 @@
+/**
+ * @type {Logger}
+ */
+export let benchLogger = undefined;
+
+/**
+ * @type {boolean}
+ */
+export let areBenchRunning = false;
+
+/**
+ * @type {BenchState[]}
+ */
+export const state = [];
+
+/**
+ * Mutate the global areBenchRunning
+ * @param {boolean} running
+ */
+export function setAreBenchRunning(running) {
+  areBenchRunning = running;
+}
+
+/**
+ * Set the bench logger
+ * @param {Logger} logger
+ */
+export function setBenchLogger(logger) {
+  benchLogger = logger;
+}

packages/cli/src/benchmarking/utils.js

@@ -0,0 +1,36 @@
+import { mainFn } from "@lbu/stdlib";
+import { printBenchResults } from "./printer.js";
+import { runBenchmarks } from "./runner.js";
+import {
+  areBenchRunning,
+  setAreBenchRunning,
+  setBenchLogger,
+  state,
+} from "./state.js";
+
+/**
+ * Wraps `mainFn` and `areBenchRunning
+ * @param {ImportMeta} meta
+ */
+export function mainBenchFn(meta) {
+  if (areBenchRunning) {
+    return;
+  }
+
+  mainFn(meta, async (logger) => {
+    setBenchLogger(logger);
+    setAreBenchRunning(true);
+
+    // Used when `mainBenchFn` is called the first thing of the process,
+    // which results in no benchmarks registered yet
+    await new Promise((r) => {
+      setTimeout(r, 2);
+    });
+
+    await runBenchmarks(state);
+
+    const exitCode = printBenchResults();
+
+    process.exit(exitCode);
+  });
+}