feat: latency and throughput advanced statistics (#119)

Signed-off-by: Jérôme Benoit <[email protected]>
tinylibs · Oct 18, 2024 · 70bc3ab · 70bc3ab
1 parent 1297a9b
commit 70bc3ab
Show file tree

Hide file tree

Showing 9 changed files with 1,357 additions and 176 deletions.
diff --git a/.size-limit.json b/.size-limit.json
@@ -1,10 +1,10 @@
 [
   {
     "path": "dist/index.js",
-    "limit": "10 kB"
+    "limit": "12 kB"
   },
   {
     "path": "dist/index.cjs",
-    "limit": "10 kB"
+    "limit": "12 kB"
   }
 ]
diff --git a/README.md b/README.md
@@ -6,8 +6,7 @@ _I'm transitioning to a full-time open source career. Your support would be grea
 [![CI](https://github.com/tinylibs/tinybench/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/tinylibs/tinybench/actions/workflows/test.yml)
 [![NPM version](https://img.shields.io/npm/v/tinybench.svg?style=flat)](https://www.npmjs.com/package/tinybench)
 
-Benchmark your code easily with Tinybench, a simple, tiny and light-weight `7KB` (`2KB` minified and gzipped)
-benchmarking library!
+Benchmark your code easily with Tinybench, a simple, tiny and light-weight `10KB` (`2KB` minified and gzipped) benchmarking library!
 You can run your benchmarks in multiple JavaScript runtimes, Tinybench is
 completely based on the Web APIs with proper timing using `process.hrtime` or
 `performance.now`.
@@ -23,7 +22,7 @@ _In case you need more tiny libraries like tinypool or tinyspy, please consider
 
 ## Installing
 
-```bash
+```shell
 $ npm install -D tinybench
 ```
 
@@ -52,12 +51,12 @@ await bench.run();
 console.table(bench.table());
 
 // Output:
-// ┌─────────┬───────────────┬──────────┬──────────────────────┬──────────┬──────────────────────────────────┬─────────┐
-// │ (index) │ Task name     │ ops/sec  │ Average time/op (ns) │ Margin   │ Median time/op (ns)              │ Samples │
-// ├─────────┼───────────────┼──────────┼──────────────────────┼──────────┼──────────────────────────────────┼─────────┤
-// │ 0       │ 'faster task' │ '38,832' │ 25751.297631307978   │ '±3.48%' │ '22016.49999997812±5.5000000145' │ 3884    │
-// │ 1       │ 'slower task' │ '669'    │ 1493338.567164177    │ '±5.98%' │ '1445076.0000000286'             │ 67      │
-// └─────────┴───────────────┴──────────┴──────────────────────┴──────────┴──────────────────────────────────┴─────────┘
+// ┌─────────┬───────────────┬────────────────────────────┬───────────────────────────┬──────────────────────┬─────────────────────┬─────────┐
+// │ (index) │ Task name     │ Throughput average (ops/s) │ Throughput median (ops/s) │ Latency average (ns) │ Latency median (ns) │ Samples │
+// ├─────────┼───────────────┼────────────────────────────┼───────────────────────────┼──────────────────────┼─────────────────────┼─────────┤
+// │ 0       │ 'faster task' │ '102906 ± 0.89%'           │ '82217 ± 14'              │ '11909.14 ± 3.95%'   │ '12163.00 ± 2.00'   │ 8398    │
+// │ 1       │ 'slower task' │ '988 ± 26.26%'             │ '710'                     │ '1379560.47 ± 6.72%' │ '1408552.00'        │ 73      │
+// └─────────┴───────────────┴────────────────────────────┴───────────────────────────┴──────────────────────┴─────────────────────┴─────────┘
 ```
 
 The `add` method accepts a task name and a task function, so it can benchmark
@@ -164,6 +163,8 @@ function has been executed.
 - `setResult(result: Partial<TaskResult>)`: change the result object values
 - `reset()`: reset the task to make the `Task.runs` a zero-value and remove the `Task.result` object
 
+FnOptions:
+
 ```ts
 export interface FnOptions {
   /**
@@ -188,118 +189,136 @@ export interface FnOptions {
 }
 ```
 
-## `TaskResult`
+### `TaskResult`
 
-the benchmark task result object.
+The benchmark task result object:
 
 ```ts
 export interface TaskResult {
   /*
-   * the last error that was thrown while running the task
+   * the last task error that was thrown
    */
-  error?: unknown;
+  error?: Error;
 
   /**
-   * The amount of time in milliseconds to run the benchmark task (cycle).
+   * the time to run the task benchmark cycle (ms)
    */
   totalTime: number;
 
   /**
-   * the minimum value in the samples
+   * how long each operation takes (ms)
    */
-  min: number;
+  period: number;
+
   /**
-   * the maximum value in the samples
+   * the task latency statistics
    */
-  max: number;
+  latency: Statistics;
+
+  /**
+   * the task throughput statistics
+   */
+  throughput: Statistics;
 
   /**
    * the number of operations per second
+   * @deprecated use `.throughput.mean` instead
    */
   hz: number;
 
   /**
-   * how long each operation takes (ms)
+   * latency samples (ms)
+   * @deprecated use `.latency.samples` instead
    */
-  period: number;
+  samples: number[];
 
   /**
-   * task samples of each task iteration time (ms)
+   * the minimum latency samples value
+   * @deprecated use `.latency.min` instead
    */
-  samples: number[];
+  min: number;
+  /**
+   * the maximum latency samples value
+   * @deprecated use `.latency.max` instead
+   */
+  max: number;
 
   /**
-   * samples mean/average (estimate of the population mean)
+   * the latency samples mean/average (estimate of the population mean/average)
+   * @deprecated use `.latency.mean` instead
    */
   mean: number;
 
   /**
-   * samples variance (estimate of the population variance)
+   * the latency samples variance (estimate of the population variance)
+   * @deprecated use `.latency.variance` instead
    */
   variance: number;
 
   /**
-   * samples standard deviation (estimate of the population standard deviation)
+   * the latency samples standard deviation (estimate of the population standard deviation)
+   * @deprecated use `.latency.sd` instead
    */
   sd: number;
 
   /**
-   * standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean)
+   * the latency standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean/average)
+   * @deprecated use `.latency.sem` instead
    */
   sem: number;
 
   /**
-   * degrees of freedom
+   * the latency samples degrees of freedom
+   * @deprecated use `.latency.df` instead
    */
   df: number;
 
   /**
-   * critical value of the samples
+   * the latency samples critical value
+   * @deprecated use `.latency.critical` instead
    */
   critical: number;
 
   /**
-   * margin of error
+   * the latency samples margin of error
+   * @deprecated use `.latency.moe` instead
    */
   moe: number;
 
   /**
-   * relative margin of error
+   * the latency samples relative margin of error
+   * @deprecated use `.latency.rme` instead
    */
   rme: number;
 
   /**
-   * median absolute deviation
-   */
-  mad: number;
-
-  /**
-   * p50/median percentile
-   */
-  p50: number;
-
-  /**
-   * p75 percentile
+   * the latency samples p75 percentile
+   * @deprecated use `.latency.p75` instead
    */
   p75: number;
 
   /**
-   * p99 percentile
+   * the latency samples p99 percentile
+   * @deprecated use `.latency.p99` instead
    */
   p99: number;
 
   /**
-   * p995 percentile
+   * the latency samples p995 percentile
+   * @deprecated use `.latency.p995` instead
    */
   p995: number;
 
   /**
-   * p999 percentile
+   * the latency samples p999 percentile
+   * @deprecated use `.latency.p999` instead
    */
   p999: number;
 }
 ```
 
+[Statistics](https://github.com/tinylibs/tinybench/blob/main/src/types.ts#L30) type definition.
+
 ### `Events`
 
 Both the `Task` and `Bench` objects extend the `EventTarget` object, so you can attach listeners to different types of events

diff --git a/examples/src/simple.ts b/examples/src/simple.ts
@@ -17,9 +17,9 @@ await bench.run();
 console.table(bench.table());
 
 // Output:
-// ┌─────────┬───────────────┬──────────┬──────────────────────┬──────────┬──────────────────────────────────┬─────────┐
-// │ (index) │ Task name     │ ops/sec  │ Average time/op (ns) │ Margin   │ Median time/op (ns)              │ Samples │
-// ├─────────┼───────────────┼──────────┼──────────────────────┼──────────┼──────────────────────────────────┼─────────┤
-// │ 0       │ 'faster task' │ '38,832' │ 25751.297631307978   │ '±3.48%' │ '22016.49999997812±5.5000000145' │ 3884    │
-// │ 1       │ 'slower task' │ '669'    │ 1493338.567164177    │ '±5.98%' │ '1445076.0000000286'             │ 67      │
-// └─────────┴───────────────┴──────────┴──────────────────────┴──────────┴──────────────────────────────────┴─────────┘
+// ┌─────────┬───────────────┬────────────────────────────┬───────────────────────────┬──────────────────────┬─────────────────────┬─────────┐
+// │ (index) │ Task name     │ Throughput average (ops/s) │ Throughput median (ops/s) │ Latency average (ns) │ Latency median (ns) │ Samples │
+// ├─────────┼───────────────┼────────────────────────────┼───────────────────────────┼──────────────────────┼─────────────────────┼─────────┤
+// │ 0       │ 'faster task' │ '102906 ± 0.89%'           │ '82217 ± 14'              │ '11909.14 ± 3.95%'   │ '12163.00 ± 2.00'   │ 8398    │
+// │ 1       │ 'slower task' │ '988 ± 26.26%'             │ '710'                     │ '1379560.47 ± 6.72%' │ '1408552.00'        │ 73      │
+// └─────────┴───────────────┴────────────────────────────┴───────────────────────────┴──────────────────────┴─────────────────────┴─────────┘
diff --git a/src/bench.ts b/src/bench.ts
@@ -1,4 +1,10 @@
 import pLimit from 'p-limit';
+import {
+  defaultMinimumIterations,
+  defaultMinimumTime,
+  defaultMinimumWarmupIterations,
+  defaultMinimumWarmupTime,
+} from './constants';
 import { createBenchEvent } from './event';
 import Task from './task';
 import type {
@@ -43,13 +49,13 @@ export default class Bench extends EventTarget {
 
   throws: boolean;
 
-  warmupTime = 100;
+  warmupTime = defaultMinimumWarmupTime;
 
-  warmupIterations = 5;
+  warmupIterations = defaultMinimumWarmupIterations;
 
-  time = 500;
+  time = defaultMinimumTime;
 
-  iterations = 10;
+  iterations = defaultMinimumIterations;
 
   now = now;
 
@@ -188,19 +194,21 @@ export default class Bench extends EventTarget {
         return (
           convert?.(task) || {
             'Task name': task.name,
-            'ops/sec': task.result.error
+            'Throughput average (ops/s)': task.result.error
               ? 'NaN'
-              : Number.parseInt(task.result.hz.toString(), 10).toLocaleString(),
-            'Average time/op (ns)': task.result.error
+              : `${task.result.throughput.mean.toFixed(0)} \xb1 ${task.result.throughput.rme.toFixed(2)}%`,
+            'Throughput median (ops/s)': task.result.error
               ? 'NaN'
-              : task.result.mean * 1e6,
-            Margin: task.result.error
+              : `${task.result.throughput.p50?.toFixed(0)}${Number.parseInt(task.result.throughput.mad!.toFixed(0), 10) > 0 ? ` \xb1 ${task.result.throughput.mad!.toFixed(0)}` : ''}`,
+            'Latency average (ns)': task.result.error
               ? 'NaN'
-              : `\xb1${task.result.rme.toFixed(2)}%`,
-            'Median time/op (ns)': task.result.error
+              : `${(task.result.latency.mean * 1e6).toFixed(2)} \xb1 ${task.result.latency.rme.toFixed(2)}%`,
+            'Latency median (ns)': task.result.error
               ? 'NaN'
-              : `${task.result.p50 * 1e6}${task.result.mad * 1e6 > 0 ? `\xb1${(task.result.mad * 1e6).toFixed(10)}` : ''}`,
-            Samples: task.result.error ? 'NaN' : task.result.samples.length,
+              : `${(task.result.latency.p50! * 1e6).toFixed(2)}${Number.parseFloat((task.result.latency.mad! * 1e6).toFixed(2)) > 0 ? ` \xb1 ${(task.result.latency.mad! * 1e6).toFixed(2)}` : ''}`,
+            Samples: task.result.error
+              ? 'NaN'
+              : task.result.latency.samples.length,
           }
         );
       }