Add p95 to stats summaries (#46)

runner/bin/regen-perf.js

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+/* global process */
+
+import '../lib/sdk/install-ses.js';
+
+import { promisify } from 'util';
+import { pipeline as pipelineCallback } from 'stream';
+import BufferLineTransform from '../lib/helpers/buffer-line-transform.js';
+import { getBlocksSummaries, getCyclesSummaries } from '../lib/stats/stages.js';
+import {
+  getLiveBlocksSummary,
+  getCyclesSummary,
+  getTotalBlockCount,
+} from '../lib/stats/run.js';
+
+const pipeline = promisify(pipelineCallback);
+
+const { stdout, stdin } = process;
+
+async function* linesToEvent(lines) {
+  for await (const line of lines) {
+    yield JSON.parse(line.toString('utf8'));
+  }
+}
+
+async function* processEvent(events) {
+  for await (const event of events) {
+    if (event.type === 'perf-finish') {
+      /** @type {import('../lib/stats/types').RunStats} */
+      const stats = event.stats;
+
+      const stages = Object.fromEntries(
+        Object.entries(stats.stages).map(([idx, stage]) => {
+          const { blocks, cycles, endedAt } = stage;
+
+          // Currently if the stage fails, no summary is generated
+          if (endedAt === undefined) {
+            return [idx, stage];
+          }
+
+          const blockValues = Object.values(blocks);
+          const cycleValues = Object.values(cycles);
+
+          const blocksSummaries = blockValues.length
+            ? getBlocksSummaries(blockValues)
+            : undefined;
+          const cyclesSummaries = cycleValues.length
+            ? getCyclesSummaries(cycleValues)
+            : undefined;
+
+          return [idx, { ...stage, blocksSummaries, cyclesSummaries }];
+        }),
+      );
+
+      const cyclesSummary = getCyclesSummary(stages);
+      const liveBlocksSummary = getLiveBlocksSummary(stages);
+      const totalBlockCount = getTotalBlockCount(stages);
+
+      yield {
+        ...event,
+        stats: {
+          ...stats,
+          stages,
+          cyclesSummary,
+          liveBlocksSummary,
+          totalBlockCount,
+        },
+      };
+    } else {
+      yield event;
+    }
+  }
+}
+
+async function* eventToLine(events) {
+  for await (const event of events) {
+    yield `${JSON.stringify(event)}\n`;
+  }
+}
+
+await pipeline(
+  stdin,
+  new BufferLineTransform(),
+  linesToEvent,
+  processEvent,
+  eventToLine,
+  stdout,
+);

runner/lib/main.js

@@ -729,8 +729,9 @@ const main = async (progName, rawArgs, powers) => {
           tasks.push(stageReady);
         }
         stats.recordStart(timeSource.getTime());
-        await sequential(...tasks)((stop) => stop);
-        stats.recordEnd(timeSource.getTime());
+        await sequential(...tasks)((stop) => stop).finally(() =>
+          stats.recordEnd(timeSource.getTime()),
+        );
       },
       async (...stageError) =>
         aggregateTryFinally(

runner/lib/stats/blocks.js

@@ -79,7 +79,7 @@ const rawBlockStatsInit = {
  */
 
 /**
- * @param {import('./helpers.js').Sums<BlockStatsSumKeys>} sums
+ * @param {import('./helpers.js').Summary<BlockStatsSumKeys>} summary
  * @returns {import('./types.js').BlockStatsSummary | undefined}
  */
 export const makeBlockStatsSummary = ({
@@ -90,6 +90,7 @@ export const makeBlockStatsSummary = ({
   items,
   mins,
   maxes,
+  p95s,
 }) =>
   blockCount
     ? {
@@ -116,6 +117,19 @@ export const makeBlockStatsSummary = ({
         avgProcessingPercentage: percentageRounder(
           averages.processingPercentage / 100,
         ),
+        p95Lag: timeRounder(p95s.lag),
+        p95BlockDuration: timeRounder(p95s.blockDuration),
+        p95ChainBlockDuration: timeRounder(p95s.chainBlockDuration),
+        p95IdleTime: timeRounder(p95s.idleTime),
+        p95CosmosTime: timeRounder(p95s.cosmosTime),
+        p95SwingsetTime: timeRounder(p95s.swingsetTime),
+        p95ProcessingTime: timeRounder(p95s.processingTime),
+        p95Deliveries: timeRounder(p95s.deliveries),
+        p95Computrons: timeRounder(p95s.computrons),
+        p95SwingsetPercentage: percentageRounder(p95s.swingsetPercentage / 100),
+        p95ProcessingPercentage: percentageRounder(
+          p95s.processingPercentage / 100,
+        ),
       }
     : undefined;
 

runner/lib/stats/cycles.js

@@ -35,16 +35,22 @@ const rawCycleStatsInit = {
 /** @typedef {'success' | 'blockCount' | 'duration'} CycleStatsSumKeys */
 
 /**
- * @param {import('./helpers.js').Sums<CycleStatsSumKeys>} sums
+ * @param {import('./helpers.js').Summary<CycleStatsSumKeys>} sums
  * @returns {import('./types.js').CycleStatsSummary | undefined}
  */
-export const makeCycleStatsSummary = ({ weights: cycleCount, averages }) =>
+export const makeCycleStatsSummary = ({
+  weights: cycleCount,
+  averages,
+  p95s,
+}) =>
   cycleCount
     ? {
         cycleCount,
         cycleSuccessRate: percentageRounder(averages.success),
         avgBlockCount: rounder(averages.blockCount),
         avgDuration: rounder(averages.duration),
+        p95BlockCount: rounder(p95s.blockCount),
+        p95Duration: rounder(p95s.duration),
       }
     : undefined;
 

runner/lib/stats/helpers.d.ts

@@ -86,7 +86,7 @@ type SumRecord<K extends string> = {
   readonly [P in K]: number;
 };
 
-export type Sums<K extends string> = {
+export type Summary<K extends string> = {
   readonly values: number;
   readonly weights: number;
   readonly items: SumRecord<K>;
@@ -95,13 +95,17 @@ export type Sums<K extends string> = {
   readonly totals: SumRecord<K>; // weighted
   readonly counts: SumRecord<K>; // weighted
   readonly averages: SumRecord<K>; // weighted
+  readonly p95s: SumRecord<K>;
 };
 
-type SummableRecord = { readonly [P in string]: number | undefined };
+type SummaryRecord = { readonly [P in string]: number | undefined };
+export type SummarizeData<T extends SummaryRecord> = ReadonlyArray<{
+  values: T;
+  weight?: number;
+}>;
 
-interface Summer<T extends SummableRecord> {
-  add(value: T, weight?: 1 | number): void;
-  getSums(): Sums<keyof T>;
-}
+export declare function summarize<T extends SummaryRecord>(
+  data: SummarizeData<T>,
+): Summary<keyof T>;
 
-export declare function makeSummer<T extends SummableRecord>(): Summer<T>;
+export declare function notUndefined<T>(x: T | undefined): x is T;

runner/lib/stats/helpers.js

@@ -145,74 +145,67 @@ export const arrayGroupBy = (source, callback) => {
   return target;
 };
 
-export const makeSummer = () => {
-  /** @type {Map<string, {min: number, max: number, values: number, weights: number, total: number}>} */
-  const sumDatas = new Map();
-  let weights = 0;
-  let values = 0;
+/** @param {unknown} val */
+export const notUndefined = (val) => val !== undefined;
 
-  /**
-   *
-   * @param {Record<string, number | undefined>} obj
-   * @param {number} [weight]
-   */
-  const add = (obj, weight = 1) => {
-    values += 1;
-    weights += weight;
-    for (const [key, value] of Object.entries(obj)) {
-      let sumData = sumDatas.get(key);
-      if (!sumData) {
-        sumData = {
-          min: NaN,
-          max: NaN,
-          values: 0,
-          weights: 0,
-          total: 0,
-        };
-        sumDatas.set(key, sumData);
-      }
-      if (value != null && !Number.isNaN(value)) {
-        sumData.values += 1;
-        sumData.weights += weight;
-        sumData.total += value * weight;
-        if (!(value > sumData.min)) {
-          sumData.min = value;
-        }
-        if (!(value < sumData.max)) {
-          sumData.max = value;
-        }
-      }
+/**
+ * @param {Array<{values: Record<string, number | undefined>, weight?: number | undefined}>} data
+ * @returns {import('./helpers.js').Summary<string>}
+ */
+export const summarize = (data) => {
+  const keys = new Set(data.flatMap(({ values }) => Object.keys(values)));
+
+  const weights = data.reduce((acc, { weight = 1 }) => acc + weight, 0);
+
+  const items = /** @type {Record<string, number>} */ ({});
+  const mins = /** @type {Record<string, number>} */ ({});
+  const maxes = /** @type {Record<string, number>} */ ({});
+  const totals = /** @type {Record<string, number>} */ ({});
+  const counts = /** @type {Record<string, number>} */ ({});
+  const averages = /** @type {Record<string, number>} */ ({});
+  const p95s = /** @type {Record<string, number>} */ ({});
+
+  for (const key of keys) {
+    const sortedData = /** @type {Array<{values: Record<string, number>, weight?: number | undefined}>} */ (data.filter(
+      ({ values }) => Number.isFinite(values[key]),
+    )).sort((a, b) => a.values[key] - b.values[key]);
+
+    items[key] = sortedData.length;
+    mins[key] = sortedData.length ? sortedData[0].values[key] : NaN;
+    maxes[key] = sortedData.length ? sortedData.slice(-1)[0].values[key] : NaN;
+    totals[key] = 0;
+    counts[key] = 0;
+    for (const { values, weight = 1 } of sortedData) {
+      totals[key] += values[key] * weight;
+      counts[key] += weight;
     }
-  };
-
-  const getSums = () => {
-    const items = /** @type {Record<string, number>} */ ({});
-    const mins = /** @type {Record<string, number>} */ ({});
-    const maxes = /** @type {Record<string, number>} */ ({});
-    const totals = /** @type {Record<string, number>} */ ({});
-    const counts = /** @type {Record<string, number>} */ ({});
-    const averages = /** @type {Record<string, number>} */ ({});
-
-    for (const [key, sumData] of sumDatas.entries()) {
-      items[key] = sumData.values;
-      mins[key] = sumData.min;
-      maxes[key] = sumData.max;
-      totals[key] = sumData.total;
-      counts[key] = sumData.weights;
-      averages[key] = sumData.total / sumData.weights;
+    averages[key] = totals[key] / counts[key];
+
+    if (
+      sortedData.length > 1 &&
+      sortedData.every(({ weight = 1 }) => weight === 1)
+    ) {
+      const rank = (95 * (sortedData.length - 1)) / 100;
+      const rankIndex = Math.floor(rank);
+      const basePercentile = sortedData[rankIndex].values[key];
+      const nextPercentile = sortedData[rankIndex + 1].values[key];
+      p95s[key] =
+        basePercentile +
+        (rankIndex - rankIndex) * (nextPercentile - basePercentile);
+    } else {
+      p95s[key] = NaN;
     }
-
-    return harden({
-      values,
-      weights,
-      items,
-      mins,
-      maxes,
-      totals,
-      counts,
-      averages,
-    });
-  };
-
-  return harden({ add, getSums });
+  }
+
+  return harden({
+    values: data.values.length,
+    weights,
+    items,
+    mins,
+    maxes,
+    totals,
+    counts,
+    averages,
+    p95s,
+  });
 };