feat(autobench): make metrics more regular; run more benchmarks

Agoric · Mar 10, 2021 · a0ec399 · a0ec399
1 parent 4f60df1
commit a0ec399
Show file tree

Hide file tree

Showing 2 changed files with 58 additions and 52 deletions.
diff --git a/packages/swingset-runner/autobench b/packages/swingset-runner/autobench
@@ -1,8 +1,8 @@
 #!/bin/sh
-set -e
-for suite in exchangeBenchmark; do
+for suite in exchangeBenchmark pingPongBenchmark swapBenchmark; do
   echo "Autobenching suite $suite ..."
   test ! -f demo/$suite/prepareContracts.js || node -r esm demo/$suite/prepareContracts.js
+  echo '{}' > benchstats.json || continue
   node --expose-gc -r esm bin/runner --init --benchmark 100 --statsfile benchstats.json --config demo/$suite/swingset.json run -- --quiet --prime
   node -r esm src/push-metrics.js $suite benchstats.json
 done

diff --git a/packages/swingset-runner/src/push-metrics.js b/packages/swingset-runner/src/push-metrics.js
@@ -81,84 +81,90 @@ function generateCommonMetrics(obj, phaseLabels) {
   return metrics;
 }
 
+function makePropMetrics(metricName, headerSpecs) {
+  return Object.fromEntries(
+    headerSpecs.map(([prop, ...args]) => [
+      prop,
+      promHeader(metricName[prop], ...args),
+    ]),
+  );
+}
+
 function generateMetricsFromPrimeData(data, labels = undefined) {
-  let metrics = '';
+  const metricName = {
+    up: 'stat_up',
+    down: 'stat_down',
+    max: 'stat_max',
+    value: 'stat_value',
+    perCrank: 'stat_per_crank',
+  };
+  const propMetrics = makePropMetrics(metricName, [
+    ['up', 'counter', `Number of increments`],
+    ['down', 'counter', `Number of decrements`],
+    ['max', 'gauge', `Maximum value`],
+    ['value', 'gauge', 'Latest value'],
+    ['perCrank', 'gauge', `Autobench value per crank`],
+  ]);
+
   const todo = new Set(Object.keys(data));
-  for (const { metricType, key, name, description } of KERNEL_STATS_METRICS) {
+  for (const { key, name } of KERNEL_STATS_METRICS) {
     if (!(key in data)) {
       // eslint-disable-next-line no-continue
       continue;
     }
     todo.delete(key);
-    if ('value' in data[key]) {
-      metrics += promHeader(name, metricType, description);
-      metrics += promValue(name, data[key].value, labels);
-    }
-    if ('up' in data[key]) {
-      const nm = `${name}_up`;
-      metrics += promHeader(
-        nm,
-        'counter',
-        `${description} (number of increments)`,
-      );
-      metrics += promValue(nm, data[key].up, labels);
-    }
-    if ('down' in data[key]) {
-      const nm = `${name}_down`;
-      metrics += promHeader(
-        nm,
-        'counter',
-        `${description} (number of decrements)`,
-      );
-      metrics += promValue(nm, data[key].down, labels);
-    }
-    if ('max' in data[key]) {
-      const nm = `${name}_max`;
-      metrics += promHeader(nm, 'gauge', `${description} (maximum value)`);
-      metrics += promValue(nm, data[key].max, labels);
-    }
-    if ('perCrank' in data[key]) {
-      const nm = `${name}_per_crank`;
-      metrics += promHeader(nm, 'gauge', `${description} (value per crank)`);
-      metrics += promValue(nm, data[key].perCrank, labels);
+    const statLabels = [...labels, ['stat', name]];
+
+    for (const prop of Object.keys(propMetrics)) {
+      if (prop in data[key]) {
+        propMetrics[prop] += promValue(
+          metricName[prop],
+          data[key][prop],
+          statLabels,
+        );
+      }
     }
   }
 
   for (const key of todo.keys()) {
     console.warn(`Unrecognized prime data property ${key}`);
   }
-  return metrics;
+  return Object.values(propMetrics).join('');
 }
 
 function generateMetricsFromBenchmarkData(data, labels = undefined) {
-  let metrics = '';
+  const metricName = {
+    delta: 'stat_delta',
+    deltaPerRound: 'stat_delta_per_round',
+  };
+  const propMetrics = makePropMetrics(metricName, [
+    ['delta', 'gauge', `Autobench benchmark delta`],
+    ['deltaPerRound', 'gauge', `Autobench benchmark delta per round`],
+  ]);
+
   const todo = new Set(Object.keys(data));
-  for (const { key, name, description } of KERNEL_STATS_METRICS) {
+  for (const { key, name } of KERNEL_STATS_METRICS) {
     if (!(key in data)) {
       // eslint-disable-next-line no-continue
       continue;
     }
     todo.delete(key);
-    if ('delta' in data[key]) {
-      const nm = `${name}_delta`;
-      metrics += promHeader(nm, 'gauge', `${description} benchmark delta`);
-      metrics += promValue(nm, data[key].delta, labels);
-    }
-    if ('deltaPerRound' in data[key]) {
-      const nm = `${name}_delta_per_round`;
-      metrics += promHeader(
-        nm,
-        'gauge',
-        `${description} benchmark delta per round`,
-      );
-      metrics += promValue(nm, data[key].deltaPerRound, labels);
+    const statLabels = [...labels, ['stat', name]];
+    for (const prop of Object.keys(propMetrics)) {
+      if (prop in data[key]) {
+        propMetrics[prop] += promValue(
+          metricName[prop],
+          data[key][prop],
+          statLabels,
+        );
+      }
     }
   }
 
   for (const key of todo.keys()) {
     console.warn(`Unrecognized benchmark data property ${key}`);
   }
-  return metrics;
+  return Object.values(propMetrics).join('');
 }
 
 function generateMetricsFromBenchStats(benchStats, labels = []) {