diff --git a/.github/workflows/test-all-packages.yml b/.github/workflows/test-all-packages.yml index ecbfdd5ab6f..871551fa2e4 100644 --- a/.github/workflows/test-all-packages.yml +++ b/.github/workflows/test-all-packages.yml @@ -124,6 +124,9 @@ jobs: - name: yarn test (assert) if: (success() || failure()) run: cd packages/assert && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT + - name: yarn test (benchmark) + if: (success() || failure()) + run: cd packages/benchmark && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT - name: yarn test (deployment) if: (success() || failure()) run: cd packages/deployment && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT diff --git a/packages/SwingSet/package.json b/packages/SwingSet/package.json index 12d05f5b211..f4ca36b97de 100644 --- a/packages/SwingSet/package.json +++ b/packages/SwingSet/package.json @@ -52,6 +52,7 @@ "@endo/patterns": "^0.2.5", "@endo/promise-kit": "^0.2.59", "@endo/ses-ava": "^0.2.43", + "@endo/stream": "^0.3.28", "@endo/zip": "^0.2.34", "ansi-styles": "^6.2.1", "anylogger": "^0.21.0", diff --git a/packages/SwingSet/tools/run-utils.ts b/packages/SwingSet/tools/run-utils.ts new file mode 100644 index 00000000000..54e283d09f1 --- /dev/null +++ b/packages/SwingSet/tools/run-utils.ts @@ -0,0 +1,156 @@ +/* eslint-disable @jessie.js/safe-await-separator */ +import { Fail, q } from '@agoric/assert'; +import { kunser } from '@agoric/kmarshal'; +import { makeQueue } from '@endo/stream'; +import type { E } from '@endo/eventual-send'; + +import type { SwingsetController } from '../src/controller/controller.js'; + +const sink = () => {}; + +export const makeRunUtils = ( + controller: SwingsetController, + log = (..._) => {}, +) => { + let cranksRun = 0; + + const mutex = makeQueue(); + + mutex.put(controller.run()); + + const runThunk = async any>( + thunk: T, + ): Promise> => { + try { + // this promise for the last lock may fail + await mutex.get(); + } catch { + // noop because the result will resolve for the previous runMethod return + } + + const thunkResult = await thunk(); + + const result = controller.run().then(cranks => { + cranksRun += cranks; + log(`kernel ran ${cranks} cranks`); + return thunkResult; + }); + mutex.put(result.then(sink, sink)); + return result; + }; + + const queueAndRun = async (deliveryThunk, voidResult = false) => { + log('queueAndRun at', cranksRun); + + const kpid = await runThunk(deliveryThunk); + + if (voidResult) { + return undefined; + } + const status = controller.kpStatus(kpid); + switch (status) { + case 'fulfilled': + return kunser(controller.kpResolution(kpid)); + case 'rejected': + throw kunser(controller.kpResolution(kpid)); + case 'unresolved': + throw Fail`unsettled value for ${q(kpid)}`; + default: + throw Fail`unknown promise status ${q(kpid)} ${q(status)}`; + } + }; + + type EVProxy = typeof E & { + sendOnly: (presence: unknown) => Record void>; + vat: (name: string) => Record Promise>; + }; + + // IMPORTANT WARNING TO USERS OF `EV` + // + // `EV` presents an abstraction that can be used (within tests only!) to get + // much of the convenience with respect to messaging that `E` provides in + // normal code. However, this convenience comes with a huge caveat that all + // users of this convenience feature MUST keep in mind. + // + // A test can drop messages onto the kernel's run queue using the + // `controller.queueToVatRoot` and `controller.queueToVatObject` methods. + // These are synchronous operations which merely place messages onto the run + // queue without causing execution. Execution, on the other hand, is + // initiated by calling `controller.run`, which will cause the kernel to begin + // delivering messages to vats from the run queue, continuing until the run + // queue is exhausted. HOWEVER, exhaustion of the run queue, which resolves + // the result promise returned by the `run` call, IS NOT coupled in any causal + // way to the resolution of result promises associated with the individual + // queued messages themselves. The status and resolution values of these + // promises can be synchronously queried (by kpid) via the + // `controller.kpStatus` and `controller.kpResolution` methods once `run` has + // completed. These queries are only available once the swingset has + // reqlinquished agency, i.e., when the work initiated by `controller.run` has + // finished. At that point, nothing is going on inside the kernel, and + // nothing WILL be going on inside the kernel until a subsequent call to + // `controller.run`, which in turn will only have an effect if additional + // messages have been placed on the kernel run queue in the meantime. You MAY + // NOT call `queueToVatRoot`, `queueToVatObject`, `kpStatus`, or + // `kpResolution` while run queue execution, triggered by a call to `run`, is + // in progress + // + // The functionality made available by `EV` looks similar to that provided by + // `E`, but it is very much not. When you send a message using `EV`, it + // places the message onto the kernel run queue and then immediately invokes + // `controller.run`. When the result of `run` resolves, the kpid returned by + // the message enqueueing operation is queried. If at that time the promise + // it identifies is resolved (or rejected), the value it was resolved (or + // rejected) to is used as the result from the `EV` invocation. However, if + // it is still pending at that time, `EV` will throw an exception, which will + // manifest as a rejection and your test will fail confusingly or abort. This + // means that if you initiate some operation via an `EV` message send, it must + // complete within a single `run` cycle for it to be of any use to you. This + // is quite different from a message sent using `E`, which will return a + // promise that can remain pending indefinitely, possibly to be settled by a + // future message delivery. + + // @ts-expect-error cast, approximate + const EV: EVProxy = presence => + new Proxy(harden({}), { + get: (_t, method, _rx) => { + const boundMethod = (...args) => + queueAndRun(() => + controller.queueToVatObject(presence, method, args), + ); + return harden(boundMethod); + }, + }); + EV.vat = vatName => + new Proxy(harden({}), { + get: (_t, method, _rx) => { + const boundMethod = (...args) => + queueAndRun(() => controller.queueToVatRoot(vatName, method, args)); + return harden(boundMethod); + }, + }); + // @ts-expect-error xxx + EV.sendOnly = presence => + new Proxy(harden({}), { + get: (_t, method, _rx) => { + const boundMethod = (...args) => + queueAndRun( + () => controller.queueToVatObject(presence, method, args), + true, + ); + return harden(boundMethod); + }, + }); + // @ts-expect-error xxx + EV.get = presence => + new Proxy(harden({}), { + get: (_t, pathElement, _rx) => + queueAndRun(() => + controller.queueToVatRoot('bootstrap', 'awaitVatObject', [ + presence, + [pathElement], + ]), + ), + }); + return harden({ runThunk, EV }); +}; +export type RunUtils = ReturnType; diff --git a/packages/agoric-cli/src/sdk-package-names.js b/packages/agoric-cli/src/sdk-package-names.js index 2db4676d928..1f2908ddf7b 100644 --- a/packages/agoric-cli/src/sdk-package-names.js +++ b/packages/agoric-cli/src/sdk-package-names.js @@ -5,6 +5,7 @@ export default [ "@agoric/access-token", "@agoric/assert", "@agoric/base-zone", + "@agoric/benchmark", "@agoric/boot", "@agoric/builders", "@agoric/cache", diff --git a/packages/benchmark/CHANGELOG.md b/packages/benchmark/CHANGELOG.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/benchmark/benchmark/benchmark-vault-adjust.js b/packages/benchmark/benchmark/benchmark-vault-adjust.js new file mode 100644 index 00000000000..f22be8047bd --- /dev/null +++ b/packages/benchmark/benchmark/benchmark-vault-adjust.js @@ -0,0 +1,70 @@ +import { bench } from '../src/benchmarkerator.js'; + +// eslint-disable-next-line import/order +import { Offers } from '@agoric/inter-protocol/src/clientSupport.js'; + +const collateralBrandKey = 'ATOM'; +const adjustOpenOfferId = 'adjust-open'; + +// The benchmark-defined option `size` (default 1) indicates how many operations +// will be performed per round. The difference between performing 10 rounds of +// 1 operation each (command line: `--rounds 10`) and performing 1 round of 10 +// operations (command line: `-o size 10`) is that while both will perform 10 +// operations, in the first case the 10 operations will be done sequentially +// while in the second case they will be done concurrently. These are two +// different modes you might want to measure. (Of course, you could do 10 +// rounds of 10 operations each: `--rounds 10 -o size 10`, and that would work +// fine also.) + +bench.addBenchmark('adjust vault balance', { + setup: async context => { + const { alice } = context.actors; + + await alice.executeOfferMaker(Offers.vaults.OpenVault, { + offerId: adjustOpenOfferId, + collateralBrandKey, + wantMinted: 5.0, + giveCollateral: 9.0, + }); + const upd = alice.getLatestUpdateRecord(); + assert( + upd.updated === 'offerStatus' && + upd.status.id === adjustOpenOfferId && + upd.status.numWantsSatisfied === 1, + ); + return undefined; + }, + + executeRound: async (context, round) => { + const { alice } = context.actors; + + const adjustVault = async (i, n, r) => { + const offerId = `adjust-vault-${i}-of-${n}=round-${r}`; + await alice.executeOfferMaker( + Offers.vaults.AdjustBalances, + { + offerId, + collateralBrandKey, + giveMinted: 0.0005, + }, + adjustOpenOfferId, + ); + const upd = alice.getLatestUpdateRecord(); + assert( + upd.updated === 'offerStatus' && + upd.status.id === offerId && + upd.status.numWantsSatisfied === 1, + ); + }; + + const adjustN = async n => { + const range = [...Array(n)].map((_, i) => i + 1); + await Promise.all(range.map(i => adjustVault(i, n, round))); + }; + + const roundSize = context.options.size ? Number(context.options.size) : 1; + await adjustN(roundSize); + }, +}); + +await bench.run(); diff --git a/packages/boot/test/bootstrapTests/benchmark-vaults.js b/packages/benchmark/benchmark/benchmark-vault-open.js similarity index 90% rename from packages/boot/test/bootstrapTests/benchmark-vaults.js rename to packages/benchmark/benchmark/benchmark-vault-open.js index 4f6e50d8656..df4a9b7bc0e 100644 --- a/packages/boot/test/bootstrapTests/benchmark-vaults.js +++ b/packages/benchmark/benchmark/benchmark-vault-open.js @@ -1,11 +1,11 @@ -import { bench } from './benchmarkerator.js'; +import { bench } from '../src/benchmarkerator.js'; // eslint-disable-next-line import/order import { Offers } from '@agoric/inter-protocol/src/clientSupport.js'; const collateralBrandKey = 'ATOM'; -bench.addBenchmark('vault open', { +bench.addBenchmark('open vault', { executeRound: async (context, round) => { const { alice } = context.actors; @@ -36,4 +36,4 @@ bench.addBenchmark('vault open', { }, }); -await bench.run('vaults'); +await bench.run(); diff --git a/packages/boot/test/bootstrapTests/benchmarkerator.md b/packages/benchmark/doc/benchmarkerator.md similarity index 91% rename from packages/boot/test/bootstrapTests/benchmarkerator.md rename to packages/benchmark/doc/benchmarkerator.md index af1b5871a24..2558ddc0c08 100644 --- a/packages/boot/test/bootstrapTests/benchmarkerator.md +++ b/packages/benchmark/doc/benchmarkerator.md @@ -39,16 +39,11 @@ yourself).[^2] Those will themselves be deprecated and/or moved into `benchmarks`. However, as of this writing that has not yet been done. -The first thing a benchmark should do is import The Benchmarkerator:[^3] +The first thing a benchmark should do is import The Benchmarkerator: ``` import { bench } from '@agoric/benchmark'; ``` -[^3]: As of this writing The Benchmarkerator actually lives in - `@agoric/boot/test/boostrapTests/benchmarkerator.js`. This should be - changed soon, hopefully before the first PR containing this writeup is - landed, in which case you will never see this footnote. - Note that this importation usually should be the very first thing you do, much as you typically import `@agoric/swingset-vat/tools/prepare-test-env-ava.js` or `@agoric/zoe/tools/prepare-test-env-ava.js` or the like as the first thing in a @@ -69,10 +64,7 @@ labelled in Ava), and `benchmark` is an object describing the benchmark itself After defining one or more benchmarks with `addBenchmark` you must then end by invoking: -```await bench.run(name);``` - -where `name` is a string naming the benchmark run for purposes of error and -result reporting. +```await bench.run();``` ### The `Benchmark` object @@ -151,7 +143,8 @@ The supported command line options are: | `-v`
`--verbose` | Enable verbose output | | `--vat-type TYPE` | Use the specified vat manager type rather than the default `xs-worker` | | `-l`
`--local` | Shorthand for `--vat-type local` (vats run in the same process as the kernel; less realistic than `xs-worker` but much faster and easier to debug) | -| `-d`
`--dump` | Output JSON-formated benchmark data to a file | +| `-d PATH`
`--dump PATH` | Output JSON-formated benchmark data into _PATH_ | +| `-s PATH`
`--slot PATH` | Output a log file into _PATH_ | | `-h`
`--help` | Output this helpful usage information and then exit | An optional `--` flag ends the options list. Any remaining command line @@ -164,11 +157,13 @@ Timing results and other collected metrics are output to _stdout_. Two batches of information are provided: one for the setup phase and one for the benchmark rounds themselves (in aggregate). -In addition, if you specify the `--dump` command line option, a JSON-formatted -(i.e., machine readable) version of this same data will be output to the file -`benchmark-NAME.json` in the current working directory (where _NAME_ is the name -that you provided as the argument to `bench.run`). +If you specify the `--dump FILEPATH` command line option, a JSON-formatted +(i.e., machine readable) version of this same data will be output to the +indicated file. Output results include execution times (according to Node's nanosecond clock), crank counts, and the various kernel resource usage data reported by `controller.getStats()`. + +In addition, if you specify the `--slog FILEPATH` command line option, a +SwingSet slog file for the run will be output to the file indicated. diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json new file mode 100644 index 00000000000..d0ac7f9484d --- /dev/null +++ b/packages/benchmark/package.json @@ -0,0 +1,66 @@ +{ + "name": "@agoric/benchmark", + "version": "0.1.0", + "private": true, + "description": "Benchmark support", + "type": "module", + "main": "./src/benchmarkerator.js", + "exports": { + ".": "./src/benchmarkerator.js" + }, + "repository": "https://github.com/Agoric/agoric-sdk", + "scripts": { + "build": "exit 0", + "test": "exit 0", + "test:xs": "exit 0", + "lint-fix": "yarn lint:eslint --fix", + "lint": "run-s --continue-on-error lint:*", + "lint:types": "tsc", + "lint:eslint": "eslint ." + }, + "keywords": [], + "author": "Agoric", + "license": "Apache-2.0", + "dependencies": { + "@agoric/assert": "^0.6.0", + "@agoric/boot": "^0.1.0", + "@agoric/cosmic-swingset": "^0.41.3", + "@agoric/internal": "^0.3.2", + "@agoric/inter-protocol": "^0.16.1", + "@agoric/vats": "^0.15.1", + "@agoric/zoe": "^0.26.2", + "@endo/init": "^0.5.59" + }, + "devDependencies": {}, + "files": [ + "CHANGELOG.md", + "src/", + "scripts/", + "tools/", + "*.json", + "globals.d.ts", + "exported.js" + ], + "engines": { + "node": ">=14.15.0" + }, + "ava": { + "extensions": { + "js": true, + "ts": "module" + }, + "files": [ + "test/**/test-*.js", + "test/**/test-*.ts" + ], + "nodeArguments": [ + "--loader=tsx", + "--no-warnings" + ], + "require": [ + "@endo/init/debug.js" + ], + "timeout": "20m", + "workerThreads": false + } +} diff --git a/packages/boot/test/bootstrapTests/benchmarkerator.js b/packages/benchmark/src/benchmarkerator.js similarity index 94% rename from packages/boot/test/bootstrapTests/benchmarkerator.js rename to packages/benchmark/src/benchmarkerator.js index eb6e4e751fd..2044911a870 100644 --- a/packages/boot/test/bootstrapTests/benchmarkerator.js +++ b/packages/benchmark/src/benchmarkerator.js @@ -4,11 +4,17 @@ import fs from 'node:fs'; import '@endo/init/pre-bundle-source.js'; import '@endo/init'; +// XXX The following four imports are present only to make `tsc` shut up. They do no actual work. +import '@agoric/vats/exported.js'; +import '@agoric/inter-protocol/exported.js'; +import '@agoric/zoe/exported.js'; +import '@agoric/cosmic-swingset/src/launch-chain.js'; + import { Fail } from '@agoric/assert'; import { eventLoopIteration } from '@agoric/internal/src/testing-utils.js'; import { makeAgoricNamesRemotesFromFakeStorage } from '@agoric/vats/tools/board-utils.js'; -import { makeSwingsetTestKit } from './supports.ts'; -import { makeWalletFactoryDriver } from './drivers.ts'; +import { makeSwingsetTestKit } from '@agoric/boot/tools/supports.ts'; +import { makeWalletFactoryDriver } from '@agoric/boot/tools/drivers.ts'; // When I was a child my family took a lot of roadtrips around California to go // camping and backpacking and so on. It was not uncommon in those days (nor is @@ -38,7 +44,7 @@ import { makeWalletFactoryDriver } from './drivers.ts'; * @typedef {{ * options: Record, * argv: string[], - * actors: Record, + * actors: Record, * title?: string, * rounds?: number, * config?: Record, @@ -100,7 +106,7 @@ import { makeWalletFactoryDriver } from './drivers.ts'; * * @typedef {{ * addBenchmark: (title: string, benchmark: Benchmark) => void, - * run: (name?: string) => Promise, + * run: () => Promise, * }} Benchmarkerator */ @@ -111,7 +117,8 @@ const argv = process.argv.slice(2); let commandLineRounds; let verbose = false; let help = false; -let dump = false; +let dumpFile; +let slogFile; /** @type ManagerType */ let defaultManagerType = 'xs-worker'; const options = {}; @@ -142,8 +149,11 @@ FLAGS may be: --local - shorthand for '--vat-type local' (less realistic perf numbers but faster and easier to debug) - -d - --dump - output JSON-formatted benchmark data to a file + -s PATH + --slog PATH - output a slog file into PATH + + -d PATH + --dump PATH - output JSON-formatted benchmark data into PATH -h --help - output this helpful usage information and then exit @@ -178,7 +188,7 @@ while (argv[0] && stillScanningArgs) { break; case '-d': case '--dump': - dump = true; + dumpFile = argv.shift(); break; case '--vat-type': { const type = argv.shift(); @@ -203,6 +213,10 @@ while (argv[0] && stillScanningArgs) { case '--help': help = true; break; + case '-s': + case '--slog': + slogFile = argv.shift(); + break; case '-o': case '--option': { const optionName = argv.shift(); @@ -427,6 +441,7 @@ export const makeBenchmarkerator = async () => { const swingsetTestKit = await makeSwingsetTestKit(console.log, undefined, { defaultManagerType, verbose, + slogFile, }); const { runUtils, @@ -520,11 +535,8 @@ export const makeBenchmarkerator = async () => { /** * Execute the benchmarks. - * - * @param {string} [name] - string identifying the set of benchmarks being - * executed. Used for logging and labeling the output data. */ - const run = async (name = 'benchmark') => { + const run = async () => { /** @type {object} */ const benchmarkContext = { ...context }; await null; @@ -569,11 +581,8 @@ export const makeBenchmarkerator = async () => { } await eventLoopIteration(); await shutdown(); - if (dump) { - fs.writeFileSync( - `benchmark-${name}.json`, - JSON.stringify(benchmarkReport, null, 2), - ); + if (dumpFile) { + fs.writeFileSync(dumpFile, JSON.stringify(benchmarkReport, null, 2)); } }; @@ -586,4 +595,5 @@ export const makeBenchmarkerator = async () => { /** * The normal singleton benchmarkerator. */ +// eslint-disable-next-line @jessie.js/safe-await-separator export const bench = await makeBenchmarkerator(); diff --git a/packages/benchmark/tsconfig.json b/packages/benchmark/tsconfig.json new file mode 100644 index 00000000000..129dfe148b1 --- /dev/null +++ b/packages/benchmark/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "allowImportingTsExtensions": true, + "allowSyntheticDefaultImports": true, + "checkJs": true, + "maxNodeModuleJsDepth": 2, + }, + "include": [ + "*.js", + "scripts/**/*.js", + "src/**/*.js", + "benchmark/**/*.js", + "benchmark/**/*.ts", + "test/**/*.js", + "test/**/*.ts", + "tools/**/*.js", + ], +} diff --git a/packages/boot/package.json b/packages/boot/package.json index 570a86cd7a7..8c0e269bf9a 100644 --- a/packages/boot/package.json +++ b/packages/boot/package.json @@ -9,7 +9,6 @@ "build": "exit 0", "test": "ava", "test:xs": "SWINGSET_WORKER_TYPE=xs-worker ava 'test/bootstrapTests/**/test-*.js' 'test/upgrading/**/test-*.js'", - "bench": "ava --config ./ava.bench.config.js", "lint-fix": "yarn lint:eslint --fix", "lint": "run-s --continue-on-error lint:*", "lint:types": "tsc", @@ -19,9 +18,17 @@ "author": "Agoric", "license": "Apache-2.0", "dependencies": { + "@agoric/assert": "^0.6.0", "@agoric/builders": "^0.1.0", + "@agoric/cosmic-swingset": "^0.41.3", "@agoric/ertp": "^0.16.2", "@agoric/internal": "^0.3.2", + "@agoric/inter-protocol": "^0.16.1", + "@agoric/kmarshal": "^0.1.0", + "@agoric/swing-store": "^0.9.1", + "@agoric/swingset-vat": "^0.32.2", + "@agoric/telemetry": "^0.6.2", + "@agoric/time": "^0.3.2", "@agoric/vat-data": "^0.5.2", "@agoric/vats": "^0.15.1", "@agoric/vm-config": "^0.1.0", @@ -37,17 +44,10 @@ "import-meta-resolve": "^2.2.1" }, "devDependencies": { - "@agoric/assert": "^0.6.0", - "@agoric/cosmic-swingset": "^0.41.3", "@agoric/deploy-script-support": "^0.10.3", "@agoric/governance": "^0.10.3", - "@agoric/inter-protocol": "^0.16.1", - "@agoric/kmarshal": "^0.1.0", "@agoric/store": "^0.9.2", - "@agoric/swing-store": "^0.9.1", "@agoric/swingset-liveslots": "^0.10.2", - "@agoric/swingset-vat": "^0.32.2", - "@agoric/time": "^0.3.2", "ava": "^5.3.0", "c8": "^7.13.0", "tsx": "^3.12.8" diff --git a/packages/boot/test/bootstrapTests/bench-vaults-performance.js b/packages/boot/test/bootstrapTests/bench-vaults-performance.js deleted file mode 100644 index cd76d1ad3eb..00000000000 --- a/packages/boot/test/bootstrapTests/bench-vaults-performance.js +++ /dev/null @@ -1,221 +0,0 @@ -// @ts-check -/** @file Bootstrap stress test of vaults */ -import { test as anyTest } from '@agoric/zoe/tools/prepare-test-env-ava.js'; -import { PerformanceObserver, performance } from 'node:perf_hooks'; -import v8 from 'node:v8'; -import process from 'node:process'; -import fs from 'node:fs'; - -import { Fail } from '@agoric/assert'; -import { Offers } from '@agoric/inter-protocol/src/clientSupport.js'; -import engineGC from '@agoric/internal/src/lib-nodejs/engine-gc.js'; - -import { eventLoopIteration } from '@agoric/internal/src/testing-utils.js'; -import { makeAgoricNamesRemotesFromFakeStorage } from '@agoric/vats/tools/board-utils.js'; -import { makeSwingsetTestKit } from './supports.ts'; -import { makeWalletFactoryDriver } from './drivers.ts'; - -/** - * @type {import('ava').TestFn< - * Awaited> - * >} - */ -const test = anyTest; - -let snapshotNum = 0; -const collectStats = async (step, dumpHeap) => { - await eventLoopIteration(); - try { - const t0 = performance.now(); - engineGC(); - const t1 = performance.now(); - const memoryUsage = process.memoryUsage(); - const t2 = performance.now(); - const heapStats = v8.getHeapStatistics(); - const t3 = performance.now(); - - const memStats = { - memoryUsage, - heapStats, - statsTime: { - forcedGcMs: t1 - t0, - memoryUsageMs: t2 - t1, - heapStatsMs: t3 - t2, - }, - }; - - if (dumpHeap) { - console.log(`Snapshotting heap at step ${step}...`); - - // process.pid increments so these will be lexically sorted pathnames. - const heapSnapshot = `Heap-${process.pid}-${snapshotNum}-${step}.heapsnapshot`; - snapshotNum += 1; - - v8.writeHeapSnapshot(heapSnapshot); - const heapSnapshotTime = performance.now() - t3; - memStats.heapSnapshot = heapSnapshot; - memStats.statsTime.heapSnapshot = heapSnapshotTime; - } - - console.log(`Heap details at step ${step} vaults: `, memStats); - return memStats; - } catch (err) { - console.warn('Failed to gather memory stats', err); - return undefined; - } -}; - -// presently all these tests use one collateral manager -const collateralBrandKey = 'ATOM'; - -const makeDefaultTestContext = async t => { - console.time('DefaultTestContext'); - const swingsetTestKit = await makeSwingsetTestKit(t.log, 'bundles/vaults'); - - const { runUtils, storage } = swingsetTestKit; - console.timeLog('DefaultTestContext', 'swingsetTestKit'); - const { EV } = runUtils; - - // Wait for ATOM to make it into agoricNames - await EV.vat('bootstrap').consumeItem('vaultFactoryKit'); - console.timeLog('DefaultTestContext', 'vaultFactoryKit'); - - // has to be late enough for agoricNames data to have been published - const agoricNamesRemotes = makeAgoricNamesRemotesFromFakeStorage( - swingsetTestKit.storage, - ); - agoricNamesRemotes.brand.ATOM || Fail`ATOM missing from agoricNames`; - console.timeLog('DefaultTestContext', 'agoricNamesRemotes'); - - const walletFactoryDriver = await makeWalletFactoryDriver( - runUtils, - storage, - agoricNamesRemotes, - ); - console.timeLog('DefaultTestContext', 'walletFactoryDriver'); - - console.timeEnd('DefaultTestContext'); - - return { ...swingsetTestKit, agoricNamesRemotes, walletFactoryDriver }; -}; - -test.before(async t => { - t.context = await makeDefaultTestContext(t); -}); -test.after.always(t => t.context.shutdown()); - -const rows = []; -const perfObserver = new PerformanceObserver(items => { - for (const entry of items.getEntries()) { - // @ts-expect-error cast - const { vaultsOpened, round } = entry.detail; - rows.push({ - name: `${round}:${vaultsOpened}`, - durationMs: entry.duration, - avgPerVaultMs: entry.duration / vaultsOpened, - }); - } -}); -perfObserver.observe({ entryTypes: ['measure'] }); - -const whereUrl = import.meta.url; -const sdkPathStart = whereUrl.lastIndexOf('agoric-sdk/'); -const where = sdkPathStart > 0 ? whereUrl.substring(sdkPathStart) : whereUrl; - -async function stressVaults(t, dumpHeap) { - rows.length = 0; - const dumpTag = dumpHeap ? '-with-dump' : ''; - const name = `stress-vaults${dumpTag}`; - - const { walletFactoryDriver } = t.context; - const wd = await walletFactoryDriver.provideSmartWallet('agoric1open'); - await walletFactoryDriver.provideSmartWallet( - 'agoric1ldmtatp24qlllgxmrsjzcpe20fvlkp448zcuce', - ); - await walletFactoryDriver.provideSmartWallet( - 'agoric140dmkrz2e42ergjj7gyvejhzmjzurvqeq82ang', - ); - await walletFactoryDriver.provideSmartWallet( - 'agoric1w8wktaur4zf8qmmtn3n7x3r0jhsjkjntcm3u6h', - ); - - /** - * @param {number} i - * @param {number} n - * @param {number} r - */ - const openVault = async (i, n, r) => { - assert.typeof(i, 'number'); - assert.typeof(n, 'number'); - assert.typeof(r, 'number'); - - const offerId = `open-vault-${i}-of-${n}-round-${r}${dumpTag}`; - await wd.executeOfferMaker(Offers.vaults.OpenVault, { - offerId, - collateralBrandKey, - wantMinted: 5, - giveCollateral: 1.0, - }); - - t.like(wd.getLatestUpdateRecord(), { - updated: 'offerStatus', - status: { id: offerId, numWantsSatisfied: 1 }, - }); - }; - - /** - * @param {number} n - * @param {number} r - */ - const openN = async (n, r) => { - t.log(`opening ${n} vaults`); - const range = [...Array(n)].map((_, i) => i + 1); - performance.mark(`start-open`); - await Promise.all(range.map(i => openVault(i, n, r))); - performance.mark(`end-open`); - performance.measure(`open-${n}-round-${r}`, { - start: 'start-open', - end: 'end-open', - detail: { vaultsOpened: n, round: r }, - }); - }; - - // clear out for a baseline - await collectStats('start', dumpHeap); - // 10 is enough to compare retention in heaps - await openN(10, 1); - await collectStats('round1', dumpHeap); - await openN(10, 2); - const memStats = await collectStats('round2', dumpHeap); - - // let perfObserver get the last measurement - await eventLoopIteration(); - - const benchmarkReport = { - ...rows[1], - memStats, - name, - test: t.title, - where, - }; - fs.writeFileSync( - `benchmark-${name}.json`, - JSON.stringify(benchmarkReport, null, 2), - ); - - console.table(rows); -} - -// Note: it is probably not useful to enable both of the two following benchmark -// tests at the same time. Nothing bad per se will happen if you do, but it -// will take longer to run with no particular benefit resulting. However, if you run -// both you *must* run them serially, so that their executions don't get -// comingled and mess up the numbers. - -test.skip('stress vaults with heap snapshots', async t => { - await stressVaults(t, true); -}); - -test.serial('stress vaults', async t => { - await stressVaults(t, false); -}); diff --git a/packages/boot/test/bootstrapTests/liquidation.ts b/packages/boot/test/bootstrapTests/liquidation.ts index f2793746f9d..e7ec6ce39cc 100644 --- a/packages/boot/test/bootstrapTests/liquidation.ts +++ b/packages/boot/test/bootstrapTests/liquidation.ts @@ -8,13 +8,13 @@ import { makeAgoricNamesRemotesFromFakeStorage, } from '@agoric/vats/tools/board-utils.js'; import { Offers } from '@agoric/inter-protocol/src/clientSupport.js'; -import { ExecutionContext } from 'ava'; +import type { ExecutionContext } from 'ava'; +import { makeSwingsetTestKit } from '../../tools/supports.ts'; import { makeGovernanceDriver, makePriceFeedDriver, makeWalletFactoryDriver, -} from './drivers.ts'; -import { makeSwingsetTestKit } from './supports.ts'; +} from '../../tools/drivers.ts'; export type LiquidationSetup = { vaults: { diff --git a/packages/boot/test/bootstrapTests/test-addAssets.ts b/packages/boot/test/bootstrapTests/test-addAssets.ts index 4c13afa6f84..eecdaffe601 100644 --- a/packages/boot/test/bootstrapTests/test-addAssets.ts +++ b/packages/boot/test/bootstrapTests/test-addAssets.ts @@ -7,7 +7,7 @@ import { LiquidationTestContext, makeLiquidationTestContext, } from './liquidation.ts'; -import { makeProposalExtractor } from './supports.ts'; +import { makeProposalExtractor } from '../../tools/supports.ts'; const test = anyTest as TestFn< LiquidationTestContext & { diff --git a/packages/boot/test/bootstrapTests/test-demo-config.ts b/packages/boot/test/bootstrapTests/test-demo-config.ts index ba561ae30b7..f04317f8b6c 100644 --- a/packages/boot/test/bootstrapTests/test-demo-config.ts +++ b/packages/boot/test/bootstrapTests/test-demo-config.ts @@ -3,7 +3,7 @@ import { test as anyTest } from '@agoric/zoe/tools/prepare-test-env-ava.js'; import { PowerFlags } from '@agoric/vats/src/walletFlags.js'; import type { TestFn } from 'ava'; -import { makeSwingsetTestKit, keyArrayEqual } from './supports.ts'; +import { makeSwingsetTestKit, keyArrayEqual } from '../../tools/supports.ts'; const { keys } = Object; diff --git a/packages/boot/test/bootstrapTests/test-vats-restart.ts b/packages/boot/test/bootstrapTests/test-vats-restart.ts index bb24718cef5..c80a0eda69f 100644 --- a/packages/boot/test/bootstrapTests/test-vats-restart.ts +++ b/packages/boot/test/bootstrapTests/test-vats-restart.ts @@ -10,8 +10,11 @@ import { makeAgoricNamesRemotesFromFakeStorage } from '@agoric/vats/tools/board- import { TestFn } from 'ava'; import { BridgeHandler } from '@agoric/vats'; import type { EconomyBootstrapSpace } from '@agoric/inter-protocol/src/proposals/econ-behaviors.js'; -import { makeProposalExtractor, makeSwingsetTestKit } from './supports.ts'; -import { makeWalletFactoryDriver } from './drivers.ts'; +import { + makeProposalExtractor, + makeSwingsetTestKit, +} from '../../tools/supports.ts'; +import { makeWalletFactoryDriver } from '../../tools/drivers.ts'; const { Fail } = assert; diff --git a/packages/boot/test/bootstrapTests/test-vaults-integration.ts b/packages/boot/test/bootstrapTests/test-vaults-integration.ts index 53df2ad119f..eda3c165a98 100644 --- a/packages/boot/test/bootstrapTests/test-vaults-integration.ts +++ b/packages/boot/test/bootstrapTests/test-vaults-integration.ts @@ -13,8 +13,8 @@ import { } from '@agoric/vats/tools/board-utils.js'; import type { TestFn } from 'ava'; import { ParamChangesOfferArgs } from '@agoric/inter-protocol/src/econCommitteeCharter.js'; -import { makeWalletFactoryDriver } from './drivers.ts'; -import { makeSwingsetTestKit } from './supports.ts'; +import { makeSwingsetTestKit } from '../../tools/supports.ts'; +import { makeWalletFactoryDriver } from '../../tools/drivers.ts'; // presently all these tests use one collateral manager const collateralBrandKey = 'ATOM'; diff --git a/packages/boot/test/bootstrapTests/test-vaults-upgrade.ts b/packages/boot/test/bootstrapTests/test-vaults-upgrade.ts index 1b2d45f2e10..d88523ad44a 100644 --- a/packages/boot/test/bootstrapTests/test-vaults-upgrade.ts +++ b/packages/boot/test/bootstrapTests/test-vaults-upgrade.ts @@ -15,8 +15,8 @@ import { makeAgoricNamesRemotesFromFakeStorage } from '@agoric/vats/tools/board- import { ExecutionContext, TestFn } from 'ava'; import { FakeStorageKit } from '@agoric/internal/src/storage-test-utils.js'; import { EconomyBootstrapSpace } from '@agoric/inter-protocol/src/proposals/econ-behaviors.js'; -import { makeWalletFactoryDriver } from './drivers.ts'; -import { makeSwingsetTestKit } from './supports.ts'; +import { makeSwingsetTestKit } from '../../tools/supports.ts'; +import { makeWalletFactoryDriver } from '../../tools/drivers.ts'; // presently all these tests use one collateral manager const collateralBrandKey = 'ATOM'; diff --git a/packages/boot/test/bootstrapTests/test-zcf-upgrade.ts b/packages/boot/test/bootstrapTests/test-zcf-upgrade.ts index ac6a812bf12..ef7a1d9aa87 100644 --- a/packages/boot/test/bootstrapTests/test-zcf-upgrade.ts +++ b/packages/boot/test/bootstrapTests/test-zcf-upgrade.ts @@ -11,9 +11,12 @@ import path from 'path'; import { BridgeHandler } from '@agoric/vats'; import { makeAgoricNamesRemotesFromFakeStorage } from '@agoric/vats/tools/board-utils.js'; import { TestFn } from 'ava'; -import { makeZoeDriver } from './drivers.ts'; -import { matchAmount } from './supports.js'; -import { makeProposalExtractor, makeSwingsetTestKit } from './supports.ts'; +import { + matchAmount, + makeProposalExtractor, + makeSwingsetTestKit, +} from '../../tools/supports.ts'; +import { makeZoeDriver } from '../../tools/drivers.ts'; const filename = new URL(import.meta.url).pathname; const dirname = path.dirname(filename); diff --git a/packages/boot/test/upgrading/test-upgrade-vats.js b/packages/boot/test/upgrading/test-upgrade-vats.js index 0594f805426..2733bcd6c4b 100644 --- a/packages/boot/test/upgrading/test-upgrade-vats.js +++ b/packages/boot/test/upgrading/test-upgrade-vats.js @@ -3,13 +3,9 @@ import { test as anyTest } from '@agoric/swingset-vat/tools/prepare-test-env-ava import { BridgeId } from '@agoric/internal'; import { buildVatController } from '@agoric/swingset-vat'; +import { makeRunUtils } from '@agoric/swingset-vat/tools/run-utils.ts'; import { resolve as importMetaResolve } from 'import-meta-resolve'; -import { - makeRunUtils, - matchAmount, - matchIter, - matchRef, -} from '../bootstrapTests/supports.ts'; +import { matchAmount, matchIter, matchRef } from '../../tools/supports.ts'; /** * @type {import('ava').TestFn<{}>} diff --git a/packages/boot/test/bootstrapTests/drivers.ts b/packages/boot/tools/drivers.ts similarity index 98% rename from packages/boot/test/bootstrapTests/drivers.ts rename to packages/boot/tools/drivers.ts index 8b494a0af4a..0f667d36e82 100644 --- a/packages/boot/test/bootstrapTests/drivers.ts +++ b/packages/boot/tools/drivers.ts @@ -22,7 +22,8 @@ import type { WalletFactoryStartResult } from '@agoric/vats/src/core/startWallet import type { OfferSpec } from '@agoric/smart-wallet/src/offers.js'; import type { TimerService } from '@agoric/time/src/types.js'; import type { OfferMaker } from '@agoric/smart-wallet/src/types.js'; -import type { RunUtils, SwingsetTestKit } from './supports.ts'; +import type { RunUtils } from '@agoric/swingset-vat/tools/run-utils.ts'; +import type { SwingsetTestKit } from './supports.ts'; export const makeWalletFactoryDriver = async ( runUtils: RunUtils, diff --git a/packages/boot/test/bootstrapTests/supports.ts b/packages/boot/tools/supports.ts similarity index 79% rename from packages/boot/test/bootstrapTests/supports.ts rename to packages/boot/tools/supports.ts index b0cc8cf39b7..18bd4e1b138 100644 --- a/packages/boot/test/bootstrapTests/supports.ts +++ b/packages/boot/tools/supports.ts @@ -14,32 +14,21 @@ import { unmarshalFromVstorage } from '@agoric/internal/src/marshal.js'; import { makeFakeStorageKit } from '@agoric/internal/src/storage-test-utils.js'; import { initSwingStore } from '@agoric/swing-store'; import { loadSwingsetConfigFile } from '@agoric/swingset-vat'; -import { krefOf, kunser } from '@agoric/kmarshal'; +import { krefOf } from '@agoric/kmarshal'; +import { makeSlogSender } from '@agoric/telemetry'; import { TimeMath, Timestamp } from '@agoric/time'; +import '@agoric/vats/exported.js'; import { boardSlottingMarshaller, slotToBoardRemote, } from '@agoric/vats/tools/board-utils.js'; -import { makeQueue } from '@endo/stream'; -import type { SwingsetController } from '@agoric/swingset-vat/src/controller/controller.js'; -import type { BootstrapRootObject } from '@agoric/vats/src/core/lib-boot'; -import type { E } from '@endo/eventual-send'; import type { ExecutionContext as AvaT } from 'ava'; -const sink = () => {}; +import { makeRunUtils } from '@agoric/swingset-vat/tools/run-utils.ts'; const trace = makeTracer('BSTSupport', false); -export const bootstrapMethods: { [P in keyof BootstrapRootObject]: P } = { - bootstrap: 'bootstrap', - consumeItem: 'consumeItem', - produceItem: 'produceItem', - resetItem: 'resetItem', - awaitVatObject: 'awaitVatObject', - snapshotStore: 'snapshotStore', -}; - const keysToObject = ( keys: K[], valueMaker: (key: K, i: number) => V, @@ -62,105 +51,6 @@ export const keyArrayEqual = ( return t.deepEqual(aobj, bobj, message); }; -export const makeRunUtils = ( - controller: SwingsetController, - log = (..._) => {}, -) => { - let cranksRun = 0; - - const mutex = makeQueue(); - - mutex.put(controller.run()); - - const runThunk = async any>( - thunk: T, - ): Promise> => { - try { - // this promise for the last lock may fail - await mutex.get(); - } catch { - // noop because the result will resolve for the previous runMethod return - } - - const thunkResult = await thunk(); - - const result = controller.run().then(cranks => { - cranksRun += cranks; - log(`kernel ran ${cranks} cranks`); - return thunkResult; - }); - mutex.put(result.then(sink, sink)); - return result; - }; - - const queueAndRun = async (deliveryThunk, voidResult = false) => { - log('queueAndRun at', cranksRun); - - const kpid = await runThunk(deliveryThunk); - - if (voidResult) { - return undefined; - } - const status = controller.kpStatus(kpid); - switch (status) { - case 'fulfilled': - return kunser(controller.kpResolution(kpid)); - case 'rejected': - throw kunser(controller.kpResolution(kpid)); - case 'unresolved': - throw Error(`unresolved value for ${kpid}`); - default: - throw Fail`unknown status ${status}`; - } - }; - - type EVProxy = typeof E & { - sendOnly: (presence: unknown) => Record void>; - vat: (name: string) => Record Promise>; - }; - // @ts-expect-error cast, approximate - const EV: EVProxy = presence => - new Proxy(harden({}), { - get: (_t, method, _rx) => - harden((...args) => - queueAndRun(() => - controller.queueToVatObject(presence, method, args), - ), - ), - }); - EV.vat = vatName => - new Proxy(harden({}), { - get: (_t, method, _rx) => - harden((...args) => - queueAndRun(() => controller.queueToVatRoot(vatName, method, args)), - ), - }); - // @ts-expect-error xxx - EV.sendOnly = presence => - new Proxy(harden({}), { - get: (_t, method, _rx) => - harden((...args) => - queueAndRun( - () => controller.queueToVatObject(presence, method, args), - true, - ), - ), - }); - // @ts-expect-error xxx - EV.get = presence => - new Proxy(harden({}), { - get: (_t, pathElement, _rx) => - queueAndRun(() => - controller.queueToVatRoot('bootstrap', 'awaitVatObject', [ - presence, - [pathElement], - ]), - ), - }); - return harden({ runThunk, EV }); -}; -export type RunUtils = ReturnType; - /** * @param {string} bundleDir * @param {string} specifier @@ -213,7 +103,7 @@ interface Powers { export const makeProposalExtractor = ({ childProcess, fs }: Powers) => { const getPkgPath = (pkg, fileName = '') => - new URL(`../../../${pkg}/${fileName}`, import.meta.url).pathname; + new URL(`../../${pkg}/${fileName}`, import.meta.url).pathname; const importSpec = spec => importMetaResolve(spec, import.meta.url).then(u => new URL(u).pathname); @@ -353,6 +243,7 @@ export const matchIter = (t: AvaT, iter, valueRef) => { * @param [options.configSpecifier] bootstrap config specifier * @param [options.storage] * @param [options.verbose] + * @param [options.slogFile] * @param [options.defaultManagerType] */ export const makeSwingsetTestKit = async ( @@ -362,6 +253,7 @@ export const makeSwingsetTestKit = async ( configSpecifier = undefined as string | undefined, storage = makeFakeStorageKit('bootstrapTests'), verbose = false, + slogFile = undefined as string | undefined, defaultManagerType = 'local' as ManagerType, } = {}, ) => { @@ -453,6 +345,17 @@ export const makeSwingsetTestKit = async ( } }; + let slogSender; + if (slogFile) { + slogSender = await makeSlogSender({ + stateDir: '.', + env: { + ...process.env, + SLOGFILE: slogFile, + SLOGSENDER: '', + }, + }); + } const { controller, timer } = await buildSwingset( new Map(), bridgeOutbound, @@ -460,7 +363,7 @@ export const makeSwingsetTestKit = async ( configPath, [], {}, - { debugName: 'TESTBOOT', verbose }, + { debugName: 'TESTBOOT', verbose, slogSender }, ); console.timeLog('makeBaseSwingsetTestKit', 'buildSwingset');