Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pre-bundle to speed up swingset unit tests #1644

Merged
merged 6 commits into from
Aug 30, 2020
Merged

pre-bundle to speed up swingset unit tests #1644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8c0cc8b
feat: allow pre-built kernelBundles for faster unit tests
warner Aug 30, 2020
36496ab
fix: don't modify the original 'config' object
warner Aug 30, 2020
2c9125f
test: speed up test-message-patterns.js with buildKernelBundles
warner Aug 30, 2020
88d82b4
test: speed up test-promises.js with buildKernelBundles
warner Aug 30, 2020
45d7299
test: speed up test-devices.js by using buildKernelBundles()
warner Aug 30, 2020
faa8be3
test: speed up test-terminate.js with buildKernelBundles
warner Aug 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 48 additions & 28 deletions packages/SwingSet/src/controller.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,6 @@ function makeConsole(tag) {
return harden(cons);
}

const ADMIN_DEVICE_PATH = require.resolve('./kernel/vatAdmin/vatAdmin-src');
const ADMIN_VAT_PATH = require.resolve('./kernel/vatAdmin/vatAdminWrapper');
const KERNEL_SOURCE_PATH = require.resolve('./kernel/kernel.js');

function byName(a, b) {
if (a.name < b.name) {
return -1;
Expand Down Expand Up @@ -205,33 +201,66 @@ export function loadSwingsetConfigFile(configPath) {
}
}

/**
* Build the kernel source bundles.
*
*/
export async function buildKernelBundles() {
// this takes 2.7s on my computer
const sources = {
kernel: require.resolve('./kernel/kernel.js'),
adminDevice: require.resolve('./kernel/vatAdmin/vatAdmin-src'),
adminVat: require.resolve('./kernel/vatAdmin/vatAdminWrapper'),
comms: require.resolve('./vats/comms'),
vattp: require.resolve('./vats/vat-tp'),
timer: require.resolve('./vats/vat-timerWrapper'),
};
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not obviously a use case we have an immediate need to support, but it strikes me that we've got most of the building blocks here to allow kernels to be built without some of this stuff in cases where the swingset being launched doesn't need them.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, #1351 is an opportunity to set up devices a bit more coherently, in which case we could defer adding the timer vat until someone gives us a timer device to work with. OTOH #1346 might manage to make devices sufficiently vat-like that we don't need wrappers anymore and that one goes away.

I'm inclined to think that comms is a sufficiently core part of the swingset experience ("what's life without IO?") that it's probably always going to be there, but yeah, who knows.

const kernelBundles = {};
for (const name of Object.keys(sources)) {
// this was harder to read with Promise.all
// eslint-disable-next-line no-await-in-loop
kernelBundles[name] = await bundleSource(sources[name]);
}
return harden(kernelBundles);
}

export async function buildVatController(
config,
argv = [],
runtimeOptions = {},
) {
const { debugPrefix = '', verbose = false } = runtimeOptions;
// build console early so we can add console.log to diagnose early problems
const { debugPrefix = '' } = runtimeOptions;
if (typeof Compartment === 'undefined') {
throw Error('SES must be installed before calling buildVatController');
}

// eslint-disable-next-line no-shadow
const console = makeConsole(`${debugPrefix}SwingSet:controller`);
// We can harden this 'console' because it's new, but if we were using the
// original 'console' object (which has a unique prototype), we'd have to
// harden(Object.getPrototypeOf(console));
// see https://github.com/Agoric/SES-shim/issues/292 for details
harden(console);

const {
verbose = false,
kernelBundles = await buildKernelBundles(),
} = runtimeOptions;

// FIXME: Put this somewhere better.
process.on('unhandledRejection', e =>
console.error('UnhandledPromiseRejectionWarning:', e),
);

// https://github.com/Agoric/SES-shim/issues/292
harden(Object.getPrototypeOf(console));
harden(console);

if (config.bootstrap && argv) {
if (!config.vats[config.bootstrap].parameters) {
config.vats[config.bootstrap].parameters = {};
}
config.vats[config.bootstrap].parameters.argv = argv;
// move 'argv' into parameters on the bootstrap vat, without changing the
// original config (which might be hardened or shared)
const bootstrapName = config.bootstrap;
const parameters = { ...config.vats[bootstrapName].parameters, argv };
const bootstrapVat = { ...config.vats[bootstrapName], parameters };
const vats = { ...config.vats, [bootstrapName]: bootstrapVat };
config = { ...config, vats };
}

function kernelRequire(what) {
Expand All @@ -248,8 +277,7 @@ export async function buildVatController(
throw Error(`kernelRequire unprepared to satisfy require(${what})`);
}
}
const kernelSource = await bundleSource(KERNEL_SOURCE_PATH);
const kernelNS = await importBundle(kernelSource, {
const kernelNS = await importBundle(kernelBundles.kernel, {
filePrefix: 'kernel',
endowments: {
console: makeConsole(`${debugPrefix}SwingSet:kernel`),
Expand Down Expand Up @@ -351,19 +379,15 @@ export async function buildVatController(
}

// the vatAdminDevice is given endowments by the kernel itself
const vatAdminVatBundle = await bundleSource(ADMIN_VAT_PATH);
kernel.addGenesisVat('vatAdmin', vatAdminVatBundle);
const vatAdminDeviceBundle = await bundleSource(ADMIN_DEVICE_PATH);
kernel.addVatAdminDevice(vatAdminDeviceBundle);
kernel.addGenesisVat('vatAdmin', kernelBundles.adminVat);
kernel.addVatAdminDevice(kernelBundles.adminDevice);

// comms vat is added automatically, but TODO: bootstraps must still
// connect it to vat-tp. TODO: test-message-patterns builds two comms and
// two vattps, must handle somehow.
const commsVatSourcePath = require.resolve('./vats/comms');
const commsVatBundle = await bundleSource(commsVatSourcePath);
kernel.addGenesisVat(
'comms',
commsVatBundle,
kernelBundles.comms,
{},
{
enablePipelining: true,
Expand All @@ -373,15 +397,11 @@ export async function buildVatController(

// vat-tp is added automatically, but TODO: bootstraps must still connect
// it to comms
const vatTPSourcePath = require.resolve('./vats/vat-tp');
const vatTPBundle = await bundleSource(vatTPSourcePath);
kernel.addGenesisVat('vattp', vatTPBundle);
kernel.addGenesisVat('vattp', kernelBundles.vattp);

// timer wrapper vat is added automatically, but TODO: bootstraps must
// still provide a timer device, and connect it to the wrapper vat
const timerWrapperSourcePath = require.resolve('./vats/vat-timerWrapper');
const timerWrapperBundle = await bundleSource(timerWrapperSourcePath);
kernel.addGenesisVat('timer', timerWrapperBundle);
kernel.addGenesisVat('timer', kernelBundles.timer);

function addGenesisVat(
name,
Expand Down
1 change: 1 addition & 0 deletions packages/SwingSet/src/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ export {
loadBasedir,
loadSwingsetConfigFile,
buildVatController,
buildKernelBundles,
} from './controller';

export { buildMailboxStateMap, buildMailbox } from './devices/mailbox';
Expand Down
1 change: 0 additions & 1 deletion packages/SwingSet/test/basedir-message-patterns/vat-a.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ export function buildRootObject(vatPowers) {
},

async run(which) {
console.log(`running alice[${which}]`);
await alice[which]();
},
});
Expand Down
22 changes: 14 additions & 8 deletions packages/SwingSet/test/test-devices.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import '@agoric/install-ses';
import test from 'ava';
import { initSwingStore, getAllState } from '@agoric/swing-store-simple';

import { buildVatController } from '../src/index';
import { buildVatController, buildKernelBundles } from '../src/index';
import { buildMailboxStateMap, buildMailbox } from '../src/devices/mailbox';
import buildCommand from '../src/devices/command';

Expand All @@ -16,6 +16,11 @@ function capargs(args, slots = []) {
return capdata(JSON.stringify(args), slots);
}

test.before(async t => {
const kernelBundles = await buildKernelBundles();
t.context.data = { kernelBundles };
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is t.context something provided by Ava or are we just doing this and hoping it will be safe?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's from AVA: https://github.com/avajs/ava/blob/master/docs/01-writing-tests.md#test-context

});

test('d0', async t => {
const config = {
bootstrap: 'bootstrap',
Expand All @@ -27,7 +32,7 @@ test('d0', async t => {
},
devices: [['d0', require.resolve('./files-devices/device-0'), {}]],
};
const c = await buildVatController(config);
const c = await buildVatController(config, [], t.context.data);
await c.step();
// console.log(util.inspect(c.dump(), { depth: null }));
t.deepEqual(JSON.parse(c.dump().log[0]), [
Expand Down Expand Up @@ -75,7 +80,7 @@ test('d1', async t => {
],
],
};
const c = await buildVatController(config);
const c = await buildVatController(config, [], t.context.data);
await c.step();
c.queueToVatExport('bootstrap', 'o+0', 'step1', capargs([]));
await c.step();
Expand All @@ -100,7 +105,7 @@ async function test2(t, mode) {
},
devices: [['d2', require.resolve('./files-devices/device-2'), {}]],
};
const c = await buildVatController(config, [mode]);
const c = await buildVatController(config, [mode], t.context.data);
await c.step();
if (mode === '1') {
t.deepEqual(c.dump().log, ['calling d2.method1', 'method1 hello', 'done']);
Expand Down Expand Up @@ -186,6 +191,7 @@ test('device state', async t => {
// from bootstrap, and read it back.
const c1 = await buildVatController(config, ['write+read'], {
hostStorage: storage,
kernelBundles: t.context.data.kernelBundles,
});
const d3 = c1.deviceNameToID('d3');
await c1.run();
Expand All @@ -208,7 +214,7 @@ test('mailbox outbound', async t => {
devices: [['mailbox', mb.srcPath, mb.endowments]],
};

const c = await buildVatController(config, ['mailbox1']);
const c = await buildVatController(config, ['mailbox1'], t.context.data);
await c.run();
t.deepEqual(s.exportToData(), {
peer1: {
Expand Down Expand Up @@ -248,7 +254,7 @@ test('mailbox inbound', async t => {

let rc;

const c = await buildVatController(config, ['mailbox2']);
const c = await buildVatController(config, ['mailbox2'], t.context.data);
await c.run();
rc = mb.deliverInbound(
'peer1',
Expand Down Expand Up @@ -361,7 +367,7 @@ test('command broadcast', async t => {
devices: [['command', cm.srcPath, cm.endowments]],
};

const c = await buildVatController(config, ['command1']);
const c = await buildVatController(config, ['command1'], t.context.data);
await c.run();
t.deepEqual(broadcasts, [{ hello: 'everybody' }]);
});
Expand All @@ -378,7 +384,7 @@ test('command deliver', async t => {
devices: [['command', cm.srcPath, cm.endowments]],
};

const c = await buildVatController(config, ['command2']);
const c = await buildVatController(config, ['command2'], t.context.data);
await c.run();

t.deepEqual(c.dump().log.length, 0);
Expand Down
93 changes: 54 additions & 39 deletions packages/SwingSet/test/test-message-patterns.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
import '@agoric/install-ses';
import test from 'ava';
import path from 'path';
import { buildVatController, loadBasedir } from '../src/index';
import bundleSource from '@agoric/bundle-source';
import { buildVatController, buildKernelBundles } from '../src/index';
import { buildLoopbox } from '../src/devices/loopbox';
import { buildPatterns } from './message-patterns';

Expand Down Expand Up @@ -38,13 +39,52 @@ async function runWithTrace(c) {
}
}

export async function runVatsLocally(t, name) {
console.log(`------ testing pattern (local) -- ${name}`);
test.before(async t => {
const kernelBundles = await buildKernelBundles();
const bdir = path.resolve(__dirname, 'basedir-message-patterns');
const config = await loadBasedir(bdir);
config.bootstrap = 'bootstrap';
config.vats.bootstrap = { sourceSpec: path.join(bdir, 'bootstrap-local.js') };
const c = await buildVatController(config, [name]);
const bundleA = await bundleSource(path.resolve(bdir, 'vat-a.js'));
const bundleB = await bundleSource(path.resolve(bdir, 'vat-b.js'));

const bootstrapLocal = path.resolve(bdir, 'bootstrap-local.js');
const bundleLocal = await bundleSource(bootstrapLocal);
const localConfig = {
bootstrap: 'bootstrap',
vats: {
bootstrap: { bundle: bundleLocal },
a: { bundle: bundleA },
b: { bundle: bundleB },
},
};

const bootstrapComms = path.resolve(bdir, 'bootstrap-comms.js');
const bundleComms = await bundleSource(bootstrapComms);
const moreComms = {
bundle: kernelBundles.comms,
creationOptions: {
enablePipelining: true,
enableSetup: true,
},
};
const moreVatTP = { bundle: kernelBundles.vattp };
const commsConfig = {
bootstrap: 'bootstrap',
vats: {
bootstrap: { bundle: bundleComms },
a: { bundle: bundleA },
b: { bundle: bundleB },
leftcomms: moreComms,
rightcomms: moreComms,
leftvattp: moreVatTP,
rightvattp: moreVatTP,
},
};

t.context.data = { localConfig, commsConfig, kernelBundles };
});

export async function runVatsLocally(t, name) {
const { localConfig: config, kernelBundles } = t.context.data;
const c = await buildVatController(config, [name], { kernelBundles });
// await runWithTrace(c);
await c.run();
return c.dump().log;
Expand All @@ -65,51 +105,26 @@ for (const name of Array.from(bp.patterns.keys()).sort()) {
test.serial('local patterns', testLocalPattern, name);
}

const commsSourcePath = require.resolve('../src/vats/comms');
const vatTPSourcePath = require.resolve('../src/vats/vat-tp');

export async function runVatsInComms(t, enablePipelining, name) {
console.log(`------ testing pattern (comms) -- ${name}`);
const enableSetup = true;
const bdir = path.resolve(__dirname, 'basedir-message-patterns');
const config = await loadBasedir(bdir);
config.bootstrap = 'bootstrap';
config.vats.bootstrap = { sourceSpec: path.join(bdir, 'bootstrap-comms.js') };
config.vats.leftcomms = {
sourceSpec: commsSourcePath,
creationOptions: {
enablePipelining,
enableSetup,
},
};
config.vats.rightcomms = {
sourceSpec: commsSourcePath,
creationOptions: {
enablePipelining,
enableSetup,
},
};
config.vats.leftvattp = { sourceSpec: vatTPSourcePath };
config.vats.rightvattp = { sourceSpec: vatTPSourcePath };
export async function runVatsInComms(t, name) {
const { commsConfig, kernelBundles } = t.context.data;
const { passOneMessage, loopboxSrcPath, loopboxEndowments } = buildLoopbox(
'queued',
);
config.devices = [['loopbox', loopboxSrcPath, loopboxEndowments]];
const c = await buildVatController(config, [name]);
const devices = [['loopbox', loopboxSrcPath, loopboxEndowments]];
const config = { ...commsConfig, devices };
const c = await buildVatController(config, [name], { kernelBundles });
// await runWithTrace(c);
await c.run();
while (passOneMessage()) {
await c.run();
}
console.log(`bootstrapResult`, c.bootstrapResult.status());
return c.dump().log;
}

async function testCommsPattern(t, name) {
const enablePipelining = true;
const logs = await runVatsInComms(t, enablePipelining, name);
const logs = await runVatsInComms(t, name);
let expected;
if (enablePipelining && name in bp.expected_pipelined) {
if (name in bp.expected_pipelined) {
expected = bp.expected_pipelined[name];
} else {
expected = bp.expected[name];
Expand Down
Loading