fix(swingset): replay dynamic vats properly

This saves the dynamic vat data (source bundle or bundleName, dynamicOptions)
to the kernel state DB when the vat is first created. Then, during replay, we
reload the dynamic vat from that data.

This refactors dynamic vat creation slightly, to merge create-by-bundle and
create-by-bundle-name into a single function that takes a `source` argument
which contains either `{ bundle }` or `{ bundleName }`. By passing `source`
to `createVatDynamically`, rather than first converting it into a full
bundle, we have enough information to store the much smaller bundleName in
the DB. Userspace code still sees the same API (`vatAdminSvc~.createVat(bundle)`
or `vatAdminSvc~.createVatByName(bundleName)`), but the kernel internals
and the vat-admin device endowments have changed.

We add a new `vat.dynamicIDs` entry to the kernelKeeper, which contains a
single JSON-encoded Array of dynamic vatIDs. This isn't great, but it's
expedient, and our DB doesn't really support enumerating anything more clever
anyways. We also add a pair of keys to the vatKeeper for each dynamic vat:
`v$NN.source` and `v$NN.options`, to record the bundle/bundleName and
dynamicOptions that were used the first time around, so we can use them on
subsequent calls too.

`createVatDynamically` normally sends a message (to the vatAdminVat) when the
new vat is ready, so it can inform the original caller and give them the new
root object. We suppress this message when reloading the kernel, because it
was already delivered the first time around. One side-effect is that failures
to reload the dynamic vat must panic the kernel, because there is nobody
ready to report the error to. This could happen if the new host application
failed to configure the same source bundles as the previous one did. At some
point we'll want to store all these bundles in the DB, which should remove
this failure mode.

closes #1480

packages/SwingSet/src/kernel/dynamicVat.js

@@ -1,3 +1,4 @@
+/* global harden */
 import { assertKnownOptions } from '../assertOptions';
 import { makeVatSlot } from '../parseVatSlots';
 
@@ -13,34 +14,56 @@ export function makeDynamicVatCreator(stuff) {
     addVatManager,
     addExport,
     queueToExport,
+    getBundle,
+    kernelKeeper,
+    panic,
   } = stuff;
 
   /** A function to be called from the vatAdmin device to create a new vat. It
    * creates the vat and sends a notification to the device. The root object
    * will be available soon, but we immediately return the vatID so the ultimate
    * requestor doesn't have to wait.
    *
-   * @param vatSourceBundle a source bundle (JSON-serializable data) which
-   * defines the vat. This should be generated by calling bundle-source on a
-   * module whose default export is makeRootObject(), which takes E as a
-   * parameter and returns a root object.
-   * @param options an options bundle. The only option defined so far is
-   * 'metered', which defaults to 'true'. If 'true', the new dynamic vat is
-   * subject to a meter that limits the amount of computation and allocation
-   * that can occur during any given crank. Stack frames are limited as well.
-   * The meter is refilled between cranks, but if the meter ever underflows,
-   * the vat is terminated. If 'false', the vat is unmetered.
+   * @param source an object which either has a `bundle` (JSON-serializable
+   * data) or a `bundleName` string. The bundle defines the vat, and should
+   * be generated by calling bundle-source on a module with an export named
+   * makeRootObject(). If `bundleName` is used, it must identify a bundle
+   * already known to the kernel (via the `config.bundles` table).
+   * @param options a "dynamicOptions" bundle. This recognizes two options so
+   * far. The first is 'metered' (defaults to 'true') which subjects the new
+   * dynamic vat to a meter that limits the amount of computation and
+   * allocation that can occur during any given crank. Stack frames are
+   * limited as well. The meter is refilled between cranks, but if the meter
+   * ever underflows, the vat is terminated. If 'false', the vat is
+   * unmetered. The other is 'vatParameters', which provides the contents of
+   * the second argument to 'buildRootObject()'
    *
    * @return { vatID } the vatID for a newly created vat. The success or
    * failure of the operation will be reported in a message to the admin vat,
    * citing this vatID
    */
 
-  function createVatDynamically(vatSourceBundle, dynamicOptions = {}) {
+  function createVatDynamically(source, dynamicOptions = {}) {
+    const vatID = allocateUnusedVatID();
+    kernelKeeper.addDynamicVatID(vatID);
+    const vatKeeper = kernelKeeper.allocateVatKeeperIfNeeded(vatID);
+    vatKeeper.setSourceAndOptions(source, dynamicOptions);
+    // eslint-disable-next-line no-use-before-define
+    return create(vatID, source, dynamicOptions, true);
+  }
+
+  function recreateVatDynamically(vatID, source, dynamicOptions) {
+    // eslint-disable-next-line no-use-before-define
+    return create(vatID, source, dynamicOptions, false);
+  }
+
+  function create(vatID, source, dynamicOptions, notifyNewVat) {
+    const vatSourceBundle = source.bundle || getBundle(source.bundleName);
+    if (!vatSourceBundle) {
+      throw Error(`Bundle ${source.bundleName} not found`);
+    }
     assertKnownOptions(dynamicOptions, ['metered', 'vatParameters']);
     const { metered = true, vatParameters = {} } = dynamicOptions;
-
-    const vatID = allocateUnusedVatID();
     let terminated = false;
 
     function notifyTermination(error) {
@@ -104,13 +127,21 @@ export function makeDynamicVatCreator(stuff) {
     }
 
     function makeErrorResponse(error) {
+      if (!notifyNewVat) {
+        // if we fail to recreate the vat during replay, crash the kernel,
+        // because we can no longer inform the original caller
+        panic(`unable to re-create dynamic vat ${vatID}`);
+      }
       return {
         body: JSON.stringify([vatID, { error: `${error}` }]),
         slots: [],
       };
     }
 
     function sendResponse(args) {
+      if (!notifyNewVat) {
+        return;
+      }
       const vatAdminVatId = vatNameToID('vatAdmin');
       const vatAdminRootObjectSlot = makeVatRootObjectSlot();
       queueToExport(
@@ -142,5 +173,5 @@ export function makeDynamicVatCreator(stuff) {
     return vatID;
   }
 
-  return createVatDynamically;
+  return harden({ createVatDynamically, recreateVatDynamically });
 }

packages/SwingSet/src/kernel/kernel.js

@@ -646,13 +646,33 @@ export default function buildKernel(kernelEndowments) {
     manager.setVatSyscallHandler(vatSyscallHandler);
   }
 
-  const createVatDynamically = makeDynamicVatCreator({
+  const knownBundles = new Map();
+
+  function hasBundle(name) {
+    return knownBundles.has(name);
+  }
+
+  function getBundle(name) {
+    return knownBundles.get(name);
+  }
+
+  function addBundle(name, bundle) {
+    knownBundles.set(name, bundle);
+  }
+
+  const {
+    createVatDynamically,
+    recreateVatDynamically,
+  } = makeDynamicVatCreator({
     allocateUnusedVatID: kernelKeeper.allocateUnusedVatID,
     vatNameToID,
     vatManagerFactory,
     addVatManager,
     addExport,
     queueToExport,
+    getBundle,
+    kernelKeeper,
+    panic,
   });
 
   function buildDeviceManager(deviceID, name, buildRootDeviceNode, endowments) {
@@ -700,20 +720,6 @@ export default function buildKernel(kernelEndowments) {
     return vatKeeper.vatStats();
   }
 
-  const knownBundles = new Map();
-
-  function hasBundle(name) {
-    return knownBundles.has(name);
-  }
-
-  function getBundle(name) {
-    return knownBundles.get(name);
-  }
-
-  function addBundle(name, bundle) {
-    knownBundles.set(name, bundle);
-  }
-
   async function start(bootstrapVatName) {
     if (started) {
       throw new Error('kernel.start already called');
@@ -727,7 +733,7 @@ export default function buildKernel(kernelEndowments) {
       kernelKeeper.createStartingKernelState();
     }
 
-    // instantiate all vats
+    // instantiate all genesis vats
     for (const name of genesisVats.keys()) {
       const managerOptions = genesisVats.get(name);
       const vatID = kernelKeeper.allocateVatIDForNameIfNeeded(name);
@@ -737,20 +743,23 @@ export default function buildKernel(kernelEndowments) {
       addVatManager(vatID, manager, managerOptions);
     }
 
-    function createVatDynamicallyByName(bundleName, options) {
-      const bundle = getBundle(bundleName);
-      if (bundle) {
-        return createVatDynamically(bundle, options);
-      } else {
-        throw Error(`Bundle ${bundleName} not found`);
-      }
+    // instantiate all dynamic vats
+    for (const vatID of kernelKeeper.getAllDynamicVatIDs()) {
+      console.debug(`Loading dynamic vat ${vatID}`);
+      const vatKeeper = kernelKeeper.allocateVatKeeperIfNeeded(vatID);
+      const {
+        source,
+        options: dynamicOptions,
+      } = vatKeeper.getSourceAndOptions();
+      // eslint-disable-next-line no-await-in-loop
+      await recreateVatDynamically(vatID, source, dynamicOptions);
+      // now the vatManager is attached and ready for transcript replay
     }
 
     if (vatAdminDeviceBundle) {
       // if we have a device bundle, then vats[vatAdmin] will be present too
       const endowments = {
         create: createVatDynamically,
-        createByName: createVatDynamicallyByName,
         stats: collectVatStats,
         /* TODO: terminate */
       };
@@ -803,12 +812,11 @@ export default function buildKernel(kernelEndowments) {
         // eslint-disable-next-line no-await-in-loop
         await vat.manager.replayTranscript();
         console.debug(`finished replaying vatID ${vatID} transcript `);
-      }
-      const newLength = kernelKeeper.getRunQueueLength();
-      if (newLength !== oldLength) {
-        throw new Error(
-          `replayTranscript added run-queue entries, wasn't supposed to`,
-        );
+        const newLength = kernelKeeper.getRunQueueLength();
+        if (newLength !== oldLength) {
+          console.log(`SPURIOUS RUNQUEUE`, kernelKeeper.dump().runQueue);
+          throw Error(`replay ${vatID} added spurious run-queue entries`);
+        }
       }
       kernelKeeper.loadStats();
     }

packages/SwingSet/src/kernel/state/kernelKeeper.js

@@ -31,12 +31,17 @@ const enableKernelPromiseGC = true;
 // The schema is:
 //
 // vat.names = JSON([names..])
+// vat.dynamicIDs = JSON([vatIDs..])
 // vat.name.$NAME = $vatID = v$NN
 // vat.nextID = $NN
 // device.names = JSON([names..])
 // device.name.$NAME = $deviceID = d$NN
 // device.nextID = $NN
 
+// dynamic vats have these too:
+// v$NN.source = JSON({ bundle }) or JSON({ bundleName })
+// v$NN.options = JSON
+
 // v$NN.o.nextID = $NN
 // v$NN.p.nextID = $NN
 // v$NN.d.nextID = $NN
@@ -221,6 +226,7 @@ export default function makeKernelKeeper(storage) {
 
   function createStartingKernelState() {
     storage.set('vat.names', '[]');
+    storage.set('vat.dynamicIDs', '[]');
     storage.set('vat.nextID', JSON.stringify(FIRST_VAT_ID));
     storage.set('device.names', '[]');
     storage.set('device.nextID', JSON.stringify(FIRST_DEVICE_ID));
@@ -498,6 +504,18 @@ export default function makeKernelKeeper(storage) {
     return storage.get(k);
   }
 
+  function addDynamicVatID(vatID) {
+    assert.typeof(vatID, 'string');
+    const KEY = 'vat.dynamicIDs';
+    const dynamicVatIDs = JSON.parse(getRequired(KEY));
+    dynamicVatIDs.push(vatID);
+    storage.set(KEY, JSON.stringify(dynamicVatIDs));
+  }
+
+  function getAllDynamicVatIDs() {
+    return JSON.parse(getRequired('vat.dynamicIDs'));
+  }
+
   const deadKernelPromises = new Set();
 
   /**
@@ -769,6 +787,8 @@ export default function makeKernelKeeper(storage) {
     allocateUnusedVatID,
     allocateVatKeeperIfNeeded,
     getAllVatNames,
+    addDynamicVatID,
+    getAllDynamicVatIDs,
 
     getDeviceIDForName,
     allocateDeviceIDForNameIfNeeded,

packages/SwingSet/src/kernel/state/vatKeeper.js

@@ -58,6 +58,20 @@ export function makeVatKeeper(
 ) {
   insistVatID(vatID);
 
+  function setSourceAndOptions(source, options) {
+    assert.typeof(source, 'object');
+    assert(source.bundle || source.bundleName);
+    assert.typeof(options, 'object');
+    storage.set(`${vatID}.source`, JSON.stringify(source));
+    storage.set(`${vatID}.options`, JSON.stringify(options));
+  }
+
+  function getSourceAndOptions() {
+    const source = JSON.parse(storage.get(`${vatID}.source`));
+    const options = JSON.parse(storage.get(`${vatID}.options`));
+    return harden({ source, options });
+  }
+
   /**
    * Provide the kernel slot corresponding to a given vat slot, creating the
    * kernel slot if it doesn't already exist.
@@ -225,6 +239,8 @@ export function makeVatKeeper(
   }
 
   return harden({
+    setSourceAndOptions,
+    getSourceAndOptions,
     mapVatSlotToKernelSlot,
     mapKernelSlotToVatSlot,
     deleteCListEntry,

packages/SwingSet/src/kernel/vatAdmin/vatAdmin-src.js

@@ -17,7 +17,6 @@
 export function buildRootDeviceNode({ endowments }) {
   const {
     create: kernelVatCreationFn,
-    createByName: kernelVatCreationByNameFn,
     stats: kernelVatStatsFn,
     // terminate: kernelTerminateFn,
   } = endowments;
@@ -28,11 +27,11 @@ export function buildRootDeviceNode({ endowments }) {
     // kernel's vat creator fn. Remember that the root object will arrive
     // separately. Clean up the outgoing and incoming arguments.
     create(bundle, options) {
-      const vatID = kernelVatCreationFn(bundle, options);
+      const vatID = kernelVatCreationFn({ bundle }, options);
       return vatID;
     },
     createByName(bundleName, options) {
-      const vatID = kernelVatCreationByNameFn(bundleName, options);
+      const vatID = kernelVatCreationFn({ bundleName }, options);
       return vatID;
     },
     terminate(_vatID) {

packages/SwingSet/test/test-state.js

@@ -273,6 +273,7 @@ test('kernel state', async t => {
     ['runQueue', '[]'],
     ['vat.nextID', '1'],
     ['vat.names', '[]'],
+    ['vat.dynamicIDs', '[]'],
     ['device.names', '[]'],
     ['device.nextID', '7'],
     ['ko.nextID', '20'],
@@ -298,6 +299,7 @@ test('kernelKeeper vat names', async t => {
     ['runQueue', '[]'],
     ['vat.nextID', '3'],
     ['vat.names', JSON.stringify(['vatname5', 'Frank'])],
+    ['vat.dynamicIDs', '[]'],
     ['device.names', '[]'],
     ['device.nextID', '7'],
     ['ko.nextID', '20'],
@@ -333,6 +335,7 @@ test('kernelKeeper device names', async t => {
     ['runQueue', '[]'],
     ['vat.nextID', '1'],
     ['vat.names', '[]'],
+    ['vat.dynamicIDs', '[]'],
     ['device.nextID', '9'],
     ['device.names', JSON.stringify(['devicename5', 'Frank'])],
     ['ko.nextID', '20'],
@@ -482,6 +485,7 @@ test('kernelKeeper promises', async t => {
     ['device.nextID', '7'],
     ['vat.nextID', '1'],
     ['vat.names', '[]'],
+    ['vat.dynamicIDs', '[]'],
     ['device.names', '[]'],
     ['runQueue', '[]'],
     ['kd.nextID', '30'],