fix(swingset): abandon/delete most non-durables during stopVat()

This deletes most non-durable data during upgrade. stopVat() delegates
to a new function `releaseOldState()`, which makes an incomplete
effort to drop everything.

The portions which are complete are:

* find all locally-decided promises and rejects them
* find all exported Remotables and virtual objects, and abandons them
* simulate finalizers for all in-RAM Presences and Representatives
* use collectionManager to delete all virtual collections
* perform a bringOutYourDead to clean up resulting dead references

After that, `deleteVirtualObjectsWithoutDecref` walks the vatstore and
deletes the data from all virtual objects, without attempting to
decref the things they pointed to. This fails to release durables and
imports which were referenced by those virtual objects (e.g. cycles
that escaped the earlier purge).

Code is written, but not yet complete, to decref those objects
properly. A later update to this file will activate that (and update
the tests to confirm it works).

The new unit test constructs a large object graph and examines it
afterwards to make sure everything was deleted appropriately. The test
knows about the limitations of `deleteVirtualObjectsWithoutDecref`, as
well as bug #5053 which causes some other objects to be retained
incorrectly.

The collectionManager was changed to keep an in-RAM set of the vrefs
for all collections, both virtual and durable. We need the virtuals to
implement `deleteAllVirtualCollections` because there's no efficient
way to enumerate them from the vatstore entries, and the code is a lot
simpler if I just track all of them. We also need the Set to tolerate
duplicate deletion attempts: `deleteAllVirtualCollections` runs first,
but just afterwards a `bringOutYourDead` might notice a zero refcount
on a virtual collection and attempt to delete it a second time. We
cannot keep this Set in RAM: if we have a very large number of
collections, it violates our RAM budget, so we need to change our DB
structure to accomodate this need (#5058).

refs #1848

packages/SwingSet/src/liveslots/collectionManager.js

@@ -31,6 +31,12 @@ export function makeCollectionManager(
   serialize,
   unserialize,
 ) {
+  // TODO(#5058): we hold a list of all collections (both virtual and
+  // durable) in RAM, so we can delete the virtual ones during
+  // stopVat(), and tolerate subsequent GC-triggered duplication
+  // deletion without crashing. This needs to move to the DB to avoid
+  // the RAM consumption of a large number of collections.
+  const allCollectionObjIDs = new Set();
   const storeKindIDToName = new Map();
 
   const storeKindInfo = {
@@ -566,9 +572,13 @@ export function makeCollectionManager(
   }
 
   function deleteCollection(vobjID) {
+    if (!allCollectionObjIDs.has(vobjID)) {
+      return false; // already deleted
+    }
     const { id, subid } = parseVatSlot(vobjID);
     const kindName = storeKindIDToName.get(`${id}`);
     const collection = summonCollectionInternal(false, 'GC', subid, kindName);
+    allCollectionObjIDs.delete(vobjID);
 
     const doMoreGC = collection.clearInternal(true);
     let priorKey = '';
@@ -583,6 +593,18 @@ export function makeCollectionManager(
     return doMoreGC;
   }
 
+  function deleteAllVirtualCollections() {
+    const vobjIDs = Array.from(allCollectionObjIDs).sort();
+    for (const vobjID of vobjIDs) {
+      const { id } = parseVatSlot(vobjID);
+      const kindName = storeKindIDToName.get(`${id}`);
+      const { durable } = storeKindInfo[kindName];
+      if (!durable) {
+        deleteCollection(vobjID);
+      }
+    }
+  }
+
   function makeCollection(label, kindName, keySchema, valueSchema) {
     assert.typeof(label, 'string');
     assert(storeKindInfo[kindName]);
@@ -606,6 +628,7 @@ export function makeCollectionManager(
       JSON.stringify(serialize(harden(schemata))),
     );
     syscall.vatstoreSet(prefixc(collectionID, '|label'), label);
+    allCollectionObjIDs.add(vobjID);
 
     return [
       vobjID,
@@ -843,6 +866,7 @@ export function makeCollectionManager(
 
   return harden({
     initializeStoreKindInfo,
+    deleteAllVirtualCollections,
     makeScalarBigMapStore,
     makeScalarBigWeakMapStore,
     makeScalarBigSetStore,

packages/SwingSet/src/liveslots/liveslots.js

@@ -16,6 +16,7 @@ import { insistMessage } from '../lib/message.js';
 import { makeVirtualReferenceManager } from './virtualReferences.js';
 import { makeVirtualObjectManager } from './virtualObjectManager.js';
 import { makeCollectionManager } from './collectionManager.js';
+import { releaseOldState } from './stop-vat.js';
 
 const DEFAULT_VIRTUAL_OBJECT_CACHE_SIZE = 3; // XXX ridiculously small value to force churn for testing
 
@@ -1265,32 +1266,6 @@ function build(
     vom.insistAllDurableKindsReconnected();
   }
 
-  /**
-   * @returns { Promise<void> }
-   */
-  async function stopVat() {
-    assert(didStartVat);
-    assert(!didStopVat);
-    didStopVat = true;
-
-    // Pretend that userspace rejected all non-durable promises. We
-    // basically do the same thing that `thenReject(p,
-    // vpid)(rejection)` would have done, but we skip ahead to the
-    // syscall.resolve part. The real `thenReject` also does
-    // pRec.reject(), which would give control to userspace (who might
-    // have re-imported the promise and attached a .then to it), and
-    // stopVat() must not allow userspace to gain agency.
-
-    const rejectCapData = m.serialize('vat upgraded');
-    const vpids = Array.from(deciderVPIDs.keys()).sort();
-    const rejections = vpids.map(vpid => [vpid, true, rejectCapData]);
-    if (rejections.length) {
-      syscall.resolve(rejections);
-    }
-    // eslint-disable-next-line no-use-before-define
-    await bringOutYourDead();
-  }
-
   /**
    * @param { VatDeliveryObject } delivery
    * @returns { void | Promise<void> }
@@ -1330,10 +1305,6 @@ function build(
         result = startVat(vpCapData);
         break;
       }
-      case 'stopVat': {
-        result = stopVat();
-        break;
-      }
       default:
         assert.fail(X`unknown delivery type ${type}`);
     }
@@ -1354,6 +1325,37 @@ function build(
     return undefined;
   }
 
+  /**
+   * @returns { Promise<void> }
+   */
+  async function stopVat() {
+    assert(didStartVat);
+    assert(!didStopVat);
+    didStopVat = true;
+
+    try {
+      await releaseOldState({
+        m,
+        deciderVPIDs,
+        syscall,
+        exportedRemotables,
+        addToPossiblyDeadSet,
+        slotToVal,
+        valToSlot,
+        dropExports,
+        retireExports,
+        vrm,
+        collectionManager,
+        bringOutYourDead,
+        vreffedObjectRegistry,
+      });
+    } catch (e) {
+      console.log(`-- error during stopVat()`);
+      console.log(e);
+      throw e;
+    }
+  }
+
   /**
    * Do things that should be done (such as flushing caches to disk) after a
    * dispatch has completed and user code has relinquished agency.
@@ -1402,9 +1404,11 @@ function build(
    */
   async function dispatch(delivery) {
     // We must short-circuit dispatch to bringOutYourDead here because it has to
-    // be async
+    // be async, same for stopVat
     if (delivery[0] === 'bringOutYourDead') {
       return meterControl.runWithoutMeteringAsync(bringOutYourDead);
+    } else if (delivery[0] === 'stopVat') {
+      return meterControl.runWithoutMeteringAsync(stopVat);
     } else {
       // Start user code running, record any internal liveslots errors. We do
       // *not* directly wait for the userspace function to complete, nor for