Merge pull request #3298 from Agoric/3109-refcounting

implement kernel-side GC

refs #3106 
closes #2646 
closes #3109

packages/SwingSet/src/kernel/kernel.js

@@ -18,6 +18,7 @@ import { makeMeterManager } from './metering.js';
 import { makeKernelSyscallHandler, doSend } from './kernelSyscall.js';
 import { makeSlogger, makeDummySlogger } from './slogger.js';
 import { getKpidsToRetire } from './cleanup.js';
+import { processNextGCAction } from './gc-actions.js';
 
 import { makeVatRootObjectSlot, makeVatLoader } from './loadVat.js';
 import { makeDeviceTranslators } from './deviceTranslator.js';
@@ -43,13 +44,17 @@ function makeError(message, name = 'Error') {
 const VAT_TERMINATION_ERROR = makeError('vat terminated');
 
 /*
- * Pretend that a vat just exported an object
+ * Pretend that a vat just exported an object, and increment the refcount on
+ * the resulting kref so nothing tries to delete it for being unreferenced.
  */
 export function doAddExport(kernelKeeper, fromVatID, vref) {
   insistVatID(fromVatID);
   assert(parseVatSlot(vref).allocatedByVat);
   const vatKeeper = kernelKeeper.provideVatKeeper(fromVatID);
   const kref = vatKeeper.mapVatSlotToKernelSlot(vref);
+  // we lie to incrementRefCount (this is really an export, but we pretend
+  // it's an import) so it will actually increment the count
+  kernelKeeper.incrementRefCount(kref, 'doAddExport', { isExport: false });
   return kref;
 }
 
@@ -524,6 +529,29 @@ export default function buildKernel(
     }
   }
 
+  async function processGCMessage(message) {
+    // used for dropExports, retireExports, and retireImports
+    const { type, vatID, krefs } = message;
+    // console.log(`-- processGCMessage(${vatID} ${type} ${krefs.join(',')})`);
+    insistVatID(vatID);
+    // eslint-disable-next-line no-use-before-define
+    if (!vatWarehouse.lookup(vatID)) {
+      return; // can't collect from the dead
+    }
+    const kd = harden([type, krefs]);
+    if (type === 'retireExports') {
+      for (const kref of krefs) {
+        // const rc = kernelKeeper.getObjectRefCount(kref);
+        // console.log(`   ${kref}: ${rc.reachable},${rc.recognizable}`);
+        kernelKeeper.deleteKernelObject(kref);
+        // console.log(`   deleted ${kref}`);
+      }
+    }
+    // eslint-disable-next-line no-use-before-define
+    const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd);
+    await deliverAndLogToVat(vatID, kd, vd);
+  }
+
   async function processCreateVat(message) {
     assert(vatAdminRootKref, `initializeKernel did not set vatAdminRootKref`);
     const { vatID, source, dynamicOptions } = message;
@@ -582,6 +610,8 @@ export default function buildKernel(
     }
   }
 
+  const gcMessages = ['dropExports', 'retireExports', 'retireImports'];
+
   let processQueueRunning;
   async function processQueueMessage(message) {
     kdebug(`processQ ${JSON.stringify(message)}`);
@@ -613,6 +643,8 @@ export default function buildKernel(
         await processNotify(message);
       } else if (message.type === 'create-vat') {
         await processCreateVat(message);
+      } else if (gcMessages.includes(message.type)) {
+        await processGCMessage(message);
       } else {
         assert.fail(X`unable to process message.type ${message.type}`);
       }
@@ -633,7 +665,7 @@ export default function buildKernel(
         kdebug(`vat terminated: ${JSON.stringify(info)}`);
       }
       if (!didAbort) {
-        kernelKeeper.purgeDeadKernelPromises();
+        kernelKeeper.processRefcounts();
         kernelKeeper.saveStats();
       }
       commitCrank();
@@ -897,6 +929,10 @@ export default function buildKernel(
   }
 
   function getNextMessage() {
+    const gcMessage = processNextGCAction(kernelKeeper);
+    if (gcMessage) {
+      return gcMessage;
+    }
     if (!kernelKeeper.isRunQueueEmpty()) {
       return kernelKeeper.getNextMsg();
     }
@@ -976,6 +1012,9 @@ export default function buildKernel(
       case 'fulfilled':
       case 'rejected':
         kernelKeeper.decrementRefCount(kpid, 'external');
+        for (const kref of p.data.slots) {
+          kernelKeeper.incrementRefCount(kref, 'external');
+        }
         return p.data;
       default:
         assert.fail(X`invalid state for ${kpid}: ${p.state}`);

packages/SwingSet/src/kernel/kernelSyscall.js

@@ -117,19 +117,28 @@ export function makeKernelSyscallHandler(tools) {
 
   function dropImports(koids) {
     assert(Array.isArray(koids), X`dropImports given non-Array ${koids}`);
-    console.log(`-- kernel ignoring dropImports ${koids.join(',')}`);
+    // all the work was done during translation, there's nothing to do here
     return OKNULL;
   }
 
   function retireImports(koids) {
     assert(Array.isArray(koids), X`retireImports given non-Array ${koids}`);
-    console.log(`-- kernel ignoring retireImports ${koids.join(',')}`);
+    // all the work was done during translation, there's nothing to do here
     return OKNULL;
   }
 
   function retireExports(koids) {
     assert(Array.isArray(koids), X`retireExports given non-Array ${koids}`);
-    console.log(`-- kernel ignoring retireExports ${koids.join(',')}`);
+    const newActions = [];
+    for (const koid of koids) {
+      const importers = kernelKeeper.getImporters(koid);
+      for (const vatID of importers) {
+        newActions.push(`${vatID} retireImport ${koid}`);
+      }
+      // TODO: decref and delete any #2069 auxdata
+      kernelKeeper.deleteKernelObject(koid);
+    }
+    kernelKeeper.addGCActions(newActions);
     return OKNULL;
   }
 

packages/SwingSet/src/kernel/state/deviceKeeper.js

@@ -27,13 +27,20 @@ export function initializeDeviceState(kvStore, deviceID) {
  *
  * @param {*} kvStore  The storage in which the persistent state will be kept
  * @param {string} deviceID  The device ID string of the device in question
- * @param {*} addKernelDeviceNode  Kernel function to add a new device node to the
- *    kernel's mapping tables.
- *
+ * @param { addKernelDeviceNode: (deviceID: string) => string,
+ *          incrementRefCount: (kernelSlot: string,
+ *                              tag: string?,
+ *                              options: {
+ *                                isExport: boolean?,
+ *                                onlyRecognizable: boolean?,
+ *                              },
+ *                             ) => undefined),
+ *         } tools
  * @returns {*} an object to hold and access the kernel's state for the given device
  */
-export function makeDeviceKeeper(kvStore, deviceID, addKernelDeviceNode) {
+export function makeDeviceKeeper(kvStore, deviceID, tools) {
   insistDeviceID(deviceID);
+  const { addKernelDeviceNode, incrementRefCount } = tools;
 
   function setSourceAndOptions(source, options) {
     assert.typeof(source, 'object');
@@ -78,6 +85,7 @@ export function makeDeviceKeeper(kvStore, deviceID, addKernelDeviceNode) {
         } else {
           assert.fail(X`unknown type ${type}`);
         }
+        // device nodes don't have refcounts: they're immortal
         const kernelKey = `${deviceID}.c.${kernelSlot}`;
         kvStore.set(kernelKey, devSlot);
         kvStore.set(devKey, kernelSlot);
@@ -119,6 +127,15 @@ export function makeDeviceKeeper(kvStore, deviceID, addKernelDeviceNode) {
       } else {
         assert.fail(X`unknown type ${type}`);
       }
+      // Use isExport=false, since this is an import. Unlike
+      // mapKernelSlotToVatSlot, we use onlyRecognizable=false, to increment
+      // both reachable and recognizable, because we aren't tracking object
+      // reachability on the device clist (deviceSlots doesn't use WeakRefs
+      // and won't emit dropImports), so we need the clist to hold a ref to
+      // the imported object forever.
+      const opts = { isExport: false, onlyRecognizable: false };
+      incrementRefCount(kernelSlot, `${deviceID}|dk|clist`, opts);
+
       const devSlot = makeVatSlot(type, false, id);
 
       const devKey = `${deviceID}.c.${devSlot}`;