Convert swingstore from LMDB to Sqlite, phase 1

packages/SwingSet/misc-tools/db-delete.js

@@ -1,5 +1,4 @@
 #!/usr/bin/env node
-import 'lmdb';
 import '@endo/init';
 import process from 'process';
 import { openSwingStore } from '@agoric/swing-store';
@@ -47,7 +46,9 @@ async function run() {
   const stateDBDir = argv.shift();
   const key = argv.shift();
 
-  const { kvStore, commit } = openSwingStore(stateDBDir);
+  const { kernelStorage, hostStorage } = openSwingStore(stateDBDir);
+  const { kvStore } = kernelStorage;
+  const { commit } = hostStorage;
 
   if (range) {
     for (const k of kvStore.getKeys(`${key}.`, `${key}/`)) {

packages/SwingSet/misc-tools/db-dump.js

@@ -1,5 +1,4 @@
 /* eslint-disable */
-import 'lmdb';
 import '@endo/init';
 import process from 'process';
 import { openSwingStore } from '@agoric/swing-store';
@@ -41,7 +40,7 @@ function run() {
   }
   const stateDBDir = argv.shift();
 
-  const { kvStore } = openSwingStore(stateDBDir);
+  const { kvStore } = openSwingStore(stateDBDir).kernelStorage;
 
   // we know all keys start with letters, so '@' is before all keys, and '{'
   // is after all keys

packages/SwingSet/misc-tools/db-get.js

@@ -1,5 +1,4 @@
 #!/usr/bin/env node
-import 'lmdb';
 import '@endo/init';
 import process from 'process';
 import { openSwingStore } from '@agoric/swing-store';
@@ -58,7 +57,7 @@ function run() {
   const stateDBDir = argv.shift();
   const key = argv.shift();
 
-  const { kvStore } = openSwingStore(stateDBDir);
+  const { kvStore } = openSwingStore(stateDBDir).kernelStorage;
 
   function pkv(k) {
     const value = kvStore.get(k);

packages/SwingSet/misc-tools/db-set.js

@@ -1,5 +1,4 @@
 #!/usr/bin/env node
-import 'lmdb';
 import '@endo/init';
 import process from 'process';
 import { openSwingStore } from '@agoric/swing-store';
@@ -33,10 +32,10 @@ async function run() {
   const key = argv.shift();
   const value = argv.shift();
 
-  const { kvStore, commit } = openSwingStore(stateDBDir);
+  const { kernelStorage, hostStorage } = openSwingStore(stateDBDir);
 
-  kvStore.set(key, value);
-  await commit();
+  kernelStorage.kvStore.set(key, value);
+  await hostStorage.commit();
 }
 
 run().then(

packages/SwingSet/misc-tools/extract-transcript-from-kerneldb.js

@@ -1,4 +1,3 @@
-import 'lmdb';
 import '@endo/init';
 import process from 'process';
 import fs from 'fs';
@@ -19,7 +18,7 @@ if (!dirPath) {
 if (!isSwingStore(dirPath)) {
   throw Error(`${dirPath} does not appear to be a swingstore (no ./data.mdb)`);
 }
-const { kvStore, streamStore } = openSwingStore(dirPath);
+const { kvStore, streamStore } = openSwingStore(dirPath).kernelStorage;
 function get(key) {
   return kvStore.get(key);
 }

packages/SwingSet/misc-tools/replace-bundle.js

@@ -1,6 +1,5 @@
 #!/usr/bin/env node
 
-import 'lmdb';
 import '@endo/init';
 import process from 'process';
 import { openSwingStore } from '@agoric/swing-store';
@@ -36,7 +35,9 @@ async function run() {
   const stateDBDir = argv.shift();
   const srcPath = argv.shift();
 
-  const { kvStore, commit } = openSwingStore(stateDBDir);
+  const { kernelStorage, hostStorage } = openSwingStore(stateDBDir);
+  const { kvStore } = kernelStorage;
+  const { commit } = hostStorage;
   log(`will use ${srcPath} in ${stateDBDir} for ${bundleName}`);
 
   if (bundleName === 'kernel') {

packages/SwingSet/misc-tools/vat-stats.js

@@ -1,5 +1,4 @@
 /* eslint-disable */
-import 'lmdb';
 import '@endo/init';
 import process from 'process';
 import { openSwingStore } from '@agoric/swing-store';
@@ -36,7 +35,7 @@ function fail(message, printUsage) {
 function run() {
   const argv = process.argv.slice(2);
   const stateDBDir = argv.shift();
-  const { kvStore: kv } = openSwingStore(stateDBDir);
+  const { kvStore: kv } = openSwingStore(stateDBDir).kernelStorage;
   const vatIDs = [];
   const vats = {};
   for (const name of JSON.parse(kv.get('vat.names'))) {

packages/SwingSet/package.json

@@ -48,7 +48,6 @@
     "@endo/zip": "^0.2.28",
     "anylogger": "^0.21.0",
     "import-meta-resolve": "^1.1.1",
-    "lmdb": "^2.4.5",
     "microtime": "^3.1.0",
     "semver": "^6.3.0"
   },

packages/SwingSet/src/controller/controller.js

@@ -15,16 +15,15 @@ import microtime from 'microtime';
 import { assert, Fail } from '@agoric/assert';
 import { importBundle } from '@endo/import-bundle';
 import { xsnap, recordXSnap } from '@agoric/xsnap';
+import { initSwingStore } from '@agoric/swing-store';
 
 import { checkBundle } from '@endo/check-bundle/lite.js';
-import { createSHA256 } from '../lib-nodejs/hasher.js';
 import engineGC from '../lib-nodejs/engine-gc.js';
 import { startSubprocessWorker } from '../lib-nodejs/spawnSubprocessWorker.js';
 import { waitUntilQuiescent } from '../lib-nodejs/waitUntilQuiescent.js';
 import { makeGcAndFinalize } from '../lib-nodejs/gc-and-finalize.js';
 import { kslot } from '../lib/kmarshal.js';
 import { insistStorageAPI } from '../lib/storageAPI.js';
-import { provideHostStorage } from './hostStorage.js';
 import {
   buildKernelBundle,
   swingsetIsInitialized,
@@ -159,7 +158,7 @@ export function makeStartXSnap(bundles, { snapStore, env, spawn }) {
 
 /**
  *
- * @param {HostStore} hostStorage
+ * @param {SwingStoreKernelStorage} kernelStorage
  * @param {Record<string, unknown>} deviceEndowments
  * @param {{
  *   verbose?: boolean,
@@ -175,11 +174,11 @@ export function makeStartXSnap(bundles, { snapStore, env, spawn }) {
  * }} runtimeOptions
  */
 export async function makeSwingsetController(
-  hostStorage = provideHostStorage(),
+  kernelStorage = initSwingStore().kernelStorage,
   deviceEndowments = {},
   runtimeOptions = {},
 ) {
-  const kvStore = hostStorage.kvStore;
+  const kvStore = kernelStorage.kvStore;
   insistStorageAPI(kvStore);
 
   // Use ambient process.env only if caller did not specify.
@@ -293,14 +292,14 @@ export async function makeSwingsetController(
     JSON.parse(kvStore.get('supervisorBundle')),
   ];
   const startXSnap = makeStartXSnap(bundles, {
-    snapStore: hostStorage.snapStore,
+    snapStore: kernelStorage.snapStore,
     env,
     spawn,
   });
 
   const kernelEndowments = {
     waitUntilQuiescent,
-    hostStorage,
+    kernelStorage,
     debugPrefix,
     vatEndowments,
     makeConsole,
@@ -312,7 +311,6 @@ export async function makeSwingsetController(
     WeakRef,
     FinalizationRegistry,
     gcAndFinalize: makeGcAndFinalize(engineGC),
-    createSHA256,
   };
 
   const kernelRuntimeOptions = {
@@ -420,7 +418,7 @@ export async function makeSwingsetController(
     },
 
     getActivityhash() {
-      return kernel.getActivityhash();
+      return kernelStorage.getActivityhash();
     },
 
     // everything beyond here is for tests, and everything should be migrated
@@ -434,6 +432,7 @@ export async function makeSwingsetController(
       const vatID = kernel.vatNameToID(vatName);
       const kref = kernel.getRootObject(vatID);
       kernel.pinObject(kref);
+      kernelStorage.emitCrankHashes();
       return kref;
     },
 
@@ -442,7 +441,10 @@ export async function makeSwingsetController(
     },
 
     kpResolution(kpid) {
-      return kernel.kpResolution(kpid);
+      const result = kernel.kpResolution(kpid);
+      // kpResolution does DB write (changes refcounts) so we need emitCrankHashes here
+      kernelStorage.emitCrankHashes();
+      return result;
     },
     vatNameToID(vatName) {
       return kernel.vatNameToID(vatName);
@@ -465,6 +467,7 @@ export async function makeSwingsetController(
       if (kpid) {
         kernel.kpRegisterInterest(kpid);
       }
+      kernelStorage.emitCrankHashes();
       return kpid;
     },
 
@@ -477,12 +480,14 @@ export async function makeSwingsetController(
       if (!options.upgradeMessage) {
         options.upgradeMessage = `vat ${vatName} upgraded`;
       }
-      return controller.queueToVatRoot(
+      const result = controller.queueToVatRoot(
         'vatAdmin',
         'upgradeStaticVat',
         [vatID, pauseTarget, bundleID, options],
         'ignore',
       );
+      // no emitCrankHashes here because queueToVatRoot did that
+      return result;
     },
   });
 
@@ -507,7 +512,7 @@ export async function makeSwingsetController(
  * @param {SwingSetConfig} config
  * @param {string[]} argv
  * @param {{
- *   hostStorage?: HostStore;
+ *   kernelStorage?: SwingStoreKernelStorage;
  *   env?: Record<string, string>;
  *   verbose?: boolean;
  *   kernelBundles?: Record<string, Bundle>;
@@ -525,7 +530,7 @@ export async function buildVatController(
   runtimeOptions = {},
 ) {
   const {
-    hostStorage = provideHostStorage(),
+    kernelStorage = initSwingStore().kernelStorage,
     env,
     verbose,
     kernelBundles: kernelAndOtherBundles = {},
@@ -548,18 +553,18 @@ export async function buildVatController(
   };
   const initializationOptions = { verbose, kernelBundles };
   let bootstrapResult;
-  if (!swingsetIsInitialized(hostStorage)) {
+  if (!swingsetIsInitialized(kernelStorage)) {
     // eslint-disable-next-line @jessie.js/no-nested-await
     bootstrapResult = await initializeSwingset(
       config,
       argv,
-      hostStorage,
+      kernelStorage,
       initializationOptions,
       runtimeOptions,
     );
   }
   const controller = await makeSwingsetController(
-    hostStorage,
+    kernelStorage,
     {},
     actualRuntimeOptions,
   );