Merge pull request #1407 from dckc/vatworker-1299

prototype XS-based VatWorker process refs #1299
Agoric · Aug 25, 2020 · 5ceb471 · 5ceb471
2 parents a97e808 + 73b9cc0
commit 5ceb471
Show file tree

Hide file tree

Showing 29 changed files with 1,189 additions and 12 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "packages/xs-vat-worker/moddable"]
+	path = packages/xs-vat-worker/moddable
+	url = https://github.com/agoric-labs/moddable.git
+	branch = ag-linux-cli
diff --git a/package.json b/package.json
@@ -38,7 +38,8 @@
     "packages/cosmic-swingset",
     "packages/agoric-cli",
     "packages/deployment",
-    "packages/notifier"
+    "packages/notifier",
+    "packages/xs-vat-worker"
   ],
   "devDependencies": {
     "ava": "^3.11.1",

diff --git a/packages/SwingSet/package.json b/packages/SwingSet/package.json
@@ -39,6 +39,7 @@
     "@agoric/tame-metering": "^1.2.3",
     "@agoric/transform-eventual-send": "^1.3.1",
     "@agoric/transform-metering": "^1.3.0",
+    "@agoric/xs-vat-worker": "^0.1.0",
     "@babel/core": "^7.5.0",
     "@babel/generator": "^7.6.4",
     "anylogger": "^0.21.0",

diff --git a/packages/SwingSet/src/controller.js b/packages/SwingSet/src/controller.js
@@ -16,6 +16,7 @@ import { importBundle } from '@agoric/import-bundle';
 import { initSwingStore } from '@agoric/swing-store-simple';
 import { makeMeteringTransformer } from '@agoric/transform-metering';
 import { makeTransform } from '@agoric/transform-eventual-send';
+import { locateWorkerBin } from '@agoric/xs-vat-worker';
 
 import { startSubprocessWorker } from './spawnSubprocessWorker';
 import { assertKnownOptions } from './assertOptions';
@@ -321,6 +322,12 @@ export async function buildVatController(
     // console.log(`--slog ${JSON.stringify(obj)}`);
   }
 
+  const startSubprocessWorkerNode = () => startSubprocessWorker();
+  const xsWorkerBin = locateWorkerBin({ resolve: path.resolve });
+  const startSubprocessWorkerXS = fs.existsSync(xsWorkerBin)
+    ? () => startSubprocessWorker({ execPath: xsWorkerBin, args: [] })
+    : undefined;
+
   const kernelEndowments = {
     waitUntilQuiescent,
     hostStorage,
@@ -331,7 +338,8 @@ export async function buildVatController(
     transformMetering,
     transformTildot,
     makeNodeWorker,
-    startSubprocessWorker,
+    startSubprocessWorkerNode,
+    startSubprocessWorkerXS,
     writeSlogObject,
   };
 

diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js
@@ -50,7 +50,8 @@ export default function buildKernel(kernelEndowments, kernelOptions = {}) {
     transformMetering,
     transformTildot,
     makeNodeWorker,
-    startSubprocessWorker,
+    startSubprocessWorkerNode,
+    startSubprocessWorkerXS,
     writeSlogObject,
   } = kernelEndowments;
   const { verbose } = kernelOptions;
@@ -612,7 +613,8 @@ export default function buildKernel(kernelEndowments, kernelOptions = {}) {
     transformMetering,
     waitUntilQuiescent,
     makeNodeWorker,
-    startSubprocessWorker,
+    startSubprocessWorkerNode,
+    startSubprocessWorkerXS,
   });
 
   function buildVatSyscallHandler(vatID, translators) {

diff --git a/packages/SwingSet/src/kernel/vatManager/factory.js b/packages/SwingSet/src/kernel/vatManager/factory.js
@@ -13,7 +13,8 @@ export function makeVatManagerFactory({
   transformMetering,
   waitUntilQuiescent,
   makeNodeWorker,
-  startSubprocessWorker,
+  startSubprocessWorkerNode,
+  startSubprocessWorkerXS,
 }) {
   const localFactory = makeLocalVatManagerFactory({
     allVatPowers,
@@ -30,7 +31,12 @@ export function makeVatManagerFactory({
   });
 
   const nodeSubprocessFactory = makeNodeSubprocessFactory({
-    startSubprocessWorker,
+    startSubprocessWorker: startSubprocessWorkerNode,
+    kernelKeeper,
+  });
+
+  const xsWorkerFactory = makeNodeSubprocessFactory({
+    startSubprocessWorker: startSubprocessWorkerXS,
     kernelKeeper,
   });
 
@@ -93,6 +99,13 @@ export function makeVatManagerFactory({
       );
     }
 
+    if (managerType === 'xs-worker') {
+      if (!startSubprocessWorkerXS) {
+        throw new Error('manager type xs-worker not available');
+      }
+      return xsWorkerFactory.createFromBundle(vatID, bundle, managerOptions);
+    }
+
     throw Error(
       `unknown type ${managerType}, not local/nodeWorker/node-subprocess`,
     );

diff --git a/packages/SwingSet/src/spawnSubprocessWorker.js b/packages/SwingSet/src/spawnSubprocessWorker.js
@@ -20,8 +20,10 @@ const supercode = require.resolve(
 // always be Node.
 const stdio = harden(['inherit', 'inherit', 'inherit', 'pipe', 'pipe']);
 
-export function startSubprocessWorker() {
-  const proc = spawn(process.execPath, ['-r', 'esm', supercode], { stdio });
+export function startSubprocessWorker(options) {
+  const execPath = options.execPath || process.execPath;
+  const args = options.args || ['-r', 'esm', supercode];
+  const proc = spawn(execPath, args, { stdio });
 
   const toChild = Netstring.writeStream();
   toChild.pipe(proc.stdio[3]);

diff --git a/packages/SwingSet/test/workers/test-worker.js b/packages/SwingSet/test/workers/test-worker.js
@@ -1,7 +1,24 @@
 import '@agoric/install-ses';
 import test from 'ava';
+import { resolve } from 'path';
+import { existsSync } from 'fs';
+import { locateWorkerBin } from '@agoric/xs-vat-worker';
 import { loadBasedir, buildVatController } from '../../src/index';
 
+const xsWorkerBin = locateWorkerBin({ resolve });
+const maybeTestXS = existsSync(xsWorkerBin) ? test : test.skip;
+
+maybeTestXS('xs vat manager', async t => {
+  const config = await loadBasedir(__dirname);
+  config.vats.target.creationOptions = { managerType: 'xs-worker' };
+  const c = await buildVatController(config, []);
+
+  await c.run();
+  t.is(c.bootstrapResult.status(), 'fulfilled');
+
+  await c.shutdown();
+});
+
 test('nodeWorker vat manager', async t => {
   const config = await loadBasedir(__dirname);
   config.vats.target.creationOptions = { managerType: 'nodeWorker' };

diff --git a/packages/xs-vat-worker/.eslintignore b/packages/xs-vat-worker/.eslintignore
@@ -0,0 +1,9 @@
+/dist/
+/scripts/
+/xs_modules/
+/swingset/
+/moddable/
+build/
+bundle-*.js
+test/bug1/
+src-native/
diff --git a/packages/xs-vat-worker/.eslintrc.js b/packages/xs-vat-worker/.eslintrc.js
@@ -0,0 +1,28 @@
+/* global module */
+module.exports = {
+  // parser: "babel-eslint",
+  extends: ['airbnb-base', 'plugin:prettier/recommended'],
+  env: {
+    es6: true, // supports new ES6 globals (e.g., new types such as Set)
+  },
+  globals: {
+    "harden": "readonly",
+    "globalThis": "writeable",
+  },
+  rules: {
+    'implicit-arrow-linebreak': 'off',
+    'function-paren-newline': 'off',
+    'arrow-parens': 'off',
+    strict: 'off',
+    'prefer-destructuring': 'off',
+    'no-else-return': 'off',
+    'no-console': 'off',
+    'no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
+    'no-return-assign': 'off',
+    'no-param-reassign': 'off',
+    'no-restricted-syntax': ['off', 'ForOfStatement'],
+    'no-unused-expressions': 'off',
+    'no-loop-func': 'off',
+    'import/prefer-default-export': 'off', // contrary to Agoric standard
+  },
+};
diff --git a/packages/xs-vat-worker/.gitignore b/packages/xs-vat-worker/.gitignore
@@ -0,0 +1,2 @@
+.envrc
+build/
diff --git a/packages/xs-vat-worker/.npmignore b/packages/xs-vat-worker/.npmignore
@@ -0,0 +1,11 @@
+# Demo
+demo
+
+# scripts
+scripts
+
+# test
+test
+
+# Travis CI
+.travis.yml
diff --git a/packages/xs-vat-worker/.prettierrc.json b/packages/xs-vat-worker/.prettierrc.json
@@ -0,0 +1,4 @@
+{
+  "trailingComma": "all",
+  "singleQuote": true
+}
diff --git a/packages/xs-vat-worker/compartmap.json b/packages/xs-vat-worker/compartmap.json
@@ -0,0 +1,127 @@
+{
+  "main": "packages/xs-vat-worker/",
+  "compartments": {
+    "packages/xs-vat-worker/": {
+      "label": "@agoric/[email protected]",
+      "location": "packages/xs-vat-worker/",
+      "contents": [
+        "./src/vatWorker.js"
+      ],
+      "modules": {
+        "@agoric/import-bundle": {
+          "compartment": "node_modules/@agoric/import-bundle/",
+          "module": "./src/index.js"
+        },
+        "@agoric/marshal": {
+          "compartment": "node_modules/@agoric/marshal/",
+          "module": "./marshal.js"
+        },
+        "@agoric/swingset-vat/src/kernel/liveSlots": {
+          "compartment": "node_modules/@agoric/swingset-vat/",
+          "module": "./src/kernel/liveSlots.js"
+        }
+      }
+    },
+    "node_modules/@agoric/import-bundle/": {
+      "label": "@agoric/[email protected]",
+      "location": "node_modules/@agoric/import-bundle/",
+      "contents": [
+        "./src/index.js",
+        "./src/compartment-wrapper.js"
+      ],
+      "modules": {}
+    },
+    "node_modules/@agoric/marshal/": {
+      "label": "@agoric/[email protected]",
+      "location": "node_modules/@agoric/marshal/",
+      "contents": [
+        "./marshal.js"
+      ],
+      "modules": {
+        "@agoric/nat": {
+          "compartment": "node_modules/@agoric/nat/",
+          "module": "./dist/nat.esm.js"
+        },
+        "@agoric/promise-kit": {
+          "compartment": "node_modules/@agoric/promise-kit/",
+          "module": "./src/promiseKit.js"
+        }
+      }
+    },
+    "node_modules/@agoric/swingset-vat/": {
+      "label": "@agoric/[email protected]",
+      "location": "node_modules/@agoric/swingset-vat/",
+      "contents": [
+        "./src/kernel/liveSlots.js",
+        "./src/parseVatSlots.js",
+        "./src/capdata.js"
+      ],
+      "modules": {
+        "@agoric/marshal": {
+          "compartment": "node_modules/@agoric/marshal/",
+          "module": "./marshal.js"
+        },
+        "@agoric/assert": {
+          "compartment": "node_modules/@agoric/assert/",
+          "module": "./src/assert.js"
+        },
+        "@agoric/promise-kit": {
+          "compartment": "node_modules/@agoric/promise-kit/",
+          "module": "./src/promiseKit.js"
+        },
+        "@agoric/nat": {
+          "compartment": "node_modules/@agoric/nat/",
+          "module": "./dist/nat.esm.js"
+        }
+      }
+    },
+    "node_modules/@agoric/nat/": {
+      "label": "@agoric/[email protected]",
+      "location": "node_modules/@agoric/nat/",
+      "contents": [
+        "./dist/nat.esm.js"
+      ],
+      "modules": {}
+    },
+    "node_modules/@agoric/promise-kit/": {
+      "label": "@agoric/[email protected]",
+      "location": "node_modules/@agoric/promise-kit/",
+      "contents": [
+        "./src/promiseKit.js"
+      ],
+      "modules": {}
+    },
+    "node_modules/@agoric/assert/": {
+      "label": "@agoric/[email protected]",
+      "location": "node_modules/@agoric/assert/",
+      "contents": [
+        "./src/assert.js",
+        "./src/types.js"
+      ],
+      "modules": {}
+    }
+  },
+  "modules": {
+    "node_modules/0_MKDIR": "$(ROOT)/node_modules/0_MKDIR",
+    "node_modules/@agoric/0_MKDIR": "$(ROOT)/node_modules/@agoric/0_MKDIR",
+    "node_modules/@agoric/assert/0_MKDIR": "$(ROOT)/node_modules/@agoric/assert/0_MKDIR",
+    "node_modules/@agoric/assert/src/assert": "$(ROOT)/node_modules/@agoric/assert/src/assert",
+    "node_modules/@agoric/assert/src/types": "$(ROOT)/node_modules/@agoric/assert/src/types",
+    "node_modules/@agoric/import-bundle/0_MKDIR": "$(ROOT)/node_modules/@agoric/import-bundle/0_MKDIR",
+    "node_modules/@agoric/import-bundle/src/compartment-wrapper": "$(ROOT)/node_modules/@agoric/import-bundle/src/compartment-wrapper",
+    "node_modules/@agoric/import-bundle/src/index": "$(ROOT)/node_modules/@agoric/import-bundle/src/index",
+    "node_modules/@agoric/marshal/marshal": "$(ROOT)/node_modules/@agoric/marshal/marshal",
+    "node_modules/@agoric/nat/0_MKDIR": "$(ROOT)/node_modules/@agoric/nat/0_MKDIR",
+    "node_modules/@agoric/nat/dist/nat.esm": "$(ROOT)/node_modules/@agoric/nat/dist/nat.esm",
+    "node_modules/@agoric/promise-kit/0_MKDIR": "$(ROOT)/node_modules/@agoric/promise-kit/0_MKDIR",
+    "node_modules/@agoric/promise-kit/src/promiseKit": "$(ROOT)/node_modules/@agoric/promise-kit/src/promiseKit",
+    "node_modules/@agoric/swingset-vat/0_MKDIR": "$(ROOT)/node_modules/@agoric/swingset-vat/0_MKDIR",
+    "node_modules/@agoric/swingset-vat/src/0_MKDIR": "$(ROOT)/node_modules/@agoric/swingset-vat/src/0_MKDIR",
+    "node_modules/@agoric/swingset-vat/src/capdata": "$(ROOT)/node_modules/@agoric/swingset-vat/src/capdata",
+    "node_modules/@agoric/swingset-vat/src/kernel/liveSlots": "$(ROOT)/node_modules/@agoric/swingset-vat/src/kernel/liveSlots",
+    "node_modules/@agoric/swingset-vat/src/parseVatSlots": "$(ROOT)/node_modules/@agoric/swingset-vat/src/parseVatSlots",
+    "packages/0_MKDIR": "$(ROOT)/packages/0_MKDIR",
+    "packages/xs-vat-worker/0_MKDIR": "$(ROOT)/packages/xs-vat-worker/0_MKDIR",
+    "packages/xs-vat-worker/src/vatWorker": "$(ROOT)/packages/xs-vat-worker/src/vatWorker"
+  }
+}
diff --git a/packages/xs-vat-worker/manifest.json b/packages/xs-vat-worker/manifest.json
@@ -0,0 +1,40 @@
+{
+  "build": {
+    "BUILD": "$(MODDABLE)/build",
+    "MODULES": "$(MODDABLE)/modules"
+  },
+  "creation": {
+    "keys": {
+      "available": 4096
+    },
+    "stack": 4096,
+    "parser": {
+      "buffer": 32768
+    }
+  },
+  "strip": [],
+  "modules": {
+    "@moddable/files": [
+      "$(MODULES)/base/instrumentation/*",
+      "$(MODULES)/files/file/*",
+      "$(MODULES)/files/file/lin/*"
+    ],
+    "@moddable/Resource": [ "$(MODULES)/files/resource/Resource" ],
+    "@moddable/timer": [
+      "$(MODULES)/base/timer/timer",
+      "$(MODULES)/base/timer/lin/*"
+    ],
+    "@agoric/eventual-send": "../eventual-send/src/index",
+
+    "main": "./start-xs",
+    "src/console": "./src/console",
+    "src/harden": "./src/harden",
+    "src-native/fdchan": [ "./src-native/fdchan" ],
+    "src/endo-load": "./src/endo-load",
+    "src/vatWorker": "./src/vatWorker"
+  },
+  "data": {
+    "compartmap": "./compartmap"
+  },
+  "include": [ "./compartmap.json" ]
+}
diff --git a/packages/xs-vat-worker/moddable b/packages/xs-vat-worker/moddable