vat worker SwingSet integration CHECKPOINT

this isn't close to right; I'm just using github as a backup service.

.gitignore

@@ -62,4 +62,4 @@ bundle-*.js
 
 .idea/
 
-packages/xs-vat-worker/build/
+build/

packages/SwingSet/package.json

@@ -43,6 +43,7 @@
     "@agoric/tame-metering": "^1.2.3",
     "@agoric/transform-eventual-send": "^1.3.1",
     "@agoric/transform-metering": "^1.3.0",
+    "@agoric/xs-vat-worker": "^0.1.0",
     "@babel/core": "^7.5.0",
     "@babel/generator": "^7.6.4",
     "anylogger": "^0.21.0",

packages/SwingSet/src/controller.js

@@ -3,6 +3,7 @@
 import fs from 'fs';
 import path from 'path';
 import re2 from 're2';
+import { spawn } from 'child_process';
 import { Worker } from 'worker_threads';
 import * as babelCore from '@babel/core';
 import * as babelParser from '@agoric/babel-parser';
@@ -17,6 +18,7 @@ import { initSwingStore } from '@agoric/swing-store-simple';
 import { HandledPromise } from '@agoric/eventual-send';
 import { makeMeteringTransformer } from '@agoric/transform-metering';
 import { makeTransform } from '@agoric/transform-eventual-send';
+import { spawnXsWorker } from '@agoric/xs-vat-worker/src/xsWorkerSpawn';
 
 import { startSubprocessWorker } from './spawnSubprocessWorker';
 import { assertKnownOptions } from './assertOptions';
@@ -289,6 +291,7 @@ export async function buildVatController(
     transformTildot,
     makeNodeWorker,
     startSubprocessWorker,
+    startXsWorker: () => spawnXsWorker({ spawn }),
   };
 
   const kernel = buildKernel(kernelEndowments);

packages/SwingSet/src/kernel/kernel.js

@@ -41,6 +41,7 @@ export default function buildKernel(kernelEndowments) {
     transformTildot,
     makeNodeWorker,
     startSubprocessWorker,
+    startXsWorker,
   } = kernelEndowments;
   insistStorageAPI(hostStorage);
   const { enhancedCrankBuffer, commitCrank } = wrapStorage(hostStorage);
@@ -569,6 +570,7 @@ export default function buildKernel(kernelEndowments) {
     waitUntilQuiescent,
     makeNodeWorker,
     startSubprocessWorker,
+    startXsWorker,
   });
 
   /*

packages/SwingSet/src/kernel/vatManager/factory.js

@@ -1,5 +1,6 @@
 /* global harden */
 import { assert } from '@agoric/assert';
+import { makeXsWorkerFactory } from '@agoric/xs-vat-worker';
 import { assertKnownOptions } from '../../assertOptions';
 import { makeLocalVatManagerFactory } from './localVatManager';
 import { makeNodeWorkerVatManagerFactory } from './nodeWorker';
@@ -14,6 +15,7 @@ export function makeVatManagerFactory({
   waitUntilQuiescent,
   makeNodeWorker,
   startSubprocessWorker,
+  startXsWorker,
 }) {
   const localFactory = makeLocalVatManagerFactory({
     allVatPowers,
@@ -34,6 +36,8 @@ export function makeVatManagerFactory({
     kernelKeeper,
   });
 
+  const xsWorkerFactory = makeXsWorkerFactory({ startXsWorker });
+
   function validateManagerOptions(managerOptions) {
     assertKnownOptions(managerOptions, [
       'enablePipelining',
@@ -92,6 +96,10 @@ export function makeVatManagerFactory({
       );
     }
 
+    if (managerType === 'xs-worker') {
+      return xsWorkerFactory.createFromBundle(vatID, bundle, managerOptions);
+    }
+
     throw Error(
       `unknown type ${managerType}, not local/nodeWorker/node-subprocess`,
     );

packages/SwingSet/test/workers/test-worker.js

@@ -1,7 +1,26 @@
 import '@agoric/install-ses';
 import tap from 'tap';
+import { workerBin } from '@agoric/xs-vat-worker/src/locate';
 import { loadBasedir, buildVatController } from '../../src/index';
 
+tap.test('xs vat manager', async t => {
+  if (!workerBin) {
+    console.warn('XS vat worker not built; skipping');
+    t.end();
+    return;
+  }
+
+  const config = await loadBasedir(__dirname);
+  config.vats.target.creationOptions = { managerType: 'xs-worker' };
+  const c = await buildVatController(config, []);
+
+  await c.run();
+  t.equal(c.bootstrapResult.status(), 'fulfilled');
+
+  await c.shutdown();
+  t.end();
+});
+
 tap.test('nodeWorker vat manager', async t => {
   const config = await loadBasedir(__dirname);
   config.vats.target.creationOptions = { managerType: 'nodeWorker' };

packages/xs-vat-worker/package.json

@@ -1,9 +1,10 @@
 {
   "name": "@agoric/xs-vat-worker",
-  "version": "0.0.1",
-  "description": "???",
-  "module": "src/locate.js",
+  "version": "0.1.0",
+  "description": "swingset-vat worker for XS js runtime",
+  "module": "src/xsWorkerFactory.js",
   "scripts": {
+    "postinstall": "cp src/locate-undefined.js src/locate.js",
     "build:xs-lin": "make -f xs-lin.mk",
     "test": "tape -r esm 'test/**/test-*.js'",
     "build": "exit 0",

packages/xs-vat-worker/src/locate-undefined.js

@@ -0,0 +1 @@
+export const workerBin = undefined;

packages/xs-vat-worker/src/locate.js

@@ -1,13 +1,2 @@
-import fs from 'fs';
-
-/**
- * return the absolute pathname of the generated xs-vat-worker
- * executable (or undefined if it doesn't exist yet, because yarn
- * build:xs-lin wasn't run)
- *
- * WARNING: uses ambient access to fs.exists, require.resolve
- */
-export function locateWorkerBin() {
-  const binPath = require.resolve('../build/bin/lin/debug/xs-vat-worker');
-  return fs.existsSync(binPath) ? binPath : undefined;
-}
+export const workerBin =
+  '/home/connolly/projects/agoric/agoric-sdk/packages/xs-vat-worker/node_modules/.bin/xs-vat-worker';

packages/xs-vat-worker/src/xsWorkerFactory.js

@@ -0,0 +1,57 @@
+function makeWorker(io) {
+  const format = obj => JSON.stringify(obj);
+  const send = obj => io.writeNetstring(format(obj));
+
+  const expect = async msgtype => {
+    const txt = await io.readNetstring();
+    let msg;
+    try {
+      msg = JSON.parse(txt);
+    } catch (badJSON) {
+      console.error('bad JSON ', txt.length, ' chars: [', txt, ']', badJSON);
+      throw badJSON;
+    }
+    if (msg.msgtype !== msgtype) {
+      throw new Error(
+        `expected ${msgtype}; found: ${msg.msgtype}; error: ${
+          msg.error
+        } [${JSON.stringify(msg)}]`,
+      );
+    }
+    return msg;
+  };
+
+  return harden({
+    async loadVat(bundle) {
+      await send({ msgtype: 'load-bundle', bundle });
+      return expect('load-bundle-ack');
+    },
+    async deliver(...args) {
+      const { d, syscalls, _crankNumber } = args;
+      await send({ msgtype: 'dispatch', type: d[0], args: d.slice(1) });
+      for (const syscall of syscalls) {
+        // eslint-disable-next-line no-await-in-loop
+        const request = await expect('syscall');
+        console.log('syscall request', request);
+        // eslint-disable-next-line no-await-in-loop
+        await send({ msgtype: 'syscall-ack', response: syscall.response });
+      }
+      return expect('dispatch-ack');
+    },
+    async finish() {
+      await send({ msgtype: 'finish' });
+      await expect('finish-ack');
+    },
+  });
+}
+
+export function makeXsWorkerFactory({ startXsWorker }) {
+  return harden({
+    async createFromBundle(vatID, bundle, _managerOptions) {
+      const io = startXsWorker();
+      const worker = makeWorker(io);
+      await worker.loadVat(bundle);
+      return worker;
+    },
+  });
+}

packages/xs-vat-worker/src/xsWorkerSpawn.js

@@ -0,0 +1,40 @@
+import { makePromiseKit } from '@agoric/promise-kit';
+import { workerBin } from './locate';
+import { readNetstring, writeNetstring } from './netstring';
+
+// eslint-disable-next-line no-unused-vars
+function parentLog(first, ...args) {
+  // console.error(`--parent: ${first}`, ...args);
+}
+
+// we send on fd3, and receive on fd4. We pass fd1/2 (stdout/err) through, so
+// console log/err from the child shows up normally.
+const stdio = harden(['inherit', 'inherit', 'inherit', 'pipe', 'pipe']);
+const INFD = 3;
+const OUTFD = 4;
+
+export function spawnXsWorker({ spawn }) {
+  console.log('spawning', { workerBin });
+  const child = spawn(workerBin, [], { stdio });
+
+  const pk = makePromiseKit();
+
+  child.once('exit', code => {
+    parentLog('child exit', code);
+    pk.resolve(code);
+  });
+  child.once('error', e => {
+    parentLog('child error', e);
+    pk.reject(e);
+  });
+  parentLog(`waiting on child`);
+
+  child.stdio[OUTFD].pause();
+
+  return harden({
+    readNetstring: () => readNetstring(child.stdio[OUTFD]),
+    writeNetstring: s => writeNetstring(child.stdio[INFD], s),
+    kill: () => child.kill(),
+    done: pk.promise,
+  });
+}

packages/xs-vat-worker/test/kernelSimulator.js

@@ -2,11 +2,6 @@
 // context: https://github.com/Agoric/agoric-sdk/issues/1299
 import '@agoric/install-ses';
 
-import { readNetstring, writeNetstring } from '../src/netstring';
-
-const INFD = 3;
-const OUTFD = 4;
-
 function options(env) {
   return {
     vat1: env.VAT1 || 'vat-target.js',
@@ -15,54 +10,6 @@ function options(env) {
   };
 }
 
-function makeWorker(child) {
-  const format = obj => JSON.stringify(obj);
-  const send = obj => writeNetstring(child.stdio[INFD], format(obj));
-
-  child.stdio[OUTFD].pause();
-
-  const expect = async msgtype => {
-    const txt = await readNetstring(child.stdio[OUTFD]);
-    let msg;
-    try {
-      msg = JSON.parse(txt);
-    } catch (badJSON) {
-      console.error('bad JSON ', txt.length, ' chars: [', txt, ']', badJSON);
-      throw badJSON;
-    }
-    if (msg.msgtype !== msgtype) {
-      throw new Error(
-        `expected ${msgtype}; found: ${msg.msgtype}; error: ${
-          msg.error
-        } [${JSON.stringify(msg)}]`,
-      );
-    }
-    return msg;
-  };
-
-  return harden({
-    async loadVat(bundle) {
-      await send({ msgtype: 'load-bundle', bundle });
-      return expect('load-bundle-ack');
-    },
-    async dispatch({ d, syscalls, _crankNumber }) {
-      await send({ msgtype: 'dispatch', type: d[0], args: d.slice(1) });
-      for (const syscall of syscalls) {
-        // eslint-disable-next-line no-await-in-loop
-        const request = await expect('syscall');
-        console.log('syscall request', request);
-        // eslint-disable-next-line no-await-in-loop
-        await send({ msgtype: 'syscall-ack', response: syscall.response });
-      }
-      return expect('dispatch-ack');
-    },
-    async finish() {
-      await send({ msgtype: 'finish' });
-      await expect('finish-ack');
-    },
-  });
-}
-
 async function runTranscript(w1, bundle, transcript) {
   await w1.loadVat(bundle);
   console.log('loadVat done.');

packages/xs-vat-worker/xs-lin.mk

@@ -1,14 +1,19 @@
 MODDABLE=$(PWD)/moddable
 TOOLS=$(MODDABLE)/build/bin/lin/release/
+NODE_MODULES=$(PWD)/node_modules
+ROOT=$(PWD)/../..
+
+$(NODE_MODULES)/.bin/xs-vat-worker: build/bin/lin/debug/xs-vat-worker
+	cp $< $@
+	echo "export const workerBin = '$@';" >src/locate.js
 
 build/bin/lin/debug/xs-vat-worker: build $(TOOLS)/mcconfig moddable/xs/platforms/lin_xs_cli.c
-	PATH=$(TOOLS):$$PATH mcconfig -o build -p x-cli-lin -m -d
+	ROOT=$(ROOT) PATH=$(TOOLS):$$PATH mcconfig -o build -p x-cli-lin -m -d
 
 build:
 	mkdir -p build
 
 moddable/xs/platforms/lin_xs_cli.c: moddable/xs/platforms/lin_xs.h
-	touch $@
 
 moddable/xs/platforms/lin_xs.h: /usr/include/glib-2.0/gio/gio.h
 
@@ -22,7 +27,7 @@ $(TOOLS)/mcconfig:
 	make headless
 
 clean:
-	rm -rf build
+	rm -rf build $(NODE_MODULES)/.bin/xs-vat-worker
 
 realclean:
 	rm -rf build