add xs-vat-worker package with 1 integration test

vatWorker runs a vat in its own process. We have drivers for both node
and xs, though only the node driver is working as of this commit.

We also have a kernelSimulator that takes a vat transcript
(as from swingset-runner/bin/kerneldump) and simulates a swingset
kernel vat manager.

Earlier vatWorker work was done in a gist:
https://gist.github.com/dckc/f8e0b5d838079a994784d599c282cce7

This commit is based on:
2020-08-02 a32f30e simple transcript runs as integration test

The xs driver was working, to some extent, in:
2020-08-01 5b80235 agoric-sdk catch-up

See also
Agoric#1299

package.json

@@ -39,7 +39,8 @@
     "packages/cosmic-swingset",
     "packages/agoric-cli",
     "packages/deployment",
-    "packages/notifier"
+    "packages/notifier",
+    "packages/xs-vat-worker"
   ],
   "devDependencies": {
     "eslint": "^6.8.0",

packages/xs-vat-worker/.eslintignore

@@ -0,0 +1,4 @@
+/dist/
+/scripts/
+/xs_modules/
+/swingset/

packages/xs-vat-worker/.eslintrc.js

@@ -0,0 +1,27 @@
+/* global module */
+module.exports = {
+  // parser: "babel-eslint",
+  extends: ['airbnb-base', 'plugin:prettier/recommended'],
+  env: {
+    es6: true, // supports new ES6 globals (e.g., new types such as Set)
+  },
+  globals: {
+    "harden": "readonly",
+  },
+  rules: {
+    'implicit-arrow-linebreak': 'off',
+    'function-paren-newline': 'off',
+    'arrow-parens': 'off',
+    strict: 'off',
+    'prefer-destructuring': 'off',
+    'no-else-return': 'off',
+    'no-console': 'off',
+    'no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
+    'no-return-assign': 'off',
+    'no-param-reassign': 'off',
+    'no-restricted-syntax': ['off', 'ForOfStatement'],
+    'no-unused-expressions': 'off',
+    'no-loop-func': 'off',
+    'import/prefer-default-export': 'off', // contrary to Agoric standard
+  },
+};

packages/xs-vat-worker/.npmignore

@@ -0,0 +1,11 @@
+# Demo
+demo
+
+# scripts
+scripts
+
+# test
+test
+
+# Travis CI
+.travis.yml

packages/xs-vat-worker/.prettierrc.json

@@ -0,0 +1,4 @@
+{
+  "trailingComma": "all",
+  "singleQuote": true
+}

packages/xs-vat-worker/bin/node-vat-worker

@@ -0,0 +1,2 @@
+#!/bin/sh
+node -r esm ./src/index.js

packages/xs-vat-worker/manifest.json

@@ -0,0 +1,48 @@
+{
+	"include": [
+        "$(MODDABLE)/examples/manifest_base.json"
+    ],
+    "$note": [
+        "horrible @agoric/E KLUDGE below",
+        "liveSlots is a bit KLUDGy too"
+    ],
+    "creation": {
+        "keys": {
+          "available": 4096
+        },
+        "stack": 4096,
+        "parser": {
+          "buffer": 32768
+        }
+    },
+    "strip": [],
+	"modules": {
+        "files": [ "$(MODULES)/files/file/*", "$(MODULES)/files/file/lin/*" ],
+        "fdchan": [ "./fdchan" ],
+        "timer": [
+            "$(MODULES)/base/timer/timer",
+            "$(MODULES)/base/timer/lin/*",
+        ],
+        "@agoric/nat": "xs_modules/@agoric/nat/nat.esm",
+        "@agoric/install-ses": "xs_modules/@agoric/install-ses/install-ses",
+        "@agoric/import-bundle": "xs_modules/@agoric/import-bundle/index",
+        "@agoric/compartment-wrapper": "xs_modules/@agoric/import-bundle/compartment-wrapper",
+        "@agoric/eventual-send": [
+            "xs_modules/@agoric/eventual-send/index"
+        ],
+        "@agoric/E": "xs_modules/@agoric/eventual-send/E",
+        "@agoric/marshal": "xs_modules/@agoric/marshal/marshal",
+        "@agoric/produce-promise": "xs_modules/@agoric/produce-promise/producePromise",
+        "@agoric/assert": "xs_modules/@agoric/assert/assert",
+        "@agoric/types": "xs_modules/@agoric/assert/types",
+        "swingset/capdata": "./swingset/capdata",
+        "swingset/parseVatSlots": "./swingset/parseVatSlots",
+        "swingset/kernel/liveSlots": "./swingset/kernel/liveSlots",
+		"*": [
+            "./main",
+            "./console",
+            "./timer-ticks",
+            "./vatWorker"
+        ]
+	}
+}

packages/xs-vat-worker/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@agoric/xs-vat-worker",
+  "version": "0.0.1",
+  "description": "???",
+  "main": "dist/vat-worker.cjs.js",
+  "module": "dist/vat-worker.esm.js",
+  "browser": "dist/vat-worker.umd.js",
+  "scripts": {
+    "xs-build": "mkdir -p build; mcconfig -o build -p x-cli-lin -m -d",
+    "xs-run": "yarn xs-build && ./build/bin/lin/debug/make-vat-transcript",
+    "xs-build-release": "mkdir -p build; mcconfig -o build -p x-cli-lin -m",
+    "xs-run-release": "yarn xs-build-release && ./build/bin/lin/release/make-vat-transcript",
+    "test": "tape -r esm 'test/**/test-*.js'",
+    "build": "exit 0",
+    "lint-fix": "eslint --fix '**/*.{js,jsx}'",
+    "lint-check": "eslint '**/*.{js,jsx}'",
+    "lint-fix-jessie": "eslint -c '.eslintrc-jessie.js' --fix '**/*.{js,jsx}'",
+    "lint-check-jessie": "eslint -c '.eslintrc-jessie.js' '**/*.{js,jsx}'"
+  },
+  "devDependencies": {
+    "eslint": "^6.1.0",
+    "eslint-config-airbnb-base": "^14.0.0",
+    "eslint-config-jessie": "0.0.3",
+    "eslint-config-prettier": "^6.0.0",
+    "eslint-plugin-import": "^2.18.2",
+    "eslint-plugin-prettier": "^3.1.0",
+    "prettier": "^1.18.2",
+    "tap-spec": "^5.0.0",
+    "tape": "^4.11.0",
+    "tape-promise": "^4.0.0"
+  },
+  "dependencies": {
+    "@agoric/assert": "^0.0.8",
+    "@agoric/bundle-source": "^1.1.6",
+    "@agoric/eventual-send": "^0.9.3",
+    "@agoric/import-bundle": "^0.0.8",
+    "@agoric/install-ses": "^0.2.0",
+    "@agoric/produce-promise": "^0.1.3",
+    "@agoric/swingset-vat": "^0.6.0",
+    "anylogger": "^1.0.4",
+    "esm": "^3.2.5"
+  },
+  "keywords": [],
+  "files": [
+    "dist"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://gist.github.com/f8e0b5d838079a994784d599c282cce7.git"
+  },
+  "author": "Agoric",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://gist.github.com/dckc/f8e0b5d838079a994784d599c282cce7"
+  },
+  "homepage": "https://gist.github.com/dckc/f8e0b5d838079a994784d599c282cce7"
+}

packages/xs-vat-worker/src-native/fdchan.c

@@ -0,0 +1,74 @@
+/**
+ * ref Netstrings 19970201 by Bernstein
+ * https://cr.yp.to/proto/netstrings.txt
+ *
+ * Moddable in C
+ * https://github.com/Moddable-OpenSource/moddable/blob/public/documentation/xs/XS%20in%20C.md
+ *
+ * stdio
+ * https://pubs.opengroup.org/onlinepubs/009695399/basedefs/stdio.h.html
+ */
+
+#include <assert.h>
+#include <stdio.h>
+
+#include "xsAll.h"
+#include "xs.h"
+
+void xs_Reader(xsMachine *the) {
+  int argc = xsToInteger(xsArgc);
+  if (argc < 1) {
+    mxTypeError("expected fd");
+  }
+  int fd = xsToInteger(xsArg(0));
+  FILE *inStream = fdopen(fd, "rb");
+  if (!inStream) {
+    mxUnknownError("fdopen failed");
+  }
+	xsSetHostData(xsThis, (void *)((uintptr_t)inStream));
+
+	// modInstrumentationAdjust(Files, +1);
+}
+
+void xs_Writer(xsMachine *the) {
+  int argc = xsToInteger(xsArgc);
+  if (argc < 1) {
+    mxTypeError("expected fd");
+  }
+  int fd = xsToInteger(xsArg(0));
+  FILE *outStream = fdopen(fd, "wb");
+  if (!outStream) {
+    mxUnknownError("fdopen failed");
+  }
+	xsSetHostData(xsThis, (void *)((uintptr_t)outStream));
+
+	// modInstrumentationAdjust(Files, +1);
+}
+
+void xs_read_netstring(xsMachine *the) {
+  size_t len;
+  char* buf = NULL;
+  FILE *inStream = xsGetHostData(xsThis);
+  assert(inStream);
+
+  if (fscanf(inStream, "%9lu", &len) < 1) { goto BARF; }  /* >999999999 bytes is bad */
+  // fprintf(stderr, "xs_stdin_read_netstring len %lu\n", len);
+  if (fgetc(inStream) != ':') { goto BARF; }
+  buf = malloc(len + 1);       /* malloc(0) is not portable */
+  if (!buf) { goto BARF; }
+  if (fread(buf, 1, len, inStream) < len) { goto BARF; }
+  if (fgetc(inStream) != ',') { goto BARF; }
+
+  xsResult = xsStringBuffer(buf, len);
+  free(buf);
+  // fprintf(stderr, "xs_stdin_read_nestring return\n");
+  return;
+
+BARF:
+  free(buf);
+  xsUnknownError("getline failed");
+}
+
+void xs_fdchan_destructor() {
+
+}

packages/xs-vat-worker/src-native/fdchan.js

@@ -0,0 +1,9 @@
+export class Reader @ "xs_fdchan_destructor" {
+  constructor(fd) @ "xs_Reader"
+  read_netstring() @ "xs_read_netstring";
+}
+
+export class Writer @ "xs_fdchan_destructor" {
+  constructor(fd) @ "xs_Writer"
+	write(...items) @ "xs_file_write";
+}

packages/xs-vat-worker/src/console.js

@@ -0,0 +1,25 @@
+/** console for xs platform */
+/* global trace, globalThis */
+const harden = x => Object.freeze(x, true);
+
+const text = it => (typeof it === 'object' ? JSON.stringify(it) : `${it}`);
+const combine = (...things) => `${things.map(text).join(' ')}\n`;
+
+export function makeConsole(write_) {
+  const write = write_ || trace; // note ocap exception for tracing / logging
+  return harden({
+    log(...things) {
+      write(combine(...things));
+    },
+    // node.js docs say this is just an alias for error
+    warn(...things) {
+      write(combine('WARNING: ', ...things));
+    },
+    // node docs say this goes to stderr
+    error(...things) {
+      write(combine('ERROR: ', ...things));
+    },
+  });
+}
+
+globalThis.console = makeConsole();

packages/xs-vat-worker/src/index.js

@@ -0,0 +1,111 @@
+// vatWorker driver for node.js
+// contrast with main.js for xs
+import '@agoric/install-ses';
+
+import { main as vatWorker } from './vatWorker';
+
+const INFD = 3;
+const OUTFD = 4;
+
+function makePipe(io, sleep) {
+  function write(data) {
+    let done = 0;
+    for (;;) {
+      try {
+        done += io.writeSync(OUTFD, data.slice(done));
+        if (done >= data.length) {
+          return done;
+        }
+      } catch (writeFailed) {
+        if (writeFailed.code === 'EAGAIN') {
+          sleep(0.1);
+          // try again
+        } else {
+          throw writeFailed;
+        }
+      }
+    }
+  }
+
+  return harden({
+    writeMessage(msg) {
+      write(`${msg.length}:`);
+      write(msg);
+      write(',');
+    },
+    readMessage(EOF) {
+      let buf = Buffer.from('999999999:', 'utf-8');
+      let len = null;
+      let colonPos = null;
+      let offset = 0;
+
+      for (;;) {
+        // console.error('readMessage', { length: buf.length, len, colonPos, offset });
+        try {
+          offset += io.readSync(INFD, buf, {
+            offset,
+            length: buf.length - offset,
+          });
+        } catch (err) {
+          if (err.code === 'EAGAIN') {
+            sleep(0.1);
+            // eslint-disable-next-line no-continue
+            continue;
+          } else if (err.code === 'EOF') {
+            throw EOF;
+          } else {
+            throw err;
+          }
+        }
+        if (len === null) {
+          colonPos = buf.indexOf(':');
+          if (colonPos > 0) {
+            const digits = buf.slice(0, colonPos).toString('utf-8');
+            len = parseInt(digits, 10);
+            const rest = Buffer.alloc(len + 1);
+            // console.error('parsed len. copy', { digits, len, targetStart: 0, sourceStart: colonPos + 1, sourceEnd: offset });
+            buf.copy(rest, 0, colonPos + 1, offset);
+            buf = rest;
+            offset -= colonPos + 1;
+          }
+        } else if (offset === len + 1) {
+          const delim = buf.slice(-1).toString('utf-8');
+          if (delim !== ',') {
+            throw new Error(
+              `bad netstring: length ${len} expected , found [${delim}]`,
+            );
+          }
+          const result = buf.slice(0, -1).toString('utf-8');
+          // console.error({ colon: colonPos, len, result: result.slice(0, 20) });
+          return result;
+        }
+      }
+    },
+  });
+}
+
+async function main({ setImmediate, fs, spawnSync }) {
+  const sleep = secs => spawnSync('sleep', [secs]);
+  const pipe = makePipe(fs, sleep);
+  return vatWorker({
+    readMessage: pipe.readMessage,
+    writeMessage: pipe.writeMessage,
+    setImmediate,
+  });
+}
+
+main({
+  setImmediate,
+  // eslint-disable-next-line global-require
+  spawnSync: require('child_process').spawnSync,
+  fs: {
+    // eslint-disable-next-line global-require
+    readSync: require('fs').readSync,
+    // eslint-disable-next-line global-require
+    writeSync: require('fs').writeSync,
+  },
+})
+  .catch(err => {
+    console.error(err);
+  })
+  .then(() => process.exit(0));