Binary require support

.circleci/config.yml

@@ -9,11 +9,14 @@ jobs:
       - run:
           name: Installing dependencies
           command: yarn install
+      - run:
+          name: Build unit test binary
+          command: yarn build-test-binary
       # - run:
       #     name: Linting
       #     command: yarn lint
       - run:
-          name: Integration tests
+          name: Unit & integration tests
           command: yarn test
       - run:
           name: Coverage

dist/ncc/loaders/readme.md

@@ -0,0 +1,10 @@
+# About this directory
+
+This directory will contain:
+
+- `node-loader.js` the ncc loader for supporting Node binary requires
+- `relocate-loader.js` the ncc loader for handling CommonJS asset and reference relocations
+
+These are generated by the `build` step defined in `../../../package.json`.
+
+These files are published to npm.

package.json

@@ -9,10 +9,11 @@
   },
   "scripts": {
     "build": "node scripts/build",
+    "build-test-binary": "cd test/binary && node-gyp rebuild && cp build/Release/hello.node ../integration/hello.node",
     "codecov": "codecov",
     "test": "npm run build && jest",
     "test-coverage": "jest --coverage --globals \"{\\\"coverage\\\":true}\" && codecov",
-    "prepublish": "npm test"
+    "prepublish": "in-publish && npm test || not-in-publish"
   },
   "devDependencies": {
     "@azure/cosmos": "^2.0.5",
@@ -42,12 +43,14 @@
     "got": "^9.3.2",
     "graphql": "^14.0.2",
     "hot-shots": "^5.9.2",
+    "in-publish": "^2.0.0",
     "ioredis": "^4.2.0",
     "isomorphic-unfetch": "^3.0.0",
     "jest": "^23.6.0",
     "jimp": "^0.5.6",
     "koa": "^2.6.2",
     "leveldown": "^4.0.1",
+    "loader-utils": "^1.1.0",
     "magic-string": "^0.25.1",
     "mailgun": "^0.5.0",
     "mariadb": "^2.0.1-beta",
@@ -56,6 +59,8 @@
     "mongoose": "^5.3.12",
     "mysql": "^2.16.0",
     "node-pre-gyp": "^0.12.0",
+    "node-gyp": "^3.8.0",
+    "node-native-loader": "^1.1.1",
     "passport": "^0.4.0",
     "passport-google-oauth": "^1.0.0",
     "path-platform": "^0.11.15",

scripts/build.js

@@ -15,21 +15,24 @@ async function main() {
     // to bundle it. even if we did want watching and a bigger
     // bundle, webpack (and therefore ncc) cannot currently bundle
     // chokidar, which is quite convenient
-    externals: ["chokidar", "./asset-relocator.js"]
+    externals: ["chokidar", "./relocate-loader.js"]
   });
 
-  const { code: assetRelocator, assets: assetRelocatorAssets } = await ncc(__dirname + "/../src/asset-relocator");
-
+  const { code: relocateLoader, assets: relocateLoaderAssets } = await ncc(__dirname + "/../src/loaders/relocate-loader");
+  const { code: nodeLoader, assets: nodeLoaderAssets } = await ncc(__dirname + "/../src/loaders/node-loader");
   const { code: sourcemapSupport, assets: sourcemapAssets } = await ncc(require.resolve("source-map-support/register"));
 
-  if (Object.keys(cliAssets).length || Object.keys(indexAssets).length || Object.keys(assetRelocatorAssets).length || Object.keys(sourcemapAssets).length) {
+  if (Object.keys(cliAssets).length || Object.keys(indexAssets).length ||
+      Object.keys(relocateLoaderAssets).length || Object.keys(nodeLoaderAssets).length ||
+      Object.keys(sourcemapAssets).length) {
     console.error('Assets emitted by core build, these need to be written into the dist directory');
   }
 
   writeFileSync(__dirname + "/../dist/ncc/cli.js", cli);
   writeFileSync(__dirname + "/../dist/ncc/index.js", index);
-  writeFileSync(__dirname + "/../dist/ncc/asset-relocator.js", assetRelocator);
   writeFileSync(__dirname + "/../dist/ncc/sourcemap-register.js", sourcemapSupport);
+  writeFileSync(__dirname + "/../dist/ncc/loaders/relocate-loader.js", relocateLoader);
+  writeFileSync(__dirname + "/../dist/ncc/loaders/node-loader.js", nodeLoader);
 
   // copy webpack buildin
   await copy(

src/index.js

@@ -18,7 +18,7 @@ WebpackParser.parse = function (source, opts = {}) {
   });
 }
 
-const SUPPORTED_EXTENSIONS = [".js", ".mjs", ".json"];
+const SUPPORTED_EXTENSIONS = [".js", ".mjs", ".json", ".node"];
 
 function resolveModule(context, request, callback, forcedExternals = []) {
   const resolveOptions = {
@@ -46,6 +46,7 @@ function resolveModule(context, request, callback, forcedExternals = []) {
 
 module.exports = async (entry, { externals = [], minify = true, sourceMap = false, filename = "index.js" } = {}) => {
   const mfs = new MemoryFS();
+  const assetNames = Object.create(null);
   const compiler = webpack({
     entry,
     optimization: {
@@ -75,8 +76,18 @@ module.exports = async (entry, { externals = [], minify = true, sourceMap = fals
           parser: { amd: false }
         },
         {
-          test: /\.(js|mjs)/,
-          use: [{ loader: __dirname + "/asset-relocator.js" }]
+          test: /\.(js|mjs)$/,
+          use: [{
+            loader: __dirname + "/loaders/relocate-loader.js",
+            options: { assetNames }
+          }]
+        },
+        {
+          test: /\.node$/,
+          use: [{
+            loader: __dirname + "/loaders/node-loader.js",
+            options: { assetNames }
+          }]
         }
       ]
     },

src/loaders/node-loader.js

@@ -0,0 +1,16 @@
+const { getOptions } = require('loader-utils');
+const getUniqueAssetName = require('../utils/dedupe-names');
+
+module.exports = function (content) {
+  if (this.cacheable)
+    this.cacheable();
+
+  const id = this.resourcePath;
+  const options = getOptions(this);
+
+  const name = getUniqueAssetName(id, options.assetNames);
+  this.emitFile(name, content);
+
+  return 'module.exports = __non_webpack_require__("./' + name + '")';
+};
+module.exports.raw = true;

src/asset-relocator.js → src/loaders/relocate-loader.js

@@ -6,6 +6,8 @@ const { attachScopes } = require('rollup-pluginutils');
 const evaluate = require('static-eval');
 const acorn = require('acorn');
 const bindings = require('bindings');
+const getUniqueAssetName = require('../utils/dedupe-names');
+const { getOptions } = require('loader-utils');
 
 // binary support for inlining logic from - node-pre-gyp/lib/pre-binding.js
 function isPregypId (id) {
@@ -55,20 +57,16 @@ module.exports = function (code) {
   if (id.endsWith('.json') || !code.match(relocateRegEx))
     return this.callback(null, code);
 
-  const assetNames = Object.create(null);
+  const options = getOptions(this);
   const emitAsset = (assetPath) => {
     // JS assets to support require(assetPath) and not fs-based handling
     // NB package.json is ambiguous here...
     if (assetPath.endsWith('.js') || assetPath.endsWith('.mjs'))
       return;
 
+    const name = getUniqueAssetName(assetPath, options.assetNames);
+
     // console.log('Emitting ' + assetPath + ' for module ' + id);
-    const basename = path.basename(assetPath);
-    const ext = path.extname(basename);
-    let name = basename, i = 0;
-    while (name in assetNames && assetNames[name] !== assetPath)
-      name = basename.substr(0, basename.length - ext.length) + ++i + ext;
-    assetNames[name] = assetPath;
 
     this.emitFile(name, fs.readFileSync(assetPath));
     return "__dirname + '/" + JSON.stringify(name).slice(1, -1) + "'";
@@ -225,9 +223,9 @@ module.exports = function (code) {
       // __dirname,  __filename, binary only currently as well as require('bindings')(...)
       // Can add require.resolve, import.meta.url, even path-like environment variables
       if (node.type === 'Identifier' && isExpressionReference(node, parent)) {
-        if ((node.name === '__dirname' ||
-            node.name === '__filename' ||
-            node.name === pregypId || node.name === bindingsId) && !shadowDepths[node.name]) {
+        if (!shadowDepths[node.name] &&
+            (node.name === '__dirname' || node.name === '__filename' ||
+            node.name === pregypId || node.name === bindingsId)) {
           staticChildValue = computeStaticValue(node, false);
           // if it computes, then we start backtracking
           if (staticChildValue) {

src/utils/dedupe-names.js

@@ -0,0 +1,11 @@
+const path = require("path");
+
+module.exports = function (assetPath, assetNames) {
+  const basename = path.basename(assetPath);
+  const ext = path.extname(basename);
+  let name = basename, i = 0;
+  while (name in assetNames && assetNames[name] !== assetPath)
+    name = basename.substr(0, basename.length - ext.length) + ++i + ext;
+  assetNames[name] = assetPath;
+  return name;
+};

test/binary/binding.gyp

@@ -0,0 +1,11 @@
+{
+  "targets": [
+    {
+      "target_name": "hello",
+      "sources": [ "hello.cc" ],
+      "include_dirs": [
+        "<!(node -e \"require('nan')\")"
+      ]
+    }
+  ]
+}

test/binary/hello.cc

@@ -0,0 +1,26 @@
+// hello.cc
+#include <node.h>
+
+namespace demo {
+
+using v8::FunctionCallbackInfo;
+using v8::Isolate;
+using v8::Local;
+using v8::NewStringType;
+using v8::Object;
+using v8::String;
+using v8::Value;
+
+void Method(const FunctionCallbackInfo<Value>& args) {
+  Isolate* isolate = args.GetIsolate();
+  args.GetReturnValue().Set(String::NewFromUtf8(
+      isolate, "world", NewStringType::kNormal).ToLocalChecked());
+}
+
+void Initialize(Local<Object> exports) {
+  NODE_SET_METHOD(exports, "hello", Method);
+}
+
+NODE_MODULE(NODE_GYP_MODULE_NAME, Initialize)
+
+}  // namespace demo

test/index.test.js

@@ -73,7 +73,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) {
       mkdirp.sync(dirname(assetPath));
       fs.writeFileSync(assetPath, assets[asset]);
     }
-    (__dirname => {
+    ((__dirname, require) => {
       try {
         eval(`${code}\n//# sourceURL=${integrationTest}`);
       }
@@ -83,7 +83,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) {
         fs.writeFileSync(__dirname + "/index.js", code);
         throw e;
       }
-    })(__dirname + "/tmp");
+    })(__dirname + "/tmp", id => require(id.startsWith('./') ? './tmp/' + id.substr(2) : id));
     if ("function" !== typeof module.exports) {
       throw new Error(
         `Integration test "${integrationTest}" evaluation failed. It does not export a function`

test/integration/binary-require.js

@@ -0,0 +1,6 @@
+const binary = require('./hello.node');
+const assert = require('assert');
+
+module.exports = () => {
+  assert.equal(binary.hello(), 'world');
+};