From 1239e0aeadf61b5b4af1214146457c73cc48d6cf Mon Sep 17 00:00:00 2001 From: guybedford Date: Fri, 30 Nov 2018 02:02:46 +0200 Subject: [PATCH] emit ".node" binaries that are required --- .circleci/config.yml | 5 +++- dist/ncc/loaders/readme.md | 10 +++++++ package.json | 7 ++++- scripts/build.js | 13 ++++++---- src/index.js | 17 +++++++++--- src/loaders/node-loader.js | 16 ++++++++++++ .../relocate-loader.js} | 18 ++++++------- src/utils/dedupe-names.js | 11 ++++++++ test/binary/binding.gyp | 11 ++++++++ test/binary/hello.cc | 26 +++++++++++++++++++ test/index.test.js | 4 +-- test/integration/binary-require.js | 6 +++++ 12 files changed, 122 insertions(+), 22 deletions(-) create mode 100644 dist/ncc/loaders/readme.md create mode 100644 src/loaders/node-loader.js rename src/{asset-relocator.js => loaders/relocate-loader.js} (96%) create mode 100644 src/utils/dedupe-names.js create mode 100644 test/binary/binding.gyp create mode 100644 test/binary/hello.cc create mode 100644 test/integration/binary-require.js diff --git a/.circleci/config.yml b/.circleci/config.yml index a5132bd9..d78f5c77 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,11 +9,14 @@ jobs: - run: name: Installing dependencies command: yarn install + - run: + name: Build unit test binary + command: yarn build-test-binary # - run: # name: Linting # command: yarn lint - run: - name: Integration tests + name: Unit & integration tests command: yarn test - run: name: Coverage diff --git a/dist/ncc/loaders/readme.md b/dist/ncc/loaders/readme.md new file mode 100644 index 00000000..a83acdc7 --- /dev/null +++ b/dist/ncc/loaders/readme.md @@ -0,0 +1,10 @@ +# About this directory + +This directory will contain: + +- `node-loader.js` the ncc loader for supporting Node binary requires +- `relocate-loader.js` the ncc loader for handling CommonJS asset and reference relocations + +These are generated by the `build` step defined in `../../../package.json`. + +These files are published to npm. diff --git a/package.json b/package.json index ca8ea3e3..7f64e7c4 100644 --- a/package.json +++ b/package.json @@ -9,10 +9,11 @@ }, "scripts": { "build": "node scripts/build", + "build-test-binary": "cd test/binary && node-gyp rebuild && cp build/Release/hello.node ../integration/hello.node", "codecov": "codecov", "test": "npm run build && jest", "test-coverage": "jest --coverage --globals \"{\\\"coverage\\\":true}\" && codecov", - "prepublish": "npm test" + "prepublish": "in-publish && npm test || not-in-publish" }, "devDependencies": { "@azure/cosmos": "^2.0.5", @@ -42,12 +43,14 @@ "got": "^9.3.2", "graphql": "^14.0.2", "hot-shots": "^5.9.2", + "in-publish": "^2.0.0", "ioredis": "^4.2.0", "isomorphic-unfetch": "^3.0.0", "jest": "^23.6.0", "jimp": "^0.5.6", "koa": "^2.6.2", "leveldown": "^4.0.1", + "loader-utils": "^1.1.0", "magic-string": "^0.25.1", "mailgun": "^0.5.0", "mariadb": "^2.0.1-beta", @@ -56,6 +59,8 @@ "mongoose": "^5.3.12", "mysql": "^2.16.0", "node-pre-gyp": "^0.12.0", + "node-gyp": "^3.8.0", + "node-native-loader": "^1.1.1", "passport": "^0.4.0", "passport-google-oauth": "^1.0.0", "path-platform": "^0.11.15", diff --git a/scripts/build.js b/scripts/build.js index 50b5c615..8f08ff00 100644 --- a/scripts/build.js +++ b/scripts/build.js @@ -15,21 +15,24 @@ async function main() { // to bundle it. even if we did want watching and a bigger // bundle, webpack (and therefore ncc) cannot currently bundle // chokidar, which is quite convenient - externals: ["chokidar", "./asset-relocator.js"] + externals: ["chokidar", "./relocate-loader.js"] }); - const { code: assetRelocator, assets: assetRelocatorAssets } = await ncc(__dirname + "/../src/asset-relocator"); - + const { code: relocateLoader, assets: relocateLoaderAssets } = await ncc(__dirname + "/../src/loaders/relocate-loader"); + const { code: nodeLoader, assets: nodeLoaderAssets } = await ncc(__dirname + "/../src/loaders/node-loader"); const { code: sourcemapSupport, assets: sourcemapAssets } = await ncc(require.resolve("source-map-support/register")); - if (Object.keys(cliAssets).length || Object.keys(indexAssets).length || Object.keys(assetRelocatorAssets).length || Object.keys(sourcemapAssets).length) { + if (Object.keys(cliAssets).length || Object.keys(indexAssets).length || + Object.keys(relocateLoaderAssets).length || Object.keys(nodeLoaderAssets).length || + Object.keys(sourcemapAssets).length) { console.error('Assets emitted by core build, these need to be written into the dist directory'); } writeFileSync(__dirname + "/../dist/ncc/cli.js", cli); writeFileSync(__dirname + "/../dist/ncc/index.js", index); - writeFileSync(__dirname + "/../dist/ncc/asset-relocator.js", assetRelocator); writeFileSync(__dirname + "/../dist/ncc/sourcemap-register.js", sourcemapSupport); + writeFileSync(__dirname + "/../dist/ncc/loaders/relocate-loader.js", relocateLoader); + writeFileSync(__dirname + "/../dist/ncc/loaders/node-loader.js", nodeLoader); // copy webpack buildin await copy( diff --git a/src/index.js b/src/index.js index 6a3d63a3..de1a8adb 100644 --- a/src/index.js +++ b/src/index.js @@ -18,7 +18,7 @@ WebpackParser.parse = function (source, opts = {}) { }); } -const SUPPORTED_EXTENSIONS = [".js", ".mjs", ".json"]; +const SUPPORTED_EXTENSIONS = [".js", ".mjs", ".json", ".node"]; function resolveModule(context, request, callback, forcedExternals = []) { const resolveOptions = { @@ -46,6 +46,7 @@ function resolveModule(context, request, callback, forcedExternals = []) { module.exports = async (entry, { externals = [], minify = true, sourceMap = false, filename = "index.js" } = {}) => { const mfs = new MemoryFS(); + const assetNames = Object.create(null); const compiler = webpack({ entry, optimization: { @@ -75,8 +76,18 @@ module.exports = async (entry, { externals = [], minify = true, sourceMap = fals parser: { amd: false } }, { - test: /\.(js|mjs)/, - use: [{ loader: __dirname + "/asset-relocator.js" }] + test: /\.(js|mjs)$/, + use: [{ + loader: __dirname + "/loaders/relocate-loader.js", + options: { assetNames } + }] + }, + { + test: /\.node$/, + use: [{ + loader: __dirname + "/loaders/node-loader.js", + options: { assetNames } + }] } ] }, diff --git a/src/loaders/node-loader.js b/src/loaders/node-loader.js new file mode 100644 index 00000000..afa7d04a --- /dev/null +++ b/src/loaders/node-loader.js @@ -0,0 +1,16 @@ +const { getOptions } = require('loader-utils'); +const getUniqueAssetName = require('../utils/dedupe-names'); + +module.exports = function (content) { + if (this.cacheable) + this.cacheable(); + + const id = this.resourcePath; + const options = getOptions(this); + + const name = getUniqueAssetName(id, options.assetNames); + this.emitFile(name, content); + + return 'module.exports = __non_webpack_require__("./' + name + '")'; +}; +module.exports.raw = true; diff --git a/src/asset-relocator.js b/src/loaders/relocate-loader.js similarity index 96% rename from src/asset-relocator.js rename to src/loaders/relocate-loader.js index ede77a42..f479c254 100644 --- a/src/asset-relocator.js +++ b/src/loaders/relocate-loader.js @@ -6,6 +6,8 @@ const { attachScopes } = require('rollup-pluginutils'); const evaluate = require('static-eval'); const acorn = require('acorn'); const bindings = require('bindings'); +const getUniqueAssetName = require('../utils/dedupe-names'); +const { getOptions } = require('loader-utils'); // binary support for inlining logic from - node-pre-gyp/lib/pre-binding.js function isPregypId (id) { @@ -55,20 +57,16 @@ module.exports = function (code) { if (id.endsWith('.json') || !code.match(relocateRegEx)) return this.callback(null, code); - const assetNames = Object.create(null); + const options = getOptions(this); const emitAsset = (assetPath) => { // JS assets to support require(assetPath) and not fs-based handling // NB package.json is ambiguous here... if (assetPath.endsWith('.js') || assetPath.endsWith('.mjs')) return; + const name = getUniqueAssetName(assetPath, options.assetNames); + // console.log('Emitting ' + assetPath + ' for module ' + id); - const basename = path.basename(assetPath); - const ext = path.extname(basename); - let name = basename, i = 0; - while (name in assetNames && assetNames[name] !== assetPath) - name = basename.substr(0, basename.length - ext.length) + ++i + ext; - assetNames[name] = assetPath; this.emitFile(name, fs.readFileSync(assetPath)); return "__dirname + '/" + JSON.stringify(name).slice(1, -1) + "'"; @@ -225,9 +223,9 @@ module.exports = function (code) { // __dirname, __filename, binary only currently as well as require('bindings')(...) // Can add require.resolve, import.meta.url, even path-like environment variables if (node.type === 'Identifier' && isExpressionReference(node, parent)) { - if ((node.name === '__dirname' || - node.name === '__filename' || - node.name === pregypId || node.name === bindingsId) && !shadowDepths[node.name]) { + if (!shadowDepths[node.name] && + (node.name === '__dirname' || node.name === '__filename' || + node.name === pregypId || node.name === bindingsId)) { staticChildValue = computeStaticValue(node, false); // if it computes, then we start backtracking if (staticChildValue) { diff --git a/src/utils/dedupe-names.js b/src/utils/dedupe-names.js new file mode 100644 index 00000000..a1df8f1c --- /dev/null +++ b/src/utils/dedupe-names.js @@ -0,0 +1,11 @@ +const path = require("path"); + +module.exports = function (assetPath, assetNames) { + const basename = path.basename(assetPath); + const ext = path.extname(basename); + let name = basename, i = 0; + while (name in assetNames && assetNames[name] !== assetPath) + name = basename.substr(0, basename.length - ext.length) + ++i + ext; + assetNames[name] = assetPath; + return name; +}; \ No newline at end of file diff --git a/test/binary/binding.gyp b/test/binary/binding.gyp new file mode 100644 index 00000000..ef90a796 --- /dev/null +++ b/test/binary/binding.gyp @@ -0,0 +1,11 @@ +{ + "targets": [ + { + "target_name": "hello", + "sources": [ "hello.cc" ], + "include_dirs": [ + " + +namespace demo { + +using v8::FunctionCallbackInfo; +using v8::Isolate; +using v8::Local; +using v8::NewStringType; +using v8::Object; +using v8::String; +using v8::Value; + +void Method(const FunctionCallbackInfo& args) { + Isolate* isolate = args.GetIsolate(); + args.GetReturnValue().Set(String::NewFromUtf8( + isolate, "world", NewStringType::kNormal).ToLocalChecked()); +} + +void Initialize(Local exports) { + NODE_SET_METHOD(exports, "hello", Method); +} + +NODE_MODULE(NODE_GYP_MODULE_NAME, Initialize) + +} // namespace demo \ No newline at end of file diff --git a/test/index.test.js b/test/index.test.js index c5b41c26..69539c48 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -73,7 +73,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) { mkdirp.sync(dirname(assetPath)); fs.writeFileSync(assetPath, assets[asset]); } - (__dirname => { + ((__dirname, require) => { try { eval(`${code}\n//# sourceURL=${integrationTest}`); } @@ -83,7 +83,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) { fs.writeFileSync(__dirname + "/index.js", code); throw e; } - })(__dirname + "/tmp"); + })(__dirname + "/tmp", id => require(id.startsWith('./') ? './tmp/' + id.substr(2) : id)); if ("function" !== typeof module.exports) { throw new Error( `Integration test "${integrationTest}" evaluation failed. It does not export a function` diff --git a/test/integration/binary-require.js b/test/integration/binary-require.js new file mode 100644 index 00000000..6d7d7ddc --- /dev/null +++ b/test/integration/binary-require.js @@ -0,0 +1,6 @@ +const binary = require('./hello.node'); +const assert = require('assert'); + +module.exports = () => { + assert.equal(binary.hello(), 'world'); +}; \ No newline at end of file