From 633e8e2a20cd1c0c72dbd974259ffa765f47cf85 Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Wed, 9 Feb 2022 23:48:26 +0100 Subject: [PATCH] feat: bundle Jest's modules with webpack [skip ci] --- package.json | 6 +- .../jest-util/src/requireOrImportModule.ts | 4 +- scripts/build.js | 162 ++++-------------- scripts/buildUtils.js | 118 ++++++++++++- yarn.lock | 75 +++++++- 5 files changed, 233 insertions(+), 132 deletions(-) diff --git a/package.json b/package.json index 64974a50de2b..7ff78b8fc47a 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@typescript-eslint/parser": "^5.14.0", "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", + "babel-loader": "^8.2.3", "camelcase": "^6.2.0", "chalk": "^4.0.0", "chokidar": "^3.3.0", @@ -82,6 +83,8 @@ "ts-node": "^10.5.0", "type-fest": "^2.11.2", "typescript": "^4.2.4", + "webpack": "^5.68.0", + "webpack-node-externals": "^3.0.0", "which": "^2.0.1" }, "scripts": { @@ -168,7 +171,8 @@ "babel-jest": "workspace:*", "jest": "workspace:*", "jest-environment-node": "workspace:*", - "react-native": "patch:react-native@npm:0.68.0#.yarn/patches/react-native-npm-0.68.0-9eb3ecb60a.patch" + "react-native": "patch:react-native@npm:0.68.0#.yarn/patches/react-native-npm-0.68.0-9eb3ecb60a.patch", + "terser-webpack-plugin/jest-worker": "27.5.0" }, "packageManager": "yarn@3.2.0" } diff --git a/packages/jest-util/src/requireOrImportModule.ts b/packages/jest-util/src/requireOrImportModule.ts index 13311cc84b4e..bf97741e53be 100644 --- a/packages/jest-util/src/requireOrImportModule.ts +++ b/packages/jest-util/src/requireOrImportModule.ts @@ -30,7 +30,9 @@ export default async function requireOrImportModule( const moduleUrl = pathToFileURL(filePath); // node `import()` supports URL, but TypeScript doesn't know that - const importedModule = await import(moduleUrl.href); + const importedModule = await import( + /* webpackIgnore: true */ moduleUrl.href + ); if (!applyInteropRequireDefault) { return importedModule; diff --git a/scripts/build.js b/scripts/build.js index fa6c025ea2cf..330dfa1a1662 100644 --- a/scripts/build.js +++ b/scripts/build.js @@ -5,149 +5,57 @@ * LICENSE file in the root directory of this source tree. */ -/** - * script to build (transpile) files. - * By default it transpiles js files for all packages and writes them - * into `build/` directory. - * Non-js files not matching IGNORE_PATTERN will be copied without transpiling. - * - * Example: - * node ./scripts/build.js - * node ./scripts/build.js /users/123/jest/packages/jest-111/src/111.js - * - * NOTE: this script is node@6 compatible - */ - 'use strict'; const assert = require('assert'); const fs = require('fs'); const path = require('path'); -const babel = require('@babel/core'); +const util = require('util'); const chalk = require('chalk'); -const glob = require('glob'); -const micromatch = require('micromatch'); -const prettier = require('prettier'); -const transformOptions = require('../babel.config.js'); -const {getPackages, adjustToTerminalWidth, OK} = require('./buildUtils'); - -const SRC_DIR = 'src'; -const BUILD_DIR = 'build'; -const JS_FILES_PATTERN = '**/*.js'; -const TS_FILES_PATTERN = '**/*.ts'; -const IGNORE_PATTERN = '**/__{tests,mocks}__/**'; -const PACKAGES_DIR = path.resolve(__dirname, '../packages'); - -const INLINE_REQUIRE_EXCLUDE_LIST = - /packages\/expect|(jest-(circus|diff|get-type|jasmine2|matcher-utils|message-util|regex-util|snapshot))|pretty-format\//; - -const prettierConfig = prettier.resolveConfig.sync(__filename); -prettierConfig.trailingComma = 'none'; -prettierConfig.parser = 'babel'; - -function getPackageName(file) { - return path.relative(PACKAGES_DIR, file).split(path.sep)[0]; -} - -function getBuildPath(file, buildFolder) { - const pkgName = getPackageName(file); - const pkgSrcPath = path.resolve(PACKAGES_DIR, pkgName, SRC_DIR); - const pkgBuildPath = path.resolve(PACKAGES_DIR, pkgName, buildFolder); - const relativeToSrcPath = path.relative(pkgSrcPath, file); - return path.resolve(pkgBuildPath, relativeToSrcPath).replace(/\.ts$/, '.js'); -} +const webpack = require('webpack'); +const {createWebpackConfigs, OK, ERROR} = require('./buildUtils'); -function buildNodePackage({packageDir, pkg}) { - const srcDir = path.resolve(packageDir, SRC_DIR); - const pattern = path.resolve(srcDir, '**/*'); - const files = glob.sync(pattern, {nodir: true}); +async function buildNodePackages() { + process.stdout.write(chalk.inverse(' Bundling packages \n')); - process.stdout.write(adjustToTerminalWidth(`${pkg.name}\n`)); - - files.forEach(file => buildFile(file, true)); - - assert.ok( - fs.existsSync(path.resolve(packageDir, pkg.main)), - `Main file "${pkg.main}" in ${pkg.name} should exist`, + const webpackConfigs = createWebpackConfigs(); + const compiler = webpack( + webpackConfigs + .map(({webpackConfig}) => webpackConfig) + .filter(config => config != null), ); - process.stdout.write(`${OK}\n`); -} - -function buildFile(file, silent) { - const destPath = getBuildPath(file, BUILD_DIR); - - if (micromatch.isMatch(file, IGNORE_PATTERN)) { - silent || - process.stdout.write( - `${ - chalk.dim(' \u2022 ') + path.relative(PACKAGES_DIR, file) - } (ignore)\n`, - ); - return; - } + let stats; + try { + stats = await util.promisify(compiler.run.bind(compiler))(); + await util.promisify(compiler.close.bind(compiler))(); - fs.mkdirSync(path.dirname(destPath), {recursive: true}); - if ( - !micromatch.isMatch(file, JS_FILES_PATTERN) && - !micromatch.isMatch(file, TS_FILES_PATTERN) - ) { - fs.createReadStream(file).pipe(fs.createWriteStream(destPath)); - silent || - process.stdout.write( - `${ - chalk.red(' \u2022 ') + - path.relative(PACKAGES_DIR, file) + - chalk.red(' \u21D2 ') + - path.relative(PACKAGES_DIR, destPath) - } (copy)\n`, - ); - } else { - const options = Object.assign({}, transformOptions); - options.plugins = options.plugins.slice(); + assert.ok(!stats.hasErrors(), 'Must not have errors or warnings'); + } catch (error) { + process.stdout.write(`${ERROR}\n\n`); - if (INLINE_REQUIRE_EXCLUDE_LIST.test(file)) { - // The excluded modules are injected into the user's sandbox - // We need to guard some globals there. - options.plugins.push( - require.resolve('./babel-plugin-jest-native-globals'), - ); - } else { - options.plugins = options.plugins.map(plugin => { - if ( - Array.isArray(plugin) && - plugin[0] === '@babel/plugin-transform-modules-commonjs' - ) { - return [plugin[0], Object.assign({}, plugin[1], {lazy: true})]; - } + if (stats) { + const info = stats.toJson(); - return plugin; - }); + if (stats.hasErrors()) { + console.error('errors', info.errors); + } } - const transformed = babel.transformFileSync(file, options).code; - const prettyCode = prettier.format(transformed, prettierConfig); - - fs.writeFileSync(destPath, prettyCode); - - silent || - process.stdout.write( - `${ - chalk.green(' \u2022 ') + - path.relative(PACKAGES_DIR, file) + - chalk.green(' \u21D2 ') + - path.relative(PACKAGES_DIR, destPath) - }\n`, - ); + throw error; } -} -const files = process.argv.slice(2); + webpackConfigs.forEach(({packageDir, pkg}) => { + assert.ok( + fs.existsSync(path.resolve(packageDir, pkg.main)), + `Main file "${pkg.main}" in "${pkg.name}" should exist`, + ); + }); -if (files.length) { - files.forEach(file => buildFile(file)); -} else { - const packages = getPackages(); - process.stdout.write(chalk.inverse(' Building packages \n')); - packages.forEach(buildNodePackage); + process.stdout.write(`${OK}\n`); } + +buildNodePackages().catch(error => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/buildUtils.js b/scripts/buildUtils.js index 1136a369308d..dc6002a6cec8 100644 --- a/scripts/buildUtils.js +++ b/scripts/buildUtils.js @@ -13,14 +13,16 @@ const path = require('path'); const chalk = require('chalk'); const {sync: readPkg} = require('read-pkg'); const stringLength = require('string-length'); +const nodeExternals = require('webpack-node-externals'); const rootPackage = require('../package.json'); const PACKAGES_DIR = path.resolve(__dirname, '../packages'); const OK = chalk.reset.inverse.bold.green(' DONE '); +const ERROR = chalk.reset.inverse.bold.red(' BOOM '); // Get absolute paths of all directories under packages/* -module.exports.getPackages = function getPackages() { +function getPackages() { const packages = fs .readdirSync(PACKAGES_DIR) .map(file => path.resolve(PACKAGES_DIR, file)) @@ -104,7 +106,9 @@ module.exports.getPackages = function getPackages() { return {packageDir, pkg}; }); -}; +} + +module.exports.getPackages = getPackages; module.exports.adjustToTerminalWidth = function adjustToTerminalWidth(str) { const columns = process.stdout.columns || 80; @@ -117,5 +121,115 @@ module.exports.adjustToTerminalWidth = function adjustToTerminalWidth(str) { return strs.slice(0, -1).concat(lastString).join('\n'); }; +const INLINE_REQUIRE_EXCLUDE_LIST = + /packages\/expect|(jest-(circus|diff|get-type|jasmine2|matcher-utils|message-util|regex-util|snapshot))|pretty-format\//; + module.exports.OK = OK; +module.exports.ERROR = ERROR; module.exports.PACKAGES_DIR = PACKAGES_DIR; + +module.exports.INLINE_REQUIRE_EXCLUDE_LIST = INLINE_REQUIRE_EXCLUDE_LIST; + +function createWebpackConfigs() { + const babelConfig = require('../babel.config.js'); + const packages = getPackages(); + + return packages.map(({packageDir, pkg}) => { + const input = `${packageDir}/src/index.ts`; + + if (!fs.existsSync(input)) { + return {packageDir, pkg}; + } + + const options = Object.assign({}, babelConfig); + options.plugins = options.plugins.slice(); + + if (INLINE_REQUIRE_EXCLUDE_LIST.test(input)) { + // The excluded modules are injected into the user's sandbox + // We need to guard some globals there. + options.plugins.push( + require.resolve('./babel-plugin-jest-native-globals'), + ); + } else { + options.plugins = options.plugins.map(plugin => { + if ( + Array.isArray(plugin) && + plugin[0] === '@babel/plugin-transform-modules-commonjs' + ) { + return [plugin[0], Object.assign({}, plugin[1], {lazy: true})]; + } + + return plugin; + }); + } + + return { + packageDir, + pkg, + webpackConfig: { + devtool: false, + entry: input, + externals: nodeExternals(), + mode: 'production', + module: { + rules: [ + { + test: /.ts$/, + use: { + loader: 'babel-loader', + options, + }, + }, + ], + }, + output: { + filename: 'index.js', + library: { + type: 'commonjs2', + }, + path: path.resolve(packageDir, 'build'), + }, + plugins: [new IgnoreDynamicRequire()], + resolve: { + extensions: ['.ts', '.js'], + }, + target: 'node', + }, + }; + }); +} + +// inspired by https://framagit.org/Glandos/webpack-ignore-dynamic-require +class IgnoreDynamicRequire { + apply(compiler) { + compiler.hooks.normalModuleFactory.tap('IgnoreDynamicRequire', factory => { + factory.hooks.parser + .for('javascript/auto') + .tap('IgnoreDynamicRequire', parser => { + // This is a SyncBailHook, so returning anything stops the parser, and nothing (undefined) allows to continue + function ignoreRequireCallExpression(expression) { + if (expression.arguments.length === 0) { + return undefined; + } + const arg = parser.evaluateExpression(expression.arguments[0]); + if (arg.isString() && !arg.string.startsWith('.')) { + return true; + } + if (!arg.isString() && !arg.isConditional()) { + return true; + } + return undefined; + } + + parser.hooks.call + .for('require') + .tap('IgnoreDynamicRequire', ignoreRequireCallExpression); + parser.hooks.call + .for('require.resolve') + .tap('IgnoreDynamicRequire', ignoreRequireCallExpression); + }); + }); + } +} + +module.exports.createWebpackConfigs = createWebpackConfigs; diff --git a/yarn.lock b/yarn.lock index 7e1c7a58f9a2..5c3dfb061f7e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2696,6 +2696,7 @@ __metadata: "@typescript-eslint/parser": ^5.14.0 ansi-regex: ^5.0.1 ansi-styles: ^5.0.0 + babel-loader: ^8.2.3 camelcase: ^6.2.0 chalk: ^4.0.0 chokidar: ^3.3.0 @@ -2752,6 +2753,8 @@ __metadata: ts-node: ^10.5.0 type-fest: ^2.11.2 typescript: ^4.2.4 + webpack: ^5.68.0 + webpack-node-externals: ^3.0.0 which: ^2.0.1 languageName: unknown linkType: soft @@ -6701,6 +6704,21 @@ __metadata: languageName: unknown linkType: soft +"babel-loader@npm:^8.2.3": + version: 8.2.5 + resolution: "babel-loader@npm:8.2.5" + dependencies: + find-cache-dir: ^3.3.1 + loader-utils: ^2.0.0 + make-dir: ^3.1.0 + schema-utils: ^2.6.5 + peerDependencies: + "@babel/core": ^7.0.0 + webpack: ">=2" + checksum: a6605557885eabbc3250412405f2c63ca87287a95a439c643fdb47d5ea3d5326f72e43ab97be070316998cb685d5dfbc70927ce1abe8be7a6a4f5919287773fb + languageName: node + linkType: hard + "babel-loader@npm:^8.2.4": version: 8.2.4 resolution: "babel-loader@npm:8.2.4" @@ -13824,6 +13842,17 @@ __metadata: languageName: unknown linkType: soft +"jest-worker@npm:27.5.0": + version: 27.5.0 + resolution: "jest-worker@npm:27.5.0" + dependencies: + "@types/node": "*" + merge-stream: ^2.0.0 + supports-color: ^8.0.0 + checksum: bfd41bef36d3c217819278d8e53b7b9e02c32d90f54149ab4ec87595e389f5caca84237cc4c84050c93a435d458150876ce1812d68cd50a5a4cbb7d80286212f + languageName: node + linkType: hard + "jest-worker@npm:^26.0.0, jest-worker@npm:^26.2.1": version: 26.6.2 resolution: "jest-worker@npm:26.6.2" @@ -13835,7 +13864,7 @@ __metadata: languageName: node linkType: hard -"jest-worker@npm:^27.0.2, jest-worker@npm:^27.0.6, jest-worker@npm:^27.4.5, jest-worker@npm:^27.5.1": +"jest-worker@npm:^27.0.2, jest-worker@npm:^27.0.6, jest-worker@npm:^27.5.1": version: 27.5.1 resolution: "jest-worker@npm:27.5.1" dependencies: @@ -22646,6 +22675,13 @@ __metadata: languageName: node linkType: hard +"webpack-node-externals@npm:^3.0.0": + version: 3.0.0 + resolution: "webpack-node-externals@npm:3.0.0" + checksum: 355080c35c821115b97dda8c93d9d0565a90a6012a532324eb0d6a64f8f0d609431fd29504fc7ce414755841ac14f601f3eef99472c2c5dc00233b504ebe73f2 + languageName: node + linkType: hard + "webpack-sources@npm:^1.4.3": version: 1.4.3 resolution: "webpack-sources@npm:1.4.3" @@ -22663,6 +22699,43 @@ __metadata: languageName: node linkType: hard +"webpack@npm:^5.68.0": + version: 5.72.0 + resolution: "webpack@npm:5.72.0" + dependencies: + "@types/eslint-scope": ^3.7.3 + "@types/estree": ^0.0.51 + "@webassemblyjs/ast": 1.11.1 + "@webassemblyjs/wasm-edit": 1.11.1 + "@webassemblyjs/wasm-parser": 1.11.1 + acorn: ^8.4.1 + acorn-import-assertions: ^1.7.6 + browserslist: ^4.14.5 + chrome-trace-event: ^1.0.2 + enhanced-resolve: ^5.9.2 + es-module-lexer: ^0.9.0 + eslint-scope: 5.1.1 + events: ^3.2.0 + glob-to-regexp: ^0.4.1 + graceful-fs: ^4.2.9 + json-parse-better-errors: ^1.0.2 + loader-runner: ^4.2.0 + mime-types: ^2.1.27 + neo-async: ^2.6.2 + schema-utils: ^3.1.0 + tapable: ^2.1.1 + terser-webpack-plugin: ^5.1.3 + watchpack: ^2.3.1 + webpack-sources: ^3.2.3 + peerDependenciesMeta: + webpack-cli: + optional: true + bin: + webpack: bin/webpack.js + checksum: 8365f1466d0f7adbf80ebc9b780f263a28eeeabcd5fb515249bfd9a56ab7fe8d29ea53df3d9364d0732ab39ae774445eb28abce694ed375b13882a6b2fe93ffc + languageName: node + linkType: hard + "webpack@npm:^5.70.0": version: 5.70.0 resolution: "webpack@npm:5.70.0"