From 12596cc925012318b299bb8f1553d9dd782667a1 Mon Sep 17 00:00:00 2001 From: Charles Lyding <19598772+clydin@users.noreply.github.com> Date: Tue, 14 Sep 2021 14:28:10 -0400 Subject: [PATCH] fix(@angular-devkit/build-angular): support WASM-based esbuild optimizer fallback In the event that the Angular CLI is executed on a platform that does not yet have native support for esbuild, the WASM-based variant of esbuild will now be used. If the first attempt to optimize a file fails to execute the native variant of esbuild, future executions will instead use the WASM-based variant instead which will execute regardless of the native platform. The WASM-based variant, unfortunately, can be significantly slower than the native version (some cases can be several times slower). For install time concerns regarding the esbuild post-install step, esbuild is now listed as an optional dependency which will allow the post-install step to fail but allow the full npm install to pass. This install scenario should only occur in the event that the esbuild native binary cannot be installed or is otherwise unavailable. --- package.json | 1 + .../angular_devkit/build_angular/BUILD.bazel | 2 + .../build_angular/esbuild-check.js | 16 ++ .../angular_devkit/build_angular/package.json | 5 +- .../webpack/plugins/css-optimizer-plugin.ts | 65 +++++--- .../src/webpack/plugins/esbuild-executor.ts | 131 +++++++++++++++ .../plugins/javascript-optimizer-plugin.ts | 5 + .../plugins/javascript-optimizer-worker.ts | 156 ++++++++++++++---- .../e2e/tests/build/esbuild-unsupported.ts | 11 ++ yarn.lock | 23 +-- 10 files changed, 348 insertions(+), 67 deletions(-) create mode 100644 packages/angular_devkit/build_angular/esbuild-check.js create mode 100644 packages/angular_devkit/build_angular/src/webpack/plugins/esbuild-executor.ts create mode 100644 tests/legacy-cli/e2e/tests/build/esbuild-unsupported.ts diff --git a/package.json b/package.json index 29386e73ebfa..53ca7fc8b0fe 100644 --- a/package.json +++ b/package.json @@ -145,6 +145,7 @@ "css-loader": "6.3.0", "debug": "^4.1.1", "esbuild": "0.12.28", + "esbuild-wasm": "0.12.28", "eslint": "7.32.0", "eslint-config-prettier": "8.3.0", "eslint-plugin-header": "3.1.1", diff --git a/packages/angular_devkit/build_angular/BUILD.bazel b/packages/angular_devkit/build_angular/BUILD.bazel index ff7517b77b30..54bf345fb49d 100644 --- a/packages/angular_devkit/build_angular/BUILD.bazel +++ b/packages/angular_devkit/build_angular/BUILD.bazel @@ -86,6 +86,7 @@ ts_library( include = [ "package.json", "builders.json", + "esbuild-check.js", "src/**/schema.json", "src/**/*.js", "src/**/*.html", @@ -143,6 +144,7 @@ ts_library( "@npm//critters", "@npm//css-loader", "@npm//esbuild", + "@npm//esbuild-wasm", "@npm//find-cache-dir", "@npm//glob", "@npm//https-proxy-agent", diff --git a/packages/angular_devkit/build_angular/esbuild-check.js b/packages/angular_devkit/build_angular/esbuild-check.js new file mode 100644 index 000000000000..10295ea96ba9 --- /dev/null +++ b/packages/angular_devkit/build_angular/esbuild-check.js @@ -0,0 +1,16 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +// If the platform does not support the native variant of esbuild, this will crash. +// This script can then be spawned by the CLI to determine if native usage is supported. +require('esbuild') + .formatMessages([], { kind: 'error ' }) + .then( + () => {}, + () => {}, + ); diff --git a/packages/angular_devkit/build_angular/package.json b/packages/angular_devkit/build_angular/package.json index 9d76e00c9ad2..4662b7407684 100644 --- a/packages/angular_devkit/build_angular/package.json +++ b/packages/angular_devkit/build_angular/package.json @@ -32,7 +32,7 @@ "core-js": "3.18.0", "critters": "0.0.10", "css-loader": "6.3.0", - "esbuild": "0.12.28", + "esbuild-wasm": "0.12.28", "find-cache-dir": "3.3.2", "glob": "7.1.7", "https-proxy-agent": "5.0.0", @@ -72,6 +72,9 @@ "webpack-merge": "5.8.0", "webpack-subresource-integrity": "5.0.0" }, + "optionalDependencies": { + "esbuild": "0.12.28" + }, "peerDependencies": { "@angular/compiler-cli": "^13.0.0 || ^13.0.0-next", "@angular/localize": "^13.0.0 || ^13.0.0-next", diff --git a/packages/angular_devkit/build_angular/src/webpack/plugins/css-optimizer-plugin.ts b/packages/angular_devkit/build_angular/src/webpack/plugins/css-optimizer-plugin.ts index 1ab09bee505a..a43eb74a3e4f 100644 --- a/packages/angular_devkit/build_angular/src/webpack/plugins/css-optimizer-plugin.ts +++ b/packages/angular_devkit/build_angular/src/webpack/plugins/css-optimizer-plugin.ts @@ -6,9 +6,11 @@ * found in the LICENSE file at https://angular.io/license */ -import { Message, formatMessages, transform } from 'esbuild'; +import type { Message, TransformResult } from 'esbuild'; import type { Compilation, Compiler, sources } from 'webpack'; import { addWarning } from '../../utils/webpack-diagnostics'; +import { EsbuildExecutor } from './esbuild-executor'; + /** * The name of the plugin provided to Webpack when tapping Webpack compiler hooks. */ @@ -26,6 +28,7 @@ export interface CssOptimizerPluginOptions { */ export class CssOptimizerPlugin { private targets: string[] | undefined; + private esbuild = new EsbuildExecutor(); constructor(options?: CssOptimizerPluginOptions) { if (options?.supportedBrowsers) { @@ -76,25 +79,13 @@ export class CssOptimizerPlugin { } const { source, map: inputMap } = styleAssetSource.sourceAndMap(); - let sourceMapLine; - if (inputMap) { - // esbuild will automatically remap the sourcemap if provided - sourceMapLine = `\n/*# sourceMappingURL=data:application/json;charset=utf-8;base64,${Buffer.from( - JSON.stringify(inputMap), - ).toString('base64')} */`; - } - const input = typeof source === 'string' ? source : source.toString(); - const { code, warnings, map } = await transform( - sourceMapLine ? input + sourceMapLine : input, - { - loader: 'css', - legalComments: 'inline', - minify: true, - sourcemap: !!inputMap && 'external', - sourcefile: asset.name, - target: this.targets, - }, + + const { code, warnings, map } = await this.optimize( + input, + asset.name, + inputMap, + this.targets, ); await this.addWarnings(compilation, warnings); @@ -114,9 +105,43 @@ export class CssOptimizerPlugin { }); } + /** + * Optimizes a CSS asset using esbuild. + * + * @param input The CSS asset source content to optimize. + * @param name The name of the CSS asset. Used to generate source maps. + * @param inputMap Optionally specifies the CSS asset's original source map that will + * be merged with the intermediate optimized source map. + * @param target Optionally specifies the target browsers for the output code. + * @returns A promise resolving to the optimized CSS, source map, and any warnings. + */ + private optimize( + input: string, + name: string, + inputMap: object, + target: string[] | undefined, + ): Promise { + let sourceMapLine; + if (inputMap) { + // esbuild will automatically remap the sourcemap if provided + sourceMapLine = `\n/*# sourceMappingURL=data:application/json;charset=utf-8;base64,${Buffer.from( + JSON.stringify(inputMap), + ).toString('base64')} */`; + } + + return this.esbuild.transform(sourceMapLine ? input + sourceMapLine : input, { + loader: 'css', + legalComments: 'inline', + minify: true, + sourcemap: !!inputMap && 'external', + sourcefile: name, + target, + }); + } + private async addWarnings(compilation: Compilation, warnings: Message[]) { if (warnings.length > 0) { - for (const warning of await formatMessages(warnings, { kind: 'warning' })) { + for (const warning of await this.esbuild.formatMessages(warnings, { kind: 'warning' })) { addWarning(compilation, warning); } } diff --git a/packages/angular_devkit/build_angular/src/webpack/plugins/esbuild-executor.ts b/packages/angular_devkit/build_angular/src/webpack/plugins/esbuild-executor.ts new file mode 100644 index 000000000000..ecc5875bc5e8 --- /dev/null +++ b/packages/angular_devkit/build_angular/src/webpack/plugins/esbuild-executor.ts @@ -0,0 +1,131 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import { spawnSync } from 'child_process'; +import type { + FormatMessagesOptions, + PartialMessage, + TransformOptions, + TransformResult, +} from 'esbuild'; +import * as path from 'path'; + +/** + * Provides the ability to execute esbuild regardless of the current platform's support + * for using the native variant of esbuild. The native variant will be preferred (assuming + * the `usingWasm` constructor option is `false) due to its inherent performance advantages. + * At first use of esbuild, a supportability test will be automatically performed and the + * WASM-variant will be used if needed by the platform. + */ +export class EsbuildExecutor + implements Pick +{ + private esbuildTransform: this['transform']; + private esbuildFormatMessages: this['formatMessages']; + private initialized = false; + + /** + * Constructs an instance of the `EsbuildExecutor` class. + * + * @param usingWasm If true, the WASM-variant will be preferred and no support test will be + * performed; if false (default), the native variant will be preferred. + */ + constructor(private usingWasm = false) { + this.esbuildTransform = this.esbuildFormatMessages = () => { + throw new Error('esbuild implementation missing'); + }; + } + + /** + * Determines whether the native variant of esbuild can be used on the current platform. + * + * @returns True, if the native variant of esbuild is support; False, if the WASM variant is required. + */ + static hasNativeSupport(): boolean { + // Try to use native variant to ensure it is functional for the platform. + // Spawning a separate esbuild check process is used to determine if the native + // variant is viable. If check fails, the WASM variant is initialized instead. + // Attempting to call one of the native esbuild functions is not a viable test + // currently since esbuild spawn errors are currently not propagated through the + // call stack for the esbuild function. If this limitation is removed in the future + // then the separate process spawn check can be removed in favor of a direct function + // call check. + try { + const { status, error } = spawnSync(process.execPath, [ + path.join(__dirname, '../../../esbuild-check.js'), + ]); + + return status === 0 && error === undefined; + } catch { + return false; + } + } + + /** + * Initializes the esbuild transform and format messages functions. + * + * @returns A promise that fulfills when esbuild has been loaded and available for use. + */ + private async ensureEsbuild(): Promise { + if (this.initialized) { + return; + } + + // If the WASM variant was preferred at class construction or native is not supported, use WASM + if (this.usingWasm || !EsbuildExecutor.hasNativeSupport()) { + await this.useWasm(); + this.initialized = true; + + return; + } + + try { + // Use the faster native variant if available. + const { transform, formatMessages } = await import('esbuild'); + + this.esbuildTransform = transform; + this.esbuildFormatMessages = formatMessages; + } catch { + // If the native variant is not installed then use the WASM-based variant + await this.useWasm(); + } + + this.initialized = true; + } + + /** + * Transitions an executor instance to use the WASM-variant of esbuild. + */ + private async useWasm(): Promise { + const { transform, formatMessages } = await import('esbuild-wasm'); + this.esbuildTransform = transform; + this.esbuildFormatMessages = formatMessages; + + // The ESBUILD_BINARY_PATH environment variable cannot exist when attempting to use the + // WASM variant. If it is then the binary located at the specified path will be used instead + // of the WASM variant. + delete process.env.ESBUILD_BINARY_PATH; + + this.usingWasm = true; + } + + async transform(input: string, options?: TransformOptions): Promise { + await this.ensureEsbuild(); + + return this.esbuildTransform(input, options); + } + + async formatMessages( + messages: PartialMessage[], + options: FormatMessagesOptions, + ): Promise { + await this.ensureEsbuild(); + + return this.esbuildFormatMessages(messages, options); + } +} diff --git a/packages/angular_devkit/build_angular/src/webpack/plugins/javascript-optimizer-plugin.ts b/packages/angular_devkit/build_angular/src/webpack/plugins/javascript-optimizer-plugin.ts index 0e7c4ba4ef59..428e80d36fed 100644 --- a/packages/angular_devkit/build_angular/src/webpack/plugins/javascript-optimizer-plugin.ts +++ b/packages/angular_devkit/build_angular/src/webpack/plugins/javascript-optimizer-plugin.ts @@ -10,6 +10,7 @@ import Piscina from 'piscina'; import { ScriptTarget } from 'typescript'; import type { Compiler, sources } from 'webpack'; import { maxWorkers } from '../../utils/environment-options'; +import { EsbuildExecutor } from './esbuild-executor'; /** * The maximum number of Workers that will be created to execute optimize tasks. @@ -160,6 +161,10 @@ export class JavaScriptOptimizerPlugin { target, removeLicenses: this.options.removeLicenses, advanced: this.options.advanced, + // Perform a single native esbuild support check. + // This removes the need for each worker to perform the check which would + // otherwise require spawning a separate process per worker. + usingWasmEsbuild: !EsbuildExecutor.hasNativeSupport(), }; // Sort scripts so larger scripts start first - worker pool uses a FIFO queue diff --git a/packages/angular_devkit/build_angular/src/webpack/plugins/javascript-optimizer-worker.ts b/packages/angular_devkit/build_angular/src/webpack/plugins/javascript-optimizer-worker.ts index 7d28e3924688..dd0cc9a0a9a7 100644 --- a/packages/angular_devkit/build_angular/src/webpack/plugins/javascript-optimizer-worker.ts +++ b/packages/angular_devkit/build_angular/src/webpack/plugins/javascript-optimizer-worker.ts @@ -7,8 +7,9 @@ */ import remapping from '@ampproject/remapping'; -import { TransformFailure, transform } from 'esbuild'; +import type { TransformFailure, TransformResult } from 'esbuild'; import { minify } from 'terser'; +import { EsbuildExecutor } from './esbuild-executor'; /** * A request to optimize JavaScript using the supplied options. @@ -18,61 +19,76 @@ interface OptimizeRequest { * The options to use when optimizing. */ options: { + /** + * Controls advanced optimizations. + * Currently these are only terser related: + * * terser compress passes are set to 2 + * * terser pure_getters option is enabled + */ advanced: boolean; + /** + * Specifies the string tokens that should be replaced with a defined value. + */ define?: Record; + /** + * Controls whether class, function, and variable names should be left intact + * throughout the output code. + */ keepNames: boolean; + /** + * Controls whether license text is removed from the output code. + * Within the CLI, this option is linked to the license extraction functionality. + */ removeLicenses: boolean; + /** + * Controls whether source maps should be generated. + */ sourcemap: boolean; + /** + * Specifies the target ECMAScript version for the output code. + */ target: 5 | 2015 | 2016 | 2017 | 2018 | 2019 | 2020; + /** + * Controls whether esbuild should only use the WASM-variant instead of trying to + * use the native variant. Some platforms may not support the native-variant and + * this option allows one support test to be conducted prior to all the workers starting. + */ + usingWasmEsbuild: boolean; }; /** * The JavaScript asset to optimize. */ asset: { + /** + * The name of the JavaScript asset (typically the filename). + */ name: string; + /** + * The source content off the JavaScript asset. + */ code: string; + /** + * The source map of the JavaScript asset, if available. + * This map is merged with all intermediate source maps during optimization. + */ map: object; }; } +/** + * The cached esbuild executor. + * This will automatically use the native or WASM version based on platform and availability + * with the native version given priority due to its superior performance. + */ +let esbuild: EsbuildExecutor | undefined; + +/** + * Handles optimization requests sent from the main thread via the `JavaScriptOptimizerPlugin`. + */ export default async function ({ asset, options }: OptimizeRequest) { // esbuild is used as a first pass - let esbuildResult; - try { - esbuildResult = await transform(asset.code, { - minifyIdentifiers: !options.keepNames, - minifySyntax: true, - // NOTE: Disabling whitespace ensures unused pure annotations are kept - minifyWhitespace: false, - pure: ['forwardRef'], - legalComments: options.removeLicenses ? 'none' : 'inline', - sourcefile: asset.name, - sourcemap: options.sourcemap && 'external', - define: options.define, - keepNames: options.keepNames, - target: `es${options.target}`, - }); - } catch (error) { - const failure = error as TransformFailure; - - // If esbuild fails with only ES5 support errors, fallback to just terser. - // This will only happen if ES5 is the output target and a global script contains ES2015+ syntax. - // In that case, the global script is technically already invalid for the target environment but - // this is and has been considered a configuration issue. Global scripts must be compatible with - // the target environment. - if ( - failure.errors?.every((error) => - error.text.includes('to the configured target environment ("es5") is not supported yet'), - ) - ) { - esbuildResult = { - code: asset.code, - }; - } else { - throw error; - } - } + const esbuildResult = await optimizeWithEsbuild(asset.code, asset.name, options); // terser is used as a second pass const terserResult = await optimizeWithTerser( @@ -106,6 +122,74 @@ export default async function ({ asset, options }: OptimizeRequest) { return { name: asset.name, code: terserResult.code, map: fullSourcemap }; } +/** + * Optimizes a JavaScript asset using esbuild. + * + * @param content The JavaScript asset source content to optimize. + * @param name The name of the JavaScript asset. Used to generate source maps. + * @param options The optimization request options to apply to the content. + * @returns A promise that resolves with the optimized code, source map, and any warnings. + */ +async function optimizeWithEsbuild( + content: string, + name: string, + options: OptimizeRequest['options'], +): Promise { + if (!esbuild) { + esbuild = new EsbuildExecutor(options.usingWasmEsbuild); + } + + let result: TransformResult; + try { + result = await esbuild.transform(content, { + minifyIdentifiers: !options.keepNames, + minifySyntax: true, + // NOTE: Disabling whitespace ensures unused pure annotations are kept + minifyWhitespace: false, + pure: ['forwardRef'], + legalComments: options.removeLicenses ? 'none' : 'inline', + sourcefile: name, + sourcemap: options.sourcemap && 'external', + define: options.define, + keepNames: options.keepNames, + target: `es${options.target}`, + }); + } catch (error) { + const failure = error as TransformFailure; + + // If esbuild fails with only ES5 support errors, fallback to just terser. + // This will only happen if ES5 is the output target and a global script contains ES2015+ syntax. + // In that case, the global script is technically already invalid for the target environment but + // this is and has been considered a configuration issue. Global scripts must be compatible with + // the target environment. + if ( + failure.errors?.every((error) => + error.text.includes('to the configured target environment ("es5") is not supported yet'), + ) + ) { + result = { + code: content, + map: '', + warnings: [], + }; + } else { + throw error; + } + } + + return result; +} + +/** + * Optimizes a JavaScript asset using terser. + * + * @param name The name of the JavaScript asset. Used to generate source maps. + * @param code The JavaScript asset source content to optimize. + * @param sourcemaps If true, generate an output source map for the optimized code. + * @param target Specifies the target ECMAScript version for the output code. + * @param advanced Controls advanced optimizations. + * @returns A promise that resolves with the optimized code and source map. + */ async function optimizeWithTerser( name: string, code: string, diff --git a/tests/legacy-cli/e2e/tests/build/esbuild-unsupported.ts b/tests/legacy-cli/e2e/tests/build/esbuild-unsupported.ts new file mode 100644 index 000000000000..0a3681549d3d --- /dev/null +++ b/tests/legacy-cli/e2e/tests/build/esbuild-unsupported.ts @@ -0,0 +1,11 @@ +import { join } from 'path'; +import { execWithEnv } from '../../utils/process'; + +export default async function () { + // Set the esbuild native binary path to a non-existent file to simulate a spawn error. + // The build should still succeed by falling back to the WASM variant of esbuild. + await execWithEnv('ng', ['build'], { + ...process.env, + 'ESBUILD_BINARY_PATH': join(__dirname, 'esbuild-bin-no-exist-xyz'), + }); +} diff --git a/yarn.lock b/yarn.lock index a9f1b5215587..c820a322fb2e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -115,8 +115,7 @@ "@angular/dev-infra-private@https://github.com/angular/dev-infra-private-builds.git#41d350a975685c83a04895f07a2edabe7abf60ae": version "0.0.0" - uid "41d350a975685c83a04895f07a2edabe7abf60ae" - resolved "https://github.com/angular/dev-infra-private-builds.git#41d350a975685c83a04895f07a2edabe7abf60ae" + resolved "https://github.com/angular/dev-infra-private-builds.git#f1e02ba26af835ba0b20b7ed1bdff55694500865" dependencies: "@actions/core" "^1.4.0" "@actions/github" "^5.0.0" @@ -129,7 +128,7 @@ "@bazel/protractor" "4.1.0" "@bazel/runfiles" "4.1.0" "@bazel/typescript" "4.1.0" - "@microsoft/api-extractor" "7.18.9" + "@microsoft/api-extractor" "7.18.7" "@octokit/auth-app" "^3.6.0" "@octokit/core" "^3.5.1" "@octokit/graphql" "^4.8.0" @@ -138,8 +137,6 @@ "@octokit/request-error" "^2.1.0" "@octokit/rest" "^18.7.0" "@octokit/types" "^6.16.6" - "@rollup/plugin-commonjs" "^20.0.0" - "@rollup/plugin-node-resolve" "^13.0.4" chalk "^4.1.0" clang-format "^1.4.0" cli-progress "^3.7.0" @@ -156,7 +153,9 @@ node-fetch "^2.6.1" prettier "^2.3.2" protractor "^7.0.0" - rollup "2.56.3" + rollup "^2.53.3" + rollup-plugin-commonjs "^10.1.0" + rollup-plugin-node-resolve "^5.2.0" rollup-plugin-sourcemaps "^0.6.3" selenium-webdriver "3.5.0" semver "^7.3.5" @@ -4212,9 +4211,9 @@ ejs@^3.1.6: jake "^10.6.1" electron-to-chromium@^1.3.830: - version "1.3.843" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.843.tgz#671489bd2f59fd49b76adddc1aa02c88cd38a5c0" - integrity sha512-OWEwAbzaVd1Lk9MohVw8LxMXFlnYd9oYTYxfX8KS++kLLjDfbovLOcEEXwRhG612dqGQ6+44SZvim0GXuBRiKg== + version "1.3.840" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.840.tgz#3f2a1df97015d9b1db5d86a4c6bd4cdb920adcbb" + integrity sha512-yRoUmTLDJnkIJx23xLY7GbSvnmDCq++NSuxHDQ0jiyDJ9YZBUGJcrdUqm+ZwZFzMbCciVzfem2N2AWiHJcWlbw== emoji-regex@^8.0.0: version "8.0.0" @@ -4400,6 +4399,11 @@ es6-weak-map@^2.0.3: es6-iterator "^2.0.3" es6-symbol "^3.1.1" +esbuild-wasm@0.12.28: + version "0.12.28" + resolved "https://registry.yarnpkg.com/esbuild-wasm/-/esbuild-wasm-0.12.28.tgz#4142a6bb1aeb1987c4b3596b247327d7bc6aa4a2" + integrity sha512-SiIIPHyPWaXRQ+IkAHeF5Pd9+n86mPqQG7mZAhGKn7Y6NfPqP1H+svVEG72pN8aoBZKxblB0ah210qQfhbQIfA== + esbuild@0.12.28, esbuild@^0.12.15: version "0.12.28" resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.12.28.tgz#84da0d2a0d0dee181281545271e0d65cf6fab1ef" @@ -9016,7 +9020,6 @@ sass@^1.32.8: "sauce-connect-proxy@https://saucelabs.com/downloads/sc-4.6.4-linux.tar.gz": version "0.0.0" - uid "992e2cb0d91e54b27a4f5bbd2049f3b774718115" resolved "https://saucelabs.com/downloads/sc-4.6.4-linux.tar.gz#992e2cb0d91e54b27a4f5bbd2049f3b774718115" saucelabs@^1.5.0: