From 2dc885304bc0722156e53f9d5f24423f92eea280 Mon Sep 17 00:00:00 2001 From: Charles <19598772+clydin@users.noreply.github.com> Date: Wed, 9 Oct 2019 14:25:23 -0400 Subject: [PATCH] refactor(@angular-devkit/build-angular): reorganize bundle processing for browser builder (#15776) --- .../build_angular/src/browser/action-cache.ts | 191 +++++++++++++++ .../src/browser/action-executor.ts | 111 +++++++-- .../build_angular/src/browser/index.ts | 226 ++---------------- 3 files changed, 296 insertions(+), 232 deletions(-) create mode 100644 packages/angular_devkit/build_angular/src/browser/action-cache.ts diff --git a/packages/angular_devkit/build_angular/src/browser/action-cache.ts b/packages/angular_devkit/build_angular/src/browser/action-cache.ts new file mode 100644 index 000000000000..f04e4053b07b --- /dev/null +++ b/packages/angular_devkit/build_angular/src/browser/action-cache.ts @@ -0,0 +1,191 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import { createHash } from 'crypto'; +import * as findCacheDirectory from 'find-cache-dir'; +import * as fs from 'fs'; +import { manglingDisabled } from '../utils/mangle-options'; +import { CacheKey, ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle'; + +const cacache = require('cacache'); +const cacheDownlevelPath = findCacheDirectory({ name: 'angular-build-dl' }); +const packageVersion = require('../../package.json').version; + +// Workaround Node.js issue prior to 10.16 with copyFile on macOS +// https://github.com/angular/angular-cli/issues/15544 & https://github.com/nodejs/node/pull/27241 +let copyFileWorkaround = false; +if (process.platform === 'darwin') { + const version = process.versions.node.split('.').map(part => Number(part)); + if (version[0] < 10 || version[0] === 11 || (version[0] === 10 && version[1] < 16)) { + copyFileWorkaround = true; + } +} + +export interface CacheEntry { + path: string; + size: number; + integrity?: string; +} + +export class BundleActionCache { + constructor(private readonly integrityAlgorithm?: string) {} + + static copyEntryContent(entry: CacheEntry | string, dest: fs.PathLike): void { + if (copyFileWorkaround) { + try { + fs.unlinkSync(dest); + } catch {} + } + + fs.copyFileSync( + typeof entry === 'string' ? entry : entry.path, + dest, + fs.constants.COPYFILE_FICLONE, + ); + if (process.platform !== 'win32') { + // The cache writes entries as readonly and when using copyFile the permissions will also be copied. + // See: https://github.com/npm/cacache/blob/073fbe1a9f789ba42d9a41de7b8429c93cf61579/lib/util/move-file.js#L36 + fs.chmodSync(dest, 0o644); + } + } + + generateBaseCacheKey(content: string): string { + // Create base cache key with elements: + // * package version - different build-angular versions cause different final outputs + // * code length/hash - ensure cached version matches the same input code + const algorithm = this.integrityAlgorithm || 'sha1'; + const codeHash = createHash(algorithm) + .update(content) + .digest('base64'); + let baseCacheKey = `${packageVersion}|${content.length}|${algorithm}-${codeHash}`; + if (manglingDisabled) { + baseCacheKey += '|MD'; + } + + return baseCacheKey; + } + + generateCacheKeys(action: ProcessBundleOptions): string[] { + const baseCacheKey = this.generateBaseCacheKey(action.code); + + // Postfix added to sourcemap cache keys when vendor sourcemaps are present + // Allows non-destructive caching of both variants + const SourceMapVendorPostfix = !!action.sourceMaps && action.vendorSourceMaps ? '|vendor' : ''; + + // Determine cache entries required based on build settings + const cacheKeys = []; + + // If optimizing and the original is not ignored, add original as required + if ((action.optimize || action.optimizeOnly) && !action.ignoreOriginal) { + cacheKeys[CacheKey.OriginalCode] = baseCacheKey + '|orig'; + + // If sourcemaps are enabled, add original sourcemap as required + if (action.sourceMaps) { + cacheKeys[CacheKey.OriginalMap] = baseCacheKey + SourceMapVendorPostfix + '|orig-map'; + } + } + // If not only optimizing, add downlevel as required + if (!action.optimizeOnly) { + cacheKeys[CacheKey.DownlevelCode] = baseCacheKey + '|dl'; + + // If sourcemaps are enabled, add downlevel sourcemap as required + if (action.sourceMaps) { + cacheKeys[CacheKey.DownlevelMap] = baseCacheKey + SourceMapVendorPostfix + '|dl-map'; + } + } + + return cacheKeys; + } + + async getCacheEntries(cacheKeys: (string | null)[]): Promise<(CacheEntry | null)[] | false> { + // Attempt to get required cache entries + const cacheEntries = []; + for (const key of cacheKeys) { + if (key) { + const entry = await cacache.get.info(cacheDownlevelPath, key); + if (!entry) { + return false; + } + cacheEntries.push({ + path: entry.path, + size: entry.size, + integrity: entry.metadata && entry.metadata.integrity, + }); + } else { + cacheEntries.push(null); + } + } + + return cacheEntries; + } + + async getCachedBundleResult(action: ProcessBundleOptions): Promise { + const entries = action.cacheKeys && await this.getCacheEntries(action.cacheKeys); + if (!entries) { + return null; + } + + const result: ProcessBundleResult = { name: action.name }; + + let cacheEntry = entries[CacheKey.OriginalCode]; + if (cacheEntry) { + result.original = { + filename: action.filename, + size: cacheEntry.size, + integrity: cacheEntry.integrity, + }; + + BundleActionCache.copyEntryContent(cacheEntry, result.original.filename); + + cacheEntry = entries[CacheKey.OriginalMap]; + if (cacheEntry) { + result.original.map = { + filename: action.filename + '.map', + size: cacheEntry.size, + }; + + BundleActionCache.copyEntryContent(cacheEntry, result.original.filename + '.map'); + } + } else if (!action.ignoreOriginal) { + // If the original wasn't processed (and therefore not cached), add info + result.original = { + filename: action.filename, + size: Buffer.byteLength(action.code, 'utf8'), + map: + action.map === undefined + ? undefined + : { + filename: action.filename + '.map', + size: Buffer.byteLength(action.map, 'utf8'), + }, + }; + } + + cacheEntry = entries[CacheKey.DownlevelCode]; + if (cacheEntry) { + result.downlevel = { + filename: action.filename.replace('es2015', 'es5'), + size: cacheEntry.size, + integrity: cacheEntry.integrity, + }; + + BundleActionCache.copyEntryContent(cacheEntry, result.downlevel.filename); + + cacheEntry = entries[CacheKey.DownlevelMap]; + if (cacheEntry) { + result.downlevel.map = { + filename: action.filename.replace('es2015', 'es5') + '.map', + size: cacheEntry.size, + }; + + BundleActionCache.copyEntryContent(cacheEntry, result.downlevel.filename + '.map'); + } + } + + return result; + } +} diff --git a/packages/angular_devkit/build_angular/src/browser/action-executor.ts b/packages/angular_devkit/build_angular/src/browser/action-executor.ts index 7471e885d626..8e24c3d5d27e 100644 --- a/packages/angular_devkit/build_angular/src/browser/action-executor.ts +++ b/packages/angular_devkit/build_angular/src/browser/action-executor.ts @@ -7,49 +7,110 @@ */ import JestWorker from 'jest-worker'; import * as os from 'os'; +import * as path from 'path'; +import { ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle'; +import { BundleActionCache } from './action-cache'; -export class ActionExecutor { - private largeWorker: JestWorker; - private smallWorker: JestWorker; +let workerFile = require.resolve('../utils/process-bundle'); +workerFile = + path.extname(workerFile) === '.ts' + ? require.resolve('../utils/process-bundle-bootstrap') + : workerFile; - private smallThreshold = 32 * 1024; +export class BundleActionExecutor { + private largeWorker?: JestWorker; + private smallWorker?: JestWorker; + private cache: BundleActionCache; + + constructor( + private workerOptions: unknown, + integrityAlgorithm?: string, + private readonly sizeThreshold = 32 * 1024, + ) { + this.cache = new BundleActionCache(integrityAlgorithm); + } + + private static executeMethod(worker: JestWorker, method: string, input: unknown): Promise { + return ((worker as unknown) as Record Promise>)[method](input); + } + + private ensureLarge(): JestWorker { + if (this.largeWorker) { + return this.largeWorker; + } - constructor(actionFile: string, private readonly actionName: string, setupOptions?: unknown) { // larger files are processed in a separate process to limit memory usage in the main process - this.largeWorker = new JestWorker(actionFile, { - exposedMethods: [actionName], - setupArgs: setupOptions === undefined ? undefined : [setupOptions], - }); + return (this.largeWorker = new JestWorker(workerFile, { + exposedMethods: ['process'], + setupArgs: [this.workerOptions], + })); + } + + private ensureSmall(): JestWorker { + if (this.smallWorker) { + return this.smallWorker; + } // small files are processed in a limited number of threads to improve speed // The limited number also prevents a large increase in memory usage for an otherwise short operation - this.smallWorker = new JestWorker(actionFile, { - exposedMethods: [actionName], - setupArgs: setupOptions === undefined ? undefined : [setupOptions], + return (this.smallWorker = new JestWorker(workerFile, { + exposedMethods: ['process'], + setupArgs: [this.workerOptions], numWorkers: os.cpus().length < 2 ? 1 : 2, // Will automatically fallback to processes if not supported enableWorkerThreads: true, - }); + })); } - execute(options: Input): Promise { - if (options.size > this.smallThreshold) { - return ((this.largeWorker as unknown) as Record Promise>)[ - this.actionName - ](options); + private executeAction(method: string, action: { code: string }): Promise { + // code.length is not an exact byte count but close enough for this + if (action.code.length > this.sizeThreshold) { + return BundleActionExecutor.executeMethod(this.ensureLarge(), method, action); } else { - return ((this.smallWorker as unknown) as Record Promise>)[ - this.actionName - ](options); + return BundleActionExecutor.executeMethod(this.ensureSmall(), method, action); } } - executeAll(options: Input[]): Promise { - return Promise.all(options.map(o => this.execute(o))); + async process(action: ProcessBundleOptions) { + const cacheKeys = this.cache.generateCacheKeys(action); + action.cacheKeys = cacheKeys; + + // Try to get cached data, if it fails fallback to processing + try { + const cachedResult = await this.cache.getCachedBundleResult(action); + if (cachedResult) { + return cachedResult; + } + } catch {} + + return this.executeAction('process', action); + } + + async *processAll(actions: Iterable) { + const executions = new Map, Promise>(); + for (const action of actions) { + const execution = this.process(action); + executions.set( + execution, + execution.then(result => { + executions.delete(execution); + + return result; + }), + ); + } + + while (executions.size > 0) { + yield Promise.race(executions.values()); + } } stop() { - this.largeWorker.end(); - this.smallWorker.end(); + if (this.largeWorker) { + this.largeWorker.end(); + } + if (this.smallWorker) { + this.smallWorker.end(); + } } } diff --git a/packages/angular_devkit/build_angular/src/browser/index.ts b/packages/angular_devkit/build_angular/src/browser/index.ts index 5d87691438e5..ff5c3e892d1a 100644 --- a/packages/angular_devkit/build_angular/src/browser/index.ts +++ b/packages/angular_devkit/build_angular/src/browser/index.ts @@ -67,9 +67,7 @@ import { } from '../utils'; import { copyAssets } from '../utils/copy-assets'; import { I18nOptions, createI18nOptions } from '../utils/i18n-options'; -import { manglingDisabled } from '../utils/mangle-options'; import { - CacheKey, ProcessBundleFile, ProcessBundleOptions, ProcessBundleResult, @@ -80,7 +78,7 @@ import { getIndexInputFile, getIndexOutputFile, } from '../utils/webpack-browser-config'; -import { ActionExecutor } from './action-executor'; +import { BundleActionExecutor } from './action-executor'; import { Schema as BrowserBuilderSchema } from './schema'; const cacache = require('cacache'); @@ -352,7 +350,8 @@ export function buildWebpackBrowser( } // If not optimizing then ES2015 polyfills do not need processing // Unlike other module scripts, it is never downleveled - if (!actionOptions.optimize && file.file.startsWith('polyfills-es2015')) { + const es2015Polyfills = file.file.startsWith('polyfills-es2015'); + if (!actionOptions.optimize && es2015Polyfills) { continue; } @@ -375,22 +374,6 @@ export function buildWebpackBrowser( filename = filename.replace('-es2015', ''); } - // ES2015 polyfills are only optimized; optimization check was performed above - if (file.file.startsWith('polyfills-es2015')) { - actions.push({ - ...actionOptions, - filename, - code, - map, - // id is always present for non-assets - // tslint:disable-next-line: no-non-null-assertion - name: file.id!, - optimizeOnly: true, - }); - - continue; - } - // Record the bundle processing action // The runtime chunk gets special processing for lazy loaded files actions.push({ @@ -403,8 +386,14 @@ export function buildWebpackBrowser( name: file.id!, runtime: file.file.startsWith('runtime'), ignoreOriginal: es5Polyfills, + optimizeOnly: es2015Polyfills, }); + // ES2015 polyfills are only optimized; optimization check was performed above + if (es2015Polyfills) { + continue; + } + // Add the newly created ES5 bundles to the index as nomodule scripts const newFilename = es5Polyfills ? file.file.replace('-es2015', '') @@ -417,205 +406,28 @@ export function buildWebpackBrowser( const processActions: typeof actions = []; let processRuntimeAction: ProcessBundleOptions | undefined; - const cacheActions: { src: string; dest: string }[] = []; const processResults: ProcessBundleResult[] = []; for (const action of actions) { - // Create base cache key with elements: - // * package version - different build-angular versions cause different final outputs - // * code length/hash - ensure cached version matches the same input code - const algorithm = action.integrityAlgorithm || 'sha1'; - const codeHash = createHash(algorithm) - .update(action.code) - .digest('base64'); - let baseCacheKey = `${packageVersion}|${action.code.length}|${algorithm}-${codeHash}`; - if (manglingDisabled) { - baseCacheKey += '|MD'; - } - - // Postfix added to sourcemap cache keys when vendor sourcemaps are present - // Allows non-destructive caching of both variants - const SourceMapVendorPostfix = - !!action.sourceMaps && action.vendorSourceMaps ? '|vendor' : ''; - - // Determine cache entries required based on build settings - const cacheKeys = []; - - // If optimizing and the original is not ignored, add original as required - if ((action.optimize || action.optimizeOnly) && !action.ignoreOriginal) { - cacheKeys[CacheKey.OriginalCode] = baseCacheKey + '|orig'; - - // If sourcemaps are enabled, add original sourcemap as required - if (action.sourceMaps) { - cacheKeys[CacheKey.OriginalMap] = - baseCacheKey + SourceMapVendorPostfix + '|orig-map'; - } - } - // If not only optimizing, add downlevel as required - if (!action.optimizeOnly) { - cacheKeys[CacheKey.DownlevelCode] = baseCacheKey + '|dl'; - - // If sourcemaps are enabled, add downlevel sourcemap as required - if (action.sourceMaps) { - cacheKeys[CacheKey.DownlevelMap] = - baseCacheKey + SourceMapVendorPostfix + '|dl-map'; - } - } - - // Attempt to get required cache entries - const cacheEntries = []; - let cached = cacheKeys.length > 0; - for (const key of cacheKeys) { - if (key) { - const entry = await cacache.get.info(cacheDownlevelPath, key); - if (!entry) { - cached = false; - break; - } - cacheEntries.push(entry); - } else { - cacheEntries.push(null); - } - } - - // If all required cached entries are present, use the cached entries - // Otherwise process the files // If SRI is enabled always process the runtime bundle // Lazy route integrity values are stored in the runtime bundle if (action.integrityAlgorithm && action.runtime) { processRuntimeAction = action; - } else if (cached) { - const result: ProcessBundleResult = { name: action.name }; - if (action.integrityAlgorithm) { - result.integrity = `${action.integrityAlgorithm}-${codeHash}`; - } - - let cacheEntry = cacheEntries[CacheKey.OriginalCode]; - if (cacheEntry) { - cacheActions.push({ - src: cacheEntry.path, - dest: action.filename, - }); - result.original = { - filename: action.filename, - size: cacheEntry.size, - integrity: cacheEntry.metadata && cacheEntry.metadata.integrity, - }; - - cacheEntry = cacheEntries[CacheKey.OriginalMap]; - if (cacheEntry) { - cacheActions.push({ - src: cacheEntry.path, - dest: action.filename + '.map', - }); - result.original.map = { - filename: action.filename + '.map', - size: cacheEntry.size, - }; - } - } else if (!action.ignoreOriginal) { - // If the original wasn't processed (and therefore not cached), add info - result.original = { - filename: action.filename, - size: Buffer.byteLength(action.code, 'utf8'), - map: - action.map === undefined - ? undefined - : { - filename: action.filename + '.map', - size: Buffer.byteLength(action.map, 'utf8'), - }, - }; - } - - cacheEntry = cacheEntries[CacheKey.DownlevelCode]; - if (cacheEntry) { - cacheActions.push({ - src: cacheEntry.path, - dest: action.filename.replace('es2015', 'es5'), - }); - result.downlevel = { - filename: action.filename.replace('es2015', 'es5'), - size: cacheEntry.size, - integrity: cacheEntry.metadata && cacheEntry.metadata.integrity, - }; - - cacheEntry = cacheEntries[CacheKey.DownlevelMap]; - if (cacheEntry) { - cacheActions.push({ - src: cacheEntry.path, - dest: action.filename.replace('es2015', 'es5') + '.map', - }); - result.downlevel.map = { - filename: action.filename.replace('es2015', 'es5') + '.map', - size: cacheEntry.size, - }; - } - } - - processResults.push(result); - } else if (action.runtime) { - processRuntimeAction = { - ...action, - cacheKeys, - }; } else { - processActions.push({ - ...action, - cacheKeys, - }); - } - } - - // Workaround Node.js issue prior to 10.16 with copyFile on macOS - // https://github.com/angular/angular-cli/issues/15544 & https://github.com/nodejs/node/pull/27241 - let copyFileWorkaround = false; - if (process.platform === 'darwin') { - const version = process.versions.node.split('.').map(part => Number(part)); - if ( - version[0] < 10 || - version[0] === 11 || - (version[0] === 10 && version[1] < 16) - ) { - copyFileWorkaround = true; + processActions.push(action); } } - for (const action of cacheActions) { - if (copyFileWorkaround) { - try { - fs.unlinkSync(action.dest); - } catch {} - } - - fs.copyFileSync(action.src, action.dest, fs.constants.COPYFILE_FICLONE); - if (process.platform !== 'win32') { - // The cache writes entries as readonly and when using copyFile the permissions will also be copied. - // See: https://github.com/npm/cacache/blob/073fbe1a9f789ba42d9a41de7b8429c93cf61579/lib/util/move-file.js#L36 - fs.chmodSync(action.dest, 0o644); - } - } - - if (processActions.length > 0) { - const workerFile = require.resolve('../utils/process-bundle'); - const executor = new ActionExecutor< - ProcessBundleOptions & { size: number }, - ProcessBundleResult - >( - path.extname(workerFile) !== '.ts' - ? workerFile - : require.resolve('../utils/process-bundle-bootstrap'), - 'process', - { cachePath: cacheDownlevelPath, i18n }, - ); + const executor = new BundleActionExecutor( + { cachePath: cacheDownlevelPath, i18n }, + options.subresourceIntegrity ? 'sha384' : undefined, + ); - try { - const results = await executor.executeAll( - processActions.map(a => ({ ...a, size: a.code.length })), - ); - results.forEach(result => processResults.push(result)); - } finally { - executor.stop(); + try { + for await (const result of executor.processAll(processActions)) { + processResults.push(result); } + } finally { + executor.stop(); } // Runtime must be processed after all other files