From 09af70743800aefdefe06e0ca32bcdde18f9eb77 Mon Sep 17 00:00:00 2001 From: Charles Lyding <19598772+clydin@users.noreply.github.com> Date: Wed, 21 Dec 2022 17:21:53 -0500 Subject: [PATCH] feat(@angular-devkit/build-angular): implement node module license extraction for esbuild builder When using the experimental esbuild-based browser application builder, the `--extract-licenses` option will now generate an output licenses file when enabled. This option extracts license information for each node module package included in the output files of the built code. This includes JavaScript and CSS output files. The esbuild metafile information generated during the bundling steps is used as the source of information regarding what input files where included and where they are located. A path segment of `node_modules` is used to indicate that a file belongs to a package and its license should be include in the output licenses file. The package name and license field are extracted from the `package.json` file for the package. If a license file (e.g., `LICENSE`) is present in the root of the package, it will also be included in the output licenses file. Custom licenses as defined by the recommended npm custom license text (`SEE LICENSE IN `) will also be extracted and included in the output license file. For additional information regarding the license field in a `package.json`, see https://docs.npmjs.com/cli/v9/configuring-npm/package-json#license. --- .../browser-esbuild/experimental-warnings.ts | 1 - .../src/builders/browser-esbuild/index.ts | 12 ++ .../browser-esbuild/license-extractor.ts | 176 ++++++++++++++++++ .../src/builders/browser-esbuild/options.ts | 2 + 4 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 packages/angular_devkit/build_angular/src/builders/browser-esbuild/license-extractor.ts diff --git a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/experimental-warnings.ts b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/experimental-warnings.ts index 1db7b0b52f31..9d0c7d309891 100644 --- a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/experimental-warnings.ts +++ b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/experimental-warnings.ts @@ -12,7 +12,6 @@ import { Schema as BrowserBuilderOptions } from '../browser/schema'; const UNSUPPORTED_OPTIONS: Array = [ 'allowedCommonJsDependencies', 'budgets', - 'extractLicenses', 'progress', 'scripts', diff --git a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/index.ts b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/index.ts index c54daf74f643..2229446f74ab 100644 --- a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/index.ts +++ b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/index.ts @@ -22,6 +22,7 @@ import { getSupportedBrowsers } from '../../utils/supported-browsers'; import { SourceFileCache, createCompilerPlugin } from './compiler-plugin'; import { bundle, logMessages } from './esbuild'; import { logExperimentalWarnings } from './experimental-warnings'; +import { extractLicenses } from './license-extractor'; import { NormalizedBrowserOptions, normalizeOptions } from './options'; import { shutdownSassWorkerPool } from './sass-plugin'; import { Schema as BrowserBuilderOptions } from './schema'; @@ -197,11 +198,20 @@ async function execute( await Promise.all( outputFiles.map((file) => fs.writeFile(path.join(outputPath, file.path), file.contents)), ); + // Write metafile if stats option is enabled if (options.stats) { await fs.writeFile(path.join(outputPath, 'stats.json'), JSON.stringify(metafile, null, 2)); } + // Extract and write licenses for used packages + if (options.extractLicenses) { + await fs.writeFile( + path.join(outputPath, '3rdpartylicenses.txt'), + await extractLicenses(metafile, workspaceRoot), + ); + } + // Augment the application with service worker support // TODO: This should eventually operate on the in-memory files prior to writing the output files if (serviceWorkerOptions) { @@ -269,6 +279,7 @@ function createCodeBundleOptions( conditions: ['es2020', 'es2015', 'module'], resolveExtensions: ['.ts', '.tsx', '.mjs', '.js'], metafile: true, + legalComments: options.extractLicenses ? 'none' : 'eof', logLevel: options.verbose ? 'debug' : 'silent', minify: optimizationOptions.scripts, pure: ['forwardRef'], @@ -397,6 +408,7 @@ function createGlobalStylesBundleOptions( includePaths: stylePreprocessorOptions?.includePaths, }); buildOptions.incremental = watch; + buildOptions.legalComments = options.extractLicenses ? 'none' : 'eof'; const namespace = 'angular:styles/global'; buildOptions.entryPoints = {}; diff --git a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/license-extractor.ts b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/license-extractor.ts new file mode 100644 index 000000000000..572da8f0d314 --- /dev/null +++ b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/license-extractor.ts @@ -0,0 +1,176 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import type { Metafile } from 'esbuild'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +/** + * The path segment used to signify that a file is part of a package. + */ +const NODE_MODULE_SEGMENT = 'node_modules'; + +/** + * String constant for the NPM recommended custom license wording. + * + * See: https://docs.npmjs.com/cli/v9/configuring-npm/package-json#license + * + * Example: + * ``` + * { + * "license" : "SEE LICENSE IN " + * } + * ``` + */ +const CUSTOM_LICENSE_TEXT = 'SEE LICENSE IN '; + +/** + * A list of commonly named license files found within packages. + */ +const LICENSE_FILES = ['LICENSE', 'LICENSE.txt', 'LICENSE.md']; + +/** + * Header text that will be added to the top of the output license extraction file. + */ +const EXTRACTION_FILE_HEADER = ''; + +/** + * The package entry separator to use within the output license extraction file. + */ +const EXTRACTION_FILE_SEPARATOR = '-'.repeat(80) + '\n'; + +/** + * Extracts license information for each node module package included in the output + * files of the built code. This includes JavaScript and CSS output files. The esbuild + * metafile generated during the bundling steps is used as the source of information + * regarding what input files where included and where they are located. A path segment + * of `node_modules` is used to indicate that a file belongs to a package and its license + * should be include in the output licenses file. + * + * The package name and license field are extracted from the `package.json` file for the + * package. If a license file (e.g., `LICENSE`) is present in the root of the package, it + * will also be included in the output licenses file. + * + * @param metafile An esbuild metafile object. + * @param rootDirectory The root directory of the workspace. + * @returns A string containing the content of the output licenses file. + */ +export async function extractLicenses(metafile: Metafile, rootDirectory: string) { + let extractedLicenseContent = `${EXTRACTION_FILE_HEADER}\n${EXTRACTION_FILE_SEPARATOR}`; + + const seenPaths = new Set(); + const seenPackages = new Set(); + + for (const entry of Object.values(metafile.outputs)) { + for (const [inputPath, { bytesInOutput }] of Object.entries(entry.inputs)) { + // Skip if not included in output + if (bytesInOutput <= 0) { + continue; + } + + // Skip already processed paths + if (seenPaths.has(inputPath)) { + continue; + } + seenPaths.add(inputPath); + + // Skip non-package paths + if (!inputPath.includes(NODE_MODULE_SEGMENT)) { + continue; + } + + // Extract the package name from the path + let baseDirectory = path.join(rootDirectory, inputPath); + let nameOrScope, nameOrFile; + let found = false; + while (baseDirectory !== path.dirname(baseDirectory)) { + const segment = path.basename(baseDirectory); + if (segment === NODE_MODULE_SEGMENT) { + found = true; + break; + } + + nameOrFile = nameOrScope; + nameOrScope = segment; + baseDirectory = path.dirname(baseDirectory); + } + + // Skip non-package path edge cases that are not caught in the includes check above + if (!found || !nameOrScope) { + continue; + } + + const packageName = nameOrScope.startsWith('@') + ? `${nameOrScope}/${nameOrFile}` + : nameOrScope; + const packageDirectory = path.join(baseDirectory, packageName); + + // Load the package's metadata to find the package's name, version, and license type + const packageJsonPath = path.join(packageDirectory, 'package.json'); + let packageJson; + try { + packageJson = JSON.parse(await readFile(packageJsonPath, 'utf-8')) as { + name: string; + version: string; + // The object form is deprecated and should only be present in old packages + license?: string | { type: string }; + }; + } catch { + // Invalid package + continue; + } + + // Skip already processed packages + const packageId = `${packageName}@${packageJson.version}`; + if (seenPackages.has(packageId)) { + continue; + } + seenPackages.add(packageId); + + // Attempt to find license text inside package + let licenseText = ''; + if ( + typeof packageJson.license === 'string' && + packageJson.license.toLowerCase().startsWith(CUSTOM_LICENSE_TEXT) + ) { + // Attempt to load the package's custom license + let customLicensePath; + const customLicenseFile = path.normalize( + packageJson.license.slice(CUSTOM_LICENSE_TEXT.length + 1).trim(), + ); + if (customLicenseFile.startsWith('..') || path.isAbsolute(customLicenseFile)) { + // Path is attempting to access files outside of the package + // TODO: Issue warning? + } else { + customLicensePath = path.join(packageDirectory, customLicenseFile); + try { + licenseText = await readFile(customLicensePath, 'utf-8'); + break; + } catch {} + } + } else { + // Search for a license file within the root of the package + for (const potentialLicense of LICENSE_FILES) { + const packageLicensePath = path.join(packageDirectory, potentialLicense); + try { + licenseText = await readFile(packageLicensePath, 'utf-8'); + break; + } catch {} + } + } + + // Generate the package's license entry in the output content + extractedLicenseContent += `Package: ${packageJson.name}\n`; + extractedLicenseContent += `License: ${JSON.stringify(packageJson.license, null, 2)}\n`; + extractedLicenseContent += `\n${licenseText}\n`; + extractedLicenseContent += EXTRACTION_FILE_SEPARATOR; + } + } + + return extractedLicenseContent; +} diff --git a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/options.ts b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/options.ts index c752b35e01af..d6d1196cdf6d 100644 --- a/packages/angular_devkit/build_angular/src/builders/browser-esbuild/options.ts +++ b/packages/angular_devkit/build_angular/src/builders/browser-esbuild/options.ts @@ -136,6 +136,7 @@ export async function normalizeOptions( buildOptimizer, crossOrigin, externalDependencies, + extractLicenses, inlineStyleLanguage = 'css', poll, preserveSymlinks, @@ -153,6 +154,7 @@ export async function normalizeOptions( cacheOptions, crossOrigin, externalDependencies, + extractLicenses, inlineStyleLanguage, stats: !!statsJson, poll,