From 31b4c1a925e3978e105c7a174cba2d2b8000e235 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 10 Sep 2024 23:46:38 +0000 Subject: [PATCH 1/4] Allow for a custom link creation script path to be used for the create-links.js file --- common/reviews/api/package-extractor.api.md | 1 + .../package-extractor/src/PackageExtractor.ts | 64 ++++++--- .../src/scripts/create-links.ts | 23 +-- .../src/test/PackageExtractor.test.ts | 135 +++++++++++++++++- 4 files changed, 193 insertions(+), 30 deletions(-) diff --git a/common/reviews/api/package-extractor.api.md b/common/reviews/api/package-extractor.api.md index 75ce36ce25d..406f7119c33 100644 --- a/common/reviews/api/package-extractor.api.md +++ b/common/reviews/api/package-extractor.api.md @@ -31,6 +31,7 @@ export interface IExtractorOptions { includeDevDependencies?: boolean; includeNpmIgnoreFiles?: boolean; linkCreation?: 'default' | 'script' | 'none'; + linkCreationScriptPath?: string; mainProjectName: string; overwriteExisting: boolean; pnpmInstallFolder?: string; diff --git a/libraries/package-extractor/src/PackageExtractor.ts b/libraries/package-extractor/src/PackageExtractor.ts index b38bee90483..c0ba0c6687b 100644 --- a/libraries/package-extractor/src/PackageExtractor.ts +++ b/libraries/package-extractor/src/PackageExtractor.ts @@ -15,7 +15,6 @@ import { FileSystem, Import, JsonFile, - AlreadyExistsBehavior, type IPackageJson } from '@rushstack/node-core-library'; import { Colorize, type ITerminal } from '@rushstack/terminal'; @@ -36,6 +35,9 @@ declare module 'npm-packlist' { } } +export const TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING: '{TARGET_ROOT_SCRIPT_RELATIVE_PATH}' = + '{TARGET_ROOT_SCRIPT_RELATIVE_PATH}'; + /** * Part of the extractor-matadata.json file format. Represents an extracted project. * @@ -188,7 +190,7 @@ export interface IExtractorOptions { /** * Whether to skip copying files to the extraction target directory, and only create an extraction - * archive. This is only supported when linkCreation is 'script' or 'none'. + * archive. This is only supported when {@link IExtractorOptions.linkCreation} is 'script' or 'none'. */ createArchiveOnly?: boolean; @@ -224,6 +226,12 @@ export interface IExtractorOptions { */ linkCreation?: 'default' | 'script' | 'none'; + /** + * The path to the generated link creation script. This is only used when {@link IExtractorOptions.linkCreation} + * is 'script'. + */ + linkCreationScriptPath?: string; + /** * An additional folder containing files which will be copied into the root of the extraction. */ @@ -363,7 +371,9 @@ export class PackageExtractor { sourceRootFolder, targetRootFolder, folderToCopy: additionalFolderToCopy, - linkCreation + linkCreation, + linkCreationScriptPath, + createArchiveOnly } = options; const { projectConfigurationsByName, foldersToCopy, symlinkAnalyzer } = state; @@ -395,7 +405,7 @@ export class PackageExtractor { await this._collectFoldersAsync(projectFolder, options, state); } - if (!options.createArchiveOnly) { + if (!createArchiveOnly) { terminal.writeLine(`Copying folders to target folder "${targetRootFolder}"`); } await Async.forEachAsync( @@ -415,24 +425,30 @@ export class PackageExtractor { const additionalFolderExtractorOptions: IExtractorOptions = { ...options, sourceRootFolder: additionalFolderPath, - targetRootFolder: options.targetRootFolder + targetRootFolder }; await this._extractFolderAsync(additionalFolderPath, additionalFolderExtractorOptions, state); } switch (linkCreation) { case 'script': { - const sourceFilePath: string = path.join(scriptsFolderPath, createLinksScriptFilename); - if (!options.createArchiveOnly) { - terminal.writeLine(`Creating ${createLinksScriptFilename}`); - await FileSystem.copyFileAsync({ - sourcePath: sourceFilePath, - destinationPath: path.join(targetRootFolder, createLinksScriptFilename), - alreadyExistsBehavior: AlreadyExistsBehavior.Error + terminal.writeLine(`Creating ${createLinksScriptFilename}`); + const createLinksSourceFilePath: string = path.join(scriptsFolderPath, createLinksScriptFilename); + const createLinksTargetFilePath: string = linkCreationScriptPath + ? path.resolve(targetRootFolder, linkCreationScriptPath) + : path.join(targetRootFolder, createLinksScriptFilename); + let createLinksScriptContent: string = await FileSystem.readFileAsync(createLinksSourceFilePath); + createLinksScriptContent = createLinksScriptContent.replace( + TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING, + Path.convertToSlashes(path.relative(path.dirname(createLinksTargetFilePath), targetRootFolder)) + ); + if (!createArchiveOnly) { + await FileSystem.writeFileAsync(createLinksTargetFilePath, createLinksScriptContent, { + ensureFolderExists: true }); } await state.archiver?.addToArchiveAsync({ - filePath: sourceFilePath, + fileData: createLinksTargetFilePath, archivePath: createLinksScriptFilename }); break; @@ -951,11 +967,18 @@ export class PackageExtractor { options: IExtractorOptions, state: IExtractorState ): Promise { - const { mainProjectName, targetRootFolder } = options; + const { mainProjectName, targetRootFolder, linkCreation, linkCreationScriptPath } = options; const { projectConfigurationsByPath } = state; const extractorMetadataFileName: string = 'extractor-metadata.json'; - const extractorMetadataFilePath: string = path.join(targetRootFolder, extractorMetadataFileName); + const extractorMetadataFolderPath: string = + linkCreation === 'script' && linkCreationScriptPath + ? path.dirname(path.resolve(targetRootFolder, linkCreationScriptPath)) + : targetRootFolder; + const extractorMetadataFilePath: string = path.join( + extractorMetadataFolderPath, + extractorMetadataFileName + ); const extractorMetadataJson: IExtractorMetadataJson = { mainProjectName, projects: [], @@ -983,12 +1006,13 @@ export class PackageExtractor { const extractorMetadataFileContent: string = JSON.stringify(extractorMetadataJson, undefined, 0); if (!options.createArchiveOnly) { - await FileSystem.writeFileAsync(extractorMetadataFilePath, extractorMetadataFileContent); + await FileSystem.writeFileAsync(extractorMetadataFilePath, extractorMetadataFileContent, { + ensureFolderExists: true + }); } - await state.archiver?.addToArchiveAsync({ - fileData: extractorMetadataFileContent, - archivePath: extractorMetadataFileName - }); + + const archivePath: string = path.relative(targetRootFolder, extractorMetadataFilePath); + await state.archiver?.addToArchiveAsync({ archivePath, fileData: extractorMetadataFileContent }); } private async _makeBinLinksAsync(options: IExtractorOptions, state: IExtractorState): Promise { diff --git a/libraries/package-extractor/src/scripts/create-links.ts b/libraries/package-extractor/src/scripts/create-links.ts index 2c65be9b36f..f6ebcdc2988 100644 --- a/libraries/package-extractor/src/scripts/create-links.ts +++ b/libraries/package-extractor/src/scripts/create-links.ts @@ -7,8 +7,15 @@ import * as fs from 'fs'; import * as path from 'path'; -import type { IExtractorMetadataJson } from '../PackageExtractor'; import type { IFileSystemCreateLinkOptions } from '@rushstack/node-core-library'; +import type { + TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING as TargetRootScriptRelativePathTemplateString, + IExtractorMetadataJson +} from '../PackageExtractor'; + +const TARGET_ROOT_SCRIPT_RELATIVE_PATH: typeof TargetRootScriptRelativePathTemplateString = + '{TARGET_ROOT_SCRIPT_RELATIVE_PATH}'; +const TARGET_ROOT_FOLDER: string = path.resolve(__dirname, TARGET_ROOT_SCRIPT_RELATIVE_PATH); // API borrowed from @rushstack/node-core-library, since this script avoids using any // NPM dependencies. @@ -105,9 +112,7 @@ function main(): boolean { return false; } - const targetRootFolder: string = __dirname; - const extractorMetadataPath: string = path.join(targetRootFolder, 'extractor-metadata.json'); - + const extractorMetadataPath: string = path.join(__dirname, 'extractor-metadata.json'); if (!fs.existsSync(extractorMetadataPath)) { throw new Error('Input file not found: ' + extractorMetadataPath); } @@ -116,12 +121,12 @@ function main(): boolean { const extractorMetadataObject: IExtractorMetadataJson = JSON.parse(extractorMetadataJson); if (args[0] === 'create') { - console.log(`\nCreating links for extraction at path "${targetRootFolder}"`); - removeLinks(targetRootFolder, extractorMetadataObject); - createLinks(targetRootFolder, extractorMetadataObject); + console.log(`\nCreating links for extraction at path "${TARGET_ROOT_FOLDER}"`); + removeLinks(TARGET_ROOT_FOLDER, extractorMetadataObject); + createLinks(TARGET_ROOT_FOLDER, extractorMetadataObject); } else { - console.log(`\nRemoving links for extraction at path "${targetRootFolder}"`); - removeLinks(targetRootFolder, extractorMetadataObject); + console.log(`\nRemoving links for extraction at path "${TARGET_ROOT_FOLDER}"`); + removeLinks(TARGET_ROOT_FOLDER, extractorMetadataObject); } console.log('The operation completed successfully.'); diff --git a/libraries/package-extractor/src/test/PackageExtractor.test.ts b/libraries/package-extractor/src/test/PackageExtractor.test.ts index db43c9b11b5..06bdf4ca67a 100644 --- a/libraries/package-extractor/src/test/PackageExtractor.test.ts +++ b/libraries/package-extractor/src/test/PackageExtractor.test.ts @@ -2,8 +2,9 @@ // See LICENSE in the project root for license information. import path from 'path'; +import type { ChildProcess } from 'child_process'; -import { FileSystem } from '@rushstack/node-core-library'; +import { Executable, FileSystem } from '@rushstack/node-core-library'; import { Terminal, StringBufferTerminalProvider } from '@rushstack/terminal'; import { PackageExtractor, type IExtractorProjectConfiguration } from '../PackageExtractor'; @@ -464,4 +465,136 @@ describe(PackageExtractor.name, () => { await expect(FileSystem.existsAsync(path.join(targetFolder, 'package.json'))).resolves.toBe(true); await expect(FileSystem.existsAsync(path.join(targetFolder, 'src', 'index.js'))).resolves.toBe(true); }); + + it('should extract project with script linkCreation', async () => { + const targetFolder: string = path.join(extractorTargetFolder, 'extractor-output-10'); + + await expect( + packageExtractor.extractAsync({ + mainProjectName: project1PackageName, + sourceRootFolder: repoRoot, + targetRootFolder: targetFolder, + overwriteExisting: true, + projectConfigurations: getDefaultProjectConfigurations(), + terminal, + includeNpmIgnoreFiles: true, + linkCreation: 'script', + includeDevDependencies: true + }) + ).resolves.not.toThrow(); + + // Validate project 1 files + await expect( + FileSystem.existsAsync(path.join(targetFolder, project1RelativePath, 'src', 'index.js')) + ).resolves.toBe(true); + + // Validate project 2 is not linked through node_modules + const project1NodeModulesPath: string = path.join(targetFolder, project1RelativePath, 'node_modules'); + await expect( + FileSystem.existsAsync(path.join(project1NodeModulesPath, project2PackageName, 'src', 'index.js')) + ).resolves.toEqual(false); + + // Validate project 3 is not linked through node_modules + await expect( + FileSystem.existsAsync(path.join(project1NodeModulesPath, project3PackageName, 'src', 'index.js')) + ).resolves.toEqual(false); + + // Run the linkCreation script + const createLinksProcess: ChildProcess = Executable.spawn(process.argv0, [ + path.join(targetFolder, 'create-links.js'), + 'create' + ]); + await expect( + Executable.waitForExitAsync(createLinksProcess, { throwOnNonZeroExitCode: true }) + ).resolves.not.toThrow(); + + // Validate project 2 is linked through node_modules + await expect( + FileSystem.getRealPathAsync(path.join(project1NodeModulesPath, project2PackageName, 'src', 'index.js')) + ).resolves.toEqual(path.join(targetFolder, project2RelativePath, 'src', 'index.js')); + + // Validate project 3 is linked through node_modules + await expect( + FileSystem.getRealPathAsync(path.join(project1NodeModulesPath, project3PackageName, 'src', 'index.js')) + ).resolves.toEqual(path.join(targetFolder, project3RelativePath, 'src', 'index.js')); + await expect( + FileSystem.getRealPathAsync( + path.join( + project1NodeModulesPath, + project2PackageName, + 'node_modules', + project3PackageName, + 'src', + 'index.js' + ) + ) + ).resolves.toEqual(path.join(targetFolder, project3RelativePath, 'src', 'index.js')); + }); + + it('should extract project with script linkCreation and custom linkCreationScriptPath', async () => { + const targetFolder: string = path.join(extractorTargetFolder, 'extractor-output-11'); + const linkCreationScriptPath: string = path.join(targetFolder, 'foo', 'bar', 'baz.js'); + + await expect( + packageExtractor.extractAsync({ + mainProjectName: project1PackageName, + sourceRootFolder: repoRoot, + targetRootFolder: targetFolder, + overwriteExisting: true, + projectConfigurations: getDefaultProjectConfigurations(), + terminal, + includeNpmIgnoreFiles: true, + linkCreation: 'script', + linkCreationScriptPath, + includeDevDependencies: true + }) + ).resolves.not.toThrow(); + + // Validate project 1 files + await expect( + FileSystem.existsAsync(path.join(targetFolder, project1RelativePath, 'src', 'index.js')) + ).resolves.toBe(true); + + // Validate project 2 is not linked through node_modules + const project1NodeModulesPath: string = path.join(targetFolder, project1RelativePath, 'node_modules'); + await expect( + FileSystem.existsAsync(path.join(project1NodeModulesPath, project2PackageName, 'src', 'index.js')) + ).resolves.toEqual(false); + + // Validate project 3 is not linked through node_modules + await expect( + FileSystem.existsAsync(path.join(project1NodeModulesPath, project3PackageName, 'src', 'index.js')) + ).resolves.toEqual(false); + + // Run the linkCreation script + const createLinksProcess: ChildProcess = Executable.spawn(process.argv0, [ + linkCreationScriptPath, + 'create' + ]); + await expect( + Executable.waitForExitAsync(createLinksProcess, { throwOnNonZeroExitCode: true }) + ).resolves.not.toThrow(); + + // Validate project 2 is linked through node_modules + await expect( + FileSystem.getRealPathAsync(path.join(project1NodeModulesPath, project2PackageName, 'src', 'index.js')) + ).resolves.toEqual(path.join(targetFolder, project2RelativePath, 'src', 'index.js')); + + // Validate project 3 is linked through node_modules + await expect( + FileSystem.getRealPathAsync(path.join(project1NodeModulesPath, project3PackageName, 'src', 'index.js')) + ).resolves.toEqual(path.join(targetFolder, project3RelativePath, 'src', 'index.js')); + await expect( + FileSystem.getRealPathAsync( + path.join( + project1NodeModulesPath, + project2PackageName, + 'node_modules', + project3PackageName, + 'src', + 'index.js' + ) + ) + ).resolves.toEqual(path.join(targetFolder, project3RelativePath, 'src', 'index.js')); + }); }); From c5b22fa2954558dbb50010a8e91d03adaaf13d46 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 10 Sep 2024 23:53:10 +0000 Subject: [PATCH 2/4] Rush change --- ...-CustomLinkCreationScriptPath_2024-09-10-23-52.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 common/changes/@rushstack/package-extractor/user-danade-CustomLinkCreationScriptPath_2024-09-10-23-52.json diff --git a/common/changes/@rushstack/package-extractor/user-danade-CustomLinkCreationScriptPath_2024-09-10-23-52.json b/common/changes/@rushstack/package-extractor/user-danade-CustomLinkCreationScriptPath_2024-09-10-23-52.json new file mode 100644 index 00000000000..18fa3574437 --- /dev/null +++ b/common/changes/@rushstack/package-extractor/user-danade-CustomLinkCreationScriptPath_2024-09-10-23-52.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/package-extractor", + "comment": "Add the ability to change where the \"create-links.js\" file and the associated metadata file are generated when running in the \"script\" linkCreation mode", + "type": "minor" + } + ], + "packageName": "@rushstack/package-extractor" +} \ No newline at end of file From 9711a683d5e541b47c6dabbdaf3d9d9939ecbe8b Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 10 Sep 2024 23:59:04 +0000 Subject: [PATCH 3/4] Fix creation of zip file with custom script path --- libraries/package-extractor/src/PackageExtractor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/package-extractor/src/PackageExtractor.ts b/libraries/package-extractor/src/PackageExtractor.ts index c0ba0c6687b..3a4ad9e20ec 100644 --- a/libraries/package-extractor/src/PackageExtractor.ts +++ b/libraries/package-extractor/src/PackageExtractor.ts @@ -448,8 +448,8 @@ export class PackageExtractor { }); } await state.archiver?.addToArchiveAsync({ - fileData: createLinksTargetFilePath, - archivePath: createLinksScriptFilename + fileData: createLinksScriptContent, + archivePath: path.relative(targetRootFolder, createLinksTargetFilePath) }); break; } From e0d3c32319440218820b08cab5c2e1ecb4d3c948 Mon Sep 17 00:00:00 2001 From: Daniel <3473356+D4N14L@users.noreply.github.com> Date: Wed, 11 Sep 2024 10:17:51 -0700 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Ian Clanton-Thuon --- libraries/package-extractor/src/PackageExtractor.ts | 4 ++-- libraries/package-extractor/src/scripts/create-links.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/package-extractor/src/PackageExtractor.ts b/libraries/package-extractor/src/PackageExtractor.ts index 3a4ad9e20ec..bc5cdc99d32 100644 --- a/libraries/package-extractor/src/PackageExtractor.ts +++ b/libraries/package-extractor/src/PackageExtractor.ts @@ -433,10 +433,10 @@ export class PackageExtractor { switch (linkCreation) { case 'script': { terminal.writeLine(`Creating ${createLinksScriptFilename}`); - const createLinksSourceFilePath: string = path.join(scriptsFolderPath, createLinksScriptFilename); + const createLinksSourceFilePath: string = `${scriptsFolderPath}/${createLinksScriptFilename}`; const createLinksTargetFilePath: string = linkCreationScriptPath ? path.resolve(targetRootFolder, linkCreationScriptPath) - : path.join(targetRootFolder, createLinksScriptFilename); + : `${targetRootFolder}/${createLinksScriptFilename}`; let createLinksScriptContent: string = await FileSystem.readFileAsync(createLinksSourceFilePath); createLinksScriptContent = createLinksScriptContent.replace( TARGET_ROOT_SCRIPT_RELATIVE_PATH_TEMPLATE_STRING, diff --git a/libraries/package-extractor/src/scripts/create-links.ts b/libraries/package-extractor/src/scripts/create-links.ts index f6ebcdc2988..fde99a9aa7d 100644 --- a/libraries/package-extractor/src/scripts/create-links.ts +++ b/libraries/package-extractor/src/scripts/create-links.ts @@ -112,7 +112,7 @@ function main(): boolean { return false; } - const extractorMetadataPath: string = path.join(__dirname, 'extractor-metadata.json'); + const extractorMetadataPath: string = `${__dirname}/extractor-metadata.json`; if (!fs.existsSync(extractorMetadataPath)) { throw new Error('Input file not found: ' + extractorMetadataPath); }