From b4e8261abbc764a24ca87c446c515570119d6aad Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Wed, 6 May 2020 13:56:28 -0700 Subject: [PATCH 01/16] Initial support for workspaces --- apps/rush-lib/src/api/PackageJsonEditor.ts | 6 +- apps/rush-lib/src/api/RushConfiguration.ts | 13 + .../rush-lib/src/logic/DependencySpecifier.ts | 11 + apps/rush-lib/src/logic/InstallManager.ts | 596 +++++++++++++++++- .../src/logic/base/BaseWorkspaceFile.ts | 67 ++ .../src/logic/pnpm/PnpmWorkspaceFile.ts | 80 +++ apps/rush-lib/src/schemas/rush.schema.json | 4 + 7 files changed, 753 insertions(+), 24 deletions(-) create mode 100644 apps/rush-lib/src/logic/base/BaseWorkspaceFile.ts create mode 100644 apps/rush-lib/src/logic/pnpm/PnpmWorkspaceFile.ts diff --git a/apps/rush-lib/src/api/PackageJsonEditor.ts b/apps/rush-lib/src/api/PackageJsonEditor.ts index f308bc7415c..67acc9d5635 100644 --- a/apps/rush-lib/src/api/PackageJsonEditor.ts +++ b/apps/rush-lib/src/api/PackageJsonEditor.ts @@ -183,7 +183,11 @@ export class PackageJsonEditor { this._onChange.bind(this) ); - if (dependencyType === DependencyType.Regular || dependencyType === DependencyType.Optional) { + if ( + dependencyType === DependencyType.Regular || + dependencyType === DependencyType.Optional || + dependencyType === DependencyType.Peer + ) { this._dependencies.set(packageName, dependency); } else { this._devDependencies.set(packageName, dependency); diff --git a/apps/rush-lib/src/api/RushConfiguration.ts b/apps/rush-lib/src/api/RushConfiguration.ts index 96c313a3199..62b800395b4 100644 --- a/apps/rush-lib/src/api/RushConfiguration.ts +++ b/apps/rush-lib/src/api/RushConfiguration.ts @@ -169,6 +169,10 @@ export interface IPnpmOptionsJson extends IPackageManagerOptionsJsonBase { * Defines the dependency resolution strategy PNPM will use */ resolutionStrategy?: ResolutionStrategy; + /** + * Should we use the workspaces feature of PNPM to install and link packages? + */ + useWorkspaces?: boolean; } /** @@ -327,6 +331,14 @@ export class PnpmOptionsConfiguration extends PackageManagerOptionsConfiguration */ public readonly resolutionStrategy: ResolutionStrategy; + /** + * If true, then Rush will use the workspaces feature to install and link packages when invoking PNPM. + * + * @remarks + * The default value is false. (For now.) + */ + public readonly useWorkspaces: boolean; + /** @internal */ public constructor(json: IPnpmOptionsJson, commonTempFolder: string) { super(json); @@ -340,6 +352,7 @@ export class PnpmOptionsConfiguration extends PackageManagerOptionsConfiguration } this.strictPeerDependencies = !!json.strictPeerDependencies; this.resolutionStrategy = json.resolutionStrategy || 'fewer-dependencies'; + this.useWorkspaces = !!json.useWorkspaces; } } diff --git a/apps/rush-lib/src/logic/DependencySpecifier.ts b/apps/rush-lib/src/logic/DependencySpecifier.ts index 51f653aad58..fc40f1a6345 100644 --- a/apps/rush-lib/src/logic/DependencySpecifier.ts +++ b/apps/rush-lib/src/logic/DependencySpecifier.ts @@ -32,6 +32,7 @@ export class DependencySpecifier { * directory - A local directory * remote - An HTTP url to a .tar.gz, .tar or .tgz file * alias - A package alias such as "npm:other-package@^1.2.3" + * workspace - A package specified using workspace protocol such as "workspace:^1.2.3" */ public readonly specifierType: string; @@ -44,6 +45,16 @@ export class DependencySpecifier { public constructor(packageName: string, versionSpecifier: string) { this.packageName = packageName; + + // Workspace ranges are a feature from PNPM and Yarn. Set the version specifier + // to the trimmed version range. + if (versionSpecifier.startsWith('workspace:')) { + this.specifierType = 'workspace' + this.versionSpecifier = versionSpecifier.slice(this.specifierType.length); + this.aliasTarget = undefined; + return; + } + this.versionSpecifier = versionSpecifier; const result: npmPackageArg.AliasResult = npmPackageArg.resolve( diff --git a/apps/rush-lib/src/logic/InstallManager.ts b/apps/rush-lib/src/logic/InstallManager.ts index 6e634b531f9..845714c7c9d 100644 --- a/apps/rush-lib/src/logic/InstallManager.ts +++ b/apps/rush-lib/src/logic/InstallManager.ts @@ -60,6 +60,7 @@ import { PackageManagerName } from '../api/packageManager/PackageManager'; import { PnpmPackageManager } from '../api/packageManager/PnpmPackageManager'; import { DependencySpecifier } from './DependencySpecifier'; import { EnvironmentConfiguration } from '../api/EnvironmentConfiguration'; +import { PnpmWorkspaceFile } from './pnpm/PnpmWorkspaceFile'; // eslint-disable-next-line @typescript-eslint/interface-name-prefix export interface CreateOptions { @@ -132,7 +133,7 @@ export interface IInstallManagerOptions { export class InstallManager { private _rushConfiguration: RushConfiguration; private _rushGlobalFolder: RushGlobalFolder; - private _commonNodeModulesMarker: LastInstallFlag; + private _commonLastInstallMarker: LastInstallFlag; private _commonTempFolderRecycler: AsyncRecycler; private _options: IInstallManagerOptions; @@ -158,7 +159,7 @@ export class InstallManager { lastInstallState.storePath = rushConfiguration.pnpmOptions.pnpmStorePath; } - this._commonNodeModulesMarker = new LastInstallFlag(this._rushConfiguration.commonTempFolder, lastInstallState); + this._commonLastInstallMarker = new LastInstallFlag(this._rushConfiguration.commonTempFolder, lastInstallState); } /** @@ -275,7 +276,7 @@ export class InstallManager { } public get commonNodeModulesMarker(): LastInstallFlag { - return this._commonNodeModulesMarker; + return this._commonLastInstallMarker; } public async doInstall(): Promise { @@ -367,10 +368,18 @@ export class InstallManager { console.log(colors.bold('Using the default variant for installation.')); } - const shrinkwrapIsUpToDate: boolean = this._createTempModulesAndCheckShrinkwrap({ - shrinkwrapFile, - variant: options.variant - }) && !options.recheckShrinkwrap; + const useWorkspaces: boolean = true; + const shrinkwrapIsUpToDate: boolean = ( + useWorkspaces ? + this._prepareWorkspaceInstallAndCheckShrinkwrap({ + shrinkwrapFile, + variant: options.variant + }): + this._createTempModulesAndCheckShrinkwrap({ + shrinkwrapFile, + variant: options.variant + }) + ) && !options.recheckShrinkwrap; if (!shrinkwrapIsUpToDate) { if (!options.allowShrinkwrapUpdates) { @@ -382,19 +391,29 @@ export class InstallManager { } } - await this._installCommonModules({ - shrinkwrapIsUpToDate, - variantIsUpToDate, - ...options - }); + // TODO: Actually install + if (!useWorkspaces) { + await this._installCommonModules({ + shrinkwrapIsUpToDate, + variantIsUpToDate, + ...options + }); - if (!options.noLink) { - const linkManager: BaseLinkManager = LinkManagerFactory.getLinkManager(this._rushConfiguration); - await linkManager.createSymlinksForProjects(false); - } else { - console.log( - os.EOL + colors.yellow('Since "--no-link" was specified, you will need to run "rush link" manually.') - ); + if (!options.noLink) { + const linkManager: BaseLinkManager = LinkManagerFactory.getLinkManager(this._rushConfiguration); + await linkManager.createSymlinksForProjects(false); + } else { + console.log( + os.EOL + colors.yellow('Since "--no-link" was specified, you will need to run "rush link" manually.') + ); + } + } + else { + await this._installWorkspace({ + shrinkwrapIsUpToDate, + variantIsUpToDate, + ...options + }); } } @@ -481,6 +500,235 @@ export class InstallManager { }); } + private _prepareWorkspaceInstallAndCheckShrinkwrap(options: { + shrinkwrapFile: BaseShrinkwrapFile | undefined; + variant: string | undefined; + }): boolean { + const { + shrinkwrapFile, + variant + } = options; + + const stopwatch: Stopwatch = Stopwatch.start(); + + console.log(os.EOL + colors.bold('Updating workspace files in ' + this._rushConfiguration.commonTempFolder)); + + const shrinkwrapWarnings: string[] = []; + + // We will start with the assumption that it's valid, and then set it to false if + // any of the checks fail + let shrinkwrapIsUpToDate: boolean = true; + + if (!shrinkwrapFile) { + shrinkwrapIsUpToDate = false; + } + + // dependency name --> version specifier + const allExplicitPreferredVersions: Map = this._rushConfiguration.getCommonVersions(variant) + .getAllPreferredVersions(); + + if (shrinkwrapFile) { + // Check any (explicitly) preferred dependencies first + allExplicitPreferredVersions.forEach((version: string, dependency: string) => { + const dependencySpecifier: DependencySpecifier = new DependencySpecifier(dependency, version); + + if (!shrinkwrapFile.hasCompatibleTopLevelDependency(dependencySpecifier)) { + shrinkwrapWarnings.push(`"${dependency}" (${version}) required by the preferred versions from ` + + RushConstants.commonVersionsFilename); + shrinkwrapIsUpToDate = false; + } + }); + + if (this._findOrphanedWorkspaceProjects(shrinkwrapFile)) { + // If there are any orphaned projects, then "npm install" would fail because the shrinkwrap + // contains references such as "resolved": "file:projects\\project1" that refer to nonexistent + // file paths. + shrinkwrapIsUpToDate = false; + } + } + + // Also copy down the committed .npmrc file, if there is one + // "common\config\rush\.npmrc" --> "common\temp\.npmrc" + // Also ensure that we remove any old one that may be hanging around + Utilities.syncNpmrc(this._rushConfiguration.commonRushConfigFolder, this._rushConfiguration.commonTempFolder); + + // also, copy the pnpmfile.js if it exists + if (this._rushConfiguration.packageManager === 'pnpm') { + const committedPnpmFilePath: string = + this._rushConfiguration.getPnpmfilePath(this._options.variant); + const tempPnpmFilePath: string + = path.join(this._rushConfiguration.commonTempFolder, RushConstants.pnpmfileFilename); + + // ensure that we remove any old one that may be hanging around + this._syncFile(committedPnpmFilePath, tempPnpmFilePath); + } + + const commonPackageJson: IPackageJson = { + dependencies: {}, + description: 'Temporary file generated by the Rush tool', + name: 'rush-common', + private: true, + version: '0.0.0' + }; + + // dependency name --> version specifier + const allPreferredVersions: Map = new Map(); + + // Should we add implicitly preferred versions? + let useImplicitlyPinnedVersions: boolean; + if (this._rushConfiguration.commonVersions.implicitlyPreferredVersions !== undefined) { + // Use the manually configured setting + useImplicitlyPinnedVersions = this._rushConfiguration.commonVersions.implicitlyPreferredVersions; + } else { + // Default to true. + useImplicitlyPinnedVersions = true; + } + + if (useImplicitlyPinnedVersions) { + // Add in the implicitly preferred versions. + // These are any first-level dependencies for which we only consume a single version range + // (e.g. every package that depends on react uses an identical specifier) + const implicitlyPreferredVersions: Map = + InstallManager.collectImplicitlyPreferredVersions(this._rushConfiguration, { variant }); + MapExtensions.mergeFromMap(allPreferredVersions, implicitlyPreferredVersions); + } + + // Add in the explicitly preferred versions. + // Note that these take precedence over implicitly preferred versions. + MapExtensions.mergeFromMap(allPreferredVersions, allExplicitPreferredVersions); + + // TODO: Use the preferred versions to validate? + // // Add any preferred versions to the top of the commonPackageJson + // // do this in alphabetical order for simpler debugging + // for (const dependency of Array.from(allPreferredVersions.keys()).sort()) { + // commonPackageJson.dependencies![dependency] = allPreferredVersions.get(dependency)!; + // } + + // // To make the common/package.json file more readable, sort alphabetically + // // according to rushProject.tempProjectName instead of packageName. + // const sortedRushProjects: RushConfigurationProject[] = this._rushConfiguration.projects.slice(0); + // Sort.sortBy(sortedRushProjects, x => x.tempProjectName); + + // To generate the workspace file, we will add each project to the file as we loop through and validate + const workspaceFile: PnpmWorkspaceFile = new PnpmWorkspaceFile( + path.join(this._rushConfiguration.commonTempFolder, 'pnpm-workspace.yaml') + ); + + // Loop through the projects and add them to the workspace file. While we're at it, also validate that + // referenced workspace projects are valid, and check if the shrinkwrap file is already up-to-date. + for (const rushProject of this._rushConfiguration.projects) { + workspaceFile.addPackage(rushProject.projectFolder); + const packageJson: PackageJsonEditor = rushProject.packageJsonEditor; + + for (const { name, version, dependencyType } of [...packageJson.dependencyList, ...packageJson.devDependencyList]) { + const dependencySpecifier: DependencySpecifier = new DependencySpecifier(name, version); + + // Is there a locally built Rush project that could satisfy this dependency? + const localProject: RushConfigurationProject | undefined = + this._rushConfiguration.getProjectByName(name); + + // Validate that local projects are referenced with workspace notation. If not, and it is not a + // cyclic dependency, then it needs to be updated to specify `workspace:*` explicitly. Currently only + // supporting versions and version ranges for specifying a local project. + if ( + (dependencySpecifier.specifierType === 'version' || dependencySpecifier.specifierType === 'range') && + localProject && + !rushProject.cyclicDependencyProjects.has(name) + ) { + if (!semver.satisfies(localProject.packageJsonEditor.version, dependencySpecifier.versionSpecifier)) { + console.log(); + console.log(colors.red( + `"${rushProject.packageName}" depends on package "${name}" (${version}) which exists within the workspace ` + + 'but cannot be fulfilled with the specified version range. Either specify a valid version range, or add ' + + 'the package as a cyclic dependency.' + )); + throw new AlreadyReportedError(); + } + + packageJson.addOrUpdateDependency(name, `workspace:${version}`, dependencyType); + continue; + } else if (dependencySpecifier.specifierType === 'workspace') { + // Already specified as a local project, no need to validate + continue; + } + + // It is not a local dependency, validate that it is compatible + if (shrinkwrapFile) { + // if (!shrinkwrapFile.tryEnsureCompatibleDependency(dependencySpecifier, rushProject.tempProjectName)) { + // shrinkwrapWarnings.push(`"${packageName}" (${packageVersion}) required by` + // + ` "${rushProject.packageName}"`); + // shrinkwrapIsUpToDate = false; + // } + if (!shrinkwrapFile.tryEnsureCompatibleDependency(dependencySpecifier, rushProject.packageName)) { + shrinkwrapWarnings.push(`"${name}" (${version}) required by "${rushProject.packageName}"`); + shrinkwrapIsUpToDate = false; + } + } + } + + // Save the package.json if we modified the version references + if (rushProject.packageJsonEditor.saveIfModified()) { + console.log(colors.yellow( + `"${rushProject.packageName}" depends on one or more workspace packages which did not use "workspace:" ` + + 'notation. The package.json has been modified and must be committed to source control.' + )); + } + } + + // Example: "C:\MyRepo\common\temp\package.json" + const commonPackageJsonFilename: string = path.join( + this._rushConfiguration.commonTempFolder, + FileConstants.PackageJson + ); + + if (shrinkwrapFile) { + // If we have a (possibly incomplete) shrinkwrap file, save it as the temporary file. + shrinkwrapFile.save(this._rushConfiguration.tempShrinkwrapFilename); + shrinkwrapFile.save(this._rushConfiguration.tempShrinkwrapPreinstallFilename); + } else { + // Otherwise delete the temporary file + FileSystem.deleteFile(this._rushConfiguration.tempShrinkwrapFilename); + + if (this._rushConfiguration.packageManager === 'pnpm') { + // Workaround for https://github.com/pnpm/pnpm/issues/1890 + // + // When "rush update --full" is run, rush deletes common/temp/pnpm-lock.yaml so that + // a new lockfile can be generated. But because of the above bug "pnpm install" would + // respect "common/temp/node_modules/.pnpm-lock.yaml" and thus would not generate a + // new lockfile. Deleting this file in addition to deleting common/temp/pnpm-lock.yaml + // ensures that a new lockfile will be generated with "rush update --full". + + const pnpmPackageManager: PnpmPackageManager = + (this._rushConfiguration.packageManagerWrapper as PnpmPackageManager); + + FileSystem.deleteFile(path.join(this._rushConfiguration.commonTempFolder, + pnpmPackageManager.internalShrinkwrapRelativePath)); + } + } + + // Don't update the file timestamp unless the content has changed, since "rush install" + // will consider this timestamp + JsonFile.save(commonPackageJson, commonPackageJsonFilename, { onlyIfChanged: true }); + workspaceFile.save(workspaceFile.workspaceFilename, { onlyIfChanged: true }); + + stopwatch.stop(); + console.log(`Finished creating workspace (${stopwatch.toString()})`); + // console.log(`Finished creating temporary modules (${stopwatch.toString()})`); + + if (shrinkwrapWarnings.length > 0) { + console.log(); + console.log(colors.yellow(Utilities.wrapWords( + `The ${this._shrinkwrapFilePhrase} is missing the following dependencies:`))); + + for (const shrinkwrapWarning of shrinkwrapWarnings) { + console.log(colors.yellow(' ' + shrinkwrapWarning)); + } + console.log(); + } + + return shrinkwrapIsUpToDate; + } + /** * Regenerates the common/package.json and all temp_modules projects. * If shrinkwrapFile is provided, this function also validates whether it contains @@ -803,6 +1051,280 @@ export class InstallManager { return shrinkwrapIsUpToDate; } + /** + * Runs "npm install" in the common folder. + */ + private _installWorkspace(options: { + shrinkwrapIsUpToDate: boolean; + variantIsUpToDate: boolean; + } & IInstallManagerOptions): Promise { + const { + shrinkwrapIsUpToDate, + variantIsUpToDate + } = options; + + const projectNodeModulesFolders: string[] = this._rushConfiguration.projects.map(project => { + return path.join(project.projectFolder, 'node_modules') + }); + + return Promise.resolve().then(() => { + // console.log(os.EOL + colors.bold('Checking node_modules in ' + this._rushConfiguration.commonTempFolder) + // + os.EOL); + + // const commonNodeModulesFolder: string = path.join(this._rushConfiguration.commonTempFolder, + // 'node_modules'); + + // This marker file indicates that the last "rush install" completed successfully + const markerFileExistedAndWasValidAtStart: boolean = this._commonLastInstallMarker.checkValidAndReportStoreIssues(); + + // If "--clean" or "--full-clean" was specified, or if the last install was interrupted, + // then we will need to delete the node_modules folder. Otherwise, we can do an incremental + // install. + const deleteNodeModules: boolean = !markerFileExistedAndWasValidAtStart; + + // Based on timestamps, can we skip this install entirely? + if (shrinkwrapIsUpToDate && !deleteNodeModules && variantIsUpToDate) { + const potentiallyChangedFiles: string[] = []; + + // Consider the timestamp on the node_modules folder; if someone tampered with it + // or deleted it entirely, then we can't skip this install + potentiallyChangedFiles.push(...projectNodeModulesFolders); + + // Additionally, if they pulled an updated npm-shrinkwrap.json file from Git, + // then we can't skip this install + potentiallyChangedFiles.push(this._rushConfiguration.getCommittedShrinkwrapFilename(options.variant)); + + if (this._rushConfiguration.packageManager === 'pnpm') { + // If the repo is using pnpmfile.js, consider that also + const pnpmFileFilename: string = this._rushConfiguration.getPnpmfilePath(options.variant); + + if (FileSystem.exists(pnpmFileFilename)) { + potentiallyChangedFiles.push(pnpmFileFilename); + } + } + + // // Also consider timestamps for all the temp tarballs. (createTempModulesAndCheckShrinkwrap() will + // // carefully preserve these timestamps unless something has changed.) + // // Example: "C:\MyRepo\common\temp\projects\my-project-2.tgz" + // potentiallyChangedFiles.push(...this._rushConfiguration.projects.map(x => { + // return this._getTarballFilePath(x); + // })); + // Also consider timestamps for all the package.json files. + // Example: "C:\MyRepo\projects\my-project-2\package.json" + potentiallyChangedFiles.push(...this._rushConfiguration.projects.map(x => x.packageJsonEditor.filePath)); + + // NOTE: If commonLastInstallMarkerFilename (or any of the potentiallyChangedFiles) does not + // exist, then isFileTimestampCurrent() returns false. + if (Utilities.isFileTimestampCurrent(this._commonLastInstallMarker.path, potentiallyChangedFiles)) { + // Nothing to do, because everything is up to date according to time stamps + return; + } + } + + return this._checkIfReleaseIsPublished() + .catch((error) => { + // If the user is working in an environment that can't reach the registry, + // don't bother them with errors. + return undefined; + }).then((publishedRelease: boolean | undefined) => { + + if (publishedRelease === false) { + console.log(colors.yellow('Warning: This release of the Rush tool was unpublished; it may be unstable.')); + } + + // Since we're going to be tampering with common/node_modules, delete the "rush link" flag file if it exists; + // this ensures that a full "rush link" is required next time + Utilities.deleteFile(this._rushConfiguration.rushLinkJsonFilename); + + // Delete the successful install file to indicate the install transaction has started + this._commonLastInstallMarker.clear(); + + // NOTE: The PNPM store is supposed to be transactionally safe, so we don't delete it automatically. + // The user must request that via the command line. + if (deleteNodeModules) { + if (this._rushConfiguration.packageManager === 'npm') { + console.log(`Deleting the "npm-cache" folder`); + // This is faster and more thorough than "npm cache clean" + this._commonTempFolderRecycler.moveFolder(this._rushConfiguration.npmCacheFolder); + + console.log(`Deleting the "npm-tmp" folder`); + this._commonTempFolderRecycler.moveFolder(this._rushConfiguration.npmTmpFolder); + } + } + + // Example: "C:\MyRepo\common\temp\npm-local\node_modules\.bin\npm" + const packageManagerFilename: string = this._rushConfiguration.packageManagerToolFilename; + + let packageManagerEnv: NodeJS.ProcessEnv = process.env; + + let configurationEnvironment: IConfigurationEnvironment | undefined = undefined; + + if (this._rushConfiguration.packageManager === 'npm') { + // if ( + // this._rushConfiguration.npmOptions && + // this._rushConfiguration.npmOptions.environmentVariables + // ) { + // configurationEnvironment = this._rushConfiguration.npmOptions.environmentVariables; + // } + } else if (this._rushConfiguration.packageManager === 'pnpm') { + if ( + this._rushConfiguration.pnpmOptions && + this._rushConfiguration.pnpmOptions.environmentVariables + ) { + configurationEnvironment = this._rushConfiguration.pnpmOptions.environmentVariables; + } + } else if (this._rushConfiguration.packageManager === 'yarn') { + if ( + this._rushConfiguration.yarnOptions && + this._rushConfiguration.yarnOptions.environmentVariables + ) { + configurationEnvironment = this._rushConfiguration.yarnOptions.environmentVariables; + } + } + + packageManagerEnv = this._mergeEnvironmentVariables( + process.env, + configurationEnvironment + ); + + for (const nodeModulesFolder of projectNodeModulesFolders) { + // Is there an existing "node_modules" folder to consider? + if (FileSystem.exists(nodeModulesFolder)) { + // Should we delete the "node_modules" folder? + if (deleteNodeModules) { + // YES: Delete "node_modules" + + // Explain to the user why we are hosing their node_modules folder + console.log('Deleting files from ' + nodeModulesFolder); + + this._commonTempFolderRecycler.moveFolder(nodeModulesFolder); + + // Utilities.createFolderWithRetry(commonNodeModulesFolder); + } else { + // NO: Prepare to do an incremental install in the "node_modules" folder + + // note: it is not necessary to run "prune" with pnpm + // if (this._rushConfiguration.packageManager === 'npm') { + // console.log(`Running "${this._rushConfiguration.packageManager} prune"` + // + ` in ${this._rushConfiguration.commonTempFolder}`); + // const args: string[] = ['prune']; + // this._pushConfigurationArgs(args, options); + + // Utilities.executeCommandWithRetry(this._options.maxInstallAttempts, packageManagerFilename, args, + // this._rushConfiguration.commonTempFolder, packageManagerEnv); + + // // Delete the (installed image of) the temp projects, since "npm install" does not + // // detect changes for "file:./" references. + // // We recognize the temp projects by their names, which always start with "rush-". + + // // Example: "C:\MyRepo\common\temp\node_modules\@rush-temp" + // const pathToDeleteWithoutStar: string = path.join(commonNodeModulesFolder, + // RushConstants.rushTempNpmScope); + // console.log(`Deleting ${pathToDeleteWithoutStar}\\*`); + // // Glob can't handle Windows paths + // const normalizedpathToDeleteWithoutStar: string = Text.replaceAll(pathToDeleteWithoutStar, '\\', '/'); + + // // Example: "C:/MyRepo/common/temp/node_modules/@rush-temp/*" + // for (const tempModulePath of glob.sync(globEscape(normalizedpathToDeleteWithoutStar) + '/*')) { + // // We could potentially use AsyncRecycler here, but in practice these folders tend + // // to be very small + // Utilities.dangerouslyDeletePath(tempModulePath); + // } + // } + } + } + } + + if (this._rushConfiguration.packageManager === 'yarn') { + // Yarn does not correctly detect changes to a tarball, so we need to forcibly clear its cache + const yarnRushTempCacheFolder: string = path.join( + this._rushConfiguration.yarnCacheFolder, 'v2', 'npm-@rush-temp' + ); + if (FileSystem.exists(yarnRushTempCacheFolder)) { + console.log('Deleting ' + yarnRushTempCacheFolder); + Utilities.dangerouslyDeletePath(yarnRushTempCacheFolder); + } + } + + // Run "npm install" in the common folder + const installArgs: string[] = ['install']; + this._pushConfigurationArgs(installArgs, options); + + console.log(os.EOL + colors.bold(`Running "${this._rushConfiguration.packageManager} install" in` + + ` ${this._rushConfiguration.commonTempFolder}`) + os.EOL); + + // If any diagnostic options were specified, then show the full command-line + // if (options.debug || options.collectLogFile || options.networkConcurrency) { + console.log(os.EOL + colors.green('Invoking package manager: ') + + FileSystem.getRealPath(packageManagerFilename) + ' ' + installArgs.join(' ') + os.EOL); + // } + + try { + Utilities.executeCommandWithRetry(this._options.maxInstallAttempts, packageManagerFilename, + installArgs, + this._rushConfiguration.commonTempFolder, + packageManagerEnv, + false, () => { + if (this._rushConfiguration.packageManager === 'pnpm') { + console.log(colors.yellow(`Deleting the "node_modules" folders`)); + for (const nodeModulesFolder of projectNodeModulesFolders) { + if (FileSystem.exists(nodeModulesFolder)) { + this._commonTempFolderRecycler.moveFolder(nodeModulesFolder); + } + } + + // Leave the pnpm-store as is for the retry. This ensures that packages that have already + // been downloaded need not be downloaded again, thereby potentially increasing the chances + // of a subsequent successful install. + + // Utilities.createFolderWithRetry(commonNodeModulesFolder); + } + }); + } catch (error) { + // All the install attempts failed. + + if ( + this._rushConfiguration.packageManager === 'pnpm' && + this._rushConfiguration.pnpmOptions.pnpmStore === 'local' + ) { + // If the installation has failed even after the retries, then pnpm store may + // have got into a corrupted, irrecoverable state. Delete the store so that a + // future install can create the store afresh. + console.log(colors.yellow(`Deleting the "pnpm-store" folder`)); + this._commonTempFolderRecycler.moveFolder(this._rushConfiguration.pnpmOptions.pnpmStorePath); + } + + throw error; + } + + // if (this._rushConfiguration.packageManager === 'npm') { + + // console.log(os.EOL + colors.bold('Running "npm shrinkwrap"...')); + // const npmArgs: string[] = ['shrinkwrap']; + // this._pushConfigurationArgs(npmArgs, options); + // Utilities.executeCommand(this._rushConfiguration.packageManagerToolFilename, + // npmArgs, this._rushConfiguration.commonTempFolder); + // console.log('"npm shrinkwrap" completed' + os.EOL); + + // this._fixupNpm5Regression(); + // } + + if (options.allowShrinkwrapUpdates && !shrinkwrapIsUpToDate) { + // Copy (or delete) common\temp\pnpm-lock.yaml --> common\config\rush\pnpm-lock.yaml + this._syncFile(this._rushConfiguration.tempShrinkwrapFilename, + this._rushConfiguration.getCommittedShrinkwrapFilename(options.variant)); + } else { + // TODO: Validate whether the package manager updated it in a nontrivial way + } + + // Finally, create the marker file to indicate a successful install + this._commonLastInstallMarker.create(); + + console.log(''); + }); + }); + } + /** * Runs "npm install" in the common folder. */ @@ -823,7 +1345,7 @@ export class InstallManager { 'node_modules'); // This marker file indicates that the last "rush install" completed successfully - const markerFileExistedAndWasValidAtStart: boolean = this._commonNodeModulesMarker.checkValidAndReportStoreIssues(); + const markerFileExistedAndWasValidAtStart: boolean = this._commonLastInstallMarker.checkValidAndReportStoreIssues(); // If "--clean" or "--full-clean" was specified, or if the last install was interrupted, // then we will need to delete the node_modules folder. Otherwise, we can do an incremental @@ -860,7 +1382,7 @@ export class InstallManager { // NOTE: If commonNodeModulesMarkerFilename (or any of the potentiallyChangedFiles) does not // exist, then isFileTimestampCurrent() returns false. - if (Utilities.isFileTimestampCurrent(this._commonNodeModulesMarker.path, potentiallyChangedFiles)) { + if (Utilities.isFileTimestampCurrent(this._commonLastInstallMarker.path, potentiallyChangedFiles)) { // Nothing to do, because everything is up to date according to time stamps return; } @@ -882,7 +1404,7 @@ export class InstallManager { Utilities.deleteFile(this._rushConfiguration.rushLinkJsonFilename); // Delete the successful install file to indicate the install transaction has started - this._commonNodeModulesMarker.clear(); + this._commonLastInstallMarker.clear(); // NOTE: The PNPM store is supposed to be transactionally safe, so we don't delete it automatically. // The user must request that via the command line. @@ -1057,7 +1579,7 @@ export class InstallManager { } // Finally, create the marker file to indicate a successful install - this._commonNodeModulesMarker.create(); + this._commonLastInstallMarker.create(); console.log(''); }); @@ -1292,6 +1814,14 @@ export class InstallManager { if ((this._rushConfiguration.packageManagerWrapper as PnpmPackageManager).supportsResolutionStrategy) { args.push('--resolution-strategy', this._rushConfiguration.pnpmOptions.resolutionStrategy); } + + // If we're using PNPM workspaces, then we need to run it recursively for each project. We also need + // to turn off the 'link-workspace-packages' option, as we only want to rely on `workspace:` notation + // to link packages + if (this._rushConfiguration.pnpmOptions.useWorkspaces) { + args.push('--recursive'); + args.push('--link-workspace-packages', 'false'); + } } else if (this._rushConfiguration.packageManager === 'yarn') { args.push('--link-folder', 'yarn-link'); args.push('--cache-folder', this._rushConfiguration.yarnCacheFolder); @@ -1398,6 +1928,26 @@ export class InstallManager { return false; // none found } + /** + * Checks for temp projects that exist in the shrinkwrap file, but don't exist + * in rush.json. This might occur, e.g. if a project was recently deleted or renamed. + * + * @returns true if orphans were found, or false if everything is okay + */ + private _findOrphanedWorkspaceProjects(shrinkwrapFile: BaseShrinkwrapFile): boolean { + // // We can recognize temp projects because they are under the "@rush-temp" NPM scope. + // for (const tempProjectName of shrinkwrapFile.getTempProjectNames()) { + // if (!this._rushConfiguration.getProjectByName(tempProjectName)) { + // console.log(os.EOL + colors.yellow(Utilities.wrapWords( + // `Your ${this._shrinkwrapFilePhrase} references a project "${tempProjectName}" which no longer exists.`)) + // + os.EOL); + // return true; // found one + // } + // } + + return false; // none found + } + private get _shrinkwrapFilePhrase(): string { return this._rushConfiguration.shrinkwrapFilePhrase; } diff --git a/apps/rush-lib/src/logic/base/BaseWorkspaceFile.ts b/apps/rush-lib/src/logic/base/BaseWorkspaceFile.ts new file mode 100644 index 00000000000..eeb3dae405a --- /dev/null +++ b/apps/rush-lib/src/logic/base/BaseWorkspaceFile.ts @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { FileSystem } from '@rushstack/node-core-library'; + +export interface IWorkspaceFileSaveOptions { + /** + * If there is an existing file, and the contents have not changed, then + * don't write anything; this preserves the old timestamp. + */ + onlyIfChanged?: boolean; + + /** + * Creates the folder recursively using FileSystem.ensureFolder() + * Defaults to false. + */ + ensureFolderExists?: boolean; +} + +/** + * This class is a parser for pnpm's pnpm-workspace.yaml file format. + */ +export abstract class BaseWorkspaceFile { + protected _alreadyWarnedSpecs: Set = new Set(); + + /** + * Serializes and saves the workspace file to specified location + */ + public save(filePath: string, options: IWorkspaceFileSaveOptions): void { + // Do we need to read the previous file contents? + let oldBuffer: Buffer | undefined = undefined; + if (options.onlyIfChanged && FileSystem.exists(filePath)) { + try { + oldBuffer = FileSystem.readFileToBuffer(filePath); + } catch (error) { + // Ignore this error, and try writing a new file. If that fails, then we should report that + // error instead. + } + } + + const newYaml: string = this.serialize(); + + const newBuffer: Buffer = Buffer.from(newYaml); // utf8 encoding happens here + + if (options.onlyIfChanged) { + // Has the file changed? + if (oldBuffer && Buffer.compare(newBuffer, oldBuffer) === 0) { + // Nothing has changed, so don't touch the file + return; + } + } + + FileSystem.writeFile(filePath, newBuffer.toString(), { + ensureFolderExists: options.ensureFolderExists + }); + } + + /** + * Adds a package path to the workspace file. + * + * @virtual + */ + public abstract addPackage(packagePath: string): void; + + /** @virtual */ + protected abstract serialize(): string; +} diff --git a/apps/rush-lib/src/logic/pnpm/PnpmWorkspaceFile.ts b/apps/rush-lib/src/logic/pnpm/PnpmWorkspaceFile.ts new file mode 100644 index 00000000000..d59e4017fae --- /dev/null +++ b/apps/rush-lib/src/logic/pnpm/PnpmWorkspaceFile.ts @@ -0,0 +1,80 @@ +import * as yaml from 'js-yaml'; +import * as os from 'os'; +import * as path from 'path'; +import { FileSystem, Sort } from '@rushstack/node-core-library'; + +import { BaseWorkspaceFile } from '../base/BaseWorkspaceFile'; + +// This is based on PNPM's own configuration: +// https://github.com/pnpm/pnpm-shrinkwrap/blob/master/src/write.ts +const WORKSPACE_YAML_FORMAT: yaml.DumpOptions = { + lineWidth: 1000, + noCompatMode: true, + noRefs: true, + sortKeys: true +}; + +/** + * This interface represents the raw pnpm-workspace.YAML file + * Example: + * { + * "packages": [ + * "../../apps/project1" + * ] + * } + */ +interface IPnpmWorkspaceYaml { + /** The list of local package directories */ + packages: string[] +} + +export class PnpmWorkspaceFile extends BaseWorkspaceFile { + /** + * The filename of the workspace file. + */ + public readonly workspaceFilename: string; + + private _workspacePackages: Set; + + /** + * + */ + public constructor(workspaceYamlFilename: string) { + super(); + + this.workspaceFilename = workspaceYamlFilename; + let workspaceYaml: IPnpmWorkspaceYaml; + try { + // Populate with the existing file, or an empty list if the file doesn't exist + workspaceYaml = FileSystem.exists(workspaceYamlFilename) + ? yaml.safeLoad(FileSystem.readFile(workspaceYamlFilename).toString()) + : { packages: [] }; + } catch (error) { + throw new Error(`Error reading "${workspaceYamlFilename}":${os.EOL} ${error.message}`); + } + + this._workspacePackages = new Set(workspaceYaml.packages); + } + + /** @override */ + public addPackage(packagePath: string): void { + // Ensure the path is relative to the pnpm-workspace.yaml file + if (path.isAbsolute(packagePath)) { + packagePath = path.relative(path.dirname(this.workspaceFilename), packagePath); + } + + // Ensure that the path is split using forward slashes + this._workspacePackages.add(path.posix.join(...packagePath.split(path.sep))); + } + + /** @override */ + protected serialize(): string { + // Ensure stable sort order when serializing + Sort.sortSet(this._workspacePackages); + + const workspaceYaml: IPnpmWorkspaceYaml = { + packages: Array.from(this._workspacePackages) + }; + return yaml.safeDump(workspaceYaml, WORKSPACE_YAML_FORMAT); + } +} \ No newline at end of file diff --git a/apps/rush-lib/src/schemas/rush.schema.json b/apps/rush-lib/src/schemas/rush.schema.json index 923467a637f..e1a246346eb 100644 --- a/apps/rush-lib/src/schemas/rush.schema.json +++ b/apps/rush-lib/src/schemas/rush.schema.json @@ -106,6 +106,10 @@ "fast" ] }, + "useWorkspaces": { + "description": "If true, then Rush will use the workspaces feature to install and link packages when invoking PNPM. The default value is false.", + "type": "boolean" + }, "environmentVariables": { "$ref": "#/definitions/environmentVariables" } From 86bac901d8efaf716fbedf22a5fd7a18d41a352b Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Wed, 6 May 2020 14:08:08 -0700 Subject: [PATCH 02/16] Include changes to API md --- common/reviews/api/rush-lib.api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index e6e9d46a87e..7f531e8f861 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -172,6 +172,7 @@ export interface _IPnpmOptionsJson extends IPackageManagerOptionsJsonBase { preventManualShrinkwrapChanges?: boolean; resolutionStrategy?: ResolutionStrategy; strictPeerDependencies?: boolean; + useWorkspaces?: boolean; } // @public @@ -285,6 +286,7 @@ export class PnpmOptionsConfiguration extends PackageManagerOptionsConfiguration readonly preventManualShrinkwrapChanges: boolean; readonly resolutionStrategy: ResolutionStrategy; readonly strictPeerDependencies: boolean; + readonly useWorkspaces: boolean; } // @public From ca31390bb865db92c11f879bac677d536c271172 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Tue, 19 May 2020 16:38:22 -0700 Subject: [PATCH 03/16] Refactor InstallManager into separate files --- .../src/cli/actions/BaseInstallAction.ts | 5 +- .../rush-lib/src/cli/actions/InstallAction.ts | 2 +- apps/rush-lib/src/cli/actions/UpdateAction.ts | 2 +- .../rush-lib/src/logic/DependencySpecifier.ts | 2 +- apps/rush-lib/src/logic/InstallManager.ts | 1986 ----------------- .../src/logic/InstallManagerFactory.ts | 41 + apps/rush-lib/src/logic/PackageJsonUpdater.ts | 9 +- apps/rush-lib/src/logic/RushInstallManager.ts | 581 +++++ .../src/logic/WorkspaceInstallManager.ts | 378 ++++ .../src/logic/base/BaseInstallManager.ts | 919 ++++++++ .../src/logic/base/BaseShrinkwrapFile.ts | 32 +- .../src/logic/npm/NpmShrinkwrapFile.ts | 17 +- .../src/logic/pnpm/PnpmLinkManager.ts | 112 +- .../src/logic/pnpm/PnpmShrinkwrapFile.ts | 103 +- .../src/logic/pnpm/PnpmWorkspaceFile.ts | 10 +- .../src/logic/yarn/YarnShrinkwrapFile.ts | 14 + apps/rush-lib/src/utilities/Utilities.ts | 31 +- 17 files changed, 2226 insertions(+), 2018 deletions(-) delete mode 100644 apps/rush-lib/src/logic/InstallManager.ts create mode 100644 apps/rush-lib/src/logic/InstallManagerFactory.ts create mode 100644 apps/rush-lib/src/logic/RushInstallManager.ts create mode 100644 apps/rush-lib/src/logic/WorkspaceInstallManager.ts create mode 100644 apps/rush-lib/src/logic/base/BaseInstallManager.ts diff --git a/apps/rush-lib/src/cli/actions/BaseInstallAction.ts b/apps/rush-lib/src/cli/actions/BaseInstallAction.ts index 577d8b5193c..2409695bc05 100644 --- a/apps/rush-lib/src/cli/actions/BaseInstallAction.ts +++ b/apps/rush-lib/src/cli/actions/BaseInstallAction.ts @@ -12,7 +12,7 @@ import { import { BaseRushAction } from './BaseRushAction'; import { Event } from '../../api/EventHooks'; -import { InstallManager, IInstallManagerOptions } from '../../logic/InstallManager'; +import { BaseInstallManager, IInstallManagerOptions } from '../../logic/base/BaseInstallManager'; import { PurgeManager } from '../../logic/PurgeManager'; import { SetupChecks } from '../../logic/SetupChecks'; import { StandardScriptUpdater } from '../../logic/StandardScriptUpdater'; @@ -20,6 +20,7 @@ import { Stopwatch } from '../../utilities/Stopwatch'; import { VersionMismatchFinder } from '../../logic/versionMismatch/VersionMismatchFinder'; import { Variants } from '../../api/Variants'; import { RushConstants } from '../../logic/RushConstants'; +import { InstallManagerFactory } from '../../logic/InstallManagerFactory'; /** * This is the common base class for InstallAction and UpdateAction. @@ -112,7 +113,7 @@ export abstract class BaseInstallAction extends BaseRushAction { const installManagerOptions: IInstallManagerOptions = this.buildInstallOptions(); - const installManager: InstallManager = new InstallManager( + const installManager: BaseInstallManager = InstallManagerFactory.getInstallManager( this.rushConfiguration, this.rushGlobalFolder, purgeManager, diff --git a/apps/rush-lib/src/cli/actions/InstallAction.ts b/apps/rush-lib/src/cli/actions/InstallAction.ts index 28ee9a9a3ab..399c4d20077 100644 --- a/apps/rush-lib/src/cli/actions/InstallAction.ts +++ b/apps/rush-lib/src/cli/actions/InstallAction.ts @@ -2,7 +2,7 @@ // See LICENSE in the project root for license information. import { BaseInstallAction } from './BaseInstallAction'; -import { IInstallManagerOptions } from '../../logic/InstallManager'; +import { IInstallManagerOptions } from '../../logic/base/BaseInstallManager'; import { RushCommandLineParser } from '../RushCommandLineParser'; export class InstallAction extends BaseInstallAction { diff --git a/apps/rush-lib/src/cli/actions/UpdateAction.ts b/apps/rush-lib/src/cli/actions/UpdateAction.ts index 957cd6b9fb5..e4b9d5121af 100644 --- a/apps/rush-lib/src/cli/actions/UpdateAction.ts +++ b/apps/rush-lib/src/cli/actions/UpdateAction.ts @@ -4,7 +4,7 @@ import { CommandLineFlagParameter } from '@rushstack/ts-command-line'; import { BaseInstallAction } from './BaseInstallAction'; -import { IInstallManagerOptions } from '../../logic/InstallManager'; +import { IInstallManagerOptions } from '../../logic/base/BaseInstallManager'; import { RushCommandLineParser } from '../RushCommandLineParser'; export class UpdateAction extends BaseInstallAction { diff --git a/apps/rush-lib/src/logic/DependencySpecifier.ts b/apps/rush-lib/src/logic/DependencySpecifier.ts index fc40f1a6345..91c37227bd7 100644 --- a/apps/rush-lib/src/logic/DependencySpecifier.ts +++ b/apps/rush-lib/src/logic/DependencySpecifier.ts @@ -50,7 +50,7 @@ export class DependencySpecifier { // to the trimmed version range. if (versionSpecifier.startsWith('workspace:')) { this.specifierType = 'workspace' - this.versionSpecifier = versionSpecifier.slice(this.specifierType.length); + this.versionSpecifier = versionSpecifier.slice(this.specifierType.length + 1).trim(); this.aliasTarget = undefined; return; } diff --git a/apps/rush-lib/src/logic/InstallManager.ts b/apps/rush-lib/src/logic/InstallManager.ts deleted file mode 100644 index 59ee34882a2..00000000000 --- a/apps/rush-lib/src/logic/InstallManager.ts +++ /dev/null @@ -1,1986 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -/* eslint max-lines: off */ - -import * as glob from 'glob'; -import * as colors from 'colors'; -import * as fetch from 'node-fetch'; -import * as http from 'http'; -import HttpsProxyAgent = require('https-proxy-agent'); -import * as os from 'os'; -import * as path from 'path'; -import * as semver from 'semver'; -import * as tar from 'tar'; -import * as globEscape from 'glob-escape'; -import { - JsonFile, - LockFile, - Text, - IPackageJson, - MapExtensions, - FileSystem, - FileConstants, - Sort, - PosixModeBits, - JsonObject -} from '@rushstack/node-core-library'; - -import { ApprovedPackagesChecker } from '../logic/ApprovedPackagesChecker'; -import { AsyncRecycler } from '../utilities/AsyncRecycler'; -import { BaseLinkManager } from '../logic/base/BaseLinkManager'; -import { BaseShrinkwrapFile } from '../logic/base/BaseShrinkwrapFile'; -import { PolicyValidator } from '../logic/policy/PolicyValidator'; -import { IRushTempPackageJson } from '../logic/base/BasePackage'; -import { Git } from '../logic/Git'; -import { LastInstallFlag } from '../api/LastInstallFlag'; -import { LinkManagerFactory } from '../logic/LinkManagerFactory'; -import { PurgeManager } from './PurgeManager'; -import { RushConfiguration, ICurrentVariantJson, IConfigurationEnvironment } from '../api/RushConfiguration'; -import { RushConfigurationProject } from '../api/RushConfigurationProject'; -import { RushConstants } from '../logic/RushConstants'; -import { ShrinkwrapFileFactory } from '../logic/ShrinkwrapFileFactory'; -import { Stopwatch } from '../utilities/Stopwatch'; -import { Utilities } from '../utilities/Utilities'; -import { Rush } from '../api/Rush'; -import { PackageJsonEditor, DependencyType, PackageJsonDependency } from '../api/PackageJsonEditor'; -import { AlreadyReportedError } from '../utilities/AlreadyReportedError'; -import { CommonVersionsConfiguration } from '../api/CommonVersionsConfiguration'; -import { RushGlobalFolder } from '../api/RushGlobalFolder'; -import { PackageManagerName } from '../api/packageManager/PackageManager'; -import { PnpmPackageManager } from '../api/packageManager/PnpmPackageManager'; -import { DependencySpecifier } from './DependencySpecifier'; -import { EnvironmentConfiguration } from '../api/EnvironmentConfiguration'; -import { PnpmWorkspaceFile } from './pnpm/PnpmWorkspaceFile'; - -// The PosixModeBits are intended to be used with bitwise operations. -/* eslint-disable no-bitwise */ - -/** - * The "noMtime" flag is new in tar@4.4.1 and not available yet for \@types/tar. - * As a temporary workaround, augment the type. - */ -declare module 'tar' { - // eslint-disable-next-line @typescript-eslint/interface-name-prefix - export interface CreateOptions { - /** - * "Set to true to omit writing mtime values for entries. Note that this prevents using other - * mtime-based features like tar.update or the keepNewer option with the resulting tar archive." - */ - noMtime?: boolean; - } -} - -export interface IInstallManagerOptions { - /** - * Whether the global "--debug" flag was specified. - */ - debug: boolean; - /** - * Whether or not Rush will automatically update the shrinkwrap file. - * True for "rush update", false for "rush install". - */ - allowShrinkwrapUpdates: boolean; - /** - * Whether to skip policy checks. - */ - bypassPolicy: boolean; - /** - * Whether to skip linking, i.e. require "rush link" to be done manually later. - */ - noLink: boolean; - /** - * Whether to delete the shrinkwrap file before installation, i.e. so that all dependencies - * will be upgraded to the latest SemVer-compatible version. - */ - fullUpgrade: boolean; - /** - * Whether to force an update to the shrinkwrap file even if it appears to be unnecessary. - * Normally Rush uses heuristics to determine when "pnpm install" can be skipped, - * but sometimes the heuristics can be inaccurate due to external influences - * (pnpmfile.js script logic, registry changes, etc). - */ - recheckShrinkwrap: boolean; - - /** - * The value of the "--network-concurrency" command-line parameter, which - * is a diagnostic option used to troubleshoot network failures. - * - * Currently only supported for PNPM. - */ - networkConcurrency: number | undefined; - - /** - * Whether or not to collect verbose logs from the package manager. - * If specified when using PNPM, the logs will be in /common/temp/pnpm.log - */ - collectLogFile: boolean; - - /** - * The variant to consider when performing installations and validating shrinkwrap updates. - */ - variant?: string | undefined; - - /** - * Retry the install the specified number of times - */ - maxInstallAttempts: number -} - -/** - * This class implements common logic between "rush install" and "rush update". - */ -export class InstallManager { - private _rushConfiguration: RushConfiguration; - private _rushGlobalFolder: RushGlobalFolder; - private _commonLastInstallMarker: LastInstallFlag; - private _commonTempFolderRecycler: AsyncRecycler; - - private _options: IInstallManagerOptions; - - public constructor( - rushConfiguration: RushConfiguration, - rushGlobalFolder: RushGlobalFolder, - purgeManager: PurgeManager, - options: IInstallManagerOptions - ) { - this._rushConfiguration = rushConfiguration; - this._rushGlobalFolder = rushGlobalFolder; - this._commonTempFolderRecycler = purgeManager.commonTempFolderRecycler; - this._options = options; - - const lastInstallState: JsonObject = { - node: process.versions.node, - packageManager: rushConfiguration.packageManager, - packageManagerVersion: rushConfiguration.packageManagerToolVersion - } - - if (lastInstallState.packageManager === 'pnpm') { - lastInstallState.storePath = rushConfiguration.pnpmOptions.pnpmStorePath; - } - - this._commonLastInstallMarker = new LastInstallFlag(this._rushConfiguration.commonTempFolder, lastInstallState); - } - - /** - * Returns a map of all direct dependencies that only have a single semantic version specifier. - * Returns a map: dependency name --> version specifier - */ - public static collectImplicitlyPreferredVersions( - rushConfiguration: RushConfiguration, - options: { - variant?: string | undefined - } = {} - ): Map { - // First, collect all the direct dependencies of all local projects, and their versions: - // direct dependency name --> set of version specifiers - const versionsForDependencies: Map> = new Map>(); - - rushConfiguration.projects.forEach((project: RushConfigurationProject) => { - InstallManager._collectVersionsForDependencies( - rushConfiguration, - { - versionsForDependencies, - dependencies: project.packageJsonEditor.dependencyList, - cyclicDependencies: project.cyclicDependencyProjects, - variant: options.variant - }); - - InstallManager._collectVersionsForDependencies( - rushConfiguration, - { - versionsForDependencies, - dependencies: project.packageJsonEditor.devDependencyList, - cyclicDependencies: project.cyclicDependencyProjects, - variant: options.variant - }); - }); - - // If any dependency has more than one version, then filter it out (since we don't know which version - // should be preferred). What remains will be the list of preferred dependencies. - // dependency --> version specifier - const implicitlyPreferred: Map = new Map(); - versionsForDependencies.forEach((versions: Set, dep: string) => { - if (versions.size === 1) { - const version: string = versions.values().next().value; - implicitlyPreferred.set(dep, version); - } - }); - return implicitlyPreferred; - } - - // Helper for collectImplicitlyPreferredVersions() - private static _updateVersionsForDependencies(versionsForDependencies: Map>, - dependency: string, version: string): void { - if (!versionsForDependencies.has(dependency)) { - versionsForDependencies.set(dependency, new Set()); - } - versionsForDependencies.get(dependency)!.add(version); - } - - // Helper for collectImplicitlyPreferredVersions() - private static _collectVersionsForDependencies( - rushConfiguration: RushConfiguration, - options: { - versionsForDependencies: Map>; - dependencies: ReadonlyArray; - cyclicDependencies: Set; - variant: string | undefined; - }): void { - const { - variant, - dependencies, - versionsForDependencies, - cyclicDependencies - } = options; - - const commonVersions: CommonVersionsConfiguration = rushConfiguration.getCommonVersions(variant); - - const allowedAlternativeVersions: Map> - = commonVersions.allowedAlternativeVersions; - - for (const dependency of dependencies) { - const alternativesForThisDependency: ReadonlyArray - = allowedAlternativeVersions.get(dependency.name) || []; - - // For each dependency, collectImplicitlyPreferredVersions() is collecting the set of all version specifiers - // that appear across the repo. If there is only one version specifier, then that's the "preferred" one. - // However, there are a few cases where additional version specifiers can be safely ignored. - let ignoreVersion: boolean = false; - - // 1. If the version specifier was listed in "allowedAlternativeVersions", then it's never a candidate. - // (Even if it's the only version specifier anywhere in the repo, we still ignore it, because - // otherwise the rule would be difficult to explain.) - if (alternativesForThisDependency.indexOf(dependency.version) > 0) { - ignoreVersion = true; - } else { - // Is it a local project? - const localProject: RushConfigurationProject | undefined = rushConfiguration.getProjectByName(dependency.name); - if (localProject) { - // 2. If it's a symlinked local project, then it's not a candidate, because the package manager will - // never even see it. - // However there are two ways that a local project can NOT be symlinked: - // - if the local project doesn't satisfy the referenced semver specifier; OR - // - if the local project was specified in "cyclicDependencyProjects" in rush.json - if (semver.satisfies(localProject.packageJsonEditor.version, dependency.version) - && !cyclicDependencies.has(dependency.name)) { - ignoreVersion = true; - } - } - - if (!ignoreVersion) { - InstallManager._updateVersionsForDependencies(versionsForDependencies, dependency.name, dependency.version); - } - } - } - } - - public get commonNodeModulesMarker(): LastInstallFlag { - return this._commonLastInstallMarker; - } - - public async doInstall(): Promise { - const options: IInstallManagerOptions = this._options; - - // Check the policies - PolicyValidator.validatePolicy(this._rushConfiguration, options); - - // Git hooks are only installed if the repo opts in by including files in /common/git-hooks - const hookSource: string = path.join(this._rushConfiguration.commonFolder, 'git-hooks'); - const hookDestination: string | undefined = Git.getHooksFolder(); - - if (FileSystem.exists(hookSource) && hookDestination) { - const hookFilenames: string[] = FileSystem.readFolder(hookSource); - if (hookFilenames.length > 0) { - console.log(os.EOL + colors.bold('Found files in the "common/git-hooks" folder.')); - - // Clear the currently installed git hooks and install fresh copies - FileSystem.ensureEmptyFolder(hookDestination); - - // Only copy files that look like Git hook names - const filteredHookFilenames: string[] = hookFilenames.filter(x => /^[a-z\-]+/.test(x)); - for (const filename of filteredHookFilenames) { - FileSystem.copyFile({ - sourcePath: path.join(hookSource, filename), - destinationPath: path.join(hookDestination, filename) - }); - FileSystem.changePosixModeBits(path.join(hookDestination, filename), - PosixModeBits.UserRead | PosixModeBits.UserExecute); - } - - console.log('Successfully installed these Git hook scripts: ' + filteredHookFilenames.join(', ') + os.EOL); - } - } - - const approvedPackagesChecker: ApprovedPackagesChecker = new ApprovedPackagesChecker(this._rushConfiguration); - if (approvedPackagesChecker.approvedPackagesFilesAreOutOfDate) { - if (this._options.allowShrinkwrapUpdates) { - approvedPackagesChecker.rewriteConfigFiles(); - console.log(colors.yellow( - 'Approved package files have been updated. These updates should be committed to source control' - )); - } else { - throw new Error(`Approved packages files are out-of date. Run "rush update" to update them.`); - } - } - - // Ensure that the package manager is installed - await this.ensureLocalPackageManager(); - let shrinkwrapFile: BaseShrinkwrapFile | undefined = undefined; - - // (If it's a full update, then we ignore the shrinkwrap from Git since it will be overwritten) - if (!options.fullUpgrade) { - try { - shrinkwrapFile = ShrinkwrapFileFactory.getShrinkwrapFile( - this._rushConfiguration.packageManager, - this._rushConfiguration.packageManagerOptions, - this._rushConfiguration.getCommittedShrinkwrapFilename(options.variant) - ); - } catch (ex) { - console.log(); - console.log(`Unable to load the ${this._shrinkwrapFilePhrase}: ${ex.message}`); - - if (!options.allowShrinkwrapUpdates) { - console.log(); - console.log(colors.red('You need to run "rush update" to fix this problem')); - throw new AlreadyReportedError(); - } - - shrinkwrapFile = undefined; - } - } - - // Write a file indicating which variant is being installed. - // This will be used by bulk scripts to determine the correct Shrinkwrap file to track. - const currentVariantJsonFilename: string = this._rushConfiguration.currentVariantJsonFilename; - const currentVariantJson: ICurrentVariantJson = { - variant: options.variant || null // eslint-disable-line @rushstack/no-null - }; - - // Determine if the variant is already current by updating current-variant.json. - // If nothing is written, the variant has not changed. - const variantIsUpToDate: boolean = !JsonFile.save(currentVariantJson, currentVariantJsonFilename, { - onlyIfChanged: true - }); - - if (options.variant) { - console.log(); - console.log(colors.bold(`Using variant '${options.variant}' for installation.`)); - } else if (!variantIsUpToDate && !options.variant) { - console.log(); - console.log(colors.bold('Using the default variant for installation.')); - } - - const useWorkspaces: boolean = true; - const shrinkwrapIsUpToDate: boolean = ( - useWorkspaces ? - this._prepareWorkspaceInstallAndCheckShrinkwrap({ - shrinkwrapFile, - variant: options.variant - }): - this._createTempModulesAndCheckShrinkwrap({ - shrinkwrapFile, - variant: options.variant - }) - ) && !options.recheckShrinkwrap; - - if (!shrinkwrapIsUpToDate) { - if (!options.allowShrinkwrapUpdates) { - console.log(); - console.log(colors.red( - `The ${this._shrinkwrapFilePhrase} is out of date. You need to run "rush update".` - )); - throw new AlreadyReportedError(); - } - } - - // TODO: Actually install - if (!useWorkspaces) { - await this._installCommonModules({ - shrinkwrapIsUpToDate, - variantIsUpToDate, - ...options - }); - - if (!options.noLink) { - const linkManager: BaseLinkManager = LinkManagerFactory.getLinkManager(this._rushConfiguration); - await linkManager.createSymlinksForProjects(false); - } else { - console.log( - os.EOL + colors.yellow('Since "--no-link" was specified, you will need to run "rush link" manually.') - ); - } - } - else { - await this._installWorkspace({ - shrinkwrapIsUpToDate, - variantIsUpToDate, - ...options - }); - } - } - - /** - * If the "(p)npm-local" symlink hasn't been set up yet, this creates it, installing the - * specified (P)npm version in the user's home directory if needed. - */ - public ensureLocalPackageManager(): Promise { - // Example: "C:\Users\YourName\.rush" - const rushUserFolder: string = this._rushGlobalFolder.nodeSpecificPath; - - if (!FileSystem.exists(rushUserFolder)) { - console.log('Creating ' + rushUserFolder); - FileSystem.ensureFolder(rushUserFolder); - } - - const packageManager: PackageManagerName = this._rushConfiguration.packageManager; - const packageManagerVersion: string = this._rushConfiguration.packageManagerToolVersion; - - const packageManagerAndVersion: string = `${packageManager}-${packageManagerVersion}`; - // Example: "C:\Users\YourName\.rush\pnpm-1.2.3" - const packageManagerToolFolder: string = path.join(rushUserFolder, packageManagerAndVersion); - - const packageManagerMarker: LastInstallFlag = new LastInstallFlag(packageManagerToolFolder, { - node: process.versions.node - }); - - console.log(`Trying to acquire lock for ${packageManagerAndVersion}`); - return LockFile.acquire(rushUserFolder, packageManagerAndVersion).then((lock: LockFile) => { - console.log(`Acquired lock for ${packageManagerAndVersion}`); - - if (!packageManagerMarker.isValid() || lock.dirtyWhenAcquired) { - console.log(colors.bold(`Installing ${packageManager} version ${packageManagerVersion}${os.EOL}`)); - - // note that this will remove the last-install flag from the directory - Utilities.installPackageInDirectory({ - directory: packageManagerToolFolder, - packageName: packageManager, - version: this._rushConfiguration.packageManagerToolVersion, - tempPackageTitle: `${packageManager}-local-install`, - maxInstallAttempts: this._options.maxInstallAttempts, - // This is using a local configuration to install a package in a shared global location. - // Generally that's a bad practice, but in this case if we can successfully install - // the package at all, we can reasonably assume it's good for all the repositories. - // In particular, we'll assume that two different NPM registries cannot have two - // different implementations of the same version of the same package. - // This was needed for: https://github.com/microsoft/rushstack/issues/691 - commonRushConfigFolder: this._rushConfiguration.commonRushConfigFolder - }); - - console.log(`Successfully installed ${packageManager} version ${packageManagerVersion}`); - } else { - console.log(`Found ${packageManager} version ${packageManagerVersion} in ${packageManagerToolFolder}`); - } - - packageManagerMarker.create(); - - // Example: "C:\MyRepo\common\temp" - FileSystem.ensureFolder(this._rushConfiguration.commonTempFolder); - - // Example: "C:\MyRepo\common\temp\pnpm-local" - const localPackageManagerToolFolder: string = - path.join(this._rushConfiguration.commonTempFolder, `${packageManager}-local`); - - console.log(os.EOL + 'Symlinking "' + localPackageManagerToolFolder + '"'); - console.log(' --> "' + packageManagerToolFolder + '"'); - - // We cannot use FileSystem.exists() to test the existence of a symlink, because it will - // return false for broken symlinks. There is no way to test without catching an exception. - try { - FileSystem.deleteFolder(localPackageManagerToolFolder); - } catch (error) { - if (error.code !== 'ENOENT') { - throw error; - } - } - - FileSystem.createSymbolicLinkJunction({ - linkTargetPath: packageManagerToolFolder, - newLinkPath: localPackageManagerToolFolder - }); - - lock.release(); - }); - } - - private _prepareWorkspaceInstallAndCheckShrinkwrap(options: { - shrinkwrapFile: BaseShrinkwrapFile | undefined; - variant: string | undefined; - }): boolean { - const { - shrinkwrapFile, - variant - } = options; - - const stopwatch: Stopwatch = Stopwatch.start(); - - console.log(os.EOL + colors.bold('Updating workspace files in ' + this._rushConfiguration.commonTempFolder)); - - const shrinkwrapWarnings: string[] = []; - - // We will start with the assumption that it's valid, and then set it to false if - // any of the checks fail - let shrinkwrapIsUpToDate: boolean = true; - - if (!shrinkwrapFile) { - shrinkwrapIsUpToDate = false; - } - - // dependency name --> version specifier - const allExplicitPreferredVersions: Map = this._rushConfiguration.getCommonVersions(variant) - .getAllPreferredVersions(); - - if (shrinkwrapFile) { - // Check any (explicitly) preferred dependencies first - allExplicitPreferredVersions.forEach((version: string, dependency: string) => { - const dependencySpecifier: DependencySpecifier = new DependencySpecifier(dependency, version); - - if (!shrinkwrapFile.hasCompatibleTopLevelDependency(dependencySpecifier)) { - shrinkwrapWarnings.push(`"${dependency}" (${version}) required by the preferred versions from ` - + RushConstants.commonVersionsFilename); - shrinkwrapIsUpToDate = false; - } - }); - - if (this._findOrphanedWorkspaceProjects(shrinkwrapFile)) { - // If there are any orphaned projects, then "npm install" would fail because the shrinkwrap - // contains references such as "resolved": "file:projects\\project1" that refer to nonexistent - // file paths. - shrinkwrapIsUpToDate = false; - } - } - - // Also copy down the committed .npmrc file, if there is one - // "common\config\rush\.npmrc" --> "common\temp\.npmrc" - // Also ensure that we remove any old one that may be hanging around - Utilities.syncNpmrc(this._rushConfiguration.commonRushConfigFolder, this._rushConfiguration.commonTempFolder); - - // also, copy the pnpmfile.js if it exists - if (this._rushConfiguration.packageManager === 'pnpm') { - const committedPnpmFilePath: string = - this._rushConfiguration.getPnpmfilePath(this._options.variant); - const tempPnpmFilePath: string - = path.join(this._rushConfiguration.commonTempFolder, RushConstants.pnpmfileFilename); - - // ensure that we remove any old one that may be hanging around - this._syncFile(committedPnpmFilePath, tempPnpmFilePath); - } - - const commonPackageJson: IPackageJson = { - dependencies: {}, - description: 'Temporary file generated by the Rush tool', - name: 'rush-common', - private: true, - version: '0.0.0' - }; - - // dependency name --> version specifier - const allPreferredVersions: Map = new Map(); - - // Should we add implicitly preferred versions? - let useImplicitlyPinnedVersions: boolean; - if (this._rushConfiguration.commonVersions.implicitlyPreferredVersions !== undefined) { - // Use the manually configured setting - useImplicitlyPinnedVersions = this._rushConfiguration.commonVersions.implicitlyPreferredVersions; - } else { - // Default to true. - useImplicitlyPinnedVersions = true; - } - - if (useImplicitlyPinnedVersions) { - // Add in the implicitly preferred versions. - // These are any first-level dependencies for which we only consume a single version range - // (e.g. every package that depends on react uses an identical specifier) - const implicitlyPreferredVersions: Map = - InstallManager.collectImplicitlyPreferredVersions(this._rushConfiguration, { variant }); - MapExtensions.mergeFromMap(allPreferredVersions, implicitlyPreferredVersions); - } - - // Add in the explicitly preferred versions. - // Note that these take precedence over implicitly preferred versions. - MapExtensions.mergeFromMap(allPreferredVersions, allExplicitPreferredVersions); - - // TODO: Use the preferred versions to validate? - // // Add any preferred versions to the top of the commonPackageJson - // // do this in alphabetical order for simpler debugging - // for (const dependency of Array.from(allPreferredVersions.keys()).sort()) { - // commonPackageJson.dependencies![dependency] = allPreferredVersions.get(dependency)!; - // } - - // // To make the common/package.json file more readable, sort alphabetically - // // according to rushProject.tempProjectName instead of packageName. - // const sortedRushProjects: RushConfigurationProject[] = this._rushConfiguration.projects.slice(0); - // Sort.sortBy(sortedRushProjects, x => x.tempProjectName); - - // To generate the workspace file, we will add each project to the file as we loop through and validate - const workspaceFile: PnpmWorkspaceFile = new PnpmWorkspaceFile( - path.join(this._rushConfiguration.commonTempFolder, 'pnpm-workspace.yaml') - ); - - // Loop through the projects and add them to the workspace file. While we're at it, also validate that - // referenced workspace projects are valid, and check if the shrinkwrap file is already up-to-date. - for (const rushProject of this._rushConfiguration.projects) { - workspaceFile.addPackage(rushProject.projectFolder); - const packageJson: PackageJsonEditor = rushProject.packageJsonEditor; - - for (const { name, version, dependencyType } of [...packageJson.dependencyList, ...packageJson.devDependencyList]) { - const dependencySpecifier: DependencySpecifier = new DependencySpecifier(name, version); - - // Is there a locally built Rush project that could satisfy this dependency? - const localProject: RushConfigurationProject | undefined = - this._rushConfiguration.getProjectByName(name); - - // Validate that local projects are referenced with workspace notation. If not, and it is not a - // cyclic dependency, then it needs to be updated to specify `workspace:*` explicitly. Currently only - // supporting versions and version ranges for specifying a local project. - if ( - (dependencySpecifier.specifierType === 'version' || dependencySpecifier.specifierType === 'range') && - localProject && - !rushProject.cyclicDependencyProjects.has(name) - ) { - if (!semver.satisfies(localProject.packageJsonEditor.version, dependencySpecifier.versionSpecifier)) { - console.log(); - console.log(colors.red( - `"${rushProject.packageName}" depends on package "${name}" (${version}) which exists within the workspace ` - + 'but cannot be fulfilled with the specified version range. Either specify a valid version range, or add ' - + 'the package as a cyclic dependency.' - )); - throw new AlreadyReportedError(); - } - - packageJson.addOrUpdateDependency(name, `workspace:${version}`, dependencyType); - continue; - } else if (dependencySpecifier.specifierType === 'workspace') { - // Already specified as a local project, no need to validate - continue; - } - - // It is not a local dependency, validate that it is compatible - if (shrinkwrapFile) { - // if (!shrinkwrapFile.tryEnsureCompatibleDependency(dependencySpecifier, rushProject.tempProjectName)) { - // shrinkwrapWarnings.push(`"${packageName}" (${packageVersion}) required by` - // + ` "${rushProject.packageName}"`); - // shrinkwrapIsUpToDate = false; - // } - if (!shrinkwrapFile.tryEnsureCompatibleDependency(dependencySpecifier, rushProject.packageName)) { - shrinkwrapWarnings.push(`"${name}" (${version}) required by "${rushProject.packageName}"`); - shrinkwrapIsUpToDate = false; - } - } - } - - // Save the package.json if we modified the version references - if (rushProject.packageJsonEditor.saveIfModified()) { - console.log(colors.yellow( - `"${rushProject.packageName}" depends on one or more workspace packages which did not use "workspace:" ` - + 'notation. The package.json has been modified and must be committed to source control.' - )); - } - } - - // Example: "C:\MyRepo\common\temp\package.json" - const commonPackageJsonFilename: string = path.join( - this._rushConfiguration.commonTempFolder, - FileConstants.PackageJson - ); - - if (shrinkwrapFile) { - // If we have a (possibly incomplete) shrinkwrap file, save it as the temporary file. - shrinkwrapFile.save(this._rushConfiguration.tempShrinkwrapFilename); - shrinkwrapFile.save(this._rushConfiguration.tempShrinkwrapPreinstallFilename); - } else { - // Otherwise delete the temporary file - FileSystem.deleteFile(this._rushConfiguration.tempShrinkwrapFilename); - - if (this._rushConfiguration.packageManager === 'pnpm') { - // Workaround for https://github.com/pnpm/pnpm/issues/1890 - // - // When "rush update --full" is run, rush deletes common/temp/pnpm-lock.yaml so that - // a new lockfile can be generated. But because of the above bug "pnpm install" would - // respect "common/temp/node_modules/.pnpm-lock.yaml" and thus would not generate a - // new lockfile. Deleting this file in addition to deleting common/temp/pnpm-lock.yaml - // ensures that a new lockfile will be generated with "rush update --full". - - const pnpmPackageManager: PnpmPackageManager = - (this._rushConfiguration.packageManagerWrapper as PnpmPackageManager); - - FileSystem.deleteFile(path.join(this._rushConfiguration.commonTempFolder, - pnpmPackageManager.internalShrinkwrapRelativePath)); - } - } - - // Don't update the file timestamp unless the content has changed, since "rush install" - // will consider this timestamp - JsonFile.save(commonPackageJson, commonPackageJsonFilename, { onlyIfChanged: true }); - workspaceFile.save(workspaceFile.workspaceFilename, { onlyIfChanged: true }); - - stopwatch.stop(); - console.log(`Finished creating workspace (${stopwatch.toString()})`); - // console.log(`Finished creating temporary modules (${stopwatch.toString()})`); - - if (shrinkwrapWarnings.length > 0) { - console.log(); - console.log(colors.yellow(Utilities.wrapWords( - `The ${this._shrinkwrapFilePhrase} is missing the following dependencies:`))); - - for (const shrinkwrapWarning of shrinkwrapWarnings) { - console.log(colors.yellow(' ' + shrinkwrapWarning)); - } - console.log(); - } - - return shrinkwrapIsUpToDate; - } - - /** - * Regenerates the common/package.json and all temp_modules projects. - * If shrinkwrapFile is provided, this function also validates whether it contains - * everything we need to install and returns true if so; in all other cases, - * the return value is false. - */ - private _createTempModulesAndCheckShrinkwrap(options: { - shrinkwrapFile: BaseShrinkwrapFile | undefined; - variant: string | undefined; - }): boolean { - const { - shrinkwrapFile, - variant - } = options; - - const stopwatch: Stopwatch = Stopwatch.start(); - - // Example: "C:\MyRepo\common\temp\projects" - const tempProjectsFolder: string = path.join(this._rushConfiguration.commonTempFolder, - RushConstants.rushTempProjectsFolderName); - - console.log(os.EOL + colors.bold('Updating temp projects in ' + tempProjectsFolder)); - - Utilities.createFolderWithRetry(tempProjectsFolder); - - const shrinkwrapWarnings: string[] = []; - - // We will start with the assumption that it's valid, and then set it to false if - // any of the checks fail - let shrinkwrapIsUpToDate: boolean = true; - - if (!shrinkwrapFile) { - shrinkwrapIsUpToDate = false; - } - - // dependency name --> version specifier - const allExplicitPreferredVersions: Map = this._rushConfiguration.getCommonVersions(variant) - .getAllPreferredVersions(); - - if (shrinkwrapFile) { - // Check any (explicitly) preferred dependencies first - allExplicitPreferredVersions.forEach((version: string, dependency: string) => { - const dependencySpecifier: DependencySpecifier = new DependencySpecifier(dependency, version); - - if (!shrinkwrapFile.hasCompatibleTopLevelDependency(dependencySpecifier)) { - shrinkwrapWarnings.push(`"${dependency}" (${version}) required by the preferred versions from ` - + RushConstants.commonVersionsFilename); - shrinkwrapIsUpToDate = false; - } - }); - - if (this._findOrphanedTempProjects(shrinkwrapFile)) { - // If there are any orphaned projects, then "npm install" would fail because the shrinkwrap - // contains references such as "resolved": "file:projects\\project1" that refer to nonexistent - // file paths. - shrinkwrapIsUpToDate = false; - } - } - - // Also copy down the committed .npmrc file, if there is one - // "common\config\rush\.npmrc" --> "common\temp\.npmrc" - // Also ensure that we remove any old one that may be hanging around - Utilities.syncNpmrc(this._rushConfiguration.commonRushConfigFolder, this._rushConfiguration.commonTempFolder); - - // also, copy the pnpmfile.js if it exists - if (this._rushConfiguration.packageManager === 'pnpm') { - const committedPnpmFilePath: string = - this._rushConfiguration.getPnpmfilePath(this._options.variant); - const tempPnpmFilePath: string - = path.join(this._rushConfiguration.commonTempFolder, RushConstants.pnpmfileFilename); - - // ensure that we remove any old one that may be hanging around - this._syncFile(committedPnpmFilePath, tempPnpmFilePath); - } - - const commonPackageJson: IPackageJson = { - dependencies: {}, - description: 'Temporary file generated by the Rush tool', - name: 'rush-common', - private: true, - version: '0.0.0' - }; - - // dependency name --> version specifier - const allPreferredVersions: Map = new Map(); - - // Should we add implicitly preferred versions? - let useImplicitlyPinnedVersions: boolean; - if (this._rushConfiguration.commonVersions.implicitlyPreferredVersions !== undefined) { - // Use the manually configured setting - useImplicitlyPinnedVersions = this._rushConfiguration.commonVersions.implicitlyPreferredVersions; - } else { - // Default to true. - useImplicitlyPinnedVersions = true; - } - - if (useImplicitlyPinnedVersions) { - // Add in the implicitly preferred versions. - // These are any first-level dependencies for which we only consume a single version range - // (e.g. every package that depends on react uses an identical specifier) - const implicitlyPreferredVersions: Map = - InstallManager.collectImplicitlyPreferredVersions(this._rushConfiguration, { variant }); - MapExtensions.mergeFromMap(allPreferredVersions, implicitlyPreferredVersions); - } - - // Add in the explicitly preferred versions. - // Note that these take precedence over implicitly preferred versions. - MapExtensions.mergeFromMap(allPreferredVersions, allExplicitPreferredVersions); - - // Add any preferred versions to the top of the commonPackageJson - // do this in alphabetical order for simpler debugging - for (const dependency of Array.from(allPreferredVersions.keys()).sort()) { - commonPackageJson.dependencies![dependency] = allPreferredVersions.get(dependency)!; - } - - // To make the common/package.json file more readable, sort alphabetically - // according to rushProject.tempProjectName instead of packageName. - const sortedRushProjects: RushConfigurationProject[] = this._rushConfiguration.projects.slice(0); - Sort.sortBy(sortedRushProjects, x => x.tempProjectName); - - for (const rushProject of sortedRushProjects) { - const packageJson: PackageJsonEditor = rushProject.packageJsonEditor; - - // Example: "C:\MyRepo\common\temp\projects\my-project-2.tgz" - const tarballFile: string = this._getTarballFilePath(rushProject); - - // Example: "my-project-2" - const unscopedTempProjectName: string = rushProject.unscopedTempProjectName; - - // Example: dependencies["@rush-temp/my-project-2"] = "file:./projects/my-project-2.tgz" - commonPackageJson.dependencies![rushProject.tempProjectName] - = `file:./${RushConstants.rushTempProjectsFolderName}/${rushProject.unscopedTempProjectName}.tgz`; - - const tempPackageJson: IRushTempPackageJson = { - name: rushProject.tempProjectName, - version: '0.0.0', - private: true, - dependencies: {} - }; - - // Collect pairs of (packageName, packageVersion) to be added as dependencies of the @rush-temp package.json - const tempDependencies: Map = new Map(); - - // These can be regular, optional, or peer dependencies (but NOT dev dependencies). - // (A given packageName will never appear more than once in this list.) - for (const dependency of packageJson.dependencyList) { - - // If there are any optional dependencies, copy directly into the optionalDependencies field. - if (dependency.dependencyType === DependencyType.Optional) { - if (!tempPackageJson.optionalDependencies) { - tempPackageJson.optionalDependencies = {}; - } - tempPackageJson.optionalDependencies[dependency.name] = dependency.version; - } else { - tempDependencies.set(dependency.name, dependency.version); - } - } - - for (const dependency of packageJson.devDependencyList) { - // If there are devDependencies, we need to merge them with the regular dependencies. If the same - // library appears in both places, then the dev dependency wins (because presumably it's saying what you - // want right now for development, not the range that you support for consumers). - tempDependencies.set(dependency.name, dependency.version); - } - Sort.sortMapKeys(tempDependencies); - - for (const [packageName, packageVersion] of tempDependencies.entries()) { - const dependencySpecifier: DependencySpecifier = new DependencySpecifier(packageName, packageVersion); - - // Is there a locally built Rush project that could satisfy this dependency? - // If so, then we will symlink to the project folder rather than to common/temp/node_modules. - // In this case, we don't want "npm install" to process this package, but we do need - // to record this decision for "rush link" later, so we add it to a special 'rushDependencies' field. - const localProject: RushConfigurationProject | undefined = - this._rushConfiguration.getProjectByName(packageName); - - if (localProject) { - // Don't locally link if it's listed in the cyclicDependencyProjects - if (!rushProject.cyclicDependencyProjects.has(packageName)) { - - // Also, don't locally link if the SemVer doesn't match - const localProjectVersion: string = localProject.packageJsonEditor.version; - if (semver.satisfies(localProjectVersion, packageVersion)) { - - // We will locally link this package, so instead add it to our special "rushDependencies" - // field in the package.json file. - if (!tempPackageJson.rushDependencies) { - tempPackageJson.rushDependencies = {}; - } - tempPackageJson.rushDependencies[packageName] = packageVersion; - continue; - } - } - } - - // We will NOT locally link this package; add it as a regular dependency. - tempPackageJson.dependencies![packageName] = packageVersion; - - if (shrinkwrapFile) { - if (!shrinkwrapFile.tryEnsureCompatibleDependency(dependencySpecifier, rushProject.tempProjectName)) { - shrinkwrapWarnings.push(`"${packageName}" (${packageVersion}) required by` - + ` "${rushProject.packageName}"`); - shrinkwrapIsUpToDate = false; - } - } - } - - // NPM expects the root of the tarball to have a directory called 'package' - const npmPackageFolder: string = 'package'; - - // Example: "C:\MyRepo\common\temp\projects\my-project-2" - const tempProjectFolder: string = path.join( - this._rushConfiguration.commonTempFolder, - RushConstants.rushTempProjectsFolderName, - unscopedTempProjectName); - - // Example: "C:\MyRepo\common\temp\projects\my-project-2\package.json" - const tempPackageJsonFilename: string = path.join(tempProjectFolder, FileConstants.PackageJson); - - // we only want to overwrite the package if the existing tarball's package.json is different from tempPackageJson - let shouldOverwrite: boolean = true; - try { - // if the tarball and the temp file still exist, then compare the contents - if (FileSystem.exists(tarballFile) && FileSystem.exists(tempPackageJsonFilename)) { - - // compare the extracted package.json with the one we are about to write - const oldBuffer: Buffer = FileSystem.readFileToBuffer(tempPackageJsonFilename); - const newBuffer: Buffer = Buffer.from(JsonFile.stringify(tempPackageJson)); - - if (Buffer.compare(oldBuffer, newBuffer) === 0) { - shouldOverwrite = false; - } - } - } catch (error) { - // ignore the error, we will go ahead and create a new tarball - } - - if (shouldOverwrite) { - try { - // ensure the folder we are about to zip exists - Utilities.createFolderWithRetry(tempProjectFolder); - - // remove the old tarball & old temp package json, this is for any cases where new tarball creation - // fails, and the shouldOverwrite logic is messed up because the my-project-2\package.json - // exists and is updated, but the tarball is not accurate - FileSystem.deleteFile(tarballFile); - FileSystem.deleteFile(tempPackageJsonFilename); - - // write the expected package.json file into the zip staging folder - JsonFile.save(tempPackageJson, tempPackageJsonFilename); - - const tarOptions: tar.CreateOptions = ({ - gzip: true, - file: tarballFile, - cwd: tempProjectFolder, - portable: true, - noMtime: true, - noPax: true, - sync: true, - prefix: npmPackageFolder, - filter: (path: string, stat: tar.FileStat): boolean => { - if (!this._rushConfiguration.experimentsConfiguration.configuration - .noChmodFieldInTarHeaderNormalization) { - stat.mode = (stat.mode & ~0x1FF) | PosixModeBits.AllRead | PosixModeBits.UserWrite - | PosixModeBits.AllExecute; - } - return true; - } - } as tar.CreateOptions); - - // create the new tarball - tar.create(tarOptions, [FileConstants.PackageJson]); - - console.log(`Updating ${tarballFile}`); - } catch (error) { - console.log(colors.yellow(error)); - // delete everything in case of any error - FileSystem.deleteFile(tarballFile); - FileSystem.deleteFile(tempPackageJsonFilename); - } - } - } - - // Example: "C:\MyRepo\common\temp\package.json" - const commonPackageJsonFilename: string = path.join(this._rushConfiguration.commonTempFolder, - FileConstants.PackageJson); - - if (shrinkwrapFile) { - // If we have a (possibly incomplete) shrinkwrap file, check to see if any shrinkwrap-specific - // changes make the shrinkwrap out-of-date, and save it as the temporary file. - if (shrinkwrapFile.shouldForceRecheck()) { - shrinkwrapIsUpToDate = false; - } - - shrinkwrapFile.save(this._rushConfiguration.tempShrinkwrapFilename); - shrinkwrapFile.save(this._rushConfiguration.tempShrinkwrapPreinstallFilename); - } else { - // Otherwise delete the temporary file - FileSystem.deleteFile(this._rushConfiguration.tempShrinkwrapFilename); - - if (this._rushConfiguration.packageManager === 'pnpm') { - // Workaround for https://github.com/pnpm/pnpm/issues/1890 - // - // When "rush update --full" is run, rush deletes common/temp/pnpm-lock.yaml so that - // a new lockfile can be generated. But because of the above bug "pnpm install" would - // respect "common/temp/node_modules/.pnpm-lock.yaml" and thus would not generate a - // new lockfile. Deleting this file in addition to deleting common/temp/pnpm-lock.yaml - // ensures that a new lockfile will be generated with "rush update --full". - - const pnpmPackageManager: PnpmPackageManager = - (this._rushConfiguration.packageManagerWrapper as PnpmPackageManager); - - FileSystem.deleteFile(path.join(this._rushConfiguration.commonTempFolder, - pnpmPackageManager.internalShrinkwrapRelativePath)); - } - } - - // Don't update the file timestamp unless the content has changed, since "rush install" - // will consider this timestamp - JsonFile.save(commonPackageJson, commonPackageJsonFilename, { onlyIfChanged: true }); - - stopwatch.stop(); - console.log(`Finished creating temporary modules (${stopwatch.toString()})`); - - if (shrinkwrapWarnings.length > 0) { - console.log(); - console.log(colors.yellow(Utilities.wrapWords( - `The ${this._shrinkwrapFilePhrase} is missing the following dependencies:`))); - - for (const shrinkwrapWarning of shrinkwrapWarnings) { - console.log(colors.yellow(' ' + shrinkwrapWarning)); - } - console.log(); - } - - return shrinkwrapIsUpToDate; - } - - /** - * Runs "npm install" in the common folder. - */ - private _installWorkspace(options: { - shrinkwrapIsUpToDate: boolean; - variantIsUpToDate: boolean; - } & IInstallManagerOptions): Promise { - const { - shrinkwrapIsUpToDate, - variantIsUpToDate - } = options; - - const projectNodeModulesFolders: string[] = this._rushConfiguration.projects.map(project => { - return path.join(project.projectFolder, 'node_modules') - }); - - return Promise.resolve().then(() => { - // console.log(os.EOL + colors.bold('Checking node_modules in ' + this._rushConfiguration.commonTempFolder) - // + os.EOL); - - // const commonNodeModulesFolder: string = path.join(this._rushConfiguration.commonTempFolder, - // 'node_modules'); - - // This marker file indicates that the last "rush install" completed successfully - const markerFileExistedAndWasValidAtStart: boolean = this._commonLastInstallMarker.checkValidAndReportStoreIssues(); - - // If "--clean" or "--full-clean" was specified, or if the last install was interrupted, - // then we will need to delete the node_modules folder. Otherwise, we can do an incremental - // install. - const deleteNodeModules: boolean = !markerFileExistedAndWasValidAtStart; - - // Based on timestamps, can we skip this install entirely? - if (shrinkwrapIsUpToDate && !deleteNodeModules && variantIsUpToDate) { - const potentiallyChangedFiles: string[] = []; - - // Consider the timestamp on the node_modules folder; if someone tampered with it - // or deleted it entirely, then we can't skip this install - potentiallyChangedFiles.push(...projectNodeModulesFolders); - - // Additionally, if they pulled an updated npm-shrinkwrap.json file from Git, - // then we can't skip this install - potentiallyChangedFiles.push(this._rushConfiguration.getCommittedShrinkwrapFilename(options.variant)); - - if (this._rushConfiguration.packageManager === 'pnpm') { - // If the repo is using pnpmfile.js, consider that also - const pnpmFileFilename: string = this._rushConfiguration.getPnpmfilePath(options.variant); - - if (FileSystem.exists(pnpmFileFilename)) { - potentiallyChangedFiles.push(pnpmFileFilename); - } - } - - // // Also consider timestamps for all the temp tarballs. (createTempModulesAndCheckShrinkwrap() will - // // carefully preserve these timestamps unless something has changed.) - // // Example: "C:\MyRepo\common\temp\projects\my-project-2.tgz" - // potentiallyChangedFiles.push(...this._rushConfiguration.projects.map(x => { - // return this._getTarballFilePath(x); - // })); - // Also consider timestamps for all the package.json files. - // Example: "C:\MyRepo\projects\my-project-2\package.json" - potentiallyChangedFiles.push(...this._rushConfiguration.projects.map(x => x.packageJsonEditor.filePath)); - - // NOTE: If commonLastInstallMarkerFilename (or any of the potentiallyChangedFiles) does not - // exist, then isFileTimestampCurrent() returns false. - if (Utilities.isFileTimestampCurrent(this._commonLastInstallMarker.path, potentiallyChangedFiles)) { - // Nothing to do, because everything is up to date according to time stamps - return; - } - } - - return this._checkIfReleaseIsPublished() - .catch((error) => { - // If the user is working in an environment that can't reach the registry, - // don't bother them with errors. - return undefined; - }).then((publishedRelease: boolean | undefined) => { - - if (publishedRelease === false) { - console.log(colors.yellow('Warning: This release of the Rush tool was unpublished; it may be unstable.')); - } - - // Since we're going to be tampering with common/node_modules, delete the "rush link" flag file if it exists; - // this ensures that a full "rush link" is required next time - Utilities.deleteFile(this._rushConfiguration.rushLinkJsonFilename); - - // Delete the successful install file to indicate the install transaction has started - this._commonLastInstallMarker.clear(); - - // NOTE: The PNPM store is supposed to be transactionally safe, so we don't delete it automatically. - // The user must request that via the command line. - if (deleteNodeModules) { - if (this._rushConfiguration.packageManager === 'npm') { - console.log(`Deleting the "npm-cache" folder`); - // This is faster and more thorough than "npm cache clean" - this._commonTempFolderRecycler.moveFolder(this._rushConfiguration.npmCacheFolder); - - console.log(`Deleting the "npm-tmp" folder`); - this._commonTempFolderRecycler.moveFolder(this._rushConfiguration.npmTmpFolder); - } - } - - // Example: "C:\MyRepo\common\temp\npm-local\node_modules\.bin\npm" - const packageManagerFilename: string = this._rushConfiguration.packageManagerToolFilename; - - let packageManagerEnv: NodeJS.ProcessEnv = process.env; - - let configurationEnvironment: IConfigurationEnvironment | undefined = undefined; - - if (this._rushConfiguration.packageManager === 'npm') { - // if ( - // this._rushConfiguration.npmOptions && - // this._rushConfiguration.npmOptions.environmentVariables - // ) { - // configurationEnvironment = this._rushConfiguration.npmOptions.environmentVariables; - // } - } else if (this._rushConfiguration.packageManager === 'pnpm') { - if ( - this._rushConfiguration.pnpmOptions && - this._rushConfiguration.pnpmOptions.environmentVariables - ) { - configurationEnvironment = this._rushConfiguration.pnpmOptions.environmentVariables; - } - } else if (this._rushConfiguration.packageManager === 'yarn') { - if ( - this._rushConfiguration.yarnOptions && - this._rushConfiguration.yarnOptions.environmentVariables - ) { - configurationEnvironment = this._rushConfiguration.yarnOptions.environmentVariables; - } - } - - packageManagerEnv = this._mergeEnvironmentVariables( - process.env, - configurationEnvironment - ); - - for (const nodeModulesFolder of projectNodeModulesFolders) { - // Is there an existing "node_modules" folder to consider? - if (FileSystem.exists(nodeModulesFolder)) { - // Should we delete the "node_modules" folder? - if (deleteNodeModules) { - // YES: Delete "node_modules" - - // Explain to the user why we are hosing their node_modules folder - console.log('Deleting files from ' + nodeModulesFolder); - - this._commonTempFolderRecycler.moveFolder(nodeModulesFolder); - - // Utilities.createFolderWithRetry(commonNodeModulesFolder); - } else { - // NO: Prepare to do an incremental install in the "node_modules" folder - - // note: it is not necessary to run "prune" with pnpm - // if (this._rushConfiguration.packageManager === 'npm') { - // console.log(`Running "${this._rushConfiguration.packageManager} prune"` - // + ` in ${this._rushConfiguration.commonTempFolder}`); - // const args: string[] = ['prune']; - // this._pushConfigurationArgs(args, options); - - // Utilities.executeCommandWithRetry(this._options.maxInstallAttempts, packageManagerFilename, args, - // this._rushConfiguration.commonTempFolder, packageManagerEnv); - - // // Delete the (installed image of) the temp projects, since "npm install" does not - // // detect changes for "file:./" references. - // // We recognize the temp projects by their names, which always start with "rush-". - - // // Example: "C:\MyRepo\common\temp\node_modules\@rush-temp" - // const pathToDeleteWithoutStar: string = path.join(commonNodeModulesFolder, - // RushConstants.rushTempNpmScope); - // console.log(`Deleting ${pathToDeleteWithoutStar}\\*`); - // // Glob can't handle Windows paths - // const normalizedpathToDeleteWithoutStar: string = Text.replaceAll(pathToDeleteWithoutStar, '\\', '/'); - - // // Example: "C:/MyRepo/common/temp/node_modules/@rush-temp/*" - // for (const tempModulePath of glob.sync(globEscape(normalizedpathToDeleteWithoutStar) + '/*')) { - // // We could potentially use AsyncRecycler here, but in practice these folders tend - // // to be very small - // Utilities.dangerouslyDeletePath(tempModulePath); - // } - // } - } - } - } - - if (this._rushConfiguration.packageManager === 'yarn') { - // Yarn does not correctly detect changes to a tarball, so we need to forcibly clear its cache - const yarnRushTempCacheFolder: string = path.join( - this._rushConfiguration.yarnCacheFolder, 'v2', 'npm-@rush-temp' - ); - if (FileSystem.exists(yarnRushTempCacheFolder)) { - console.log('Deleting ' + yarnRushTempCacheFolder); - Utilities.dangerouslyDeletePath(yarnRushTempCacheFolder); - } - } - - // Run "npm install" in the common folder - const installArgs: string[] = ['install']; - this._pushConfigurationArgs(installArgs, options); - - console.log(os.EOL + colors.bold(`Running "${this._rushConfiguration.packageManager} install" in` - + ` ${this._rushConfiguration.commonTempFolder}`) + os.EOL); - - // If any diagnostic options were specified, then show the full command-line - // if (options.debug || options.collectLogFile || options.networkConcurrency) { - console.log(os.EOL + colors.green('Invoking package manager: ') - + FileSystem.getRealPath(packageManagerFilename) + ' ' + installArgs.join(' ') + os.EOL); - // } - - try { - Utilities.executeCommandWithRetry(this._options.maxInstallAttempts, packageManagerFilename, - installArgs, - this._rushConfiguration.commonTempFolder, - packageManagerEnv, - false, () => { - if (this._rushConfiguration.packageManager === 'pnpm') { - console.log(colors.yellow(`Deleting the "node_modules" folders`)); - for (const nodeModulesFolder of projectNodeModulesFolders) { - if (FileSystem.exists(nodeModulesFolder)) { - this._commonTempFolderRecycler.moveFolder(nodeModulesFolder); - } - } - - // Leave the pnpm-store as is for the retry. This ensures that packages that have already - // been downloaded need not be downloaded again, thereby potentially increasing the chances - // of a subsequent successful install. - - // Utilities.createFolderWithRetry(commonNodeModulesFolder); - } - }); - } catch (error) { - // All the install attempts failed. - - if ( - this._rushConfiguration.packageManager === 'pnpm' && - this._rushConfiguration.pnpmOptions.pnpmStore === 'local' - ) { - // If the installation has failed even after the retries, then pnpm store may - // have got into a corrupted, irrecoverable state. Delete the store so that a - // future install can create the store afresh. - console.log(colors.yellow(`Deleting the "pnpm-store" folder`)); - this._commonTempFolderRecycler.moveFolder(this._rushConfiguration.pnpmOptions.pnpmStorePath); - } - - throw error; - } - - // if (this._rushConfiguration.packageManager === 'npm') { - - // console.log(os.EOL + colors.bold('Running "npm shrinkwrap"...')); - // const npmArgs: string[] = ['shrinkwrap']; - // this._pushConfigurationArgs(npmArgs, options); - // Utilities.executeCommand(this._rushConfiguration.packageManagerToolFilename, - // npmArgs, this._rushConfiguration.commonTempFolder); - // console.log('"npm shrinkwrap" completed' + os.EOL); - - // this._fixupNpm5Regression(); - // } - - if (options.allowShrinkwrapUpdates && !shrinkwrapIsUpToDate) { - // Copy (or delete) common\temp\pnpm-lock.yaml --> common\config\rush\pnpm-lock.yaml - this._syncFile(this._rushConfiguration.tempShrinkwrapFilename, - this._rushConfiguration.getCommittedShrinkwrapFilename(options.variant)); - } else { - // TODO: Validate whether the package manager updated it in a nontrivial way - } - - // Finally, create the marker file to indicate a successful install - this._commonLastInstallMarker.create(); - - console.log(''); - }); - }); - } - - /** - * Runs "npm install" in the common folder. - */ - private _installCommonModules(options: { - shrinkwrapIsUpToDate: boolean; - variantIsUpToDate: boolean; - } & IInstallManagerOptions): Promise { - const { - shrinkwrapIsUpToDate, - variantIsUpToDate - } = options; - - return Promise.resolve().then(() => { - console.log(os.EOL + colors.bold('Checking node_modules in ' + this._rushConfiguration.commonTempFolder) - + os.EOL); - - const commonNodeModulesFolder: string = path.join(this._rushConfiguration.commonTempFolder, - 'node_modules'); - - // This marker file indicates that the last "rush install" completed successfully - const markerFileExistedAndWasValidAtStart: boolean = this._commonLastInstallMarker.checkValidAndReportStoreIssues(); - - // If "--clean" or "--full-clean" was specified, or if the last install was interrupted, - // then we will need to delete the node_modules folder. Otherwise, we can do an incremental - // install. - const deleteNodeModules: boolean = !markerFileExistedAndWasValidAtStart; - - // Based on timestamps, can we skip this install entirely? - if (shrinkwrapIsUpToDate && !deleteNodeModules && variantIsUpToDate) { - const potentiallyChangedFiles: string[] = []; - - // Consider the timestamp on the node_modules folder; if someone tampered with it - // or deleted it entirely, then we can't skip this install - potentiallyChangedFiles.push(commonNodeModulesFolder); - - // Additionally, if they pulled an updated npm-shrinkwrap.json file from Git, - // then we can't skip this install - potentiallyChangedFiles.push(this._rushConfiguration.getCommittedShrinkwrapFilename(options.variant)); - - // Add common-versions.json file in potentially changed file list. - potentiallyChangedFiles.push(this._rushConfiguration.getCommonVersionsFilePath(options.variant)); - - if (this._rushConfiguration.packageManager === 'pnpm') { - // If the repo is using pnpmfile.js, consider that also - const pnpmFileFilename: string = this._rushConfiguration.getPnpmfilePath(options.variant); - - if (FileSystem.exists(pnpmFileFilename)) { - potentiallyChangedFiles.push(pnpmFileFilename); - } - } - - // Also consider timestamps for all the temp tarballs. (createTempModulesAndCheckShrinkwrap() will - // carefully preserve these timestamps unless something has changed.) - // Example: "C:\MyRepo\common\temp\projects\my-project-2.tgz" - potentiallyChangedFiles.push(...this._rushConfiguration.projects.map(x => { - return this._getTarballFilePath(x); - })); - - // NOTE: If commonNodeModulesMarkerFilename (or any of the potentiallyChangedFiles) does not - // exist, then isFileTimestampCurrent() returns false. - if (Utilities.isFileTimestampCurrent(this._commonLastInstallMarker.path, potentiallyChangedFiles)) { - // Nothing to do, because everything is up to date according to time stamps - return; - } - } - - return this._checkIfReleaseIsPublished() - .catch((error) => { - // If the user is working in an environment that can't reach the registry, - // don't bother them with errors. - return undefined; - }).then((publishedRelease: boolean | undefined) => { - - if (publishedRelease === false) { - console.log(colors.yellow('Warning: This release of the Rush tool was unpublished; it may be unstable.')); - } - - // Since we're going to be tampering with common/node_modules, delete the "rush link" flag file if it exists; - // this ensures that a full "rush link" is required next time - Utilities.deleteFile(this._rushConfiguration.rushLinkJsonFilename); - - // Delete the successful install file to indicate the install transaction has started - this._commonLastInstallMarker.clear(); - - // NOTE: The PNPM store is supposed to be transactionally safe, so we don't delete it automatically. - // The user must request that via the command line. - if (deleteNodeModules) { - if (this._rushConfiguration.packageManager === 'npm') { - console.log(`Deleting the "npm-cache" folder`); - // This is faster and more thorough than "npm cache clean" - this._commonTempFolderRecycler.moveFolder(this._rushConfiguration.npmCacheFolder); - - console.log(`Deleting the "npm-tmp" folder`); - this._commonTempFolderRecycler.moveFolder(this._rushConfiguration.npmTmpFolder); - } - } - - // Example: "C:\MyRepo\common\temp\npm-local\node_modules\.bin\npm" - const packageManagerFilename: string = this._rushConfiguration.packageManagerToolFilename; - - let packageManagerEnv: NodeJS.ProcessEnv = process.env; - - let configurationEnvironment: IConfigurationEnvironment | undefined = undefined; - - if (this._rushConfiguration.packageManager === 'npm') { - if ( - this._rushConfiguration.npmOptions && - this._rushConfiguration.npmOptions.environmentVariables - ) { - configurationEnvironment = this._rushConfiguration.npmOptions.environmentVariables; - } - } else if (this._rushConfiguration.packageManager === 'pnpm') { - if ( - this._rushConfiguration.pnpmOptions && - this._rushConfiguration.pnpmOptions.environmentVariables - ) { - configurationEnvironment = this._rushConfiguration.pnpmOptions.environmentVariables; - } - } else if (this._rushConfiguration.packageManager === 'yarn') { - if ( - this._rushConfiguration.yarnOptions && - this._rushConfiguration.yarnOptions.environmentVariables - ) { - configurationEnvironment = this._rushConfiguration.yarnOptions.environmentVariables; - } - } - - packageManagerEnv = this._mergeEnvironmentVariables( - process.env, - configurationEnvironment - ); - - // Is there an existing "node_modules" folder to consider? - if (FileSystem.exists(commonNodeModulesFolder)) { - // Should we delete the entire "node_modules" folder? - if (deleteNodeModules) { - // YES: Delete "node_modules" - - // Explain to the user why we are hosing their node_modules folder - console.log('Deleting files from ' + commonNodeModulesFolder); - - this._commonTempFolderRecycler.moveFolder(commonNodeModulesFolder); - - Utilities.createFolderWithRetry(commonNodeModulesFolder); - } else { - // NO: Prepare to do an incremental install in the "node_modules" folder - - // note: it is not necessary to run "prune" with pnpm - if (this._rushConfiguration.packageManager === 'npm') { - console.log(`Running "${this._rushConfiguration.packageManager} prune"` - + ` in ${this._rushConfiguration.commonTempFolder}`); - const args: string[] = ['prune']; - this._pushConfigurationArgs(args, options); - - Utilities.executeCommandWithRetry(this._options.maxInstallAttempts, packageManagerFilename, args, - this._rushConfiguration.commonTempFolder, packageManagerEnv); - - // Delete the (installed image of) the temp projects, since "npm install" does not - // detect changes for "file:./" references. - // We recognize the temp projects by their names, which always start with "rush-". - - // Example: "C:\MyRepo\common\temp\node_modules\@rush-temp" - const pathToDeleteWithoutStar: string = path.join(commonNodeModulesFolder, - RushConstants.rushTempNpmScope); - console.log(`Deleting ${pathToDeleteWithoutStar}\\*`); - // Glob can't handle Windows paths - const normalizedpathToDeleteWithoutStar: string = Text.replaceAll(pathToDeleteWithoutStar, '\\', '/'); - - // Example: "C:/MyRepo/common/temp/node_modules/@rush-temp/*" - for (const tempModulePath of glob.sync(globEscape(normalizedpathToDeleteWithoutStar) + '/*')) { - // We could potentially use AsyncRecycler here, but in practice these folders tend - // to be very small - Utilities.dangerouslyDeletePath(tempModulePath); - } - } - } - } - - if (this._rushConfiguration.packageManager === 'yarn') { - // Yarn does not correctly detect changes to a tarball, so we need to forcibly clear its cache - const yarnRushTempCacheFolder: string = path.join( - this._rushConfiguration.yarnCacheFolder, 'v2', 'npm-@rush-temp' - ); - if (FileSystem.exists(yarnRushTempCacheFolder)) { - console.log('Deleting ' + yarnRushTempCacheFolder); - Utilities.dangerouslyDeletePath(yarnRushTempCacheFolder); - } - } - - // Run "npm install" in the common folder - const installArgs: string[] = ['install']; - this._pushConfigurationArgs(installArgs, options); - - console.log(os.EOL + colors.bold(`Running "${this._rushConfiguration.packageManager} install" in` - + ` ${this._rushConfiguration.commonTempFolder}`) + os.EOL); - - // If any diagnostic options were specified, then show the full command-line - if (options.debug || options.collectLogFile || options.networkConcurrency) { - console.log(os.EOL + colors.green('Invoking package manager: ') - + FileSystem.getRealPath(packageManagerFilename) + ' ' + installArgs.join(' ') + os.EOL); - } - - try { - Utilities.executeCommandWithRetry(this._options.maxInstallAttempts, packageManagerFilename, - installArgs, - this._rushConfiguration.commonTempFolder, - packageManagerEnv, - false, () => { - if (this._rushConfiguration.packageManager === 'pnpm') { - console.log(colors.yellow(`Deleting the "node_modules" folder`)); - this._commonTempFolderRecycler.moveFolder(commonNodeModulesFolder); - - // Leave the pnpm-store as is for the retry. This ensures that packages that have already - // been downloaded need not be downloaded again, thereby potentially increasing the chances - // of a subsequent successful install. - - Utilities.createFolderWithRetry(commonNodeModulesFolder); - } - }); - } catch (error) { - // All the install attempts failed. - - if ( - this._rushConfiguration.packageManager === 'pnpm' && - this._rushConfiguration.pnpmOptions.pnpmStore === 'local' - ) { - // If the installation has failed even after the retries, then pnpm store may - // have got into a corrupted, irrecoverable state. Delete the store so that a - // future install can create the store afresh. - console.log(colors.yellow(`Deleting the "pnpm-store" folder`)); - this._commonTempFolderRecycler.moveFolder(this._rushConfiguration.pnpmOptions.pnpmStorePath); - } - - throw error; - } - - if (this._rushConfiguration.packageManager === 'npm') { - - console.log(os.EOL + colors.bold('Running "npm shrinkwrap"...')); - const npmArgs: string[] = ['shrinkwrap']; - this._pushConfigurationArgs(npmArgs, options); - Utilities.executeCommand(this._rushConfiguration.packageManagerToolFilename, - npmArgs, this._rushConfiguration.commonTempFolder); - console.log('"npm shrinkwrap" completed' + os.EOL); - - this._fixupNpm5Regression(); - } - - if (options.allowShrinkwrapUpdates && !shrinkwrapIsUpToDate) { - // Shrinkwrap files may need to be post processed after install, so load and save it - const tempShrinkwrapFile: BaseShrinkwrapFile | undefined = ShrinkwrapFileFactory.getShrinkwrapFile( - this._rushConfiguration.packageManager, - this._rushConfiguration.packageManagerOptions, - this._rushConfiguration.tempShrinkwrapFilename); - if (tempShrinkwrapFile) { - tempShrinkwrapFile.save(this._rushConfiguration.tempShrinkwrapFilename); - } - - // Copy (or delete) common\temp\pnpm-lock.yaml --> common\config\rush\pnpm-lock.yaml - this._syncFile(this._rushConfiguration.tempShrinkwrapFilename, - this._rushConfiguration.getCommittedShrinkwrapFilename(options.variant)); - } else { - // TODO: Validate whether the package manager updated it in a nontrivial way - } - - // Finally, create the marker file to indicate a successful install - this._commonLastInstallMarker.create(); - - console.log(''); - }); - }); - } - - private _mergeEnvironmentVariables( - baseEnv: NodeJS.ProcessEnv, - environmentVariables?: IConfigurationEnvironment - ): NodeJS.ProcessEnv { - const packageManagerEnv: NodeJS.ProcessEnv = baseEnv; - - if (environmentVariables) { - // eslint-disable-next-line guard-for-in - for (const envVar in environmentVariables) { - let setEnvironmentVariable: boolean = true; - console.log(`\nProcessing definition for environment variable: ${envVar}`); - - if (baseEnv.hasOwnProperty(envVar)) { - setEnvironmentVariable = false; - console.log(`Environment variable already defined:`); - console.log(` Name: ${envVar}`); - console.log(` Existing value: ${baseEnv[envVar]}`); - console.log(` Value set in rush.json: ${environmentVariables[envVar].value}`); - - if (environmentVariables[envVar].override) { - setEnvironmentVariable = true; - console.log(`Overriding the environment variable with the value set in rush.json.`); - } - else { - console.log(colors.yellow(`WARNING: Not overriding the value of the environment variable.`)); - } - } - - if (setEnvironmentVariable) { - if (this._options.debug) { - console.log(`Setting environment variable for package manager.`); - console.log(` Name: ${envVar}`); - console.log(` Value: ${environmentVariables[envVar].value}`); - } - packageManagerEnv[envVar] = environmentVariables[envVar].value; - } - } - } - - return packageManagerEnv; - } - - private _checkIfReleaseIsPublished(): Promise { - return Promise.resolve().then(() => { - const lastCheckFile: string = path.join(this._rushGlobalFolder.nodeSpecificPath, - 'rush-' + Rush.version, 'last-check.flag'); - - if (FileSystem.exists(lastCheckFile)) { - let cachedResult: boolean | 'error' | undefined = undefined; - try { - // NOTE: mtimeMs is not supported yet in Node.js 6.x - const nowMs: number = new Date().getTime(); - const ageMs: number = nowMs - FileSystem.getStatistics(lastCheckFile).mtime.getTime(); - const HOUR: number = 60 * 60 * 1000; - - // Is the cache too old? - if (ageMs < 24 * HOUR) { - // No, read the cached result - cachedResult = JsonFile.load(lastCheckFile); - } - } catch (e) { - // Unable to parse file - } - if (cachedResult === 'error') { - return Promise.reject(new Error('Unable to contact server')); - } - if (cachedResult === true || cachedResult === false) { - return cachedResult; - } - } - - // Before we start the network operation, record a failed state. If the process exits for some reason, - // this will record the error. It will also update the timestamp to prevent other Rush instances - // from attempting to update the file. - JsonFile.save('error', lastCheckFile, { ensureFolderExists: true }); - - // For this check we use the official registry, not the private registry - return this._queryIfReleaseIsPublished('https://registry.npmjs.org:443') - .then((publishedRelease: boolean) => { - // Cache the result - JsonFile.save(publishedRelease, lastCheckFile, { ensureFolderExists: true }); - return publishedRelease; - }) - .catch((error: Error) => { - JsonFile.save('error', lastCheckFile, { ensureFolderExists: true }); - return Promise.reject(error); - }); - }); - } - - private _queryIfReleaseIsPublished(registryUrl: string): Promise { - let queryUrl: string = registryUrl; - if (queryUrl[-1] !== '/') { - queryUrl += '/'; - } - // Note that the "@" symbol does not normally get URL-encoded - queryUrl += RushConstants.rushPackageName.replace('/', '%2F'); - - const userAgent: string = `pnpm/? npm/? node/${process.version} ${os.platform()} ${os.arch()}`; - - const headers: fetch.Headers = new fetch.Headers(); - headers.append('user-agent', userAgent); - headers.append('accept', 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*'); - - let agent: http.Agent | undefined = undefined; - if (process.env.HTTP_PROXY) { - agent = new HttpsProxyAgent(process.env.HTTP_PROXY); - } - - return fetch.default(queryUrl, { - headers: headers, - agent: agent - }) - .then((response: fetch.Response) => { - if (!response.ok) { - return Promise.reject(new Error('Failed to query')); - } - return response - .json() - .then((data) => { - let url: string; - try { - if (!data.versions[Rush.version]) { - // Version was not published - return false; - } - url = data.versions[Rush.version].dist.tarball; - if (!url) { - return Promise.reject(new Error(`URL not found`)); - } - } catch (e) { - return Promise.reject(new Error('Error parsing response')); - } - - // Make sure the tarball wasn't deleted from the CDN - headers.set('accept', '*/*'); - return fetch.default(url, { - headers: headers, - agent: agent - }) - .then((response2: fetch.Response) => { - if (!response2.ok) { - if (response2.status === 404) { - return false; - } else { - return Promise.reject(new Error('Failed to fetch')); - } - } - return true; - }); - }); - }); - } - - /** - * Used when invoking the NPM tool. Appends the common configuration options - * to the command-line. - */ - private _pushConfigurationArgs(args: string[], options: IInstallManagerOptions): void { - if (this._rushConfiguration.packageManager === 'npm') { - if (semver.lt(this._rushConfiguration.packageManagerToolVersion, '5.0.0')) { - // NOTE: - // - // When using an npm version older than v5.0.0, we do NOT install optional dependencies for - // Rush, because npm does not generate the shrinkwrap file consistently across platforms. - // - // Consider the "fsevents" package. This is a Mac specific package - // which is an optional second-order dependency. Optional dependencies work by attempting to install - // the package, but removes the package if the install failed. - // This means that someone running generate on a Mac WILL have fsevents included in their shrinkwrap. - // When someone using Windows attempts to install from the shrinkwrap, the install will fail. - // - // If someone generates the shrinkwrap using Windows, then fsevents will NOT be listed in the shrinkwrap. - // When someone using Mac attempts to install from the shrinkwrap, they will NOT have the - // optional dependency installed. - // - // This issue has been fixed as of npm v5.0.0: https://github.com/npm/npm/releases/tag/v5.0.0 - // - // For more context, see https://github.com/microsoft/rushstack/issues/761#issuecomment-428689600 - args.push('--no-optional'); - } - args.push('--cache', this._rushConfiguration.npmCacheFolder); - args.push('--tmp', this._rushConfiguration.npmTmpFolder); - - if (options.collectLogFile) { - args.push('--verbose'); - } - } else if (this._rushConfiguration.packageManager === 'pnpm') { - // Only explicitly define the store path if `pnpmStore` is using the default, or has been set to - // 'local'. If `pnpmStore` = 'global', then allow PNPM to use the system's default - // path. In all cases, this will be overridden by RUSH_PNPM_STORE_PATH - if ( - this._rushConfiguration.pnpmOptions.pnpmStore === 'local' || - EnvironmentConfiguration.pnpmStorePathOverride - ) { - args.push('--store', this._rushConfiguration.pnpmOptions.pnpmStorePath); - } - - // we are using the --no-lock flag for now, which unfortunately prints a warning, but should be OK - // since rush already has its own install lock file which will invalidate the cache for us. - // we theoretically could use the lock file, but we would need to clean the store if the - // lockfile existed, otherwise PNPM would hang indefinitely. it is simpler to rely on Rush's - // last install flag, which encapsulates the entire installation - args.push('--no-lock'); - - // Ensure that Rush's tarball dependencies get synchronized properly with the pnpm-lock.yaml file. - // See this GitHub issue: https://github.com/pnpm/pnpm/issues/1342 - - if (semver.gte(this._rushConfiguration.packageManagerToolVersion, '3.0.0')) { - args.push('--no-prefer-frozen-lockfile'); - } else { - args.push('--no-prefer-frozen-shrinkwrap'); - } - - if (options.collectLogFile) { - args.push('--reporter', 'ndjson'); - } - - if (options.networkConcurrency) { - args.push('--network-concurrency', options.networkConcurrency.toString()); - } - - if (this._rushConfiguration.pnpmOptions.strictPeerDependencies) { - args.push('--strict-peer-dependencies'); - } - - if ((this._rushConfiguration.packageManagerWrapper as PnpmPackageManager).supportsResolutionStrategy) { - args.push('--resolution-strategy', this._rushConfiguration.pnpmOptions.resolutionStrategy); - } - - // If we're using PNPM workspaces, then we need to run it recursively for each project. We also need - // to turn off the 'link-workspace-packages' option, as we only want to rely on `workspace:` notation - // to link packages - if (this._rushConfiguration.pnpmOptions.useWorkspaces) { - args.push('--recursive'); - args.push('--link-workspace-packages', 'false'); - } - } else if (this._rushConfiguration.packageManager === 'yarn') { - args.push('--link-folder', 'yarn-link'); - args.push('--cache-folder', this._rushConfiguration.yarnCacheFolder); - - // Without this option, Yarn will sometimes stop and ask for user input on STDIN - // (e.g. "Which command would you like to run?"). - args.push('--non-interactive'); - - if (options.networkConcurrency) { - args.push('--network-concurrency', options.networkConcurrency.toString()); - } - - if (this._rushConfiguration.yarnOptions.ignoreEngines) { - args.push('--ignore-engines'); - } - } - } - - /** - * Copies the file "sourcePath" to "destinationPath", overwriting the target file location. - * If the source file does not exist, then the target file is deleted. - */ - private _syncFile(sourcePath: string, destinationPath: string): void { - if (FileSystem.exists(sourcePath)) { - console.log('Updating ' + destinationPath); - FileSystem.copyFile({ sourcePath, destinationPath }); - } else { - if (FileSystem.exists(destinationPath)) { - console.log('Deleting ' + destinationPath); - FileSystem.deleteFile(destinationPath); - } - } - } - - /** - * Gets the path to the tarball - * Example: "C:\MyRepo\common\temp\projects\my-project-2.tgz" - */ - private _getTarballFilePath(project: RushConfigurationProject): string { - return path.join( - this._rushConfiguration.commonTempFolder, - RushConstants.rushTempProjectsFolderName, - `${project.unscopedTempProjectName}.tgz`); - } - - /** - * This is a workaround for a bug introduced in NPM 5 (and still unfixed as of NPM 5.5.1): - * https://github.com/npm/npm/issues/19006 - * - * The regression is that "npm install" sets the package.json "version" field for the - * @rush-temp projects to a value like "file:projects/example.tgz", when it should be "0.0.0". - * This causes "rush link" to fail later, when read-package-tree tries to parse the bad version. - * The error looks like this: - * - * ERROR: Failed to parse package.json for foo: Invalid version: "file:projects/example.tgz" - * - * Our workaround is to rewrite the package.json files for each of the @rush-temp projects - * in the node_modules folder, after "npm install" completes. - */ - private _fixupNpm5Regression(): void { - const pathToDeleteWithoutStar: string = path.join(this._rushConfiguration.commonTempFolder, - 'node_modules', RushConstants.rushTempNpmScope); - // Glob can't handle Windows paths - const normalizedPathToDeleteWithoutStar: string = Text.replaceAll(pathToDeleteWithoutStar, '\\', '/'); - - let anyChanges: boolean = false; - - // Example: "C:/MyRepo/common/temp/node_modules/@rush-temp/*/package.json" - for (const packageJsonPath of glob.sync(globEscape(normalizedPathToDeleteWithoutStar) + '/*/package.json')) { - // Example: "C:/MyRepo/common/temp/node_modules/@rush-temp/example/package.json" - const packageJsonObject: IRushTempPackageJson = JsonFile.load(packageJsonPath); - - // The temp projects always use "0.0.0" as their version - packageJsonObject.version = '0.0.0'; - - if (JsonFile.save(packageJsonObject, packageJsonPath, { onlyIfChanged: true })) { - anyChanges = true; - } - } - - if (anyChanges) { - console.log(os.EOL + colors.yellow(Utilities.wrapWords(`Applied workaround for NPM 5 bug`)) + os.EOL); - } - } - - /** - * Checks for temp projects that exist in the shrinkwrap file, but don't exist - * in rush.json. This might occur, e.g. if a project was recently deleted or renamed. - * - * @returns true if orphans were found, or false if everything is okay - */ - private _findOrphanedTempProjects(shrinkwrapFile: BaseShrinkwrapFile): boolean { - - // We can recognize temp projects because they are under the "@rush-temp" NPM scope. - for (const tempProjectName of shrinkwrapFile.getTempProjectNames()) { - if (!this._rushConfiguration.findProjectByTempName(tempProjectName)) { - console.log(os.EOL + colors.yellow(Utilities.wrapWords( - `Your ${this._shrinkwrapFilePhrase} references a project "${tempProjectName}" which no longer exists.`)) - + os.EOL); - return true; // found one - } - } - - return false; // none found - } - - /** - * Checks for temp projects that exist in the shrinkwrap file, but don't exist - * in rush.json. This might occur, e.g. if a project was recently deleted or renamed. - * - * @returns true if orphans were found, or false if everything is okay - */ - private _findOrphanedWorkspaceProjects(shrinkwrapFile: BaseShrinkwrapFile): boolean { - // // We can recognize temp projects because they are under the "@rush-temp" NPM scope. - // for (const tempProjectName of shrinkwrapFile.getTempProjectNames()) { - // if (!this._rushConfiguration.getProjectByName(tempProjectName)) { - // console.log(os.EOL + colors.yellow(Utilities.wrapWords( - // `Your ${this._shrinkwrapFilePhrase} references a project "${tempProjectName}" which no longer exists.`)) - // + os.EOL); - // return true; // found one - // } - // } - - return false; // none found - } - - private get _shrinkwrapFilePhrase(): string { - return this._rushConfiguration.shrinkwrapFilePhrase; - } -} diff --git a/apps/rush-lib/src/logic/InstallManagerFactory.ts b/apps/rush-lib/src/logic/InstallManagerFactory.ts new file mode 100644 index 00000000000..d5cde17258d --- /dev/null +++ b/apps/rush-lib/src/logic/InstallManagerFactory.ts @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as colors from 'colors'; +import * as semver from 'semver'; + +import { AlreadyReportedError } from '../utilities/AlreadyReportedError'; +import { BaseInstallManager, IInstallManagerOptions } from './base/BaseInstallManager'; +import { PurgeManager } from './PurgeManager'; +import { RushConfiguration } from '../api/RushConfiguration' +import { RushGlobalFolder } from '../api/RushGlobalFolder'; +import { RushInstallManager } from './RushInstallManager'; +import { WorkspaceInstallManager } from './WorkspaceInstallManager'; + +export class InstallManagerFactory { + + public static getInstallManager( + rushConfiguration: RushConfiguration, + rushGlobalFolder: RushGlobalFolder, + purgeManager: PurgeManager, + options: IInstallManagerOptions + ): BaseInstallManager { + if ( + rushConfiguration.packageManager === 'pnpm' && + rushConfiguration.pnpmOptions && + rushConfiguration.pnpmOptions.useWorkspaces + ) { + if (!semver.satisfies(rushConfiguration.packageManagerToolVersion, '>=4.14.3')) { + console.log(); + console.log(colors.red( + 'Workspaces are only supported in Rush for PNPM >=4.14.3. Upgrade PNPM to use the workspaces feature.' + )); + throw new AlreadyReportedError(); + } + + return new WorkspaceInstallManager(rushConfiguration, rushGlobalFolder, purgeManager, options); + } + + return new RushInstallManager(rushConfiguration, rushGlobalFolder, purgeManager, options); + } +} diff --git a/apps/rush-lib/src/logic/PackageJsonUpdater.ts b/apps/rush-lib/src/logic/PackageJsonUpdater.ts index 61a8dba3ac3..d42e6ab7346 100644 --- a/apps/rush-lib/src/logic/PackageJsonUpdater.ts +++ b/apps/rush-lib/src/logic/PackageJsonUpdater.ts @@ -5,7 +5,8 @@ import * as colors from 'colors'; import * as semver from 'semver'; import { RushConfiguration } from '../api/RushConfiguration'; -import { InstallManager, IInstallManagerOptions } from './InstallManager'; +import { BaseInstallManager, IInstallManagerOptions } from './base/BaseInstallManager'; +import { InstallManagerFactory } from './InstallManagerFactory'; import { VersionMismatchFinder } from './versionMismatch/VersionMismatchFinder'; import { PurgeManager } from './PurgeManager'; import { Utilities } from '../utilities/Utilities'; @@ -125,7 +126,7 @@ export class PackageJsonUpdater { variant } = options; - const implicitlyPinned: Map = InstallManager.collectImplicitlyPreferredVersions( + const implicitlyPinned: Map = BaseInstallManager.collectImplicitlyPreferredVersions( this._rushConfiguration, { variant @@ -145,7 +146,7 @@ export class PackageJsonUpdater { variant: variant, maxInstallAttempts: RushConstants.defaultMaxInstallAttempts }; - const installManager: InstallManager = new InstallManager( + const installManager: BaseInstallManager = InstallManagerFactory.getInstallManager( this._rushConfiguration, this._rushGlobalFolder, purgeManager, @@ -280,7 +281,7 @@ export class PackageJsonUpdater { */ private async _getNormalizedVersionSpec( projects: RushConfigurationProject[], - installManager: InstallManager, + installManager: BaseInstallManager, packageName: string, initialSpec: string | undefined, implicitlyPinnedVersion: string | undefined, diff --git a/apps/rush-lib/src/logic/RushInstallManager.ts b/apps/rush-lib/src/logic/RushInstallManager.ts new file mode 100644 index 00000000000..1567329d8b1 --- /dev/null +++ b/apps/rush-lib/src/logic/RushInstallManager.ts @@ -0,0 +1,581 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/* eslint max-lines: off */ + +import * as glob from 'glob'; +import * as colors from 'colors'; +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import * as tar from 'tar'; +import * as globEscape from 'glob-escape'; +import { + JsonFile, + Text, + IPackageJson, + FileSystem, + FileConstants, + Sort, + PosixModeBits, +} from '@rushstack/node-core-library'; + +import { BaseInstallManager, IInstallManagerOptions } from '../logic/base/BaseInstallManager'; +import { BaseShrinkwrapFile } from '../logic/base/BaseShrinkwrapFile'; +import { IRushTempPackageJson } from '../logic/base/BasePackage'; +import { PurgeManager } from './PurgeManager'; +import { RushConfiguration } from '../api/RushConfiguration'; +import { RushConfigurationProject } from '../api/RushConfigurationProject'; +import { RushConstants } from '../logic/RushConstants'; +import { Stopwatch } from '../utilities/Stopwatch'; +import { Utilities } from '../utilities/Utilities'; +import { PackageJsonEditor, DependencyType } from '../api/PackageJsonEditor'; +import { RushGlobalFolder } from '../api/RushGlobalFolder';; +import { DependencySpecifier } from './DependencySpecifier'; + +// The PosixModeBits are intended to be used with bitwise operations. +/* eslint-disable no-bitwise */ + +/** + * The "noMtime" flag is new in tar@4.4.1 and not available yet for \@types/tar. + * As a temporary workaround, augment the type. + */ +declare module 'tar' { + // eslint-disable-next-line @typescript-eslint/interface-name-prefix + export interface CreateOptions { + /** + * "Set to true to omit writing mtime values for entries. Note that this prevents using other + * mtime-based features like tar.update or the keepNewer option with the resulting tar archive." + */ + noMtime?: boolean; + } +} + +/** + * This class implements common logic between "rush install" and "rush update". + */ +export class RushInstallManager extends BaseInstallManager { + + public constructor( + rushConfiguration: RushConfiguration, + rushGlobalFolder: RushGlobalFolder, + purgeManager: PurgeManager, + options: IInstallManagerOptions + ) { + super(rushConfiguration, rushGlobalFolder, purgeManager, options); + } + + /** + * Regenerates the common/package.json and all temp_modules projects. + * If shrinkwrapFile is provided, this function also validates whether it contains + * everything we need to install and returns true if so; in all other cases, + * the return value is false. + */ + protected async prepareAndCheckShrinkwrap( + shrinkwrapFile: BaseShrinkwrapFile | undefined + ): Promise<{ shrinkwrapIsUpToDate: boolean, shrinkwrapWarnings: string[] }> { + const stopwatch: Stopwatch = Stopwatch.start(); + + // Example: "C:\MyRepo\common\temp\projects" + const tempProjectsFolder: string = path.join(this.rushConfiguration.commonTempFolder, + RushConstants.rushTempProjectsFolderName); + + console.log(os.EOL + colors.bold('Updating temp projects in ' + tempProjectsFolder)); + + Utilities.createFolderWithRetry(tempProjectsFolder); + + const shrinkwrapWarnings: string[] = []; + + // We will start with the assumption that it's valid, and then set it to false if + // any of the checks fail + let shrinkwrapIsUpToDate: boolean = true; + + if (!shrinkwrapFile) { + shrinkwrapIsUpToDate = false; + } + + // dependency name --> version specifier + const allExplicitPreferredVersions: Map = + this.rushConfiguration.getCommonVersions(this.options.variant).getAllPreferredVersions(); + + if (shrinkwrapFile) { + // Check any (explicitly) preferred dependencies first + allExplicitPreferredVersions.forEach((version: string, dependency: string) => { + const dependencySpecifier: DependencySpecifier = new DependencySpecifier(dependency, version); + + if (!shrinkwrapFile.hasCompatibleTopLevelDependency(dependencySpecifier)) { + shrinkwrapWarnings.push(`Missing dependency "${dependency}" (${version}) required by the preferred versions from ` + + RushConstants.commonVersionsFilename); + shrinkwrapIsUpToDate = false; + } + }); + + if (this._findOrphanedTempProjects(shrinkwrapFile)) { + // If there are any orphaned projects, then "npm install" would fail because the shrinkwrap + // contains references such as "resolved": "file:projects\\project1" that refer to nonexistent + // file paths. + shrinkwrapIsUpToDate = false; + } + } + + const commonPackageJson: IPackageJson = { + dependencies: {}, + description: 'Temporary file generated by the Rush tool', + name: 'rush-common', + private: true, + version: '0.0.0' + }; + + // dependency name --> version specifier + const allPreferredVersions: Map = + BaseInstallManager.collectPreferredVersions(this.rushConfiguration, this.options.variant); + + // Add any preferred versions to the top of the commonPackageJson + // do this in alphabetical order for simpler debugging + for (const dependency of Array.from(allPreferredVersions.keys()).sort()) { + commonPackageJson.dependencies![dependency] = allPreferredVersions.get(dependency)!; + } + + // To make the common/package.json file more readable, sort alphabetically + // according to rushProject.tempProjectName instead of packageName. + const sortedRushProjects: RushConfigurationProject[] = this.rushConfiguration.projects.slice(0); + Sort.sortBy(sortedRushProjects, x => x.tempProjectName); + + for (const rushProject of sortedRushProjects) { + const packageJson: PackageJsonEditor = rushProject.packageJsonEditor; + + // Example: "C:\MyRepo\common\temp\projects\my-project-2.tgz" + const tarballFile: string = this._getTarballFilePath(rushProject); + + // Example: "my-project-2" + const unscopedTempProjectName: string = rushProject.unscopedTempProjectName; + + // Example: dependencies["@rush-temp/my-project-2"] = "file:./projects/my-project-2.tgz" + commonPackageJson.dependencies![rushProject.tempProjectName] + = `file:./${RushConstants.rushTempProjectsFolderName}/${rushProject.unscopedTempProjectName}.tgz`; + + const tempPackageJson: IRushTempPackageJson = { + name: rushProject.tempProjectName, + version: '0.0.0', + private: true, + dependencies: {} + }; + + // Collect pairs of (packageName, packageVersion) to be added as dependencies of the @rush-temp package.json + const tempDependencies: Map = new Map(); + + // These can be regular, optional, or peer dependencies (but NOT dev dependencies). + // (A given packageName will never appear more than once in this list.) + for (const dependency of packageJson.dependencyList) { + + // If there are any optional dependencies, copy directly into the optionalDependencies field. + if (dependency.dependencyType === DependencyType.Optional) { + if (!tempPackageJson.optionalDependencies) { + tempPackageJson.optionalDependencies = {}; + } + tempPackageJson.optionalDependencies[dependency.name] = dependency.version; + } else { + tempDependencies.set(dependency.name, dependency.version); + } + } + + for (const dependency of packageJson.devDependencyList) { + // If there are devDependencies, we need to merge them with the regular dependencies. If the same + // library appears in both places, then the dev dependency wins (because presumably it's saying what you + // want right now for development, not the range that you support for consumers). + tempDependencies.set(dependency.name, dependency.version); + } + Sort.sortMapKeys(tempDependencies); + + for (const [packageName, packageVersion] of tempDependencies.entries()) { + const dependencySpecifier: DependencySpecifier = new DependencySpecifier(packageName, packageVersion); + + // Is there a locally built Rush project that could satisfy this dependency? + // If so, then we will symlink to the project folder rather than to common/temp/node_modules. + // In this case, we don't want "npm install" to process this package, but we do need + // to record this decision for "rush link" later, so we add it to a special 'rushDependencies' field. + const localProject: RushConfigurationProject | undefined = + this.rushConfiguration.getProjectByName(packageName); + + if (localProject) { + // Don't locally link if it's listed in the cyclicDependencyProjects + if (!rushProject.cyclicDependencyProjects.has(packageName)) { + + // Also, don't locally link if the SemVer doesn't match + const localProjectVersion: string = localProject.packageJsonEditor.version; + if (semver.satisfies(localProjectVersion, packageVersion)) { + + // We will locally link this package, so instead add it to our special "rushDependencies" + // field in the package.json file. + if (!tempPackageJson.rushDependencies) { + tempPackageJson.rushDependencies = {}; + } + tempPackageJson.rushDependencies[packageName] = packageVersion; + continue; + } + } + } + + // We will NOT locally link this package; add it as a regular dependency. + tempPackageJson.dependencies![packageName] = packageVersion; + + if (shrinkwrapFile) { + if (!shrinkwrapFile.tryEnsureCompatibleDependency(dependencySpecifier, rushProject.tempProjectName)) { + shrinkwrapWarnings.push(`Missing dependency "${packageName}" (${packageVersion}) required by` + + ` "${rushProject.packageName}"`); + shrinkwrapIsUpToDate = false; + } + } + } + + // NPM expects the root of the tarball to have a directory called 'package' + const npmPackageFolder: string = 'package'; + + // Example: "C:\MyRepo\common\temp\projects\my-project-2" + const tempProjectFolder: string = path.join( + this.rushConfiguration.commonTempFolder, + RushConstants.rushTempProjectsFolderName, + unscopedTempProjectName); + + // Example: "C:\MyRepo\common\temp\projects\my-project-2\package.json" + const tempPackageJsonFilename: string = path.join(tempProjectFolder, FileConstants.PackageJson); + + // we only want to overwrite the package if the existing tarball's package.json is different from tempPackageJson + let shouldOverwrite: boolean = true; + try { + // if the tarball and the temp file still exist, then compare the contents + if (FileSystem.exists(tarballFile) && FileSystem.exists(tempPackageJsonFilename)) { + + // compare the extracted package.json with the one we are about to write + const oldBuffer: Buffer = FileSystem.readFileToBuffer(tempPackageJsonFilename); + const newBuffer: Buffer = Buffer.from(JsonFile.stringify(tempPackageJson)); + + if (Buffer.compare(oldBuffer, newBuffer) === 0) { + shouldOverwrite = false; + } + } + } catch (error) { + // ignore the error, we will go ahead and create a new tarball + } + + if (shouldOverwrite) { + try { + // ensure the folder we are about to zip exists + Utilities.createFolderWithRetry(tempProjectFolder); + + // remove the old tarball & old temp package json, this is for any cases where new tarball creation + // fails, and the shouldOverwrite logic is messed up because the my-project-2\package.json + // exists and is updated, but the tarball is not accurate + FileSystem.deleteFile(tarballFile); + FileSystem.deleteFile(tempPackageJsonFilename); + + // write the expected package.json file into the zip staging folder + JsonFile.save(tempPackageJson, tempPackageJsonFilename); + + const tarOptions: tar.CreateOptions = ({ + gzip: true, + file: tarballFile, + cwd: tempProjectFolder, + portable: true, + noMtime: true, + noPax: true, + sync: true, + prefix: npmPackageFolder, + filter: (path: string, stat: tar.FileStat): boolean => { + if (!this.rushConfiguration.experimentsConfiguration.configuration + .noChmodFieldInTarHeaderNormalization) { + stat.mode = (stat.mode & ~0x1FF) | PosixModeBits.AllRead | PosixModeBits.UserWrite + | PosixModeBits.AllExecute; + } + return true; + } + } as tar.CreateOptions); + + // create the new tarball + tar.create(tarOptions, [FileConstants.PackageJson]); + + console.log(`Updating ${tarballFile}`); + } catch (error) { + console.log(colors.yellow(error)); + // delete everything in case of any error + FileSystem.deleteFile(tarballFile); + FileSystem.deleteFile(tempPackageJsonFilename); + } + } + } + + // Example: "C:\MyRepo\common\temp\package.json" + const commonPackageJsonFilename: string = path.join(this.rushConfiguration.commonTempFolder, + FileConstants.PackageJson); + + // Don't update the file timestamp unless the content has changed, since "rush install" + // will consider this timestamp + JsonFile.save(commonPackageJson, commonPackageJsonFilename, { onlyIfChanged: true }); + + stopwatch.stop(); + console.log(`Finished creating temporary modules (${stopwatch.toString()})`); + + return { shrinkwrapIsUpToDate, shrinkwrapWarnings }; + } + + protected canSkipInstall(lastModifiedDate: Date): boolean { + console.log(os.EOL + colors.bold('Checking node_modules in ' + this.rushConfiguration.commonTempFolder) + + os.EOL); + + // Based on timestamps, can we skip this install entirely? + const potentiallyChangedFiles: string[] = []; + + // Consider the timestamp on the node_modules folder; if someone tampered with it + // or deleted it entirely, then we can't skip this install + potentiallyChangedFiles.push(path.join(this.rushConfiguration.commonTempFolder, 'node_modules')); + + // Additionally, if they pulled an updated npm-shrinkwrap.json file from Git, + // then we can't skip this install + potentiallyChangedFiles.push(this.rushConfiguration.getCommittedShrinkwrapFilename(this.options.variant)); + + // Add common-versions.json file in potentially changed file list. + potentiallyChangedFiles.push(this.rushConfiguration.getCommonVersionsFilePath(this.options.variant)); + + if (this.rushConfiguration.packageManager === 'pnpm') { + // If the repo is using pnpmfile.js, consider that also + const pnpmFileFilename: string = this.rushConfiguration.getPnpmfilePath(this.options.variant); + + if (FileSystem.exists(pnpmFileFilename)) { + potentiallyChangedFiles.push(pnpmFileFilename); + } + } + + // Also consider timestamps for all the temp tarballs. (createTempModulesAndCheckShrinkwrap() will + // carefully preserve these timestamps unless something has changed.) + // Example: "C:\MyRepo\common\temp\projects\my-project-2.tgz" + potentiallyChangedFiles.push(...this.rushConfiguration.projects.map(x => this._getTarballFilePath(x))); + + // NOTE: If any of the potentiallyChangedFiles does not exist, then isFileTimestampCurrent() + // returns false. + return Utilities.isFileTimestampCurrent(lastModifiedDate, potentiallyChangedFiles); + } + + /** + * Runs "npm install" in the common folder. + */ + protected async install(cleanInstall: boolean) : Promise { + if (cleanInstall) { + // NOTE: The PNPM store is supposed to be transactionally safe, so we don't delete it automatically. + // The user must request that via the command line. + if (this.rushConfiguration.packageManager === 'npm') { + console.log(`Deleting the "npm-cache" folder`); + // This is faster and more thorough than "npm cache clean" + this.installRecycler.moveFolder(this.rushConfiguration.npmCacheFolder); + + console.log(`Deleting the "npm-tmp" folder`); + this.installRecycler.moveFolder(this.rushConfiguration.npmTmpFolder); + } + } + + const commonNodeModulesFolder: string = path.join( + this.rushConfiguration.commonTempFolder, + 'node_modules' + ); + + const packageManagerEnv: NodeJS.ProcessEnv = this.getPackageManagerEnvironment(); + + // Is there an existing "node_modules" folder to consider? + if (FileSystem.exists(commonNodeModulesFolder)) { + // Should we delete the entire "node_modules" folder? + if (cleanInstall) { + // YES: Delete "node_modules" + + // Explain to the user why we are hosing their node_modules folder + console.log('Deleting files from ' + commonNodeModulesFolder); + + this.installRecycler.moveFolder(commonNodeModulesFolder); + + Utilities.createFolderWithRetry(commonNodeModulesFolder); + } else { + // NO: Prepare to do an incremental install in the "node_modules" folder + + // note: it is not necessary to run "prune" with pnpm + if (this.rushConfiguration.packageManager === 'npm') { + console.log(`Running "${this.rushConfiguration.packageManager} prune"` + + ` in ${this.rushConfiguration.commonTempFolder}`); + const args: string[] = ['prune']; + this.pushConfigurationArgs(args, this.options); + + Utilities.executeCommandWithRetry( + this.options.maxInstallAttempts, + this.rushConfiguration.packageManagerToolFilename, + args, + this.rushConfiguration.commonTempFolder, + packageManagerEnv + ); + + // Delete the (installed image of) the temp projects, since "npm install" does not + // detect changes for "file:./" references. + // We recognize the temp projects by their names, which always start with "rush-". + + // Example: "C:\MyRepo\common\temp\node_modules\@rush-temp" + const pathToDeleteWithoutStar: string = path.join(commonNodeModulesFolder, + RushConstants.rushTempNpmScope); + console.log(`Deleting ${pathToDeleteWithoutStar}\\*`); + // Glob can't handle Windows paths + const normalizedpathToDeleteWithoutStar: string = Text.replaceAll(pathToDeleteWithoutStar, '\\', '/'); + + // Example: "C:/MyRepo/common/temp/node_modules/@rush-temp/*" + for (const tempModulePath of glob.sync(globEscape(normalizedpathToDeleteWithoutStar) + '/*')) { + // We could potentially use AsyncRecycler here, but in practice these folders tend + // to be very small + Utilities.dangerouslyDeletePath(tempModulePath); + } + } + } + } + + if (this.rushConfiguration.packageManager === 'yarn') { + // Yarn does not correctly detect changes to a tarball, so we need to forcibly clear its cache + const yarnRushTempCacheFolder: string = path.join( + this.rushConfiguration.yarnCacheFolder, 'v2', 'npm-@rush-temp' + ); + if (FileSystem.exists(yarnRushTempCacheFolder)) { + console.log('Deleting ' + yarnRushTempCacheFolder); + Utilities.dangerouslyDeletePath(yarnRushTempCacheFolder); + } + } + + // Run "npm install" in the common folder + const installArgs: string[] = ['install']; + this.pushConfigurationArgs(installArgs, this.options); + + console.log(os.EOL + colors.bold(`Running "${this.rushConfiguration.packageManager} install" in` + + ` ${this.rushConfiguration.commonTempFolder}`) + os.EOL); + + // If any diagnostic options were specified, then show the full command-line + if (this.options.debug || this.options.collectLogFile || this.options.networkConcurrency) { + console.log(os.EOL + colors.green('Invoking package manager: ') + + FileSystem.getRealPath(this.rushConfiguration.packageManagerToolFilename) + ' ' + + installArgs.join(' ') + os.EOL); + } + + try { + Utilities.executeCommandWithRetry( + this.options.maxInstallAttempts, + this.rushConfiguration.packageManagerToolFilename, + installArgs, + this.rushConfiguration.commonTempFolder, + packageManagerEnv, + false, + () => { + if (this.rushConfiguration.packageManager === 'pnpm') { + console.log(colors.yellow(`Deleting the "node_modules" folder`)); + this.installRecycler.moveFolder(commonNodeModulesFolder); + + // Leave the pnpm-store as is for the retry. This ensures that packages that have already + // been downloaded need not be downloaded again, thereby potentially increasing the chances + // of a subsequent successful install. + + Utilities.createFolderWithRetry(commonNodeModulesFolder); + } + }); + } catch (error) { + // All the install attempts failed. + if ( + this.rushConfiguration.packageManager === 'pnpm' && + this.rushConfiguration.pnpmOptions.pnpmStore === 'local' + ) { + // If the installation has failed even after the retries, then pnpm store may + // have got into a corrupted, irrecoverable state. Delete the store so that a + // future install can create the store afresh. + console.log(colors.yellow(`Deleting the "pnpm-store" folder`)); + this.installRecycler.moveFolder(this.rushConfiguration.pnpmOptions.pnpmStorePath); + } + + throw error; + } + + if (this.rushConfiguration.packageManager === 'npm') { + console.log(os.EOL + colors.bold('Running "npm shrinkwrap"...')); + const npmArgs: string[] = ['shrinkwrap']; + this.pushConfigurationArgs(npmArgs, this.options); + Utilities.executeCommand( + this.rushConfiguration.packageManagerToolFilename, + npmArgs, + this.rushConfiguration.commonTempFolder + ); + console.log('"npm shrinkwrap" completed' + os.EOL); + + this._fixupNpm5Regression(); + } + } + + /** + * Gets the path to the tarball + * Example: "C:\MyRepo\common\temp\projects\my-project-2.tgz" + */ + private _getTarballFilePath(project: RushConfigurationProject): string { + return path.join( + this.rushConfiguration.commonTempFolder, + RushConstants.rushTempProjectsFolderName, + `${project.unscopedTempProjectName}.tgz`); + } + + /** + * This is a workaround for a bug introduced in NPM 5 (and still unfixed as of NPM 5.5.1): + * https://github.com/npm/npm/issues/19006 + * + * The regression is that "npm install" sets the package.json "version" field for the + * @rush-temp projects to a value like "file:projects/example.tgz", when it should be "0.0.0". + * This causes "rush link" to fail later, when read-package-tree tries to parse the bad version. + * The error looks like this: + * + * ERROR: Failed to parse package.json for foo: Invalid version: "file:projects/example.tgz" + * + * Our workaround is to rewrite the package.json files for each of the @rush-temp projects + * in the node_modules folder, after "npm install" completes. + */ + private _fixupNpm5Regression(): void { + const pathToDeleteWithoutStar: string = path.join(this.rushConfiguration.commonTempFolder, + 'node_modules', RushConstants.rushTempNpmScope); + // Glob can't handle Windows paths + const normalizedPathToDeleteWithoutStar: string = Text.replaceAll(pathToDeleteWithoutStar, '\\', '/'); + + let anyChanges: boolean = false; + + // Example: "C:/MyRepo/common/temp/node_modules/@rush-temp/*/package.json" + for (const packageJsonPath of glob.sync(globEscape(normalizedPathToDeleteWithoutStar) + '/*/package.json')) { + // Example: "C:/MyRepo/common/temp/node_modules/@rush-temp/example/package.json" + const packageJsonObject: IRushTempPackageJson = JsonFile.load(packageJsonPath); + + // The temp projects always use "0.0.0" as their version + packageJsonObject.version = '0.0.0'; + + if (JsonFile.save(packageJsonObject, packageJsonPath, { onlyIfChanged: true })) { + anyChanges = true; + } + } + + if (anyChanges) { + console.log(os.EOL + colors.yellow(Utilities.wrapWords(`Applied workaround for NPM 5 bug`)) + os.EOL); + } + } + + /** + * Checks for temp projects that exist in the shrinkwrap file, but don't exist + * in rush.json. This might occur, e.g. if a project was recently deleted or renamed. + * + * @returns true if orphans were found, or false if everything is okay + */ + private _findOrphanedTempProjects(shrinkwrapFile: BaseShrinkwrapFile): boolean { + + // We can recognize temp projects because they are under the "@rush-temp" NPM scope. + for (const tempProjectName of shrinkwrapFile.getTempProjectNames()) { + if (!this.rushConfiguration.findProjectByTempName(tempProjectName)) { + console.log(os.EOL + colors.yellow(Utilities.wrapWords( + `Your ${this.rushConfiguration.shrinkwrapFilePhrase} references a project "${tempProjectName}" ` + + 'which no longer exists.')) + + os.EOL); + return true; // found one + } + } + + return false; // none found + } +} diff --git a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts new file mode 100644 index 00000000000..bc13eec02d5 --- /dev/null +++ b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts @@ -0,0 +1,378 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/* eslint max-lines: off */ + +import * as colors from 'colors'; +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import { + JsonFile, + IPackageJson, + FileSystem, + FileConstants +} from '@rushstack/node-core-library'; + +import { AlreadyReportedError } from '../utilities/AlreadyReportedError'; +import { BaseInstallManager, IInstallManagerOptions } from './base/BaseInstallManager'; +import { BaseShrinkwrapFile } from '../logic/base/BaseShrinkwrapFile'; +import { DependencySpecifier } from './DependencySpecifier'; +import { PackageJsonEditor } from '../api/PackageJsonEditor'; +import { PnpmWorkspaceFile } from './pnpm/PnpmWorkspaceFile'; +import { PurgeManager } from './PurgeManager'; +import { RushConfiguration } from '../api/RushConfiguration'; +import { RushConfigurationProject } from '../api/RushConfigurationProject'; +import { RushConstants } from '../logic/RushConstants';; +import { RushGlobalFolder } from '../api/RushGlobalFolder'; +import { Stopwatch } from '../utilities/Stopwatch'; +import { Utilities } from '../utilities/Utilities'; + +/** + * This class implements common logic between "rush install" and "rush update". + */ +export class WorkspaceInstallManager extends BaseInstallManager { + + public constructor( + rushConfiguration: RushConfiguration, + rushGlobalFolder: RushGlobalFolder, + purgeManager: PurgeManager, + options: IInstallManagerOptions + ) { + super(rushConfiguration, rushGlobalFolder, purgeManager, options); + } + + public async doInstall(): Promise { + // Workspaces do not support the noLink option, so throw if this is passed + if (this.options.noLink) { + console.log(); + console.log(colors.red( + 'The "--no-link" option was provided but is not supported when using workspaces. Run the command again ' + + 'without specifying this argument.' + )); + throw new AlreadyReportedError(); + } + + await super.doInstall(); + } + + protected async prepareAndCheckShrinkwrap( + shrinkwrapFile: BaseShrinkwrapFile | undefined + ): Promise<{ shrinkwrapIsUpToDate: boolean, shrinkwrapWarnings: string[] }> { + + const stopwatch: Stopwatch = Stopwatch.start(); + + console.log(os.EOL + colors.bold('Updating workspace files in ' + this.rushConfiguration.commonTempFolder)); + + const shrinkwrapWarnings: string[] = []; + + // We will start with the assumption that it's valid, and then set it to false if + // any of the checks fail + let shrinkwrapIsUpToDate: boolean = true; + + if (!shrinkwrapFile) { + shrinkwrapIsUpToDate = false; + } + + // dependency name --> version specifier + const allExplicitPreferredVersions: Map = + this.rushConfiguration.getCommonVersions(this.options.variant).getAllPreferredVersions(); + + if (shrinkwrapFile) { + // Check any (explicitly) preferred dependencies first + allExplicitPreferredVersions.forEach((version: string, dependency: string) => { + const dependencySpecifier: DependencySpecifier = new DependencySpecifier(dependency, version); + + if (!shrinkwrapFile.hasCompatibleTopLevelDependency(dependencySpecifier)) { + shrinkwrapWarnings.push(`Missing dependency "${dependency}" (${version}) required by the preferred versions from ` + + RushConstants.commonVersionsFilename); + shrinkwrapIsUpToDate = false; + } + }); + + if (this._findOrphanedWorkspaceProjects(shrinkwrapFile)) { + // If there are any orphaned projects, then "npm install" would fail because the shrinkwrap + // contains references such as "resolved": "file:projects\\project1" that refer to nonexistent + // file paths. + shrinkwrapIsUpToDate = false; + } + } + + const commonPackageJson: IPackageJson = { + dependencies: {}, + description: 'Temporary file generated by the Rush tool', + name: 'rush-common', + private: true, + version: '0.0.0' + }; + + // dependency name --> version specifier + const allPreferredVersions: Map = + BaseInstallManager.collectPreferredVersions(this.rushConfiguration, this.options.variant); + + // Add any preferred versions to the top of the commonPackageJson + // do this in alphabetical order for simpler debugging + for (const dependency of Array.from(allPreferredVersions.keys()).sort()) { + commonPackageJson.dependencies![dependency] = allPreferredVersions.get(dependency)!; + } + + // To generate the workspace file, we will add each project to the file as we loop through and validate + const workspaceFile: PnpmWorkspaceFile = new PnpmWorkspaceFile( + path.join(this.rushConfiguration.commonTempFolder, 'pnpm-workspace.yaml') + ); + + // Loop through the projects and add them to the workspace file. While we're at it, also validate that + // referenced workspace projects are valid, and check if the shrinkwrap file is already up-to-date. + for (const rushProject of this.rushConfiguration.projects) { + workspaceFile.addPackage(rushProject.projectFolder); + const packageJson: PackageJsonEditor = rushProject.packageJsonEditor; + + for (const { name, version, dependencyType } of [...packageJson.dependencyList, ...packageJson.devDependencyList]) { + const dependencySpecifier: DependencySpecifier = new DependencySpecifier(name, version); + + // Is there a locally built Rush project that could satisfy this dependency? + const localProject: RushConfigurationProject | undefined = + this.rushConfiguration.getProjectByName(name); + + // Validate that local projects are referenced with workspace notation. If not, and it is not a + // cyclic dependency, then it needs to be updated to specify `workspace:*` explicitly. Currently only + // supporting versions and version ranges for specifying a local project. + if ( + (dependencySpecifier.specifierType === 'version' || dependencySpecifier.specifierType === 'range') && + localProject && + !rushProject.cyclicDependencyProjects.has(name) + ) { + // Make sure that this version is intended to target a local package. If not, then we will fail since it + // is not explicitly specified as a cyclic dependency. + if (!semver.satisfies(localProject.packageJsonEditor.version, dependencySpecifier.versionSpecifier)) { + console.log(); + console.log(colors.red( + `"${rushProject.packageName}" depends on package "${name}" (${version}) which exists within the workspace ` + + 'but cannot be fulfilled with the specified version range. Either specify a valid version range, or add ' + + 'the package as a cyclic dependency.' + )); + throw new AlreadyReportedError(); + } + + if (!this.options.allowShrinkwrapUpdates) { + console.log(); + console.log(colors.red( + `"${rushProject.packageName}" depends on package "${name}" (${version}) which exists within the workspace. ` + + 'Run "rush update" to update workspace references for this package.' + )); + throw new AlreadyReportedError(); + } + + // We will update to `workspace:*` by default to ensure we're always using the workspace package. + packageJson.addOrUpdateDependency(name, 'workspace:*', dependencyType); + shrinkwrapIsUpToDate = false; + continue; + } else if (dependencySpecifier.specifierType === 'workspace') { + // Already specified as a local project, let's just validate that the specifier is valid. + if (!semver.satisfies(localProject!.packageJsonEditor.version, dependencySpecifier.versionSpecifier)) { + console.log(); + console.log(colors.red( + `"${rushProject.packageName}" depends on package "${name}" (${version}) which exists within the workspace ` + + 'but cannot be fulfilled with the specified version range. Specify a valid workspace version range.' + )); + throw new AlreadyReportedError(); + } + continue; + } + + // It is not a local dependency, validate that it is compatible + if ( + shrinkwrapFile && + !shrinkwrapFile.tryEnsureCompatibleWorkspaceDependency( + dependencySpecifier, + rushProject.packageName, + this.rushConfiguration + ) + ) { + shrinkwrapWarnings.push(`Missing dependency "${name}" (${version}) required by "${rushProject.packageName}"`); + shrinkwrapIsUpToDate = false; + } + } + + // Save the package.json if we modified the version references + if (rushProject.packageJsonEditor.saveIfModified()) { + console.log(colors.yellow( + `"${rushProject.packageName}" depends on one or more workspace packages which did not use "workspace:" ` + + 'notation. The package.json has been modified and must be committed to source control.' + )); + } + } + + // Example: "C:\MyRepo\common\temp\package.json" + const commonPackageJsonFilename: string = path.join(this.rushConfiguration.commonTempFolder, + FileConstants.PackageJson); + + // Don't update the file timestamp unless the content has changed, since "rush install" + // will consider this timestamp + JsonFile.save(commonPackageJson, commonPackageJsonFilename, { onlyIfChanged: true }); + workspaceFile.save(workspaceFile.workspaceFilename, { onlyIfChanged: true }); + + stopwatch.stop(); + console.log(`Finished creating workspace (${stopwatch.toString()})`); + + return { shrinkwrapIsUpToDate, shrinkwrapWarnings }; + } + + protected canSkipInstall(lastModifiedDate: Date): boolean { + console.log(os.EOL + colors.bold('Checking workspace node_modules') + os.EOL); + + // Based on timestamps, can we skip this install entirely? + const potentiallyChangedFiles: string[] = []; + + // Consider the timestamp on the node_modules folder; if someone tampered with it + // or deleted it entirely, then we can't skip this install + potentiallyChangedFiles.push(path.join(this.rushConfiguration.commonTempFolder, RushConstants.nodeModulesFolderName)); + + // Additionally, if they pulled an updated npm-shrinkwrap.json file from Git, + // then we can't skip this install + potentiallyChangedFiles.push(this.rushConfiguration.getCommittedShrinkwrapFilename(this.options.variant)); + + // Add common-versions.json file in potentially changed file list. + potentiallyChangedFiles.push(this.rushConfiguration.getCommonVersionsFilePath(this.options.variant)); + + if (this.rushConfiguration.packageManager === 'pnpm') { + // If the repo is using pnpmfile.js, consider that also + const pnpmFileFilename: string = this.rushConfiguration.getPnpmfilePath(this.options.variant); + + if (FileSystem.exists(pnpmFileFilename)) { + potentiallyChangedFiles.push(pnpmFileFilename); + } + + // Add workspace file. This file is only modified when workspace packages change. + potentiallyChangedFiles.push(path.join(this.rushConfiguration.commonTempFolder, 'pnpm-workspace.yaml')); + } + + // Also consider timestamps for all the project node_modules folders. + // Example: "C:\MyRepo\projects\projectA\node_modules" + potentiallyChangedFiles.push(...this.rushConfiguration.projects.map(x => path.join( + x.projectFolder, RushConstants.nodeModulesFolderName))); + + // NOTE: If any of the potentiallyChangedFiles does not exist, then isFileTimestampCurrent() + // returns false. + return Utilities.isFileTimestampCurrent(lastModifiedDate, potentiallyChangedFiles); + } + + /** + * Runs "npm install" in the common folder. + */ + protected async install(cleanInstall: boolean): Promise { + // Example: "C:\MyRepo\common\temp\npm-local\node_modules\.bin\npm" + const packageManagerFilename: string = this.rushConfiguration.packageManagerToolFilename; + const packageManagerEnv: NodeJS.ProcessEnv = this.getPackageManagerEnvironment(); + + const projectNodeModulesFolders: string[] = [ + path.join(this.rushConfiguration.commonTempFolder, RushConstants.nodeModulesFolderName), + ...this.rushConfiguration.projects.map(project => { + return path.join(project.projectFolder, RushConstants.nodeModulesFolderName) + }), + ]; + + // Should we delete the "node_modules" folder? + if (cleanInstall) { + // Explain to the user why we are hosing their node_modules folder + console.log(`Deleting files from project "${ RushConstants.nodeModulesFolderName }" folders`); + for (const nodeModulesFolder of projectNodeModulesFolders) { + this.installRecycler.moveFolder(nodeModulesFolder); + } + } + + // Run "npm install" in the common folder + const installArgs: string[] = ['install']; + this.pushConfigurationArgs(installArgs, this.options); + + console.log(os.EOL + colors.bold(`Running "${this.rushConfiguration.packageManager} install" in` + + ` ${this.rushConfiguration.commonTempFolder}`) + os.EOL); + + // If any diagnostic options were specified, then show the full command-line + if (this.options.debug || this.options.collectLogFile || this.options.networkConcurrency) { + console.log(os.EOL + colors.green('Invoking package manager: ') + + FileSystem.getRealPath(packageManagerFilename) + ' ' + installArgs.join(' ') + os.EOL); + } + + try { + Utilities.executeCommandWithRetry( + this.options.maxInstallAttempts, + packageManagerFilename, + installArgs, + this.rushConfiguration.commonTempFolder, + packageManagerEnv, + false, + () => { + if (this.rushConfiguration.packageManager === 'pnpm') { + // Leave the pnpm-store as is for the retry. This ensures that packages that have already + // been downloaded need not be downloaded again, thereby potentially increasing the chances + // of a subsequent successful install. We will only remove the node_modules folders for + // local projects + console.log(colors.yellow(`Deleting files from project "${RushConstants.nodeModulesFolderName}" folders`)); + for (const nodeModulesFolder of projectNodeModulesFolders) { + this.installRecycler.moveFolder(nodeModulesFolder); + } + } + }); + + // Ensure that node_modules folders exist after install, since the timestamps on these folders are used + // to determine if the install can be skipped + for (const nodeModulesFolder of projectNodeModulesFolders) { + FileSystem.ensureFolder(nodeModulesFolder); + } + } catch (error) { + // All the install attempts failed. + + if ( + this.rushConfiguration.packageManager === 'pnpm' && + this.rushConfiguration.pnpmOptions.pnpmStore === 'local' + ) { + // If the installation has failed even after the retries, then pnpm store may + // have got into a corrupted, irrecoverable state. Delete the store so that a + // future install can create the store afresh. + console.log(colors.yellow(`Deleting the "pnpm-store" folder`)); + this.installRecycler.moveFolder(this.rushConfiguration.pnpmOptions.pnpmStorePath); + } + + throw error; + } + + console.log(''); + } + + /** + * Used when invoking the NPM tool. Appends the common configuration options + * to the command-line. + */ + protected pushConfigurationArgs(args: string[], options: IInstallManagerOptions): void { + super.pushConfigurationArgs(args, options); + + // Add workspace-specific args + if (this.rushConfiguration.packageManager === 'pnpm') { + args.push('--recursive'); + args.push('--link-workspace-packages', 'false'); + } + } + + /** + * Checks for projects that exist in the shrinkwrap file, but don't exist + * in rush.json. This might occur, e.g. if a project was recently deleted or renamed. + * + * @returns true if orphans were found, or false if everything is okay + */ + private _findOrphanedWorkspaceProjects(shrinkwrapFile: BaseShrinkwrapFile): boolean { + + for (const workspacePath of shrinkwrapFile.getWorkspacePaths()) { + const projectPath: string = path.resolve(this.rushConfiguration.commonTempFolder, workspacePath); + if (!this.rushConfiguration.tryGetProjectForPath(projectPath)) { + console.log(os.EOL + colors.yellow(Utilities.wrapWords( + `Your ${this.rushConfiguration.shrinkwrapFilePhrase} references a project at "${projectPath}" ` + + 'which no longer exists.')) + + os.EOL); + return true; // found one + } + } + + return false; // none found + } +} diff --git a/apps/rush-lib/src/logic/base/BaseInstallManager.ts b/apps/rush-lib/src/logic/base/BaseInstallManager.ts new file mode 100644 index 00000000000..a52e63a15fe --- /dev/null +++ b/apps/rush-lib/src/logic/base/BaseInstallManager.ts @@ -0,0 +1,919 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +/* eslint max-lines: off */ + +import * as colors from 'colors'; +import * as fetch from 'node-fetch'; +import * as fs from 'fs'; +import * as http from 'http'; +import HttpsProxyAgent = require('https-proxy-agent'); +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import { + FileSystem, + JsonFile, + JsonObject, + LockFile, + MapExtensions, + PosixModeBits, +} from '@rushstack/node-core-library'; + +import { AlreadyReportedError } from '../../utilities/AlreadyReportedError'; +import { ApprovedPackagesChecker } from '../ApprovedPackagesChecker'; +import { AsyncRecycler } from '../../utilities/AsyncRecycler'; +import { BaseLinkManager } from './BaseLinkManager'; +import { BaseShrinkwrapFile } from '../base/BaseShrinkwrapFile'; +import { CommonVersionsConfiguration } from '../../api/CommonVersionsConfiguration'; +import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; +import { Git } from '../Git'; +import { LastInstallFlag } from '../../api/LastInstallFlag'; +import { LinkManagerFactory } from '../LinkManagerFactory'; +import { PackageJsonDependency } from '../../api/PackageJsonEditor'; +import { PackageManagerName } from '../../api/packageManager/PackageManager'; +import { PnpmPackageManager } from '../../api/packageManager/PnpmPackageManager'; +import { PolicyValidator } from '../policy/PolicyValidator'; +import { PurgeManager } from '../PurgeManager'; +import { Rush } from '../../api/Rush'; +import { RushConfiguration, IConfigurationEnvironment, ICurrentVariantJson } from '../../api/RushConfiguration'; +import { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import { RushGlobalFolder } from '../../api/RushGlobalFolder'; +import { RushConstants } from '../RushConstants'; +import { ShrinkwrapFileFactory } from '../ShrinkwrapFileFactory'; +import { Utilities } from '../../utilities/Utilities'; +import { DependencySpecifier } from '../DependencySpecifier'; + +// The PosixModeBits are intended to be used with bitwise operations. +/* eslint-disable no-bitwise */ + +export interface IInstallManagerOptions { + /** + * Whether the global "--debug" flag was specified. + */ + debug: boolean; + /** + * Whether or not Rush will automatically update the shrinkwrap file. + * True for "rush update", false for "rush install". + */ + allowShrinkwrapUpdates: boolean; + /** + * Whether to skip policy checks. + */ + bypassPolicy: boolean; + /** + * Whether to skip linking, i.e. require "rush link" to be done manually later. + */ + noLink: boolean; + /** + * Whether to delete the shrinkwrap file before installation, i.e. so that all dependencies + * will be upgraded to the latest SemVer-compatible version. + */ + fullUpgrade: boolean; + /** + * Whether to force an update to the shrinkwrap file even if it appears to be unnecessary. + * Normally Rush uses heuristics to determine when "pnpm install" can be skipped, + * but sometimes the heuristics can be inaccurate due to external influences + * (pnpmfile.js script logic, registry changes, etc). + */ + recheckShrinkwrap: boolean; + + /** + * The value of the "--network-concurrency" command-line parameter, which + * is a diagnostic option used to troubleshoot network failures. + * + * Currently only supported for PNPM. + */ + networkConcurrency: number | undefined; + + /** + * Whether or not to collect verbose logs from the package manager. + * If specified when using PNPM, the logs will be in /common/temp/pnpm.log + */ + collectLogFile: boolean; + + /** + * The variant to consider when performing installations and validating shrinkwrap updates. + */ + variant?: string | undefined; + + /** + * Retry the install the specified number of times + */ + maxInstallAttempts: number +} + +/** + * This class implements common logic between "rush install" and "rush update". + */ +export abstract class BaseInstallManager { + private _rushConfiguration: RushConfiguration; + private _rushGlobalFolder: RushGlobalFolder; + private _installRecycler: AsyncRecycler; + private _options: IInstallManagerOptions; + private _commonTempInstallFlag: LastInstallFlag; + + public constructor ( + rushConfiguration: RushConfiguration, + rushGlobalFolder: RushGlobalFolder, + purgeManager: PurgeManager, + options: IInstallManagerOptions + ) { + this._rushConfiguration = rushConfiguration; + this._rushGlobalFolder = rushGlobalFolder; + this._installRecycler = purgeManager.commonTempFolderRecycler; + this._options = options; + + const lastInstallState: JsonObject = { + node: process.versions.node, + packageManager: rushConfiguration.packageManager, + packageManagerVersion: rushConfiguration.packageManagerToolVersion + } + + if (lastInstallState.packageManager === 'pnpm') { + lastInstallState.storePath = rushConfiguration.pnpmOptions.pnpmStorePath; + } + + this._commonTempInstallFlag = new LastInstallFlag( + this._rushConfiguration.commonTempFolder, + lastInstallState + ); + } + + protected get rushConfiguration(): RushConfiguration { return this._rushConfiguration; } + + protected get rushGlobalFolder(): RushGlobalFolder { return this._rushGlobalFolder; } + + protected get installRecycler(): AsyncRecycler { return this._installRecycler; } + + protected get options(): IInstallManagerOptions { return this._options; } + + public async doInstall(): Promise { + const { shrinkwrapIsUpToDate, variantIsUpToDate } = await this.prepare(); + + // This marker file indicates that the last "rush install" completed successfully. + // If "--purge" was specified, or if the last install was interrupted, then we will need to + // perform a clean install. Otherwise, we can do an incremental install. + const cleanInstall: boolean = !this._commonTempInstallFlag.checkValidAndReportStoreIssues(); + + // Allow us to defer the file read until we need it + const canSkipInstall: () => boolean = () => { + const outputStats: fs.Stats = FileSystem.getStatistics(this._commonTempInstallFlag.path); + return this.canSkipInstall(outputStats.mtime); + } + + if (cleanInstall || !shrinkwrapIsUpToDate || !variantIsUpToDate || !canSkipInstall()) { + let publishedRelease: boolean | undefined; + try { + publishedRelease = await this.checkIfReleaseIsPublished(); + } catch { + // If the user is working in an environment that can't reach the registry, + // don't bother them with errors. + } + + if (publishedRelease === false) { + console.log(colors.yellow('Warning: This release of the Rush tool was unpublished; it may be unstable.')); + } + + // Since we're going to be tampering with common/node_modules, delete the "rush link" flag file if it exists; + // this ensures that a full "rush link" is required next time + Utilities.deleteFile(this.rushConfiguration.rushLinkJsonFilename); + + // Delete the successful install file to indicate the install transaction has started + this._commonTempInstallFlag.clear(); + + // Perform the actual install + await this.install(cleanInstall); + + if (this.options.allowShrinkwrapUpdates && !shrinkwrapIsUpToDate) { + // Shrinkwrap files may need to be post processed after install, so load and save it + const tempShrinkwrapFile: BaseShrinkwrapFile | undefined = ShrinkwrapFileFactory.getShrinkwrapFile( + this.rushConfiguration.packageManager, + this.rushConfiguration.packageManagerOptions, + this.rushConfiguration.tempShrinkwrapFilename); + if (tempShrinkwrapFile) { + tempShrinkwrapFile.save(this.rushConfiguration.tempShrinkwrapFilename); + } + + // Copy (or delete) common\temp\pnpm-lock.yaml --> common\config\rush\pnpm-lock.yaml + Utilities.syncFile( + this.rushConfiguration.tempShrinkwrapFilename, + this.rushConfiguration.getCommittedShrinkwrapFilename(this.options.variant) + ); + } else { + // TODO: Validate whether the package manager updated it in a nontrivial way + } + + // Create the marker file to indicate a successful install + this._commonTempInstallFlag.create(); + + console.log(''); + } + + if (!this.options.noLink) { + await this.link(); + } else { + console.log( + os.EOL + colors.yellow('Since "--no-link" was specified, you will need to run "rush link" manually.') + ); + } + } + + protected abstract prepareAndCheckShrinkwrap( + shrinkwrapFile: BaseShrinkwrapFile | undefined + ): Promise<{ shrinkwrapIsUpToDate: boolean, shrinkwrapWarnings: string[] }>; + + protected abstract canSkipInstall(lastInstallDate: Date): boolean; + + protected abstract install(cleanInstall: boolean): Promise; + + protected async prepare(): Promise<{ variantIsUpToDate: boolean, shrinkwrapIsUpToDate: boolean }> { + // Check the policies + PolicyValidator.validatePolicy(this.rushConfiguration, this.options); + + // Git hooks are only installed if the repo opts in by including files in /common/git-hooks + const hookSource: string = path.join(this.rushConfiguration.commonFolder, 'git-hooks'); + const hookDestination: string | undefined = Git.getHooksFolder(); + + if (FileSystem.exists(hookSource) && hookDestination) { + const hookFilenames: string[] = FileSystem.readFolder(hookSource); + if (hookFilenames.length > 0) { + console.log(os.EOL + colors.bold('Found files in the "common/git-hooks" folder.')); + + // Clear the currently installed git hooks and install fresh copies + FileSystem.ensureEmptyFolder(hookDestination); + + // Only copy files that look like Git hook names + const filteredHookFilenames: string[] = hookFilenames.filter(x => /^[a-z\-]+/.test(x)); + for (const filename of filteredHookFilenames) { + FileSystem.copyFile({ + sourcePath: path.join(hookSource, filename), + destinationPath: path.join(hookDestination, filename) + }); + FileSystem.changePosixModeBits(path.join(hookDestination, filename), + PosixModeBits.UserRead | PosixModeBits.UserExecute); + } + + console.log('Successfully installed these Git hook scripts: ' + filteredHookFilenames.join(', ') + os.EOL); + } + } + + const approvedPackagesChecker: ApprovedPackagesChecker = new ApprovedPackagesChecker(this.rushConfiguration); + if (approvedPackagesChecker.approvedPackagesFilesAreOutOfDate) { + if (this.options.allowShrinkwrapUpdates) { + approvedPackagesChecker.rewriteConfigFiles(); + console.log(colors.yellow( + 'Approved package files have been updated. These updates should be committed to source control' + )); + } else { + throw new Error(`Approved packages files are out-of date. Run "rush update" to update them.`); + } + } + + // Ensure that the package manager is installed + await this.ensureLocalPackageManager(); + + // Write a file indicating which variant is being installed. + // This will be used by bulk scripts to determine the correct Shrinkwrap file to track. + const currentVariantJsonFilename: string = this.rushConfiguration.currentVariantJsonFilename; + const currentVariantJson: ICurrentVariantJson = { + variant: this.options.variant || null // eslint-disable-line @rushstack/no-null + }; + + // Determine if the variant is already current by updating current-variant.json. + // If nothing is written, the variant has not changed. + const variantIsUpToDate: boolean = !JsonFile.save(currentVariantJson, currentVariantJsonFilename, { + onlyIfChanged: true + }); + + if (this.options.variant) { + console.log(); + console.log(colors.bold(`Using variant '${this.options.variant}' for installation.`)); + } else if (!variantIsUpToDate && !this.options.variant) { + console.log(); + console.log(colors.bold('Using the default variant for installation.')); + } + + let shrinkwrapFile: BaseShrinkwrapFile | undefined = undefined; + + // (If it's a full update, then we ignore the shrinkwrap from Git since it will be overwritten) + if (!this.options.fullUpgrade) { + try { + shrinkwrapFile = ShrinkwrapFileFactory.getShrinkwrapFile( + this.rushConfiguration.packageManager, + this.rushConfiguration.packageManagerOptions, + this.rushConfiguration.getCommittedShrinkwrapFilename(this.options.variant) + ); + } catch (ex) { + console.log(); + console.log(`Unable to load the ${this.rushConfiguration.shrinkwrapFilePhrase}: ${ex.message}`); + + if (!this.options.allowShrinkwrapUpdates) { + console.log(); + console.log(colors.red('You need to run "rush update" to fix this problem')); + throw new AlreadyReportedError(); + } + + shrinkwrapFile = undefined; + } + } + + // Also copy down the committed .npmrc file, if there is one + // "common\config\rush\.npmrc" --> "common\temp\.npmrc" + // Also ensure that we remove any old one that may be hanging around + Utilities.syncNpmrc(this.rushConfiguration.commonRushConfigFolder, this.rushConfiguration.commonTempFolder); + + // also, copy the pnpmfile.js if it exists + if (this.rushConfiguration.packageManager === 'pnpm') { + const committedPnpmFilePath: string = this.rushConfiguration.getPnpmfilePath(this.options.variant); + const tempPnpmFilePath: string = path.join( + this.rushConfiguration.commonTempFolder, + RushConstants.pnpmfileFilename + ); + + // ensure that we remove any old one that may be hanging around + Utilities.syncFile(committedPnpmFilePath, tempPnpmFilePath); + } + + // Allow for package managers to do their own preparation and check that the shrinkwrap is up to date + // eslint-disable-next-line prefer-const + let { shrinkwrapIsUpToDate, shrinkwrapWarnings } = await this.prepareAndCheckShrinkwrap(shrinkwrapFile); + shrinkwrapIsUpToDate = shrinkwrapIsUpToDate && !( + this.options.recheckShrinkwrap || (shrinkwrapFile && shrinkwrapFile.shouldForceRecheck()) + ); + + // Write out the reported warnings + if (shrinkwrapWarnings.length > 0) { + console.log(); + console.log(colors.yellow(Utilities.wrapWords( + `The ${this.rushConfiguration.shrinkwrapFilePhrase} contains the following issues:`))); + + for (const shrinkwrapWarning of shrinkwrapWarnings) { + console.log(colors.yellow(' ' + shrinkwrapWarning)); + } + console.log(); + } + + this._syncTempShrinkwrap(shrinkwrapFile); + + // Force update if the shrinkwrap is out of date + if (!shrinkwrapIsUpToDate) { + if (!this.options.allowShrinkwrapUpdates) { + console.log(); + console.log(colors.red( + `The ${this.rushConfiguration.shrinkwrapFilePhrase} is out of date. You need to run "rush update".` + )); + throw new AlreadyReportedError(); + } + } + + return { shrinkwrapIsUpToDate, variantIsUpToDate }; + } + + protected link(): Promise { + const linkManager: BaseLinkManager = LinkManagerFactory.getLinkManager(this.rushConfiguration); + return linkManager.createSymlinksForProjects(false); + } + + protected checkIfReleaseIsPublished(): Promise { + return Promise.resolve().then(() => { + const lastCheckFile: string = path.join(this.rushGlobalFolder.nodeSpecificPath, + 'rush-' + Rush.version, 'last-check.flag'); + + if (FileSystem.exists(lastCheckFile)) { + let cachedResult: boolean | 'error' | undefined = undefined; + try { + // NOTE: mtimeMs is not supported yet in Node.js 6.x + const nowMs: number = new Date().getTime(); + const ageMs: number = nowMs - FileSystem.getStatistics(lastCheckFile).mtime.getTime(); + const HOUR: number = 60 * 60 * 1000; + + // Is the cache too old? + if (ageMs < 24 * HOUR) { + // No, read the cached result + cachedResult = JsonFile.load(lastCheckFile); + } + } catch (e) { + // Unable to parse file + } + if (cachedResult === 'error') { + return Promise.reject(new Error('Unable to contact server')); + } + if (cachedResult === true || cachedResult === false) { + return cachedResult; + } + } + + // Before we start the network operation, record a failed state. If the process exits for some reason, + // this will record the error. It will also update the timestamp to prevent other Rush instances + // from attempting to update the file. + JsonFile.save('error', lastCheckFile, { ensureFolderExists: true }); + + // For this check we use the official registry, not the private registry + return BaseInstallManager._queryIfReleaseIsPublished('https://registry.npmjs.org:443') + .then((publishedRelease: boolean) => { + // Cache the result + JsonFile.save(publishedRelease, lastCheckFile, { ensureFolderExists: true }); + return publishedRelease; + }) + .catch((error: Error) => { + JsonFile.save('error', lastCheckFile, { ensureFolderExists: true }); + return Promise.reject(error); + }); + }); + } + + /** + * If the "(p)npm-local" symlink hasn't been set up yet, this creates it, installing the + * specified (P)npm version in the user's home directory if needed. + */ + public ensureLocalPackageManager(): Promise { + // Example: "C:\Users\YourName\.rush" + const rushUserFolder: string = this.rushGlobalFolder.nodeSpecificPath; + + if (!FileSystem.exists(rushUserFolder)) { + console.log('Creating ' + rushUserFolder); + FileSystem.ensureFolder(rushUserFolder); + } + + const packageManager: PackageManagerName = this.rushConfiguration.packageManager; + const packageManagerVersion: string = this.rushConfiguration.packageManagerToolVersion; + + const packageManagerAndVersion: string = `${packageManager}-${packageManagerVersion}`; + // Example: "C:\Users\YourName\.rush\pnpm-1.2.3" + const packageManagerToolFolder: string = path.join(rushUserFolder, packageManagerAndVersion); + + const packageManagerMarker: LastInstallFlag = new LastInstallFlag(packageManagerToolFolder, { + node: process.versions.node + }); + + console.log(`Trying to acquire lock for ${packageManagerAndVersion}`); + return LockFile.acquire(rushUserFolder, packageManagerAndVersion).then((lock: LockFile) => { + console.log(`Acquired lock for ${packageManagerAndVersion}`); + + if (!packageManagerMarker.isValid() || lock.dirtyWhenAcquired) { + console.log(colors.bold(`Installing ${packageManager} version ${packageManagerVersion}${os.EOL}`)); + + // note that this will remove the last-install flag from the directory + Utilities.installPackageInDirectory({ + directory: packageManagerToolFolder, + packageName: packageManager, + version: this.rushConfiguration.packageManagerToolVersion, + tempPackageTitle: `${packageManager}-local-install`, + maxInstallAttempts: this.options.maxInstallAttempts, + // This is using a local configuration to install a package in a shared global location. + // Generally that's a bad practice, but in this case if we can successfully install + // the package at all, we can reasonably assume it's good for all the repositories. + // In particular, we'll assume that two different NPM registries cannot have two + // different implementations of the same version of the same package. + // This was needed for: https://github.com/microsoft/rushstack/issues/691 + commonRushConfigFolder: this.rushConfiguration.commonRushConfigFolder + }); + + console.log(`Successfully installed ${packageManager} version ${packageManagerVersion}`); + } else { + console.log(`Found ${packageManager} version ${packageManagerVersion} in ${packageManagerToolFolder}`); + } + + packageManagerMarker.create(); + + // Example: "C:\MyRepo\common\temp" + FileSystem.ensureFolder(this.rushConfiguration.commonTempFolder); + + // Example: "C:\MyRepo\common\temp\pnpm-local" + const localPackageManagerToolFolder: string = path.join( + this.rushConfiguration.commonTempFolder, + `${packageManager}-local` + ); + + console.log(os.EOL + 'Symlinking "' + localPackageManagerToolFolder + '"'); + console.log(' --> "' + packageManagerToolFolder + '"'); + + // We cannot use FileSystem.exists() to test the existence of a symlink, because it will + // return false for broken symlinks. There is no way to test without catching an exception. + try { + FileSystem.deleteFolder(localPackageManagerToolFolder); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + FileSystem.createSymbolicLinkJunction({ + linkTargetPath: packageManagerToolFolder, + newLinkPath: localPackageManagerToolFolder + }); + + lock.release(); + }); + } + + protected getPackageManagerEnvironment(): NodeJS.ProcessEnv { + let configurationEnvironment: IConfigurationEnvironment | undefined = undefined; + if (this.rushConfiguration.packageManager === 'npm') { + if ( + this.rushConfiguration.npmOptions && + this.rushConfiguration.npmOptions.environmentVariables + ) { + configurationEnvironment = this.rushConfiguration.npmOptions.environmentVariables; + } + } else if (this.rushConfiguration.packageManager === 'pnpm') { + if ( + this.rushConfiguration.pnpmOptions && + this.rushConfiguration.pnpmOptions.environmentVariables + ) { + configurationEnvironment = this.rushConfiguration.pnpmOptions.environmentVariables; + } + } else if (this.rushConfiguration.packageManager === 'yarn') { + if ( + this.rushConfiguration.yarnOptions && + this.rushConfiguration.yarnOptions.environmentVariables + ) { + configurationEnvironment = this.rushConfiguration.yarnOptions.environmentVariables; + } + } + + return this._mergeEnvironmentVariables( + process.env, + configurationEnvironment + ); + } + + /** + * Used when invoking the NPM tool. Appends the common configuration options + * to the command-line. + */ + protected pushConfigurationArgs(args: string[], options: IInstallManagerOptions): void { + if (this._rushConfiguration.packageManager === 'npm') { + if (semver.lt(this._rushConfiguration.packageManagerToolVersion, '5.0.0')) { + // NOTE: + // + // When using an npm version older than v5.0.0, we do NOT install optional dependencies for + // Rush, because npm does not generate the shrinkwrap file consistently across platforms. + // + // Consider the "fsevents" package. This is a Mac specific package + // which is an optional second-order dependency. Optional dependencies work by attempting to install + // the package, but removes the package if the install failed. + // This means that someone running generate on a Mac WILL have fsevents included in their shrinkwrap. + // When someone using Windows attempts to install from the shrinkwrap, the install will fail. + // + // If someone generates the shrinkwrap using Windows, then fsevents will NOT be listed in the shrinkwrap. + // When someone using Mac attempts to install from the shrinkwrap, they will NOT have the + // optional dependency installed. + // + // This issue has been fixed as of npm v5.0.0: https://github.com/npm/npm/releases/tag/v5.0.0 + // + // For more context, see https://github.com/microsoft/rushstack/issues/761#issuecomment-428689600 + args.push('--no-optional'); + } + args.push('--cache', this._rushConfiguration.npmCacheFolder); + args.push('--tmp', this._rushConfiguration.npmTmpFolder); + + if (options.collectLogFile) { + args.push('--verbose'); + } + } else if (this._rushConfiguration.packageManager === 'pnpm') { + // Only explicitly define the store path if `pnpmStore` is using the default, or has been set to + // 'local'. If `pnpmStore` = 'global', then allow PNPM to use the system's default + // path. In all cases, this will be overridden by RUSH_PNPM_STORE_PATH + if ( + this._rushConfiguration.pnpmOptions.pnpmStore === 'local' || + EnvironmentConfiguration.pnpmStorePathOverride + ) { + args.push('--store', this._rushConfiguration.pnpmOptions.pnpmStorePath); + } + + // we are using the --no-lock flag for now, which unfortunately prints a warning, but should be OK + // since rush already has its own install lock file which will invalidate the cache for us. + // we theoretically could use the lock file, but we would need to clean the store if the + // lockfile existed, otherwise PNPM would hang indefinitely. it is simpler to rely on Rush's + // last install flag, which encapsulates the entire installation + args.push('--no-lock'); + + // Ensure that Rush's tarball dependencies get synchronized properly with the pnpm-lock.yaml file. + // See this GitHub issue: https://github.com/pnpm/pnpm/issues/1342 + + if (semver.gte(this._rushConfiguration.packageManagerToolVersion, '3.0.0')) { + args.push('--no-prefer-frozen-lockfile'); + } else { + args.push('--no-prefer-frozen-shrinkwrap'); + } + + if (options.collectLogFile) { + args.push('--reporter', 'ndjson'); + } + + if (options.networkConcurrency) { + args.push('--network-concurrency', options.networkConcurrency.toString()); + } + + if (this._rushConfiguration.pnpmOptions.strictPeerDependencies) { + args.push('--strict-peer-dependencies'); + } + + if ((this._rushConfiguration.packageManagerWrapper as PnpmPackageManager).supportsResolutionStrategy) { + args.push('--resolution-strategy', this._rushConfiguration.pnpmOptions.resolutionStrategy); + } + } else if (this._rushConfiguration.packageManager === 'yarn') { + args.push('--link-folder', 'yarn-link'); + args.push('--cache-folder', this._rushConfiguration.yarnCacheFolder); + + // Without this option, Yarn will sometimes stop and ask for user input on STDIN + // (e.g. "Which command would you like to run?"). + args.push('--non-interactive'); + + if (options.networkConcurrency) { + args.push('--network-concurrency', options.networkConcurrency.toString()); + } + + if (this._rushConfiguration.yarnOptions.ignoreEngines) { + args.push('--ignore-engines'); + } + } + } + + private _syncTempShrinkwrap(shrinkwrapFile: BaseShrinkwrapFile | undefined): void { + if (shrinkwrapFile) { + // If we have a (possibly incomplete) shrinkwrap file, save it as the temporary file. + shrinkwrapFile.save(this.rushConfiguration.tempShrinkwrapFilename); + shrinkwrapFile.save(this.rushConfiguration.tempShrinkwrapPreinstallFilename); + } else { + // Otherwise delete the temporary file + FileSystem.deleteFile(this.rushConfiguration.tempShrinkwrapFilename); + + if (this.rushConfiguration.packageManager === 'pnpm') { + // Workaround for https://github.com/pnpm/pnpm/issues/1890 + // + // When "rush update --full" is run, rush deletes common/temp/pnpm-lock.yaml so that + // a new lockfile can be generated. But because of the above bug "pnpm install" would + // respect "common/temp/node_modules/.pnpm-lock.yaml" and thus would not generate a + // new lockfile. Deleting this file in addition to deleting common/temp/pnpm-lock.yaml + // ensures that a new lockfile will be generated with "rush update --full". + + const pnpmPackageManager: PnpmPackageManager = + (this.rushConfiguration.packageManagerWrapper as PnpmPackageManager); + + FileSystem.deleteFile(path.join(this.rushConfiguration.commonTempFolder, + pnpmPackageManager.internalShrinkwrapRelativePath)); + } + } + } + + private _mergeEnvironmentVariables( + baseEnv: NodeJS.ProcessEnv, + environmentVariables?: IConfigurationEnvironment + ): NodeJS.ProcessEnv { + const packageManagerEnv: NodeJS.ProcessEnv = baseEnv; + + if (environmentVariables) { + // eslint-disable-next-line guard-for-in + for (const envVar in environmentVariables) { + let setEnvironmentVariable: boolean = true; + console.log(`\nProcessing definition for environment variable: ${envVar}`); + + if (baseEnv.hasOwnProperty(envVar)) { + setEnvironmentVariable = false; + console.log(`Environment variable already defined:`); + console.log(` Name: ${envVar}`); + console.log(` Existing value: ${baseEnv[envVar]}`); + console.log(` Value set in rush.json: ${environmentVariables[envVar].value}`); + + if (environmentVariables[envVar].override) { + setEnvironmentVariable = true; + console.log(`Overriding the environment variable with the value set in rush.json.`); + } + else { + console.log(colors.yellow(`WARNING: Not overriding the value of the environment variable.`)); + } + } + + if (setEnvironmentVariable) { + if (this.options.debug) { + console.log(`Setting environment variable for package manager.`); + console.log(` Name: ${envVar}`); + console.log(` Value: ${environmentVariables[envVar].value}`); + } + packageManagerEnv[envVar] = environmentVariables[envVar].value; + } + } + } + + return packageManagerEnv; + } + + public static collectPreferredVersions( + rushConfiguration: RushConfiguration, + variant: string | undefined + ): Map { + // dependency name --> version specifier + const allExplicitPreferredVersions: Map = rushConfiguration + .getCommonVersions(variant) + .getAllPreferredVersions(); + + // dependency name --> version specifier + const allPreferredVersions: Map = new Map(); + + // Should we add implicitly preferred versions? + let useImplicitlyPinnedVersions: boolean; + if (rushConfiguration.commonVersions.implicitlyPreferredVersions !== undefined) { + // Use the manually configured setting + useImplicitlyPinnedVersions = rushConfiguration.commonVersions.implicitlyPreferredVersions; + } else { + // Default to true. + useImplicitlyPinnedVersions = true; + } + + if (useImplicitlyPinnedVersions) { + // Add in the implicitly preferred versions. + // These are any first-level dependencies for which we only consume a single version range + // (e.g. every package that depends on react uses an identical specifier) + const implicitlyPreferredVersions: Map = + BaseInstallManager.collectImplicitlyPreferredVersions(rushConfiguration, { variant }); + MapExtensions.mergeFromMap(allPreferredVersions, implicitlyPreferredVersions); + } + + // Add in the explicitly preferred versions. + // Note that these take precedence over implicitly preferred versions. + MapExtensions.mergeFromMap(allPreferredVersions, allExplicitPreferredVersions); + return allPreferredVersions; + } + + /** + * Returns a map of all direct dependencies that only have a single semantic version specifier. + * Returns a map: dependency name --> version specifier + */ + public static collectImplicitlyPreferredVersions( + rushConfiguration: RushConfiguration, + options: { + variant?: string | undefined + } = {} + ): Map { + // First, collect all the direct dependencies of all local projects, and their versions: + // direct dependency name --> set of version specifiers + const versionsForDependencies: Map> = new Map>(); + + rushConfiguration.projects.forEach((project: RushConfigurationProject) => { + BaseInstallManager._collectVersionsForDependencies( + rushConfiguration, + { + versionsForDependencies, + dependencies: project.packageJsonEditor.dependencyList, + cyclicDependencies: project.cyclicDependencyProjects, + variant: options.variant + }); + + BaseInstallManager._collectVersionsForDependencies( + rushConfiguration, + { + versionsForDependencies, + dependencies: project.packageJsonEditor.devDependencyList, + cyclicDependencies: project.cyclicDependencyProjects, + variant: options.variant + }); + }); + + // If any dependency has more than one version, then filter it out (since we don't know which version + // should be preferred). What remains will be the list of preferred dependencies. + // dependency --> version specifier + const implicitlyPreferred: Map = new Map(); + versionsForDependencies.forEach((versions: Set, dep: string) => { + if (versions.size === 1) { + const version: string = versions.values().next().value; + implicitlyPreferred.set(dep, version); + } + }); + return implicitlyPreferred; + } + + // Helper for checkIfReleaseIsPublished() + private static _queryIfReleaseIsPublished(registryUrl: string): Promise { + let queryUrl: string = registryUrl; + if (queryUrl[-1] !== '/') { + queryUrl += '/'; + } + // Note that the "@" symbol does not normally get URL-encoded + queryUrl += RushConstants.rushPackageName.replace('/', '%2F'); + + const userAgent: string = `pnpm/? npm/? node/${process.version} ${os.platform()} ${os.arch()}`; + + const headers: fetch.Headers = new fetch.Headers(); + headers.append('user-agent', userAgent); + headers.append('accept', 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*'); + + let agent: http.Agent | undefined = undefined; + if (process.env.HTTP_PROXY) { + agent = new HttpsProxyAgent(process.env.HTTP_PROXY); + } + + return fetch.default(queryUrl, { + headers: headers, + agent: agent + }) + .then((response: fetch.Response) => { + if (!response.ok) { + return Promise.reject(new Error('Failed to query')); + } + return response + .json() + .then((data) => { + let url: string; + try { + if (!data.versions[Rush.version]) { + // Version was not published + return false; + } + url = data.versions[Rush.version].dist.tarball; + if (!url) { + return Promise.reject(new Error(`URL not found`)); + } + } catch (e) { + return Promise.reject(new Error('Error parsing response')); + } + + // Make sure the tarball wasn't deleted from the CDN + headers.set('accept', '*/*'); + return fetch.default(url, { + headers: headers, + agent: agent + }) + .then((response2: fetch.Response) => { + if (!response2.ok) { + if (response2.status === 404) { + return false; + } else { + return Promise.reject(new Error('Failed to fetch')); + } + } + return true; + }); + }); + }); + } + + // Helper for collectImplicitlyPreferredVersions() + private static _updateVersionsForDependencies(versionsForDependencies: Map>, + dependency: string, version: string): void { + if (!versionsForDependencies.has(dependency)) { + versionsForDependencies.set(dependency, new Set()); + } + versionsForDependencies.get(dependency)!.add(version); + } + + // Helper for collectImplicitlyPreferredVersions() + private static _collectVersionsForDependencies( + rushConfiguration: RushConfiguration, + options: { + versionsForDependencies: Map>; + dependencies: ReadonlyArray; + cyclicDependencies: Set; + variant: string | undefined; + }): void { + const { + variant, + dependencies, + versionsForDependencies, + cyclicDependencies + } = options; + + const commonVersions: CommonVersionsConfiguration = rushConfiguration.getCommonVersions(variant); + + const allowedAlternativeVersions: Map> + = commonVersions.allowedAlternativeVersions; + + for (const dependency of dependencies) { + const alternativesForThisDependency: ReadonlyArray + = allowedAlternativeVersions.get(dependency.name) || []; + + // For each dependency, collectImplicitlyPreferredVersions() is collecting the set of all version specifiers + // that appear across the repo. If there is only one version specifier, then that's the "preferred" one. + // However, there are a few cases where additional version specifiers can be safely ignored. + let ignoreVersion: boolean = false; + + // 1. If the version specifier was listed in "allowedAlternativeVersions", then it's never a candidate. + // (Even if it's the only version specifier anywhere in the repo, we still ignore it, because + // otherwise the rule would be difficult to explain.) + if (alternativesForThisDependency.indexOf(dependency.version) > 0) { + ignoreVersion = true; + } else { + // Is it a local project? + const localProject: RushConfigurationProject | undefined = rushConfiguration.getProjectByName(dependency.name); + if (localProject) { + // 2. If it's a symlinked local project, then it's not a candidate, because the package manager will + // never even see it. + // However there are two ways that a local project can NOT be symlinked: + // - if the local project doesn't satisfy the referenced semver specifier; OR + // - if the local project was specified in "cyclicDependencyProjects" in rush.json + const specifier: DependencySpecifier = new DependencySpecifier(dependency.name, dependency.version); + if (semver.satisfies(localProject.packageJsonEditor.version, specifier.versionSpecifier) + && !cyclicDependencies.has(dependency.name)) { + ignoreVersion = true; + } + } + + if (!ignoreVersion) { + BaseInstallManager._updateVersionsForDependencies(versionsForDependencies, dependency.name, dependency.version); + } + } + } + } +} diff --git a/apps/rush-lib/src/logic/base/BaseShrinkwrapFile.ts b/apps/rush-lib/src/logic/base/BaseShrinkwrapFile.ts index fa3f03e239f..60f000513f3 100644 --- a/apps/rush-lib/src/logic/base/BaseShrinkwrapFile.ts +++ b/apps/rush-lib/src/logic/base/BaseShrinkwrapFile.ts @@ -8,7 +8,7 @@ import { PackageName, FileSystem } from '@rushstack/node-core-library'; import { RushConstants } from '../../logic/RushConstants'; import { DependencySpecifier } from '../DependencySpecifier'; import { IPolicyValidatorOptions } from '../policy/PolicyValidator'; -import { PackageManagerOptionsConfigurationBase } from '../../api/RushConfiguration'; +import { PackageManagerOptionsConfigurationBase, RushConfiguration } from '../../api/RushConfiguration'; /** * This class is a parser for both npm's npm-shrinkwrap.json and pnpm's pnpm-lock.yaml file formats. @@ -95,6 +95,20 @@ export abstract class BaseShrinkwrapFile { return this._checkDependencyVersion(dependencySpecifier, shrinkwrapDependency); } + public tryEnsureCompatibleWorkspaceDependency( + dependencySpecifier: DependencySpecifier, + projectName: string, + rushConfiguration: RushConfiguration + ): boolean { + const shrinkwrapDependency: DependencySpecifier | undefined = + this.tryEnsureWorkspaceDependencyVersion(dependencySpecifier, projectName, rushConfiguration); + if (!shrinkwrapDependency) { + return false; + } + + return this._checkDependencyVersion(dependencySpecifier, shrinkwrapDependency); + } + /** * Returns the list of temp projects defined in this file. * Example: [ '@rush-temp/project1', '@rush-temp/project2' ] @@ -103,10 +117,26 @@ export abstract class BaseShrinkwrapFile { */ public abstract getTempProjectNames(): ReadonlyArray; + /** + * Returns the list of paths to Rush projects relative to the + * install root. + * Example: [ '../../apps/project1', '../../apps/project2' ] + * + * @virtual + */ + public abstract getWorkspacePaths(): ReadonlyArray; + /** @virtual */ protected abstract tryEnsureDependencyVersion(dependencySpecifier: DependencySpecifier, tempProjectName: string): DependencySpecifier | undefined; + /** @virtual */ + protected abstract tryEnsureWorkspaceDependencyVersion( + dependencySpecifier: DependencySpecifier, + projectName: string, + rushConfiguration: RushConfiguration + ): DependencySpecifier | undefined; + /** @virtual */ protected abstract getTopLevelDependencyVersion(dependencyName: string): DependencySpecifier | undefined; diff --git a/apps/rush-lib/src/logic/npm/NpmShrinkwrapFile.ts b/apps/rush-lib/src/logic/npm/NpmShrinkwrapFile.ts index 8ac80a66539..4de2f4379d9 100644 --- a/apps/rush-lib/src/logic/npm/NpmShrinkwrapFile.ts +++ b/apps/rush-lib/src/logic/npm/NpmShrinkwrapFile.ts @@ -2,13 +2,15 @@ import * as os from 'os'; import { JsonFile, - FileSystem + FileSystem, + InternalError } from '@rushstack/node-core-library'; import { BaseShrinkwrapFile } from '../base/BaseShrinkwrapFile'; import { DependencySpecifier } from '../DependencySpecifier'; +import { RushConfiguration } from '../../api/RushConfiguration'; interface INpmShrinkwrapDependencyJson { version: string; @@ -111,4 +113,17 @@ export class NpmShrinkwrapFile extends BaseShrinkwrapFile { return new DependencySpecifier(dependencySpecifier.packageName, dependencyJson.version); } + /** @override */ + protected tryEnsureWorkspaceDependencyVersion( + dependencySpecifier: DependencySpecifier, + projectName: string, + rushConfiguration: RushConfiguration + ): DependencySpecifier | undefined { + throw new InternalError('Not implemented'); + } + + /** @override */ + public getWorkspacePaths(): ReadonlyArray { + throw new InternalError('Not implemented'); + } } \ No newline at end of file diff --git a/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts b/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts index 532859212af..c26fafee010 100644 --- a/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts +++ b/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts @@ -26,6 +26,7 @@ import { IRushLinkJson } from '../../api/RushConfiguration'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { PnpmShrinkwrapFile, IPnpmShrinkwrapDependencyYaml } from './PnpmShrinkwrapFile'; import { PnpmProjectDependencyManifest } from './PnpmProjectDependencyManifest'; +import { PackageJsonDependency } from '../../api/PackageJsonEditor'; // special flag for debugging, will print extra diagnostic information, // but comes with performance cost @@ -51,15 +52,124 @@ export class PnpmLinkManager extends BaseLinkManager { throw new InternalError(`Cannot load shrinkwrap at "${this._rushConfiguration.tempShrinkwrapFilename}"`); } + const useWorkspaces: boolean = ( + this._rushConfiguration.packageManager === 'pnpm' && + this._rushConfiguration.pnpmOptions && + this._rushConfiguration.pnpmOptions.useWorkspaces + ); + for (const rushProject of this._rushConfiguration.projects) { console.log(os.EOL + 'LINKING: ' + rushProject.packageName); - await this._linkProject(rushProject, rushLinkJson, pnpmShrinkwrapFile); + + if (useWorkspaces) { + await this._linkWorkspaceProject(rushProject, rushLinkJson, pnpmShrinkwrapFile); + } else { + await this._linkProject(rushProject, rushLinkJson, pnpmShrinkwrapFile); + } } console.log(`Writing "${this._rushConfiguration.rushLinkJsonFilename}"`); JsonFile.save(rushLinkJson, this._rushConfiguration.rushLinkJsonFilename); } + /** + * This is called once for each local project from Rush.json. + * @param project The local project that we will create symlinks for + * @param rushLinkJson The common/temp/rush-link.json output file + */ + private async _linkWorkspaceProject( + project: RushConfigurationProject, + rushLinkJson: IRushLinkJson, + pnpmShrinkwrapFile: PnpmShrinkwrapFile + ): Promise { + // First, generate the local dependency graph. When using workspaces, Rush forces `workspace:` + // notation for all locally-referenced projects. + const localDependencies: PackageJsonDependency[] = [ + ...project.packageJsonEditor.dependencyList, + ...project.packageJsonEditor.devDependencyList + ].filter(x => x.version.startsWith('workspace:')); + + for (const { name } of localDependencies) { + const matchedRushPackage: RushConfigurationProject | undefined = + this._rushConfiguration.getProjectByName(name); + + if (matchedRushPackage) { + // We found a suitable match, so add the local package as a local link + let localLinks: string[] = rushLinkJson.localLinks[project.packageName]; + if (!localLinks) { + localLinks = []; + rushLinkJson.localLinks[project.packageName] = localLinks; + } + localLinks.push(name); + } else { + throw new InternalError( + `Cannot find dependency "${name}" for "${project.packageName}" in the Rush configuration` + ); + } + } + + // // We won't be using the package to actually create symlinks, this is just to bootstrap + // // the shrinkwrap-deps.json generation logic. + // const localPackage: BasePackage = BasePackage.createLinkedPackage( + // project.packageName, + // project.packageJsonEditor.version, + // project.projectFolder + // ); + + // // Iterate through all the regular dependencies + // const parentShrinkwrapEntry: IPnpmShrinkwrapDependencyYaml | undefined = + // pnpmShrinkwrapFile.getShrinkwrapEntryFromTempProjectDependencyKey(tempProjectDependencyKey); + // if (!parentShrinkwrapEntry) { + // throw new InternalError( + // 'Cannot find shrinkwrap entry using dependency key for temp project: ' + + // `${project.tempProjectName}`); + // } + + // const pnpmProjectDependencyManifest: PnpmProjectDependencyManifest = new PnpmProjectDependencyManifest({ + // pnpmShrinkwrapFile, + // project + // }); + + // const dependencies: PackageJsonDependency[] = [ + // ...project.packageJsonEditor.dependencyList, + // ...project.packageJsonEditor.devDependencyList + // ].filter(x => !x.version.startsWith('workspace:')); + + // for (const { name, dependencyType } of dependencies) { + + // // read the version number from the shrinkwrap entry + // const isOptional: boolean = dependencyType === DependencyType.Optional; + // const version: string | undefined = isOptional + // ? (parentShrinkwrapEntry.optionalDependencies || {})[name] + // : (parentShrinkwrapEntry.dependencies || {})[name]; + // if (!version) { + // if (!isOptional) { + // throw new InternalError( + // `Cannot find shrinkwrap entry dependency "${name}" for workspace project: ` + + // `${project.packageName}`); + // } + // continue; + // } + + // const newLocalFolderPath: string = path.join(localPackage.folderPath, 'node_modules', name); + // const newLocalPackage: BasePackage = BasePackage.createLinkedPackage( + // name, + // version, + // newLocalFolderPath + // ); + + // if (!this._rushConfiguration.experimentsConfiguration.configuration.legacyIncrementalBuildDependencyDetection) { + // pnpmProjectDependencyManifest.addDependency(newLocalPackage, parentShrinkwrapEntry); + // } + // } + + // if (!this._rushConfiguration.experimentsConfiguration.configuration.legacyIncrementalBuildDependencyDetection) { + // pnpmProjectDependencyManifest.save(); + // } else { + // pnpmProjectDependencyManifest.deleteIfExists(); + // } + } + /** * This is called once for each local project from Rush.json. * @param project The local project that we will create symlinks for diff --git a/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts b/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts index 80eee3a694d..6f5094e353e 100644 --- a/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts +++ b/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts @@ -1,5 +1,6 @@ import * as yaml from 'js-yaml'; import * as os from 'os'; +import * as path from 'path'; import * as semver from 'semver'; import * as crypto from 'crypto'; import * as colors from 'colors'; @@ -7,7 +8,7 @@ import { FileSystem } from '@rushstack/node-core-library'; import { BaseShrinkwrapFile } from '../base/BaseShrinkwrapFile'; import { DependencySpecifier } from '../DependencySpecifier'; -import { PackageManagerOptionsConfigurationBase, PnpmOptionsConfiguration } from '../../api/RushConfiguration'; +import { PackageManagerOptionsConfigurationBase, PnpmOptionsConfiguration, RushConfiguration } from '../../api/RushConfiguration'; import { IPolicyValidatorOptions } from '../policy/PolicyValidator'; import { AlreadyReportedError } from '../../utilities/AlreadyReportedError'; @@ -45,6 +46,15 @@ export interface IPnpmShrinkwrapDependencyYaml { peerDependenciesMeta: { [dependency: string]: IPeerDependenciesMetaYaml }; } +export interface IPnpmShrinkwrapImporterYaml { + /** The list of resolved version numbers for direct dependencies */ + dependencies: { [dependency: string]: string } + /** The list of resolved version numbers for dev dependencies */ + devDependencies: { [dependency: string]: string } + /** The list of specifiers used to resolve direct dependency versions */ + specifiers: { [dependency: string]: string } +} + /** * This interface represents the raw pnpm-lock.YAML file * Example: @@ -76,15 +86,17 @@ export interface IPnpmShrinkwrapDependencyYaml { * } * } */ -interface IPnpmShrinkwrapYaml { +interface IPnpmShrinkwrapYaml extends IPnpmShrinkwrapImporterYaml { /** The list of resolved version numbers for direct dependencies */ - dependencies: { [dependency: string]: string }; + dependencies: { [dependency: string]: string } + /** The list of importers for local workspace projects */ + importers: { [relativePath: string]: IPnpmShrinkwrapImporterYaml }; /** The description of the solved graph */ packages: { [dependencyVersion: string]: IPnpmShrinkwrapDependencyYaml }; /** URL of the registry which was used */ registry: string; /** The list of specifiers used to resolve direct dependency versions */ - specifiers: { [dependency: string]: string }; + specifiers: { [dependency: string]: string } } /** @@ -278,6 +290,19 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { return this._getTempProjectNames(this._shrinkwrapJson.dependencies); } + /** @override */ + public getWorkspacePaths(): ReadonlyArray { + const result: string[] = []; + for (const key of Object.keys(this._shrinkwrapJson.importers)) { + // If it starts with @rush-temp, then include it: + if (key !== '.') { + result.push(key); + } + } + result.sort(); // make the result deterministic + return result; + } + /** * Gets the path to the tarball file if the package is a tarball. * Returns undefined if the package entry doesn't exist or the package isn't a tarball. @@ -294,7 +319,14 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { } public getTopLevelDependencyKey(dependencyName: string): string | undefined { - return BaseShrinkwrapFile.tryGetValue(this._shrinkwrapJson.dependencies, dependencyName); + // For workspaces, top-level dependencies aren't populated, so instead we use the root workspace + // for common package versions and consider this the 'top level' + const dependenciesToCheck: { [dependency: string]: string } = ( + this._shrinkwrapJson.importers && this._shrinkwrapJson.importers.hasOwnProperty('.') + ) ? this._shrinkwrapJson.importers['.'].dependencies + : this._shrinkwrapJson.dependencies; + + return BaseShrinkwrapFile.tryGetValue(dependenciesToCheck, dependencyName); } /** @@ -308,7 +340,14 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { * @override */ public getTopLevelDependencyVersion(dependencyName: string): DependencySpecifier | undefined { - let value: string | undefined = BaseShrinkwrapFile.tryGetValue(this._shrinkwrapJson.dependencies, dependencyName); + // For workspaces, top-level dependencies aren't populated, so instead we use the root workspace + // for common package versions and consider this the 'top level' + const dependenciesToCheck: { [dependency: string]: string } = ( + this._shrinkwrapJson.importers && this._shrinkwrapJson.importers.hasOwnProperty('.') + ) ? this._shrinkwrapJson.importers['.'].dependencies + : this._shrinkwrapJson.dependencies; + + let value: string | undefined = BaseShrinkwrapFile.tryGetValue(dependenciesToCheck, dependencyName); if (value) { // Getting the top level dependency version from a PNPM lockfile version 5.1 @@ -384,6 +423,13 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { return undefined; } + public getImporter(importerPath: string): IPnpmShrinkwrapImporterYaml | undefined { + const importer: IPnpmShrinkwrapImporterYaml | undefined = + BaseShrinkwrapFile.tryGetValue(this._shrinkwrapJson.importers, importerPath); + + return importer && importer.dependencies ? importer : undefined; + } + public getShrinkwrapEntryFromTempProjectDependencyKey( tempProjectDependencyKey: string ): IPnpmShrinkwrapDependencyYaml | undefined { @@ -484,6 +530,51 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { return this._parsePnpmDependencyKey(packageName, dependencyKey); } + /** + * Gets the resolved version number of a dependency for a specific temp project. + * For PNPM, we can reuse the version that another project is using. + * Note that this function modifies the shrinkwrap data. + * + * @override + */ + protected tryEnsureWorkspaceDependencyVersion( + dependencySpecifier: DependencySpecifier, + projectName: string, + rushConfiguration: RushConfiguration + ): DependencySpecifier | undefined { + + // PNPM doesn't have the same advantage of NPM, where we can skip generate as long as the + // shrinkwrap file puts our dependency in either the top of the node_modules folder + // or underneath the package we are looking at. + // This is because the PNPM shrinkwrap file describes the exact links that need to be created + // to recreate the graph.. + // Because of this, we actually need to check for a version that this package is directly + // linked to. + + const packageName: string = dependencySpecifier.packageName; + + // Importer paths use forward slashes + const importerPath: string = path.relative( + rushConfiguration.commonTempFolder, + rushConfiguration.getProjectByName(projectName)!.projectFolder + ).replace(new RegExp(`\\${path.sep}`, 'g'), '/'); + const projectImporter: IPnpmShrinkwrapImporterYaml | undefined = this.getImporter(importerPath); + if (!projectImporter) { + return undefined; + } + + const allDependencies: { [dependency: string]: string } = { + ...(projectImporter.dependencies || {}), + ...(projectImporter.devDependencies || {}) + } + if (!allDependencies.hasOwnProperty(packageName)) { + return undefined; + } + + const dependencyKey: string = allDependencies[packageName]; + return this._parsePnpmDependencyKey(packageName, dependencyKey); + } + /** * Returns the version of a dependency being used by a given project */ diff --git a/apps/rush-lib/src/logic/pnpm/PnpmWorkspaceFile.ts b/apps/rush-lib/src/logic/pnpm/PnpmWorkspaceFile.ts index d59e4017fae..4ec2ffa6634 100644 --- a/apps/rush-lib/src/logic/pnpm/PnpmWorkspaceFile.ts +++ b/apps/rush-lib/src/logic/pnpm/PnpmWorkspaceFile.ts @@ -1,7 +1,8 @@ -import * as yaml from 'js-yaml'; +import * as globEscape from 'glob-escape'; import * as os from 'os'; import * as path from 'path'; -import { FileSystem, Sort } from '@rushstack/node-core-library'; +import * as yaml from 'js-yaml'; +import { FileSystem, Sort, Text } from '@rushstack/node-core-library'; import { BaseWorkspaceFile } from '../base/BaseWorkspaceFile'; @@ -63,8 +64,9 @@ export class PnpmWorkspaceFile extends BaseWorkspaceFile { packagePath = path.relative(path.dirname(this.workspaceFilename), packagePath); } - // Ensure that the path is split using forward slashes - this._workspacePackages.add(path.posix.join(...packagePath.split(path.sep))); + // Glob can't handle Windows paths + const globPath: string = Text.replaceAll(packagePath, '\\', '/'); + this._workspacePackages.add(globEscape(globPath)); } /** @override */ diff --git a/apps/rush-lib/src/logic/yarn/YarnShrinkwrapFile.ts b/apps/rush-lib/src/logic/yarn/YarnShrinkwrapFile.ts index 706787cea1a..1716f936cc8 100644 --- a/apps/rush-lib/src/logic/yarn/YarnShrinkwrapFile.ts +++ b/apps/rush-lib/src/logic/yarn/YarnShrinkwrapFile.ts @@ -6,6 +6,7 @@ import { import { FileSystem, PackageName, IParsedPackageNameOrError, InternalError } from '@rushstack/node-core-library'; import { RushConstants } from '../RushConstants'; import { DependencySpecifier } from '../DependencySpecifier'; +import { RushConfiguration } from '../../api/RushConfiguration'; /** * Used with YarnShrinkwrapFile._encodePackageNameAndSemVer() and _decodePackageNameAndSemVer(). @@ -195,6 +196,11 @@ export class YarnShrinkwrapFile extends BaseShrinkwrapFile { return this._tempProjectNames; } + /** @override */ + public getWorkspacePaths(): ReadonlyArray { + throw new InternalError('Not implemented'); + } + /** @override */ public hasCompatibleTopLevelDependency(dependencySpecifier: DependencySpecifier): boolean { // It seems like we should normalize the key somehow, but Yarn apparently does not @@ -229,4 +235,12 @@ export class YarnShrinkwrapFile extends BaseShrinkwrapFile { throw new InternalError('Not implemented'); } + + protected tryEnsureWorkspaceDependencyVersion( + dependencySpecifier: DependencySpecifier, + projectName: string, + rushConfiguration: RushConfiguration + ): DependencySpecifier | undefined { + throw new InternalError('Not implemented'); + } } diff --git a/apps/rush-lib/src/utilities/Utilities.ts b/apps/rush-lib/src/utilities/Utilities.ts index 41abb827dee..b8e38aa9fa8 100644 --- a/apps/rush-lib/src/utilities/Utilities.ts +++ b/apps/rush-lib/src/utilities/Utilities.ts @@ -264,25 +264,20 @@ export class Utilities { } /* - * Returns true if outputFilename has a more recent last modified timestamp - * than all of the inputFilenames, which would imply that we don't need to rebuild it. - * Returns false if any of the files does not exist. + * Returns true if dateToCompare is more recent than all of the inputFilenames, which + * would imply that we don't need to rebuild it. Returns false if any of the files + * does not exist. * NOTE: The filenames can also be paths for directories, in which case the directory * timestamp is compared. */ - public static isFileTimestampCurrent(outputFilename: string, inputFilenames: string[]): boolean { - if (!FileSystem.exists(outputFilename)) { - return false; - } - const outputStats: fs.Stats = FileSystem.getStatistics(outputFilename); - + public static isFileTimestampCurrent(dateToCompare: Date, inputFilenames: string[]): boolean { for (const inputFilename of inputFilenames) { if (!FileSystem.exists(inputFilename)) { return false; } const inputStats: fs.Stats = FileSystem.getStatistics(inputFilename); - if (outputStats.mtime < inputStats.mtime) { + if (dateToCompare < inputStats.mtime) { return false; } } @@ -550,6 +545,22 @@ export class Utilities { FileSystem.writeFile(targetNpmrcPath, resultLines.join(os.EOL)); } + /** + * Copies the file "sourcePath" to "destinationPath", overwriting the target file location. + * If the source file does not exist, then the target file is deleted. + */ + public static syncFile(sourcePath: string, destinationPath: string): void { + if (FileSystem.exists(sourcePath)) { + console.log('Updating ' + destinationPath); + FileSystem.copyFile({ sourcePath, destinationPath }); + } else { + if (FileSystem.exists(destinationPath)) { + console.log('Deleting ' + destinationPath); + FileSystem.deleteFile(destinationPath); + } + } + } + /** * syncNpmrc() copies the .npmrc file to the target folder, and also trims unusable lines from the .npmrc file. * If the source .npmrc file not exist, then syncNpmrc() will delete an .npmrc that is found in the target folder. From beff2d820c07bd8921f6d7e507f9a192865ad99c Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Fri, 22 May 2020 15:45:00 -0700 Subject: [PATCH 04/16] Support shrinkwrap-deps.json and cleanup --- .../src/logic/WorkspaceInstallManager.ts | 141 +++++++++++------- .../src/logic/base/BaseShrinkwrapFile.ts | 72 +++++---- .../src/logic/npm/NpmShrinkwrapFile.ts | 17 ++- .../src/logic/pnpm/PnpmLinkManager.ts | 132 ++++++++-------- .../pnpm/PnpmProjectDependencyManifest.ts | 32 ++-- .../src/logic/pnpm/PnpmShrinkwrapFile.ts | 85 +++++------ .../src/logic/yarn/YarnShrinkwrapFile.ts | 24 +-- 7 files changed, 282 insertions(+), 221 deletions(-) diff --git a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts index bc13eec02d5..e02834cdfc5 100644 --- a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts +++ b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts @@ -11,14 +11,15 @@ import { JsonFile, IPackageJson, FileSystem, - FileConstants + FileConstants, + InternalError } from '@rushstack/node-core-library'; import { AlreadyReportedError } from '../utilities/AlreadyReportedError'; import { BaseInstallManager, IInstallManagerOptions } from './base/BaseInstallManager'; import { BaseShrinkwrapFile } from '../logic/base/BaseShrinkwrapFile'; import { DependencySpecifier } from './DependencySpecifier'; -import { PackageJsonEditor } from '../api/PackageJsonEditor'; +import { PackageJsonEditor, DependencyType } from '../api/PackageJsonEditor'; import { PnpmWorkspaceFile } from './pnpm/PnpmWorkspaceFile'; import { PurgeManager } from './PurgeManager'; import { RushConfiguration } from '../api/RushConfiguration'; @@ -42,6 +43,15 @@ export class WorkspaceInstallManager extends BaseInstallManager { super(rushConfiguration, rushGlobalFolder, purgeManager, options); } + public static getCommonWorkspaceKey(rushConfiguration: RushConfiguration): string { + switch (rushConfiguration.packageManager) { + case 'pnpm': + return '.' + default: + throw new InternalError('Not implemented'); + } + } + public async doInstall(): Promise { // Workspaces do not support the noLink option, so throw if this is passed if (this.options.noLink) { @@ -72,6 +82,19 @@ export class WorkspaceInstallManager extends BaseInstallManager { if (!shrinkwrapFile) { shrinkwrapIsUpToDate = false; + } else { + if ( + shrinkwrapFile.getWorkspaceKeys().length === 0 && + this.rushConfiguration.projects.length !== 0 && + !this.options.fullUpgrade + ) { + console.log(); + console.log(colors.red( + 'The shrinkwrap file has not been updated to support workspaces. Run "rush update --full" to update ' + + 'the shrinkwrap file.' + )); + throw new AlreadyReportedError(); + } } // dependency name --> version specifier @@ -83,7 +106,14 @@ export class WorkspaceInstallManager extends BaseInstallManager { allExplicitPreferredVersions.forEach((version: string, dependency: string) => { const dependencySpecifier: DependencySpecifier = new DependencySpecifier(dependency, version); - if (!shrinkwrapFile.hasCompatibleTopLevelDependency(dependencySpecifier)) { + // The common package.json is used to ensure common versions are installed, so look for this workspace + // and validate that the requested dependency is specified + if ( + !shrinkwrapFile.hasCompatibleWorkspaceDependency( + dependencySpecifier, + WorkspaceInstallManager.getCommonWorkspaceKey(this.rushConfiguration) + ) + ) { shrinkwrapWarnings.push(`Missing dependency "${dependency}" (${version}) required by the preferred versions from ` + RushConstants.commonVersionsFilename); shrinkwrapIsUpToDate = false; @@ -91,31 +121,12 @@ export class WorkspaceInstallManager extends BaseInstallManager { }); if (this._findOrphanedWorkspaceProjects(shrinkwrapFile)) { - // If there are any orphaned projects, then "npm install" would fail because the shrinkwrap - // contains references such as "resolved": "file:projects\\project1" that refer to nonexistent - // file paths. + // If there are any orphaned projects, then install would fail because the shrinkwrap + // contains references that refer to nonexistent file paths. shrinkwrapIsUpToDate = false; } } - const commonPackageJson: IPackageJson = { - dependencies: {}, - description: 'Temporary file generated by the Rush tool', - name: 'rush-common', - private: true, - version: '0.0.0' - }; - - // dependency name --> version specifier - const allPreferredVersions: Map = - BaseInstallManager.collectPreferredVersions(this.rushConfiguration, this.options.variant); - - // Add any preferred versions to the top of the commonPackageJson - // do this in alphabetical order for simpler debugging - for (const dependency of Array.from(allPreferredVersions.keys()).sort()) { - commonPackageJson.dependencies![dependency] = allPreferredVersions.get(dependency)!; - } - // To generate the workspace file, we will add each project to the file as we loop through and validate const workspaceFile: PnpmWorkspaceFile = new PnpmWorkspaceFile( path.join(this.rushConfiguration.commonTempFolder, 'pnpm-workspace.yaml') @@ -131,20 +142,19 @@ export class WorkspaceInstallManager extends BaseInstallManager { const dependencySpecifier: DependencySpecifier = new DependencySpecifier(name, version); // Is there a locally built Rush project that could satisfy this dependency? - const localProject: RushConfigurationProject | undefined = - this.rushConfiguration.getProjectByName(name); + const referencedLocalProject: RushConfigurationProject | undefined = this.rushConfiguration.getProjectByName(name); // Validate that local projects are referenced with workspace notation. If not, and it is not a // cyclic dependency, then it needs to be updated to specify `workspace:*` explicitly. Currently only // supporting versions and version ranges for specifying a local project. if ( (dependencySpecifier.specifierType === 'version' || dependencySpecifier.specifierType === 'range') && - localProject && + referencedLocalProject && !rushProject.cyclicDependencyProjects.has(name) ) { // Make sure that this version is intended to target a local package. If not, then we will fail since it // is not explicitly specified as a cyclic dependency. - if (!semver.satisfies(localProject.packageJsonEditor.version, dependencySpecifier.versionSpecifier)) { + if (!semver.satisfies(referencedLocalProject.packageJsonEditor.version, dependencySpecifier.versionSpecifier)) { console.log(); console.log(colors.red( `"${rushProject.packageName}" depends on package "${name}" (${version}) which exists within the workspace ` @@ -168,25 +178,21 @@ export class WorkspaceInstallManager extends BaseInstallManager { shrinkwrapIsUpToDate = false; continue; } else if (dependencySpecifier.specifierType === 'workspace') { - // Already specified as a local project, let's just validate that the specifier is valid. - if (!semver.satisfies(localProject!.packageJsonEditor.version, dependencySpecifier.versionSpecifier)) { - console.log(); - console.log(colors.red( - `"${rushProject.packageName}" depends on package "${name}" (${version}) which exists within the workspace ` - + 'but cannot be fulfilled with the specified version range. Specify a valid workspace version range.' - )); - throw new AlreadyReportedError(); + // Already specified as a local project. Allow the package manager to validate this + continue; } + + // PNPM does not specify peer dependencies for workspaces in the shrinkwrap, so skip validating these + if (this.rushConfiguration.packageManager === 'pnpm' && dependencyType === DependencyType.Peer) { continue; } // It is not a local dependency, validate that it is compatible if ( shrinkwrapFile && - !shrinkwrapFile.tryEnsureCompatibleWorkspaceDependency( + !shrinkwrapFile.hasCompatibleWorkspaceDependency( dependencySpecifier, - rushProject.packageName, - this.rushConfiguration + shrinkwrapFile.getWorkspaceKeyByPath(this.rushConfiguration.commonTempFolder, rushProject.projectFolder) ) ) { shrinkwrapWarnings.push(`Missing dependency "${name}" (${version}) required by "${rushProject.packageName}"`); @@ -194,8 +200,8 @@ export class WorkspaceInstallManager extends BaseInstallManager { } } - // Save the package.json if we modified the version references - if (rushProject.packageJsonEditor.saveIfModified()) { + // Save the package.json if we modified the version references and warn that the package.json was modified + if (packageJson.saveIfModified()) { console.log(colors.yellow( `"${rushProject.packageName}" depends on one or more workspace packages which did not use "workspace:" ` + 'notation. The package.json has been modified and must be committed to source control.' @@ -203,14 +209,37 @@ export class WorkspaceInstallManager extends BaseInstallManager { } } + // Update the common package.json to contain all preferred versions + const commonPackageJson: IPackageJson = { + dependencies: {}, + description: 'Temporary file generated by the Rush tool', + name: 'rush-common', + private: true, + version: '0.0.0' + }; + + // dependency name --> version specifier + const allPreferredVersions: Map = BaseInstallManager.collectPreferredVersions( + this.rushConfiguration, + this.options.variant + ); + + // Add any preferred versions to the top of the commonPackageJson + // do this in alphabetical order for simpler debugging + for (const dependency of Array.from(allPreferredVersions.keys()).sort()) { + commonPackageJson.dependencies![dependency] = allPreferredVersions.get(dependency)!; + } + // Example: "C:\MyRepo\common\temp\package.json" - const commonPackageJsonFilename: string = path.join(this.rushConfiguration.commonTempFolder, - FileConstants.PackageJson); + const commonPackageJsonFilename: string = path.join( + this.rushConfiguration.commonTempFolder, + FileConstants.PackageJson + ); - // Don't update the file timestamp unless the content has changed, since "rush install" - // will consider this timestamp - JsonFile.save(commonPackageJson, commonPackageJsonFilename, { onlyIfChanged: true }); + // Save the generated files. Don't update the file timestamp unless the content has changed, + // since "rush install" will consider this timestamp workspaceFile.save(workspaceFile.workspaceFilename, { onlyIfChanged: true }); + JsonFile.save(commonPackageJson, commonPackageJsonFilename, { onlyIfChanged: true }); stopwatch.stop(); console.log(`Finished creating workspace (${stopwatch.toString()})`); @@ -362,13 +391,21 @@ export class WorkspaceInstallManager extends BaseInstallManager { */ private _findOrphanedWorkspaceProjects(shrinkwrapFile: BaseShrinkwrapFile): boolean { - for (const workspacePath of shrinkwrapFile.getWorkspacePaths()) { - const projectPath: string = path.resolve(this.rushConfiguration.commonTempFolder, workspacePath); - if (!this.rushConfiguration.tryGetProjectForPath(projectPath)) { + for (const workspaceKey of shrinkwrapFile.getWorkspaceKeys()) { + + // Look for the RushConfigurationProject using the workspace key + let rushProjectPath: string; + if (this.rushConfiguration.packageManager === 'pnpm') { + // PNPM workspace keys are relative paths from the workspace root, which is the common temp folder + rushProjectPath = path.resolve(this.rushConfiguration.commonTempFolder, workspaceKey); + } else { + throw new InternalError('Orphaned workspaces cannot be checked for the provided package manager'); + } + + if (!this.rushConfiguration.tryGetProjectForPath(rushProjectPath)) { console.log(os.EOL + colors.yellow(Utilities.wrapWords( - `Your ${this.rushConfiguration.shrinkwrapFilePhrase} references a project at "${projectPath}" ` - + 'which no longer exists.')) - + os.EOL); + `Your ${this.rushConfiguration.shrinkwrapFilePhrase} references a project at "${rushProjectPath}" ` + + 'which no longer exists.')) + os.EOL); return true; // found one } } diff --git a/apps/rush-lib/src/logic/base/BaseShrinkwrapFile.ts b/apps/rush-lib/src/logic/base/BaseShrinkwrapFile.ts index 60f000513f3..d8d695b7ff0 100644 --- a/apps/rush-lib/src/logic/base/BaseShrinkwrapFile.ts +++ b/apps/rush-lib/src/logic/base/BaseShrinkwrapFile.ts @@ -8,7 +8,7 @@ import { PackageName, FileSystem } from '@rushstack/node-core-library'; import { RushConstants } from '../../logic/RushConstants'; import { DependencySpecifier } from '../DependencySpecifier'; import { IPolicyValidatorOptions } from '../policy/PolicyValidator'; -import { PackageManagerOptionsConfigurationBase, RushConfiguration } from '../../api/RushConfiguration'; +import { PackageManagerOptionsConfigurationBase } from '../../api/RushConfiguration'; /** * This class is a parser for both npm's npm-shrinkwrap.json and pnpm's pnpm-lock.yaml file formats. @@ -95,20 +95,6 @@ export abstract class BaseShrinkwrapFile { return this._checkDependencyVersion(dependencySpecifier, shrinkwrapDependency); } - public tryEnsureCompatibleWorkspaceDependency( - dependencySpecifier: DependencySpecifier, - projectName: string, - rushConfiguration: RushConfiguration - ): boolean { - const shrinkwrapDependency: DependencySpecifier | undefined = - this.tryEnsureWorkspaceDependencyVersion(dependencySpecifier, projectName, rushConfiguration); - if (!shrinkwrapDependency) { - return false; - } - - return this._checkDependencyVersion(dependencySpecifier, shrinkwrapDependency); - } - /** * Returns the list of temp projects defined in this file. * Example: [ '@rush-temp/project1', '@rush-temp/project2' ] @@ -117,29 +103,61 @@ export abstract class BaseShrinkwrapFile { */ public abstract getTempProjectNames(): ReadonlyArray; + /** @virtual */ + protected abstract tryEnsureDependencyVersion(dependencySpecifier: DependencySpecifier, + tempProjectName: string): DependencySpecifier | undefined; + + /** @virtual */ + protected abstract getTopLevelDependencyVersion(dependencyName: string): DependencySpecifier | undefined; + + /** + * Returns true if the specified workspace in the shrinkwrap file includes a package that would + * satisfy the specified SemVer version range. + * + * Consider this example: + * + * - project-a\ + * - lib-a@1.2.3 + * - lib-b@1.0.0 + * - lib-b@2.0.0 + * + * In this example, hasCompatibleWorkspaceDependency("lib-b", ">= 1.1.0", "workspace-key-for-project-a") + * would fail because it finds lib-b@1.0.0 which does not satisfy the pattern ">= 1.1.0". + * + * @virtual + */ + public hasCompatibleWorkspaceDependency(dependencySpecifier: DependencySpecifier, workspaceKey: string): boolean { + const shrinkwrapDependency: DependencySpecifier | undefined = this.getWorkspaceDependencyVersion( + dependencySpecifier, + workspaceKey + ); + return shrinkwrapDependency + ? this._checkDependencyVersion(dependencySpecifier, shrinkwrapDependency) + : false; + } + /** - * Returns the list of paths to Rush projects relative to the - * install root. + * Returns the list of keys to workspace projects specified in the shrinkwrap. * Example: [ '../../apps/project1', '../../apps/project2' ] * * @virtual */ - public abstract getWorkspacePaths(): ReadonlyArray; + public abstract getWorkspaceKeys(): ReadonlyArray; - /** @virtual */ - protected abstract tryEnsureDependencyVersion(dependencySpecifier: DependencySpecifier, - tempProjectName: string): DependencySpecifier | undefined; + /** + * Returns the key to the project in the workspace specified by the shrinkwrap. + * Example: '../../apps/project1' + * + * @virtual + */ + public abstract getWorkspaceKeyByPath(workspaceRoot: string, projectFolder: string): string /** @virtual */ - protected abstract tryEnsureWorkspaceDependencyVersion( + protected abstract getWorkspaceDependencyVersion( dependencySpecifier: DependencySpecifier, - projectName: string, - rushConfiguration: RushConfiguration + workspaceKey: string ): DependencySpecifier | undefined; - /** @virtual */ - protected abstract getTopLevelDependencyVersion(dependencyName: string): DependencySpecifier | undefined; - /** @virtual */ protected abstract serialize(): string; diff --git a/apps/rush-lib/src/logic/npm/NpmShrinkwrapFile.ts b/apps/rush-lib/src/logic/npm/NpmShrinkwrapFile.ts index 4de2f4379d9..2b6dc3a767d 100644 --- a/apps/rush-lib/src/logic/npm/NpmShrinkwrapFile.ts +++ b/apps/rush-lib/src/logic/npm/NpmShrinkwrapFile.ts @@ -10,7 +10,6 @@ import { BaseShrinkwrapFile } from '../base/BaseShrinkwrapFile'; import { DependencySpecifier } from '../DependencySpecifier'; -import { RushConfiguration } from '../../api/RushConfiguration'; interface INpmShrinkwrapDependencyJson { version: string; @@ -114,16 +113,20 @@ export class NpmShrinkwrapFile extends BaseShrinkwrapFile { } /** @override */ - protected tryEnsureWorkspaceDependencyVersion( - dependencySpecifier: DependencySpecifier, - projectName: string, - rushConfiguration: RushConfiguration - ): DependencySpecifier | undefined { + public getWorkspaceKeys(): ReadonlyArray { + throw new InternalError('Not implemented'); + } + + /** @override */ + public getWorkspaceKeyByPath(workspaceRoot: string, projectFolder: string): string { throw new InternalError('Not implemented'); } /** @override */ - public getWorkspacePaths(): ReadonlyArray { + protected getWorkspaceDependencyVersion( + dependencySpecifier: DependencySpecifier, + workspaceKey: string + ): DependencySpecifier | undefined { throw new InternalError('Not implemented'); } } \ No newline at end of file diff --git a/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts b/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts index c26fafee010..e2560963d7a 100644 --- a/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts +++ b/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts @@ -24,9 +24,9 @@ import { BasePackage } from '../base/BasePackage'; import { RushConstants } from '../../logic/RushConstants'; import { IRushLinkJson } from '../../api/RushConfiguration'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import { PnpmShrinkwrapFile, IPnpmShrinkwrapDependencyYaml } from './PnpmShrinkwrapFile'; +import { PnpmShrinkwrapFile, IPnpmShrinkwrapDependencyYaml, IPnpmShrinkwrapImporterYaml } from './PnpmShrinkwrapFile'; import { PnpmProjectDependencyManifest } from './PnpmProjectDependencyManifest'; -import { PackageJsonDependency } from '../../api/PackageJsonEditor'; +import { PackageJsonDependency, DependencyType } from '../../api/PackageJsonEditor'; // special flag for debugging, will print extra diagnostic information, // but comes with performance cost @@ -108,66 +108,70 @@ export class PnpmLinkManager extends BaseLinkManager { } } - // // We won't be using the package to actually create symlinks, this is just to bootstrap - // // the shrinkwrap-deps.json generation logic. - // const localPackage: BasePackage = BasePackage.createLinkedPackage( - // project.packageName, - // project.packageJsonEditor.version, - // project.projectFolder - // ); - - // // Iterate through all the regular dependencies - // const parentShrinkwrapEntry: IPnpmShrinkwrapDependencyYaml | undefined = - // pnpmShrinkwrapFile.getShrinkwrapEntryFromTempProjectDependencyKey(tempProjectDependencyKey); - // if (!parentShrinkwrapEntry) { - // throw new InternalError( - // 'Cannot find shrinkwrap entry using dependency key for temp project: ' + - // `${project.tempProjectName}`); - // } + const importerKey: string = pnpmShrinkwrapFile.getWorkspaceKeyByPath( + this._rushConfiguration.commonTempFolder, + project.projectFolder + ); + const workspaceImporter: IPnpmShrinkwrapImporterYaml | undefined = pnpmShrinkwrapFile.getWorkspaceImporter(importerKey); + if (!workspaceImporter) { + throw new InternalError(`Cannot find shrinkwrap entry using importer key for workspace project: ${importerKey}`); + } - // const pnpmProjectDependencyManifest: PnpmProjectDependencyManifest = new PnpmProjectDependencyManifest({ - // pnpmShrinkwrapFile, - // project - // }); - - // const dependencies: PackageJsonDependency[] = [ - // ...project.packageJsonEditor.dependencyList, - // ...project.packageJsonEditor.devDependencyList - // ].filter(x => !x.version.startsWith('workspace:')); - - // for (const { name, dependencyType } of dependencies) { - - // // read the version number from the shrinkwrap entry - // const isOptional: boolean = dependencyType === DependencyType.Optional; - // const version: string | undefined = isOptional - // ? (parentShrinkwrapEntry.optionalDependencies || {})[name] - // : (parentShrinkwrapEntry.dependencies || {})[name]; - // if (!version) { - // if (!isOptional) { - // throw new InternalError( - // `Cannot find shrinkwrap entry dependency "${name}" for workspace project: ` + - // `${project.packageName}`); - // } - // continue; - // } + const pnpmProjectDependencyManifest: PnpmProjectDependencyManifest = new PnpmProjectDependencyManifest({ + pnpmShrinkwrapFile, + project + }); - // const newLocalFolderPath: string = path.join(localPackage.folderPath, 'node_modules', name); - // const newLocalPackage: BasePackage = BasePackage.createLinkedPackage( - // name, - // version, - // newLocalFolderPath - // ); + // Dev dependen + const dependencies: PackageJsonDependency[] = [ + ...project.packageJsonEditor.dependencyList, + ...project.packageJsonEditor.devDependencyList + ].filter(x => !x.version.startsWith('workspace:')); + + for (const { name, dependencyType } of dependencies) { + // read the version number from the shrinkwrap entry + let version: string | undefined; + switch (dependencyType) { + case DependencyType.Regular: + version = (workspaceImporter.dependencies || {})[name]; + break; + case DependencyType.Dev: + version = (workspaceImporter.devDependencies || {})[name]; + break; + case DependencyType.Optional: + version = (workspaceImporter.optionalDependencies || {})[name]; + break; + case DependencyType.Peer: + // Peer dependencies do not need to be considered + continue; + } - // if (!this._rushConfiguration.experimentsConfiguration.configuration.legacyIncrementalBuildDependencyDetection) { - // pnpmProjectDependencyManifest.addDependency(newLocalPackage, parentShrinkwrapEntry); - // } - // } + if (!version) { + if (dependencyType !== DependencyType.Optional) { + throw new InternalError( + `Cannot find shrinkwrap entry dependency "${name}" for workspace project: ${project.packageName}` + ); + } + continue; + } - // if (!this._rushConfiguration.experimentsConfiguration.configuration.legacyIncrementalBuildDependencyDetection) { - // pnpmProjectDependencyManifest.save(); - // } else { - // pnpmProjectDependencyManifest.deleteIfExists(); - // } + if (!this._rushConfiguration.experimentsConfiguration.configuration.legacyIncrementalBuildDependencyDetection) { + pnpmProjectDependencyManifest.addDependency( + name, + version, + { + ...(workspaceImporter.optionalDependencies || {}), + ...(workspaceImporter.dependencies || {}), + ...(workspaceImporter.devDependencies || {}) + }); + } + } + + if (!this._rushConfiguration.experimentsConfiguration.configuration.legacyIncrementalBuildDependencyDetection) { + pnpmProjectDependencyManifest.save(); + } else { + pnpmProjectDependencyManifest.deleteIfExists(); + } } /** @@ -425,7 +429,8 @@ export class PnpmLinkManager extends BaseLinkManager { throw new InternalError(`Dependency "${dependencyName}" is not a symlink in "${pathToLocalInstallation}`); } - // read the version number from the shrinkwrap entry + // read the version number from the shrinkwrap entry and return if no version is specified + // and the dependency is optional const version: string | undefined = isOptional ? (parentShrinkwrapEntry.optionalDependencies || {})[dependencyName] : (parentShrinkwrapEntry.dependencies || {})[dependencyName]; @@ -450,7 +455,14 @@ export class PnpmLinkManager extends BaseLinkManager { newLocalPackage.symlinkTargetFolderPath = FileSystem.getRealPath(dependencyLocalInstallationSymlink); if (!this._rushConfiguration.experimentsConfiguration.configuration.legacyIncrementalBuildDependencyDetection) { - pnpmProjectDependencyManifest.addDependency(newLocalPackage, parentShrinkwrapEntry); + pnpmProjectDependencyManifest.addDependency( + newLocalPackage.name, + newLocalPackage.version!, + { + ...(parentShrinkwrapEntry.optionalDependencies || {}), + ...(parentShrinkwrapEntry.dependencies || {}) + } + ); } return newLocalPackage; diff --git a/apps/rush-lib/src/logic/pnpm/PnpmProjectDependencyManifest.ts b/apps/rush-lib/src/logic/pnpm/PnpmProjectDependencyManifest.ts index f3c8d644d89..00d02fe72df 100644 --- a/apps/rush-lib/src/logic/pnpm/PnpmProjectDependencyManifest.ts +++ b/apps/rush-lib/src/logic/pnpm/PnpmProjectDependencyManifest.ts @@ -16,7 +16,6 @@ import { } from './PnpmShrinkwrapFile'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { RushConstants } from '../RushConstants'; -import { BasePackage } from '../base/BasePackage'; import { DependencySpecifier } from '../DependencySpecifier'; export interface IPnpmProjectDependencyManifestOptions { @@ -63,12 +62,8 @@ export class PnpmProjectDependencyManifest { ); } - public addDependency(pkg: BasePackage, parentShrinkwrapEntry: IPnpmShrinkwrapDependencyYaml): void { - if (!pkg.version) { - throw new InternalError(`Version missing from dependency ${pkg.name}`); - } - - this._addDependencyInternal(pkg.name, pkg.version, parentShrinkwrapEntry); + public addDependency(name: string, version: string, parentDependencies: { [dependency: string]: string }): void { + this._addDependencyInternal(name, version, parentDependencies); } /** @@ -98,7 +93,7 @@ export class PnpmProjectDependencyManifest { private _addDependencyInternal( name: string, version: string, - parentShrinkwrapEntry: IPnpmShrinkwrapDependencyYaml, + parentDependencies: { [dependency: string]: string }, throwIfShrinkwrapEntryMissing: boolean = true ): void { const shrinkwrapEntry: IPnpmShrinkwrapDependencyYaml | undefined = this._pnpmShrinkwrapFile.getShrinkwrapEntry( @@ -125,11 +120,17 @@ export class PnpmProjectDependencyManifest { // Add the current dependency this._projectDependencyManifestFile.set(specifier, integrity); + // Collect the shrinkwrap dependencies + const shrinkwrapDependencies: { [dependency: string]: string } = { + ...(shrinkwrapEntry.optionalDependencies || {}), + ...(shrinkwrapEntry.dependencies || {}) + } + // Add the dependencies of the dependency for (const dependencyName in shrinkwrapEntry.dependencies) { if (shrinkwrapEntry.dependencies.hasOwnProperty(dependencyName)) { const dependencyVersion: string = shrinkwrapEntry.dependencies[dependencyName]; - this._addDependencyInternal(dependencyName, dependencyVersion, shrinkwrapEntry); + this._addDependencyInternal(dependencyName, dependencyVersion, shrinkwrapDependencies); } } @@ -141,7 +142,7 @@ export class PnpmProjectDependencyManifest { this._addDependencyInternal( optionalDependencyName, dependencyVersion, - shrinkwrapEntry, + shrinkwrapDependencies, throwIfShrinkwrapEntryMissing = false); } } @@ -171,13 +172,10 @@ export class PnpmProjectDependencyManifest { } // If not, check the parent. - if ( - parentShrinkwrapEntry.dependencies && - parentShrinkwrapEntry.dependencies.hasOwnProperty(peerDependencyName) - ) { + if (parentDependencies.hasOwnProperty(peerDependencyName)) { const dependencySpecifier: DependencySpecifier | undefined = parsePnpmDependencyKey( peerDependencyName, - parentShrinkwrapEntry.dependencies[peerDependencyName] + parentDependencies[peerDependencyName] ); if (dependencySpecifier) { if (!semver.valid(dependencySpecifier.versionSpecifier)) { @@ -204,7 +202,7 @@ export class PnpmProjectDependencyManifest { this._addDependencyInternal( peerDependencyName, peerDependencyKeys[peerDependencyName], - shrinkwrapEntry + shrinkwrapDependencies ); continue; } @@ -231,7 +229,7 @@ export class PnpmProjectDependencyManifest { this._addDependencyInternal( peerDependencyName, this._pnpmShrinkwrapFile.getTopLevelDependencyKey(peerDependencyName)!, - shrinkwrapEntry + shrinkwrapDependencies ); } } diff --git a/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts b/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts index 6f5094e353e..0b1de2a115b 100644 --- a/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts +++ b/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts @@ -8,7 +8,7 @@ import { FileSystem } from '@rushstack/node-core-library'; import { BaseShrinkwrapFile } from '../base/BaseShrinkwrapFile'; import { DependencySpecifier } from '../DependencySpecifier'; -import { PackageManagerOptionsConfigurationBase, PnpmOptionsConfiguration, RushConfiguration } from '../../api/RushConfiguration'; +import { PackageManagerOptionsConfigurationBase, PnpmOptionsConfiguration } from '../../api/RushConfiguration'; import { IPolicyValidatorOptions } from '../policy/PolicyValidator'; import { AlreadyReportedError } from '../../utilities/AlreadyReportedError'; @@ -51,7 +51,9 @@ export interface IPnpmShrinkwrapImporterYaml { dependencies: { [dependency: string]: string } /** The list of resolved version numbers for dev dependencies */ devDependencies: { [dependency: string]: string } - /** The list of specifiers used to resolve direct dependency versions */ + /** The list of resolved version numbers for optional dependencies */ + optionalDependencies: { [dependency: string]: string } + /** The list of specifiers used to resolve dependency versions */ specifiers: { [dependency: string]: string } } @@ -86,7 +88,7 @@ export interface IPnpmShrinkwrapImporterYaml { * } * } */ -interface IPnpmShrinkwrapYaml extends IPnpmShrinkwrapImporterYaml { +interface IPnpmShrinkwrapYaml { /** The list of resolved version numbers for direct dependencies */ dependencies: { [dependency: string]: string } /** The list of importers for local workspace projects */ @@ -206,6 +208,9 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { if (!this._shrinkwrapJson.packages) { this._shrinkwrapJson.packages = { }; } + if (!this._shrinkwrapJson.importers) { + this._shrinkwrapJson.importers = { }; + } } public static loadFromFile( @@ -290,19 +295,6 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { return this._getTempProjectNames(this._shrinkwrapJson.dependencies); } - /** @override */ - public getWorkspacePaths(): ReadonlyArray { - const result: string[] = []; - for (const key of Object.keys(this._shrinkwrapJson.importers)) { - // If it starts with @rush-temp, then include it: - if (key !== '.') { - result.push(key); - } - } - result.sort(); // make the result deterministic - return result; - } - /** * Gets the path to the tarball file if the package is a tarball. * Returns undefined if the package entry doesn't exist or the package isn't a tarball. @@ -319,14 +311,7 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { } public getTopLevelDependencyKey(dependencyName: string): string | undefined { - // For workspaces, top-level dependencies aren't populated, so instead we use the root workspace - // for common package versions and consider this the 'top level' - const dependenciesToCheck: { [dependency: string]: string } = ( - this._shrinkwrapJson.importers && this._shrinkwrapJson.importers.hasOwnProperty('.') - ) ? this._shrinkwrapJson.importers['.'].dependencies - : this._shrinkwrapJson.dependencies; - - return BaseShrinkwrapFile.tryGetValue(dependenciesToCheck, dependencyName); + return BaseShrinkwrapFile.tryGetValue(this._shrinkwrapJson.dependencies, dependencyName); } /** @@ -340,14 +325,7 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { * @override */ public getTopLevelDependencyVersion(dependencyName: string): DependencySpecifier | undefined { - // For workspaces, top-level dependencies aren't populated, so instead we use the root workspace - // for common package versions and consider this the 'top level' - const dependenciesToCheck: { [dependency: string]: string } = ( - this._shrinkwrapJson.importers && this._shrinkwrapJson.importers.hasOwnProperty('.') - ) ? this._shrinkwrapJson.importers['.'].dependencies - : this._shrinkwrapJson.dependencies; - - let value: string | undefined = BaseShrinkwrapFile.tryGetValue(dependenciesToCheck, dependencyName); + let value: string | undefined = BaseShrinkwrapFile.tryGetValue(this._shrinkwrapJson.dependencies, dependencyName); if (value) { // Getting the top level dependency version from a PNPM lockfile version 5.1 @@ -423,13 +401,6 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { return undefined; } - public getImporter(importerPath: string): IPnpmShrinkwrapImporterYaml | undefined { - const importer: IPnpmShrinkwrapImporterYaml | undefined = - BaseShrinkwrapFile.tryGetValue(this._shrinkwrapJson.importers, importerPath); - - return importer && importer.dependencies ? importer : undefined; - } - public getShrinkwrapEntryFromTempProjectDependencyKey( tempProjectDependencyKey: string ): IPnpmShrinkwrapDependencyYaml | undefined { @@ -530,6 +501,28 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { return this._parsePnpmDependencyKey(packageName, dependencyKey); } + /** @override */ + public getWorkspaceKeys(): ReadonlyArray { + const result: string[] = []; + for (const key of Object.keys(this._shrinkwrapJson.importers)) { + // Avoid including the common workspace + if (key !== '.') { + result.push(key); + } + } + result.sort(); // make the result deterministic + return result; + } + + /** @override */ + public getWorkspaceKeyByPath(workspaceRoot: string, projectFolder: string): string { + return path.relative(workspaceRoot, projectFolder).replace(new RegExp(`\\${path.sep}`, 'g'), '/'); + } + + public getWorkspaceImporter(importerPath: string): IPnpmShrinkwrapImporterYaml | undefined { + return BaseShrinkwrapFile.tryGetValue(this._shrinkwrapJson.importers, importerPath); + } + /** * Gets the resolved version number of a dependency for a specific temp project. * For PNPM, we can reuse the version that another project is using. @@ -537,10 +530,9 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { * * @override */ - protected tryEnsureWorkspaceDependencyVersion( + protected getWorkspaceDependencyVersion( dependencySpecifier: DependencySpecifier, - projectName: string, - rushConfiguration: RushConfiguration + workspaceKey: string ): DependencySpecifier | undefined { // PNPM doesn't have the same advantage of NPM, where we can skip generate as long as the @@ -552,18 +544,13 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { // linked to. const packageName: string = dependencySpecifier.packageName; - - // Importer paths use forward slashes - const importerPath: string = path.relative( - rushConfiguration.commonTempFolder, - rushConfiguration.getProjectByName(projectName)!.projectFolder - ).replace(new RegExp(`\\${path.sep}`, 'g'), '/'); - const projectImporter: IPnpmShrinkwrapImporterYaml | undefined = this.getImporter(importerPath); + const projectImporter: IPnpmShrinkwrapImporterYaml | undefined = this.getWorkspaceImporter(workspaceKey); if (!projectImporter) { return undefined; } const allDependencies: { [dependency: string]: string } = { + ...(projectImporter.optionalDependencies || {}), ...(projectImporter.dependencies || {}), ...(projectImporter.devDependencies || {}) } diff --git a/apps/rush-lib/src/logic/yarn/YarnShrinkwrapFile.ts b/apps/rush-lib/src/logic/yarn/YarnShrinkwrapFile.ts index 1716f936cc8..b8be508aad8 100644 --- a/apps/rush-lib/src/logic/yarn/YarnShrinkwrapFile.ts +++ b/apps/rush-lib/src/logic/yarn/YarnShrinkwrapFile.ts @@ -1,12 +1,13 @@ import * as os from 'os'; import * as lockfile from '@yarnpkg/lockfile'; + import { BaseShrinkwrapFile } from '../base/BaseShrinkwrapFile'; import { FileSystem, PackageName, IParsedPackageNameOrError, InternalError } from '@rushstack/node-core-library'; import { RushConstants } from '../RushConstants'; import { DependencySpecifier } from '../DependencySpecifier'; -import { RushConfiguration } from '../../api/RushConfiguration'; +// import { RushConfigurationProject } from '../../api/RushConfigurationProject'; /** * Used with YarnShrinkwrapFile._encodePackageNameAndSemVer() and _decodePackageNameAndSemVer(). @@ -196,11 +197,6 @@ export class YarnShrinkwrapFile extends BaseShrinkwrapFile { return this._tempProjectNames; } - /** @override */ - public getWorkspacePaths(): ReadonlyArray { - throw new InternalError('Not implemented'); - } - /** @override */ public hasCompatibleTopLevelDependency(dependencySpecifier: DependencySpecifier): boolean { // It seems like we should normalize the key somehow, but Yarn apparently does not @@ -236,10 +232,20 @@ export class YarnShrinkwrapFile extends BaseShrinkwrapFile { throw new InternalError('Not implemented'); } - protected tryEnsureWorkspaceDependencyVersion( + /** @override */ + public getWorkspaceKeys(): ReadonlyArray { + throw new InternalError('Not implemented'); + } + + /** @override */ + public getWorkspaceKeyByPath(workspaceRoot: string, projectFolder: string): string { + throw new InternalError('Not implemented'); + } + + /** @override */ + protected getWorkspaceDependencyVersion( dependencySpecifier: DependencySpecifier, - projectName: string, - rushConfiguration: RushConfiguration + workspaceKey: string ): DependencySpecifier | undefined { throw new InternalError('Not implemented'); } From 2780ef9f241b4d6f51fdbd389c22e4fa1c69f324 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Fri, 22 May 2020 15:47:38 -0700 Subject: [PATCH 05/16] Add --to and --from support to rush install --- .../src/cli/actions/BaseRushAction.ts | 54 ++++++++++++++++- .../rush-lib/src/cli/actions/InstallAction.ts | 40 ++++++++++++- apps/rush-lib/src/cli/actions/UpdateAction.ts | 4 +- .../src/cli/scriptActions/BulkScriptAction.ts | 59 +------------------ apps/rush-lib/src/logic/PackageJsonUpdater.ts | 4 +- .../src/logic/WorkspaceInstallManager.ts | 28 ++++++++- .../src/logic/base/BaseInstallManager.ts | 10 ++++ 7 files changed, 136 insertions(+), 63 deletions(-) diff --git a/apps/rush-lib/src/cli/actions/BaseRushAction.ts b/apps/rush-lib/src/cli/actions/BaseRushAction.ts index 6618bd64469..a3f31fe734e 100644 --- a/apps/rush-lib/src/cli/actions/BaseRushAction.ts +++ b/apps/rush-lib/src/cli/actions/BaseRushAction.ts @@ -7,16 +7,19 @@ import * as path from 'path'; import { CommandLineAction, + CommandLineStringListParameter, ICommandLineActionOptions } from '@rushstack/ts-command-line'; import { LockFile } from '@rushstack/node-core-library'; -import { RushConfiguration } from '../../api/RushConfiguration'; +import { AlreadyReportedError } from '../../utilities/AlreadyReportedError'; import { EventHooksManager } from '../../logic/EventHooksManager'; +import { PackageJsonLookup, IPackageJson } from '@rushstack/node-core-library'; import { RushCommandLineParser } from './../RushCommandLineParser'; +import { RushConfiguration } from '../../api/RushConfiguration'; +import { RushGlobalFolder } from '../../api/RushGlobalFolder' import { Utilities } from '../../utilities/Utilities'; -import { RushGlobalFolder } from '../../api/RushGlobalFolder'; export interface IBaseRushActionOptions extends ICommandLineActionOptions { /** @@ -120,4 +123,51 @@ export abstract class BaseRushAction extends BaseConfiglessRushAction { return this._eventHooksManager; } + + protected mergeProjectsWithVersionPolicy( + projectsParameters: CommandLineStringListParameter, + versionPoliciesParameters: CommandLineStringListParameter + ): string[] { + const packageJsonLookup: PackageJsonLookup = new PackageJsonLookup(); + + const projects: string[] = []; + for (const projectParameter of projectsParameters.values) { + if (projectParameter === '.') { + const packageJson: IPackageJson | undefined = packageJsonLookup.tryLoadPackageJsonFor(process.cwd()); + if (packageJson) { + const projectName: string = packageJson.name; + if (this.rushConfiguration.projectsByName.has(projectName)) { + projects.push(projectName); + } else { + console.log(colors.red( + 'Rush is not currently running in a project directory specified in rush.json. ' + + `The "." value for the ${projectsParameters.longName} parameter is not allowed.` + )); + throw new AlreadyReportedError(); + } + } else { + console.log(colors.red( + 'Rush is not currently running in a project directory. ' + + `The "." value for the ${projectsParameters.longName} parameter is not allowed.` + )); + throw new AlreadyReportedError(); + } + } else { + projects.push(projectParameter); + } + } + + if (versionPoliciesParameters.values && versionPoliciesParameters.values.length > 0) { + this.rushConfiguration.projects.forEach(project => { + const matches: boolean = versionPoliciesParameters.values.some(policyName => { + return project.versionPolicyName === policyName; + }); + if (matches) { + projects.push(project.packageName); + } + }); + } + + return projects; + } } diff --git a/apps/rush-lib/src/cli/actions/InstallAction.ts b/apps/rush-lib/src/cli/actions/InstallAction.ts index 399c4d20077..51ed463fc6b 100644 --- a/apps/rush-lib/src/cli/actions/InstallAction.ts +++ b/apps/rush-lib/src/cli/actions/InstallAction.ts @@ -4,8 +4,14 @@ import { BaseInstallAction } from './BaseInstallAction'; import { IInstallManagerOptions } from '../../logic/base/BaseInstallManager'; import { RushCommandLineParser } from '../RushCommandLineParser'; +import { CommandLineStringListParameter } from '@rushstack/ts-command-line'; export class InstallAction extends BaseInstallAction { + protected _fromFlag: CommandLineStringListParameter; + protected _toFlag: CommandLineStringListParameter; + protected _fromVersionPolicy: CommandLineStringListParameter; + protected _toVersionPolicy: CommandLineStringListParameter; + public constructor(parser: RushCommandLineParser) { super({ actionName: 'install', @@ -24,6 +30,36 @@ export class InstallAction extends BaseInstallAction { }); } + protected onDefineParameters(): void { + super.onDefineParameters(); + this._toFlag = this.defineStringListParameter({ + parameterLongName: '--to', + parameterShortName: '-t', + argumentName: 'PROJECT1', + description: 'Run install in the specified project and all of its dependencies. "." can be used as shorthand ' + + 'to specify the project in the current working directory.' + }); + this._fromVersionPolicy = this.defineStringListParameter({ + parameterLongName: '--from-version-policy', + argumentName: 'VERSION_POLICY_NAME', + description: 'Run install in all projects with the specified version policy ' + + 'and all projects that directly or indirectly depend on projects with the specified version policy' + }); + this._toVersionPolicy = this.defineStringListParameter({ + parameterLongName: '--to-version-policy', + argumentName: 'VERSION_POLICY_NAME', + description: 'Run install in all projects with the specified version policy and all of their dependencies' + }); + this._fromFlag = this.defineStringListParameter({ + parameterLongName: '--from', + parameterShortName: '-f', + argumentName: 'PROJECT2', + description: 'Run install in all projects that directly or indirectly depend on the specified project. ' + + '"." can be used as shorthand to specify the project in the current working directory.' + }); + + } + protected buildInstallOptions(): IInstallManagerOptions { return { debug: this.parser.isDebug, @@ -37,7 +73,9 @@ export class InstallAction extends BaseInstallAction { variant: this._variant.value, // Because the 'defaultValue' option on the _maxInstallAttempts parameter is set, // it is safe to assume that the value is not null - maxInstallAttempts: this._maxInstallAttempts.value! + maxInstallAttempts: this._maxInstallAttempts.value!, + toFlags: this.mergeProjectsWithVersionPolicy(this._toFlag, this._toVersionPolicy), + fromFlags: this.mergeProjectsWithVersionPolicy(this._fromFlag, this._fromVersionPolicy) }; } } diff --git a/apps/rush-lib/src/cli/actions/UpdateAction.ts b/apps/rush-lib/src/cli/actions/UpdateAction.ts index e4b9d5121af..42ea4844f60 100644 --- a/apps/rush-lib/src/cli/actions/UpdateAction.ts +++ b/apps/rush-lib/src/cli/actions/UpdateAction.ts @@ -65,7 +65,9 @@ export class UpdateAction extends BaseInstallAction { variant: this._variant.value, // Because the 'defaultValue' option on the _maxInstallAttempts parameter is set, // it is safe to assume that the value is not null - maxInstallAttempts: this._maxInstallAttempts.value! + maxInstallAttempts: this._maxInstallAttempts.value!, + toFlags: [], // Do not support partial updates + fromFlags: [] // Do not support partial updates }; } } diff --git a/apps/rush-lib/src/cli/scriptActions/BulkScriptAction.ts b/apps/rush-lib/src/cli/scriptActions/BulkScriptAction.ts index bcc32af5744..c9f855f4d5d 100644 --- a/apps/rush-lib/src/cli/scriptActions/BulkScriptAction.ts +++ b/apps/rush-lib/src/cli/scriptActions/BulkScriptAction.ts @@ -10,11 +10,7 @@ import { CommandLineStringListParameter, CommandLineParameterKind } from '@rushstack/ts-command-line'; -import { - FileSystem, - PackageJsonLookup, - IPackageJson -} from '@rushstack/node-core-library'; +import { FileSystem } from '@rushstack/node-core-library'; import { Event } from '../../index'; import { SetupChecks } from '../../logic/SetupChecks'; @@ -105,8 +101,8 @@ export class BulkScriptAction extends BaseScriptAction { const taskSelector: TaskSelector = new TaskSelector({ rushConfiguration: this.rushConfiguration, - toFlags: this._mergeProjectsWithVersionPolicy(this._toFlag, this._toVersionPolicy), - fromFlags: this._mergeProjectsWithVersionPolicy(this._fromFlag, this._fromVersionPolicy), + toFlags: this.mergeProjectsWithVersionPolicy(this._toFlag, this._toVersionPolicy), + fromFlags: this.mergeProjectsWithVersionPolicy(this._fromFlag, this._fromVersionPolicy), commandToRun: this._commandToRun, customParameterValues, isQuietMode: isQuietMode, @@ -203,55 +199,6 @@ export class BulkScriptAction extends BaseScriptAction { this.defineScriptParameters(); } - private _mergeProjectsWithVersionPolicy( - projectsParameters: CommandLineStringListParameter, - versionPoliciesParameters: CommandLineStringListParameter - ): string[] { - const packageJsonLookup: PackageJsonLookup = new PackageJsonLookup(); - - const projects: string[] = []; - for (const projectParameter of projectsParameters.values) { - if (projectParameter === '.') { - const packageJson: IPackageJson | undefined = packageJsonLookup.tryLoadPackageJsonFor(process.cwd()); - if (packageJson) { - const projectName: string = packageJson.name; - if (this.rushConfiguration.projectsByName.has(projectName)) { - projects.push(projectName); - } else { - console.log(colors.red( - 'Rush is not currently running in a project directory specified in rush.json. ' + - `The "." value for the ${this._toFlag.longName} parameter or the ${this._fromFlag.longName} parameter ` + - 'is not allowed.' - )); - throw new AlreadyReportedError(); - } - } else { - console.log(colors.red( - 'Rush is not currently running in a project directory. ' + - `The "." value for the ${this._toFlag.longName} parameter or the ${this._fromFlag.longName} parameter ` + - 'is not allowed.' - )); - throw new AlreadyReportedError(); - } - } else { - projects.push(projectParameter); - } - } - - if (versionPoliciesParameters.values && versionPoliciesParameters.values.length > 0) { - this.rushConfiguration.projects.forEach(project => { - const matches: boolean = versionPoliciesParameters.values.some(policyName => { - return project.versionPolicyName === policyName; - }); - if (matches) { - projects.push(project.packageName); - } - }); - } - - return projects; - } - private _doBeforeTask(): void { if (this.actionName !== RushConstants.buildCommandName && this.actionName !== RushConstants.rebuildCommandName) { // Only collects information for built-in tasks like build or rebuild. diff --git a/apps/rush-lib/src/logic/PackageJsonUpdater.ts b/apps/rush-lib/src/logic/PackageJsonUpdater.ts index d42e6ab7346..e0ae0247ef1 100644 --- a/apps/rush-lib/src/logic/PackageJsonUpdater.ts +++ b/apps/rush-lib/src/logic/PackageJsonUpdater.ts @@ -144,7 +144,9 @@ export class PackageJsonUpdater { networkConcurrency: undefined, collectLogFile: false, variant: variant, - maxInstallAttempts: RushConstants.defaultMaxInstallAttempts + maxInstallAttempts: RushConstants.defaultMaxInstallAttempts, + toFlags: [], + fromFlags: [] }; const installManager: BaseInstallManager = InstallManagerFactory.getInstallManager( this._rushConfiguration, diff --git a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts index e02834cdfc5..c3fabe89e0f 100644 --- a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts +++ b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts @@ -94,7 +94,7 @@ export class WorkspaceInstallManager extends BaseInstallManager { + 'the shrinkwrap file.' )); throw new AlreadyReportedError(); - } + } } // dependency name --> version specifier @@ -180,7 +180,7 @@ export class WorkspaceInstallManager extends BaseInstallManager { } else if (dependencySpecifier.specifierType === 'workspace') { // Already specified as a local project. Allow the package manager to validate this continue; - } + } // PNPM does not specify peer dependencies for workspaces in the shrinkwrap, so skip validating these if (this.rushConfiguration.packageManager === 'pnpm' && dependencyType === DependencyType.Peer) { @@ -380,7 +380,31 @@ export class WorkspaceInstallManager extends BaseInstallManager { if (this.rushConfiguration.packageManager === 'pnpm') { args.push('--recursive'); args.push('--link-workspace-packages', 'false'); + + const toPackages: Set = this._findPackageNames(options.toFlags); + for (const toPackage of toPackages) { + args.push('--filter', `${toPackage}...`); + } + + const fromPackages: Set = this._findPackageNames(options.fromFlags); + for (const fromPackage of fromPackages) { + args.push('--filter', `...${fromPackage}`); + } + } + } + + private _findPackageNames(shorthandNames: ReadonlyArray): Set { + + const dependencies: Set = new Set(); + for (const name of shorthandNames) { + const project: RushConfigurationProject | undefined = + this.rushConfiguration.findProjectByShorthandName(name); + if (!project) { + throw new Error(`The project '${name}' does not exist in rush.json`); + } + dependencies.add(project.packageName); } + return dependencies; } /** diff --git a/apps/rush-lib/src/logic/base/BaseInstallManager.ts b/apps/rush-lib/src/logic/base/BaseInstallManager.ts index a52e63a15fe..76c80b63aea 100644 --- a/apps/rush-lib/src/logic/base/BaseInstallManager.ts +++ b/apps/rush-lib/src/logic/base/BaseInstallManager.ts @@ -101,6 +101,16 @@ export interface IInstallManagerOptions { * Retry the install the specified number of times */ maxInstallAttempts: number + + /** + * Run install in the specified projects as well as all dependencies + */ + toFlags: ReadonlyArray; + + /** + * Run install in the specified projects as well as all dependent projects + */ + fromFlags: ReadonlyArray; } /** From 3f741a34a90b9f78441157390ab01e95780225a7 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Fri, 22 May 2020 16:56:36 -0700 Subject: [PATCH 06/16] Missed changes --- apps/rush-lib/src/logic/RushInstallManager.ts | 2 +- .../src/logic/WorkspaceInstallManager.ts | 2 +- .../src/logic/base/BaseInstallManager.ts | 29 +++++++++++++++++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/apps/rush-lib/src/logic/RushInstallManager.ts b/apps/rush-lib/src/logic/RushInstallManager.ts index 84747859a1a..452c3fa62c9 100644 --- a/apps/rush-lib/src/logic/RushInstallManager.ts +++ b/apps/rush-lib/src/logic/RushInstallManager.ts @@ -347,7 +347,7 @@ export class RushInstallManager extends BaseInstallManager { // then we can't skip this install potentiallyChangedFiles.push(this.rushConfiguration.getCommittedShrinkwrapFilename(this.options.variant)); - // Add common-versions.json file in potentially changed file list. + // Add common-versions.json file to the potentially changed files list. potentiallyChangedFiles.push(this.rushConfiguration.getCommonVersionsFilePath(this.options.variant)); if (this.rushConfiguration.packageManager === 'pnpm') { diff --git a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts index c3fabe89e0f..7b8f5ef2575 100644 --- a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts +++ b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts @@ -261,7 +261,7 @@ export class WorkspaceInstallManager extends BaseInstallManager { // then we can't skip this install potentiallyChangedFiles.push(this.rushConfiguration.getCommittedShrinkwrapFilename(this.options.variant)); - // Add common-versions.json file in potentially changed file list. + // Add common-versions.json file to the potentially changed files list. potentiallyChangedFiles.push(this.rushConfiguration.getCommonVersionsFilePath(this.options.variant)); if (this.rushConfiguration.packageManager === 'pnpm') { diff --git a/apps/rush-lib/src/logic/base/BaseInstallManager.ts b/apps/rush-lib/src/logic/base/BaseInstallManager.ts index ef509c0f07a..cf98377fad8 100644 --- a/apps/rush-lib/src/logic/base/BaseInstallManager.ts +++ b/apps/rush-lib/src/logic/base/BaseInstallManager.ts @@ -238,6 +238,9 @@ export abstract class BaseInstallManager { protected abstract canSkipInstall(lastInstallDate: Date): boolean; + /** + * Runs "npm/pnpm/yarn install" in the "common/temp" folder. + */ protected abstract install(cleanInstall: boolean): Promise; protected async prepare(): Promise<{ variantIsUpToDate: boolean, shrinkwrapIsUpToDate: boolean }> { @@ -606,10 +609,30 @@ export abstract class BaseInstallManager { // Ensure that Rush's tarball dependencies get synchronized properly with the pnpm-lock.yaml file. // See this GitHub issue: https://github.com/pnpm/pnpm/issues/1342 - if (semver.gte(this._rushConfiguration.packageManagerToolVersion, '3.0.0')) { - args.push('--no-prefer-frozen-lockfile'); + if (this.rushConfiguration.experimentsConfiguration.configuration.usePnpmFrozenLockfileForRushInstall) { + if (!this.options.allowShrinkwrapUpdates) { + if (semver.gte(this._rushConfiguration.packageManagerToolVersion, '3.0.0')) { + args.push('--frozen-lockfile'); + } else { + args.push('--frozen-shrinkwrap'); + } + } else { + // Ensure that Rush's tarball dependencies get synchronized properly with the pnpm-lock.yaml file. + // See this GitHub issue: https://github.com/pnpm/pnpm/issues/1342 + if (semver.gte(this._rushConfiguration.packageManagerToolVersion, '3.0.0')) { + args.push('--no-prefer-frozen-lockfile'); + } else { + args.push('--no-prefer-frozen-shrinkwrap'); + } + } } else { - args.push('--no-prefer-frozen-shrinkwrap'); + // Ensure that Rush's tarball dependencies get synchronized properly with the pnpm-lock.yaml file. + // See this GitHub issue: https://github.com/pnpm/pnpm/issues/1342 + if (semver.gte(this._rushConfiguration.packageManagerToolVersion, '3.0.0')) { + args.push('--no-prefer-frozen-lockfile'); + } else { + args.push('--no-prefer-frozen-shrinkwrap'); + } } if (options.collectLogFile) { From ec792d6f96b58437525fb2893e18942e17d618df Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Fri, 22 May 2020 16:57:50 -0700 Subject: [PATCH 07/16] Revert "Add --to and --from support to rush install" This reverts commit 2780ef9f241b4d6f51fdbd389c22e4fa1c69f324. --- .../src/cli/actions/BaseRushAction.ts | 54 +---------------- .../rush-lib/src/cli/actions/InstallAction.ts | 40 +------------ apps/rush-lib/src/cli/actions/UpdateAction.ts | 4 +- .../src/cli/scriptActions/BulkScriptAction.ts | 59 ++++++++++++++++++- apps/rush-lib/src/logic/PackageJsonUpdater.ts | 4 +- .../src/logic/WorkspaceInstallManager.ts | 28 +-------- .../src/logic/base/BaseInstallManager.ts | 10 ---- 7 files changed, 63 insertions(+), 136 deletions(-) diff --git a/apps/rush-lib/src/cli/actions/BaseRushAction.ts b/apps/rush-lib/src/cli/actions/BaseRushAction.ts index a3f31fe734e..6618bd64469 100644 --- a/apps/rush-lib/src/cli/actions/BaseRushAction.ts +++ b/apps/rush-lib/src/cli/actions/BaseRushAction.ts @@ -7,19 +7,16 @@ import * as path from 'path'; import { CommandLineAction, - CommandLineStringListParameter, ICommandLineActionOptions } from '@rushstack/ts-command-line'; import { LockFile } from '@rushstack/node-core-library'; -import { AlreadyReportedError } from '../../utilities/AlreadyReportedError'; +import { RushConfiguration } from '../../api/RushConfiguration'; import { EventHooksManager } from '../../logic/EventHooksManager'; -import { PackageJsonLookup, IPackageJson } from '@rushstack/node-core-library'; import { RushCommandLineParser } from './../RushCommandLineParser'; -import { RushConfiguration } from '../../api/RushConfiguration'; -import { RushGlobalFolder } from '../../api/RushGlobalFolder' import { Utilities } from '../../utilities/Utilities'; +import { RushGlobalFolder } from '../../api/RushGlobalFolder'; export interface IBaseRushActionOptions extends ICommandLineActionOptions { /** @@ -123,51 +120,4 @@ export abstract class BaseRushAction extends BaseConfiglessRushAction { return this._eventHooksManager; } - - protected mergeProjectsWithVersionPolicy( - projectsParameters: CommandLineStringListParameter, - versionPoliciesParameters: CommandLineStringListParameter - ): string[] { - const packageJsonLookup: PackageJsonLookup = new PackageJsonLookup(); - - const projects: string[] = []; - for (const projectParameter of projectsParameters.values) { - if (projectParameter === '.') { - const packageJson: IPackageJson | undefined = packageJsonLookup.tryLoadPackageJsonFor(process.cwd()); - if (packageJson) { - const projectName: string = packageJson.name; - if (this.rushConfiguration.projectsByName.has(projectName)) { - projects.push(projectName); - } else { - console.log(colors.red( - 'Rush is not currently running in a project directory specified in rush.json. ' + - `The "." value for the ${projectsParameters.longName} parameter is not allowed.` - )); - throw new AlreadyReportedError(); - } - } else { - console.log(colors.red( - 'Rush is not currently running in a project directory. ' + - `The "." value for the ${projectsParameters.longName} parameter is not allowed.` - )); - throw new AlreadyReportedError(); - } - } else { - projects.push(projectParameter); - } - } - - if (versionPoliciesParameters.values && versionPoliciesParameters.values.length > 0) { - this.rushConfiguration.projects.forEach(project => { - const matches: boolean = versionPoliciesParameters.values.some(policyName => { - return project.versionPolicyName === policyName; - }); - if (matches) { - projects.push(project.packageName); - } - }); - } - - return projects; - } } diff --git a/apps/rush-lib/src/cli/actions/InstallAction.ts b/apps/rush-lib/src/cli/actions/InstallAction.ts index 51ed463fc6b..399c4d20077 100644 --- a/apps/rush-lib/src/cli/actions/InstallAction.ts +++ b/apps/rush-lib/src/cli/actions/InstallAction.ts @@ -4,14 +4,8 @@ import { BaseInstallAction } from './BaseInstallAction'; import { IInstallManagerOptions } from '../../logic/base/BaseInstallManager'; import { RushCommandLineParser } from '../RushCommandLineParser'; -import { CommandLineStringListParameter } from '@rushstack/ts-command-line'; export class InstallAction extends BaseInstallAction { - protected _fromFlag: CommandLineStringListParameter; - protected _toFlag: CommandLineStringListParameter; - protected _fromVersionPolicy: CommandLineStringListParameter; - protected _toVersionPolicy: CommandLineStringListParameter; - public constructor(parser: RushCommandLineParser) { super({ actionName: 'install', @@ -30,36 +24,6 @@ export class InstallAction extends BaseInstallAction { }); } - protected onDefineParameters(): void { - super.onDefineParameters(); - this._toFlag = this.defineStringListParameter({ - parameterLongName: '--to', - parameterShortName: '-t', - argumentName: 'PROJECT1', - description: 'Run install in the specified project and all of its dependencies. "." can be used as shorthand ' + - 'to specify the project in the current working directory.' - }); - this._fromVersionPolicy = this.defineStringListParameter({ - parameterLongName: '--from-version-policy', - argumentName: 'VERSION_POLICY_NAME', - description: 'Run install in all projects with the specified version policy ' - + 'and all projects that directly or indirectly depend on projects with the specified version policy' - }); - this._toVersionPolicy = this.defineStringListParameter({ - parameterLongName: '--to-version-policy', - argumentName: 'VERSION_POLICY_NAME', - description: 'Run install in all projects with the specified version policy and all of their dependencies' - }); - this._fromFlag = this.defineStringListParameter({ - parameterLongName: '--from', - parameterShortName: '-f', - argumentName: 'PROJECT2', - description: 'Run install in all projects that directly or indirectly depend on the specified project. ' + - '"." can be used as shorthand to specify the project in the current working directory.' - }); - - } - protected buildInstallOptions(): IInstallManagerOptions { return { debug: this.parser.isDebug, @@ -73,9 +37,7 @@ export class InstallAction extends BaseInstallAction { variant: this._variant.value, // Because the 'defaultValue' option on the _maxInstallAttempts parameter is set, // it is safe to assume that the value is not null - maxInstallAttempts: this._maxInstallAttempts.value!, - toFlags: this.mergeProjectsWithVersionPolicy(this._toFlag, this._toVersionPolicy), - fromFlags: this.mergeProjectsWithVersionPolicy(this._fromFlag, this._fromVersionPolicy) + maxInstallAttempts: this._maxInstallAttempts.value! }; } } diff --git a/apps/rush-lib/src/cli/actions/UpdateAction.ts b/apps/rush-lib/src/cli/actions/UpdateAction.ts index 42ea4844f60..e4b9d5121af 100644 --- a/apps/rush-lib/src/cli/actions/UpdateAction.ts +++ b/apps/rush-lib/src/cli/actions/UpdateAction.ts @@ -65,9 +65,7 @@ export class UpdateAction extends BaseInstallAction { variant: this._variant.value, // Because the 'defaultValue' option on the _maxInstallAttempts parameter is set, // it is safe to assume that the value is not null - maxInstallAttempts: this._maxInstallAttempts.value!, - toFlags: [], // Do not support partial updates - fromFlags: [] // Do not support partial updates + maxInstallAttempts: this._maxInstallAttempts.value! }; } } diff --git a/apps/rush-lib/src/cli/scriptActions/BulkScriptAction.ts b/apps/rush-lib/src/cli/scriptActions/BulkScriptAction.ts index aba5ff0edd9..b58faf595da 100644 --- a/apps/rush-lib/src/cli/scriptActions/BulkScriptAction.ts +++ b/apps/rush-lib/src/cli/scriptActions/BulkScriptAction.ts @@ -10,7 +10,11 @@ import { CommandLineStringListParameter, CommandLineParameterKind } from '@rushstack/ts-command-line'; -import { FileSystem } from '@rushstack/node-core-library'; +import { + FileSystem, + PackageJsonLookup, + IPackageJson +} from '@rushstack/node-core-library'; import { Event } from '../../index'; import { SetupChecks } from '../../logic/SetupChecks'; @@ -102,8 +106,8 @@ export class BulkScriptAction extends BaseScriptAction { const taskSelector: TaskSelector = new TaskSelector({ rushConfiguration: this.rushConfiguration, - toFlags: this.mergeProjectsWithVersionPolicy(this._toFlag, this._toVersionPolicy), - fromFlags: this.mergeProjectsWithVersionPolicy(this._fromFlag, this._fromVersionPolicy), + toFlags: this._mergeProjectsWithVersionPolicy(this._toFlag, this._toVersionPolicy), + fromFlags: this._mergeProjectsWithVersionPolicy(this._fromFlag, this._fromVersionPolicy), commandToRun: this._commandToRun, customParameterValues, isQuietMode: isQuietMode, @@ -202,6 +206,55 @@ export class BulkScriptAction extends BaseScriptAction { this.defineScriptParameters(); } + private _mergeProjectsWithVersionPolicy( + projectsParameters: CommandLineStringListParameter, + versionPoliciesParameters: CommandLineStringListParameter + ): string[] { + const packageJsonLookup: PackageJsonLookup = new PackageJsonLookup(); + + const projects: string[] = []; + for (const projectParameter of projectsParameters.values) { + if (projectParameter === '.') { + const packageJson: IPackageJson | undefined = packageJsonLookup.tryLoadPackageJsonFor(process.cwd()); + if (packageJson) { + const projectName: string = packageJson.name; + if (this.rushConfiguration.projectsByName.has(projectName)) { + projects.push(projectName); + } else { + console.log(colors.red( + 'Rush is not currently running in a project directory specified in rush.json. ' + + `The "." value for the ${this._toFlag.longName} parameter or the ${this._fromFlag.longName} parameter ` + + 'is not allowed.' + )); + throw new AlreadyReportedError(); + } + } else { + console.log(colors.red( + 'Rush is not currently running in a project directory. ' + + `The "." value for the ${this._toFlag.longName} parameter or the ${this._fromFlag.longName} parameter ` + + 'is not allowed.' + )); + throw new AlreadyReportedError(); + } + } else { + projects.push(projectParameter); + } + } + + if (versionPoliciesParameters.values && versionPoliciesParameters.values.length > 0) { + this.rushConfiguration.projects.forEach(project => { + const matches: boolean = versionPoliciesParameters.values.some(policyName => { + return project.versionPolicyName === policyName; + }); + if (matches) { + projects.push(project.packageName); + } + }); + } + + return projects; + } + private _doBeforeTask(): void { if (this.actionName !== RushConstants.buildCommandName && this.actionName !== RushConstants.rebuildCommandName) { // Only collects information for built-in tasks like build or rebuild. diff --git a/apps/rush-lib/src/logic/PackageJsonUpdater.ts b/apps/rush-lib/src/logic/PackageJsonUpdater.ts index e0ae0247ef1..d42e6ab7346 100644 --- a/apps/rush-lib/src/logic/PackageJsonUpdater.ts +++ b/apps/rush-lib/src/logic/PackageJsonUpdater.ts @@ -144,9 +144,7 @@ export class PackageJsonUpdater { networkConcurrency: undefined, collectLogFile: false, variant: variant, - maxInstallAttempts: RushConstants.defaultMaxInstallAttempts, - toFlags: [], - fromFlags: [] + maxInstallAttempts: RushConstants.defaultMaxInstallAttempts }; const installManager: BaseInstallManager = InstallManagerFactory.getInstallManager( this._rushConfiguration, diff --git a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts index 7b8f5ef2575..e8b80fa26fe 100644 --- a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts +++ b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts @@ -94,7 +94,7 @@ export class WorkspaceInstallManager extends BaseInstallManager { + 'the shrinkwrap file.' )); throw new AlreadyReportedError(); - } + } } // dependency name --> version specifier @@ -180,7 +180,7 @@ export class WorkspaceInstallManager extends BaseInstallManager { } else if (dependencySpecifier.specifierType === 'workspace') { // Already specified as a local project. Allow the package manager to validate this continue; - } + } // PNPM does not specify peer dependencies for workspaces in the shrinkwrap, so skip validating these if (this.rushConfiguration.packageManager === 'pnpm' && dependencyType === DependencyType.Peer) { @@ -380,31 +380,7 @@ export class WorkspaceInstallManager extends BaseInstallManager { if (this.rushConfiguration.packageManager === 'pnpm') { args.push('--recursive'); args.push('--link-workspace-packages', 'false'); - - const toPackages: Set = this._findPackageNames(options.toFlags); - for (const toPackage of toPackages) { - args.push('--filter', `${toPackage}...`); - } - - const fromPackages: Set = this._findPackageNames(options.fromFlags); - for (const fromPackage of fromPackages) { - args.push('--filter', `...${fromPackage}`); - } - } - } - - private _findPackageNames(shorthandNames: ReadonlyArray): Set { - - const dependencies: Set = new Set(); - for (const name of shorthandNames) { - const project: RushConfigurationProject | undefined = - this.rushConfiguration.findProjectByShorthandName(name); - if (!project) { - throw new Error(`The project '${name}' does not exist in rush.json`); - } - dependencies.add(project.packageName); } - return dependencies; } /** diff --git a/apps/rush-lib/src/logic/base/BaseInstallManager.ts b/apps/rush-lib/src/logic/base/BaseInstallManager.ts index cf98377fad8..2820a4dd3a6 100644 --- a/apps/rush-lib/src/logic/base/BaseInstallManager.ts +++ b/apps/rush-lib/src/logic/base/BaseInstallManager.ts @@ -101,16 +101,6 @@ export interface IInstallManagerOptions { * Retry the install the specified number of times */ maxInstallAttempts: number - - /** - * Run install in the specified projects as well as all dependencies - */ - toFlags: ReadonlyArray; - - /** - * Run install in the specified projects as well as all dependent projects - */ - fromFlags: ReadonlyArray; } /** From 31a4eeffdc59bc33a234351edf3c0030737404ef Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 25 May 2020 14:35:24 -0700 Subject: [PATCH 08/16] Fix frozen shrinkwrap compatibility with workspaces --- apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts b/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts index fcb812ec78a..131d0486b26 100644 --- a/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts +++ b/apps/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts @@ -421,7 +421,19 @@ export class PnpmShrinkwrapFile extends BaseShrinkwrapFile { * @override */ protected serialize(): string { - let shrinkwrapContent: string = yaml.safeDump(this._shrinkwrapJson, SHRINKWRAP_YAML_FORMAT); + // Ensure that if any of the top-level properties are provided but empty that they are removed. We populate the object + // properties when we read the shrinkwrap but PNPM does not set these top-level properties unless they are present. + const shrinkwrapToSerialize: Partial = { ...this._shrinkwrapJson } + for (const key of Object.keys(shrinkwrapToSerialize).filter(key => shrinkwrapToSerialize.hasOwnProperty(key))) { + if ( + typeof shrinkwrapToSerialize[key] === 'object' && + Object.entries(shrinkwrapToSerialize[key] || {}).length === 0 + ) { + delete shrinkwrapToSerialize[key]; + } + } + + let shrinkwrapContent: string = yaml.safeDump(shrinkwrapToSerialize, SHRINKWRAP_YAML_FORMAT); if (this._shrinkwrapHashEnabled) { this._shrinkwrapHash = crypto.createHash('sha1').update(shrinkwrapContent).digest('hex'); shrinkwrapContent = From d54f5b509c05ea11e55f179145a110972daf7a2b Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Tue, 26 May 2020 13:32:56 -0700 Subject: [PATCH 09/16] Dependency cleanup --- apps/rush-lib/src/cli/actions/BaseInstallAction.ts | 2 +- apps/rush-lib/src/logic/InstallManagerFactory.ts | 6 +++--- apps/rush-lib/src/logic/WorkspaceInstallManager.ts | 3 +-- apps/rush-lib/src/logic/base/BaseInstallManager.ts | 7 +++---- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/rush-lib/src/cli/actions/BaseInstallAction.ts b/apps/rush-lib/src/cli/actions/BaseInstallAction.ts index 2409695bc05..38a91dc8754 100644 --- a/apps/rush-lib/src/cli/actions/BaseInstallAction.ts +++ b/apps/rush-lib/src/cli/actions/BaseInstallAction.ts @@ -13,6 +13,7 @@ import { import { BaseRushAction } from './BaseRushAction'; import { Event } from '../../api/EventHooks'; import { BaseInstallManager, IInstallManagerOptions } from '../../logic/base/BaseInstallManager'; +import { InstallManagerFactory } from '../../logic/InstallManagerFactory'; import { PurgeManager } from '../../logic/PurgeManager'; import { SetupChecks } from '../../logic/SetupChecks'; import { StandardScriptUpdater } from '../../logic/StandardScriptUpdater'; @@ -20,7 +21,6 @@ import { Stopwatch } from '../../utilities/Stopwatch'; import { VersionMismatchFinder } from '../../logic/versionMismatch/VersionMismatchFinder'; import { Variants } from '../../api/Variants'; import { RushConstants } from '../../logic/RushConstants'; -import { InstallManagerFactory } from '../../logic/InstallManagerFactory'; /** * This is the common base class for InstallAction and UpdateAction. diff --git a/apps/rush-lib/src/logic/InstallManagerFactory.ts b/apps/rush-lib/src/logic/InstallManagerFactory.ts index d5cde17258d..2c4bb5ee569 100644 --- a/apps/rush-lib/src/logic/InstallManagerFactory.ts +++ b/apps/rush-lib/src/logic/InstallManagerFactory.ts @@ -4,13 +4,13 @@ import * as colors from 'colors'; import * as semver from 'semver'; -import { AlreadyReportedError } from '../utilities/AlreadyReportedError'; import { BaseInstallManager, IInstallManagerOptions } from './base/BaseInstallManager'; +import { RushInstallManager } from './RushInstallManager'; +import { WorkspaceInstallManager } from './WorkspaceInstallManager'; +import { AlreadyReportedError } from '../utilities/AlreadyReportedError'; import { PurgeManager } from './PurgeManager'; import { RushConfiguration } from '../api/RushConfiguration' import { RushGlobalFolder } from '../api/RushGlobalFolder'; -import { RushInstallManager } from './RushInstallManager'; -import { WorkspaceInstallManager } from './WorkspaceInstallManager'; export class InstallManagerFactory { diff --git a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts index e8b80fa26fe..31d6b6ac641 100644 --- a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts +++ b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts @@ -1,12 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -/* eslint max-lines: off */ - import * as colors from 'colors'; import * as os from 'os'; import * as path from 'path'; import * as semver from 'semver'; + import { JsonFile, IPackageJson, diff --git a/apps/rush-lib/src/logic/base/BaseInstallManager.ts b/apps/rush-lib/src/logic/base/BaseInstallManager.ts index 2820a4dd3a6..2931f171be8 100644 --- a/apps/rush-lib/src/logic/base/BaseInstallManager.ts +++ b/apps/rush-lib/src/logic/base/BaseInstallManager.ts @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -/* eslint max-lines: off */ - import * as colors from 'colors'; import * as fetch from 'node-fetch'; import * as fs from 'fs'; @@ -11,6 +9,7 @@ import HttpsProxyAgent = require('https-proxy-agent'); import * as os from 'os'; import * as path from 'path'; import * as semver from 'semver'; + import { FileSystem, JsonFile, @@ -26,6 +25,7 @@ import { AsyncRecycler } from '../../utilities/AsyncRecycler'; import { BaseLinkManager } from './BaseLinkManager'; import { BaseShrinkwrapFile } from '../base/BaseShrinkwrapFile'; import { CommonVersionsConfiguration } from '../../api/CommonVersionsConfiguration'; +import { DependencySpecifier } from '../DependencySpecifier'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; import { Git } from '../Git'; import { LastInstallFlag } from '../../api/LastInstallFlag'; @@ -35,14 +35,13 @@ import { PackageManagerName } from '../../api/packageManager/PackageManager'; import { PnpmPackageManager } from '../../api/packageManager/PnpmPackageManager'; import { PolicyValidator } from '../policy/PolicyValidator'; import { PurgeManager } from '../PurgeManager'; -import { Rush } from '../../api/Rush'; import { RushConfiguration, IConfigurationEnvironment, ICurrentVariantJson } from '../../api/RushConfiguration'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import { Rush } from '../../api/Rush'; import { RushGlobalFolder } from '../../api/RushGlobalFolder'; import { RushConstants } from '../RushConstants'; import { ShrinkwrapFileFactory } from '../ShrinkwrapFileFactory'; import { Utilities } from '../../utilities/Utilities'; -import { DependencySpecifier } from '../DependencySpecifier'; // The PosixModeBits are intended to be used with bitwise operations. /* eslint-disable no-bitwise */ From 79e06d20d8df8284354b9bd50dd05b312e38aab7 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Tue, 26 May 2020 17:57:11 -0700 Subject: [PATCH 10/16] Change some imports to dynamic imports to elimate a circular dependency. --- apps/rush-lib/src/api/Rush.ts | 14 +++++++++----- apps/rush-lib/src/logic/base/BaseInstallManager.ts | 10 +++++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/rush-lib/src/api/Rush.ts b/apps/rush-lib/src/api/Rush.ts index 8f437d780e4..8f3d738aa49 100644 --- a/apps/rush-lib/src/api/Rush.ts +++ b/apps/rush-lib/src/api/Rush.ts @@ -4,8 +4,8 @@ import { EOL } from 'os'; import * as colors from 'colors'; import { PackageJsonLookup } from '@rushstack/node-core-library'; +import { CommandLineParser } from '@rushstack/ts-command-line'; -import { RushCommandLineParser } from '../cli/RushCommandLineParser'; import { RushConstants } from '../logic/RushConstants'; import { RushXCommandLine } from '../cli/RushXCommandLine'; import { CommandLineMigrationAdvisor } from '../cli/CommandLineMigrationAdvisor'; @@ -61,10 +61,14 @@ export class Rush { return; } - const parser: RushCommandLineParser = new RushCommandLineParser({ - alreadyReportedNodeTooNewError: options.alreadyReportedNodeTooNewError - }); - parser.execute().catch(console.error); // CommandLineParser.execute() should never reject the promise + // Use a dynamic import here to avoid an issue where a class's (RushInstallManager) base class (BaseInstallManager) + // isn't defined at its class declaration because of a circular import dependency + import('../cli/RushCommandLineParser').then(({ RushCommandLineParser }) => { + const parser: CommandLineParser = new RushCommandLineParser({ + alreadyReportedNodeTooNewError: options.alreadyReportedNodeTooNewError + }); + return parser.execute() + }).catch(console.error); // CommandLineParser.execute() should never reject the promise } /** diff --git a/apps/rush-lib/src/logic/base/BaseInstallManager.ts b/apps/rush-lib/src/logic/base/BaseInstallManager.ts index 2931f171be8..64ca1d44256 100644 --- a/apps/rush-lib/src/logic/base/BaseInstallManager.ts +++ b/apps/rush-lib/src/logic/base/BaseInstallManager.ts @@ -29,11 +29,9 @@ import { DependencySpecifier } from '../DependencySpecifier'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; import { Git } from '../Git'; import { LastInstallFlag } from '../../api/LastInstallFlag'; -import { LinkManagerFactory } from '../LinkManagerFactory'; import { PackageJsonDependency } from '../../api/PackageJsonEditor'; import { PackageManagerName } from '../../api/packageManager/PackageManager'; import { PnpmPackageManager } from '../../api/packageManager/PnpmPackageManager'; -import { PolicyValidator } from '../policy/PolicyValidator'; import { PurgeManager } from '../PurgeManager'; import { RushConfiguration, IConfigurationEnvironment, ICurrentVariantJson } from '../../api/RushConfiguration'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; @@ -233,6 +231,9 @@ export abstract class BaseInstallManager { protected abstract install(cleanInstall: boolean): Promise; protected async prepare(): Promise<{ variantIsUpToDate: boolean, shrinkwrapIsUpToDate: boolean }> { + // Use a dynamic import here to avoid an issue where a class's (RushInstallManager) base class (BaseInstallManager) + // isn't defined at its class declaration because of a circular import dependency + const { PolicyValidator } = await import('../policy/PolicyValidator'); // Check the policies PolicyValidator.validatePolicy(this.rushConfiguration, this.options); @@ -375,7 +376,10 @@ export abstract class BaseInstallManager { return { shrinkwrapIsUpToDate, variantIsUpToDate }; } - protected link(): Promise { + protected async link(): Promise { + // Use a dynamic import here to avoid an issue where a class's (RushInstallManager) base class (BaseInstallManager) + // isn't defined at its class declaration because of a circular import dependency + const { LinkManagerFactory } = await import('../LinkManagerFactory'); const linkManager: BaseLinkManager = LinkManagerFactory.getLinkManager(this.rushConfiguration); return linkManager.createSymlinksForProjects(false); } From 75cde9ae6a7ab2ea2f1dbbe25abd51ff59f9dc83 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Thu, 28 May 2020 13:10:46 -0700 Subject: [PATCH 11/16] Use DependencySpecifier to determine the specifier type --- .../src/logic/WorkspaceInstallManager.ts | 4 ++-- .../src/logic/pnpm/PnpmLinkManager.ts | 20 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts index 31d6b6ac641..338c0a693d9 100644 --- a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts +++ b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts @@ -93,7 +93,7 @@ export class WorkspaceInstallManager extends BaseInstallManager { + 'the shrinkwrap file.' )); throw new AlreadyReportedError(); - } + } } // dependency name --> version specifier @@ -179,7 +179,7 @@ export class WorkspaceInstallManager extends BaseInstallManager { } else if (dependencySpecifier.specifierType === 'workspace') { // Already specified as a local project. Allow the package manager to validate this continue; - } + } // PNPM does not specify peer dependencies for workspaces in the shrinkwrap, so skip validating these if (this.rushConfiguration.packageManager === 'pnpm' && dependencyType === DependencyType.Peer) { diff --git a/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts b/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts index ab59a2f8e33..c574f16ceea 100644 --- a/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts +++ b/apps/rush-lib/src/logic/pnpm/PnpmLinkManager.ts @@ -26,6 +26,7 @@ import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { PnpmShrinkwrapFile, IPnpmShrinkwrapDependencyYaml, IPnpmShrinkwrapImporterYaml } from './PnpmShrinkwrapFile'; import { PnpmProjectDependencyManifest } from './PnpmProjectDependencyManifest'; import { PackageJsonDependency, DependencyType } from '../../api/PackageJsonEditor'; +import { DependencySpecifier } from '../DependencySpecifier'; // special flag for debugging, will print extra diagnostic information, // but comes with performance cost @@ -86,7 +87,7 @@ export class PnpmLinkManager extends BaseLinkManager { const localDependencies: PackageJsonDependency[] = [ ...project.packageJsonEditor.dependencyList, ...project.packageJsonEditor.devDependencyList - ].filter(x => x.version.startsWith('workspace:')); + ].filter(x => new DependencySpecifier(x.name, x.version).specifierType === 'workspace'); for (const { name } of localDependencies) { const matchedRushPackage: RushConfigurationProject | undefined = @@ -115,17 +116,18 @@ export class PnpmLinkManager extends BaseLinkManager { if (!workspaceImporter) { throw new InternalError(`Cannot find shrinkwrap entry using importer key for workspace project: ${importerKey}`); } - const pnpmProjectDependencyManifest: PnpmProjectDependencyManifest = new PnpmProjectDependencyManifest({ pnpmShrinkwrapFile, project }); + const useProjectDependencyManifest: boolean = + !this._rushConfiguration.experimentsConfiguration.configuration.legacyIncrementalBuildDependencyDetection; - // Dev dependen + // Dev dependencies take priority over normal dependencies const dependencies: PackageJsonDependency[] = [ ...project.packageJsonEditor.dependencyList, ...project.packageJsonEditor.devDependencyList - ].filter(x => !x.version.startsWith('workspace:')); + ].filter(x => new DependencySpecifier(x.name, x.version).specifierType !== 'workspace'); for (const { name, dependencyType } of dependencies) { // read the version number from the shrinkwrap entry @@ -141,11 +143,14 @@ export class PnpmLinkManager extends BaseLinkManager { version = (workspaceImporter.optionalDependencies || {})[name]; break; case DependencyType.Peer: - // Peer dependencies do not need to be considered + // Peer dependencies do not need to be considered since they aren't a true + // dependency, and would be satisfied in the consuming package. They are + // also not specified in the workspace importer continue; } if (!version) { + // Optional dependencies by definition may not exist, so avoid throwing on these if (dependencyType !== DependencyType.Optional) { throw new InternalError( `Cannot find shrinkwrap entry dependency "${name}" for workspace project: ${project.packageName}` @@ -154,7 +159,8 @@ export class PnpmLinkManager extends BaseLinkManager { continue; } - if (!this._rushConfiguration.experimentsConfiguration.configuration.legacyIncrementalBuildDependencyDetection) { + if (useProjectDependencyManifest) { + // Add to the manifest and provide all the parent dependencies pnpmProjectDependencyManifest.addDependency( name, version, @@ -166,7 +172,7 @@ export class PnpmLinkManager extends BaseLinkManager { } } - if (!this._rushConfiguration.experimentsConfiguration.configuration.legacyIncrementalBuildDependencyDetection) { + if (useProjectDependencyManifest) { pnpmProjectDependencyManifest.save(); } else { pnpmProjectDependencyManifest.deleteIfExists(); From 57fb0ebb86140f051da728772d28d7f560a1e5c0 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Fri, 5 Jun 2020 23:46:54 -0700 Subject: [PATCH 12/16] Bump semver to 7.3.0 --- apps/rush-lib/package.json | 4 ++-- apps/rush-lib/src/cli/actions/VersionAction.ts | 4 ++-- apps/rush-lib/src/logic/PublishUtilities.ts | 14 +++++++------- apps/rush/package.json | 4 ++-- build-tests/api-extractor-test-02/package.json | 4 ++-- core-build/gulp-core-build/package.json | 4 ++-- libraries/node-core-library/package.json | 4 ++-- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/apps/rush-lib/package.json b/apps/rush-lib/package.json index fec9888328f..dd3f0308fb0 100644 --- a/apps/rush-lib/package.json +++ b/apps/rush-lib/package.json @@ -38,7 +38,7 @@ "node-fetch": "~2.1.2", "npm-package-arg": "~6.1.0", "read-package-tree": "~5.1.5", - "semver": "~5.3.0", + "semver": "~7.3.0", "strict-uri-encode": "~2.0.0", "tar": "~5.0.5", "true-case-path": "~2.2.1", @@ -57,7 +57,7 @@ "@types/minimatch": "2.0.29", "@types/node": "10.17.13", "@types/node-fetch": "1.6.9", - "@types/semver": "5.3.33", + "@types/semver": "~7.2.0", "@types/tar": "4.0.3", "@types/npm-package-arg": "6.1.0", "@types/strict-uri-encode": "2.0.0", diff --git a/apps/rush-lib/src/cli/actions/VersionAction.ts b/apps/rush-lib/src/cli/actions/VersionAction.ts index e06fa0c0fec..df2641c11e4 100644 --- a/apps/rush-lib/src/cli/actions/VersionAction.ts +++ b/apps/rush-lib/src/cli/actions/VersionAction.ts @@ -150,10 +150,10 @@ export class VersionAction extends BaseRushAction { const newPolicyVersion: semver.SemVer = new semver.SemVer(policy.version); if (newPolicyVersion.prerelease.length) { // Update 1.5.0-alpha.10 to 1.5.0-beta.10 - newPolicyVersion.prerelease[0] = this._prereleaseIdentifier.value; + newPolicyVersion.prerelease = [ this._prereleaseIdentifier.value, ...newPolicyVersion.prerelease.slice(1) ]; } else { // Update 1.5.0 to 1.5.0-beta - newPolicyVersion.prerelease.push(this._prereleaseIdentifier.value); + newPolicyVersion.prerelease = [ this._prereleaseIdentifier.value ]; } newVersion = newPolicyVersion.format(); } diff --git a/apps/rush-lib/src/logic/PublishUtilities.ts b/apps/rush-lib/src/logic/PublishUtilities.ts index 583c1fe1150..0b7da6c9a68 100644 --- a/apps/rush-lib/src/logic/PublishUtilities.ts +++ b/apps/rush-lib/src/logic/PublishUtilities.ts @@ -102,7 +102,7 @@ export class PublishUtilities { } else { // For hotfix changes, do not re-write new version change.newVersion = (change.changeType! >= ChangeType.patch) ? - semver.inc(pkg.version, PublishUtilities._getReleaseType(change.changeType!)) : + semver.inc(pkg.version, PublishUtilities._getReleaseType(change.changeType!))! : (change.changeType === ChangeType.hotfix ? change.newVersion : pkg.version); } @@ -262,9 +262,9 @@ export class PublishUtilities { let upperLimit: string = newVersion; if (semver.prerelease(newVersion)) { // Remove the prerelease first, then bump major. - upperLimit = semver.inc(newVersion, 'patch'); + upperLimit = semver.inc(newVersion, 'patch')!; } - upperLimit = semver.inc(upperLimit, 'major'); + upperLimit = semver.inc(upperLimit, 'major')!; return `>=${newVersion} <${upperLimit}`; } @@ -448,7 +448,7 @@ export class PublishUtilities { return newVersion; } if (prereleaseToken.isPrerelease && change.changeType === ChangeType.dependency) { - newVersion = semver.inc(newVersion, 'patch'); + newVersion = semver.inc(newVersion, 'patch')!; } return `${newVersion}-${prereleaseToken.name}`; } else { @@ -521,7 +521,7 @@ export class PublishUtilities { currentChange.changeType = ChangeType.none; } else { if (change.changeType === ChangeType.hotfix) { - const prereleaseComponents: string[] = semver.prerelease(pkg.version); + const prereleaseComponents: ReadonlyArray | null = semver.prerelease(pkg.version); if (!rushConfiguration.hotfixChangeEnabled) { throw new Error(`Cannot add hotfix change; hotfixChangeEnabled is false in configuration.`); } @@ -530,7 +530,7 @@ export class PublishUtilities { if (!prereleaseComponents) { currentChange.newVersion += '-hotfix'; } - currentChange.newVersion = semver.inc(currentChange.newVersion, 'prerelease'); + currentChange.newVersion = semver.inc(currentChange.newVersion, 'prerelease')!; } else { // When there are multiple changes of this package, the final value of new version // should not depend on the order of the changes. @@ -539,7 +539,7 @@ export class PublishUtilities { packageVersion = currentChange.newVersion; } currentChange.newVersion = change.changeType! >= ChangeType.patch ? - semver.inc(pkg.version, PublishUtilities._getReleaseType(currentChange.changeType!)) : + semver.inc(pkg.version, PublishUtilities._getReleaseType(currentChange.changeType!))! : packageVersion; } diff --git a/apps/rush/package.json b/apps/rush/package.json index 515a7f12cc8..9af79973f4d 100644 --- a/apps/rush/package.json +++ b/apps/rush/package.json @@ -34,7 +34,7 @@ "@rushstack/node-core-library": "3.22.0", "@microsoft/rush-lib": "5.23.4", "colors": "~1.2.1", - "semver": "~5.3.0" + "semver": "~7.3.0" }, "devDependencies": { "@microsoft/rush-stack-compiler-3.5": "0.4.15", @@ -43,7 +43,7 @@ "@types/chai": "3.4.34", "@types/mocha": "5.2.5", "@types/node": "10.17.13", - "@types/semver": "5.3.33", + "@types/semver": "~7.2.0", "@types/sinon": "1.16.34", "chai": "~3.5.0", "gulp": "~4.0.2", diff --git a/build-tests/api-extractor-test-02/package.json b/build-tests/api-extractor-test-02/package.json index 65ac0fb9a0a..82e1f53b1f8 100644 --- a/build-tests/api-extractor-test-02/package.json +++ b/build-tests/api-extractor-test-02/package.json @@ -9,13 +9,13 @@ "build": "node build.js" }, "dependencies": { - "@types/semver": "5.3.33", "api-extractor-test-01": "1.0.0", - "semver": "~5.3.0" + "semver": "~7.3.0" }, "devDependencies": { "@microsoft/api-extractor": "7.8.5", "@types/node": "10.17.13", + "@types/semver": "~7.2.0", "fs-extra": "~7.0.1", "typescript": "~3.7.2" } diff --git a/core-build/gulp-core-build/package.json b/core-build/gulp-core-build/package.json index 238266ba0af..5e0372e552f 100644 --- a/core-build/gulp-core-build/package.json +++ b/core-build/gulp-core-build/package.json @@ -20,7 +20,6 @@ "@types/node": "10.17.13", "@types/node-notifier": "0.0.28", "@types/orchestrator": "0.0.30", - "@types/semver": "5.3.33", "@types/through2": "2.0.32", "@types/vinyl": "2.0.3", "@types/yargs": "0.0.34", @@ -46,7 +45,7 @@ "object-assign": "~4.1.0", "orchestrator": "~0.3.8", "pretty-hrtime": "~1.0.2", - "semver": "~5.3.0", + "semver": "~7.3.0", "through2": "~2.0.1", "vinyl": "~2.2.0", "xml": "~1.0.1", @@ -60,6 +59,7 @@ "@types/chai": "3.4.34", "@types/glob": "7.1.1", "@types/mocha": "5.2.5", + "@types/semver": "~7.2.0", "@types/z-schema": "3.16.31", "chai": "~3.5.0" } diff --git a/libraries/node-core-library/package.json b/libraries/node-core-library/package.json index 3d6c2590e4a..1b16234b2a4 100644 --- a/libraries/node-core-library/package.json +++ b/libraries/node-core-library/package.json @@ -16,7 +16,7 @@ "colors": "~1.2.1", "fs-extra": "~7.0.1", "jju": "~1.4.0", - "semver": "~5.3.0", + "semver": "~7.3.0", "timsort": "~0.3.0", "z-schema": "~3.18.3" }, @@ -27,7 +27,7 @@ "@types/fs-extra": "7.0.0", "@types/jest": "25.2.1", "@types/jju": "1.4.1", - "@types/semver": "5.3.33", + "@types/semver": "~7.2.0", "@types/timsort": "0.3.0", "@types/z-schema": "3.16.31", "gulp": "~4.0.2" From a48ca0b887134be198aabcb3e1d31fb1581836c1 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Fri, 5 Jun 2020 23:47:43 -0700 Subject: [PATCH 13/16] Remove common versions from package.json --- .../src/logic/WorkspaceInstallManager.ts | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts index 338c0a693d9..fec8f4fdda4 100644 --- a/apps/rush-lib/src/logic/WorkspaceInstallManager.ts +++ b/apps/rush-lib/src/logic/WorkspaceInstallManager.ts @@ -217,28 +217,17 @@ export class WorkspaceInstallManager extends BaseInstallManager { version: '0.0.0' }; - // dependency name --> version specifier - const allPreferredVersions: Map = BaseInstallManager.collectPreferredVersions( - this.rushConfiguration, - this.options.variant - ); - - // Add any preferred versions to the top of the commonPackageJson - // do this in alphabetical order for simpler debugging - for (const dependency of Array.from(allPreferredVersions.keys()).sort()) { - commonPackageJson.dependencies![dependency] = allPreferredVersions.get(dependency)!; - } - // Example: "C:\MyRepo\common\temp\package.json" const commonPackageJsonFilename: string = path.join( this.rushConfiguration.commonTempFolder, FileConstants.PackageJson ); + JsonFile.save(commonPackageJson, commonPackageJsonFilename, { onlyIfChanged: true }); + // Save the generated files. Don't update the file timestamp unless the content has changed, // since "rush install" will consider this timestamp workspaceFile.save(workspaceFile.workspaceFilename, { onlyIfChanged: true }); - JsonFile.save(commonPackageJson, commonPackageJsonFilename, { onlyIfChanged: true }); stopwatch.stop(); console.log(`Finished creating workspace (${stopwatch.toString()})`); @@ -341,7 +330,8 @@ export class WorkspaceInstallManager extends BaseInstallManager { this.installRecycler.moveFolder(nodeModulesFolder); } } - }); + } + ); // Ensure that node_modules folders exist after install, since the timestamps on these folders are used // to determine if the install can be skipped From c4ffc52fea03743e78ac6bb263c38b5fbc5b7e3a Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Fri, 5 Jun 2020 23:55:24 -0700 Subject: [PATCH 14/16] Add pnpmfile shimming for common versions --- apps/rush-lib/src/api/RushConfiguration.ts | 13 +++ .../src/logic/base/BaseInstallManager.ts | 91 ++++++++++++++++++- .../pnpm/PnpmProjectDependencyManifest.ts | 2 +- apps/rush-lib/src/schemas/rush.schema.json | 4 + .../rush/nonbrowser-approved-packages.json | 4 + common/reviews/api/node-core-library.api.md | 3 + common/reviews/api/rush-lib.api.md | 2 + .../node-core-library/src/MapExtensions.ts | 17 ++++ 8 files changed, 134 insertions(+), 2 deletions(-) diff --git a/apps/rush-lib/src/api/RushConfiguration.ts b/apps/rush-lib/src/api/RushConfiguration.ts index f2947cca987..f94075195d1 100644 --- a/apps/rush-lib/src/api/RushConfiguration.ts +++ b/apps/rush-lib/src/api/RushConfiguration.ts @@ -178,6 +178,10 @@ export interface IPnpmOptionsJson extends IPackageManagerOptionsJsonBase { * {@inheritDoc PnpmOptionsConfiguration.useWorkspaces} */ useWorkspaces?: boolean; + /** + * {@inheritDoc PnpmOptionsConfiguration.useShimPnpmfile} + */ + useShimPnpmfile?: boolean; } /** @@ -364,6 +368,14 @@ export class PnpmOptionsConfiguration extends PackageManagerOptionsConfiguration */ public readonly useWorkspaces: boolean; + /** + * If true, then Rush will shim common versions support into the pnpmfile. + * + * @remarks + * The default value is false. (For now.) + */ + public readonly useShimPnpmfile: boolean; + /** @internal */ public constructor(json: IPnpmOptionsJson, commonTempFolder: string) { super(json); @@ -379,6 +391,7 @@ export class PnpmOptionsConfiguration extends PackageManagerOptionsConfiguration this.resolutionStrategy = json.resolutionStrategy || 'fewer-dependencies'; this.preventManualShrinkwrapChanges = !!json.preventManualShrinkwrapChanges; this.useWorkspaces = !!json.useWorkspaces; + this.useShimPnpmfile = !!json.useShimPnpmfile; } } diff --git a/apps/rush-lib/src/logic/base/BaseInstallManager.ts b/apps/rush-lib/src/logic/base/BaseInstallManager.ts index 64ca1d44256..9cd0b627062 100644 --- a/apps/rush-lib/src/logic/base/BaseInstallManager.ts +++ b/apps/rush-lib/src/logic/base/BaseInstallManager.ts @@ -16,7 +16,7 @@ import { JsonObject, LockFile, MapExtensions, - PosixModeBits, + PosixModeBits } from '@rushstack/node-core-library'; import { AlreadyReportedError } from '../../utilities/AlreadyReportedError'; @@ -339,6 +339,14 @@ export abstract class BaseInstallManager { // ensure that we remove any old one that may be hanging around Utilities.syncFile(committedPnpmFilePath, tempPnpmFilePath); + + // shim support for common versions resolution into the pnpmfile + if ( + this.rushConfiguration.pnpmOptions && + (this.rushConfiguration.pnpmOptions.useShimPnpmfile || this.rushConfiguration.pnpmOptions.useWorkspaces) + ) { + await BaseInstallManager.createShimPnpmfile(tempPnpmFilePath, this.rushConfiguration, this.options.variant); + } } // Allow for package managers to do their own preparation and check that the shrinkwrap is up to date @@ -517,6 +525,87 @@ export abstract class BaseInstallManager { }); } + protected static async createShimPnpmfile( + filename: string, + rushConfiguration: RushConfiguration, + variant: string | undefined + ): Promise { + + const allPreferredVersions: Map = + BaseInstallManager.collectPreferredVersions(rushConfiguration, variant); + const allowedAlternativeVersions: Map> = + rushConfiguration.getCommonVersions(variant).allowedAlternativeVersions; + + // Setup the header of the shim + const clientPnpmfileName: string = 'clientPnpmfile.js'; + const pnpmfileContent: string[] = [ + '// THIS IS A GENERATED FILE. DO NOT MODIFY.', + '"use strict";', + 'module.exports = { hooks: { readPackage, afterAllResolved } };', + + 'const { existsSync } = require("fs");', + 'const originalPnpmfile = ', + ` existsSync("${clientPnpmfileName}") ? require("./${path.basename(clientPnpmfileName, '.js')}") : undefined;`, + `const semver = require(${JSON.stringify(require.resolve('semver'))});`, + `const allPreferredVersions = ${JSON.stringify(MapExtensions.toObject(allPreferredVersions))};`, + `const allowedAlternativeVersions = ${JSON.stringify(MapExtensions.toObject(allowedAlternativeVersions))};`, + + 'function afterAllResolved(lockfile, context) {', + ' return (originalPnpmfile && originalPnpmfile.hooks && originalPnpmfile.hooks.afterAllResolved)', + ' ? originalPnpmfile.hooks.afterAllResolved(lockfile, context)', + ' : lockfile;', + '}', + + 'function readPackage(pkg, context) {', + ' setPreferredVersions(pkg.dependencies);', + ' setPreferredVersions(pkg.devDependencies);', + ' setPreferredVersions(pkg.optionalDependencies);', + ' return (originalPnpmfile && originalPnpmfile.hooks && originalPnpmfile.hooks.readPackage)', + ' ? originalPnpmfile.hooks.readPackage(pkg, context)', + ' : pkg;', + '}', + + 'function setPreferredVersions(dependencies) {', + ' for (const name of Object.keys(dependencies || {})) {', + ' if (allPreferredVersions.hasOwnProperty(name)) {', + ' const preferredVersion = allPreferredVersions[name];', + ' const version = dependencies[name];', + ' if (allowedAlternativeVersions.hasOwnProperty(name)) {', + ' const allowedAlternatives = allowedAlternativeVersions[name];', + ' if (allowedAlternatives && allowedAlternatives.indexOf(version) > -1) {', + ' continue;', + ' }', + ' }', + ' let isValidRange = false;', + ' try {', + ' isValidRange = !!semver.validRange(preferredVersion) && !!semver.validRange(version);', + ' } catch {', + ' }', + ' if (isValidRange && semver.subset(preferredVersion, version)) {', + ' dependencies[name] = preferredVersion;', + ' }', + ' }', + ' }', + '}', + ]; + + // Attempt to move the existing pnpmfile if there is one + try { + const pnpmfileDir: string = path.dirname(filename); + await FileSystem.moveAsync({ + sourcePath: filename, + destinationPath: path.join(pnpmfileDir, clientPnpmfileName) + }); + } catch (error) { + if (!FileSystem.isNotExistError(error)) { + throw error; + } + } + + // Write the shim pnpmfile to the original file path + await FileSystem.writeFileAsync(filename, pnpmfileContent.join('\n'), { ensureFolderExists: true }); + } + protected getPackageManagerEnvironment(): NodeJS.ProcessEnv { let configurationEnvironment: IConfigurationEnvironment | undefined = undefined; if (this.rushConfiguration.packageManager === 'npm') { diff --git a/apps/rush-lib/src/logic/pnpm/PnpmProjectDependencyManifest.ts b/apps/rush-lib/src/logic/pnpm/PnpmProjectDependencyManifest.ts index 00d02fe72df..4a8651fc14d 100644 --- a/apps/rush-lib/src/logic/pnpm/PnpmProjectDependencyManifest.ts +++ b/apps/rush-lib/src/logic/pnpm/PnpmProjectDependencyManifest.ts @@ -221,7 +221,7 @@ export class PnpmProjectDependencyManifest { // We couldn't find the peer dependency, but we determined it's by design, skip this dependency... continue; } - throw new InternalError( + throw new Error( `Could not find peer dependency '${peerDependencyName}' that satisfies version '${dependencySemVer}'` ); } diff --git a/apps/rush-lib/src/schemas/rush.schema.json b/apps/rush-lib/src/schemas/rush.schema.json index e5f49a467e2..6a24846369a 100644 --- a/apps/rush-lib/src/schemas/rush.schema.json +++ b/apps/rush-lib/src/schemas/rush.schema.json @@ -110,6 +110,10 @@ "description": "If true, then Rush will use the workspaces feature to install and link packages when invoking PNPM. The default value is false.", "type": "boolean" }, + "useShimPnpmfile": { + "description": "If true, then Rush will use the workspaces feature to install and link packages when invoking PNPM. The default value is false.", + "type": "boolean" + }, "environmentVariables": { "$ref": "#/definitions/environmentVariables" }, diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index 71bd1929599..0b56ad4393d 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -422,6 +422,10 @@ "name": "https-proxy-agent", "allowedCategories": [ "libraries" ] }, + { + "name": "import-lazy", + "allowedCategories": [ "libraries" ] + }, { "name": "inquirer", "allowedCategories": [ "libraries" ] diff --git a/common/reviews/api/node-core-library.api.md b/common/reviews/api/node-core-library.api.md index f8c1d69796f..61da3fb10e7 100644 --- a/common/reviews/api/node-core-library.api.md +++ b/common/reviews/api/node-core-library.api.md @@ -441,6 +441,9 @@ export class LockFile { // @public export class MapExtensions { static mergeFromMap(targetMap: Map, sourceMap: ReadonlyMap): void; + static toObject(map: Map): { + [key: string]: TValue; + }; } // @public diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 26ded2132fd..0f3cf5d2c8b 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -175,6 +175,7 @@ export interface _IPnpmOptionsJson extends IPackageManagerOptionsJsonBase { preventManualShrinkwrapChanges?: boolean; resolutionStrategy?: ResolutionStrategy; strictPeerDependencies?: boolean; + useShimPnpmfile?: boolean; useWorkspaces?: boolean; } @@ -289,6 +290,7 @@ export class PnpmOptionsConfiguration extends PackageManagerOptionsConfiguration readonly preventManualShrinkwrapChanges: boolean; readonly resolutionStrategy: ResolutionStrategy; readonly strictPeerDependencies: boolean; + readonly useShimPnpmfile: boolean; readonly useWorkspaces: boolean; } diff --git a/libraries/node-core-library/src/MapExtensions.ts b/libraries/node-core-library/src/MapExtensions.ts index b84ba96a960..38452657c3c 100644 --- a/libraries/node-core-library/src/MapExtensions.ts +++ b/libraries/node-core-library/src/MapExtensions.ts @@ -19,4 +19,21 @@ export class MapExtensions { targetMap.set(pair[0], pair[1]); } } + + /** + * Converts a string-keyed map to an object. + * @remarks + * This function has the same effect as Object.fromEntries(map.entries()) + * in supported versions of Node (\>= 12.0.0). + * @param map - The map that the object properties will be sourced from + */ + public static toObject(map: Map): { [key: string]: TValue } { + return Array.from(map.entries()).reduce<{ [key: string]: TValue }>( + (previous: { [key: string]: TValue }, current: [ string, TValue ]) => { + previous[current[0]] = current[1]; + return previous; + }, + {} + ); + } } From 6e0a30957b7015c4a741c420afc1bee6a8f64af8 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Mon, 8 Jun 2020 11:53:33 -0700 Subject: [PATCH 15/16] Re-integrate some changes from master --- apps/rush-lib/src/logic/RushInstallManager.ts | 80 ++++++++++++------- .../src/logic/base/BaseInstallManager.ts | 2 +- 2 files changed, 50 insertions(+), 32 deletions(-) diff --git a/apps/rush-lib/src/logic/RushInstallManager.ts b/apps/rush-lib/src/logic/RushInstallManager.ts index 452c3fa62c9..5b8b7792126 100644 --- a/apps/rush-lib/src/logic/RushInstallManager.ts +++ b/apps/rush-lib/src/logic/RushInstallManager.ts @@ -147,9 +147,6 @@ export class RushInstallManager extends BaseInstallManager { // Example: "C:\MyRepo\common\temp\projects\my-project-2.tgz" const tarballFile: string = this._getTarballFilePath(rushProject); - // Example: "my-project-2" - const unscopedTempProjectName: string = rushProject.unscopedTempProjectName; - // Example: dependencies["@rush-temp/my-project-2"] = "file:./projects/my-project-2.tgz" commonPackageJson.dependencies![rushProject.tempProjectName] = `file:./${RushConstants.rushTempProjectsFolderName}/${rushProject.unscopedTempProjectName}.tgz`; @@ -242,14 +239,8 @@ export class RushInstallManager extends BaseInstallManager { } } - // NPM expects the root of the tarball to have a directory called 'package' - const npmPackageFolder: string = 'package'; - // Example: "C:\MyRepo\common\temp\projects\my-project-2" - const tempProjectFolder: string = path.join( - this.rushConfiguration.commonTempFolder, - RushConstants.rushTempProjectsFolderName, - unscopedTempProjectName); + const tempProjectFolder: string = this._getTempProjectFolder(rushProject); // Example: "C:\MyRepo\common\temp\projects\my-project-2\package.json" const tempPackageJsonFilename: string = path.join(tempProjectFolder, FileConstants.PackageJson); @@ -286,27 +277,8 @@ export class RushInstallManager extends BaseInstallManager { // write the expected package.json file into the zip staging folder JsonFile.save(tempPackageJson, tempPackageJsonFilename); - const tarOptions: tar.CreateOptions = ({ - gzip: true, - file: tarballFile, - cwd: tempProjectFolder, - portable: true, - noMtime: true, - noPax: true, - sync: true, - prefix: npmPackageFolder, - filter: (path: string, stat: tar.FileStat): boolean => { - if (!this.rushConfiguration.experimentsConfiguration.configuration - .noChmodFieldInTarHeaderNormalization) { - stat.mode = (stat.mode & ~0x1FF) | PosixModeBits.AllRead | PosixModeBits.UserWrite - | PosixModeBits.AllExecute; - } - return true; - } - } as tar.CreateOptions); - - // create the new tarball - tar.create(tarOptions, [FileConstants.PackageJson]); + // Delete the existing tarball and create a new one + this._createTempProjectTarball(rushProject); console.log(`Updating ${tarballFile}`); } catch (error) { @@ -386,6 +358,13 @@ export class RushInstallManager extends BaseInstallManager { } } + // Since we are actually running npm/pnpm/yarn install, recreate all the temp project tarballs. + // This ensures that any existing tarballs with older header bits will be regenerated. + // It is safe to assume that temp project pacakge.jsons already exist. + for (const rushProject of this.rushConfiguration.projects) { + this._createTempProjectTarball(rushProject); + } + const commonNodeModulesFolder: string = path.join( this.rushConfiguration.commonTempFolder, 'node_modules' @@ -520,6 +499,11 @@ export class RushInstallManager extends BaseInstallManager { } } + private _getTempProjectFolder(rushProject: RushConfigurationProject): string { + const unscopedTempProjectName: string = rushProject.unscopedTempProjectName; + return path.join(this.rushConfiguration.commonTempFolder, RushConstants.rushTempProjectsFolderName, unscopedTempProjectName); + } + /** * Gets the path to the tarball * Example: "C:\MyRepo\common\temp\projects\my-project-2.tgz" @@ -531,6 +515,40 @@ export class RushInstallManager extends BaseInstallManager { `${project.unscopedTempProjectName}.tgz`); } + /** + * Deletes the existing tarball and creates a tarball for the given rush project + */ + private _createTempProjectTarball(rushProject: RushConfigurationProject): void { + const tarballFile: string = this._getTarballFilePath(rushProject); + const tempProjectFolder: string = this._getTempProjectFolder(rushProject); + + FileSystem.deleteFile(tarballFile); + + // NPM expects the root of the tarball to have a directory called 'package' + const npmPackageFolder: string = 'package'; + + const tarOptions: tar.CreateOptions = ({ + gzip: true, + file: tarballFile, + cwd: tempProjectFolder, + portable: true, + noMtime: true, + noPax: true, + sync: true, + prefix: npmPackageFolder, + filter: (path: string, stat: tar.FileStat): boolean => { + if (!this.rushConfiguration.experimentsConfiguration.configuration + .noChmodFieldInTarHeaderNormalization) { + stat.mode = (stat.mode & ~0x1FF) | PosixModeBits.AllRead | PosixModeBits.UserWrite + | PosixModeBits.AllExecute; + } + return true; + } + } as tar.CreateOptions); + // create the new tarball + tar.create(tarOptions, [FileConstants.PackageJson]); + } + /** * This is a workaround for a bug introduced in NPM 5 (and still unfixed as of NPM 5.5.1): * https://github.com/npm/npm/issues/19006 diff --git a/apps/rush-lib/src/logic/base/BaseInstallManager.ts b/apps/rush-lib/src/logic/base/BaseInstallManager.ts index 9cd0b627062..9376920fe76 100644 --- a/apps/rush-lib/src/logic/base/BaseInstallManager.ts +++ b/apps/rush-lib/src/logic/base/BaseInstallManager.ts @@ -730,7 +730,7 @@ export abstract class BaseInstallManager { } if ((this._rushConfiguration.packageManagerWrapper as PnpmPackageManager).supportsResolutionStrategy) { - args.push('--resolution-strategy', this._rushConfiguration.pnpmOptions.resolutionStrategy); + args.push(`--resolution-strategy=${this._rushConfiguration.pnpmOptions.resolutionStrategy}`); } } else if (this._rushConfiguration.packageManager === 'yarn') { args.push('--link-folder', 'yarn-link'); From d00f583fc89cabdf81ad96a8571c958760e0f264 Mon Sep 17 00:00:00 2001 From: Daniel Nadeau <3473356+D4N14L@users.noreply.github.com> Date: Tue, 9 Jun 2020 11:20:23 -0700 Subject: [PATCH 16/16] Add fix for recycling invalid symlinks --- apps/rush-lib/src/utilities/AsyncRecycler.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/apps/rush-lib/src/utilities/AsyncRecycler.ts b/apps/rush-lib/src/utilities/AsyncRecycler.ts index 579694c8459..e3987d21117 100644 --- a/apps/rush-lib/src/utilities/AsyncRecycler.ts +++ b/apps/rush-lib/src/utilities/AsyncRecycler.ts @@ -73,7 +73,25 @@ export class AsyncRecycler { } Utilities.retryUntilTimeout( - () => FileSystem.move({ sourcePath: folderPath, destinationPath: newFolderPath }), + () => { + try { + FileSystem.move({ sourcePath: folderPath, destinationPath: newFolderPath }); + } catch (e) { + if (FileSystem.isErrnoException(e)) { + // It could be an invalid symlink, so try to obtain the stats + try { + const stats: fs.Stats = FileSystem.getLinkStatistics(folderPath); + if (stats.isSymbolicLink() && stats.isDirectory()) { + FileSystem.deleteFolder(folderPath); + return; + } + } catch (e2) { + // Throw original error below + } + } + throw e; + } + }, maxWaitTimeMs, (e) => new Error(`Error: ${e}${os.EOL}Often this is caused by a file lock ` + 'from a process like the virus scanner.'),