From c6af46b96aca8d445a3262a03c0b41bf9473b8d8 Mon Sep 17 00:00:00 2001 From: Miroslav Jonas Date: Tue, 4 Apr 2023 14:55:26 +0200 Subject: [PATCH] fix(core): include pr notes --- .../plugins/js/lock-file/npm-parser.spec.ts | 8 +- .../plugins/js/lock-file/pnpm-parser.spec.ts | 36 +- .../src/plugins/js/lock-file/pnpm-parser.ts | 70 +- .../js/lock-file/utils/pnpm-normalizer.ts | 660 +++++++++--------- .../plugins/js/lock-file/yarn-parser.spec.ts | 8 +- 5 files changed, 399 insertions(+), 383 deletions(-) diff --git a/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts index 1177263d98b894..47770a0dd74eb5 100644 --- a/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts @@ -27,7 +27,7 @@ describe('NPM lock file utility', () => { }); it('should parse root lock file', async () => { - expect(Object.keys(graph.externalNodes).length).toEqual(1285); // 1143 + expect(Object.keys(graph.externalNodes).length).toEqual(1285); }); it('should prune lock file', async () => { @@ -151,7 +151,7 @@ describe('NPM lock file utility', () => { const builder = new ProjectGraphBuilder(); parseNpmLockfile(JSON.stringify(rootV2LockFile), builder); const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202 + expect(Object.keys(graph.externalNodes).length).toEqual(212); expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` Object { @@ -336,7 +336,7 @@ describe('NPM lock file utility', () => { const builder = new ProjectGraphBuilder(); parseNpmLockfile(JSON.stringify(rootLockFile), builder); const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(369); // 338 + expect(Object.keys(graph.externalNodes).length).toEqual(369); }); it('should parse v3', async () => { const rootLockFile = require(joinPathFragments( @@ -347,7 +347,7 @@ describe('NPM lock file utility', () => { const builder = new ProjectGraphBuilder(); parseNpmLockfile(JSON.stringify(rootLockFile), builder); const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(369); //338 + expect(Object.keys(graph.externalNodes).length).toEqual(369); }); }); diff --git a/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts index f0e123b05a7e7a..ad663033cbfa1b 100644 --- a/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts @@ -124,7 +124,7 @@ describe('pnpm LockFile utility', () => { }); it('should parse root lock file', async () => { - expect(Object.keys(graph.externalNodes).length).toEqual(1280); ///1143 + expect(Object.keys(graph.externalNodes).length).toEqual(1280); }); it('should prune lock file', async () => { @@ -132,18 +132,10 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/nextjs/app/package.json' )); - // this is original generated lock file - // const appLockFile = require(joinPathFragments( - // __dirname, - // '__fixtures__/nextjs/app/pnpm-lock.yaml' - // )).default; - - // const appGraph = parsePnpmLockfile(appLockFile); - // expect(Object.keys(appGraph.externalNodes).length).toEqual(863); // this is our pruned lock file structure const prunedGraph = pruneProjectGraph(graph, appPackageJson); - // meeroslav: our pruning keep all transitive peer deps, mainly cypress and eslint + // our pruning keep all transitive peer deps, mainly cypress and eslint // which adds 119 more deps // but it's still possible to run `pnpm install --frozen-lockfile` on it (there are e2e tests for that) expect(Object.keys(prunedGraph.externalNodes).length).toEqual( @@ -151,11 +143,9 @@ describe('pnpm LockFile utility', () => { ); // this should not fail - const result = stringifyPnpmLockfile( - prunedGraph, - lockFile, - appPackageJson - ); + expect(() => + stringifyPnpmLockfile(prunedGraph, lockFile, appPackageJson) + ).not.toThrow(); }); }); @@ -168,12 +158,10 @@ describe('pnpm LockFile utility', () => { const builder = new ProjectGraphBuilder(); parsePnpmLockfile(lockFile, builder); graph = builder.getUpdatedProjectGraph(); - - // console.log(JSON.stringify(Object.keys(graph.externalNodes).sort().map(k => ({ p: k, v: graph.externalNodes[k].data.version })), null, 2)); }); it('should parse root lock file', async () => { - expect(Object.keys(graph.externalNodes).length).toEqual(1296); ///1143 + expect(Object.keys(graph.externalNodes).length).toEqual(1296); }); it('should prune lock file', async () => { @@ -199,11 +187,9 @@ describe('pnpm LockFile utility', () => { ); // this should not fail - const result = stringifyPnpmLockfile( - prunedGraph, - lockFile, - appPackageJson - ); + expect(() => + stringifyPnpmLockfile(prunedGraph, lockFile, appPackageJson) + ).not.toThrow(); }); }); }); @@ -241,7 +227,7 @@ describe('pnpm LockFile utility', () => { const builder = new ProjectGraphBuilder(); parsePnpmLockfile(lockFile, builder); const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(213); //202 + expect(Object.keys(graph.externalNodes).length).toEqual(213); expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` Object { @@ -362,7 +348,7 @@ describe('pnpm LockFile utility', () => { const builder = new ProjectGraphBuilder(); parsePnpmLockfile(lockFile, builder); const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(370); //337 + expect(Object.keys(graph.externalNodes).length).toEqual(370); expect(Object.keys(graph.dependencies).length).toEqual(213); expect(graph.dependencies['npm:@nrwl/devkit'].length).toEqual(6); }); diff --git a/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts b/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts index 764777bbae9996..0b9c6cc715ac0e 100644 --- a/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts +++ b/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts @@ -5,6 +5,7 @@ import type { PackageSnapshots, } from '@pnpm/lockfile-types'; import { + isV6Lockfile, loadPnpmHoistedDepsDefinition, parseAndNormalizePnpmLockfile, stringifyToPnpmYaml, @@ -31,8 +32,6 @@ export function parsePnpmLockfile( addDependencies(data, builder, keyMap); } -const MATCH_ADDITIONAL_VERSION_INFO = /_|\(/; - function addNodes( data: Lockfile, builder: ProjectGraphBuilder, @@ -42,9 +41,10 @@ function addNodes( Object.entries(data.packages).forEach(([key, snapshot]) => { const packageName = findPackageName(key, snapshot, data); - const version = findVersion(key, packageName).split( - MATCH_ADDITIONAL_VERSION_INFO - )[0]; + const rawVersion = findVersion(key, packageName); + const version = isV6Lockfile(data) + ? rawVersion.split('(')[0] + : rawVersion.split('_')[0]; // we don't need to keep duplicates, we can just track the keys const existingNode = nodes.get(packageName)?.get(version); @@ -107,7 +107,7 @@ function getHoistedVersion( k.startsWith(`/${packageName}/`) ); if (key) { - version = key.slice(key.lastIndexOf('/') + 1).split('_')[0]; + version = getVersion(key, packageName).split('_')[0]; } else { // pnpm might not hoist every package // similarly those packages will not be available to be used via import @@ -149,11 +149,12 @@ export function stringifyPnpmLockfile( packageJson: NormalizedPackageJson ): string { const data = parseAndNormalizePnpmLockfile(rootLockFileContent); + const { lockfileVersion, packages } = data; const output: Lockfile = { - lockfileVersion: normalizeLockfileVersion(data.lockfileVersion), + lockfileVersion, importers: { - '.': mapRootSnapshot(packageJson, data.packages, graph.externalNodes), + '.': mapRootSnapshot(packageJson, packages, graph.externalNodes), }, packages: sortObjectByKeys( mapSnapshots(data.packages, graph.externalNodes) @@ -163,13 +164,6 @@ export function stringifyPnpmLockfile( return stringifyToPnpmYaml(output); } -function normalizeLockfileVersion(version: string | number) { - if (typeof version === 'string' || version < 6) { - return version; - } - return version.toFixed(1); -} - function mapSnapshots( packages: PackageSnapshots, nodes: Record @@ -200,7 +194,7 @@ function findOriginalKeys( // standard package if (key.startsWith(`/${packageName}/${version}`)) { matchedKeys.push([ - returnFullKey ? key : key.slice(packageName.length + 2), + returnFullKey ? key : getVersion(key, packageName), snapshot, ]); } @@ -209,20 +203,25 @@ function findOriginalKeys( matchedKeys.push([version, snapshot]); } // alias package - if ( - version.startsWith('npm:') && - key.startsWith( - `/${version.slice(4, version.indexOf('@', 6))}/${version.slice( - version.indexOf('@', 6) + 1 - )}` - ) - ) { + if (versionIsAlias(key, version)) { matchedKeys.push([key, snapshot]); } } return matchedKeys; } +// check if version has a form of npm:packageName@version and +// key starts with /packageName/version +function versionIsAlias(key: string, versionExpr: string): boolean { + const PREFIX = 'npm:'; + if (!versionExpr.startsWith(PREFIX)) return false; + + const indexOfVersionSeparator = versionExpr.indexOf('@', PREFIX.length + 1); + const packageName = versionExpr.slice(PREFIX.length, indexOfVersionSeparator); + const version = versionExpr.slice(indexOfVersionSeparator + 1); + return key.startsWith(`/${packageName}/${version}`); +} + function mapRootSnapshot( packageJson: NormalizedPackageJson, packages: PackageSnapshots, @@ -258,12 +257,12 @@ function mapRootSnapshot( function findVersion(key: string, packageName: string): string { if (key.startsWith(`/${packageName}/`)) { - return key.slice(packageName.length + 2); + return getVersion(key, packageName); } // for alias packages prepend with "npm:" if (key.startsWith('/')) { const aliasName = key.slice(1, key.lastIndexOf('/')); - const version = key.slice(key.lastIndexOf('/') + 1); + const version = getVersion(key, aliasName); return `npm:${aliasName}@${version}`; } @@ -316,14 +315,25 @@ function findPackageName( return dependencyName; } } - // if package contains org e.g. "/@babel/runtime/7.12.5" + return extractNameFromKey(key); +} + +function getVersion(key: string, packageName: string): string { + const KEY_NAME_SEPARATOR_LENGTH = 2; // leading and trailing slash + + return key.slice(packageName.length + KEY_NAME_SEPARATOR_LENGTH); +} + +function extractNameFromKey(key: string): string { + // if package name contains org e.g. "/@babel/runtime/7.12.5" // we want slice until the third slash if (key.startsWith('/@')) { - const nameIndex = key.indexOf('/', 1); - return key.slice(1, key.indexOf('/', nameIndex + 1)); + // find the position of the '/' after org name + const startFrom = key.indexOf('/', 1); + return key.slice(1, key.indexOf('/', startFrom + 1)); } - // if package name doesnt contain org we slice at the second slash if (key.startsWith('/')) { + // if package has just a name e.g. "/react/7.12.5..." return key.slice(1, key.indexOf('/', 1)); } return key; diff --git a/packages/nx/src/plugins/js/lock-file/utils/pnpm-normalizer.ts b/packages/nx/src/plugins/js/lock-file/utils/pnpm-normalizer.ts index 135cb1b6f4bd93..2c11104b606d16 100644 --- a/packages/nx/src/plugins/js/lock-file/utils/pnpm-normalizer.ts +++ b/packages/nx/src/plugins/js/lock-file/utils/pnpm-normalizer.ts @@ -14,55 +14,10 @@ import { existsSync, readFileSync } from 'fs'; import { workspaceRoot } from '../../../../utils/workspace-root'; import { valid } from 'semver'; -const LOCKFILE_YAML_FORMAT = { - blankLines: true, - lineWidth: 1000, - noCompatMode: true, - noRefs: true, - sortKeys: false, -}; - -const ROOT_KEYS_ORDER = { - lockfileVersion: 1, - // only and never are conflict options. - neverBuiltDependencies: 2, - onlyBuiltDependencies: 2, - overrides: 3, - packageExtensionsChecksum: 4, - patchedDependencies: 5, - specifiers: 10, - dependencies: 11, - optionalDependencies: 12, - devDependencies: 13, - dependenciesMeta: 14, - importers: 15, - packages: 16, -}; - -function isV6Lockfile(data: InlineSpecifiersLockfile | Lockfile) { +export function isV6Lockfile(data: InlineSpecifiersLockfile | Lockfile) { return data.lockfileVersion.toString().startsWith('6.'); } -export function stringifyToPnpmYaml(lockfile: Lockfile): string { - const isLockfileV6 = isV6Lockfile(lockfile); - const adaptedLockfile = isLockfileV6 - ? convertToInlineSpecifiersFormat(lockfile) - : lockfile; - return dump( - sortLockfileKeys( - normalizeLockfile(adaptedLockfile as Lockfile, isLockfileV6) - ), - LOCKFILE_YAML_FORMAT - ); -} - -export function parseAndNormalizePnpmLockfile(content: string): Lockfile { - const lockFileData = load(content); - return revertFromInlineSpecifiersFormatIfNecessary( - convertFromLockfileFileMutable(lockFileData) - ); -} - export function loadPnpmHoistedDepsDefinition() { const fullPath = `${workspaceRoot}/node_modules/.modules.yaml`; @@ -74,44 +29,32 @@ export function loadPnpmHoistedDepsDefinition() { } } +/************************************************************************* + * THE FOLLOWING CODE IS COPIED & simplified FROM @pnpm/lockfile-file for convenience + *************************************************************************/ + /** - * THE FOLLOWING CODE IS COPIED FROM @pnpm/lockfile-file for convenience + * Parsing and mapping logic from pnpm lockfile `read` function + * https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/read.ts#L91 */ - -function isMutableLockfile( - lockfileFile: - | (Omit & - Partial & - Partial>) - | InlineSpecifiersLockfile - | Lockfile -): lockfileFile is Omit & - Partial & - Partial> { - return typeof lockfileFile['importers'] === 'undefined'; +export function parseAndNormalizePnpmLockfile(content: string): Lockfile { + const lockFileData = load(content); + return revertFromInlineSpecifiersFormatIfNecessary( + convertFromLockfileFileMutable(lockFileData) + ); } /** * Reverts changes from the "forceSharedFormat" write option if necessary. + * https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/read.ts#L234 */ -function convertFromLockfileFileMutable( - lockfileFile: - | (Omit & - Partial & - Partial>) - | InlineSpecifiersLockfile - | Lockfile -): InlineSpecifiersLockfile | Lockfile { - if (isMutableLockfile(lockfileFile)) { +function convertFromLockfileFileMutable(lockfileFile: LockfileFile): Lockfile { + if (typeof lockfileFile?.['importers'] === 'undefined') { lockfileFile.importers = { '.': { specifiers: lockfileFile['specifiers'] ?? {}, - ...(lockfileFile['dependenciesMeta'] && { - dependenciesMeta: lockfileFile['dependenciesMeta'], - }), - ...(lockfileFile['publishDirectory'] && { - publishDirectory: lockfileFile['publishDirectory'], - }), + dependenciesMeta: lockfileFile['dependenciesMeta'], + publishDirectory: lockfileFile['publishDirectory'], }, }; delete lockfileFile.specifiers; @@ -121,209 +64,42 @@ function convertFromLockfileFileMutable( delete lockfileFile[depType]; } } - return lockfileFile as Lockfile; - } else { - return lockfileFile; } + return lockfileFile as Lockfile; } -interface InlineSpecifiersLockfile - extends Omit { - lockfileVersion: string; - importers: Record; -} - -interface InlineSpecifiersProjectSnapshot { - dependencies?: InlineSpecifiersResolvedDependencies; - devDependencies?: InlineSpecifiersResolvedDependencies; - optionalDependencies?: InlineSpecifiersResolvedDependencies; - dependenciesMeta?: ProjectSnapshot['dependenciesMeta']; -} - -interface InlineSpecifiersResolvedDependencies { - [depName: string]: SpecifierAndResolution; -} - -interface SpecifierAndResolution { - specifier: string; - version: string; -} - -const INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX = '-inlineSpecifiers'; +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/write.ts#L27 +const LOCKFILE_YAML_FORMAT = { + blankLines: true, + lineWidth: 1000, + noCompatMode: true, + noRefs: true, + sortKeys: false, +}; -function isInlineSpecifierLockfile( - lockfile: InlineSpecifiersLockfile | Lockfile -): lockfile is InlineSpecifiersLockfile { - const { lockfileVersion } = lockfile; - return ( - isV6Lockfile(lockfile) || - (typeof lockfileVersion === 'string' && - lockfileVersion.endsWith( - INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX - )) +/** + * Mapping and writing logic from pnpm lockfile `write` function + * https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/write.ts#L77 + */ +export function stringifyToPnpmYaml(lockfile: Lockfile): string { + const isLockfileV6 = isV6Lockfile(lockfile); + const adaptedLockfile = isLockfileV6 + ? convertToInlineSpecifiersFormat(lockfile) + : lockfile; + return dump( + sortLockfileKeys( + normalizeLockfile(adaptedLockfile as Lockfile, isLockfileV6) + ), + LOCKFILE_YAML_FORMAT ); } -function revertFromInlineSpecifiersFormatIfNecessary( - lockfile: InlineSpecifiersLockfile | Lockfile -): Lockfile { - if (isInlineSpecifierLockfile(lockfile)) { - const { lockfileVersion, importers, ...rest } = lockfile; - - const originalVersionStr = lockfileVersion.replace( - INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX, - '' - ); - const originalVersion = Number(originalVersionStr); - if (isNaN(originalVersion)) { - throw new Error( - `Unable to revert lockfile from inline specifiers format. Invalid version parsed: ${originalVersionStr}` - ); - } - - let revertedImporters = mapValues(importers, revertProjectSnapshot); - let packages = lockfile.packages; - if (originalVersion === 6) { - revertedImporters = Object.fromEntries( - Object.entries(revertedImporters ?? {}).map( - ([importerId, pkgSnapshot]: [string, ProjectSnapshot]) => { - const newSnapshot = { ...pkgSnapshot }; - if (newSnapshot.dependencies != null) { - newSnapshot.dependencies = mapValues( - newSnapshot.dependencies, - convertNewRefToOldRef - ); - } - if (newSnapshot.optionalDependencies != null) { - newSnapshot.optionalDependencies = mapValues( - newSnapshot.optionalDependencies, - convertNewRefToOldRef - ); - } - if (newSnapshot.devDependencies != null) { - newSnapshot.devDependencies = mapValues( - newSnapshot.devDependencies, - convertNewRefToOldRef - ); - } - return [importerId, newSnapshot]; - } - ) - ); - packages = Object.fromEntries( - Object.entries(lockfile.packages ?? {}).map( - ([depPath, pkgSnapshot]) => { - const newSnapshot = { ...pkgSnapshot }; - if (newSnapshot.dependencies != null) { - newSnapshot.dependencies = mapValues( - newSnapshot.dependencies, - convertNewRefToOldRef - ); - } - if (newSnapshot.optionalDependencies != null) { - newSnapshot.optionalDependencies = mapValues( - newSnapshot.optionalDependencies, - convertNewRefToOldRef - ); - } - return [convertNewDepPathToOldDepPath(depPath), newSnapshot]; - } - ) - ); - } - const newLockfile = { - ...rest, - lockfileVersion: lockfileVersion.endsWith( - INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX - ) - ? originalVersion - : lockfileVersion, - packages, - importers: revertedImporters, - }; - if (originalVersion === 6 && newLockfile.time) { - newLockfile.time = Object.fromEntries( - Object.entries(newLockfile.time).map(([depPath, time]) => [ - convertNewDepPathToOldDepPath(depPath), - time, - ]) - ); - } - return newLockfile; - } - return lockfile; -} - -function convertNewDepPathToOldDepPath(oldDepPath: string) { - if (!oldDepPath.includes('@', 2)) return oldDepPath; - const index = oldDepPath.indexOf('@', oldDepPath.indexOf('/@') + 2); - if (oldDepPath.includes('(') && index > oldDepPath.indexOf('(')) - return oldDepPath; - return `${oldDepPath.substring(0, index)}/${oldDepPath.substring(index + 1)}`; -} - -function convertNewRefToOldRef(oldRef: string) { - if (oldRef.startsWith('link:') || oldRef.startsWith('file:')) { - return oldRef; - } - if (oldRef.includes('@')) { - return convertNewDepPathToOldDepPath(oldRef); - } - return oldRef; -} - -function revertProjectSnapshot( - from: InlineSpecifiersProjectSnapshot -): ProjectSnapshot { - const specifiers: ResolvedDependencies = {}; - - function moveSpecifiers( - from: InlineSpecifiersResolvedDependencies - ): ResolvedDependencies { - const resolvedDependencies: ResolvedDependencies = {}; - for (const [depName, { specifier, version }] of Object.entries(from)) { - const existingValue = specifiers[depName]; - if (existingValue != null && existingValue !== specifier) { - throw new Error( - `Project snapshot lists the same dependency more than once with conflicting versions: ${depName}` - ); - } - - specifiers[depName] = specifier; - resolvedDependencies[depName] = version; - } - return resolvedDependencies; - } - - const dependencies: ResolvedDependencies = from.dependencies - ? moveSpecifiers(from.dependencies) - : null; - const devDependencies: ResolvedDependencies = from.devDependencies - ? moveSpecifiers(from.devDependencies) - : null; - const optionalDependencies: ResolvedDependencies = from.optionalDependencies - ? moveSpecifiers(from.optionalDependencies) - : null; - - return { - ...from, - specifiers, - dependencies, - devDependencies, - optionalDependencies, - }; -} - -export const DEPENDENCIES_FIELDS = [ - 'optionalDependencies', - 'dependencies', - 'devDependencies', -]; - +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/write.ts#L99 type LockfileFile = Omit & Partial & Partial>; +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/write.ts#L106 function normalizeLockfile(lockfile: Lockfile, isLockfileV6: boolean) { let lockfileToSave!: LockfileFile; if (Object.keys(lockfile.importers).length === 1 && lockfile.importers['.']) { @@ -402,9 +178,10 @@ function normalizeLockfile(lockfile: Lockfile, isLockfileV6: boolean) { return lockfileToSave; } -function pruneTime( +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/write.ts#L173 +function pruneTimeInLockfileV6( time: Record, - importers: Record + importers: Record ): Record { const rootDepPaths = new Set(); for (const importer of Object.values(importers)) { @@ -416,21 +193,39 @@ function pruneTime( } else { version = ref as string; } - const suffixStart = version.indexOf('_'); + const suffixStart = version.indexOf('('); const refWithoutPeerSuffix = suffixStart === -1 ? version : version.slice(0, suffixStart); - const depPath = dpRefToRelative(refWithoutPeerSuffix, depName); + const depPath = refToRelative(refWithoutPeerSuffix, depName); if (!depPath) continue; rootDepPaths.add(depPath); } } } - return pickBy((depPath) => rootDepPaths.has(depPath), time); + return pickBy((prop) => rootDepPaths.has(prop), time); } -function pruneTimeInLockfileV6( +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/write.ts#L191 +function refToRelative(reference: string, pkgName: string): string | null { + if (reference.startsWith('link:')) { + return null; + } + if (reference.startsWith('file:')) { + return reference; + } + if ( + !reference.includes('/') || + !reference.replace(/(\([^)]+\))+$/, '').includes('/') + ) { + return `/${pkgName}@${reference}`; + } + return reference; +} + +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/write.ts#L207 +function pruneTime( time: Record, - importers: Record + importers: Record ): Record { const rootDepPaths = new Set(); for (const importer of Object.values(importers)) { @@ -442,34 +237,91 @@ function pruneTimeInLockfileV6( } else { version = ref as string; } - const suffixStart = version.indexOf('('); + const suffixStart = version.indexOf('_'); const refWithoutPeerSuffix = suffixStart === -1 ? version : version.slice(0, suffixStart); - const depPath = refToRelative(refWithoutPeerSuffix, depName); + const depPath = dpRefToRelative(refWithoutPeerSuffix, depName); if (!depPath) continue; rootDepPaths.add(depPath); } } } - return pickBy((prop) => rootDepPaths.has(prop), time); + return pickBy((depPath) => rootDepPaths.has(depPath), time); } -function refToRelative(reference: string, pkgName: string): string | null { - if (reference.startsWith('link:')) { - return null; - } - if (reference.startsWith('file:')) { - return reference; - } - if ( - !reference.includes('/') || - !reference.replace(/(\([^)]+\))+$/, '').includes('/') - ) { - return `/${pkgName}@${reference}`; +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/sortLockfileKeys.ts#L34 +const ROOT_KEYS_ORDER = { + lockfileVersion: 1, + // only and never are conflict options. + neverBuiltDependencies: 2, + onlyBuiltDependencies: 2, + overrides: 3, + packageExtensionsChecksum: 4, + patchedDependencies: 5, + specifiers: 10, + dependencies: 11, + optionalDependencies: 12, + devDependencies: 13, + dependenciesMeta: 14, + importers: 15, + packages: 16, +}; + +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/sortLockfileKeys.ts#L60 +function sortLockfileKeys(lockfile: LockfileFile): LockfileFile { + let sortedLockfile = {} as Lockfile; + const sortedKeys = Object.keys(lockfile).sort( + (a, b) => ROOT_KEYS_ORDER[a] - ROOT_KEYS_ORDER[b] + ); + for (const key of sortedKeys) { + sortedLockfile[key] = lockfile[key]; } - return reference; + return sortedLockfile; +} + +/** + * Types from https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/InlineSpecifiersLockfile.ts + */ + +const INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX = '-inlineSpecifiers'; + +interface InlineSpecifiersLockfile + extends Omit { + lockfileVersion: string; + importers: Record; +} + +interface InlineSpecifiersProjectSnapshot { + dependencies?: InlineSpecifiersResolvedDependencies; + devDependencies?: InlineSpecifiersResolvedDependencies; + optionalDependencies?: InlineSpecifiersResolvedDependencies; + dependenciesMeta?: ProjectSnapshot['dependenciesMeta']; +} + +interface InlineSpecifiersResolvedDependencies { + [depName: string]: SpecifierAndResolution; +} + +interface SpecifierAndResolution { + specifier: string; + version: string; +} + +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L10 +function isExperimentalInlineSpecifiersFormat( + lockfile: InlineSpecifiersLockfile | Lockfile +): lockfile is InlineSpecifiersLockfile { + const { lockfileVersion } = lockfile; + return ( + lockfileVersion.toString().startsWith('6.') || + (typeof lockfileVersion === 'string' && + lockfileVersion.endsWith( + INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX + )) + ); } +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L17 function convertToInlineSpecifiersFormat( lockfile: Lockfile ): InlineSpecifiersLockfile { @@ -547,6 +399,25 @@ function convertToInlineSpecifiersFormat( return newLockfile; } +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L72 +function convertOldDepPathToNewDepPath(oldDepPath: string) { + const parsedDepPath = dpParse(oldDepPath); + if (!parsedDepPath.name || !parsedDepPath.version) return oldDepPath; + let newDepPath = `/${parsedDepPath.name}@${parsedDepPath.version}`; + if (parsedDepPath.peersSuffix) { + if (parsedDepPath.peersSuffix.startsWith('(')) { + newDepPath += parsedDepPath.peersSuffix; + } else { + newDepPath += `_${parsedDepPath.peersSuffix}`; + } + } + if (parsedDepPath.host) { + newDepPath = `${parsedDepPath.host}${newDepPath}`; + } + return newDepPath; +} + +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L89 function convertOldRefToNewRef(oldRef: string) { if (oldRef.startsWith('link:') || oldRef.startsWith('file:')) { return oldRef; @@ -557,6 +428,122 @@ function convertOldRefToNewRef(oldRef: string) { return oldRef; } +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L99 +function revertFromInlineSpecifiersFormatIfNecessary( + lockfile: Lockfile | InlineSpecifiersLockfile +): Lockfile { + return isExperimentalInlineSpecifiersFormat(lockfile) + ? revertFromInlineSpecifiersFormat(lockfile) + : lockfile; +} + +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L105 +function revertFromInlineSpecifiersFormat( + lockfile: InlineSpecifiersLockfile +): Lockfile { + const { lockfileVersion, importers, ...rest } = lockfile; + + const originalVersionStr = lockfileVersion.replace( + INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX, + '' + ); + const originalVersion = Number(originalVersionStr); + if (isNaN(originalVersion)) { + throw new Error( + `Unable to revert lockfile from inline specifiers format. Invalid version parsed: ${originalVersionStr}` + ); + } + + let revertedImporters = mapValues(importers, revertProjectSnapshot); + let packages = lockfile.packages; + if (originalVersion === 6) { + revertedImporters = Object.fromEntries( + Object.entries(revertedImporters ?? {}).map( + ([importerId, pkgSnapshot]: [string, ProjectSnapshot]) => { + const newSnapshot = { ...pkgSnapshot }; + if (newSnapshot.dependencies != null) { + newSnapshot.dependencies = mapValues( + newSnapshot.dependencies, + convertNewRefToOldRef + ); + } + if (newSnapshot.optionalDependencies != null) { + newSnapshot.optionalDependencies = mapValues( + newSnapshot.optionalDependencies, + convertNewRefToOldRef + ); + } + if (newSnapshot.devDependencies != null) { + newSnapshot.devDependencies = mapValues( + newSnapshot.devDependencies, + convertNewRefToOldRef + ); + } + return [importerId, newSnapshot]; + } + ) + ); + packages = Object.fromEntries( + Object.entries(lockfile.packages ?? {}).map(([depPath, pkgSnapshot]) => { + const newSnapshot = { ...pkgSnapshot }; + if (newSnapshot.dependencies != null) { + newSnapshot.dependencies = mapValues( + newSnapshot.dependencies, + convertNewRefToOldRef + ); + } + if (newSnapshot.optionalDependencies != null) { + newSnapshot.optionalDependencies = mapValues( + newSnapshot.optionalDependencies, + convertNewRefToOldRef + ); + } + return [convertNewDepPathToOldDepPath(depPath), newSnapshot]; + }) + ); + } + const newLockfile = { + ...rest, + lockfileVersion: lockfileVersion.endsWith( + INLINE_SPECIFIERS_FORMAT_LOCKFILE_VERSION_SUFFIX + ) + ? originalVersion + : lockfileVersion, + packages, + importers: revertedImporters, + }; + if (originalVersion === 6 && newLockfile.time) { + newLockfile.time = Object.fromEntries( + Object.entries(newLockfile.time).map(([depPath, time]) => [ + convertNewDepPathToOldDepPath(depPath), + time, + ]) + ); + } + return newLockfile; +} + +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L162 +function convertNewDepPathToOldDepPath(oldDepPath: string) { + if (!oldDepPath.includes('@', 2)) return oldDepPath; + const index = oldDepPath.indexOf('@', oldDepPath.indexOf('/@') + 2); + if (oldDepPath.includes('(') && index > oldDepPath.indexOf('(')) + return oldDepPath; + return `${oldDepPath.substring(0, index)}/${oldDepPath.substring(index + 1)}`; +} + +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L169 +function convertNewRefToOldRef(oldRef: string) { + if (oldRef.startsWith('link:') || oldRef.startsWith('file:')) { + return oldRef; + } + if (oldRef.includes('@')) { + return convertNewDepPathToOldDepPath(oldRef); + } + return oldRef; +} + +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L179 function convertProjectSnapshotToInlineSpecifiersFormat( projectSnapshot: ProjectSnapshot ): InlineSpecifiersProjectSnapshot { @@ -575,6 +562,7 @@ function convertProjectSnapshotToInlineSpecifiersFormat( } as InlineSpecifiersProjectSnapshot; } +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L195 function convertResolvedDependenciesToInlineSpecifiersFormat( resolvedDependencies: ResolvedDependencies, { specifiers }: { specifiers: ResolvedDependencies } @@ -585,23 +573,50 @@ function convertResolvedDependenciesToInlineSpecifiersFormat( })); } -function convertOldDepPathToNewDepPath(oldDepPath: string) { - const parsedDepPath = dpParse(oldDepPath); - if (!parsedDepPath.name || !parsedDepPath.version) return oldDepPath; - let newDepPath = `/${parsedDepPath.name}@${parsedDepPath.version}`; - if (parsedDepPath.peersSuffix) { - if (parsedDepPath.peersSuffix.startsWith('(')) { - newDepPath += parsedDepPath.peersSuffix; - } else { - newDepPath += `_${parsedDepPath.peersSuffix}`; +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L205 +function revertProjectSnapshot( + from: InlineSpecifiersProjectSnapshot +): ProjectSnapshot { + const specifiers: ResolvedDependencies = {}; + + function moveSpecifiers( + from: InlineSpecifiersResolvedDependencies + ): ResolvedDependencies { + const resolvedDependencies: ResolvedDependencies = {}; + for (const [depName, { specifier, version }] of Object.entries(from)) { + const existingValue = specifiers[depName]; + if (existingValue != null && existingValue !== specifier) { + throw new Error( + `Project snapshot lists the same dependency more than once with conflicting versions: ${depName}` + ); + } + + specifiers[depName] = specifier; + resolvedDependencies[depName] = version; } + return resolvedDependencies; } - if (parsedDepPath.host) { - newDepPath = `${parsedDepPath.host}${newDepPath}`; - } - return newDepPath; + + const dependencies: ResolvedDependencies = from.dependencies + ? moveSpecifiers(from.dependencies) + : null; + const devDependencies: ResolvedDependencies = from.devDependencies + ? moveSpecifiers(from.devDependencies) + : null; + const optionalDependencies: ResolvedDependencies = from.optionalDependencies + ? moveSpecifiers(from.optionalDependencies) + : null; + + return { + ...from, + specifiers, + dependencies, + devDependencies, + optionalDependencies, + }; } +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/lockfile/lockfile-file/src/experiments/inlineSpecifiersLockfileConverters.ts#L241 function mapValues( obj: Record, mapper: (val: T, key: string) => U @@ -613,21 +628,27 @@ function mapValues( return result; } -function sortLockfileKeys(lockfile: LockfileFile): LockfileFile { - let sortedLockfile = {} as Lockfile; - const sortedKeys = Object.keys(lockfile).sort( - (a, b) => ROOT_KEYS_ORDER[a] - ROOT_KEYS_ORDER[b] - ); - for (const key of sortedKeys) { - sortedLockfile[key] = lockfile[key]; - } - return sortedLockfile; -} +/************************************************************************* + * THE FOLLOWING CODE IS COPIED FROM @pnpm/types for convenience + *************************************************************************/ -/** +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/packages/types/src/misc.ts#L6 +const DEPENDENCIES_FIELDS = [ + 'optionalDependencies', + 'dependencies', + 'devDependencies', +]; + +/************************************************************************* * THE FOLLOWING CODE IS COPIED FROM @pnpm/dependency-path for convenience - */ + *************************************************************************/ + +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/packages/dependency-path/src/index.ts#L6 +function isAbsolute(dependencyPath) { + return dependencyPath[0] !== '/'; +} +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/packages/dependency-path/src/index.ts#L80 function dpRefToRelative(reference, pkgName) { if (reference.startsWith('link:')) { return null; @@ -645,10 +666,7 @@ function dpRefToRelative(reference, pkgName) { return reference; } -function isAbsolute(dependencyPath) { - return dependencyPath[0] !== '/'; -} - +// https://github.com/pnpm/pnpm/blob/af3e5559d377870d4c3d303429b3ed1a4e64fedc/packages/dependency-path/src/index.ts#L96 function dpParse(dependencyPath) { // eslint-disable-next-line: strict-type-predicates if (typeof dependencyPath !== 'string') { @@ -706,10 +724,11 @@ function dpParse(dependencyPath) { }; } -/** +/******************************************************************************** * THE FOLLOWING CODE IS COPIED AND SIMPLIFIED FROM @pnpm/ramda for convenience - */ + *******************************************************************************/ +// https://github.com/pnpm/ramda/blob/50c6b57110b2f3631ed8633141f12012b7768d85/source/pickBy.js#L24 function pickBy(test: (prop: string) => boolean, obj: Record) { let result = {}; @@ -722,6 +741,7 @@ function pickBy(test: (prop: string) => boolean, obj: Record) { return result; } +// https://github.com/pnpm/ramda/blob/50c6b57110b2f3631ed8633141f12012b7768d85/source/isEmpty.js#L28 function isEmpty(obj: object) { return obj != null && Object.keys(obj).length === 0; } diff --git a/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts index 649e49641e1622..66232f568782d0 100644 --- a/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts @@ -167,7 +167,7 @@ describe('yarn LockFile utility', () => { }); it('should parse root lock file', async () => { - expect(Object.keys(graph.externalNodes).length).toEqual(1244); // 1104 + expect(Object.keys(graph.externalNodes).length).toEqual(1244); }); it('should prune lock file', async () => { @@ -231,7 +231,7 @@ describe('yarn LockFile utility', () => { const builder = new ProjectGraphBuilder(); parseYarnLockfile(classicLockFile, builder); const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(127); // 124 hoisted + expect(Object.keys(graph.externalNodes).length).toEqual(127); expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` Object { @@ -369,7 +369,7 @@ describe('yarn LockFile utility', () => { const builder = new ProjectGraphBuilder(); parseYarnLockfile(berryLockFile, builder); const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(129); //124 hoisted + expect(Object.keys(graph.externalNodes).length).toEqual(129); expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` Object { @@ -506,7 +506,7 @@ describe('yarn LockFile utility', () => { const builder = new ProjectGraphBuilder(); parseYarnLockfile(classicLockFile, builder); const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(371); //337 hoisted + expect(Object.keys(graph.externalNodes).length).toEqual(371); }); });