From 2e47682fdcfe2714c2c69264620dccb813938fc5 Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Wed, 26 Apr 2023 13:57:37 -0400 Subject: [PATCH] feat(core): add project specific inputs to hasher --- .../shared/reference/project-configuration.md | 4 +- nx-dev/nx-dev/project.json | 1 + packages/nx/src/hasher/hasher.spec.ts | 19 +++- packages/nx/src/hasher/hasher.ts | 102 ++++++++++++++---- packages/nx/src/tasks-runner/utils.ts | 8 +- 5 files changed, 102 insertions(+), 32 deletions(-) diff --git a/docs/shared/reference/project-configuration.md b/docs/shared/reference/project-configuration.md index 653fd72c36eab7..6921290657b58a 100644 --- a/docs/shared/reference/project-configuration.md +++ b/docs/shared/reference/project-configuration.md @@ -163,7 +163,7 @@ _Named Inputs_ Examples: - `inputs: ["production"]` -- same as `inputs: [{input: "production", projects: "self"}]` +- same as `"inputs": [{"input": "production", "projects": "self"}]` in versions prior to Nx 16, or `"inputs": [{"input": "production"}]` after version 16. Often the same glob will appear in many places (e.g., prod fileset will exclude spec files for all projects). Because keeping them in sync is error-prone, we recommend defining `namedInputs`, which you can then reference in all of those @@ -174,7 +174,7 @@ places. Examples: - `inputs: ["^production"]` -- same as `inputs: [{input: "production", projects: "dependencies"}]` +- same as `inputs: [{"input": "production", "projects": "dependencies"}]` prior to Nx 16, or `"inputs": [{"input": "production", "dependencies": true }]` after version 16. Similar to `dependsOn`, the "^" symbols means "dependencies". This is a very important idea, so let's illustrate it with an example. diff --git a/nx-dev/nx-dev/project.json b/nx-dev/nx-dev/project.json index 4c10c84f9b51df..09fdd316650509 100644 --- a/nx-dev/nx-dev/project.json +++ b/nx-dev/nx-dev/project.json @@ -8,6 +8,7 @@ "dependsOn": [ { "target": "build-base", + "projects": "{self}" } ], "executor": "nx:run-commands", diff --git a/packages/nx/src/hasher/hasher.spec.ts b/packages/nx/src/hasher/hasher.spec.ts index 39471543382361..833e45ee6b0324 100644 --- a/packages/nx/src/hasher/hasher.spec.ts +++ b/packages/nx/src/hasher/hasher.spec.ts @@ -88,12 +88,22 @@ describe('Hasher', () => { { runtime: 'echo runtime123' }, { env: 'TESTENV' }, { env: 'NONEXISTENTENV' }, + { input: 'default', projects: ['unrelated'] }, ], }, }, files: [{ file: '/file', hash: 'file.hash' }], }, }, + unrelated: { + name: 'unrelated', + type: 'lib', + data: { + root: 'libs/unrelated', + targets: { build: {} }, + files: [{ file: 'libs/unrelated/filec.ts', hash: 'filec.hash' }], + }, + }, }, dependencies: { parent: [], @@ -121,12 +131,15 @@ describe('Hasher', () => { expect(hash.value).toContain('runtime123'); expect(hash.value).toContain('runtime456'); expect(hash.value).toContain('env123'); + expect(hash.value).toContain('filec.hash'); expect(hash.details.command).toEqual('parent|build||{"prop":"prop-value"}'); expect(hash.details.nodes).toEqual({ 'parent:{projectRoot}/**/*': - '/file|file.hash|{"root":"libs/parent","targets":{"build":{"executor":"unknown","inputs":["default","^default",{"runtime":"echo runtime123"},{"env":"TESTENV"},{"env":"NONEXISTENTENV"}]}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}', + '/file|file.hash|{"root":"libs/parent","targets":{"build":{"executor":"unknown","inputs":["default","^default",{"runtime":"echo runtime123"},{"env":"TESTENV"},{"env":"NONEXISTENTENV"},{"input":"default","projects":["unrelated"]}]}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}', parent: 'unknown', + 'unrelated:{projectRoot}/**/*': + 'libs/unrelated/filec.ts|filec.hash|{"root":"libs/unrelated","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}', '{workspaceRoot}/nx.json': 'nx.json.hash', '{workspaceRoot}/.gitignore': '', '{workspaceRoot}/.nxignore': '', @@ -774,7 +787,7 @@ describe('Hasher', () => { const expanded = expandNamedInput('c', { a: ['a.txt', { fileset: 'myfileset' }], b: ['b.txt'], - c: ['a', { input: 'b', projects: 'self' }], + c: ['a', { input: 'b' }], }); expect(expanded).toEqual([ { fileset: 'a.txt' }, @@ -787,7 +800,7 @@ describe('Hasher', () => { expect(() => expandNamedInput('c', {})).toThrow(); expect(() => expandNamedInput('b', { - b: [{ input: 'c', projects: 'self' }], + b: [{ input: 'c' }], }) ).toThrow(); }); diff --git a/packages/nx/src/hasher/hasher.ts b/packages/nx/src/hasher/hasher.ts index b52784c7cbd06c..1174462e7cab75 100644 --- a/packages/nx/src/hasher/hasher.ts +++ b/packages/nx/src/hasher/hasher.ts @@ -164,11 +164,10 @@ export class Hasher { const DEFAULT_INPUTS: ReadonlyArray = [ { - projects: 'self', fileset: '{projectRoot}/**/*', }, { - projects: 'dependencies', + dependencies: true, input: 'default', }, ]; @@ -201,15 +200,19 @@ class TaskHasher { const targetDefaults = (this.nxJson.targetDefaults || {})[ task.target.target ]; - const { selfInputs, depsInputs } = splitInputsIntoSelfAndDependencies( - targetData.inputs || targetDefaults?.inputs || (DEFAULT_INPUTS as any), - namedInputs - ); + const { selfInputs, depsInputs, projectInputs } = + splitInputsIntoSelfAndDependencies( + targetData.inputs || + targetDefaults?.inputs || + (DEFAULT_INPUTS as any), + namedInputs + ); const selfAndInputs = await this.hashSelfAndDepsInputs( task.target.project, selfInputs, depsInputs, + projectInputs, visited ); @@ -224,7 +227,7 @@ class TaskHasher { }); } - private async hashNamedInput( + private async hashNamedInputForDependencies( projectName: string, namedInput: string, visited: string[] @@ -240,11 +243,12 @@ class TaskHasher { }; const selfInputs = expandNamedInput(namedInput, namedInputs); - const depsInputs = [{ input: namedInput }]; + const depsInputs = [{ input: namedInput, dependencies: true as true }]; // true is boolean by default return this.hashSelfAndDepsInputs( projectName, selfInputs, depsInputs, + [], visited ); } @@ -252,19 +256,21 @@ class TaskHasher { private async hashSelfAndDepsInputs( projectName: string, selfInputs: ExpandedSelfInput[], - depsInputs: { input: string }[], + depsInputs: { input: string; dependencies: true }[], + projectInputs: { input: string; projects: string[] }[], visited: string[] ) { const projectGraphDeps = this.projectGraph.dependencies[projectName] ?? []; // we don't want random order of dependencies to change the hash projectGraphDeps.sort((a, b) => a.target.localeCompare(b.target)); - const self = await this.hashSelfInputs(projectName, selfInputs); + const self = await this.hashSingleProjectInputs(projectName, selfInputs); const deps = await this.hashDepsInputs( depsInputs, projectGraphDeps, visited ); + const projects = await this.hashProjectInputs(projectInputs, visited); let details = {}; for (const s of self) { @@ -273,10 +279,14 @@ class TaskHasher { for (const s of deps) { details = { ...details, ...s.details }; } + for (const s of projects) { + details = { ...details, ...s.details }; + } const value = this.hashing.hashArray([ ...self.map((d) => d.value), ...deps.map((d) => d.value), + ...projects.map((d) => d.value), ]); return { value, details }; @@ -296,7 +306,7 @@ class TaskHasher { return null; } else { visited.push(d.target); - return await this.hashNamedInput( + return await this.hashNamedInputForDependencies( d.target, input.input || 'default', visited @@ -366,7 +376,7 @@ class TaskHasher { }; } - private async hashSelfInputs( + private async hashSingleProjectInputs( projectName: string, inputs: ExpandedSelfInput[] ): Promise { @@ -422,6 +432,29 @@ class TaskHasher { ]); } + private async hashProjectInputs( + projectInputs: { input: string; projects: string[] }[], + visited: string[] + ): Promise { + const partialHashes: Promise[] = []; + for (const input of projectInputs) { + for (const project of input.projects) { + const namedInputs = getNamedInputs( + this.nxJson, + this.projectGraph.nodes[project] + ); + const expandedInput = expandSingleProjectInputs( + [{ input: input.input }], + namedInputs + ); + partialHashes.push( + this.hashSingleProjectInputs(project, expandedInput) + ); + } + } + return Promise.all(partialHashes).then((hashes) => hashes.flat()); + } + private async hashRootFileset(fileset: string): Promise { const mapKey = fileset; const withoutWorkspaceRoot = fileset.substring(16); @@ -559,30 +592,55 @@ export function splitInputsIntoSelfAndDependencies( inputs: ReadonlyArray, namedInputs: { [inputName: string]: ReadonlyArray } ): { - depsInputs: { input: string }[]; + depsInputs: { input: string; dependencies: true }[]; + projectInputs: { input: string; projects: string[] }[]; selfInputs: ExpandedSelfInput[]; } { - const depsInputs = []; + const depsInputs: { input: string; dependencies: true }[] = []; + const projectInputs: { input: string; projects: string[] }[] = []; const selfInputs = []; for (const d of inputs) { if (typeof d === 'string') { if (d.startsWith('^')) { - depsInputs.push({ input: d.substring(1) }); + depsInputs.push({ input: d.substring(1), dependencies: true }); } else { selfInputs.push(d); } } else { - if ((d as any).projects === 'dependencies') { - depsInputs.push(d as any); + if ( + ('dependencies' in d && d.dependencies) || + // Todo(@AgentEnder): Remove check in v17 + ('projects' in d && + typeof d.projects === 'string' && + d.projects === 'dependencies') + ) { + depsInputs.push({ + input: d.input, + dependencies: true, + }); + } else if ( + 'projects' in d && + d.projects && + // Todo(@AgentEnder): Remove check in v17 + !(d.projects === 'self') + ) { + projectInputs.push({ + input: d.input, + projects: Array.isArray(d.projects) ? d.projects : [d.projects], + }); } else { selfInputs.push(d); } } } - return { depsInputs, selfInputs: expandSelfInputs(selfInputs, namedInputs) }; + return { + depsInputs, + projectInputs, + selfInputs: expandSingleProjectInputs(selfInputs, namedInputs), + }; } -function expandSelfInputs( +function expandSingleProjectInputs( inputs: ReadonlyArray, namedInputs: { [inputName: string]: ReadonlyArray } ): ExpandedSelfInput[] { @@ -598,9 +656,9 @@ function expandSelfInputs( expanded.push({ fileset: d }); } } else { - if ((d as any).projects === 'dependencies') { + if ((d as any).projects || (d as any).dependencies) { throw new Error( - `namedInputs definitions cannot contain any inputs with projects == 'dependencies'` + `namedInputs definitions can only refer to other namedInputs definitions within the same project.` ); } if ((d as any).fileset || (d as any).env || (d as any).runtime) { @@ -619,7 +677,7 @@ export function expandNamedInput( ): ExpandedSelfInput[] { namedInputs ||= {}; if (!namedInputs[input]) throw new Error(`Input '${input}' is not defined`); - return expandSelfInputs(namedInputs[input], namedInputs); + return expandSingleProjectInputs(namedInputs[input], namedInputs); } export function filterUsingGlobPatterns( diff --git a/packages/nx/src/tasks-runner/utils.ts b/packages/nx/src/tasks-runner/utils.ts index 3d87cffae00adb..f81a95be0a5b56 100644 --- a/packages/nx/src/tasks-runner/utils.ts +++ b/packages/nx/src/tasks-runner/utils.ts @@ -35,14 +35,12 @@ export function getDependencyConfigs( const specifiers = typeof dependencyConfig.projects === 'string' ? [dependencyConfig.projects] - : dependencyConfig.projects ?? []; - for (const specifier of specifiers) { + : dependencyConfig.projects; + for (const specifier of specifiers ?? []) { if ( !(specifier in projectGraph.nodes) && // Todo(@agentender): Remove the check for self / dependencies in v17 - !['self', 'dependencies'].includes( - specifier - ) + !['self', 'dependencies'].includes(specifier) ) { output.error({ title: `dependsOn is improperly configured for ${project}:${target}`,