From f5ae9958eff61d59c0c54705fc59ecf8662e952a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Jona=C5=A1?= Date: Mon, 15 May 2023 17:30:17 +0200 Subject: [PATCH] feat(core): add commandExternalDependencies hash input (#16916) Co-authored-by: FrozenPandaz --- .../shared/reference/project-configuration.md | 44 +++++ packages/nx/schemas/nx-schema.json | 10 ++ packages/nx/schemas/project-schema.json | 10 ++ .../src/config/workspace-json-project-json.ts | 1 + packages/nx/src/hasher/hasher.spec.ts | 154 ++++++++++++++++-- packages/nx/src/hasher/hasher.ts | 72 ++++++-- 6 files changed, 271 insertions(+), 20 deletions(-) diff --git a/docs/shared/reference/project-configuration.md b/docs/shared/reference/project-configuration.md index 3a66789833dec..cecd6fa7dfee3 100644 --- a/docs/shared/reference/project-configuration.md +++ b/docs/shared/reference/project-configuration.md @@ -158,6 +158,50 @@ Examples: Note the result value is hashed, so it is never displayed. +_External Dependencies_ + +For official plugins, Nx intelligently finds a set of external dependencies which it hashes for the target. `nx:run-commands` is an exception to this. +Because you may specify any command to be run, it is not possible to determine which, if any, external dependencies are used by the target. +To be safe, Nx assumes that updating any external dependency invalidates the cache for the target. + +> Note: Community plugins are also treated like `nx:run-commands` + +This input type allows for you to override this cautious behavior by specifying a set of external dependencies to hash for the target. + +Examples: + +Targets that only use commands natively available in the terminal will not depend on any external dependencies. Specify an empty array to not hash any external dependencies. + +```json +{ + "copyFiles": { + "inputs": [ + { + "externalDependencies": [] + } + ], + "executor": "nx:run-commands", + "command": "cp src/assets dist" + } +} +``` + +If a target uses a command from a npm package, that package should be listed. + +```json +{ + "copyFiles": { + "inputs": [ + { + "externalDependencies": ["lerna"] + } + ], + "executor": "nx:run-commands", + "command": "npx lerna publish" + } +} +``` + _Named Inputs_ Examples: diff --git a/packages/nx/schemas/nx-schema.json b/packages/nx/schemas/nx-schema.json index 7a21c6dc244e9..ddbfbe78eb8f2 100644 --- a/packages/nx/schemas/nx-schema.json +++ b/packages/nx/schemas/nx-schema.json @@ -157,6 +157,16 @@ } }, "additionalProperties": false + }, + { + "type": "object", + "properties": { + "externalDependencies": { + "type": "string", + "description": "The list of external dependencies that our target depends on for `nx:run-commands` and community plugins." + } + }, + "additionalProperties": false } ] } diff --git a/packages/nx/schemas/project-schema.json b/packages/nx/schemas/project-schema.json index 34e9305c998f7..1961c37b65493 100644 --- a/packages/nx/schemas/project-schema.json +++ b/packages/nx/schemas/project-schema.json @@ -206,6 +206,16 @@ } }, "additionalProperties": false + }, + { + "type": "object", + "properties": { + "externalDependencies": { + "type": "string", + "description": "The list of external dependencies that our target depends on for `nx:run-commands` and community plugins." + } + }, + "additionalProperties": false } ] } diff --git a/packages/nx/src/config/workspace-json-project-json.ts b/packages/nx/src/config/workspace-json-project-json.ts index 4b3bd06128fec..8fdec6e61a0e1 100644 --- a/packages/nx/src/config/workspace-json-project-json.ts +++ b/packages/nx/src/config/workspace-json-project-json.ts @@ -131,6 +131,7 @@ export type InputDefinition = | { input: string } | { fileset: string } | { runtime: string } + | { externalDependencies: string[] } | { env: string }; /** diff --git a/packages/nx/src/hasher/hasher.spec.ts b/packages/nx/src/hasher/hasher.spec.ts index e4155ec57392e..38b9eac6a8990 100644 --- a/packages/nx/src/hasher/hasher.spec.ts +++ b/packages/nx/src/hasher/hasher.spec.ts @@ -81,7 +81,7 @@ describe('Hasher', () => { root: 'libs/parent', targets: { build: { - executor: 'unknown', + executor: 'nx:run-commands', inputs: [ 'default', '^default', @@ -136,8 +136,8 @@ describe('Hasher', () => { expect(hash.details.command).toEqual('parent|build||{"prop":"prop-value"}'); expect(hash.details.nodes).toEqual({ 'parent:{projectRoot}/**/*': - '/file|file.hash|{"root":"libs/parent","targets":{"build":{"executor":"unknown","inputs":["default","^default",{"runtime":"echo runtime123"},{"env":"TESTENV"},{"env":"NONEXISTENTENV"},{"input":"default","projects":["unrelated"]}]}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}', - parent: 'unknown', + '/file|file.hash|{"root":"libs/parent","targets":{"build":{"executor":"nx:run-commands","inputs":["default","^default",{"runtime":"echo runtime123"},{"env":"TESTENV"},{"env":"NONEXISTENTENV"},{"input":"default","projects":["unrelated"]}]}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}', + parent: 'nx:run-commands', 'unrelated:{projectRoot}/**/*': 'libs/unrelated/filec.ts|filec.hash|{"root":"libs/unrelated","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}', '{workspaceRoot}/nx.json': 'nx.json.hash', @@ -159,7 +159,7 @@ describe('Hasher', () => { type: 'lib', data: { root: 'libs/parent', - targets: { build: { executor: 'unknown' } }, + targets: { build: { executor: 'nx:run-commands' } }, files: [ { file: '/filea.ts', hash: 'a.hash' }, { file: '/filea.spec.ts', hash: 'a.spec.hash' }, @@ -219,7 +219,7 @@ describe('Hasher', () => { targets: { build: { inputs: ['prod', '^prod'], - executor: 'unknown', + executor: 'nx:run-commands', }, }, files: [ @@ -236,7 +236,7 @@ describe('Hasher', () => { namedInputs: { prod: ['default'], }, - targets: { build: { executor: 'unknown' } }, + targets: { build: { executor: 'nx:run-commands' } }, files: [ { file: 'libs/child/fileb.ts', hash: 'b.hash' }, { file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash' }, @@ -288,12 +288,12 @@ describe('Hasher', () => { targets: { build: { inputs: ['prod'], - executor: 'unknown', + executor: 'nx:run-commands', }, test: { inputs: ['default'], dependsOn: ['build'], - executor: 'unknown', + executor: 'nx:run-commands', }, }, files: [ @@ -356,7 +356,7 @@ describe('Hasher', () => { targets: { test: { inputs: ['default', '^prod'], - executor: 'unknown', + executor: 'nx:run-commands', }, }, files: [ @@ -380,7 +380,7 @@ describe('Hasher', () => { targets: { test: { inputs: ['default'], - executor: 'unknown', + executor: 'nx:run-commands', }, }, files: [ @@ -965,6 +965,140 @@ describe('Hasher', () => { expect(hash.details.nodes['npm:nx']).not.toBeDefined(); expect(hash.details.nodes['app']).toEqual('nx:run-commands'); }); + + it('should use externalDependencies to override nx:run-commands', async () => { + const hasher = new Hasher( + { + nodes: { + app: { + name: 'app', + type: 'app', + data: { + root: 'apps/app', + targets: { + build: { + executor: 'nx:run-commands', + inputs: [ + { fileset: '{projectRoot}/**/*' }, + { externalDependencies: ['webpack', 'react'] }, + ], + }, + }, + files: [{ file: '/filea.ts', hash: 'a.hash' }], + }, + }, + }, + externalNodes: { + 'npm:nx': { + name: 'npm:nx', + type: 'npm', + data: { + packageName: 'nx', + version: '16.0.0', + }, + }, + 'npm:webpack': { + name: 'npm:webpack', + type: 'npm', + data: { + packageName: 'webpack', + version: '5.0.0', + }, + }, + 'npm:react': { + name: 'npm:react', + type: 'npm', + data: { + packageName: 'react', + version: '17.0.0', + }, + }, + }, + dependencies: {}, + allWorkspaceFiles, + }, + {} as any, + {}, + createHashing() + ); + + const hash = await hasher.hashTask({ + target: { project: 'app', target: 'build' }, + id: 'app-build', + overrides: { prop: 'prop-value' }, + }); + + expect(hash.details.nodes['npm:nx']).not.toBeDefined(); + expect(hash.details.nodes['app']).not.toBeDefined(); + expect(hash.details.nodes['npm:webpack']).toEqual('5.0.0'); + expect(hash.details.nodes['npm:react']).toEqual('17.0.0'); + }); + + it('should use externalDependencies with empty array to ignore all deps', async () => { + const hasher = new Hasher( + { + nodes: { + app: { + name: 'app', + type: 'app', + data: { + root: 'apps/app', + targets: { + build: { + executor: 'nx:run-commands', + inputs: [ + { fileset: '{projectRoot}/**/*' }, + { externalDependencies: [] }, // intentionally empty + ], + }, + }, + files: [{ file: '/filea.ts', hash: 'a.hash' }], + }, + }, + }, + externalNodes: { + 'npm:nx': { + name: 'npm:nx', + type: 'npm', + data: { + packageName: 'nx', + version: '16.0.0', + }, + }, + 'npm:webpack': { + name: 'npm:webpack', + type: 'npm', + data: { + packageName: 'webpack', + version: '5.0.0', + }, + }, + 'npm:react': { + name: 'npm:react', + type: 'npm', + data: { + packageName: 'react', + version: '17.0.0', + }, + }, + }, + dependencies: {}, + allWorkspaceFiles, + }, + {} as any, + {}, + createHashing() + ); + + const hash = await hasher.hashTask({ + target: { project: 'app', target: 'build' }, + id: 'app-build', + overrides: { prop: 'prop-value' }, + }); + + expect(hash.details.nodes['npm:nx']).not.toBeDefined(); + expect(hash.details.nodes['app']).not.toBeDefined(); + }); }); describe('expandNamedInput', () => { diff --git a/packages/nx/src/hasher/hasher.ts b/packages/nx/src/hasher/hasher.ts index c41a365f59ae6..0f6b450b66965 100644 --- a/packages/nx/src/hasher/hasher.ts +++ b/packages/nx/src/hasher/hasher.ts @@ -15,7 +15,8 @@ import { hashTsConfig } from '../plugins/js/hasher/hasher'; type ExpandedSelfInput = | { fileset: string } | { runtime: string } - | { env: string }; + | { env: string } + | { externalDependencies: string[] }; /** * A data structure returned by the default hasher. @@ -217,7 +218,11 @@ class TaskHasher { visited ); - const target = this.hashTarget(task.target.project, task.target.target); + const target = this.hashTarget( + task.target.project, + task.target.target, + selfInputs + ); if (target) { return { value: this.hashing.hashArray([selfAndInputs.value, target.value]), @@ -377,7 +382,11 @@ class TaskHasher { return partialHash; } - private hashTarget(projectName: string, targetName: string): PartialHash { + private hashTarget( + projectName: string, + targetName: string, + selfInputs: ExpandedSelfInput[] + ): PartialHash { const projectNode = this.projectGraph.nodes[projectName]; const target = projectNode.data.targets[targetName]; @@ -385,19 +394,41 @@ class TaskHasher { return; } - // we can only vouch for @nrwl packages's executors - // if it's "run commands" we skip traversing since we have no info what this command depends on - // for everything else we take the hash of the @nrwl package dependency tree + // we can only vouch for @nx packages's executor dependencies + // if it's "run commands" or third-party we skip traversing since we have no info what this command depends on if ( target.executor.startsWith(`@nrwl/`) || target.executor.startsWith(`@nx/`) ) { const executorPackage = target.executor.split(':')[0]; - const executorNode = `npm:${executorPackage}`; - if (this.projectGraph.externalNodes?.[executorNode]) { - return this.hashExternalDependency(executorNode); + const executorNodeName = + this.findExternalDependencyNodeName(executorPackage); + return this.hashExternalDependency(executorNodeName); + } + + // use command external dependencies if available to construct the hash + const partialHashes: PartialHash[] = []; + let hasCommandExternalDependencies = false; + for (const input of selfInputs) { + if (input['externalDependencies']) { + // if we have externalDependencies with empty array we still want to override the default hash + hasCommandExternalDependencies = true; + const externalDependencies = input['externalDependencies']; + for (let dep of externalDependencies) { + dep = this.findExternalDependencyNodeName(dep); + partialHashes.push(this.hashExternalDependency(dep)); + } } } + if (hasCommandExternalDependencies) { + return { + value: this.hashing.hashArray(partialHashes.map((h) => h.value)), + details: partialHashes.reduce( + (acc, c) => ({ ...acc, ...c.details }), + {} + ), + }; + } const hash = this.hashing.hashArray([ JSON.stringify(this.projectGraph.externalNodes), @@ -410,6 +441,22 @@ class TaskHasher { }; } + private findExternalDependencyNodeName(packageName: string): string { + if (this.projectGraph.externalNodes[packageName]) { + return packageName; + } + if (this.projectGraph.externalNodes[`npm:${packageName}`]) { + return `npm:${packageName}`; + } + for (const node of Object.values(this.projectGraph.externalNodes)) { + if (node.data.packageName === packageName) { + return node.name; + } + } + // not found, just return the package name + return packageName; + } + private async hashSingleProjectInputs( projectName: string, inputs: ExpandedSelfInput[] @@ -695,7 +742,12 @@ function expandSingleProjectInputs( `namedInputs definitions can only refer to other namedInputs definitions within the same project.` ); } - if ((d as any).fileset || (d as any).env || (d as any).runtime) { + if ( + (d as any).fileset || + (d as any).env || + (d as any).runtime || + (d as any).externalDependencies + ) { expanded.push(d); } else { expanded.push(...expandNamedInput((d as any).input, namedInputs));