From 2e04be3db5da944753ba244cc77693ad8c2cf148 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli Date: Wed, 19 Jul 2023 11:48:55 -0400 Subject: [PATCH 1/7] fix(core): migrate old invalid glob syntax in target outputs --- .../shared/reference/project-configuration.md | 4 +- packages/nx/migrations.json | 6 +++ .../update-16-5-4/update-output-globs.spec.ts | 54 +++++++++++++++++++ .../update-16-5-4/update-output-globs.ts | 34 ++++++++++++ .../nx/src/native/cache/expand_outputs.rs | 16 ++++++ 5 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 packages/nx/src/migrations/update-16-5-4/update-output-globs.spec.ts create mode 100644 packages/nx/src/migrations/update-16-5-4/update-output-globs.ts diff --git a/docs/shared/reference/project-configuration.md b/docs/shared/reference/project-configuration.md index b812fd1f39ae7..8a2062338fd5b 100644 --- a/docs/shared/reference/project-configuration.md +++ b/docs/shared/reference/project-configuration.md @@ -313,13 +313,13 @@ Sometimes, multiple targets might write to the same directory. When possible it } ``` -But if the above is not possible, globs (parsed with the [minimatch](https://github.com/isaacs/minimatch) library) can be specified as outputs to only cache a set of files rather than the whole directory. +But if the above is not possible, globs (parsed by the [GlobSet](https://docs.rs/globset/0.4.5/globset/#syntax) Rust library) can be specified as outputs to only cache a set of files rather than the whole directory. ```json { "targets": { "build-js": { - "outputs": ["{workspaceRoot}/dist/libs/mylib/**/*.js"] + "outputs": ["{workspaceRoot}/dist/libs/mylib/**/*.{js,map}"] }, "build-css": { "outputs": ["{workspaceRoot}/dist/libs/mylib/**/*.css"] diff --git a/packages/nx/migrations.json b/packages/nx/migrations.json index f287f739d45ba..6e74b4d901043 100644 --- a/packages/nx/migrations.json +++ b/packages/nx/migrations.json @@ -77,6 +77,12 @@ "version": "16.2.0-beta.0", "description": "Remove outputPath from run commands", "implementation": "./src/migrations/update-16-2-0/remove-run-commands-output-path" + }, + "16.5.4-update-output-globs": { + "cli": "nx", + "version": "16.5.4-beta.0", + "description": "Update outdated non-standard globs to unix standard", + "implementation": "./src/migrations/update-16-5-4/update-output-globs" } } } diff --git a/packages/nx/src/migrations/update-16-5-4/update-output-globs.spec.ts b/packages/nx/src/migrations/update-16-5-4/update-output-globs.spec.ts new file mode 100644 index 0000000000000..4dc86a5a6c173 --- /dev/null +++ b/packages/nx/src/migrations/update-16-5-4/update-output-globs.spec.ts @@ -0,0 +1,54 @@ +import { createTreeWithEmptyWorkspace } from '../../generators/testing-utils/create-tree-with-empty-workspace'; +import { TargetConfiguration } from '../../config/workspace-json-project-json'; +import { + addProjectConfiguration, + readProjectConfiguration, +} from '../../generators/utils/project-configuration'; +import updateOutputsGlobs from './update-output-globs'; + +describe('update output globs', () => { + it('should update output globs', () => { + const tree = createTreeWithEmptyWorkspace(); + const targets: Record = { + build: { + outputs: ['{options.outputPath}', 'dist/apps/my-app/*.(js|map|ts)'], + }, + lint: {}, + test: { + outputs: ['dist/apps/my-app/main.(js|map|ts)'], + }, + run: { + outputs: ['dist/apps/my-app'], + }, + }; + addProjectConfiguration(tree, 'my-app', { + root: 'apps/my-app', + targets, + }); + + updateOutputsGlobs(tree); + + const migratedTargets = readProjectConfiguration(tree, 'my-app').targets; + expect(migratedTargets).toMatchInlineSnapshot(` + { + "build": { + "outputs": [ + "{options.outputPath}", + "dist/apps/my-app/*.{js,map,ts}", + ], + }, + "lint": {}, + "run": { + "outputs": [ + "dist/apps/my-app", + ], + }, + "test": { + "outputs": [ + "dist/apps/my-app/main.{js,map,ts}", + ], + }, + } + `); + }); +}); diff --git a/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts b/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts new file mode 100644 index 0000000000000..0af15e24449fc --- /dev/null +++ b/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts @@ -0,0 +1,34 @@ +import { Tree } from '../../generators/tree'; +import { + getProjects, + updateProjectConfiguration, +} from '../../generators/utils/project-configuration'; + +export default async function updateOutputsGlobs(tree: Tree) { + for (const [projectName, projectConfiguration] of getProjects( + tree + ).entries()) { + for (const [targetName, targetConfiguration] of Object.entries( + projectConfiguration.targets + )) { + if (!Array.isArray(targetConfiguration.outputs)) { + continue; + } + + let outputs = []; + for (const output of targetConfiguration.outputs ?? []) { + // replace {projectRoot}/folder/*.(js|map|ts) to {projectRoot}/folder/*.{js,map,ts} + const regex = /\(([^)]+)\)/g; + const newOutput = output.replace(regex, (match, group1) => { + let replacements = group1.split('|').join(','); + return `{${replacements}}`; + }); + + outputs.push(newOutput); + } + + targetConfiguration.outputs = outputs; + } + updateProjectConfiguration(tree, projectName, projectConfiguration); + } +} diff --git a/packages/nx/src/native/cache/expand_outputs.rs b/packages/nx/src/native/cache/expand_outputs.rs index f478d7a995a0e..b039b92ba12d2 100644 --- a/packages/nx/src/native/cache/expand_outputs.rs +++ b/packages/nx/src/native/cache/expand_outputs.rs @@ -59,6 +59,10 @@ mod test { .child("nx.darwin-arm64.node") .touch() .unwrap(); + temp.child("multi").child("file.js").touch().unwrap(); + temp.child("multi").child("src.ts").touch().unwrap(); + temp.child("multi").child("file.map").touch().unwrap(); + temp.child("multi").child("file.txt").touch().unwrap(); temp } #[test] @@ -80,4 +84,16 @@ mod test { ] ); } + + #[test] + fn should_handle_multiple_extensions() { + let temp = setup_fs(); + let entries = vec!["multi/*.{js,map,ts}".to_string()]; + let mut result = expand_outputs(temp.display().to_string(), entries).unwrap(); + result.sort(); + assert_eq!( + result, + vec!["multi/file.js", "multi/file.map", "multi/src.ts"] + ); + } } From e65c0bbe2c4b5d6788b2812604ac56abec1eee7f Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli Date: Wed, 19 Jul 2023 12:04:07 -0400 Subject: [PATCH 2/7] fix(core): format with prettier --- packages/nx/src/migrations/update-16-5-4/update-output-globs.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts b/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts index 0af15e24449fc..4c363990c10a1 100644 --- a/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts +++ b/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts @@ -3,6 +3,7 @@ import { getProjects, updateProjectConfiguration, } from '../../generators/utils/project-configuration'; +import { formatChangedFilesWithPrettierIfAvailable } from '../../generators/internal-utils/format-changed-files-with-prettier-if-available'; export default async function updateOutputsGlobs(tree: Tree) { for (const [projectName, projectConfiguration] of getProjects( @@ -31,4 +32,5 @@ export default async function updateOutputsGlobs(tree: Tree) { } updateProjectConfiguration(tree, projectName, projectConfiguration); } + await formatChangedFilesWithPrettierIfAvailable(tree); } From 0edf215c255236c82fa544d643ae3c29ae77d021 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli Date: Wed, 19 Jul 2023 12:38:22 -0400 Subject: [PATCH 3/7] fix(core): handle {projectRoot} {workspaceRoot} {options.*} explicitly in getting outputs --- packages/nx/src/tasks-runner/utils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/nx/src/tasks-runner/utils.ts b/packages/nx/src/tasks-runner/utils.ts index b39f139c79a17..704cac692aa2b 100644 --- a/packages/nx/src/tasks-runner/utils.ts +++ b/packages/nx/src/tasks-runner/utils.ts @@ -182,7 +182,10 @@ export function getOutputsForTargetAndConfiguration( options, }); }) - .filter((output) => !!output && !output.match(/{.*}/)); + .filter( + (output) => + !!output && !output.match(/{(projectRoot|workspaceRoot|(options.*))}/) + ); } // Keep backwards compatibility in case `outputs` doesn't exist From cb8c147f6f146879e31fa3f235c637af2c5422c9 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli Date: Wed, 19 Jul 2023 14:38:19 -0400 Subject: [PATCH 4/7] fix(core): update targetDefaults in nx.json --- .../update-16-5-4/update-output-globs.spec.ts | 23 +++++++++ .../update-16-5-4/update-output-globs.ts | 48 ++++++++++++++----- packages/nx/src/tasks-runner/utils.ts | 3 +- 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/packages/nx/src/migrations/update-16-5-4/update-output-globs.spec.ts b/packages/nx/src/migrations/update-16-5-4/update-output-globs.spec.ts index 4dc86a5a6c173..6798e3ad7bb8a 100644 --- a/packages/nx/src/migrations/update-16-5-4/update-output-globs.spec.ts +++ b/packages/nx/src/migrations/update-16-5-4/update-output-globs.spec.ts @@ -5,6 +5,8 @@ import { readProjectConfiguration, } from '../../generators/utils/project-configuration'; import updateOutputsGlobs from './update-output-globs'; +import { readJson, updateJson } from '../../generators/utils/json'; +import { NxJsonConfiguration } from '../../config/nx-json'; describe('update output globs', () => { it('should update output globs', () => { @@ -26,6 +28,15 @@ describe('update output globs', () => { targets, }); + updateJson(tree, 'nx.json', (json) => { + json.targetDefaults = { + lint: { + outputs: ['dist/apps/my-app', '*.(js|map|ts)'], + }, + }; + return json; + }); + updateOutputsGlobs(tree); const migratedTargets = readProjectConfiguration(tree, 'my-app').targets; @@ -50,5 +61,17 @@ describe('update output globs', () => { }, } `); + + const nxJson = readJson(tree, 'nx.json'); + expect(nxJson.targetDefaults).toMatchInlineSnapshot(` + { + "lint": { + "outputs": [ + "dist/apps/my-app", + "*.{js,map,ts}", + ], + }, + } + `); }); }); diff --git a/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts b/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts index 4c363990c10a1..487ca898a9a02 100644 --- a/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts +++ b/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts @@ -4,6 +4,24 @@ import { updateProjectConfiguration, } from '../../generators/utils/project-configuration'; import { formatChangedFilesWithPrettierIfAvailable } from '../../generators/internal-utils/format-changed-files-with-prettier-if-available'; +import { TargetConfiguration } from '../../config/workspace-json-project-json'; +import { updateJson } from '../../generators/utils/json'; +import { NxJsonConfiguration } from '../../config/nx-json'; + +function replaceOutputs(targetConfiguration: TargetConfiguration) { + let outputs = []; + for (const output of targetConfiguration.outputs ?? []) { + // replace {projectRoot}/folder/*.(js|map|ts) to {projectRoot}/folder/*.{js,map,ts} + const regex = /\(([^)]+)\)/g; + const newOutput = output.replace(regex, (match, group1) => { + let replacements = group1.split('|').join(','); + return `{${replacements}}`; + }); + + outputs.push(newOutput); + } + return outputs; +} export default async function updateOutputsGlobs(tree: Tree) { for (const [projectName, projectConfiguration] of getProjects( @@ -16,21 +34,25 @@ export default async function updateOutputsGlobs(tree: Tree) { continue; } - let outputs = []; - for (const output of targetConfiguration.outputs ?? []) { - // replace {projectRoot}/folder/*.(js|map|ts) to {projectRoot}/folder/*.{js,map,ts} - const regex = /\(([^)]+)\)/g; - const newOutput = output.replace(regex, (match, group1) => { - let replacements = group1.split('|').join(','); - return `{${replacements}}`; - }); - - outputs.push(newOutput); - } - - targetConfiguration.outputs = outputs; + targetConfiguration.outputs = replaceOutputs(targetConfiguration); } updateProjectConfiguration(tree, projectName, projectConfiguration); } + + if (tree.exists('nx.json')) { + updateJson(tree, 'nx.json', (json) => { + for (const [, targetConfiguration] of Object.entries( + json.targetDefaults ?? {} + )) { + if (!Array.isArray(targetConfiguration.outputs)) { + continue; + } + + targetConfiguration.outputs = replaceOutputs(targetConfiguration); + } + return json; + }); + } + await formatChangedFilesWithPrettierIfAvailable(tree); } diff --git a/packages/nx/src/tasks-runner/utils.ts b/packages/nx/src/tasks-runner/utils.ts index 704cac692aa2b..ff3c9c48aa1e6 100644 --- a/packages/nx/src/tasks-runner/utils.ts +++ b/packages/nx/src/tasks-runner/utils.ts @@ -184,7 +184,8 @@ export function getOutputsForTargetAndConfiguration( }) .filter( (output) => - !!output && !output.match(/{(projectRoot|workspaceRoot|(options.*))}/) + !!output && + !output.match(/{(projectRoot|workspaceRoot|(options\.*))}/) ); } From 675b946457ff12810c403d781153f24a3b168a0c Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli Date: Wed, 19 Jul 2023 16:29:40 -0400 Subject: [PATCH 5/7] fix(core): fix regex --- packages/nx/src/tasks-runner/utils.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/nx/src/tasks-runner/utils.ts b/packages/nx/src/tasks-runner/utils.ts index ff3c9c48aa1e6..704cac692aa2b 100644 --- a/packages/nx/src/tasks-runner/utils.ts +++ b/packages/nx/src/tasks-runner/utils.ts @@ -184,8 +184,7 @@ export function getOutputsForTargetAndConfiguration( }) .filter( (output) => - !!output && - !output.match(/{(projectRoot|workspaceRoot|(options\.*))}/) + !!output && !output.match(/{(projectRoot|workspaceRoot|(options.*))}/) ); } From 870a9bd52e348586325caa05c959f64c94a01f4d Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli Date: Wed, 19 Jul 2023 17:26:42 -0400 Subject: [PATCH 6/7] fix(core): review comments --- .../update-16-5-4/update-output-globs.ts | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts b/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts index 487ca898a9a02..47803469a4c3f 100644 --- a/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts +++ b/packages/nx/src/migrations/update-16-5-4/update-output-globs.ts @@ -8,19 +8,13 @@ import { TargetConfiguration } from '../../config/workspace-json-project-json'; import { updateJson } from '../../generators/utils/json'; import { NxJsonConfiguration } from '../../config/nx-json'; -function replaceOutputs(targetConfiguration: TargetConfiguration) { - let outputs = []; - for (const output of targetConfiguration.outputs ?? []) { - // replace {projectRoot}/folder/*.(js|map|ts) to {projectRoot}/folder/*.{js,map,ts} - const regex = /\(([^)]+)\)/g; - const newOutput = output.replace(regex, (match, group1) => { - let replacements = group1.split('|').join(','); - return `{${replacements}}`; - }); - - outputs.push(newOutput); - } - return outputs; +function replaceOutput(output: string) { + // replace {projectRoot}/folder/*.(js|map|ts) to {projectRoot}/folder/*.{js,map,ts} + const regex = /\(([^)]+)\)/g; + return output.replace(regex, (match, group1) => { + let replacements = group1.split('|').join(','); + return `{${replacements}}`; + }); } export default async function updateOutputsGlobs(tree: Tree) { @@ -28,13 +22,14 @@ export default async function updateOutputsGlobs(tree: Tree) { tree ).entries()) { for (const [targetName, targetConfiguration] of Object.entries( - projectConfiguration.targets + projectConfiguration.targets ?? {} )) { if (!Array.isArray(targetConfiguration.outputs)) { continue; } - targetConfiguration.outputs = replaceOutputs(targetConfiguration); + targetConfiguration.outputs = + targetConfiguration.outputs.map(replaceOutput); } updateProjectConfiguration(tree, projectName, projectConfiguration); } @@ -48,7 +43,8 @@ export default async function updateOutputsGlobs(tree: Tree) { continue; } - targetConfiguration.outputs = replaceOutputs(targetConfiguration); + targetConfiguration.outputs = + targetConfiguration.outputs.map(replaceOutput); } return json; }); From 6eb988dd5181da596b66d934164b3c9f25cf70a4 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli Date: Wed, 19 Jul 2023 18:16:55 -0400 Subject: [PATCH 7/7] fix(core): update e2e to test glob cache with multiple extensions --- e2e/nx-run/src/cache.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/e2e/nx-run/src/cache.test.ts b/e2e/nx-run/src/cache.test.ts index d571676106d29..cbebdc873770d 100644 --- a/e2e/nx-run/src/cache.test.ts +++ b/e2e/nx-run/src/cache.test.ts @@ -157,7 +157,7 @@ describe('cache', () => { updateProjectConfig(mylib, (c) => { c.targets.build = { executor: 'nx:run-commands', - outputs: ['{workspaceRoot}/dist/*.txt'], + outputs: ['{workspaceRoot}/dist/*.{txt,md}'], options: { commands: [ 'rm -rf dist', @@ -167,7 +167,8 @@ describe('cache', () => { 'echo c > dist/c.txt', 'echo d > dist/d.txt', 'echo e > dist/e.txt', - 'echo f > dist/f.txt', + 'echo f > dist/f.md', + 'echo g > dist/g.html', ], parallel: false, }, @@ -188,7 +189,8 @@ describe('cache', () => { expect(outputsWithUntouchedOutputs).toContain('c.txt'); expect(outputsWithUntouchedOutputs).toContain('d.txt'); expect(outputsWithUntouchedOutputs).toContain('e.txt'); - expect(outputsWithUntouchedOutputs).toContain('f.txt'); + expect(outputsWithUntouchedOutputs).toContain('f.md'); + expect(outputsWithUntouchedOutputs).toContain('g.html'); // Create a file in the dist that does not match output glob updateFile('dist/c.ts', ''); @@ -202,7 +204,8 @@ describe('cache', () => { expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('c.txt'); expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('d.txt'); expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('e.txt'); - expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('f.txt'); + expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('f.md'); + expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('g.html'); expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('c.ts'); // Clear Dist @@ -217,8 +220,9 @@ describe('cache', () => { expect(outputsWithoutOutputs).toContain('c.txt'); expect(outputsWithoutOutputs).toContain('d.txt'); expect(outputsWithoutOutputs).toContain('e.txt'); - expect(outputsWithoutOutputs).toContain('f.txt'); + expect(outputsWithoutOutputs).toContain('f.md'); expect(outputsWithoutOutputs).not.toContain('c.ts'); + expect(outputsWithoutOutputs).not.toContain('g.html'); }); it('should use consider filesets when hashing', async () => {