From a71a12ccfd4bdbcc3de5188451f1c46b9d18ff52 Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Mon, 24 Jul 2023 10:17:14 -0500 Subject: [PATCH] feat(core): more progress on v2 projects --- docs/generated/devkit/nx_devkit.md | 9 +- .../packages/devkit/documents/nx_devkit.md | 9 +- .../migrators/projects/e2e.migrator.spec.ts | 1 + .../nx/plugins/package-json-workspaces.ts | 154 +++++++++++++ packages/nx/plugins/project-json.ts | 43 ++++ packages/nx/src/config/workspaces.spec.ts | 3 +- packages/nx/src/config/workspaces.ts | 209 ++++-------------- packages/nx/src/generators/tree.spec.ts | 2 +- .../utils/project-configuration.spec.ts | 12 +- .../generators/utils/project-configuration.ts | 95 ++++++-- .../update-15-0-0/migrate-to-inputs.spec.ts | 6 +- .../src/native/tests/workspace_files.spec.ts | 13 +- .../affected/locators/project-glob-changes.ts | 6 +- .../build-nodes/workspace-projects.ts | 4 - .../src/project-graph/build-project-graph.ts | 20 +- .../project-graph/project-graph-builder.ts | 176 +++++++-------- .../utils/retrieve-workspace-files.ts | 12 +- .../nx/src/utils/find-matching-projects.ts | 3 +- packages/nx/src/utils/globs.ts | 6 + packages/nx/src/utils/nx-plugin.ts | 2 +- 20 files changed, 469 insertions(+), 316 deletions(-) create mode 100644 packages/nx/plugins/package-json-workspaces.ts create mode 100644 packages/nx/plugins/project-json.ts create mode 100644 packages/nx/src/utils/globs.ts diff --git a/docs/generated/devkit/nx_devkit.md b/docs/generated/devkit/nx_devkit.md index d29196005db043..0bf49e2d8df409 100644 --- a/docs/generated/devkit/nx_devkit.md +++ b/docs/generated/devkit/nx_devkit.md @@ -1422,7 +1422,7 @@ but it can also be passed in explicitly. ### getProjects -▸ **getProjects**(`tree`): `Map`<`string`, [`ProjectConfiguration`](../../devkit/documents/nx_devkit#projectconfiguration)\> +▸ **getProjects**(`tree`, `includeInferredProjects?`): `Map`<`string`, [`ProjectConfiguration`](../../devkit/documents/nx_devkit#projectconfiguration)\> Get a map of all projects in a workspace. @@ -1430,9 +1430,10 @@ Use [readProjectConfiguration](../../devkit/documents/nx_devkit#readprojectconfi #### Parameters -| Name | Type | -| :----- | :---------------------------------------------- | -| `tree` | [`Tree`](../../devkit/documents/nx_devkit#tree) | +| Name | Type | Default value | +| :------------------------ | :---------------------------------------------- | :------------ | +| `tree` | [`Tree`](../../devkit/documents/nx_devkit#tree) | `undefined` | +| `includeInferredProjects` | `boolean` | `true` | #### Returns diff --git a/docs/generated/packages/devkit/documents/nx_devkit.md b/docs/generated/packages/devkit/documents/nx_devkit.md index d29196005db043..0bf49e2d8df409 100644 --- a/docs/generated/packages/devkit/documents/nx_devkit.md +++ b/docs/generated/packages/devkit/documents/nx_devkit.md @@ -1422,7 +1422,7 @@ but it can also be passed in explicitly. ### getProjects -▸ **getProjects**(`tree`): `Map`<`string`, [`ProjectConfiguration`](../../devkit/documents/nx_devkit#projectconfiguration)\> +▸ **getProjects**(`tree`, `includeInferredProjects?`): `Map`<`string`, [`ProjectConfiguration`](../../devkit/documents/nx_devkit#projectconfiguration)\> Get a map of all projects in a workspace. @@ -1430,9 +1430,10 @@ Use [readProjectConfiguration](../../devkit/documents/nx_devkit#readprojectconfi #### Parameters -| Name | Type | -| :----- | :---------------------------------------------- | -| `tree` | [`Tree`](../../devkit/documents/nx_devkit#tree) | +| Name | Type | Default value | +| :------------------------ | :---------------------------------------------- | :------------ | +| `tree` | [`Tree`](../../devkit/documents/nx_devkit#tree) | `undefined` | +| `includeInferredProjects` | `boolean` | `true` | #### Returns diff --git a/packages/angular/src/generators/ng-add/migrators/projects/e2e.migrator.spec.ts b/packages/angular/src/generators/ng-add/migrators/projects/e2e.migrator.spec.ts index 2103b73204f11c..0a94d76a7ef877 100644 --- a/packages/angular/src/generators/ng-add/migrators/projects/e2e.migrator.spec.ts +++ b/packages/angular/src/generators/ng-add/migrators/projects/e2e.migrator.spec.ts @@ -377,6 +377,7 @@ describe('e2e migrator', () => { await migrator.migrate(); + console.log(tree.listChanges().map((c) => `${c.path} ${c.type}`)); expect(tree.exists('protractor.conf.js')).toBe(false); expect(tree.exists('apps/app1-e2e/protractor.conf.js')).toBe(true); expect(tree.exists('e2e/tsconfig.json')).toBe(false); diff --git a/packages/nx/plugins/package-json-workspaces.ts b/packages/nx/plugins/package-json-workspaces.ts new file mode 100644 index 00000000000000..18ed8a59c2ae8b --- /dev/null +++ b/packages/nx/plugins/package-json-workspaces.ts @@ -0,0 +1,154 @@ +import { existsSync } from 'node:fs'; +import { dirname, join } from 'node:path'; + +import { NxJsonConfiguration } from '../src/config/nx-json'; +import { ProjectConfiguration } from '../src/config/workspace-json-project-json'; +import { toProjectName } from '../src/config/workspaces'; +import { readJsonFile, readYamlFile } from '../src/utils/fileutils'; +import { combineGlobPatterns } from '../src/utils/globs'; +import { NX_PREFIX } from '../src/utils/logger'; +import { NxPluginV2 } from '../src/utils/nx-plugin'; +import { output } from '../src/utils/output'; +import { PackageJson } from '../src/utils/package-json'; +import { joinPathFragments } from '../src/utils/path'; + +export function getPackageJsonWorkspacesPlugin( + root: string, + nxJson: NxJsonConfiguration, + readJson: (string) => T = (string) => + readJsonFile(string) +): NxPluginV2 { + const globPatternsFromPackageManagerWorkspaces = + getGlobPatternsFromPackageManagerWorkspaces(root, readJson); + return { + name: 'nx-core-build-package-json-nodes', + processProjectNodes: { + // Load projects from pnpm / npm workspaces + ...(globPatternsFromPackageManagerWorkspaces.length + ? { + [combineGlobPatterns(globPatternsFromPackageManagerWorkspaces)]: ( + pkgJsonPath + ) => { + const json = readJson(pkgJsonPath); + return { + projectNodes: { + [json.name]: buildProjectConfigurationFromPackageJson( + pkgJsonPath, + json, + nxJson + ), + }, + }; + }, + } + : {}), + }, + }; +} + +function buildProjectConfigurationFromPackageJson( + path: string, + packageJson: { name: string }, + nxJson: NxJsonConfiguration +): ProjectConfiguration & { name: string } { + const normalizedPath = path.split('\\').join('/'); + const directory = dirname(normalizedPath); + + if (!packageJson.name && directory === '.') { + throw new Error( + 'Nx requires the root package.json to specify a name if it is being used as an Nx project.' + ); + } + + let name = packageJson.name ?? toProjectName(normalizedPath); + if (nxJson?.npmScope) { + const npmPrefix = `@${nxJson.npmScope}/`; + if (name.startsWith(npmPrefix)) { + name = name.replace(npmPrefix, ''); + } + } + const projectType = + nxJson?.workspaceLayout?.appsDir != nxJson?.workspaceLayout?.libsDir && + nxJson?.workspaceLayout?.appsDir && + directory.startsWith(nxJson.workspaceLayout.appsDir) + ? 'application' + : 'library'; + + return { + root: directory, + sourceRoot: directory, + name, + projectType, + }; +} + +/** + * Get the package.json globs from package manager workspaces + */ +export function getGlobPatternsFromPackageManagerWorkspaces( + root: string, + readJson: (path: string) => T = (path) => + readJsonFile(join(root, path)) // making this an arg allows us to reuse in devkit +): string[] { + try { + const patterns: string[] = []; + const packageJson = readJson('package.json'); + + patterns.push( + ...normalizePatterns( + Array.isArray(packageJson.workspaces) + ? packageJson.workspaces + : packageJson.workspaces?.packages ?? [] + ) + ); + + if (existsSync(join(root, 'pnpm-workspace.yaml'))) { + try { + const { packages } = readYamlFile<{ packages: string[] }>( + join(root, 'pnpm-workspace.yaml') + ); + patterns.push(...normalizePatterns(packages || [])); + } catch (e: unknown) { + output.warn({ + title: `${NX_PREFIX} Unable to parse pnpm-workspace.yaml`, + bodyLines: [e.toString()], + }); + } + } + + if (existsSync(join(root, 'lerna.json'))) { + try { + const { packages } = readJson('lerna.json'); + patterns.push( + ...normalizePatterns(packages?.length > 0 ? packages : ['packages/*']) + ); + } catch (e: unknown) { + output.warn({ + title: `${NX_PREFIX} Unable to parse lerna.json`, + bodyLines: [e.toString()], + }); + } + } + + // Merge patterns from workspaces definitions + // TODO(@AgentEnder): update logic after better way to determine root project inclusion + // Include the root project + return packageJson.nx ? patterns.concat('package.json') : patterns; + } catch { + return []; + } +} + +function normalizePatterns(patterns: string[]): string[] { + return patterns.map((pattern) => + removeRelativePath( + pattern.endsWith('/package.json') + ? pattern + : joinPathFragments(pattern, 'package.json') + ) + ); +} + +function removeRelativePath(pattern: string): string { + return pattern.startsWith('./') ? pattern.substring(2) : pattern; +} diff --git a/packages/nx/plugins/project-json.ts b/packages/nx/plugins/project-json.ts new file mode 100644 index 00000000000000..a158515c851499 --- /dev/null +++ b/packages/nx/plugins/project-json.ts @@ -0,0 +1,43 @@ +import { dirname } from 'node:path'; + +import { ProjectConfiguration } from '../src/config/workspace-json-project-json'; +import { toProjectName } from '../src/config/workspaces'; +import { readJsonFile } from '../src/utils/fileutils'; +import { NxPluginV2 } from '../src/utils/nx-plugin'; + +export function getProjectJsonPlugin( + readJson: (string) => T = (string) => + readJsonFile(string) +): NxPluginV2 { + // making this an arg allows us to reuse in devkit): NxPluginV2 { + return { + name: 'nx-core-build-project-json-nodes', + processProjectNodes: { + // Load projects from project.json files. These will be read second, since + // they are listed last in the plugin, so they will overwrite things from the package.json + // based projects. + '{project.json,**/project.json}': (file) => { + const json = readJson(file); + const project = buildProjectFromProjectJson(json, file); + return { + projectNodes: { + [project.name]: project, + }, + }; + }, + }, + }; +} + +export function buildProjectFromProjectJson( + json: Partial, + path: string +): ProjectConfiguration { + return { + ...{ + name: toProjectName(path), + root: dirname(path), + }, + ...json, + }; +} diff --git a/packages/nx/src/config/workspaces.spec.ts b/packages/nx/src/config/workspaces.spec.ts index 6611385dc6b283..35a2d08014ca99 100644 --- a/packages/nx/src/config/workspaces.spec.ts +++ b/packages/nx/src/config/workspaces.spec.ts @@ -14,6 +14,7 @@ jest.mock('fs', () => require('memfs').fs); const libConfig = (root, name?: string) => ({ name: name ?? toProjectName(`${root}/some-file`), + projectType: 'library', root: `libs/${root}`, sourceRoot: `libs/${root}/src`, }); @@ -68,7 +69,6 @@ describe('Workspaces', () => { const workspaces = new Workspaces('/root'); const resolved = workspaces.readProjectsConfigurations(); - console.log(resolved); expect(resolved.projects.lib1).toEqual(standaloneConfig); }); @@ -100,7 +100,6 @@ describe('Workspaces', () => { const workspaces = new Workspaces('/root'); const { projects } = workspaces.readProjectsConfigurations(); - console.log(projects); // projects got merged for lib1 expect(projects['lib1']).toEqual({ diff --git a/packages/nx/src/config/workspaces.ts b/packages/nx/src/config/workspaces.ts index 506104556aa8e7..00dd1ca5da0714 100644 --- a/packages/nx/src/config/workspaces.ts +++ b/packages/nx/src/config/workspaces.ts @@ -35,6 +35,12 @@ import { } from '../project-graph/utils/find-project-for-path'; import { readNxJson } from './nx-json'; import { ProjectGraphExternalNode } from './project-graph'; +import { combineGlobPatterns } from '../utils/globs'; +import { + getGlobPatternsFromPackageManagerWorkspaces, + getPackageJsonWorkspacesPlugin, +} from '../../plugins/package-json-workspaces'; +import { getProjectJsonPlugin } from '../../plugins/project-json'; export class Workspaces { private cachedProjectsConfig: ProjectsConfigurations; @@ -216,79 +222,12 @@ export async function getGlobPatternsFromPluginsAsync( ); } -/** - * Get the package.json globs from package manager workspaces - */ -export function getGlobPatternsFromPackageManagerWorkspaces( - root: string -): string[] { - try { - const patterns: string[] = []; - const packageJson = readJsonFile(join(root, 'package.json')); - - patterns.push( - ...normalizePatterns( - Array.isArray(packageJson.workspaces) - ? packageJson.workspaces - : packageJson.workspaces?.packages ?? [] - ) - ); - - if (existsSync(join(root, 'pnpm-workspace.yaml'))) { - try { - const { packages } = readYamlFile<{ packages: string[] }>( - join(root, 'pnpm-workspace.yaml') - ); - patterns.push(...normalizePatterns(packages || [])); - } catch (e: unknown) { - output.warn({ - title: `${NX_PREFIX} Unable to parse pnpm-workspace.yaml`, - bodyLines: [e.toString()], - }); - } - } - - if (existsSync(join(root, 'lerna.json'))) { - try { - const { packages } = readJsonFile(join(root, 'lerna.json')); - patterns.push( - ...normalizePatterns(packages?.length > 0 ? packages : ['packages/*']) - ); - } catch (e: unknown) { - output.warn({ - title: `${NX_PREFIX} Unable to parse lerna.json`, - bodyLines: [e.toString()], - }); - } - } - - // Merge patterns from workspaces definitions - // TODO(@AgentEnder): update logic after better way to determine root project inclusion - // Include the root project - return packageJson.nx ? patterns.concat('package.json') : patterns; - } catch { - return []; - } -} - -function normalizePatterns(patterns: string[]): string[] { - return patterns.map((pattern) => - removeRelativePath( - pattern.endsWith('/package.json') - ? pattern - : joinPathFragments(pattern, 'package.json') - ) - ); -} - -function removeRelativePath(pattern: string): string { - return pattern.startsWith('./') ? pattern.substring(2) : pattern; -} - export function globForProjectFiles( root: string, pluginsGlobPatterns: string[], - nxJson?: NxJsonConfiguration + nxJson?: NxJsonConfiguration, + readJson: (path: string) => T = (path) => + readJsonFile(join(root, path)) // making this an arg allows us to reuse in devkit ) { // Deal w/ Caching const cacheKey = [root, ...pluginsGlobPatterns].join(','); @@ -302,7 +241,7 @@ export function globForProjectFiles( projectGlobCacheKey = cacheKey; const _globPatternsFromPackageManagerWorkspaces = - getGlobPatternsFromPackageManagerWorkspaces(root); + getGlobPatternsFromPackageManagerWorkspaces(root, readJson); const globPatternsFromPackageManagerWorkspaces = _globPatternsFromPackageManagerWorkspaces ?? []; @@ -382,73 +321,38 @@ export function globForProjectFiles( return projectGlobCache; } -const combineGlobPatterns = (patterns: string[]) => - patterns.length > 1 - ? '{' + patterns.join(',') + '}' - : patterns.length === 1 - ? patterns[0] - : ''; - -function buildProjectConfigurationFromPackageJson( - path: string, - packageJson: { name: string }, - nxJson: NxJsonConfiguration -): ProjectConfiguration & { name: string } { - const normalizedPath = path.split('\\').join('/'); - const directory = dirname(normalizedPath); - - if (!packageJson.name && directory === '.') { - throw new Error( - 'Nx requires the root package.json to specify a name if it is being used as an Nx project.' - ); - } - - let name = packageJson.name ?? toProjectName(normalizedPath); - if (nxJson?.npmScope) { - const npmPrefix = `@${nxJson.npmScope}/`; - if (name.startsWith(npmPrefix)) { - name = name.replace(npmPrefix, ''); - } - } - const projectType = - nxJson?.workspaceLayout?.appsDir != nxJson?.workspaceLayout?.libsDir && - nxJson?.workspaceLayout?.appsDir && - directory.startsWith(nxJson.workspaceLayout.appsDir) - ? 'application' - : 'library'; - - return { - root: directory, - sourceRoot: directory, - name, - projectType, - }; -} - -export function inferProjectFromNonStandardFile( - file: string -): ProjectConfiguration & { name: string } { - const directory = dirname(file).split('\\').join('/'); - - return { - name: toProjectName(file), - root: directory, - }; -} - function mergeProjectConfigurationIntoWorkspace( // projectName -> ProjectConfiguration existingProjects: Record, // projectRoot -> projectName existingProjectRootMap: Map, - project: ProjectConfiguration + project: ProjectConfiguration, + // project.json is a special case, so we need to detect it. + file: string ): void { - const matchingProjectName = existingProjectRootMap.get(project.root); + let matchingProjectName = existingProjectRootMap.get(project.root); if (!matchingProjectName) { existingProjects[project.name] = project; existingProjectRootMap.set(project.root, project.name); return; + // There are some special cases for handling project.json - mainly + // that it should override any name the project already has. + } else if ( + project.name && + project.name !== matchingProjectName && + basename(file) === 'project.json' + ) { + // Copy config to new name + existingProjects[project.name] = existingProjects[matchingProjectName]; + // Update name in project config + existingProjects[project.name].name = project.name; + // Update root map to point to new name + existingProjectRootMap[project.root] = project.name; + // Remove entry for old name + delete existingProjects[matchingProjectName]; + // Update name that config should be merged to + matchingProjectName = project.name; } const matchingProject = existingProjects[matchingProjectName]; @@ -508,48 +412,16 @@ export function buildProjectsConfigurationsFromProjectPaths( const externalNodes: Record = {}; // We go in reverse here s.t. plugins listed first in the plugins array have highest priority - they overwrite // whatever configuration was added by plugins later in the array. - const plugins = loadNxPluginsSync(nxJson.plugins).reverse(); + const plugins = nxJson?.plugins?.length + ? loadNxPluginsSync(nxJson.plugins).reverse() + : []; // We push the nx core node builder onto the end, s.t. it overwrites any user specified behavior - const globPatternsFromPackageManagerWorkspaces = - getGlobPatternsFromPackageManagerWorkspaces(root); - const nxCorePlugin: NxPluginV2 = { - name: 'nx-core-build-nodes', - processProjectNodes: { - // Load projects from pnpm / npm workspaces - ...(globPatternsFromPackageManagerWorkspaces.length - ? { - [combineGlobPatterns(globPatternsFromPackageManagerWorkspaces)]: ( - pkgJsonPath - ) => { - const json = readJson(pkgJsonPath); - return { - projectNodes: { - [json.name]: buildProjectConfigurationFromPackageJson( - pkgJsonPath, - json, - nxJson - ), - }, - }; - }, - } - : {}), - // Load projects from project.json files. These will be read second, since - // they are listed last in the plugin, so they will overwrite things from the package.json - // based projects. - '{project.json,**/project.json}': (file) => { - const json = readJson(file); - json.name ??= toProjectName(file); - return { - projectNodes: { - [json.name]: json, - }, - }; - }, - }, - }; - plugins.push(nxCorePlugin); + + const nxCorePlugin: NxPluginV2 = getProjectJsonPlugin(readJson); + const nxPackageManagerWorkspacesPlugin: NxPluginV2 = + getPackageJsonWorkspacesPlugin(root, nxJson, readJson); + plugins.push(nxPackageManagerWorkspacesPlugin, nxCorePlugin); // We iterate over plugins first - this ensures that plugins specified first take precedence. for (const plugin of plugins) { @@ -567,7 +439,8 @@ export function buildProjectsConfigurationsFromProjectPaths( mergeProjectConfigurationIntoWorkspace( projects, projectRootMap, - projectNodes[node] + projectNodes[node], + file ); } Object.assign(externalNodes, pluginExternalNodes); diff --git a/packages/nx/src/generators/tree.spec.ts b/packages/nx/src/generators/tree.spec.ts index 91dee8465a653b..dbfd6c603a26df 100644 --- a/packages/nx/src/generators/tree.spec.ts +++ b/packages/nx/src/generators/tree.spec.ts @@ -19,7 +19,7 @@ describe('tree', () => { beforeEach(() => { console.error = jest.fn(); - // console.log = jest.fn(); + console.log = jest.fn(); dir = dirSync().name; ensureDirSync(path.join(dir, 'parent/child')); diff --git a/packages/nx/src/generators/utils/project-configuration.spec.ts b/packages/nx/src/generators/utils/project-configuration.spec.ts index 9f2c1853986b65..83be6d8a058eda 100644 --- a/packages/nx/src/generators/utils/project-configuration.spec.ts +++ b/packages/nx/src/generators/utils/project-configuration.spec.ts @@ -15,6 +15,7 @@ import { import * as projectSchema from '../../../schemas/project-schema.json'; import { joinPathFragments } from '../../utils/path'; +import { PackageJson } from '../../utils/package-json'; const projectConfiguration: ProjectConfiguration = { name: 'test', @@ -171,6 +172,11 @@ describe('project configuration', () => { describe('for npm workspaces', () => { beforeEach(() => { tree = createTree(); + writeJson(tree, 'package.json', { + name: '@testing/root', + version: '0.0.1', + workspaces: ['*/**/package.json'], + }); }); it('should read project configuration from package.json files', () => { @@ -182,9 +188,10 @@ describe('project configuration', () => { const proj = readProjectConfiguration(tree, 'proj'); expect(proj).toEqual({ + name: 'proj', + projectType: 'library', root: 'proj', sourceRoot: 'proj', - projectType: 'library', }); }); @@ -197,9 +204,10 @@ describe('project configuration', () => { expect(projects.size).toEqual(1); expect(projects.get('proj')).toEqual({ + name: 'proj', + projectType: 'library', root: 'proj', sourceRoot: 'proj', - projectType: 'library', }); }); }); diff --git a/packages/nx/src/generators/utils/project-configuration.ts b/packages/nx/src/generators/utils/project-configuration.ts index 7ca14acbb3a909..ae52b4a7c658cc 100644 --- a/packages/nx/src/generators/utils/project-configuration.ts +++ b/packages/nx/src/generators/utils/project-configuration.ts @@ -18,6 +18,9 @@ import { PackageJson } from '../../utils/package-json'; import { readNxJson } from './nx-json'; import { output } from '../../utils/output'; import { getNxRequirePaths } from '../../utils/installation-directory'; +import minimatch = require('minimatch'); +import { buildProjectFromProjectJson } from '../../../plugins/project-json'; +import { getGlobPatternsFromPackageManagerWorkspaces } from '../../../plugins/package-json-workspaces'; export { readNxJson, updateNxJson } from './nx-json'; export { @@ -88,7 +91,7 @@ export function updateProjectConfiguration( if (!tree.exists(projectConfigFile)) { throw new Error( - `Cannot update Project ${projectName} at ${projectConfiguration.root}. It doesn't exist or uses package.json configuration.` + `Cannot update Project ${projectName} at ${projectConfiguration.root}. It either doesn't exist yet, or may not use project.json for configuration. Use \`addProjectConfiguration()\` instead if you want to create a new project.` ); } writeJson(tree, projectConfigFile, { @@ -134,6 +137,7 @@ export function readProjectConfiguration( projectName: string ): ProjectConfiguration { const allProjects = readAndCombineAllProjectConfigurations(tree); + console.log('allProjects', allProjects); if (!allProjects[projectName]) { // temporary polyfill to make sure our generators work for existing angularcli workspaces if (tree.exists('angular.json')) { @@ -151,8 +155,14 @@ export function readProjectConfiguration( * * Use {@link readProjectConfiguration} if only one project is needed. */ -export function getProjects(tree: Tree): Map { - let allProjects = readAndCombineAllProjectConfigurations(tree); +export function getProjects( + tree: Tree, + includeInferredProjects: boolean = true +): Map { + let allProjects = readAndCombineAllProjectConfigurations( + tree, + includeInferredProjects + ); // temporary polyfill to make sure our generators work for existing angularcli workspaces if (tree.exists('angular.json')) { const angularJson = toNewFormat(readJson(tree, 'angular.json')); @@ -177,28 +187,83 @@ export function getRelativeProjectJsonSchemaPath( ); } -function readAndCombineAllProjectConfigurations(tree: Tree): { +function readAndCombineAllProjectConfigurations( + tree: Tree, + includeInferredProjects: boolean = false +): { [name: string]: ProjectConfiguration; } { const nxJson = readNxJson(tree); + const globPatterns = ( + includeInferredProjects + ? getGlobPatternsFromPlugins( + nxJson, + getNxRequirePaths(tree.root), + tree.root + ).concat( + ...getGlobPatternsFromPackageManagerWorkspaces(tree.root, (p) => + readJson(tree, p) + ) + ) + : [] + ).concat('**/project.json', 'project.json'); + const globbedFiles = globForProjectFiles( tree.root, - getGlobPatternsFromPlugins(nxJson, getNxRequirePaths(tree.root), tree.root), - nxJson + globPatterns, + nxJson, + (p) => readJson(tree, p) ).map(normalizePath); - const createdFiles = findCreatedProjectFiles(tree); - const deletedFiles = findDeletedProjectFiles(tree); + + const createdFiles = findCreatedProjectFiles(tree, globPatterns); + const deletedFiles = findDeletedProjectFiles(tree, globPatterns); const projectFiles = [...globbedFiles, ...createdFiles].filter( (r) => deletedFiles.indexOf(r) === -1 ); - return buildProjectsConfigurationsFromProjectPaths( + console.log('projectFiles', projectFiles); + + const { projects } = buildProjectsConfigurationsFromProjectPaths( nxJson, projectFiles, tree.root, (file) => readJson(tree, file) - ).projects; + ); + + console.log(projects); + + for (const project in projects) { + const config = projects[project]; + const maybeProjectJsonPath = joinPathFragments(config.root, 'project.json'); + if (tree.exists(maybeProjectJsonPath)) { + const json = readJson(tree, maybeProjectJsonPath); + projects[project] = buildProjectFromProjectJson( + json, + maybeProjectJsonPath + ); + } else { + // TODO - We should enable this. It's currently disabled because + // it causes issues with the migrator (e.g. e2e.migrator.ts) code inside @nx/angular. With this disabled, + // inferred projects will be visible in their final state, which could be confusing for folks + // who are writing migration generators. + // + // The reason this breaks the angular migrator stuff is that we write a project.json file + // where projectConfig.root !== dirname(path/to/project.json). We don't support this, but the + // migrator does it as a "staging" step. When we try to read the project.json above its not + // present at the expected location, so the below code would wipe the targets from the project + // Wiping the targets causes the migrator to fail. + // + // // Inferred targets, tags, etc don't show up when running generators + // // This is to help avoid running into issues when trying to update the workspace + // projects[project] = { + // name: project, + // root: config.root, + // }; + } + } + + return projects; } /** @@ -210,14 +275,13 @@ function readAndCombineAllProjectConfigurations(tree: Tree): { * We exclude the root `package.json` from this list unless * considered a project during workspace generation */ -function findCreatedProjectFiles(tree: Tree) { +function findCreatedProjectFiles(tree: Tree, globPatterns: string[]) { const createdProjectFiles = []; for (const change of tree.listChanges()) { if (change.type === 'CREATE') { const fileName = basename(change.path); - // all created project json files are created projects - if (fileName === 'project.json') { + if (globPatterns.some((pattern) => minimatch(change.path, pattern))) { createdProjectFiles.push(change.path); } else if (fileName === 'package.json') { try { @@ -238,14 +302,13 @@ function findCreatedProjectFiles(tree: Tree) { * there is no project.json file, as `glob` * cannot find them. */ -function findDeletedProjectFiles(tree: Tree) { +function findDeletedProjectFiles(tree: Tree, globPatterns: string[]) { return tree .listChanges() .filter((f) => { - const fileName = basename(f.path); return ( f.type === 'DELETE' && - (fileName === 'project.json' || fileName === 'package.json') + globPatterns.some((pattern) => minimatch(f.path, pattern)) ); }) .map((r) => r.path); diff --git a/packages/nx/src/migrations/update-15-0-0/migrate-to-inputs.spec.ts b/packages/nx/src/migrations/update-15-0-0/migrate-to-inputs.spec.ts index c68aeb8c0d4dd2..74f861657df3e6 100644 --- a/packages/nx/src/migrations/update-15-0-0/migrate-to-inputs.spec.ts +++ b/packages/nx/src/migrations/update-15-0-0/migrate-to-inputs.spec.ts @@ -7,7 +7,7 @@ import { readProjectConfiguration, updateNxJson, } from '../../generators/utils/project-configuration'; -import { readJson, writeJson } from '../../generators/utils/json'; +import { readJson, updateJson, writeJson } from '../../generators/utils/json'; import migrateToInputs from './migrate-to-inputs'; import { NxJsonConfiguration } from '../../config/nx-json'; @@ -206,6 +206,10 @@ describe('15.0.0 migration (migrate-to-inputs)', () => { }); it('should add project specific implicit dependencies to projects with package.json', async () => { + updateJson(tree, 'package.json', (j) => ({ + ...j, + workspaces: ['**/package.json'], + })); updateNxJson(tree, { implicitDependencies: { 'tools/scripts/build-app.js': ['app1', 'app2'], diff --git a/packages/nx/src/native/tests/workspace_files.spec.ts b/packages/nx/src/native/tests/workspace_files.spec.ts index 9e067a911bb5c9..e37ac8cc26f8b3 100644 --- a/packages/nx/src/native/tests/workspace_files.spec.ts +++ b/packages/nx/src/native/tests/workspace_files.spec.ts @@ -15,7 +15,10 @@ describe('workspace files', () => { root: dirname(filename), }; } - return res; + return { + projectNodes: res, + externalNodes: {} + }; }; } @@ -234,7 +237,7 @@ describe('workspace files', () => { let globs = ['project.json', '**/project.json', '**/package.json']; - let projectConfigurations = getProjectConfigurations( + let nodes = getProjectConfigurations( fs.tempDir, globs, (filenames) => { @@ -246,10 +249,12 @@ describe('workspace files', () => { root: dirname(filename), }; } - return res; + return { + externalNodes: {}, projectNodes: res + }; } ); - expect(projectConfigurations).toMatchInlineSnapshot(` + expect(nodes.projectNodes).toMatchInlineSnapshot(` { "project1": { "name": "project1", diff --git a/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts b/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts index f5fd0955e9bbf6..35f36643ef1a3e 100644 --- a/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts +++ b/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts @@ -1,13 +1,11 @@ import { TouchedProjectLocator } from '../affected-project-graph-models'; import minimatch = require('minimatch'); -import { - getGlobPatternsFromPackageManagerWorkspaces, - getGlobPatternsFromPluginsAsync, -} from '../../../config/workspaces'; +import { getGlobPatternsFromPluginsAsync } from '../../../config/workspaces'; import { workspaceRoot } from '../../../utils/workspace-root'; import { getNxRequirePaths } from '../../../utils/installation-directory'; import { join } from 'path'; import { existsSync } from 'fs'; +import { getGlobPatternsFromPackageManagerWorkspaces } from '../../../../plugins/package-json-workspaces'; export const getTouchedProjectsFromProjectGlobChanges: TouchedProjectLocator = async (touchedFiles, projectGraphNodes, nxJson): Promise => { diff --git a/packages/nx/src/project-graph/build-nodes/workspace-projects.ts b/packages/nx/src/project-graph/build-nodes/workspace-projects.ts index a83bf1abb65c25..20e412829c36eb 100644 --- a/packages/nx/src/project-graph/build-nodes/workspace-projects.ts +++ b/packages/nx/src/project-graph/build-nodes/workspace-projects.ts @@ -1,10 +1,6 @@ import { join } from 'path'; import { existsSync } from 'fs'; import { workspaceRoot } from '../../utils/workspace-root'; -import { - loadNxPlugins, - mergePluginTargetsWithNxTargets, -} from '../../utils/nx-plugin'; import { ProjectGraphProcessorContext, ProjectGraphProjectNode, diff --git a/packages/nx/src/project-graph/build-project-graph.ts b/packages/nx/src/project-graph/build-project-graph.ts index 65b7f747c931cc..e0169ea9c23fd2 100644 --- a/packages/nx/src/project-graph/build-project-graph.ts +++ b/packages/nx/src/project-graph/build-project-graph.ts @@ -22,7 +22,10 @@ import { } from '../config/project-graph'; import { readJsonFile } from '../utils/fileutils'; import { NxJsonConfiguration } from '../config/nx-json'; -import { ProjectGraphBuilder } from './project-graph-builder'; +import { + ProjectDependencyBuilder, + ProjectGraphBuilder, +} from './project-graph-builder'; import { ProjectConfiguration, ProjectsConfigurations, @@ -216,13 +219,20 @@ async function updateProjectGraphWithPlugins( context: ProjectGraphProcessorContext, initProjectGraph: ProjectGraph ) { - const plugins = ( - await loadNxPlugins(context.nxJsonConfiguration.plugins) - ).filter((x) => !!x.processProjectGraph); + const plugins = await loadNxPlugins(context.nxJsonConfiguration.plugins); let graph = initProjectGraph; for (const plugin of plugins) { try { - graph = await plugin.processProjectGraph(graph, context); + if (plugin.processProjectDependencies) { + const builder = new ProjectDependencyBuilder(graph); + await plugin.processProjectDependencies(builder, { + ...context, + graph, + }); + graph = builder.getUpdatedProjectGraph(); + } else if (plugin.processProjectGraph) { + graph = await plugin.processProjectGraph(graph, context); + } } catch (e) { let message = `Failed to process the project graph with "${plugin.name}".`; if (e instanceof Error) { diff --git a/packages/nx/src/project-graph/project-graph-builder.ts b/packages/nx/src/project-graph/project-graph-builder.ts index 21f3f8f379fa3f..9b9b783b8dcd0d 100644 --- a/packages/nx/src/project-graph/project-graph-builder.ts +++ b/packages/nx/src/project-graph/project-graph-builder.ts @@ -21,6 +21,93 @@ export class ProjectDependencyBuilder { protected readonly fileMap?: ProjectFileMap ) {} + getUpdatedProjectGraph(): ProjectGraph { + for (const sourceProject of Object.keys(this._graph.nodes)) { + const alreadySetTargetProjects = + this.calculateAlreadySetTargetDeps(sourceProject); + this._graph.dependencies[sourceProject] = [ + ...alreadySetTargetProjects.values(), + ].flatMap((depsMap) => [...depsMap.values()]); + + const fileDeps = this.calculateTargetDepsFromFiles(sourceProject); + for (const [targetProject, types] of fileDeps.entries()) { + // only add known nodes + if ( + !this._graph.nodes[targetProject] && + !this._graph.externalNodes[targetProject] + ) { + continue; + } + for (const type of types.values()) { + if ( + !alreadySetTargetProjects.has(targetProject) || + !alreadySetTargetProjects.get(targetProject).has(type) + ) { + if ( + !this.removedEdges[sourceProject] || + !this.removedEdges[sourceProject].has(targetProject) + ) { + this._graph.dependencies[sourceProject].push({ + source: sourceProject, + target: targetProject, + type, + }); + } + } + } + } + } + return this._graph; + } + + private calculateTargetDepsFromFiles( + sourceProject: string + ): Map> { + const fileDeps = new Map>(); + const files = this.fileMap[sourceProject] || []; + if (!files) { + return fileDeps; + } + for (let f of files) { + if (f.deps) { + for (let d of f.deps) { + const target = fileDataDepTarget(d); + if (!fileDeps.has(target)) { + fileDeps.set(target, new Set([fileDataDepType(d)])); + } else { + fileDeps.get(target).add(fileDataDepType(d)); + } + } + } + } + return fileDeps; + } + + private calculateAlreadySetTargetDeps( + sourceProject: string + ): Map> { + const alreadySetTargetProjects = new Map< + string, + Map + >(); + if (this._graph.dependencies[sourceProject]) { + const removed = this.removedEdges[sourceProject]; + for (const d of this._graph.dependencies[sourceProject]) { + // static and dynamic dependencies of internal projects + // will be rebuilt based on the file dependencies + // we only need to keep the implicit dependencies + if (d.type === DependencyType.implicit && !removed?.has(d.target)) { + if (!alreadySetTargetProjects.has(d.target)) { + alreadySetTargetProjects.set(d.target, new Map([[d.type, d]])); + } else { + alreadySetTargetProjects.get(d.target).set(d.type, d); + } + } + } + } + return alreadySetTargetProjects; + } + protected addDependency( sourceProjectName: string, targetProjectName: string, @@ -271,45 +358,6 @@ export class ProjectGraphBuilder extends ProjectDependencyBuilder { this.graph.version = version; } - getUpdatedProjectGraph(): ProjectGraph { - for (const sourceProject of Object.keys(this.graph.nodes)) { - const alreadySetTargetProjects = - this.calculateAlreadySetTargetDeps(sourceProject); - this.graph.dependencies[sourceProject] = [ - ...alreadySetTargetProjects.values(), - ].flatMap((depsMap) => [...depsMap.values()]); - - const fileDeps = this.calculateTargetDepsFromFiles(sourceProject); - for (const [targetProject, types] of fileDeps.entries()) { - // only add known nodes - if ( - !this.graph.nodes[targetProject] && - !this.graph.externalNodes[targetProject] - ) { - continue; - } - for (const type of types.values()) { - if ( - !alreadySetTargetProjects.has(targetProject) || - !alreadySetTargetProjects.get(targetProject).has(type) - ) { - if ( - !this.removedEdges[sourceProject] || - !this.removedEdges[sourceProject].has(targetProject) - ) { - this.graph.dependencies[sourceProject].push({ - source: sourceProject, - target: targetProject, - type, - }); - } - } - } - } - } - return this.graph; - } - private removeDependenciesWithNode(name: string) { // remove all source dependencies delete this.graph.dependencies[name]; @@ -330,54 +378,6 @@ export class ProjectGraphBuilder extends ProjectDependencyBuilder { } } } - - private calculateTargetDepsFromFiles( - sourceProject: string - ): Map> { - const fileDeps = new Map>(); - const files = this.fileMap[sourceProject] || []; - if (!files) { - return fileDeps; - } - for (let f of files) { - if (f.deps) { - for (let d of f.deps) { - const target = fileDataDepTarget(d); - if (!fileDeps.has(target)) { - fileDeps.set(target, new Set([fileDataDepType(d)])); - } else { - fileDeps.get(target).add(fileDataDepType(d)); - } - } - } - } - return fileDeps; - } - - private calculateAlreadySetTargetDeps( - sourceProject: string - ): Map> { - const alreadySetTargetProjects = new Map< - string, - Map - >(); - if (this.graph.dependencies[sourceProject]) { - const removed = this.removedEdges[sourceProject]; - for (const d of this.graph.dependencies[sourceProject]) { - // static and dynamic dependencies of internal projects - // will be rebuilt based on the file dependencies - // we only need to keep the implicit dependencies - if (d.type === DependencyType.implicit && !removed?.has(d.target)) { - if (!alreadySetTargetProjects.has(d.target)) { - alreadySetTargetProjects.set(d.target, new Map([[d.type, d]])); - } else { - alreadySetTargetProjects.get(d.target).set(d.type, d); - } - } - } - } - return alreadySetTargetProjects; - } } interface CandidateDependency { @@ -437,7 +437,7 @@ function validateDynamicDependency( graph: ProjectGraph, d: CandidateDependency ) { - if (this.graph.externalNodes[d.sourceProjectName]) { + if (graph.externalNodes[d.sourceProjectName]) { throw new Error(`External projects can't have "dynamic" dependencies`); } // dynamic dependency is always bound to a file diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index 47b54673596d06..a58ff6a7aa7190 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -1,7 +1,6 @@ import { performance } from 'perf_hooks'; import { buildProjectsConfigurationsFromProjectPaths, - getGlobPatternsFromPackageManagerWorkspaces, getGlobPatternsFromPluginsAsync, mergeTargetConfigurations, readTargetDefaultsForTarget, @@ -24,6 +23,7 @@ import { ProjectGraphExternalNode, } from '../../config/project-graph'; import type { NxWorkspaceFiles } from '../../native'; +import { getGlobPatternsFromPackageManagerWorkspaces } from '../../../plugins/package-json-workspaces'; /** * Walks the workspace directory to create the `projectFileMap`, `ProjectConfigurations` and `allWorkspaceFiles` @@ -57,11 +57,6 @@ export async function retrieveWorkspaceFiles( configs ); - console.log('FOUND 2', { - projectNodes: projectConfigurations.projects, - externalNodes: projectConfigurations.externalNodes, - }); - return { projectNodes: projectConfigurations.projects, externalNodes: projectConfigurations.externalNodes, @@ -108,11 +103,6 @@ export async function retrieveProjectConfigurations( configs ); - console.log('FOUND', { - projectNodes: projectConfigurations, - externalNodes: projectConfigurations.externalNodes, - }); - return { projectNodes: projectConfigurations.projects, externalNodes: projectConfigurations.externalNodes, diff --git a/packages/nx/src/utils/find-matching-projects.ts b/packages/nx/src/utils/find-matching-projects.ts index 04f733e870ac29..e497dc57050c10 100644 --- a/packages/nx/src/utils/find-matching-projects.ts +++ b/packages/nx/src/utils/find-matching-projects.ts @@ -248,8 +248,9 @@ export const getMatchingStringsWithCache = (() => { const regex = minimatch.makeRe(pattern); if (regex) { regexCache.set(pattern, regex); + } else { + throw new Error('Unable to build regex for glob pattern ' + pattern); } - throw new Error('Unable to build regex for glob pattern ' + pattern); } const matcher = regexCache.get(pattern); return items.filter((item) => { diff --git a/packages/nx/src/utils/globs.ts b/packages/nx/src/utils/globs.ts new file mode 100644 index 00000000000000..acefe5aa71878e --- /dev/null +++ b/packages/nx/src/utils/globs.ts @@ -0,0 +1,6 @@ +export const combineGlobPatterns = (patterns: string[]) => + patterns.length > 1 + ? '{' + patterns.join(',') + '}' + : patterns.length === 1 + ? patterns[0] + : ''; diff --git a/packages/nx/src/utils/nx-plugin.ts b/packages/nx/src/utils/nx-plugin.ts index f14ef16b4f5f2e..5cb16c96eb9c40 100644 --- a/packages/nx/src/utils/nx-plugin.ts +++ b/packages/nx/src/utils/nx-plugin.ts @@ -79,7 +79,7 @@ export type DependencyLocator = ( */ filesToProcess: ProjectFileMap; } -) => {}; +) => void | Promise; export type NxPluginV2 = { name: string;