From d3a0f7a9af4b0119ead5b06b2ee3018578e6f039 Mon Sep 17 00:00:00 2001 From: AgentEnder Date: Fri, 7 Jul 2023 18:27:35 -0400 Subject: [PATCH] feat(core): add initial draft of v2 project inference --- package.json | 2 +- packages/nx/src/config/project-graph.ts | 1 + packages/nx/src/config/workspaces.spec.ts | 23 +- packages/nx/src/config/workspaces.ts | 212 ++++++++++-------- packages/nx/src/generators/tree.spec.ts | 2 +- .../generators/utils/project-configuration.ts | 4 +- .../build-nodes/workspace-projects.ts | 11 +- .../utils/retrieve-workspace-files.ts | 7 +- .../nx/src/utils/find-matching-projects.ts | 6 +- packages/nx/src/utils/nx-plugin.deprecated.ts | 32 +++ packages/nx/src/utils/nx-plugin.ts | 72 ++++-- pnpm-lock.yaml | 3 + 12 files changed, 243 insertions(+), 132 deletions(-) create mode 100644 packages/nx/src/utils/nx-plugin.deprecated.ts diff --git a/package.json b/package.json index 72295477ea91c1..6cb6268ed9c23a 100644 --- a/package.json +++ b/package.json @@ -298,6 +298,7 @@ "@tailwindcss/line-clamp": "^0.4.2", "@tailwindcss/typography": "^0.5.7", "@types/license-checker": "^25.0.3", + "@types/minimatch": "^5.1.2", "@yarnpkg/lockfile": "^1.1.0", "@yarnpkg/parsers": "3.0.0-rc.46", "@zkochan/js-yaml": "0.0.6", @@ -354,4 +355,3 @@ } } } - diff --git a/packages/nx/src/config/project-graph.ts b/packages/nx/src/config/project-graph.ts index 68a1ad0a82fb90..8c7d880a15169d 100644 --- a/packages/nx/src/config/project-graph.ts +++ b/packages/nx/src/config/project-graph.ts @@ -138,6 +138,7 @@ export interface ProjectGraphProcessorContext { /** * A function that produces an updated ProjectGraph + * @deprecated(v18) Use `buildProjectDependencies` and `buildProjectNodes` instead. */ export type ProjectGraphProcessor = ( currentGraph: ProjectGraph, diff --git a/packages/nx/src/config/workspaces.spec.ts b/packages/nx/src/config/workspaces.spec.ts index 90e10d357d87cd..6611385dc6b283 100644 --- a/packages/nx/src/config/workspaces.spec.ts +++ b/packages/nx/src/config/workspaces.spec.ts @@ -12,12 +12,14 @@ import { TargetConfiguration } from './workspace-json-project-json'; jest.mock('fs', () => require('memfs').fs); -const libConfig = (name) => ({ - root: `libs/${name}`, - sourceRoot: `libs/${name}/src`, +const libConfig = (root, name?: string) => ({ + name: name ?? toProjectName(`${root}/some-file`), + root: `libs/${root}`, + sourceRoot: `libs/${root}/src`, }); -const packageLibConfig = (root) => ({ +const packageLibConfig = (root, name?: string) => ({ + name: name ?? toProjectName(`${root}/some-file`), root, sourceRoot: root, projectType: 'library', @@ -66,13 +68,17 @@ describe('Workspaces', () => { const workspaces = new Workspaces('/root'); const resolved = workspaces.readProjectsConfigurations(); + console.log(resolved); expect(resolved.projects.lib1).toEqual(standaloneConfig); }); it('should build project configurations from glob', () => { const lib1Config = libConfig('lib1'); const lib2Config = packageLibConfig('libs/lib2'); - const domainPackageConfig = packageLibConfig('libs/domain/lib3'); + const domainPackageConfig = packageLibConfig( + 'libs/domain/lib3', + 'domain-lib3' + ); const domainLibConfig = libConfig('domain/lib4'); vol.fromJSON( @@ -94,10 +100,14 @@ describe('Workspaces', () => { const workspaces = new Workspaces('/root'); const { projects } = workspaces.readProjectsConfigurations(); - // projects got deduped so the workspace one remained + console.log(projects); + + // projects got merged for lib1 expect(projects['lib1']).toEqual({ + name: 'lib1', root: 'libs/lib1', sourceRoot: 'libs/lib1/src', + projectType: 'library', }); expect(projects.lib2).toEqual(lib2Config); expect(projects['domain-lib3']).toEqual(domainPackageConfig); @@ -137,6 +147,7 @@ describe('Workspaces', () => { const workspaces = new Workspaces('/root2'); const resolved = workspaces.readProjectsConfigurations(); expect(resolved.projects['my-package']).toEqual({ + name: 'my-package', root: 'packages/my-package', sourceRoot: 'packages/my-package', projectType: 'library', diff --git a/packages/nx/src/config/workspaces.ts b/packages/nx/src/config/workspaces.ts index acb529bd28e3d0..7d070b1c4c6751 100644 --- a/packages/nx/src/config/workspaces.ts +++ b/packages/nx/src/config/workspaces.ts @@ -1,5 +1,5 @@ import { sync as globSync } from 'fast-glob'; -import { existsSync, readFileSync } from 'fs'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; import * as path from 'path'; import { basename, dirname, extname, join } from 'path'; import { performance } from 'perf_hooks'; @@ -9,9 +9,11 @@ import { logger, NX_PREFIX } from '../utils/logger'; import { loadNxPlugins, loadNxPluginsSync, + NxPluginV2, readPluginPackageJson, registerPluginTSTranspiler, } from '../utils/nx-plugin'; +import minimatch = require('minimatch'); import type { NxJsonConfiguration, TargetDefaults } from './nx-json'; import { @@ -108,6 +110,7 @@ export class Workspaces { ), nxJson ), + this.root, (path) => readJsonFile(join(this.root, path)) ); if ( @@ -543,17 +546,9 @@ export function getGlobPatternsFromPlugins( ): string[] { const plugins = loadNxPluginsSync(nxJson?.plugins, paths, root); - const patterns = []; - for (const plugin of plugins) { - if (!plugin.projectFilePatterns) { - continue; - } - for (const filePattern of plugin.projectFilePatterns) { - patterns.push('*/**/' + filePattern); - } - } - - return patterns; + return plugins.flatMap((plugin) => + plugin.buildProjectNodes ? Object.keys(plugin.buildProjectNodes) : [] + ); } export async function getGlobPatternsFromPluginsAsync( @@ -563,17 +558,9 @@ export async function getGlobPatternsFromPluginsAsync( ): Promise { const plugins = await loadNxPlugins(nxJson?.plugins, paths, root); - const patterns = []; - for (const plugin of plugins) { - if (!plugin.projectFilePatterns) { - continue; - } - for (const filePattern of plugin.projectFilePatterns) { - patterns.push('*/**/' + filePattern); - } - } - - return patterns; + return plugins.flatMap((plugin) => + plugin.buildProjectNodes ? Object.keys(plugin.buildProjectNodes) : [] + ); } /** @@ -684,7 +671,7 @@ export function globForProjectFiles( projectGlobPatterns.push(...pluginsGlobPatterns); - const combinedProjectGlobPattern = '{' + projectGlobPatterns.join(',') + '}'; + const combinedProjectGlobPattern = combineGlobPatterns(projectGlobPatterns); performance.mark('start-glob-for-projects'); /** @@ -719,7 +706,7 @@ export function globForProjectFiles( const globResults = globSync(combinedProjectGlobPattern, opts); - projectGlobCache = deduplicateProjectFiles(globResults); + projectGlobCache = globResults; // TODO @vsavkin remove after Nx 16 if ( @@ -742,23 +729,12 @@ export function globForProjectFiles( return projectGlobCache; } -/** - * @description Loops through files and reduces them to 1 file per project. - * @param files Array of files that may represent projects - */ -export function deduplicateProjectFiles(files: string[]): string[] { - const filtered = new Map(); - files.forEach((file) => { - const projectFolder = dirname(file); - const projectFile = basename(file); - if (filtered.has(projectFolder) && projectFile !== 'project.json') return; - filtered.set(projectFolder, projectFile); - }); - - return Array.from(filtered.entries()).map(([folder, file]) => - join(folder, file) - ); -} +const combineGlobPatterns = (patterns: string[]) => + patterns.length > 1 + ? '{' + patterns.join(',') + '}' + : patterns.length === 1 + ? patterns[0] + : ''; function buildProjectConfigurationFromPackageJson( path: string, @@ -787,6 +763,7 @@ function buildProjectConfigurationFromPackageJson( directory.startsWith(nxJson.workspaceLayout.appsDir) ? 'application' : 'library'; + return { root: directory, sourceRoot: directory, @@ -806,66 +783,115 @@ export function inferProjectFromNonStandardFile( }; } +function mergeProjectConfigurationIntoWorkspace( + project: ProjectConfiguration, + existingProjects: Record +): void { + const matchingProject = + existingProjects[project.name]?.root === project.root + ? existingProjects[project.name] + : Object.values(existingProjects).find((c) => c.root === project.root); + + if (!matchingProject) { + existingProjects[project.name] = project; + return; + } + + // This handles top level properties that are overwritten. `srcRoot`, `projectType`, or fields that Nx doesn't know about. + const updatedProjectConfiguration = { ...matchingProject, ...project }; + + // The next blocks handle properties that should be themselves merged (e.g. targets, tags, and implicit dependencies) + + if (project.tags && matchingProject.tags) { + updatedProjectConfiguration.tags = matchingProject.tags.concat( + project.tags + ); + } + + if (project.implicitDependencies && matchingProject.tags) { + updatedProjectConfiguration.implicitDependencies = + matchingProject.implicitDependencies.concat(project.implicitDependencies); + } + + if (project.generators && matchingProject.generators) { + updatedProjectConfiguration.generators = { + ...matchingProject.generators, + ...project.generators, + }; + } + + if (project.targets && matchingProject.targets) { + updatedProjectConfiguration.targets = { + ...matchingProject.targets, + ...project.targets, + }; + } + + if (updatedProjectConfiguration.name !== matchingProject.name) { + delete existingProjects[matchingProject.name]; + } + existingProjects[updatedProjectConfiguration.name] = + updatedProjectConfiguration; +} + export function buildProjectsConfigurationsFromProjectPaths( nxJson: NxJsonConfiguration, projectFiles: string[], // making this parameter allows devkit to pick up newly created projects + root: string = workspaceRoot, readJson: (string) => T = (string) => readJsonFile(string) // making this an arg allows us to reuse in devkit ): Record { const projects: Record = {}; + // We go in reverse here s.t. plugins listed first in the plugins array have highest priority - they overwrite + // whatever configuration was added by plugins later in the array. + const plugins = loadNxPluginsSync(nxJson.plugins).reverse(); - for (const file of projectFiles) { - const directory = dirname(file).split('\\').join('/'); - const fileName = basename(file); - - if (fileName === 'project.json') { - // Nx specific project configuration (`project.json` files) in the same - // directory as a package.json should overwrite the inferred package.json - // project configuration. - const configuration = readJson(file); - - configuration.root = directory; + // We push the nx core node builder onto the end, s.t. it overwrites any user specified behavior + const globPatternsFromPackageManagerWorkspaces = + getGlobPatternsFromPackageManagerWorkspaces(root); + plugins.push({ + name: 'nx-core-build-nodes', + buildProjectNodes: { + // Load projects from pnpm / npm workspaces + ...(globPatternsFromPackageManagerWorkspaces.length + ? ({ + [combineGlobPatterns(globPatternsFromPackageManagerWorkspaces)]: ( + pkgJsonPath + ) => { + const json = readJson(pkgJsonPath); + return { + [json.name]: buildProjectConfigurationFromPackageJson( + pkgJsonPath, + json, + nxJson + ), + }; + }, + } as NxPluginV2['buildProjectNodes']) + : {}), + // Load projects from project.json files. These will be read second, since + // they are listed last in the plugin, so they will overwrite things from the package.json + // based projects. + '{project.json,**/project.json}': (file) => { + const json = readJson(file); + json.name ??= toProjectName(file); + return { + [json.name]: json, + }; + }, + }, + }); - let name = configuration.name; - if (!configuration.name) { - name = toProjectName(file); - } - if (!projects[name]) { - projects[name] = configuration; - } else { - logger.warn( - `Skipping project found at ${directory} since project ${name} already exists at ${projects[name].root}! Specify a unique name for the project to allow Nx to differentiate between the two projects.` - ); - } - } else { - // We can infer projects from package.json files, - // if a package.json file is in a directory w/o a `project.json` file. - // this results in targets being inferred by Nx from package scripts, - // and the root / sourceRoot both being the directory. - if (fileName === 'package.json') { - const projectPackageJson = readJson(file); - const { name, ...config } = buildProjectConfigurationFromPackageJson( - file, - projectPackageJson, - nxJson - ); - if (!projects[name]) { - projects[name] = config; - } else { - logger.warn( - `Skipping project found at ${directory} since project ${name} already exists at ${projects[name].root}! Specify a unique name for the project to allow Nx to differentiate between the two projects.` - ); - } - } else { - // This project was created from an nx plugin. - // The only thing we know about the file is its location - const { name, ...config } = inferProjectFromNonStandardFile(file); - if (!projects[name]) { - projects[name] = config; - } else { - logger.warn( - `Skipping project inferred from ${file} since project ${name} already exists.` - ); + // We iterate over plugins first - this ensures that plugins specified first take precedence. + for (const plugin of plugins) { + // Within a plugin patterns specified later overwrite info from earlier matches. + for (const pattern in plugin.buildProjectNodes ?? {}) { + for (const file of projectFiles) { + if (minimatch(file, pattern)) { + const nodes = plugin.buildProjectNodes[pattern](file, projects); + for (const node in nodes) { + mergeProjectConfigurationIntoWorkspace(nodes[node], projects); + } } } } diff --git a/packages/nx/src/generators/tree.spec.ts b/packages/nx/src/generators/tree.spec.ts index dbfd6c603a26df..91dee8465a653b 100644 --- a/packages/nx/src/generators/tree.spec.ts +++ b/packages/nx/src/generators/tree.spec.ts @@ -19,7 +19,7 @@ describe('tree', () => { beforeEach(() => { console.error = jest.fn(); - console.log = jest.fn(); + // console.log = jest.fn(); dir = dirSync().name; ensureDirSync(path.join(dir, 'parent/child')); diff --git a/packages/nx/src/generators/utils/project-configuration.ts b/packages/nx/src/generators/utils/project-configuration.ts index 6a3e5b7b4113e1..c54f110538a307 100644 --- a/packages/nx/src/generators/utils/project-configuration.ts +++ b/packages/nx/src/generators/utils/project-configuration.ts @@ -5,7 +5,6 @@ import { } from '../../config/workspace-json-project-json'; import { buildProjectsConfigurationsFromProjectPaths, - deduplicateProjectFiles, getGlobPatternsFromPlugins, globForProjectFiles, renamePropertyWithStableKeys, @@ -197,6 +196,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): { return buildProjectsConfigurationsFromProjectPaths( nxJson, projectFiles, + tree.root, (file) => readJson(tree, file) ); } @@ -229,7 +229,7 @@ function findCreatedProjectFiles(tree: Tree) { } } } - return deduplicateProjectFiles(createdProjectFiles).map(normalizePath); + return createdProjectFiles.map(normalizePath); } /** diff --git a/packages/nx/src/project-graph/build-nodes/workspace-projects.ts b/packages/nx/src/project-graph/build-nodes/workspace-projects.ts index 8112a779930e22..a83bf1abb65c25 100644 --- a/packages/nx/src/project-graph/build-nodes/workspace-projects.ts +++ b/packages/nx/src/project-graph/build-nodes/workspace-projects.ts @@ -47,6 +47,11 @@ export async function buildWorkspaceProjectNodes( const p = ctx.projectsConfigurations.projects[key]; const projectRoot = join(workspaceRoot, p.root); + // Todo(@AgentEnder) we can move a lot of this to the buildProjectNodes + // builtin plugin inside workspaces.ts, but there would be some functional differences + // - The plugin would only apply to package.json files found via the workspaces globs + // - This means that scripts / tags / etc from the `nx` property wouldn't be read if a project + // is being found by project.json and not included in the workspaces configuration. Maybe this is fine? if (existsSync(join(projectRoot, 'package.json'))) { p.targets = mergeNpmScriptsWithTargets(projectRoot, p.targets); @@ -77,12 +82,6 @@ export async function buildWorkspaceProjectNodes( partialProjectGraphNodes ); - p.targets = mergePluginTargetsWithNxTargets( - p.root, - p.targets, - await loadNxPlugins(ctx.nxJsonConfiguration.plugins) - ); - p.targets = normalizeProjectTargets(p, nxJson.targetDefaults, key); // TODO: remove in v16 diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index b2df26cd1f039c..01367600f23fea 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -128,8 +128,11 @@ function createProjectConfigurations( performance.mark('build-project-configs:start'); let projectConfigurations = mergeTargetDefaultsIntoProjectDescriptions( - buildProjectsConfigurationsFromProjectPaths(nxJson, configFiles, (path) => - readJsonFile(join(workspaceRoot, path)) + buildProjectsConfigurationsFromProjectPaths( + nxJson, + configFiles, + workspaceRoot, + (path) => readJsonFile(join(workspaceRoot, path)) ), nxJson ); diff --git a/packages/nx/src/utils/find-matching-projects.ts b/packages/nx/src/utils/find-matching-projects.ts index b2888047f2cf8d..04f733e870ac29 100644 --- a/packages/nx/src/utils/find-matching-projects.ts +++ b/packages/nx/src/utils/find-matching-projects.ts @@ -245,7 +245,11 @@ export const getMatchingStringsWithCache = (() => { } const patternCache = minimatchCache.get(pattern)!; if (!regexCache.has(pattern)) { - regexCache.set(pattern, minimatch.makeRe(pattern)); + const regex = minimatch.makeRe(pattern); + if (regex) { + regexCache.set(pattern, regex); + } + throw new Error('Unable to build regex for glob pattern ' + pattern); } const matcher = regexCache.get(pattern); return items.filter((item) => { diff --git a/packages/nx/src/utils/nx-plugin.deprecated.ts b/packages/nx/src/utils/nx-plugin.deprecated.ts new file mode 100644 index 00000000000000..5bc7b843d07977 --- /dev/null +++ b/packages/nx/src/utils/nx-plugin.deprecated.ts @@ -0,0 +1,32 @@ +import { ProjectGraphProcessor } from '../config/project-graph'; +import { TargetConfiguration } from '../config/workspace-json-project-json'; + +/** + * @deprecated(v18) Add targets to the nodes when building them instead. + */ +export type ProjectTargetConfigurator = ( + file: string +) => Record; + +/** + * @deprecated(v18) Use v2 plugins w/ buildProjectNodes and buildProjectDependencies instead. + */ +export type NxPluginV1 = { + name: string; + /** + * @deprecated(v18) Use buildProjectNodes and buildProjectDependencies instead. + */ + processProjectGraph?: ProjectGraphProcessor; + + /** + * @deprecated(v18) Add targets to the nodes inside of buildProjectNodes instead. + */ + registerProjectTargets?: ProjectTargetConfigurator; + + /** + * A glob pattern to search for non-standard project files. + * @example: ["*.csproj", "pom.xml"] + * @deprecated(v18) Use buildProjectNodes instead. + */ + projectFilePatterns?: string[]; +}; diff --git a/packages/nx/src/utils/nx-plugin.ts b/packages/nx/src/utils/nx-plugin.ts index a5ca08f05b60d7..0e8199cd5f1eac 100644 --- a/packages/nx/src/utils/nx-plugin.ts +++ b/packages/nx/src/utils/nx-plugin.ts @@ -2,7 +2,7 @@ import { sync } from 'fast-glob'; import { existsSync } from 'fs'; import * as path from 'path'; import { ProjectGraphProcessor } from '../config/project-graph'; -import { Workspaces } from '../config/workspaces'; +import { toProjectName, Workspaces } from '../config/workspaces'; import { workspaceRoot } from './workspace-root'; import { readJsonFile } from '../utils/fileutils'; @@ -25,30 +25,35 @@ import { findProjectForPath, } from '../project-graph/utils/find-project-for-path'; import { normalizePath } from './path'; -import { join } from 'path'; +import { dirname, join } from 'path'; import { getNxRequirePaths } from './installation-directory'; import { readTsConfig } from '../plugins/js/utils/typescript'; import type * as ts from 'typescript'; +import { NxPluginV1 } from './nx-plugin.deprecated'; -export type ProjectTargetConfigurator = ( - file: string -) => Record; +export type ProjectConfigurationBuilder = ( + projectConfigurationFile: string, + currentProjectConfigurations: Record +) => Record; -/** - * A plugin for Nx - */ -export interface NxPlugin { +export type NxPluginV2 = { name: string; - processProjectGraph?: ProjectGraphProcessor; - registerProjectTargets?: ProjectTargetConfigurator; /** - * A glob pattern to search for non-standard project files. - * @example: ["*.csproj", "pom.xml"] + * Provides a map between file patterns and functions that retrieve configuration info from + * those files. e.g. { '**\/*.csproj': buildProjectsFromCsProjFile } */ - projectFilePatterns?: string[]; -} + buildProjectNodes?: Record; + + // Todo(@AgentEnder): This shouldn't be a full processor, since its only responsible for defining edges between projects. What do we want the API to be? + buildProjectDependencies?: ProjectGraphProcessor; +}; + +/** + * A plugin for Nx + */ +export type NxPlugin = NxPluginV1 | NxPluginV2; // Short lived cache (cleared between cmd runs) // holding resolved nx plugin objects. @@ -129,7 +134,7 @@ export function loadNxPluginsSync( plugins?: string[], paths = getNxRequirePaths(), root = workspaceRoot -): NxPlugin[] { +): (NxPluginV2 & Pick)[] { const result: NxPlugin[] = []; // TODO: This should be specified in nx.json @@ -153,14 +158,14 @@ export function loadNxPluginsSync( } } - return result; + return result.map(ensurePluginIsV2); } export async function loadNxPlugins( plugins?: string[], paths = getNxRequirePaths(), root = workspaceRoot -): Promise { +): Promise<(NxPluginV2 & Pick)[]> { const result: NxPlugin[] = []; // TODO: This should be specified in nx.json @@ -175,7 +180,30 @@ export async function loadNxPlugins( result.push(await loadNxPluginAsync(plugin, paths, root)); } - return result; + return result.map(ensurePluginIsV2); +} + +function ensurePluginIsV2(plugin: NxPlugin): NxPluginV2 { + if ('projectFilePatterns' in plugin && !('buildProjectNodes' in plugin)) { + return { + ...plugin, + buildProjectNodes: { + [`*/**/{${(plugin.projectFilePatterns ?? []).join(',')}}`]: ( + configFilePath + ) => { + const name = toProjectName(configFilePath); + return { + [name]: { + name, + root: dirname(configFilePath), + targets: plugin.registerProjectTargets?.(configFilePath), + }, + }; + }, + }, + }; + } + return plugin; } export function mergePluginTargetsWithNxTargets( @@ -185,7 +213,11 @@ export function mergePluginTargetsWithNxTargets( ): Record { let newTargets: Record = {}; for (const plugin of plugins) { - if (!plugin.projectFilePatterns?.length || !plugin.registerProjectTargets) { + if ( + !('projectFilePatterns' in plugin) || + !plugin.projectFilePatterns?.length || + !plugin.registerProjectTargets + ) { continue; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09a6392aee2642..94765b93a5d692 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,6 +50,9 @@ dependencies: '@types/license-checker': specifier: ^25.0.3 version: 25.0.3 + '@types/minimatch': + specifier: ^5.1.2 + version: 5.1.2 '@yarnpkg/lockfile': specifier: ^1.1.0 version: 1.1.0