diff --git a/docs/generated/devkit/nx_devkit.md b/docs/generated/devkit/nx_devkit.md index 1c40cbf1f07cb5..232f98c1e2787e 100644 --- a/docs/generated/devkit/nx_devkit.md +++ b/docs/generated/devkit/nx_devkit.md @@ -181,6 +181,8 @@ Type of dependency between projects • **ProjectGraphBuilder**: `Object` +@deprecated(v18): General project graph processors are deprecated. Replace usage with a plugin that utilizes `processProjectNodes` and `processProjectDependencies`. + --- ### Workspaces diff --git a/docs/generated/packages/devkit/documents/nx_devkit.md b/docs/generated/packages/devkit/documents/nx_devkit.md index 1c40cbf1f07cb5..232f98c1e2787e 100644 --- a/docs/generated/packages/devkit/documents/nx_devkit.md +++ b/docs/generated/packages/devkit/documents/nx_devkit.md @@ -181,6 +181,8 @@ Type of dependency between projects • **ProjectGraphBuilder**: `Object` +@deprecated(v18): General project graph processors are deprecated. Replace usage with a plugin that utilizes `processProjectNodes` and `processProjectDependencies`. + --- ### Workspaces diff --git a/packages/nx/src/config/workspaces.ts b/packages/nx/src/config/workspaces.ts index 9e5d9633003297..cb2686d2a58b87 100644 --- a/packages/nx/src/config/workspaces.ts +++ b/packages/nx/src/config/workspaces.ts @@ -39,6 +39,7 @@ import { normalizeProjectRoot, } from '../project-graph/utils/find-project-for-path'; import { getImplementationFactory, resolveSchema } from './schema-utils'; +import { ProjectGraphExternalNode } from './project-graph'; export class Workspaces { private cachedProjectsConfig: ProjectsConfigurations; @@ -107,7 +108,7 @@ export class Workspaces { ), this.root, (path) => readJsonFile(join(this.root, path)) - ); + ).projects; if ( shouldMergeAngularProjects( this.root, @@ -648,7 +649,7 @@ function mergeProjectConfigurationIntoWorkspace( ); } - if (project.implicitDependencies && matchingProject.tags) { + if (project.implicitDependencies && matchingProject.implicitDependencies) { updatedProjectConfiguration.implicitDependencies = matchingProject.implicitDependencies.concat(project.implicitDependencies); } @@ -680,9 +681,13 @@ export function buildProjectsConfigurationsFromProjectPaths( root: string = workspaceRoot, readJson: (string) => T = (string) => readJsonFile(string) // making this an arg allows us to reuse in devkit -): Record { +): { + projects: Record; + externalNodes: Record; +} { const projectRootMap: Map = new Map(); const projects: Record = {}; + const externalNodes: Record = {}; // We go in reverse here s.t. plugins listed first in the plugins array have highest priority - they overwrite // whatever configuration was added by plugins later in the array. const plugins = loadNxPluginsSync(nxJson.plugins).reverse(); @@ -690,25 +695,27 @@ export function buildProjectsConfigurationsFromProjectPaths( // We push the nx core node builder onto the end, s.t. it overwrites any user specified behavior const globPatternsFromPackageManagerWorkspaces = getGlobPatternsFromPackageManagerWorkspaces(root); - plugins.push({ + const nxCorePlugin: NxPluginV2 = { name: 'nx-core-build-nodes', processProjectNodes: { // Load projects from pnpm / npm workspaces ...(globPatternsFromPackageManagerWorkspaces.length - ? ({ + ? { [combineGlobPatterns(globPatternsFromPackageManagerWorkspaces)]: ( pkgJsonPath ) => { const json = readJson(pkgJsonPath); return { - [json.name]: buildProjectConfigurationFromPackageJson( - pkgJsonPath, - json, - nxJson - ), + projectNodes: { + [json.name]: buildProjectConfigurationFromPackageJson( + pkgJsonPath, + json, + nxJson + ), + }, }; }, - } as NxPluginV2['processProjectNodes']) + } : {}), // Load projects from project.json files. These will be read second, since // they are listed last in the plugin, so they will overwrite things from the package.json @@ -716,12 +723,16 @@ export function buildProjectsConfigurationsFromProjectPaths( '{project.json,**/project.json}': (file) => { const json = readJson(file); json.name ??= toProjectName(file); + json.root ??= dirname(file).split('\\').join('/'); return { - [json.name]: json, + projectNodes: { + [json.name]: json, + }, }; }, }, - }); + }; + plugins.push(nxCorePlugin); // We iterate over plugins first - this ensures that plugins specified first take precedence. for (const plugin of plugins) { @@ -729,24 +740,26 @@ export function buildProjectsConfigurationsFromProjectPaths( for (const pattern in plugin.processProjectNodes ?? {}) { for (const file of projectFiles) { if (minimatch(file, pattern)) { - const nodes = plugin.processProjectNodes[pattern](file, { - projectsConfigurations: projects, - nxJsonConfiguration: nxJson, - workspaceRoot: root - }); - for (const node in nodes) { + const { projectNodes, externalNodes: pluginExternalNodes } = + plugin.processProjectNodes[pattern](file, { + projectsConfigurations: projects, + nxJsonConfiguration: nxJson, + workspaceRoot: root, + }); + for (const node in projectNodes) { mergeProjectConfigurationIntoWorkspace( projects, projectRootMap, - nodes[node] + projectNodes[node] ); } + Object.assign(externalNodes, pluginExternalNodes); } } } } - return projects; + return { projects, externalNodes }; } export function mergeTargetConfigurations( diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts index b8e86f96b18bfa..9b9a3cbe1cad49 100644 --- a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts +++ b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts @@ -3,6 +3,7 @@ import { FileData, ProjectFileMap, ProjectGraph, + ProjectGraphExternalNode, } from '../../config/project-graph'; import { buildProjectGraphUsingProjectFileMap } from '../../project-graph/build-project-graph'; import { updateProjectFileMap } from '../../project-graph/file-map-utils'; @@ -23,6 +24,7 @@ import { retrieveProjectConfigurations, } from '../../project-graph/utils/retrieve-workspace-files'; import { ProjectConfiguration } from '../../config/workspace-json-project-json'; +import { ProjectsConfigurations } from '../../config/workspace-json-project-json'; let cachedSerializedProjectGraphPromise: Promise<{ error: Error | null; @@ -42,6 +44,7 @@ const collectedDeletedFiles = new Set(); let storedWorkspaceConfigHash: string | undefined; let waitPeriod = 100; let scheduledTimeoutId; +let knownExternalNodes: Record = {}; export async function getCachedSerializedProjectGraphPromise() { try { @@ -172,14 +175,12 @@ async function processCollectedUpdatedAndDeletedFiles() { let nxJson = new Workspaces(workspaceRoot).readNxJson(); - const projectConfigurations = await retrieveProjectConfigurations( + const { projectNodes } = await retrieveProjectConfigurations( workspaceRoot, nxJson ); - const workspaceConfigHash = computeWorkspaceConfigHash( - projectConfigurations - ); + const workspaceConfigHash = computeWorkspaceConfigHash(projectNodes); serverLogger.requestLog( `Updated file-hasher based on watched changes, recomputing project graph...` ); @@ -190,14 +191,12 @@ async function processCollectedUpdatedAndDeletedFiles() { if (workspaceConfigHash !== storedWorkspaceConfigHash) { storedWorkspaceConfigHash = workspaceConfigHash; - projectFileMapWithFiles = await retrieveWorkspaceFiles( - workspaceRoot, - nxJson - ); + ({ externalNodes: knownExternalNodes, ...projectFileMapWithFiles } = + await retrieveWorkspaceFiles(workspaceRoot, nxJson)); } else { if (projectFileMapWithFiles) { projectFileMapWithFiles = updateProjectFileMap( - projectConfigurations, + projectNodes, projectFileMapWithFiles.projectFileMap, projectFileMapWithFiles.allWorkspaceFiles, updatedFiles, @@ -275,7 +274,8 @@ async function createAndSerializeProjectGraph(): Promise<{ const { projectGraph, projectFileMapCache } = await buildProjectGraphUsingProjectFileMap( projectsConfigurations, - projectFileMap, + knownExternalNodes, + projectFileMapWithFiles.projectFileMap, allWorkspaceFiles, currentProjectFileMapCache || readProjectFileMapCache(), true diff --git a/packages/nx/src/generators/utils/project-configuration.ts b/packages/nx/src/generators/utils/project-configuration.ts index c54f110538a307..7ca14acbb3a909 100644 --- a/packages/nx/src/generators/utils/project-configuration.ts +++ b/packages/nx/src/generators/utils/project-configuration.ts @@ -198,7 +198,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): { projectFiles, tree.root, (file) => readJson(tree, file) - ); + ).projects; } /** diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts index d5445ab9341d42..03cab44261f8fb 100644 --- a/packages/nx/src/native/index.d.ts +++ b/packages/nx/src/native/index.d.ts @@ -37,13 +37,18 @@ export const enum WorkspaceErrors { Generic = 'Generic' } /** Get workspace config files based on provided globs */ -export function getProjectConfigurations(workspaceRoot: string, globs: Array, parseConfigurations: (arg0: Array) => Record): Record +export function getProjectConfigurations(workspaceRoot: string, globs: Array, parseConfigurations: (arg0: Array) => ConfigurationParserResult): ConfigurationParserResult export interface NxWorkspaceFiles { projectFileMap: Record> globalFiles: Array projectConfigurations: Record + externalNodes: Record +} +export function getWorkspaceFilesNative(workspaceRoot: string, globs: Array, parseConfigurations: (arg0: Array) => ConfigurationParserResult): NxWorkspaceFiles +export interface ConfigurationParserResult { + projectNodes: Record + externalNodes: Record } -export function getWorkspaceFilesNative(workspaceRoot: string, globs: Array, parseConfigurations: (arg0: Array) => Record): NxWorkspaceFiles export class Watcher { origin: string /** diff --git a/packages/nx/src/native/workspace/get_config_files.rs b/packages/nx/src/native/workspace/get_config_files.rs index 518cc3b60827fa..1d1c59e8e8825b 100644 --- a/packages/nx/src/native/workspace/get_config_files.rs +++ b/packages/nx/src/native/workspace/get_config_files.rs @@ -1,9 +1,9 @@ use crate::native::utils::glob::build_glob_set; use crate::native::utils::path::Normalize; use crate::native::walker::nx_walker; +use crate::native::workspace::types::ConfigurationParserResult; use globset::GlobSet; -use napi::JsObject; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::path::{Path, PathBuf}; @@ -13,11 +13,10 @@ use std::path::{Path, PathBuf}; pub fn get_project_configurations( workspace_root: String, globs: Vec, - parse_configurations: ConfigurationParser, -) -> napi::Result> +) -> napi::Result where - ConfigurationParser: Fn(Vec) -> napi::Result>, +ConfigurationParser: Fn(Vec) -> napi::Result { let globs = build_glob_set(globs)?; let config_paths: Vec = nx_walker(workspace_root, move |rec| { diff --git a/packages/nx/src/native/workspace/get_nx_workspace_files.rs b/packages/nx/src/native/workspace/get_nx_workspace_files.rs index 68973833035add..6dfbfcd7bf8fac 100644 --- a/packages/nx/src/native/workspace/get_nx_workspace_files.rs +++ b/packages/nx/src/native/workspace/get_nx_workspace_files.rs @@ -1,6 +1,4 @@ -use itertools::Itertools; -use napi::threadsafe_function::{ErrorStrategy, ThreadsafeFunction, ThreadsafeFunctionCallMode}; -use napi::{JsFunction, JsObject, JsUnknown, Status}; +use napi::JsObject; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -13,15 +11,15 @@ use crate::native::types::FileData; use crate::native::utils::glob::build_glob_set; use crate::native::utils::path::Normalize; use crate::native::walker::nx_walker; -use crate::native::workspace::errors::{InternalWorkspaceErrors, WorkspaceErrors}; -use crate::native::workspace::get_config_files::insert_config_file_into_map; -use crate::native::workspace::types::FileLocation; +use crate::native::workspace::errors::WorkspaceErrors; +use crate::native::workspace::types::{ConfigurationParserResult, FileLocation}; #[napi(object)] pub struct NxWorkspaceFiles { pub project_file_map: HashMap>, pub global_files: Vec, pub project_configurations: HashMap, + pub external_nodes: HashMap, } #[napi] @@ -31,7 +29,7 @@ pub fn get_workspace_files_native( parse_configurations: ConfigurationParser, ) -> napi::Result where - ConfigurationParser: Fn(Vec) -> napi::Result>, + ConfigurationParser: Fn(Vec) -> napi::Result, { enable_logger(); @@ -42,10 +40,10 @@ where let projects_vec: Vec = projects.iter().map(|p| p.to_normalized_string()).collect(); - let project_configurations = parse_configurations(projects_vec) + let parsed_graph_nodes = parse_configurations(projects_vec) .map_err(|e| napi::Error::new(WorkspaceErrors::ParseError, e.to_string()))?; - let root_map = create_root_map(&project_configurations); + let root_map = create_root_map(&parsed_graph_nodes.project_nodes); trace!(?root_map); @@ -96,7 +94,8 @@ where Ok(NxWorkspaceFiles { project_file_map, global_files, - project_configurations, + external_nodes: parsed_graph_nodes.external_nodes, + project_configurations: parsed_graph_nodes.project_nodes, }) } @@ -116,16 +115,18 @@ type WorkspaceData = (HashSet, Vec); fn get_file_data(workspace_root: &str, globs: Vec) -> anyhow::Result { let globs = build_glob_set(globs)?; let (projects, file_data) = nx_walker(workspace_root, move |rec| { - let mut projects: HashMap = HashMap::new(); + let mut projects: HashSet = HashSet::new(); let mut file_hashes: Vec = vec![]; for (path, content) in rec { file_hashes.push(FileData { file: path.to_normalized_string(), hash: xxh3::xxh3_64(&content).to_string(), }); - insert_config_file_into_map(path, &mut projects, &globs) + if globs.is_match(&path) { + projects.insert(path); + } } (projects, file_hashes) }); - Ok((projects.into_values().collect(), file_data)) + Ok((projects, file_data)) } diff --git a/packages/nx/src/native/workspace/types.rs b/packages/nx/src/native/workspace/types.rs index dc460dabcec666..3390cd396d8bf9 100644 --- a/packages/nx/src/native/workspace/types.rs +++ b/packages/nx/src/native/workspace/types.rs @@ -1,5 +1,15 @@ +use std::collections::HashMap; + +use napi::JsObject; + #[derive(Debug, Eq, PartialEq)] pub enum FileLocation { Global, Project(String), } + +#[napi(object)] +pub struct ConfigurationParserResult { + pub project_nodes: HashMap, + pub external_nodes: HashMap, +} \ No newline at end of file diff --git a/packages/nx/src/project-graph/build-project-graph.ts b/packages/nx/src/project-graph/build-project-graph.ts index a5e5485f792c79..e0169ea9c23fd2 100644 --- a/packages/nx/src/project-graph/build-project-graph.ts +++ b/packages/nx/src/project-graph/build-project-graph.ts @@ -17,11 +17,15 @@ import { getRootTsConfigPath } from '../plugins/js/utils/typescript'; import { ProjectFileMap, ProjectGraph, + ProjectGraphExternalNode, ProjectGraphProcessorContext, } from '../config/project-graph'; import { readJsonFile } from '../utils/fileutils'; import { NxJsonConfiguration } from '../config/nx-json'; -import { ProjectGraphBuilder } from './project-graph-builder'; +import { + ProjectDependencyBuilder, + ProjectGraphBuilder, +} from './project-graph-builder'; import { ProjectConfiguration, ProjectsConfigurations, @@ -49,6 +53,7 @@ export function getProjectFileMap(): { export async function buildProjectGraphUsingProjectFileMap( projectsConfigurations: ProjectsConfigurations, + externalNodes: Record, projectFileMap: ProjectFileMap, allWorkspaceFiles: FileData[], fileMap: ProjectFileMapCache | null, @@ -94,6 +99,7 @@ export async function buildProjectGraphUsingProjectFileMap( ); let projectGraph = await buildProjectGraphUsingContext( nxJson, + externalNodes, context, cachedFileData, projectGraphVersion @@ -139,6 +145,7 @@ function readCombinedDeps() { async function buildProjectGraphUsingContext( nxJson: NxJsonConfiguration, + knownExternalNodes: Record, ctx: ProjectGraphProcessorContext, cachedFileData: { [project: string]: { [file: string]: FileData } }, projectGraphVersion: string @@ -147,6 +154,9 @@ async function buildProjectGraphUsingContext( const builder = new ProjectGraphBuilder(null, ctx.fileMap); builder.setVersion(projectGraphVersion); + for (const node in knownExternalNodes) { + builder.addExternalNode(knownExternalNodes[node]); + } await buildWorkspaceProjectNodes(ctx, builder, nxJson); const initProjectGraph = builder.getUpdatedProjectGraph(); @@ -209,13 +219,20 @@ async function updateProjectGraphWithPlugins( context: ProjectGraphProcessorContext, initProjectGraph: ProjectGraph ) { - const plugins = ( - await loadNxPlugins(context.nxJsonConfiguration.plugins) - ).filter((x) => !!x.processProjectGraph); + const plugins = await loadNxPlugins(context.nxJsonConfiguration.plugins); let graph = initProjectGraph; for (const plugin of plugins) { try { - graph = await plugin.processProjectGraph(graph, context); + if (plugin.processProjectDependencies) { + const builder = new ProjectDependencyBuilder(graph); + await plugin.processProjectDependencies(builder, { + ...context, + graph, + }); + graph = builder.getUpdatedProjectGraph(); + } else if (plugin.processProjectGraph) { + graph = await plugin.processProjectGraph(graph, context); + } } catch (e) { let message = `Failed to process the project graph with "${plugin.name}".`; if (e instanceof Error) { diff --git a/packages/nx/src/project-graph/project-graph-builder.ts b/packages/nx/src/project-graph/project-graph-builder.ts index 939ddbc21c0fb5..88ed654ce015fc 100644 --- a/packages/nx/src/project-graph/project-graph-builder.ts +++ b/packages/nx/src/project-graph/project-graph-builder.ts @@ -13,24 +13,214 @@ import { } from '../config/project-graph'; import { getProjectFileMap } from './build-project-graph'; -export class ProjectGraphBuilder { +export class ProjectDependencyBuilder { + protected readonly removedEdges: { [source: string]: Set } = {}; + + constructor( + private readonly _graph: ProjectGraph, + protected readonly fileMap?: ProjectFileMap + ) {} + + getUpdatedProjectGraph(): ProjectGraph { + for (const sourceProject of Object.keys(this._graph.nodes)) { + const alreadySetTargetProjects = + this.calculateAlreadySetTargetDeps(sourceProject); + this._graph.dependencies[sourceProject] = [ + ...alreadySetTargetProjects.values(), + ].flatMap((depsMap) => [...depsMap.values()]); + + const fileDeps = this.calculateTargetDepsFromFiles(sourceProject); + for (const [targetProject, types] of fileDeps.entries()) { + // only add known nodes + if ( + !this._graph.nodes[targetProject] && + !this._graph.externalNodes[targetProject] + ) { + continue; + } + for (const type of types.values()) { + if ( + !alreadySetTargetProjects.has(targetProject) || + !alreadySetTargetProjects.get(targetProject).has(type) + ) { + if ( + !this.removedEdges[sourceProject] || + !this.removedEdges[sourceProject].has(targetProject) + ) { + this._graph.dependencies[sourceProject].push({ + source: sourceProject, + target: targetProject, + type, + }); + } + } + } + } + } + return this._graph; + } + + private calculateTargetDepsFromFiles( + sourceProject: string + ): Map> { + const fileDeps = new Map>(); + const files = this.fileMap[sourceProject] || []; + if (!files) { + return fileDeps; + } + for (let f of files) { + if (f.deps) { + for (let d of f.deps) { + const target = fileDataDepTarget(d); + if (!fileDeps.has(target)) { + fileDeps.set(target, new Set([fileDataDepType(d)])); + } else { + fileDeps.get(target).add(fileDataDepType(d)); + } + } + } + } + return fileDeps; + } + + private calculateAlreadySetTargetDeps( + sourceProject: string + ): Map> { + const alreadySetTargetProjects = new Map< + string, + Map + >(); + if (this._graph.dependencies[sourceProject]) { + const removed = this.removedEdges[sourceProject]; + for (const d of this._graph.dependencies[sourceProject]) { + // static and dynamic dependencies of internal projects + // will be rebuilt based on the file dependencies + // we only need to keep the implicit dependencies + if (d.type === DependencyType.implicit && !removed?.has(d.target)) { + if (!alreadySetTargetProjects.has(d.target)) { + alreadySetTargetProjects.set(d.target, new Map([[d.type, d]])); + } else { + alreadySetTargetProjects.get(d.target).set(d.type, d); + } + } + } + } + return alreadySetTargetProjects; + } + + protected addDependency( + sourceProjectName: string, + targetProjectName: string, + type: DependencyType, + sourceProjectFile?: string + ): void { + if (sourceProjectName === targetProjectName) { + return; + } + + validateNewDependency(this._graph, { + sourceProjectName, + targetProjectName, + dependencyType: type, + sourceProjectFile, + }); + + if (!this._graph.dependencies[sourceProjectName]) { + this._graph.dependencies[sourceProjectName] = []; + } + const isDuplicate = !!this._graph.dependencies[sourceProjectName].find( + (d) => d.target === targetProjectName && d.type === type + ); + + if (sourceProjectFile) { + const source = this._graph.nodes[sourceProjectName]; + if (!source) { + throw new Error( + `Source project is not a project node: ${sourceProjectName}` + ); + } + const fileData = (this.fileMap[sourceProjectName] || []).find( + (f) => f.file === sourceProjectFile + ); + if (!fileData) { + throw new Error( + `Source project ${sourceProjectName} does not have a file: ${sourceProjectFile}` + ); + } + + if (!fileData.deps) { + fileData.deps = []; + } + if ( + !fileData.deps.find( + (t) => + fileDataDepTarget(t) === targetProjectName && + fileDataDepType(t) === type + ) + ) { + const dep: string | [string, string] = + type === 'static' ? targetProjectName : [targetProjectName, type]; + fileData.deps.push(dep); + } + } else if (!isDuplicate) { + // only add to dependencies section if the source file is not specified + // and not already added + this._graph.dependencies[sourceProjectName].push({ + source: sourceProjectName, + target: targetProjectName, + type, + }); + } + } + + /** + * Removes a dependency from source project to target project + */ + removeDependency(sourceProjectName: string, targetProjectName: string): void { + if (sourceProjectName === targetProjectName) { + return; + } + if (!this._graph.nodes[sourceProjectName]) { + throw new Error(`Source project does not exist: ${sourceProjectName}`); + } + if ( + !this._graph.nodes[targetProjectName] && + !this._graph.externalNodes[targetProjectName] + ) { + throw new Error(`Target project does not exist: ${targetProjectName}`); + } + // this._graph.dependencies[sourceProjectName] = this._graph.dependencies[ + // sourceProjectName + // ].filter((d) => d.target !== targetProjectName); + if (!this.removedEdges[sourceProjectName]) { + this.removedEdges[sourceProjectName] = new Set(); + } + this.removedEdges[sourceProjectName].add(targetProjectName); + } +} + +/** + * @deprecated(v18): General project graph processors are deprecated. Replace usage with a plugin that utilizes `processProjectNodes` and `processProjectDependencies`. + */ +export class ProjectGraphBuilder extends ProjectDependencyBuilder { // TODO(FrozenPandaz): make this private readonly graph: ProjectGraph; - private readonly fileMap: ProjectFileMap; - readonly removedEdges: { [source: string]: Set } = {}; constructor(g?: ProjectGraph, fileMap?: ProjectFileMap) { - if (g) { - this.graph = g; - this.fileMap = fileMap || getProjectFileMap().projectFileMap; - } else { - this.graph = { - nodes: {}, - externalNodes: {}, - dependencies: {}, - }; - this.fileMap = fileMap || {}; - } + const graph = g + ? g + : { + nodes: {}, + externalNodes: {}, + dependencies: {}, + }; + const normalizedFileMap = g + ? fileMap ?? getProjectFileMap().projectFileMap + : fileMap ?? {}; + super(graph, normalizedFileMap); + + //TODO(@FrozenPandaz) Remove whenever we make this private. + this.graph = graph; } /** @@ -44,6 +234,7 @@ export class ProjectGraphBuilder { }; this.graph.dependencies = { ...this.graph.dependencies, ...p.dependencies }; } + /** * Adds a project node to the project graph */ @@ -106,11 +297,6 @@ export class ProjectGraphBuilder { targetProjectName: string, sourceProjectFile?: string ): void { - // internal nodes must provide sourceProjectFile when creating static dependency - // externalNodes do not have sourceProjectFile - if (this.graph.nodes[sourceProjectName] && !sourceProjectFile) { - throw new Error(`Source project file is required`); - } this.addDependency( sourceProjectName, targetProjectName, @@ -127,13 +313,6 @@ export class ProjectGraphBuilder { targetProjectName: string, sourceProjectFile: string ): void { - if (this.graph.externalNodes[sourceProjectName]) { - throw new Error(`External projects can't have "dynamic" dependencies`); - } - // dynamic dependency is always bound to a file - if (!sourceProjectFile) { - throw new Error(`Source project file is required`); - } this.addDependency( sourceProjectName, targetProjectName, @@ -149,9 +328,6 @@ export class ProjectGraphBuilder { sourceProjectName: string, targetProjectName: string ): void { - if (this.graph.externalNodes[sourceProjectName]) { - throw new Error(`External projects can't have "implicit" dependencies`); - } this.addDependency( sourceProjectName, targetProjectName, @@ -159,31 +335,6 @@ export class ProjectGraphBuilder { ); } - /** - * Removes a dependency from source project to target project - */ - removeDependency(sourceProjectName: string, targetProjectName: string): void { - if (sourceProjectName === targetProjectName) { - return; - } - if (!this.graph.nodes[sourceProjectName]) { - throw new Error(`Source project does not exist: ${sourceProjectName}`); - } - if ( - !this.graph.nodes[targetProjectName] && - !this.graph.externalNodes[targetProjectName] - ) { - throw new Error(`Target project does not exist: ${targetProjectName}`); - } - // this.graph.dependencies[sourceProjectName] = this.graph.dependencies[ - // sourceProjectName - // ].filter((d) => d.target !== targetProjectName); - if (!this.removedEdges[sourceProjectName]) { - this.removedEdges[sourceProjectName] = new Set(); - } - this.removedEdges[sourceProjectName].add(targetProjectName); - } - /** * Add an explicit dependency from a file in source project to target project * @deprecated this method will be removed in v17. Use {@link addStaticDependency} or {@link addDynamicDependency} instead @@ -207,121 +358,6 @@ export class ProjectGraphBuilder { this.graph.version = version; } - getUpdatedProjectGraph(): ProjectGraph { - for (const sourceProject of Object.keys(this.graph.nodes)) { - const alreadySetTargetProjects = - this.calculateAlreadySetTargetDeps(sourceProject); - this.graph.dependencies[sourceProject] = [ - ...alreadySetTargetProjects.values(), - ].flatMap((depsMap) => [...depsMap.values()]); - - const fileDeps = this.calculateTargetDepsFromFiles(sourceProject); - for (const [targetProject, types] of fileDeps.entries()) { - // only add known nodes - if ( - !this.graph.nodes[targetProject] && - !this.graph.externalNodes[targetProject] - ) { - continue; - } - for (const type of types.values()) { - if ( - !alreadySetTargetProjects.has(targetProject) || - !alreadySetTargetProjects.get(targetProject).has(type) - ) { - if ( - !this.removedEdges[sourceProject] || - !this.removedEdges[sourceProject].has(targetProject) - ) { - this.graph.dependencies[sourceProject].push({ - source: sourceProject, - target: targetProject, - type, - }); - } - } - } - } - } - return this.graph; - } - - private addDependency( - sourceProjectName: string, - targetProjectName: string, - type: DependencyType, - sourceProjectFile?: string - ): void { - if (sourceProjectName === targetProjectName) { - return; - } - if ( - !this.graph.nodes[sourceProjectName] && - !this.graph.externalNodes[sourceProjectName] - ) { - throw new Error(`Source project does not exist: ${sourceProjectName}`); - } - if ( - !this.graph.nodes[targetProjectName] && - !this.graph.externalNodes[targetProjectName] && - !sourceProjectFile - ) { - throw new Error(`Target project does not exist: ${targetProjectName}`); - } - if ( - this.graph.externalNodes[sourceProjectName] && - this.graph.nodes[targetProjectName] - ) { - throw new Error(`External projects can't depend on internal projects`); - } - if (!this.graph.dependencies[sourceProjectName]) { - this.graph.dependencies[sourceProjectName] = []; - } - const isDuplicate = !!this.graph.dependencies[sourceProjectName].find( - (d) => d.target === targetProjectName && d.type === type - ); - - if (sourceProjectFile) { - const source = this.graph.nodes[sourceProjectName]; - if (!source) { - throw new Error( - `Source project is not a project node: ${sourceProjectName}` - ); - } - const fileData = (this.fileMap[sourceProjectName] || []).find( - (f) => f.file === sourceProjectFile - ); - if (!fileData) { - throw new Error( - `Source project ${sourceProjectName} does not have a file: ${sourceProjectFile}` - ); - } - - if (!fileData.deps) { - fileData.deps = []; - } - if ( - !fileData.deps.find( - (t) => - fileDataDepTarget(t) === targetProjectName && - fileDataDepType(t) === type - ) - ) { - const dep: string | [string, string] = - type === 'static' ? targetProjectName : [targetProjectName, type]; - fileData.deps.push(dep); - } - } else if (!isDuplicate) { - // only add to dependencies section if the source file is not specified - // and not already added - this.graph.dependencies[sourceProjectName].push({ - source: sourceProjectName, - target: targetProjectName, - type, - }); - } - } - private removeDependenciesWithNode(name: string) { // remove all source dependencies delete this.graph.dependencies[name]; @@ -342,52 +378,80 @@ export class ProjectGraphBuilder { } } } +} - private calculateTargetDepsFromFiles( - sourceProject: string - ): Map> { - const fileDeps = new Map>(); - const files = this.fileMap[sourceProject] || []; - if (!files) { - return fileDeps; - } - for (let f of files) { - if (f.deps) { - for (let d of f.deps) { - const target = fileDataDepTarget(d); - if (!fileDeps.has(target)) { - fileDeps.set(target, new Set([fileDataDepType(d)])); - } else { - fileDeps.get(target).add(fileDataDepType(d)); - } - } - } - } - return fileDeps; +interface CandidateDependency { + sourceProjectName: string; + targetProjectName: string; + sourceProjectFile?: string; + dependencyType: DependencyType; +} + +function validateNewDependency(graph: ProjectGraph, d: CandidateDependency) { + if (d.dependencyType === DependencyType.implicit) { + validateImplicitDependency(graph, d); + } else if (d.dependencyType === DependencyType.dynamic) { + validateDynamicDependency(graph, d); + } else if (d.dependencyType === DependencyType.static) { + validateStaticDependency(graph, d); } - private calculateAlreadySetTargetDeps( - sourceProject: string - ): Map> { - const alreadySetTargetProjects = new Map< - string, - Map - >(); - if (this.graph.dependencies[sourceProject]) { - const removed = this.removedEdges[sourceProject]; - for (const d of this.graph.dependencies[sourceProject]) { - // static and dynamic dependencies of internal projects - // will be rebuilt based on the file dependencies - // we only need to keep the implicit dependencies - if (d.type === DependencyType.implicit && !removed?.has(d.target)) { - if (!alreadySetTargetProjects.has(d.target)) { - alreadySetTargetProjects.set(d.target, new Map([[d.type, d]])); - } else { - alreadySetTargetProjects.get(d.target).set(d.type, d); - } - } - } - } - return alreadySetTargetProjects; + validateCommonDependencyRules(graph, d); +} + +function validateCommonDependencyRules( + graph: ProjectGraph, + d: CandidateDependency +) { + if ( + !graph.nodes[d.sourceProjectName] && + !graph.externalNodes[d.sourceProjectName] + ) { + throw new Error(`Source project does not exist: ${d.sourceProjectName}`); + } + if ( + !graph.nodes[d.targetProjectName] && + !graph.externalNodes[d.targetProjectName] && + !d.sourceProjectFile + ) { + throw new Error(`Target project does not exist: ${d.targetProjectName}`); + } + if ( + graph.externalNodes[d.sourceProjectName] && + graph.nodes[d.targetProjectName] + ) { + throw new Error(`External projects can't depend on internal projects`); + } +} + +function validateImplicitDependency( + graph: ProjectGraph, + d: CandidateDependency +) { + if (graph.externalNodes[d.sourceProjectName]) { + throw new Error(`External projects can't have "implicit" dependencies`); + } +} + +function validateDynamicDependency( + graph: ProjectGraph, + d: CandidateDependency +) { + if (this.graph.externalNodes[d.sourceProjectName]) { + throw new Error(`External projects can't have "dynamic" dependencies`); + } + // dynamic dependency is always bound to a file + if (!d.sourceProjectFile) { + throw new Error( + `Source project file is required for "dynamic" dependencies` + ); + } +} + +function validateStaticDependency(graph: ProjectGraph, d: CandidateDependency) { + // internal nodes must provide sourceProjectFile when creating static dependency + // externalNodes do not have sourceProjectFile + if (graph.nodes[d.sourceProjectName] && !d.sourceProjectFile) { + throw new Error(`Source project file is required`); } } diff --git a/packages/nx/src/project-graph/project-graph.ts b/packages/nx/src/project-graph/project-graph.ts index 4e23b58e6fac04..d6be106211ddfb 100644 --- a/packages/nx/src/project-graph/project-graph.ts +++ b/packages/nx/src/project-graph/project-graph.ts @@ -71,13 +71,18 @@ export function readProjectsConfigurationFromProjectGraph( export async function buildProjectGraphWithoutDaemon() { const nxJson = readNxJson(); - const { allWorkspaceFiles, projectFileMap, projectConfigurations } = - await retrieveWorkspaceFiles(workspaceRoot, nxJson); + const { + allWorkspaceFiles, + projectFileMap, + projectConfigurations, + externalNodes, + } = await retrieveWorkspaceFiles(workspaceRoot, nxJson); const cacheEnabled = process.env.NX_CACHE_PROJECT_GRAPH !== 'false'; return ( await buildProjectGraphUsingProjectFileMap( projectConfigurations, + externalNodes, projectFileMap, allWorkspaceFiles, cacheEnabled ? readProjectFileMapCache() : null, diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index 7674b946892b1b..056834cf82607e 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -18,7 +18,11 @@ import { shouldMergeAngularProjects, } from '../../adapter/angular-json'; import { NxJsonConfiguration } from '../../config/nx-json'; -import { FileData, ProjectFileMap } from '../../config/project-graph'; +import { + FileData, + ProjectFileMap, + ProjectGraphExternalNode, +} from '../../config/project-graph'; import type { NxWorkspaceFiles } from '../../native'; /** @@ -31,7 +35,8 @@ export async function retrieveWorkspaceFiles( workspaceRoot: string, nxJson: NxJsonConfiguration ) { - const { getWorkspaceFilesNative } = require('../../native'); + const { getWorkspaceFilesNative } = + require('../../native') as typeof import('../../native'); performance.mark('native-file-deps:start'); let globs = await configurationGlobs(workspaceRoot, nxJson); @@ -44,20 +49,19 @@ export async function retrieveWorkspaceFiles( performance.mark('get-workspace-files:start'); - const { projectConfigurations, projectFileMap, globalFiles } = - getWorkspaceFilesNative( - workspaceRoot, - globs, - (configs: string[]): Record => { - const projectConfigurations = createProjectConfigurations( - workspaceRoot, - nxJson, - configs - ); + const { projectConfigurations, projectFileMap, globalFiles, externalNodes } = + getWorkspaceFilesNative(workspaceRoot, globs, (configs: string[]) => { + const projectConfigurations = createProjectConfigurations( + workspaceRoot, + nxJson, + configs + ); - return projectConfigurations.projects; - } - ) as NxWorkspaceFiles; + return { + projectNodes: projectConfigurations.projects, + externalNodes: projectConfigurations.externalNodes, + }; + }) as NxWorkspaceFiles; performance.mark('get-workspace-files:end'); performance.measure( 'get-workspace-files', @@ -72,6 +76,7 @@ export async function retrieveWorkspaceFiles( version: 2, projects: projectConfigurations, } as ProjectsConfigurations, + externalNodes: externalNodes as Record, }; } @@ -84,22 +89,28 @@ export async function retrieveWorkspaceFiles( export async function retrieveProjectConfigurations( workspaceRoot: string, nxJson: NxJsonConfiguration -): Promise> { - const { getProjectConfigurations } = require('../../native'); +): Promise<{ + externalNodes: Record; + projectNodes: Record; +}> { + const { getProjectConfigurations } = + require('../../native') as typeof import('../../native'); const globs = await configurationGlobs(workspaceRoot, nxJson); - return getProjectConfigurations( - workspaceRoot, - globs, - (configs: string[]): Record => { - const projectConfigurations = createProjectConfigurations( - workspaceRoot, - nxJson, - configs - ); + return getProjectConfigurations(workspaceRoot, globs, (configs: string[]) => { + const projectConfigurations = createProjectConfigurations( + workspaceRoot, + nxJson, + configs + ); - return projectConfigurations.projects; - } - ); + return { + projectNodes: projectConfigurations.projects, + externalNodes: projectConfigurations.externalNodes, + }; + }) as { + externalNodes: Record; + projectNodes: Record; + }; } function buildAllWorkspaceFiles( @@ -124,16 +135,21 @@ function createProjectConfigurations( workspaceRoot: string, nxJson: NxJsonConfiguration, configFiles: string[] -): ProjectsConfigurations { +): ProjectsConfigurations & { + externalNodes: Record; +} { performance.mark('build-project-configs:start'); - let projectConfigurations = mergeTargetDefaultsIntoProjectDescriptions( + const { projects, externalNodes } = buildProjectsConfigurationsFromProjectPaths( nxJson, configFiles, workspaceRoot, (path) => readJsonFile(join(workspaceRoot, path)) - ), + ); + + let projectConfigurations = mergeTargetDefaultsIntoProjectDescriptions( + projects, nxJson ); @@ -153,6 +169,7 @@ function createProjectConfigurations( return { version: 2, projects: projectConfigurations, + externalNodes, }; } diff --git a/packages/nx/src/utils/nx-plugin.ts b/packages/nx/src/utils/nx-plugin.ts index a6153fcf8336cc..5cb16c96eb9c40 100644 --- a/packages/nx/src/utils/nx-plugin.ts +++ b/packages/nx/src/utils/nx-plugin.ts @@ -1,7 +1,12 @@ import { sync } from 'fast-glob'; import { existsSync } from 'fs'; import * as path from 'path'; -import { ProjectGraphProcessor } from '../config/project-graph'; +import { + ProjectFileMap, + ProjectGraph, + ProjectGraphExternalNode, + ProjectGraphProcessor, +} from '../config/project-graph'; import { toProjectName, Workspaces } from '../config/workspaces'; import { workspaceRoot } from './workspace-root'; @@ -32,15 +37,49 @@ import { NxJsonConfiguration } from '../config/nx-json'; import type * as ts from 'typescript'; import { NxPluginV1 } from './nx-plugin.deprecated'; +import { ProjectDependencyBuilder } from '../project-graph/project-graph-builder'; export type ProjectConfigurationBuilder = ( projectConfigurationFile: string, context: { - projectsConfigurations: Record, - nxJsonConfiguration: NxJsonConfiguration, - workspaceRoot: string + projectsConfigurations: Record; + nxJsonConfiguration: NxJsonConfiguration; + workspaceRoot: string; + } +) => { + projectNodes?: Record; + externalNodes?: Record; +}; + +export type DependencyLocator = ( + builder: ProjectDependencyBuilder, + context: { + /** + * The current project graph, + */ + readonly graph: ProjectGraph; + + /** + * The configuration of each project in the workspace + */ + projectsConfigurations: ProjectsConfigurations; + + /** + * The `nx.json` configuration from the workspace + */ + nxJsonConfiguration: NxJsonConfiguration; + + /** + * All files in the workspace + */ + fileMap: ProjectFileMap; + + /** + * Files changes since last invocation + */ + filesToProcess: ProjectFileMap; } -) => Record; +) => void | Promise; export type NxPluginV2 = { name: string; @@ -52,7 +91,7 @@ export type NxPluginV2 = { processProjectNodes?: Record; // Todo(@AgentEnder): This shouldn't be a full processor, since its only responsible for defining edges between projects. What do we want the API to be? - processProjectDependencies?: ProjectGraphProcessor; + processProjectDependencies?: DependencyLocator; }; export * from './nx-plugin.deprecated'; @@ -200,10 +239,12 @@ function ensurePluginIsV2(plugin: NxPlugin): NxPluginV2 { ) => { const name = toProjectName(configFilePath); return { - [name]: { - name, - root: dirname(configFilePath), - targets: plugin.registerProjectTargets?.(configFilePath), + projectNodes: { + [name]: { + name, + root: dirname(configFilePath), + targets: plugin.registerProjectTargets?.(configFilePath), + }, }, }; },