feat(core): add dependency builder and external node capability to v2…

… api
nrwl · Jul 20, 2023 · 77c430e · 77c430e
1 parent 1297568
commit 77c430e
Show file tree

Hide file tree

Showing 14 changed files with 489 additions and 313 deletions.
diff --git a/docs/generated/devkit/nx_devkit.md b/docs/generated/devkit/nx_devkit.md
@@ -181,6 +181,8 @@ Type of dependency between projects
 
 • **ProjectGraphBuilder**: `Object`
 
+@deprecated(v18): General project graph processors are deprecated. Replace usage with a plugin that utilizes `processProjectNodes` and `processProjectDependencies`.
+
 ---
 
 ### Workspaces

diff --git a/docs/generated/packages/devkit/documents/nx_devkit.md b/docs/generated/packages/devkit/documents/nx_devkit.md
@@ -181,6 +181,8 @@ Type of dependency between projects
 
 • **ProjectGraphBuilder**: `Object`
 
+@deprecated(v18): General project graph processors are deprecated. Replace usage with a plugin that utilizes `processProjectNodes` and `processProjectDependencies`.
+
 ---
 
 ### Workspaces

diff --git a/packages/nx/src/config/workspaces.ts b/packages/nx/src/config/workspaces.ts
@@ -39,6 +39,7 @@ import {
   normalizeProjectRoot,
 } from '../project-graph/utils/find-project-for-path';
 import { getImplementationFactory, resolveSchema } from './schema-utils';
+import { ProjectGraphExternalNode } from './project-graph';
 
 export class Workspaces {
   private cachedProjectsConfig: ProjectsConfigurations;
@@ -107,7 +108,7 @@ export class Workspaces {
       ),
       this.root,
       (path) => readJsonFile(join(this.root, path))
-    );
+    ).projects;
     if (
       shouldMergeAngularProjects(
         this.root,
@@ -648,7 +649,7 @@ function mergeProjectConfigurationIntoWorkspace(
     );
   }
 
-  if (project.implicitDependencies && matchingProject.tags) {
+  if (project.implicitDependencies && matchingProject.implicitDependencies) {
     updatedProjectConfiguration.implicitDependencies =
       matchingProject.implicitDependencies.concat(project.implicitDependencies);
   }
@@ -680,73 +681,85 @@ export function buildProjectsConfigurationsFromProjectPaths(
   root: string = workspaceRoot,
   readJson: <T extends Object>(string) => T = <T extends Object>(string) =>
     readJsonFile<T>(string) // making this an arg allows us to reuse in devkit
-): Record<string, ProjectConfiguration> {
+): {
+  projects: Record<string, ProjectConfiguration>;
+  externalNodes: Record<string, ProjectGraphExternalNode>;
+} {
   const projectRootMap: Map<string, string> = new Map();
   const projects: Record<string, ProjectConfiguration> = {};
+  const externalNodes: Record<string, ProjectGraphExternalNode> = {};
   // We go in reverse here s.t. plugins listed first in the plugins array have highest priority - they overwrite
   // whatever configuration was added by plugins later in the array.
   const plugins = loadNxPluginsSync(nxJson.plugins).reverse();
 
   // We push the nx core node builder onto the end, s.t. it overwrites any user specified behavior
   const globPatternsFromPackageManagerWorkspaces =
     getGlobPatternsFromPackageManagerWorkspaces(root);
-  plugins.push({
+  const nxCorePlugin: NxPluginV2 = {
     name: 'nx-core-build-nodes',
     processProjectNodes: {
       // Load projects from pnpm / npm workspaces
       ...(globPatternsFromPackageManagerWorkspaces.length
-        ? ({
+        ? {
             [combineGlobPatterns(globPatternsFromPackageManagerWorkspaces)]: (
               pkgJsonPath
             ) => {
               const json = readJson<PackageJson>(pkgJsonPath);
               return {
-                [json.name]: buildProjectConfigurationFromPackageJson(
-                  pkgJsonPath,
-                  json,
-                  nxJson
-                ),
+                projectNodes: {
+                  [json.name]: buildProjectConfigurationFromPackageJson(
+                    pkgJsonPath,
+                    json,
+                    nxJson
+                  ),
+                },
               };
             },
-          } as NxPluginV2['processProjectNodes'])
+          }
         : {}),
       // Load projects from project.json files. These will be read second, since
       // they are listed last in the plugin, so they will overwrite things from the package.json
       // based projects.
       '{project.json,**/project.json}': (file) => {
         const json = readJson<ProjectConfiguration>(file);
         json.name ??= toProjectName(file);
+        json.root ??= dirname(file).split('\\').join('/');
         return {
-          [json.name]: json,
+          projectNodes: {
+            [json.name]: json,
+          },
         };
       },
     },
-  });
+  };
+  plugins.push(nxCorePlugin);
 
   // We iterate over plugins first - this ensures that plugins specified first take precedence.
   for (const plugin of plugins) {
     // Within a plugin patterns specified later overwrite info from earlier matches.
     for (const pattern in plugin.processProjectNodes ?? {}) {
       for (const file of projectFiles) {
         if (minimatch(file, pattern)) {
-          const nodes = plugin.processProjectNodes[pattern](file, {
-            projectsConfigurations: projects,
-            nxJsonConfiguration: nxJson,
-            workspaceRoot: root
-          });
-          for (const node in nodes) {
+          const { projectNodes, externalNodes: pluginExternalNodes } =
+            plugin.processProjectNodes[pattern](file, {
+              projectsConfigurations: projects,
+              nxJsonConfiguration: nxJson,
+              workspaceRoot: root,
+            });
+          for (const node in projectNodes) {
             mergeProjectConfigurationIntoWorkspace(
               projects,
               projectRootMap,
-              nodes[node]
+              projectNodes[node]
             );
           }
+          Object.assign(externalNodes, pluginExternalNodes);
         }
       }
     }
   }
 
-  return projects;
+  return { projects, externalNodes };
 }
 
 export function mergeTargetConfigurations(

diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts
@@ -3,6 +3,7 @@ import {
   FileData,
   ProjectFileMap,
   ProjectGraph,
+  ProjectGraphExternalNode,
 } from '../../config/project-graph';
 import { buildProjectGraphUsingProjectFileMap } from '../../project-graph/build-project-graph';
 import { updateProjectFileMap } from '../../project-graph/file-map-utils';
@@ -23,6 +24,7 @@ import {
   retrieveProjectConfigurations,
 } from '../../project-graph/utils/retrieve-workspace-files';
 import { ProjectConfiguration } from '../../config/workspace-json-project-json';
+import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
 
 let cachedSerializedProjectGraphPromise: Promise<{
   error: Error | null;
@@ -42,6 +44,7 @@ const collectedDeletedFiles = new Set<string>();
 let storedWorkspaceConfigHash: string | undefined;
 let waitPeriod = 100;
 let scheduledTimeoutId;
+let knownExternalNodes: Record<string, ProjectGraphExternalNode> = {};
 
 export async function getCachedSerializedProjectGraphPromise() {
   try {
@@ -172,14 +175,12 @@ async function processCollectedUpdatedAndDeletedFiles() {
 
     let nxJson = new Workspaces(workspaceRoot).readNxJson();
 
-    const projectConfigurations = await retrieveProjectConfigurations(
+    const { projectNodes } = await retrieveProjectConfigurations(
       workspaceRoot,
       nxJson
     );
 
-    const workspaceConfigHash = computeWorkspaceConfigHash(
-      projectConfigurations
-    );
+    const workspaceConfigHash = computeWorkspaceConfigHash(projectNodes);
     serverLogger.requestLog(
       `Updated file-hasher based on watched changes, recomputing project graph...`
     );
@@ -190,14 +191,12 @@ async function processCollectedUpdatedAndDeletedFiles() {
     if (workspaceConfigHash !== storedWorkspaceConfigHash) {
       storedWorkspaceConfigHash = workspaceConfigHash;
 
-      projectFileMapWithFiles = await retrieveWorkspaceFiles(
-        workspaceRoot,
-        nxJson
-      );
+      ({ externalNodes: knownExternalNodes, ...projectFileMapWithFiles } =
+        await retrieveWorkspaceFiles(workspaceRoot, nxJson));
     } else {
       if (projectFileMapWithFiles) {
         projectFileMapWithFiles = updateProjectFileMap(
-          projectConfigurations,
+          projectNodes,
           projectFileMapWithFiles.projectFileMap,
           projectFileMapWithFiles.allWorkspaceFiles,
           updatedFiles,
@@ -275,7 +274,8 @@ async function createAndSerializeProjectGraph(): Promise<{
     const { projectGraph, projectFileMapCache } =
       await buildProjectGraphUsingProjectFileMap(
         projectsConfigurations,
-        projectFileMap,
+        knownExternalNodes,
+        projectFileMapWithFiles.projectFileMap,
         allWorkspaceFiles,
         currentProjectFileMapCache || readProjectFileMapCache(),
         true

diff --git a/packages/nx/src/generators/utils/project-configuration.ts b/packages/nx/src/generators/utils/project-configuration.ts
@@ -198,7 +198,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): {
     projectFiles,
     tree.root,
     (file) => readJson(tree, file)
-  );
+  ).projects;
 }
 
 /**

diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts
@@ -37,13 +37,18 @@ export const enum WorkspaceErrors {
   Generic = 'Generic'
 }
 /** Get workspace config files based on provided globs */
-export function getProjectConfigurations(workspaceRoot: string, globs: Array<string>, parseConfigurations: (arg0: Array<string>) => Record<string, object>): Record<string, object>
+export function getProjectConfigurations(workspaceRoot: string, globs: Array<string>, parseConfigurations: (arg0: Array<string>) => ConfigurationParserResult): ConfigurationParserResult
 export interface NxWorkspaceFiles {
   projectFileMap: Record<string, Array<FileData>>
   globalFiles: Array<FileData>
   projectConfigurations: Record<string, object>
+  externalNodes: Record<string, object>
+}
+export function getWorkspaceFilesNative(workspaceRoot: string, globs: Array<string>, parseConfigurations: (arg0: Array<string>) => ConfigurationParserResult): NxWorkspaceFiles
+export interface ConfigurationParserResult {
+  projectNodes: Record<string, object>
+  externalNodes: Record<string, object>
 }
-export function getWorkspaceFilesNative(workspaceRoot: string, globs: Array<string>, parseConfigurations: (arg0: Array<string>) => Record<string, object>): NxWorkspaceFiles
 export class Watcher {
   origin: string
   /**

diff --git a/packages/nx/src/native/workspace/get_config_files.rs b/packages/nx/src/native/workspace/get_config_files.rs
@@ -1,9 +1,9 @@
 use crate::native::utils::glob::build_glob_set;
 use crate::native::utils::path::Normalize;
 use crate::native::walker::nx_walker;
+use crate::native::workspace::types::ConfigurationParserResult;
 use globset::GlobSet;
 
-use napi::JsObject;
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
@@ -13,11 +13,10 @@ use std::path::{Path, PathBuf};
 pub fn get_project_configurations<ConfigurationParser>(
     workspace_root: String,
     globs: Vec<String>,
-
     parse_configurations: ConfigurationParser,
-) -> napi::Result<HashMap<String, JsObject>>
+) -> napi::Result<ConfigurationParserResult>
 where
-    ConfigurationParser: Fn(Vec<String>) -> napi::Result<HashMap<String, JsObject>>,
+ConfigurationParser: Fn(Vec<String>) -> napi::Result<ConfigurationParserResult>
 {
     let globs = build_glob_set(globs)?;
     let config_paths: Vec<String> = nx_walker(workspace_root, move |rec| {

diff --git a/packages/nx/src/native/workspace/get_nx_workspace_files.rs b/packages/nx/src/native/workspace/get_nx_workspace_files.rs
@@ -1,6 +1,4 @@
-use itertools::Itertools;
-use napi::threadsafe_function::{ErrorStrategy, ThreadsafeFunction, ThreadsafeFunctionCallMode};
-use napi::{JsFunction, JsObject, JsUnknown, Status};
+use napi::JsObject;
 use std::collections::{HashMap, HashSet};
 use std::path::{Path, PathBuf};
 
@@ -13,15 +11,15 @@ use crate::native::types::FileData;
 use crate::native::utils::glob::build_glob_set;
 use crate::native::utils::path::Normalize;
 use crate::native::walker::nx_walker;
-use crate::native::workspace::errors::{InternalWorkspaceErrors, WorkspaceErrors};
-use crate::native::workspace::get_config_files::insert_config_file_into_map;
-use crate::native::workspace::types::FileLocation;
+use crate::native::workspace::errors::WorkspaceErrors;
+use crate::native::workspace::types::{ConfigurationParserResult, FileLocation};
 
 #[napi(object)]
 pub struct NxWorkspaceFiles {
     pub project_file_map: HashMap<String, Vec<FileData>>,
     pub global_files: Vec<FileData>,
     pub project_configurations: HashMap<String, JsObject>,
+    pub external_nodes: HashMap<String, JsObject>,
 }
 
 #[napi]
@@ -31,7 +29,7 @@ pub fn get_workspace_files_native<ConfigurationParser>(
     parse_configurations: ConfigurationParser,
 ) -> napi::Result<NxWorkspaceFiles, WorkspaceErrors>
 where
-    ConfigurationParser: Fn(Vec<String>) -> napi::Result<HashMap<String, JsObject>>,
+    ConfigurationParser: Fn(Vec<String>) -> napi::Result<ConfigurationParserResult>,
 {
     enable_logger();
 
@@ -42,10 +40,10 @@ where
 
     let projects_vec: Vec<String> = projects.iter().map(|p| p.to_normalized_string()).collect();
 
-    let project_configurations = parse_configurations(projects_vec)
+    let parsed_graph_nodes = parse_configurations(projects_vec)
         .map_err(|e| napi::Error::new(WorkspaceErrors::ParseError, e.to_string()))?;
 
-    let root_map = create_root_map(&project_configurations);
+    let root_map = create_root_map(&parsed_graph_nodes.project_nodes);
 
     trace!(?root_map);
 
@@ -96,7 +94,8 @@ where
     Ok(NxWorkspaceFiles {
         project_file_map,
         global_files,
-        project_configurations,
+        external_nodes: parsed_graph_nodes.external_nodes,
+        project_configurations: parsed_graph_nodes.project_nodes,
     })
 }
 
@@ -116,16 +115,18 @@ type WorkspaceData = (HashSet<PathBuf>, Vec<FileData>);
 fn get_file_data(workspace_root: &str, globs: Vec<String>) -> anyhow::Result<WorkspaceData> {
     let globs = build_glob_set(globs)?;
     let (projects, file_data) = nx_walker(workspace_root, move |rec| {
-        let mut projects: HashMap<PathBuf, PathBuf> = HashMap::new();
+        let mut projects: HashSet<PathBuf> = HashSet::new();
         let mut file_hashes: Vec<FileData> = vec![];
         for (path, content) in rec {
             file_hashes.push(FileData {
                 file: path.to_normalized_string(),
                 hash: xxh3::xxh3_64(&content).to_string(),
             });
-            insert_config_file_into_map(path, &mut projects, &globs)
+            if globs.is_match(&path) {
+                projects.insert(path);
+            }
         }
         (projects, file_hashes)
     });
-    Ok((projects.into_values().collect(), file_data))
+    Ok((projects, file_data))
 }
diff --git a/packages/nx/src/native/workspace/types.rs b/packages/nx/src/native/workspace/types.rs
@@ -1,5 +1,15 @@
+use std::collections::HashMap;
+
+use napi::JsObject;
+
 #[derive(Debug, Eq, PartialEq)]
 pub enum FileLocation {
     Global,
     Project(String),
 }
+
+#[napi(object)]
+pub struct ConfigurationParserResult {
+    pub project_nodes: HashMap<String, JsObject>,
+    pub external_nodes: HashMap<String, JsObject>,
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -181,6 +181,8 @@ Type of dependency between projects @@
     • **ProjectGraphBuilder**: `Object`
+    @deprecated(v18): General project graph processors are deprecated. Replace usage with a plugin that utilizes `processProjectNodes` and `processProjectDependencies`.
     ---
     ### Workspaces
@@ Expand Down @@