feat(core): add dependency builder and external node capability to v2…

… api
nrwl · Jul 21, 2023 · 43f4f47 · 43f4f47
1 parent 609e2ca
commit 43f4f47
Show file tree

Hide file tree

Showing 20 changed files with 553 additions and 334 deletions.
diff --git a/docs/generated/devkit/nx_devkit.md b/docs/generated/devkit/nx_devkit.md
@@ -181,6 +181,8 @@ Type of dependency between projects
 
 • **ProjectGraphBuilder**: `Object`
 
+@deprecated(v18): General project graph processors are deprecated. Replace usage with a plugin that utilizes `processProjectNodes` and `processProjectDependencies`.
+
 ---
 
 ### Workspaces

diff --git a/docs/generated/packages/devkit/documents/nx_devkit.md b/docs/generated/packages/devkit/documents/nx_devkit.md
@@ -181,6 +181,8 @@ Type of dependency between projects
 
 • **ProjectGraphBuilder**: `Object`
 
+@deprecated(v18): General project graph processors are deprecated. Replace usage with a plugin that utilizes `processProjectNodes` and `processProjectDependencies`.
+
 ---
 
 ### Workspaces

diff --git a/packages/nx/src/config/workspaces.spec.ts b/packages/nx/src/config/workspaces.spec.ts
@@ -14,6 +14,7 @@ jest.mock('fs', () => require('memfs').fs);
 
 const libConfig = (root, name?: string) => ({
   name: name ?? toProjectName(`${root}/some-file`),
+  projectType: 'library',
   root: `libs/${root}`,
   sourceRoot: `libs/${root}/src`,
 });
@@ -68,7 +69,6 @@ describe('Workspaces', () => {
 
       const workspaces = new Workspaces('/root');
       const resolved = workspaces.readProjectsConfigurations();
-      console.log(resolved);
       expect(resolved.projects.lib1).toEqual(standaloneConfig);
     });
 
@@ -100,7 +100,6 @@ describe('Workspaces', () => {
 
       const workspaces = new Workspaces('/root');
       const { projects } = workspaces.readProjectsConfigurations();
-      console.log(projects);
 
       // projects got merged for lib1
       expect(projects['lib1']).toEqual({

diff --git a/packages/nx/src/config/workspaces.ts b/packages/nx/src/config/workspaces.ts
@@ -32,6 +32,7 @@ import {
   findProjectForPath,
   normalizeProjectRoot,
 } from '../project-graph/utils/find-project-for-path';
+import { ProjectGraphExternalNode } from './project-graph';
 
 export class Workspaces {
   private cachedProjectsConfig: ProjectsConfigurations;
@@ -100,7 +101,7 @@ export class Workspaces {
       ),
       this.root,
       (path) => readJsonFile(join(this.root, path))
-    );
+    ).projects;
     if (
       shouldMergeAngularProjects(
         this.root,
@@ -253,12 +254,13 @@ export async function getGlobPatternsFromPluginsAsync(
  * Get the package.json globs from package manager workspaces
  */
 export function getGlobPatternsFromPackageManagerWorkspaces(
-  root: string
+  root: string,
+  readJson: <T extends Object = any>(f: string) => T = <T extends Object>(f) =>
+    readJsonFile<T>(join(root, f))
 ): string[] {
   try {
     const patterns: string[] = [];
-    const packageJson = readJsonFile<PackageJson>(join(root, 'package.json'));
-
+    const packageJson = readJson<PackageJson>('package.json');
     patterns.push(
       ...normalizePatterns(
         Array.isArray(packageJson.workspaces)
@@ -283,7 +285,7 @@ export function getGlobPatternsFromPackageManagerWorkspaces(
 
     if (existsSync(join(root, 'lerna.json'))) {
       try {
-        const { packages } = readJsonFile<any>(join(root, 'lerna.json'));
+        const { packages } = readJson('lerna.json');
         patterns.push(
           ...normalizePatterns(packages?.length > 0 ? packages : ['packages/*'])
         );
@@ -299,7 +301,7 @@ export function getGlobPatternsFromPackageManagerWorkspaces(
     // TODO(@AgentEnder): update logic after better way to determine root project inclusion
     // Include the root project
     return packageJson.nx ? patterns.concat('package.json') : patterns;
-  } catch {
+  } catch (e) {
     return [];
   }
 }
@@ -321,7 +323,9 @@ function removeRelativePath(pattern: string): string {
 export function globForProjectFiles(
   root: string,
   pluginsGlobPatterns: string[],
-  nxJson?: NxJsonConfiguration
+  nxJson?: NxJsonConfiguration,
+  readJson: <T extends Object>(string) => T = <T extends Object>(string) =>
+    readJsonFile<T>(string) // making this an arg allows us to reuse in devkit
 ) {
   // Deal w/ Caching
   const cacheKey = [root, ...pluginsGlobPatterns].join(',');
@@ -335,7 +339,7 @@ export function globForProjectFiles(
   projectGlobCacheKey = cacheKey;
 
   const _globPatternsFromPackageManagerWorkspaces =
-    getGlobPatternsFromPackageManagerWorkspaces(root);
+    getGlobPatternsFromPackageManagerWorkspaces(root, readJson);
 
   const globPatternsFromPackageManagerWorkspaces =
     _globPatternsFromPackageManagerWorkspaces ?? [];
@@ -474,14 +478,33 @@ function mergeProjectConfigurationIntoWorkspace(
   existingProjects: Record<string, ProjectConfiguration>,
   // projectRoot -> projectName
   existingProjectRootMap: Map<string, string>,
-  project: ProjectConfiguration
+  project: ProjectConfiguration,
+  // project.json is a special case, so we need to detect it.
+  file: string
 ): void {
-  const matchingProjectName = existingProjectRootMap.get(project.root);
+  let matchingProjectName = existingProjectRootMap.get(project.root);
 
   if (!matchingProjectName) {
     existingProjects[project.name] = project;
     existingProjectRootMap.set(project.root, project.name);
     return;
+    // There are some special cases for handling project.json - mainly
+    // that it should override any name the project already has.
+  } else if (
+    project.name &&
+    project.name !== matchingProjectName &&
+    basename(file) === 'project.json'
+  ) {
+    // Copy config to new name
+    existingProjects[project.name] = existingProjects[matchingProjectName];
+    // Update name in project config
+    existingProjects[project.name].name = project.name;
+    // Update root map to point to new name
+    existingProjectRootMap[project.root] = project.name;
+    // Remove entry for old name
+    delete existingProjects[matchingProjectName];
+    // Update name that config should be merged to
+    matchingProjectName = project.name;
   }
 
   const matchingProject = existingProjects[matchingProjectName];
@@ -500,7 +523,7 @@ function mergeProjectConfigurationIntoWorkspace(
     );
   }
 
-  if (project.implicitDependencies && matchingProject.tags) {
+  if (project.implicitDependencies && matchingProject.implicitDependencies) {
     updatedProjectConfiguration.implicitDependencies =
       matchingProject.implicitDependencies.concat(project.implicitDependencies);
   }
@@ -519,86 +542,100 @@ function mergeProjectConfigurationIntoWorkspace(
     };
   }
 
-  if (updatedProjectConfiguration.name !== matchingProject.name) {
-    delete existingProjects[matchingProject.name];
-  }
   existingProjects[updatedProjectConfiguration.name] =
     updatedProjectConfiguration;
 }
 
 export function buildProjectsConfigurationsFromProjectPaths(
-  nxJson: NxJsonConfiguration,
+  nxJson: NxJsonConfiguration | null,
   projectFiles: string[], // making this parameter allows devkit to pick up newly created projects
   root: string = workspaceRoot,
   readJson: <T extends Object>(string) => T = <T extends Object>(string) =>
     readJsonFile<T>(string) // making this an arg allows us to reuse in devkit
-): Record<string, ProjectConfiguration> {
+): {
+  projects: Record<string, ProjectConfiguration>;
+  externalNodes: Record<string, ProjectGraphExternalNode>;
+} {
   const projectRootMap: Map<string, string> = new Map();
   const projects: Record<string, ProjectConfiguration> = {};
+  const externalNodes: Record<string, ProjectGraphExternalNode> = {};
   // We go in reverse here s.t. plugins listed first in the plugins array have highest priority - they overwrite
   // whatever configuration was added by plugins later in the array.
-  const plugins = loadNxPluginsSync(nxJson.plugins).reverse();
+  const plugins = loadNxPluginsSync(
+    nxJson?.plugins ?? [],
+    getNxRequirePaths(root),
+    root
+  ).reverse();
 
   // We push the nx core node builder onto the end, s.t. it overwrites any user specified behavior
   const globPatternsFromPackageManagerWorkspaces =
-    getGlobPatternsFromPackageManagerWorkspaces(root);
-  plugins.push({
+    getGlobPatternsFromPackageManagerWorkspaces(root, readJson);
+  const nxCorePlugin: NxPluginV2 = {
     name: 'nx-core-build-nodes',
     processProjectNodes: {
       // Load projects from pnpm / npm workspaces
       ...(globPatternsFromPackageManagerWorkspaces.length
-        ? ({
+        ? {
             [combineGlobPatterns(globPatternsFromPackageManagerWorkspaces)]: (
               pkgJsonPath
             ) => {
               const json = readJson<PackageJson>(pkgJsonPath);
               return {
-                [json.name]: buildProjectConfigurationFromPackageJson(
-                  pkgJsonPath,
-                  json,
-                  nxJson
-                ),
+                projectNodes: {
+                  [json.name]: buildProjectConfigurationFromPackageJson(
+                    pkgJsonPath,
+                    json,
+                    nxJson
+                  ),
+                },
               };
             },
-          } as NxPluginV2['processProjectNodes'])
+          }
         : {}),
       // Load projects from project.json files. These will be read second, since
       // they are listed last in the plugin, so they will overwrite things from the package.json
       // based projects.
       '{project.json,**/project.json}': (file) => {
         const json = readJson<ProjectConfiguration>(file);
         json.name ??= toProjectName(file);
+        json.root ??= dirname(file).split('\\').join('/');
         return {
-          [json.name]: json,
+          projectNodes: {
+            [json.name]: json,
+          },
         };
       },
     },
-  });
+  };
+  plugins.push(nxCorePlugin);
 
   // We iterate over plugins first - this ensures that plugins specified first take precedence.
   for (const plugin of plugins) {
     // Within a plugin patterns specified later overwrite info from earlier matches.
     for (const pattern in plugin.processProjectNodes ?? {}) {
       for (const file of projectFiles) {
         if (minimatch(file, pattern)) {
-          const nodes = plugin.processProjectNodes[pattern](file, {
-            projectsConfigurations: projects,
-            nxJsonConfiguration: nxJson,
-            workspaceRoot: root
-          });
-          for (const node in nodes) {
+          const { projectNodes, externalNodes: pluginExternalNodes } =
+            plugin.processProjectNodes[pattern](file, {
+              projectsConfigurations: projects,
+              nxJsonConfiguration: nxJson,
+              workspaceRoot: root,
+            });
+          for (const node in projectNodes) {
             mergeProjectConfigurationIntoWorkspace(
               projects,
               projectRootMap,
-              nodes[node]
+              projectNodes[node],
+              file
             );
           }
+          Object.assign(externalNodes, pluginExternalNodes);
         }
       }
     }
   }
 
-  return projects;
+  return { projects, externalNodes };
 }
 
 export function mergeTargetConfigurations(

diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts
@@ -3,6 +3,7 @@ import {
   FileData,
   ProjectFileMap,
   ProjectGraph,
+  ProjectGraphExternalNode,
 } from '../../config/project-graph';
 import { buildProjectGraphUsingProjectFileMap } from '../../project-graph/build-project-graph';
 import { updateProjectFileMap } from '../../project-graph/file-map-utils';
@@ -23,6 +24,7 @@ import {
   retrieveProjectConfigurations,
 } from '../../project-graph/utils/retrieve-workspace-files';
 import { ProjectConfiguration } from '../../config/workspace-json-project-json';
+import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
 
 let cachedSerializedProjectGraphPromise: Promise<{
   error: Error | null;
@@ -42,6 +44,7 @@ const collectedDeletedFiles = new Set<string>();
 let storedWorkspaceConfigHash: string | undefined;
 let waitPeriod = 100;
 let scheduledTimeoutId;
+let knownExternalNodes: Record<string, ProjectGraphExternalNode> = {};
 
 export async function getCachedSerializedProjectGraphPromise() {
   try {
@@ -172,14 +175,12 @@ async function processCollectedUpdatedAndDeletedFiles() {
 
     let nxJson = new Workspaces(workspaceRoot).readNxJson();
 
-    const projectConfigurations = await retrieveProjectConfigurations(
+    const { projectNodes } = await retrieveProjectConfigurations(
       workspaceRoot,
       nxJson
     );
 
-    const workspaceConfigHash = computeWorkspaceConfigHash(
-      projectConfigurations
-    );
+    const workspaceConfigHash = computeWorkspaceConfigHash(projectNodes);
     serverLogger.requestLog(
       `Updated file-hasher based on watched changes, recomputing project graph...`
     );
@@ -190,14 +191,12 @@ async function processCollectedUpdatedAndDeletedFiles() {
     if (workspaceConfigHash !== storedWorkspaceConfigHash) {
       storedWorkspaceConfigHash = workspaceConfigHash;
 
-      projectFileMapWithFiles = await retrieveWorkspaceFiles(
-        workspaceRoot,
-        nxJson
-      );
+      ({ externalNodes: knownExternalNodes, ...projectFileMapWithFiles } =
+        await retrieveWorkspaceFiles(workspaceRoot, nxJson));
     } else {
       if (projectFileMapWithFiles) {
         projectFileMapWithFiles = updateProjectFileMap(
-          projectConfigurations,
+          projectNodes,
           projectFileMapWithFiles.projectFileMap,
           projectFileMapWithFiles.allWorkspaceFiles,
           updatedFiles,
@@ -275,7 +274,8 @@ async function createAndSerializeProjectGraph(): Promise<{
     const { projectGraph, projectFileMapCache } =
       await buildProjectGraphUsingProjectFileMap(
         projectsConfigurations,
-        projectFileMap,
+        knownExternalNodes,
+        projectFileMapWithFiles.projectFileMap,
         allWorkspaceFiles,
         currentProjectFileMapCache || readProjectFileMapCache(),
         true

diff --git a/packages/nx/src/generators/tree.spec.ts b/packages/nx/src/generators/tree.spec.ts
@@ -19,7 +19,7 @@ describe('tree', () => {
 
     beforeEach(() => {
       console.error = jest.fn();
-      // console.log = jest.fn();
+      console.log = jest.fn();
 
       dir = dirSync().name;
       ensureDirSync(path.join(dir, 'parent/child'));

diff --git a/packages/nx/src/generators/utils/project-configuration.spec.ts b/packages/nx/src/generators/utils/project-configuration.spec.ts
@@ -15,6 +15,7 @@ import {
 
 import * as projectSchema from '../../../schemas/project-schema.json';
 import { joinPathFragments } from '../../utils/path';
+import { PackageJson } from '../../utils/package-json';
 
 const projectConfiguration: ProjectConfiguration = {
   name: 'test',
@@ -171,6 +172,11 @@ describe('project configuration', () => {
   describe('for npm workspaces', () => {
     beforeEach(() => {
       tree = createTree();
+      writeJson<PackageJson>(tree, 'package.json', {
+        name: '@testing/root',
+        version: '0.0.1',
+        workspaces: ['**/package.json'],
+      });
     });
 
     it('should read project configuration from package.json files', () => {
@@ -182,6 +188,7 @@ describe('project configuration', () => {
       const proj = readProjectConfiguration(tree, 'proj');
 
       expect(proj).toEqual({
+        name: 'proj',
         root: 'proj',
         sourceRoot: 'proj',
         projectType: 'library',
@@ -197,6 +204,7 @@ describe('project configuration', () => {
 
       expect(projects.size).toEqual(1);
       expect(projects.get('proj')).toEqual({
+        name: 'proj',
         root: 'proj',
         sourceRoot: 'proj',
         projectType: 'library',
-Original file line number
+Diff line change
@@ Expand Up / @@ -181,6 +181,8 @@ Type of dependency between projects @@
     • **ProjectGraphBuilder**: `Object`
+    @deprecated(v18): General project graph processors are deprecated. Replace usage with a plugin that utilizes `processProjectNodes` and `processProjectDependencies`.
     ---
     ### Workspaces
@@ Expand Down @@