fix(core): only start plugin workers once (#22222)

nrwl · Mar 8, 2024 · ec12e67 · ec12e67
1 parent f98b18d
commit ec12e67
Show file tree

Hide file tree

Showing 22 changed files with 299 additions and 228 deletions.
diff --git a/packages/devkit/src/utils/convert-nx-executor.ts b/packages/devkit/src/utils/convert-nx-executor.ts
@@ -8,7 +8,6 @@ const {
   Workspaces,
   readNxJsonFromDisk,
   retrieveProjectConfigurationsWithAngularProjects,
-  shutdownPluginWorkers,
 } = requireNx();
 
 /**
@@ -39,7 +38,6 @@ export function convertNxExecutor(executor: Executor) {
             (workspaces as any).readProjectsConfigurations({
               _includeProjectsFromAngularJson: true,
             });
-      shutdownPluginWorkers?.();
 
       const context: ExecutorContext = {
         root: builderContext.workspaceRoot,

diff --git a/packages/nx/src/config/workspaces.spec.ts b/packages/nx/src/config/workspaces.spec.ts
@@ -3,7 +3,7 @@ import { TempFs } from '../internal-testing-utils/temp-fs';
 import { withEnvironmentVariables } from '../internal-testing-utils/with-environment';
 import { retrieveProjectConfigurations } from '../project-graph/utils/retrieve-workspace-files';
 import { readNxJson } from './configuration';
-import { shutdownPluginWorkers } from '../project-graph/plugins/plugin-pool';
+import { loadNxPluginsInIsolation } from '../project-graph/plugins/internal-api';
 
 describe('Workspaces', () => {
   let fs: TempFs;
@@ -37,7 +37,19 @@ describe('Workspaces', () => {
         {
           NX_WORKSPACE_ROOT_PATH: fs.tempDir,
         },
-        () => retrieveProjectConfigurations(fs.tempDir, readNxJson(fs.tempDir))
+        async () => {
+          const [plugins, cleanup] = await loadNxPluginsInIsolation(
+            readNxJson(fs.tempDir).plugins,
+            fs.tempDir
+          );
+          const res = retrieveProjectConfigurations(
+            plugins,
+            fs.tempDir,
+            readNxJson(fs.tempDir)
+          );
+          cleanup();
+          return res;
+        }
       );
       expect(projects['my-package']).toEqual({
         name: 'my-package',

diff --git a/packages/nx/src/daemon/server/handle-request-project-graph.ts b/packages/nx/src/daemon/server/handle-request-project-graph.ts
@@ -3,6 +3,8 @@ import { serializeResult } from '../socket-utils';
 import { serverLogger } from './logger';
 import { getCachedSerializedProjectGraphPromise } from './project-graph-incremental-recomputation';
 import { HandlerResult } from './server';
+import { getPlugins } from './plugins';
+import { readNxJson } from '../../config/nx-json';
 
 export async function handleRequestProjectGraph(): Promise<HandlerResult> {
   try {

diff --git a/packages/nx/src/daemon/server/plugins.ts b/packages/nx/src/daemon/server/plugins.ts
@@ -0,0 +1,26 @@
+import { readNxJson } from '../../config/nx-json';
+import {
+  RemotePlugin,
+  loadNxPluginsInIsolation,
+} from '../../project-graph/plugins/internal-api';
+import { workspaceRoot } from '../../utils/workspace-root';
+
+let loadedPlugins: Promise<RemotePlugin[]>;
+let cleanup: () => void;
+
+export async function getPlugins() {
+  if (loadedPlugins) {
+    return loadedPlugins;
+  }
+  const pluginsConfiguration = readNxJson().plugins ?? [];
+  const [result, cleanupFn] = await loadNxPluginsInIsolation(
+    pluginsConfiguration,
+    workspaceRoot
+  );
+  cleanup = cleanupFn;
+  return result;
+}
+
+export function cleanupPlugins() {
+  cleanup();
+}
diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts
@@ -29,6 +29,8 @@ import { workspaceRoot } from '../../utils/workspace-root';
 import { notifyFileWatcherSockets } from './file-watching/file-watcher-sockets';
 import { serverLogger } from './logger';
 import { NxWorkspaceFilesExternals } from '../../native';
+import { RemotePlugin } from '../../project-graph/plugins/internal-api';
+import { getPlugins } from './plugins';
 
 interface SerializedProjectGraph {
   error: Error | null;
@@ -69,14 +71,15 @@ export async function getCachedSerializedProjectGraphPromise(): Promise<Serializ
     // reset the wait time
     waitPeriod = 100;
     await resetInternalStateIfNxDepsMissing();
+    const plugins = await getPlugins();
     if (collectedUpdatedFiles.size == 0 && collectedDeletedFiles.size == 0) {
       if (!cachedSerializedProjectGraphPromise) {
         cachedSerializedProjectGraphPromise =
-          processFilesAndCreateAndSerializeProjectGraph();
+          processFilesAndCreateAndSerializeProjectGraph(plugins);
       }
     } else {
       cachedSerializedProjectGraphPromise =
-        processFilesAndCreateAndSerializeProjectGraph();
+        processFilesAndCreateAndSerializeProjectGraph(plugins);
     }
     return await cachedSerializedProjectGraphPromise;
   } catch (e) {
@@ -123,7 +126,7 @@ export function addUpdatedAndDeletedFiles(
       }
 
       cachedSerializedProjectGraphPromise =
-        processFilesAndCreateAndSerializeProjectGraph();
+        processFilesAndCreateAndSerializeProjectGraph(await getPlugins());
       await cachedSerializedProjectGraphPromise;
 
       if (createdFiles.length > 0) {
@@ -199,7 +202,9 @@ async function processCollectedUpdatedAndDeletedFiles(
   }
 }
 
-async function processFilesAndCreateAndSerializeProjectGraph(): Promise<SerializedProjectGraph> {
+async function processFilesAndCreateAndSerializeProjectGraph(
+  plugins: RemotePlugin[]
+): Promise<SerializedProjectGraph> {
   try {
     performance.mark('hash-watched-changes-start');
     const updatedFiles = [...collectedUpdatedFiles.values()];
@@ -219,6 +224,7 @@ async function processFilesAndCreateAndSerializeProjectGraph(): Promise<Serializ
     const nxJson = readNxJson(workspaceRoot);
     global.NX_GRAPH_CREATION = true;
     const graphNodes = await retrieveProjectConfigurations(
+      plugins,
       workspaceRoot,
       nxJson
     );
@@ -275,7 +281,8 @@ async function createAndSerializeProjectGraph({
         allWorkspaceFiles,
         rustReferences,
         currentProjectFileMapCache || readFileMapCache(),
-        true
+        true,
+        await getPlugins()
       );
     currentProjectFileMapCache = projectFileMapCache;
     currentProjectGraph = projectGraph;

diff --git a/packages/nx/src/daemon/server/shutdown-utils.ts b/packages/nx/src/daemon/server/shutdown-utils.ts
@@ -4,21 +4,26 @@ import { serverLogger } from './logger';
 import { serializeResult } from '../socket-utils';
 import { deleteDaemonJsonProcessCache } from '../cache';
 import type { Watcher } from '../../native';
+import { cleanupPlugins } from './plugins';
 
 export const SERVER_INACTIVITY_TIMEOUT_MS = 10800000 as const; // 10800000 ms = 3 hours
 
 let watcherInstance: Watcher | undefined;
+
 export function storeWatcherInstance(instance: Watcher) {
   watcherInstance = instance;
 }
+
 export function getWatcherInstance() {
   return watcherInstance;
 }
 
 let outputWatcherInstance: Watcher | undefined;
+
 export function storeOutputWatcherInstance(instance: Watcher) {
   outputWatcherInstance = instance;
 }
+
 export function getOutputWatcherInstance() {
   return outputWatcherInstance;
 }
@@ -35,6 +40,7 @@ export async function handleServerProcessTermination({
   try {
     server.close();
     deleteDaemonJsonProcessCache();
+    cleanupPlugins();
 
     if (watcherInstance) {
       await watcherInstance.stop();

diff --git a/packages/nx/src/devkit-internals.ts b/packages/nx/src/devkit-internals.ts
@@ -21,4 +21,3 @@ export {
   findProjectForPath,
 } from './project-graph/utils/find-project-for-path';
 export { registerTsProject } from './plugins/js/utils/register';
-export { shutdownPluginWorkers } from './project-graph/plugins/plugin-pool';
diff --git a/packages/nx/src/executors/utils/convert-nx-executor.ts b/packages/nx/src/executors/utils/convert-nx-executor.ts
@@ -7,6 +7,7 @@ import { readNxJson } from '../../config/nx-json';
 import { Executor, ExecutorContext } from '../../config/misc-interfaces';
 import { retrieveProjectConfigurations } from '../../project-graph/utils/retrieve-workspace-files';
 import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
+import { loadNxPluginsInIsolation } from '../../project-graph/plugins/internal-api';
 
 /**
  * Convert an Nx Executor into an Angular Devkit Builder
@@ -17,15 +18,22 @@ export function convertNxExecutor(executor: Executor) {
   const builderFunction = (options, builderContext) => {
     const promise = async () => {
       const nxJsonConfiguration = readNxJson(builderContext.workspaceRoot);
+
+      const [plugins, cleanup] = await loadNxPluginsInIsolation(
+        nxJsonConfiguration.plugins,
+        builderContext.workspaceRoot
+      );
       const projectsConfigurations: ProjectsConfigurations = {
         version: 2,
         projects: (
           await retrieveProjectConfigurations(
+            plugins,
             builderContext.workspaceRoot,
             nxJsonConfiguration
           )
         ).projects,
       };
+      cleanup();
       const context: ExecutorContext = {
         root: builderContext.workspaceRoot,
         projectName: builderContext.target.project,

diff --git a/packages/nx/src/generators/utils/project-configuration.ts b/packages/nx/src/generators/utils/project-configuration.ts
@@ -28,6 +28,7 @@ import { readJson, writeJson } from './json';
 import { readNxJson } from './nx-json';
 
 import type { Tree } from '../tree';
+import { NxPlugin } from '../../project-graph/plugins';
 
 export { readNxJson, updateNxJson } from './nx-json';
 
@@ -201,7 +202,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): {
   ];
   const projectGlobPatterns = configurationGlobs([
     ProjectJsonProjectsPlugin,
-    { createNodes: packageJsonWorkspacesCreateNodes },
+    { createNodes: packageJsonWorkspacesCreateNodes } as NxPlugin,
   ]);
   const globbedFiles = globWithWorkspaceContext(tree.root, projectGlobPatterns);
   const createdFiles = findCreatedProjectFiles(tree, patterns);

diff --git a/packages/nx/src/migrations/update-15-1-0/set-project-names.ts b/packages/nx/src/migrations/update-15-1-0/set-project-names.ts
@@ -4,16 +4,14 @@ import { dirname } from 'path';
 import { readJson, writeJson } from '../../generators/utils/json';
 import { formatChangedFilesWithPrettierIfAvailable } from '../../generators/internal-utils/format-changed-files-with-prettier-if-available';
 import { retrieveProjectConfigurationPaths } from '../../project-graph/utils/retrieve-workspace-files';
-import { loadNxPlugins } from '../../project-graph/plugins/internal-api';
-import { shutdownPluginWorkers } from '../../project-graph/plugins/plugin-pool';
+import { loadPlugins } from '../../project-graph/plugins/internal-api';
 
 export default async function (tree: Tree) {
   const nxJson = readNxJson(tree);
-  const projectFiles = await retrieveProjectConfigurationPaths(
+  const projectFiles = retrieveProjectConfigurationPaths(
     tree.root,
-    await loadNxPlugins(nxJson?.plugins)
+    (await loadPlugins(nxJson?.plugins ?? [], tree.root)).map((p) => p.plugin)
   );
-  await shutdownPluginWorkers();
   const projectJsons = projectFiles.filter((f) => f.endsWith('project.json'));
 
   for (let f of projectJsons) {

diff --git a/.../nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts b/.../nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts
@@ -1,4 +1,5 @@
 import { TempFs } from '../../../../internal-testing-utils/temp-fs';
+
 const tempFs = new TempFs('explicit-project-deps');
 
 import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
@@ -9,7 +10,8 @@ import {
 } from '../../../../project-graph/utils/retrieve-workspace-files';
 import { CreateDependenciesContext } from '../../../../project-graph/plugins';
 import { setupWorkspaceContext } from '../../../../utils/workspace-context';
-import { shutdownPluginWorkers } from '../../../../project-graph/plugins/plugin-pool';
+import ProjectJsonProjectsPlugin from '../../../project-json/build-nodes/project-json';
+import { loadNxPluginsInIsolation } from '../../../../project-graph/plugins/internal-api';
 
 // projectName => tsconfig import path
 const dependencyProjectNamesToImportPaths = {
@@ -23,10 +25,6 @@ describe('explicit project dependencies', () => {
     tempFs.reset();
   });
 
-  afterEach(async () => {
-    await shutdownPluginWorkers();
-  });
-
   describe('static imports, dynamic imports, and commonjs requires', () => {
     it('should build explicit dependencies for static imports, and top-level dynamic imports and commonjs requires', async () => {
       const source = 'proj';
@@ -568,10 +566,13 @@ async function createContext(
 
   setupWorkspaceContext(tempFs.tempDir);
 
+  const [plugins, cleanup] = await loadNxPluginsInIsolation([], tempFs.tempDir);
   const { projects, projectRootMap } = await retrieveProjectConfigurations(
+    plugins,
     tempFs.tempDir,
     nxJson
   );
+  cleanup();
 
   const { fileMap } = await retrieveWorkspaceFiles(
     tempFs.tempDir,

diff --git a/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts b/packages/nx/src/project-graph/affected/locators/project-glob-changes.ts
@@ -4,13 +4,14 @@ import { workspaceRoot } from '../../../utils/workspace-root';
 import { join } from 'path';
 import { existsSync } from 'fs';
 import { configurationGlobs } from '../../utils/retrieve-workspace-files';
-import { loadNxPlugins } from '../../plugins/internal-api';
+import { loadPlugins } from '../../plugins/internal-api';
 import { combineGlobPatterns } from '../../../utils/globs';
 
 export const getTouchedProjectsFromProjectGlobChanges: TouchedProjectLocator =
   async (touchedFiles, projectGraphNodes, nxJson): Promise<string[]> => {
+    const plugins = await loadPlugins(nxJson?.plugins ?? [], workspaceRoot);
     const globPattern = combineGlobPatterns(
-      configurationGlobs(await loadNxPlugins(nxJson?.plugins, workspaceRoot))
+      configurationGlobs(plugins.map((p) => p.plugin))
     );
 
     const touchedProjects = new Set<string>();

diff --git a/packages/nx/src/project-graph/build-project-graph.ts b/packages/nx/src/project-graph/build-project-graph.ts
@@ -13,7 +13,7 @@ import {
 } from './nx-deps-cache';
 import { applyImplicitDependencies } from './utils/implicit-project-dependencies';
 import { normalizeProjectNodes } from './utils/normalize-project-nodes';
-import { loadNxPlugins } from './plugins/internal-api';
+import { RemotePlugin } from './plugins/internal-api';
 import { isNxPluginV1, isNxPluginV2 } from './plugins/utils';
 import { CreateDependenciesContext } from './plugins';
 import { getRootTsConfigPath } from '../plugins/js/utils/typescript';
@@ -29,10 +29,8 @@ import { ProjectConfiguration } from '../config/workspace-json-project-json';
 import { readNxJson } from '../config/configuration';
 import { existsSync } from 'fs';
 import { PackageJson } from '../utils/package-json';
-import { getNxRequirePaths } from '../utils/installation-directory';
 import { output } from '../utils/output';
-import { ExternalObject, NxWorkspaceFilesExternals } from '../native';
-import { shutdownPluginWorkers } from './plugins/plugin-pool';
+import { NxWorkspaceFilesExternals } from '../native';
 
 let storedFileMap: FileMap | null = null;
 let storedAllWorkspaceFiles: FileData[] | null = null;
@@ -68,7 +66,8 @@ export async function buildProjectGraphUsingProjectFileMap(
   allWorkspaceFiles: FileData[],
   rustReferences: NxWorkspaceFilesExternals,
   fileMapCache: FileMapCache | null,
-  shouldWriteCache: boolean
+  shouldWriteCache: boolean,
+  plugins: RemotePlugin[]
 ): Promise<{
   projectGraph: ProjectGraph;
   projectFileMapCache: FileMapCache;
@@ -118,7 +117,8 @@ export async function buildProjectGraphUsingProjectFileMap(
     externalNodes,
     context,
     cachedFileData,
-    projectGraphVersion
+    projectGraphVersion,
+    plugins
   );
   const projectFileMapCache = createProjectFileMapCache(
     nxJson,
@@ -129,7 +129,6 @@ export async function buildProjectGraphUsingProjectFileMap(
   if (shouldWriteCache) {
     writeCache(projectFileMapCache, projectGraph);
   }
-  await shutdownPluginWorkers();
   return {
     projectGraph,
     projectFileMapCache,
@@ -165,7 +164,8 @@ async function buildProjectGraphUsingContext(
   knownExternalNodes: Record<string, ProjectGraphExternalNode>,
   ctx: CreateDependenciesContext,
   cachedFileData: CachedFileData,
-  projectGraphVersion: string
+  projectGraphVersion: string,
+  plugins: RemotePlugin[]
 ) {
   performance.mark('build project graph:start');
 
@@ -178,7 +178,7 @@ async function buildProjectGraphUsingContext(
   await normalizeProjectNodes(ctx, builder);
   const initProjectGraph = builder.getUpdatedProjectGraph();
 
-  const r = await updateProjectGraphWithPlugins(ctx, initProjectGraph);
+  const r = await updateProjectGraphWithPlugins(ctx, initProjectGraph, plugins);
 
   const updatedBuilder = new ProjectGraphBuilder(r, ctx.fileMap.projectFileMap);
   for (const proj of Object.keys(cachedFileData.projectFileMap)) {
@@ -235,12 +235,9 @@ function createContext(
 
 async function updateProjectGraphWithPlugins(
   context: CreateDependenciesContext,
-  initProjectGraph: ProjectGraph
+  initProjectGraph: ProjectGraph,
+  plugins: RemotePlugin[]
 ) {
-  const plugins = await loadNxPlugins(
-    context.nxJsonConfiguration?.plugins,
-    context.workspaceRoot
-  );
   let graph = initProjectGraph;
   for (const plugin of plugins) {
     try {