feat(core): error when running atomized tasks outside of DTE

docs/nx-cloud/features/split-e2e-tasks.md

@@ -394,3 +394,13 @@ With more granular e2e tasks, all the other features of Nx become more powerful.
 ### More Precise Flaky Task Identification
 
 Nx Agents [automatically re-run failed flaky e2e tests](/ci/features/flaky-tasks) on a separate agent without a developer needing to manually re-run the CI pipeline. Leveraging e2e task splitting, Nx identifies the specific flaky test file - this way you can quickly fix the offending test file. Without e2e splitting, Nx identifies that at least one of the e2e tests are flaky - requiring you to find the flaky test on your own.
+
+## Use Atomizer only with Nx Cloud Distribution!
+
+{% callout type="warning" title="To benefit from the performance improvements of Atomizer, distribution with Nx Cloud is required." %}
+{% /callout %}
+
+When running an atomized task like `e2e-ci`, Nx will spawn a new instance of Cypress, Playwright or Jest for each file to facilitate the benefits listed above.
+When running tasks on a single machine, this setup can lead to degraded performance. Each process comes with some overhead, which will slow things down.
+
+When running locally or in CI without distribution, use the non-atomized target (`e2e` in the example above).

packages/nx/src/tasks-runner/run-command.ts

@@ -1,36 +1,40 @@
-import { TasksRunner, TaskStatus } from './tasks-runner';
 import { join } from 'path';
-import { workspaceRoot } from '../utils/workspace-root';
-import { NxArgs } from '../utils/command-line-utils';
-import { isRelativePath } from '../utils/fileutils';
-import { output } from '../utils/output';
-import { shouldStreamOutput } from './utils';
-import { CompositeLifeCycle, LifeCycle } from './life-cycle';
-import { StaticRunManyTerminalOutputLifeCycle } from './life-cycles/static-run-many-terminal-output-life-cycle';
-import { StaticRunOneTerminalOutputLifeCycle } from './life-cycles/static-run-one-terminal-output-life-cycle';
-import { TaskTimingsLifeCycle } from './life-cycles/task-timings-life-cycle';
-import { createRunManyDynamicOutputRenderer } from './life-cycles/dynamic-run-many-terminal-output-life-cycle';
-import { TaskProfilingLifeCycle } from './life-cycles/task-profiling-life-cycle';
-import { isCI } from '../utils/is-ci';
-import { createRunOneDynamicOutputRenderer } from './life-cycles/dynamic-run-one-terminal-output-life-cycle';
-import { ProjectGraph, ProjectGraphProjectNode } from '../config/project-graph';
 import {
   NxJsonConfiguration,
   readNxJson,
   TargetDefaults,
   TargetDependencies,
 } from '../config/nx-json';
+import { ProjectGraph, ProjectGraphProjectNode } from '../config/project-graph';
 import { Task, TaskGraph } from '../config/task-graph';
-import { createTaskGraph } from './create-task-graph';
-import { findCycle, makeAcyclic } from './task-graph-utils';
 import { TargetDependencyConfig } from '../config/workspace-json-project-json';
-import { handleErrors } from '../utils/params';
-import { hashTasksThatDoNotDependOnOutputsOfOtherTasks } from '../hasher/hash-task';
 import { daemonClient } from '../daemon/client/client';
-import { StoreRunInformationLifeCycle } from './life-cycles/store-run-information-life-cycle';
 import { createTaskHasher } from '../hasher/create-task-hasher';
-import { TaskHistoryLifeCycle } from './life-cycles/task-history-life-cycle';
+import { hashTasksThatDoNotDependOnOutputsOfOtherTasks } from '../hasher/hash-task';
+import { NxArgs } from '../utils/command-line-utils';
+import { isRelativePath } from '../utils/fileutils';
+import { isCI } from '../utils/is-ci';
 import { isNxCloudUsed } from '../utils/nx-cloud-utils';
+import { output } from '../utils/output';
+import { handleErrors } from '../utils/params';
+import { workspaceRoot } from '../utils/workspace-root';
+import { createTaskGraph } from './create-task-graph';
+import { CompositeLifeCycle, LifeCycle } from './life-cycle';
+import { createRunManyDynamicOutputRenderer } from './life-cycles/dynamic-run-many-terminal-output-life-cycle';
+import { createRunOneDynamicOutputRenderer } from './life-cycles/dynamic-run-one-terminal-output-life-cycle';
+import { StaticRunManyTerminalOutputLifeCycle } from './life-cycles/static-run-many-terminal-output-life-cycle';
+import { StaticRunOneTerminalOutputLifeCycle } from './life-cycles/static-run-one-terminal-output-life-cycle';
+import { StoreRunInformationLifeCycle } from './life-cycles/store-run-information-life-cycle';
+import { TaskHistoryLifeCycle } from './life-cycles/task-history-life-cycle';
+import { TaskProfilingLifeCycle } from './life-cycles/task-profiling-life-cycle';
+import { TaskTimingsLifeCycle } from './life-cycles/task-timings-life-cycle';
+import {
+  findCycle,
+  makeAcyclic,
+  validateAtomizedTasks,
+} from './task-graph-utils';
+import { TasksRunner, TaskStatus } from './tasks-runner';
+import { shouldStreamOutput } from './utils';
 
 async function getTerminalOutputLifeCycle(
   initiatingProject: string,
@@ -91,7 +95,7 @@ async function getTerminalOutputLifeCycle(
   }
 }
 
-function createTaskGraphAndValidateCycles(
+function createTaskGraphAndRunValidations(
   projectGraph: ProjectGraph,
   extraTargetDependencies: TargetDependencies,
   projectNames: string[],
@@ -129,6 +133,8 @@ function createTaskGraphAndValidateCycles(
     }
   }
 
+  validateAtomizedTasks(taskGraph, projectGraph);
+
   return taskGraph;
 }
 
@@ -147,7 +153,7 @@ export async function runCommand(
     async () => {
       const projectNames = projectsToRun.map((t) => t.name);
 
-      const taskGraph = createTaskGraphAndValidateCycles(
+      const taskGraph = createTaskGraphAndRunValidations(
         projectGraph,
         extraTargetDependencies ?? {},
         projectNames,

packages/nx/src/tasks-runner/task-graph-utils.spec.ts

@@ -1,4 +1,8 @@
-import { findCycle, makeAcyclic } from './task-graph-utils';
+import {
+  findCycle,
+  makeAcyclic,
+  validateAtomizedTasks,
+} from './task-graph-utils';
 
 describe('task graph utils', () => {
   describe('findCycles', () => {
@@ -57,4 +61,141 @@ describe('task graph utils', () => {
       expect(graph.roots).toEqual(['d', 'e']);
     });
   });
+
+  describe('validateAtomizedTasks', () => {
+    let mockProcessExit: jest.SpyInstance;
+    let env: NodeJS.ProcessEnv;
+
+    beforeEach(() => {
+      env = process.env;
+      process.env = {};
+
+      mockProcessExit = jest
+        .spyOn(process, 'exit')
+        .mockImplementation((code: number) => {
+          return undefined as any as never;
+        });
+    });
+
+    afterEach(() => {
+      process.env = env;
+
+      mockProcessExit.mockRestore();
+    });
+
+    it('should do nothing if no tasks are atomized', () => {
+      const taskGraph = {
+        tasks: {
+          'e2e:e2e': {
+            target: {
+              project: 'e2e',
+              target: 'e2e',
+            },
+          },
+        },
+      };
+      const projectGraph = {
+        nodes: {
+          e2e: {
+            data: {
+              targets: {
+                e2e: {},
+              },
+            },
+          },
+        },
+      };
+      validateAtomizedTasks(taskGraph as any, projectGraph as any);
+      expect(mockProcessExit).not.toHaveBeenCalled();
+    });
+    it('should exit if atomized task is present but no DTE', () => {
+      const taskGraph = {
+        tasks: {
+          'e2e:e2e-ci': {
+            target: {
+              project: 'e2e',
+              target: 'e2e-ci',
+            },
+          },
+        },
+      };
+      const projectGraph = {
+        nodes: {
+          e2e: {
+            data: {
+              targets: {
+                'e2e-ci': {
+                  metadata: {
+                    nonAtomizedTarget: 'e2e',
+                  },
+                },
+              },
+            },
+          },
+        },
+      };
+      validateAtomizedTasks(taskGraph as any, projectGraph as any);
+      expect(mockProcessExit).toHaveBeenCalledWith(1);
+    });
+    it('should do nothing if atomized task is present in DTE', () => {
+      process.env['NX_CLOUD_DISTRIBUTED_EXECUTION_ID'] = '123';
+      const taskGraph = {
+        tasks: {
+          'e2e:e2e-ci': {
+            target: {
+              project: 'e2e',
+              target: 'e2e-ci',
+            },
+          },
+        },
+      };
+      const projectGraph = {
+        nodes: {
+          e2e: {
+            data: {
+              targets: {
+                'e2e-ci': {
+                  metadata: {
+                    nonAtomizedTarget: 'e2e',
+                  },
+                },
+              },
+            },
+          },
+        },
+      };
+      validateAtomizedTasks(taskGraph as any, projectGraph as any);
+      expect(mockProcessExit).not.toHaveBeenCalled();
+    });
+    it('should do nothing if atomized task is present but escape hatch env var is set', () => {
+      process.env['NX_SKIP_ATOMIZER_VALIDATION'] = 'true';
+      const taskGraph = {
+        tasks: {
+          'e2e:e2e-ci': {
+            target: {
+              project: 'e2e',
+              target: 'e2e-ci',
+            },
+          },
+        },
+      };
+      const projectGraph = {
+        nodes: {
+          e2e: {
+            data: {
+              targets: {
+                'e2e-ci': {
+                  metadata: {
+                    nonAtomizedTarget: 'e2e',
+                  },
+                },
+              },
+            },
+          },
+        },
+      };
+      validateAtomizedTasks(taskGraph as any, projectGraph as any);
+      expect(mockProcessExit).not.toHaveBeenCalled();
+    });
+  });
 });

packages/nx/src/tasks-runner/task-graph-utils.ts

@@ -1,3 +1,7 @@
+import { ProjectGraph } from '../config/project-graph';
+import { TaskGraph } from '../config/task-graph';
+import { output } from '../utils/output';
+
 function _findCycle(
   graph: { dependencies: Record<string, string[]> },
   id: string,
@@ -66,3 +70,42 @@ export function makeAcyclic(graph: {
     (t) => graph.dependencies[t].length === 0
   );
 }
+
+export function validateAtomizedTasks(
+  taskGraph: TaskGraph,
+  projectGraph: ProjectGraph
+): void {
+  if (process.env['NX_SKIP_ATOMIZER_VALIDATION']) {
+    return;
+  }
+  const tasksWithAtomizer = Object.values(taskGraph.tasks).filter(
+    (task) =>
+      projectGraph.nodes[task.target.project]?.data?.targets?.[
+        task.target.target
+      ]?.metadata?.nonAtomizedTarget !== undefined
+  );
+
+  const isInDTE =
+    process.env['NX_CLOUD_DISTRIBUTED_EXECUTION_ID'] ||
+    process.env['NX_AGENT_NAME'];
+
+  if (tasksWithAtomizer.length > 0 && !isInDTE) {
+    const linkLine =
+      'Learn more at https://nx.dev/ci/features/split-e2e-tasks#use-atomizer-only-with-nx-cloud-distribution';
+    if (tasksWithAtomizer.length === 1) {
+      output.error({
+        title: `The ${tasksWithAtomizer[0].id} task uses the atomizer and should only be run with Nx Cloud distribution.`,
+        bodyLines: [linkLine],
+      });
+    } else {
+      output.error({
+        title: `The following tasks use the atomizer and should only be run with Nx Cloud distribution:`,
+        bodyLines: [
+          `${tasksWithAtomizer.map((task) => task.id).join(', ')}`,
+          linkLine,
+        ],
+      });
+    }
+    process.exit(1);
+  }
+}