From 0693e21aa8a384cfd86fd32b76d0ab72d229dc07 Mon Sep 17 00:00:00 2001 From: Max Kless Date: Thu, 11 Jul 2024 13:26:13 +0200 Subject: [PATCH] feat(core): error when running atomized tasks outside of DTE --- docs/nx-cloud/features/split-e2e-tasks.md | 10 ++ packages/nx/src/tasks-runner/run-command.ts | 52 ++++--- .../src/tasks-runner/task-graph-utils.spec.ts | 143 +++++++++++++++++- .../nx/src/tasks-runner/task-graph-utils.ts | 43 ++++++ 4 files changed, 224 insertions(+), 24 deletions(-) diff --git a/docs/nx-cloud/features/split-e2e-tasks.md b/docs/nx-cloud/features/split-e2e-tasks.md index 2a7960539b3d4..4322767427f95 100644 --- a/docs/nx-cloud/features/split-e2e-tasks.md +++ b/docs/nx-cloud/features/split-e2e-tasks.md @@ -394,3 +394,13 @@ With more granular e2e tasks, all the other features of Nx become more powerful. ### More Precise Flaky Task Identification Nx Agents [automatically re-run failed flaky e2e tests](/ci/features/flaky-tasks) on a separate agent without a developer needing to manually re-run the CI pipeline. Leveraging e2e task splitting, Nx identifies the specific flaky test file - this way you can quickly fix the offending test file. Without e2e splitting, Nx identifies that at least one of the e2e tests are flaky - requiring you to find the flaky test on your own. + +## Use Atomizer only with Nx Cloud Distribution! + +{% callout type="warning" title="To benefit from the performance improvements of Atomizer, distribution with Nx Cloud is required." %} +{% /callout %} + +When running an atomized task like `e2e-ci`, Nx will spawn a new instance of Cypress, Playwright or Jest for each file to facilitate the benefits listed above. +When running tasks on a single machine, this setup can lead to degraded performance. Each process comes with some overhead, which will slow things down. + +When running locally or in CI without distribution, use the non-atomized target (`e2e` in the example above). diff --git a/packages/nx/src/tasks-runner/run-command.ts b/packages/nx/src/tasks-runner/run-command.ts index 06a22716e8ab7..6beb321a0f782 100644 --- a/packages/nx/src/tasks-runner/run-command.ts +++ b/packages/nx/src/tasks-runner/run-command.ts @@ -1,36 +1,40 @@ -import { TasksRunner, TaskStatus } from './tasks-runner'; import { join } from 'path'; -import { workspaceRoot } from '../utils/workspace-root'; -import { NxArgs } from '../utils/command-line-utils'; -import { isRelativePath } from '../utils/fileutils'; -import { output } from '../utils/output'; -import { shouldStreamOutput } from './utils'; -import { CompositeLifeCycle, LifeCycle } from './life-cycle'; -import { StaticRunManyTerminalOutputLifeCycle } from './life-cycles/static-run-many-terminal-output-life-cycle'; -import { StaticRunOneTerminalOutputLifeCycle } from './life-cycles/static-run-one-terminal-output-life-cycle'; -import { TaskTimingsLifeCycle } from './life-cycles/task-timings-life-cycle'; -import { createRunManyDynamicOutputRenderer } from './life-cycles/dynamic-run-many-terminal-output-life-cycle'; -import { TaskProfilingLifeCycle } from './life-cycles/task-profiling-life-cycle'; -import { isCI } from '../utils/is-ci'; -import { createRunOneDynamicOutputRenderer } from './life-cycles/dynamic-run-one-terminal-output-life-cycle'; -import { ProjectGraph, ProjectGraphProjectNode } from '../config/project-graph'; import { NxJsonConfiguration, readNxJson, TargetDefaults, TargetDependencies, } from '../config/nx-json'; +import { ProjectGraph, ProjectGraphProjectNode } from '../config/project-graph'; import { Task, TaskGraph } from '../config/task-graph'; -import { createTaskGraph } from './create-task-graph'; -import { findCycle, makeAcyclic } from './task-graph-utils'; import { TargetDependencyConfig } from '../config/workspace-json-project-json'; -import { handleErrors } from '../utils/params'; -import { hashTasksThatDoNotDependOnOutputsOfOtherTasks } from '../hasher/hash-task'; import { daemonClient } from '../daemon/client/client'; -import { StoreRunInformationLifeCycle } from './life-cycles/store-run-information-life-cycle'; import { createTaskHasher } from '../hasher/create-task-hasher'; -import { TaskHistoryLifeCycle } from './life-cycles/task-history-life-cycle'; +import { hashTasksThatDoNotDependOnOutputsOfOtherTasks } from '../hasher/hash-task'; +import { NxArgs } from '../utils/command-line-utils'; +import { isRelativePath } from '../utils/fileutils'; +import { isCI } from '../utils/is-ci'; import { isNxCloudUsed } from '../utils/nx-cloud-utils'; +import { output } from '../utils/output'; +import { handleErrors } from '../utils/params'; +import { workspaceRoot } from '../utils/workspace-root'; +import { createTaskGraph } from './create-task-graph'; +import { CompositeLifeCycle, LifeCycle } from './life-cycle'; +import { createRunManyDynamicOutputRenderer } from './life-cycles/dynamic-run-many-terminal-output-life-cycle'; +import { createRunOneDynamicOutputRenderer } from './life-cycles/dynamic-run-one-terminal-output-life-cycle'; +import { StaticRunManyTerminalOutputLifeCycle } from './life-cycles/static-run-many-terminal-output-life-cycle'; +import { StaticRunOneTerminalOutputLifeCycle } from './life-cycles/static-run-one-terminal-output-life-cycle'; +import { StoreRunInformationLifeCycle } from './life-cycles/store-run-information-life-cycle'; +import { TaskHistoryLifeCycle } from './life-cycles/task-history-life-cycle'; +import { TaskProfilingLifeCycle } from './life-cycles/task-profiling-life-cycle'; +import { TaskTimingsLifeCycle } from './life-cycles/task-timings-life-cycle'; +import { + findCycle, + makeAcyclic, + validateAtomizedTasks, +} from './task-graph-utils'; +import { TasksRunner, TaskStatus } from './tasks-runner'; +import { shouldStreamOutput } from './utils'; async function getTerminalOutputLifeCycle( initiatingProject: string, @@ -91,7 +95,7 @@ async function getTerminalOutputLifeCycle( } } -function createTaskGraphAndValidateCycles( +function createTaskGraphAndRunValidations( projectGraph: ProjectGraph, extraTargetDependencies: TargetDependencies, projectNames: string[], @@ -129,6 +133,8 @@ function createTaskGraphAndValidateCycles( } } + validateAtomizedTasks(taskGraph, projectGraph); + return taskGraph; } @@ -147,7 +153,7 @@ export async function runCommand( async () => { const projectNames = projectsToRun.map((t) => t.name); - const taskGraph = createTaskGraphAndValidateCycles( + const taskGraph = createTaskGraphAndRunValidations( projectGraph, extraTargetDependencies ?? {}, projectNames, diff --git a/packages/nx/src/tasks-runner/task-graph-utils.spec.ts b/packages/nx/src/tasks-runner/task-graph-utils.spec.ts index 2d413e8929c70..25a1cd81a4119 100644 --- a/packages/nx/src/tasks-runner/task-graph-utils.spec.ts +++ b/packages/nx/src/tasks-runner/task-graph-utils.spec.ts @@ -1,4 +1,8 @@ -import { findCycle, makeAcyclic } from './task-graph-utils'; +import { + findCycle, + makeAcyclic, + validateAtomizedTasks, +} from './task-graph-utils'; describe('task graph utils', () => { describe('findCycles', () => { @@ -57,4 +61,141 @@ describe('task graph utils', () => { expect(graph.roots).toEqual(['d', 'e']); }); }); + + describe('validateAtomizedTasks', () => { + let mockProcessExit: jest.SpyInstance; + let env: NodeJS.ProcessEnv; + + beforeEach(() => { + env = process.env; + process.env = {}; + + mockProcessExit = jest + .spyOn(process, 'exit') + .mockImplementation((code: number) => { + return undefined as any as never; + }); + }); + + afterEach(() => { + process.env = env; + + mockProcessExit.mockRestore(); + }); + + it('should do nothing if no tasks are atomized', () => { + const taskGraph = { + tasks: { + 'e2e:e2e': { + target: { + project: 'e2e', + target: 'e2e', + }, + }, + }, + }; + const projectGraph = { + nodes: { + e2e: { + data: { + targets: { + e2e: {}, + }, + }, + }, + }, + }; + validateAtomizedTasks(taskGraph as any, projectGraph as any); + expect(mockProcessExit).not.toHaveBeenCalled(); + }); + it('should exit if atomized task is present but no DTE', () => { + const taskGraph = { + tasks: { + 'e2e:e2e-ci': { + target: { + project: 'e2e', + target: 'e2e-ci', + }, + }, + }, + }; + const projectGraph = { + nodes: { + e2e: { + data: { + targets: { + 'e2e-ci': { + metadata: { + nonAtomizedTarget: 'e2e', + }, + }, + }, + }, + }, + }, + }; + validateAtomizedTasks(taskGraph as any, projectGraph as any); + expect(mockProcessExit).toHaveBeenCalledWith(1); + }); + it('should do nothing if atomized task is present in DTE', () => { + process.env['NX_CLOUD_DISTRIBUTED_EXECUTION_ID'] = '123'; + const taskGraph = { + tasks: { + 'e2e:e2e-ci': { + target: { + project: 'e2e', + target: 'e2e-ci', + }, + }, + }, + }; + const projectGraph = { + nodes: { + e2e: { + data: { + targets: { + 'e2e-ci': { + metadata: { + nonAtomizedTarget: 'e2e', + }, + }, + }, + }, + }, + }, + }; + validateAtomizedTasks(taskGraph as any, projectGraph as any); + expect(mockProcessExit).not.toHaveBeenCalled(); + }); + it('should do nothing if atomized task is present but escape hatch env var is set', () => { + process.env['NX_SKIP_ATOMIZER_VALIDATION'] = 'true'; + const taskGraph = { + tasks: { + 'e2e:e2e-ci': { + target: { + project: 'e2e', + target: 'e2e-ci', + }, + }, + }, + }; + const projectGraph = { + nodes: { + e2e: { + data: { + targets: { + 'e2e-ci': { + metadata: { + nonAtomizedTarget: 'e2e', + }, + }, + }, + }, + }, + }, + }; + validateAtomizedTasks(taskGraph as any, projectGraph as any); + expect(mockProcessExit).not.toHaveBeenCalled(); + }); + }); }); diff --git a/packages/nx/src/tasks-runner/task-graph-utils.ts b/packages/nx/src/tasks-runner/task-graph-utils.ts index c2f6fe4f43c86..059a449bb3e9e 100644 --- a/packages/nx/src/tasks-runner/task-graph-utils.ts +++ b/packages/nx/src/tasks-runner/task-graph-utils.ts @@ -1,3 +1,7 @@ +import { ProjectGraph } from '../config/project-graph'; +import { TaskGraph } from '../config/task-graph'; +import { output } from '../utils/output'; + function _findCycle( graph: { dependencies: Record }, id: string, @@ -66,3 +70,42 @@ export function makeAcyclic(graph: { (t) => graph.dependencies[t].length === 0 ); } + +export function validateAtomizedTasks( + taskGraph: TaskGraph, + projectGraph: ProjectGraph +): void { + if (process.env['NX_SKIP_ATOMIZER_VALIDATION']) { + return; + } + const tasksWithAtomizer = Object.values(taskGraph.tasks).filter( + (task) => + projectGraph.nodes[task.target.project]?.data?.targets?.[ + task.target.target + ]?.metadata?.nonAtomizedTarget !== undefined + ); + + const isInDTE = + process.env['NX_CLOUD_DISTRIBUTED_EXECUTION_ID'] || + process.env['NX_AGENT_NAME']; + + if (tasksWithAtomizer.length > 0 && !isInDTE) { + const linkLine = + 'Learn more at https://nx.dev/ci/features/split-e2e-tasks#use-atomizer-only-with-nx-cloud-distribution'; + if (tasksWithAtomizer.length === 1) { + output.error({ + title: `The ${tasksWithAtomizer[0].id} task uses the atomizer and should only be run with Nx Cloud distribution.`, + bodyLines: [linkLine], + }); + } else { + output.error({ + title: `The following tasks use the atomizer and should only be run with Nx Cloud distribution:`, + bodyLines: [ + `${tasksWithAtomizer.map((task) => task.id).join(', ')}`, + linkLine, + ], + }); + } + process.exit(1); + } +}