From 8f6fcf23e5f113a478da1c17b6aff39715ea163f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leosvel=20P=C3=A9rez=20Espinosa?= Date: Wed, 24 May 2023 08:50:41 +0100 Subject: [PATCH] feat(core): expose the task graph in the executor context (#17111) --- docs/generated/devkit/nx_devkit.md | 13 +- .../packages/devkit/documents/nx_devkit.md | 13 +- .../devkit/src/utils/convert-nx-executor.ts | 13 +- packages/nx/bin/run-executor.ts | 54 ++++---- packages/nx/src/command-line/run/run-one.ts | 4 +- packages/nx/src/command-line/run/run.ts | 122 +++++++++++------- packages/nx/src/config/misc-interfaces.ts | 6 + .../executors/utils/convert-nx-executor.ts | 6 +- .../src/tasks-runner/batch/batch-messages.ts | 3 +- .../nx/src/tasks-runner/batch/run-batch.ts | 17 ++- .../forked-process-task-runner.ts | 42 ++++-- .../nx/src/tasks-runner/task-orchestrator.ts | 5 +- 12 files changed, 178 insertions(+), 120 deletions(-) diff --git a/docs/generated/devkit/nx_devkit.md b/docs/generated/devkit/nx_devkit.md index 2a7b74407b5b3..4b0a25ae9ae8f 100644 --- a/docs/generated/devkit/nx_devkit.md +++ b/docs/generated/devkit/nx_devkit.md @@ -2064,14 +2064,11 @@ Note that the return value is a promise of an iterator, so you need to await bef #### Parameters -| Name | Type | -| :--------------------------------- | :-------------------------------------------------------------------- | -| `targetDescription` | `Object` | -| `targetDescription.configuration?` | `string` | -| `targetDescription.project` | `string` | -| `targetDescription.target` | `string` | -| `overrides` | `Object` | -| `context` | [`ExecutorContext`](../../devkit/documents/nx_devkit#executorcontext) | +| Name | Type | +| :------------------ | :-------------------------------------------------------------------- | +| `targetDescription` | [`Target`](../../devkit/documents/nx_devkit#target) | +| `overrides` | `Object` | +| `context` | [`ExecutorContext`](../../devkit/documents/nx_devkit#executorcontext) | #### Returns diff --git a/docs/generated/packages/devkit/documents/nx_devkit.md b/docs/generated/packages/devkit/documents/nx_devkit.md index 2a7b74407b5b3..4b0a25ae9ae8f 100644 --- a/docs/generated/packages/devkit/documents/nx_devkit.md +++ b/docs/generated/packages/devkit/documents/nx_devkit.md @@ -2064,14 +2064,11 @@ Note that the return value is a promise of an iterator, so you need to await bef #### Parameters -| Name | Type | -| :--------------------------------- | :-------------------------------------------------------------------- | -| `targetDescription` | `Object` | -| `targetDescription.configuration?` | `string` | -| `targetDescription.project` | `string` | -| `targetDescription.target` | `string` | -| `overrides` | `Object` | -| `context` | [`ExecutorContext`](../../devkit/documents/nx_devkit#executorcontext) | +| Name | Type | +| :------------------ | :-------------------------------------------------------------------- | +| `targetDescription` | [`Target`](../../devkit/documents/nx_devkit#target) | +| `overrides` | `Object` | +| `context` | [`ExecutorContext`](../../devkit/documents/nx_devkit#executorcontext) | #### Returns diff --git a/packages/devkit/src/utils/convert-nx-executor.ts b/packages/devkit/src/utils/convert-nx-executor.ts index 22c7d2416fc2c..a2f92aa88a3d2 100644 --- a/packages/devkit/src/utils/convert-nx-executor.ts +++ b/packages/devkit/src/utils/convert-nx-executor.ts @@ -1,10 +1,8 @@ import type { Observable } from 'rxjs'; import type { Executor, ExecutorContext } from 'nx/src/config/misc-interfaces'; -import type { ProjectGraph } from 'nx/src/config/project-graph'; import { requireNx } from '../../nx'; -const { createProjectGraphAsync, readCachedProjectGraph, Workspaces } = - requireNx(); +const { Workspaces } = requireNx(); /** * Convert an Nx Executor into an Angular Devkit Builder @@ -21,12 +19,6 @@ export function convertNxExecutor(executor: Executor) { }); const promise = async () => { - let projectGraph: ProjectGraph; - try { - projectGraph = readCachedProjectGraph(); - } catch { - projectGraph = await createProjectGraphAsync(); - } const context: ExecutorContext = { root: builderContext.workspaceRoot, projectName: builderContext.target.project, @@ -36,7 +28,8 @@ export function convertNxExecutor(executor: Executor) { projectsConfigurations, nxJsonConfiguration, cwd: process.cwd(), - projectGraph, + projectGraph: null, + taskGraph: null, isVerbose: false, }; return executor(options, context); diff --git a/packages/nx/bin/run-executor.ts b/packages/nx/bin/run-executor.ts index 2350ec28e3855..23ed41d06f3e6 100644 --- a/packages/nx/bin/run-executor.ts +++ b/packages/nx/bin/run-executor.ts @@ -1,5 +1,6 @@ import { appendFileSync, openSync, writeFileSync } from 'fs'; -import { run } from '../src/command-line/run/run'; +import { Target, run } from '../src/command-line/run/run'; +import { TaskGraph } from '../src/config/task-graph'; if (process.env.NX_TERMINAL_OUTPUT_PATH) { setUpOutputWatching( @@ -12,32 +13,8 @@ if (!process.env.NX_WORKSPACE_ROOT) { console.error('Invalid Nx command invocation'); process.exit(1); } -requireCli(); -function requireCli() { - process.env.NX_CLI_SET = 'true'; - try { - const args = JSON.parse(process.argv[2]); - run( - process.cwd(), - process.env.NX_WORKSPACE_ROOT, - args.targetDescription, - args.overrides, - args.isVerbose, - false - ) - .then((statusCode) => { - process.exit(statusCode); - }) - .catch((e) => { - console.error(`Unexpected error`); - console.error(e); - }); - } catch (e) { - console.error(`Could not find 'nx' module in this workspace.`, e); - process.exit(1); - } -} +process.env.NX_CLI_SET = 'true'; /** * We need to collect all stdout and stderr and store it, so the caching mechanism @@ -93,3 +70,28 @@ function setUpOutputWatching(captureStderr: boolean, streamOutput: boolean) { } }); } + +process.on( + 'message', + async (message: { + targetDescription: Target; + overrides: Record; + taskGraph: TaskGraph; + isVerbose: boolean; + }) => { + try { + const statusCode = await run( + process.cwd(), + process.env.NX_WORKSPACE_ROOT, + message.targetDescription, + message.overrides, + message.isVerbose, + message.taskGraph + ); + process.exit(statusCode); + } catch (e) { + console.error(`Could not find 'nx' module in this workspace.`, e); + process.exit(1); + } + } +); diff --git a/packages/nx/src/command-line/run/run-one.ts b/packages/nx/src/command-line/run/run-one.ts index 68beffaa06b4e..bb4800a26524a 100644 --- a/packages/nx/src/command-line/run/run-one.ts +++ b/packages/nx/src/command-line/run/run-one.ts @@ -57,9 +57,7 @@ export async function runOne( process.env.NX_VERBOSE_LOGGING = 'true'; } if (nxArgs.help) { - await ( - await import('./run') - ).run(cwd, workspaceRoot, opts, {}, false, true); + await (await import('./run')).printTargetRunHelp(opts, workspaceRoot); process.exit(0); } diff --git a/packages/nx/src/command-line/run/run.ts b/packages/nx/src/command-line/run/run.ts index 012086af8877a..1b7d5620606a9 100644 --- a/packages/nx/src/command-line/run/run.ts +++ b/packages/nx/src/command-line/run/run.ts @@ -20,6 +20,7 @@ import { ProjectsConfigurations, } from '../../config/workspace-json-project-json'; import { Executor, ExecutorContext } from '../../config/misc-interfaces'; +import { TaskGraph } from '../../config/task-graph'; import { serializeOverridesIntoCommandLine } from '../../utils/serialize-overrides-into-command-line'; import { readCachedProjectGraph, @@ -123,28 +124,13 @@ function createImplicitTargetConfig( return buildTargetFromScript(targetName, nx); } -async function runExecutorInternal( - { - project, - target, - configuration, - }: { - project: string; - target: string; - configuration?: string; - }, - overrides: { [k: string]: any }, +async function parseExecutorAndTarget( + ws: Workspaces, + { project, target, configuration }: Target, root: string, - cwd: string, projectsConfigurations: ProjectsConfigurations, - nxJsonConfiguration: NxJsonConfiguration, - projectGraph: ProjectGraph, - isVerbose: boolean, - printHelp: boolean -): Promise> { - validateProject(projectsConfigurations, project); - - const ws = new Workspaces(root); + nxJsonConfiguration: NxJsonConfiguration +) { const proj = projectsConfigurations.projects[project]; const targetConfig = proj.targets?.[target] || @@ -159,21 +145,60 @@ async function runExecutorInternal( throw new Error(`Cannot find target '${target}' for project '${project}'`); } - configuration = configuration ?? targetConfig.defaultConfiguration; - const [nodeModule, executor] = targetConfig.executor.split(':'); const { schema, implementationFactory } = ws.readExecutor( nodeModule, executor ); - if (printHelp) { - printRunHelp({ project, target }, schema, { - plugin: nodeModule, - entity: executor, - }); - process.exit(0); - } + return { executor, implementationFactory, nodeModule, schema, targetConfig }; +} + +async function printTargetRunHelpInternal( + { project, target, configuration }: Target, + root: string, + projectsConfigurations: ProjectsConfigurations, + nxJsonConfiguration: NxJsonConfiguration +) { + const ws = new Workspaces(root); + const { executor, nodeModule, schema } = await parseExecutorAndTarget( + ws, + { project, target, configuration }, + root, + projectsConfigurations, + nxJsonConfiguration + ); + + printRunHelp({ project, target }, schema, { + plugin: nodeModule, + entity: executor, + }); + process.exit(0); +} + +async function runExecutorInternal( + { project, target, configuration }: Target, + overrides: { [k: string]: any }, + root: string, + cwd: string, + projectsConfigurations: ProjectsConfigurations, + nxJsonConfiguration: NxJsonConfiguration, + projectGraph: ProjectGraph, + taskGraph: TaskGraph, + isVerbose: boolean +): Promise> { + validateProject(projectsConfigurations, project); + + const ws = new Workspaces(root); + const { executor, implementationFactory, nodeModule, schema, targetConfig } = + await parseExecutorAndTarget( + ws, + { project, target, configuration }, + root, + projectsConfigurations, + nxJsonConfiguration + ); + configuration ??= targetConfig.defaultConfiguration; const combinedOptions = combineOptionsForExecutor( overrides, @@ -197,6 +222,7 @@ async function runExecutorInternal( targetName: target, configurationName: configuration, projectGraph, + taskGraph, cwd, isVerbose, }) as Promise | AsyncIterableIterator; @@ -254,11 +280,7 @@ async function runExecutorInternal( * Note that the return value is a promise of an iterator, so you need to await before iterating over it. */ export async function runExecutor( - targetDescription: { - project: string; - target: string; - configuration?: string; - }, + targetDescription: Target, overrides: { [k: string]: any }, context: ExecutorContext ): Promise> { @@ -273,22 +295,34 @@ export async function runExecutor( context.projectsConfigurations, context.nxJsonConfiguration, context.projectGraph, - context.isVerbose, - false + context.taskGraph, + context.isVerbose ); } +export function printTargetRunHelp(targetDescription: Target, root: string) { + const projectGraph = readCachedProjectGraph(); + return handleErrors(false, async () => { + const projectsConfigurations = + readProjectsConfigurationFromProjectGraph(projectGraph); + const nxJsonConfiguration = readNxJson(); + + printTargetRunHelpInternal( + targetDescription, + root, + projectsConfigurations, + nxJsonConfiguration + ); + }); +} + export function run( cwd: string, root: string, - targetDescription: { - project: string; - target: string; - configuration?: string; - }, + targetDescription: Target, overrides: { [k: string]: any }, isVerbose: boolean, - isHelp: boolean + taskGraph: TaskGraph ) { const projectGraph = readCachedProjectGraph(); return handleErrors(isVerbose, async () => { @@ -303,8 +337,8 @@ export function run( projectsConfigurations, readNxJson(), projectGraph, - isVerbose, - isHelp + taskGraph, + isVerbose ) ); }); diff --git a/packages/nx/src/config/misc-interfaces.ts b/packages/nx/src/config/misc-interfaces.ts index 86960a4786099..175da59241eff 100644 --- a/packages/nx/src/config/misc-interfaces.ts +++ b/packages/nx/src/config/misc-interfaces.ts @@ -210,6 +210,12 @@ export interface ExecutorContext { */ projectGraph?: ProjectGraph; + /** + * A snapshot of the task graph as + * it existed when the Nx command was kicked off + */ + taskGraph?: TaskGraph; + /** * Deprecated. Use projectsConfigurations or nxJsonConfiguration * The full workspace configuration diff --git a/packages/nx/src/executors/utils/convert-nx-executor.ts b/packages/nx/src/executors/utils/convert-nx-executor.ts index 7fc6784d0f33e..6d6baffd37a10 100644 --- a/packages/nx/src/executors/utils/convert-nx-executor.ts +++ b/packages/nx/src/executors/utils/convert-nx-executor.ts @@ -5,11 +5,6 @@ import type { Observable } from 'rxjs'; import { Workspaces } from '../../config/workspaces'; import { Executor, ExecutorContext } from '../../config/misc-interfaces'; -import { - createProjectGraphAsync, - readCachedProjectGraph, -} from '../../project-graph/project-graph'; -import { ProjectGraph } from '../../config/project-graph'; /** * Convert an Nx Executor into an Angular Devkit Builder @@ -34,6 +29,7 @@ export function convertNxExecutor(executor: Executor) { nxJsonConfiguration, cwd: process.cwd(), projectGraph: null, + taskGraph: null, isVerbose: false, }; return executor(options, context); diff --git a/packages/nx/src/tasks-runner/batch/batch-messages.ts b/packages/nx/src/tasks-runner/batch/batch-messages.ts index 8636b7f0f6308..062490c0a50c1 100644 --- a/packages/nx/src/tasks-runner/batch/batch-messages.ts +++ b/packages/nx/src/tasks-runner/batch/batch-messages.ts @@ -8,7 +8,8 @@ export enum BatchMessageType { export interface BatchTasksMessage { type: BatchMessageType.Tasks; executorName: string; - taskGraph: TaskGraph; + batchTaskGraph: TaskGraph; + fullTaskGraph: TaskGraph; } /** * Results of running the batch. Mapped from task id to results diff --git a/packages/nx/src/tasks-runner/batch/run-batch.ts b/packages/nx/src/tasks-runner/batch/run-batch.ts index adb5da07cc854..37b0a16dfe154 100644 --- a/packages/nx/src/tasks-runner/batch/run-batch.ts +++ b/packages/nx/src/tasks-runner/batch/run-batch.ts @@ -20,14 +20,18 @@ function getBatchExecutor(executorName: string) { return workspace.readExecutor(nodeModule, exportName); } -async function runTasks(executorName: string, taskGraph: TaskGraph) { +async function runTasks( + executorName: string, + batchTaskGraph: TaskGraph, + fullTaskGraph: TaskGraph +) { const input: Record = {}; const projectGraph = await createProjectGraphAsync(); const projectsConfigurations = readProjectsConfigurationFromProjectGraph(projectGraph); const nxJsonConfiguration = readNxJson(); const batchExecutor = getBatchExecutor(executorName); - const tasks = Object.values(taskGraph.tasks); + const tasks = Object.values(batchTaskGraph.tasks); const context: ExecutorContext = { root: workspaceRoot, cwd: process.cwd(), @@ -36,6 +40,7 @@ async function runTasks(executorName: string, taskGraph: TaskGraph) { workspace: { ...projectsConfigurations, ...nxJsonConfiguration }, isVerbose: false, projectGraph, + taskGraph: fullTaskGraph, }; for (const task of tasks) { const projectConfiguration = @@ -54,7 +59,7 @@ async function runTasks(executorName: string, taskGraph: TaskGraph) { try { const results = await batchExecutor.batchImplementationFactory()( - taskGraph, + batchTaskGraph, input, tasks[0].overrides, context @@ -75,7 +80,11 @@ async function runTasks(executorName: string, taskGraph: TaskGraph) { process.on('message', async (message: BatchMessage) => { switch (message.type) { case BatchMessageType.Tasks: { - const results = await runTasks(message.executorName, message.taskGraph); + const results = await runTasks( + message.executorName, + message.batchTaskGraph, + message.fullTaskGraph + ); process.send({ type: BatchMessageType.Complete, results, diff --git a/packages/nx/src/tasks-runner/forked-process-task-runner.ts b/packages/nx/src/tasks-runner/forked-process-task-runner.ts index 0d9e77633132b..702d22beec8bb 100644 --- a/packages/nx/src/tasks-runner/forked-process-task-runner.ts +++ b/packages/nx/src/tasks-runner/forked-process-task-runner.ts @@ -19,7 +19,7 @@ import { BatchResults, } from './batch/batch-messages'; import { stripIndents } from '../utils/strip-indents'; -import { Task } from '../config/task-graph'; +import { Task, TaskGraph } from '../config/task-graph'; import { Transform } from 'stream'; const workerPath = join(__dirname, './batch/run-batch.js'); @@ -36,10 +36,13 @@ export class ForkedProcessTaskRunner { } // TODO: vsavkin delegate terminal output printing - public forkProcessForBatch({ executorName, taskGraph }: Batch) { + public forkProcessForBatch( + { executorName, taskGraph: batchTaskGraph }: Batch, + fullTaskGraph: TaskGraph + ) { return new Promise((res, rej) => { try { - const count = Object.keys(taskGraph.tasks).length; + const count = Object.keys(batchTaskGraph.tasks).length; if (count > 1) { output.logSingleLine( `Running ${output.bold(count)} ${output.bold( @@ -48,7 +51,7 @@ export class ForkedProcessTaskRunner { ); } else { const args = getPrintableCommandArgsForTask( - Object.values(taskGraph.tasks)[0] + Object.values(batchTaskGraph.tasks)[0] ); output.logCommand(args.join(' ')); output.addNewline(); @@ -65,7 +68,7 @@ export class ForkedProcessTaskRunner { if (code === null) code = this.signalToCode(signal); if (code !== 0) { const results: BatchResults = {}; - for (const rootTaskId of taskGraph.roots) { + for (const rootTaskId of batchTaskGraph.roots) { results[rootTaskId] = { success: false, }; @@ -99,8 +102,9 @@ export class ForkedProcessTaskRunner { // Start the tasks p.send({ type: BatchMessageType.Tasks, - taskGraph, executorName, + batchTaskGraph, + fullTaskGraph, }); } catch (e) { rej(e); @@ -113,21 +117,22 @@ export class ForkedProcessTaskRunner { { streamOutput, temporaryOutputPath, + taskGraph, }: { streamOutput: boolean; temporaryOutputPath: string; + taskGraph: TaskGraph; } ) { return new Promise<{ code: number; terminalOutput: string }>((res, rej) => { try { const args = getPrintableCommandArgsForTask(task); - const serializedArgs = getSerializedArgsForTask(task, this.verbose); if (streamOutput) { output.logCommand(args.join(' ')); output.addNewline(); } - const p = fork(this.cliPath, serializedArgs, { + const p = fork(this.cliPath, { stdio: ['inherit', 'pipe', 'pipe', 'ipc'], env: this.getEnvVariablesForTask( task, @@ -147,6 +152,14 @@ export class ForkedProcessTaskRunner { } }); + // Send message to run the executor + p.send({ + targetDescription: task.target, + overrides: task.overrides, + taskGraph, + isVerbose: this.verbose, + }); + if (streamOutput) { if (process.env.NX_PREFIX_OUTPUT === 'true') { const color = getColor(task.target.project); @@ -205,20 +218,21 @@ export class ForkedProcessTaskRunner { { streamOutput, temporaryOutputPath, + taskGraph, }: { streamOutput: boolean; temporaryOutputPath: string; + taskGraph: TaskGraph; } ) { return new Promise<{ code: number; terminalOutput: string }>((res, rej) => { try { const args = getPrintableCommandArgsForTask(task); - const serializedArgs = getSerializedArgsForTask(task, this.verbose); if (streamOutput) { output.logCommand(args.join(' ')); output.addNewline(); } - const p = fork(this.cliPath, serializedArgs, { + const p = fork(this.cliPath, { stdio: ['inherit', 'inherit', 'inherit', 'ipc'], env: this.getEnvVariablesForTask( task, @@ -236,6 +250,14 @@ export class ForkedProcessTaskRunner { } }); + // Send message to run the executor + p.send({ + targetDescription: task.target, + overrides: task.overrides, + taskGraph, + isVerbose: this.verbose, + }); + p.on('exit', (code, signal) => { if (code === null) code = this.signalToCode(signal); // we didn't print any output as we were running the command diff --git a/packages/nx/src/tasks-runner/task-orchestrator.ts b/packages/nx/src/tasks-runner/task-orchestrator.ts index 3f707feff178a..8f54c9a8c78eb 100644 --- a/packages/nx/src/tasks-runner/task-orchestrator.ts +++ b/packages/nx/src/tasks-runner/task-orchestrator.ts @@ -229,7 +229,8 @@ export class TaskOrchestrator { private async runBatch(batch: Batch) { try { const results = await this.forkedProcessTaskRunner.forkProcessForBatch( - batch + batch, + this.taskGraph ); const batchResultEntries = Object.entries(results); return batchResultEntries.map(([taskId, result]) => ({ @@ -296,6 +297,7 @@ export class TaskOrchestrator { { temporaryOutputPath, streamOutput, + taskGraph: this.taskGraph, } ) : await this.forkedProcessTaskRunner.forkProcessDirectOutputCapture( @@ -303,6 +305,7 @@ export class TaskOrchestrator { { temporaryOutputPath, streamOutput, + taskGraph: this.taskGraph, } );