From aad5e49e103a9435298dd6ccf41a5344a6d1039d Mon Sep 17 00:00:00 2001 From: Emily Xiong Date: Wed, 20 Sep 2023 20:02:24 -0400 Subject: [PATCH] feat(core): change exec to run adhoc tasks --- docs/generated/cli/exec.md | 26 +-- docs/generated/packages/nx/documents/exec.md | 26 +-- e2e/nx-run/src/run.test.ts | 58 +++++- .../src/command-line/exec/command-object.ts | 5 +- packages/nx/src/command-line/exec/exec.ts | 178 +++++++++++++----- .../nx/src/commands-runner/command-graph.ts | 13 ++ .../commands-runner/create-command-graph.ts | 52 +++++ .../commands-runner/get-command-projects.ts | 36 ++++ .../nx/src/tasks-runner/task-graph-utils.ts | 45 ++--- packages/nx/src/tasks-runner/utils.ts | 33 +++- 10 files changed, 362 insertions(+), 110 deletions(-) create mode 100644 packages/nx/src/commands-runner/command-graph.ts create mode 100644 packages/nx/src/commands-runner/create-command-graph.ts create mode 100644 packages/nx/src/commands-runner/get-command-projects.ts diff --git a/docs/generated/cli/exec.md b/docs/generated/cli/exec.md index a0fa423a92df2a..2b137591029304 100644 --- a/docs/generated/cli/exec.md +++ b/docs/generated/cli/exec.md @@ -17,11 +17,13 @@ Install `nx` globally to invoke the command directly using `nx`, or use `npx nx` ## Options -### configuration +### all -Type: `string` +Type: `boolean` + +Default: `true` -This is the configuration to use when performing tasks on projects +[deprecated] `run-many` runs all targets on all projects in the workspace if no projects are provided. This option is no longer required. ### exclude @@ -35,6 +37,12 @@ Type: `string` Show the task graph of the command. Pass a file path to save the graph data instead of viewing it in the browser. +### help + +Type: `boolean` + +Show help + ### nxBail Type: `boolean` @@ -51,25 +59,17 @@ Default: `false` Ignore cycles in the task graph -### output-style - -Type: `string` - -Choices: [dynamic, static, stream, stream-without-prefixes, compact] - -Defines how Nx emits outputs tasks logs - ### parallel Type: `string` Max number of parallel processes [default is 3] -### project +### projects Type: `string` -Target project +Projects to run. (comma/space delimited project names and/or patterns) ### runner diff --git a/docs/generated/packages/nx/documents/exec.md b/docs/generated/packages/nx/documents/exec.md index a0fa423a92df2a..2b137591029304 100644 --- a/docs/generated/packages/nx/documents/exec.md +++ b/docs/generated/packages/nx/documents/exec.md @@ -17,11 +17,13 @@ Install `nx` globally to invoke the command directly using `nx`, or use `npx nx` ## Options -### configuration +### all -Type: `string` +Type: `boolean` + +Default: `true` -This is the configuration to use when performing tasks on projects +[deprecated] `run-many` runs all targets on all projects in the workspace if no projects are provided. This option is no longer required. ### exclude @@ -35,6 +37,12 @@ Type: `string` Show the task graph of the command. Pass a file path to save the graph data instead of viewing it in the browser. +### help + +Type: `boolean` + +Show help + ### nxBail Type: `boolean` @@ -51,25 +59,17 @@ Default: `false` Ignore cycles in the task graph -### output-style - -Type: `string` - -Choices: [dynamic, static, stream, stream-without-prefixes, compact] - -Defines how Nx emits outputs tasks logs - ### parallel Type: `string` Max number of parallel processes [default is 3] -### project +### projects Type: `string` -Target project +Projects to run. (comma/space delimited project names and/or patterns) ### runner diff --git a/e2e/nx-run/src/run.test.ts b/e2e/nx-run/src/run.test.ts index 051f56ddc6d14c..146aedde14db9f 100644 --- a/e2e/nx-run/src/run.test.ts +++ b/e2e/nx-run/src/run.test.ts @@ -638,13 +638,21 @@ describe('Nx Running Tests', () => { describe('exec', () => { let pkg: string; + let pkg2: string; let pkgRoot: string; + let pkg2Root: string; let originalRootPackageJson: PackageJson; beforeAll(() => { originalRootPackageJson = readJson('package.json'); pkg = uniq('package'); + pkg2 = uniq('package'); pkgRoot = tmpProjPath(path.join('libs', pkg)); + pkg2Root = tmpProjPath(path.join('libs', pkg2)); + runCLI(`generate @nx/js:lib ${pkg} --bundler=none --unitTestRunner=none`); + runCLI( + `generate @nx/js:lib ${pkg2} --bundler=none --unitTestRunner=none` + ); updateJson('package.json', (v) => { v.workspaces = ['libs/*']; @@ -662,6 +670,22 @@ describe('Nx Running Tests', () => { }, }) ); + + updateFile( + `libs/${pkg2}/package.json`, + JSON.stringify({ + name: pkg2, + version: '0.0.1', + scripts: { + build: "nx exec -- echo '$NX_PROJECT_NAME'", + }, + }) + ); + + updateJson(`libs/${pkg2}/project.json`, (content) => { + content['implicitDependencies'] = [pkg]; + return content; + }); }); afterAll(() => { @@ -676,6 +700,36 @@ describe('Nx Running Tests', () => { expect(output).toContain(`nx run ${pkg}:build`); }); + it('should run adhoc tasks in topological order', () => { + let output = runCLI('exec -- echo HELLO'); + expect(output).toContain('HELLO'); + + output = runCLI(`build ${pkg}`); + expect(output).toContain(pkg); + expect(output).not.toContain(pkg2); + + output = runCommand('npm run build', { + cwd: pkgRoot, + }); + expect(output).toContain(pkg); + expect(output).not.toContain(pkg2); + + output = runCLI(`exec -- echo '$NX_PROJECT_NAME'`); + expect(output.replace(/\s+/g, ' ')).toContain(`${pkg} ${pkg2}`); + + output = runCLI("exec -- echo '$NX_PROJECT_ROOT_PATH'"); + expect(output.replace(/\s+/g, ' ')).toContain( + `${path.join('libs', pkg)} ${path.join('libs', pkg2)}` + ); + + output = runCLI(`exec --projects ${pkg} -- echo WORLD`); + expect(output).toContain('WORLD'); + + output = runCLI(`exec --projects ${pkg} -- echo '$NX_PROJECT_NAME'`); + expect(output).toContain(pkg); + expect(output).not.toContain(pkg2); + }); + it('should work for npm scripts with delimiter', () => { const output = runCommand('npm run build:option', { cwd: pkgRoot }); expect(output).toContain('HELLO WITH OPTION'); @@ -701,7 +755,6 @@ describe('Nx Running Tests', () => { }); it('should read outputs', () => { - console.log(pkgRoot); const nodeCommands = [ "const fs = require('fs')", "fs.mkdirSync('../../tmp/exec-outputs-test', {recursive: true})", @@ -724,10 +777,9 @@ describe('Nx Running Tests', () => { }, }) ); - const out = runCommand('npm run build', { + runCommand('npm run build', { cwd: pkgRoot, }); - console.log(out); expect( fileExists(tmpProjPath('tmp/exec-outputs-test/file.txt')) ).toBeTruthy(); diff --git a/packages/nx/src/command-line/exec/command-object.ts b/packages/nx/src/command-line/exec/command-object.ts index 522ee48b7ce737..65a656feedfa88 100644 --- a/packages/nx/src/command-line/exec/command-object.ts +++ b/packages/nx/src/command-line/exec/command-object.ts @@ -1,18 +1,19 @@ import { CommandModule } from 'yargs'; import { - withRunOneOptions, withOverrides, + withRunManyOptions, } from '../yargs-utils/shared-options'; export const yargsExecCommand: CommandModule = { command: 'exec', describe: 'Executes any command as if it was a target on the project', - builder: (yargs) => withRunOneOptions(yargs), + builder: (yargs) => withRunManyOptions(yargs), handler: async (args) => { try { await (await import('./exec')).nxExecCommand(withOverrides(args) as any); process.exit(0); } catch (e) { + console.error(e); process.exit(1); } }, diff --git a/packages/nx/src/command-line/exec/exec.ts b/packages/nx/src/command-line/exec/exec.ts index fef4670c64fa76..00697ec3311a75 100644 --- a/packages/nx/src/command-line/exec/exec.ts +++ b/packages/nx/src/command-line/exec/exec.ts @@ -2,7 +2,10 @@ import { execSync } from 'child_process'; import { join } from 'path'; import { exit } from 'process'; import * as yargs from 'yargs-parser'; +import { Arguments } from 'yargs'; +import { existsSync } from 'fs'; +import { findMatchingProjects } from '../../utils/find-matching-projects'; import { readNxJson } from '../../config/configuration'; import { ProjectGraph, @@ -12,18 +15,33 @@ import { createProjectGraphAsync, readProjectsConfigurationFromProjectGraph, } from '../../project-graph/project-graph'; -import { splitArgsIntoNxArgsAndOverrides } from '../../utils/command-line-utils'; +import { + NxArgs, + splitArgsIntoNxArgsAndOverrides, +} from '../../utils/command-line-utils'; import { readJsonFile } from '../../utils/fileutils'; import { output } from '../../utils/output'; import { PackageJson } from '../../utils/package-json'; import { getPackageManagerCommand } from '../../utils/package-manager'; import { workspaceRoot } from '../../utils/workspace-root'; import { calculateDefaultProjectName } from '../../config/calculate-default-project-name'; +import { getCommandProjects } from '../../commands-runner/get-command-projects'; export async function nxExecCommand( - args: Record + args: Record ): Promise { - const scriptArgV: string[] = readScriptArgV(args); + const nxJson = readNxJson(); + const { nxArgs, overrides } = splitArgsIntoNxArgsAndOverrides( + args, + 'run-many', + { printWarnings: args.graph !== 'stdout' }, + nxJson + ); + if (nxArgs.verbose) { + process.env.NX_VERBOSE_LOGGING = 'true'; + } + const scriptArgV: string[] = readScriptArgV(overrides); + const projectGraph = await createProjectGraphAsync({ exitOnError: true }); // NX is already running if (process.env.NX_TASK_TARGET_PROJECT) { @@ -32,23 +50,68 @@ export async function nxExecCommand( .trim(); execSync(command, { stdio: 'inherit', - env: process.env, + env: { + ...process.env, + NX_PROJECT_NAME: process.env.NX_TASK_TARGET_PROJECT, + NX_PROJECT_ROOT_PATH: + projectGraph.nodes?.[process.env.NX_TASK_TARGET_PROJECT]?.data?.root, + }, }); } else { // nx exec is being ran inside of Nx's context - return runScriptAsNxTarget(scriptArgV); + return runScriptAsNxTarget(projectGraph, scriptArgV, nxArgs); } } -async function runScriptAsNxTarget(argv: string[]) { - const projectGraph = await createProjectGraphAsync(); - - const { projectName, project } = getProject(projectGraph); - +async function runScriptAsNxTarget( + projectGraph: ProjectGraph, + argv: string[], + nxArgs: NxArgs +) { // NPM, Yarn, and PNPM set this to the name of the currently executing script. Lets use it if we can. const targetName = process.env.npm_lifecycle_event; - const scriptDefinition = getScriptDefinition(project, targetName); + if (targetName) { + const defaultPorject = getDefaultProject(projectGraph); + const scriptDefinition = getScriptDefinition(targetName, defaultPorject); + if (scriptDefinition) { + runTargetOnProject( + scriptDefinition, + targetName, + defaultPorject, + defaultPorject.name, + argv + ); + return; + } + } + + const projects = getProjects(projectGraph, nxArgs); + const projectsToRun: string[] = getCommandProjects( + projectGraph, + projects, + nxArgs + ); + projectsToRun.forEach((projectName) => { + const command = argv.reduce((cmd, arg) => cmd + `"${arg}" `, '').trim(); + execSync(command, { + stdio: 'inherit', + env: { + ...process.env, + NX_PROJECT_NAME: projectGraph.nodes?.[projectName]?.name, + NX_PROJECT_ROOT_PATH: projectGraph.nodes?.[projectName]?.data?.root, + }, + cwd: projectGraph.nodes?.[projectName]?.data?.root ?? workspaceRoot, + }); + }); +} +function runTargetOnProject( + scriptDefinition: string, + targetName: string, + project: ProjectGraphProjectNode, + projectName: string, + argv: string[] +) { ensureNxTarget(project, targetName); // Get ArgV that is provided in npm script definition @@ -58,20 +121,15 @@ async function runScriptAsNxTarget(argv: string[]) { const pm = getPackageManagerCommand(); // `targetName` might be an npm script with `:` like: `start:dev`, `start:debug`. - let command = `${ + const command = `${ pm.exec } nx run ${projectName}:\\\"${targetName}\\\" ${extraArgs.join(' ')}`; - return execSync(command, { stdio: 'inherit' }); + execSync(command, { stdio: 'inherit' }); } -function readScriptArgV(args: Record) { - const { overrides } = splitArgsIntoNxArgsAndOverrides( - args, - 'run-one', - { printWarnings: false }, - readNxJson() - ); - +function readScriptArgV( + overrides: Arguments & { __overrides_unparsed__: string[] } +) { const scriptSeparatorIdx = process.argv.findIndex((el) => el === '--'); if (scriptSeparatorIdx === -1) { output.error({ @@ -84,25 +142,22 @@ function readScriptArgV(args: Record) { } function getScriptDefinition( - project: ProjectGraphProjectNode, - targetName: string + targetName: string, + project?: ProjectGraphProjectNode ): PackageJson['scripts'][string] { - const scriptDefinition = readJsonFile( - join(workspaceRoot, project.data.root, 'package.json') - ).scripts[targetName]; - - if (!scriptDefinition) { - output.error({ - title: - "`nx exec` is meant to be used in a project's package.json scripts", - bodyLines: [ - `Nx was unable to find a npm script matching ${targetName} for ${project.name}`, - ], - }); - process.exit(1); + if (!project) { + return; + } + const packageJsonPath = join( + workspaceRoot, + project.data.root, + 'package.json' + ); + if (existsSync(packageJsonPath)) { + const scriptDefinition = + readJsonFile(packageJsonPath).scripts?.[targetName]; + return scriptDefinition; } - - return scriptDefinition; } function ensureNxTarget(project: ProjectGraphProjectNode, targetName: string) { @@ -117,26 +172,47 @@ function ensureNxTarget(project: ProjectGraphProjectNode, targetName: string) { } } -function getProject(projectGraph: ProjectGraph) { - const projectName = calculateDefaultProjectName( +function getDefaultProject( + projectGraph: ProjectGraph +): ProjectGraphProjectNode | undefined { + const defaultProjectName = calculateDefaultProjectName( process.cwd(), workspaceRoot, readProjectsConfigurationFromProjectGraph(projectGraph), readNxJson() ); + if (defaultProjectName && projectGraph.nodes[defaultProjectName]) { + return projectGraph.nodes[defaultProjectName]; + } +} - if (!projectName) { - output.error({ - title: 'Unable to determine project name for `nx exec`', - bodyLines: [ - "`nx exec` should be ran from within an Nx project's root directory.", - 'Does this package.json belong to an Nx project?', - ], - }); - process.exit(1); +function getProjects( + projectGraph: ProjectGraph, + nxArgs: NxArgs +): ProjectGraphProjectNode[] { + let selectedProjects = {}; + + // get projects matched + if (nxArgs.projects?.length) { + const matchingProjects = findMatchingProjects( + nxArgs.projects, + projectGraph.nodes + ); + for (const project of matchingProjects) { + selectedProjects[project] = projectGraph.nodes[project]; + } + } else { + // if no project specified, return all projects + selectedProjects = { ...projectGraph.nodes }; } - const project: ProjectGraphProjectNode = projectGraph.nodes[projectName]; + const excludedProjects = findMatchingProjects( + nxArgs.exclude, + selectedProjects + ); + for (const excludedProject of excludedProjects) { + delete selectedProjects[excludedProject]; + } - return { projectName, project }; + return Object.values(selectedProjects); } diff --git a/packages/nx/src/commands-runner/command-graph.ts b/packages/nx/src/commands-runner/command-graph.ts new file mode 100644 index 00000000000000..89001bd5d4d8f2 --- /dev/null +++ b/packages/nx/src/commands-runner/command-graph.ts @@ -0,0 +1,13 @@ +/** + * Graph of Tasks to be executed + */ +export interface CommandGraph { + /** + * Projects that do not have any dependencies and are thus ready to execute immediately + */ + roots: string[]; + /** + * Map of projects to projects which the task depends on + */ + dependencies: Record; +} diff --git a/packages/nx/src/commands-runner/create-command-graph.ts b/packages/nx/src/commands-runner/create-command-graph.ts new file mode 100644 index 00000000000000..59459321de68f0 --- /dev/null +++ b/packages/nx/src/commands-runner/create-command-graph.ts @@ -0,0 +1,52 @@ +import { ProjectGraph } from '../config/project-graph'; +import { findCycle, makeAcyclic } from '../tasks-runner/task-graph-utils'; +import { NxArgs } from '../utils/command-line-utils'; +import { output } from '../utils/output'; +import { CommandGraph } from './command-graph'; + +export function createCommandGraph( + projectGraph: ProjectGraph, + projectNames: string[], + nxArgs: NxArgs +): CommandGraph { + const dependencies: Record = {}; + for (const projectName of projectNames) { + if (projectGraph.dependencies[projectName].length >= 1) { + dependencies[projectName] = [ + ...new Set( + projectGraph.dependencies[projectName] + .map((projectDep) => projectDep.target) + .filter((projectDep) => projectGraph.nodes[projectDep]) + ).values(), + ]; + } else { + dependencies[projectName] = []; + } + } + const roots = Object.keys(dependencies).filter( + (d) => dependencies[d].length === 0 + ); + const commandGraph = { + dependencies, + roots, + }; + + const cycle = findCycle(commandGraph); + if (cycle) { + if (process.env.NX_IGNORE_CYCLES === 'true' || nxArgs.nxIgnoreCycles) { + output.warn({ + title: `The command graph has a circular dependency`, + bodyLines: [`${cycle.join(' --> ')}`], + }); + makeAcyclic(commandGraph); + } else { + output.error({ + title: `Could not execute command because the project graph has a circular dependency`, + bodyLines: [`${cycle.join(' --> ')}`], + }); + process.exit(1); + } + } + + return commandGraph; +} diff --git a/packages/nx/src/commands-runner/get-command-projects.ts b/packages/nx/src/commands-runner/get-command-projects.ts new file mode 100644 index 00000000000000..3d9c669e85fb5b --- /dev/null +++ b/packages/nx/src/commands-runner/get-command-projects.ts @@ -0,0 +1,36 @@ +import { ProjectGraph, ProjectGraphProjectNode } from '../config/project-graph'; +import { removeIdsFromGraph } from '../tasks-runner/utils'; +import { NxArgs } from '../utils/command-line-utils'; +import { CommandGraph } from './command-graph'; +import { createCommandGraph } from './create-command-graph'; + +export function getCommandProjects( + projectGraph: ProjectGraph, + projects: ProjectGraphProjectNode[], + nxArgs: NxArgs +) { + const commandGraph = createCommandGraph( + projectGraph, + projects.map((project) => project.name), + nxArgs + ); + return getSortedProjects(commandGraph); +} + +function getSortedProjects( + commandGraph: CommandGraph, + sortedProjects: string[] = [] +): string[] { + const roots = commandGraph.roots; + if (!roots.length) { + return sortedProjects; + } + sortedProjects.push(...roots); + const newGraph: CommandGraph = removeIdsFromGraph( + commandGraph, + roots, + commandGraph.dependencies + ); + + return getSortedProjects(newGraph, sortedProjects); +} diff --git a/packages/nx/src/tasks-runner/task-graph-utils.ts b/packages/nx/src/tasks-runner/task-graph-utils.ts index be1204a9f79ae6..770f98d79dee96 100644 --- a/packages/nx/src/tasks-runner/task-graph-utils.ts +++ b/packages/nx/src/tasks-runner/task-graph-utils.ts @@ -1,23 +1,23 @@ -import { TaskGraph } from '../config/task-graph'; - function _findCycle( - taskGraph: TaskGraph, - taskId: string, + graph: { dependencies: Record }, + id: string, visited: { [taskId: string]: boolean }, path: string[] ) { - if (visited[taskId]) return null; - visited[taskId] = true; + if (visited[id]) return null; + visited[id] = true; - for (const d of taskGraph.dependencies[taskId]) { + for (const d of graph.dependencies[id]) { if (path.includes(d)) return [...path, d]; - const cycle = _findCycle(taskGraph, d, visited, [...path, d]); + const cycle = _findCycle(graph, d, visited, [...path, d]); if (cycle) return cycle; } return null; } -export function findCycle(taskGraph: TaskGraph): string[] | null { +export function findCycle(taskGraph: { + dependencies: Record; +}): string[] | null { const visited = {}; for (const t of Object.keys(taskGraph.dependencies)) { visited[t] = false; @@ -32,34 +32,37 @@ export function findCycle(taskGraph: TaskGraph): string[] | null { } function _makeAcyclic( - taskGraph: TaskGraph, - taskId: string, + graph: { dependencies: Record }, + id: string, visited: { [taskId: string]: boolean }, path: string[] ) { - if (visited[taskId]) return; - visited[taskId] = true; + if (visited[id]) return; + visited[id] = true; - const deps = taskGraph.dependencies[taskId]; + const deps = graph.dependencies[id]; for (const d of [...deps]) { if (path.includes(d)) { deps.splice(deps.indexOf(d), 1); } else { - _makeAcyclic(taskGraph, d, visited, [...path, d]); + _makeAcyclic(graph, d, visited, [...path, d]); } } return null; } -export function makeAcyclic(taskGraph: TaskGraph): void { +export function makeAcyclic(graph: { + roots: string[]; + dependencies: Record; +}): void { const visited = {}; - for (const t of Object.keys(taskGraph.dependencies)) { + for (const t of Object.keys(graph.dependencies)) { visited[t] = false; } - for (const t of Object.keys(taskGraph.dependencies)) { - _makeAcyclic(taskGraph, t, visited, [t]); + for (const t of Object.keys(graph.dependencies)) { + _makeAcyclic(graph, t, visited, [t]); } - taskGraph.roots = Object.keys(taskGraph.dependencies).filter( - (t) => taskGraph.dependencies[t].length === 0 + graph.roots = Object.keys(graph.dependencies).filter( + (t) => graph.dependencies[t].length === 0 ); } diff --git a/packages/nx/src/tasks-runner/utils.ts b/packages/nx/src/tasks-runner/utils.ts index 52164a39bb5483..381551147c45ce 100644 --- a/packages/nx/src/tasks-runner/utils.ts +++ b/packages/nx/src/tasks-runner/utils.ts @@ -277,19 +277,38 @@ export function removeTasksFromTaskGraph( graph: TaskGraph, ids: string[] ): TaskGraph { - const tasks = {}; + const newGraph = removeIdsFromGraph(graph, ids, graph.tasks); + return { + ...newGraph, + tasks: newGraph.mapWithIds, + }; +} + +export function removeIdsFromGraph( + graph: { + roots: string[]; + dependencies: Record; + }, + ids: string[], + mapWithIds: Record +): { + mapWithIds: Record; + roots: string[]; + dependencies: Record; +} { + const filteredMapWithIds = {}; const dependencies = {}; const removedSet = new Set(ids); - for (let taskId of Object.keys(graph.tasks)) { - if (!removedSet.has(taskId)) { - tasks[taskId] = graph.tasks[taskId]; - dependencies[taskId] = graph.dependencies[taskId].filter( - (depTaskId) => !removedSet.has(depTaskId) + for (let id of Object.keys(mapWithIds)) { + if (!removedSet.has(id)) { + filteredMapWithIds[id] = mapWithIds[id]; + dependencies[id] = graph.dependencies[id].filter( + (depId) => !removedSet.has(depId) ); } } return { - tasks, + mapWithIds, dependencies: dependencies, roots: Object.keys(dependencies).filter( (k) => dependencies[k].length === 0