Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(core): ability to save task graph to a file when running --graph #16350

Merged
merged 3 commits into from
May 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/generated/cli/affected.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,11 @@ Change the way Nx is calculating the affected command by providing directly chan

### graph

Type: `boolean`
Type: `string`

Default: `false`

Show the task graph of the command
Show the task graph of the command. Pass a file path to save the graph data instead of viewing it in the browser.

### head

Expand Down
4 changes: 2 additions & 2 deletions docs/generated/cli/exec.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ Exclude certain projects from being processed

### graph
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

followup: Why is this here?


Type: `boolean`
Type: `string`

Default: `false`

Show the task graph of the command
Show the task graph of the command. Pass a file path to save the graph data instead of viewing it in the browser.

### nx-bail

Expand Down
4 changes: 2 additions & 2 deletions docs/generated/cli/run-many.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,11 @@ Exclude certain projects from being processed

### graph

Type: `boolean`
Type: `string`

Default: `false`

Show the task graph of the command
Show the task graph of the command. Pass a file path to save the graph data instead of viewing it in the browser.

### help

Expand Down
4 changes: 2 additions & 2 deletions docs/generated/packages/nx/documents/affected.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,11 @@ Change the way Nx is calculating the affected command by providing directly chan

### graph

Type: `boolean`
Type: `string`

Default: `false`

Show the task graph of the command
Show the task graph of the command. Pass a file path to save the graph data instead of viewing it in the browser.

### head

Expand Down
4 changes: 2 additions & 2 deletions docs/generated/packages/nx/documents/exec.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ Exclude certain projects from being processed

### graph

Type: `boolean`
Type: `string`

Default: `false`

Show the task graph of the command
Show the task graph of the command. Pass a file path to save the graph data instead of viewing it in the browser.

### nx-bail

Expand Down
4 changes: 2 additions & 2 deletions docs/generated/packages/nx/documents/run-many.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,11 @@ Exclude certain projects from being processed

### graph

Type: `boolean`
Type: `string`

Default: `false`

Show the task graph of the command
Show the task graph of the command. Pass a file path to save the graph data instead of viewing it in the browser.

### help

Expand Down
8 changes: 8 additions & 0 deletions e2e/nx-run/src/affected-graph.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,14 @@ describe('Nx Affected and Graph Tests', () => {
expect(environmentJs).toContain('"affected":[]');
});

it('graph should output valid json when stdout is specified', () => {
const result = runCLI(`affected -t build --graph stdout`);
let model;
expect(() => (model = JSON.parse(result))).not.toThrow();
expect(model).toHaveProperty('graph');
expect(model).toHaveProperty('tasks');
});

it('affected:graph should include affected projects in environment file', () => {
runCLI(`affected:graph --file=project-graph.html`);

Expand Down
10 changes: 8 additions & 2 deletions packages/nx/src/command-line/affected/affected.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ import { output } from '../../utils/output';
import { generateGraph } from '../graph/graph';
import { printAffected } from './print-affected';
import { connectToNxCloudIfExplicitlyAsked } from '../connect/connect-to-nx-cloud';
import type { NxArgs } from '../../utils/command-line-utils';
import {
NxArgs,
readGraphFileFromGraphArg,
} from '../../utils/command-line-utils';
import {
parseFiles,
splitArgsIntoNxArgsAndOverrides,
Expand Down Expand Up @@ -38,7 +41,8 @@ export async function affected(
args,
'affected',
{
printWarnings: command !== 'print-affected' && !args.plain,
printWarnings:
command !== 'print-affected' && !args.plain && args.graph !== 'stdout',
},
nxJson
);
Expand Down Expand Up @@ -83,6 +87,7 @@ export async function affected(
const projectsWithTarget = allProjectsWithTarget(projects, nxArgs);
if (nxArgs.graph) {
const projectNames = projectsWithTarget.map((t) => t.name);
const file = readGraphFileFromGraphArg(nxArgs);

return await generateGraph(
{
Expand All @@ -91,6 +96,7 @@ export async function affected(
view: 'tasks',
targets: nxArgs.targets,
projects: projectNames,
file,
},
projectNames
);
Expand Down
81 changes: 76 additions & 5 deletions packages/nx/src/command-line/graph/graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,12 @@ export async function generateGraph(
},
affectedProjects: string[]
): Promise<void> {
if (Array.isArray(args.targets) && args.targets.length > 1) {
if (
Array.isArray(args.targets) &&
args.targets.length > 1 &&
args.file &&
!(args.file === 'stdout' || args.file.endsWith('.json'))
) {
output.warn({
title: 'Showing Multiple Targets is not supported yet',
bodyLines: [
Expand Down Expand Up @@ -246,6 +251,18 @@ export async function generateGraph(
graph = filterGraph(graph, args.focus || null, args.exclude || []);

if (args.file) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to alter this output to include tasks and omit affectedProjects and criticalPath.

Option 1:
tasks is the same as what print-affected has currently which is tasks: Task[]

Option 2:
tasks is the task graph so we have taskGraph: { roots: string[], tasks: Record<string, Task>, dependencies: Record<string, string[]> }

Copy link
Member Author

@AgentEnder AgentEnder May 12, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • Updated with option 2 🎉

There is a caveat. print-affected included task hashes, but they were sometimes not matching with the task hash as compared against the cache... See: #16153

Should we try to get accurate hashes this time, or just be ok with no hash?

// stdout is a magical constant that doesn't actually write a file
if (args.file === 'stdout') {
console.log(
JSON.stringify(
createJsonOutput(graph, args.projects, args.targets),
null,
2
)
);
process.exit(0);
}

const workspaceFolder = workspaceRoot;
const ext = extname(args.file);
const fullFilePath = isAbsolute(args.file)
Expand Down Expand Up @@ -294,10 +311,20 @@ export async function generateGraph(
} else if (ext === '.json') {
ensureDirSync(dirname(fullFilePath));

writeJsonFile(fullFilePath, {
graph,
affectedProjects,
criticalPath: affectedProjects,
const json = createJsonOutput(graph, args.projects, args.targets);
json.affectedProjects = affectedProjects;
json.criticalPath = affectedProjects;

writeJsonFile(fullFilePath, json);

output.warn({
title: 'JSON output contains deprecated fields:',
bodyLines: [
'- affectedProjects',
'- criticalPath',
'',
'These fields will be removed in Nx 18. If you need to see which projects were affected, use `nx show projects --affected`.',
],
});

output.success({
Expand Down Expand Up @@ -685,3 +712,47 @@ function createTaskId(
return `${projectId}:${targetId}`;
}
}

interface GraphJsonResponse {
tasks?: TaskGraph;
graph: ProjectGraph;

/**
* @deprecated To see affected projects, use `nx show projects --affected`. This will be removed in Nx 18.
*/
affectedProjects?: string[];

/**
* @deprecated To see affected projects, use `nx show projects --affected`. This will be removed in Nx 18.
*/
criticalPath?: string[];
}

function createJsonOutput(
graph: ProjectGraph,
projects: string[],
targets?: string[]
): GraphJsonResponse {
const response: GraphJsonResponse = {
graph,
};

if (targets?.length) {
const nxJson = readNxJson();

const defaultDependencyConfigs = mapTargetDefaultsToDependencies(
nxJson.targetDefaults
);

response.tasks = createTaskGraph(
graph,
defaultDependencyConfigs,
projects,
targets,
undefined,
{}
);
}

return response;
}
9 changes: 7 additions & 2 deletions packages/nx/src/command-line/run-many/run-many.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import { runCommand } from '../../tasks-runner/run-command';
import type { NxArgs } from '../../utils/command-line-utils';
import {
NxArgs,
readGraphFileFromGraphArg,
} from '../../utils/command-line-utils';
import { splitArgsIntoNxArgsAndOverrides } from '../../utils/command-line-utils';
import { projectHasTarget } from '../../utils/project-graph-utils';
import { connectToNxCloudIfExplicitlyAsked } from '../connect/connect-to-nx-cloud';
Expand Down Expand Up @@ -33,7 +36,7 @@ export async function runMany(
const { nxArgs, overrides } = splitArgsIntoNxArgsAndOverrides(
args,
'run-many',
{ printWarnings: true },
{ printWarnings: args.graph !== 'stdout' },
nxJson
);
if (nxArgs.verbose) {
Expand All @@ -46,6 +49,7 @@ export async function runMany(
const projects = projectsToRun(nxArgs, projectGraph);

if (nxArgs.graph) {
const file = readGraphFileFromGraphArg(nxArgs);
const projectNames = projects.map((t) => t.name);
return await generateGraph(
{
Expand All @@ -55,6 +59,7 @@ export async function runMany(
all: nxArgs.all,
targets: nxArgs.targets,
projects: projectNames,
file,
},
projectNames
);
Expand Down
9 changes: 7 additions & 2 deletions packages/nx/src/command-line/run/run-one.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import { runCommand } from '../../tasks-runner/run-command';
import { splitArgsIntoNxArgsAndOverrides } from '../../utils/command-line-utils';
import {
readGraphFileFromGraphArg,
splitArgsIntoNxArgsAndOverrides,
} from '../../utils/command-line-utils';
import { connectToNxCloudIfExplicitlyAsked } from '../connect/connect-to-nx-cloud';
import { performance } from 'perf_hooks';
import {
Expand Down Expand Up @@ -47,7 +50,7 @@ export async function runOne(
targets: [opts.target],
},
'run-one',
{ printWarnings: true },
{ printWarnings: args.graph !== 'stdout' },
nxJson
);
if (nxArgs.verbose) {
Expand All @@ -66,6 +69,7 @@ export async function runOne(

if (nxArgs.graph) {
const projectNames = projects.map((t) => t.name);
const file = readGraphFileFromGraphArg(nxArgs);

return await generateGraph(
{
Expand All @@ -74,6 +78,7 @@ export async function runOne(
view: 'tasks',
targets: nxArgs.targets,
projects: projectNames,
file,
},
projectNames
);
Expand Down
11 changes: 9 additions & 2 deletions packages/nx/src/command-line/yargs-utils/shared-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,16 @@ export function withRunOptions(yargs: Argv): Argv {
hidden: true,
})
.option('graph', {
type: 'boolean',
describe: 'Show the task graph of the command',
type: 'string',
describe:
'Show the task graph of the command. Pass a file path to save the graph data instead of viewing it in the browser.',
default: false,
coerce: (value) =>
value === 'true' || value === true
? true
: value === 'false' || value === false
? false
: value,
})
.option('verbose', {
type: 'boolean',
Expand Down
10 changes: 7 additions & 3 deletions packages/nx/src/utils/command-line-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export interface NxArgs {
plain?: boolean;
projects?: string[];
select?: string;
graph?: boolean;
graph?: string | boolean;
skipNxCache?: boolean;
outputStyle?: string;
nxBail?: boolean;
Expand Down Expand Up @@ -288,8 +288,6 @@ function getUncommittedFiles(): string[] {
return parseGitOutput(`git diff --name-only --no-renames --relative HEAD .`);
}

``;

function getUntrackedFiles(): string[] {
return parseGitOutput(`git ls-files --others --exclude-standard`);
}
Expand Down Expand Up @@ -338,3 +336,9 @@ export function getProjectRoots(
): string[] {
return projectNames.map((name) => nodes[name].data.root);
}

export function readGraphFileFromGraphArg({ graph }: NxArgs) {
return typeof graph === 'string' && graph !== 'true' && graph !== ''
? graph
: undefined;
}