Skip to content

Commit

Permalink
feat(core): add project specific inputs to hasher
Browse files Browse the repository at this point in the history
  • Loading branch information
AgentEnder committed Apr 26, 2023
1 parent 23bc60c commit 2e47682
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 32 deletions.
4 changes: 2 additions & 2 deletions docs/shared/reference/project-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ _Named Inputs_
Examples:

- `inputs: ["production"]`
- same as `inputs: [{input: "production", projects: "self"}]`
- same as `"inputs": [{"input": "production", "projects": "self"}]` in versions prior to Nx 16, or `"inputs": [{"input": "production"}]` after version 16.

Often the same glob will appear in many places (e.g., prod fileset will exclude spec files for all projects). Because
keeping them in sync is error-prone, we recommend defining `namedInputs`, which you can then reference in all of those
Expand All @@ -174,7 +174,7 @@ places.
Examples:

- `inputs: ["^production"]`
- same as `inputs: [{input: "production", projects: "dependencies"}]`
- same as `inputs: [{"input": "production", "projects": "dependencies"}]` prior to Nx 16, or `"inputs": [{"input": "production", "dependencies": true }]` after version 16.

Similar to `dependsOn`, the "^" symbols means "dependencies". This is a very important idea, so let's illustrate it with
an example.
Expand Down
1 change: 1 addition & 0 deletions nx-dev/nx-dev/project.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"dependsOn": [
{
"target": "build-base",
"projects": "{self}"
}
],
"executor": "nx:run-commands",
Expand Down
19 changes: 16 additions & 3 deletions packages/nx/src/hasher/hasher.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,22 @@ describe('Hasher', () => {
{ runtime: 'echo runtime123' },
{ env: 'TESTENV' },
{ env: 'NONEXISTENTENV' },
{ input: 'default', projects: ['unrelated'] },
],
},
},
files: [{ file: '/file', hash: 'file.hash' }],
},
},
unrelated: {
name: 'unrelated',
type: 'lib',
data: {
root: 'libs/unrelated',
targets: { build: {} },
files: [{ file: 'libs/unrelated/filec.ts', hash: 'filec.hash' }],
},
},
},
dependencies: {
parent: [],
Expand Down Expand Up @@ -121,12 +131,15 @@ describe('Hasher', () => {
expect(hash.value).toContain('runtime123');
expect(hash.value).toContain('runtime456');
expect(hash.value).toContain('env123');
expect(hash.value).toContain('filec.hash');

expect(hash.details.command).toEqual('parent|build||{"prop":"prop-value"}');
expect(hash.details.nodes).toEqual({
'parent:{projectRoot}/**/*':
'/file|file.hash|{"root":"libs/parent","targets":{"build":{"executor":"unknown","inputs":["default","^default",{"runtime":"echo runtime123"},{"env":"TESTENV"},{"env":"NONEXISTENTENV"}]}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}',
'/file|file.hash|{"root":"libs/parent","targets":{"build":{"executor":"unknown","inputs":["default","^default",{"runtime":"echo runtime123"},{"env":"TESTENV"},{"env":"NONEXISTENTENV"},{"input":"default","projects":["unrelated"]}]}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}',
parent: 'unknown',
'unrelated:{projectRoot}/**/*':
'libs/unrelated/filec.ts|filec.hash|{"root":"libs/unrelated","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}',
'{workspaceRoot}/nx.json': 'nx.json.hash',
'{workspaceRoot}/.gitignore': '',
'{workspaceRoot}/.nxignore': '',
Expand Down Expand Up @@ -774,7 +787,7 @@ describe('Hasher', () => {
const expanded = expandNamedInput('c', {
a: ['a.txt', { fileset: 'myfileset' }],
b: ['b.txt'],
c: ['a', { input: 'b', projects: 'self' }],
c: ['a', { input: 'b' }],
});
expect(expanded).toEqual([
{ fileset: 'a.txt' },
Expand All @@ -787,7 +800,7 @@ describe('Hasher', () => {
expect(() => expandNamedInput('c', {})).toThrow();
expect(() =>
expandNamedInput('b', {
b: [{ input: 'c', projects: 'self' }],
b: [{ input: 'c' }],
})
).toThrow();
});
Expand Down
102 changes: 80 additions & 22 deletions packages/nx/src/hasher/hasher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -164,11 +164,10 @@ export class Hasher {

const DEFAULT_INPUTS: ReadonlyArray<InputDefinition> = [
{
projects: 'self',
fileset: '{projectRoot}/**/*',
},
{
projects: 'dependencies',
dependencies: true,
input: 'default',
},
];
Expand Down Expand Up @@ -201,15 +200,19 @@ class TaskHasher {
const targetDefaults = (this.nxJson.targetDefaults || {})[
task.target.target
];
const { selfInputs, depsInputs } = splitInputsIntoSelfAndDependencies(
targetData.inputs || targetDefaults?.inputs || (DEFAULT_INPUTS as any),
namedInputs
);
const { selfInputs, depsInputs, projectInputs } =
splitInputsIntoSelfAndDependencies(
targetData.inputs ||
targetDefaults?.inputs ||
(DEFAULT_INPUTS as any),
namedInputs
);

const selfAndInputs = await this.hashSelfAndDepsInputs(
task.target.project,
selfInputs,
depsInputs,
projectInputs,
visited
);

Expand All @@ -224,7 +227,7 @@ class TaskHasher {
});
}

private async hashNamedInput(
private async hashNamedInputForDependencies(
projectName: string,
namedInput: string,
visited: string[]
Expand All @@ -240,31 +243,34 @@ class TaskHasher {
};

const selfInputs = expandNamedInput(namedInput, namedInputs);
const depsInputs = [{ input: namedInput }];
const depsInputs = [{ input: namedInput, dependencies: true as true }]; // true is boolean by default
return this.hashSelfAndDepsInputs(
projectName,
selfInputs,
depsInputs,
[],
visited
);
}

private async hashSelfAndDepsInputs(
projectName: string,
selfInputs: ExpandedSelfInput[],
depsInputs: { input: string }[],
depsInputs: { input: string; dependencies: true }[],
projectInputs: { input: string; projects: string[] }[],
visited: string[]
) {
const projectGraphDeps = this.projectGraph.dependencies[projectName] ?? [];
// we don't want random order of dependencies to change the hash
projectGraphDeps.sort((a, b) => a.target.localeCompare(b.target));

const self = await this.hashSelfInputs(projectName, selfInputs);
const self = await this.hashSingleProjectInputs(projectName, selfInputs);
const deps = await this.hashDepsInputs(
depsInputs,
projectGraphDeps,
visited
);
const projects = await this.hashProjectInputs(projectInputs, visited);

let details = {};
for (const s of self) {
Expand All @@ -273,10 +279,14 @@ class TaskHasher {
for (const s of deps) {
details = { ...details, ...s.details };
}
for (const s of projects) {
details = { ...details, ...s.details };
}

const value = this.hashing.hashArray([
...self.map((d) => d.value),
...deps.map((d) => d.value),
...projects.map((d) => d.value),
]);

return { value, details };
Expand All @@ -296,7 +306,7 @@ class TaskHasher {
return null;
} else {
visited.push(d.target);
return await this.hashNamedInput(
return await this.hashNamedInputForDependencies(
d.target,
input.input || 'default',
visited
Expand Down Expand Up @@ -366,7 +376,7 @@ class TaskHasher {
};
}

private async hashSelfInputs(
private async hashSingleProjectInputs(
projectName: string,
inputs: ExpandedSelfInput[]
): Promise<PartialHash[]> {
Expand Down Expand Up @@ -422,6 +432,29 @@ class TaskHasher {
]);
}

private async hashProjectInputs(
projectInputs: { input: string; projects: string[] }[],
visited: string[]
): Promise<PartialHash[]> {
const partialHashes: Promise<PartialHash[]>[] = [];
for (const input of projectInputs) {
for (const project of input.projects) {
const namedInputs = getNamedInputs(
this.nxJson,
this.projectGraph.nodes[project]
);
const expandedInput = expandSingleProjectInputs(
[{ input: input.input }],
namedInputs
);
partialHashes.push(
this.hashSingleProjectInputs(project, expandedInput)
);
}
}
return Promise.all(partialHashes).then((hashes) => hashes.flat());
}

private async hashRootFileset(fileset: string): Promise<PartialHash> {
const mapKey = fileset;
const withoutWorkspaceRoot = fileset.substring(16);
Expand Down Expand Up @@ -559,30 +592,55 @@ export function splitInputsIntoSelfAndDependencies(
inputs: ReadonlyArray<InputDefinition | string>,
namedInputs: { [inputName: string]: ReadonlyArray<InputDefinition | string> }
): {
depsInputs: { input: string }[];
depsInputs: { input: string; dependencies: true }[];
projectInputs: { input: string; projects: string[] }[];
selfInputs: ExpandedSelfInput[];
} {
const depsInputs = [];
const depsInputs: { input: string; dependencies: true }[] = [];
const projectInputs: { input: string; projects: string[] }[] = [];
const selfInputs = [];
for (const d of inputs) {
if (typeof d === 'string') {
if (d.startsWith('^')) {
depsInputs.push({ input: d.substring(1) });
depsInputs.push({ input: d.substring(1), dependencies: true });
} else {
selfInputs.push(d);
}
} else {
if ((d as any).projects === 'dependencies') {
depsInputs.push(d as any);
if (
('dependencies' in d && d.dependencies) ||
// Todo(@AgentEnder): Remove check in v17
('projects' in d &&
typeof d.projects === 'string' &&
d.projects === 'dependencies')
) {
depsInputs.push({
input: d.input,
dependencies: true,
});
} else if (
'projects' in d &&
d.projects &&
// Todo(@AgentEnder): Remove check in v17
!(d.projects === 'self')
) {
projectInputs.push({
input: d.input,
projects: Array.isArray(d.projects) ? d.projects : [d.projects],
});
} else {
selfInputs.push(d);
}
}
}
return { depsInputs, selfInputs: expandSelfInputs(selfInputs, namedInputs) };
return {
depsInputs,
projectInputs,
selfInputs: expandSingleProjectInputs(selfInputs, namedInputs),
};
}

function expandSelfInputs(
function expandSingleProjectInputs(
inputs: ReadonlyArray<InputDefinition | string>,
namedInputs: { [inputName: string]: ReadonlyArray<InputDefinition | string> }
): ExpandedSelfInput[] {
Expand All @@ -598,9 +656,9 @@ function expandSelfInputs(
expanded.push({ fileset: d });
}
} else {
if ((d as any).projects === 'dependencies') {
if ((d as any).projects || (d as any).dependencies) {
throw new Error(
`namedInputs definitions cannot contain any inputs with projects == 'dependencies'`
`namedInputs definitions can only refer to other namedInputs definitions within the same project.`
);
}
if ((d as any).fileset || (d as any).env || (d as any).runtime) {
Expand All @@ -619,7 +677,7 @@ export function expandNamedInput(
): ExpandedSelfInput[] {
namedInputs ||= {};
if (!namedInputs[input]) throw new Error(`Input '${input}' is not defined`);
return expandSelfInputs(namedInputs[input], namedInputs);
return expandSingleProjectInputs(namedInputs[input], namedInputs);
}

export function filterUsingGlobPatterns(
Expand Down
8 changes: 3 additions & 5 deletions packages/nx/src/tasks-runner/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,12 @@ export function getDependencyConfigs(
const specifiers =
typeof dependencyConfig.projects === 'string'
? [dependencyConfig.projects]
: dependencyConfig.projects ?? [];
for (const specifier of specifiers) {
: dependencyConfig.projects;
for (const specifier of specifiers ?? []) {
if (
!(specifier in projectGraph.nodes) &&
// Todo(@agentender): Remove the check for self / dependencies in v17
!['self', 'dependencies'].includes(
specifier
)
!['self', 'dependencies'].includes(specifier)
) {
output.error({
title: `dependsOn is improperly configured for ${project}:${target}`,
Expand Down

0 comments on commit 2e47682

Please sign in to comment.