From 1039bc8744f9fbff1e50979286abcff898ef7826 Mon Sep 17 00:00:00 2001 From: Emily Xiong Date: Wed, 3 Jul 2024 13:28:10 -0400 Subject: [PATCH] feat(core): add parallelism to target configuration --- docs/generated/devkit/TargetConfiguration.md | 10 + docs/generated/devkit/Task.md | 9 + packages/cypress/src/plugins/plugin.spec.ts | 4 + packages/cypress/src/plugins/plugin.ts | 3 + .../js/src/utils/buildable-libs-utils.spec.ts | 13 + packages/nx/schemas/project-schema.json | 26 ++ packages/nx/src/config/task-graph.ts | 5 + .../nx/src/config/to-project-name.spec.ts | 1 + .../src/config/workspace-json-project-json.ts | 6 + packages/nx/src/hasher/task-hasher.spec.ts | 54 +++ .../utils/project-configuration-utils.spec.ts | 2 + .../utils/project-configuration-utils.ts | 2 + .../nx/src/tasks-runner/create-task-graph.ts | 3 +- .../src/tasks-runner/tasks-schedule.spec.ts | 360 +++++++++++++++++- .../nx/src/tasks-runner/tasks-schedule.ts | 54 ++- .../playwright/src/plugins/plugin.spec.ts | 7 + packages/playwright/src/plugins/plugin.ts | 2 + .../files/hasher/hasher.spec.ts.template | 3 +- 18 files changed, 553 insertions(+), 11 deletions(-) diff --git a/docs/generated/devkit/TargetConfiguration.md b/docs/generated/devkit/TargetConfiguration.md index ef01902f857fcc..ac5ef2b463a361 100644 --- a/docs/generated/devkit/TargetConfiguration.md +++ b/docs/generated/devkit/TargetConfiguration.md @@ -22,6 +22,7 @@ Target's configuration - [metadata](../../devkit/documents/TargetConfiguration#metadata): TargetMetadata - [options](../../devkit/documents/TargetConfiguration#options): T - [outputs](../../devkit/documents/TargetConfiguration#outputs): string[] +- [parallelism](../../devkit/documents/TargetConfiguration#parallelism): boolean ## Properties @@ -109,3 +110,12 @@ Target's options. They are passed in to the executor. List of the target's outputs. The outputs will be cached by the Nx computation caching engine. + +--- + +### parallelism + +• `Optional` **parallelism**: `boolean` + +Whether this target can be run in parallel with other tasks +Default is true diff --git a/docs/generated/devkit/Task.md b/docs/generated/devkit/Task.md index 686bac2a5e6f20..10f6d600f80f1d 100644 --- a/docs/generated/devkit/Task.md +++ b/docs/generated/devkit/Task.md @@ -13,6 +13,7 @@ A representation of the invocation of an Executor - [id](../../devkit/documents/Task#id): string - [outputs](../../devkit/documents/Task#outputs): string[] - [overrides](../../devkit/documents/Task#overrides): any +- [parallelism](../../devkit/documents/Task#parallelism): boolean - [projectRoot](../../devkit/documents/Task#projectroot): string - [startTime](../../devkit/documents/Task#starttime): number - [target](../../devkit/documents/Task#target): Object @@ -84,6 +85,14 @@ Overrides for the configured options of the target --- +### parallelism + +• **parallelism**: `boolean` + +Determines if a given task should be parallelizable. + +--- + ### projectRoot • `Optional` **projectRoot**: `string` diff --git a/packages/cypress/src/plugins/plugin.spec.ts b/packages/cypress/src/plugins/plugin.spec.ts index aced265f5ef534..6393344224965c 100644 --- a/packages/cypress/src/plugins/plugin.spec.ts +++ b/packages/cypress/src/plugins/plugin.spec.ts @@ -121,6 +121,7 @@ describe('@nx/cypress/plugin', () => { "{projectRoot}/dist/videos", "{projectRoot}/dist/screenshots", ], + "parallelism": false, }, "open-cypress": { "command": "cypress open", @@ -329,6 +330,7 @@ describe('@nx/cypress/plugin', () => { "{projectRoot}/dist/videos", "{projectRoot}/dist/screenshots", ], + "parallelism": false, }, "e2e-ci": { "cache": true, @@ -369,6 +371,7 @@ describe('@nx/cypress/plugin', () => { "{projectRoot}/dist/videos", "{projectRoot}/dist/screenshots", ], + "parallelism": false, }, "e2e-ci--src/test.cy.ts": { "cache": true, @@ -404,6 +407,7 @@ describe('@nx/cypress/plugin', () => { "{projectRoot}/dist/videos", "{projectRoot}/dist/screenshots", ], + "parallelism": false, }, "open-cypress": { "command": "cypress open", diff --git a/packages/cypress/src/plugins/plugin.ts b/packages/cypress/src/plugins/plugin.ts index d43489cf500f62..15a094bbaf3b58 100644 --- a/packages/cypress/src/plugins/plugin.ts +++ b/packages/cypress/src/plugins/plugin.ts @@ -211,6 +211,7 @@ async function buildCypressTargets( cache: true, inputs: getInputs(namedInputs), outputs: getOutputs(projectRoot, cypressConfig, 'e2e'), + parallelism: false, metadata: { technologies: ['cypress'], description: 'Runs Cypress Tests', @@ -276,6 +277,7 @@ async function buildCypressTargets( options: { cwd: projectRoot, }, + parallelism: false, metadata: { technologies: ['cypress'], description: `Runs Cypress Tests in ${relativeSpecFilePath} in CI`, @@ -300,6 +302,7 @@ async function buildCypressTargets( inputs, outputs, dependsOn, + parallelism: false, metadata: { technologies: ['cypress'], description: 'Runs Cypress Tests in CI', diff --git a/packages/js/src/utils/buildable-libs-utils.spec.ts b/packages/js/src/utils/buildable-libs-utils.spec.ts index bf987d362344e1..84f9f331f533aa 100644 --- a/packages/js/src/utils/buildable-libs-utils.spec.ts +++ b/packages/js/src/utils/buildable-libs-utils.spec.ts @@ -389,30 +389,35 @@ describe('calculateDependenciesFromTaskGraph', () => { overrides: {}, target: { project: 'lib1', target: 'build' }, outputs: [], + parallelism: true, }, 'lib2:build': { id: 'lib2:build', overrides: {}, target: { project: 'lib2', target: 'build' }, outputs: [], + parallelism: true, }, 'lib2:build-base': { id: 'lib2:build-base', overrides: {}, target: { project: 'lib2', target: 'build-base' }, outputs: [], + parallelism: true, }, 'lib3:build': { id: 'lib3:build', overrides: {}, target: { project: 'lib3', target: 'build' }, outputs: [], + parallelism: true, }, 'lib4:build': { id: 'lib4:build', overrides: {}, target: { project: 'lib4', target: 'build' }, outputs: [], + parallelism: true, }, }, }; @@ -559,48 +564,56 @@ describe('calculateDependenciesFromTaskGraph', () => { overrides: {}, target: { project: 'lib1', target: 'build' }, outputs: [], + parallelism: true, }, 'lib1:build-base': { id: 'lib1:build-base', overrides: {}, target: { project: 'lib1', target: 'build-base' }, outputs: [], + parallelism: true, }, 'lib2:build': { id: 'lib2:build', overrides: {}, target: { project: 'lib2', target: 'build' }, outputs: [], + parallelism: true, }, 'lib2:build-base': { id: 'lib2:build-base', overrides: {}, target: { project: 'lib2', target: 'build-base' }, outputs: [], + parallelism: true, }, 'lib3:build': { id: 'lib3:build', overrides: {}, target: { project: 'lib3', target: 'build' }, outputs: [], + parallelism: true, }, 'lib3:build-base': { id: 'lib3:build-base', overrides: {}, target: { project: 'lib3', target: 'build-base' }, outputs: [], + parallelism: true, }, 'lib4:build': { id: 'lib4:build', overrides: {}, target: { project: 'lib4', target: 'build' }, outputs: [], + parallelism: true, }, 'lib4:build-base': { id: 'lib4:build-base', overrides: {}, target: { project: 'lib4', target: 'build-base' }, outputs: [], + parallelism: true, }, }, }; diff --git a/packages/nx/schemas/project-schema.json b/packages/nx/schemas/project-schema.json index 969ddbf126dd29..cb8fa2b220b5f3 100644 --- a/packages/nx/schemas/project-schema.json +++ b/packages/nx/schemas/project-schema.json @@ -4,6 +4,27 @@ "title": "JSON schema for Nx projects", "type": "object", "properties": { + "name": { + "type": "string", + "description": "Project's name. Optional if specified in workspace.json" + }, + "root": { + "type": "string", + "description": "Project's location relative to the root of the workspace" + }, + "sourceRoot": { + "type": "string", + "description": "The location of project's sources relative to the root of the workspace" + }, + "projectType": { + "type": "string", + "description": "Type of project supported", + "enum": ["library", "application"] + }, + "generators": { + "type": "object", + "description": "List of default values used by generators" + }, "namedInputs": { "type": "object", "description": "Named inputs used by inputs defined in targets", @@ -112,6 +133,11 @@ "cache": { "type": "boolean", "description": "Specifies if the given target should be cacheable" + }, + "parallelism": { + "type": "boolean", + "default": true, + "description": "Whether this target can be run in parallel with other tasks" } } } diff --git a/packages/nx/src/config/task-graph.ts b/packages/nx/src/config/task-graph.ts index 573f691ccc220d..9829af409e10f7 100644 --- a/packages/nx/src/config/task-graph.ts +++ b/packages/nx/src/config/task-graph.ts @@ -76,6 +76,11 @@ export interface Task { * Determines if a given task should be cacheable. */ cache?: boolean; + + /** + * Determines if a given task should be parallelizable. + */ + parallelism: boolean; } /** diff --git a/packages/nx/src/config/to-project-name.spec.ts b/packages/nx/src/config/to-project-name.spec.ts index 208997128a1181..0815e9237b3d49 100644 --- a/packages/nx/src/config/to-project-name.spec.ts +++ b/packages/nx/src/config/to-project-name.spec.ts @@ -73,6 +73,7 @@ describe('Workspaces', () => { ], "executor": "@nx/js:release-publish", "options": {}, + "parallelism": true, }, }, } diff --git a/packages/nx/src/config/workspace-json-project-json.ts b/packages/nx/src/config/workspace-json-project-json.ts index 4d7b1e4ef4fd02..eff58d64c48708 100644 --- a/packages/nx/src/config/workspace-json-project-json.ts +++ b/packages/nx/src/config/workspace-json-project-json.ts @@ -230,4 +230,10 @@ export interface TargetConfiguration { * Metadata about the target */ metadata?: TargetMetadata; + + /** + * Whether this target can be run in parallel with other tasks + * Default is true + */ + parallelism?: boolean; } diff --git a/packages/nx/src/hasher/task-hasher.spec.ts b/packages/nx/src/hasher/task-hasher.spec.ts index 29fa359bb6bd14..45b695515babc5 100644 --- a/packages/nx/src/hasher/task-hasher.spec.ts +++ b/packages/nx/src/hasher/task-hasher.spec.ts @@ -121,6 +121,7 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['parent-build'], @@ -130,6 +131,7 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -188,6 +190,7 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['child-build'], @@ -197,12 +200,14 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, 'child-build': { id: 'child-build', target: { project: 'child', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: { @@ -275,6 +280,7 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['child-build'], @@ -284,12 +290,14 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, 'child-build': { id: 'child-build', target: { project: 'child', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: { @@ -354,6 +362,7 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'test' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -365,6 +374,7 @@ describe('TaskHasher', () => { id: 'parent-test', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, taskGraph, {} @@ -378,6 +388,7 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, taskGraph, {} @@ -459,12 +470,14 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'test' }, overrides: {}, outputs: [], + parallelism: true, }, 'child-test': { id: 'child-test', target: { project: 'child', target: 'test' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: { @@ -478,6 +491,7 @@ describe('TaskHasher', () => { id: 'parent-test', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, taskGraph, { MY_TEST_HASH_ENV: 'MY_TEST_HASH_ENV_VALUE' } @@ -491,6 +505,7 @@ describe('TaskHasher', () => { id: 'child-test', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, taskGraph, { MY_TEST_HASH_ENV: 'MY_TEST_HASH_ENV_VALUE' } @@ -559,6 +574,7 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['child-build'], @@ -568,12 +584,14 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, 'child-build': { id: 'child-build', target: { project: 'child', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: { @@ -622,6 +640,7 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['parent:build'], @@ -631,6 +650,7 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -687,12 +707,14 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, 'child-build': { id: 'child-build', target: { project: 'child', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: { @@ -706,6 +728,7 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, taskGraph, {} @@ -719,6 +742,7 @@ describe('TaskHasher', () => { id: 'child-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, taskGraph, {} @@ -763,6 +787,7 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: {}, outputs: [], + parallelism: true, }, { roots: ['parent:build'], @@ -772,6 +797,7 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -834,6 +860,7 @@ describe('TaskHasher', () => { id: 'app-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['app-build'], @@ -843,6 +870,7 @@ describe('TaskHasher', () => { target: { project: 'app', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -902,6 +930,7 @@ describe('TaskHasher', () => { id: 'app-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['app-build'], @@ -911,6 +940,7 @@ describe('TaskHasher', () => { target: { project: 'app', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -961,6 +991,7 @@ describe('TaskHasher', () => { id: 'app-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['app-build'], @@ -970,6 +1001,7 @@ describe('TaskHasher', () => { target: { project: 'app', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -1051,12 +1083,14 @@ describe('TaskHasher', () => { target: { project: 'a', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, 'b-build': { id: 'b-build', target: { project: 'b', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -1071,6 +1105,7 @@ describe('TaskHasher', () => { target: { project: 'a', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, taskGraph, {} @@ -1081,6 +1116,7 @@ describe('TaskHasher', () => { target: { project: 'b', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, taskGraph, {} @@ -1092,6 +1128,7 @@ describe('TaskHasher', () => { target: { project: 'b', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, taskGraph, {} @@ -1102,6 +1139,7 @@ describe('TaskHasher', () => { target: { project: 'a', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, taskGraph, {} @@ -1201,6 +1239,7 @@ describe('TaskHasher', () => { id: 'app-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['app-build'], @@ -1210,6 +1249,7 @@ describe('TaskHasher', () => { target: { project: 'app', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -1418,6 +1458,7 @@ describe('TaskHasher', () => { id: 'app-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['app-build'], @@ -1427,6 +1468,7 @@ describe('TaskHasher', () => { target: { project: 'app', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -1502,6 +1544,7 @@ describe('TaskHasher', () => { id: 'app-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['app-build'], @@ -1511,6 +1554,7 @@ describe('TaskHasher', () => { target: { project: 'app', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -1584,6 +1628,7 @@ describe('TaskHasher', () => { id: 'app-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['app-build'], @@ -1593,6 +1638,7 @@ describe('TaskHasher', () => { target: { project: 'app', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: {}, @@ -1710,6 +1756,7 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['grandchild-build'], @@ -1719,18 +1766,21 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'build' }, overrides: {}, outputs: ['dist/libs/libs/parent'], + parallelism: true, }, 'child-build': { id: 'child-build', target: { project: 'child', target: 'build' }, overrides: {}, outputs: ['dist/libs/libs/child'], + parallelism: true, }, 'grandchild-build': { id: 'grandchild-build', target: { project: 'grandchild', target: 'build' }, overrides: {}, outputs: ['dist/libs/libs/grandchild'], + parallelism: true, }, }, dependencies: { @@ -1850,6 +1900,7 @@ describe('TaskHasher', () => { id: 'parent-build', overrides: { prop: 'prop-value' }, outputs: [], + parallelism: true, }, { roots: ['grandchild-build'], @@ -1859,18 +1910,21 @@ describe('TaskHasher', () => { target: { project: 'parent', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, 'child-build': { id: 'child-build', target: { project: 'child', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, 'grandchild-build': { id: 'grandchild-build', target: { project: 'grandchild', target: 'build' }, overrides: {}, outputs: [], + parallelism: true, }, }, dependencies: { diff --git a/packages/nx/src/project-graph/utils/project-configuration-utils.spec.ts b/packages/nx/src/project-graph/utils/project-configuration-utils.spec.ts index c24db50ef41893..a722abf6fc521a 100644 --- a/packages/nx/src/project-graph/utils/project-configuration-utils.spec.ts +++ b/packages/nx/src/project-graph/utils/project-configuration-utils.spec.ts @@ -1502,6 +1502,7 @@ describe('project-configuration-utils', () => { "options": { "command": "echo libs/project", }, + "parallelism": true, } `); }); @@ -1658,6 +1659,7 @@ describe('project-configuration-utils', () => { "options": { "command": "echo a @ libs/a", }, + "parallelism": true, } `); }); diff --git a/packages/nx/src/project-graph/utils/project-configuration-utils.ts b/packages/nx/src/project-graph/utils/project-configuration-utils.ts index e1a38c2c12eaf7..9eeae9a695f35a 100644 --- a/packages/nx/src/project-graph/utils/project-configuration-utils.ts +++ b/packages/nx/src/project-graph/utils/project-configuration-utils.ts @@ -1104,5 +1104,7 @@ export function normalizeTarget( ); } + target.parallelism ??= true; + return target; } diff --git a/packages/nx/src/tasks-runner/create-task-graph.ts b/packages/nx/src/tasks-runner/create-task-graph.ts index 8989476b4b584a..909fc411beba16 100644 --- a/packages/nx/src/tasks-runner/create-task-graph.ts +++ b/packages/nx/src/tasks-runner/create-task-graph.ts @@ -35,7 +35,7 @@ export class ProcessTasks { configuration: string, overrides: Object, excludeTaskDependencies: boolean - ) { + ): string[] { for (const projectName of projectNames) { for (const target of targets) { const project = this.projectGraph.nodes[projectName]; @@ -319,6 +319,7 @@ export class ProcessTasks { interpolatedOverrides ), cache: project.data.targets[target].cache, + parallelism: project.data.targets[target].parallelism ?? true, }; } diff --git a/packages/nx/src/tasks-runner/tasks-schedule.spec.ts b/packages/nx/src/tasks-runner/tasks-schedule.spec.ts index d849fccdd21e8d..32664d95f4c6a5 100644 --- a/packages/nx/src/tasks-runner/tasks-schedule.spec.ts +++ b/packages/nx/src/tasks-runner/tasks-schedule.spec.ts @@ -5,7 +5,7 @@ import { DependencyType, ProjectGraph } from '../config/project-graph'; import * as nxJsonUtils from '../config/nx-json'; import * as executorUtils from '../command-line/run/executor-utils'; -function createMockTask(id: string): Task { +function createMockTask(id: string, parallelism: boolean = true): Task { const [project, target] = id.split(':'); return { id, @@ -15,6 +15,7 @@ function createMockTask(id: string): Task { }, outputs: [], overrides: {}, + parallelism, }; } @@ -411,4 +412,361 @@ describe('TasksSchedule', () => { }); }); }); + + describe('tasks with parallelism false', () => { + describe('dependent tasks', () => { + let taskSchedule: TasksSchedule; + let taskGraph: TaskGraph; + let app1Build: Task; + let app2Build: Task; + let lib1Build: Task; + let lifeCycle: any; + beforeEach(() => { + // app1 depends on lib1 + // app2 does not depend on anything + // lib1 does not depend on anything + // all tasks have parallelism set to false + app1Build = createMockTask('app1:build', false); + app2Build = createMockTask('app2:build', false); + lib1Build = createMockTask('lib1:build', false); + + taskGraph = { + tasks: { + 'app1:build': app1Build, + 'app2:build': app2Build, + 'lib1:build': lib1Build, + }, + dependencies: { + 'app1:build': ['lib1:build'], + 'app2:build': [], + 'lib1:build': [], + }, + roots: ['lib1:build', 'app2:build'], + }; + jest.spyOn(nxJsonUtils, 'readNxJson').mockReturnValue({}); + jest.spyOn(executorUtils, 'getExecutorInformation').mockReturnValue({ + schema: { + version: 2, + properties: {}, + }, + implementationFactory: jest.fn(), + batchImplementationFactory: jest.fn(), + isNgCompat: true, + isNxExecutor: true, + }); + + const projectGraph: ProjectGraph = { + nodes: { + app1: { + data: { + root: 'app1', + targets: { + build: { + executor: 'awesome-executors:build', + }, + }, + }, + name: 'app1', + type: 'app', + }, + app2: { + name: 'app2', + type: 'app', + data: { + root: 'app2', + targets: { + build: { + executor: 'awesome-executors:app2-build', + }, + }, + }, + }, + lib1: { + name: 'lib1', + type: 'lib', + data: { + root: 'lib1', + targets: { + build: { + executor: 'awesome-executors:build', + }, + }, + }, + }, + } as any, + dependencies: { + app1: [ + { + source: 'app1', + target: 'lib1', + type: DependencyType.static, + }, + ], + app2: [ + { + source: 'app2', + target: 'lib1', + type: DependencyType.static, + }, + ], + }, + externalNodes: {}, + version: '5', + }; + + lifeCycle = { + startTask: jest.fn(), + endTask: jest.fn(), + scheduleTask: jest.fn(), + }; + taskSchedule = new TasksSchedule(projectGraph, taskGraph, { + lifeCycle, + }); + }); + + describe('Without Batch Mode', () => { + let original; + beforeEach(() => { + original = process.env['NX_BATCH_MODE']; + process.env['NX_BATCH_MODE'] = 'false'; + }); + + afterEach(() => { + process.env['NX_BATCH_MODE'] = original; + }); + + it('should begin with no scheduled tasks', () => { + expect(taskSchedule.nextBatch()).toBeNull(); + expect(taskSchedule.nextTask()).toBeNull(); + }); + + it('should schedule root tasks first', async () => { + // app1 depends on lib1, app2 has no dependencies + await taskSchedule.scheduleNextTasks(); + expect(taskSchedule.nextTask()).toEqual(lib1Build); + // since lib1 is not parallel, app2 should not be scheduled even though it has no dependencies + expect(taskSchedule.nextTask()).toBeNull(); + taskSchedule.complete([lib1Build.id]); + await taskSchedule.scheduleNextTasks(); + expect(taskSchedule.nextTask()).toEqual(app1Build); + expect(taskSchedule.nextTask()).toBeNull(); // app2 should not be scheduled since app1 is not parallel and not completed + taskSchedule.complete([app1Build.id]); + await taskSchedule.scheduleNextTasks(); + expect(taskSchedule.nextTask()).toEqual(app2Build); + taskSchedule.complete([app2Build.id]); + expect(taskSchedule.hasTasks()).toEqual(false); + }); + + it('should not schedule batches', async () => { + await taskSchedule.scheduleNextTasks(); + + expect(taskSchedule.nextTask()).not.toBeNull(); + + expect(taskSchedule.nextBatch()).toBeNull(); + }); + }); + + describe('With Batch Mode', () => { + let original; + beforeEach(() => { + original = process.env['NX_BATCH_MODE']; + process.env['NX_BATCH_MODE'] = 'true'; + }); + + afterEach(() => { + process.env['NX_BATCH_MODE'] = original; + }); + + it('should not schedule batches of tasks by different executors if task has parallelism false', async () => { + await taskSchedule.scheduleNextTasks(); + + // since all tasks have parallelism false, they should not be batched + expect(taskSchedule.nextTask()).toEqual(lib1Build); + + expect(taskSchedule.nextBatch()).toBeNull(); + }); + }); + }); + + describe('non-dependent tasks', () => { + let taskSchedule: TasksSchedule; + let taskGraph: TaskGraph; + let app1Test: Task; + let app2Test: Task; + let lib1Test: Task; + let lifeCycle: any; + beforeEach(() => { + // app1, app2, and lib1 do not depend on each other + // all tasks have parallelism set to false + app1Test = createMockTask('app1:test', false); + app2Test = createMockTask('app2:test', false); + lib1Test = createMockTask('lib1:test', false); + + taskGraph = { + tasks: { + 'app1:test': app1Test, + 'app2:test': app2Test, + 'lib1:test': lib1Test, + }, + dependencies: { + 'app1:test': [], + 'app2:test': [], + 'lib1:test': [], + }, + roots: ['app1:test', 'app2:test', 'lib1:test'], + }; + jest.spyOn(nxJsonUtils, 'readNxJson').mockReturnValue({}); + jest.spyOn(executorUtils, 'getExecutorInformation').mockReturnValue({ + schema: { + version: 2, + properties: {}, + }, + implementationFactory: jest.fn(), + batchImplementationFactory: jest.fn(), + isNgCompat: true, + isNxExecutor: true, + }); + + const projectGraph: ProjectGraph = { + nodes: { + app1: { + data: { + root: 'app1', + targets: { + test: { + executor: 'awesome-executors:test', + parallelism: false, + }, + }, + }, + name: 'app1', + type: 'app', + }, + app2: { + name: 'app2', + type: 'app', + data: { + root: 'app2', + targets: { + test: { + executor: 'awesome-executors:app2-test', + parallelism: false, + }, + }, + }, + }, + lib1: { + name: 'lib1', + type: 'lib', + data: { + root: 'lib1', + targets: { + test: { + executor: 'awesome-executors:test', + }, + }, + }, + }, + } as any, + dependencies: { + app1: [ + { + source: 'app1', + target: 'lib1', + type: DependencyType.static, + }, + ], + app2: [ + { + source: 'app2', + target: 'lib1', + type: DependencyType.static, + }, + ], + }, + externalNodes: {}, + version: '5', + }; + + lifeCycle = { + startTask: jest.fn(), + endTask: jest.fn(), + scheduleTask: jest.fn(), + }; + taskSchedule = new TasksSchedule(projectGraph, taskGraph, { + lifeCycle, + }); + }); + + describe('Without Batch Mode', () => { + let original; + beforeEach(() => { + original = process.env['NX_BATCH_MODE']; + process.env['NX_BATCH_MODE'] = 'false'; + }); + + afterEach(() => { + process.env['NX_BATCH_MODE'] = original; + }); + + it('should begin with no scheduled tasks', () => { + expect(taskSchedule.nextBatch()).toBeNull(); + expect(taskSchedule.nextTask()).toBeNull(); + }); + + it('should schedule root tasks in topological order', async () => { + await taskSchedule.scheduleNextTasks(); + expect(taskSchedule.nextTask()).toEqual(app1Test); + let nextTask = taskSchedule.nextTask(); + expect(nextTask).not.toEqual(app2Test); // app2 should not be scheduled since app1 is not parallel and not completed + expect(nextTask).not.toEqual(lib1Test); // lib1 should not be scheduled since app1 is not parallel and not completed + expect(nextTask).toBeNull(); + + taskSchedule.complete([app1Test.id]); + await taskSchedule.scheduleNextTasks(); + nextTask = taskSchedule.nextTask(); + expect(nextTask).toEqual(app2Test); // app2 should be scheduled since app1 is completed now + + nextTask = taskSchedule.nextTask(); + expect(nextTask).not.toEqual(lib1Test); // lib1 should not be scheduled since app2 is not parallel and not completed + expect(nextTask).toBeNull(); + + taskSchedule.complete([app2Test.id]); + await taskSchedule.scheduleNextTasks(); + expect(taskSchedule.nextTask()).toEqual(lib1Test); // lib1 should be scheduled since app2 is completed now + taskSchedule.complete([lib1Test.id]); + expect(taskSchedule.hasTasks()).toEqual(false); + }); + + it('should not schedule batches', async () => { + await taskSchedule.scheduleNextTasks(); + + expect(taskSchedule.nextTask()).not.toBeNull(); + + expect(taskSchedule.nextBatch()).toBeNull(); + }); + }); + + describe('With Batch Mode', () => { + let original; + beforeEach(() => { + original = process.env['NX_BATCH_MODE']; + process.env['NX_BATCH_MODE'] = 'true'; + }); + + afterEach(() => { + process.env['NX_BATCH_MODE'] = original; + }); + + it('should not schedule batches of tasks by different executors if task have parallelism false', async () => { + await taskSchedule.scheduleNextTasks(); + + // app1, app2, and lib1 are not parallel, so they should not be batched + expect(taskSchedule.nextTask()).toEqual(app1Test); + + expect(taskSchedule.nextBatch()).toBeNull(); + }); + }); + }); + }); }); diff --git a/packages/nx/src/tasks-runner/tasks-schedule.ts b/packages/nx/src/tasks-runner/tasks-schedule.ts index 5d63f3a3173027..3c1af43fd7c1ca 100644 --- a/packages/nx/src/tasks-runner/tasks-schedule.ts +++ b/packages/nx/src/tasks-runner/tasks-schedule.ts @@ -45,6 +45,17 @@ export class TasksSchedule { ); } + private hasRunningTasks(): boolean { + const totalTasksNumbber = Object.keys(this.taskGraph.tasks).length; + const notScheduledTaskNumber = Object.keys( + this.notScheduledTaskGraph.tasks + ).length; + const completedTasksNumbers = this.completedTasks.size; + return ( + totalTasksNumbber - notScheduledTaskNumber - completedTasksNumbers > 0 + ); + } + public complete(taskIds: string[]) { for (const taskId of taskIds) { this.completedTasks.add(taskId); @@ -146,8 +157,8 @@ export class TasksSchedule { task: Task, rootExecutorName: string, isRoot: boolean - ) { - if (!this.canBatchTaskBeScheduled(task.id, batches[rootExecutorName])) { + ): Promise { + if (!this.canBatchTaskBeScheduled(task, batches[rootExecutorName])) { return; } @@ -191,18 +202,45 @@ export class TasksSchedule { } private canBatchTaskBeScheduled( - taskId: string, + task: Task, batchTaskGraph: TaskGraph | undefined ): boolean { + // task self needs to have parallelism true // all deps have either completed or belong to the same batch - return this.taskGraph.dependencies[taskId].every( - (id) => this.completedTasks.has(id) || !!batchTaskGraph?.tasks[id] + return ( + task.parallelism === true && + this.taskGraph.dependencies[task.id].every( + (id) => this.completedTasks.has(id) || !!batchTaskGraph?.tasks[id] + ) ); } - private canBeScheduled(taskId: string) { - return this.taskGraph.dependencies[taskId].every((id) => - this.completedTasks.has(id) + private canBeScheduled(taskId: string): boolean { + const hasDependenciesCompleted = this.taskGraph.dependencies[taskId].every( + (id) => this.completedTasks.has(id) ); + + // if dependencies have not completed, cannot schedule + if (!hasDependenciesCompleted) { + return false; + } + + // if there are no running tasks, can schedule anything + if (this.hasRunningTasks() === false) { + return true; + } + + const runningTasksNotSupportParallelism = this.scheduledTasks.some( + (taskId) => { + return this.taskGraph.tasks[taskId].parallelism === false; + } + ); + if (runningTasksNotSupportParallelism) { + // if any running tasks do not support parallelism, no other tasks can be scheduled + return false; + } else { + // if all running tasks support parallelism, can only schedule task with parallelism + return this.taskGraph.tasks[taskId].parallelism === true; + } } } diff --git a/packages/playwright/src/plugins/plugin.spec.ts b/packages/playwright/src/plugins/plugin.spec.ts index f6740f0a172978..60c36e4f2c4a0c 100644 --- a/packages/playwright/src/plugins/plugin.spec.ts +++ b/packages/playwright/src/plugins/plugin.spec.ts @@ -91,6 +91,7 @@ describe('@nx/playwright/plugin', () => { "outputs": [ "{projectRoot}/test-results", ], + "parallelism": false, }, "e2e-ci": { "cache": true, @@ -123,6 +124,7 @@ describe('@nx/playwright/plugin', () => { "outputs": [ "{projectRoot}/test-results", ], + "parallelism": false, }, }, }, @@ -200,6 +202,7 @@ describe('@nx/playwright/plugin', () => { "{projectRoot}/test-results/html", "{projectRoot}/test-results", ], + "parallelism": false, }, "e2e-ci": { "cache": true, @@ -235,6 +238,7 @@ describe('@nx/playwright/plugin', () => { "{projectRoot}/test-results/html", "{projectRoot}/test-results", ], + "parallelism": false, }, }, }, @@ -323,6 +327,7 @@ describe('@nx/playwright/plugin', () => { "outputs": [ "{projectRoot}/test-results", ], + "parallelism": false, } `); expect(targets['e2e-ci--tests/run-me.spec.ts']).toMatchInlineSnapshot(` @@ -358,6 +363,7 @@ describe('@nx/playwright/plugin', () => { "outputs": [ "{projectRoot}/test-results", ], + "parallelism": false, } `); expect(targets['e2e-ci--tests/run-me-2.spec.ts']).toMatchInlineSnapshot(` @@ -393,6 +399,7 @@ describe('@nx/playwright/plugin', () => { "outputs": [ "{projectRoot}/test-results", ], + "parallelism": false, } `); expect(targets['e2e-ci--tests/skip-me.spec.ts']).not.toBeDefined(); diff --git a/packages/playwright/src/plugins/plugin.ts b/packages/playwright/src/plugins/plugin.ts index 9f64ffc8a6513b..dc45d1ccdf2a28 100644 --- a/packages/playwright/src/plugins/plugin.ts +++ b/packages/playwright/src/plugins/plugin.ts @@ -162,6 +162,7 @@ async function buildPlaywrightTargets( options: { cwd: '{projectRoot}', }, + parallelism: false, metadata: { technologies: ['playwright'], description: 'Runs Playwright Tests', @@ -257,6 +258,7 @@ async function buildPlaywrightTargets( inputs: ciBaseTargetConfig.inputs, outputs: ciBaseTargetConfig.outputs, dependsOn, + parallelism: false, metadata: { technologies: ['playwright'], description: 'Runs Playwright Tests in CI', diff --git a/packages/plugin/src/generators/executor/files/hasher/hasher.spec.ts.template b/packages/plugin/src/generators/executor/files/hasher/hasher.spec.ts.template index bcdaac89f331ad..92fc67ddccdd96 100644 --- a/packages/plugin/src/generators/executor/files/hasher/hasher.spec.ts.template +++ b/packages/plugin/src/generators/executor/files/hasher/hasher.spec.ts.template @@ -14,7 +14,8 @@ describe('<%=propertyName%>Hasher', () => { target: 'target' }, overrides: {}, - outputs: [] + outputs: [], + parallelism: true }, { hasher: mockHasher } as unknown as HasherContext)