From c5ee9c4196bd7aa7ad0c0309bacf16da3786e9c2 Mon Sep 17 00:00:00 2001 From: Jason Jean Date: Mon, 21 Nov 2022 11:16:26 -0500 Subject: [PATCH] feat(core): run tasks with no dependencies in topological order --- .../src/tasks-runner/tasks-schedule.spec.ts | 562 ++++++++++++------ .../nx/src/tasks-runner/tasks-schedule.ts | 29 +- packages/nx/src/utils/project-graph-utils.ts | 2 +- 3 files changed, 418 insertions(+), 175 deletions(-) diff --git a/packages/nx/src/tasks-runner/tasks-schedule.spec.ts b/packages/nx/src/tasks-runner/tasks-schedule.spec.ts index 9278368e02db4..6345dd832c925 100644 --- a/packages/nx/src/tasks-runner/tasks-schedule.spec.ts +++ b/packages/nx/src/tasks-runner/tasks-schedule.spec.ts @@ -2,8 +2,7 @@ import { TasksSchedule } from './tasks-schedule'; import { Workspaces } from '../config/workspaces'; import { removeTasksFromTaskGraph } from './utils'; import { Task, TaskGraph } from '../config/task-graph'; -import { ProjectGraph } from '../config/project-graph'; -import { ExecutorConfig } from '../config/misc-interfaces'; +import { DependencyType, ProjectGraph } from '../config/project-graph'; function createMockTask(id: string): Task { const [project, target] = id.split(':'); @@ -18,213 +17,438 @@ function createMockTask(id: string): Task { } describe('TasksSchedule', () => { - let taskSchedule: TasksSchedule; - let taskGraph: TaskGraph; - let app1Build: Task; - let app2Build: Task; - let lib1Build: Task; - let lifeCycle: any; - beforeEach(() => { - app1Build = createMockTask('app1:build'); - app2Build = createMockTask('app2:build'); - lib1Build = createMockTask('lib1:build'); - - taskGraph = { - tasks: { - 'app1:build': app1Build, - 'app2:build': app2Build, - 'lib1:build': lib1Build, - }, - dependencies: { - 'app1:build': ['lib1:build'], - 'app2:build': [], - 'lib1:build': [], - }, - roots: ['lib1:build', 'app2:build'], - }; - const workspace: Partial = { - readExecutor() { - return { - schema: { - version: 2, - properties: {}, - }, - implementationFactory: jest.fn(), - batchImplementationFactory: jest.fn(), - } as any; - }, - readNxJson() { - return {}; - }, - }; - - const projectGraph: ProjectGraph = { - nodes: { - app1: { - data: { - root: 'app1', - targets: { - build: { - executor: 'awesome-executors:build', + describe('dependent tasks', () => { + let taskSchedule: TasksSchedule; + let taskGraph: TaskGraph; + let app1Build: Task; + let app2Build: Task; + let lib1Build: Task; + let lifeCycle: any; + beforeEach(() => { + app1Build = createMockTask('app1:build'); + app2Build = createMockTask('app2:build'); + lib1Build = createMockTask('lib1:build'); + + taskGraph = { + tasks: { + 'app1:build': app1Build, + 'app2:build': app2Build, + 'lib1:build': lib1Build, + }, + dependencies: { + 'app1:build': ['lib1:build'], + 'app2:build': [], + 'lib1:build': [], + }, + roots: ['lib1:build', 'app2:build'], + }; + const workspace: Partial = { + readExecutor() { + return { + schema: { + version: 2, + properties: {}, + }, + implementationFactory: jest.fn(), + batchImplementationFactory: jest.fn(), + } as any; + }, + readNxJson() { + return {}; + }, + }; + + const projectGraph: ProjectGraph = { + nodes: { + app1: { + data: { + root: 'app1', + targets: { + build: { + executor: 'awesome-executors:build', + }, }, }, + name: 'app1', + type: 'app', }, - name: 'app1', - type: 'app', - }, - app2: { - name: 'app2', - type: 'app', - data: { - root: 'app2', - targets: { - build: { - executor: 'awesome-executors:app2-build', + app2: { + name: 'app2', + type: 'app', + data: { + root: 'app2', + targets: { + build: { + executor: 'awesome-executors:app2-build', + }, }, }, }, - }, - lib1: { - name: 'lib1', - type: 'lib', - data: { - root: 'lib1', - targets: { - build: { - executor: 'awesome-executors:build', + lib1: { + name: 'lib1', + type: 'lib', + data: { + root: 'lib1', + targets: { + build: { + executor: 'awesome-executors:build', + }, }, }, }, }, - }, - dependencies: {}, - allWorkspaceFiles: [], - externalNodes: {}, - version: '5', - }; - - const hasher = { - hashTask: () => 'hash', - } as any; - - lifeCycle = { - startTask: jest.fn(), - endTask: jest.fn(), - scheduleTask: jest.fn(), - }; - taskSchedule = new TasksSchedule( - hasher, - {}, - projectGraph, - taskGraph, - workspace as Workspaces, - { - lifeCycle, - } - ); - }); + dependencies: { + app1: [ + { + source: 'app1', + target: 'lib1', + type: DependencyType.static, + }, + ], + app2: [ + { + source: 'app2', + target: 'lib1', + type: DependencyType.static, + }, + ], + }, + allWorkspaceFiles: [], + externalNodes: {}, + version: '5', + }; - describe('Without Batch Mode', () => { - let original; - beforeEach(() => { - original = process.env['NX_BATCH_MODE']; - process.env['NX_BATCH_MODE'] = 'false'; - }); + const hasher = { + hashTask: () => 'hash', + } as any; - afterEach(() => { - process.env['NX_BATCH_MODE'] = original; + lifeCycle = { + startTask: jest.fn(), + endTask: jest.fn(), + scheduleTask: jest.fn(), + }; + taskSchedule = new TasksSchedule( + hasher, + {}, + projectGraph, + taskGraph, + workspace as Workspaces, + { + lifeCycle, + } + ); }); - it('should begin with no scheduled tasks', () => { - expect(taskSchedule.nextBatch()).toBeNull(); - expect(taskSchedule.nextTask()).toBeNull(); - }); + describe('Without Batch Mode', () => { + let original; + beforeEach(() => { + original = process.env['NX_BATCH_MODE']; + process.env['NX_BATCH_MODE'] = 'false'; + }); - it('should schedule root tasks first', async () => { - await taskSchedule.scheduleNextTasks(); - expect(taskSchedule.nextTask()).toEqual(lib1Build); - expect(taskSchedule.nextTask()).toEqual(app2Build); - }); + afterEach(() => { + process.env['NX_BATCH_MODE'] = original; + }); - it('should invoke lifeCycle.scheduleTask', async () => { - await taskSchedule.scheduleNextTasks(); - expect(lifeCycle.scheduleTask).toHaveBeenCalled(); - }); + it('should begin with no scheduled tasks', () => { + expect(taskSchedule.nextBatch()).toBeNull(); + expect(taskSchedule.nextTask()).toBeNull(); + }); - it('should not schedule any tasks that still have uncompleted dependencies', async () => { - await taskSchedule.scheduleNextTasks(); - taskSchedule.nextTask(); - taskSchedule.nextTask(); - expect(taskSchedule.nextTask()).toBeNull(); + it('should schedule root tasks first', async () => { + await taskSchedule.scheduleNextTasks(); + expect(taskSchedule.nextTask()).toEqual(lib1Build); + expect(taskSchedule.nextTask()).toEqual(app2Build); + }); - taskSchedule.complete([app2Build.id]); + it('should invoke lifeCycle.scheduleTask', async () => { + await taskSchedule.scheduleNextTasks(); + expect(lifeCycle.scheduleTask).toHaveBeenCalled(); + }); - expect(taskSchedule.nextTask()).toBeNull(); - }); + it('should not schedule any tasks that still have uncompleted dependencies', async () => { + await taskSchedule.scheduleNextTasks(); + taskSchedule.nextTask(); + taskSchedule.nextTask(); + expect(taskSchedule.nextTask()).toBeNull(); - it('should continue to schedule tasks that have completed dependencies', async () => { - await taskSchedule.scheduleNextTasks(); - taskSchedule.nextTask(); - taskSchedule.nextTask(); - taskSchedule.complete([lib1Build.id]); + taskSchedule.complete([app2Build.id]); - await taskSchedule.scheduleNextTasks(); - expect(taskSchedule.nextTask()).toEqual(app1Build); - }); + expect(taskSchedule.nextTask()).toBeNull(); + }); - it('should run out of tasks when they are all complete', async () => { - await taskSchedule.scheduleNextTasks(); - taskSchedule.nextTask(); - taskSchedule.nextTask(); - taskSchedule.complete([lib1Build.id, app1Build.id, app2Build.id]); + it('should continue to schedule tasks that have completed dependencies', async () => { + await taskSchedule.scheduleNextTasks(); + taskSchedule.nextTask(); + taskSchedule.nextTask(); + taskSchedule.complete([lib1Build.id]); + + await taskSchedule.scheduleNextTasks(); + expect(taskSchedule.nextTask()).toEqual(app1Build); + }); + + it('should run out of tasks when they are all complete', async () => { + await taskSchedule.scheduleNextTasks(); + taskSchedule.nextTask(); + taskSchedule.nextTask(); + taskSchedule.complete([lib1Build.id, app1Build.id, app2Build.id]); + + expect(taskSchedule.hasTasks()).toEqual(false); + }); - expect(taskSchedule.hasTasks()).toEqual(false); + it('should not schedule batches', async () => { + await taskSchedule.scheduleNextTasks(); + + expect(taskSchedule.nextTask()).not.toBeNull(); + + expect(taskSchedule.nextBatch()).toBeNull(); + }); }); - it('should not schedule batches', async () => { - await taskSchedule.scheduleNextTasks(); + describe('With Batch Mode', () => { + let original; + beforeEach(() => { + original = process.env['NX_BATCH_MODE']; + process.env['NX_BATCH_MODE'] = 'true'; + }); + + afterEach(() => { + process.env['NX_BATCH_MODE'] = original; + }); + + it('should schedule batches of tasks by different executors', async () => { + await taskSchedule.scheduleNextTasks(); - expect(taskSchedule.nextTask()).not.toBeNull(); + expect(taskSchedule.nextTask()).toBeNull(); - expect(taskSchedule.nextBatch()).toBeNull(); + expect(taskSchedule.nextBatch()).toEqual({ + executorName: 'awesome-executors:build', + taskGraph: removeTasksFromTaskGraph(taskGraph, ['app2:build']), + }); + expect(taskSchedule.nextBatch()).toEqual({ + executorName: 'awesome-executors:app2-build', + taskGraph: removeTasksFromTaskGraph(taskGraph, [ + 'app1:build', + 'lib1:build', + ]), + }); + }); + + it('should run out of tasks when all batches are done', async () => { + await taskSchedule.scheduleNextTasks(); + taskSchedule.nextBatch(); + taskSchedule.nextBatch(); + taskSchedule.complete(['app1:build', 'lib1:build', 'app2:build']); + expect(taskSchedule.hasTasks()).toEqual(false); + }); }); }); - describe('With Batch Mode', () => { - let original; + describe('non-dependent tasks', () => { + let taskSchedule: TasksSchedule; + let taskGraph: TaskGraph; + let app1Test: Task; + let app2Test: Task; + let lib1Test: Task; + let lifeCycle: any; beforeEach(() => { - original = process.env['NX_BATCH_MODE']; - process.env['NX_BATCH_MODE'] = 'true'; - }); + app1Test = createMockTask('app1:test'); + app2Test = createMockTask('app2:test'); + lib1Test = createMockTask('lib1:test'); + + taskGraph = { + tasks: { + 'app1:test': app1Test, + 'app2:test': app2Test, + 'lib1:test': lib1Test, + }, + dependencies: { + 'app1:test': [], + 'app2:test': [], + 'lib1:test': [], + }, + roots: ['app1:test', 'app2:test', 'lib1:test'], + }; + const workspace: Partial = { + readExecutor() { + return { + schema: { + version: 2, + properties: {}, + }, + implementationFactory: jest.fn(), + batchImplementationFactory: jest.fn(), + } as any; + }, + readNxJson() { + return {}; + }, + }; + + const projectGraph: ProjectGraph = { + nodes: { + app1: { + data: { + root: 'app1', + targets: { + test: { + executor: 'awesome-executors:test', + }, + }, + }, + name: 'app1', + type: 'app', + }, + app2: { + name: 'app2', + type: 'app', + data: { + root: 'app2', + targets: { + test: { + executor: 'awesome-executors:app2-test', + }, + }, + }, + }, + lib1: { + name: 'lib1', + type: 'lib', + data: { + root: 'lib1', + targets: { + test: { + executor: 'awesome-executors:test', + }, + }, + }, + }, + }, + dependencies: { + app1: [ + { + source: 'app1', + target: 'lib1', + type: DependencyType.static, + }, + ], + app2: [ + { + source: 'app2', + target: 'lib1', + type: DependencyType.static, + }, + ], + }, + allWorkspaceFiles: [], + externalNodes: {}, + version: '5', + }; + + const hasher = { + hashTask: () => 'hash', + } as any; - afterEach(() => { - process.env['NX_BATCH_MODE'] = original; + lifeCycle = { + startTask: jest.fn(), + endTask: jest.fn(), + scheduleTask: jest.fn(), + }; + taskSchedule = new TasksSchedule( + hasher, + {}, + projectGraph, + taskGraph, + workspace as Workspaces, + { + lifeCycle, + } + ); }); - it('should schedule batches of tasks by different executors', async () => { - await taskSchedule.scheduleNextTasks(); + describe('Without Batch Mode', () => { + let original; + beforeEach(() => { + original = process.env['NX_BATCH_MODE']; + process.env['NX_BATCH_MODE'] = 'false'; + }); + + afterEach(() => { + process.env['NX_BATCH_MODE'] = original; + }); - expect(taskSchedule.nextTask()).toBeNull(); + it('should begin with no scheduled tasks', () => { + expect(taskSchedule.nextBatch()).toBeNull(); + expect(taskSchedule.nextTask()).toBeNull(); + }); - expect(taskSchedule.nextBatch()).toEqual({ - executorName: 'awesome-executors:build', - taskGraph: removeTasksFromTaskGraph(taskGraph, ['app2:build']), + it('should schedule root tasks in topological order', async () => { + await taskSchedule.scheduleNextTasks(); + expect(taskSchedule.nextTask()).toEqual(lib1Test); + expect(taskSchedule.nextTask()).toEqual(app1Test); + expect(taskSchedule.nextTask()).toEqual(app2Test); }); - expect(taskSchedule.nextBatch()).toEqual({ - executorName: 'awesome-executors:app2-build', - taskGraph: removeTasksFromTaskGraph(taskGraph, [ - 'app1:build', - 'lib1:build', - ]), + + it('should invoke lifeCycle.scheduleTask', async () => { + await taskSchedule.scheduleNextTasks(); + expect(lifeCycle.scheduleTask).toHaveBeenCalled(); + }); + + it('should run out of tasks when they are all complete', async () => { + await taskSchedule.scheduleNextTasks(); + taskSchedule.nextTask(); + taskSchedule.nextTask(); + taskSchedule.nextTask(); + taskSchedule.complete([lib1Test.id, app1Test.id, app2Test.id]); + + expect(taskSchedule.hasTasks()).toEqual(false); + }); + + it('should not schedule batches', async () => { + await taskSchedule.scheduleNextTasks(); + + expect(taskSchedule.nextTask()).not.toBeNull(); + + expect(taskSchedule.nextBatch()).toBeNull(); }); }); - it('should run out of tasks when all batches are done', async () => { - await taskSchedule.scheduleNextTasks(); - taskSchedule.nextBatch(); - taskSchedule.nextBatch(); - taskSchedule.complete(['app1:build', 'lib1:build', 'app2:build']); - expect(taskSchedule.hasTasks()).toEqual(false); + describe('With Batch Mode', () => { + let original; + beforeEach(() => { + original = process.env['NX_BATCH_MODE']; + process.env['NX_BATCH_MODE'] = 'true'; + }); + + afterEach(() => { + process.env['NX_BATCH_MODE'] = original; + }); + + it('should schedule batches of tasks by different executors', async () => { + await taskSchedule.scheduleNextTasks(); + + expect(taskSchedule.nextTask()).toBeNull(); + + expect(taskSchedule.nextBatch()).toEqual({ + executorName: 'awesome-executors:test', + taskGraph: removeTasksFromTaskGraph(taskGraph, ['app2:test']), + }); + expect(taskSchedule.nextBatch()).toEqual({ + executorName: 'awesome-executors:app2-test', + taskGraph: removeTasksFromTaskGraph(taskGraph, [ + 'app1:test', + 'lib1:test', + ]), + }); + }); + + it('should run out of tasks when all batches are done', async () => { + await taskSchedule.scheduleNextTasks(); + taskSchedule.nextBatch(); + taskSchedule.nextBatch(); + taskSchedule.complete(['app1:test', 'lib1:test', 'app2:test']); + expect(taskSchedule.hasTasks()).toEqual(false); + }); }); }); }); diff --git a/packages/nx/src/tasks-runner/tasks-schedule.ts b/packages/nx/src/tasks-runner/tasks-schedule.ts index 53c6166e68aad..345f61940e857 100644 --- a/packages/nx/src/tasks-runner/tasks-schedule.ts +++ b/packages/nx/src/tasks-runner/tasks-schedule.ts @@ -12,6 +12,8 @@ import { Task, TaskGraph } from '../config/task-graph'; import { ProjectGraph } from '../config/project-graph'; import { NxJsonConfiguration } from '../config/nx-json'; import { hashTask } from '../hasher/hash-task'; +import { findAllProjectNodeDependencies } from '../utils/project-graph-utils'; +import { reverse } from '../project-graph/operators'; export interface Batch { executorName: string; @@ -21,7 +23,7 @@ export interface Batch { export class TasksSchedule { private notScheduledTaskGraph = this.taskGraph; private reverseTaskDeps = calculateReverseDeps(this.taskGraph); - + private reverseProjectGraph = reverse(this.projectGraph); private scheduledBatches: Batch[] = []; private scheduledTasks: string[] = []; @@ -102,11 +104,28 @@ export class TasksSchedule { this.scheduledTasks = this.scheduledTasks .concat(taskId) // NOTE: sort task by most dependent on first - .sort( - (taskId1, taskId2) => + .sort((taskId1, taskId2) => { + // First compare the length of task dependencies. + const taskDifference = this.reverseTaskDeps[taskId2].length - - this.reverseTaskDeps[taskId1].length - ); + this.reverseTaskDeps[taskId1].length; + + if (taskDifference !== 0) { + return taskDifference; + } + + // Tie-breaker for tasks with equal number of task dependencies. + // Most likely tasks with no dependencies such as test + const project1 = this.taskGraph.tasks[taskId1].target.project; + const project2 = this.taskGraph.tasks[taskId2].target.project; + + return ( + findAllProjectNodeDependencies(project2, this.reverseProjectGraph) + .length - + findAllProjectNodeDependencies(project1, this.reverseProjectGraph) + .length + ); + }); } private scheduleBatches() { diff --git a/packages/nx/src/utils/project-graph-utils.ts b/packages/nx/src/utils/project-graph-utils.ts index e13966a62769b..90a88709d396c 100644 --- a/packages/nx/src/utils/project-graph-utils.ts +++ b/packages/nx/src/utils/project-graph-utils.ts @@ -113,7 +113,7 @@ export function getProjectNameFromDirPath( * @param {ProjectGraph} projectGraph * @returns {string[]} */ -function findAllProjectNodeDependencies( +export function findAllProjectNodeDependencies( parentNodeName: string, projectGraph = readCachedProjectGraph() ): string[] {