From 65199bc1e5f3d76d265f8140b7ff5471601875c1 Mon Sep 17 00:00:00 2001 From: AgentEnder Date: Tue, 19 Sep 2023 16:34:39 -0400 Subject: [PATCH] feat(core): accept various task runner options from root of nx.json --- docs/generated/devkit/NxJsonConfiguration.md | 37 +++++++ docs/generated/devkit/TargetConfiguration.md | 10 ++ docs/generated/devkit/Task.md | 9 ++ docs/generated/devkit/Workspace.md | 53 ++++++++++ packages/nx/src/config/nx-json.ts | 21 ++++ packages/nx/src/config/task-graph.ts | 5 + .../src/config/workspace-json-project-json.ts | 6 ++ packages/nx/src/daemon/client/client.ts | 1 + .../nx/src/generators/utils/deprecated.ts | 8 ++ .../nx/src/tasks-runner/create-task-graph.ts | 1 + .../nx/src/tasks-runner/run-command.spec.ts | 51 ++++++++++ packages/nx/src/tasks-runner/run-command.ts | 99 ++++++++++++++----- .../nx/src/tasks-runner/task-orchestrator.ts | 6 +- packages/nx/src/tasks-runner/utils.ts | 13 ++- packages/nx/src/utils/cache-directory.ts | 5 +- packages/nx/src/utils/nx-cloud-utils.ts | 24 +++-- 16 files changed, 305 insertions(+), 44 deletions(-) diff --git a/docs/generated/devkit/NxJsonConfiguration.md b/docs/generated/devkit/NxJsonConfiguration.md index 18c5c9ddc2f2f8..9f6ead99b22568 100644 --- a/docs/generated/devkit/NxJsonConfiguration.md +++ b/docs/generated/devkit/NxJsonConfiguration.md @@ -18,7 +18,9 @@ Nx.json configuration ### Properties +- [accessToken](../../devkit/documents/NxJsonConfiguration#accesstoken): string - [affected](../../devkit/documents/NxJsonConfiguration#affected): NxAffectedConfig +- [cacheDirectory](../../devkit/documents/NxJsonConfiguration#cachedirectory): string - [cli](../../devkit/documents/NxJsonConfiguration#cli): Object - [defaultProject](../../devkit/documents/NxJsonConfiguration#defaultproject): string - [extends](../../devkit/documents/NxJsonConfiguration#extends): string @@ -27,15 +29,26 @@ Nx.json configuration - [installation](../../devkit/documents/NxJsonConfiguration#installation): NxInstallationConfiguration - [namedInputs](../../devkit/documents/NxJsonConfiguration#namedinputs): Object - [npmScope](../../devkit/documents/NxJsonConfiguration#npmscope): string +- [parallel](../../devkit/documents/NxJsonConfiguration#parallel): number - [plugins](../../devkit/documents/NxJsonConfiguration#plugins): string[] - [pluginsConfig](../../devkit/documents/NxJsonConfiguration#pluginsconfig): Record<string, unknown> - [release](../../devkit/documents/NxJsonConfiguration#release): NxReleaseConfiguration - [targetDefaults](../../devkit/documents/NxJsonConfiguration#targetdefaults): TargetDefaults - [tasksRunnerOptions](../../devkit/documents/NxJsonConfiguration#tasksrunneroptions): Object +- [useDaemonProcess](../../devkit/documents/NxJsonConfiguration#usedaemonprocess): boolean - [workspaceLayout](../../devkit/documents/NxJsonConfiguration#workspacelayout): Object ## Properties +### accessToken + +• `Optional` **accessToken**: `string` + +If specified Nx will use nx-cloud by default with the given token. +To use a different runner that accepts an access token, define it in [tasksRunnerOptions](../../devkit/documents/Workspace#tasksrunneroptions) + +--- + ### affected • `Optional` **affected**: [`NxAffectedConfig`](../../devkit/documents/NxAffectedConfig) @@ -44,6 +57,14 @@ Default options for `nx affected` --- +### cacheDirectory + +• `Optional` **cacheDirectory**: `string` + +Changes the default location of the cache directory. + +--- + ### cli • `Optional` **cli**: `Object` @@ -148,6 +169,14 @@ NPM Scope that the workspace uses --- +### parallel + +• `Optional` **parallel**: `number` + +Specifies how many tasks are ran in parallel by Nx for the default tasks runner. + +--- + ### plugins • `Optional` **plugins**: `string`[] @@ -192,6 +221,14 @@ Available Task Runners --- +### useDaemonProcess + +• `Optional` **useDaemonProcess**: `boolean` + +Allows turning the daemon off if set to false explicitly. + +--- + ### workspaceLayout • `Optional` **workspaceLayout**: `Object` diff --git a/docs/generated/devkit/TargetConfiguration.md b/docs/generated/devkit/TargetConfiguration.md index 251878c7850c02..ae0a478bdf5226 100644 --- a/docs/generated/devkit/TargetConfiguration.md +++ b/docs/generated/devkit/TargetConfiguration.md @@ -12,6 +12,7 @@ Target's configuration ### Properties +- [cache](../../devkit/documents/TargetConfiguration#cache): boolean - [command](../../devkit/documents/TargetConfiguration#command): string - [configurations](../../devkit/documents/TargetConfiguration#configurations): Object - [defaultConfiguration](../../devkit/documents/TargetConfiguration#defaultconfiguration): string @@ -23,6 +24,15 @@ Target's configuration ## Properties +### cache + +• `Optional` **cache**: `boolean` + +Determines if Nx is able to cache a given target. +Currently only supported in `targetDefaults`. + +--- + ### command • `Optional` **command**: `string` diff --git a/docs/generated/devkit/Task.md b/docs/generated/devkit/Task.md index 389893a3f59987..bd068534e58e4e 100644 --- a/docs/generated/devkit/Task.md +++ b/docs/generated/devkit/Task.md @@ -6,6 +6,7 @@ A representation of the invocation of an Executor ### Properties +- [cache](../../devkit/documents/Task#cache): boolean - [endTime](../../devkit/documents/Task#endtime): number - [hash](../../devkit/documents/Task#hash): string - [hashDetails](../../devkit/documents/Task#hashdetails): Object @@ -17,6 +18,14 @@ A representation of the invocation of an Executor ## Properties +### cache + +• `Optional` **cache**: `boolean` + +Determines if a given task should be cacheable. + +--- + ### endTime • `Optional` **endTime**: `number` diff --git a/docs/generated/devkit/Workspace.md b/docs/generated/devkit/Workspace.md index 7f2f09e29a1070..07a89a98551655 100644 --- a/docs/generated/devkit/Workspace.md +++ b/docs/generated/devkit/Workspace.md @@ -16,7 +16,9 @@ use ProjectsConfigurations or NxJsonConfiguration ### Properties +- [accessToken](../../devkit/documents/Workspace#accesstoken): string - [affected](../../devkit/documents/Workspace#affected): NxAffectedConfig +- [cacheDirectory](../../devkit/documents/Workspace#cachedirectory): string - [cli](../../devkit/documents/Workspace#cli): Object - [defaultProject](../../devkit/documents/Workspace#defaultproject): string - [extends](../../devkit/documents/Workspace#extends): string @@ -25,17 +27,32 @@ use ProjectsConfigurations or NxJsonConfiguration - [installation](../../devkit/documents/Workspace#installation): NxInstallationConfiguration - [namedInputs](../../devkit/documents/Workspace#namedinputs): Object - [npmScope](../../devkit/documents/Workspace#npmscope): string +- [parallel](../../devkit/documents/Workspace#parallel): number - [plugins](../../devkit/documents/Workspace#plugins): string[] - [pluginsConfig](../../devkit/documents/Workspace#pluginsconfig): Record<string, unknown> - [projects](../../devkit/documents/Workspace#projects): Record<string, ProjectConfiguration> - [release](../../devkit/documents/Workspace#release): NxReleaseConfiguration - [targetDefaults](../../devkit/documents/Workspace#targetdefaults): TargetDefaults - [tasksRunnerOptions](../../devkit/documents/Workspace#tasksrunneroptions): Object +- [useDaemonProcess](../../devkit/documents/Workspace#usedaemonprocess): boolean - [version](../../devkit/documents/Workspace#version): number - [workspaceLayout](../../devkit/documents/Workspace#workspacelayout): Object ## Properties +### accessToken + +• `Optional` **accessToken**: `string` + +If specified Nx will use nx-cloud by default with the given token. +To use a different runner that accepts an access token, define it in [tasksRunnerOptions](../../devkit/documents/Workspace#tasksrunneroptions) + +#### Inherited from + +[NxJsonConfiguration](../../devkit/documents/NxJsonConfiguration).[accessToken](../../devkit/documents/NxJsonConfiguration#accesstoken) + +--- + ### affected • `Optional` **affected**: [`NxAffectedConfig`](../../devkit/documents/NxAffectedConfig) @@ -48,6 +65,18 @@ Default options for `nx affected` --- +### cacheDirectory + +• `Optional` **cacheDirectory**: `string` + +Changes the default location of the cache directory. + +#### Inherited from + +[NxJsonConfiguration](../../devkit/documents/NxJsonConfiguration).[cacheDirectory](../../devkit/documents/NxJsonConfiguration#cachedirectory) + +--- + ### cli • `Optional` **cli**: `Object` @@ -184,6 +213,18 @@ NPM Scope that the workspace uses --- +### parallel + +• `Optional` **parallel**: `number` + +Specifies how many tasks are ran in parallel by Nx for the default tasks runner. + +#### Inherited from + +[NxJsonConfiguration](../../devkit/documents/NxJsonConfiguration).[parallel](../../devkit/documents/NxJsonConfiguration#parallel) + +--- + ### plugins • `Optional` **plugins**: `string`[] @@ -260,6 +301,18 @@ Available Task Runners --- +### useDaemonProcess + +• `Optional` **useDaemonProcess**: `boolean` + +Allows turning the daemon off if set to false explicitly. + +#### Inherited from + +[NxJsonConfiguration](../../devkit/documents/NxJsonConfiguration).[useDaemonProcess](../../devkit/documents/NxJsonConfiguration#usedaemonprocess) + +--- + ### version • **version**: `number` diff --git a/packages/nx/src/config/nx-json.ts b/packages/nx/src/config/nx-json.ts index 0823f002395fa0..0b8d9aec3754a7 100644 --- a/packages/nx/src/config/nx-json.ts +++ b/packages/nx/src/config/nx-json.ts @@ -191,6 +191,27 @@ export interface NxJsonConfiguration { * **ALPHA**: Configuration for `nx release` (versioning and publishing of applications and libraries) */ release?: NxReleaseConfiguration; + + /** + * If specified Nx will use nx-cloud by default with the given token. + * To use a different runner that accepts an access token, define it in {@link tasksRunnerOptions} + */ + accessToken?: string; + + /** + * Specifies how many tasks are ran in parallel by Nx for the default tasks runner. + */ + parallel?: number; + + /** + * Changes the default location of the cache directory. + */ + cacheDirectory?: string; + + /** + * Allows turning the daemon off if set to false explicitly. + */ + useDaemonProcess?: boolean; } export function readNxJson(root: string = workspaceRoot): NxJsonConfiguration { diff --git a/packages/nx/src/config/task-graph.ts b/packages/nx/src/config/task-graph.ts index 96fb672b1176d6..00ec80f94ec280 100644 --- a/packages/nx/src/config/task-graph.ts +++ b/packages/nx/src/config/task-graph.ts @@ -66,6 +66,11 @@ export interface Task { * Unix timestamp of when a Batch Task ends **/ endTime?: number; + + /** + * Determines if a given task should be cacheable. + */ + cache?: boolean; } /** diff --git a/packages/nx/src/config/workspace-json-project-json.ts b/packages/nx/src/config/workspace-json-project-json.ts index 86a2904908a1f6..1466774b769938 100644 --- a/packages/nx/src/config/workspace-json-project-json.ts +++ b/packages/nx/src/config/workspace-json-project-json.ts @@ -181,4 +181,10 @@ export interface TargetConfiguration { * A default named configuration to use when a target configuration is not provided. */ defaultConfiguration?: string; + + /** + * Determines if Nx is able to cache a given target. + * Currently only supported in `targetDefaults`. + */ + cache?: boolean; } diff --git a/packages/nx/src/daemon/client/client.ts b/packages/nx/src/daemon/client/client.ts index 925d7d507f9226..778df66051a13e 100644 --- a/packages/nx/src/daemon/client/client.ts +++ b/packages/nx/src/daemon/client/client.ts @@ -65,6 +65,7 @@ export class DaemonClient { enabled() { if (this._enabled === undefined) { const useDaemonProcessOption = + this.nxJson.useDaemonProcess ?? this.nxJson.tasksRunnerOptions?.['default']?.options?.useDaemonProcess; const env = process.env.NX_DAEMON; diff --git a/packages/nx/src/generators/utils/deprecated.ts b/packages/nx/src/generators/utils/deprecated.ts index a49cd05057ed50..eb363c1704bd10 100644 --- a/packages/nx/src/generators/utils/deprecated.ts +++ b/packages/nx/src/generators/utils/deprecated.ts @@ -40,6 +40,10 @@ export function updateWorkspaceConfiguration( extends: ext, installation, release, + accessToken, + cacheDirectory, + parallel, + useDaemonProcess, } = workspaceConfig; const nxJson: Required = { @@ -58,6 +62,10 @@ export function updateWorkspaceConfiguration( extends: ext, installation, release, + accessToken, + cacheDirectory, + parallel, + useDaemonProcess, }; updateNxJson(tree, nxJson); diff --git a/packages/nx/src/tasks-runner/create-task-graph.ts b/packages/nx/src/tasks-runner/create-task-graph.ts index 97572012dd4f80..4676730bc45dd7 100644 --- a/packages/nx/src/tasks-runner/create-task-graph.ts +++ b/packages/nx/src/tasks-runner/create-task-graph.ts @@ -334,6 +334,7 @@ export class ProcessTasks { target: qualifiedTarget, projectRoot: project.data.root, overrides: interpolateOverrides(overrides, project.name, project.data), + cache: project.data.targets[target].cache, }; } diff --git a/packages/nx/src/tasks-runner/run-command.spec.ts b/packages/nx/src/tasks-runner/run-command.spec.ts index b835cd3ea0298c..086d61a7bae6b7 100644 --- a/packages/nx/src/tasks-runner/run-command.spec.ts +++ b/packages/nx/src/tasks-runner/run-command.spec.ts @@ -1,6 +1,7 @@ import { TasksRunner } from './tasks-runner'; import { getRunner } from './run-command'; import { NxJsonConfiguration } from '../config/nx-json'; +import { join } from 'path'; describe('getRunner', () => { let nxJson: NxJsonConfiguration; @@ -73,4 +74,54 @@ describe('getRunner', () => { expect(tasksRunner).toEqual(mockRunner); }); + + it('uses default runner when no tasksRunnerOptions are present', () => { + jest.mock(join(__dirname, './default-tasks-runner.ts'), () => mockRunner); + + const { tasksRunner } = getRunner({}, {}); + + expect(tasksRunner).toEqual(mockRunner); + }); + + it('uses nx-cloud when no tasksRunnerOptions are present and accessToken is specified', () => { + jest.mock('nx-cloud', () => mockRunner); + + const { tasksRunner } = getRunner( + {}, + { + accessToken: 'XXXX-XXX-XXXX', + } + ); + + expect(tasksRunner).toEqual(mockRunner); + }); + + it('reads options from base properties if no runner options provided', () => { + jest.mock(join(__dirname, './default-tasks-runner.ts'), () => mockRunner); + + const { runnerOptions } = getRunner( + {}, + { + cacheDirectory: '.nx/cache', + parallel: 3, + useDaemonProcess: false, + targetDefaults: { + build: { + cache: true, + }, + }, + } + ); + + expect(runnerOptions).toMatchInlineSnapshot(` + { + "cacheDirectory": ".nx/cache", + "cacheableOperations": [ + "build", + ], + "parallel": 3, + "useDaemonProcess": false, + } + `); + }); }); diff --git a/packages/nx/src/tasks-runner/run-command.ts b/packages/nx/src/tasks-runner/run-command.ts index b651dc6a754905..1fbc629363bca7 100644 --- a/packages/nx/src/tasks-runner/run-command.ts +++ b/packages/nx/src/tasks-runner/run-command.ts @@ -390,7 +390,7 @@ function shouldUseDynamicLifeCycle( if (isCI()) return false; if (outputStyle === 'static' || outputStyle === 'stream') return false; - return !tasks.find((t) => shouldStreamOutput(t, null, options)); + return !tasks.find((t) => shouldStreamOutput(t, null)); } export function getRunner( @@ -402,35 +402,82 @@ export function getRunner( } { let runner = nxArgs.runner; runner = runner || 'default'; - if (!nxJson.tasksRunnerOptions) { - throw new Error(`Could not find any runner configurations in nx.json`); + + if (runner !== 'default' && !nxJson.tasksRunnerOptions[runner]) { + throw new Error(`Could not find runner configuration for ${runner}`); } - if (nxJson.tasksRunnerOptions[runner]) { - let modulePath: string = nxJson.tasksRunnerOptions[runner].runner; - let tasksRunner; - if (modulePath) { - if (isRelativePath(modulePath)) { - modulePath = join(workspaceRoot, modulePath); - } + const modulePath: string = getTasksRunnerPath(runner, nxJson); - tasksRunner = require(modulePath); - // to support both babel and ts formats - if (tasksRunner.default) { - tasksRunner = tasksRunner.default; - } - } else { - tasksRunner = require('./default-tasks-runner').defaultTasksRunner; + let tasksRunner = require(modulePath); + // to support both babel and ts formats + if (tasksRunner.default) { + tasksRunner = tasksRunner.default; + } + + return { + tasksRunner, + runnerOptions: getRunnerOptions(runner, nxJson, nxArgs), + }; +} +function getTasksRunnerPath( + runner: string, + nxJson: NxJsonConfiguration +) { + let modulePath: string = nxJson.tasksRunnerOptions?.[runner]?.runner; + + if (modulePath) { + if (isRelativePath(modulePath)) { + return join(workspaceRoot, modulePath); } + return modulePath; + } - return { - tasksRunner, - runnerOptions: { - ...nxJson.tasksRunnerOptions[runner].options, - ...nxArgs, - }, - }; - } else { - throw new Error(`Could not find runner configuration for ${runner}`); + // No runner prop in tasks runner options, check if access token is set. + const isCloudRunner = + nxJson.accessToken || + nxJson.tasksRunnerOptions?.[runner]?.options?.accessToken; + + return isCloudRunner ? 'nx-cloud' : require.resolve('./default-tasks-runner'); +} + +function getRunnerOptions( + runner: string, + nxJson: NxJsonConfiguration, + nxArgs: NxArgs +): any { + const defaultCacheableOperations = []; + + for (const key in nxJson.targetDefaults) { + if (nxJson.targetDefaults[key].cache) { + defaultCacheableOperations.push(key); + } + } + + const result = { + ...nxJson.tasksRunnerOptions?.[runner]?.options, + ...nxArgs, + }; + + if (nxJson.accessToken) { + result.accessToken ??= nxJson.accessToken; + } + + if (nxJson.parallel) { + result.parallel ??= nxJson.parallel; + } + + if (nxJson.cacheDirectory) { + result.cacheDirectory ??= nxJson.cacheDirectory; + } + + if (defaultCacheableOperations.length) { + result.cacheableOperations ??= defaultCacheableOperations; + } + + if (nxJson.useDaemonProcess !== undefined) { + result.useDaemonProcess ??= nxJson.useDaemonProcess; } + + return result; } diff --git a/packages/nx/src/tasks-runner/task-orchestrator.ts b/packages/nx/src/tasks-runner/task-orchestrator.ts index 6b1c3c757576b2..9aa2539ccddc5b 100644 --- a/packages/nx/src/tasks-runner/task-orchestrator.ts +++ b/packages/nx/src/tasks-runner/task-orchestrator.ts @@ -282,11 +282,7 @@ export class TaskOrchestrator { try { // obtain metadata const temporaryOutputPath = this.cache.temporaryOutputPath(task); - const streamOutput = shouldStreamOutput( - task, - this.initiatingProject, - this.options - ); + const streamOutput = shouldStreamOutput(task, this.initiatingProject); const pipeOutput = await this.pipeOutputCapture(task); diff --git a/packages/nx/src/tasks-runner/utils.ts b/packages/nx/src/tasks-runner/utils.ts index 9ffd006f9cef21..722d9f3c8f048f 100644 --- a/packages/nx/src/tasks-runner/utils.ts +++ b/packages/nx/src/tasks-runner/utils.ts @@ -328,11 +328,7 @@ export function getSerializedArgsForTask(task: Task, isVerbose: boolean) { export function shouldStreamOutput( task: Task, - initiatingProject: string | null, - options: { - cacheableOperations?: string[] | null; - cacheableTargets?: string[] | null; - } + initiatingProject: string | null ): boolean { if (process.env.NX_STREAM_OUTPUT === 'true') return true; if (longRunningTask(task)) return true; @@ -347,6 +343,13 @@ export function isCacheableTask( cacheableTargets?: string[] | null; } ): boolean { + if ( + task.cache !== undefined && + process.env.NX_ALLOW_PROJECT_LEVEL_CACHE === 'true' + ) { + return task.cache; + } + const cacheable = options.cacheableOperations || options.cacheableTargets; return ( cacheable && diff --git a/packages/nx/src/utils/cache-directory.ts b/packages/nx/src/utils/cache-directory.ts index a8780b8fb879ad..2c9b04cb1ad001 100644 --- a/packages/nx/src/utils/cache-directory.ts +++ b/packages/nx/src/utils/cache-directory.ts @@ -7,7 +7,10 @@ import { workspaceRoot } from './workspace-root'; function readCacheDirectoryProperty(root: string): string | undefined { try { const nxJson = readJsonFile(join(root, 'nx.json')); - return nxJson.tasksRunnerOptions.default.options.cacheDirectory; + return ( + nxJson.cacheDirectory ?? + nxJson.tasksRunnerOptions.default.options.cacheDirectory + ); } catch { return undefined; } diff --git a/packages/nx/src/utils/nx-cloud-utils.ts b/packages/nx/src/utils/nx-cloud-utils.ts index 3cb4545feb90df..ded0d079992e50 100644 --- a/packages/nx/src/utils/nx-cloud-utils.ts +++ b/packages/nx/src/utils/nx-cloud-utils.ts @@ -1,19 +1,29 @@ import { readNxJson } from '../config/configuration'; -export function isNxCloudUsed() { - const nxJson = readNxJson(); - return Object.values(nxJson.tasksRunnerOptions).find( - (r) => r.runner == '@nrwl/nx-cloud' || r.runner == 'nx-cloud' +export function isNxCloudUsed(nxJson = readNxJson()) { + return ( + !!nxJson.accessToken || + Object.values(nxJson.tasksRunnerOptions ?? {}).find( + (r) => r.runner == '@nrwl/nx-cloud' || r.runner == 'nx-cloud' + ) ); } export function getNxCloudUrl(): string { const taskRunner = isNxCloudUsed(); if (!taskRunner) throw new Error('nx-cloud runner not find in nx.json'); - return taskRunner.options.url || 'https://nx.app'; + return ( + (typeof taskRunner === 'object' ? taskRunner.options.url : null) ?? + 'https://nx.app' + ); } export function getNxCloudToken(): string { - const taskRunner = isNxCloudUsed(); + const nxJson = readNxJson(); + const taskRunner = isNxCloudUsed(nxJson); + if (!taskRunner) throw new Error('nx-cloud runner not find in nx.json'); - return taskRunner.options.accessToken; + + return typeof taskRunner === 'object' + ? taskRunner.options.accessToken + : nxJson.accessToken; }