From dc7ad937859e14e812eb164c11a6c71cc08ed6d2 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 2 Feb 2023 23:53:16 +0800 Subject: [PATCH 01/55] feat: AsyncOperationQueue supports remote executing --- .../logic/operations/AsyncOperationQueue.ts | 87 ++++++++++++++++--- .../logic/operations/ConsoleTimelinePlugin.ts | 4 + .../src/logic/operations/OperationStatus.ts | 11 ++- .../test/AsyncOperationQueue.test.ts | 56 +++++++++++- .../AsyncOperationQueue.test.ts.snap | 2 +- 5 files changed, 147 insertions(+), 13 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 006df21d840..8dff124792f 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -5,7 +5,7 @@ import { OperationExecutionRecord } from './OperationExecutionRecord'; import { OperationStatus } from './OperationStatus'; /** - * Implmentation of the async iteration protocol for a collection of IOperation objects. + * Implementation of the async iteration protocol for a collection of IOperation objects. * The async iterator will wait for an operation to be ready for execution, or terminate if there are no more operations. * * @remarks @@ -18,6 +18,10 @@ export class AsyncOperationQueue { private readonly _queue: OperationExecutionRecord[]; private readonly _pendingIterators: ((result: IteratorResult) => void)[]; + private readonly _totalOperations: number; + + private _completedOperations: number; + private _isDone: boolean; /** * @param operations - The set of operations to be executed @@ -29,6 +33,9 @@ export class AsyncOperationQueue public constructor(operations: Iterable, sortFn: IOperationSortFunction) { this._queue = computeTopologyAndSort(operations, sortFn); this._pendingIterators = []; + this._totalOperations = this._queue.length; + this._isDone = false; + this._completedOperations = 0; } /** @@ -49,6 +56,17 @@ export class AsyncOperationQueue return promise; } + /** + * Set a callback to be invoked when one operation is completed. + * If all operations are completed, set the queue to done, resolve all pending iterators in next cycle. + */ + public complete(): void { + this._completedOperations++; + if (this._completedOperations === this._totalOperations) { + this._isDone = true; + } + } + /** * Routes ready operations with 0 dependencies to waiting iterators. Normally invoked as part of `next()`, but * if the caller does not update operation dependencies prior to calling `next()`, may need to be invoked manually. @@ -56,19 +74,42 @@ export class AsyncOperationQueue public assignOperations(): void { const { _queue: queue, _pendingIterators: waitingIterators } = this; + if (this._isDone) { + for (const resolveAsyncIterator of waitingIterators.splice(0)) { + resolveAsyncIterator({ + value: undefined, + done: true + }); + } + return; + } + // By iterating in reverse order we do less array shuffling when removing operations for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { const operation: OperationExecutionRecord = queue[i]; - if (operation.status === OperationStatus.Blocked) { + if ( + operation.status === OperationStatus.Blocked || + operation.status === OperationStatus.Success || + operation.status === OperationStatus.SuccessWithWarning || + operation.status === OperationStatus.FromCache || + operation.status === OperationStatus.NoOp || + operation.status === OperationStatus.Failure + ) { // It shouldn't be on the queue, remove it queue.splice(i, 1); + } else if ( + operation.status === OperationStatus.RemotePending || + operation.status === OperationStatus.RemoteExecuting + ) { + // This operation is not ready to execute yet, but it may become ready later + // next one plz :) + continue; } else if (operation.status !== OperationStatus.Ready) { // Sanity check throw new Error(`Unexpected status "${operation.status}" for queued operation: ${operation.name}`); } else if (operation.dependencies.size === 0) { // This task is ready to process, hand it to the iterator. - queue.splice(i, 1); // Needs to have queue semantics, otherwise tools that iterate it get confused waitingIterators.shift()!({ value: operation, @@ -78,13 +119,26 @@ export class AsyncOperationQueue // Otherwise operation is still waiting } - if (queue.length === 0) { - // Queue is empty, flush - for (const resolveAsyncIterator of waitingIterators.splice(0)) { - resolveAsyncIterator({ - value: undefined, - done: true - }); + if (waitingIterators.length > 0) { + // cycle through the queue again to find the next operation that is executed remotely + for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { + const operation: OperationExecutionRecord = queue[i]; + + if (operation.status === OperationStatus.RemoteExecuting) { + // try to attempt to get the lock again + waitingIterators.shift()!({ + value: operation, + done: false + }); + } + } + + if (waitingIterators.length > 0) { + // Queue is not empty, but no operations are ready to process + // Pause for a second and start over + setTimeout(() => { + this.assignOperations(); + }, 1000); } } } @@ -96,6 +150,19 @@ export class AsyncOperationQueue public [Symbol.asyncIterator](): AsyncIterator { return this; } + + /** + * Recursively sets the status of all operations that consume the specified operation. + */ + public static setOperationConsumersStatusRecursively( + operation: OperationExecutionRecord, + operationStatus: OperationStatus + ): void { + for (const consumer of operation.consumers) { + consumer.status = operationStatus; + AsyncOperationQueue.setOperationConsumersStatusRecursively(consumer, operationStatus); + } + } } export interface IOperationSortFunction { diff --git a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts index aff052574b4..ac5cb13baa9 100644 --- a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts @@ -72,6 +72,8 @@ const TIMELINE_WIDTH: number = 109; const TIMELINE_CHART_SYMBOLS: Record = { [OperationStatus.Ready]: '?', [OperationStatus.Executing]: '?', + [OperationStatus.RemoteExecuting]: '?', + [OperationStatus.RemotePending]: '?', [OperationStatus.Success]: '#', [OperationStatus.SuccessWithWarning]: '!', [OperationStatus.Failure]: '!', @@ -87,6 +89,8 @@ const TIMELINE_CHART_SYMBOLS: Record = { const TIMELINE_CHART_COLORIZER: Record string> = { [OperationStatus.Ready]: colors.yellow, [OperationStatus.Executing]: colors.yellow, + [OperationStatus.RemoteExecuting]: colors.yellow, + [OperationStatus.RemotePending]: colors.yellow, [OperationStatus.Success]: colors.green, [OperationStatus.SuccessWithWarning]: colors.yellow, [OperationStatus.Failure]: colors.red, diff --git a/libraries/rush-lib/src/logic/operations/OperationStatus.ts b/libraries/rush-lib/src/logic/operations/OperationStatus.ts index 3fdff28ba84..43047308cd1 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStatus.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStatus.ts @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. /** @@ -14,6 +14,15 @@ export enum OperationStatus { * The Operation is currently executing */ Executing = 'EXECUTING', + /** + * The Operation is currently executing by a remote process + */ + RemoteExecuting = 'REMOTE EXECUTING', + /** + * The Operation is pending because one of the upstream operation is + * executing by a remote process + */ + RemotePending = 'REMOTE PENDING', /** * The Operation completed successfully and did not write to standard output */ diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 1270b19c262..3c030e619d4 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -5,6 +5,7 @@ import { Operation } from '../Operation'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from '../OperationExecutionRecord'; import { MockOperationRunner } from './MockOperationRunner'; import { AsyncOperationQueue, IOperationSortFunction } from '../AsyncOperationQueue'; +import { OperationStatus } from '../OperationStatus'; function addDependency(consumer: OperationExecutionRecord, dependency: OperationExecutionRecord): void { consumer.dependencies.add(dependency); @@ -40,6 +41,8 @@ describe(AsyncOperationQueue.name, () => { for (const consumer of operation.consumers) { consumer.dependencies.delete(operation); } + operation.status = OperationStatus.Success; + queue.complete(); } expect(actualOrder).toEqual(expectedOrder); @@ -68,7 +71,7 @@ describe(AsyncOperationQueue.name, () => { expect(actualOrder).toEqual(expectedOrder); }); - it('detects cyles', async () => { + it('detects cycles', async () => { const operations = [createRecord('a'), createRecord('b'), createRecord('c'), createRecord('d')]; addDependency(operations[0], operations[2]); @@ -124,6 +127,8 @@ describe(AsyncOperationQueue.name, () => { } --concurrency; + operation.status = OperationStatus.Success; + queue.complete(); } }) ); @@ -132,4 +137,53 @@ describe(AsyncOperationQueue.name, () => { expect(actualConcurrency.get(operation)).toEqual(operationConcurrency); } }); + + it('handles remote executed operations', async () => { + const operations = [ + createRecord('a'), + createRecord('b'), + createRecord('c'), + createRecord('d'), + createRecord('e') + ]; + + addDependency(operations[2], operations[1]); + addDependency(operations[3], operations[1]); + addDependency(operations[4], operations[1]); + addDependency(operations[3], operations[2]); + addDependency(operations[4], operations[3]); + + // b remote executing -> a -> b (remote executed) -> c -> d -> e + const expectedOrder: string[] = ['b', 'a', 'b', 'c', 'd', 'e']; + + const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, nullSort); + + const actualOrder: string[] = []; + let remoteExecuted: boolean = false; + for await (const operation of queue) { + actualOrder.push(operation.name); + + if (operation === operations[1]) { + if (!remoteExecuted) { + operations[1].status = OperationStatus.RemoteExecuting; + AsyncOperationQueue.setOperationConsumersStatusRecursively( + operations[1], + OperationStatus.RemotePending + ); + // remote executed operation is finished later + remoteExecuted = true; + continue; + } else { + AsyncOperationQueue.setOperationConsumersStatusRecursively(operations[1], OperationStatus.Ready); + } + } + for (const consumer of operation.consumers) { + consumer.dependencies.delete(operation); + } + operation.status = OperationStatus.Success; + queue.complete(); + } + + expect(actualOrder).toEqual(expectedOrder); + }); }); diff --git a/libraries/rush-lib/src/logic/operations/test/__snapshots__/AsyncOperationQueue.test.ts.snap b/libraries/rush-lib/src/logic/operations/test/__snapshots__/AsyncOperationQueue.test.ts.snap index 7a4c086efd4..c1642f68137 100644 --- a/libraries/rush-lib/src/logic/operations/test/__snapshots__/AsyncOperationQueue.test.ts.snap +++ b/libraries/rush-lib/src/logic/operations/test/__snapshots__/AsyncOperationQueue.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`AsyncOperationQueue detects cyles 1`] = ` +exports[`AsyncOperationQueue detects cycles 1`] = ` "A cyclic dependency was encountered: a -> c From 7b8a521472235ca8b1e17a2572d65d46dd63f5c7 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 9 Feb 2023 19:52:55 +0800 Subject: [PATCH 02/55] feat: cobuildlock & cobuildconfiguration --- common/reviews/api/rush-lib.api.md | 29 ++++ .../rush-lib/src/api/CobuildConfiguration.ts | 111 +++++++++++++ .../src/api/EnvironmentConfiguration.ts | 31 ++++ .../cli/scriptActions/PhasedScriptAction.ts | 8 + libraries/rush-lib/src/index.ts | 4 +- libraries/rush-lib/src/logic/RushConstants.ts | 11 ++ .../src/logic/buildCache/ProjectBuildCache.ts | 12 +- .../rush-lib/src/logic/cobuild/CobuildLock.ts | 115 ++++++++++++++ .../src/logic/cobuild/ICobuildLockProvider.ts | 31 ++++ .../logic/operations/AsyncOperationQueue.ts | 26 ++- .../logic/operations/ConsoleTimelinePlugin.ts | 4 +- .../src/logic/operations/IOperationRunner.ts | 7 + .../operations/OperationExecutionManager.ts | 11 +- .../operations/OperationExecutionRecord.ts | 13 +- .../src/logic/operations/OperationStatus.ts | 11 +- .../src/logic/operations/RunnerWatcher.ts | 56 +++++++ .../logic/operations/ShellOperationRunner.ts | 148 +++++++++++++++--- .../operations/ShellOperationRunnerPlugin.ts | 2 + .../test/AsyncOperationQueue.test.ts | 6 - .../src/pluginFramework/PhasedCommandHooks.ts | 8 +- .../src/pluginFramework/RushSession.ts | 31 +++- .../rush-lib/src/schemas/cobuild.schema.json | 39 +++++ 22 files changed, 652 insertions(+), 62 deletions(-) create mode 100644 libraries/rush-lib/src/api/CobuildConfiguration.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/CobuildLock.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts create mode 100644 libraries/rush-lib/src/logic/operations/RunnerWatcher.ts create mode 100644 libraries/rush-lib/src/schemas/cobuild.schema.json diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 9aa7b02ce99..242dffd798d 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -95,6 +95,23 @@ export class ChangeManager { // @beta (undocumented) export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) => ICloudBuildCacheProvider; +// @beta +export class CobuildConfiguration { + readonly cobuildEnabled: boolean; + // Warning: (ae-forgotten-export) The symbol "ICobuildLockProvider" needs to be exported by the entry point index.d.ts + readonly cobuildLockProvider: ICobuildLockProvider; + // (undocumented) + get contextId(): string; + // (undocumented) + static getCobuildConfigFilePath(rushConfiguration: RushConfiguration): string; + static tryLoadAsync(terminal: ITerminal, rushConfiguration: RushConfiguration, rushSession: RushSession): Promise; +} + +// Warning: (ae-forgotten-export) The symbol "ICobuildJson" needs to be exported by the entry point index.d.ts +// +// @beta (undocumented) +export type CobuildLockProviderFactory = (cobuildJson: ICobuildJson) => ICobuildLockProvider; + // @public export class CommonVersionsConfiguration { readonly allowedAlternativeVersions: Map>; @@ -149,6 +166,7 @@ export class EnvironmentConfiguration { static get buildCacheCredential(): string | undefined; static get buildCacheEnabled(): boolean | undefined; static get buildCacheWriteAllowed(): boolean | undefined; + static get cobuildEnabled(): boolean | undefined; // Warning: (ae-forgotten-export) The symbol "IEnvironment" needs to be exported by the entry point index.d.ts // // @internal @@ -173,6 +191,7 @@ export enum EnvironmentVariableNames { RUSH_BUILD_CACHE_CREDENTIAL = "RUSH_BUILD_CACHE_CREDENTIAL", RUSH_BUILD_CACHE_ENABLED = "RUSH_BUILD_CACHE_ENABLED", RUSH_BUILD_CACHE_WRITE_ALLOWED = "RUSH_BUILD_CACHE_WRITE_ALLOWED", + RUSH_COBUILD_ENABLED = "RUSH_COBUILD_ENABLED", RUSH_DEPLOY_TARGET_FOLDER = "RUSH_DEPLOY_TARGET_FOLDER", RUSH_GIT_BINARY_PATH = "RUSH_GIT_BINARY_PATH", RUSH_GLOBAL_FOLDER = "RUSH_GLOBAL_FOLDER", @@ -258,6 +277,7 @@ export interface IConfigurationEnvironmentVariable { // @alpha export interface ICreateOperationsContext { readonly buildCacheConfiguration: BuildCacheConfiguration | undefined; + readonly cobuildConfiguration: CobuildConfiguration | undefined; readonly customParameters: ReadonlyMap; readonly isIncrementalBuildAllowed: boolean; readonly isInitial: boolean; @@ -432,6 +452,7 @@ export interface IOperationRunnerContext { // @internal _operationMetadataManager?: _OperationMetadataManager; quietMode: boolean; + status: OperationStatus; stdioSummarizer: StdioSummarizer; stopwatch: IStopwatchResult; } @@ -672,7 +693,9 @@ export enum OperationStatus { Failure = "FAILURE", FromCache = "FROM CACHE", NoOp = "NO OP", + Queued = "Queued", Ready = "READY", + RemoteExecuting = "REMOTE EXECUTING", Skipped = "SKIPPED", Success = "SUCCESS", SuccessWithWarning = "SUCCESS WITH WARNINGS" @@ -954,6 +977,8 @@ export class RushConstants { static readonly buildCommandName: string; static readonly bulkCommandKind: 'bulk'; static readonly changeFilesFolderName: string; + static readonly cobuildFilename: string; + static readonly cobuildLockVersion: number; static readonly commandLineFilename: string; static readonly commonFolderName: string; static readonly commonVersionsFilename: string; @@ -1026,12 +1051,16 @@ export class RushSession { // (undocumented) getCloudBuildCacheProviderFactory(cacheProviderName: string): CloudBuildCacheProviderFactory | undefined; // (undocumented) + getCobuildLockProviderFactory(cobuildLockProviderName: string): CobuildLockProviderFactory | undefined; + // (undocumented) getLogger(name: string): ILogger; // (undocumented) readonly hooks: RushLifecycleHooks; // (undocumented) registerCloudBuildCacheProviderFactory(cacheProviderName: string, factory: CloudBuildCacheProviderFactory): void; // (undocumented) + registerCobuildLockProviderFactory(cobuildLockProviderName: string, factory: CobuildLockProviderFactory): void; + // (undocumented) get terminalProvider(): ITerminalProvider; } diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts new file mode 100644 index 00000000000..3f9d2a12dac --- /dev/null +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as path from 'path'; +import { FileSystem, ITerminal, JsonFile, JsonSchema } from '@rushstack/node-core-library'; +import schemaJson from '../schemas/cobuild.schema.json'; +import { EnvironmentConfiguration } from './EnvironmentConfiguration'; +import { CobuildLockProviderFactory, RushSession } from '../pluginFramework/RushSession'; +import { RushConstants } from '../logic/RushConstants'; + +import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; +import type { RushConfiguration } from './RushConfiguration'; + +export interface ICobuildJson { + cobuildEnabled: boolean; + cobuildLockProvider: string; + cobuildContextIdPattern?: string; +} + +export interface ICobuildConfigurationOptions { + cobuildJson: ICobuildJson; + rushConfiguration: RushConfiguration; + rushSession: RushSession; +} + +/** + * Use this class to load and save the "common/config/rush/cobuild.json" config file. + * This file provides configuration options for the Rush Cobuild feature. + * @beta + */ +export class CobuildConfiguration { + private static _jsonSchema: JsonSchema = JsonSchema.fromLoadedObject(schemaJson); + + /** + * Indicates whether the cobuild feature is enabled. + * Typically it is enabled in the cobuild.json config file. + */ + public readonly cobuildEnabled: boolean; + /** + * Method to calculate the cobuild context id + * FIXME: + */ + // public readonly getCacheEntryId: GetCacheEntryIdFunction; + public readonly cobuildLockProvider: ICobuildLockProvider; + + private constructor(options: ICobuildConfigurationOptions) { + this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? options.cobuildJson.cobuildEnabled; + + const { cobuildJson } = options; + + const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = + options.rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); + if (!cobuildLockProviderFactory) { + throw new Error(`Unexpected cobuild lock provider: ${cobuildJson.cobuildLockProvider}`); + } + this.cobuildLockProvider = cobuildLockProviderFactory(cobuildJson); + } + + /** + * Attempts to load the cobuild.json data from the standard file path `common/config/rush/cobuild.json`. + * If the file has not been created yet, then undefined is returned. + */ + public static async tryLoadAsync( + terminal: ITerminal, + rushConfiguration: RushConfiguration, + rushSession: RushSession + ): Promise { + const jsonFilePath: string = CobuildConfiguration.getCobuildConfigFilePath(rushConfiguration); + if (!FileSystem.exists(jsonFilePath)) { + return undefined; + } + return await CobuildConfiguration._loadAsync(jsonFilePath, terminal, rushConfiguration, rushSession); + } + + public static getCobuildConfigFilePath(rushConfiguration: RushConfiguration): string { + return path.resolve(rushConfiguration.commonRushConfigFolder, RushConstants.cobuildFilename); + } + private static async _loadAsync( + jsonFilePath: string, + terminal: ITerminal, + rushConfiguration: RushConfiguration, + rushSession: RushSession + ): Promise { + const cobuildJson: ICobuildJson = await JsonFile.loadAndValidateAsync( + jsonFilePath, + CobuildConfiguration._jsonSchema + ); + + // FIXME: + // let getCacheEntryId: GetCacheEntryIdFunction; + // try { + // getCacheEntryId = CacheEntryId.parsePattern(cobuildJson.cacheEntryNamePattern); + // } catch (e) { + // terminal.writeErrorLine( + // `Error parsing cache entry name pattern "${cobuildJson.cacheEntryNamePattern}": ${e}` + // ); + // throw new AlreadyReportedError(); + // } + + return new CobuildConfiguration({ + cobuildJson, + rushConfiguration, + rushSession + }); + } + + public get contextId(): string { + // FIXME: hardcode + return '123'; + } +} diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index 3bd511504b7..269b571d983 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -143,6 +143,17 @@ export enum EnvironmentVariableNames { */ RUSH_BUILD_CACHE_WRITE_ALLOWED = 'RUSH_BUILD_CACHE_WRITE_ALLOWED', + /** + * Setting this environment variable overrides the value of `cobuildEnabled` in the `cobuild.json` + * configuration file. + * + * @remarks + * Specify `1` to enable the cobuild or `0` to disable it. + * + * If there is no build cache configured, then this environment variable is ignored. + */ + RUSH_COBUILD_ENABLED = 'RUSH_COBUILD_ENABLED', + /** * Explicitly specifies the path for the Git binary that is invoked by certain Rush operations. */ @@ -196,6 +207,8 @@ export class EnvironmentConfiguration { private static _buildCacheWriteAllowed: boolean | undefined; + private static _cobuildEnabled: boolean | undefined; + private static _gitBinaryPath: string | undefined; private static _tarBinaryPath: string | undefined; @@ -293,6 +306,15 @@ export class EnvironmentConfiguration { return EnvironmentConfiguration._buildCacheWriteAllowed; } + /** + * If set, enables or disables the cobuild feature. + * See {@link EnvironmentVariableNames.RUSH_COBUILD_ENABLED} + */ + public static get cobuildEnabled(): boolean | undefined { + EnvironmentConfiguration._ensureValidated(); + return EnvironmentConfiguration._cobuildEnabled; + } + /** * Allows the git binary path to be explicitly provided. * See {@link EnvironmentVariableNames.RUSH_GIT_BINARY_PATH} @@ -423,6 +445,15 @@ export class EnvironmentConfiguration { break; } + case EnvironmentVariableNames.RUSH_COBUILD_ENABLED: { + EnvironmentConfiguration._cobuildEnabled = + EnvironmentConfiguration.parseBooleanEnvironmentVariable( + EnvironmentVariableNames.RUSH_COBUILD_ENABLED, + value + ); + break; + } + case EnvironmentVariableNames.RUSH_GIT_BINARY_PATH: { EnvironmentConfiguration._gitBinaryPath = value; break; diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 67c14ff22a5..99eb4c503b6 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -37,6 +37,7 @@ import { IExecutionResult } from '../../logic/operations/IOperationExecutionResu import { OperationResultSummarizerPlugin } from '../../logic/operations/OperationResultSummarizerPlugin'; import type { ITelemetryOperationResult } from '../../logic/Telemetry'; import { parseParallelism } from '../parsing/ParseParallelism'; +import { CobuildConfiguration } from '../../api/CobuildConfiguration'; /** * Constructor parameters for PhasedScriptAction. @@ -299,12 +300,18 @@ export class PhasedScriptAction extends BaseScriptAction { const changedProjectsOnly: boolean = !!this._changedProjectsOnly?.value; let buildCacheConfiguration: BuildCacheConfiguration | undefined; + let cobuildConfiguration: CobuildConfiguration | undefined; if (!this._disableBuildCache) { buildCacheConfiguration = await BuildCacheConfiguration.tryLoadAsync( terminal, this.rushConfiguration, this.rushSession ); + cobuildConfiguration = await CobuildConfiguration.tryLoadAsync( + terminal, + this.rushConfiguration, + this.rushSession + ); } const projectSelection: Set = @@ -325,6 +332,7 @@ export class PhasedScriptAction extends BaseScriptAction { const projectChangeAnalyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); const initialCreateOperationsContext: ICreateOperationsContext = { buildCacheConfiguration, + cobuildConfiguration, customParameters: customParametersByName, isIncrementalBuildAllowed: this._isIncrementalBuildAllowed, isInitial: true, diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index c98c9d20d56..18fbed045f0 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -31,6 +31,7 @@ export { } from './logic/pnpm/PnpmOptionsConfiguration'; export { BuildCacheConfiguration } from './api/BuildCacheConfiguration'; +export { CobuildConfiguration } from './api/CobuildConfiguration'; export { GetCacheEntryIdFunction, IGenerateCacheEntryIdOptions } from './logic/buildCache/CacheEntryId'; export { FileSystemBuildCacheProvider, @@ -98,7 +99,8 @@ export { OperationStatus } from './logic/operations/OperationStatus'; export { RushSession, IRushSessionOptions, - CloudBuildCacheProviderFactory + CloudBuildCacheProviderFactory, + CobuildLockProviderFactory } from './pluginFramework/RushSession'; export { diff --git a/libraries/rush-lib/src/logic/RushConstants.ts b/libraries/rush-lib/src/logic/RushConstants.ts index bdbdff0b3a7..04e5bcb40ae 100644 --- a/libraries/rush-lib/src/logic/RushConstants.ts +++ b/libraries/rush-lib/src/logic/RushConstants.ts @@ -182,6 +182,17 @@ export class RushConstants { */ public static readonly buildCacheVersion: number = 1; + /** + * Cobuild configuration file. + */ + public static readonly cobuildFilename: string = 'cobuild.json'; + + /** + * Cobuild version number, incremented when the logic to create cobuild lock changes. + * Changing this ensures that lock generated by an old version will no longer access as a cobuild lock. + */ + public static readonly cobuildLockVersion: number = 1; + /** * Per-project configuration filename. */ diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index 658bd3217b0..f8a7ea21a07 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -74,6 +74,10 @@ export class ProjectBuildCache { return ProjectBuildCache._tarUtilityPromise; } + public get cacheId(): string | undefined { + return this._cacheId; + } + public static async tryGetProjectBuildCache( options: IProjectBuildCacheOptions ): Promise { @@ -133,8 +137,8 @@ export class ProjectBuildCache { } } - public async tryRestoreFromCacheAsync(terminal: ITerminal): Promise { - const cacheId: string | undefined = this._cacheId; + public async tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise { + const cacheId: string | undefined = specifiedCacheId || this._cacheId; if (!cacheId) { terminal.writeWarningLine('Unable to get cache ID. Ensure Git is installed.'); return false; @@ -213,13 +217,13 @@ export class ProjectBuildCache { return restoreSuccess; } - public async trySetCacheEntryAsync(terminal: ITerminal): Promise { + public async trySetCacheEntryAsync(terminal: ITerminal, specifiedCacheId?: string): Promise { if (!this._cacheWriteEnabled) { // Skip writing local and cloud build caches, without any noise return true; } - const cacheId: string | undefined = this._cacheId; + const cacheId: string | undefined = specifiedCacheId || this._cacheId; if (!cacheId) { terminal.writeWarningLine('Unable to get cache ID. Ensure Git is installed.'); return false; diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts new file mode 100644 index 00000000000..6746c3d3b52 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { RushConstants } from '../RushConstants'; + +import type { ITerminal } from '@rushstack/node-core-library'; +import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import type { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; +import { OperationStatus } from '../operations/OperationStatus'; + +export interface ICobuildLockOptions { + cobuildConfiguration: CobuildConfiguration; + projectBuildCache: ProjectBuildCache; + terminal: ITerminal; +} + +export interface ICobuildCompletedState { + status: OperationStatus.Success | OperationStatus.SuccessWithWarning | OperationStatus.Failure; + cacheId: string; +} + +const KEY_SEPARATOR: string = ':'; +const COMPLETED_STATE_SEPARATOR: string = ';'; + +export class CobuildLock { + public readonly options: ICobuildLockOptions; + public readonly lockKey: string; + public readonly completedKey: string; + + public readonly projectBuildCache: ProjectBuildCache; + public readonly cobuildConfiguration: CobuildConfiguration; + + public constructor(options: ICobuildLockOptions) { + this.options = options; + const { cobuildConfiguration, projectBuildCache } = options; + this.projectBuildCache = projectBuildCache; + this.cobuildConfiguration = cobuildConfiguration; + + const { contextId } = cobuildConfiguration; + const { cacheId } = projectBuildCache; + // Example: cobuild:v1:::lock + this.lockKey = ['cobuild', `v${RushConstants.cobuildLockVersion}`, contextId, cacheId, 'lock'].join( + KEY_SEPARATOR + ); + // Example: cobuild:v1:::completed + this.completedKey = [ + 'cobuild', + `v${RushConstants.cobuildLockVersion}`, + contextId, + cacheId, + 'completed' + ].join(KEY_SEPARATOR); + } + + public async setCompletedStateAsync(state: ICobuildCompletedState): Promise { + const { terminal } = this.options; + const serializedState: string = this._serializeCompletedState(state); + terminal.writeDebugLine(`Set completed state by key ${this.completedKey}: ${serializedState}`); + await this.cobuildConfiguration.cobuildLockProvider.setCompletedStateAsync({ + key: this.completedKey, + value: serializedState, + terminal + }); + } + + public async getCompletedStateAsync(): Promise { + const { terminal } = this.options; + const state: string | undefined = + await this.cobuildConfiguration.cobuildLockProvider.getCompletedStateAsync({ + key: this.completedKey, + terminal + }); + terminal.writeDebugLine(`Get completed state by key ${this.completedKey}: ${state}`); + if (!state) { + return; + } + return this._deserializeCompletedState(state); + } + + public async tryAcquireLockAsync(): Promise { + const { terminal } = this.options; + // const result: boolean = true; + // const result: boolean = false; + // const result: boolean = Math.random() > 0.5; + const acquireLockResult: boolean = await this.cobuildConfiguration.cobuildLockProvider.acquireLockAsync({ + lockKey: this.lockKey, + terminal + }); + terminal.writeDebugLine(`Acquired lock for ${this.lockKey}, result: ${acquireLockResult}`); + return acquireLockResult; + } + + public async releaseLockAsync(): Promise { + const { terminal } = this.options; + terminal.writeDebugLine(`Released lock for ${this.lockKey}`); + return; + } + + public async renewLockAsync(): Promise { + const { terminal } = this.options; + terminal.writeDebugLine(`Renewed lock for ${this.lockKey}`); + return; + } + + private _serializeCompletedState(state: ICobuildCompletedState): string { + // Example: SUCCESS;1234567890 + // Example: FAILURE;1234567890 + return `${state.status}${COMPLETED_STATE_SEPARATOR}${state.cacheId}`; + } + + private _deserializeCompletedState(state: string): ICobuildCompletedState | undefined { + const [status, cacheId] = state.split(COMPLETED_STATE_SEPARATOR); + return { status: status as ICobuildCompletedState['status'], cacheId }; + } +} diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts new file mode 100644 index 00000000000..4735fc1542f --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { ITerminal } from '@rushstack/node-core-library'; + +export interface ILockOptions { + lockKey: string; + terminal: ITerminal; +} + +export interface IGetCompletedStateOptions { + key: string; + terminal: ITerminal; +} + +export interface ISetCompletedStateOptions { + key: string; + value: string; + terminal: ITerminal; +} + +/** + * @beta + */ +export interface ICobuildLockProvider { + acquireLockAsync(options: ILockOptions): Promise; + renewLockAsync(options: ILockOptions): Promise; + releaseLockAsync(options: ILockOptions): Promise; + setCompletedStateAsync(options: ISetCompletedStateOptions): Promise; + getCompletedStateAsync(options: IGetCompletedStateOptions): Promise; +} diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 8dff124792f..ffea8cc71e4 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -84,6 +84,10 @@ export class AsyncOperationQueue return; } + queue.forEach((q) => { + console.log(q.name, q.status); + }); + // By iterating in reverse order we do less array shuffling when removing operations for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { const operation: OperationExecutionRecord = queue[i]; @@ -99,9 +103,13 @@ export class AsyncOperationQueue // It shouldn't be on the queue, remove it queue.splice(i, 1); } else if ( - operation.status === OperationStatus.RemotePending || - operation.status === OperationStatus.RemoteExecuting + operation.status === OperationStatus.Queued || + operation.status === OperationStatus.Executing ) { + // This operation is currently executing + // next one plz :) + continue; + } else if (operation.status === OperationStatus.RemoteExecuting) { // This operation is not ready to execute yet, but it may become ready later // next one plz :) continue; @@ -111,6 +119,7 @@ export class AsyncOperationQueue } else if (operation.dependencies.size === 0) { // This task is ready to process, hand it to the iterator. // Needs to have queue semantics, otherwise tools that iterate it get confused + operation.status = OperationStatus.Queued; waitingIterators.shift()!({ value: operation, done: false @@ -150,19 +159,6 @@ export class AsyncOperationQueue public [Symbol.asyncIterator](): AsyncIterator { return this; } - - /** - * Recursively sets the status of all operations that consume the specified operation. - */ - public static setOperationConsumersStatusRecursively( - operation: OperationExecutionRecord, - operationStatus: OperationStatus - ): void { - for (const consumer of operation.consumers) { - consumer.status = operationStatus; - AsyncOperationQueue.setOperationConsumersStatusRecursively(consumer, operationStatus); - } - } } export interface IOperationSortFunction { diff --git a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts index ac5cb13baa9..31c4b56a48e 100644 --- a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts @@ -71,9 +71,9 @@ const TIMELINE_WIDTH: number = 109; */ const TIMELINE_CHART_SYMBOLS: Record = { [OperationStatus.Ready]: '?', + [OperationStatus.Queued]: '?', [OperationStatus.Executing]: '?', [OperationStatus.RemoteExecuting]: '?', - [OperationStatus.RemotePending]: '?', [OperationStatus.Success]: '#', [OperationStatus.SuccessWithWarning]: '!', [OperationStatus.Failure]: '!', @@ -88,9 +88,9 @@ const TIMELINE_CHART_SYMBOLS: Record = { */ const TIMELINE_CHART_COLORIZER: Record string> = { [OperationStatus.Ready]: colors.yellow, + [OperationStatus.Queued]: colors.yellow, [OperationStatus.Executing]: colors.yellow, [OperationStatus.RemoteExecuting]: colors.yellow, - [OperationStatus.RemotePending]: colors.yellow, [OperationStatus.Success]: colors.green, [OperationStatus.SuccessWithWarning]: colors.yellow, [OperationStatus.Failure]: colors.red, diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index 22062e82b71..5428e6c0c2a 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -40,6 +40,13 @@ export interface IOperationRunnerContext { * Object used to track elapsed time. */ stopwatch: IStopwatchResult; + /** + * The current execution status of an operation. Operations start in the 'ready' state, + * but can be 'blocked' if an upstream operation failed. It is 'executing' when + * the operation is executing. Once execution is complete, it is either 'success' or + * 'failure'. + */ + status: OperationStatus; } /** diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index a84a0cb67d6..d61c6c196fb 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -189,6 +189,11 @@ export class OperationExecutionManager { record: OperationExecutionRecord ) => { this._onOperationComplete(record); + + if (record.status !== OperationStatus.RemoteExecuting) { + // If the operation was not remote, then we can notify queue that it is complete + executionQueue.complete(); + } }; await Async.forEachAsync( @@ -328,8 +333,10 @@ export class OperationExecutionManager { item.runner.isSkipAllowed = false; } - // Remove this operation from the dependencies, to unblock the scheduler - item.dependencies.delete(record); + if (status !== OperationStatus.RemoteExecuting) { + // Remove this operation from the dependencies, to unblock the scheduler + item.dependencies.delete(record); + } } } } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 11cd208e8bd..f20f05fe25f 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -138,6 +138,11 @@ export class OperationExecutionRecord implements IOperationRunnerContext { try { this.status = await this.runner.executeAsync(this); + + if (this.status === OperationStatus.RemoteExecuting) { + this.stopwatch.reset(); + } + // Delegate global state reporting onResult(this); } catch (error) { @@ -146,9 +151,11 @@ export class OperationExecutionRecord implements IOperationRunnerContext { // Delegate global state reporting onResult(this); } finally { - this._collatedWriter?.close(); - this.stdioSummarizer.close(); - this.stopwatch.stop(); + if (this.status !== OperationStatus.RemoteExecuting) { + this._collatedWriter?.close(); + this.stdioSummarizer.close(); + this.stopwatch.stop(); + } } } } diff --git a/libraries/rush-lib/src/logic/operations/OperationStatus.ts b/libraries/rush-lib/src/logic/operations/OperationStatus.ts index 43047308cd1..18e7204e78c 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStatus.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStatus.ts @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. /** @@ -10,6 +10,10 @@ export enum OperationStatus { * The Operation is on the queue, ready to execute (but may be waiting for dependencies) */ Ready = 'READY', + /** + * The Operation is Queued + */ + Queued = 'Queued', /** * The Operation is currently executing */ @@ -18,11 +22,6 @@ export enum OperationStatus { * The Operation is currently executing by a remote process */ RemoteExecuting = 'REMOTE EXECUTING', - /** - * The Operation is pending because one of the upstream operation is - * executing by a remote process - */ - RemotePending = 'REMOTE PENDING', /** * The Operation completed successfully and did not write to standard output */ diff --git a/libraries/rush-lib/src/logic/operations/RunnerWatcher.ts b/libraries/rush-lib/src/logic/operations/RunnerWatcher.ts new file mode 100644 index 00000000000..e5823454cf6 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/RunnerWatcher.ts @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +export type ICallbackFn = () => Promise | void; + +export interface IRunnerWatcherOptions { + interval: number; +} + +/** + * A help class to run callbacks in a loop with a specified interval. + * + * @beta + */ +export class RunnerWatcher { + private _callbacks: ICallbackFn[]; + private _interval: number; + private _timeoutId: NodeJS.Timeout | undefined; + private _isRunning: boolean; + + public constructor(options: IRunnerWatcherOptions) { + this._callbacks = []; + this._interval = options.interval; + this._isRunning = false; + } + + public addCallback(callback: ICallbackFn): void { + if (this._isRunning) { + throw new Error('Can not add callback while watcher is running'); + } + this._callbacks.push(callback); + } + + public start(): void { + if (this._timeoutId) { + throw new Error('Watcher already started'); + } + if (this._callbacks.length === 0) { + return; + } + this._isRunning = true; + this._timeoutId = setTimeout(() => { + this._callbacks.forEach((callback) => callback()); + this._timeoutId = undefined; + this.start(); + }, this._interval); + } + + public stop(): void { + if (this._timeoutId) { + clearTimeout(this._timeoutId); + this._timeoutId = undefined; + this._isRunning = false; + } + } +} diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 2dca3e01b37..fedab5e5fe2 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -35,11 +35,14 @@ import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvid import { RushConstants } from '../RushConstants'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; import { OperationMetadataManager } from './OperationMetadataManager'; +import { RunnerWatcher } from './RunnerWatcher'; +import { CobuildLock, ICobuildCompletedState } from '../cobuild/CobuildLock'; import type { RushConfiguration } from '../../api/RushConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { ProjectChangeAnalyzer, IRawRepoState } from '../ProjectChangeAnalyzer'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; +import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; import type { IPhase } from '../../api/CommandLineConfiguration'; export interface IProjectDeps { @@ -51,6 +54,7 @@ export interface IOperationRunnerOptions { rushProject: RushConfigurationProject; rushConfiguration: RushConfiguration; buildCacheConfiguration: BuildCacheConfiguration | undefined; + cobuildConfiguration: CobuildConfiguration | undefined; commandToRun: string; isIncrementalBuildAllowed: boolean; projectChangeAnalyzer: ProjectChangeAnalyzer; @@ -95,6 +99,7 @@ export class ShellOperationRunner implements IOperationRunner { private readonly _phase: IPhase; private readonly _rushConfiguration: RushConfiguration; private readonly _buildCacheConfiguration: BuildCacheConfiguration | undefined; + private readonly _cobuildConfiguration: CobuildConfiguration | undefined; private readonly _commandName: string; private readonly _commandToRun: string; private readonly _isCacheReadAllowed: boolean; @@ -108,6 +113,7 @@ export class ShellOperationRunner implements IOperationRunner { * undefined === we didn't create one because the feature is not enabled */ private _projectBuildCache: ProjectBuildCache | undefined | UNINITIALIZED = UNINITIALIZED; + private _cobuildLock: CobuildLock | undefined | UNINITIALIZED = UNINITIALIZED; public constructor(options: IOperationRunnerOptions) { const { phase } = options; @@ -117,6 +123,7 @@ export class ShellOperationRunner implements IOperationRunner { this._phase = phase; this._rushConfiguration = options.rushConfiguration; this._buildCacheConfiguration = options.buildCacheConfiguration; + this._cobuildConfiguration = options.cobuildConfiguration; this._commandName = phase.name; this._commandToRun = options.commandToRun; this._isCacheReadAllowed = options.isIncrementalBuildAllowed; @@ -150,6 +157,10 @@ export class ShellOperationRunner implements IOperationRunner { context.collatedWriter.terminal, this._logFilenameIdentifier ); + const runnerWatcher: RunnerWatcher = new RunnerWatcher({ + interval: 10 * 1000 + // interval: 1000 + }); try { const removeColorsTransform: TextRewriterTransform = new TextRewriterTransform({ @@ -247,6 +258,16 @@ export class ShellOperationRunner implements IOperationRunner { }); } + // Try to acquire the cobuild lock + let cobuildLock: CobuildLock | undefined; + if (this._cobuildConfiguration?.cobuildEnabled) { + const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync( + terminal, + trackedFiles + ); + cobuildLock = await this._tryGetCobuildLockAsync(terminal, projectBuildCache); + } + // If possible, we want to skip this operation -- either by restoring it from the // cache, if caching is enabled, or determining that the project // is unchanged (using the older incremental execution logic). These two approaches, @@ -264,8 +285,30 @@ export class ShellOperationRunner implements IOperationRunner { // false if a dependency wasn't able to be skipped. // let buildCacheReadAttempted: boolean = false; - if (this._isCacheReadAllowed) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ + if (cobuildLock) { + // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to + // the build cache once completed, but does not retrieve them (since the "incremental" + // flag is disabled). However, we still need a cobuild to be able to retrieve a finished + // build from another cobuild in this case. + const cobuildCompletedState: ICobuildCompletedState | undefined = + await cobuildLock.getCompletedStateAsync(); + if (cobuildCompletedState) { + const { status, cacheId } = cobuildCompletedState; + + const restoreFromCacheSuccess: boolean | undefined = + await cobuildLock.projectBuildCache.tryRestoreFromCacheAsync(terminal, cacheId); + + if (restoreFromCacheSuccess) { + // Restore the original state of the operation without cache + await context._operationStateFile?.tryRestoreAsync(); + if (cobuildCompletedState) { + return cobuildCompletedState.status; + } + return status; + } + } + } else if (this._isCacheReadAllowed) { + const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync( terminal, trackedProjectFiles, operationMetadataManager: context._operationMetadataManager @@ -317,8 +360,25 @@ export class ShellOperationRunner implements IOperationRunner { return OperationStatus.Success; } + if (this.isCacheWriteAllowed && cobuildLock) { + const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); + if (acquireSuccess) { + if (context.status === OperationStatus.RemoteExecuting) { + // This operation is used to marked remote executing, now change it to executing + context.status = OperationStatus.Executing; + } + runnerWatcher.addCallback(async () => { + await cobuildLock?.renewLockAsync(); + }); + } else { + // failed to acquire the lock, mark current operation to remote executing + return OperationStatus.RemoteExecuting; + } + } + // Run the operation terminal.writeLine('Invoking: ' + this._commandToRun); + runnerWatcher.start(); const subProcess: child_process.ChildProcess = Utilities.executeLifecycleCommandAsync( this._commandToRun, @@ -366,6 +426,37 @@ export class ShellOperationRunner implements IOperationRunner { } ); + let setCompletedStatePromise: Promise | undefined; + let setCacheEntryPromise: Promise | undefined; + if (cobuildLock && this.isCacheWriteAllowed) { + const { projectBuildCache } = cobuildLock; + const cacheId: string | undefined = projectBuildCache.cacheId; + const contextId: string = cobuildLock.cobuildConfiguration.contextId; + + if (cacheId) { + const finalCacheId: string = + status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; + switch (status) { + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: + case OperationStatus.Failure: { + setCompletedStatePromise = cobuildLock + .setCompletedStateAsync({ + status, + cacheId: finalCacheId + }) + .then(() => { + return cobuildLock?.releaseLockAsync(); + }); + setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( + terminal, + finalCacheId + ); + } + } + } + } + const taskIsSuccessful: boolean = status === OperationStatus.Success || (status === OperationStatus.SuccessWithWarning && @@ -373,9 +464,10 @@ export class ShellOperationRunner implements IOperationRunner { !!this._rushConfiguration.experimentsConfiguration.configuration .buildCacheWithAllowWarningsInSuccessfulBuild); + let writeProjectStatePromise: Promise | undefined; if (taskIsSuccessful && projectDeps) { // Write deps on success. - const writeProjectStatePromise: Promise = JsonFile.saveAsync(projectDeps, currentDepsPath, { + writeProjectStatePromise = JsonFile.saveAsync(projectDeps, currentDepsPath, { ensureFolderExists: true }); @@ -389,24 +481,23 @@ export class ShellOperationRunner implements IOperationRunner { // If the command is successful, we can calculate project hash, and no dependencies were skipped, // write a new cache entry. - const setCacheEntryPromise: Promise | undefined = this.isCacheWriteAllowed - ? ( - await this._tryGetProjectBuildCacheAsync({ - terminal, - trackedProjectFiles, - operationMetadataManager: context._operationMetadataManager - }) - )?.trySetCacheEntryAsync(terminal) - : undefined; - - const [, cacheWriteSuccess] = await Promise.all([writeProjectStatePromise, setCacheEntryPromise]); - - if (terminalProvider.hasErrors) { - status = OperationStatus.Failure; - } else if (cacheWriteSuccess === false) { - status = OperationStatus.SuccessWithWarning; + if (!setCacheEntryPromise && this.isCacheWriteAllowed) { + setCacheEntryPromise = ( + await this._tryGetProjectBuildCacheAsync(terminal, trackedFiles) + )?.trySetCacheEntryAsync(terminal); } } + const [, cacheWriteSuccess] = await Promise.all([ + writeProjectStatePromise, + setCacheEntryPromise, + setCompletedStatePromise + ]); + + if (terminalProvider.hasErrors) { + status = OperationStatus.Failure; + } else if (cacheWriteSuccess === false) { + status = OperationStatus.SuccessWithWarning; + } normalizeNewlineTransform.close(); @@ -419,6 +510,7 @@ export class ShellOperationRunner implements IOperationRunner { return status; } finally { projectLogWritable.close(); + runnerWatcher.stop(); } } @@ -513,6 +605,24 @@ export class ShellOperationRunner implements IOperationRunner { return this._projectBuildCache; } + + private async _tryGetCobuildLockAsync( + terminal: ITerminal, + projectBuildCache: ProjectBuildCache | undefined + ): Promise { + if (this._cobuildLock === UNINITIALIZED) { + this._cobuildLock = undefined; + + if (projectBuildCache && this._cobuildConfiguration && this._cobuildConfiguration.cobuildEnabled) { + this._cobuildLock = new CobuildLock({ + cobuildConfiguration: this._cobuildConfiguration, + projectBuildCache: projectBuildCache, + terminal + }); + } + } + return this._cobuildLock; + } } /** diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts index d070fc1588b..1d1a425feef 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts @@ -31,6 +31,7 @@ function createShellOperations( ): Set { const { buildCacheConfiguration, + cobuildConfiguration, isIncrementalBuildAllowed, phaseSelection: selectedPhases, projectChangeAnalyzer, @@ -79,6 +80,7 @@ function createShellOperations( if (commandToRun) { operation.runner = new ShellOperationRunner({ buildCacheConfiguration, + cobuildConfiguration, commandToRun: commandToRun || '', displayName, isIncrementalBuildAllowed, diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 3c030e619d4..41af93fe6ea 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -166,15 +166,9 @@ describe(AsyncOperationQueue.name, () => { if (operation === operations[1]) { if (!remoteExecuted) { operations[1].status = OperationStatus.RemoteExecuting; - AsyncOperationQueue.setOperationConsumersStatusRecursively( - operations[1], - OperationStatus.RemotePending - ); // remote executed operation is finished later remoteExecuted = true; continue; - } else { - AsyncOperationQueue.setOperationConsumersStatusRecursively(operations[1], OperationStatus.Ready); } } for (const consumer of operation.consumers) { diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 1c2e893b67d..237fcfd7eff 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -8,10 +8,10 @@ import type { BuildCacheConfiguration } from '../api/BuildCacheConfiguration'; import type { IPhase } from '../api/CommandLineConfiguration'; import type { RushConfiguration } from '../api/RushConfiguration'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; - import type { Operation } from '../logic/operations/Operation'; import type { ProjectChangeAnalyzer } from '../logic/ProjectChangeAnalyzer'; -import { IExecutionResult } from '../logic/operations/IOperationExecutionResult'; +import type { IExecutionResult } from '../logic/operations/IOperationExecutionResult'; +import type { CobuildConfiguration } from '../api/CobuildConfiguration'; /** * A plugin that interacts with a phased commands. @@ -33,6 +33,10 @@ export interface ICreateOperationsContext { * The configuration for the build cache, if the feature is enabled. */ readonly buildCacheConfiguration: BuildCacheConfiguration | undefined; + /** + * The configuration for the cobuild, if cobuild feature and build cache feature are both enabled. + */ + readonly cobuildConfiguration: CobuildConfiguration | undefined; /** * The set of custom parameters for the executing command. * Maps from the `longName` field in command-line.json to the parser configuration in ts-command-line. diff --git a/libraries/rush-lib/src/pluginFramework/RushSession.ts b/libraries/rush-lib/src/pluginFramework/RushSession.ts index aa644ccc4d5..d01a16f533a 100644 --- a/libraries/rush-lib/src/pluginFramework/RushSession.ts +++ b/libraries/rush-lib/src/pluginFramework/RushSession.ts @@ -2,11 +2,14 @@ // See LICENSE in the project root for license information. import { InternalError, ITerminalProvider } from '@rushstack/node-core-library'; -import { IBuildCacheJson } from '../api/BuildCacheConfiguration'; -import { ICloudBuildCacheProvider } from '../logic/buildCache/ICloudBuildCacheProvider'; import { ILogger, ILoggerOptions, Logger } from './logging/Logger'; import { RushLifecycleHooks } from './RushLifeCycle'; +import type { IBuildCacheJson } from '../api/BuildCacheConfiguration'; +import type { ICloudBuildCacheProvider } from '../logic/buildCache/ICloudBuildCacheProvider'; +import type { ICobuildJson } from '../api/CobuildConfiguration'; +import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; + /** * @beta */ @@ -20,12 +23,18 @@ export interface IRushSessionOptions { */ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) => ICloudBuildCacheProvider; +/** + * @beta + */ +export type CobuildLockProviderFactory = (cobuildJson: ICobuildJson) => ICobuildLockProvider; + /** * @beta */ export class RushSession { private readonly _options: IRushSessionOptions; private readonly _cloudBuildCacheProviderFactories: Map = new Map(); + private readonly _cobuildLockProviderFactories: Map = new Map(); public readonly hooks: RushLifecycleHooks; @@ -68,4 +77,22 @@ export class RushSession { ): CloudBuildCacheProviderFactory | undefined { return this._cloudBuildCacheProviderFactories.get(cacheProviderName); } + + public registerCobuildLockProviderFactory( + cobuildLockProviderName: string, + factory: CobuildLockProviderFactory + ): void { + if (this._cobuildLockProviderFactories.has(cobuildLockProviderName)) { + throw new Error( + `A cobuild lock provider factory for ${cobuildLockProviderName} has already been registered` + ); + } + this._cobuildLockProviderFactories.set(cobuildLockProviderName, factory); + } + + public getCobuildLockProviderFactory( + cobuildLockProviderName: string + ): CobuildLockProviderFactory | undefined { + return this._cobuildLockProviderFactories.get(cobuildLockProviderName); + } } diff --git a/libraries/rush-lib/src/schemas/cobuild.schema.json b/libraries/rush-lib/src/schemas/cobuild.schema.json new file mode 100644 index 00000000000..5b13a4ec631 --- /dev/null +++ b/libraries/rush-lib/src/schemas/cobuild.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Configuration for Rush's cobuild.", + "description": "For use with the Rush tool, this file provides configuration options for cobuild feature. See http://rushjs.io for details.", + "definitions": { + "anything": { + "type": ["array", "boolean", "integer", "number", "object", "string"], + "items": { + "$ref": "#/definitions/anything" + } + } + }, + "type": "object", + "allOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["cobuildEnabled", "cobuildLockProvider"], + "properties": { + "$schema": { + "description": "Part of the JSON Schema standard, this optional keyword declares the URL of the schema that the file conforms to. Editors may download the schema and use it to perform syntax highlighting.", + "type": "string" + }, + "cobuildEnabled": { + "description": "Set this to true to enable the cobuild feature.", + "type": "boolean" + }, + "cobuildLockProvider": { + "description": "Specify the cobuild lock provider to use", + "type": "string" + }, + "cobuildContextIdPattern": { + "type": "string", + "description": "Setting this property overrides the cobuild context ID." + } + } + } + ] +} From 2382a94e5a75f53d7fcbdcb2df194c5882be9cff Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 10 Feb 2023 11:47:39 +0800 Subject: [PATCH 03/55] chore --- libraries/rush-lib/src/api/EnvironmentConfiguration.ts | 2 +- libraries/rush-lib/src/index.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index 269b571d983..345f98f5fd1 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -150,7 +150,7 @@ export enum EnvironmentVariableNames { * @remarks * Specify `1` to enable the cobuild or `0` to disable it. * - * If there is no build cache configured, then this environment variable is ignored. + * If there is no cobuild configured, then this environment variable is ignored. */ RUSH_COBUILD_ENABLED = 'RUSH_COBUILD_ENABLED', diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index 18fbed045f0..3b6022658c4 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -118,6 +118,7 @@ export { IRushPluginConfigurationBase as _IRushPluginConfigurationBase } from '. export { ILogger } from './pluginFramework/logging/Logger'; export { ICloudBuildCacheProvider } from './logic/buildCache/ICloudBuildCacheProvider'; +export { ICobuildLockProvider } from './logic/cobuild/ICobuildLockProvider'; export { ICredentialCacheOptions, ICredentialCacheEntry, CredentialCache } from './logic/CredentialCache'; From fedf61ad6e0d1bf1bcf255655d3b8e86bd3697e5 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 10 Feb 2023 23:45:52 +0800 Subject: [PATCH 04/55] fix: async queue --- .../rush-lib/src/logic/operations/AsyncOperationQueue.ts | 4 ---- .../src/logic/operations/test/AsyncOperationQueue.test.ts | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index ffea8cc71e4..8bd625da4e2 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -84,10 +84,6 @@ export class AsyncOperationQueue return; } - queue.forEach((q) => { - console.log(q.name, q.status); - }); - // By iterating in reverse order we do less array shuffling when removing operations for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { const operation: OperationExecutionRecord = queue[i]; diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 41af93fe6ea..212c7c2edd3 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -66,6 +66,8 @@ describe(AsyncOperationQueue.name, () => { for (const consumer of operation.consumers) { consumer.dependencies.delete(operation); } + operation.status = OperationStatus.Success; + queue.complete(); } expect(actualOrder).toEqual(expectedOrder); From 1678c2c79d3b297a70c2022a5bfae68d3cacf385 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Sat, 11 Feb 2023 00:11:22 +0800 Subject: [PATCH 05/55] feat: registry of cobuild lock provider & redis cobuild plugin --- .../rush/nonbrowser-approved-packages.json | 4 + common/config/rush/pnpm-lock.yaml | 42 ++++++ common/config/rush/repo-state.json | 2 +- common/reviews/api/rush-lib.api.md | 34 ++++- .../api/rush-redis-cobuild-plugin.api.md | 29 +++++ libraries/rush-lib/src/index.ts | 6 +- .../rush-lib/src/logic/cobuild/CobuildLock.ts | 91 ++++--------- .../src/logic/cobuild/ICobuildLockProvider.ts | 41 +++--- .../rush-redis-cobuild-plugin/.eslintrc.js | 10 ++ .../rush-redis-cobuild-plugin/.npmignore | 32 +++++ .../rush-redis-cobuild-plugin/LICENSE | 24 ++++ .../rush-redis-cobuild-plugin/README.md | 9 ++ .../config/api-extractor.json | 16 +++ .../config/jest.config.json | 3 + .../rush-redis-cobuild-plugin/config/rig.json | 7 + .../rush-redis-cobuild-plugin/package.json | 34 +++++ .../rush-plugin-manifest.json | 11 ++ .../src/RedisCobuildLockProvider.ts | 121 ++++++++++++++++++ .../src/RushRedisCobuildPlugin.ts | 40 ++++++ .../rush-redis-cobuild-plugin/src/index.ts | 7 + .../src/schemas/redis-config.schema.json | 70 ++++++++++ .../src/test/RedisCobuildLockProvider.test.ts | 93 ++++++++++++++ .../RedisCobuildLockProvider.test.ts.snap | 5 + .../rush-redis-cobuild-plugin/tsconfig.json | 8 ++ rush.json | 6 + 25 files changed, 659 insertions(+), 86 deletions(-) create mode 100644 common/reviews/api/rush-redis-cobuild-plugin.api.md create mode 100644 rush-plugins/rush-redis-cobuild-plugin/.eslintrc.js create mode 100644 rush-plugins/rush-redis-cobuild-plugin/.npmignore create mode 100644 rush-plugins/rush-redis-cobuild-plugin/LICENSE create mode 100644 rush-plugins/rush-redis-cobuild-plugin/README.md create mode 100644 rush-plugins/rush-redis-cobuild-plugin/config/api-extractor.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/config/jest.config.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/config/rig.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/package.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/rush-plugin-manifest.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/index.ts create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap create mode 100644 rush-plugins/rush-redis-cobuild-plugin/tsconfig.json diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index d28a686b68e..0bc0e61f0eb 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -74,6 +74,10 @@ "name": "@pnpm/logger", "allowedCategories": [ "libraries" ] }, + { + "name": "@redis/client", + "allowedCategories": [ "libraries" ] + }, { "name": "@rushstack/debug-certificate-manager", "allowedCategories": [ "libraries" ] diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 2a1a2e1f46e..50f93c9f999 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -2416,6 +2416,29 @@ importers: '@types/heft-jest': 1.0.1 '@types/node': 14.18.36 + ../../rush-plugins/rush-redis-cobuild-plugin: + specifiers: + '@microsoft/rush-lib': workspace:* + '@redis/client': ~1.5.5 + '@rushstack/eslint-config': workspace:* + '@rushstack/heft': workspace:* + '@rushstack/heft-node-rig': workspace:* + '@rushstack/node-core-library': workspace:* + '@rushstack/rush-sdk': workspace:* + '@types/heft-jest': 1.0.1 + '@types/node': 14.18.36 + dependencies: + '@redis/client': 1.5.5 + '@rushstack/node-core-library': link:../../libraries/node-core-library + '@rushstack/rush-sdk': link:../../libraries/rush-sdk + devDependencies: + '@microsoft/rush-lib': link:../../libraries/rush-lib + '@rushstack/eslint-config': link:../../eslint/eslint-config + '@rushstack/heft': link:../../apps/heft + '@rushstack/heft-node-rig': link:../../rigs/heft-node-rig + '@types/heft-jest': 1.0.1 + '@types/node': 14.18.36 + ../../rush-plugins/rush-serve-plugin: specifiers: '@rushstack/debug-certificate-manager': workspace:* @@ -6444,6 +6467,15 @@ packages: react: 16.13.1 dev: true + /@redis/client/1.5.5: + resolution: {integrity: sha512-fuMnpDYSjT5JXR9rrCW1YWA4L8N/9/uS4ImT3ZEC/hcaQRI1D/9FvwjriRj1UvepIgzZXthFVKMNRzP/LNL7BQ==} + engines: {node: '>=14'} + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + dev: false + /@reduxjs/toolkit/1.8.6_qfynotfwlyrsyq662adyrweaoe: resolution: {integrity: sha512-4Ia/Loc6WLmdSOzi7k5ff7dLK8CgG2b8aqpLsCAJhazAzGdp//YBUSaj0ceW6a3kDBDNRrq5CRwyCS0wBiL1ig==} peerDependencies: @@ -11236,6 +11268,11 @@ packages: engines: {node: '>=6'} dev: true + /cluster-key-slot/1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + dev: false + /cmd-extension/1.0.2: resolution: {integrity: sha512-iWDjmP8kvsMdBmLTHxFaqXikO8EdFRDfim7k6vUHglY/2xJ5jLrPsnQGijdfp4U+sr/BeecG0wKm02dSIAeQ1g==} engines: {node: '>=10'} @@ -14076,6 +14113,11 @@ packages: loader-utils: 1.4.2 dev: false + /generic-pool/3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + dev: false + /gensync/1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index 9d0e76159ca..f31d8076457 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "b7c7c11ce97089924eb38eb913f8062e986ae06a", + "pnpmShrinkwrapHash": "b40972486600f4a66b11abdd57d7ff9a0032c357", "preferredVersionsHash": "5222ca779ae69ebfd201e39c17f48ce9eaf8c3c2" } diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 242dffd798d..cac66ad13c1 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -98,7 +98,6 @@ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) = // @beta export class CobuildConfiguration { readonly cobuildEnabled: boolean; - // Warning: (ae-forgotten-export) The symbol "ICobuildLockProvider" needs to be exported by the entry point index.d.ts readonly cobuildLockProvider: ICobuildLockProvider; // (undocumented) get contextId(): string; @@ -263,6 +262,39 @@ export interface ICloudBuildCacheProvider { updateCachedCredentialInteractiveAsync(terminal: ITerminal): Promise; } +// @beta (undocumented) +export interface ICobuildCompletedState { + cacheId: string; + // (undocumented) + status: OperationStatus.Success | OperationStatus.SuccessWithWarning | OperationStatus.Failure; +} + +// @beta (undocumented) +export interface ICobuildContext { + // (undocumented) + cacheId: string; + // (undocumented) + contextId: string; + // (undocumented) + terminal: ITerminal; + // (undocumented) + version: number; +} + +// @beta (undocumented) +export interface ICobuildLockProvider { + // (undocumented) + acquireLockAsync(context: ICobuildContext): Promise; + // (undocumented) + getCompletedStateAsync(context: ICobuildContext): Promise; + // (undocumented) + releaseLockAsync(context: ICobuildContext): Promise; + // (undocumented) + renewLockAsync(context: ICobuildContext): Promise; + // (undocumented) + setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; +} + // @public export interface IConfigurationEnvironment { [environmentVariableName: string]: IConfigurationEnvironmentVariable; diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md new file mode 100644 index 00000000000..6a73c0c36bc --- /dev/null +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -0,0 +1,29 @@ +## API Report File for "@rushstack/rush-redis-cobuild-plugin" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import type { IRushPlugin } from '@rushstack/rush-sdk'; +import type { RedisClientOptions } from '@redis/client'; +import type { RushConfiguration } from '@rushstack/rush-sdk'; +import type { RushSession } from '@rushstack/rush-sdk'; + +// @public +export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { +} + +// @public (undocumented) +class RushRedisCobuildPlugin implements IRushPlugin { + // Warning: (ae-forgotten-export) The symbol "IRushRedisCobuildPluginOptions" needs to be exported by the entry point index.d.ts + constructor(options: IRushRedisCobuildPluginOptions); + // (undocumented) + apply(rushSession: RushSession, rushConfiguration: RushConfiguration): void; + // (undocumented) + pluginName: string; +} +export default RushRedisCobuildPlugin; + +// (No @packageDocumentation comment for this package) + +``` diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index 3b6022658c4..55550568353 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -118,7 +118,11 @@ export { IRushPluginConfigurationBase as _IRushPluginConfigurationBase } from '. export { ILogger } from './pluginFramework/logging/Logger'; export { ICloudBuildCacheProvider } from './logic/buildCache/ICloudBuildCacheProvider'; -export { ICobuildLockProvider } from './logic/cobuild/ICobuildLockProvider'; +export { + ICobuildLockProvider, + ICobuildContext, + ICobuildCompletedState +} from './logic/cobuild/ICobuildLockProvider'; export { ICredentialCacheOptions, ICredentialCacheEntry, CredentialCache } from './logic/CredentialCache'; diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index 6746c3d3b52..9343640b573 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -1,12 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import { InternalError, ITerminal } from '@rushstack/node-core-library'; import { RushConstants } from '../RushConstants'; -import type { ITerminal } from '@rushstack/node-core-library'; import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; import type { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; -import { OperationStatus } from '../operations/OperationStatus'; +import type { OperationStatus } from '../operations/OperationStatus'; +import type { ICobuildContext } from './ICobuildLockProvider'; export interface ICobuildLockOptions { cobuildConfiguration: CobuildConfiguration; @@ -19,97 +20,55 @@ export interface ICobuildCompletedState { cacheId: string; } -const KEY_SEPARATOR: string = ':'; -const COMPLETED_STATE_SEPARATOR: string = ';'; - export class CobuildLock { - public readonly options: ICobuildLockOptions; - public readonly lockKey: string; - public readonly completedKey: string; - public readonly projectBuildCache: ProjectBuildCache; public readonly cobuildConfiguration: CobuildConfiguration; + private _cobuildContext: ICobuildContext; + public constructor(options: ICobuildLockOptions) { - this.options = options; - const { cobuildConfiguration, projectBuildCache } = options; + const { cobuildConfiguration, projectBuildCache, terminal } = options; this.projectBuildCache = projectBuildCache; this.cobuildConfiguration = cobuildConfiguration; const { contextId } = cobuildConfiguration; const { cacheId } = projectBuildCache; - // Example: cobuild:v1:::lock - this.lockKey = ['cobuild', `v${RushConstants.cobuildLockVersion}`, contextId, cacheId, 'lock'].join( - KEY_SEPARATOR - ); - // Example: cobuild:v1:::completed - this.completedKey = [ - 'cobuild', - `v${RushConstants.cobuildLockVersion}`, + + if (!cacheId) { + // This should never happen + throw new InternalError(`Cache id is require for cobuild lock`); + } + + this._cobuildContext = { + terminal, contextId, cacheId, - 'completed' - ].join(KEY_SEPARATOR); + version: RushConstants.cobuildLockVersion + }; } public async setCompletedStateAsync(state: ICobuildCompletedState): Promise { - const { terminal } = this.options; - const serializedState: string = this._serializeCompletedState(state); - terminal.writeDebugLine(`Set completed state by key ${this.completedKey}: ${serializedState}`); - await this.cobuildConfiguration.cobuildLockProvider.setCompletedStateAsync({ - key: this.completedKey, - value: serializedState, - terminal - }); + await this.cobuildConfiguration.cobuildLockProvider.setCompletedStateAsync(this._cobuildContext, state); } public async getCompletedStateAsync(): Promise { - const { terminal } = this.options; - const state: string | undefined = - await this.cobuildConfiguration.cobuildLockProvider.getCompletedStateAsync({ - key: this.completedKey, - terminal - }); - terminal.writeDebugLine(`Get completed state by key ${this.completedKey}: ${state}`); - if (!state) { - return; - } - return this._deserializeCompletedState(state); + const state: ICobuildCompletedState | undefined = + await this.cobuildConfiguration.cobuildLockProvider.getCompletedStateAsync(this._cobuildContext); + return state; } public async tryAcquireLockAsync(): Promise { - const { terminal } = this.options; - // const result: boolean = true; - // const result: boolean = false; - // const result: boolean = Math.random() > 0.5; - const acquireLockResult: boolean = await this.cobuildConfiguration.cobuildLockProvider.acquireLockAsync({ - lockKey: this.lockKey, - terminal - }); - terminal.writeDebugLine(`Acquired lock for ${this.lockKey}, result: ${acquireLockResult}`); + const acquireLockResult: boolean = await this.cobuildConfiguration.cobuildLockProvider.acquireLockAsync( + this._cobuildContext + ); return acquireLockResult; } public async releaseLockAsync(): Promise { - const { terminal } = this.options; - terminal.writeDebugLine(`Released lock for ${this.lockKey}`); - return; + await this.cobuildConfiguration.cobuildLockProvider.releaseLockAsync(this._cobuildContext); } public async renewLockAsync(): Promise { - const { terminal } = this.options; - terminal.writeDebugLine(`Renewed lock for ${this.lockKey}`); - return; - } - - private _serializeCompletedState(state: ICobuildCompletedState): string { - // Example: SUCCESS;1234567890 - // Example: FAILURE;1234567890 - return `${state.status}${COMPLETED_STATE_SEPARATOR}${state.cacheId}`; - } - - private _deserializeCompletedState(state: string): ICobuildCompletedState | undefined { - const [status, cacheId] = state.split(COMPLETED_STATE_SEPARATOR); - return { status: status as ICobuildCompletedState['status'], cacheId }; + await this.cobuildConfiguration.cobuildLockProvider.renewLockAsync(this._cobuildContext); } } diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index 4735fc1542f..b95327373b6 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -1,31 +1,38 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { ITerminal } from '@rushstack/node-core-library'; +import type { ITerminal } from '@rushstack/node-core-library'; +import type { OperationStatus } from '../operations/OperationStatus'; -export interface ILockOptions { - lockKey: string; - terminal: ITerminal; -} - -export interface IGetCompletedStateOptions { - key: string; +/** + * @beta + */ +export interface ICobuildContext { + contextId: string; + cacheId: string; + version: number; terminal: ITerminal; } -export interface ISetCompletedStateOptions { - key: string; - value: string; - terminal: ITerminal; +/** + * @beta + */ +export interface ICobuildCompletedState { + status: OperationStatus.Success | OperationStatus.SuccessWithWarning | OperationStatus.Failure; + /** + * Completed state points to the cache id that was used to store the build cache. + * Note: Cache failed builds in a separate cache id + */ + cacheId: string; } /** * @beta */ export interface ICobuildLockProvider { - acquireLockAsync(options: ILockOptions): Promise; - renewLockAsync(options: ILockOptions): Promise; - releaseLockAsync(options: ILockOptions): Promise; - setCompletedStateAsync(options: ISetCompletedStateOptions): Promise; - getCompletedStateAsync(options: IGetCompletedStateOptions): Promise; + acquireLockAsync(context: ICobuildContext): Promise; + renewLockAsync(context: ICobuildContext): Promise; + releaseLockAsync(context: ICobuildContext): Promise; + setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; + getCompletedStateAsync(context: ICobuildContext): Promise; } diff --git a/rush-plugins/rush-redis-cobuild-plugin/.eslintrc.js b/rush-plugins/rush-redis-cobuild-plugin/.eslintrc.js new file mode 100644 index 00000000000..4c934799d67 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/.eslintrc.js @@ -0,0 +1,10 @@ +// This is a workaround for https://github.com/eslint/eslint/issues/3458 +require('@rushstack/eslint-config/patch/modern-module-resolution'); + +module.exports = { + extends: [ + '@rushstack/eslint-config/profile/node-trusted-tool', + '@rushstack/eslint-config/mixins/friendly-locals' + ], + parserOptions: { tsconfigRootDir: __dirname } +}; diff --git a/rush-plugins/rush-redis-cobuild-plugin/.npmignore b/rush-plugins/rush-redis-cobuild-plugin/.npmignore new file mode 100644 index 00000000000..fcd991b60fa --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/.npmignore @@ -0,0 +1,32 @@ +# THIS IS A STANDARD TEMPLATE FOR .npmignore FILES IN THIS REPO. + +# Ignore all files by default, to avoid accidentally publishing unintended files. +* + +# Use negative patterns to bring back the specific things we want to publish. +!/bin/** +!/lib/** +!/lib-*/** +!/dist/** +!ThirdPartyNotice.txt + +# Ignore certain patterns that should not get published. +/dist/*.stats.* +/lib/**/test/ +/lib-*/**/test/ +*.test.js + +# NOTE: These don't need to be specified, because NPM includes them automatically. +# +# package.json +# README (and its variants) +# CHANGELOG (and its variants) +# LICENSE / LICENCE + +#-------------------------------------------- +# DO NOT MODIFY THE TEMPLATE ABOVE THIS LINE +#-------------------------------------------- + +# (Add your project-specific overrides here) +!/includes/** +!rush-plugin-manifest.json diff --git a/rush-plugins/rush-redis-cobuild-plugin/LICENSE b/rush-plugins/rush-redis-cobuild-plugin/LICENSE new file mode 100644 index 00000000000..0fbcd4e5857 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/LICENSE @@ -0,0 +1,24 @@ +@rushstack/rush-redis-cobuild-plugin + +Copyright (c) Microsoft Corporation. All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/rush-plugins/rush-redis-cobuild-plugin/README.md b/rush-plugins/rush-redis-cobuild-plugin/README.md new file mode 100644 index 00000000000..bfb2d49760b --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/README.md @@ -0,0 +1,9 @@ +# @rushstack/rush-amazon-s3-build-cache-plugin + +This is a Rush plugin for using Redis as cobuild lock provider during the "build" + +## Links + +- [CHANGELOG.md]( + https://github.com/microsoft/rushstack/blob/main/rush-plugins/rush-redis-cobuild-plugin/CHANGELOG.md) - Find + out what's new in the latest version diff --git a/rush-plugins/rush-redis-cobuild-plugin/config/api-extractor.json b/rush-plugins/rush-redis-cobuild-plugin/config/api-extractor.json new file mode 100644 index 00000000000..74590d3c4f8 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/config/api-extractor.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + + "mainEntryPointFilePath": "/lib/index.d.ts", + "apiReport": { + "enabled": true, + "reportFolder": "../../../common/reviews/api" + }, + "docModel": { + "enabled": false + }, + "dtsRollup": { + "enabled": true, + "betaTrimmedFilePath": "/dist/.d.ts" + } +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/config/jest.config.json b/rush-plugins/rush-redis-cobuild-plugin/config/jest.config.json new file mode 100644 index 00000000000..4bb17bde3ee --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/config/jest.config.json @@ -0,0 +1,3 @@ +{ + "extends": "@rushstack/heft-node-rig/profiles/default/config/jest.config.json" +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/config/rig.json b/rush-plugins/rush-redis-cobuild-plugin/config/rig.json new file mode 100644 index 00000000000..6ac88a96368 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/config/rig.json @@ -0,0 +1,7 @@ +{ + // The "rig.json" file directs tools to look for their config files in an external package. + // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + + "rigPackageName": "@rushstack/heft-node-rig" +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/package.json b/rush-plugins/rush-redis-cobuild-plugin/package.json new file mode 100644 index 00000000000..ce88b68a818 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/package.json @@ -0,0 +1,34 @@ +{ + "name": "@rushstack/rush-redis-cobuild-plugin", + "version": "5.88.2", + "description": "Rush plugin for Redis cobuild lock", + "repository": { + "type": "git", + "url": "https://github.com/microsoft/rushstack", + "directory": "rush-plugins/rush-redis-cobuild-plugin" + }, + "homepage": "https://rushjs.io", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "license": "MIT", + "scripts": { + "build": "heft build --clean", + "start": "heft test --clean --watch", + "test": "heft test", + "_phase:build": "heft build --clean", + "_phase:test": "heft test --no-build" + }, + "dependencies": { + "@redis/client": "~1.5.5", + "@rushstack/node-core-library": "workspace:*", + "@rushstack/rush-sdk": "workspace:*" + }, + "devDependencies": { + "@microsoft/rush-lib": "workspace:*", + "@rushstack/eslint-config": "workspace:*", + "@rushstack/heft": "workspace:*", + "@rushstack/heft-node-rig": "workspace:*", + "@types/heft-jest": "1.0.1", + "@types/node": "14.18.36" + } +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/rush-plugin-manifest.json b/rush-plugins/rush-redis-cobuild-plugin/rush-plugin-manifest.json new file mode 100644 index 00000000000..64ebf15d963 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/rush-plugin-manifest.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush-plugin-manifest.schema.json", + "plugins": [ + { + "pluginName": "rush-redis-cobuild-plugin", + "description": "Rush plugin for Redis cobuild lock", + "entryPoint": "lib/index.js", + "optionsSchema": "lib/schemas/redis-config.schema.json" + } + ] +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts new file mode 100644 index 00000000000..bef340d9abd --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { createClient } from '@redis/client'; + +import type { ICobuildLockProvider, ICobuildContext, ICobuildCompletedState } from '@rushstack/rush-sdk'; +import type { + RedisClientOptions, + RedisClientType, + RedisFunctions, + RedisModules, + RedisScripts +} from '@redis/client'; + +/** + * The redis client options + * @public + */ +export interface IRedisCobuildLockProviderOptions extends RedisClientOptions {} + +const KEY_SEPARATOR: string = ':'; +const COMPLETED_STATE_SEPARATOR: string = ';'; + +export class RedisCobuildLockProvider implements ICobuildLockProvider { + private readonly _options: IRedisCobuildLockProviderOptions; + + private _redisClient: RedisClientType; + private _lockKeyMap: WeakMap = new WeakMap(); + private _completedKeyMap: WeakMap = new WeakMap(); + + public constructor(options: IRedisCobuildLockProviderOptions) { + this._options = options; + this._redisClient = createClient(this._options); + } + + public async acquireLockAsync(context: ICobuildContext): Promise { + const { terminal } = context; + const lockKey: string = this.getLockKey(context); + const incrResult: number = await this._redisClient.incr(lockKey); + const result: boolean = incrResult === 1; + terminal.writeDebugLine(`Acquired lock for ${lockKey}: ${incrResult}, 1 is success`); + if (result) { + await this.renewLockAsync(context); + } + return result; + } + + public async renewLockAsync(context: ICobuildContext): Promise { + const { terminal } = context; + const lockKey: string = this.getLockKey(context); + await this._redisClient.expire(lockKey, 30); + terminal.writeDebugLine(`Renewed lock for ${lockKey}`); + } + + public async releaseLockAsync(context: ICobuildContext): Promise { + const { terminal } = context; + const lockKey: string = this.getLockKey(context); + await this._redisClient.set(lockKey, 0); + terminal.writeDebugLine(`Released lock for ${lockKey}`); + } + + public async setCompletedStateAsync( + context: ICobuildContext, + state: ICobuildCompletedState + ): Promise { + const { terminal } = context; + const key: string = this.getCompletedStateKey(context); + const value: string = this._serializeCompletedState(state); + await this._redisClient.set(key, value); + terminal.writeDebugLine(`Set completed state for ${key}: ${value}`); + } + + public async getCompletedStateAsync(context: ICobuildContext): Promise { + const key: string = this.getCompletedStateKey(context); + let state: ICobuildCompletedState | undefined; + const value: string | null = await this._redisClient.get(key); + if (value) { + state = this._deserializeCompletedState(value); + } + return state; + } + + /** + * Returns the lock key for the given context + * Example: cobuild:v1:::lock + */ + public getLockKey(context: ICobuildContext): string { + const { version, contextId, cacheId } = context; + let lockKey: string | undefined = this._lockKeyMap.get(context); + if (!lockKey) { + lockKey = ['cobuild', `v${version}`, contextId, cacheId, 'lock'].join(KEY_SEPARATOR); + this._completedKeyMap.set(context, lockKey); + } + return lockKey; + } + + /** + * Returns the completed key for the given context + * Example: cobuild:v1:::completed + */ + public getCompletedStateKey(context: ICobuildContext): string { + const { version, contextId, cacheId } = context; + let completedKey: string | undefined = this._completedKeyMap.get(context); + if (!completedKey) { + completedKey = ['cobuild', `v${version}`, contextId, cacheId, 'completed'].join(KEY_SEPARATOR); + this._completedKeyMap.set(context, completedKey); + } + return completedKey; + } + + private _serializeCompletedState(state: ICobuildCompletedState): string { + // Example: SUCCESS;1234567890 + // Example: FAILURE;1234567890 + return `${state.status}${COMPLETED_STATE_SEPARATOR}${state.cacheId}`; + } + + private _deserializeCompletedState(state: string): ICobuildCompletedState | undefined { + const [status, cacheId] = state.split(COMPLETED_STATE_SEPARATOR); + return { status: status as ICobuildCompletedState['status'], cacheId }; + } +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts new file mode 100644 index 00000000000..1d1d69fdaeb --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { Import } from '@rushstack/node-core-library'; +import type { IRushPlugin, RushSession, RushConfiguration } from '@rushstack/rush-sdk'; +import type { IRedisCobuildLockProviderOptions, RedisCobuildLockProvider } from './RedisCobuildLockProvider'; + +const RedisCobuildLockProviderModule: typeof import('./RedisCobuildLockProvider') = Import.lazy( + './RedisCobuildLockProvider', + require +); + +const PLUGIN_NAME: string = 'RedisCobuildPlugin'; + +/** + * @public + */ +export type IRushRedisCobuildPluginOptions = IRedisCobuildLockProviderOptions; + +/** + * @public + */ +export class RushRedisCobuildPlugin implements IRushPlugin { + public pluginName: string = PLUGIN_NAME; + + private _options: IRushRedisCobuildPluginOptions; + + public constructor(options: IRushRedisCobuildPluginOptions) { + this._options = options; + } + + public apply(rushSession: RushSession, rushConfiguration: RushConfiguration): void { + rushSession.hooks.initialize.tap(PLUGIN_NAME, () => { + rushSession.registerCobuildLockProviderFactory('redis', (): RedisCobuildLockProvider => { + const options: IRushRedisCobuildPluginOptions = this._options; + return new RedisCobuildLockProviderModule.RedisCobuildLockProvider(options); + }); + }); + } +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts new file mode 100644 index 00000000000..d4466255b4c --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { RushRedisCobuildPlugin } from './RushRedisCobuildPlugin'; + +export default RushRedisCobuildPlugin; +export type { IRedisCobuildLockProviderOptions } from './RedisCobuildLockProvider'; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json b/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json new file mode 100644 index 00000000000..4d984d81b3e --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json @@ -0,0 +1,70 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Configuration for cobuild lock with Redis configuration\n\nhttps://github.com/redis/node-redis/blob/master/docs/client-configuration.md", + "type": "object", + "additionalProperties": false, + "properties": { + "url": { + "type": "string", + "description": "redis[s]://[[username][:password]@][host][:port][/db-number]\n\n See the following links for more information:\n\nredis: https://www.iana.org/assignments/uri-schemes/prov/redis\n\nrediss: https://www.iana.org/assignments/uri-schemes/prov/rediss" + }, + "socket": { + "type": "object", + "description": "Socket connection properties. Unlisted net.connect properties (and tls.connect) are also supported", + "properties": { + "port": { + "description": "Redis server port. Default value is 6379", + "type": "number" + }, + "host": { + "description": "Redis server host. Default value is localhost", + "type": "string" + }, + "family": { + "description": "IP Stack version (one of 4 | 6 | 0). Default value is 0", + "type": "number" + }, + "path": { + "description": "path to the UNIX Socket", + "type": "string" + }, + "connectTimeout": { + "description": "Connection timeout in milliseconds. Default value is 5000", + "type": "number" + }, + "noDelay": { + "description": "Toggle Nagle's algorithm. Default value is true", + "type": "boolean" + }, + "keepAlive": { + "description": "Toggle keep alive on the socket", + "type": "boolean" + } + } + }, + "username": { + "description": "ACL username", + "type": "string" + }, + "password": { + "description": "ACL password", + "type": "string" + }, + "name": { + "description": "Redis client name", + "type": "string" + }, + "database": { + "description": "Redis database number", + "type": "number" + }, + "legacyMode": { + "description": "Maintain some backwards compatibility", + "type": "boolean" + }, + "pingInterval": { + "description": "Send PING command at interval (in ms). Useful with \"Azure Cache for Redis\".", + "type": "number" + } + } +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts new file mode 100644 index 00000000000..90f0cbe0aea --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -0,0 +1,93 @@ +/* eslint-disable @typescript-eslint/no-floating-promises */ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { Terminal, ConsoleTerminalProvider } from '@rushstack/node-core-library'; +import { ICobuildCompletedState, ICobuildContext, OperationStatus } from '@rushstack/rush-sdk'; +import { IRedisCobuildLockProviderOptions, RedisCobuildLockProvider } from '../RedisCobuildLockProvider'; + +import * as redisAPI from '@redis/client'; +import type { RedisClientType } from '@redis/client'; + +const terminal = new Terminal(new ConsoleTerminalProvider()); + +describe(RedisCobuildLockProvider.name, () => { + let storage: Record = {}; + beforeEach(() => { + jest.spyOn(redisAPI, 'createClient').mockImplementation(() => { + return { + incr: jest.fn().mockImplementation((key: string) => { + storage[key] = (Number(storage[key]) || 0) + 1; + return storage[key]; + }), + expire: jest.fn().mockResolvedValue(undefined), + set: jest.fn().mockImplementation((key: string, value: string) => { + storage[key] = value; + }), + get: jest.fn().mockImplementation((key: string) => { + return storage[key]; + }) + } as unknown as RedisClientType; + }); + }); + + afterEach(() => { + jest.resetAllMocks(); + storage = {}; + }); + + function prepareSubject(): RedisCobuildLockProvider { + return new RedisCobuildLockProvider({} as IRedisCobuildLockProviderOptions); + } + const context: ICobuildContext = { + contextId: '123', + cacheId: 'abc', + version: 1, + terminal + }; + + it('getLockKey works', () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const lockKey: string = subject.getLockKey(context); + expect(lockKey).toMatchSnapshot(); + }); + + it('getCompletedStateKey works', () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const completedStateKey: string = subject.getCompletedStateKey(context); + expect(completedStateKey).toMatchSnapshot(); + }); + + it('acquires lock success', async () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const result: boolean = await subject.acquireLockAsync(context); + expect(result).toBe(true); + }); + + it('acquires lock fails at the second time', async () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const cobuildContext: ICobuildContext = { + ...context, + contextId: 'abc' + }; + const result1: boolean = await subject.acquireLockAsync(cobuildContext); + expect(result1).toBe(true); + const result2: boolean = await subject.acquireLockAsync(cobuildContext); + expect(result2).toBe(false); + }); + + it('releaseLockAsync works', async () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + expect(() => subject.releaseLockAsync(context)).not.toThrowError(); + }); + + it('set and get completedState works', async () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const cacheId: string = 'foo'; + const status: ICobuildCompletedState['status'] = OperationStatus.SuccessWithWarning; + expect(() => subject.setCompletedStateAsync(context, { status, cacheId })).not.toThrowError(); + const actualState: ICobuildCompletedState | undefined = await subject.getCompletedStateAsync(context); + expect(actualState?.cacheId).toBe(cacheId); + expect(actualState?.status).toBe(status); + }); +}); diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap new file mode 100644 index 00000000000..33aa6130bbf --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap @@ -0,0 +1,5 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`RedisCobuildLockProvider getCompletedStateKey works 1`] = `"cobuild:v1:123:abc:completed"`; + +exports[`RedisCobuildLockProvider getLockKey works 1`] = `"cobuild:v1:123:abc:lock"`; diff --git a/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json b/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json new file mode 100644 index 00000000000..b3d3ff2a64f --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./node_modules/@rushstack/heft-node-rig/profiles/default/tsconfig-base.json", + + "compilerOptions": { + "lib": ["DOM"], + "types": ["heft-jest", "node"] + } +} diff --git a/rush.json b/rush.json index 085a872ecf9..db2d4a025db 100644 --- a/rush.json +++ b/rush.json @@ -1026,6 +1026,12 @@ "reviewCategory": "libraries", "shouldPublish": false }, + { + "packageName": "@rushstack/rush-redis-cobuild-plugin", + "projectFolder": "rush-plugins/rush-redis-cobuild-plugin", + "reviewCategory": "libraries", + "versionPolicyName": "rush" + }, { "packageName": "@rushstack/rush-serve-plugin", "projectFolder": "rush-plugins/rush-serve-plugin", From c2bb3e3453987e8bb7394e4bd8bad30d8dd85895 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Sat, 11 Feb 2023 00:14:41 +0800 Subject: [PATCH 06/55] chore --- rush.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rush.json b/rush.json index db2d4a025db..8202810d639 100644 --- a/rush.json +++ b/rush.json @@ -1030,7 +1030,7 @@ "packageName": "@rushstack/rush-redis-cobuild-plugin", "projectFolder": "rush-plugins/rush-redis-cobuild-plugin", "reviewCategory": "libraries", - "versionPolicyName": "rush" + "shouldPublish": true }, { "packageName": "@rushstack/rush-serve-plugin", From e3839da56851a17ba8f00f9eac4f5adb856b20e7 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Wed, 15 Feb 2023 10:55:43 +0800 Subject: [PATCH 07/55] feat: rush-redis-cobuild-plugin-integration-test --- .../.eslintrc.js | 7 +++ .../.gitignore | 1 + .../README.md | 18 +++++++ .../config/heft.json | 51 +++++++++++++++++++ .../config/rush-project.json | 8 +++ .../docker-compose.yml | 10 ++++ .../package.json | 24 +++++++++ .../src/testLockProvider.ts | 43 ++++++++++++++++ .../tsconfig.json | 25 +++++++++ .../rush/nonbrowser-approved-packages.json | 4 ++ common/config/rush/pnpm-lock.yaml | 24 +++++++++ common/reviews/api/rush-lib.api.md | 8 +++ .../api/rush-redis-cobuild-plugin.api.md | 26 +++++++++- .../rush-lib/src/api/CobuildConfiguration.ts | 9 ++++ .../cli/scriptActions/PhasedScriptAction.ts | 3 ++ .../src/logic/cobuild/ICobuildLockProvider.ts | 2 + .../src/RedisCobuildLockProvider.ts | 15 +++++- .../rush-redis-cobuild-plugin/src/index.ts | 1 + .../src/test/RedisCobuildLockProvider.test.ts | 1 + rush.json | 6 +++ 20 files changed, 283 insertions(+), 3 deletions(-) create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/.eslintrc.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/README.md create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.eslintrc.js b/build-tests/rush-redis-cobuild-plugin-integration-test/.eslintrc.js new file mode 100644 index 00000000000..60160b354c4 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.eslintrc.js @@ -0,0 +1,7 @@ +// This is a workaround for https://github.com/eslint/eslint/issues/3458 +require('@rushstack/eslint-config/patch/modern-module-resolution'); + +module.exports = { + extends: ['@rushstack/eslint-config/profile/node'], + parserOptions: { tsconfigRootDir: __dirname } +}; diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore b/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore new file mode 100644 index 00000000000..16a0ee1e146 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore @@ -0,0 +1 @@ +redis-data/dump.rdb diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md new file mode 100644 index 00000000000..87b7b6e0af9 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -0,0 +1,18 @@ +# About +This package enables integration testing of the `RedisCobuildLockProvider` by connecting to an actual Redis created using an [redis](https://hub.docker.com/_/redis) docker image. + +# Prerequisites +Docker and docker compose must be installed + +# Start the Redis +In this folder run `docker-compose up -d` + +# Stop the Redis +In this folder run `docker-compose down` + +# Run the test +```sh +# start the docker container: docker-compose up -d +# build the code: rushx build +rushx test-lock-provider +``` diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json b/build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json new file mode 100644 index 00000000000..99e058540fb --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json @@ -0,0 +1,51 @@ +/** + * Defines configuration used by core Heft. + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/heft/heft.schema.json", + + "eventActions": [ + { + /** + * The kind of built-in operation that should be performed. + * The "deleteGlobs" action deletes files or folders that match the + * specified glob patterns. + */ + "actionKind": "deleteGlobs", + + /** + * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json + * occur at the end of the stage of the Heft run. + */ + "heftEvent": "clean", + + /** + * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other + * configs. + */ + "actionId": "defaultClean", + + /** + * Glob patterns to be deleted. The paths are resolved relative to the project folder. + */ + "globsToDelete": ["dist", "lib", "temp"] + } + ], + + /** + * The list of Heft plugins to be loaded. + */ + "heftPlugins": [ + // { + // /** + // * The path to the plugin package. + // */ + // "plugin": "path/to/my-plugin", + // + // /** + // * An optional object that provides additional settings that may be defined by the plugin. + // */ + // // "options": { } + // } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/config/rush-project.json new file mode 100644 index 00000000000..247dc17187a --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/config/rush-project.json @@ -0,0 +1,8 @@ +{ + "operationSettings": [ + { + "operationName": "build", + "outputFolderNames": ["lib", "dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml b/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml new file mode 100644 index 00000000000..4514fcd8223 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3.7' + +services: + redis: + image: redis:6.2.10-alpine + command: redis-server --save 20 1 --loglevel warning --requirepass redis123 + ports: + - '6379:6379' + volumes: + - ./redis-data:/data diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/package.json new file mode 100644 index 00000000000..a667eec050e --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/package.json @@ -0,0 +1,24 @@ +{ + "name": "rush-redis-cobuild-plugin-integration-test", + "description": "Tests connecting to an redis server", + "version": "1.0.0", + "private": true, + "license": "MIT", + "scripts": { + "build": "heft build --clean", + "_phase:build": "heft build --clean", + "test-lock-provider": "node ./lib/testLockProvider.js" + }, + "devDependencies": { + "@rushstack/eslint-config": "workspace:*", + "@rushstack/heft": "workspace:*", + "@microsoft/rush-lib": "workspace:*", + "@rushstack/rush-redis-cobuild-plugin": "workspace:*", + "@rushstack/node-core-library": "workspace:*", + "@types/node": "12.20.24", + "eslint": "~8.7.0", + "typescript": "~4.8.4", + "http-proxy": "~1.18.1", + "@types/http-proxy": "~1.17.8" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts new file mode 100644 index 00000000000..820532437f1 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts @@ -0,0 +1,43 @@ +import { + RedisCobuildLockProvider, + IRedisCobuildLockProviderOptions +} from '@rushstack/rush-redis-cobuild-plugin'; +import { ConsoleTerminalProvider, ITerminal, Terminal } from '@rushstack/node-core-library'; +import { OperationStatus, ICobuildContext } from '@microsoft/rush-lib'; + +const options: IRedisCobuildLockProviderOptions = { + url: 'redis://localhost:6379', + password: 'redis123' +}; + +const terminal: ITerminal = new Terminal( + new ConsoleTerminalProvider({ + verboseEnabled: true, + debugEnabled: true + }) +); + +async function main(): Promise { + const lockProvider: RedisCobuildLockProvider = new RedisCobuildLockProvider(options); + await lockProvider.connectAsync(); + const context: ICobuildContext = { + terminal, + contextId: 'test-context-id', + version: 1, + cacheId: 'test-cache-id' + }; + await lockProvider.acquireLockAsync(context); + await lockProvider.renewLockAsync(context); + await lockProvider.setCompletedStateAsync(context, { + status: OperationStatus.Success, + cacheId: 'test-cache-id' + }); + await lockProvider.releaseLockAsync(context); + const completedState = await lockProvider.getCompletedStateAsync(context); + console.log('Completed state: ', completedState); + await lockProvider.disconnectAsync(); +} +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json b/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json new file mode 100644 index 00000000000..6314e94a07d --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json.schemastore.org/tsconfig", + + "compilerOptions": { + "outDir": "lib", + "rootDir": "src", + + "forceConsistentCasingInFileNames": true, + "jsx": "react", + "declaration": true, + "sourceMap": true, + "declarationMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictNullChecks": true, + "noUnusedLocals": true, + "types": ["node"], + + "module": "commonjs", + "target": "es2017", + "lib": ["es2017", "DOM"] + }, + "include": ["src/**/*.ts", "src/**/*.tsx"], + "exclude": ["node_modules", "lib"] +} diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index 0bc0e61f0eb..3294b64fdae 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -170,6 +170,10 @@ "name": "@rushstack/package-deps-hash", "allowedCategories": [ "libraries" ] }, + { + "name": "@rushstack/rush-redis-cobuild-plugin", + "allowedCategories": [ "tests" ] + }, { "name": "@rushstack/rig-package", "allowedCategories": [ "libraries" ] diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 50f93c9f999..8d3c15d3bb0 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1346,6 +1346,30 @@ importers: '@rushstack/heft-node-rig': link:../../rigs/heft-node-rig '@types/node': 14.18.36 + ../../build-tests/rush-redis-cobuild-plugin-integration-test: + specifiers: + '@microsoft/rush-lib': workspace:* + '@rushstack/eslint-config': workspace:* + '@rushstack/heft': workspace:* + '@rushstack/node-core-library': workspace:* + '@rushstack/rush-redis-cobuild-plugin': workspace:* + '@types/http-proxy': ~1.17.8 + '@types/node': 12.20.24 + eslint: ~8.7.0 + http-proxy: ~1.18.1 + typescript: ~4.8.4 + devDependencies: + '@microsoft/rush-lib': link:../../libraries/rush-lib + '@rushstack/eslint-config': link:../../eslint/eslint-config + '@rushstack/heft': link:../../apps/heft + '@rushstack/node-core-library': link:../../libraries/node-core-library + '@rushstack/rush-redis-cobuild-plugin': link:../../rush-plugins/rush-redis-cobuild-plugin + '@types/http-proxy': 1.17.9 + '@types/node': 12.20.24 + eslint: 8.7.0 + http-proxy: 1.18.1 + typescript: 4.8.4 + ../../build-tests/set-webpack-public-path-plugin-webpack4-test: specifiers: '@rushstack/eslint-config': workspace:* diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index cac66ad13c1..b967a5aeac1 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -100,8 +100,12 @@ export class CobuildConfiguration { readonly cobuildEnabled: boolean; readonly cobuildLockProvider: ICobuildLockProvider; // (undocumented) + connectLockProviderAsync(): Promise; + // (undocumented) get contextId(): string; // (undocumented) + disconnectLockProviderAsync(): Promise; + // (undocumented) static getCobuildConfigFilePath(rushConfiguration: RushConfiguration): string; static tryLoadAsync(terminal: ITerminal, rushConfiguration: RushConfiguration, rushSession: RushSession): Promise; } @@ -286,6 +290,10 @@ export interface ICobuildLockProvider { // (undocumented) acquireLockAsync(context: ICobuildContext): Promise; // (undocumented) + connectAsync(): Promise; + // (undocumented) + disconnectAsync(): Promise; + // (undocumented) getCompletedStateAsync(context: ICobuildContext): Promise; // (undocumented) releaseLockAsync(context: ICobuildContext): Promise; diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index 6a73c0c36bc..4a2f99398e4 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -4,15 +4,39 @@ ```ts +import type { ICobuildCompletedState } from '@rushstack/rush-sdk'; +import type { ICobuildContext } from '@rushstack/rush-sdk'; +import type { ICobuildLockProvider } from '@rushstack/rush-sdk'; import type { IRushPlugin } from '@rushstack/rush-sdk'; import type { RedisClientOptions } from '@redis/client'; import type { RushConfiguration } from '@rushstack/rush-sdk'; import type { RushSession } from '@rushstack/rush-sdk'; -// @public +// @beta export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { } +// @beta (undocumented) +export class RedisCobuildLockProvider implements ICobuildLockProvider { + constructor(options: IRedisCobuildLockProviderOptions); + // (undocumented) + acquireLockAsync(context: ICobuildContext): Promise; + // (undocumented) + connectAsync(): Promise; + // (undocumented) + disconnectAsync(): Promise; + // (undocumented) + getCompletedStateAsync(context: ICobuildContext): Promise; + getCompletedStateKey(context: ICobuildContext): string; + getLockKey(context: ICobuildContext): string; + // (undocumented) + releaseLockAsync(context: ICobuildContext): Promise; + // (undocumented) + renewLockAsync(context: ICobuildContext): Promise; + // (undocumented) + setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; +} + // @public (undocumented) class RushRedisCobuildPlugin implements IRushPlugin { // Warning: (ae-forgotten-export) The symbol "IRushRedisCobuildPluginOptions" needs to be exported by the entry point index.d.ts diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 3f9d2a12dac..1b4b636c390 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -75,6 +75,7 @@ export class CobuildConfiguration { public static getCobuildConfigFilePath(rushConfiguration: RushConfiguration): string { return path.resolve(rushConfiguration.commonRushConfigFolder, RushConstants.cobuildFilename); } + private static async _loadAsync( jsonFilePath: string, terminal: ITerminal, @@ -108,4 +109,12 @@ export class CobuildConfiguration { // FIXME: hardcode return '123'; } + + public async connectLockProviderAsync(): Promise { + await this.cobuildLockProvider.connectAsync(); + } + + public async disconnectLockProviderAsync(): Promise { + await this.cobuildLockProvider.disconnectAsync(); + } } diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 99eb4c503b6..8cb5064fa3c 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -312,6 +312,7 @@ export class PhasedScriptAction extends BaseScriptAction { this.rushConfiguration, this.rushSession ); + await cobuildConfiguration?.connectLockProviderAsync(); } const projectSelection: Set = @@ -376,6 +377,8 @@ export class PhasedScriptAction extends BaseScriptAction { await this._runWatchPhases(internalOptions); } + + await cobuildConfiguration?.disconnectLockProviderAsync(); } private async _runInitialPhases(options: IRunPhasesOptions): Promise { diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index b95327373b6..a36bc8ab702 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -30,6 +30,8 @@ export interface ICobuildCompletedState { * @beta */ export interface ICobuildLockProvider { + connectAsync(): Promise; + disconnectAsync(): Promise; acquireLockAsync(context: ICobuildContext): Promise; renewLockAsync(context: ICobuildContext): Promise; releaseLockAsync(context: ICobuildContext): Promise; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index bef340d9abd..3af20ccbb66 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -14,13 +14,16 @@ import type { /** * The redis client options - * @public + * @beta */ export interface IRedisCobuildLockProviderOptions extends RedisClientOptions {} const KEY_SEPARATOR: string = ':'; const COMPLETED_STATE_SEPARATOR: string = ';'; +/** + * @beta + */ export class RedisCobuildLockProvider implements ICobuildLockProvider { private readonly _options: IRedisCobuildLockProviderOptions; @@ -33,6 +36,14 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { this._redisClient = createClient(this._options); } + public async connectAsync(): Promise { + await this._redisClient.connect(); + } + + public async disconnectAsync(): Promise { + await this._redisClient.disconnect(); + } + public async acquireLockAsync(context: ICobuildContext): Promise { const { terminal } = context; const lockKey: string = this.getLockKey(context); @@ -89,7 +100,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { let lockKey: string | undefined = this._lockKeyMap.get(context); if (!lockKey) { lockKey = ['cobuild', `v${version}`, contextId, cacheId, 'lock'].join(KEY_SEPARATOR); - this._completedKeyMap.set(context, lockKey); + this._lockKeyMap.set(context, lockKey); } return lockKey; } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts index d4466255b4c..f627507d614 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts @@ -4,4 +4,5 @@ import { RushRedisCobuildPlugin } from './RushRedisCobuildPlugin'; export default RushRedisCobuildPlugin; +export { RedisCobuildLockProvider } from './RedisCobuildLockProvider'; export type { IRedisCobuildLockProviderOptions } from './RedisCobuildLockProvider'; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index 90f0cbe0aea..5cd78099676 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -39,6 +39,7 @@ describe(RedisCobuildLockProvider.name, () => { function prepareSubject(): RedisCobuildLockProvider { return new RedisCobuildLockProvider({} as IRedisCobuildLockProviderOptions); } + const context: ICobuildContext = { contextId: '123', cacheId: 'abc', diff --git a/rush.json b/rush.json index 8202810d639..b3a91990c4f 100644 --- a/rush.json +++ b/rush.json @@ -796,6 +796,12 @@ "reviewCategory": "tests", "shouldPublish": false }, + { + "packageName": "rush-redis-cobuild-plugin-integration-test", + "projectFolder": "build-tests/rush-redis-cobuild-plugin-integration-test", + "reviewCategory": "tests", + "shouldPublish": false + }, { "packageName": "set-webpack-public-path-plugin-webpack4-test", "projectFolder": "build-tests/set-webpack-public-path-plugin-webpack4-test", From dbf758f282d475a5e8a66a5de2e249236f1e3051 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 16 Feb 2023 16:06:21 +0800 Subject: [PATCH 08/55] fix: rush redis cobuild --- .gitignore | 7 +- .prettierignore | 3 + .vscode/redis-cobuild.code-workspace | 91 +++ .../.gitignore | 2 +- .../docker-compose.yml | 2 +- .../package.json | 14 +- .../sandbox/repo/.gitignore | 4 + .../rush-redis-cobuild-plugin.json | 4 + .../repo/common/config/rush/build-cache.json | 92 +++ .../repo/common/config/rush/cobuild.json | 25 + .../repo/common/config/rush/command-line.json | 308 +++++++++ .../repo/common/config/rush/pnpm-lock.yaml | 34 + .../repo/common/config/rush/repo-state.json | 4 + .../common/scripts/install-run-rush-pnpm.js | 28 + .../repo/common/scripts/install-run-rush.js | 214 ++++++ .../repo/common/scripts/install-run-rushx.js | 28 + .../repo/common/scripts/install-run.js | 645 ++++++++++++++++++ .../repo/projects/a/config/rush-project.json | 12 + .../sandbox/repo/projects/a/package.json | 8 + .../repo/projects/b/config/rush-project.json | 12 + .../sandbox/repo/projects/b/package.json | 8 + .../sandbox/repo/projects/build.js | 12 + .../repo/projects/c/config/rush-project.json | 12 + .../sandbox/repo/projects/c/package.json | 11 + .../repo/projects/d/config/rush-project.json | 12 + .../sandbox/repo/projects/d/package.json | 12 + .../repo/projects/e/config/rush-project.json | 12 + .../sandbox/repo/projects/e/package.json | 12 + .../sandbox/repo/rush.json | 29 + .../src/paths.ts | 5 + .../src/runRush.ts | 37 + .../src/testLockProvider.ts | 18 +- .../tsconfig.json | 1 + common/config/rush/pnpm-lock.yaml | 4 +- common/reviews/api/rush-lib.api.md | 2 - .../api/rush-redis-cobuild-plugin.api.md | 2 +- .../rush-init/common/config/rush/cobuild.json | 25 + .../rush-lib/src/api/RushConfiguration.ts | 1 + .../rush-lib/src/cli/actions/InitAction.ts | 1 + .../rush-lib/src/logic/cobuild/CobuildLock.ts | 6 +- .../src/logic/cobuild/ICobuildLockProvider.ts | 2 - .../logic/operations/AsyncOperationQueue.ts | 1 + .../operations/OperationExecutionManager.ts | 15 +- .../logic/operations/ShellOperationRunner.ts | 38 +- .../src/RedisCobuildLockProvider.ts | 34 +- .../src/RushRedisCobuildPlugin.ts | 2 +- .../src/test/RedisCobuildLockProvider.test.ts | 14 +- 47 files changed, 1793 insertions(+), 72 deletions(-) create mode 100644 .vscode/redis-cobuild.code-workspace create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/build-cache.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/repo-state.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush-pnpm.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rushx.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/src/paths.ts create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts create mode 100644 libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json diff --git a/.gitignore b/.gitignore index 3d02c0474a5..9e44f747b7b 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,12 @@ jspm_packages/ *.iml # Visual Studio Code -.vscode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!*.code-workspace # Rush temporary files common/deploy/ diff --git a/.prettierignore b/.prettierignore index f577e87f844..5355633673c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -104,5 +104,8 @@ libraries/rush-lib/assets/rush-init/ # These are intentionally invalid files libraries/heft-config-file/src/test/errorCases/invalidJson/config.json +# common scripts in sandbox repo +build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/ + # We'll consider enabling this later; Prettier reformats code blocks, which affects end-user content *.md diff --git a/.vscode/redis-cobuild.code-workspace b/.vscode/redis-cobuild.code-workspace new file mode 100644 index 00000000000..9ab11bb4b4b --- /dev/null +++ b/.vscode/redis-cobuild.code-workspace @@ -0,0 +1,91 @@ +{ + "folders": [ + { + "name": "rush-redis-cobuild-plugin-integration-test", + "path": "../build-tests/rush-redis-cobuild-plugin-integration-test" + }, + { + "name": "rush-redis-cobuild-plugin", + "path": "../rush-plugins/rush-redis-cobuild-plugin" + }, + { + "name": "rush-lib", + "path": "../libraries/rush-lib" + }, + { + "name": ".vscode", + "path": "../.vscode" + } + ], + "tasks": { + "version": "2.0.0", + "tasks": [ + { + "type": "shell", + "label": "cobuild", + "dependsOrder": "sequence", + "dependsOn": ["update 1", "_cobuild"], + "problemMatcher": [] + }, + { + "type": "shell", + "label": "_cobuild", + "dependsOn": ["build 1", "build 2"], + "problemMatcher": [] + }, + { + "type": "shell", + "label": "update", + "command": "node ../../lib/runRush.js update", + "problemMatcher": [], + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": false + }, + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + } + }, + { + "type": "shell", + "label": "build 1", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": true + }, + "group": "build" + }, + { + "type": "shell", + "label": "build 2", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": true + }, + "group": "build" + } + ] + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore b/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore index 16a0ee1e146..97e8499abcc 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore @@ -1 +1 @@ -redis-data/dump.rdb +redis-data/dump.rdb \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml b/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml index 4514fcd8223..2b9a3f3722b 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.7' services: redis: image: redis:6.2.10-alpine - command: redis-server --save 20 1 --loglevel warning --requirepass redis123 + command: redis-server --save "" --loglevel warning --requirepass redis123 ports: - '6379:6379' volumes: diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/package.json index a667eec050e..52ffe8f3cfb 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/package.json @@ -1,24 +1,24 @@ { "name": "rush-redis-cobuild-plugin-integration-test", - "description": "Tests connecting to an redis server", "version": "1.0.0", "private": true, + "description": "Tests connecting to an redis server", "license": "MIT", "scripts": { - "build": "heft build --clean", "_phase:build": "heft build --clean", + "build": "heft build --clean", "test-lock-provider": "node ./lib/testLockProvider.js" }, "devDependencies": { + "@microsoft/rush-lib": "workspace:*", "@rushstack/eslint-config": "workspace:*", "@rushstack/heft": "workspace:*", - "@microsoft/rush-lib": "workspace:*", - "@rushstack/rush-redis-cobuild-plugin": "workspace:*", "@rushstack/node-core-library": "workspace:*", - "@types/node": "12.20.24", + "@rushstack/rush-redis-cobuild-plugin": "workspace:*", + "@types/http-proxy": "~1.17.8", + "@types/node": "14.18.36", "eslint": "~8.7.0", - "typescript": "~4.8.4", "http-proxy": "~1.18.1", - "@types/http-proxy": "~1.17.8" + "typescript": "~4.8.4" } } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore new file mode 100644 index 00000000000..484950696c9 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore @@ -0,0 +1,4 @@ +# Rush temporary files +common/deploy/ +common/temp/ +common/autoinstallers/*/.npmrc \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json new file mode 100644 index 00000000000..625dff477fc --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json @@ -0,0 +1,4 @@ +{ + "url": "redis://localhost:6379", + "password": "redis123" +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/build-cache.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/build-cache.json new file mode 100644 index 00000000000..d09eaa6a04c --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/build-cache.json @@ -0,0 +1,92 @@ +/** + * This configuration file manages Rush's build cache feature. + * More documentation is available on the Rush website: https://rushjs.io + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/build-cache.schema.json", + + /** + * (Required) EXPERIMENTAL - Set this to true to enable the build cache feature. + * + * See https://rushjs.io/pages/maintainer/build_cache/ for details about this experimental feature. + */ + "buildCacheEnabled": true, + + /** + * (Required) Choose where project build outputs will be cached. + * + * Possible values: "local-only", "azure-blob-storage", "amazon-s3" + */ + "cacheProvider": "local-only", + + /** + * Setting this property overrides the cache entry ID. If this property is set, it must contain + * a [hash] token. + * + * Other available tokens: + * - [projectName] + * - [projectName:normalize] + * - [phaseName] + * - [phaseName:normalize] + * - [phaseName:trimPrefix] + */ + // "cacheEntryNamePattern": "[projectName:normalize]-[phaseName:normalize]-[hash]" + + /** + * Use this configuration with "cacheProvider"="azure-blob-storage" + */ + "azureBlobStorageConfiguration": { + /** + * (Required) The name of the the Azure storage account to use for build cache. + */ + // "storageAccountName": "example", + /** + * (Required) The name of the container in the Azure storage account to use for build cache. + */ + // "storageContainerName": "my-container", + /** + * The Azure environment the storage account exists in. Defaults to AzurePublicCloud. + * + * Possible values: "AzurePublicCloud", "AzureChina", "AzureGermany", "AzureGovernment" + */ + // "azureEnvironment": "AzurePublicCloud", + /** + * An optional prefix for cache item blob names. + */ + // "blobPrefix": "my-prefix", + /** + * If set to true, allow writing to the cache. Defaults to false. + */ + // "isCacheWriteAllowed": true + }, + + /** + * Use this configuration with "cacheProvider"="amazon-s3" + */ + "amazonS3Configuration": { + /** + * (Required unless s3Endpoint is specified) The name of the bucket to use for build cache. + * Example: "my-bucket" + */ + // "s3Bucket": "my-bucket", + /** + * (Required unless s3Bucket is specified) The Amazon S3 endpoint of the bucket to use for build cache. + * This should not include any path; use the s3Prefix to set the path. + * Examples: "my-bucket.s3.us-east-2.amazonaws.com" or "http://localhost:9000" + */ + // "s3Endpoint": "https://my-bucket.s3.us-east-2.amazonaws.com", + /** + * (Required) The Amazon S3 region of the bucket to use for build cache. + * Example: "us-east-1" + */ + // "s3Region": "us-east-1", + /** + * An optional prefix ("folder") for cache items. It should not start with "/". + */ + // "s3Prefix": "my-prefix", + /** + * If set to true, allow writing to the cache. Defaults to false. + */ + // "isCacheWriteAllowed": true + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json new file mode 100644 index 00000000000..29a49cbd86c --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json @@ -0,0 +1,25 @@ +/** + * This configuration file manages Rush's cobuild feature. + * More documentation is available on the Rush website: https://rushjs.io + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/cobuild.schema.json", + + /** + * (Required) EXPERIMENTAL - Set this to true to enable the cobuild feature. + */ + "cobuildEnabled": true, + + /** + * (Required) Choose where cobuild lock will be acquired. + * + * The lock provider is registered by the rush plugins. + * For example, @rushstack/rush-redis-cobuild-plugin registers the "redis" lock provider. + */ + "cobuildLockProvider": "redis" + + /** + * Setting this property overrides the cobuild context ID. + */ + // "cobuildContextIdPattern": "" +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json new file mode 100644 index 00000000000..1a8f7837fa3 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json @@ -0,0 +1,308 @@ +/** + * This configuration file defines custom commands for the "rush" command-line. + * More documentation is available on the Rush website: https://rushjs.io + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json", + + /** + * Custom "commands" introduce new verbs for the command-line. To see the help for these + * example commands, try "rush --help", "rush my-bulk-command --help", or + * "rush my-global-command --help". + */ + "commands": [ + { + "commandKind": "bulk", + "summary": "Concurrent version of rush build", + "name": "cobuild", + "safeForSimultaneousRushProcesses": true, + "enableParallelism": true, + "incremental": true + } + + // { + // /** + // * (Required) Determines the type of custom command. + // * Rush's "bulk" commands are invoked separately for each project. Rush will look in + // * each project's package.json file for a "scripts" entry whose name matches the + // * command name. By default, the command will run for every project in the repo, + // * according to the dependency graph (similar to how "rush build" works). + // * The set of projects can be restricted e.g. using the "--to" or "--from" parameters. + // */ + // "commandKind": "bulk", + // + // /** + // * (Required) The name that will be typed as part of the command line. This is also the name + // * of the "scripts" hook in the project's package.json file. + // * The name should be comprised of lower case words separated by hyphens or colons. The name should include an + // * English verb (e.g. "deploy"). Use a hyphen to separate words (e.g. "upload-docs"). A group of related commands + // * can be prefixed with a colon (e.g. "docs:generate", "docs:deploy", "docs:serve", etc). + // * + // * Note that if the "rebuild" command is overridden here, it becomes separated from the "build" command + // * and will call the "rebuild" script instead of the "build" script. + // */ + // "name": "my-bulk-command", + // + // /** + // * (Required) A short summary of the custom command to be shown when printing command line + // * help, e.g. "rush --help". + // */ + // "summary": "Example bulk custom command", + // + // /** + // * A detailed description of the command to be shown when printing command line + // * help (e.g. "rush --help my-command"). + // * If omitted, the "summary" text will be shown instead. + // * + // * Whenever you introduce commands/parameters, taking a little time to write meaningful + // * documentation can make a big difference for the developer experience in your repo. + // */ + // "description": "This is an example custom command that runs separately for each project", + // + // /** + // * By default, Rush operations acquire a lock file which prevents multiple commands from executing simultaneously + // * in the same repo folder. (For example, it would be a mistake to run "rush install" and "rush build" at the + // * same time.) If your command makes sense to run concurrently with other operations, + // * set "safeForSimultaneousRushProcesses" to true to disable this protection. + // * + // * In particular, this is needed for custom scripts that invoke other Rush commands. + // */ + // "safeForSimultaneousRushProcesses": false, + // + // /** + // * (Required) If true, then this command is safe to be run in parallel, i.e. executed + // * simultaneously for multiple projects. Similar to "rush build", regardless of parallelism + // * projects will not start processing until their dependencies have completed processing. + // */ + // "enableParallelism": false, + // + // /** + // * Normally projects will be processed according to their dependency order: a given project will not start + // * processing the command until all of its dependencies have completed. This restriction doesn't apply for + // * certain operations, for example a "clean" task that deletes output files. In this case + // * you can set "ignoreDependencyOrder" to true to increase parallelism. + // */ + // "ignoreDependencyOrder": false, + // + // /** + // * Normally Rush requires that each project's package.json has a "scripts" entry matching + // * the custom command name. To disable this check, set "ignoreMissingScript" to true; + // * projects with a missing definition will be skipped. + // */ + // "ignoreMissingScript": false, + // + // /** + // * When invoking shell scripts, Rush uses a heuristic to distinguish errors from warnings: + // * - If the shell script returns a nonzero process exit code, Rush interprets this as "one or more errors". + // * Error output is displayed in red, and it prevents Rush from attempting to process any downstream projects. + // * - If the shell script returns a zero process exit code but writes something to its stderr stream, + // * Rush interprets this as "one or more warnings". Warning output is printed in yellow, but does NOT prevent + // * Rush from processing downstream projects. + // * + // * Thus, warnings do not interfere with local development, but they will cause a CI job to fail, because + // * the Rush process itself returns a nonzero exit code if there are any warnings or errors. This is by design. + // * In an active monorepo, we've found that if you allow any warnings in your main branch, it inadvertently + // * teaches developers to ignore warnings, which quickly leads to a situation where so many "expected" warnings + // * have accumulated that warnings no longer serve any useful purpose. + // * + // * Sometimes a poorly behaved task will write output to stderr even though its operation was successful. + // * In that case, it's strongly recommended to fix the task. However, as a workaround you can set + // * allowWarningsInSuccessfulBuild=true, which causes Rush to return a nonzero exit code for errors only. + // * + // * Note: The default value is false. In Rush 5.7.x and earlier, the default value was true. + // */ + // "allowWarningsInSuccessfulBuild": false, + // + // /** + // * If true then this command will be incremental like the built-in "build" command + // */ + // "incremental": false, + // + // /** + // * (EXPERIMENTAL) Normally Rush terminates after the command finishes. If this option is set to "true" Rush + // * will instead enter a loop where it watches the file system for changes to the selected projects. Whenever a + // * change is detected, the command will be invoked again for the changed project and any selected projects that + // * directly or indirectly depend on it. + // * + // * For details, refer to the website article "Using watch mode". + // */ + // "watchForChanges": false, + // + // /** + // * (EXPERIMENTAL) Disable cache for this action. This may be useful if this command affects state outside of + // * projects' own folders. + // */ + // "disableBuildCache": false + // }, + // + // { + // /** + // * (Required) Determines the type of custom command. + // * Rush's "global" commands are invoked once for the entire repo. + // */ + // "commandKind": "global", + // + // "name": "my-global-command", + // "summary": "Example global custom command", + // "description": "This is an example custom command that runs once for the entire repo", + // + // "safeForSimultaneousRushProcesses": false, + // + // /** + // * (Required) A script that will be invoked using the OS shell. The working directory will be + // * the folder that contains rush.json. If custom parameters are associated with this command, their + // * values will be appended to the end of this string. + // */ + // "shellCommand": "node common/scripts/my-global-command.js", + // + // /** + // * If your "shellCommand" script depends on NPM packages, the recommended best practice is + // * to make it into a regular Rush project that builds using your normal toolchain. In cases where + // * the command needs to work without first having to run "rush build", the recommended practice + // * is to publish the project to an NPM registry and use common/scripts/install-run.js to launch it. + // * + // * Autoinstallers offer another possibility: They are folders under "common/autoinstallers" with + // * a package.json file and shrinkwrap file. Rush will automatically invoke the package manager to + // * install these dependencies before an associated command is invoked. Autoinstallers have the + // * advantage that they work even in a branch where "rush install" is broken, which makes them a + // * good solution for Git hook scripts. But they have the disadvantages of not being buildable + // * projects, and of increasing the overall installation footprint for your monorepo. + // * + // * The "autoinstallerName" setting must not contain a path and must be a valid NPM package name. + // * For example, the name "my-task" would map to "common/autoinstallers/my-task/package.json", and + // * the "common/autoinstallers/my-task/node_modules/.bin" folder would be added to the shell PATH when + // * invoking the "shellCommand". + // */ + // // "autoinstallerName": "my-task" + // } + ], + + "phases": [], + + /** + * Custom "parameters" introduce new parameters for specified Rush command-line commands. + * For example, you might define a "--production" parameter for the "rush build" command. + */ + "parameters": [ + // { + // /** + // * (Required) Determines the type of custom parameter. + // * A "flag" is a custom command-line parameter whose presence acts as an on/off switch. + // */ + // "parameterKind": "flag", + // + // /** + // * (Required) The long name of the parameter. It must be lower-case and use dash delimiters. + // */ + // "longName": "--my-flag", + // + // /** + // * An optional alternative short name for the parameter. It must be a dash followed by a single + // * lower-case or upper-case letter, which is case-sensitive. + // * + // * NOTE: The Rush developers recommend that automation scripts should always use the long name + // * to improve readability. The short name is only intended as a convenience for humans. + // * The alphabet letters run out quickly, and are difficult to memorize, so *only* use + // * a short name if you expect the parameter to be needed very often in everyday operations. + // */ + // "shortName": "-m", + // + // /** + // * (Required) A long description to be shown in the command-line help. + // * + // * Whenever you introduce commands/parameters, taking a little time to write meaningful + // * documentation can make a big difference for the developer experience in your repo. + // */ + // "description": "A custom flag parameter that is passed to the scripts that are invoked when building projects", + // + // /** + // * (Required) A list of custom commands and/or built-in Rush commands that this parameter may + // * be used with. The parameter will be appended to the shell command that Rush invokes. + // */ + // "associatedCommands": ["build", "rebuild"] + // }, + // + // { + // /** + // * (Required) Determines the type of custom parameter. + // * A "string" is a custom command-line parameter whose value is a simple text string. + // */ + // "parameterKind": "string", + // "longName": "--my-string", + // "description": "A custom string parameter for the \"my-global-command\" custom command", + // + // "associatedCommands": ["my-global-command"], + // + // /** + // * The name of the argument, which will be shown in the command-line help. + // * + // * For example, if the parameter name is '--count" and the argument name is "NUMBER", + // * then the command-line help would display "--count NUMBER". The argument name must + // * be comprised of upper-case letters, numbers, and underscores. It should be kept short. + // */ + // "argumentName": "SOME_TEXT", + // + // /** + // * If true, this parameter must be included with the command. The default is false. + // */ + // "required": false + // }, + // + // { + // /** + // * (Required) Determines the type of custom parameter. + // * A "choice" is a custom command-line parameter whose argument must be chosen from a list of + // * allowable alternatives. + // */ + // "parameterKind": "choice", + // "longName": "--my-choice", + // "description": "A custom choice parameter for the \"my-global-command\" custom command", + // + // "associatedCommands": ["my-global-command"], + // + // /** + // * If true, this parameter must be included with the command. The default is false. + // */ + // "required": false, + // + // /** + // * Normally if a parameter is omitted from the command line, it will not be passed + // * to the shell command. this value will be inserted by default. Whereas if a "defaultValue" + // * is defined, the parameter will always be passed to the shell command, and will use the + // * default value if unspecified. The value must be one of the defined alternatives. + // */ + // "defaultValue": "vanilla", + // + // /** + // * (Required) A list of alternative argument values that can be chosen for this parameter. + // */ + // "alternatives": [ + // { + // /** + // * A token that is one of the alternatives that can be used with the choice parameter, + // * e.g. "vanilla" in "--flavor vanilla". + // */ + // "name": "vanilla", + // + // /** + // * A detailed description for the alternative that can be shown in the command-line help. + // * + // * Whenever you introduce commands/parameters, taking a little time to write meaningful + // * documentation can make a big difference for the developer experience in your repo. + // */ + // "description": "Use the vanilla flavor (the default)" + // }, + // + // { + // "name": "chocolate", + // "description": "Use the chocolate flavor" + // }, + // + // { + // "name": "strawberry", + // "description": "Use the strawberry flavor" + // } + // ] + // } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml new file mode 100644 index 00000000000..5918b7e8af7 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml @@ -0,0 +1,34 @@ +lockfileVersion: 5.4 + +importers: + + .: + specifiers: {} + + ../../projects/a: + specifiers: {} + + ../../projects/b: + specifiers: {} + + ../../projects/c: + specifiers: + b: workspace:* + dependencies: + b: link:../b + + ../../projects/d: + specifiers: + b: workspace:* + c: workspace:* + dependencies: + b: link:../b + c: link:../c + + ../../projects/e: + specifiers: + b: workspace:* + d: workspace:* + dependencies: + b: link:../b + d: link:../d diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/repo-state.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/repo-state.json new file mode 100644 index 00000000000..0e7b144099d --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/repo-state.json @@ -0,0 +1,4 @@ +// DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. +{ + "preferredVersionsHash": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f" +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush-pnpm.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush-pnpm.js new file mode 100644 index 00000000000..5c149955de6 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush-pnpm.js @@ -0,0 +1,28 @@ +// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED. +// +// This script is intended for usage in an automated build environment where the Rush command may not have +// been preinstalled, or may have an unpredictable version. This script will automatically install the version of Rush +// specified in the rush.json configuration file (if not already installed), and then pass a command-line to the +// rush-pnpm command. +// +// An example usage would be: +// +// node common/scripts/install-run-rush-pnpm.js pnpm-command +// +// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/ + +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +var __webpack_exports__ = {}; +/*!*****************************************************!*\ + !*** ./lib-esnext/scripts/install-run-rush-pnpm.js ***! + \*****************************************************/ + +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See the @microsoft/rush package's LICENSE file for license information. +require('./install-run-rush'); +//# sourceMappingURL=install-run-rush-pnpm.js.map +module.exports = __webpack_exports__; +/******/ })() +; +//# sourceMappingURL=install-run-rush-pnpm.js.map \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush.js new file mode 100644 index 00000000000..cada1eded21 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush.js @@ -0,0 +1,214 @@ +// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED. +// +// This script is intended for usage in an automated build environment where the Rush command may not have +// been preinstalled, or may have an unpredictable version. This script will automatically install the version of Rush +// specified in the rush.json configuration file (if not already installed), and then pass a command-line to it. +// An example usage would be: +// +// node common/scripts/install-run-rush.js install +// +// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/ + +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ({ + +/***/ 657147: +/*!*********************!*\ + !*** external "fs" ***! + \*********************/ +/***/ ((module) => { + +module.exports = require("fs"); + +/***/ }), + +/***/ 371017: +/*!***********************!*\ + !*** external "path" ***! + \***********************/ +/***/ ((module) => { + +module.exports = require("path"); + +/***/ }) + +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ var cachedModule = __webpack_module_cache__[moduleId]; +/******/ if (cachedModule !== undefined) { +/******/ return cachedModule.exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ __webpack_modules__[moduleId](module, module.exports, __webpack_require__); +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ /* webpack/runtime/compat get default export */ +/******/ (() => { +/******/ // getDefaultExport function for compatibility with non-harmony modules +/******/ __webpack_require__.n = (module) => { +/******/ var getter = module && module.__esModule ? +/******/ () => (module['default']) : +/******/ () => (module); +/******/ __webpack_require__.d(getter, { a: getter }); +/******/ return getter; +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/define property getters */ +/******/ (() => { +/******/ // define getter functions for harmony exports +/******/ __webpack_require__.d = (exports, definition) => { +/******/ for(var key in definition) { +/******/ if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) { +/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] }); +/******/ } +/******/ } +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/hasOwnProperty shorthand */ +/******/ (() => { +/******/ __webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop)) +/******/ })(); +/******/ +/******/ /* webpack/runtime/make namespace object */ +/******/ (() => { +/******/ // define __esModule on exports +/******/ __webpack_require__.r = (exports) => { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ })(); +/******/ +/************************************************************************/ +var __webpack_exports__ = {}; +// This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk. +(() => { +/*!************************************************!*\ + !*** ./lib-esnext/scripts/install-run-rush.js ***! + \************************************************/ +__webpack_require__.r(__webpack_exports__); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! path */ 371017); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_0__); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! fs */ 657147); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_1___default = /*#__PURE__*/__webpack_require__.n(fs__WEBPACK_IMPORTED_MODULE_1__); +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See the @microsoft/rush package's LICENSE file for license information. + + +const { installAndRun, findRushJsonFolder, RUSH_JSON_FILENAME, runWithErrorAndStatusCode } = require('./install-run'); +const PACKAGE_NAME = '@microsoft/rush'; +const RUSH_PREVIEW_VERSION = 'RUSH_PREVIEW_VERSION'; +const INSTALL_RUN_RUSH_LOCKFILE_PATH_VARIABLE = 'INSTALL_RUN_RUSH_LOCKFILE_PATH'; +function _getRushVersion(logger) { + const rushPreviewVersion = process.env[RUSH_PREVIEW_VERSION]; + if (rushPreviewVersion !== undefined) { + logger.info(`Using Rush version from environment variable ${RUSH_PREVIEW_VERSION}=${rushPreviewVersion}`); + return rushPreviewVersion; + } + const rushJsonFolder = findRushJsonFolder(); + const rushJsonPath = path__WEBPACK_IMPORTED_MODULE_0__.join(rushJsonFolder, RUSH_JSON_FILENAME); + try { + const rushJsonContents = fs__WEBPACK_IMPORTED_MODULE_1__.readFileSync(rushJsonPath, 'utf-8'); + // Use a regular expression to parse out the rushVersion value because rush.json supports comments, + // but JSON.parse does not and we don't want to pull in more dependencies than we need to in this script. + const rushJsonMatches = rushJsonContents.match(/\"rushVersion\"\s*\:\s*\"([0-9a-zA-Z.+\-]+)\"/); + return rushJsonMatches[1]; + } + catch (e) { + throw new Error(`Unable to determine the required version of Rush from rush.json (${rushJsonFolder}). ` + + "The 'rushVersion' field is either not assigned in rush.json or was specified " + + 'using an unexpected syntax.'); + } +} +function _getBin(scriptName) { + switch (scriptName.toLowerCase()) { + case 'install-run-rush-pnpm.js': + return 'rush-pnpm'; + case 'install-run-rushx.js': + return 'rushx'; + default: + return 'rush'; + } +} +function _run() { + const [nodePath /* Ex: /bin/node */, scriptPath /* /repo/common/scripts/install-run-rush.js */, ...packageBinArgs /* [build, --to, myproject] */] = process.argv; + // Detect if this script was directly invoked, or if the install-run-rushx script was invokved to select the + // appropriate binary inside the rush package to run + const scriptName = path__WEBPACK_IMPORTED_MODULE_0__.basename(scriptPath); + const bin = _getBin(scriptName); + if (!nodePath || !scriptPath) { + throw new Error('Unexpected exception: could not detect node path or script path'); + } + let commandFound = false; + let logger = { info: console.log, error: console.error }; + for (const arg of packageBinArgs) { + if (arg === '-q' || arg === '--quiet') { + // The -q/--quiet flag is supported by both `rush` and `rushx`, and will suppress + // any normal informational/diagnostic information printed during startup. + // + // To maintain the same user experience, the install-run* scripts pass along this + // flag but also use it to suppress any diagnostic information normally printed + // to stdout. + logger = { + info: () => { }, + error: console.error + }; + } + else if (!arg.startsWith('-') || arg === '-h' || arg === '--help') { + // We either found something that looks like a command (i.e. - doesn't start with a "-"), + // or we found the -h/--help flag, which can be run without a command + commandFound = true; + } + } + if (!commandFound) { + console.log(`Usage: ${scriptName} [args...]`); + if (scriptName === 'install-run-rush-pnpm.js') { + console.log(`Example: ${scriptName} pnpm-command`); + } + else if (scriptName === 'install-run-rush.js') { + console.log(`Example: ${scriptName} build --to myproject`); + } + else { + console.log(`Example: ${scriptName} custom-command`); + } + process.exit(1); + } + runWithErrorAndStatusCode(logger, () => { + const version = _getRushVersion(logger); + logger.info(`The rush.json configuration requests Rush version ${version}`); + const lockFilePath = process.env[INSTALL_RUN_RUSH_LOCKFILE_PATH_VARIABLE]; + if (lockFilePath) { + logger.info(`Found ${INSTALL_RUN_RUSH_LOCKFILE_PATH_VARIABLE}="${lockFilePath}", installing with lockfile.`); + } + return installAndRun(logger, PACKAGE_NAME, version, bin, packageBinArgs, lockFilePath); + }); +} +_run(); +//# sourceMappingURL=install-run-rush.js.map +})(); + +module.exports = __webpack_exports__; +/******/ })() +; +//# sourceMappingURL=install-run-rush.js.map \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rushx.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rushx.js new file mode 100644 index 00000000000..b05df262bc2 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rushx.js @@ -0,0 +1,28 @@ +// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED. +// +// This script is intended for usage in an automated build environment where the Rush command may not have +// been preinstalled, or may have an unpredictable version. This script will automatically install the version of Rush +// specified in the rush.json configuration file (if not already installed), and then pass a command-line to the +// rushx command. +// +// An example usage would be: +// +// node common/scripts/install-run-rushx.js custom-command +// +// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/ + +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +var __webpack_exports__ = {}; +/*!*************************************************!*\ + !*** ./lib-esnext/scripts/install-run-rushx.js ***! + \*************************************************/ + +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See the @microsoft/rush package's LICENSE file for license information. +require('./install-run-rush'); +//# sourceMappingURL=install-run-rushx.js.map +module.exports = __webpack_exports__; +/******/ })() +; +//# sourceMappingURL=install-run-rushx.js.map \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js new file mode 100644 index 00000000000..bcd982b369e --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js @@ -0,0 +1,645 @@ +// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED. +// +// This script is intended for usage in an automated build environment where a Node tool may not have +// been preinstalled, or may have an unpredictable version. This script will automatically install the specified +// version of the specified tool (if not already installed), and then pass a command-line to it. +// An example usage would be: +// +// node common/scripts/install-run.js qrcode@1.2.2 qrcode https://rushjs.io +// +// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/ + +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ({ + +/***/ 679877: +/*!************************************************!*\ + !*** ./lib-esnext/utilities/npmrcUtilities.js ***! + \************************************************/ +/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => { + +__webpack_require__.r(__webpack_exports__); +/* harmony export */ __webpack_require__.d(__webpack_exports__, { +/* harmony export */ "syncNpmrc": () => (/* binding */ syncNpmrc) +/* harmony export */ }); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! fs */ 657147); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(fs__WEBPACK_IMPORTED_MODULE_0__); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! path */ 371017); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_1___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_1__); +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. +// IMPORTANT - do not use any non-built-in libraries in this file + + +/** + * As a workaround, copyAndTrimNpmrcFile() copies the .npmrc file to the target folder, and also trims + * unusable lines from the .npmrc file. + * + * Why are we trimming the .npmrc lines? NPM allows environment variables to be specified in + * the .npmrc file to provide different authentication tokens for different registry. + * However, if the environment variable is undefined, it expands to an empty string, which + * produces a valid-looking mapping with an invalid URL that causes an error. Instead, + * we'd prefer to skip that line and continue looking in other places such as the user's + * home directory. + * + * @returns + * The text of the the .npmrc. + */ +function _copyAndTrimNpmrcFile(logger, sourceNpmrcPath, targetNpmrcPath) { + logger.info(`Transforming ${sourceNpmrcPath}`); // Verbose + logger.info(` --> "${targetNpmrcPath}"`); + let npmrcFileLines = fs__WEBPACK_IMPORTED_MODULE_0__.readFileSync(sourceNpmrcPath).toString().split('\n'); + npmrcFileLines = npmrcFileLines.map((line) => (line || '').trim()); + const resultLines = []; + // This finds environment variable tokens that look like "${VAR_NAME}" + const expansionRegExp = /\$\{([^\}]+)\}/g; + // Comment lines start with "#" or ";" + const commentRegExp = /^\s*[#;]/; + // Trim out lines that reference environment variables that aren't defined + for (const line of npmrcFileLines) { + let lineShouldBeTrimmed = false; + // Ignore comment lines + if (!commentRegExp.test(line)) { + const environmentVariables = line.match(expansionRegExp); + if (environmentVariables) { + for (const token of environmentVariables) { + // Remove the leading "${" and the trailing "}" from the token + const environmentVariableName = token.substring(2, token.length - 1); + // Is the environment variable defined? + if (!process.env[environmentVariableName]) { + // No, so trim this line + lineShouldBeTrimmed = true; + break; + } + } + } + } + if (lineShouldBeTrimmed) { + // Example output: + // "; MISSING ENVIRONMENT VARIABLE: //my-registry.com/npm/:_authToken=${MY_AUTH_TOKEN}" + resultLines.push('; MISSING ENVIRONMENT VARIABLE: ' + line); + } + else { + resultLines.push(line); + } + } + const combinedNpmrc = resultLines.join('\n'); + fs__WEBPACK_IMPORTED_MODULE_0__.writeFileSync(targetNpmrcPath, combinedNpmrc); + return combinedNpmrc; +} +/** + * syncNpmrc() copies the .npmrc file to the target folder, and also trims unusable lines from the .npmrc file. + * If the source .npmrc file not exist, then syncNpmrc() will delete an .npmrc that is found in the target folder. + * + * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._syncNpmrc() + * + * @returns + * The text of the the synced .npmrc, if one exists. If one does not exist, then undefined is returned. + */ +function syncNpmrc(sourceNpmrcFolder, targetNpmrcFolder, useNpmrcPublish, logger = { + info: console.log, + error: console.error +}) { + const sourceNpmrcPath = path__WEBPACK_IMPORTED_MODULE_1__.join(sourceNpmrcFolder, !useNpmrcPublish ? '.npmrc' : '.npmrc-publish'); + const targetNpmrcPath = path__WEBPACK_IMPORTED_MODULE_1__.join(targetNpmrcFolder, '.npmrc'); + try { + if (fs__WEBPACK_IMPORTED_MODULE_0__.existsSync(sourceNpmrcPath)) { + return _copyAndTrimNpmrcFile(logger, sourceNpmrcPath, targetNpmrcPath); + } + else if (fs__WEBPACK_IMPORTED_MODULE_0__.existsSync(targetNpmrcPath)) { + // If the source .npmrc doesn't exist and there is one in the target, delete the one in the target + logger.info(`Deleting ${targetNpmrcPath}`); // Verbose + fs__WEBPACK_IMPORTED_MODULE_0__.unlinkSync(targetNpmrcPath); + } + } + catch (e) { + throw new Error(`Error syncing .npmrc file: ${e}`); + } +} +//# sourceMappingURL=npmrcUtilities.js.map + +/***/ }), + +/***/ 532081: +/*!********************************!*\ + !*** external "child_process" ***! + \********************************/ +/***/ ((module) => { + +module.exports = require("child_process"); + +/***/ }), + +/***/ 657147: +/*!*********************!*\ + !*** external "fs" ***! + \*********************/ +/***/ ((module) => { + +module.exports = require("fs"); + +/***/ }), + +/***/ 822037: +/*!*********************!*\ + !*** external "os" ***! + \*********************/ +/***/ ((module) => { + +module.exports = require("os"); + +/***/ }), + +/***/ 371017: +/*!***********************!*\ + !*** external "path" ***! + \***********************/ +/***/ ((module) => { + +module.exports = require("path"); + +/***/ }) + +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ var cachedModule = __webpack_module_cache__[moduleId]; +/******/ if (cachedModule !== undefined) { +/******/ return cachedModule.exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ __webpack_modules__[moduleId](module, module.exports, __webpack_require__); +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ /* webpack/runtime/compat get default export */ +/******/ (() => { +/******/ // getDefaultExport function for compatibility with non-harmony modules +/******/ __webpack_require__.n = (module) => { +/******/ var getter = module && module.__esModule ? +/******/ () => (module['default']) : +/******/ () => (module); +/******/ __webpack_require__.d(getter, { a: getter }); +/******/ return getter; +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/define property getters */ +/******/ (() => { +/******/ // define getter functions for harmony exports +/******/ __webpack_require__.d = (exports, definition) => { +/******/ for(var key in definition) { +/******/ if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) { +/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] }); +/******/ } +/******/ } +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/hasOwnProperty shorthand */ +/******/ (() => { +/******/ __webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop)) +/******/ })(); +/******/ +/******/ /* webpack/runtime/make namespace object */ +/******/ (() => { +/******/ // define __esModule on exports +/******/ __webpack_require__.r = (exports) => { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ })(); +/******/ +/************************************************************************/ +var __webpack_exports__ = {}; +// This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk. +(() => { +/*!*******************************************!*\ + !*** ./lib-esnext/scripts/install-run.js ***! + \*******************************************/ +__webpack_require__.r(__webpack_exports__); +/* harmony export */ __webpack_require__.d(__webpack_exports__, { +/* harmony export */ "RUSH_JSON_FILENAME": () => (/* binding */ RUSH_JSON_FILENAME), +/* harmony export */ "getNpmPath": () => (/* binding */ getNpmPath), +/* harmony export */ "findRushJsonFolder": () => (/* binding */ findRushJsonFolder), +/* harmony export */ "installAndRun": () => (/* binding */ installAndRun), +/* harmony export */ "runWithErrorAndStatusCode": () => (/* binding */ runWithErrorAndStatusCode) +/* harmony export */ }); +/* harmony import */ var child_process__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! child_process */ 532081); +/* harmony import */ var child_process__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(child_process__WEBPACK_IMPORTED_MODULE_0__); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! fs */ 657147); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_1___default = /*#__PURE__*/__webpack_require__.n(fs__WEBPACK_IMPORTED_MODULE_1__); +/* harmony import */ var os__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! os */ 822037); +/* harmony import */ var os__WEBPACK_IMPORTED_MODULE_2___default = /*#__PURE__*/__webpack_require__.n(os__WEBPACK_IMPORTED_MODULE_2__); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! path */ 371017); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_3___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_3__); +/* harmony import */ var _utilities_npmrcUtilities__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(/*! ../utilities/npmrcUtilities */ 679877); +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See the @microsoft/rush package's LICENSE file for license information. + + + + + +const RUSH_JSON_FILENAME = 'rush.json'; +const RUSH_TEMP_FOLDER_ENV_VARIABLE_NAME = 'RUSH_TEMP_FOLDER'; +const INSTALL_RUN_LOCKFILE_PATH_VARIABLE = 'INSTALL_RUN_LOCKFILE_PATH'; +const INSTALLED_FLAG_FILENAME = 'installed.flag'; +const NODE_MODULES_FOLDER_NAME = 'node_modules'; +const PACKAGE_JSON_FILENAME = 'package.json'; +/** + * Parse a package specifier (in the form of name\@version) into name and version parts. + */ +function _parsePackageSpecifier(rawPackageSpecifier) { + rawPackageSpecifier = (rawPackageSpecifier || '').trim(); + const separatorIndex = rawPackageSpecifier.lastIndexOf('@'); + let name; + let version = undefined; + if (separatorIndex === 0) { + // The specifier starts with a scope and doesn't have a version specified + name = rawPackageSpecifier; + } + else if (separatorIndex === -1) { + // The specifier doesn't have a version + name = rawPackageSpecifier; + } + else { + name = rawPackageSpecifier.substring(0, separatorIndex); + version = rawPackageSpecifier.substring(separatorIndex + 1); + } + if (!name) { + throw new Error(`Invalid package specifier: ${rawPackageSpecifier}`); + } + return { name, version }; +} +let _npmPath = undefined; +/** + * Get the absolute path to the npm executable + */ +function getNpmPath() { + if (!_npmPath) { + try { + if (os__WEBPACK_IMPORTED_MODULE_2__.platform() === 'win32') { + // We're on Windows + const whereOutput = child_process__WEBPACK_IMPORTED_MODULE_0__.execSync('where npm', { stdio: [] }).toString(); + const lines = whereOutput.split(os__WEBPACK_IMPORTED_MODULE_2__.EOL).filter((line) => !!line); + // take the last result, we are looking for a .cmd command + // see https://github.com/microsoft/rushstack/issues/759 + _npmPath = lines[lines.length - 1]; + } + else { + // We aren't on Windows - assume we're on *NIX or Darwin + _npmPath = child_process__WEBPACK_IMPORTED_MODULE_0__.execSync('command -v npm', { stdio: [] }).toString(); + } + } + catch (e) { + throw new Error(`Unable to determine the path to the NPM tool: ${e}`); + } + _npmPath = _npmPath.trim(); + if (!fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(_npmPath)) { + throw new Error('The NPM executable does not exist'); + } + } + return _npmPath; +} +function _ensureFolder(folderPath) { + if (!fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(folderPath)) { + const parentDir = path__WEBPACK_IMPORTED_MODULE_3__.dirname(folderPath); + _ensureFolder(parentDir); + fs__WEBPACK_IMPORTED_MODULE_1__.mkdirSync(folderPath); + } +} +/** + * Create missing directories under the specified base directory, and return the resolved directory. + * + * Does not support "." or ".." path segments. + * Assumes the baseFolder exists. + */ +function _ensureAndJoinPath(baseFolder, ...pathSegments) { + let joinedPath = baseFolder; + try { + for (let pathSegment of pathSegments) { + pathSegment = pathSegment.replace(/[\\\/]/g, '+'); + joinedPath = path__WEBPACK_IMPORTED_MODULE_3__.join(joinedPath, pathSegment); + if (!fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(joinedPath)) { + fs__WEBPACK_IMPORTED_MODULE_1__.mkdirSync(joinedPath); + } + } + } + catch (e) { + throw new Error(`Error building local installation folder (${path__WEBPACK_IMPORTED_MODULE_3__.join(baseFolder, ...pathSegments)}): ${e}`); + } + return joinedPath; +} +function _getRushTempFolder(rushCommonFolder) { + const rushTempFolder = process.env[RUSH_TEMP_FOLDER_ENV_VARIABLE_NAME]; + if (rushTempFolder !== undefined) { + _ensureFolder(rushTempFolder); + return rushTempFolder; + } + else { + return _ensureAndJoinPath(rushCommonFolder, 'temp'); + } +} +/** + * Resolve a package specifier to a static version + */ +function _resolvePackageVersion(logger, rushCommonFolder, { name, version }) { + if (!version) { + version = '*'; // If no version is specified, use the latest version + } + if (version.match(/^[a-zA-Z0-9\-\+\.]+$/)) { + // If the version contains only characters that we recognize to be used in static version specifiers, + // pass the version through + return version; + } + else { + // version resolves to + try { + const rushTempFolder = _getRushTempFolder(rushCommonFolder); + const sourceNpmrcFolder = path__WEBPACK_IMPORTED_MODULE_3__.join(rushCommonFolder, 'config', 'rush'); + (0,_utilities_npmrcUtilities__WEBPACK_IMPORTED_MODULE_4__.syncNpmrc)(sourceNpmrcFolder, rushTempFolder, undefined, logger); + const npmPath = getNpmPath(); + // This returns something that looks like: + // @microsoft/rush@3.0.0 '3.0.0' + // @microsoft/rush@3.0.1 '3.0.1' + // ... + // @microsoft/rush@3.0.20 '3.0.20' + // + const npmVersionSpawnResult = child_process__WEBPACK_IMPORTED_MODULE_0__.spawnSync(npmPath, ['view', `${name}@${version}`, 'version', '--no-update-notifier'], { + cwd: rushTempFolder, + stdio: [] + }); + if (npmVersionSpawnResult.status !== 0) { + throw new Error(`"npm view" returned error code ${npmVersionSpawnResult.status}`); + } + const npmViewVersionOutput = npmVersionSpawnResult.stdout.toString(); + const versionLines = npmViewVersionOutput.split('\n').filter((line) => !!line); + const latestVersion = versionLines[versionLines.length - 1]; + if (!latestVersion) { + throw new Error('No versions found for the specified version range.'); + } + const versionMatches = latestVersion.match(/^.+\s\'(.+)\'$/); + if (!versionMatches) { + throw new Error(`Invalid npm output ${latestVersion}`); + } + return versionMatches[1]; + } + catch (e) { + throw new Error(`Unable to resolve version ${version} of package ${name}: ${e}`); + } + } +} +let _rushJsonFolder; +/** + * Find the absolute path to the folder containing rush.json + */ +function findRushJsonFolder() { + if (!_rushJsonFolder) { + let basePath = __dirname; + let tempPath = __dirname; + do { + const testRushJsonPath = path__WEBPACK_IMPORTED_MODULE_3__.join(basePath, RUSH_JSON_FILENAME); + if (fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(testRushJsonPath)) { + _rushJsonFolder = basePath; + break; + } + else { + basePath = tempPath; + } + } while (basePath !== (tempPath = path__WEBPACK_IMPORTED_MODULE_3__.dirname(basePath))); // Exit the loop when we hit the disk root + if (!_rushJsonFolder) { + throw new Error('Unable to find rush.json.'); + } + } + return _rushJsonFolder; +} +/** + * Detects if the package in the specified directory is installed + */ +function _isPackageAlreadyInstalled(packageInstallFolder) { + try { + const flagFilePath = path__WEBPACK_IMPORTED_MODULE_3__.join(packageInstallFolder, INSTALLED_FLAG_FILENAME); + if (!fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(flagFilePath)) { + return false; + } + const fileContents = fs__WEBPACK_IMPORTED_MODULE_1__.readFileSync(flagFilePath).toString(); + return fileContents.trim() === process.version; + } + catch (e) { + return false; + } +} +/** + * Delete a file. Fail silently if it does not exist. + */ +function _deleteFile(file) { + try { + fs__WEBPACK_IMPORTED_MODULE_1__.unlinkSync(file); + } + catch (err) { + if (err.code !== 'ENOENT' && err.code !== 'ENOTDIR') { + throw err; + } + } +} +/** + * Removes the following files and directories under the specified folder path: + * - installed.flag + * - + * - node_modules + */ +function _cleanInstallFolder(rushTempFolder, packageInstallFolder, lockFilePath) { + try { + const flagFile = path__WEBPACK_IMPORTED_MODULE_3__.resolve(packageInstallFolder, INSTALLED_FLAG_FILENAME); + _deleteFile(flagFile); + const packageLockFile = path__WEBPACK_IMPORTED_MODULE_3__.resolve(packageInstallFolder, 'package-lock.json'); + if (lockFilePath) { + fs__WEBPACK_IMPORTED_MODULE_1__.copyFileSync(lockFilePath, packageLockFile); + } + else { + // Not running `npm ci`, so need to cleanup + _deleteFile(packageLockFile); + const nodeModulesFolder = path__WEBPACK_IMPORTED_MODULE_3__.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME); + if (fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(nodeModulesFolder)) { + const rushRecyclerFolder = _ensureAndJoinPath(rushTempFolder, 'rush-recycler'); + fs__WEBPACK_IMPORTED_MODULE_1__.renameSync(nodeModulesFolder, path__WEBPACK_IMPORTED_MODULE_3__.join(rushRecyclerFolder, `install-run-${Date.now().toString()}`)); + } + } + } + catch (e) { + throw new Error(`Error cleaning the package install folder (${packageInstallFolder}): ${e}`); + } +} +function _createPackageJson(packageInstallFolder, name, version) { + try { + const packageJsonContents = { + name: 'ci-rush', + version: '0.0.0', + dependencies: { + [name]: version + }, + description: "DON'T WARN", + repository: "DON'T WARN", + license: 'MIT' + }; + const packageJsonPath = path__WEBPACK_IMPORTED_MODULE_3__.join(packageInstallFolder, PACKAGE_JSON_FILENAME); + fs__WEBPACK_IMPORTED_MODULE_1__.writeFileSync(packageJsonPath, JSON.stringify(packageJsonContents, undefined, 2)); + } + catch (e) { + throw new Error(`Unable to create package.json: ${e}`); + } +} +/** + * Run "npm install" in the package install folder. + */ +function _installPackage(logger, packageInstallFolder, name, version, command) { + try { + logger.info(`Installing ${name}...`); + const npmPath = getNpmPath(); + const result = child_process__WEBPACK_IMPORTED_MODULE_0__.spawnSync(npmPath, [command], { + stdio: 'inherit', + cwd: packageInstallFolder, + env: process.env + }); + if (result.status !== 0) { + throw new Error(`"npm ${command}" encountered an error`); + } + logger.info(`Successfully installed ${name}@${version}`); + } + catch (e) { + throw new Error(`Unable to install package: ${e}`); + } +} +/** + * Get the ".bin" path for the package. + */ +function _getBinPath(packageInstallFolder, binName) { + const binFolderPath = path__WEBPACK_IMPORTED_MODULE_3__.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME, '.bin'); + const resolvedBinName = os__WEBPACK_IMPORTED_MODULE_2__.platform() === 'win32' ? `${binName}.cmd` : binName; + return path__WEBPACK_IMPORTED_MODULE_3__.resolve(binFolderPath, resolvedBinName); +} +/** + * Write a flag file to the package's install directory, signifying that the install was successful. + */ +function _writeFlagFile(packageInstallFolder) { + try { + const flagFilePath = path__WEBPACK_IMPORTED_MODULE_3__.join(packageInstallFolder, INSTALLED_FLAG_FILENAME); + fs__WEBPACK_IMPORTED_MODULE_1__.writeFileSync(flagFilePath, process.version); + } + catch (e) { + throw new Error(`Unable to create installed.flag file in ${packageInstallFolder}`); + } +} +function installAndRun(logger, packageName, packageVersion, packageBinName, packageBinArgs, lockFilePath = process.env[INSTALL_RUN_LOCKFILE_PATH_VARIABLE]) { + const rushJsonFolder = findRushJsonFolder(); + const rushCommonFolder = path__WEBPACK_IMPORTED_MODULE_3__.join(rushJsonFolder, 'common'); + const rushTempFolder = _getRushTempFolder(rushCommonFolder); + const packageInstallFolder = _ensureAndJoinPath(rushTempFolder, 'install-run', `${packageName}@${packageVersion}`); + if (!_isPackageAlreadyInstalled(packageInstallFolder)) { + // The package isn't already installed + _cleanInstallFolder(rushTempFolder, packageInstallFolder, lockFilePath); + const sourceNpmrcFolder = path__WEBPACK_IMPORTED_MODULE_3__.join(rushCommonFolder, 'config', 'rush'); + (0,_utilities_npmrcUtilities__WEBPACK_IMPORTED_MODULE_4__.syncNpmrc)(sourceNpmrcFolder, packageInstallFolder, undefined, logger); + _createPackageJson(packageInstallFolder, packageName, packageVersion); + const command = lockFilePath ? 'ci' : 'install'; + _installPackage(logger, packageInstallFolder, packageName, packageVersion, command); + _writeFlagFile(packageInstallFolder); + } + const statusMessage = `Invoking "${packageBinName} ${packageBinArgs.join(' ')}"`; + const statusMessageLine = new Array(statusMessage.length + 1).join('-'); + logger.info('\n' + statusMessage + '\n' + statusMessageLine + '\n'); + const binPath = _getBinPath(packageInstallFolder, packageBinName); + const binFolderPath = path__WEBPACK_IMPORTED_MODULE_3__.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME, '.bin'); + // Windows environment variables are case-insensitive. Instead of using SpawnSyncOptions.env, we need to + // assign via the process.env proxy to ensure that we append to the right PATH key. + const originalEnvPath = process.env.PATH || ''; + let result; + try { + // Node.js on Windows can not spawn a file when the path has a space on it + // unless the path gets wrapped in a cmd friendly way and shell mode is used + const shouldUseShell = binPath.includes(' ') && os__WEBPACK_IMPORTED_MODULE_2__.platform() === 'win32'; + const platformBinPath = shouldUseShell ? `"${binPath}"` : binPath; + process.env.PATH = [binFolderPath, originalEnvPath].join(path__WEBPACK_IMPORTED_MODULE_3__.delimiter); + result = child_process__WEBPACK_IMPORTED_MODULE_0__.spawnSync(platformBinPath, packageBinArgs, { + stdio: 'inherit', + windowsVerbatimArguments: false, + shell: shouldUseShell, + cwd: process.cwd(), + env: process.env + }); + } + finally { + process.env.PATH = originalEnvPath; + } + if (result.status !== null) { + return result.status; + } + else { + throw result.error || new Error('An unknown error occurred.'); + } +} +function runWithErrorAndStatusCode(logger, fn) { + process.exitCode = 1; + try { + const exitCode = fn(); + process.exitCode = exitCode; + } + catch (e) { + logger.error('\n\n' + e.toString() + '\n\n'); + } +} +function _run() { + const [nodePath /* Ex: /bin/node */, scriptPath /* /repo/common/scripts/install-run-rush.js */, rawPackageSpecifier /* qrcode@^1.2.0 */, packageBinName /* qrcode */, ...packageBinArgs /* [-f, myproject/lib] */] = process.argv; + if (!nodePath) { + throw new Error('Unexpected exception: could not detect node path'); + } + if (path__WEBPACK_IMPORTED_MODULE_3__.basename(scriptPath).toLowerCase() !== 'install-run.js') { + // If install-run.js wasn't directly invoked, don't execute the rest of this function. Return control + // to the script that (presumably) imported this file + return; + } + if (process.argv.length < 4) { + console.log('Usage: install-run.js @ [args...]'); + console.log('Example: install-run.js qrcode@1.2.2 qrcode https://rushjs.io'); + process.exit(1); + } + const logger = { info: console.log, error: console.error }; + runWithErrorAndStatusCode(logger, () => { + const rushJsonFolder = findRushJsonFolder(); + const rushCommonFolder = _ensureAndJoinPath(rushJsonFolder, 'common'); + const packageSpecifier = _parsePackageSpecifier(rawPackageSpecifier); + const name = packageSpecifier.name; + const version = _resolvePackageVersion(logger, rushCommonFolder, packageSpecifier); + if (packageSpecifier.version !== version) { + console.log(`Resolved to ${name}@${version}`); + } + return installAndRun(logger, name, version, packageBinName, packageBinArgs); + }); +} +_run(); +//# sourceMappingURL=install-run.js.map +})(); + +module.exports = __webpack_exports__; +/******/ })() +; +//# sourceMappingURL=install-run.js.map \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json new file mode 100644 index 00000000000..f0036196559 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json @@ -0,0 +1,12 @@ +{ + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json new file mode 100644 index 00000000000..9472fdbd7e5 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json @@ -0,0 +1,8 @@ +{ + "name": "a", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json new file mode 100644 index 00000000000..f0036196559 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json @@ -0,0 +1,12 @@ +{ + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json new file mode 100644 index 00000000000..a8cd24f8006 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json @@ -0,0 +1,8 @@ +{ + "name": "b", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js new file mode 100644 index 00000000000..14855ff8c72 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js @@ -0,0 +1,12 @@ +/* eslint-env es6 */ +const path = require('path'); +const { FileSystem } = require('@rushstack/node-core-library'); + +console.log('start'); +setTimeout(() => { + const outputFolder = path.resolve(process.cwd(), 'dist'); + const outputFile = path.resolve(outputFolder, 'output.txt'); + FileSystem.ensureFolder(outputFolder); + FileSystem.writeFile(outputFile, 'Hello world!'); + console.log('done'); +}, 5000); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json new file mode 100644 index 00000000000..f0036196559 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json @@ -0,0 +1,12 @@ +{ + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json new file mode 100644 index 00000000000..b25880f2c84 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json @@ -0,0 +1,11 @@ +{ + "name": "c", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + }, + "dependencies": { + "b": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json new file mode 100644 index 00000000000..f0036196559 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json @@ -0,0 +1,12 @@ +{ + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json new file mode 100644 index 00000000000..6580cb02700 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json @@ -0,0 +1,12 @@ +{ + "name": "d", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + }, + "dependencies": { + "b": "workspace:*", + "c": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json new file mode 100644 index 00000000000..f0036196559 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json @@ -0,0 +1,12 @@ +{ + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json new file mode 100644 index 00000000000..69ac8b1cc97 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json @@ -0,0 +1,12 @@ +{ + "name": "e", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + }, + "dependencies": { + "b": "workspace:*", + "d": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json new file mode 100644 index 00000000000..9f839d273ed --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json @@ -0,0 +1,29 @@ +{ + "rushVersion": "5.80.0", + "pnpmVersion": "7.13.0", + "pnpmOptions": { + "useWorkspaces": true + }, + "projects": [ + { + "packageName": "a", + "projectFolder": "projects/a" + }, + { + "packageName": "b", + "projectFolder": "projects/b" + }, + { + "packageName": "c", + "projectFolder": "projects/c" + }, + { + "packageName": "d", + "projectFolder": "projects/d" + }, + { + "packageName": "e", + "projectFolder": "projects/e" + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/paths.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/paths.ts new file mode 100644 index 00000000000..858c5f62257 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/paths.ts @@ -0,0 +1,5 @@ +import * as path from 'path'; + +const sandboxRepoFolder: string = path.resolve(__dirname, '../sandbox/repo'); + +export { sandboxRepoFolder }; diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts new file mode 100644 index 00000000000..50c44d60968 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts @@ -0,0 +1,37 @@ +import { RushCommandLineParser } from '@microsoft/rush-lib/lib/cli/RushCommandLineParser'; +import * as rushLib from '@microsoft/rush-lib'; + +// Setup redis cobuild plugin +const builtInPluginConfigurations: rushLib._IBuiltInPluginConfiguration[] = []; + +const rushConfiguration: rushLib.RushConfiguration = rushLib.RushConfiguration.loadFromDefaultLocation({ + startingFolder: __dirname +}); +const project: rushLib.RushConfigurationProject | undefined = rushConfiguration.getProjectByName( + '@rushstack/rush-redis-cobuild-plugin' +); +if (!project) { + throw new Error('Project @rushstack/rush-redis-cobuild-plugin not found'); +} +builtInPluginConfigurations.push({ + packageName: '@rushstack/rush-redis-cobuild-plugin', + pluginName: 'rush-redis-cobuild-plugin', + pluginPackageFolder: project.projectFolder +}); + +async function rushRush(args: string[]): Promise { + const options: rushLib.ILaunchOptions = { + isManaged: false, + alreadyReportedNodeTooNewError: false, + builtInPluginConfigurations + }; + const parser: RushCommandLineParser = new RushCommandLineParser({ + alreadyReportedNodeTooNewError: options.alreadyReportedNodeTooNewError, + builtInPluginConfigurations: options.builtInPluginConfigurations + }); + console.log(`Executing: rush ${args.join(' ')}`); + await parser.execute(args).catch(console.error); // CommandLineParser.execute() should never reject the promise +} + +/* eslint-disable-next-line @typescript-eslint/no-floating-promises */ +rushRush(process.argv.slice(2)); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts index 820532437f1..474abae5bd9 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts @@ -2,26 +2,24 @@ import { RedisCobuildLockProvider, IRedisCobuildLockProviderOptions } from '@rushstack/rush-redis-cobuild-plugin'; -import { ConsoleTerminalProvider, ITerminal, Terminal } from '@rushstack/node-core-library'; -import { OperationStatus, ICobuildContext } from '@microsoft/rush-lib'; +import { ConsoleTerminalProvider } from '@rushstack/node-core-library'; +import { OperationStatus, ICobuildContext, RushSession } from '@microsoft/rush-lib'; const options: IRedisCobuildLockProviderOptions = { url: 'redis://localhost:6379', password: 'redis123' }; -const terminal: ITerminal = new Terminal( - new ConsoleTerminalProvider({ - verboseEnabled: true, - debugEnabled: true - }) -); +const rushSession: RushSession = new RushSession({ + terminalProvider: new ConsoleTerminalProvider(), + getIsDebugMode: () => true +}); async function main(): Promise { - const lockProvider: RedisCobuildLockProvider = new RedisCobuildLockProvider(options); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const lockProvider: RedisCobuildLockProvider = new RedisCobuildLockProvider(options, rushSession as any); await lockProvider.connectAsync(); const context: ICobuildContext = { - terminal, contextId: 'test-context-id', version: 1, cacheId: 'test-cache-id' diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json b/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json index 6314e94a07d..599b3beb19e 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json @@ -12,6 +12,7 @@ "declarationMap": true, "inlineSources": true, "experimentalDecorators": true, + "esModuleInterop": true, "strictNullChecks": true, "noUnusedLocals": true, "types": ["node"], diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 8d3c15d3bb0..585a7ec75b8 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1354,7 +1354,7 @@ importers: '@rushstack/node-core-library': workspace:* '@rushstack/rush-redis-cobuild-plugin': workspace:* '@types/http-proxy': ~1.17.8 - '@types/node': 12.20.24 + '@types/node': 14.18.36 eslint: ~8.7.0 http-proxy: ~1.18.1 typescript: ~4.8.4 @@ -1365,7 +1365,7 @@ importers: '@rushstack/node-core-library': link:../../libraries/node-core-library '@rushstack/rush-redis-cobuild-plugin': link:../../rush-plugins/rush-redis-cobuild-plugin '@types/http-proxy': 1.17.9 - '@types/node': 12.20.24 + '@types/node': 14.18.36 eslint: 8.7.0 http-proxy: 1.18.1 typescript: 4.8.4 diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index b967a5aeac1..09900023ba0 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -280,8 +280,6 @@ export interface ICobuildContext { // (undocumented) contextId: string; // (undocumented) - terminal: ITerminal; - // (undocumented) version: number; } diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index 4a2f99398e4..b19ae9aa26b 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -18,7 +18,7 @@ export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { // @beta (undocumented) export class RedisCobuildLockProvider implements ICobuildLockProvider { - constructor(options: IRedisCobuildLockProviderOptions); + constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession); // (undocumented) acquireLockAsync(context: ICobuildContext): Promise; // (undocumented) diff --git a/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json b/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json new file mode 100644 index 00000000000..13cce367de6 --- /dev/null +++ b/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json @@ -0,0 +1,25 @@ +/** + * This configuration file manages Rush's cobuild feature. + * More documentation is available on the Rush website: https://rushjs.io + */ + { + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/cobuild.schema.json", + + /** + * (Required) EXPERIMENTAL - Set this to true to enable the cobuild feature. + */ + "cobuildEnabled": false, + + /** + * (Required) Choose where cobuild lock will be acquired. + * + * The lock provider is registered by the rush plugins. + * For example, @rushstack/rush-redis-cobuild-plugin registers the "redis" lock provider. + */ + "cobuildLockProvider": "redis" + + /** + * Setting this property overrides the cobuild context ID. + */ + // "cobuildContextIdPattern": "" +} diff --git a/libraries/rush-lib/src/api/RushConfiguration.ts b/libraries/rush-lib/src/api/RushConfiguration.ts index 052e3e4ee29..c8892a9675c 100644 --- a/libraries/rush-lib/src/api/RushConfiguration.ts +++ b/libraries/rush-lib/src/api/RushConfiguration.ts @@ -57,6 +57,7 @@ const knownRushConfigFilenames: string[] = [ RushConstants.artifactoryFilename, RushConstants.browserApprovedPackagesFilename, RushConstants.buildCacheFilename, + RushConstants.cobuildFilename, RushConstants.commandLineFilename, RushConstants.commonVersionsFilename, RushConstants.experimentsFilename, diff --git a/libraries/rush-lib/src/cli/actions/InitAction.ts b/libraries/rush-lib/src/cli/actions/InitAction.ts index 8bb6f914ec9..3d3dc5b084f 100644 --- a/libraries/rush-lib/src/cli/actions/InitAction.ts +++ b/libraries/rush-lib/src/cli/actions/InitAction.ts @@ -156,6 +156,7 @@ export class InitAction extends BaseConfiglessRushAction { 'common/config/rush/[dot]npmrc-publish', 'common/config/rush/artifactory.json', 'common/config/rush/build-cache.json', + 'common/config/rush/cobuild.json', 'common/config/rush/command-line.json', 'common/config/rush/common-versions.json', 'common/config/rush/experiments.json', diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index 9343640b573..aa7875ff687 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { InternalError, ITerminal } from '@rushstack/node-core-library'; +import { InternalError } from '@rushstack/node-core-library'; import { RushConstants } from '../RushConstants'; import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; @@ -12,7 +12,6 @@ import type { ICobuildContext } from './ICobuildLockProvider'; export interface ICobuildLockOptions { cobuildConfiguration: CobuildConfiguration; projectBuildCache: ProjectBuildCache; - terminal: ITerminal; } export interface ICobuildCompletedState { @@ -27,7 +26,7 @@ export class CobuildLock { private _cobuildContext: ICobuildContext; public constructor(options: ICobuildLockOptions) { - const { cobuildConfiguration, projectBuildCache, terminal } = options; + const { cobuildConfiguration, projectBuildCache } = options; this.projectBuildCache = projectBuildCache; this.cobuildConfiguration = cobuildConfiguration; @@ -40,7 +39,6 @@ export class CobuildLock { } this._cobuildContext = { - terminal, contextId, cacheId, version: RushConstants.cobuildLockVersion diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index a36bc8ab702..be00a297c11 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import type { ITerminal } from '@rushstack/node-core-library'; import type { OperationStatus } from '../operations/OperationStatus'; /** @@ -11,7 +10,6 @@ export interface ICobuildContext { contextId: string; cacheId: string; version: number; - terminal: ITerminal; } /** diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 8bd625da4e2..9e0cd9ef5eb 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -90,6 +90,7 @@ export class AsyncOperationQueue if ( operation.status === OperationStatus.Blocked || + operation.status === OperationStatus.Skipped || operation.status === OperationStatus.Success || operation.status === OperationStatus.SuccessWithWarning || operation.status === OperationStatus.FromCache || diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index d61c6c196fb..fb44dcf46cf 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -188,12 +188,7 @@ export class OperationExecutionManager { const onOperationComplete: (record: OperationExecutionRecord) => void = ( record: OperationExecutionRecord ) => { - this._onOperationComplete(record); - - if (record.status !== OperationStatus.RemoteExecuting) { - // If the operation was not remote, then we can notify queue that it is complete - executionQueue.complete(); - } + this._onOperationComplete(record, executionQueue); }; await Async.forEachAsync( @@ -221,7 +216,7 @@ export class OperationExecutionManager { /** * Handles the result of the operation and propagates any relevant effects. */ - private _onOperationComplete(record: OperationExecutionRecord): void { + private _onOperationComplete(record: OperationExecutionRecord, executionQueue: AsyncOperationQueue): void { const { runner, name, status } = record; let blockCacheWrite: boolean = !runner.isCacheWriteAllowed; @@ -246,6 +241,7 @@ export class OperationExecutionManager { const blockedQueue: Set = new Set(record.consumers); for (const blockedRecord of blockedQueue) { if (blockedRecord.status === OperationStatus.Ready) { + executionQueue.complete(); this._completedOperations++; // Now that we have the concept of architectural no-ops, we could implement this by replacing @@ -338,5 +334,10 @@ export class OperationExecutionManager { item.dependencies.delete(record); } } + + if (record.status !== OperationStatus.RemoteExecuting) { + // If the operation was not remote, then we can notify queue that it is complete + executionQueue.complete(); + } } } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index fedab5e5fe2..063b76c6e4a 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -265,7 +265,7 @@ export class ShellOperationRunner implements IOperationRunner { terminal, trackedFiles ); - cobuildLock = await this._tryGetCobuildLockAsync(terminal, projectBuildCache); + cobuildLock = await this._tryGetCobuildLockAsync(projectBuildCache); } // If possible, we want to skip this operation -- either by restoring it from the @@ -341,6 +341,22 @@ export class ShellOperationRunner implements IOperationRunner { } } + if (this.isCacheWriteAllowed && cobuildLock) { + const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); + if (acquireSuccess) { + if (context.status === OperationStatus.RemoteExecuting) { + // This operation is used to marked remote executing, now change it to executing + context.status = OperationStatus.Executing; + } + runnerWatcher.addCallback(async () => { + await cobuildLock?.renewLockAsync(); + }); + } else { + // failed to acquire the lock, mark current operation to remote executing + return OperationStatus.RemoteExecuting; + } + } + // If the deps file exists, remove it before starting execution. FileSystem.deleteFile(currentDepsPath); @@ -360,22 +376,6 @@ export class ShellOperationRunner implements IOperationRunner { return OperationStatus.Success; } - if (this.isCacheWriteAllowed && cobuildLock) { - const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); - if (acquireSuccess) { - if (context.status === OperationStatus.RemoteExecuting) { - // This operation is used to marked remote executing, now change it to executing - context.status = OperationStatus.Executing; - } - runnerWatcher.addCallback(async () => { - await cobuildLock?.renewLockAsync(); - }); - } else { - // failed to acquire the lock, mark current operation to remote executing - return OperationStatus.RemoteExecuting; - } - } - // Run the operation terminal.writeLine('Invoking: ' + this._commandToRun); runnerWatcher.start(); @@ -607,7 +607,6 @@ export class ShellOperationRunner implements IOperationRunner { } private async _tryGetCobuildLockAsync( - terminal: ITerminal, projectBuildCache: ProjectBuildCache | undefined ): Promise { if (this._cobuildLock === UNINITIALIZED) { @@ -616,8 +615,7 @@ export class ShellOperationRunner implements IOperationRunner { if (projectBuildCache && this._cobuildConfiguration && this._cobuildConfiguration.cobuildEnabled) { this._cobuildLock = new CobuildLock({ cobuildConfiguration: this._cobuildConfiguration, - projectBuildCache: projectBuildCache, - terminal + projectBuildCache: projectBuildCache }); } } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index 3af20ccbb66..d7d22d2898c 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -3,7 +3,12 @@ import { createClient } from '@redis/client'; -import type { ICobuildLockProvider, ICobuildContext, ICobuildCompletedState } from '@rushstack/rush-sdk'; +import type { + ICobuildLockProvider, + ICobuildContext, + ICobuildCompletedState, + RushSession +} from '@rushstack/rush-sdk'; import type { RedisClientOptions, RedisClientType, @@ -11,6 +16,7 @@ import type { RedisModules, RedisScripts } from '@redis/client'; +import type { ITerminal } from '@rushstack/node-core-library'; /** * The redis client options @@ -26,18 +32,30 @@ const COMPLETED_STATE_SEPARATOR: string = ';'; */ export class RedisCobuildLockProvider implements ICobuildLockProvider { private readonly _options: IRedisCobuildLockProviderOptions; + private _terminal: ITerminal; private _redisClient: RedisClientType; private _lockKeyMap: WeakMap = new WeakMap(); private _completedKeyMap: WeakMap = new WeakMap(); - public constructor(options: IRedisCobuildLockProviderOptions) { + public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { this._options = options; - this._redisClient = createClient(this._options); + this._terminal = rushSession.getLogger('RedisCobuildLockProvider').terminal; + try { + this._redisClient = createClient(this._options); + } catch (e) { + throw new Error(`Failed to create redis client: ${e.message}`); + } } public async connectAsync(): Promise { await this._redisClient.connect(); + // Check the connection works at early stage + try { + await this._redisClient.ping(); + } catch (e) { + throw new Error(`Failed to connect to redis server: ${e.message}`); + } } public async disconnectAsync(): Promise { @@ -45,7 +63,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } public async acquireLockAsync(context: ICobuildContext): Promise { - const { terminal } = context; + const { _terminal: terminal } = this; const lockKey: string = this.getLockKey(context); const incrResult: number = await this._redisClient.incr(lockKey); const result: boolean = incrResult === 1; @@ -57,14 +75,14 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } public async renewLockAsync(context: ICobuildContext): Promise { - const { terminal } = context; + const { _terminal: terminal } = this; const lockKey: string = this.getLockKey(context); await this._redisClient.expire(lockKey, 30); terminal.writeDebugLine(`Renewed lock for ${lockKey}`); } public async releaseLockAsync(context: ICobuildContext): Promise { - const { terminal } = context; + const { _terminal: terminal } = this; const lockKey: string = this.getLockKey(context); await this._redisClient.set(lockKey, 0); terminal.writeDebugLine(`Released lock for ${lockKey}`); @@ -74,7 +92,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { context: ICobuildContext, state: ICobuildCompletedState ): Promise { - const { terminal } = context; + const { _terminal: terminal } = this; const key: string = this.getCompletedStateKey(context); const value: string = this._serializeCompletedState(state); await this._redisClient.set(key, value); @@ -82,12 +100,14 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } public async getCompletedStateAsync(context: ICobuildContext): Promise { + const { _terminal: terminal } = this; const key: string = this.getCompletedStateKey(context); let state: ICobuildCompletedState | undefined; const value: string | null = await this._redisClient.get(key); if (value) { state = this._deserializeCompletedState(value); } + terminal.writeDebugLine(`Get completed state for ${key}: ${value}`); return state; } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts index 1d1d69fdaeb..4e8f07d7f70 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts @@ -33,7 +33,7 @@ export class RushRedisCobuildPlugin implements IRushPlugin { rushSession.hooks.initialize.tap(PLUGIN_NAME, () => { rushSession.registerCobuildLockProviderFactory('redis', (): RedisCobuildLockProvider => { const options: IRushRedisCobuildPluginOptions = this._options; - return new RedisCobuildLockProviderModule.RedisCobuildLockProvider(options); + return new RedisCobuildLockProviderModule.RedisCobuildLockProvider(options, rushSession); }); }); } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index 5cd78099676..f908f762a1c 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -2,14 +2,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { Terminal, ConsoleTerminalProvider } from '@rushstack/node-core-library'; -import { ICobuildCompletedState, ICobuildContext, OperationStatus } from '@rushstack/rush-sdk'; +import { ConsoleTerminalProvider } from '@rushstack/node-core-library'; +import { ICobuildCompletedState, ICobuildContext, OperationStatus, RushSession } from '@rushstack/rush-sdk'; import { IRedisCobuildLockProviderOptions, RedisCobuildLockProvider } from '../RedisCobuildLockProvider'; import * as redisAPI from '@redis/client'; import type { RedisClientType } from '@redis/client'; -const terminal = new Terminal(new ConsoleTerminalProvider()); +const rushSession: RushSession = new RushSession({ + terminalProvider: new ConsoleTerminalProvider(), + getIsDebugMode: () => false +}); describe(RedisCobuildLockProvider.name, () => { let storage: Record = {}; @@ -37,14 +40,13 @@ describe(RedisCobuildLockProvider.name, () => { }); function prepareSubject(): RedisCobuildLockProvider { - return new RedisCobuildLockProvider({} as IRedisCobuildLockProviderOptions); + return new RedisCobuildLockProvider({} as IRedisCobuildLockProviderOptions, rushSession); } const context: ICobuildContext = { contextId: '123', cacheId: 'abc', - version: 1, - terminal + version: 1 }; it('getLockKey works', () => { From fe0a50c5064f8bbfc28aeaad12f45d1f7914c304 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 16 Feb 2023 16:57:05 +0800 Subject: [PATCH 09/55] fix: changes after rebase --- .../logic/operations/ShellOperationRunner.ts | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 063b76c6e4a..a5c9d960964 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -261,10 +261,11 @@ export class ShellOperationRunner implements IOperationRunner { // Try to acquire the cobuild lock let cobuildLock: CobuildLock | undefined; if (this._cobuildConfiguration?.cobuildEnabled) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync( + const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ terminal, - trackedFiles - ); + trackedProjectFiles, + operationMetadataManager: context._operationMetadataManager + }); cobuildLock = await this._tryGetCobuildLockAsync(projectBuildCache); } @@ -300,7 +301,11 @@ export class ShellOperationRunner implements IOperationRunner { if (restoreFromCacheSuccess) { // Restore the original state of the operation without cache - await context._operationStateFile?.tryRestoreAsync(); + await context._operationMetadataManager?.tryRestoreAsync({ + terminal, + logPath: projectLogWritable.logPath, + errorLogPath: projectLogWritable.errorLogPath + }); if (cobuildCompletedState) { return cobuildCompletedState.status; } @@ -308,7 +313,7 @@ export class ShellOperationRunner implements IOperationRunner { } } } else if (this._isCacheReadAllowed) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync( + const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ terminal, trackedProjectFiles, operationMetadataManager: context._operationMetadataManager @@ -483,7 +488,11 @@ export class ShellOperationRunner implements IOperationRunner { // write a new cache entry. if (!setCacheEntryPromise && this.isCacheWriteAllowed) { setCacheEntryPromise = ( - await this._tryGetProjectBuildCacheAsync(terminal, trackedFiles) + await this._tryGetProjectBuildCacheAsync({ + terminal, + trackedProjectFiles, + operationMetadataManager: context._operationMetadataManager + }) )?.trySetCacheEntryAsync(terminal); } } From 44a1e802c871a7ee5ca4d79720e8a1a4bb63a69b Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 14:48:10 +0800 Subject: [PATCH 10/55] fix: cobuild integration test --- .vscode/redis-cobuild.code-workspace | 73 +------------------ .../.vscode/tasks.json | 71 ++++++++++++++++++ .../repo/common/scripts/install-run.js | 2 +- .../sandbox/repo/projects/a/package.json | 4 +- .../sandbox/repo/projects/build.js | 6 +- .../src/runRush.ts | 5 +- .../src/testLockProvider.ts | 1 - common/reviews/api/rush-lib.api.md | 2 - .../api/rush-redis-cobuild-plugin.api.md | 2 - .../rush-lib/src/cli/RushCommandLineParser.ts | 3 +- .../rush-lib/src/logic/cobuild/CobuildLock.ts | 4 - .../src/logic/cobuild/ICobuildLockProvider.ts | 1 - .../logic/operations/AsyncOperationQueue.ts | 34 ++++----- .../logic/operations/OperationStateFile.ts | 2 +- .../logic/operations/ShellOperationRunner.ts | 50 ++++++------- .../src/RedisCobuildLockProvider.ts | 56 ++++++++------ .../src/test/RedisCobuildLockProvider.test.ts | 5 -- 17 files changed, 160 insertions(+), 161 deletions(-) create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json diff --git a/.vscode/redis-cobuild.code-workspace b/.vscode/redis-cobuild.code-workspace index 9ab11bb4b4b..c51d9ed6da0 100644 --- a/.vscode/redis-cobuild.code-workspace +++ b/.vscode/redis-cobuild.code-workspace @@ -16,76 +16,5 @@ "name": ".vscode", "path": "../.vscode" } - ], - "tasks": { - "version": "2.0.0", - "tasks": [ - { - "type": "shell", - "label": "cobuild", - "dependsOrder": "sequence", - "dependsOn": ["update 1", "_cobuild"], - "problemMatcher": [] - }, - { - "type": "shell", - "label": "_cobuild", - "dependsOn": ["build 1", "build 2"], - "problemMatcher": [] - }, - { - "type": "shell", - "label": "update", - "command": "node ../../lib/runRush.js update", - "problemMatcher": [], - "presentation": { - "echo": true, - "reveal": "always", - "focus": false, - "panel": "dedicated", - "showReuseMessage": true, - "clear": false - }, - "options": { - "cwd": "${workspaceFolder}/sandbox/repo" - } - }, - { - "type": "shell", - "label": "build 1", - "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", - "problemMatcher": [], - "options": { - "cwd": "${workspaceFolder}/sandbox/repo" - }, - "presentation": { - "echo": true, - "reveal": "always", - "focus": false, - "panel": "dedicated", - "showReuseMessage": true, - "clear": true - }, - "group": "build" - }, - { - "type": "shell", - "label": "build 2", - "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", - "problemMatcher": [], - "options": { - "cwd": "${workspaceFolder}/sandbox/repo" - }, - "presentation": { - "echo": true, - "reveal": "always", - "focus": false, - "panel": "dedicated", - "showReuseMessage": true, - "clear": true - }, - "group": "build" - } - ] - } + ] } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json new file mode 100644 index 00000000000..917c7ccf6c9 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -0,0 +1,71 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "shell", + "label": "cobuild", + "dependsOrder": "sequence", + "dependsOn": ["update 1", "_cobuild"], + "problemMatcher": [] + }, + { + "type": "shell", + "label": "_cobuild", + "dependsOn": ["build 1", "build 2"], + "problemMatcher": [] + }, + { + "type": "shell", + "label": "update", + "command": "node ../../lib/runRush.js update", + "problemMatcher": [], + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": false + }, + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + } + }, + { + "type": "shell", + "label": "build 1", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": true + }, + "group": "build" + }, + { + "type": "shell", + "label": "build 2", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": true + }, + "group": "build" + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js index bcd982b369e..68b1b56fc58 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js @@ -238,8 +238,8 @@ var __webpack_exports__ = {}; __webpack_require__.r(__webpack_exports__); /* harmony export */ __webpack_require__.d(__webpack_exports__, { /* harmony export */ "RUSH_JSON_FILENAME": () => (/* binding */ RUSH_JSON_FILENAME), -/* harmony export */ "getNpmPath": () => (/* binding */ getNpmPath), /* harmony export */ "findRushJsonFolder": () => (/* binding */ findRushJsonFolder), +/* harmony export */ "getNpmPath": () => (/* binding */ getNpmPath), /* harmony export */ "installAndRun": () => (/* binding */ installAndRun), /* harmony export */ "runWithErrorAndStatusCode": () => (/* binding */ runWithErrorAndStatusCode) /* harmony export */ }); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json index 9472fdbd7e5..25b54c04e8d 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json @@ -2,7 +2,7 @@ "name": "a", "version": "1.0.0", "scripts": { - "cobuild": "node ../build.js", - "build": "node ../build.js" + "cobuild": "node ../build.js a", + "build": "node ../build.js a" } } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js index 14855ff8c72..20f983ed146 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js @@ -2,11 +2,13 @@ const path = require('path'); const { FileSystem } = require('@rushstack/node-core-library'); -console.log('start'); +const args = process.argv.slice(2); + +console.log('start', args.join(' ')); setTimeout(() => { const outputFolder = path.resolve(process.cwd(), 'dist'); const outputFile = path.resolve(outputFolder, 'output.txt'); FileSystem.ensureFolder(outputFolder); - FileSystem.writeFile(outputFile, 'Hello world!'); + FileSystem.writeFile(outputFile, `Hello world! ${args.join(' ')}`); console.log('done'); }, 5000); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts index 50c44d60968..c758ec650a0 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts @@ -1,5 +1,6 @@ -import { RushCommandLineParser } from '@microsoft/rush-lib/lib/cli/RushCommandLineParser'; -import * as rushLib from '@microsoft/rush-lib'; +// Import from lib-commonjs for easy debugging +import { RushCommandLineParser } from '@microsoft/rush-lib/lib-commonjs/cli/RushCommandLineParser'; +import * as rushLib from '@microsoft/rush-lib/lib-commonjs'; // Setup redis cobuild plugin const builtInPluginConfigurations: rushLib._IBuiltInPluginConfiguration[] = []; diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts index 474abae5bd9..859135e6eda 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts @@ -30,7 +30,6 @@ async function main(): Promise { status: OperationStatus.Success, cacheId: 'test-cache-id' }); - await lockProvider.releaseLockAsync(context); const completedState = await lockProvider.getCompletedStateAsync(context); console.log('Completed state: ', completedState); await lockProvider.disconnectAsync(); diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 09900023ba0..debe1604379 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -294,8 +294,6 @@ export interface ICobuildLockProvider { // (undocumented) getCompletedStateAsync(context: ICobuildContext): Promise; // (undocumented) - releaseLockAsync(context: ICobuildContext): Promise; - // (undocumented) renewLockAsync(context: ICobuildContext): Promise; // (undocumented) setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index b19ae9aa26b..93da87162d0 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -30,8 +30,6 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { getCompletedStateKey(context: ICobuildContext): string; getLockKey(context: ICobuildContext): string; // (undocumented) - releaseLockAsync(context: ICobuildContext): Promise; - // (undocumented) renewLockAsync(context: ICobuildContext): Promise; // (undocumented) setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; diff --git a/libraries/rush-lib/src/cli/RushCommandLineParser.ts b/libraries/rush-lib/src/cli/RushCommandLineParser.ts index 2f027d1f09e..d5ba58362ca 100644 --- a/libraries/rush-lib/src/cli/RushCommandLineParser.ts +++ b/libraries/rush-lib/src/cli/RushCommandLineParser.ts @@ -187,7 +187,8 @@ export class RushCommandLineParser extends CommandLineParser { } public async execute(args?: string[]): Promise { - this._terminalProvider.verboseEnabled = this.isDebug; + // debugParameter will be correctly parsed during super.execute(), so manually parse here. + this._terminalProvider.debugEnabled = process.argv.indexOf('--debug') >= 0; await this.pluginManager.tryInitializeUnassociatedPluginsAsync(); diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index aa7875ff687..22d1f7818d3 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -62,10 +62,6 @@ export class CobuildLock { return acquireLockResult; } - public async releaseLockAsync(): Promise { - await this.cobuildConfiguration.cobuildLockProvider.releaseLockAsync(this._cobuildContext); - } - public async renewLockAsync(): Promise { await this.cobuildConfiguration.cobuildLockProvider.renewLockAsync(this._cobuildContext); } diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index be00a297c11..73092467805 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -32,7 +32,6 @@ export interface ICobuildLockProvider { disconnectAsync(): Promise; acquireLockAsync(context: ICobuildContext): Promise; renewLockAsync(context: ICobuildContext): Promise; - releaseLockAsync(context: ICobuildContext): Promise; setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; getCompletedStateAsync(context: ICobuildContext): Promise; } diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 9e0cd9ef5eb..c4bd76ba542 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -126,26 +126,26 @@ export class AsyncOperationQueue } if (waitingIterators.length > 0) { - // cycle through the queue again to find the next operation that is executed remotely - for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { - const operation: OperationExecutionRecord = queue[i]; - - if (operation.status === OperationStatus.RemoteExecuting) { - // try to attempt to get the lock again - waitingIterators.shift()!({ - value: operation, - done: false - }); + // Pause for a few time + setTimeout(() => { + // cycle through the queue again to find the next operation that is executed remotely + for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { + const operation: OperationExecutionRecord = queue[i]; + + if (operation.status === OperationStatus.RemoteExecuting) { + // try to attempt to get the lock again + waitingIterators.shift()!({ + value: operation, + done: false + }); + } } - } - if (waitingIterators.length > 0) { - // Queue is not empty, but no operations are ready to process - // Pause for a second and start over - setTimeout(() => { + if (waitingIterators.length > 0) { + // Queue is not empty, but no operations are ready to process, start over this.assignOperations(); - }, 1000); - } + } + }, 5000); } } diff --git a/libraries/rush-lib/src/logic/operations/OperationStateFile.ts b/libraries/rush-lib/src/logic/operations/OperationStateFile.ts index de8b6ed7321..021c66aca60 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStateFile.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStateFile.ts @@ -53,7 +53,7 @@ export class OperationStateFile { } public async writeAsync(json: IOperationStateJson): Promise { - await JsonFile.saveAsync(json, this.filepath, { ensureFolderExists: true, updateExistingFile: true }); + await JsonFile.saveAsync(json, this.filepath, { ensureFolderExists: true }); this._state = json; } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index a5c9d960964..cc452034a81 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -431,7 +431,22 @@ export class ShellOperationRunner implements IOperationRunner { } ); - let setCompletedStatePromise: Promise | undefined; + const taskIsSuccessful: boolean = + status === OperationStatus.Success || + (status === OperationStatus.SuccessWithWarning && + this.warningsAreAllowed && + !!this._rushConfiguration.experimentsConfiguration.configuration + .buildCacheWithAllowWarningsInSuccessfulBuild); + + // Save the metadata to disk + const { duration: durationInSeconds } = context.stopwatch; + await context._operationMetadataManager?.saveAsync({ + durationInSeconds, + logPath: projectLogWritable.logPath, + errorLogPath: projectLogWritable.errorLogPath + }); + + let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; let setCacheEntryPromise: Promise | undefined; if (cobuildLock && this.isCacheWriteAllowed) { const { projectBuildCache } = cobuildLock; @@ -445,14 +460,13 @@ export class ShellOperationRunner implements IOperationRunner { case OperationStatus.SuccessWithWarning: case OperationStatus.Success: case OperationStatus.Failure: { - setCompletedStatePromise = cobuildLock - .setCompletedStateAsync({ - status, + const currentStatus: ICobuildCompletedState['status'] = status; + setCompletedStatePromiseFunction = () => { + return cobuildLock?.setCompletedStateAsync({ + status: currentStatus, cacheId: finalCacheId - }) - .then(() => { - return cobuildLock?.releaseLockAsync(); }); + }; setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( terminal, finalCacheId @@ -462,13 +476,6 @@ export class ShellOperationRunner implements IOperationRunner { } } - const taskIsSuccessful: boolean = - status === OperationStatus.Success || - (status === OperationStatus.SuccessWithWarning && - this.warningsAreAllowed && - !!this._rushConfiguration.experimentsConfiguration.configuration - .buildCacheWithAllowWarningsInSuccessfulBuild); - let writeProjectStatePromise: Promise | undefined; if (taskIsSuccessful && projectDeps) { // Write deps on success. @@ -476,14 +483,6 @@ export class ShellOperationRunner implements IOperationRunner { ensureFolderExists: true }); - // If the operation without cache was successful, we can save the metadata to disk - const { duration: durationInSeconds } = context.stopwatch; - await context._operationMetadataManager?.saveAsync({ - durationInSeconds, - logPath: projectLogWritable.logPath, - errorLogPath: projectLogWritable.errorLogPath - }); - // If the command is successful, we can calculate project hash, and no dependencies were skipped, // write a new cache entry. if (!setCacheEntryPromise && this.isCacheWriteAllowed) { @@ -496,11 +495,8 @@ export class ShellOperationRunner implements IOperationRunner { )?.trySetCacheEntryAsync(terminal); } } - const [, cacheWriteSuccess] = await Promise.all([ - writeProjectStatePromise, - setCacheEntryPromise, - setCompletedStatePromise - ]); + const [, cacheWriteSuccess] = await Promise.all([writeProjectStatePromise, setCacheEntryPromise]); + await setCompletedStatePromiseFunction?.(); if (terminalProvider.hasErrors) { status = OperationStatus.Failure; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index d7d22d2898c..c6ae989db55 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -49,9 +49,9 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } public async connectAsync(): Promise { - await this._redisClient.connect(); - // Check the connection works at early stage try { + await this._redisClient.connect(); + // Check the connection works at early stage await this._redisClient.ping(); } catch (e) { throw new Error(`Failed to connect to redis server: ${e.message}`); @@ -59,17 +59,26 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } public async disconnectAsync(): Promise { - await this._redisClient.disconnect(); + try { + await this._redisClient.disconnect(); + } catch (e) { + throw new Error(`Failed to disconnect to redis server: ${e.message}`); + } } public async acquireLockAsync(context: ICobuildContext): Promise { const { _terminal: terminal } = this; const lockKey: string = this.getLockKey(context); - const incrResult: number = await this._redisClient.incr(lockKey); - const result: boolean = incrResult === 1; - terminal.writeDebugLine(`Acquired lock for ${lockKey}: ${incrResult}, 1 is success`); - if (result) { - await this.renewLockAsync(context); + let result: boolean = false; + try { + const incrResult: number = await this._redisClient.incr(lockKey); + result = incrResult === 1; + terminal.writeDebugLine(`Acquired lock for ${lockKey}: ${incrResult}, 1 is success`); + if (result) { + await this.renewLockAsync(context); + } + } catch (e) { + throw new Error(`Failed to acquire lock for ${lockKey}: ${e.message}`); } return result; } @@ -77,17 +86,14 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { public async renewLockAsync(context: ICobuildContext): Promise { const { _terminal: terminal } = this; const lockKey: string = this.getLockKey(context); - await this._redisClient.expire(lockKey, 30); + try { + await this._redisClient.expire(lockKey, 30); + } catch (e) { + throw new Error(`Failed to renew lock for ${lockKey}: ${e.message}`); + } terminal.writeDebugLine(`Renewed lock for ${lockKey}`); } - public async releaseLockAsync(context: ICobuildContext): Promise { - const { _terminal: terminal } = this; - const lockKey: string = this.getLockKey(context); - await this._redisClient.set(lockKey, 0); - terminal.writeDebugLine(`Released lock for ${lockKey}`); - } - public async setCompletedStateAsync( context: ICobuildContext, state: ICobuildCompletedState @@ -95,7 +101,11 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { const { _terminal: terminal } = this; const key: string = this.getCompletedStateKey(context); const value: string = this._serializeCompletedState(state); - await this._redisClient.set(key, value); + try { + await this._redisClient.set(key, value); + } catch (e) { + throw new Error(`Failed to set completed state for ${key}: ${e.message}`); + } terminal.writeDebugLine(`Set completed state for ${key}: ${value}`); } @@ -103,11 +113,15 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { const { _terminal: terminal } = this; const key: string = this.getCompletedStateKey(context); let state: ICobuildCompletedState | undefined; - const value: string | null = await this._redisClient.get(key); - if (value) { - state = this._deserializeCompletedState(value); + try { + const value: string | null = await this._redisClient.get(key); + if (value) { + state = this._deserializeCompletedState(value); + } + terminal.writeDebugLine(`Get completed state for ${key}: ${value}`); + } catch (e) { + throw new Error(`Failed to get completed state for ${key}: ${e.message}`); } - terminal.writeDebugLine(`Get completed state for ${key}: ${value}`); return state; } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index f908f762a1c..3425e83c31a 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -79,11 +79,6 @@ describe(RedisCobuildLockProvider.name, () => { expect(result2).toBe(false); }); - it('releaseLockAsync works', async () => { - const subject: RedisCobuildLockProvider = prepareSubject(); - expect(() => subject.releaseLockAsync(context)).not.toThrowError(); - }); - it('set and get completedState works', async () => { const subject: RedisCobuildLockProvider = prepareSubject(); const cacheId: string = 'foo'; From 662b5efa45750a9843b92bc1a3785dd35b851564 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 14:56:32 +0800 Subject: [PATCH 11/55] :memo: update readme --- .../README.md | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md index 87b7b6e0af9..b2bbdd801cf 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -10,9 +10,27 @@ In this folder run `docker-compose up -d` # Stop the Redis In this folder run `docker-compose down` -# Run the test +# Install and build the integration test code + +```sh +rush update +rush build -t rush-redis-cobuild-plugin-integration-test +``` + +# Run the test for lock provider + ```sh # start the docker container: docker-compose up -d # build the code: rushx build rushx test-lock-provider ``` + +# Testing cobuild + +> Note: This test requires Visual Studio Code to be installed. + +1. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. + +2. Open Command Palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`. + +3. Two new terminal windows will open. Both running `rush cobuild` command under sandbox repo. From 1a5085f80f73cf24558e137860907c84cdb2f5bf Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 15:02:18 +0800 Subject: [PATCH 12/55] chore: rush change --- .../@microsoft/rush/feat-cobuild_2023-02-17-07-02.json | 10 ++++++++++ .../feat-cobuild_2023-02-17-07-02.json | 10 ++++++++++ rush-plugins/rush-redis-cobuild-plugin/package.json | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json create mode 100644 common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json diff --git a/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json b/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json new file mode 100644 index 00000000000..c569757d07d --- /dev/null +++ b/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "(EXPERIMENTAL) Add a cheap way to get distributed builds called \"cobuild\"", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json b/common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json new file mode 100644 index 00000000000..77f64cbf75a --- /dev/null +++ b/common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-redis-cobuild-plugin", + "comment": "Implement a redis lock provider for cobuild feature", + "type": "minor" + } + ], + "packageName": "@rushstack/rush-redis-cobuild-plugin" +} \ No newline at end of file diff --git a/rush-plugins/rush-redis-cobuild-plugin/package.json b/rush-plugins/rush-redis-cobuild-plugin/package.json index ce88b68a818..19caebf3bcd 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/package.json +++ b/rush-plugins/rush-redis-cobuild-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@rushstack/rush-redis-cobuild-plugin", - "version": "5.88.2", + "version": "0.0.0", "description": "Rush plugin for Redis cobuild lock", "repository": { "type": "git", From d0871f4b803ebd6788a3958057672fb11cda4d6c Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 15:21:14 +0800 Subject: [PATCH 13/55] chore: tasks.json --- .../.vscode/tasks.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json index 917c7ccf6c9..8e1982bd3f9 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -5,7 +5,7 @@ "type": "shell", "label": "cobuild", "dependsOrder": "sequence", - "dependsOn": ["update 1", "_cobuild"], + "dependsOn": ["update", "_cobuild"], "problemMatcher": [] }, { @@ -34,7 +34,7 @@ { "type": "shell", "label": "build 1", - "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", "problemMatcher": [], "options": { "cwd": "${workspaceFolder}/sandbox/repo" @@ -52,7 +52,7 @@ { "type": "shell", "label": "build 2", - "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", "problemMatcher": [], "options": { "cwd": "${workspaceFolder}/sandbox/repo" From c40bfc93553b7432572f6ff712f94565b84dc7c2 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 17:34:13 +0800 Subject: [PATCH 14/55] feat: allow failing build to be cached when cobuilding --- .../sandbox/repo/projects/a/package.json | 1 + .../src/logic/operations/ShellOperationRunner.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json index 25b54c04e8d..f8b84111f98 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "scripts": { "cobuild": "node ../build.js a", + "_cobuild": "sleep 5 && exit 1", "build": "node ../build.js a" } } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index cc452034a81..5b9dea9446c 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -418,7 +418,13 @@ export class ShellOperationRunner implements IOperationRunner { subProcess.on('close', (code: number) => { try { if (code !== 0) { - reject(new OperationError('error', `Returned error code: ${code}`)); + if (cobuildLock) { + // In order to preventing the worst case that all cobuild tasks go through the same failure, + // allowing a failing build to be cached and retrieved + resolve(OperationStatus.Failure); + } else { + reject(new OperationError('error', `Returned error code: ${code}`)); + } } else if (hasWarningOrError) { resolve(OperationStatus.SuccessWithWarning); } else { @@ -500,7 +506,7 @@ export class ShellOperationRunner implements IOperationRunner { if (terminalProvider.hasErrors) { status = OperationStatus.Failure; - } else if (cacheWriteSuccess === false) { + } else if (cacheWriteSuccess === false && status === OperationStatus.Success) { status = OperationStatus.SuccessWithWarning; } From 17754b493799b5b9798efd1a7042b8b92c22454f Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 19:24:10 +0800 Subject: [PATCH 15/55] feat: cobuild context id --- common/reviews/api/rush-lib.api.md | 2 + .../rush-lib/src/api/CobuildConfiguration.ts | 42 ++++++++----- .../src/logic/cobuild/CobuildContextId.ts | 59 +++++++++++++++++++ .../rush-lib/src/logic/cobuild/CobuildLock.ts | 4 ++ .../cobuild/test/CobuildContextId.test.ts | 37 ++++++++++++ .../logic/cobuild/test/CobuildLock.test.ts | 28 +++++++++ .../CobuildContextId.test.ts.snap | 5 ++ 7 files changed, 161 insertions(+), 16 deletions(-) create mode 100644 libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index debe1604379..e6bb418cb7f 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -97,7 +97,9 @@ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) = // @beta export class CobuildConfiguration { + readonly cobuildContextId: string; readonly cobuildEnabled: boolean; + // (undocumented) readonly cobuildLockProvider: ICobuildLockProvider; // (undocumented) connectLockProviderAsync(): Promise; diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 1b4b636c390..f571feec6a8 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -2,7 +2,13 @@ // See LICENSE in the project root for license information. import * as path from 'path'; -import { FileSystem, ITerminal, JsonFile, JsonSchema } from '@rushstack/node-core-library'; +import { + AlreadyReportedError, + FileSystem, + ITerminal, + JsonFile, + JsonSchema +} from '@rushstack/node-core-library'; import schemaJson from '../schemas/cobuild.schema.json'; import { EnvironmentConfiguration } from './EnvironmentConfiguration'; import { CobuildLockProviderFactory, RushSession } from '../pluginFramework/RushSession'; @@ -10,6 +16,7 @@ import { RushConstants } from '../logic/RushConstants'; import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; import type { RushConfiguration } from './RushConfiguration'; +import { CobuildContextId, GetCobuildContextIdFunction } from '../logic/cobuild/CobuildContextId'; export interface ICobuildJson { cobuildEnabled: boolean; @@ -19,6 +26,7 @@ export interface ICobuildJson { export interface ICobuildConfigurationOptions { cobuildJson: ICobuildJson; + getCobuildContextId: GetCobuildContextIdFunction; rushConfiguration: RushConfiguration; rushSession: RushSession; } @@ -38,15 +46,18 @@ export class CobuildConfiguration { public readonly cobuildEnabled: boolean; /** * Method to calculate the cobuild context id - * FIXME: */ - // public readonly getCacheEntryId: GetCacheEntryIdFunction; + public readonly cobuildContextId: string; public readonly cobuildLockProvider: ICobuildLockProvider; private constructor(options: ICobuildConfigurationOptions) { this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? options.cobuildJson.cobuildEnabled; - const { cobuildJson } = options; + const { cobuildJson, getCobuildContextId } = options; + + this.cobuildContextId = getCobuildContextId({ + environment: process.env + }); const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = options.rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); @@ -87,27 +98,26 @@ export class CobuildConfiguration { CobuildConfiguration._jsonSchema ); - // FIXME: - // let getCacheEntryId: GetCacheEntryIdFunction; - // try { - // getCacheEntryId = CacheEntryId.parsePattern(cobuildJson.cacheEntryNamePattern); - // } catch (e) { - // terminal.writeErrorLine( - // `Error parsing cache entry name pattern "${cobuildJson.cacheEntryNamePattern}": ${e}` - // ); - // throw new AlreadyReportedError(); - // } + let getCobuildContextId: GetCobuildContextIdFunction; + try { + getCobuildContextId = CobuildContextId.parsePattern(cobuildJson.cobuildContextIdPattern); + } catch (e) { + terminal.writeErrorLine( + `Error parsing cobuild context id pattern "${cobuildJson.cobuildContextIdPattern}": ${e}` + ); + throw new AlreadyReportedError(); + } return new CobuildConfiguration({ cobuildJson, + getCobuildContextId, rushConfiguration, rushSession }); } public get contextId(): string { - // FIXME: hardcode - return '123'; + return this.cobuildContextId; } public async connectLockProviderAsync(): Promise { diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts b/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts new file mode 100644 index 00000000000..2c191eca242 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { RushConstants } from '../RushConstants'; + +export interface IGenerateCobuildContextIdOptions { + environment: NodeJS.ProcessEnv; +} + +/** + * Calculates the cache entry id string for an operation. + * @beta + */ +export type GetCobuildContextIdFunction = (options: IGenerateCobuildContextIdOptions) => string; + +export class CobuildContextId { + private constructor() {} + + public static parsePattern(pattern?: string): GetCobuildContextIdFunction { + if (!pattern) { + return () => ''; + } else { + const resolvedPattern: string = pattern.trim(); + + return (options: IGenerateCobuildContextIdOptions) => { + const { environment } = options; + return this._expandWithEnvironmentVariables(resolvedPattern, environment); + }; + } + } + + private static _expandWithEnvironmentVariables(pattern: string, environment: NodeJS.ProcessEnv): string { + const missingEnvironmentVariables: Set = new Set(); + const expandedPattern: string = pattern.replace( + /\$\{([^\}]+)\}/g, + (match: string, variableName: string): string => { + const variable: string | undefined = + variableName in environment ? environment[variableName] : undefined; + if (variable !== undefined) { + return variable; + } else { + missingEnvironmentVariables.add(variableName); + return match; + } + } + ); + if (missingEnvironmentVariables.size) { + throw new Error( + `The "cobuildContextIdPattern" value in ${ + RushConstants.cobuildFilename + } contains missing environment variable${ + missingEnvironmentVariables.size > 1 ? 's' : '' + }: ${Array.from(missingEnvironmentVariables).join(', ')}` + ); + } + + return expandedPattern; + } +} diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index 22d1f7818d3..9b656a9877d 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -65,4 +65,8 @@ export class CobuildLock { public async renewLockAsync(): Promise { await this.cobuildConfiguration.cobuildLockProvider.renewLockAsync(this._cobuildContext); } + + public get cobuildContext(): ICobuildContext { + return this._cobuildContext; + } } diff --git a/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts b/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts new file mode 100644 index 00000000000..ea18a0e9242 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { CobuildContextId } from '../CobuildContextId'; + +describe(CobuildContextId.name, () => { + describe('Valid pattern names', () => { + it('expands a environment variable', () => { + const contextId: string = CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ + environment: { + MR_ID: '123', + AUTHOR_NAME: 'Mr.example' + } + }); + expect(contextId).toEqual('context-123-Mr.example'); + }); + }); + + describe('Invalid pattern names', () => { + it('throws an error if a environment variable is missing', () => { + expect(() => + CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ + environment: { + MR_ID: '123' + } + }) + ).toThrowErrorMatchingSnapshot(); + }); + it('throws an error if multiple environment variables are missing', () => { + expect(() => + CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ + environment: {} + }) + ).toThrowErrorMatchingSnapshot(); + }); + }); +}); diff --git a/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts b/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts new file mode 100644 index 00000000000..2603dbafb93 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { CobuildConfiguration } from '../../../api/CobuildConfiguration'; +import { ProjectBuildCache } from '../../buildCache/ProjectBuildCache'; +import { CobuildLock } from '../CobuildLock'; + +describe(CobuildLock.name, () => { + function prepareSubject(): CobuildLock { + const subject: CobuildLock = new CobuildLock({ + cobuildConfiguration: { + contextId: 'foo' + } as unknown as CobuildConfiguration, + projectBuildCache: { + cacheId: 'bar' + } as unknown as ProjectBuildCache + }); + return subject; + } + it('returns cobuild context', () => { + const subject: CobuildLock = prepareSubject(); + expect(subject.cobuildContext).toEqual({ + contextId: 'foo', + cacheId: 'bar', + version: 1 + }); + }); +}); diff --git a/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap b/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap new file mode 100644 index 00000000000..c2495bc8537 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap @@ -0,0 +1,5 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`CobuildContextId Invalid pattern names throws an error if a environment variable is missing 1`] = `"The \\"cobuildContextIdPattern\\" value in cobuild.json contains missing environment variable: AUTHOR_NAME"`; + +exports[`CobuildContextId Invalid pattern names throws an error if multiple environment variables are missing 1`] = `"The \\"cobuildContextIdPattern\\" value in cobuild.json contains missing environment variables: MR_ID, AUTHOR_NAME"`; From 7c2ec9eff372c1a362f4ebd8cc9fd8f6bfdb16aa Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 21:45:07 +0800 Subject: [PATCH 16/55] feat: add RUSH_COBUILD_CONTEXT_ID environment variable --- .../rush-lib/src/api/CobuildConfiguration.ts | 8 +++--- .../src/api/EnvironmentConfiguration.ts | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index f571feec6a8..214e61c0a03 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -55,9 +55,11 @@ export class CobuildConfiguration { const { cobuildJson, getCobuildContextId } = options; - this.cobuildContextId = getCobuildContextId({ - environment: process.env - }); + this.cobuildContextId = + EnvironmentConfiguration.cobuildContextId ?? + getCobuildContextId({ + environment: process.env + }); const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = options.rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index 345f98f5fd1..e2224ce3d7f 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -154,6 +154,15 @@ export enum EnvironmentVariableNames { */ RUSH_COBUILD_ENABLED = 'RUSH_COBUILD_ENABLED', + /** + * Setting this environment variable overrides the value of `cobuildContextId` calculated by + * `cobuildContextIdPattern` in the `cobuild.json` configuration file. + * + * @remarks + * If there is no cobuild configured, then this environment variable is ignored. + */ + RUSH_COBUILD_CONTEXT_ID = 'RUSH_COBUILD_CONTEXT_ID', + /** * Explicitly specifies the path for the Git binary that is invoked by certain Rush operations. */ @@ -209,6 +218,8 @@ export class EnvironmentConfiguration { private static _cobuildEnabled: boolean | undefined; + private static _cobuildContextId: string | undefined; + private static _gitBinaryPath: string | undefined; private static _tarBinaryPath: string | undefined; @@ -315,6 +326,15 @@ export class EnvironmentConfiguration { return EnvironmentConfiguration._cobuildEnabled; } + /** + * Provides a determined cobuild context id if configured + * See {@link EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID} + */ + public static get cobuildContextId(): string | undefined { + EnvironmentConfiguration._ensureValidated(); + return EnvironmentConfiguration._cobuildContextId; + } + /** * Allows the git binary path to be explicitly provided. * See {@link EnvironmentVariableNames.RUSH_GIT_BINARY_PATH} @@ -454,6 +474,11 @@ export class EnvironmentConfiguration { break; } + case EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID: { + EnvironmentConfiguration._cobuildContextId = value; + break; + } + case EnvironmentVariableNames.RUSH_GIT_BINARY_PATH: { EnvironmentConfiguration._gitBinaryPath = value; break; From 64fd7a4b45ab1b636c50a6c2590922aab530c708 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 22:31:23 +0800 Subject: [PATCH 17/55] :memo: --- .../README.md | 106 +++++++++++++++++- 1 file changed, 102 insertions(+), 4 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md index b2bbdd801cf..5a72f9010b5 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -1,13 +1,17 @@ # About + This package enables integration testing of the `RedisCobuildLockProvider` by connecting to an actual Redis created using an [redis](https://hub.docker.com/_/redis) docker image. # Prerequisites + Docker and docker compose must be installed # Start the Redis + In this folder run `docker-compose up -d` # Stop the Redis + In this folder run `docker-compose down` # Install and build the integration test code @@ -25,12 +29,106 @@ rush build -t rush-redis-cobuild-plugin-integration-test rushx test-lock-provider ``` -# Testing cobuild +# Integration test in sandbox repo + +Sandbox repo folder: **build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo** + +```sh +cd sandbox/repo +rush update +``` + +## Case 1: Disable cobuild by setting `RUSH_COBUILD_ENABLED=0` + +```sh +rm -rf common/temp/build-cache && RUSH_COBUILD_ENABLED=0 node ../../lib/runRush.js --debug cobuild +``` + +Expected behavior: Cobuild feature is disabled. Run command successfully. + +```sh +RUSH_COBUILD_ENABLED=0 node ../../lib/runRush.js --debug cobuild +``` + +Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. + +## Case 2: Cobuild enabled, run one cobuild command only + +1. Clear redis server + +```sh +(cd ../.. && docker compose down && docker compose up -d) +``` + +2. Run `rush cobuild` command + +```sh +rm -rf common/temp/build-cache && node ../../lib/runRush.js --debug cobuild +``` + +Expected behavior: Cobuild feature is enabled. Run command successfully. +You can also see cobuild related logs in the terminal. + +```sh +Get completed state for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: null +Acquired lock for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:lock: 1, 1 is success +Set completed state for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: SUCCESS;c2df36270ec5faa8ef6497fa7367a476de3e2861 +``` + +## Case 3: Cobuild enabled, run two cobuild commands in parallel > Note: This test requires Visual Studio Code to be installed. -1. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. +1. Clear redis server + +```sh +(cd ../.. && docker compose down && docker compose up -d) +``` + +2. Clear build cache + +```sh +rm -rf common/temp/build-cache +``` + +3. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. + +4. Open command palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`. + +> In this step, two dedicated terminal windows will open. Running `rush cobuild` command under sandbox repo respectively. + +Expected behavior: Cobuild feature is enabled, cobuild related logs out in both terminals. + +## Case 4: Cobuild enabled, run two cobuild commands in parallel, one of them failed + +> Note: This test requires Visual Studio Code to be installed. + +1. Making the cobuild command of project "A" fails + +**sandbox/repo/projects/a/package.json** + +```diff + "scripts": { +- "cobuild": "node ../build.js a", ++ "cobuild": "sleep 5 && exit 1", + "build": "node ../build.js a" + } +``` + +2. Clear redis server + +```sh +(cd ../.. && docker compose down && docker compose up -d) +``` + +3. Clear build cache + +```sh +rm -rf common/temp/build-cache +``` + +4. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. -2. Open Command Palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`. +5. Open command palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`. -3. Two new terminal windows will open. Both running `rush cobuild` command under sandbox repo. +Expected behavior: Cobuild feature is enabled, cobuild related logs out in both terminals. These two cobuild commands fail because of the failing build of project "A". And, one of them restored the failing build cache created by the other one. From e4cfabcc00784ace92430fa6a264b2970ff1df2b Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 23:28:46 +0800 Subject: [PATCH 18/55] fix: test --- common/reviews/api/rush-lib.api.md | 2 ++ libraries/rush-lib/src/cli/actions/test/removeRepo/.gitignore | 1 + .../src/cli/actions/test/removeRepo/common/temp/rush#90625.lock | 1 - .../src/logic/operations/test/AsyncOperationQueue.test.ts | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 libraries/rush-lib/src/cli/actions/test/removeRepo/.gitignore delete mode 100644 libraries/rush-lib/src/cli/actions/test/removeRepo/common/temp/rush#90625.lock diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index e6bb418cb7f..fa04eb4266a 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -171,6 +171,7 @@ export class EnvironmentConfiguration { static get buildCacheCredential(): string | undefined; static get buildCacheEnabled(): boolean | undefined; static get buildCacheWriteAllowed(): boolean | undefined; + static get cobuildContextId(): string | undefined; static get cobuildEnabled(): boolean | undefined; // Warning: (ae-forgotten-export) The symbol "IEnvironment" needs to be exported by the entry point index.d.ts // @@ -196,6 +197,7 @@ export enum EnvironmentVariableNames { RUSH_BUILD_CACHE_CREDENTIAL = "RUSH_BUILD_CACHE_CREDENTIAL", RUSH_BUILD_CACHE_ENABLED = "RUSH_BUILD_CACHE_ENABLED", RUSH_BUILD_CACHE_WRITE_ALLOWED = "RUSH_BUILD_CACHE_WRITE_ALLOWED", + RUSH_COBUILD_CONTEXT_ID = "RUSH_COBUILD_CONTEXT_ID", RUSH_COBUILD_ENABLED = "RUSH_COBUILD_ENABLED", RUSH_DEPLOY_TARGET_FOLDER = "RUSH_DEPLOY_TARGET_FOLDER", RUSH_GIT_BINARY_PATH = "RUSH_GIT_BINARY_PATH", diff --git a/libraries/rush-lib/src/cli/actions/test/removeRepo/.gitignore b/libraries/rush-lib/src/cli/actions/test/removeRepo/.gitignore new file mode 100644 index 00000000000..b37486fa4a9 --- /dev/null +++ b/libraries/rush-lib/src/cli/actions/test/removeRepo/.gitignore @@ -0,0 +1 @@ +common/temp \ No newline at end of file diff --git a/libraries/rush-lib/src/cli/actions/test/removeRepo/common/temp/rush#90625.lock b/libraries/rush-lib/src/cli/actions/test/removeRepo/common/temp/rush#90625.lock deleted file mode 100644 index b2b0d4002b8..00000000000 --- a/libraries/rush-lib/src/cli/actions/test/removeRepo/common/temp/rush#90625.lock +++ /dev/null @@ -1 +0,0 @@ -Fri Jan 27 01:50:33 2023 \ No newline at end of file diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 212c7c2edd3..e0cd272a802 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -181,5 +181,5 @@ describe(AsyncOperationQueue.name, () => { } expect(actualOrder).toEqual(expectedOrder); - }); + }, 6000); }); From e2353fc4f084b94297d1573aa1baad824b9d3045 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 20 Feb 2023 14:51:53 +0800 Subject: [PATCH 19/55] chore --- libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 5b9dea9446c..678325448e5 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -159,7 +159,6 @@ export class ShellOperationRunner implements IOperationRunner { ); const runnerWatcher: RunnerWatcher = new RunnerWatcher({ interval: 10 * 1000 - // interval: 1000 }); try { From 27f78f0a23a441f37b266aee18415daddfe31868 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Wed, 22 Feb 2023 16:51:32 +0800 Subject: [PATCH 20/55] chore --- rush-plugins/rush-redis-cobuild-plugin/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rush-plugins/rush-redis-cobuild-plugin/README.md b/rush-plugins/rush-redis-cobuild-plugin/README.md index bfb2d49760b..9ffa0231a46 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/README.md +++ b/rush-plugins/rush-redis-cobuild-plugin/README.md @@ -1,4 +1,4 @@ -# @rushstack/rush-amazon-s3-build-cache-plugin +# @rushstack/rush-redis-cobuild-plugin This is a Rush plugin for using Redis as cobuild lock provider during the "build" From afd26fc814a33318f2d0a35b074033d6ca446d6c Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 28 Feb 2023 19:19:15 +0800 Subject: [PATCH 21/55] refact(ShellOperationRunner): extract build cache related logic to plugin --- common/reviews/api/rush-lib.api.md | 6 +- .../CacheableOperationRunnerPlugin.ts | 449 ++++++++++++++++++ .../src/logic/operations/IOperationRunner.ts | 20 +- .../operations/IOperationRunnerPlugin.ts | 14 + .../operations/OperationExecutionManager.ts | 26 +- .../operations/OperationExecutionRecord.ts | 5 - .../logic/operations/OperationLifecycle.ts | 60 +++ .../{RunnerWatcher.ts => PeriodicCallback.ts} | 6 +- .../logic/operations/ShellOperationRunner.ts | 404 +++------------- .../operations/ShellOperationRunnerPlugin.ts | 26 +- 10 files changed, 647 insertions(+), 369 deletions(-) create mode 100644 libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts create mode 100644 libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts create mode 100644 libraries/rush-lib/src/logic/operations/OperationLifecycle.ts rename libraries/rush-lib/src/logic/operations/{RunnerWatcher.ts => PeriodicCallback.ts} (90%) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index fa04eb4266a..901545dfa60 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -477,8 +477,6 @@ export interface IOperationOptions { // @beta export interface IOperationRunner { executeAsync(context: IOperationRunnerContext): Promise; - isCacheWriteAllowed: boolean; - isSkipAllowed: boolean; readonly name: string; reportTiming: boolean; silent: boolean; @@ -489,12 +487,14 @@ export interface IOperationRunner { export interface IOperationRunnerContext { collatedWriter: CollatedWriter; debugMode: boolean; + error?: Error; // @internal _operationMetadataManager?: _OperationMetadataManager; quietMode: boolean; status: OperationStatus; stdioSummarizer: StdioSummarizer; - stopwatch: IStopwatchResult; + // Warning: (ae-forgotten-export) The symbol "Stopwatch" needs to be exported by the entry point index.d.ts + stopwatch: Stopwatch; } // @internal (undocumented) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts new file mode 100644 index 00000000000..4005e754d27 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts @@ -0,0 +1,449 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { CobuildLock, ICobuildCompletedState } from '../cobuild/CobuildLock'; +import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; +import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; +import { OperationStatus } from './OperationStatus'; +import { ColorValue, InternalError, ITerminal, JsonObject } from '@rushstack/node-core-library'; +import { RushConstants } from '../RushConstants'; +import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; +import { PrintUtilities } from '@rushstack/terminal'; + +import type { IOperationRunnerPlugin } from './IOperationRunnerPlugin'; +import type { + IOperationRunnerAfterExecuteContext, + IOperationRunnerBeforeExecuteContext, + OperationRunnerLifecycleHooks +} from './OperationLifecycle'; +import type { OperationMetadataManager } from './OperationMetadataManager'; +import type { IOperationRunner } from './IOperationRunner'; +import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; +import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import type { IPhase } from '../../api/CommandLineConfiguration'; +import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import type { IRawRepoState, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; + +const PLUGIN_NAME: 'CacheableOperationRunnerPlugin' = 'CacheableOperationRunnerPlugin'; + +export interface ICacheableOperationRunnerPluginOptions { + buildCacheConfiguration: BuildCacheConfiguration; + cobuildConfiguration: CobuildConfiguration | undefined; + isIncrementalBuildAllowed: boolean; +} + +export interface IOperationBuildCacheContext { + isCacheWriteAllowed: boolean; + isCacheReadAllowed: boolean; + isSkipAllowed: boolean; + projectBuildCache: ProjectBuildCache | undefined; + cobuildLock: CobuildLock | undefined; +} + +export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { + private static _runnerBuildCacheContextMap: Map = new Map< + IOperationRunner, + IOperationBuildCacheContext + >(); + private readonly _buildCacheConfiguration: BuildCacheConfiguration; + private readonly _cobuildConfiguration: CobuildConfiguration | undefined; + + public constructor(options: ICacheableOperationRunnerPluginOptions) { + this._buildCacheConfiguration = options.buildCacheConfiguration; + this._cobuildConfiguration = options.cobuildConfiguration; + } + + public static getBuildCacheContextByRunner( + runner: IOperationRunner + ): IOperationBuildCacheContext | undefined { + const buildCacheContext: IOperationBuildCacheContext | undefined = + CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.get(runner); + return buildCacheContext; + } + + public static getBuildCacheContextByRunnerOrThrow(runner: IOperationRunner): IOperationBuildCacheContext { + const buildCacheContext: IOperationBuildCacheContext | undefined = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(runner); + if (!buildCacheContext) { + // This should not happen + throw new InternalError(`Build cache context for runner ${runner.name} should be defined`); + } + return buildCacheContext; + } + + public static setBuildCacheContextByRunner( + runner: IOperationRunner, + buildCacheContext: IOperationBuildCacheContext + ): void { + CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.set(runner, buildCacheContext); + } + + public static clearAllBuildCacheContexts(): void { + CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.clear(); + } + + public apply(hooks: OperationRunnerLifecycleHooks): void { + hooks.beforeExecute.tapPromise( + PLUGIN_NAME, + async (beforeExecuteContext: IOperationRunnerBeforeExecuteContext) => { + const earlyReturnStatus: OperationStatus | undefined = await (async () => { + const { + context, + runner, + terminal, + lastProjectDeps, + projectDeps, + trackedProjectFiles, + logPath, + errorLogPath, + rushProject, + phase, + selectedPhases, + projectChangeAnalyzer, + commandName, + commandToRun, + earlyReturnStatus + } = beforeExecuteContext; + if (earlyReturnStatus) { + // If there is existing early return status, we don't need to do anything + return earlyReturnStatus; + } + const buildCacheContext: IOperationBuildCacheContext = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + + if (!projectDeps && buildCacheContext.isSkipAllowed) { + // To test this code path: + // Remove the `.git` folder then run "rush build --verbose" + terminal.writeLine({ + text: PrintUtilities.wrapWords( + 'This workspace does not appear to be tracked by Git. ' + + 'Rush will proceed without incremental execution, caching, and change detection.' + ), + foregroundColor: ColorValue.Cyan + }); + } + + const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ + runner, + rushProject, + phase, + selectedPhases, + projectChangeAnalyzer, + commandName, + commandToRun, + terminal, + trackedProjectFiles, + operationMetadataManager: context._operationMetadataManager + }); + buildCacheContext.projectBuildCache = projectBuildCache; + + // Try to acquire the cobuild lock + let cobuildLock: CobuildLock | undefined; + if (this._cobuildConfiguration?.cobuildEnabled) { + cobuildLock = await this._tryGetCobuildLockAsync({ runner, projectBuildCache }); + } + buildCacheContext.cobuildLock = cobuildLock; + + // If possible, we want to skip this operation -- either by restoring it from the + // cache, if caching is enabled, or determining that the project + // is unchanged (using the older incremental execution logic). These two approaches, + // "caching" and "skipping", are incompatible, so only one applies. + // + // Note that "caching" and "skipping" take two different approaches + // to tracking dependents: + // + // - For caching, "isCacheReadAllowed" is set if a project supports + // incremental builds, and determining whether this project or a dependent + // has changed happens inside the hashing logic. + // + // - For skipping, "isSkipAllowed" is set to true initially, and during + // the process of running dependents, it will be changed by OperationExecutionManager to + // false if a dependency wasn't able to be skipped. + // + let buildCacheReadAttempted: boolean = false; + + if (cobuildLock) { + // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to + // the build cache once completed, but does not retrieve them (since the "incremental" + // flag is disabled). However, we still need a cobuild to be able to retrieve a finished + // build from another cobuild in this case. + const cobuildCompletedState: ICobuildCompletedState | undefined = + await cobuildLock.getCompletedStateAsync(); + if (cobuildCompletedState) { + const { status, cacheId } = cobuildCompletedState; + + const restoreFromCacheSuccess: boolean | undefined = + await cobuildLock.projectBuildCache.tryRestoreFromCacheAsync(terminal, cacheId); + + if (restoreFromCacheSuccess) { + // Restore the original state of the operation without cache + await context._operationMetadataManager?.tryRestoreAsync({ + terminal, + logPath, + errorLogPath + }); + if (cobuildCompletedState) { + return cobuildCompletedState.status; + } + return status; + } + } + } else if (buildCacheContext.isCacheReadAllowed) { + buildCacheReadAttempted = !!projectBuildCache; + const restoreFromCacheSuccess: boolean | undefined = + await projectBuildCache?.tryRestoreFromCacheAsync(terminal); + + if (restoreFromCacheSuccess) { + // Restore the original state of the operation without cache + await context._operationMetadataManager?.tryRestoreAsync({ + terminal, + logPath, + errorLogPath + }); + return OperationStatus.FromCache; + } + } + if (buildCacheContext.isSkipAllowed && !buildCacheReadAttempted) { + const isPackageUnchanged: boolean = !!( + lastProjectDeps && + projectDeps && + projectDeps.arguments === lastProjectDeps.arguments && + _areShallowEqual(projectDeps.files, lastProjectDeps.files) + ); + + if (isPackageUnchanged) { + return OperationStatus.Skipped; + } + } + + if (buildCacheContext.isCacheWriteAllowed && cobuildLock) { + const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); + if (acquireSuccess) { + if (context.status === OperationStatus.RemoteExecuting) { + // This operation is used to marked remote executing, now change it to executing + context.status = OperationStatus.Executing; + } + runner.periodicCallback.addCallback(async () => { + await cobuildLock?.renewLockAsync(); + }); + } else { + // failed to acquire the lock, mark current operation to remote executing + context.stopwatch.reset(); + return OperationStatus.RemoteExecuting; + } + } + })(); + if (earlyReturnStatus) { + beforeExecuteContext.earlyReturnStatus = earlyReturnStatus; + } + return beforeExecuteContext; + } + ); + + hooks.afterExecute.tapPromise( + PLUGIN_NAME, + async (afterExecuteContext: IOperationRunnerAfterExecuteContext) => { + const { context, runner, terminal, status, taskIsSuccessful } = afterExecuteContext; + const buildCacheContext: IOperationBuildCacheContext = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + + const { cobuildLock, projectBuildCache, isCacheWriteAllowed } = buildCacheContext; + + let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; + let setCacheEntryPromise: Promise | undefined; + if (cobuildLock && isCacheWriteAllowed) { + if (context.error) { + // In order to preventing the worst case that all cobuild tasks go through the same failure, + // allowing a failing build to be cached and retrieved, print the error message to the terminal + // and clear the error in context. + const message: string | undefined = context.error?.message; + if (message) { + context.collatedWriter.terminal.writeStderrLine(message); + } + context.error = undefined; + } + const cacheId: string | undefined = cobuildLock.projectBuildCache.cacheId; + const contextId: string = cobuildLock.cobuildConfiguration.contextId; + + if (cacheId) { + const finalCacheId: string = + status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; + switch (status) { + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: + case OperationStatus.Failure: { + const currentStatus: ICobuildCompletedState['status'] = status; + setCompletedStatePromiseFunction = () => { + return cobuildLock?.setCompletedStateAsync({ + status: currentStatus, + cacheId: finalCacheId + }); + }; + setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( + terminal, + finalCacheId + ); + } + } + } + } + + // If the command is successful, we can calculate project hash, and no dependencies were skipped, + // write a new cache entry. + if (!setCacheEntryPromise && taskIsSuccessful && isCacheWriteAllowed && projectBuildCache) { + setCacheEntryPromise = projectBuildCache.trySetCacheEntryAsync(terminal); + } + const cacheWriteSuccess: boolean | undefined = await setCacheEntryPromise; + await setCompletedStatePromiseFunction?.(); + + if (cacheWriteSuccess === false && afterExecuteContext.status === OperationStatus.Success) { + afterExecuteContext.status = OperationStatus.SuccessWithWarning; + } + + return afterExecuteContext; + } + ); + } + + private async _tryGetProjectBuildCacheAsync({ + runner, + rushProject, + phase, + selectedPhases, + projectChangeAnalyzer, + commandName, + commandToRun, + terminal, + trackedProjectFiles, + operationMetadataManager + }: { + runner: IOperationRunner; + rushProject: RushConfigurationProject; + phase: IPhase; + selectedPhases: Iterable; + projectChangeAnalyzer: ProjectChangeAnalyzer; + commandName: string; + commandToRun: string; + terminal: ITerminal; + trackedProjectFiles: string[] | undefined; + operationMetadataManager: OperationMetadataManager | undefined; + }): Promise { + const buildCacheContext: IOperationBuildCacheContext = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + if (!buildCacheContext.projectBuildCache) { + if (this._buildCacheConfiguration && this._buildCacheConfiguration.buildCacheEnabled) { + // Disable legacy skip logic if the build cache is in play + buildCacheContext.isSkipAllowed = false; + + const projectConfiguration: RushProjectConfiguration | undefined = + await RushProjectConfiguration.tryLoadForProjectAsync(rushProject, terminal); + if (projectConfiguration) { + projectConfiguration.validatePhaseConfiguration(selectedPhases, terminal); + if (projectConfiguration.disableBuildCacheForProject) { + terminal.writeVerboseLine('Caching has been disabled for this project.'); + } else { + const operationSettings: IOperationSettings | undefined = + projectConfiguration.operationSettingsByOperationName.get(commandName); + if (!operationSettings) { + terminal.writeVerboseLine( + `This project does not define the caching behavior of the "${commandName}" command, so caching has been disabled.` + ); + } else if (operationSettings.disableBuildCacheForOperation) { + terminal.writeVerboseLine( + `Caching has been disabled for this project's "${commandName}" command.` + ); + } else { + const projectOutputFolderNames: ReadonlyArray = + operationSettings.outputFolderNames || []; + const additionalProjectOutputFilePaths: ReadonlyArray = [ + ...(operationMetadataManager?.relativeFilepaths || []) + ]; + const additionalContext: Record = {}; + if (operationSettings.dependsOnEnvVars) { + for (const varName of operationSettings.dependsOnEnvVars) { + additionalContext['$' + varName] = process.env[varName] || ''; + } + } + + if (operationSettings.dependsOnAdditionalFiles) { + const repoState: IRawRepoState | undefined = + await projectChangeAnalyzer._ensureInitializedAsync(terminal); + + const additionalFiles: Map = await getHashesForGlobsAsync( + operationSettings.dependsOnAdditionalFiles, + rushProject.projectFolder, + repoState + ); + + terminal.writeDebugLine( + `Including additional files to calculate build cache hash:\n ${Array.from( + additionalFiles.keys() + ).join('\n ')} ` + ); + + for (const [filePath, fileHash] of additionalFiles) { + additionalContext['file://' + filePath] = fileHash; + } + } + buildCacheContext.projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCache({ + projectConfiguration, + projectOutputFolderNames, + additionalProjectOutputFilePaths, + additionalContext, + buildCacheConfiguration: this._buildCacheConfiguration, + terminal, + command: commandToRun, + trackedProjectFiles: trackedProjectFiles, + projectChangeAnalyzer: projectChangeAnalyzer, + phaseName: phase.name + }); + } + } + } else { + terminal.writeVerboseLine( + `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + + 'or one provided by a rig, so it does not support caching.' + ); + } + } + } + + return buildCacheContext.projectBuildCache; + } + + private async _tryGetCobuildLockAsync({ + runner, + projectBuildCache + }: { + runner: IOperationRunner; + projectBuildCache: ProjectBuildCache | undefined; + }): Promise { + const buildCacheContext: IOperationBuildCacheContext = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + if (!buildCacheContext.cobuildLock) { + buildCacheContext.cobuildLock = undefined; + + if (projectBuildCache && this._cobuildConfiguration && this._cobuildConfiguration.cobuildEnabled) { + buildCacheContext.cobuildLock = new CobuildLock({ + cobuildConfiguration: this._cobuildConfiguration, + projectBuildCache: projectBuildCache + }); + } + } + return buildCacheContext.cobuildLock; + } +} + +function _areShallowEqual(object1: JsonObject, object2: JsonObject): boolean { + for (const n in object1) { + if (!(n in object2) || object1[n] !== object2[n]) { + return false; + } + } + for (const n in object2) { + if (!(n in object1)) { + return false; + } + } + return true; +} diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index 5428e6c0c2a..d9d6c02cf38 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -6,7 +6,7 @@ import type { CollatedWriter } from '@rushstack/stream-collator'; import type { OperationStatus } from './OperationStatus'; import type { OperationMetadataManager } from './OperationMetadataManager'; -import type { IStopwatchResult } from '../../utilities/Stopwatch'; +import type { Stopwatch } from '../../utilities/Stopwatch'; /** * Information passed to the executing `IOperationRunner` @@ -39,7 +39,7 @@ export interface IOperationRunnerContext { /** * Object used to track elapsed time. */ - stopwatch: IStopwatchResult; + stopwatch: Stopwatch; /** * The current execution status of an operation. Operations start in the 'ready' state, * but can be 'blocked' if an upstream operation failed. It is 'executing' when @@ -47,6 +47,12 @@ export interface IOperationRunnerContext { * 'failure'. */ status: OperationStatus; + + /** + * Error which occurred while executing this operation, this is stored in case we need + * it later (for example to re-print errors at end of execution). + */ + error?: Error; } /** @@ -62,11 +68,6 @@ export interface IOperationRunner { */ readonly name: string; - /** - * This flag determines if the operation is allowed to be skipped if up to date. - */ - isSkipAllowed: boolean; - /** * Indicates that this runner's duration has meaning. */ @@ -83,11 +84,6 @@ export interface IOperationRunner { */ warningsAreAllowed: boolean; - /** - * Indicates if the output of this operation may be written to the cache - */ - isCacheWriteAllowed: boolean; - /** * Method to be executed for the operation. */ diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts new file mode 100644 index 00000000000..31580484ce6 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { OperationRunnerLifecycleHooks } from './OperationLifecycle'; + +/** + * A plugin tht interacts with a operation runner + */ +export interface IOperationRunnerPlugin { + /** + * Applies this plugin. + */ + apply(hooks: OperationRunnerLifecycleHooks): void; +} diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index fb44dcf46cf..8c25bd1925e 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -11,6 +11,10 @@ import { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; import { IExecutionResult } from './IOperationExecutionResult'; +import { + CacheableOperationRunnerPlugin, + IOperationBuildCacheContext +} from './CacheableOperationRunnerPlugin'; export interface IOperationExecutionManagerOptions { quietMode: boolean; @@ -219,8 +223,11 @@ export class OperationExecutionManager { private _onOperationComplete(record: OperationExecutionRecord, executionQueue: AsyncOperationQueue): void { const { runner, name, status } = record; - let blockCacheWrite: boolean = !runner.isCacheWriteAllowed; - let blockSkip: boolean = !runner.isSkipAllowed; + const buildCacheContext: IOperationBuildCacheContext | undefined = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(runner); + + let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; + let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; const silent: boolean = runner.silent; @@ -321,12 +328,15 @@ export class OperationExecutionManager { // Apply status changes to direct dependents for (const item of record.consumers) { - if (blockCacheWrite) { - item.runner.isCacheWriteAllowed = false; - } - - if (blockSkip) { - item.runner.isSkipAllowed = false; + const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(item.runner); + if (itemRunnerBuildCacheContext) { + if (blockCacheWrite) { + itemRunnerBuildCacheContext.isCacheWriteAllowed = false; + } + if (blockSkip) { + itemRunnerBuildCacheContext.isSkipAllowed = false; + } } if (status !== OperationStatus.RemoteExecuting) { diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index f20f05fe25f..127aeb59252 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -138,11 +138,6 @@ export class OperationExecutionRecord implements IOperationRunnerContext { try { this.status = await this.runner.executeAsync(this); - - if (this.status === OperationStatus.RemoteExecuting) { - this.stopwatch.reset(); - } - // Delegate global state reporting onResult(this); } catch (error) { diff --git a/libraries/rush-lib/src/logic/operations/OperationLifecycle.ts b/libraries/rush-lib/src/logic/operations/OperationLifecycle.ts new file mode 100644 index 00000000000..fbec2c25ef7 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/OperationLifecycle.ts @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { AsyncSeriesWaterfallHook } from 'tapable'; + +import type { ITerminal } from '@rushstack/node-core-library'; +import type { IOperationRunnerContext } from './IOperationRunner'; +import type { OperationStatus } from './OperationStatus'; +import type { IProjectDeps, ShellOperationRunner } from './ShellOperationRunner'; +import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import type { IPhase } from '../../api/CommandLineConfiguration'; +import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; + +export interface IOperationRunnerBeforeExecuteContext { + context: IOperationRunnerContext; + runner: ShellOperationRunner; + earlyReturnStatus: OperationStatus | undefined; + terminal: ITerminal; + projectDeps: IProjectDeps | undefined; + lastProjectDeps: IProjectDeps | undefined; + trackedProjectFiles: string[] | undefined; + logPath: string; + errorLogPath: string; + rushProject: RushConfigurationProject; + phase: IPhase; + selectedPhases: Iterable; + projectChangeAnalyzer: ProjectChangeAnalyzer; + commandName: string; + commandToRun: string; +} + +export interface IOperationRunnerAfterExecuteContext { + context: IOperationRunnerContext; + runner: ShellOperationRunner; + terminal: ITerminal; + /** + * Exit code of the operation command + */ + exitCode: number; + status: OperationStatus; + taskIsSuccessful: boolean; +} + +/** + * Hooks into the lifecycle of the operation runner + * + */ +export class OperationRunnerLifecycleHooks { + public beforeExecute: AsyncSeriesWaterfallHook = + new AsyncSeriesWaterfallHook( + ['beforeExecuteContext'], + 'beforeExecute' + ); + + public afterExecute: AsyncSeriesWaterfallHook = + new AsyncSeriesWaterfallHook( + ['afterExecuteContext'], + 'afterExecute' + ); +} diff --git a/libraries/rush-lib/src/logic/operations/RunnerWatcher.ts b/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts similarity index 90% rename from libraries/rush-lib/src/logic/operations/RunnerWatcher.ts rename to libraries/rush-lib/src/logic/operations/PeriodicCallback.ts index e5823454cf6..f3cc9f1e141 100644 --- a/libraries/rush-lib/src/logic/operations/RunnerWatcher.ts +++ b/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts @@ -3,7 +3,7 @@ export type ICallbackFn = () => Promise | void; -export interface IRunnerWatcherOptions { +export interface IPeriodicCallbackOptions { interval: number; } @@ -12,13 +12,13 @@ export interface IRunnerWatcherOptions { * * @beta */ -export class RunnerWatcher { +export class PeriodicCallback { private _callbacks: ICallbackFn[]; private _interval: number; private _timeoutId: NodeJS.Timeout | undefined; private _isRunning: boolean; - public constructor(options: IRunnerWatcherOptions) { + public constructor(options: IPeriodicCallbackOptions) { this._callbacks = []; this._interval = options.interval; this._isRunning = false; diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 678325448e5..e6871be2fd6 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -7,10 +7,8 @@ import { JsonFile, Text, FileSystem, - JsonObject, NewlineKind, InternalError, - ITerminal, Terminal, ColorValue } from '@rushstack/node-core-library'; @@ -19,30 +17,26 @@ import { TextRewriterTransform, StderrLineTransform, SplitterTransform, - DiscardStdoutTransform, - PrintUtilities + DiscardStdoutTransform } from '@rushstack/terminal'; import { CollatedTerminal } from '@rushstack/stream-collator'; -import { Utilities, UNINITIALIZED } from '../../utilities/Utilities'; +import { Utilities } from '../../utilities/Utilities'; import { OperationStatus } from './OperationStatus'; import { OperationError } from './OperationError'; import { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; import { ProjectLogWritable } from './ProjectLogWritable'; -import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; -import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; -import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; -import { RushConstants } from '../RushConstants'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; -import { OperationMetadataManager } from './OperationMetadataManager'; -import { RunnerWatcher } from './RunnerWatcher'; -import { CobuildLock, ICobuildCompletedState } from '../cobuild/CobuildLock'; +import { PeriodicCallback } from './PeriodicCallback'; +import { + IOperationRunnerAfterExecuteContext, + IOperationRunnerBeforeExecuteContext, + OperationRunnerLifecycleHooks +} from './OperationLifecycle'; import type { RushConfiguration } from '../../api/RushConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import type { ProjectChangeAnalyzer, IRawRepoState } from '../ProjectChangeAnalyzer'; -import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; -import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import type { IPhase } from '../../api/CommandLineConfiguration'; export interface IProjectDeps { @@ -53,10 +47,7 @@ export interface IProjectDeps { export interface IOperationRunnerOptions { rushProject: RushConfigurationProject; rushConfiguration: RushConfiguration; - buildCacheConfiguration: BuildCacheConfiguration | undefined; - cobuildConfiguration: CobuildConfiguration | undefined; commandToRun: string; - isIncrementalBuildAllowed: boolean; projectChangeAnalyzer: ProjectChangeAnalyzer; displayName: string; phase: IPhase; @@ -66,20 +57,6 @@ export interface IOperationRunnerOptions { selectedPhases: Iterable; } -function _areShallowEqual(object1: JsonObject, object2: JsonObject): boolean { - for (const n in object1) { - if (!(n in object2) || object1[n] !== object2[n]) { - return false; - } - } - for (const n in object2) { - if (!(n in object1)) { - return false; - } - } - return true; -} - /** * An `IOperationRunner` subclass that performs an operation via a shell command. * Currently contains the build cache logic, pending extraction as separate operations. @@ -88,33 +65,23 @@ function _areShallowEqual(object1: JsonObject, object2: JsonObject): boolean { export class ShellOperationRunner implements IOperationRunner { public readonly name: string; - // This runner supports cache writes by default. - public isCacheWriteAllowed: boolean = true; - public isSkipAllowed: boolean; public readonly reportTiming: boolean = true; public readonly silent: boolean = false; public readonly warningsAreAllowed: boolean; + public readonly hooks: OperationRunnerLifecycleHooks; + public readonly periodicCallback: PeriodicCallback; + private readonly _rushProject: RushConfigurationProject; private readonly _phase: IPhase; private readonly _rushConfiguration: RushConfiguration; - private readonly _buildCacheConfiguration: BuildCacheConfiguration | undefined; - private readonly _cobuildConfiguration: CobuildConfiguration | undefined; private readonly _commandName: string; private readonly _commandToRun: string; - private readonly _isCacheReadAllowed: boolean; private readonly _projectChangeAnalyzer: ProjectChangeAnalyzer; private readonly _packageDepsFilename: string; private readonly _logFilenameIdentifier: string; private readonly _selectedPhases: Iterable; - /** - * UNINITIALIZED === we haven't tried to initialize yet - * undefined === we didn't create one because the feature is not enabled - */ - private _projectBuildCache: ProjectBuildCache | undefined | UNINITIALIZED = UNINITIALIZED; - private _cobuildLock: CobuildLock | undefined | UNINITIALIZED = UNINITIALIZED; - public constructor(options: IOperationRunnerOptions) { const { phase } = options; @@ -122,18 +89,19 @@ export class ShellOperationRunner implements IOperationRunner { this._rushProject = options.rushProject; this._phase = phase; this._rushConfiguration = options.rushConfiguration; - this._buildCacheConfiguration = options.buildCacheConfiguration; - this._cobuildConfiguration = options.cobuildConfiguration; this._commandName = phase.name; this._commandToRun = options.commandToRun; - this._isCacheReadAllowed = options.isIncrementalBuildAllowed; - this.isSkipAllowed = options.isIncrementalBuildAllowed; this._projectChangeAnalyzer = options.projectChangeAnalyzer; this._packageDepsFilename = `package-deps_${phase.logFilenameIdentifier}.json`; this.warningsAreAllowed = EnvironmentConfiguration.allowWarningsInSuccessfulBuild || phase.allowWarningsOnSuccess || false; this._logFilenameIdentifier = phase.logFilenameIdentifier; this._selectedPhases = options.selectedPhases; + + this.hooks = new OperationRunnerLifecycleHooks(); + this.periodicCallback = new PeriodicCallback({ + interval: 10 * 1000 + }); } public async executeAsync(context: IOperationRunnerContext): Promise { @@ -157,9 +125,6 @@ export class ShellOperationRunner implements IOperationRunner { context.collatedWriter.terminal, this._logFilenameIdentifier ); - const runnerWatcher: RunnerWatcher = new RunnerWatcher({ - interval: 10 * 1000 - }); try { const removeColorsTransform: TextRewriterTransform = new TextRewriterTransform({ @@ -236,16 +201,6 @@ export class ShellOperationRunner implements IOperationRunner { files, arguments: this._commandToRun }; - } else if (this.isSkipAllowed) { - // To test this code path: - // Remove the `.git` folder then run "rush build --verbose" - terminal.writeLine({ - text: PrintUtilities.wrapWords( - 'This workspace does not appear to be tracked by Git. ' + - 'Rush will proceed without incremental execution, caching, and change detection.' - ), - foregroundColor: ColorValue.Cyan - }); } } catch (error) { // To test this code path: @@ -257,108 +212,28 @@ export class ShellOperationRunner implements IOperationRunner { }); } - // Try to acquire the cobuild lock - let cobuildLock: CobuildLock | undefined; - if (this._cobuildConfiguration?.cobuildEnabled) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ - terminal, - trackedProjectFiles, - operationMetadataManager: context._operationMetadataManager - }); - cobuildLock = await this._tryGetCobuildLockAsync(projectBuildCache); - } - - // If possible, we want to skip this operation -- either by restoring it from the - // cache, if caching is enabled, or determining that the project - // is unchanged (using the older incremental execution logic). These two approaches, - // "caching" and "skipping", are incompatible, so only one applies. - // - // Note that "caching" and "skipping" take two different approaches - // to tracking dependents: - // - // - For caching, "isCacheReadAllowed" is set if a project supports - // incremental builds, and determining whether this project or a dependent - // has changed happens inside the hashing logic. - // - // - For skipping, "isSkipAllowed" is set to true initially, and during - // the process of running dependents, it will be changed by OperationExecutionManager to - // false if a dependency wasn't able to be skipped. - // - let buildCacheReadAttempted: boolean = false; - if (cobuildLock) { - // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to - // the build cache once completed, but does not retrieve them (since the "incremental" - // flag is disabled). However, we still need a cobuild to be able to retrieve a finished - // build from another cobuild in this case. - const cobuildCompletedState: ICobuildCompletedState | undefined = - await cobuildLock.getCompletedStateAsync(); - if (cobuildCompletedState) { - const { status, cacheId } = cobuildCompletedState; - - const restoreFromCacheSuccess: boolean | undefined = - await cobuildLock.projectBuildCache.tryRestoreFromCacheAsync(terminal, cacheId); - - if (restoreFromCacheSuccess) { - // Restore the original state of the operation without cache - await context._operationMetadataManager?.tryRestoreAsync({ - terminal, - logPath: projectLogWritable.logPath, - errorLogPath: projectLogWritable.errorLogPath - }); - if (cobuildCompletedState) { - return cobuildCompletedState.status; - } - return status; - } - } - } else if (this._isCacheReadAllowed) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ - terminal, - trackedProjectFiles, - operationMetadataManager: context._operationMetadataManager - }); - - buildCacheReadAttempted = !!projectBuildCache; - const restoreFromCacheSuccess: boolean | undefined = - await projectBuildCache?.tryRestoreFromCacheAsync(terminal); - - if (restoreFromCacheSuccess) { - // Restore the original state of the operation without cache - await context._operationMetadataManager?.tryRestoreAsync({ - terminal, - logPath: projectLogWritable.logPath, - errorLogPath: projectLogWritable.errorLogPath - }); - return OperationStatus.FromCache; - } - } - if (this.isSkipAllowed && !buildCacheReadAttempted) { - const isPackageUnchanged: boolean = !!( - lastProjectDeps && - projectDeps && - projectDeps.arguments === lastProjectDeps.arguments && - _areShallowEqual(projectDeps.files, lastProjectDeps.files) - ); - - if (isPackageUnchanged) { - return OperationStatus.Skipped; - } - } - - if (this.isCacheWriteAllowed && cobuildLock) { - const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); - if (acquireSuccess) { - if (context.status === OperationStatus.RemoteExecuting) { - // This operation is used to marked remote executing, now change it to executing - context.status = OperationStatus.Executing; - } - runnerWatcher.addCallback(async () => { - await cobuildLock?.renewLockAsync(); - }); - } else { - // failed to acquire the lock, mark current operation to remote executing - return OperationStatus.RemoteExecuting; - } + const beforeExecuteContext: IOperationRunnerBeforeExecuteContext = { + context, + runner: this, + terminal, + projectDeps, + lastProjectDeps, + trackedProjectFiles, + logPath: projectLogWritable.logPath, + errorLogPath: projectLogWritable.errorLogPath, + rushProject: this._rushProject, + phase: this._phase, + selectedPhases: this._selectedPhases, + projectChangeAnalyzer: this._projectChangeAnalyzer, + commandName: this._commandName, + commandToRun: this._commandToRun, + earlyReturnStatus: undefined + }; + + await this.hooks.beforeExecute.promise(beforeExecuteContext); + + if (beforeExecuteContext.earlyReturnStatus) { + return beforeExecuteContext.earlyReturnStatus; } // If the deps file exists, remove it before starting execution. @@ -382,7 +257,7 @@ export class ShellOperationRunner implements IOperationRunner { // Run the operation terminal.writeLine('Invoking: ' + this._commandToRun); - runnerWatcher.start(); + this.periodicCallback.start(); const subProcess: child_process.ChildProcess = Utilities.executeLifecycleCommandAsync( this._commandToRun, @@ -412,18 +287,16 @@ export class ShellOperationRunner implements IOperationRunner { }); } + let exitCode: number = 1; let status: OperationStatus = await new Promise( (resolve: (status: OperationStatus) => void, reject: (error: OperationError) => void) => { subProcess.on('close', (code: number) => { + exitCode = code; try { if (code !== 0) { - if (cobuildLock) { - // In order to preventing the worst case that all cobuild tasks go through the same failure, - // allowing a failing build to be cached and retrieved - resolve(OperationStatus.Failure); - } else { - reject(new OperationError('error', `Returned error code: ${code}`)); - } + // Do NOT reject here immediately, give a chance for hooks to suppress the error + context.error = new OperationError('error', `Returned error code: ${code}`); + resolve(OperationStatus.Failure); } else if (hasWarningOrError) { resolve(OperationStatus.SuccessWithWarning); } else { @@ -436,13 +309,6 @@ export class ShellOperationRunner implements IOperationRunner { } ); - const taskIsSuccessful: boolean = - status === OperationStatus.Success || - (status === OperationStatus.SuccessWithWarning && - this.warningsAreAllowed && - !!this._rushConfiguration.experimentsConfiguration.configuration - .buildCacheWithAllowWarningsInSuccessfulBuild); - // Save the metadata to disk const { duration: durationInSeconds } = context.stopwatch; await context._operationMetadataManager?.saveAsync({ @@ -451,62 +317,40 @@ export class ShellOperationRunner implements IOperationRunner { errorLogPath: projectLogWritable.errorLogPath }); - let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; - let setCacheEntryPromise: Promise | undefined; - if (cobuildLock && this.isCacheWriteAllowed) { - const { projectBuildCache } = cobuildLock; - const cacheId: string | undefined = projectBuildCache.cacheId; - const contextId: string = cobuildLock.cobuildConfiguration.contextId; - - if (cacheId) { - const finalCacheId: string = - status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; - switch (status) { - case OperationStatus.SuccessWithWarning: - case OperationStatus.Success: - case OperationStatus.Failure: { - const currentStatus: ICobuildCompletedState['status'] = status; - setCompletedStatePromiseFunction = () => { - return cobuildLock?.setCompletedStateAsync({ - status: currentStatus, - cacheId: finalCacheId - }); - }; - setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( - terminal, - finalCacheId - ); - } - } - } - } + const taskIsSuccessful: boolean = + status === OperationStatus.Success || + (status === OperationStatus.SuccessWithWarning && + this.warningsAreAllowed && + !!this._rushConfiguration.experimentsConfiguration.configuration + .buildCacheWithAllowWarningsInSuccessfulBuild); - let writeProjectStatePromise: Promise | undefined; if (taskIsSuccessful && projectDeps) { // Write deps on success. - writeProjectStatePromise = JsonFile.saveAsync(projectDeps, currentDepsPath, { + await JsonFile.saveAsync(projectDeps, currentDepsPath, { ensureFolderExists: true }); + } - // If the command is successful, we can calculate project hash, and no dependencies were skipped, - // write a new cache entry. - if (!setCacheEntryPromise && this.isCacheWriteAllowed) { - setCacheEntryPromise = ( - await this._tryGetProjectBuildCacheAsync({ - terminal, - trackedProjectFiles, - operationMetadataManager: context._operationMetadataManager - }) - )?.trySetCacheEntryAsync(terminal); - } + const afterExecuteContext: IOperationRunnerAfterExecuteContext = { + context, + runner: this, + terminal, + exitCode, + status, + taskIsSuccessful + }; + + await this.hooks.afterExecute.promise(afterExecuteContext); + + if (context.error) { + throw context.error; } - const [, cacheWriteSuccess] = await Promise.all([writeProjectStatePromise, setCacheEntryPromise]); - await setCompletedStatePromiseFunction?.(); + + // Sync the status in case it was changed by the hook + status = afterExecuteContext.status; if (terminalProvider.hasErrors) { status = OperationStatus.Failure; - } else if (cacheWriteSuccess === false && status === OperationStatus.Success) { - status = OperationStatus.SuccessWithWarning; } normalizeNewlineTransform.close(); @@ -520,116 +364,8 @@ export class ShellOperationRunner implements IOperationRunner { return status; } finally { projectLogWritable.close(); - runnerWatcher.stop(); - } - } - - private async _tryGetProjectBuildCacheAsync({ - terminal, - trackedProjectFiles, - operationMetadataManager - }: { - terminal: ITerminal; - trackedProjectFiles: string[] | undefined; - operationMetadataManager: OperationMetadataManager | undefined; - }): Promise { - if (this._projectBuildCache === UNINITIALIZED) { - this._projectBuildCache = undefined; - - if (this._buildCacheConfiguration && this._buildCacheConfiguration.buildCacheEnabled) { - // Disable legacy skip logic if the build cache is in play - this.isSkipAllowed = false; - - const projectConfiguration: RushProjectConfiguration | undefined = - await RushProjectConfiguration.tryLoadForProjectAsync(this._rushProject, terminal); - if (projectConfiguration) { - projectConfiguration.validatePhaseConfiguration(this._selectedPhases, terminal); - if (projectConfiguration.disableBuildCacheForProject) { - terminal.writeVerboseLine('Caching has been disabled for this project.'); - } else { - const operationSettings: IOperationSettings | undefined = - projectConfiguration.operationSettingsByOperationName.get(this._commandName); - if (!operationSettings) { - terminal.writeVerboseLine( - `This project does not define the caching behavior of the "${this._commandName}" command, so caching has been disabled.` - ); - } else if (operationSettings.disableBuildCacheForOperation) { - terminal.writeVerboseLine( - `Caching has been disabled for this project's "${this._commandName}" command.` - ); - } else { - const projectOutputFolderNames: ReadonlyArray = - operationSettings.outputFolderNames || []; - const additionalProjectOutputFilePaths: ReadonlyArray = [ - ...(operationMetadataManager?.relativeFilepaths || []) - ]; - const additionalContext: Record = {}; - if (operationSettings.dependsOnEnvVars) { - for (const varName of operationSettings.dependsOnEnvVars) { - additionalContext['$' + varName] = process.env[varName] || ''; - } - } - - if (operationSettings.dependsOnAdditionalFiles) { - const repoState: IRawRepoState | undefined = - await this._projectChangeAnalyzer._ensureInitializedAsync(terminal); - - const additionalFiles: Map = await getHashesForGlobsAsync( - operationSettings.dependsOnAdditionalFiles, - this._rushProject.projectFolder, - repoState - ); - - terminal.writeDebugLine( - `Including additional files to calculate build cache hash:\n ${Array.from( - additionalFiles.keys() - ).join('\n ')} ` - ); - - for (const [filePath, fileHash] of additionalFiles) { - additionalContext['file://' + filePath] = fileHash; - } - } - this._projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCache({ - projectConfiguration, - projectOutputFolderNames, - additionalProjectOutputFilePaths, - additionalContext, - buildCacheConfiguration: this._buildCacheConfiguration, - terminal, - command: this._commandToRun, - trackedProjectFiles: trackedProjectFiles, - projectChangeAnalyzer: this._projectChangeAnalyzer, - phaseName: this._phase.name - }); - } - } - } else { - terminal.writeVerboseLine( - `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + - 'or one provided by a rig, so it does not support caching.' - ); - } - } - } - - return this._projectBuildCache; - } - - private async _tryGetCobuildLockAsync( - projectBuildCache: ProjectBuildCache | undefined - ): Promise { - if (this._cobuildLock === UNINITIALIZED) { - this._cobuildLock = undefined; - - if (projectBuildCache && this._cobuildConfiguration && this._cobuildConfiguration.cobuildEnabled) { - this._cobuildLock = new CobuildLock({ - cobuildConfiguration: this._cobuildConfiguration, - projectBuildCache: projectBuildCache - }); - } + this.periodicCallback.stop(); } - return this._cobuildLock; } } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts index 1d1a425feef..f00e7951f5c 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts @@ -13,6 +13,7 @@ import type { PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; import { Operation } from './Operation'; +import { CacheableOperationRunnerPlugin } from './CacheableOperationRunnerPlugin'; const PLUGIN_NAME: 'ShellOperationRunnerPlugin' = 'ShellOperationRunnerPlugin'; @@ -22,6 +23,9 @@ const PLUGIN_NAME: 'ShellOperationRunnerPlugin' = 'ShellOperationRunnerPlugin'; export class ShellOperationRunnerPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { hooks.createOperations.tap(PLUGIN_NAME, createShellOperations); + hooks.afterExecuteOperations.tap(PLUGIN_NAME, () => { + CacheableOperationRunnerPlugin.clearAllBuildCacheContexts(); + }); } } @@ -78,18 +82,32 @@ function createShellOperations( const displayName: string = getDisplayName(phase, project); if (commandToRun) { - operation.runner = new ShellOperationRunner({ - buildCacheConfiguration, - cobuildConfiguration, + const shellOperationRunner: ShellOperationRunner = new ShellOperationRunner({ commandToRun: commandToRun || '', displayName, - isIncrementalBuildAllowed, phase, projectChangeAnalyzer, rushConfiguration, rushProject: project, selectedPhases }); + + if (buildCacheConfiguration) { + new CacheableOperationRunnerPlugin({ + buildCacheConfiguration, + cobuildConfiguration, + isIncrementalBuildAllowed + }).apply(shellOperationRunner.hooks); + CacheableOperationRunnerPlugin.setBuildCacheContextByRunner(shellOperationRunner, { + // This runner supports cache writes by default. + isCacheWriteAllowed: true, + isCacheReadAllowed: isIncrementalBuildAllowed, + isSkipAllowed: isIncrementalBuildAllowed, + projectBuildCache: undefined, + cobuildLock: undefined + }); + } + operation.runner = shellOperationRunner; } else { // Empty build script indicates a no-op, so use a no-op runner operation.runner = new NullOperationRunner({ From 1daf2bbad3238307e4a31cfd7f3299a232bc819d Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 14:49:34 +0800 Subject: [PATCH 22/55] chore: improve onComplete in AsyncOperationQueue --- .../src/logic/operations/AsyncOperationQueue.ts | 10 +++++----- .../src/logic/operations/OperationExecutionManager.ts | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index c4bd76ba542..bf2e1a34442 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -19,8 +19,8 @@ export class AsyncOperationQueue private readonly _queue: OperationExecutionRecord[]; private readonly _pendingIterators: ((result: IteratorResult) => void)[]; private readonly _totalOperations: number; + private readonly _completedOperations: Set; - private _completedOperations: number; private _isDone: boolean; /** @@ -35,7 +35,7 @@ export class AsyncOperationQueue this._pendingIterators = []; this._totalOperations = this._queue.length; this._isDone = false; - this._completedOperations = 0; + this._completedOperations = new Set(); } /** @@ -60,9 +60,9 @@ export class AsyncOperationQueue * Set a callback to be invoked when one operation is completed. * If all operations are completed, set the queue to done, resolve all pending iterators in next cycle. */ - public complete(): void { - this._completedOperations++; - if (this._completedOperations === this._totalOperations) { + public complete(record: OperationExecutionRecord): void { + this._completedOperations.add(record); + if (this._completedOperations.size === this._totalOperations) { this._isDone = true; } } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 8c25bd1925e..0b433b8dfcc 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -248,7 +248,7 @@ export class OperationExecutionManager { const blockedQueue: Set = new Set(record.consumers); for (const blockedRecord of blockedQueue) { if (blockedRecord.status === OperationStatus.Ready) { - executionQueue.complete(); + executionQueue.complete(blockedRecord); this._completedOperations++; // Now that we have the concept of architectural no-ops, we could implement this by replacing @@ -347,7 +347,7 @@ export class OperationExecutionManager { if (record.status !== OperationStatus.RemoteExecuting) { // If the operation was not remote, then we can notify queue that it is complete - executionQueue.complete(); + executionQueue.complete(record); } } } From 6d250a428b2184174786485654a05ea7d6655d96 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 14:52:44 +0800 Subject: [PATCH 23/55] chore: use setInterval in PeriodicCallback --- .../src/logic/operations/PeriodicCallback.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts b/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts index f3cc9f1e141..26aa1814f55 100644 --- a/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts +++ b/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts @@ -15,7 +15,7 @@ export interface IPeriodicCallbackOptions { export class PeriodicCallback { private _callbacks: ICallbackFn[]; private _interval: number; - private _timeoutId: NodeJS.Timeout | undefined; + private _intervalId: NodeJS.Timeout | undefined; private _isRunning: boolean; public constructor(options: IPeriodicCallbackOptions) { @@ -32,24 +32,22 @@ export class PeriodicCallback { } public start(): void { - if (this._timeoutId) { + if (this._intervalId) { throw new Error('Watcher already started'); } if (this._callbacks.length === 0) { return; } this._isRunning = true; - this._timeoutId = setTimeout(() => { + this._intervalId = setInterval(() => { this._callbacks.forEach((callback) => callback()); - this._timeoutId = undefined; - this.start(); }, this._interval); } public stop(): void { - if (this._timeoutId) { - clearTimeout(this._timeoutId); - this._timeoutId = undefined; + if (this._intervalId) { + clearInterval(this._intervalId); + this._intervalId = undefined; this._isRunning = false; } } From 00b87cbfded73cf5c366d29356a5585204da3077 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 19:44:55 +0800 Subject: [PATCH 24/55] refact: CacheableOperationPlugin --- common/reviews/api/rush-lib.api.md | 10 +- .../cli/scriptActions/PhasedScriptAction.ts | 4 + ...rPlugin.ts => CacheableOperationPlugin.ts} | 214 ++++++++++++------ .../operations/IOperationRunnerPlugin.ts | 14 -- .../operations/OperationExecutionManager.ts | 38 +--- .../operations/OperationExecutionRecord.ts | 10 +- ...onLifecycle.ts => OperationRunnerHooks.ts} | 13 +- .../logic/operations/PhasedOperationHooks.ts | 29 +++ .../logic/operations/ShellOperationRunner.ts | 9 +- .../operations/ShellOperationRunnerPlugin.ts | 29 +-- .../test/AsyncOperationQueue.test.ts | 8 +- .../src/pluginFramework/PhasedCommandHooks.ts | 13 ++ 12 files changed, 234 insertions(+), 157 deletions(-) rename libraries/rush-lib/src/logic/operations/{CacheableOperationRunnerPlugin.ts => CacheableOperationPlugin.ts} (72%) delete mode 100644 libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts rename libraries/rush-lib/src/logic/operations/{OperationLifecycle.ts => OperationRunnerHooks.ts} (90%) create mode 100644 libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 901545dfa60..fb32629c02f 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -9,7 +9,7 @@ import { AsyncParallelHook } from 'tapable'; import { AsyncSeriesHook } from 'tapable'; import { AsyncSeriesWaterfallHook } from 'tapable'; -import type { CollatedWriter } from '@rushstack/stream-collator'; +import { CollatedWriter } from '@rushstack/stream-collator'; import type { CommandLineParameter } from '@rushstack/ts-command-line'; import { HookMap } from 'tapable'; import { IPackageJson } from '@rushstack/node-core-library'; @@ -17,9 +17,11 @@ import { ITerminal } from '@rushstack/node-core-library'; import { ITerminalProvider } from '@rushstack/node-core-library'; import { JsonObject } from '@rushstack/node-core-library'; import { PackageNameParser } from '@rushstack/node-core-library'; -import type { StdioSummarizer } from '@rushstack/terminal'; +import { StdioSummarizer } from '@rushstack/terminal'; +import { StreamCollator } from '@rushstack/stream-collator'; import { SyncHook } from 'tapable'; import { Terminal } from '@rushstack/node-core-library'; +import { TerminalWritable } from '@rushstack/terminal'; // @public export class ApprovedPackagesConfiguration { @@ -807,6 +809,10 @@ export abstract class PackageManagerOptionsConfigurationBase implements IPackage export class PhasedCommandHooks { readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, ICreateOperationsContext]>; readonly createOperations: AsyncSeriesWaterfallHook<[Set, ICreateOperationsContext]>; + // Warning: (ae-forgotten-export) The symbol "OperationExecutionManager" needs to be exported by the entry point index.d.ts + // + // @internal + readonly operationExecutionManager: AsyncSeriesHook; readonly waitingForChanges: SyncHook; } diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 8cb5064fa3c..883c3a5916c 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -38,6 +38,7 @@ import { OperationResultSummarizerPlugin } from '../../logic/operations/Operatio import type { ITelemetryOperationResult } from '../../logic/Telemetry'; import { parseParallelism } from '../parsing/ParseParallelism'; import { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import { CacheableOperationPlugin } from '../../logic/operations/CacheableOperationPlugin'; /** * Constructor parameters for PhasedScriptAction. @@ -141,6 +142,8 @@ export class PhasedScriptAction extends BaseScriptAction { new PhasedOperationPlugin().apply(this.hooks); // Applies the Shell Operation Runner to selected operations new ShellOperationRunnerPlugin().apply(this.hooks); + // Applies the build cache related logic to the selected operations + new CacheableOperationPlugin().apply(this.hooks); if (this._enableParallelism) { this._parallelismParameter = this.defineStringParameter({ @@ -508,6 +511,7 @@ export class PhasedScriptAction extends BaseScriptAction { operations, executionManagerOptions ); + await this.hooks.operationExecutionManager.promise(executionManager); const { isInitial, isWatch } = options.createOperationsContext; diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts similarity index 72% rename from libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts rename to libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 4005e754d27..3b9dde61605 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -1,36 +1,37 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import { ColorValue, InternalError, ITerminal, JsonObject } from '@rushstack/node-core-library'; +import { ShellOperationRunner } from './ShellOperationRunner'; +import { OperationStatus } from './OperationStatus'; import { CobuildLock, ICobuildCompletedState } from '../cobuild/CobuildLock'; import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; -import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; -import { OperationStatus } from './OperationStatus'; -import { ColorValue, InternalError, ITerminal, JsonObject } from '@rushstack/node-core-library'; +import { PrintUtilities } from '@rushstack/terminal'; import { RushConstants } from '../RushConstants'; +import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; -import { PrintUtilities } from '@rushstack/terminal'; -import type { IOperationRunnerPlugin } from './IOperationRunnerPlugin'; +import type { Operation } from './Operation'; +import type { OperationExecutionManager } from './OperationExecutionManager'; +import type { OperationExecutionRecord } from './OperationExecutionRecord'; import type { IOperationRunnerAfterExecuteContext, - IOperationRunnerBeforeExecuteContext, - OperationRunnerLifecycleHooks -} from './OperationLifecycle'; -import type { OperationMetadataManager } from './OperationMetadataManager'; + IOperationRunnerBeforeExecuteContext +} from './OperationRunnerHooks'; import type { IOperationRunner } from './IOperationRunner'; -import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; -import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; -import type { IPhase } from '../../api/CommandLineConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import type { + ICreateOperationsContext, + IPhasedCommandPlugin, + PhasedCommandHooks +} from '../../pluginFramework/PhasedCommandHooks'; +import type { IPhase } from '../../api/CommandLineConfiguration'; import type { IRawRepoState, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; +import type { OperationMetadataManager } from './OperationMetadataManager'; +import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; +import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; -const PLUGIN_NAME: 'CacheableOperationRunnerPlugin' = 'CacheableOperationRunnerPlugin'; - -export interface ICacheableOperationRunnerPluginOptions { - buildCacheConfiguration: BuildCacheConfiguration; - cobuildConfiguration: CobuildConfiguration | undefined; - isIncrementalBuildAllowed: boolean; -} +const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; export interface IOperationBuildCacheContext { isCacheWriteAllowed: boolean; @@ -40,49 +41,102 @@ export interface IOperationBuildCacheContext { cobuildLock: CobuildLock | undefined; } -export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { - private static _runnerBuildCacheContextMap: Map = new Map< +export class CacheableOperationPlugin implements IPhasedCommandPlugin { + private _buildCacheContextByOperationRunner: Map = new Map< IOperationRunner, IOperationBuildCacheContext >(); - private readonly _buildCacheConfiguration: BuildCacheConfiguration; - private readonly _cobuildConfiguration: CobuildConfiguration | undefined; - public constructor(options: ICacheableOperationRunnerPluginOptions) { - this._buildCacheConfiguration = options.buildCacheConfiguration; - this._cobuildConfiguration = options.cobuildConfiguration; - } + public apply(hooks: PhasedCommandHooks): void { + hooks.createOperations.tapPromise( + PLUGIN_NAME, + async (operations: Set, context: ICreateOperationsContext): Promise> => { + const { buildCacheConfiguration, isIncrementalBuildAllowed } = context; + if (!buildCacheConfiguration) { + return operations; + } - public static getBuildCacheContextByRunner( - runner: IOperationRunner - ): IOperationBuildCacheContext | undefined { - const buildCacheContext: IOperationBuildCacheContext | undefined = - CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.get(runner); - return buildCacheContext; - } + for (const operation of operations) { + if (operation.runner) { + if (operation.runner instanceof ShellOperationRunner) { + const buildCacheContext: IOperationBuildCacheContext = { + // ShellOperationRunner supports cache writes by default. + isCacheWriteAllowed: true, + isCacheReadAllowed: isIncrementalBuildAllowed, + isSkipAllowed: isIncrementalBuildAllowed, + projectBuildCache: undefined, + cobuildLock: undefined + }; + // Upstream runners may mutate the property of build cache context for downstream runners + this._buildCacheContextByOperationRunner.set(operation.runner, buildCacheContext); + + this._applyOperationRunner(operation.runner, context); + } + } + } - public static getBuildCacheContextByRunnerOrThrow(runner: IOperationRunner): IOperationBuildCacheContext { - const buildCacheContext: IOperationBuildCacheContext | undefined = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(runner); - if (!buildCacheContext) { - // This should not happen - throw new InternalError(`Build cache context for runner ${runner.name} should be defined`); - } - return buildCacheContext; - } + return operations; + } + ); - public static setBuildCacheContextByRunner( - runner: IOperationRunner, - buildCacheContext: IOperationBuildCacheContext - ): void { - CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.set(runner, buildCacheContext); - } + hooks.operationExecutionManager.tap( + PLUGIN_NAME, + (operationExecutionManager: OperationExecutionManager) => { + operationExecutionManager.hooks.afterExecuteOperation.tapPromise( + PLUGIN_NAME, + async (operation: OperationExecutionRecord): Promise => { + const { runner, status, consumers } = operation; + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByRunner(runner); + + let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; + let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; + + switch (status) { + case OperationStatus.Skipped: { + // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. + blockCacheWrite = true; + break; + } + + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: { + // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. + blockSkip ||= !operationExecutionManager.changedProjectsOnly; + break; + } + } - public static clearAllBuildCacheContexts(): void { - CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.clear(); + // Apply status changes to direct dependents + for (const item of consumers) { + const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByRunner(item.runner); + if (itemRunnerBuildCacheContext) { + if (blockCacheWrite) { + itemRunnerBuildCacheContext.isCacheWriteAllowed = false; + } + if (blockSkip) { + itemRunnerBuildCacheContext.isSkipAllowed = false; + } + } + } + return operation; + } + ); + } + ); + + hooks.afterExecuteOperations.tapPromise(PLUGIN_NAME, async () => { + this._buildCacheContextByOperationRunner.clear(); + }); } - public apply(hooks: OperationRunnerLifecycleHooks): void { + private _applyOperationRunner(runner: ShellOperationRunner, context: ICreateOperationsContext): void { + const { buildCacheConfiguration, cobuildConfiguration } = context; + const { hooks } = runner; + + const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + hooks.beforeExecute.tapPromise( PLUGIN_NAME, async (beforeExecuteContext: IOperationRunnerBeforeExecuteContext) => { @@ -108,8 +162,6 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { // If there is existing early return status, we don't need to do anything return earlyReturnStatus; } - const buildCacheContext: IOperationBuildCacheContext = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); if (!projectDeps && buildCacheContext.isSkipAllowed) { // To test this code path: @@ -124,6 +176,7 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { } const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ + buildCacheConfiguration, runner, rushProject, phase, @@ -135,13 +188,20 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { trackedProjectFiles, operationMetadataManager: context._operationMetadataManager }); + // eslint-disable-next-line require-atomic-updates -- we are mutating the build cache context intentionally buildCacheContext.projectBuildCache = projectBuildCache; // Try to acquire the cobuild lock let cobuildLock: CobuildLock | undefined; - if (this._cobuildConfiguration?.cobuildEnabled) { - cobuildLock = await this._tryGetCobuildLockAsync({ runner, projectBuildCache }); + if (cobuildConfiguration?.cobuildEnabled) { + cobuildLock = await this._tryGetCobuildLockAsync({ + runner, + projectBuildCache, + cobuildConfiguration + }); } + + // eslint-disable-next-line require-atomic-updates -- we are mutating the build cache context intentionally buildCacheContext.cobuildLock = cobuildLock; // If possible, we want to skip this operation -- either by restoring it from the @@ -240,12 +300,10 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { } ); - hooks.afterExecute.tapPromise( + runner.hooks.afterExecute.tapPromise( PLUGIN_NAME, async (afterExecuteContext: IOperationRunnerAfterExecuteContext) => { - const { context, runner, terminal, status, taskIsSuccessful } = afterExecuteContext; - const buildCacheContext: IOperationBuildCacheContext = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + const { context, terminal, status, taskIsSuccessful } = afterExecuteContext; const { cobuildLock, projectBuildCache, isCacheWriteAllowed } = buildCacheContext; @@ -305,7 +363,24 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { ); } + private _getBuildCacheContextByRunner(runner: IOperationRunner): IOperationBuildCacheContext | undefined { + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._buildCacheContextByOperationRunner.get(runner); + return buildCacheContext; + } + + private _getBuildCacheContextByRunnerOrThrow(runner: IOperationRunner): IOperationBuildCacheContext { + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByRunner(runner); + if (!buildCacheContext) { + // This should not happen + throw new InternalError(`Build cache context for runner ${runner.name} should be defined`); + } + return buildCacheContext; + } + private async _tryGetProjectBuildCacheAsync({ + buildCacheConfiguration, runner, rushProject, phase, @@ -317,6 +392,7 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { trackedProjectFiles, operationMetadataManager }: { + buildCacheConfiguration: BuildCacheConfiguration | undefined; runner: IOperationRunner; rushProject: RushConfigurationProject; phase: IPhase; @@ -328,10 +404,9 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { trackedProjectFiles: string[] | undefined; operationMetadataManager: OperationMetadataManager | undefined; }): Promise { - const buildCacheContext: IOperationBuildCacheContext = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); if (!buildCacheContext.projectBuildCache) { - if (this._buildCacheConfiguration && this._buildCacheConfiguration.buildCacheEnabled) { + if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { // Disable legacy skip logic if the build cache is in play buildCacheContext.isSkipAllowed = false; @@ -390,7 +465,7 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { projectOutputFolderNames, additionalProjectOutputFilePaths, additionalContext, - buildCacheConfiguration: this._buildCacheConfiguration, + buildCacheConfiguration, terminal, command: commandToRun, trackedProjectFiles: trackedProjectFiles, @@ -412,21 +487,22 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { } private async _tryGetCobuildLockAsync({ + cobuildConfiguration, runner, projectBuildCache }: { + cobuildConfiguration: CobuildConfiguration | undefined; runner: IOperationRunner; projectBuildCache: ProjectBuildCache | undefined; }): Promise { - const buildCacheContext: IOperationBuildCacheContext = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); if (!buildCacheContext.cobuildLock) { buildCacheContext.cobuildLock = undefined; - if (projectBuildCache && this._cobuildConfiguration && this._cobuildConfiguration.cobuildEnabled) { + if (projectBuildCache && cobuildConfiguration && cobuildConfiguration.cobuildEnabled) { buildCacheContext.cobuildLock = new CobuildLock({ - cobuildConfiguration: this._cobuildConfiguration, - projectBuildCache: projectBuildCache + cobuildConfiguration, + projectBuildCache }); } } diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts deleted file mode 100644 index 31580484ce6..00000000000 --- a/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import type { OperationRunnerLifecycleHooks } from './OperationLifecycle'; - -/** - * A plugin tht interacts with a operation runner - */ -export interface IOperationRunnerPlugin { - /** - * Applies this plugin. - */ - apply(hooks: OperationRunnerLifecycleHooks): void; -} diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 0b433b8dfcc..4cce3332824 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -11,10 +11,7 @@ import { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; import { IExecutionResult } from './IOperationExecutionResult'; -import { - CacheableOperationRunnerPlugin, - IOperationBuildCacheContext -} from './CacheableOperationRunnerPlugin'; +import { PhasedOperationHooks } from './PhasedOperationHooks'; export interface IOperationExecutionManagerOptions { quietMode: boolean; @@ -36,7 +33,7 @@ const ASCII_HEADER_WIDTH: number = 79; * tasks are complete, or prematurely fails if any of the tasks fail. */ export class OperationExecutionManager { - private readonly _changedProjectsOnly: boolean; + public readonly changedProjectsOnly: boolean; private readonly _executionRecords: Map; private readonly _quietMode: boolean; private readonly _parallelism: number; @@ -53,13 +50,15 @@ export class OperationExecutionManager { private _hasAnyNonAllowedWarnings: boolean; private _completedOperations: number; + public readonly hooks: PhasedOperationHooks = new PhasedOperationHooks(); + public constructor(operations: Set, options: IOperationExecutionManagerOptions) { const { quietMode, debugMode, parallelism, changedProjectsOnly } = options; this._completedOperations = 0; this._quietMode = quietMode; this._hasAnyFailures = false; this._hasAnyNonAllowedWarnings = false; - this._changedProjectsOnly = changedProjectsOnly; + this.changedProjectsOnly = changedProjectsOnly; this._parallelism = parallelism; // TERMINAL PIPELINE: @@ -189,15 +188,17 @@ export class OperationExecutionManager { // This function is a callback because it may write to the collatedWriter before // operation.executeAsync returns (and cleans up the writer) - const onOperationComplete: (record: OperationExecutionRecord) => void = ( + const onOperationComplete: (record: OperationExecutionRecord) => Promise = async ( record: OperationExecutionRecord ) => { this._onOperationComplete(record, executionQueue); + await this.hooks.afterExecuteOperation.promise(record); }; await Async.forEachAsync( executionQueue, async (operation: OperationExecutionRecord) => { + await this.hooks.beforeExecuteOperation.promise(operation); await operation.executeAsync(onOperationComplete); }, { @@ -223,12 +224,6 @@ export class OperationExecutionManager { private _onOperationComplete(record: OperationExecutionRecord, executionQueue: AsyncOperationQueue): void { const { runner, name, status } = record; - const buildCacheContext: IOperationBuildCacheContext | undefined = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(runner); - - let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; - let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; - const silent: boolean = runner.silent; switch (status) { @@ -287,8 +282,6 @@ export class OperationExecutionManager { if (!silent) { record.collatedWriter.terminal.writeStdoutLine(colors.green(`"${name}" was skipped.`)); } - // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. - blockCacheWrite = true; break; } @@ -308,8 +301,6 @@ export class OperationExecutionManager { colors.green(`"${name}" completed successfully in ${record.stopwatch.toString()}.`) ); } - // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. - blockSkip ||= !this._changedProjectsOnly; break; } @@ -319,8 +310,6 @@ export class OperationExecutionManager { colors.yellow(`"${name}" completed with warnings in ${record.stopwatch.toString()}.`) ); } - // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. - blockSkip ||= !this._changedProjectsOnly; this._hasAnyNonAllowedWarnings = this._hasAnyNonAllowedWarnings || !runner.warningsAreAllowed; break; } @@ -328,17 +317,6 @@ export class OperationExecutionManager { // Apply status changes to direct dependents for (const item of record.consumers) { - const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(item.runner); - if (itemRunnerBuildCacheContext) { - if (blockCacheWrite) { - itemRunnerBuildCacheContext.isCacheWriteAllowed = false; - } - if (blockSkip) { - itemRunnerBuildCacheContext.isSkipAllowed = false; - } - } - if (status !== OperationStatus.RemoteExecuting) { // Remove this operation from the dependencies, to unblock the scheduler item.dependencies.delete(record); diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 127aeb59252..1fe67a79859 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -20,6 +20,8 @@ export interface IOperationExecutionRecordContext { /** * Internal class representing everything about executing an operation + * + * @internal */ export class OperationExecutionRecord implements IOperationRunnerContext { /** @@ -46,6 +48,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { * operation to execute, the operation with the highest criticalPathLength is chosen. * * Example: + * ``` * (0) A * \ * (1) B C (0) (applications) @@ -62,6 +65,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { * X has a score of 1, since the only package which depends on it is A * Z has a score of 2, since only X depends on it, and X has a score of 1 * Y has a score of 2, since the chain Y->X->C is longer than Y->C + * ``` * * The algorithm is implemented in AsyncOperationQueue.ts as calculateCriticalPathLength() */ @@ -132,19 +136,19 @@ export class OperationExecutionRecord implements IOperationRunnerContext { return this._operationMetadataManager?.stateFile.state?.nonCachedDurationMs; } - public async executeAsync(onResult: (record: OperationExecutionRecord) => void): Promise { + public async executeAsync(onResult: (record: OperationExecutionRecord) => Promise): Promise { this.status = OperationStatus.Executing; this.stopwatch.start(); try { this.status = await this.runner.executeAsync(this); // Delegate global state reporting - onResult(this); + await onResult(this); } catch (error) { this.status = OperationStatus.Failure; this.error = error; // Delegate global state reporting - onResult(this); + await onResult(this); } finally { if (this.status !== OperationStatus.RemoteExecuting) { this._collatedWriter?.close(); diff --git a/libraries/rush-lib/src/logic/operations/OperationLifecycle.ts b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts similarity index 90% rename from libraries/rush-lib/src/logic/operations/OperationLifecycle.ts rename to libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts index fbec2c25ef7..813e310283f 100644 --- a/libraries/rush-lib/src/logic/operations/OperationLifecycle.ts +++ b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts @@ -11,6 +11,16 @@ import type { RushConfigurationProject } from '../../api/RushConfigurationProjec import type { IPhase } from '../../api/CommandLineConfiguration'; import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; +/** + * A plugin tht interacts with a operation runner + */ +export interface IOperationRunnerPlugin { + /** + * Applies this plugin. + */ + apply(hooks: OperationRunnerHooks): void; +} + export interface IOperationRunnerBeforeExecuteContext { context: IOperationRunnerContext; runner: ShellOperationRunner; @@ -31,7 +41,6 @@ export interface IOperationRunnerBeforeExecuteContext { export interface IOperationRunnerAfterExecuteContext { context: IOperationRunnerContext; - runner: ShellOperationRunner; terminal: ITerminal; /** * Exit code of the operation command @@ -45,7 +54,7 @@ export interface IOperationRunnerAfterExecuteContext { * Hooks into the lifecycle of the operation runner * */ -export class OperationRunnerLifecycleHooks { +export class OperationRunnerHooks { public beforeExecute: AsyncSeriesWaterfallHook = new AsyncSeriesWaterfallHook( ['beforeExecuteContext'], diff --git a/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts b/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts new file mode 100644 index 00000000000..05e67ac3c48 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { AsyncSeriesWaterfallHook } from 'tapable'; + +import type { OperationExecutionRecord } from './OperationExecutionRecord'; + +/** + * A plugin that interacts with a phased commands. + * @alpha + */ +export interface IPhasedOperationPlugin { + /** + * Applies this plugin. + */ + apply(hooks: PhasedOperationHooks): void; +} + +/** + * Hooks into the execution process for phased operation + * @alpha + */ +export class PhasedOperationHooks { + public beforeExecuteOperation: AsyncSeriesWaterfallHook = + new AsyncSeriesWaterfallHook(['operation'], 'beforeExecuteOperation'); + + public afterExecuteOperation: AsyncSeriesWaterfallHook = + new AsyncSeriesWaterfallHook(['operation'], 'afterExecuteOperation'); +} diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index e6871be2fd6..b29e68e4f2c 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -31,8 +31,8 @@ import { PeriodicCallback } from './PeriodicCallback'; import { IOperationRunnerAfterExecuteContext, IOperationRunnerBeforeExecuteContext, - OperationRunnerLifecycleHooks -} from './OperationLifecycle'; + OperationRunnerHooks +} from './OperationRunnerHooks'; import type { RushConfiguration } from '../../api/RushConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; @@ -69,7 +69,7 @@ export class ShellOperationRunner implements IOperationRunner { public readonly silent: boolean = false; public readonly warningsAreAllowed: boolean; - public readonly hooks: OperationRunnerLifecycleHooks; + public readonly hooks: OperationRunnerHooks; public readonly periodicCallback: PeriodicCallback; private readonly _rushProject: RushConfigurationProject; @@ -98,7 +98,7 @@ export class ShellOperationRunner implements IOperationRunner { this._logFilenameIdentifier = phase.logFilenameIdentifier; this._selectedPhases = options.selectedPhases; - this.hooks = new OperationRunnerLifecycleHooks(); + this.hooks = new OperationRunnerHooks(); this.periodicCallback = new PeriodicCallback({ interval: 10 * 1000 }); @@ -333,7 +333,6 @@ export class ShellOperationRunner implements IOperationRunner { const afterExecuteContext: IOperationRunnerAfterExecuteContext = { context, - runner: this, terminal, exitCode, status, diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts index f00e7951f5c..3e711cbf6bf 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts @@ -13,7 +13,6 @@ import type { PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; import { Operation } from './Operation'; -import { CacheableOperationRunnerPlugin } from './CacheableOperationRunnerPlugin'; const PLUGIN_NAME: 'ShellOperationRunnerPlugin' = 'ShellOperationRunnerPlugin'; @@ -23,9 +22,6 @@ const PLUGIN_NAME: 'ShellOperationRunnerPlugin' = 'ShellOperationRunnerPlugin'; export class ShellOperationRunnerPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { hooks.createOperations.tap(PLUGIN_NAME, createShellOperations); - hooks.afterExecuteOperations.tap(PLUGIN_NAME, () => { - CacheableOperationRunnerPlugin.clearAllBuildCacheContexts(); - }); } } @@ -33,14 +29,7 @@ function createShellOperations( operations: Set, context: ICreateOperationsContext ): Set { - const { - buildCacheConfiguration, - cobuildConfiguration, - isIncrementalBuildAllowed, - phaseSelection: selectedPhases, - projectChangeAnalyzer, - rushConfiguration - } = context; + const { phaseSelection: selectedPhases, projectChangeAnalyzer, rushConfiguration } = context; const customParametersByPhase: Map = new Map(); @@ -91,22 +80,6 @@ function createShellOperations( rushProject: project, selectedPhases }); - - if (buildCacheConfiguration) { - new CacheableOperationRunnerPlugin({ - buildCacheConfiguration, - cobuildConfiguration, - isIncrementalBuildAllowed - }).apply(shellOperationRunner.hooks); - CacheableOperationRunnerPlugin.setBuildCacheContextByRunner(shellOperationRunner, { - // This runner supports cache writes by default. - isCacheWriteAllowed: true, - isCacheReadAllowed: isIncrementalBuildAllowed, - isSkipAllowed: isIncrementalBuildAllowed, - projectBuildCache: undefined, - cobuildLock: undefined - }); - } operation.runner = shellOperationRunner; } else { // Empty build script indicates a no-op, so use a no-op runner diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index e0cd272a802..e245ced36b6 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -42,7 +42,7 @@ describe(AsyncOperationQueue.name, () => { consumer.dependencies.delete(operation); } operation.status = OperationStatus.Success; - queue.complete(); + queue.complete(operation); } expect(actualOrder).toEqual(expectedOrder); @@ -67,7 +67,7 @@ describe(AsyncOperationQueue.name, () => { consumer.dependencies.delete(operation); } operation.status = OperationStatus.Success; - queue.complete(); + queue.complete(operation); } expect(actualOrder).toEqual(expectedOrder); @@ -130,7 +130,7 @@ describe(AsyncOperationQueue.name, () => { --concurrency; operation.status = OperationStatus.Success; - queue.complete(); + queue.complete(operation); } }) ); @@ -177,7 +177,7 @@ describe(AsyncOperationQueue.name, () => { consumer.dependencies.delete(operation); } operation.status = OperationStatus.Success; - queue.complete(); + queue.complete(operation); } expect(actualOrder).toEqual(expectedOrder); diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 237fcfd7eff..e1914f4cc6b 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -12,6 +12,7 @@ import type { Operation } from '../logic/operations/Operation'; import type { ProjectChangeAnalyzer } from '../logic/ProjectChangeAnalyzer'; import type { IExecutionResult } from '../logic/operations/IOperationExecutionResult'; import type { CobuildConfiguration } from '../api/CobuildConfiguration'; +import type { OperationExecutionManager } from '../logic/operations/OperationExecutionManager'; /** * A plugin that interacts with a phased commands. @@ -99,6 +100,18 @@ export class PhasedCommandHooks { public readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, ICreateOperationsContext]> = new AsyncSeriesHook(['results', 'context']); + /** + * Hook invoked after the operationExecutionManager has been created. + * Maybe used to tap into the lifecycle of operation execution process. + * + * @internal + */ + public readonly operationExecutionManager: AsyncSeriesHook = + new AsyncSeriesHook( + ['operationExecutionManager'], + 'operationExecutionManager' + ); + /** * Hook invoked after a run has finished and the command is watching for changes. * May be used to display additional relevant data to the user. From 67b3c454e2b376270d00f912868c805c528e270c Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 20:37:45 +0800 Subject: [PATCH 25/55] chore: store executionQueue in OperationExecutionManager --- .../operations/OperationExecutionManager.ts | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 4cce3332824..f470ae6312d 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -26,6 +26,13 @@ export interface IOperationExecutionManagerOptions { */ const ASCII_HEADER_WIDTH: number = 79; +const prioritySort: IOperationSortFunction = ( + a: OperationExecutionRecord, + b: OperationExecutionRecord +): number => { + return a.criticalPathLength! - b.criticalPathLength!; +}; + /** * A class which manages the execution of a set of tasks with interdependencies. * Initially, and at the end of each task execution, all unblocked tasks @@ -49,6 +56,7 @@ export class OperationExecutionManager { private _hasAnyFailures: boolean; private _hasAnyNonAllowedWarnings: boolean; private _completedOperations: number; + private _executionQueue: AsyncOperationQueue; public readonly hooks: PhasedOperationHooks = new PhasedOperationHooks(); @@ -112,6 +120,12 @@ export class OperationExecutionManager { dependencyRecord.consumers.add(consumer); } } + + const executionQueue: AsyncOperationQueue = new AsyncOperationQueue( + this._executionRecords.values(), + prioritySort + ); + this._executionQueue = executionQueue; } private _streamCollator_onWriterActive = (writer: CollatedWriter | undefined): void => { @@ -175,28 +189,18 @@ export class OperationExecutionManager { this._terminal.writeStdoutLine(`Executing a maximum of ${this._parallelism} simultaneous processes...`); const maxParallelism: number = Math.min(totalOperations, this._parallelism); - const prioritySort: IOperationSortFunction = ( - a: OperationExecutionRecord, - b: OperationExecutionRecord - ): number => { - return a.criticalPathLength! - b.criticalPathLength!; - }; - const executionQueue: AsyncOperationQueue = new AsyncOperationQueue( - this._executionRecords.values(), - prioritySort - ); // This function is a callback because it may write to the collatedWriter before // operation.executeAsync returns (and cleans up the writer) const onOperationComplete: (record: OperationExecutionRecord) => Promise = async ( record: OperationExecutionRecord ) => { - this._onOperationComplete(record, executionQueue); + this._onOperationComplete(record); await this.hooks.afterExecuteOperation.promise(record); }; await Async.forEachAsync( - executionQueue, + this._executionQueue, async (operation: OperationExecutionRecord) => { await this.hooks.beforeExecuteOperation.promise(operation); await operation.executeAsync(onOperationComplete); @@ -221,7 +225,7 @@ export class OperationExecutionManager { /** * Handles the result of the operation and propagates any relevant effects. */ - private _onOperationComplete(record: OperationExecutionRecord, executionQueue: AsyncOperationQueue): void { + private _onOperationComplete(record: OperationExecutionRecord): void { const { runner, name, status } = record; const silent: boolean = runner.silent; @@ -243,7 +247,7 @@ export class OperationExecutionManager { const blockedQueue: Set = new Set(record.consumers); for (const blockedRecord of blockedQueue) { if (blockedRecord.status === OperationStatus.Ready) { - executionQueue.complete(blockedRecord); + this._executionQueue.complete(blockedRecord); this._completedOperations++; // Now that we have the concept of architectural no-ops, we could implement this by replacing @@ -325,7 +329,7 @@ export class OperationExecutionManager { if (record.status !== OperationStatus.RemoteExecuting) { // If the operation was not remote, then we can notify queue that it is complete - executionQueue.complete(record); + this._executionQueue.complete(record); } } } From 3649b5252d68eb6d9244ea2dd3f6e32e57d2128c Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 22:57:12 +0800 Subject: [PATCH 26/55] feat: add an unassigned operation in AsyncOperationQueue --- .../logic/operations/AsyncOperationQueue.ts | 60 +++++++++++-------- .../operations/OperationExecutionManager.ts | 27 +++++++-- .../test/AsyncOperationQueue.test.ts | 37 +++++++++--- 3 files changed, 86 insertions(+), 38 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index bf2e1a34442..5fd7d34984d 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -4,6 +4,16 @@ import { OperationExecutionRecord } from './OperationExecutionRecord'; import { OperationStatus } from './OperationStatus'; +/** + * When the queue returns an unassigned operation, it means there is no workable operation at the time, + * and the caller has a chance to make a decision synchronously or asynchronously: + * 1. Manually invoke `tryGetRemoteExecutingOperation()` to get a remote executing operation. + * 2. Or, return in callback or continue the for-loop, which internally invoke `assignOperations()` to assign new operations. + */ +export const UNASSIGNED_OPERATION: 'UNASSIGNED_OPERATION' = 'UNASSIGNED_OPERATION'; + +export type IOperationIteratorResult = OperationExecutionRecord | typeof UNASSIGNED_OPERATION; + /** * Implementation of the async iteration protocol for a collection of IOperation objects. * The async iterator will wait for an operation to be ready for execution, or terminate if there are no more operations. @@ -14,10 +24,10 @@ import { OperationStatus } from './OperationStatus'; * stall until another operations completes. */ export class AsyncOperationQueue - implements AsyncIterable, AsyncIterator + implements AsyncIterable, AsyncIterator { private readonly _queue: OperationExecutionRecord[]; - private readonly _pendingIterators: ((result: IteratorResult) => void)[]; + private readonly _pendingIterators: ((result: IteratorResult) => void)[]; private readonly _totalOperations: number; private readonly _completedOperations: Set; @@ -42,11 +52,11 @@ export class AsyncOperationQueue * For use with `for await (const operation of taskQueue)` * @see {AsyncIterator} */ - public next(): Promise> { + public next(): Promise> { const { _pendingIterators: waitingIterators } = this; - const promise: Promise> = new Promise( - (resolve: (result: IteratorResult) => void) => { + const promise: Promise> = new Promise( + (resolve: (result: IteratorResult) => void) => { waitingIterators.push(resolve); } ); @@ -126,34 +136,32 @@ export class AsyncOperationQueue } if (waitingIterators.length > 0) { - // Pause for a few time - setTimeout(() => { - // cycle through the queue again to find the next operation that is executed remotely - for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { - const operation: OperationExecutionRecord = queue[i]; - - if (operation.status === OperationStatus.RemoteExecuting) { - // try to attempt to get the lock again - waitingIterators.shift()!({ - value: operation, - done: false - }); - } - } - - if (waitingIterators.length > 0) { - // Queue is not empty, but no operations are ready to process, start over - this.assignOperations(); - } - }, 5000); + // Queue is not empty, but no operations are ready to process, returns a unassigned operation to let caller decide + waitingIterators.shift()!({ + value: UNASSIGNED_OPERATION, + done: false + }); + } + } + + public tryGetRemoteExecutingOperation(): OperationExecutionRecord | undefined { + const { _queue: queue } = this; + // cycle through the queue to find the next operation that is executed remotely + for (let i: number = queue.length - 1; i >= 0; i--) { + const operation: OperationExecutionRecord = queue[i]; + + if (operation.status === OperationStatus.RemoteExecuting) { + return operation; + } } + return undefined; } /** * Returns this queue as an async iterator, such that multiple functions iterating this object concurrently * receive distinct iteration results. */ - public [Symbol.asyncIterator](): AsyncIterator { + public [Symbol.asyncIterator](): AsyncIterator { return this; } } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index f470ae6312d..1f7ee8c323f 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -6,7 +6,12 @@ import { TerminalWritable, StdioWritable, TextRewriterTransform } from '@rushsta import { StreamCollator, CollatedTerminal, CollatedWriter } from '@rushstack/stream-collator'; import { NewlineKind, Async } from '@rushstack/node-core-library'; -import { AsyncOperationQueue, IOperationSortFunction } from './AsyncOperationQueue'; +import { + AsyncOperationQueue, + IOperationIteratorResult, + IOperationSortFunction, + UNASSIGNED_OPERATION +} from './AsyncOperationQueue'; import { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; @@ -201,9 +206,23 @@ export class OperationExecutionManager { await Async.forEachAsync( this._executionQueue, - async (operation: OperationExecutionRecord) => { - await this.hooks.beforeExecuteOperation.promise(operation); - await operation.executeAsync(onOperationComplete); + async (operation: IOperationIteratorResult) => { + let record: OperationExecutionRecord | undefined; + if (operation === UNASSIGNED_OPERATION) { + // Pause for a few time + await Async.sleep(5000); + record = this._executionQueue.tryGetRemoteExecutingOperation(); + } else { + record = operation; + } + + if (!record) { + // Fail to assign a operation, start over again + return; + } else { + await this.hooks.beforeExecuteOperation.promise(record); + await record.executeAsync(onOperationComplete); + } }, { concurrency: maxParallelism diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index e245ced36b6..9436b0c73f4 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -4,8 +4,9 @@ import { Operation } from '../Operation'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from '../OperationExecutionRecord'; import { MockOperationRunner } from './MockOperationRunner'; -import { AsyncOperationQueue, IOperationSortFunction } from '../AsyncOperationQueue'; +import { AsyncOperationQueue, IOperationSortFunction, UNASSIGNED_OPERATION } from '../AsyncOperationQueue'; import { OperationStatus } from '../OperationStatus'; +import { Async } from '@rushstack/node-core-library'; function addDependency(consumer: OperationExecutionRecord, dependency: OperationExecutionRecord): void { consumer.dependencies.add(dependency); @@ -38,6 +39,9 @@ describe(AsyncOperationQueue.name, () => { const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, nullSort); for await (const operation of queue) { actualOrder.push(operation); + if (operation === UNASSIGNED_OPERATION) { + continue; + } for (const consumer of operation.consumers) { consumer.dependencies.delete(operation); } @@ -63,6 +67,9 @@ describe(AsyncOperationQueue.name, () => { const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, customSort); for await (const operation of queue) { actualOrder.push(operation); + if (operation === UNASSIGNED_OPERATION) { + continue; + } for (const consumer of operation.consumers) { consumer.dependencies.delete(operation); } @@ -117,6 +124,9 @@ describe(AsyncOperationQueue.name, () => { await Promise.all( Array.from({ length: 3 }, async () => { for await (const operation of queue) { + if (operation === UNASSIGNED_OPERATION) { + continue; + } ++concurrency; await Promise.resolve(); @@ -163,9 +173,20 @@ describe(AsyncOperationQueue.name, () => { const actualOrder: string[] = []; let remoteExecuted: boolean = false; for await (const operation of queue) { - actualOrder.push(operation.name); + let record: OperationExecutionRecord | undefined; + if (operation === UNASSIGNED_OPERATION) { + await Async.sleep(100); + record = queue.tryGetRemoteExecutingOperation(); + } else { + record = operation; + } + if (!record) { + continue; + } + + actualOrder.push(record.name); - if (operation === operations[1]) { + if (record === operations[1]) { if (!remoteExecuted) { operations[1].status = OperationStatus.RemoteExecuting; // remote executed operation is finished later @@ -173,13 +194,13 @@ describe(AsyncOperationQueue.name, () => { continue; } } - for (const consumer of operation.consumers) { - consumer.dependencies.delete(operation); + for (const consumer of record.consumers) { + consumer.dependencies.delete(record); } - operation.status = OperationStatus.Success; - queue.complete(operation); + record.status = OperationStatus.Success; + queue.complete(record); } expect(actualOrder).toEqual(expectedOrder); - }, 6000); + }); }); From c62b646e1c4c61b7828c82fa72004883913eed58 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 23:39:12 +0800 Subject: [PATCH 27/55] feat: expand redis-cobuild-plugin configuration with env vars --- .../api/rush-redis-cobuild-plugin.api.md | 4 ++ .../src/RedisCobuildLockProvider.ts | 39 ++++++++++++++++++- .../src/test/RedisCobuildLockProvider.test.ts | 30 ++++++++++++++ .../RedisCobuildLockProvider.test.ts.snap | 5 +++ 4 files changed, 77 insertions(+), 1 deletion(-) diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index 93da87162d0..fa3242d63c9 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -4,6 +4,8 @@ ```ts +/// + import type { ICobuildCompletedState } from '@rushstack/rush-sdk'; import type { ICobuildContext } from '@rushstack/rush-sdk'; import type { ICobuildLockProvider } from '@rushstack/rush-sdk'; @@ -26,6 +28,8 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { // (undocumented) disconnectAsync(): Promise; // (undocumented) + static expandOptionsWithEnvironmentVariables(options: IRedisCobuildLockProviderOptions, environment?: NodeJS.ProcessEnv): IRedisCobuildLockProviderOptions; + // (undocumented) getCompletedStateAsync(context: ICobuildContext): Promise; getCompletedStateKey(context: ICobuildContext): string; getLockKey(context: ICobuildContext): string; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index c6ae989db55..ca39dc5bf7f 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -39,7 +39,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { private _completedKeyMap: WeakMap = new WeakMap(); public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { - this._options = options; + this._options = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables(options); this._terminal = rushSession.getLogger('RedisCobuildLockProvider').terminal; try { this._redisClient = createClient(this._options); @@ -48,6 +48,43 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } } + public static expandOptionsWithEnvironmentVariables( + options: IRedisCobuildLockProviderOptions, + environment: NodeJS.ProcessEnv = process.env + ): IRedisCobuildLockProviderOptions { + const finalOptions: IRedisCobuildLockProviderOptions = { ...options }; + const missingEnvironmentVariables: Set = new Set(); + for (const [key, value] of Object.entries(finalOptions)) { + if (typeof value === 'string') { + const expandedValue: string = value.replace( + /\$\{([^\}]+)\}/g, + (match: string, variableName: string): string => { + const variable: string | undefined = + variableName in environment ? environment[variableName] : undefined; + if (variable !== undefined) { + return variable; + } else { + missingEnvironmentVariables.add(variableName); + return match; + } + } + ); + (finalOptions as Record)[key] = expandedValue; + } + } + + if (missingEnvironmentVariables.size) { + throw new Error( + `The "RedisCobuildLockProvider" tries to access missing environment variable${ + missingEnvironmentVariables.size > 1 ? 's' : '' + }: ${Array.from(missingEnvironmentVariables).join( + ', ' + )}\nPlease check the configuration in rush-redis-cobuild-plugin.json file` + ); + } + return finalOptions; + } + public async connectAsync(): Promise { try { await this._redisClient.connect(); diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index 3425e83c31a..8aab4c72db2 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -49,6 +49,36 @@ describe(RedisCobuildLockProvider.name, () => { version: 1 }; + it('expands options with environment variables', () => { + const expectedOptions = { + username: 'redisuser', + password: 'redis123' + }; + const actualOptions = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables( + { + username: '${REDIS_USERNAME}', + password: '${REDIS_PASS}' + }, + { + REDIS_USERNAME: 'redisuser', + REDIS_PASS: 'redis123' + } + ); + expect(actualOptions).toEqual(expectedOptions); + }); + + it('throws error with missing environment variables', () => { + expect(() => { + RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables( + { + username: '${REDIS_USERNAME}', + password: '${REDIS_PASS}' + }, + {} + ); + }).toThrowErrorMatchingSnapshot(); + }); + it('getLockKey works', () => { const subject: RedisCobuildLockProvider = prepareSubject(); const lockKey: string = subject.getLockKey(context); diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap index 33aa6130bbf..be0ade872fd 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap @@ -3,3 +3,8 @@ exports[`RedisCobuildLockProvider getCompletedStateKey works 1`] = `"cobuild:v1:123:abc:completed"`; exports[`RedisCobuildLockProvider getLockKey works 1`] = `"cobuild:v1:123:abc:lock"`; + +exports[`RedisCobuildLockProvider throws error with missing environment variables 1`] = ` +"The \\"RedisCobuildLockProvider\\" tries to access missing environment variables: REDIS_USERNAME, REDIS_PASS +Please check the configuration in rush-redis-cobuild-plugin.json file" +`; From 6f2991078c9c0a1db50b4f244c00307a43913a90 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 23:57:54 +0800 Subject: [PATCH 28/55] feat: RUSH_COBUILD_CONTEXT_ID is required to opt into running with cobuilds --- .../.vscode/tasks.json | 10 +++- .../repo/common/config/rush/cobuild.json | 7 +-- common/reviews/api/rush-lib.api.md | 4 +- .../rush-init/common/config/rush/cobuild.json | 7 +-- .../rush-lib/src/api/CobuildConfiguration.ts | 42 ++++++------- .../src/api/EnvironmentConfiguration.ts | 3 +- .../src/logic/cobuild/CobuildContextId.ts | 59 ------------------- .../rush-lib/src/logic/cobuild/CobuildLock.ts | 5 ++ .../cobuild/test/CobuildContextId.test.ts | 37 ------------ .../CobuildContextId.test.ts.snap | 5 -- .../operations/CacheableOperationPlugin.ts | 41 ++++++------- 11 files changed, 56 insertions(+), 164 deletions(-) delete mode 100644 libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts delete mode 100644 libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts delete mode 100644 libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json index 8e1982bd3f9..2f17191e324 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -37,7 +37,10 @@ "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", "problemMatcher": [], "options": { - "cwd": "${workspaceFolder}/sandbox/repo" + "cwd": "${workspaceFolder}/sandbox/repo", + "env": { + "RUSH_COBUILD_CONTEXT_ID": "integration-test" + } }, "presentation": { "echo": true, @@ -55,7 +58,10 @@ "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", "problemMatcher": [], "options": { - "cwd": "${workspaceFolder}/sandbox/repo" + "cwd": "${workspaceFolder}/sandbox/repo", + "env": { + "RUSH_COBUILD_CONTEXT_ID": "integration-test" + } }, "presentation": { "echo": true, diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json index 29a49cbd86c..f16a120b1a6 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json @@ -7,6 +7,8 @@ /** * (Required) EXPERIMENTAL - Set this to true to enable the cobuild feature. + * RUSH_COBUILD_CONTEXT_ID should always be specified as an environment variable with an non-empty string, + * otherwise the cobuild feature will be disabled. */ "cobuildEnabled": true, @@ -17,9 +19,4 @@ * For example, @rushstack/rush-redis-cobuild-plugin registers the "redis" lock provider. */ "cobuildLockProvider": "redis" - - /** - * Setting this property overrides the cobuild context ID. - */ - // "cobuildContextIdPattern": "" } diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index fb32629c02f..f13e1c954e5 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -99,14 +99,14 @@ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) = // @beta export class CobuildConfiguration { - readonly cobuildContextId: string; + readonly cobuildContextId: string | undefined; readonly cobuildEnabled: boolean; // (undocumented) readonly cobuildLockProvider: ICobuildLockProvider; // (undocumented) connectLockProviderAsync(): Promise; // (undocumented) - get contextId(): string; + get contextId(): string | undefined; // (undocumented) disconnectLockProviderAsync(): Promise; // (undocumented) diff --git a/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json b/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json index 13cce367de6..15a874bebb4 100644 --- a/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json +++ b/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json @@ -7,6 +7,8 @@ /** * (Required) EXPERIMENTAL - Set this to true to enable the cobuild feature. + * RUSH_COBUILD_CONTEXT_ID should always be specified as an environment variable with an non-empty string, + * otherwise the cobuild feature will be disabled. */ "cobuildEnabled": false, @@ -17,9 +19,4 @@ * For example, @rushstack/rush-redis-cobuild-plugin registers the "redis" lock provider. */ "cobuildLockProvider": "redis" - - /** - * Setting this property overrides the cobuild context ID. - */ - // "cobuildContextIdPattern": "" } diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 214e61c0a03..d5814bf19c0 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -16,17 +16,14 @@ import { RushConstants } from '../logic/RushConstants'; import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; import type { RushConfiguration } from './RushConfiguration'; -import { CobuildContextId, GetCobuildContextIdFunction } from '../logic/cobuild/CobuildContextId'; export interface ICobuildJson { cobuildEnabled: boolean; cobuildLockProvider: string; - cobuildContextIdPattern?: string; } export interface ICobuildConfigurationOptions { cobuildJson: ICobuildJson; - getCobuildContextId: GetCobuildContextIdFunction; rushConfiguration: RushConfiguration; rushSession: RushSession; } @@ -42,24 +39,30 @@ export class CobuildConfiguration { /** * Indicates whether the cobuild feature is enabled. * Typically it is enabled in the cobuild.json config file. + * + * Note: The orchestrator (or local users) should always have to opt into running with cobuilds by + * providing a cobuild context id. Even if cobuilds are "enabled" as a feature, they don't + * actually turn on for that particular build unless the cobuild context id is provided as an + * non-empty string. */ public readonly cobuildEnabled: boolean; /** - * Method to calculate the cobuild context id + * Cobuild context id + * + * @remark + * The cobuild feature won't be enabled until the context id is provided as an non-empty string. */ - public readonly cobuildContextId: string; + public readonly cobuildContextId: string | undefined; public readonly cobuildLockProvider: ICobuildLockProvider; private constructor(options: ICobuildConfigurationOptions) { - this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? options.cobuildJson.cobuildEnabled; + const { cobuildJson } = options; - const { cobuildJson, getCobuildContextId } = options; - - this.cobuildContextId = - EnvironmentConfiguration.cobuildContextId ?? - getCobuildContextId({ - environment: process.env - }); + this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? cobuildJson.cobuildEnabled; + this.cobuildContextId = EnvironmentConfiguration.cobuildContextId; + if (!this.cobuildContextId) { + this.cobuildEnabled = false; + } const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = options.rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); @@ -100,25 +103,14 @@ export class CobuildConfiguration { CobuildConfiguration._jsonSchema ); - let getCobuildContextId: GetCobuildContextIdFunction; - try { - getCobuildContextId = CobuildContextId.parsePattern(cobuildJson.cobuildContextIdPattern); - } catch (e) { - terminal.writeErrorLine( - `Error parsing cobuild context id pattern "${cobuildJson.cobuildContextIdPattern}": ${e}` - ); - throw new AlreadyReportedError(); - } - return new CobuildConfiguration({ cobuildJson, - getCobuildContextId, rushConfiguration, rushSession }); } - public get contextId(): string { + public get contextId(): string | undefined { return this.cobuildContextId; } diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index e2224ce3d7f..e53dfa97927 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -155,8 +155,7 @@ export enum EnvironmentVariableNames { RUSH_COBUILD_ENABLED = 'RUSH_COBUILD_ENABLED', /** - * Setting this environment variable overrides the value of `cobuildContextId` calculated by - * `cobuildContextIdPattern` in the `cobuild.json` configuration file. + * Setting this environment variable opt into running with cobuilds. * * @remarks * If there is no cobuild configured, then this environment variable is ignored. diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts b/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts deleted file mode 100644 index 2c191eca242..00000000000 --- a/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { RushConstants } from '../RushConstants'; - -export interface IGenerateCobuildContextIdOptions { - environment: NodeJS.ProcessEnv; -} - -/** - * Calculates the cache entry id string for an operation. - * @beta - */ -export type GetCobuildContextIdFunction = (options: IGenerateCobuildContextIdOptions) => string; - -export class CobuildContextId { - private constructor() {} - - public static parsePattern(pattern?: string): GetCobuildContextIdFunction { - if (!pattern) { - return () => ''; - } else { - const resolvedPattern: string = pattern.trim(); - - return (options: IGenerateCobuildContextIdOptions) => { - const { environment } = options; - return this._expandWithEnvironmentVariables(resolvedPattern, environment); - }; - } - } - - private static _expandWithEnvironmentVariables(pattern: string, environment: NodeJS.ProcessEnv): string { - const missingEnvironmentVariables: Set = new Set(); - const expandedPattern: string = pattern.replace( - /\$\{([^\}]+)\}/g, - (match: string, variableName: string): string => { - const variable: string | undefined = - variableName in environment ? environment[variableName] : undefined; - if (variable !== undefined) { - return variable; - } else { - missingEnvironmentVariables.add(variableName); - return match; - } - } - ); - if (missingEnvironmentVariables.size) { - throw new Error( - `The "cobuildContextIdPattern" value in ${ - RushConstants.cobuildFilename - } contains missing environment variable${ - missingEnvironmentVariables.size > 1 ? 's' : '' - }: ${Array.from(missingEnvironmentVariables).join(', ')}` - ); - } - - return expandedPattern; - } -} diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index 9b656a9877d..2b36c66f777 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -38,6 +38,11 @@ export class CobuildLock { throw new InternalError(`Cache id is require for cobuild lock`); } + if (!contextId) { + // This should never happen + throw new InternalError(`Cobuild context id is require for cobuild lock`); + } + this._cobuildContext = { contextId, cacheId, diff --git a/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts b/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts deleted file mode 100644 index ea18a0e9242..00000000000 --- a/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { CobuildContextId } from '../CobuildContextId'; - -describe(CobuildContextId.name, () => { - describe('Valid pattern names', () => { - it('expands a environment variable', () => { - const contextId: string = CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ - environment: { - MR_ID: '123', - AUTHOR_NAME: 'Mr.example' - } - }); - expect(contextId).toEqual('context-123-Mr.example'); - }); - }); - - describe('Invalid pattern names', () => { - it('throws an error if a environment variable is missing', () => { - expect(() => - CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ - environment: { - MR_ID: '123' - } - }) - ).toThrowErrorMatchingSnapshot(); - }); - it('throws an error if multiple environment variables are missing', () => { - expect(() => - CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ - environment: {} - }) - ).toThrowErrorMatchingSnapshot(); - }); - }); -}); diff --git a/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap b/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap deleted file mode 100644 index c2495bc8537..00000000000 --- a/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap +++ /dev/null @@ -1,5 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`CobuildContextId Invalid pattern names throws an error if a environment variable is missing 1`] = `"The \\"cobuildContextIdPattern\\" value in cobuild.json contains missing environment variable: AUTHOR_NAME"`; - -exports[`CobuildContextId Invalid pattern names throws an error if multiple environment variables are missing 1`] = `"The \\"cobuildContextIdPattern\\" value in cobuild.json contains missing environment variables: MR_ID, AUTHOR_NAME"`; diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 3b9dde61605..cb35690d8e3 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -320,28 +320,25 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } context.error = undefined; } - const cacheId: string | undefined = cobuildLock.projectBuildCache.cacheId; - const contextId: string = cobuildLock.cobuildConfiguration.contextId; - - if (cacheId) { - const finalCacheId: string = - status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; - switch (status) { - case OperationStatus.SuccessWithWarning: - case OperationStatus.Success: - case OperationStatus.Failure: { - const currentStatus: ICobuildCompletedState['status'] = status; - setCompletedStatePromiseFunction = () => { - return cobuildLock?.setCompletedStateAsync({ - status: currentStatus, - cacheId: finalCacheId - }); - }; - setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( - terminal, - finalCacheId - ); - } + const { cacheId, contextId } = cobuildLock.cobuildContext; + + const finalCacheId: string = + status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; + switch (status) { + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: + case OperationStatus.Failure: { + const currentStatus: ICobuildCompletedState['status'] = status; + setCompletedStatePromiseFunction = () => { + return cobuildLock?.setCompletedStateAsync({ + status: currentStatus, + cacheId: finalCacheId + }); + }; + setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( + terminal, + finalCacheId + ); } } } From 8b2933a4cf59cffa68260d29237f2dbce33d4bbb Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Wed, 15 Mar 2023 00:04:14 +0800 Subject: [PATCH 29/55] :memo: --- .../README.md | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md index 5a72f9010b5..a8466abf387 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -52,7 +52,17 @@ RUSH_COBUILD_ENABLED=0 node ../../lib/runRush.js --debug cobuild Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. -## Case 2: Cobuild enabled, run one cobuild command only +## Case 2: Cobuild enabled without specifying RUSH_COBUILD_CONTEXT_ID + +Run `rush cobuild` command without specifying cobuild context id. + +```sh +rm -rf common/temp/build-cache && node ../../lib/runRush.js --debug cobuild +``` + +Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. + +## Case 3: Cobuild enabled, run one cobuild command only 1. Clear redis server @@ -63,19 +73,19 @@ Expected behavior: Cobuild feature is disabled. Build cache was restored success 2. Run `rush cobuild` command ```sh -rm -rf common/temp/build-cache && node ../../lib/runRush.js --debug cobuild +rm -rf common/temp/build-cache && RUSH_COBUILD_CONTEXT_ID=foo node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is enabled. Run command successfully. You can also see cobuild related logs in the terminal. ```sh -Get completed state for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: null -Acquired lock for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:lock: 1, 1 is success -Set completed state for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: SUCCESS;c2df36270ec5faa8ef6497fa7367a476de3e2861 +Get completed state for cobuild:v1:foo:c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: null +Acquired lock for cobuild:v1:foo:c2df36270ec5faa8ef6497fa7367a476de3e2861:lock: 1, 1 is success +Set completed state for cobuild:v1:foo:c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: SUCCESS;c2df36270ec5faa8ef6497fa7367a476de3e2861 ``` -## Case 3: Cobuild enabled, run two cobuild commands in parallel +## Case 4: Cobuild enabled, run two cobuild commands in parallel > Note: This test requires Visual Studio Code to be installed. @@ -99,7 +109,7 @@ rm -rf common/temp/build-cache Expected behavior: Cobuild feature is enabled, cobuild related logs out in both terminals. -## Case 4: Cobuild enabled, run two cobuild commands in parallel, one of them failed +## Case 5: Cobuild enabled, run two cobuild commands in parallel, one of them failed > Note: This test requires Visual Studio Code to be installed. From bb6c51af5f16d5ba2ce010dccfda07f81c131c9f Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Wed, 15 Mar 2023 18:02:36 +0800 Subject: [PATCH 30/55] chore --- libraries/rush-lib/src/api/CobuildConfiguration.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index d5814bf19c0..953b330e090 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -2,13 +2,7 @@ // See LICENSE in the project root for license information. import * as path from 'path'; -import { - AlreadyReportedError, - FileSystem, - ITerminal, - JsonFile, - JsonSchema -} from '@rushstack/node-core-library'; +import { FileSystem, ITerminal, JsonFile, JsonSchema } from '@rushstack/node-core-library'; import schemaJson from '../schemas/cobuild.schema.json'; import { EnvironmentConfiguration } from './EnvironmentConfiguration'; import { CobuildLockProviderFactory, RushSession } from '../pluginFramework/RushSession'; @@ -49,7 +43,7 @@ export class CobuildConfiguration { /** * Cobuild context id * - * @remark + * @remarks * The cobuild feature won't be enabled until the context id is provided as an non-empty string. */ public readonly cobuildContextId: string | undefined; From 6014f5db781baeac0bb83d2b5635df7ca3c6af6c Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Wed, 15 Mar 2023 20:21:53 +0800 Subject: [PATCH 31/55] chore: update snapshots --- libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap b/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap index 36bce37042e..4758750c3b3 100644 --- a/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap +++ b/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap @@ -13,6 +13,7 @@ Loaded @microsoft/rush-lib from process.env._RUSH_LIB_PATH 'BuildCacheConfiguration', 'BumpType', 'ChangeManager', + 'CobuildConfiguration', 'CommonVersionsConfiguration', 'CredentialCache', 'DependencyType', From e3ce661bc754774daf115b5b403d81d789e3fd64 Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 17 Mar 2023 14:59:28 +0800 Subject: [PATCH 32/55] Apply suggestions from code review Co-authored-by: David Michon --- .../src/logic/operations/OperationStatus.ts | 2 +- .../src/RedisCobuildLockProvider.ts | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/OperationStatus.ts b/libraries/rush-lib/src/logic/operations/OperationStatus.ts index 18e7204e78c..38a6955296c 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStatus.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStatus.ts @@ -13,7 +13,7 @@ export enum OperationStatus { /** * The Operation is Queued */ - Queued = 'Queued', + Queued = 'QUEUED', /** * The Operation is currently executing */ diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index ca39dc5bf7f..0ab229b7be0 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -24,19 +24,19 @@ import type { ITerminal } from '@rushstack/node-core-library'; */ export interface IRedisCobuildLockProviderOptions extends RedisClientOptions {} -const KEY_SEPARATOR: string = ':'; -const COMPLETED_STATE_SEPARATOR: string = ';'; +const KEY_SEPARATOR: ':' = ':'; +const COMPLETED_STATE_SEPARATOR: ';' = ';'; /** * @beta */ export class RedisCobuildLockProvider implements ICobuildLockProvider { private readonly _options: IRedisCobuildLockProviderOptions; - private _terminal: ITerminal; + private readonly _terminal: ITerminal; - private _redisClient: RedisClientType; - private _lockKeyMap: WeakMap = new WeakMap(); - private _completedKeyMap: WeakMap = new WeakMap(); + private readonly _redisClient: RedisClientType; + private readonly _lockKeyMap: WeakMap = new WeakMap(); + private readonly _completedKeyMap: WeakMap = new WeakMap(); public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { this._options = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables(options); @@ -59,8 +59,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { const expandedValue: string = value.replace( /\$\{([^\}]+)\}/g, (match: string, variableName: string): string => { - const variable: string | undefined = - variableName in environment ? environment[variableName] : undefined; + const variable: string | undefined = environment[variableName]; if (variable !== undefined) { return variable; } else { From 7e8d7936fad0fb05d793ee9ef501b92023f18c42 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Mar 2023 14:48:59 +0800 Subject: [PATCH 33/55] chore: add catch and reanble no-floating-promise rule --- .../rush-redis-cobuild-plugin-integration-test/src/runRush.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts index c758ec650a0..2547cf233fc 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts @@ -34,5 +34,4 @@ async function rushRush(args: string[]): Promise { await parser.execute(args).catch(console.error); // CommandLineParser.execute() should never reject the promise } -/* eslint-disable-next-line @typescript-eslint/no-floating-promises */ -rushRush(process.argv.slice(2)); +rushRush(process.argv.slice(2)).catch(console.error); From 3f40c36e43b908bdc4891d84e9725cd67cfe2c40 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Mar 2023 14:57:27 +0800 Subject: [PATCH 34/55] chore: remove DOM lib in tsconfig.json --- rush-plugins/rush-redis-cobuild-plugin/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json b/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json index b3d3ff2a64f..fbc2f5c0a6c 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json +++ b/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json @@ -2,7 +2,6 @@ "extends": "./node_modules/@rushstack/heft-node-rig/profiles/default/tsconfig-base.json", "compilerOptions": { - "lib": ["DOM"], "types": ["heft-jest", "node"] } } From 1cdf9a57b513cfe3c33917fff50b899ed6a73ef2 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Mar 2023 15:00:19 +0800 Subject: [PATCH 35/55] chore: remove context id pattern property in Schema --- libraries/rush-lib/src/schemas/cobuild.schema.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libraries/rush-lib/src/schemas/cobuild.schema.json b/libraries/rush-lib/src/schemas/cobuild.schema.json index 5b13a4ec631..1a3b4720d42 100644 --- a/libraries/rush-lib/src/schemas/cobuild.schema.json +++ b/libraries/rush-lib/src/schemas/cobuild.schema.json @@ -28,10 +28,6 @@ "cobuildLockProvider": { "description": "Specify the cobuild lock provider to use", "type": "string" - }, - "cobuildContextIdPattern": { - "type": "string", - "description": "Setting this property overrides the cobuild context ID." } } } From 130bb5572a8c4ed94d09e49074648022ee29cec4 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Mar 2023 16:58:56 +0800 Subject: [PATCH 36/55] chore: comments the usage of unassigned operation --- .../src/logic/operations/OperationExecutionManager.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 1f7ee8c323f..ae59877605f 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -208,6 +208,11 @@ export class OperationExecutionManager { this._executionQueue, async (operation: IOperationIteratorResult) => { let record: OperationExecutionRecord | undefined; + /** + * If the operation is UNASSIGNED_OPERATION, it means that the queue is not able to assign a operation. + * This happens when some operations run remotely. So, we should try to get a remote executing operation + * from the queue manually here. + */ if (operation === UNASSIGNED_OPERATION) { // Pause for a few time await Async.sleep(5000); From 796aa5d413c104912b0153f93649acfa682f573e Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 20 Mar 2023 16:14:25 +0800 Subject: [PATCH 37/55] chore: get projectChangeAnalyzer and selectedPhases from createContext --- common/reviews/api/rush-lib.api.md | 2 +- .../src/logic/operations/CacheableOperationPlugin.ts | 9 ++++++--- .../src/logic/operations/OperationRunnerHooks.ts | 3 --- .../src/logic/operations/ShellOperationRunner.ts | 2 -- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index fb6cd7788f3..1e890bc34fd 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -736,7 +736,7 @@ export enum OperationStatus { Failure = "FAILURE", FromCache = "FROM CACHE", NoOp = "NO OP", - Queued = "Queued", + Queued = "QUEUED", Ready = "READY", RemoteExecuting = "REMOTE EXECUTING", Skipped = "SKIPPED", diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index cb35690d8e3..32f3a4842e4 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -132,7 +132,12 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } private _applyOperationRunner(runner: ShellOperationRunner, context: ICreateOperationsContext): void { - const { buildCacheConfiguration, cobuildConfiguration } = context; + const { + buildCacheConfiguration, + cobuildConfiguration, + phaseSelection: selectedPhases, + projectChangeAnalyzer + } = context; const { hooks } = runner; const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); @@ -152,8 +157,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { errorLogPath, rushProject, phase, - selectedPhases, - projectChangeAnalyzer, commandName, commandToRun, earlyReturnStatus diff --git a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts index 813e310283f..a2b32af9759 100644 --- a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts +++ b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts @@ -9,7 +9,6 @@ import type { OperationStatus } from './OperationStatus'; import type { IProjectDeps, ShellOperationRunner } from './ShellOperationRunner'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { IPhase } from '../../api/CommandLineConfiguration'; -import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; /** * A plugin tht interacts with a operation runner @@ -33,8 +32,6 @@ export interface IOperationRunnerBeforeExecuteContext { errorLogPath: string; rushProject: RushConfigurationProject; phase: IPhase; - selectedPhases: Iterable; - projectChangeAnalyzer: ProjectChangeAnalyzer; commandName: string; commandToRun: string; } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index b29e68e4f2c..42b833a7480 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -223,8 +223,6 @@ export class ShellOperationRunner implements IOperationRunner { errorLogPath: projectLogWritable.errorLogPath, rushProject: this._rushProject, phase: this._phase, - selectedPhases: this._selectedPhases, - projectChangeAnalyzer: this._projectChangeAnalyzer, commandName: this._commandName, commandToRun: this._commandToRun, earlyReturnStatus: undefined From e0781cd27e92e0c1de15b848f31df1159ca5cd25 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 20 Mar 2023 16:36:45 +0800 Subject: [PATCH 38/55] feat(rush-redis-cobuild-plugin): passwordEnvrionmentVariable --- .../.vscode/tasks.json | 6 ++-- .../README.md | 8 ++--- .../rush-redis-cobuild-plugin.json | 2 +- .../api/rush-redis-cobuild-plugin.api.md | 1 + .../src/RedisCobuildLockProvider.ts | 35 ++++++++++--------- .../src/schemas/redis-config.schema.json | 4 +-- .../src/test/RedisCobuildLockProvider.test.ts | 8 ++--- .../RedisCobuildLockProvider.test.ts.snap | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json index 2f17191e324..e55c4f0e872 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -39,7 +39,8 @@ "options": { "cwd": "${workspaceFolder}/sandbox/repo", "env": { - "RUSH_COBUILD_CONTEXT_ID": "integration-test" + "RUSH_COBUILD_CONTEXT_ID": "integration-test", + "REDIS_PASS": "redis123" } }, "presentation": { @@ -60,7 +61,8 @@ "options": { "cwd": "${workspaceFolder}/sandbox/repo", "env": { - "RUSH_COBUILD_CONTEXT_ID": "integration-test" + "RUSH_COBUILD_CONTEXT_ID": "integration-test", + "REDIS_PASS": "redis123" } }, "presentation": { diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md index a8466abf387..2cb561c5168 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -41,13 +41,13 @@ rush update ## Case 1: Disable cobuild by setting `RUSH_COBUILD_ENABLED=0` ```sh -rm -rf common/temp/build-cache && RUSH_COBUILD_ENABLED=0 node ../../lib/runRush.js --debug cobuild +rm -rf common/temp/build-cache && RUSH_COBUILD_ENABLED=0 REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is disabled. Run command successfully. ```sh -RUSH_COBUILD_ENABLED=0 node ../../lib/runRush.js --debug cobuild +RUSH_COBUILD_ENABLED=0 REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. @@ -57,7 +57,7 @@ Expected behavior: Cobuild feature is disabled. Build cache was restored success Run `rush cobuild` command without specifying cobuild context id. ```sh -rm -rf common/temp/build-cache && node ../../lib/runRush.js --debug cobuild +rm -rf common/temp/build-cache && REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. @@ -73,7 +73,7 @@ Expected behavior: Cobuild feature is disabled. Build cache was restored success 2. Run `rush cobuild` command ```sh -rm -rf common/temp/build-cache && RUSH_COBUILD_CONTEXT_ID=foo node ../../lib/runRush.js --debug cobuild +rm -rf common/temp/build-cache && RUSH_COBUILD_CONTEXT_ID=foo REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is enabled. Run command successfully. diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json index 625dff477fc..c27270adc35 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json @@ -1,4 +1,4 @@ { "url": "redis://localhost:6379", - "password": "redis123" + "passwordEnvironmentVariable": "REDIS_PASS" } diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index fa3242d63c9..743f0b95b65 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -16,6 +16,7 @@ import type { RushSession } from '@rushstack/rush-sdk'; // @beta export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { + passwordEnvironmentVariable?: string; } // @beta (undocumented) diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index 0ab229b7be0..62810471855 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -22,7 +22,12 @@ import type { ITerminal } from '@rushstack/node-core-library'; * The redis client options * @beta */ -export interface IRedisCobuildLockProviderOptions extends RedisClientOptions {} +export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { + /** + * The environment variable name for the redis password + */ + passwordEnvironmentVariable?: string; +} const KEY_SEPARATOR: ':' = ':'; const COMPLETED_STATE_SEPARATOR: ';' = ';'; @@ -36,7 +41,10 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { private readonly _redisClient: RedisClientType; private readonly _lockKeyMap: WeakMap = new WeakMap(); - private readonly _completedKeyMap: WeakMap = new WeakMap(); + private readonly _completedKeyMap: WeakMap = new WeakMap< + ICobuildContext, + string + >(); public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { this._options = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables(options); @@ -54,22 +62,15 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { ): IRedisCobuildLockProviderOptions { const finalOptions: IRedisCobuildLockProviderOptions = { ...options }; const missingEnvironmentVariables: Set = new Set(); - for (const [key, value] of Object.entries(finalOptions)) { - if (typeof value === 'string') { - const expandedValue: string = value.replace( - /\$\{([^\}]+)\}/g, - (match: string, variableName: string): string => { - const variable: string | undefined = environment[variableName]; - if (variable !== undefined) { - return variable; - } else { - missingEnvironmentVariables.add(variableName); - return match; - } - } - ); - (finalOptions as Record)[key] = expandedValue; + + if (finalOptions.passwordEnvironmentVariable) { + const password: string | undefined = environment[finalOptions.passwordEnvironmentVariable]; + if (password !== undefined) { + finalOptions.password = password; + } else { + missingEnvironmentVariables.add(finalOptions.passwordEnvironmentVariable); } + delete finalOptions.passwordEnvironmentVariable; } if (missingEnvironmentVariables.size) { diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json b/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json index 4d984d81b3e..d4283ba7be2 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json +++ b/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json @@ -46,8 +46,8 @@ "description": "ACL username", "type": "string" }, - "password": { - "description": "ACL password", + "passwordEnvironmentVariable": { + "description": "The environment variable used to get the ACL password", "type": "string" }, "name": { diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index 8aab4c72db2..8b275b9875a 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -51,16 +51,13 @@ describe(RedisCobuildLockProvider.name, () => { it('expands options with environment variables', () => { const expectedOptions = { - username: 'redisuser', password: 'redis123' }; const actualOptions = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables( { - username: '${REDIS_USERNAME}', - password: '${REDIS_PASS}' + passwordEnvironmentVariable: 'REDIS_PASS' }, { - REDIS_USERNAME: 'redisuser', REDIS_PASS: 'redis123' } ); @@ -71,8 +68,7 @@ describe(RedisCobuildLockProvider.name, () => { expect(() => { RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables( { - username: '${REDIS_USERNAME}', - password: '${REDIS_PASS}' + passwordEnvironmentVariable: 'REDIS_PASS' }, {} ); diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap index be0ade872fd..f2e6e72eb4c 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap @@ -5,6 +5,6 @@ exports[`RedisCobuildLockProvider getCompletedStateKey works 1`] = `"cobuild:v1: exports[`RedisCobuildLockProvider getLockKey works 1`] = `"cobuild:v1:123:abc:lock"`; exports[`RedisCobuildLockProvider throws error with missing environment variables 1`] = ` -"The \\"RedisCobuildLockProvider\\" tries to access missing environment variables: REDIS_USERNAME, REDIS_PASS +"The \\"RedisCobuildLockProvider\\" tries to access missing environment variable: REDIS_PASS Please check the configuration in rush-redis-cobuild-plugin.json file" `; From 0903effd2dfbde9bb2d97290023f54b603977456 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 20 Mar 2023 16:42:59 +0800 Subject: [PATCH 39/55] feat: always define build cache context, whether shell operation or not --- .../operations/CacheableOperationPlugin.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 32f3a4842e4..66307bcf070 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -58,19 +58,19 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { for (const operation of operations) { if (operation.runner) { - if (operation.runner instanceof ShellOperationRunner) { - const buildCacheContext: IOperationBuildCacheContext = { - // ShellOperationRunner supports cache writes by default. - isCacheWriteAllowed: true, - isCacheReadAllowed: isIncrementalBuildAllowed, - isSkipAllowed: isIncrementalBuildAllowed, - projectBuildCache: undefined, - cobuildLock: undefined - }; - // Upstream runners may mutate the property of build cache context for downstream runners - this._buildCacheContextByOperationRunner.set(operation.runner, buildCacheContext); + const buildCacheContext: IOperationBuildCacheContext = { + // ShellOperationRunner supports cache writes by default. + isCacheWriteAllowed: true, + isCacheReadAllowed: isIncrementalBuildAllowed, + isSkipAllowed: isIncrementalBuildAllowed, + projectBuildCache: undefined, + cobuildLock: undefined + }; + // Upstream runners may mutate the property of build cache context for downstream runners + this._buildCacheContextByOperationRunner.set(operation.runner, buildCacheContext); - this._applyOperationRunner(operation.runner, context); + if (operation.runner instanceof ShellOperationRunner) { + this._applyShellOperationRunner(operation.runner, context); } } } @@ -131,7 +131,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); } - private _applyOperationRunner(runner: ShellOperationRunner, context: ICreateOperationsContext): void { + private _applyShellOperationRunner(runner: ShellOperationRunner, context: ICreateOperationsContext): void { const { buildCacheConfiguration, cobuildConfiguration, From cce3342eee7866567f71d8e8295ef5951cf87a96 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 20 Mar 2023 17:56:13 +0800 Subject: [PATCH 40/55] refact: before/afterExecuteOperaiton hook --- common/reviews/api/rush-lib.api.md | 15 ++-- .../cli/scriptActions/PhasedScriptAction.ts | 10 ++- .../operations/CacheableOperationPlugin.ts | 76 +++++++++---------- .../src/logic/operations/IOperationRunner.ts | 18 +++++ .../operations/OperationExecutionManager.ts | 29 +++++-- .../operations/OperationExecutionRecord.ts | 5 ++ .../logic/operations/PhasedOperationHooks.ts | 29 ------- .../src/pluginFramework/PhasedCommandHooks.ts | 22 +++--- 8 files changed, 106 insertions(+), 98 deletions(-) delete mode 100644 libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 1e890bc34fd..62631fd47ed 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -9,7 +9,7 @@ import { AsyncParallelHook } from 'tapable'; import { AsyncSeriesHook } from 'tapable'; import { AsyncSeriesWaterfallHook } from 'tapable'; -import { CollatedWriter } from '@rushstack/stream-collator'; +import type { CollatedWriter } from '@rushstack/stream-collator'; import type { CommandLineParameter } from '@rushstack/ts-command-line'; import { HookMap } from 'tapable'; import { IPackageJson } from '@rushstack/node-core-library'; @@ -17,11 +17,9 @@ import { ITerminal } from '@rushstack/node-core-library'; import { ITerminalProvider } from '@rushstack/node-core-library'; import { JsonObject } from '@rushstack/node-core-library'; import { PackageNameParser } from '@rushstack/node-core-library'; -import { StdioSummarizer } from '@rushstack/terminal'; -import { StreamCollator } from '@rushstack/stream-collator'; +import type { StdioSummarizer } from '@rushstack/terminal'; import { SyncHook } from 'tapable'; import { Terminal } from '@rushstack/node-core-library'; -import { TerminalWritable } from '@rushstack/terminal'; // @public export class ApprovedPackagesConfiguration { @@ -488,12 +486,15 @@ export interface IOperationRunner { // @beta export interface IOperationRunnerContext { + readonly changedProjectsOnly: boolean; collatedWriter: CollatedWriter; + readonly consumers: Set; debugMode: boolean; error?: Error; // @internal _operationMetadataManager?: _OperationMetadataManager; quietMode: boolean; + readonly runner: IOperationRunner; status: OperationStatus; stdioSummarizer: StdioSummarizer; // Warning: (ae-forgotten-export) The symbol "Stopwatch" needs to be exported by the entry point index.d.ts @@ -808,12 +809,10 @@ export abstract class PackageManagerOptionsConfigurationBase implements IPackage // @alpha export class PhasedCommandHooks { + readonly afterExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]>; readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, ICreateOperationsContext]>; + readonly beforeExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]>; readonly createOperations: AsyncSeriesWaterfallHook<[Set, ICreateOperationsContext]>; - // Warning: (ae-forgotten-export) The symbol "OperationExecutionManager" needs to be exported by the entry point index.d.ts - // - // @internal - readonly operationExecutionManager: AsyncSeriesHook; readonly waitingForChanges: SyncHook; } diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 883c3a5916c..fdda90d3238 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -39,6 +39,7 @@ import type { ITelemetryOperationResult } from '../../logic/Telemetry'; import { parseParallelism } from '../parsing/ParseParallelism'; import { CobuildConfiguration } from '../../api/CobuildConfiguration'; import { CacheableOperationPlugin } from '../../logic/operations/CacheableOperationPlugin'; +import type { IOperationRunnerContext } from '../../logic/operations/IOperationRunner'; /** * Constructor parameters for PhasedScriptAction. @@ -352,7 +353,13 @@ export class PhasedScriptAction extends BaseScriptAction { quietMode: isQuietMode, debugMode: this.parser.isDebug, parallelism, - changedProjectsOnly + changedProjectsOnly, + beforeExecuteOperation: async (record: IOperationRunnerContext) => { + await this.hooks.beforeExecuteOperation.promise(record); + }, + afterExecuteOperation: async (record: IOperationRunnerContext) => { + await this.hooks.afterExecuteOperation.promise(record); + } }; const internalOptions: IRunPhasesOptions = { @@ -511,7 +518,6 @@ export class PhasedScriptAction extends BaseScriptAction { operations, executionManagerOptions ); - await this.hooks.operationExecutionManager.promise(executionManager); const { isInitial, isWatch } = options.createOperationsContext; diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 66307bcf070..ea08dd56189 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -12,13 +12,11 @@ import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProj import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; import type { Operation } from './Operation'; -import type { OperationExecutionManager } from './OperationExecutionManager'; -import type { OperationExecutionRecord } from './OperationExecutionRecord'; import type { IOperationRunnerAfterExecuteContext, IOperationRunnerBeforeExecuteContext } from './OperationRunnerHooks'; -import type { IOperationRunner } from './IOperationRunner'; +import type { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { ICreateOperationsContext, @@ -79,50 +77,44 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } ); - hooks.operationExecutionManager.tap( + hooks.afterExecuteOperation.tapPromise( PLUGIN_NAME, - (operationExecutionManager: OperationExecutionManager) => { - operationExecutionManager.hooks.afterExecuteOperation.tapPromise( - PLUGIN_NAME, - async (operation: OperationExecutionRecord): Promise => { - const { runner, status, consumers } = operation; - const buildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByRunner(runner); - - let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; - let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; - - switch (status) { - case OperationStatus.Skipped: { - // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. - blockCacheWrite = true; - break; - } + async (runnerContext: IOperationRunnerContext): Promise => { + const { runner, status, consumers, changedProjectsOnly } = runnerContext; + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByRunner(runner); + + let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; + let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; + + switch (status) { + case OperationStatus.Skipped: { + // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. + blockCacheWrite = true; + break; + } - case OperationStatus.SuccessWithWarning: - case OperationStatus.Success: { - // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. - blockSkip ||= !operationExecutionManager.changedProjectsOnly; - break; - } - } + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: { + // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. + blockSkip ||= !changedProjectsOnly; + break; + } + } - // Apply status changes to direct dependents - for (const item of consumers) { - const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByRunner(item.runner); - if (itemRunnerBuildCacheContext) { - if (blockCacheWrite) { - itemRunnerBuildCacheContext.isCacheWriteAllowed = false; - } - if (blockSkip) { - itemRunnerBuildCacheContext.isSkipAllowed = false; - } - } + // Apply status changes to direct dependents + for (const item of consumers) { + const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByRunner(item.runner); + if (itemRunnerBuildCacheContext) { + if (blockCacheWrite) { + itemRunnerBuildCacheContext.isCacheWriteAllowed = false; + } + if (blockSkip) { + itemRunnerBuildCacheContext.isSkipAllowed = false; } - return operation; } - ); + } } ); diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index d9d6c02cf38..49ac8f109a8 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -53,6 +53,24 @@ export interface IOperationRunnerContext { * it later (for example to re-print errors at end of execution). */ error?: Error; + + /** + * The set of operations that depend on this operation. + */ + readonly consumers: Set; + + /** + * The operation runner that is executing this operation. + */ + readonly runner: IOperationRunner; + + /** + * Normally the incremental build logic will rebuild changed projects as well as + * any projects that directly or indirectly depend on a changed project. + * If true, then the incremental build logic will only rebuild changed projects and + * ignore dependent projects. + */ + readonly changedProjectsOnly: boolean; } /** diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index ae59877605f..e1ef4102a81 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -16,7 +16,6 @@ import { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; import { IExecutionResult } from './IOperationExecutionResult'; -import { PhasedOperationHooks } from './PhasedOperationHooks'; export interface IOperationExecutionManagerOptions { quietMode: boolean; @@ -24,6 +23,8 @@ export interface IOperationExecutionManagerOptions { parallelism: number; changedProjectsOnly: boolean; destination?: TerminalWritable; + beforeExecuteOperation?: (operation: OperationExecutionRecord) => Promise; + afterExecuteOperation?: (operation: OperationExecutionRecord) => Promise; } /** @@ -50,6 +51,12 @@ export class OperationExecutionManager { private readonly _quietMode: boolean; private readonly _parallelism: number; private readonly _totalOperations: number; + private readonly _beforeExecuteOperation: + | ((operation: OperationExecutionRecord) => Promise) + | undefined; + private readonly _afterExecuteOperation: + | ((operation: OperationExecutionRecord) => Promise) + | undefined; private readonly _outputWritable: TerminalWritable; private readonly _colorsNewlinesTransform: TextRewriterTransform; @@ -63,16 +70,23 @@ export class OperationExecutionManager { private _completedOperations: number; private _executionQueue: AsyncOperationQueue; - public readonly hooks: PhasedOperationHooks = new PhasedOperationHooks(); - public constructor(operations: Set, options: IOperationExecutionManagerOptions) { - const { quietMode, debugMode, parallelism, changedProjectsOnly } = options; + const { + quietMode, + debugMode, + parallelism, + changedProjectsOnly, + beforeExecuteOperation, + afterExecuteOperation + } = options; this._completedOperations = 0; this._quietMode = quietMode; this._hasAnyFailures = false; this._hasAnyNonAllowedWarnings = false; this.changedProjectsOnly = changedProjectsOnly; this._parallelism = parallelism; + this._beforeExecuteOperation = beforeExecuteOperation; + this._afterExecuteOperation = afterExecuteOperation; // TERMINAL PIPELINE: // @@ -94,7 +108,8 @@ export class OperationExecutionManager { const executionRecordContext: IOperationExecutionRecordContext = { streamCollator: this._streamCollator, debugMode, - quietMode + quietMode, + changedProjectsOnly }; let totalOperations: number = 0; @@ -201,7 +216,7 @@ export class OperationExecutionManager { record: OperationExecutionRecord ) => { this._onOperationComplete(record); - await this.hooks.afterExecuteOperation.promise(record); + await this._afterExecuteOperation?.(record); }; await Async.forEachAsync( @@ -225,7 +240,7 @@ export class OperationExecutionManager { // Fail to assign a operation, start over again return; } else { - await this.hooks.beforeExecuteOperation.promise(record); + await this._beforeExecuteOperation?.(record); await record.executeAsync(onOperationComplete); } }, diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 1fe67a79859..16590abeef8 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -16,6 +16,7 @@ export interface IOperationExecutionRecordContext { debugMode: boolean; quietMode: boolean; + changedProjectsOnly: boolean; } /** @@ -123,6 +124,10 @@ export class OperationExecutionRecord implements IOperationRunnerContext { return this._context.quietMode; } + public get changedProjectsOnly(): boolean { + return this._context.changedProjectsOnly; + } + public get collatedWriter(): CollatedWriter { // Lazy instantiate because the registerTask() call affects display ordering if (!this._collatedWriter) { diff --git a/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts b/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts deleted file mode 100644 index 05e67ac3c48..00000000000 --- a/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { AsyncSeriesWaterfallHook } from 'tapable'; - -import type { OperationExecutionRecord } from './OperationExecutionRecord'; - -/** - * A plugin that interacts with a phased commands. - * @alpha - */ -export interface IPhasedOperationPlugin { - /** - * Applies this plugin. - */ - apply(hooks: PhasedOperationHooks): void; -} - -/** - * Hooks into the execution process for phased operation - * @alpha - */ -export class PhasedOperationHooks { - public beforeExecuteOperation: AsyncSeriesWaterfallHook = - new AsyncSeriesWaterfallHook(['operation'], 'beforeExecuteOperation'); - - public afterExecuteOperation: AsyncSeriesWaterfallHook = - new AsyncSeriesWaterfallHook(['operation'], 'afterExecuteOperation'); -} diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index e1914f4cc6b..9bc403b4d74 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -12,7 +12,7 @@ import type { Operation } from '../logic/operations/Operation'; import type { ProjectChangeAnalyzer } from '../logic/ProjectChangeAnalyzer'; import type { IExecutionResult } from '../logic/operations/IOperationExecutionResult'; import type { CobuildConfiguration } from '../api/CobuildConfiguration'; -import type { OperationExecutionManager } from '../logic/operations/OperationExecutionManager'; +import type { IOperationRunnerContext } from '../logic/operations/IOperationRunner'; /** * A plugin that interacts with a phased commands. @@ -101,16 +101,18 @@ export class PhasedCommandHooks { new AsyncSeriesHook(['results', 'context']); /** - * Hook invoked after the operationExecutionManager has been created. - * Maybe used to tap into the lifecycle of operation execution process. - * - * @internal + * Hook invoked before executing a operation. */ - public readonly operationExecutionManager: AsyncSeriesHook = - new AsyncSeriesHook( - ['operationExecutionManager'], - 'operationExecutionManager' - ); + public readonly beforeExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]> = new AsyncSeriesHook< + [IOperationRunnerContext] + >(['runnerContext'], 'beforeExecuteOperation'); + + /** + * Hook invoked after executing a operation. + */ + public readonly afterExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]> = new AsyncSeriesHook< + [IOperationRunnerContext] + >(['runnerContext'], 'afterExecuteOperation'); /** * Hook invoked after a run has finished and the command is watching for changes. From 10456808992d795d9ec1a8dce83109dbcc261e97 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 23 Mar 2023 16:18:23 +0800 Subject: [PATCH 41/55] fix: async operation queue in non cobuild --- .../logic/operations/AsyncOperationQueue.ts | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 5fd7d34984d..423fbb4a8c7 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -5,10 +5,12 @@ import { OperationExecutionRecord } from './OperationExecutionRecord'; import { OperationStatus } from './OperationStatus'; /** - * When the queue returns an unassigned operation, it means there is no workable operation at the time, - * and the caller has a chance to make a decision synchronously or asynchronously: - * 1. Manually invoke `tryGetRemoteExecutingOperation()` to get a remote executing operation. - * 2. Or, return in callback or continue the for-loop, which internally invoke `assignOperations()` to assign new operations. + * When the queue returns an unassigned operation, it means there is at least one remote executing operation, + * at this time, the caller has a chance to make a decision: + * 1. Manually invoke `tryGetRemoteExecutingOperation()` to get the remote executing operation. + * 2. If there is no remote executing operation available, wait for some time and return in callback, which + * internally invoke `assignOperations()` to assign new operations. + * NOTE: the caller must wait for some time to avoid busy loop and burn CPU cycles. */ export const UNASSIGNED_OPERATION: 'UNASSIGNED_OPERATION' = 'UNASSIGNED_OPERATION'; @@ -136,11 +138,14 @@ export class AsyncOperationQueue } if (waitingIterators.length > 0) { - // Queue is not empty, but no operations are ready to process, returns a unassigned operation to let caller decide - waitingIterators.shift()!({ - value: UNASSIGNED_OPERATION, - done: false - }); + // returns an unassigned operation to let caller decide when there is at least one + // remote executing operation which is not ready to process. + if (queue.some((operation) => operation.status === OperationStatus.RemoteExecuting)) { + waitingIterators.shift()!({ + value: UNASSIGNED_OPERATION, + done: false + }); + } } } From b2c2e6ce290f5343399e1d4e7631068795c68d91 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 27 Mar 2023 11:43:15 +0800 Subject: [PATCH 42/55] :memo: --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5f5e0ec89f5..38fa3f5acbb 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ These GitHub repositories provide supplementary resources for Rush Stack: | [/rigs/heft-web-rig](./rigs/heft-web-rig/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig.svg)](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig) | [changelog](./rigs/heft-web-rig/CHANGELOG.md) | [@rushstack/heft-web-rig](https://www.npmjs.com/package/@rushstack/heft-web-rig) | | [/rush-plugins/rush-amazon-s3-build-cache-plugin](./rush-plugins/rush-amazon-s3-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin) | | [@rushstack/rush-amazon-s3-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-amazon-s3-build-cache-plugin) | | [/rush-plugins/rush-azure-storage-build-cache-plugin](./rush-plugins/rush-azure-storage-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin) | | [@rushstack/rush-azure-storage-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-azure-storage-build-cache-plugin) | +| [/rush-plugins/rush-redis-cobuild-plugin](./rush-plugins/rush-redis-cobuild-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin) | [changelog](./rush-plugins/rush-redis-cobuild-plugin/CHANGELOG.md) | [@rushstack/rush-redis-cobuild-plugin](https://www.npmjs.com/package/@rushstack/rush-redis-cobuild-plugin) | | [/rush-plugins/rush-serve-plugin](./rush-plugins/rush-serve-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-serve-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-serve-plugin) | [changelog](./rush-plugins/rush-serve-plugin/CHANGELOG.md) | [@rushstack/rush-serve-plugin](https://www.npmjs.com/package/@rushstack/rush-serve-plugin) | | [/webpack/hashed-folder-copy-plugin](./webpack/hashed-folder-copy-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fhashed-folder-copy-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Fhashed-folder-copy-plugin) | [changelog](./webpack/hashed-folder-copy-plugin/CHANGELOG.md) | [@rushstack/hashed-folder-copy-plugin](https://www.npmjs.com/package/@rushstack/hashed-folder-copy-plugin) | | [/webpack/loader-load-themed-styles](./webpack/loader-load-themed-styles/) | [![npm version](https://badge.fury.io/js/%40microsoft%2Floader-load-themed-styles.svg)](https://badge.fury.io/js/%40microsoft%2Floader-load-themed-styles) | [changelog](./webpack/loader-load-themed-styles/CHANGELOG.md) | [@microsoft/loader-load-themed-styles](https://www.npmjs.com/package/@microsoft/loader-load-themed-styles) | @@ -151,6 +152,7 @@ These GitHub repositories provide supplementary resources for Rush Stack: | [/build-tests/rush-amazon-s3-build-cache-plugin-integration-test](./build-tests/rush-amazon-s3-build-cache-plugin-integration-test/) | Tests connecting to an amazon S3 endpoint | | [/build-tests/rush-lib-declaration-paths-test](./build-tests/rush-lib-declaration-paths-test/) | This project ensures all of the paths in rush-lib/lib/... have imports that resolve correctly. If this project builds, all `lib/**/*.d.ts` files in the `@microsoft/rush-lib` package are valid. | | [/build-tests/rush-project-change-analyzer-test](./build-tests/rush-project-change-analyzer-test/) | This is an example project that uses rush-lib's ProjectChangeAnalyzer to | +| [/build-tests/rush-redis-cobuild-plugin-integration-test](./build-tests/rush-redis-cobuild-plugin-integration-test/) | Tests connecting to an redis server | | [/build-tests/set-webpack-public-path-plugin-webpack4-test](./build-tests/set-webpack-public-path-plugin-webpack4-test/) | Building this project tests the set-webpack-public-path-plugin using Webpack 4 | | [/build-tests/ts-command-line-test](./build-tests/ts-command-line-test/) | Building this project is a regression test for ts-command-line | | [/libraries/rush-themed-ui](./libraries/rush-themed-ui/) | Rush Component Library: a set of themed components for rush projects | From 51bef482504f09436f70e4674831ac6dec18a5ba Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 3 Apr 2023 16:43:04 +0800 Subject: [PATCH 43/55] chore: fix missing exports --- common/reviews/api/rush-lib.api.md | 10 ++++++++-- libraries/rush-lib/src/api/CobuildConfiguration.ts | 6 ++++++ libraries/rush-lib/src/index.ts | 2 +- rush-plugins/rush-redis-cobuild-plugin/src/index.ts | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 786d2eeed48..d37a0d4f881 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -112,8 +112,6 @@ export class CobuildConfiguration { static tryLoadAsync(terminal: ITerminal, rushConfiguration: RushConfiguration, rushSession: RushSession): Promise; } -// Warning: (ae-forgotten-export) The symbol "ICobuildJson" needs to be exported by the entry point index.d.ts -// // @beta (undocumented) export type CobuildLockProviderFactory = (cobuildJson: ICobuildJson) => ICobuildLockProvider; @@ -288,6 +286,14 @@ export interface ICobuildContext { version: number; } +// @beta (undocumented) +export interface ICobuildJson { + // (undocumented) + cobuildEnabled: boolean; + // (undocumented) + cobuildLockProvider: string; +} + // @beta (undocumented) export interface ICobuildLockProvider { // (undocumented) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 953b330e090..e9ae1101647 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -11,11 +11,17 @@ import { RushConstants } from '../logic/RushConstants'; import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; import type { RushConfiguration } from './RushConfiguration'; +/** + * @beta + */ export interface ICobuildJson { cobuildEnabled: boolean; cobuildLockProvider: string; } +/** + * @beta + */ export interface ICobuildConfigurationOptions { cobuildJson: ICobuildJson; rushConfiguration: RushConfiguration; diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index 8c1b574c850..226cbfe7a75 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -31,7 +31,7 @@ export { } from './logic/pnpm/PnpmOptionsConfiguration'; export { BuildCacheConfiguration } from './api/BuildCacheConfiguration'; -export { CobuildConfiguration } from './api/CobuildConfiguration'; +export { CobuildConfiguration, ICobuildJson } from './api/CobuildConfiguration'; export { GetCacheEntryIdFunction, IGenerateCacheEntryIdOptions } from './logic/buildCache/CacheEntryId'; export { FileSystemBuildCacheProvider, diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts index f627507d614..b07442ac86d 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts @@ -5,4 +5,4 @@ import { RushRedisCobuildPlugin } from './RushRedisCobuildPlugin'; export default RushRedisCobuildPlugin; export { RedisCobuildLockProvider } from './RedisCobuildLockProvider'; -export type { IRedisCobuildLockProviderOptions } from './RedisCobuildLockProvider'; +export { IRedisCobuildLockProviderOptions } from './RedisCobuildLockProvider'; From 4eadd4b6cd5555689412a5440fa9ace7749e8042 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 3 Apr 2023 17:27:51 +0800 Subject: [PATCH 44/55] chore: code changes according to code review --- common/reviews/api/rush-lib.api.md | 3 --- .../src/logic/cobuild/ICobuildLockProvider.ts | 14 ++++++++++++++ .../operations/CacheableOperationPlugin.ts | 6 ++---- .../operations/OperationExecutionManager.ts | 18 ++++++++---------- .../test/AsyncOperationQueue.test.ts | 14 ++++++++++++++ .../src/RedisCobuildLockProvider.ts | 2 +- 6 files changed, 39 insertions(+), 18 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index d37a0d4f881..edb4710ed7c 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -278,11 +278,8 @@ export interface ICobuildCompletedState { // @beta (undocumented) export interface ICobuildContext { - // (undocumented) cacheId: string; - // (undocumented) contextId: string; - // (undocumented) version: number; } diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index 73092467805..817dd719dcd 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -7,8 +7,22 @@ import type { OperationStatus } from '../operations/OperationStatus'; * @beta */ export interface ICobuildContext { + /** + * The contextId is provided by the monorepo maintainer, it reads from environment variable {@link EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID}. + * It ensure only the builds from the same given contextId cooperated. If user was more permissive, + * and wanted all PR and CI builds building anything with the same contextId to cooperate, then just + * set it to a static value. + */ contextId: string; + /** + * The id of cache. It should be keep same as the normal cacheId from ProjectBuildCache. + * Otherwise, there is a discrepancy in the success case then turning on cobuilds will + * fail to populate the normal build cache. + */ cacheId: string; + /** + * {@inheritdoc RushConstants.cobuildLockVersion} + */ version: number; } diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index ea08dd56189..4aa87c9bc8f 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -274,10 +274,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (buildCacheContext.isCacheWriteAllowed && cobuildLock) { const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); if (acquireSuccess) { - if (context.status === OperationStatus.RemoteExecuting) { - // This operation is used to marked remote executing, now change it to executing - context.status = OperationStatus.Executing; - } + // The operation may be used to marked remote executing, now change it to executing + context.status = OperationStatus.Executing; runner.periodicCallback.addCallback(async () => { await cobuildLock?.renewLockAsync(); }); diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 66c43c3b689..85450514ac2 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -49,7 +49,7 @@ const prioritySort: IOperationSortFunction = ( * tasks are complete, or prematurely fails if any of the tasks fail. */ export class OperationExecutionManager { - public readonly changedProjectsOnly: boolean; + private readonly _changedProjectsOnly: boolean; private readonly _executionRecords: Map; private readonly _quietMode: boolean; private readonly _parallelism: number; @@ -89,7 +89,7 @@ export class OperationExecutionManager { this._quietMode = quietMode; this._hasAnyFailures = false; this._hasAnyNonAllowedWarnings = false; - this.changedProjectsOnly = changedProjectsOnly; + this._changedProjectsOnly = changedProjectsOnly; this._parallelism = parallelism; this._beforeExecuteOperation = beforeExecuteOperation; @@ -371,17 +371,15 @@ export class OperationExecutionManager { } } - // Apply status changes to direct dependents - for (const item of record.consumers) { - if (status !== OperationStatus.RemoteExecuting) { - // Remove this operation from the dependencies, to unblock the scheduler - item.dependencies.delete(record); - } - } - if (record.status !== OperationStatus.RemoteExecuting) { // If the operation was not remote, then we can notify queue that it is complete this._executionQueue.complete(record); + + // Apply status changes to direct dependents + for (const item of record.consumers) { + // Remove this operation from the dependencies, to unblock the scheduler + item.dependencies.delete(record); + } } } } diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 9436b0c73f4..3ba19c3d7b7 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -36,10 +36,13 @@ describe(AsyncOperationQueue.name, () => { const expectedOrder = [operations[2], operations[0], operations[1], operations[3]]; const actualOrder = []; + // Nothing sets the RemoteExecuting status, this should be a error if it happens + let hasUnassignedOperation: boolean = false; const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, nullSort); for await (const operation of queue) { actualOrder.push(operation); if (operation === UNASSIGNED_OPERATION) { + hasUnassignedOperation = true; continue; } for (const consumer of operation.consumers) { @@ -50,6 +53,7 @@ describe(AsyncOperationQueue.name, () => { } expect(actualOrder).toEqual(expectedOrder); + expect(hasUnassignedOperation).toEqual(false); }); it('respects the sort predicate', async () => { @@ -63,11 +67,14 @@ describe(AsyncOperationQueue.name, () => { ): number => { return expectedOrder.indexOf(b) - expectedOrder.indexOf(a); }; + // Nothing sets the RemoteExecuting status, this should be a error if it happens + let hasUnassignedOperation: boolean = false; const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, customSort); for await (const operation of queue) { actualOrder.push(operation); if (operation === UNASSIGNED_OPERATION) { + hasUnassignedOperation = true; continue; } for (const consumer of operation.consumers) { @@ -78,6 +85,8 @@ describe(AsyncOperationQueue.name, () => { } expect(actualOrder).toEqual(expectedOrder); + + expect(hasUnassignedOperation).toEqual(false); }); it('detects cycles', async () => { @@ -119,12 +128,15 @@ describe(AsyncOperationQueue.name, () => { const actualConcurrency: Map = new Map(); const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, nullSort); let concurrency: number = 0; + // Nothing sets the RemoteExecuting status, this should be a error if it happens + let hasUnassignedOperation: boolean = false; // Use 3 concurrent iterators to verify that it handles having more than the operation concurrency await Promise.all( Array.from({ length: 3 }, async () => { for await (const operation of queue) { if (operation === UNASSIGNED_OPERATION) { + hasUnassignedOperation = true; continue; } ++concurrency; @@ -148,6 +160,8 @@ describe(AsyncOperationQueue.name, () => { for (const [operation, operationConcurrency] of expectedConcurrency) { expect(actualConcurrency.get(operation)).toEqual(operationConcurrency); } + + expect(hasUnassignedOperation).toEqual(false); }); it('handles remote executed operations', async () => { diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index 62810471855..f6d6cc297b4 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -70,7 +70,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } else { missingEnvironmentVariables.add(finalOptions.passwordEnvironmentVariable); } - delete finalOptions.passwordEnvironmentVariable; + finalOptions.passwordEnvironmentVariable = undefined; } if (missingEnvironmentVariables.size) { From a8950a0801065afe8f19945fe63bffbeabf53e6f Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Sat, 8 Apr 2023 22:48:46 +0800 Subject: [PATCH 45/55] feat: cobuild leaf project log only allowed --- .../.vscode/tasks.json | 2 + .../sandbox/repo/.gitignore | 5 +- .../repo/common/config/rush/pnpm-lock.yaml | 12 ++ .../repo/projects/f/config/rush-project.json | 13 ++ .../sandbox/repo/projects/f/package.json | 11 ++ .../sandbox/repo/projects/g/package.json | 11 ++ .../sandbox/repo/rush.json | 8 ++ common/reviews/api/rush-lib.api.md | 3 + .../rush-lib/src/api/CobuildConfiguration.ts | 9 ++ .../src/api/EnvironmentConfiguration.ts | 27 ++++ .../src/logic/buildCache/ProjectBuildCache.ts | 32 ++--- .../buildCache/test/ProjectBuildCache.test.ts | 14 +- .../operations/CacheableOperationPlugin.ts | 122 +++++++++++++++++- 13 files changed, 234 insertions(+), 35 deletions(-) create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json index e55c4f0e872..1029c807a58 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -40,6 +40,7 @@ "cwd": "${workspaceFolder}/sandbox/repo", "env": { "RUSH_COBUILD_CONTEXT_ID": "integration-test", + "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED": "1", "REDIS_PASS": "redis123" } }, @@ -62,6 +63,7 @@ "cwd": "${workspaceFolder}/sandbox/repo", "env": { "RUSH_COBUILD_CONTEXT_ID": "integration-test", + "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED": "1", "REDIS_PASS": "redis123" } }, diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore index 484950696c9..6200fef459b 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore @@ -1,4 +1,7 @@ # Rush temporary files common/deploy/ common/temp/ -common/autoinstallers/*/.npmrc \ No newline at end of file +common/autoinstallers/*/.npmrc +projects/*/dist/ +*.log +node_modules/ \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml index 5918b7e8af7..36f86a80a56 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml @@ -32,3 +32,15 @@ importers: dependencies: b: link:../b d: link:../d + + ../../projects/f: + specifiers: + b: workspace:* + dependencies: + b: link:../b + + ../../projects/g: + specifiers: + b: workspace:* + dependencies: + b: link:../b diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json new file mode 100644 index 00000000000..23e6a93085e --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json @@ -0,0 +1,13 @@ +{ + "disableBuildCacheForProject": true, + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json new file mode 100644 index 00000000000..f703b70f3b2 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json @@ -0,0 +1,11 @@ +{ + "name": "f", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + }, + "dependencies": { + "b": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json new file mode 100644 index 00000000000..14c42344694 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json @@ -0,0 +1,11 @@ +{ + "name": "g", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + }, + "dependencies": { + "b": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json index 9f839d273ed..70c6c4890de 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json @@ -24,6 +24,14 @@ { "packageName": "e", "projectFolder": "projects/e" + }, + { + "packageName": "f", + "projectFolder": "projects/f" + }, + { + "packageName": "g", + "projectFolder": "projects/g" } ] } diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 7d1584e2286..1a2397f29c1 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -99,6 +99,7 @@ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) = export class CobuildConfiguration { readonly cobuildContextId: string | undefined; readonly cobuildEnabled: boolean; + readonly cobuildLeafProjectLogOnlyAllowed: boolean; // (undocumented) readonly cobuildLockProvider: ICobuildLockProvider; // (undocumented) @@ -171,6 +172,7 @@ export class EnvironmentConfiguration { static get buildCacheWriteAllowed(): boolean | undefined; static get cobuildContextId(): string | undefined; static get cobuildEnabled(): boolean | undefined; + static get cobuildLeafProjectLogOnlyAllowed(): boolean | undefined; // Warning: (ae-forgotten-export) The symbol "IEnvironment" needs to be exported by the entry point index.d.ts // // @internal @@ -197,6 +199,7 @@ export enum EnvironmentVariableNames { RUSH_BUILD_CACHE_WRITE_ALLOWED = "RUSH_BUILD_CACHE_WRITE_ALLOWED", RUSH_COBUILD_CONTEXT_ID = "RUSH_COBUILD_CONTEXT_ID", RUSH_COBUILD_ENABLED = "RUSH_COBUILD_ENABLED", + RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED = "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED", RUSH_DEPLOY_TARGET_FOLDER = "RUSH_DEPLOY_TARGET_FOLDER", RUSH_GIT_BINARY_PATH = "RUSH_GIT_BINARY_PATH", RUSH_GLOBAL_FOLDER = "RUSH_GLOBAL_FOLDER", diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index e9ae1101647..7c50e649299 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -53,6 +53,13 @@ export class CobuildConfiguration { * The cobuild feature won't be enabled until the context id is provided as an non-empty string. */ public readonly cobuildContextId: string | undefined; + /** + * If true, Rush will automatically handle the leaf project with build cache "disabled" by writing + * to the cache in a special "log files only mode". This is useful when you want to use Cobuilds + * to improve the performance in CI validations and the leaf projects have not enabled cache. + */ + public readonly cobuildLeafProjectLogOnlyAllowed: boolean; + public readonly cobuildLockProvider: ICobuildLockProvider; private constructor(options: ICobuildConfigurationOptions) { @@ -60,6 +67,8 @@ export class CobuildConfiguration { this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? cobuildJson.cobuildEnabled; this.cobuildContextId = EnvironmentConfiguration.cobuildContextId; + this.cobuildLeafProjectLogOnlyAllowed = + EnvironmentConfiguration.cobuildLeafProjectLogOnlyAllowed ?? false; if (!this.cobuildContextId) { this.cobuildEnabled = false; } diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index f7a6167c8e8..b34d7649d4a 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -162,6 +162,13 @@ export enum EnvironmentVariableNames { */ RUSH_COBUILD_CONTEXT_ID = 'RUSH_COBUILD_CONTEXT_ID', + /** + * If this variable is set to "1", When getting distributed builds, Rush will automatically handle the leaf project + * with build cache "disabled" by writing to the cache in a special "log files only mode". This is useful when you + * want to use Cobuilds to improve the performance in CI validations and the leaf projects have not enabled cache. + */ + RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED = 'RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED', + /** * Explicitly specifies the path for the Git binary that is invoked by certain Rush operations. */ @@ -225,6 +232,8 @@ export class EnvironmentConfiguration { private static _cobuildContextId: string | undefined; + private static _cobuildLeafProjectLogOnlyAllowed: boolean | undefined; + private static _gitBinaryPath: string | undefined; private static _tarBinaryPath: string | undefined; @@ -340,6 +349,15 @@ export class EnvironmentConfiguration { return EnvironmentConfiguration._cobuildContextId; } + /** + * If set, enables or disables the cobuild leaf project log only feature. + * See {@link EnvironmentVariableNames.RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED} + */ + public static get cobuildLeafProjectLogOnlyAllowed(): boolean | undefined { + EnvironmentConfiguration._ensureValidated(); + return EnvironmentConfiguration._cobuildLeafProjectLogOnlyAllowed; + } + /** * Allows the git binary path to be explicitly provided. * See {@link EnvironmentVariableNames.RUSH_GIT_BINARY_PATH} @@ -484,6 +502,15 @@ export class EnvironmentConfiguration { break; } + case EnvironmentVariableNames.RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED: { + EnvironmentConfiguration._cobuildLeafProjectLogOnlyAllowed = + EnvironmentConfiguration.parseBooleanEnvironmentVariable( + EnvironmentVariableNames.RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED, + value + ); + break; + } + case EnvironmentVariableNames.RUSH_GIT_BINARY_PATH: { EnvironmentConfiguration._gitBinaryPath = value; break; diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index f8a7ea21a07..b8c224e53df 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -7,7 +7,6 @@ import { FileSystem, Path, ITerminal, FolderItem, InternalError, Async } from '@ import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; -import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { RushConstants } from '../RushConstants'; import { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider'; @@ -17,7 +16,7 @@ import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration'; export interface IProjectBuildCacheOptions { buildCacheConfiguration: BuildCacheConfiguration; - projectConfiguration: RushProjectConfiguration; + project: RushConfigurationProject; projectOutputFolderNames: ReadonlyArray; additionalProjectOutputFilePaths?: ReadonlyArray; additionalContext?: Record; @@ -50,13 +49,9 @@ export class ProjectBuildCache { private _cacheId: string | undefined; private constructor(cacheId: string | undefined, options: IProjectBuildCacheOptions) { - const { - buildCacheConfiguration, - projectConfiguration, - projectOutputFolderNames, - additionalProjectOutputFilePaths - } = options; - this._project = projectConfiguration.project; + const { buildCacheConfiguration, project, projectOutputFolderNames, additionalProjectOutputFilePaths } = + options; + this._project = project; this._localBuildCacheProvider = buildCacheConfiguration.localCacheProvider; this._cloudBuildCacheProvider = buildCacheConfiguration.cloudCacheProvider; this._buildCacheEnabled = buildCacheConfiguration.buildCacheEnabled; @@ -81,18 +76,13 @@ export class ProjectBuildCache { public static async tryGetProjectBuildCache( options: IProjectBuildCacheOptions ): Promise { - const { terminal, projectConfiguration, projectOutputFolderNames, trackedProjectFiles } = options; + const { terminal, project, projectOutputFolderNames, trackedProjectFiles } = options; if (!trackedProjectFiles) { return undefined; } if ( - !ProjectBuildCache._validateProject( - terminal, - projectConfiguration, - projectOutputFolderNames, - trackedProjectFiles - ) + !ProjectBuildCache._validateProject(terminal, project, projectOutputFolderNames, trackedProjectFiles) ) { return undefined; } @@ -103,13 +93,11 @@ export class ProjectBuildCache { private static _validateProject( terminal: ITerminal, - projectConfiguration: RushProjectConfiguration, + project: RushConfigurationProject, projectOutputFolderNames: ReadonlyArray, trackedProjectFiles: string[] ): boolean { - const normalizedProjectRelativeFolder: string = Path.convertToSlashes( - projectConfiguration.project.projectRelativeFolder - ); + const normalizedProjectRelativeFolder: string = Path.convertToSlashes(project.projectRelativeFolder); const outputFolders: string[] = []; if (projectOutputFolderNames) { for (const outputFolderName of projectOutputFolderNames) { @@ -434,7 +422,7 @@ export class ProjectBuildCache { const projectStates: string[] = []; const projectsThatHaveBeenProcessed: Set = new Set(); let projectsToProcess: Set = new Set(); - projectsToProcess.add(options.projectConfiguration.project); + projectsToProcess.add(options.project); while (projectsToProcess.size > 0) { const newProjectsToProcess: Set = new Set(); @@ -491,7 +479,7 @@ export class ProjectBuildCache { const projectStateHash: string = hash.digest('hex'); return options.buildCacheConfiguration.getCacheEntryId({ - projectName: options.projectConfiguration.project.packageName, + projectName: options.project.packageName, projectStateHash, phaseName: options.phaseName }); diff --git a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts index c690607a45a..afbae532838 100644 --- a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts +++ b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts @@ -3,7 +3,7 @@ import { StringBufferTerminalProvider, Terminal } from '@rushstack/node-core-library'; import { BuildCacheConfiguration } from '../../../api/BuildCacheConfiguration'; -import { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; +import { RushConfigurationProject } from '../../../api/RushConfigurationProject'; import { ProjectChangeAnalyzer } from '../../ProjectChangeAnalyzer'; import { IGenerateCacheEntryIdOptions } from '../CacheEntryId'; import { FileSystemBuildCacheProvider } from '../FileSystemBuildCacheProvider'; @@ -36,13 +36,11 @@ describe(ProjectBuildCache.name, () => { } } as unknown as BuildCacheConfiguration, projectOutputFolderNames: ['dist'], - projectConfiguration: { - project: { - packageName: 'acme-wizard', - projectRelativeFolder: 'apps/acme-wizard', - dependencyProjects: [] - } - } as unknown as RushProjectConfiguration, + project: { + packageName: 'acme-wizard', + projectRelativeFolder: 'apps/acme-wizard', + dependencyProjects: [] + } as unknown as RushConfigurationProject, command: 'build', trackedProjectFiles: options.hasOwnProperty('trackedProjectFiles') ? options.trackedProjectFiles : [], projectChangeAnalyzer, diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 4aa87c9bc8f..eddcdd9ad0f 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -170,7 +170,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); } - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ + let projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ buildCacheConfiguration, runner, rushProject, @@ -183,12 +183,40 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { trackedProjectFiles, operationMetadataManager: context._operationMetadataManager }); - // eslint-disable-next-line require-atomic-updates -- we are mutating the build cache context intentionally - buildCacheContext.projectBuildCache = projectBuildCache; // Try to acquire the cobuild lock let cobuildLock: CobuildLock | undefined; if (cobuildConfiguration?.cobuildEnabled) { + if ( + cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && + rushProject.consumingProjects.size === 0 && + !projectBuildCache + ) { + // When the leaf project log only is allowed and the leaf project is build cache "disabled", try to get + // a log files only project build cache + projectBuildCache = await this._tryGetLogOnlyProjectBuildCacheAsync({ + buildCacheConfiguration, + runner, + rushProject, + phase, + projectChangeAnalyzer, + commandName, + commandToRun, + terminal, + trackedProjectFiles, + operationMetadataManager: context._operationMetadataManager + }); + if (projectBuildCache) { + terminal.writeVerboseLine( + `Log files only build cache is enabled for the project "${rushProject.packageName}" because the cobuild leaf project log only is allowed` + ); + } else { + terminal.writeWarningLine( + `Failed to get log files only build cache for the project "${rushProject.packageName}"` + ); + } + } + cobuildLock = await this._tryGetCobuildLockAsync({ runner, projectBuildCache, @@ -451,7 +479,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } buildCacheContext.projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCache({ - projectConfiguration, + project: rushProject, projectOutputFolderNames, additionalProjectOutputFilePaths, additionalContext, @@ -476,6 +504,92 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext.projectBuildCache; } + private async _tryGetLogOnlyProjectBuildCacheAsync({ + runner, + rushProject, + terminal, + commandName, + commandToRun, + buildCacheConfiguration, + phase, + trackedProjectFiles, + projectChangeAnalyzer, + operationMetadataManager + }: { + buildCacheConfiguration: BuildCacheConfiguration | undefined; + runner: IOperationRunner; + rushProject: RushConfigurationProject; + phase: IPhase; + commandToRun: string; + commandName: string; + terminal: ITerminal; + trackedProjectFiles: string[] | undefined; + projectChangeAnalyzer: ProjectChangeAnalyzer; + operationMetadataManager: OperationMetadataManager | undefined; + }): Promise { + const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { + // Disable legacy skip logic if the build cache is in play + buildCacheContext.isSkipAllowed = false; + const projectConfiguration: RushProjectConfiguration | undefined = + await RushProjectConfiguration.tryLoadForProjectAsync(rushProject, terminal); + + let projectOutputFolderNames: ReadonlyArray = []; + const additionalProjectOutputFilePaths: ReadonlyArray = [ + ...(operationMetadataManager?.relativeFilepaths || []) + ]; + const additionalContext: Record = { + // Force the cache to be a log files only cache + logFilesOnly: '1' + }; + if (projectConfiguration) { + const operationSettings: IOperationSettings | undefined = + projectConfiguration.operationSettingsByOperationName.get(commandName); + if (operationSettings) { + if (operationSettings.outputFolderNames) { + projectOutputFolderNames = operationSettings.outputFolderNames; + } + if (operationSettings.dependsOnEnvVars) { + for (const varName of operationSettings.dependsOnEnvVars) { + additionalContext['$' + varName] = process.env[varName] || ''; + } + } + + if (operationSettings.dependsOnAdditionalFiles) { + const repoState: IRawRepoState | undefined = await projectChangeAnalyzer._ensureInitializedAsync( + terminal + ); + + const additionalFiles: Map = await getHashesForGlobsAsync( + operationSettings.dependsOnAdditionalFiles, + rushProject.projectFolder, + repoState + ); + + for (const [filePath, fileHash] of additionalFiles) { + additionalContext['file://' + filePath] = fileHash; + } + } + } + } + const projectBuildCache: ProjectBuildCache | undefined = + await ProjectBuildCache.tryGetProjectBuildCache({ + project: rushProject, + projectOutputFolderNames, + additionalProjectOutputFilePaths, + additionalContext, + buildCacheConfiguration, + terminal, + command: commandToRun, + trackedProjectFiles, + projectChangeAnalyzer: projectChangeAnalyzer, + phaseName: phase.name + }); + buildCacheContext.projectBuildCache = projectBuildCache; + return projectBuildCache; + } + } + private async _tryGetCobuildLockAsync({ cobuildConfiguration, runner, From 082f3503af8883ac4421a4d9eb02c5ae8bfc65a6 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 11 Apr 2023 11:39:22 +0800 Subject: [PATCH 46/55] chore --- libraries/rush-lib/src/api/EnvironmentConfiguration.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index f802d05cfa2..327cf1aae74 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -153,7 +153,7 @@ export const EnvironmentVariableNames = { * * If there is no cobuild configured, then this environment variable is ignored. */ - RUSH_COBUILD_ENABLED = 'RUSH_COBUILD_ENABLED', + RUSH_COBUILD_ENABLED: 'RUSH_COBUILD_ENABLED', /** * Setting this environment variable opt into running with cobuilds. @@ -161,14 +161,14 @@ export const EnvironmentVariableNames = { * @remarks * If there is no cobuild configured, then this environment variable is ignored. */ - RUSH_COBUILD_CONTEXT_ID = 'RUSH_COBUILD_CONTEXT_ID', + RUSH_COBUILD_CONTEXT_ID: 'RUSH_COBUILD_CONTEXT_ID', /** * If this variable is set to "1", When getting distributed builds, Rush will automatically handle the leaf project * with build cache "disabled" by writing to the cache in a special "log files only mode". This is useful when you * want to use Cobuilds to improve the performance in CI validations and the leaf projects have not enabled cache. */ - RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED = 'RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED', + RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED: 'RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED', /** * Explicitly specifies the path for the Git binary that is invoked by certain Rush operations. From 820aa3c33d9318ed7b71194a8e98c04136b2503d Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Fri, 21 Apr 2023 19:50:34 -0700 Subject: [PATCH 47/55] Fix merge conflict --- README.md | 2 +- common/config/rush/repo-state.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 164584ff6a2..d9a5c9b3f8b 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,8 @@ These GitHub repositories provide supplementary resources for Rush Stack: | [/rigs/heft-web-rig](./rigs/heft-web-rig/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig.svg)](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig) | [changelog](./rigs/heft-web-rig/CHANGELOG.md) | [@rushstack/heft-web-rig](https://www.npmjs.com/package/@rushstack/heft-web-rig) | | [/rush-plugins/rush-amazon-s3-build-cache-plugin](./rush-plugins/rush-amazon-s3-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin) | | [@rushstack/rush-amazon-s3-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-amazon-s3-build-cache-plugin) | | [/rush-plugins/rush-azure-storage-build-cache-plugin](./rush-plugins/rush-azure-storage-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin) | | [@rushstack/rush-azure-storage-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-azure-storage-build-cache-plugin) | -| [/rush-plugins/rush-redis-cobuild-plugin](./rush-plugins/rush-redis-cobuild-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin) | [changelog](./rush-plugins/rush-redis-cobuild-plugin/CHANGELOG.md) | [@rushstack/rush-redis-cobuild-plugin](https://www.npmjs.com/package/@rushstack/rush-redis-cobuild-plugin) | | [/rush-plugins/rush-http-build-cache-plugin](./rush-plugins/rush-http-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-http-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-http-build-cache-plugin) | | [@rushstack/rush-http-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-http-build-cache-plugin) | +| [/rush-plugins/rush-redis-cobuild-plugin](./rush-plugins/rush-redis-cobuild-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin) | [changelog](./rush-plugins/rush-redis-cobuild-plugin/CHANGELOG.md) | [@rushstack/rush-redis-cobuild-plugin](https://www.npmjs.com/package/@rushstack/rush-redis-cobuild-plugin) | | [/rush-plugins/rush-serve-plugin](./rush-plugins/rush-serve-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-serve-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-serve-plugin) | [changelog](./rush-plugins/rush-serve-plugin/CHANGELOG.md) | [@rushstack/rush-serve-plugin](https://www.npmjs.com/package/@rushstack/rush-serve-plugin) | | [/webpack/hashed-folder-copy-plugin](./webpack/hashed-folder-copy-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fhashed-folder-copy-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Fhashed-folder-copy-plugin) | [changelog](./webpack/hashed-folder-copy-plugin/CHANGELOG.md) | [@rushstack/hashed-folder-copy-plugin](https://www.npmjs.com/package/@rushstack/hashed-folder-copy-plugin) | | [/webpack/loader-load-themed-styles](./webpack/loader-load-themed-styles/) | [![npm version](https://badge.fury.io/js/%40microsoft%2Floader-load-themed-styles.svg)](https://badge.fury.io/js/%40microsoft%2Floader-load-themed-styles) | [changelog](./webpack/loader-load-themed-styles/CHANGELOG.md) | [@microsoft/loader-load-themed-styles](https://www.npmjs.com/package/@microsoft/loader-load-themed-styles) | diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index 8bf829eaecf..3792953e049 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "a1681a4c0e1d3c156fa5e501a94688a520955940", + "pnpmShrinkwrapHash": "ec82d5af16b25e9021d898f5e7a1f5ee34317770", "preferredVersionsHash": "5222ca779ae69ebfd201e39c17f48ce9eaf8c3c2" } From 9c32bf8aac1bb3490c78406085377b453264e43b Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Sat, 22 Apr 2023 21:00:35 +0800 Subject: [PATCH 48/55] chore: connect/disconnect lock provider only cobuild enabled --- libraries/rush-lib/src/api/CobuildConfiguration.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 7c50e649299..a5025e61d8a 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -124,10 +124,14 @@ export class CobuildConfiguration { } public async connectLockProviderAsync(): Promise { - await this.cobuildLockProvider.connectAsync(); + if (this.cobuildEnabled) { + await this.cobuildLockProvider.connectAsync(); + } } public async disconnectLockProviderAsync(): Promise { - await this.cobuildLockProvider.disconnectAsync(); + if (this.cobuildEnabled) { + await this.cobuildLockProvider.disconnectAsync(); + } } } From 7182a2ee3e7c7e690d0460a0b163c5ac18c24e0f Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Sat, 22 Apr 2023 11:40:05 -0700 Subject: [PATCH 49/55] Bump versions for merged packages --- rush-plugins/rush-http-build-cache-plugin/package.json | 2 +- rush-plugins/rush-redis-cobuild-plugin/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rush-plugins/rush-http-build-cache-plugin/package.json b/rush-plugins/rush-http-build-cache-plugin/package.json index 9e289c5a2b0..7d565a4576d 100644 --- a/rush-plugins/rush-http-build-cache-plugin/package.json +++ b/rush-plugins/rush-http-build-cache-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@rushstack/rush-http-build-cache-plugin", - "version": "5.92.0", + "version": "5.97.1-pr3481.18", "description": "Rush plugin for generic HTTP cloud build cache", "repository": { "type": "git", diff --git a/rush-plugins/rush-redis-cobuild-plugin/package.json b/rush-plugins/rush-redis-cobuild-plugin/package.json index 19caebf3bcd..aebb8eb3f6d 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/package.json +++ b/rush-plugins/rush-redis-cobuild-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@rushstack/rush-redis-cobuild-plugin", - "version": "0.0.0", + "version": "5.97.1-pr3481.18", "description": "Rush plugin for Redis cobuild lock", "repository": { "type": "git", From ea364fdacf5b09b7bc569acd262d904428a3c3a5 Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Sat, 22 Apr 2023 11:44:27 -0700 Subject: [PATCH 50/55] Remove "workspace:5.93.1-pr3481.17" workaround --- apps/lockfile-explorer/package.json | 2 +- build-tests/install-test-workspace/package.json | 4 ++-- .../package.json | 2 +- build-tests/rush-lib-declaration-paths-test/package.json | 2 +- build-tests/rush-project-change-analyzer-test/package.json | 2 +- repo-scripts/repo-toolbox/package.json | 2 +- rush-plugins/rush-litewatch-plugin/package.json | 2 +- rush-plugins/rush-serve-plugin/package.json | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/lockfile-explorer/package.json b/apps/lockfile-explorer/package.json index cdc208b9f47..662bc9693e1 100644 --- a/apps/lockfile-explorer/package.json +++ b/apps/lockfile-explorer/package.json @@ -37,7 +37,7 @@ "_phase:test": "heft test --no-build" }, "devDependencies": { - "@microsoft/rush-lib": "workspace:5.93.1-pr3481.17", + "@microsoft/rush-lib": "workspace:*", "@rushstack/eslint-config": "workspace:*", "@rushstack/heft-node-rig": "workspace:*", "@rushstack/heft": "workspace:*", diff --git a/build-tests/install-test-workspace/package.json b/build-tests/install-test-workspace/package.json index dd9a5b939c5..6a994fe8341 100644 --- a/build-tests/install-test-workspace/package.json +++ b/build-tests/install-test-workspace/package.json @@ -8,8 +8,8 @@ "_phase:build": "node build.js" }, "devDependencies": { - "@microsoft/rush-lib": "workspace:5.93.1-pr3481.17", + "@microsoft/rush-lib": "workspace:*", "@rushstack/node-core-library": "workspace:*", - "@rushstack/rush-sdk": "workspace:5.93.1-pr3481.17" + "@rushstack/rush-sdk": "workspace:*" } } diff --git a/build-tests/rush-amazon-s3-build-cache-plugin-integration-test/package.json b/build-tests/rush-amazon-s3-build-cache-plugin-integration-test/package.json index cd8580d807a..0fccc4c2fbe 100644 --- a/build-tests/rush-amazon-s3-build-cache-plugin-integration-test/package.json +++ b/build-tests/rush-amazon-s3-build-cache-plugin-integration-test/package.json @@ -13,7 +13,7 @@ "devDependencies": { "@rushstack/eslint-config": "workspace:*", "@rushstack/heft": "workspace:*", - "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:5.93.1-pr3481.17", + "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:*", "@rushstack/node-core-library": "workspace:*", "@types/node": "14.18.36", "eslint": "~8.7.0", diff --git a/build-tests/rush-lib-declaration-paths-test/package.json b/build-tests/rush-lib-declaration-paths-test/package.json index 8acb60ff2bb..63559034233 100644 --- a/build-tests/rush-lib-declaration-paths-test/package.json +++ b/build-tests/rush-lib-declaration-paths-test/package.json @@ -8,7 +8,7 @@ "_phase:build": "heft build --clean" }, "dependencies": { - "@microsoft/rush-lib": "workspace:5.93.1-pr3481.17" + "@microsoft/rush-lib": "workspace:*" }, "devDependencies": { "@rushstack/eslint-config": "workspace:*", diff --git a/build-tests/rush-project-change-analyzer-test/package.json b/build-tests/rush-project-change-analyzer-test/package.json index 331f633b9ce..3d0a488c0d6 100644 --- a/build-tests/rush-project-change-analyzer-test/package.json +++ b/build-tests/rush-project-change-analyzer-test/package.json @@ -9,7 +9,7 @@ "_phase:build": "heft build --clean" }, "dependencies": { - "@microsoft/rush-lib": "workspace:5.93.1-pr3481.17", + "@microsoft/rush-lib": "workspace:*", "@rushstack/node-core-library": "workspace:*" }, "devDependencies": { diff --git a/repo-scripts/repo-toolbox/package.json b/repo-scripts/repo-toolbox/package.json index b297f2f1917..329f5cc8043 100644 --- a/repo-scripts/repo-toolbox/package.json +++ b/repo-scripts/repo-toolbox/package.json @@ -10,7 +10,7 @@ "_phase:build": "heft build --clean" }, "dependencies": { - "@microsoft/rush-lib": "workspace:5.93.1-pr3481.17", + "@microsoft/rush-lib": "workspace:*", "@rushstack/node-core-library": "workspace:*", "@rushstack/ts-command-line": "workspace:*", "diff": "~5.0.0" diff --git a/rush-plugins/rush-litewatch-plugin/package.json b/rush-plugins/rush-litewatch-plugin/package.json index cefbc9f9e8d..1d9d276e2eb 100644 --- a/rush-plugins/rush-litewatch-plugin/package.json +++ b/rush-plugins/rush-litewatch-plugin/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@rushstack/node-core-library": "workspace:*", - "@rushstack/rush-sdk": "workspace:5.93.1-pr3481.17" + "@rushstack/rush-sdk": "workspace:*" }, "devDependencies": { "@rushstack/eslint-config": "workspace:*", diff --git a/rush-plugins/rush-serve-plugin/package.json b/rush-plugins/rush-serve-plugin/package.json index 2b3c4503240..fbbe9eb5130 100644 --- a/rush-plugins/rush-serve-plugin/package.json +++ b/rush-plugins/rush-serve-plugin/package.json @@ -20,7 +20,7 @@ "@rushstack/heft-config-file": "workspace:*", "@rushstack/node-core-library": "workspace:*", "@rushstack/rig-package": "workspace:*", - "@rushstack/rush-sdk": "workspace:5.93.1-pr3481.17", + "@rushstack/rush-sdk": "workspace:*", "@rushstack/ts-command-line": "workspace:*", "express": "4.18.1" }, From 4d37b1a4cfd5e23077a7a5e8830179f8bbed8b29 Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Sat, 22 Apr 2023 11:45:05 -0700 Subject: [PATCH 51/55] rush update --- common/config/rush/repo-state.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index 8e773966443..a1d2598d5e6 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "cd591f4ff4f6bc918abe7c916fbbd0e0162d8ca0", + "pnpmShrinkwrapHash": "18fb63959a78aa2a51d942a78a367823623380ea", "preferredVersionsHash": "5222ca779ae69ebfd201e39c17f48ce9eaf8c3c2" } From 8a3b45d871c547b0d01ef4557f2f38083ca9aabd Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Sat, 22 Apr 2023 11:49:24 -0700 Subject: [PATCH 52/55] rebuild install-test-workspace --- .../install-test-workspace/workspace/common/pnpm-lock.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/build-tests/install-test-workspace/workspace/common/pnpm-lock.yaml b/build-tests/install-test-workspace/workspace/common/pnpm-lock.yaml index dcab06481f7..47825f31ca4 100644 --- a/build-tests/install-test-workspace/workspace/common/pnpm-lock.yaml +++ b/build-tests/install-test-workspace/workspace/common/pnpm-lock.yaml @@ -111,10 +111,6 @@ packages: resolution: {integrity: sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw==} engines: {node: '>=6.9.0'} - /@babel/helper-validator-identifier/7.14.0: - resolution: {integrity: sha512-V3ts7zMSu5lfiwWDVWzRDGIN+lnCEUdaXgtVHJgLb1rGaA6jMrtB9EmE7L18foXJIE8Un/A/h6NJfGQp/e1J4A==} - dev: true - /@babel/helper-validator-identifier/7.19.1: resolution: {integrity: sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==} engines: {node: '>=6.9.0'} @@ -122,7 +118,7 @@ packages: /@babel/highlight/7.14.0: resolution: {integrity: sha512-YSCOwxvTYEIMSGaBQb5kDDsCopDdiUGsqpatp3fOlI4+2HQSkTmEVWnVuySdAC5EWCqSWWTv0ib63RjR7dTBdg==} dependencies: - '@babel/helper-validator-identifier': 7.14.0 + '@babel/helper-validator-identifier': 7.19.1 chalk: 2.4.2 js-tokens: 4.0.0 dev: true @@ -3685,6 +3681,7 @@ packages: jszip: 3.8.0 lodash: 4.17.21 node-fetch: 2.6.7 + normalize-path: 3.0.0 npm-check: 6.0.1 npm-package-arg: 6.1.1 npm-packlist: 2.1.5 From 2b61d767cb02b400b6c7a77ec6a7e6979e796b63 Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Sat, 22 Apr 2023 11:52:43 -0700 Subject: [PATCH 53/55] Fix incorrect version policy for Rush plugins --- rush.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rush.json b/rush.json index 7219f34916b..4a5759d1e83 100644 --- a/rush.json +++ b/rush.json @@ -1042,13 +1042,13 @@ "packageName": "@rushstack/rush-redis-cobuild-plugin", "projectFolder": "rush-plugins/rush-redis-cobuild-plugin", "reviewCategory": "libraries", - "shouldPublish": true + "versionPolicyName": "rush" }, { "packageName": "@rushstack/rush-serve-plugin", "projectFolder": "rush-plugins/rush-serve-plugin", "reviewCategory": "libraries", - "shouldPublish": true + "versionPolicyName": "rush" }, // "webpack" folder (alphabetical order) From fff7b7590fded12a45357eadda6a32a40a1d90b1 Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Sat, 22 Apr 2023 12:12:01 -0700 Subject: [PATCH 54/55] rush update --recheck --- common/config/rush/pnpm-lock.yaml | 5 ++++- common/config/rush/repo-state.json | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index e800f34a344..8568042057b 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1366,7 +1366,7 @@ importers: '@rushstack/heft': link:../../apps/heft '@rushstack/node-core-library': link:../../libraries/node-core-library '@rushstack/rush-redis-cobuild-plugin': link:../../rush-plugins/rush-redis-cobuild-plugin - '@types/http-proxy': 1.17.9 + '@types/http-proxy': 1.17.10 '@types/node': 14.18.36 eslint: 8.7.0 http-proxy: 1.18.1 @@ -10530,6 +10530,7 @@ packages: /bindings/1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + requiresBuild: true dependencies: file-uri-to-path: 1.0.0 optional: true @@ -13694,6 +13695,7 @@ packages: /file-uri-to-path/1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + requiresBuild: true optional: true /file-uri-to-path/2.0.0: @@ -17690,6 +17692,7 @@ packages: /nan/2.17.0: resolution: {integrity: sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==} + requiresBuild: true optional: true /nanoid/3.3.4: diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index a1d2598d5e6..8f925a526a2 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "18fb63959a78aa2a51d942a78a367823623380ea", + "pnpmShrinkwrapHash": "3fe190e1dbf2ec1521eb033205ab2e873dc3b259", "preferredVersionsHash": "5222ca779ae69ebfd201e39c17f48ce9eaf8c3c2" } From 441e6cd5088fb1ff9ed2f4573fffdf8c5999c3ff Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Sat, 22 Apr 2023 12:19:21 -0700 Subject: [PATCH 55/55] Ensure all plugins get built --- common/config/azure-pipelines/templates/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/config/azure-pipelines/templates/build.yaml b/common/config/azure-pipelines/templates/build.yaml index 30616940808..42f1556c0bd 100644 --- a/common/config/azure-pipelines/templates/build.yaml +++ b/common/config/azure-pipelines/templates/build.yaml @@ -12,7 +12,7 @@ steps: # displayName: 'Verify Change Logs' - script: 'node common/scripts/install-run-rush.js install' displayName: 'Rush Install' - - script: 'node common/scripts/install-run-rush.js retest --verbose --production --to rush --to rush-lib' + - script: 'node common/scripts/install-run-rush.js retest --verbose --production --to rush --from rush-lib' displayName: 'Rush retest (install-run-rush) --to rush' env: # Prevent time-based browserslist update warning