From f1f9894f8866943ad3fb2a611a879908254831e6 Mon Sep 17 00:00:00 2001 From: David Michon Date: Thu, 14 Sep 2023 13:17:25 -0700 Subject: [PATCH 1/3] [rush] Add "waiting" status, fix status updates --- .../rush-serve-socket_2023-09-14-20-17.json | 10 ++++ common/reviews/api/rush-lib.api.md | 3 +- .../logic/operations/AsyncOperationQueue.ts | 49 ++++++++++++------- .../logic/operations/ConsoleTimelinePlugin.ts | 2 + .../operations/OperationExecutionManager.ts | 6 --- .../operations/OperationExecutionRecord.ts | 31 +++++++----- .../src/logic/operations/OperationStatus.ts | 6 ++- .../test/AsyncOperationQueue.test.ts | 14 +----- 8 files changed, 71 insertions(+), 50 deletions(-) create mode 100644 common/changes/@microsoft/rush/rush-serve-socket_2023-09-14-20-17.json diff --git a/common/changes/@microsoft/rush/rush-serve-socket_2023-09-14-20-17.json b/common/changes/@microsoft/rush/rush-serve-socket_2023-09-14-20-17.json new file mode 100644 index 00000000000..c9d0d1b1b96 --- /dev/null +++ b/common/changes/@microsoft/rush/rush-serve-socket_2023-09-14-20-17.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Add \"Waiting\" operation status for operations that have one or more dependencies still pending. Ensure that the `onOperationStatusChanged` hook fires for every status change.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 80494d1084e..213c6813729 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -877,7 +877,8 @@ export enum OperationStatus { RemoteExecuting = "REMOTE EXECUTING", Skipped = "SKIPPED", Success = "SUCCESS", - SuccessWithWarning = "SUCCESS WITH WARNINGS" + SuccessWithWarning = "SUCCESS WITH WARNINGS", + Waiting = "WAITING" } // @public (undocumented) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 5bd47bfd597..2189002ae5d 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -74,6 +74,19 @@ export class AsyncOperationQueue */ public complete(record: OperationExecutionRecord): void { this._completedOperations.add(record); + + // Apply status changes to direct dependents + for (const item of record.consumers) { + // Remove this operation from the dependencies, to unblock the scheduler + if ( + item.dependencies.delete(record) && + item.dependencies.size === 0 && + item.status === OperationStatus.Waiting + ) { + item.status = OperationStatus.Ready; + } + } + if (this._completedOperations.size === this._totalOperations) { this._isDone = true; } @@ -88,39 +101,39 @@ export class AsyncOperationQueue // By iterating in reverse order we do less array shuffling when removing operations for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { - const operation: OperationExecutionRecord = queue[i]; + const record: OperationExecutionRecord = queue[i]; if ( - operation.status === OperationStatus.Blocked || - operation.status === OperationStatus.Skipped || - operation.status === OperationStatus.Success || - operation.status === OperationStatus.SuccessWithWarning || - operation.status === OperationStatus.FromCache || - operation.status === OperationStatus.NoOp || - operation.status === OperationStatus.Failure + record.status === OperationStatus.Blocked || + record.status === OperationStatus.Skipped || + record.status === OperationStatus.Success || + record.status === OperationStatus.SuccessWithWarning || + record.status === OperationStatus.FromCache || + record.status === OperationStatus.NoOp || + record.status === OperationStatus.Failure ) { // It shouldn't be on the queue, remove it queue.splice(i, 1); - } else if ( - operation.status === OperationStatus.Queued || - operation.status === OperationStatus.Executing - ) { + } else if (record.status === OperationStatus.Queued || record.status === OperationStatus.Executing) { // This operation is currently executing // next one plz :) + } else if (record.status === OperationStatus.Waiting) { + // This operation is not yet ready to be executed + // next one plz :) continue; - } else if (operation.status === OperationStatus.RemoteExecuting) { + } else if (record.status === OperationStatus.RemoteExecuting) { // This operation is not ready to execute yet, but it may become ready later // next one plz :) continue; - } else if (operation.status !== OperationStatus.Ready) { + } else if (record.status !== OperationStatus.Ready) { // Sanity check - throw new Error(`Unexpected status "${operation.status}" for queued operation: ${operation.name}`); - } else if (operation.dependencies.size === 0) { + throw new Error(`Unexpected status "${record.status}" for queued operation: ${record.name}`); + } else { // This task is ready to process, hand it to the iterator. // Needs to have queue semantics, otherwise tools that iterate it get confused - operation.status = OperationStatus.Queued; + record.status = OperationStatus.Queued; waitingIterators.shift()!({ - value: operation, + value: record, done: false }); } diff --git a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts index 27a73a78b5c..39c950f501c 100644 --- a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts @@ -75,6 +75,7 @@ const TIMELINE_WIDTH: number = 109; * Timeline - symbols representing each operation status */ const TIMELINE_CHART_SYMBOLS: Record = { + [OperationStatus.Waiting]: '?', [OperationStatus.Ready]: '?', [OperationStatus.Queued]: '?', [OperationStatus.Executing]: '?', @@ -92,6 +93,7 @@ const TIMELINE_CHART_SYMBOLS: Record = { * Timeline - colorizer for each operation status */ const TIMELINE_CHART_COLORIZER: Record string> = { + [OperationStatus.Waiting]: colors.yellow, [OperationStatus.Ready]: colors.yellow, [OperationStatus.Queued]: colors.yellow, [OperationStatus.Executing]: colors.yellow, diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index b46dc315caf..61cd0f6fdfc 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -394,12 +394,6 @@ export class OperationExecutionManager { if (record.status !== OperationStatus.RemoteExecuting) { // If the operation was not remote, then we can notify queue that it is complete this._executionQueue.complete(record); - - // Apply status changes to direct dependents - for (const item of record.consumers) { - // Remove this operation from the dependencies, to unblock the scheduler - item.dependencies.delete(record); - } } } } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 1d025de4d31..d7816e93c8b 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -33,14 +33,6 @@ export class OperationExecutionRecord implements IOperationRunnerContext { */ public readonly operation: Operation; - /** - * The current execution status of an operation. Operations start in the 'ready' state, - * but can be 'blocked' if an upstream operation failed. It is 'executing' when - * the operation is executing. Once execution is complete, it is either 'success' or - * 'failure'. - */ - public status: OperationStatus = OperationStatus.Ready; - /** * The error which occurred while executing this operation, this is stored in case we need * it later (for example to re-print errors at end of execution). @@ -101,6 +93,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { private readonly _context: IOperationExecutionRecordContext; private _collatedWriter: CollatedWriter | undefined = undefined; + private _status: OperationStatus; public constructor(operation: Operation, context: IOperationExecutionRecordContext) { const { runner, associatedPhase, associatedProject } = operation; @@ -123,6 +116,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { }); } this._context = context; + this._status = operation.dependencies.size > 0 ? OperationStatus.Waiting : OperationStatus.Ready; } public get name(): string { @@ -159,6 +153,23 @@ export class OperationExecutionRecord implements IOperationRunnerContext { return this._operationMetadataManager?.stateFile.state?.cobuildRunnerId; } + /** + * The current execution status of an operation. Operations start in the 'ready' state, + * but can be 'blocked' if an upstream operation failed. It is 'executing' when + * the operation is executing. Once execution is complete, it is either 'success' or + * 'failure'. + */ + public get status(): OperationStatus { + return this._status; + } + public set status(newStatus: OperationStatus) { + if (newStatus === this._status) { + return; + } + this._status = newStatus; + this._context.onOperationStatusChanged?.(this); + } + public async executeAsync({ onStart, onResult @@ -169,9 +180,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext { if (this.status === OperationStatus.RemoteExecuting) { this.stopwatch.reset(); } - this.status = OperationStatus.Executing; this.stopwatch.start(); - this._context.onOperationStatusChanged?.(this); + this.status = OperationStatus.Executing; try { const earlyReturnStatus: OperationStatus | undefined = await onStart(this); @@ -194,7 +204,6 @@ export class OperationExecutionRecord implements IOperationRunnerContext { this.stdioSummarizer.close(); this.stopwatch.stop(); } - this._context.onOperationStatusChanged?.(this); } } } diff --git a/libraries/rush-lib/src/logic/operations/OperationStatus.ts b/libraries/rush-lib/src/logic/operations/OperationStatus.ts index 38a6955296c..12532ce4398 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStatus.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStatus.ts @@ -7,9 +7,13 @@ */ export enum OperationStatus { /** - * The Operation is on the queue, ready to execute (but may be waiting for dependencies) + * The Operation is ready to execute. All its dependencies have succeeded. */ Ready = 'READY', + /** + * The Operation is waiting for one or more dependencies to complete. + */ + Waiting = 'WAITING', /** * The Operation is Queued */ diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 7c97cf3f248..9a7d6c12b49 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -16,6 +16,7 @@ import { Async } from '@rushstack/node-core-library'; function addDependency(consumer: OperationExecutionRecord, dependency: OperationExecutionRecord): void { consumer.dependencies.add(dependency); dependency.consumers.add(consumer); + consumer.status = OperationStatus.Waiting; } function nullSort(a: OperationExecutionRecord, b: OperationExecutionRecord): number { @@ -50,9 +51,6 @@ describe(AsyncOperationQueue.name, () => { hasUnassignedOperation = true; continue; } - for (const consumer of operation.consumers) { - consumer.dependencies.delete(operation); - } operation.status = OperationStatus.Success; queue.complete(operation); } @@ -82,9 +80,6 @@ describe(AsyncOperationQueue.name, () => { hasUnassignedOperation = true; continue; } - for (const consumer of operation.consumers) { - consumer.dependencies.delete(operation); - } operation.status = OperationStatus.Success; queue.complete(operation); } @@ -151,10 +146,6 @@ describe(AsyncOperationQueue.name, () => { await Promise.resolve(); - for (const consumer of operation.consumers) { - consumer.dependencies.delete(operation); - } - --concurrency; operation.status = OperationStatus.Success; queue.complete(operation); @@ -213,9 +204,6 @@ describe(AsyncOperationQueue.name, () => { continue; } } - for (const consumer of record.consumers) { - consumer.dependencies.delete(record); - } record.status = OperationStatus.Success; queue.complete(record); } From a3e79eb9fe56366425197129728e000b328065e6 Mon Sep 17 00:00:00 2001 From: David Michon Date: Thu, 14 Sep 2023 14:16:57 -0700 Subject: [PATCH 2/3] [rush-serve-plugin] Add WebSocket support --- .../rush-serve-socket_2023-09-14-20-58.json | 10 + .../rush/nonbrowser-approved-packages.json | 12 +- rush-plugins/rush-serve-plugin/README.md | 74 ++++++ rush-plugins/rush-serve-plugin/package.json | 27 ++- .../rush-serve-plugin/src/RushServePlugin.ts | 14 +- .../rush-serve-plugin/src/api.types.ts | 127 ++++++++++ .../src/phasedCommandHandler.ts | 228 +++++++++++++++++- .../rush-serve-plugin-options.schema.json | 6 + 8 files changed, 486 insertions(+), 12 deletions(-) create mode 100644 common/changes/@microsoft/rush/rush-serve-socket_2023-09-14-20-58.json create mode 100644 rush-plugins/rush-serve-plugin/src/api.types.ts diff --git a/common/changes/@microsoft/rush/rush-serve-socket_2023-09-14-20-58.json b/common/changes/@microsoft/rush/rush-serve-socket_2023-09-14-20-58.json new file mode 100644 index 00000000000..974e08e052f --- /dev/null +++ b/common/changes/@microsoft/rush/rush-serve-socket_2023-09-14-20-58.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "Add support for optional build status notifications over a web socket connection to `@rushstack/rush-serve-plugin`.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index fdbf7174814..b34f3601442 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -406,14 +406,14 @@ "name": "compression", "allowedCategories": [ "libraries" ] }, - { - "name": "cors", - "allowedCategories": [ "libraries" ] - }, { "name": "constructs", "allowedCategories": [ "tests" ] }, + { + "name": "cors", + "allowedCategories": [ "libraries" ] + }, { "name": "css-loader", "allowedCategories": [ "libraries", "tests" ] @@ -826,6 +826,10 @@ "name": "wordwrap", "allowedCategories": [ "libraries" ] }, + { + "name": "ws", + "allowedCategories": [ "libraries" ] + }, { "name": "xmldoc", "allowedCategories": [ "libraries" ] diff --git a/rush-plugins/rush-serve-plugin/README.md b/rush-plugins/rush-serve-plugin/README.md index 01bc2e0094d..c8faa8b295d 100644 --- a/rush-plugins/rush-serve-plugin/README.md +++ b/rush-plugins/rush-serve-plugin/README.md @@ -14,3 +14,77 @@ What happens: - Rush uses the configuration in the aforementioned files to configure an Express server to serve project outputs as static (but not cached) content - When a change happens to a source file, Rush's normal watch-mode machinery will rebuild all affected project phases, resulting in new files on disk - The next time one of these files is requested, Rush will serve the new version. Optionally, may support signals for automatic refresh. + +## Live Build Status via Web Socket + +This plugin also provides a web socket server that notifies clients of the build status in real time. To use the server, configure the `buildStatusWebSocketPath` option in `common/config/rush-plugins/rush-serve-plugin.json`. Specifying `/` will make the web socket server available at `wss://localhost:/`. + +The recommended way to connect to the web socket is to serve a static HTML page from the serve plugin using the `globalRouting` configuration. + +To use the socket: +```ts +import type { + IWebSocketEventMessage, + IOperationInfo, + IRushSessionInfo, + ReadableOperationStatus +} from '@rushstack/rush-serve-plugin/api'; + +const socket: WebSocket = new WebSocket(`wss://${self.location.host}${buildStatusWebSocketPath}`); + +const operationsByName: Map = new Map(); +let buildStatus: ReadableOperationStatus = 'Ready'; + +function updateOperations(operations): void { + for (const operation of operations) { + operationsByName.set(operation.name, operation); + } + + for (const [operationName, operation] of operationsByName) { + // Do something with the operation + } +} + +function updateSessionInfo(sessionInfo: IRushSessionInfo): void { + const { actionName, repositoryIdentifier } = sessionInfo; +} + +function updateBuildStatus(newStatus: ReadableOperationStatus): void { + buildStatus = newStatus; + // Render +} + +socket.addEventListener('message', (ev) => { + const message: IWebSocketEventMessage = JSON.parse(ev.data); + + switch (message.event) { + case 'before-execute': { + const { operations } = message; + updateOperations(operations); + updateBuildStatus('Executing'); + break; + } + + case 'status-change': { + const { operations } = message; + updateOperations(operations); + break; + } + + case 'after-execute': { + const { status } = message; + updateBuildStatus(status); + break; + } + + case 'sync': { + operationsByName.clear(); + const { operations, status, sessionInfo } = message; + updateOperations(operations); + updateSessionInfo(sessionInfo); + updateBuildStatus(status); + break; + } + } +}); +``` diff --git a/rush-plugins/rush-serve-plugin/package.json b/rush-plugins/rush-serve-plugin/package.json index 3ff344818ae..8ad39e0a4b1 100644 --- a/rush-plugins/rush-serve-plugin/package.json +++ b/rush-plugins/rush-serve-plugin/package.json @@ -8,7 +8,7 @@ "type": "git", "directory": "rush-plugins/rush-serve-plugin" }, - "main": "lib/index.js", + "main": "lib-commonjs/index.js", "types": "dist/index.d.ts", "scripts": { "build": "heft test --clean", @@ -25,7 +25,8 @@ "compression": "~1.7.4", "cors": "~2.8.5", "express": "4.18.1", - "http2-express-bridge": "~1.0.7" + "http2-express-bridge": "~1.0.7", + "ws": "~8.14.1" }, "devDependencies": { "@rushstack/eslint-config": "workspace:*", @@ -35,6 +36,26 @@ "@types/cors": "~2.8.12", "@types/express": "4.17.13", "@types/heft-jest": "1.0.1", - "@types/node": "18.17.15" + "@types/node": "18.17.15", + "@types/ws": "8.5.5" + }, + "exports": { + ".": { + "require": "./lib/index.js", + "types": "./dist/rush-serve-plugin.d.ts" + }, + "./api": { + "types": "./lib/api.types.d.ts" + } + }, + "typesVersions": { + "*": { + ".": [ + "dist/rush-serve-plugin.d.ts" + ], + "api": [ + "lib/api.types.d.ts" + ] + } } } diff --git a/rush-plugins/rush-serve-plugin/src/RushServePlugin.ts b/rush-plugins/rush-serve-plugin/src/RushServePlugin.ts index 3174161f874..a3d2b7bfbd2 100644 --- a/rush-plugins/rush-serve-plugin/src/RushServePlugin.ts +++ b/rush-plugins/rush-serve-plugin/src/RushServePlugin.ts @@ -25,12 +25,21 @@ export interface IRushServePluginOptions { * The names of phased commands to which the plugin should be applied. */ phasedCommands: ReadonlyArray; + + /** + * The URL path at which to host the web socket connection for monitoring build status. If not specified, the web socket interface will not be enabled. + */ + buildStatusWebSocketPath?: string; + /** * The name of a parameter that Rush is configured to use to pass a port number to underlying operations. * If specified, the plugin will ensure the value is synchronized with the port used for its server. */ portParameterLongName?: string | undefined; + /** + * Routing rules for files that are associated with the entire workspace, rather than a single project (e.g. files output by Rush plugins). + */ globalRouting?: IGlobalRoutingRuleJson[]; } @@ -43,11 +52,13 @@ export class RushServePlugin implements IRushPlugin { private readonly _phasedCommands: Set; private readonly _portParameterLongName: string | undefined; private readonly _globalRoutingRules: IGlobalRoutingRuleJson[]; + private readonly _buildStatusWebSocketPath: string | undefined; public constructor(options: IRushServePluginOptions) { this._phasedCommands = new Set(options.phasedCommands); this._portParameterLongName = options.portParameterLongName; this._globalRoutingRules = options.globalRouting ?? []; + this._buildStatusWebSocketPath = options.buildStatusWebSocketPath; } public apply(rushSession: RushSession, rushConfiguration: RushConfiguration): void { @@ -73,7 +84,8 @@ export class RushServePlugin implements IRushPlugin { rushConfiguration, command, portParameterLongName: this._portParameterLongName, - globalRoutingRules + globalRoutingRules, + buildStatusWebSocketPath: this._buildStatusWebSocketPath }); }; diff --git a/rush-plugins/rush-serve-plugin/src/api.types.ts b/rush-plugins/rush-serve-plugin/src/api.types.ts new file mode 100644 index 00000000000..93882996945 --- /dev/null +++ b/rush-plugins/rush-serve-plugin/src/api.types.ts @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { OperationStatus } from '@rushstack/rush-sdk'; + +/** + * Human readable status values. These are the PascalCase keys of the `OperationStatus` enumeration. + */ +export type ReadableOperationStatus = keyof typeof OperationStatus; + +/** + * Information about an operation in the graph. + */ +export interface IOperationInfo { + /** + * The display name of the operation. + */ + name: string; + + /** + * The npm package name of the containing Rush Project. + */ + packageName: string; + + /** + * The name of the containing phase. + */ + phaseName: string; + + /** + * If true, this operation is configured to be silent and is included for completeness. + */ + silent: boolean; + /** + * If true, this operation is configured to be a noop and is included for graph completeness. + */ + noop: boolean; + + /** + * The current status of the operation. This value is in PascalCase and is the key of the corresponding `OperationStatus` constant. + */ + status: ReadableOperationStatus; + + /** + * The start time of the operation, if it has started, in milliseconds. Not wall clock time. + */ + startTime: number | undefined; + + /** + * The end time of the operation, if it has finished, in milliseconds. Not wall clock time. + */ + endTime: number | undefined; +} + +/** + * Information about the current Rush session. + */ +export interface IRushSessionInfo { + /** + * The name of the command being run. + */ + actionName: string; + + /** + * A unique identifier for the repository in which this Rush is running. + */ + repositoryIdentifier: string; +} + +/** + * Message sent to a WebSocket client at the start of an execution pass. + */ +export interface IWebSocketBeforeExecuteEventMessage { + event: 'before-execute'; + operations: IOperationInfo[]; +} + +/** + * Message sent to a WebSocket client at the end of an execution pass. + */ +export interface IWebSocketAfterExecuteEventMessage { + event: 'after-execute'; + status: ReadableOperationStatus; +} + +/** + * Message sent to a WebSocket client when one or more operations change status. + * + * Batched to reduce noise and improve throughput. + */ +export interface IWebSocketBatchStatusChangeEventMessage { + event: 'status-change'; + operations: IOperationInfo[]; +} + +/** + * Message sent to a WebSocket client upon initial connection, or when explicitly requested. + * + * @see IWebSocketSyncCommandMessage + */ +export interface IWebSocketSyncEventMessage { + event: 'sync'; + operations: IOperationInfo[]; + sessionInfo: IRushSessionInfo; + status: ReadableOperationStatus; +} + +/** + * The set of possible messages sent to a WebSocket client. + */ +export type IWebSocketEventMessage = + | IWebSocketBeforeExecuteEventMessage + | IWebSocketAfterExecuteEventMessage + | IWebSocketBatchStatusChangeEventMessage + | IWebSocketSyncEventMessage; + +/** + * Message received from a WebSocket client to request a sync. + */ +export interface IWebSocketSyncCommandMessage { + command: 'sync'; +} + +/** + * The set of possible messages received from a WebSocket client. + */ +export type IWebSocketCommandMessage = IWebSocketSyncCommandMessage; diff --git a/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts b/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts index 8d09d5fc1ad..27f9de84171 100644 --- a/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts +++ b/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts @@ -2,36 +2,55 @@ // See LICENSE in the project root for license information. import { once } from 'node:events'; -import http2 from 'node:http2'; +import type { Server as HTTPSecureServer } from 'node:https'; +import http2, { Http2SecureServer } from 'node:http2'; import type { AddressInfo } from 'node:net'; +import os from 'node:os'; import express, { type Application } from 'express'; import http2express from 'http2-express-bridge'; import cors from 'cors'; import compression from 'compression'; +import { WebSocketServer, type WebSocket, type MessageEvent } from 'ws'; import { CertificateManager, ICertificate } from '@rushstack/debug-certificate-manager'; -import { AlreadyReportedError } from '@rushstack/node-core-library'; -import type { +import { AlreadyReportedError, Sort } from '@rushstack/node-core-library'; +import { ILogger, RushConfiguration, RushConfigurationProject, RushSession, IPhasedCommand, Operation, - ICreateOperationsContext + ICreateOperationsContext, + IOperationExecutionResult, + OperationStatus, + IExecutionResult } from '@rushstack/rush-sdk'; import type { CommandLineStringParameter } from '@rushstack/ts-command-line'; import { PLUGIN_NAME } from './constants'; import { IRoutingRule, RushServeConfiguration } from './RushProjectServeConfigFile'; +import type { + IOperationInfo, + IWebSocketAfterExecuteEventMessage, + IWebSocketBeforeExecuteEventMessage, + IWebSocketEventMessage, + IWebSocketBatchStatusChangeEventMessage, + IWebSocketSyncEventMessage, + ReadableOperationStatus, + IWebSocketCommandMessage, + IRushSessionInfo +} from './api.types'; + export interface IPhasedCommandHandlerOptions { rushSession: RushSession; rushConfiguration: RushConfiguration; command: IPhasedCommand; portParameterLongName: string | undefined; globalRoutingRules: IRoutingRule[]; + buildStatusWebSocketPath: string | undefined; } export async function phasedCommandHandler(options: IPhasedCommandHandlerOptions): Promise { @@ -54,6 +73,9 @@ export async function phasedCommandHandler(options: IPhasedCommandHandlerOptions } } + const webSocketServerUpgrader: WebSocketServerUpgrader | undefined = + tryEnableBuildStatusWebSocketServer(options); + command.hooks.createOperations.tapPromise( { name: PLUGIN_NAME, @@ -194,6 +216,8 @@ export async function phasedCommandHandler(options: IPhasedCommandHandlerOptions app ); + webSocketServerUpgrader?.(server); + server.listen(requestedPort); await once(server, 'listening'); @@ -216,3 +240,199 @@ export async function phasedCommandHandler(options: IPhasedCommandHandlerOptions command.hooks.waitingForChanges.tap(PLUGIN_NAME, logHost); } + +type WebSocketServerUpgrader = (server: Http2SecureServer) => void; + +/** + * + */ +function tryEnableBuildStatusWebSocketServer( + options: IPhasedCommandHandlerOptions +): WebSocketServerUpgrader | undefined { + const { buildStatusWebSocketPath } = options; + if (!buildStatusWebSocketPath) { + return; + } + + let operationStates: Map | undefined; + let buildStatus: ReadableOperationStatus = 'Ready'; + + const webSockets: Set = new Set(); + + // Map from OperationStatus enum values back to the names of the constants + const readableStatusFromStatus: { [K in OperationStatus]: ReadableOperationStatus } = { + [OperationStatus.Waiting]: 'Waiting', + [OperationStatus.Ready]: 'Ready', + [OperationStatus.Queued]: 'Queued', + [OperationStatus.Executing]: 'Executing', + [OperationStatus.RemoteExecuting]: 'RemoteExecuting', + [OperationStatus.Success]: 'Success', + [OperationStatus.SuccessWithWarning]: 'SuccessWithWarning', + [OperationStatus.Skipped]: 'Skipped', + [OperationStatus.FromCache]: 'FromCache', + [OperationStatus.Failure]: 'Failure', + [OperationStatus.Blocked]: 'Blocked', + [OperationStatus.NoOp]: 'NoOp' + }; + + /** + * Maps the internal Rush record down to a subset that is JSON-friendly and human readable. + */ + function convertToOperationInfo(record: IOperationExecutionResult): IOperationInfo | undefined { + const { operation } = record; + const { name, associatedPhase, associatedProject, runner } = operation; + + if (!name || !associatedPhase || !associatedProject || !runner) { + return; + } + + return { + name, + packageName: associatedProject.packageName, + phaseName: associatedPhase.name, + + silent: !!runner.silent, + noop: !!runner.isNoOp, + + status: readableStatusFromStatus[record.status], + startTime: record.stopwatch.startTime, + endTime: record.stopwatch.endTime + }; + } + + function convertToOperationInfoArray(records: Iterable): IOperationInfo[] { + const operations: IOperationInfo[] = []; + + for (const record of records) { + const info: IOperationInfo | undefined = convertToOperationInfo(record); + + if (info) { + operations.push(info); + } + } + + Sort.sortBy(operations, (x) => x.name); + return operations; + } + + function sendWebSocketMessage(message: IWebSocketEventMessage): void { + const stringifiedMessage: string = JSON.stringify(message); + for (const socket of webSockets) { + socket.send(stringifiedMessage); + } + } + + const { command } = options; + const sessionInfo: IRushSessionInfo = { + actionName: command.actionName, + repositoryIdentifier: getRepositoryIdentifier(options.rushConfiguration) + }; + + function sendSyncMessage(webSocket: WebSocket): void { + const syncMessage: IWebSocketSyncEventMessage = { + event: 'sync', + operations: convertToOperationInfoArray(operationStates?.values() ?? []), + sessionInfo, + status: buildStatus + }; + + webSocket.send(JSON.stringify(syncMessage)); + } + + const { hooks } = command; + + hooks.beforeExecuteOperations.tap( + PLUGIN_NAME, + (operationsToExecute: Map): void => { + operationStates = operationsToExecute; + + const beforeExecuteMessage: IWebSocketBeforeExecuteEventMessage = { + event: 'before-execute', + operations: convertToOperationInfoArray(operationsToExecute.values()) + }; + buildStatus = 'Executing'; + sendWebSocketMessage(beforeExecuteMessage); + } + ); + + hooks.afterExecuteOperations.tap(PLUGIN_NAME, (result: IExecutionResult): void => { + buildStatus = readableStatusFromStatus[result.status]; + const afterExecuteMessage: IWebSocketAfterExecuteEventMessage = { + event: 'after-execute', + status: buildStatus + }; + sendWebSocketMessage(afterExecuteMessage); + }); + + const pendingStatusChanges: Map = new Map(); + let statusChangeTimeout: NodeJS.Immediate | undefined; + function sendBatchedStatusChange(): void { + statusChangeTimeout = undefined; + const infos: IOperationInfo[] = convertToOperationInfoArray(pendingStatusChanges.values()); + pendingStatusChanges.clear(); + const message: IWebSocketBatchStatusChangeEventMessage = { + event: 'status-change', + operations: infos + }; + sendWebSocketMessage(message); + } + + hooks.onOperationStatusChanged.tap(PLUGIN_NAME, (record: IOperationExecutionResult): void => { + pendingStatusChanges.set(record.operation, record); + if (!statusChangeTimeout) { + statusChangeTimeout = setImmediate(sendBatchedStatusChange); + } + }); + + const connector: WebSocketServerUpgrader = (server: Http2SecureServer) => { + const wss: WebSocketServer = new WebSocketServer({ + server: server as unknown as HTTPSecureServer, + path: buildStatusWebSocketPath + }); + wss.addListener('connection', (webSocket: WebSocket): void => { + webSockets.add(webSocket); + + sendSyncMessage(webSocket); + + webSocket.addEventListener('message', (ev: MessageEvent) => { + const parsedMessage: IWebSocketCommandMessage = JSON.parse(ev.data.toString()); + switch (parsedMessage.command) { + case 'sync': { + sendSyncMessage(webSocket); + break; + } + + default: { + // Unknown message. Ignore. + } + } + }); + + webSocket.addEventListener( + 'close', + () => { + webSockets.delete(webSocket); + }, + { once: true } + ); + }); + }; + + return connector; +} + +function getRepositoryIdentifier(rushConfiguration: RushConfiguration): string { + const { env } = process; + const { CODESPACE_NAME: codespaceName, GITHUB_USER: githubUserName } = env; + + if (codespaceName) { + const usernamePrefix: string | undefined = githubUserName?.replace(/_|$/g, '-'); + const startIndex: number = + usernamePrefix && codespaceName.startsWith(usernamePrefix) ? usernamePrefix.length : 0; + const endIndex: number = codespaceName.lastIndexOf('-'); + const normalizedName: string = codespaceName.slice(startIndex, endIndex).replace(/-/g, ' '); + return `Codespace "${normalizedName}"`; + } + + return `${os.hostname()} - ${rushConfiguration.rushJsonFolder}`; +} diff --git a/rush-plugins/rush-serve-plugin/src/schemas/rush-serve-plugin-options.schema.json b/rush-plugins/rush-serve-plugin/src/schemas/rush-serve-plugin-options.schema.json index f8e440b8e10..f0184060284 100644 --- a/rush-plugins/rush-serve-plugin/src/schemas/rush-serve-plugin-options.schema.json +++ b/rush-plugins/rush-serve-plugin/src/schemas/rush-serve-plugin-options.schema.json @@ -27,6 +27,12 @@ "pattern": "^--(?:[a-z0-9]+)(?:-[a-z0-9]+)*$" }, + "buildStatusWebSocketPath": { + "type": "string", + "description": "The URL path at which to host the web socket connection for monitoring build status. If not specified, the web socket interface will not be enabled.", + "pattern": "^/(?:[a-zA-Z0-9_$-]+(?:/[a-zA-Z0-9_$-]+)*)?$" + }, + "globalRouting": { "type": "array", "description": "Routing rules for files that are associated with the entire workspace, rather than a single project (e.g. files output by Rush plugins).", From 668f03293a0ae1e341cf11d16464c15fcaee1421 Mon Sep 17 00:00:00 2001 From: David Michon Date: Thu, 14 Sep 2023 16:21:55 -0700 Subject: [PATCH 3/3] chore: rush update --- common/config/rush/pnpm-lock.yaml | 26 ++++++++++++++++---------- common/config/rush/repo-state.json | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 4fb5405e78c..a830fc3983c 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -2866,10 +2866,12 @@ importers: '@types/express': 4.17.13 '@types/heft-jest': 1.0.1 '@types/node': 18.17.15 + '@types/ws': 8.5.5 compression: ~1.7.4 cors: ~2.8.5 express: 4.18.1 http2-express-bridge: ~1.0.7 + ws: ~8.14.1 dependencies: '@rushstack/debug-certificate-manager': link:../../libraries/debug-certificate-manager '@rushstack/heft-config-file': link:../../libraries/heft-config-file @@ -2881,6 +2883,7 @@ importers: cors: 2.8.5 express: 4.18.1 http2-express-bridge: 1.0.7_@types+express@4.17.13 + ws: 8.14.1 devDependencies: '@rushstack/eslint-config': link:../../eslint/eslint-config '@rushstack/heft': link:../../apps/heft @@ -2890,6 +2893,7 @@ importers: '@types/express': 4.17.13 '@types/heft-jest': 1.0.1 '@types/node': 18.17.15 + '@types/ws': 8.5.5 ../../vscode-extensions/rush-vscode-command-webview: specifiers: @@ -9711,7 +9715,7 @@ packages: util-deprecate: 1.0.2 watchpack: 2.4.0 webpack: 4.47.0 - ws: 8.13.0 + ws: 8.14.1 transitivePeerDependencies: - '@types/react' - bufferutil @@ -9784,7 +9788,7 @@ packages: util-deprecate: 1.0.2 watchpack: 2.4.0 webpack: 4.47.0 - ws: 8.13.0 + ws: 8.14.1 transitivePeerDependencies: - '@types/react' - bufferutil @@ -10720,6 +10724,9 @@ packages: resolution: {integrity: sha512-FAYBGwC+W6F9+huFIDtn43cpy7+SzG+atzRiTfdp3inUKL2hXnd4rG8hylJLIh4+hqrQy1P17kvJByE/z825hA==} dev: true + /@types/node/14.18.36: + resolution: {integrity: sha512-FXKWbsJ6a1hIrRxv+FoukuHnGTgEzKYGi7kilfMae96AL9UNkPFNWJEEYWzdRI9ooIkbr4AKldyuSTLql06vLQ==} + /@types/node/14.18.59: resolution: {integrity: sha512-NWJMpBL2Xs3MY93yrD6YrrTKep8eIA6iMnfG4oIc6LrTRlBZgiSCGiY3V/Owlp6umIBLyKb4F8Q7hxWatjYH5A==} dev: true @@ -10963,8 +10970,7 @@ packages: /@types/ws/8.5.5: resolution: {integrity: sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg==} dependencies: - '@types/node': 18.17.15 - dev: false + '@types/node': 14.18.36 /@types/xmldoc/1.1.4: resolution: {integrity: sha512-a/ONNCf9itbmzEz1ohx0Fv5TLJzXIPQTapxFu+DlYlDtn9UcAa1OhnrOOMwbU8125hFjrkJKL3qllD7vO5Bivw==} @@ -18379,7 +18385,7 @@ packages: whatwg-encoding: 2.0.0 whatwg-mimetype: 3.0.0 whatwg-url: 11.0.0 - ws: 8.13.0 + ws: 8.14.1 xml-name-validator: 4.0.0 transitivePeerDependencies: - bufferutil @@ -24710,7 +24716,7 @@ packages: webpack: 4.47.0_webpack-cli@3.3.12 webpack-cli: 3.3.12_webpack@4.47.0 webpack-dev-middleware: 5.3.3_webpack@4.47.0 - ws: 8.13.0 + ws: 8.14.1 transitivePeerDependencies: - bufferutil - debug @@ -24764,7 +24770,7 @@ packages: spdy: 4.0.2 webpack: 4.47.0 webpack-dev-middleware: 5.3.3_bs575e6uz6qqyriedrrkqiwy2m - ws: 8.13.0 + ws: 8.14.1 transitivePeerDependencies: - bufferutil - debug @@ -24817,7 +24823,7 @@ packages: spdy: 4.0.2 webpack: 5.82.1 webpack-dev-middleware: 5.3.3_webpack@5.82.1 - ws: 8.13.0 + ws: 8.14.1 transitivePeerDependencies: - bufferutil - debug @@ -25206,8 +25212,8 @@ packages: utf-8-validate: optional: true - /ws/8.13.0: - resolution: {integrity: sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==} + /ws/8.14.1: + resolution: {integrity: sha512-4OOseMUq8AzRBI/7SLMUwO+FEDnguetSk7KMb1sHwvF2w2Wv5Hoj0nlifx8vtGsftE/jWHojPy8sMMzYLJ2G/A==} engines: {node: '>=10.0.0'} peerDependencies: bufferutil: ^4.0.1 diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index 7244dc55c2b..50165990d5b 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "5ee5e5b7fa262ef5096e77b7d86ef24ade887c5b", + "pnpmShrinkwrapHash": "af97ead68ee0632cb19c93ef203e84fc0c1e1375", "preferredVersionsHash": "1926a5b12ac8f4ab41e76503a0d1d0dccc9c0e06" }