[rush] Basic support of --watch for BulkScriptAction (build, rebuild, etc.)

apps/rush-lib/package.json

@@ -30,6 +30,7 @@
     "@rushstack/ts-command-line": "workspace:*",
     "@yarnpkg/lockfile": "~1.0.2",
     "builtin-modules": "~3.1.0",
+    "chokidar": "~3.4.0",
     "cli-table": "~0.3.1",
     "colors": "~1.2.1",
     "git-repo-info": "~2.1.0",

apps/rush-lib/src/cli/scriptActions/BulkScriptAction.ts

@@ -15,18 +15,18 @@ import { PackageName } from '@rushstack/node-core-library';
 
 import { Event } from '../../index';
 import { SetupChecks } from '../../logic/SetupChecks';
-import { TaskSelector } from '../../logic/TaskSelector';
-import { Stopwatch } from '../../utilities/Stopwatch';
+import { ITaskSelectorConstructor, TaskSelector } from '../../logic/TaskSelector';
+import { Stopwatch, StopwatchState } from '../../utilities/Stopwatch';
 import { BaseScriptAction, IBaseScriptActionOptions } from './BaseScriptAction';
-import { TaskRunner } from '../../logic/taskRunner/TaskRunner';
-import { TaskCollection } from '../../logic/taskRunner/TaskCollection';
+import { ITaskRunnerOptions, TaskRunner } from '../../logic/taskRunner/TaskRunner';
 import { Utilities } from '../../utilities/Utilities';
 import { RushConstants } from '../../logic/RushConstants';
 import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration';
 import { LastLinkFlag, LastLinkFlagFactory } from '../../api/LastLinkFlag';
 import { IRushConfigurationProjectJson, RushConfigurationProject } from '../../api/RushConfigurationProject';
 import { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration';
 import { Selection } from '../../logic/Selection';
+import { IProjectChangeResult, ProjectWatcher } from '../../logic/ProjectWatcher';
 
 /**
  * Constructor parameters for BulkScriptAction.
@@ -44,6 +44,14 @@ export interface IBulkScriptActionOptions extends IBaseScriptActionOptions {
   commandToRun?: string;
 }
 
+interface IExecuteInternalOptions {
+  taskSelectorOptions: ITaskSelectorConstructor;
+  taskRunnerOptions: ITaskRunnerOptions;
+  stopwatch: Stopwatch;
+  ignoreHooks?: boolean;
+  terminal: Terminal;
+}
+
 /**
  * This class implements bulk commands which are run individually for each project in the repo,
  * possibly in parallel.  The action executes a script found in the project's package.json file.
@@ -68,6 +76,7 @@ export class BulkScriptAction extends BaseScriptAction {
   private _impactedByExceptProject!: CommandLineStringListParameter;
   private _fromVersionPolicy!: CommandLineStringListParameter;
   private _toVersionPolicy!: CommandLineStringListParameter;
+  private _watchParameter!: CommandLineFlagParameter;
   private _verboseParameter!: CommandLineFlagParameter;
   private _parallelismParameter: CommandLineStringParameter | undefined;
   private _ignoreHooksParameter!: CommandLineFlagParameter;
@@ -158,7 +167,14 @@ export class BulkScriptAction extends BaseScriptAction {
       Selection.expandAllConsumers(impactedByProjects)
     );
 
-    const taskSelector: TaskSelector = new TaskSelector({
+    // If no projects selected, select everything.
+    if (selection.size === 0) {
+      for (const project of this.rushConfiguration.projects) {
+        selection.add(project);
+      }
+    }
+
+    const taskSelectorOptions: ITaskSelectorConstructor = {
       rushConfiguration: this.rushConfiguration,
       buildCacheConfiguration,
       selection,
@@ -169,44 +185,105 @@ export class BulkScriptAction extends BaseScriptAction {
       ignoreMissingScript: this._ignoreMissingScript,
       ignoreDependencyOrder: this._ignoreDependencyOrder,
       packageDepsFilename: Utilities.getPackageDepsFilenameForCommand(this._commandToRun)
-    });
-
-    // Register all tasks with the task collection
-    const taskCollection: TaskCollection = taskSelector.registerTasks();
+    };
 
-    const taskRunner: TaskRunner = new TaskRunner(taskCollection.getOrderedTasks(), {
+    const taskRunnerOptions: ITaskRunnerOptions = {
       quietMode: isQuietMode,
       parallelism: parallelism,
       changedProjectsOnly: changedProjectsOnly,
       allowWarningsInSuccessfulBuild: this._allowWarningsInSuccessfulBuild
-    });
+    };
 
-    try {
-      await taskRunner.executeAsync();
+    const executeOptions: IExecuteInternalOptions = {
+      taskSelectorOptions,
+      taskRunnerOptions,
+      stopwatch,
+      terminal
+    };
 
-      stopwatch.stop();
-      console.log(colors.green(`rush ${this.actionName} (${stopwatch.toString()})`));
+    if (this._watchParameter.value) {
+      await this.runWatch(executeOptions);
+    } else {
+      await this._runOnce(executeOptions);
+    }
+  }
 
-      this._doAfterTask(stopwatch, true);
-    } catch (error) {
-      stopwatch.stop();
+  /**
+   * Runs the command in watch mode. Fundamentally is a simple loop:
+   * 1) Wait for a change to one or more projects in the selection (skipped initially)
+   * 2) Invoke the command on the changed projects, and, if applicable, downstream projects
+   * 3) Goto (1)
+   */
+  protected async runWatch(options: IExecuteInternalOptions): Promise<void> {
+    const {
+      taskSelectorOptions: { selection: initialSelection },
+      stopwatch,
+      terminal
+    } = options;
+
+    const projectWatcher: ProjectWatcher = new ProjectWatcher({
+      debounceMilliseconds: 1000,
+      rushConfiguration: this.rushConfiguration,
+      selection: initialSelection
+    });
 
-      if (error instanceof AlreadyReportedError) {
-        console.log(`rush ${this.actionName} (${stopwatch.toString()})`);
-      } else {
-        if (error && error.message) {
-          if (this.parser.isDebug) {
-            console.log('Error: ' + error.stack);
-          } else {
-            console.log('Error: ' + error.message);
-          }
-        }
+    // Loop until Ctrl+C
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      // Report so that the developer can always see that it is in watch mode.
+      terminal.writeLine(
+        `Watching for changes to ${initialSelection.size} ${
+          initialSelection.size === 1 ? 'project' : 'projects'
+        }. Press Ctrl+C to exit.`
+      );
+
+      // On the initial invocation, this promise will return immediately with the full set of projects
+      const change: IProjectChangeResult = await projectWatcher.waitForChange();
+
+      let selection: ReadonlySet<RushConfigurationProject> = change.changedProjects;
+
+      if (stopwatch.state === StopwatchState.Stopped) {
+        // Clear and reset the stopwatch so that we only report time from a single execution at a time
+        stopwatch.reset();
+        stopwatch.start();
+      }
 
-        console.log(colors.red(`rush ${this.actionName} - Errors! (${stopwatch.toString()})`));
+      terminal.writeLine(`Detected changes in ${selection.size} project${selection.size === 1 ? '' : 's'}:`);
+      const names: string[] = [...selection].map((x) => x.packageName).sort();
+      for (const name of names) {
+        terminal.writeLine(`    ${colors.cyan(name)}`);
       }
 
-      this._doAfterTask(stopwatch, false);
-      throw new AlreadyReportedError();
+      // If the command ignores dependency order, that means that only the changed projects should be affected
+      // That said, running watch for commands that ignore dependency order may have unexpected results
+      if (!this._ignoreDependencyOrder) {
+        selection = Selection.intersection(Selection.expandAllConsumers(selection), initialSelection);
+      }
+
+      const executeOptions: IExecuteInternalOptions = {
+        taskSelectorOptions: {
+          ...options.taskSelectorOptions,
+          // Revise down the set of projects to execute the command on
+          selection,
+          // Pass the PackageChangeAnalyzer from the state differ to save a bit of overhead
+          packageChangeAnalyzer: change.state
+        },
+        taskRunnerOptions: options.taskRunnerOptions,
+        stopwatch,
+        // For now, don't run pre-build or post-build in watch mode
+        ignoreHooks: true,
+        terminal
+      };
+
+      try {
+        // Delegate the the underlying command, for only the projects that need reprocessing
+        await this._runOnce(executeOptions);
+      } catch (err) {
+        // In watch mode, we want to rebuild even if the original build failed.
+        if (!(err instanceof AlreadyReportedError)) {
+          throw err;
+        }
+      }
     }
   }
 
@@ -329,6 +406,18 @@ export class BulkScriptAction extends BaseScriptAction {
         ' For details, refer to the website article "Selecting subsets of projects".'
     });
 
+    this._watchParameter = this.defineFlagParameter({
+      parameterLongName: '--watch',
+      parameterShortName: '-w',
+      description:
+        'Normally Rush terminates after the command finishes. The "--watch" parameter will instead cause Rush' +
+        ' to enter a loop where it watches the file system for changes to the selected projects.' +
+        ' Whenever a change is detected, the command will be invoked again for the changed project and' +
+        ' any selected projects that directly or indirectly depend on it.' +
+        ' This parameter may be combined with "--changed-projects-only" to ignore dependent projects.' +
+        ' For details, refer to the website article "Using watch mode".'
+    });
+
     this._verboseParameter = this.defineFlagParameter({
       parameterLongName: '--verbose',
       parameterShortName: '-v',
@@ -339,8 +428,11 @@ export class BulkScriptAction extends BaseScriptAction {
         parameterLongName: '--changed-projects-only',
         parameterShortName: '-c',
         description:
-          'If specified, the incremental build will only rebuild projects that have changed, ' +
-          'but not any projects that directly or indirectly depend on the changed package.'
+          'Normally the incremental build logic will rebuild changed projects as well as' +
+          ' any projects that directly or indirectly depend on a changed project. Specify "--changed-projects-only"' +
+          ' to ignore dependent projects, only rebuilding those projects whose files were changed.' +
+          ' Note that this parameter is "unsafe"; it is up to the developer to ensure that the ignored projects' +
+          ' are okay to ignore.'
       });
     }
     this._ignoreHooksParameter = this.defineFlagParameter({
@@ -351,6 +443,54 @@ export class BulkScriptAction extends BaseScriptAction {
     this.defineScriptParameters();
   }
 
+  /**
+   * Runs a single invocation of the command
+   */
+  private async _runOnce(options: IExecuteInternalOptions): Promise<void> {
+    const taskSelector: TaskSelector = new TaskSelector(options.taskSelectorOptions);
+
+    // Register all tasks with the task collection
+
+    const taskRunner: TaskRunner = new TaskRunner(
+      taskSelector.registerTasks().getOrderedTasks(),
+      options.taskRunnerOptions
+    );
+
+    const { ignoreHooks, stopwatch } = options;
+
+    try {
+      await taskRunner.executeAsync();
+
+      stopwatch.stop();
+      console.log(colors.green(`rush ${this.actionName} (${stopwatch.toString()})`));
+
+      if (!ignoreHooks) {
+        this._doAfterTask(stopwatch, true);
+      }
+    } catch (error) {
+      stopwatch.stop();
+
+      if (error instanceof AlreadyReportedError) {
+        console.log(`rush ${this.actionName} (${stopwatch.toString()})`);
+      } else {
+        if (error && error.message) {
+          if (this.parser.isDebug) {
+            console.log('Error: ' + error.stack);
+          } else {
+            console.log('Error: ' + error.message);
+          }
+        }
+
+        console.log(colors.red(`rush ${this.actionName} - Errors! (${stopwatch.toString()})`));
+      }
+
+      if (!ignoreHooks) {
+        this._doAfterTask(stopwatch, false);
+      }
+      throw new AlreadyReportedError();
+    }
+  }
+
   private async _getProjectNames(): Promise<string[]> {
     const unscopedNamesMap: Map<string, number> = new Map<string, number>();
 

apps/rush-lib/src/cli/test/__snapshots__/CommandLineHelp.test.ts.snap

@@ -113,7 +113,7 @@ exports[`CommandLineHelp prints the help for each action: build 1`] = `
 "usage: rush build [-h] [-p COUNT] [-t PROJECT] [-T PROJECT] [-f PROJECT]
                   [-o PROJECT] [-i PROJECT] [-I PROJECT]
                   [--to-version-policy VERSION_POLICY_NAME]
-                  [--from-version-policy VERSION_POLICY_NAME] [-v] [-c]
+                  [--from-version-policy VERSION_POLICY_NAME] [-w] [-v] [-c]
                   [--ignore-hooks] [-s] [-m]
 
 
@@ -217,12 +217,27 @@ Optional arguments:
                         each of the projects belonging to VERSION_POLICY_NAME.
                          For details, refer to the website article \\"Selecting 
                         subsets of projects\\".
+  -w, --watch           Normally Rush terminates after the command finishes. 
+                        The \\"--watch\\" parameter will instead cause Rush to 
+                        enter a loop where it watches the file system for 
+                        changes to the selected projects. Whenever a change 
+                        is detected, the command will be invoked again for 
+                        the changed project and any selected projects that 
+                        directly or indirectly depend on it. This parameter 
+                        may be combined with \\"--changed-projects-only\\" to 
+                        ignore dependent projects. For details, refer to the 
+                        website article \\"Using watch mode\\".
   -v, --verbose         Display the logs during the build, rather than just 
                         displaying the build status summary
   -c, --changed-projects-only
-                        If specified, the incremental build will only rebuild 
-                        projects that have changed, but not any projects that 
-                        directly or indirectly depend on the changed package.
+                        Normally the incremental build logic will rebuild 
+                        changed projects as well as any projects that 
+                        directly or indirectly depend on a changed project. 
+                        Specify \\"--changed-projects-only\\" to ignore dependent 
+                        projects, only rebuilding those projects whose files 
+                        were changed. Note that this parameter is \\"unsafe\\"; 
+                        it is up to the developer to ensure that the ignored 
+                        projects are okay to ignore.
   --ignore-hooks        Skips execution of the \\"eventHooks\\" scripts defined 
                         in rush.json. Make sure you know what you are 
                         skipping.
@@ -349,8 +364,8 @@ exports[`CommandLineHelp prints the help for each action: import-strings 1`] = `
 "usage: rush import-strings [-h] [-p COUNT] [-t PROJECT] [-T PROJECT]
                            [-f PROJECT] [-o PROJECT] [-i PROJECT] [-I PROJECT]
                            [--to-version-policy VERSION_POLICY_NAME]
-                           [--from-version-policy VERSION_POLICY_NAME] [-v]
-                           [--ignore-hooks]
+                           [--from-version-policy VERSION_POLICY_NAME] [-w]
+                           [-v] [--ignore-hooks]
                            [--locale {en-us,fr-fr,es-es,zh-cn}]
 
 
@@ -445,6 +460,16 @@ Optional arguments:
                         each of the projects belonging to VERSION_POLICY_NAME.
                          For details, refer to the website article \\"Selecting 
                         subsets of projects\\".
+  -w, --watch           Normally Rush terminates after the command finishes. 
+                        The \\"--watch\\" parameter will instead cause Rush to 
+                        enter a loop where it watches the file system for 
+                        changes to the selected projects. Whenever a change 
+                        is detected, the command will be invoked again for 
+                        the changed project and any selected projects that 
+                        directly or indirectly depend on it. This parameter 
+                        may be combined with \\"--changed-projects-only\\" to 
+                        ignore dependent projects. For details, refer to the 
+                        website article \\"Using watch mode\\".
   -v, --verbose         Display the logs during the build, rather than just 
                         displaying the build status summary
   --ignore-hooks        Skips execution of the \\"eventHooks\\" scripts defined 
@@ -740,7 +765,7 @@ exports[`CommandLineHelp prints the help for each action: rebuild 1`] = `
 "usage: rush rebuild [-h] [-p COUNT] [-t PROJECT] [-T PROJECT] [-f PROJECT]
                     [-o PROJECT] [-i PROJECT] [-I PROJECT]
                     [--to-version-policy VERSION_POLICY_NAME]
-                    [--from-version-policy VERSION_POLICY_NAME] [-v]
+                    [--from-version-policy VERSION_POLICY_NAME] [-w] [-v]
                     [--ignore-hooks] [-s] [-m]
 
 
@@ -841,6 +866,16 @@ Optional arguments:
                         each of the projects belonging to VERSION_POLICY_NAME.
                          For details, refer to the website article \\"Selecting 
                         subsets of projects\\".
+  -w, --watch           Normally Rush terminates after the command finishes. 
+                        The \\"--watch\\" parameter will instead cause Rush to 
+                        enter a loop where it watches the file system for 
+                        changes to the selected projects. Whenever a change 
+                        is detected, the command will be invoked again for 
+                        the changed project and any selected projects that 
+                        directly or indirectly depend on it. This parameter 
+                        may be combined with \\"--changed-projects-only\\" to 
+                        ignore dependent projects. For details, refer to the 
+                        website article \\"Using watch mode\\".
   -v, --verbose         Display the logs during the build, rather than just 
                         displaying the build status summary
   --ignore-hooks        Skips execution of the \\"eventHooks\\" scripts defined