[rush] Optimize the execution speed of Rush

common/changes/@microsoft/rush/main_2024-11-18-08-13.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Add a new experiment flag `enableSubpathScan` that, when invoking phased script commands with project selection parameters, such as `--to` or `--from`, only hashes files that are needed to compute the cache ids for the selected projects.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/changes/@rushstack/package-deps-hash/main_2024-11-18-08-13.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/package-deps-hash",
+      "comment": "Add a new optional parameter `filterPath` to `getRepoStateAsync` that limits the scope of the git query to only the specified subpaths. This can significantly improve the performance of the function when only part of the full repo data is necessary.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@rushstack/package-deps-hash"
+}

common/reviews/api/package-deps-hash.api.md

@@ -20,7 +20,7 @@ export function getRepoChanges(currentWorkingDirectory: string, revision?: strin
 export function getRepoRoot(currentWorkingDirectory: string, gitPath?: string): string;
 
 // @beta
-export function getRepoStateAsync(rootDirectory: string, additionalRelativePathsToHash?: string[], gitPath?: string): Promise<Map<string, string>>;
+export function getRepoStateAsync(rootDirectory: string, additionalRelativePathsToHash?: string[], gitPath?: string, filterPath?: string[]): Promise<Map<string, string>>;
 
 // @beta
 export function hashFilesAsync(rootDirectory: string, filesToHash: Iterable<string> | AsyncIterable<string>, gitPath?: string): Promise<Iterable<[string, string]>>;

common/reviews/api/rush-lib.api.md

@@ -481,6 +481,7 @@ export interface IExperimentsJson {
     buildCacheWithAllowWarningsInSuccessfulBuild?: boolean;
     buildSkipWithAllowWarningsInSuccessfulBuild?: boolean;
     cleanInstallAfterNpmrcChanges?: boolean;
+    enableSubpathScan?: boolean;
     forbidPhantomResolvableNodeModulesFolders?: boolean;
     generateProjectImpactGraphDuringRushUpdate?: boolean;
     noChmodFieldInTarHeaderNormalization?: boolean;
@@ -1129,7 +1130,7 @@ export class ProjectChangeAnalyzer {
     // (undocumented)
     protected getChangesByProject(lookup: LookupByPath<RushConfigurationProject>, changedFiles: Map<string, IFileDiffStatus>): Map<RushConfigurationProject, Map<string, IFileDiffStatus>>;
     // @internal
-    _tryGetSnapshotProviderAsync(projectConfigurations: ReadonlyMap<RushConfigurationProject, RushProjectConfiguration>, terminal: ITerminal): Promise<GetInputsSnapshotAsyncFn | undefined>;
+    _tryGetSnapshotProviderAsync(projectConfigurations: ReadonlyMap<RushConfigurationProject, RushProjectConfiguration>, terminal: ITerminal, projectSelection?: ReadonlySet<RushConfigurationProject>): Promise<GetInputsSnapshotAsyncFn | undefined>;
 }
 
 // @public

libraries/package-deps-hash/src/getRepoState.ts

@@ -364,7 +364,8 @@ export async function hashFilesAsync(
 export async function getRepoStateAsync(
   rootDirectory: string,
   additionalRelativePathsToHash?: string[],
-  gitPath?: string
+  gitPath?: string,
+  filterPath?: string[]
 ): Promise<Map<string, string>> {
   const statePromise: Promise<IGitTreeState> = spawnGitAsync(
     gitPath,
@@ -378,7 +379,8 @@ export async function getRepoStateAsync(
       '--full-name',
       // As of last commit
       'HEAD',
-      '--'
+      '--',
+      ...(filterPath ?? [])
     ]),
     rootDirectory
   ).then(parseGitLsTree);
@@ -396,7 +398,8 @@ export async function getRepoStateAsync(
       '--ignore-submodules',
       // Don't compare against the remote
       '--no-ahead-behind',
-      '--'
+      '--',
+      ...(filterPath ?? [])
     ]),
     rootDirectory
   ).then(parseGitStatus);

libraries/rush-lib/assets/rush-init/common/config/rush/experiments.json

@@ -102,5 +102,11 @@
   /**
    * When using cobuilds, this experiment allows uncacheable operations to benefit from cobuild orchestration without using the build cache.
    */
-   /*[LINE "HYPOTHETICAL"]*/ "allowCobuildWithoutCache": true
+   /*[LINE "HYPOTHETICAL"]*/ "allowCobuildWithoutCache": true,
+
+   /**
+   * By default, rush perform a full scan of the entire repository. For example, Rush runs `git status` to check for local file changes.
+   * When this toggle is enabled, Rush will only scan specific paths, significantly speeding up Git operations.
+   */
+  /*[LINE "HYPOTHETICAL"]*/ "enableSubpathScan": true
 }

libraries/rush-lib/src/api/ExperimentsConfiguration.ts

@@ -113,6 +113,12 @@ export interface IExperimentsJson {
    * This is useful when you want to speed up operations that can't (or shouldn't) be cached.
    */
   allowCobuildWithoutCache?: boolean;
+
+  /**
+   * By default, rush perform a full scan of the entire repository. For example, Rush runs `git status` to check for local file changes.
+   * When this toggle is enabled, Rush will only scan specific paths, significantly speeding up Git operations.
+   */
+  enableSubpathScan?: boolean;
 }
 
 const _EXPERIMENTS_JSON_SCHEMA: JsonSchema = JsonSchema.fromLoadedObject(schemaJson);

libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts

@@ -546,6 +546,7 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
     } = options;
 
     const { projectConfigurations } = initialCreateOperationsContext;
+    const { projectSelection } = initialCreateOperationsContext;
 
     const operations: Set<Operation> = await this.hooks.createOperations.promise(
       new Set(),
@@ -558,7 +559,7 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> {
 
     const analyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration);
     const getInputsSnapshotAsync: GetInputsSnapshotAsyncFn | undefined =
-      await analyzer._tryGetSnapshotProviderAsync(projectConfigurations, terminal);
+      await analyzer._tryGetSnapshotProviderAsync(projectConfigurations, terminal, projectSelection);
     const initialSnapshot: IInputsSnapshot | undefined = await getInputsSnapshotAsync?.();
 
     repoStateStopwatch.stop();

libraries/rush-lib/src/logic/ProjectChangeAnalyzer.ts

@@ -201,7 +201,8 @@ export class ProjectChangeAnalyzer {
    */
   public async _tryGetSnapshotProviderAsync(
     projectConfigurations: ReadonlyMap<RushConfigurationProject, RushProjectConfiguration>,
-    terminal: ITerminal
+    terminal: ITerminal,
+    projectSelection?: ReadonlySet<RushConfigurationProject>
   ): Promise<GetInputsSnapshotAsyncFn | undefined> {
     try {
       const gitPath: string = this._git.getGitPathOrThrow();
@@ -295,10 +296,19 @@ export class ProjectChangeAnalyzer {
       const lookupByPath: IReadonlyLookupByPath<RushConfigurationProject> =
         this._rushConfiguration.getProjectLookupForRoot(rootDirectory);
 
+      let filterPath: string[] = [];
+
+      if (
+        projectSelection &&
+        this._rushConfiguration.experimentsConfiguration.configuration.enableSubpathScan
+      ) {
+        filterPath = Array.from(projectSelection).map(({ projectFolder }) => projectFolder);
+      }
+
       return async function tryGetSnapshotAsync(): Promise<IInputsSnapshot | undefined> {
         try {
           const [hashes, additionalFiles] = await Promise.all([
-            getRepoStateAsync(rootDirectory, additionalRelativePathsToHash, gitPath),
+            getRepoStateAsync(rootDirectory, additionalRelativePathsToHash, gitPath, filterPath),
             getAdditionalFilesFromRushProjectConfigurationAsync(
               additionalGlobs,
               lookupByPath,

libraries/rush-lib/src/schemas/experiments.schema.json

@@ -73,6 +73,10 @@
     "rushAlerts": {
       "description": "(UNDER DEVELOPMENT) The Rush alerts feature provides a way to send announcements to engineers working in the monorepo, by printing directly in the user's shell window when they invoke Rush commands. This ensures that important notices will be seen by anyone doing active development, since people often ignore normal discussion group messages or don't know to subscribe.",
       "type": "boolean"
+    },
+    "enableSubpathScan": {
+      "description": "By default, rush perform a full scan of the entire repository. For example, Rush runs `git status` to check for local file changes. When this toggle is enabled, Rush will only scan specific paths, significantly speeding up Git operations.",
+      "type": "boolean"
     }
   },
   "additionalProperties": false