Deduplicate WebClient. (#5025)

* Add support for loading modules that only exist in async chunks.

* Use WebClient in the plugins.

* fixup! Use WebClient in the plugins.

* fixup! Use WebClient in the plugins.

* Remove node-fetch from rush-amazon-s3-build-cache-plugin.

* Rush change.

* fixup! Use WebClient in the plugins.

* fixup! Use WebClient in the plugins.

* Rush update.

build-tests/rush-amazon-s3-build-cache-plugin-integration-test/package.json

@@ -11,17 +11,16 @@
     "start-proxy-server": "node ./lib/startProxyServer.js"
   },
   "devDependencies": {
+    "@microsoft/rush-lib": "workspace:*",
     "@rushstack/heft": "workspace:*",
-    "local-node-rig": "workspace:*",
     "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:*",
     "@rushstack/node-core-library": "workspace:*",
+    "@rushstack/terminal": "workspace:*",
+    "@types/http-proxy": "~1.17.8",
     "@types/node": "18.17.15",
     "eslint": "~8.57.0",
-    "typescript": "~5.4.2",
     "http-proxy": "~1.18.1",
-    "@types/http-proxy": "~1.17.8"
-  },
-  "dependencies": {
-    "@rushstack/terminal": "workspace:*"
+    "local-node-rig": "workspace:*",
+    "typescript": "~5.4.2"
   }
 }

build-tests/rush-amazon-s3-build-cache-plugin-integration-test/src/readObject.ts

@@ -2,7 +2,7 @@
 // See LICENSE in the project root for license information.
 
 import { AmazonS3Client } from '@rushstack/rush-amazon-s3-build-cache-plugin';
-import { WebClient } from '@rushstack/rush-amazon-s3-build-cache-plugin';
+import { WebClient } from '@microsoft/rush-lib/lib/utilities/WebClient';
 import { ConsoleTerminalProvider, type ITerminal, Terminal } from '@rushstack/terminal';
 
 const webClient: WebClient = new WebClient();

build-tests/rush-lib-declaration-paths-test/config/heft.json

@@ -18,24 +18,8 @@
           }
         },
 
-        "copy-src-typings": {
-          "taskPlugin": {
-            "pluginPackage": "@rushstack/heft",
-            "pluginName": "copy-files-plugin",
-            "options": {
-              "copyOperations": [
-                {
-                  "sourcePath": "node_modules/@microsoft/rush-lib/src",
-                  "destinationFolders": ["src"],
-                  "includeGlobs": ["npm-check-typings.d.ts"]
-                }
-              ]
-            }
-          }
-        },
-
         "typescript": {
-          "taskDependencies": ["create-src", "copy-src-typings"]
+          "taskDependencies": ["create-src"]
         }
       }
     }

build-tests/rush-lib-declaration-paths-test/scripts/createSrc.js

@@ -26,7 +26,7 @@ module.exports = {
       }
     }
 
-    const indexFileLines = ['/// <reference path="./npm-check-typings.d.ts" />', ''];
+    const indexFileLines = [];
     for await (const dtsPath of collectDtsPaths(`${rushLibPath}/lib`, '@microsoft/rush-lib/lib')) {
       indexFileLines.push(`import '${dtsPath}';`);
     }

common/changes/@microsoft/rush/deduplicate-webclient_2024-12-04-01-17.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Remove the `node-fetch` dependency from @rushstack/rush-amazon-s3-build-cache-plugin.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/changes/@microsoft/rush/deduplicate-webclient_2024-12-04-01-18.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "(BREAKING API CHANGE) Remove the exported `WebClient` API from @rushstack/rush-amazon-s3-build-cache-plugin.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/reviews/api/rush-amazon-s3-build-cache-plugin.api.md

@@ -6,11 +6,11 @@
 
 /// <reference types="node" />
 
-import * as fetch from 'node-fetch';
 import type { IRushPlugin } from '@rushstack/rush-sdk';
 import { ITerminal } from '@rushstack/terminal';
 import type { RushConfiguration } from '@rushstack/rush-sdk';
 import type { RushSession } from '@rushstack/rush-sdk';
+import { WebClient } from '@rushstack/rush-sdk/lib/utilities/WebClient';
 
 // @public
 export class AmazonS3Client {
@@ -61,22 +61,6 @@ export interface IAmazonS3Credentials {
     sessionToken: string | undefined;
 }
 
-// Warning: (ae-forgotten-export) The symbol "IWebFetchOptionsBase" needs to be exported by the entry point index.d.ts
-//
-// @public
-export interface IGetFetchOptions extends IWebFetchOptionsBase {
-    // (undocumented)
-    verb: 'GET' | never;
-}
-
-// @public
-export interface IPutFetchOptions extends IWebFetchOptionsBase {
-    // (undocumented)
-    body?: Buffer;
-    // (undocumented)
-    verb: 'PUT';
-}
-
 // @public (undocumented)
 class RushAmazonS3BuildCachePlugin implements IRushPlugin {
     // (undocumented)
@@ -86,30 +70,6 @@ class RushAmazonS3BuildCachePlugin implements IRushPlugin {
 }
 export default RushAmazonS3BuildCachePlugin;
 
-// @public
-export class WebClient {
-    constructor();
-    // (undocumented)
-    accept: string | undefined;
-    // (undocumented)
-    addBasicAuthHeader(userName: string, password: string): void;
-    // (undocumented)
-    fetchAsync(url: string, options?: IGetFetchOptions | IPutFetchOptions): Promise<WebClientResponse>;
-    // (undocumented)
-    static mergeHeaders(target: fetch.Headers, source: fetch.Headers): void;
-    // Warning: (ae-forgotten-export) The symbol "WebClientProxy" needs to be exported by the entry point index.d.ts
-    //
-    // (undocumented)
-    proxy: WebClientProxy;
-    // (undocumented)
-    readonly standardHeaders: fetch.Headers;
-    // (undocumented)
-    userAgent: string | undefined;
-}
-
-// @public
-export type WebClientResponse = fetch.Response;
-
 // (No @packageDocumentation comment for this package)
 
 ```

libraries/rush-lib/src/utilities/WebClient.ts

@@ -7,25 +7,27 @@ import * as fetch from 'node-fetch';
 import type * as http from 'http';
 import { Import } from '@rushstack/node-core-library';
 
-// ===================================================================================================================
-// AS A TEMPORARY WORKAROUND, THIS FILE WAS COPY+PASTED INTO THE "rush-amazon-s3-build-cache-plugin" PROJECT.
-// See that copy for notes.
-// ===================================================================================================================
-
 const createHttpsProxyAgent: typeof import('https-proxy-agent') = Import.lazy('https-proxy-agent', require);
 
 /**
  * For use with {@link WebClient}.
  */
 export type WebClientResponse = fetch.Response;
 
+/**
+ * For use with {@link WebClient}.
+ */
+export type WebClientHeaders = fetch.Headers;
+// eslint-disable-next-line @typescript-eslint/no-redeclare
+export const WebClientHeaders: typeof fetch.Headers = fetch.Headers;
+
 /**
  * For use with {@link WebClient}.
  */
 export interface IWebFetchOptionsBase {
   timeoutMs?: number;
-  verb?: 'GET' | 'PUT';
-  headers?: fetch.Headers;
+  headers?: WebClientHeaders | Record<string, string>;
+  redirect?: fetch.RequestInit['redirect'];
 }
 
 /**
@@ -38,8 +40,8 @@ export interface IGetFetchOptions extends IWebFetchOptionsBase {
 /**
  * For use with {@link WebClient}.
  */
-export interface IPutFetchOptions extends IWebFetchOptionsBase {
-  verb: 'PUT';
+export interface IFetchOptionsWithBody extends IWebFetchOptionsBase {
+  verb: 'PUT' | 'POST' | 'PATCH';
   body?: Buffer;
 }
 
@@ -56,19 +58,30 @@ export enum WebClientProxy {
  * A helper for issuing HTTP requests.
  */
 export class WebClient {
+  private static _requestFn: typeof fetch.default = fetch.default;
+
   public readonly standardHeaders: fetch.Headers = new fetch.Headers();
 
   public accept: string | undefined = '*/*';
   public userAgent: string | undefined = `rush node/${process.version} ${os.platform()} ${os.arch()}`;
 
   public proxy: WebClientProxy = WebClientProxy.Detect;
 
-  public constructor() {}
+  public static mockRequestFn(fn: typeof fetch.default): void {
+    WebClient._requestFn = fn;
+  }
+
+  public static resetMockRequestFn(): void {
+    WebClient._requestFn = fetch.default;
+  }
 
-  public static mergeHeaders(target: fetch.Headers, source: fetch.Headers): void {
-    source.forEach((value, name) => {
+  public static mergeHeaders(target: fetch.Headers, source: fetch.Headers | Record<string, string>): void {
+    const iterator: Iterable<[string, string]> =
+      'entries' in source && typeof source.entries === 'function' ? source.entries() : Object.entries(source);
+
+    for (const [name, value] of iterator) {
       target.set(name, value);
-    });
+    }
   }
 
   public addBasicAuthHeader(userName: string, password: string): void {
@@ -80,7 +93,7 @@ export class WebClient {
 
   public async fetchAsync(
     url: string,
-    options?: IGetFetchOptions | IPutFetchOptions
+    options?: IGetFetchOptions | IFetchOptionsWithBody
   ): Promise<WebClientResponse> {
     const headers: fetch.Headers = new fetch.Headers();
 
@@ -93,6 +106,7 @@ export class WebClient {
     if (this.userAgent) {
       headers.set('user-agent', this.userAgent);
     }
+
     if (this.accept) {
       headers.set('accept', this.accept);
     }
@@ -126,13 +140,14 @@ export class WebClient {
       method: options?.verb,
       headers: headers,
       agent: agent,
-      timeout: timeoutMs
+      timeout: timeoutMs,
+      redirect: options?.redirect
     };
-    const putOptions: IPutFetchOptions | undefined = options as IPutFetchOptions | undefined;
-    if (putOptions?.body) {
-      requestInit.body = putOptions.body;
+    const optionsWithBody: IFetchOptionsWithBody | undefined = options as IFetchOptionsWithBody | undefined;
+    if (optionsWithBody?.body) {
+      requestInit.body = optionsWithBody.body;
     }
 
-    return await fetch.default(url, requestInit);
+    return await WebClient._requestFn(url, requestInit);
   }
 }