[Perf Framework] Allow connecting to the proxy-tool with https too

sdk/test-utils/perfstress/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## 1.0.0 (Unreleased)
 
+### 2021-09-29
+
+- Allows connecting to the proxy-tool with https with the "insecure" boolean option.
+  [#17898](https://github.com/Azure/azure-sdk-for-js/pull/17898)
+
+- [Bug Fix] Fixes [#17954](https://github.com/Azure/azure-sdk-for-js/issues/17954), boolean options parsed incorrectly as strings is rectified.
+
 ### 2021-09-24
 
 - Instead of using the cached proxy-clients(to leverage the proxy-tool), we now get a new client for each of the instantiated PerfStressTest classes. [#17832](https://github.com/Azure/azure-sdk-for-js/pull/17832)

sdk/test-utils/perfstress/GettingStarted.md

@@ -287,7 +287,7 @@ To be able to leverage the powers of playing back the requests using the test pr
 
 Run this command
 
-- `docker run -p 5000:5000 azsdkengsys.azurecr.io/engsys/ubuntu_testproxy_server:latest`
+- `docker run -p 5000:5000 -p 5001:5001 azsdkengsys.azurecr.io/engsys/testproxy-lin:latest`
 
 Reference: https://github.com/Azure/azure-sdk-tools/tree/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy#via-docker-image
 
@@ -299,10 +299,14 @@ Sample command(using storage-blob perf tests as example (Core-v1)!)
 
 > npm run perf-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --parallel 2 --test-proxy http://localhost:5000
 
+> npm run perf-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --parallel 2 --test-proxy https://localhost:5001 --insecure true
+
 Sample command(using data-tables perf tests as example (Core-v2)!)
 
 > npm run perf-test:node -- ListComplexEntitiesTest --duration 7 --iterations 2 --parallel 2 --test-proxy http://localhost:5000
 
+> npm run perf-test:node -- ListComplexEntitiesTest --duration 7 --iterations 2 --parallel 2 --test-proxy https://localhost:5001 --insecure true
+
 > npm run perf-test:node -- ListComplexEntitiesTest --duration 7 --iterations 2 --parallel 2
 
 **Using proxy-tool** part is still under construction. Please reach out to the owners/team if you face issues.

sdk/test-utils/perfstress/src/options.ts

@@ -67,6 +67,7 @@ export interface DefaultPerfStressOptions {
   "no-cleanup": boolean;
   "milliseconds-to-log": number;
   "test-proxy": string;
+  insecure: boolean;
 }
 
 /**
@@ -104,6 +105,12 @@ export const defaultPerfStressOptions: PerfStressOptionDictionary<DefaultPerfStr
     description: "URI of TestProxy server",
     defaultValue: undefined
   },
+  insecure: {
+    description:
+      "Applied when test-proxy option is defined, connects with https(insecurely by disabling SSL validation)",
+    shortName: "ins",
+    defaultValue: false
+  },
   "milliseconds-to-log": {
     description: "Log frequency in milliseconds",
     shortName: "mtl",
@@ -121,7 +128,10 @@ export const defaultPerfStressOptions: PerfStressOptionDictionary<DefaultPerfStr
 export function parsePerfStressOption<TOptions>(
   options: PerfStressOptionDictionary<TOptions>
 ): Required<PerfStressOptionDictionary<TOptions>> {
-  const minimistResult: MinimistParsedArgs = minimist(process.argv);
+  const minimistResult: MinimistParsedArgs = minimist(
+    process.argv,
+    getBooleanOptionDetails(options)
+  );
   const result: Partial<PerfStressOptionDictionary<TOptions>> = {};
 
   for (const longName of Object.keys(options)) {
@@ -150,3 +160,19 @@ export function parsePerfStressOption<TOptions>(
 
   return result as Required<PerfStressOptionDictionary<TOptions>>;
 }
+
+function getBooleanOptionDetails<TOptions>(options: PerfStressOptionDictionary<TOptions>) {
+  let booleanProps: { boolean: string[]; default: { [key: string]: boolean } } = {
+    boolean: [],
+    default: {}
+  };
+
+  for (const key in options) {
+    const defaultValue = options[key].defaultValue;
+    if (typeof defaultValue === "boolean") {
+      booleanProps.boolean.push(key);
+      booleanProps.default[key] = defaultValue;
+    }
+  }
+  return booleanProps;
+}

sdk/test-utils/perfstress/src/testProxyHttpClient.ts

@@ -10,7 +10,7 @@ import {
   SendRequest
 } from "@azure/core-rest-pipeline";
 import { RequestOptions } from "http";
-import { makeRequest } from "./utils";
+import { getCachedHttpsAgent, makeRequest } from "./utils";
 
 const paths = {
   playback: "/playback",
@@ -85,9 +85,11 @@ export class TestProxyHttpClient {
   public _recordingId?: string;
   public _mode!: string;
   private stateManager: RecordingStateManager = new RecordingStateManager();
+  public insecure: boolean;
 
-  constructor(uri: string) {
+  constructor(uri: string, insecure: boolean) {
     this._uri = uri;
+    this.insecure = insecure;
   }
   // For core-v1
   redirectRequest(request: WebResourceLike, recordingId: string): WebResourceLike;
@@ -114,7 +116,7 @@ export class TestProxyHttpClient {
   async modifyRequest(request: PipelineRequest): Promise<PipelineRequest> {
     if (this._recordingId && (this._mode === "record" || this._mode === "playback")) {
       request = this.redirectRequest(request, this._recordingId);
-      request.allowInsecureConnection = true;
+      request.allowInsecureConnection = this._uri.startsWith("http:");
     }
 
     return request;
@@ -125,7 +127,7 @@ export class TestProxyHttpClient {
     const options = this._createRecordingRequestOptions({
       path: paths.record + paths.start
     });
-    const rsp = await makeRequest(this._uri, options);
+    const rsp = await makeRequest(this._uri, options, this.insecure);
     if (rsp.statusCode !== 200) {
       throw new Error("Start request failed.");
     }
@@ -152,7 +154,7 @@ export class TestProxyHttpClient {
       ...options.headers,
       "x-recording-id": this._recordingId
     };
-    await makeRequest(this._uri, options);
+    await makeRequest(this._uri, options, this.insecure);
     this.stateManager.setState("stopped-recording");
   }
 
@@ -165,7 +167,7 @@ export class TestProxyHttpClient {
       ...options.headers,
       "x-recording-id": this._recordingId
     };
-    const rsp = await makeRequest(this._uri, options);
+    const rsp = await makeRequest(this._uri, options, this.insecure);
     if (rsp.statusCode !== 200) {
       throw new Error("Start request failed.");
     }
@@ -193,7 +195,7 @@ export class TestProxyHttpClient {
       "x-recording-id": this._recordingId,
       "x-purge-inmemory-recording": "true"
     };
-    await makeRequest(this._uri, options);
+    await makeRequest(this._uri, options, this.insecure);
     this._mode = "live";
     this._recordingId = undefined;
     this.stateManager.setState("stopped-playback");
@@ -210,27 +212,51 @@ export class TestProxyHttpClient {
   }
 }
 
-export function testProxyHttpPolicy(testProxyHttpClient: TestProxyHttpClient): PipelinePolicy {
+export function testProxyHttpPolicy(
+  testProxyHttpClient: TestProxyHttpClient,
+  isHttps: boolean,
+  insecure: boolean
+): PipelinePolicy {
   return {
     name: "recording policy",
     async sendRequest(request: PipelineRequest, next: SendRequest): Promise<PipelineResponse> {
       const modifiedRequest = await testProxyHttpClient.modifyRequest(request);
+      if (isHttps) {
+        modifiedRequest.agent = getCachedHttpsAgent(insecure);
+      }
       return next(modifiedRequest);
     }
   };
 }
 
 export class TestProxyHttpClientV1 extends TestProxyHttpClient {
   public _httpClient: HttpClient;
-  constructor(uri: string) {
-    super(uri);
-    this._httpClient = new DefaultHttpClient();
+  constructor(uri: string, insecure: boolean) {
+    super(uri, insecure);
+    this._httpClient = new DefaultHttpClientCoreV1(uri.startsWith("https"), insecure);
   }
 
   async sendRequest(request: WebResourceLike): Promise<HttpOperationResponse> {
     if (this._recordingId && (this._mode === "record" || this._mode === "playback")) {
       request = this.redirectRequest(request, this._recordingId);
     }
-    return await this._httpClient.sendRequest(request);
+    return this._httpClient.sendRequest(request);
+  }
+}
+
+class DefaultHttpClientCoreV1 extends DefaultHttpClient {
+  constructor(private isHttps: boolean, private insecure: boolean) {
+    super();
+  }
+
+  async prepareRequest(httpRequest: WebResourceLike): Promise<Partial<RequestInit>> {
+    const req: Partial<RequestInit & {
+      agent?: any;
+      compress?: boolean;
+    }> = await super.prepareRequest(httpRequest);
+    if (this.isHttps) {
+      req.agent = getCachedHttpsAgent(this.insecure);
+    }
+    return req;
   }
 }

sdk/test-utils/perfstress/src/tests.ts

@@ -69,7 +69,8 @@ export abstract class PerfStressTest<TOptions = {}> {
   public configureClientOptionsCoreV1<T>(options: T & { httpClient?: HttpClient }): T {
     if (this.parsedOptions["test-proxy"].value) {
       this.testProxyHttpClientV1 = new TestProxyHttpClientV1(
-        this.parsedOptions["test-proxy"].value
+        this.parsedOptions["test-proxy"].value,
+        this.parsedOptions["insecure"].value!
       );
       options.httpClient = this.testProxyHttpClientV1;
     }
@@ -85,9 +86,19 @@ export abstract class PerfStressTest<TOptions = {}> {
    * Note: Client must expose the pipeline property which is required for the perf framework to add its policies correctly
    */
   public configureClient<T>(client: T & { pipeline: Pipeline }): T {
-    if (this.parsedOptions["test-proxy"].value) {
-      this.testProxyHttpClient = new TestProxyHttpClient(this.parsedOptions["test-proxy"].value);
-      client.pipeline.addPolicy(testProxyHttpPolicy(this.testProxyHttpClient));
+    const url = this.parsedOptions["test-proxy"].value;
+    if (url) {
+      this.testProxyHttpClient = new TestProxyHttpClient(
+        url,
+        this.parsedOptions["insecure"].value!
+      );
+      client.pipeline.addPolicy(
+        testProxyHttpPolicy(
+          this.testProxyHttpClient,
+          url.startsWith("https"),
+          this.parsedOptions["insecure"].value!
+        )
+      );
     }
     return client;
   }

sdk/test-utils/perfstress/src/utils.ts

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
-import { IncomingMessage, RequestOptions, request } from "http";
+import { IncomingMessage, RequestOptions } from "http";
+import https from "https";
+import http from "http";
 
 /**
  * Returns the environment variable, throws an error if not defined.
@@ -16,6 +18,27 @@ export function getEnvVar(name: string) {
   return val;
 }
 
+let cachedHttpsAgent: https.Agent;
+/**
+ * Returns https Agent to allow connecting to the proxy tool with "https" protocol.
+ *
+ * @export
+ * @param {string} name
+ */
+export const getCachedHttpsAgent = (insecure: boolean) => {
+  if (!cachedHttpsAgent) {
+    cachedHttpsAgent = new https.Agent({
+      rejectUnauthorized: !insecure
+      // TODO: Doesn't work currently
+      // pfx: require("fs").readFileSync(
+      //   "/workspaces/azure-sdk-for-js/eng/common/testproxy/dotnet-devcert.pfx"
+      // ),
+      // passphrase: "password"}
+    });
+  }
+  return cachedHttpsAgent;
+};
+
 /**
  * Reads a readable stream. Doesn't save to a buffer.
  *
@@ -31,11 +54,23 @@ export async function drainStream(stream: NodeJS.ReadableStream) {
 }
 export async function makeRequest(
   uri: string,
-  requestOptions: RequestOptions
+  requestOptions: RequestOptions,
+  insecure: boolean
 ): Promise<IncomingMessage> {
   return new Promise<IncomingMessage>((resolve, reject) => {
-    const req = request(uri, requestOptions, resolve);
-
+    let req: http.ClientRequest;
+    if (uri.startsWith("https")) {
+      req = https.request(
+        uri,
+        {
+          ...requestOptions,
+          agent: getCachedHttpsAgent(insecure)
+        },
+        resolve
+      );
+    } else {
+      req = http.request(uri, requestOptions, resolve);
+    }
     req.once("error", reject);
 
     req.end();