[Perf] Add profiling support to the perf framework (#24240)

Fixes https://github.com/Azure/azure-sdk-for-js/issues/14146

Validation:
https://dev.azure.com/azure-sdk/internal/_build/results?buildId=2201082&view=results

## What's in the PR?
Adds two new options to add profiling support to the perf framework
```
  profile = "Set to true to profile the perf test. When set to true, `cpus` will be overriden to 1." (defaults to false)
  profile-filepath = "Used as the artifact path" (optional)
```
Counterpart in the tools repo
https://github.com/Azure/azure-sdk-tools/pull/5369

## Example generated profile
<img width="1125" alt="image"
src="https://user-images.githubusercontent.com/10452642/218958379-4c62386b-530a-4f3f-84bd-49d51d41bfd4.png">

---------

Co-authored-by: Jeff Fisher <jeffish@microsoft.com>
Co-authored-by: Timo van Veenendaal <me@timo.nz>

.gitignore

@@ -168,3 +168,6 @@ code-model-*
 # code workspaces
 *.code-workspace
 !/dataplane.code-workspace
+
+# CPU profiles
+*.cpuprofile

sdk/storage/perf-tests/storage-blob/README.md

@@ -21,3 +21,5 @@
      - `npm run perf-test:node -- CoreHTTPDownloadWithSASTest --warmup 2 --duration 7 --iterations 2 --parallel 2`
    - download using sas with core-rest-pipeline
      - `npm run perf-test:node -- CoreHTTPSDownloadWithSASTest --warmup 2 --duration 7 --iterations 2 --parallel 2`
+   - download test with profiling
+     - `npm run perf-test:node -- StorageBlobDownloadTest --duration 10 --profile --parallel 64` 

sdk/test-utils/perf/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## 1.0.0 (Unreleased)
 
+### 2023-02-16
+
+- [#14146](https://github.com/Azure/azure-sdk-for-js/pull/14146) Adds profiling support to the perf framework. Two new options "profile" and "profile-path" introduced.
+
+  [#24240](https://github.com/Azure/azure-sdk-for-js/pull/24240) 
+
 ### 2023-01-18
 
 - [#24518](https://github.com/Azure/azure-sdk-for-js/issues/24518) Fixes the issue where the `console.logs` that are part of the test are not being propagated as expected from the child process(test instance).

sdk/test-utils/perf/package.json

@@ -66,7 +66,8 @@
     "tslib": "^2.2.0",
     "node-fetch": "^2.6.6",
     "minimist": "~1.2.5",
-    "@types/minimist": "~1.2.0"
+    "@types/minimist": "~1.2.0",
+    "fs-extra": "^10.0.0"
   },
   "devDependencies": {
     "@azure/core-client": "^1.3.1",
@@ -81,6 +82,7 @@
     "prettier": "^2.5.1",
     "rimraf": "^3.0.0",
     "typescript": "~4.8.0",
-    "ts-node": "^8.3.0"
+    "ts-node": "^8.3.0",
+    "@types/fs-extra": "^9.0.0"
   }
 }

sdk/test-utils/perf/src/index.ts

@@ -8,4 +8,4 @@ export * from "./options";
 export * from "./policy";
 export * from "./program";
 export * from "./parallel";
-export { getEnvVar, drainStream } from "./utils";
+export { getEnvVar, drainStream } from "./utils/utils";

sdk/test-utils/perf/src/managerProgram.ts

@@ -11,7 +11,7 @@ import { DefaultPerfOptions, ParsedPerfOptions } from "./options";
 import { Snapshot } from "./snapshot";
 import { PerfTestBase, PerfTestConstructor } from "./perfTestBase";
 import { PerfProgram } from "./program";
-import { formatDuration, formatNumber } from "./utils";
+import { formatDuration, formatNumber } from "./utils/utils";
 
 /**
  * The manager program which is responsible for spawning workers which run the actual perf test.

sdk/test-utils/perf/src/options.ts

@@ -3,6 +3,7 @@
 
 import { default as minimist, ParsedArgs as MinimistParsedArgs } from "minimist";
 import { isDefined } from "@azure/core-util";
+import { getFormattedDate } from "./utils/utils";
 
 /**
  * The structure of a Perf option. They represent command line parameters.
@@ -79,6 +80,8 @@ export interface DefaultPerfOptions {
   "list-transitive-dependencies": boolean;
   cpus: number;
   "use-worker-threads": boolean;
+  profile: boolean;
+  "profile-path": string;
 }
 
 /**
@@ -143,8 +146,46 @@ export const defaultPerfOptions: PerfOptionDictionary<DefaultPerfOptions> = {
       "Set to true to use the Node worker_thread API when running tests across multiple CPUs. Set to false to use child_process (default).",
     defaultValue: false,
   },
+  profile: {
+    description:
+      "Set to true to profile the perf test. When set to true, `cpus` will be overriden to 1.",
+    defaultValue: false,
+  },
+  "profile-path": {
+    description: "Used as the artifact path",
+    defaultValue: `./profile/${getFormattedDate()}-perfProgram.cpuprofile`,
+    // If none provided, profiles get generated at the "/sdk/<service>/perf-tests/<package>/profile/"
+  },
 };
 
+/**
+ * Overrides the "cpus" option to 1, when "profile" is set to true by the user.
+ *
+ * Warns the user when profile is true, and cpus is set to something other than 1.
+ */
+function maybeOverrideCPUsOption<TOptions>(
+  minimistResult: MinimistParsedArgs,
+  result: Partial<PerfOptionDictionary<TOptions>>
+) {
+  if (!isDefined(minimistResult["profile"]) || !minimistResult["profile"]) {
+    return;
+  }
+
+  if (isDefined(minimistResult["cpus"]) && minimistResult["cpus"] !== 1) {
+    throw new Error(
+      `Unexpected value for "cpus" provided, you can only set "cpus = 1" when "profile" is set to true. 
+      Please re-run the test command without the "cpus" option.`
+    );
+  }
+
+  result["cpus" as keyof TOptions] = {
+    ...result["cpus" as keyof TOptions],
+    value: 1,
+    // Overriding to 1 core
+    // since there is no point in observing profiling artifacts of all the cores that do the same thing
+  };
+}
+
 /**
  * Parses the given options by extracting their values through `minimist`, or setting the default value defined in each option.
  * It also overwrites any present longName with the property name of each option.
@@ -183,11 +224,24 @@ export function parsePerfOption<TOptions>(
       throw new Error(`Option ${longName} is required`);
     }
 
-    result[optionName as keyof TOptions] = {
-      ...option,
-      longName,
-      value,
-    };
+    if (
+      ["profile", "cpus"].includes(optionName) &&
+      !isDefined(result["profile" as keyof TOptions]) &&
+      !isDefined(result["cpus" as keyof TOptions])
+    ) {
+      result[optionName as keyof TOptions] = {
+        ...option,
+        longName,
+        value,
+      };
+      maybeOverrideCPUsOption(minimistResult, result);
+    } else {
+      result[optionName as keyof TOptions] = {
+        ...option,
+        longName,
+        value,
+      };
+    }
   }
 
   return result as ParsedPerfOptions<TOptions>;

sdk/test-utils/perf/src/testProxyHttpClient.ts

@@ -11,7 +11,7 @@ import {
 } from "@azure/core-rest-pipeline";
 import { RequestOptions } from "http";
 import { Agent as HttpsAgent } from "https";
-import { getCachedHttpsAgent, makeRequest } from "./utils";
+import { getCachedHttpsAgent, makeRequest } from "./utils/utils";
 
 const paths = {
   playback: "/playback",

sdk/test-utils/perf/src/utils/profiling.ts

@@ -0,0 +1,27 @@
+import { Session } from "node:inspector";
+import * as fs from "fs-extra";
+
+export async function runWithCpuProfile(
+  functionToProfile: () => Promise<void>,
+  profileFilePath: string
+) {
+  const session = new Session();
+  session.connect();
+  session.post("Profiler.enable", () => {
+    session.post("Profiler.start", async () => {
+      // Invoke the logic
+      await functionToProfile();
+      // some time later...
+      session.post("Profiler.stop", (err, { profile }) => {
+        // Write profile to disk, upload, etc.
+        if (!err) {
+          fs.ensureDirSync(profileFilePath.substring(0, profileFilePath.lastIndexOf("/") + 1));
+          fs.writeFileSync(profileFilePath, JSON.stringify(profile));
+          console.log(`...CPUProfile saved to ${profileFilePath}...`);
+        } else {
+          console.log(err);
+        }
+      });
+    });
+  });
+}

sdk/test-utils/perf/src/utils/utils.ts

@@ -127,3 +127,7 @@ export function formatNumber(value: number, minSignificantDigits: number) {
     maximumSignificantDigits: significantDigits,
   });
 }
+
+export function getFormattedDate() {
+  return new Date().toISOString().replace(/[:\-.]/g, "_");
+}

sdk/test-utils/perf/src/workerProgram.ts

@@ -4,6 +4,7 @@ import { multicoreUtils, WorkerData, WorkerMulticoreUtils } from "./multicore";
 import { PerfTestBase, PerfTestConstructor } from "./perfTestBase";
 import { PerfProgram } from "./program";
 import { DefaultPerfOptions, ParsedPerfOptions } from "./options";
+import { runWithCpuProfile } from "./utils/profiling";
 
 export class WorkerPerfProgram implements PerfProgram {
   private testClass: PerfTestConstructor;
@@ -75,15 +76,21 @@ export class WorkerPerfProgram implements PerfProgram {
     await Promise.all(this.tests.map((test) => test.postSetup?.()));
     await exitStage("postSetup");
 
-    if (this.options.warmup.value! > 0) {
+    if (this.options.warmup.value > 0) {
       await enterStage("warmup");
       await this.runTests(this.options.warmup.value!);
       await exitStage("warmup");
     }
 
-    for (let iteration = 0; iteration < this.options.iterations.value!; ++iteration) {
+    for (let iteration = 0; iteration < this.options.iterations.value; ++iteration) {
       await enterStage("test");
-      await this.runTests(this.options.duration.value!);
+      const duration = this.options.duration.value;
+      const testRunner = () => this.runTests(duration);
+      if (this.options.profile.value) {
+        await runWithCpuProfile(testRunner, this.options["profile-path"].value);
+      } else {
+        await testRunner();
+      }
       await exitStage("test");
     }
 

sdk/test-utils/perf/test/serviceClientGet.spec.ts

@@ -4,7 +4,7 @@
 import { createPipelineRequest, PipelineRequest } from "@azure/core-rest-pipeline";
 import { ServiceClient } from "@azure/core-client";
 import { PerfTest, PerfOptionDictionary, drainStream } from "../src";
-import { getCachedHttpsAgent } from "../src/utils";
+import { getCachedHttpsAgent } from "../src/utils/utils";
 
 interface ServiceClientGetOptions {
   "first-run-extra-requests": number;