Integrate Asset-Sync Features (#23405)

* updating create recording request

* first draft. need to integrate node tests and then reach out to harsha

* commiting prettier update

* repair ascension algo

* add initial assets.json. update gitignore.

* move textanalytics recordings. time to try the node tests!

* remaining recorder updates. properly setting the assets.json path now

* using a different version of the proxy to ensure that we include all the bugfixes from recently

* bunch of changes for RECORDING_ASSETS_PATH

* lint applied

* Update sdk/test-utils/recorder/src/utils/utils.ts

Co-authored-by: Timo van Veenendaal <timov@microsoft.com>

* Update sdk/test-utils/recorder/src/utils/createRecordingRequest.ts

Co-authored-by: Timo van Veenendaal <timov@microsoft.com>

* Update sdk/test-utils/recorder/src/utils/relativePathCalculator.browser.ts

Co-authored-by: Timo van Veenendaal <timov@microsoft.com>

* Update sdk/test-utils/recorder/src/utils/utils.ts

Co-authored-by: Timo van Veenendaal <timov@microsoft.com>

* Update sdk/test-utils/recorder/src/utils/utils.ts

* Update sdk/test-utils/recorder/src/utils/createRecordingRequest.ts

Co-authored-by: Timo van Veenendaal <timov@microsoft.com>

* repair imports

* linting commit

* resolve failing node tests

* handle undefined set in environment variable

* commit recordings update now that the source has been updated

* fix pipeline

* if (!fs.existsSync(assetsPath)) return undefined;

* format

* fix lint

* Some refactors; calculate assets path in browser using existing environment variable

* Re-add second environment variable

* undo assets changes

Co-authored-by: Timo van Veenendaal <timov@microsoft.com>
Co-authored-by: Harsha Nalluru <sanallur@microsoft.com>

.gitignore

@@ -93,7 +93,6 @@ TestResults/*
 .vscode/*
 ./**/.vscode/*
 
-
 # Node #
 **/node_modules/
 **/cjs/
@@ -155,6 +154,9 @@ sdk/cosmosdb/cosmos/lib
 *lintReport.html
 tsdoc-metadata.json
 
+# locally cloned assets
+.assets
+
 # autorest generated files
 swagger/*.json
 

sdk/test-utils/recorder/karma.conf.js

@@ -1,9 +1,10 @@
 // https://github.com/karma-runner/karma-chrome-launcher
-const { relativeRecordingsPath } = require("./dist/index.js");
+const { relativeRecordingsPath, relativeAssetsPath } = require("./dist/index.js");
 process.env.CHROME_BIN = require("puppeteer").executablePath();
 require("dotenv").config({ path: "../.env" });
 
 process.env.RECORDINGS_RELATIVE_PATH = relativeRecordingsPath();
+process.env.RECORDING_ASSETS_PATH = relativeAssetsPath();
 
 module.exports = function (config) {
   config.set({
@@ -49,7 +50,7 @@ module.exports = function (config) {
     // inject following environment values into browser testing with window.__env__
     // environment values MUST be exported or set with same console running "karma start"
     // https://www.npmjs.com/package/karma-env-preprocessor
-    envPreprocessor: ["RECORDINGS_RELATIVE_PATH", "PROXY_MANUAL_START"],
+    envPreprocessor: ["RECORDINGS_RELATIVE_PATH", "PROXY_MANUAL_START", "RECORDING_ASSETS_PATH"],
 
     // test results reporter to use
     // possible values: 'dots', 'progress'

sdk/test-utils/recorder/src/index.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT license.
 
 export { Recorder } from "./recorder";
-export { relativeRecordingsPath } from "./utils/relativePathCalculator";
+export { relativeRecordingsPath, relativeAssetsPath } from "./utils/relativePathCalculator";
 export {
   SanitizerOptions,
   RecorderStartOptions,

sdk/test-utils/recorder/src/recorder.ts

@@ -42,6 +42,7 @@ import { setRecordingOptions } from "./options";
 import { isNode } from "@azure/core-util";
 import { env } from "./utils/env";
 import { decodeBase64 } from "./utils/encoding";
+import { relativeAssetsPath } from "./utils/relativePathCalculator";
 
 /**
  * This client manages the recorder life cycle and interacts with the proxy-tool to do the recording,
@@ -60,6 +61,7 @@ export class Recorder {
   private stateManager = new RecordingStateManager();
   private httpClient?: HttpClient;
   private sessionFile?: string;
+  private assetsJson?: string;
   private variables: Record<string, string>;
 
   constructor(private testContext?: Test | undefined) {
@@ -67,6 +69,8 @@ export class Recorder {
     if (isRecordMode() || isPlaybackMode()) {
       if (this.testContext) {
         this.sessionFile = sessionFilePath(this.testContext);
+        this.assetsJson = relativeAssetsPath();
+
         logger.info(`[Recorder#constructor] Using a session file located at ${this.sessionFile}`);
         this.httpClient = createDefaultHttpClient();
       } else {
@@ -214,7 +218,14 @@ export class Recorder {
       const startUri = `${Recorder.url}${isPlaybackMode() ? paths.playback : paths.record}${
         paths.start
       }`;
-      const req = createRecordingRequest(startUri, this.sessionFile, this.recordingId);
+
+      const req = createRecordingRequest(
+        startUri,
+        this.sessionFile,
+        this.recordingId,
+        "POST",
+        this.assetsJson
+      );
 
       if (ensureExistence(this.httpClient, "TestProxyHttpClient.httpClient")) {
         logger.verbose("[Recorder#start] Setting redirect mode");
@@ -271,6 +282,7 @@ export class Recorder {
       const stopUri = `${Recorder.url}${isPlaybackMode() ? paths.playback : paths.record}${
         paths.stop
       }`;
+
       const req = createRecordingRequest(stopUri, undefined, this.recordingId);
       req.headers.set("x-recording-save", "true");
 

sdk/test-utils/recorder/src/utils/createRecordingRequest.ts

@@ -11,16 +11,26 @@ export function createRecordingRequest(
   url: string,
   sessionFile?: string,
   recordingId?: string,
-  method: HttpMethods = "POST"
+  method: HttpMethods = "POST",
+  assetsJson?: string
 ) {
   const req = createPipelineRequest({ url: url, method });
 
   if (sessionFile !== undefined) {
-    req.body = JSON.stringify({ "x-recording-file": sessionFile });
+    const body: Record<string, string> = { "x-recording-file": sessionFile };
+
+    // during browser tests the non-presence of an assets.json will result in the value "undefined" being set
+    // its easier to just explicitly handle this case rather than ensure that folks update their karma conf properly.
+    if (assetsJson && assetsJson !== "undefined") {
+      body["x-recording-assets-file"] = assetsJson;
+    }
+
+    req.body = JSON.stringify(body);
   }
 
   if (recordingId !== undefined) {
     req.headers.set("x-recording-id", recordingId);
   }
+
   return req;
 }

sdk/test-utils/recorder/src/utils/relativePathCalculator.browser.ts

@@ -1,6 +1,42 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-export function relativeRecordingsPath(): never {
+import { env } from "./env";
-  throw new Error("Attempted to use the function meant for node in a browser.");
+import { RecorderError } from "./utils";
+
+export function relativeRecordingsPath(): string {
+  if (env.RECORDINGS_RELATIVE_PATH) {
+    return env.RECORDINGS_RELATIVE_PATH;
+  } else {
+    throw new RecorderError(
+      "RECORDINGS_RELATIVE_PATH was not set while in browser mode. Ensure that process.env.RELATIVE_RECORDINGS_PATH has been set properly in your Karma configuration."
+    );
+  }
+}
+
+/**
+ * ONLY WORKS IN THE NODE.JS ENVIRONMENT
+ *
+ * Returns the potential assets.json for the project using `process.cwd()`.
+ *
+ * Note for browser tests:
+ *    1. Supposed to be called from karma.conf.js in the package for which the testing is being done.
+ *    2. Set this `RECORDING_ASSETS_PATH` as an env variable
+ *      ```js
+ *        const { relativeRecordingsPathForBrowser } = require("@azure-tools/test-recorder-new");
+ *        process.env.RECORDING_ASSETS_PATH = relativeRecordingsPathForBrowser();
+ *      ```
+ *    3. Add "RECORDING_ASSETS_PATH" in the `envPreprocessor` array to let this be loaded in the browser environment.
+ *      ```
+ *        envPreprocessor: ["RECORDING_ASSETS_PATH"],
+ *      ```
+ *
+ * `RECORDING_ASSETS_PATH` in the browser environment is used in the recorder to tell the proxy-tool about whether or not to pass additional body argument
+ * `x-recording-assets-file` to playback|record/Start. Doing so enables the proxy to auto-restore files from a remote location.
+ *
+ * @export
+ * @returns {string} location of the relative path to discovered assets.json - `sdk/storage/storage-blob/assets.json` for example.
+ */
+export function relativeAssetsPath(): string | undefined {
+  return env.RECORDING_ASSETS_PATH;
 }

sdk/test-utils/recorder/src/utils/relativePathCalculator.ts

@@ -6,28 +6,19 @@ import fs from "fs";
 import { RecorderError } from "./utils";
 
 /**
- * ONLY WORKS IN THE NODE.JS ENVIRONMENT
+ * Replace backslashes in a path with forward slashes so they are not treated as escape characters
- *
+ * in the browser tests.
- * Returns the potential `recordings` folder(relative path) for the project using `process.cwd()`.
+ * @param filePath The path to replace
- *
+ * @returns A path without backslashes
- * Note for browser tests:
- *    1. Supposed to be called from karma.conf.js in the package for which the testing is being done.
- *    2. Set this `RECORDINGS_RELATIVE_PATH` as an env variable
- *      ```js
- *        const { relativeRecordingsPathForBrowser } = require("@azure-tools/test-recorder-new");
- *        process.env.RECORDINGS_RELATIVE_PATH = relativeRecordingsPathForBrowser();
- *      ```
- *    3. Add "RECORDINGS_RELATIVE_PATH" in the `envPreprocessor` array to let this be loaded in the browser environment.
- *      ```
- *        envPreprocessor: ["RECORDINGS_RELATIVE_PATH"],
- *      ```
- *
- * `RECORDINGS_RELATIVE_PATH` in the browser environment is used in the recorder to tell the proxy-tool about the location to generate the browser recordings at.
- *
- * @export
- * @returns {string} location of the relative `recordings` folder path - `sdk/storage/storage-blob/recordings/` example
  */
-export function relativeRecordingsPath(): string {
+function toSafePath(filePath: string): string {
+  return filePath.split(path.sep).join(path.posix.sep);
+}
+
+/**
+ * Determines the path of the package being tested relative to the repository root.
+ */
+function relativePackagePath() {
   const currentPath = process.cwd(); // Gives the current working directory
 
   let rootPath = undefined;
@@ -53,14 +44,61 @@ export function relativeRecordingsPath(): string {
   if (!(rootPath === undefined || expectedProjectPath === undefined)) {
     // <root>/
     // <root>/sdk/service/project/
-    return path
+    // => sdk/service/project
-      .join(path.relative(rootPath, expectedProjectPath), "recordings")
+    return path.relative(rootPath, expectedProjectPath);
-      .split(path.sep)
-      .join(path.posix.sep); // Converting "\" to "/" (needed for windows) so that the path.sep("\") is not treated as an escape character in the browsers
-    // => sdk/service/project/recordings
   } else {
     throw new RecorderError(
       "rootPath or expectedProjectPath could not be calculated properly from process.cwd()"
     );
   }
 }
+
+/**
+ * Returns the potential `recordings` folder(relative path) for the project using `process.cwd()`.
+ *
+ * Note for browser tests:
+ *    1. Supposed to be called from karma.conf.js in the package for which the testing is being done.
+ *    2. Set this `RECORDINGS_RELATIVE_PATH` as an env variable
+ *      ```js
+ *        const { relativeRecordingsPathForBrowser } = require("@azure-tools/test-recorder-new");
+ *        process.env.RECORDINGS_RELATIVE_PATH = relativeRecordingsPathForBrowser();
+ *      ```
+ *    3. Add "RECORDINGS_RELATIVE_PATH" in the `envPreprocessor` array to let this be loaded in the browser environment.
+ *      ```
+ *        envPreprocessor: ["RECORDINGS_RELATIVE_PATH"],
+ *      ```
+ *
+ * `RECORDINGS_RELATIVE_PATH` in the browser environment is used in the recorder to tell the proxy-tool about the location to generate the browser recordings at.
+ *
+ * @export
+ * @returns {string} location of the relative `recordings` folder path - `sdk/storage/storage-blob/recordings/` example
+ */
+export function relativeRecordingsPath(): string {
+  return toSafePath(path.join(relativePackagePath(), "recordings"));
+}
+
+/**
+ * Returns the potential assets.json for the project using `process.cwd()`.
+ *
+ * Note for browser tests:
+ *    1. Supposed to be called from karma.conf.js in the package for which the testing is being done.
+ *    2. Set this `RECORDING_ASSETS_PATH` as an env variable
+ *      ```js
+ *        const { relativeRecordingsPathForBrowser } = require("@azure-tools/test-recorder-new");
+ *        process.env.RECORDING_ASSETS_PATH = relativeRecordingsPathForBrowser();
+ *      ```
+ *    3. Add "RECORDING_ASSETS_PATH" in the `envPreprocessor` array to let this be loaded in the browser environment.
+ *      ```
+ *        envPreprocessor: ["RECORDING_ASSETS_PATH"],
+ *      ```
+ *
+ * `RECORDING_ASSETS_PATH` in the browser environment is used in the recorder to tell the proxy-tool about whether or not to pass additional body argument
+ * `x-recording-assets-file` to playback|record/Start. Doing so enables the proxy to auto-restore files from a remote location.
+ *
+ * @export
+ * @returns {string} location of the relative path to discovered assets.json - `sdk/storage/storage-blob/assets.json` for example, or undefined if the path does not exist
+ */
+export function relativeAssetsPath(): string | undefined {
+  const assetsJsonPath = path.join(relativePackagePath(), "assets.json");
+  return fs.existsSync(assetsJsonPath) ? toSafePath(assetsJsonPath) : undefined;
+}

sdk/test-utils/recorder/src/utils/sessionFilePath.ts

@@ -2,25 +2,13 @@
 // Licensed under the MIT license.
 
 import { isNode } from "@azure/core-util";
-import { env } from "./env";
 import { generateTestRecordingFilePath } from "./filePathGenerator";
 import { relativeRecordingsPath } from "./relativePathCalculator";
 import { RecorderError } from "./utils";
 
 export function sessionFilePath(testContext: Mocha.Test): string {
-  let recordingsFolder: string;
-  if (isNode) {
-    recordingsFolder = relativeRecordingsPath(); // sdk/service/project/recordings
-  } else if (env.RECORDINGS_RELATIVE_PATH) {
-    recordingsFolder = env.RECORDINGS_RELATIVE_PATH;
-  } else {
-    throw new RecorderError(
-      "RECORDINGS_RELATIVE_PATH was not set while in browser mode. Ensure that process.env.RELATIVE_RECORDINGS_PATH has been set properly in your Karma configuration."
-    );
-  }
-
-  return `${recordingsFolder}/${recordingFilePath(testContext)}`;
   // sdk/service/project/recordings/{node|browsers}/<describe-block-title>/recording_<test-title>.json
+  return `${relativeRecordingsPath()}/${recordingFilePath(testContext)}`;
 }
 
 /**

sdk/test-utils/recorder/src/utils/utils.ts

@@ -1,6 +1,5 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
-
 import { env } from "./env";
 /**
  * A custom error type for failed pipeline requests.