BUG FIXED: mismatch of workspace and error message printing twice. (#31126)

### Packages impacted by this PR
`
@azure/microsoft-playwright-testing`

### Issues associated with this PR
1. Error message printing twice one for scalable and one for reporting
2. When using PLAYWRIGHT_SERVICE_URL and PLAYWRIGHT_SERVICE_ACCESS_TOKEN
of different workspace, scalable fails as expected but reporting works

### Describe the problem that is addressed by this PR


### What are the possible designs available to address the problem? If
there are more than one possible design, why was the one in this PR
chosen?


### Are there test cases added in this PR? _(If not, why?)_


### Provide a list of related PRs _(if any)_


### Command used to generate this PR:**_(Applicable only to SDK release
request PRs)_

### Checklists
- [ ] Added impacted package name to the issue description
- [ ] Does this PR needs any fixes in the SDK Generator?** _(If so,
create an Issue in the
[Autorest/typescript](https://github.com/Azure/autorest.typescript)
repository and link it here)_
- [ ] Added a changelog (if necessary)

sdk/playwrighttesting/microsoft-playwright-testing/src/common/constants.ts

@@ -100,7 +100,6 @@ export class Constants {
   public static readonly patchTestRunShardEnd: string = "patchTestRunShardEnd";
   public static readonly postTestResults: string = "postTestResults";
   public static readonly getStorageUri: string = "getStorageUri";
-
   public static readonly ERROR_MESSAGE: ApiErrorMessage = {
     patchTestRun: {
       400: "The request made to the server is invalid. Please check the request parameters and try again.",
@@ -165,7 +164,7 @@ export const TestErrorType = {
 
 export const TestResultErrorConstants = [
   {
-    key: "Unauthorized_Scalable",
+    key: "401",
     message: "The authentication token provided is invalid. Please check the token and try again.",
     pattern: /(?=.*browserType\.connect)(?=.*401 Unauthorized)/i,
     type: TestErrorType.Scalable,
@@ -189,7 +188,7 @@ export const TestResultErrorConstants = [
     type: TestErrorType.Scalable,
   },
   {
-    key: "InvalidAccessToken_Scalable",
+    key: "InvalidAccessToken",
     message:
       "The provided access token does not match the specified workspace URL. Please verify that both values are correct.",
     pattern: /(?=.*browserType\.connect)(?=.*403 Forbidden)(?=[\s\S]*InvalidAccessToken)/i,
@@ -211,13 +210,13 @@ export const TestResultErrorConstants = [
     type: TestErrorType.Scalable,
   },
   {
-    key: "ServiceUnavailable_Scalable",
+    key: "503",
     message: "The service is currently unavailable. Please check the service status and try again.",
     pattern: /(?=.*browserType\.connect)(?=.*503 Service Unavailable)/i,
     type: TestErrorType.Scalable,
   },
   {
-    key: "GatewayTimeout_Scalable",
+    key: "504",
     message: "The request to the service timed out. Please try again later.",
     pattern: /(?=.*browserType\.connect)(?=.*504 Gateway Timeout)/i,
     type: TestErrorType.Scalable,

sdk/playwrighttesting/microsoft-playwright-testing/src/common/entraIdAccessToken.ts

@@ -46,7 +46,7 @@ class EntraIdAccessToken {
     } catch (err) {
       coreLogger.error(err);
       process.env[InternalEnvironmentVariables.MPT_SETUP_FATAL_ERROR] = "true";
-      throw new Error(ServiceErrorMessageConstants.NO_AUTH_ERROR);
+      throw new Error(ServiceErrorMessageConstants.NO_AUTH_ERROR.message);
     }
   };
 
@@ -79,6 +79,7 @@ class EntraIdAccessToken {
       const expiry = new Date(claims.exp! * 1000);
       this.token = token;
       this._expiryTimestamp = expiry.getTime();
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
     } catch (_) {
       return;
     }

sdk/playwrighttesting/microsoft-playwright-testing/src/common/executor.ts

@@ -60,7 +60,9 @@ const requireOrImportDefaultFunction = async (file: string): Promise<any> => {
   }
   if (typeof func !== "function") {
     // match playwright's error style
-    const error = new Error(`${fileName}: ${ServiceErrorMessageConstants.INVALID_GLOBAL_FUNCTION}`);
+    const error = new Error(
+      `${fileName}: ${ServiceErrorMessageConstants.INVALID_GLOBAL_FUNCTION.message}`,
+    );
     error.stack = "";
     throw error;
   }

sdk/playwrighttesting/microsoft-playwright-testing/src/common/messages.ts

@@ -2,14 +2,36 @@
 // Licensed under the MIT License.
 
 export const ServiceErrorMessageConstants = {
-  NO_AUTH_ERROR:
-    "Could not authenticate with the service. Please refer to https://aka.ms/mpt/authentication for more information.", // no mpt pat set and could not fetch entra token
-  NO_SERVICE_URL_ERROR:
-    "The value for the PLAYWRIGHT_SERVICE_URL variable is not set correctly. Please verify the URL and try again.",
-  INVALID_MPT_PAT_ERROR:
-    "The authentication token provided is invalid. Check the token and try again.",
-  EXPIRED_MPT_PAT_ERROR: "Your authentication token has expired. Create a new token.",
-  INVALID_GLOBAL_FUNCTION: "file must export a single function",
-  INVALID_PLAYWRIGHT_VERSION_ERROR:
-    "The Playwright version you are using is not supported. See the list of supported versions at https://aka.ms/mpt/supported-versions.",
+  NO_SERVICE_URL_ERROR: {
+    key: "NoServiceUrlError",
+    message:
+      "The value for the PLAYWRIGHT_SERVICE_URL variable is not set correctly. Please verify the URL and try again.",
+  },
+  INVALID_GLOBAL_FUNCTION: {
+    key: "InvalidGlobalFunction",
+    message: "File must export a single function.",
+  },
+  INVALID_PLAYWRIGHT_VERSION_ERROR: {
+    key: "InvalidPlaywrightVersionError",
+    message:
+      "The Playwright version you are using is not supported. See the list of supported versions at https://aka.ms/mpt/supported-versions.",
+  },
+  WORKSPACE_MISMATCH_ERROR: {
+    key: "InvalidAccessToken",
+    message:
+      "The provided access token does not match the specified workspace URL. Please verify that both values are correct.",
+  },
+  NO_AUTH_ERROR: {
+    key: "NoAuthError",
+    message:
+      "Could not authenticate with the service. Please refer to https://aka.ms/mpt/authentication for more information.",
+  },
+  INVALID_MPT_PAT_ERROR: {
+    key: "InvalidMptPatError",
+    message: "The authentication token provided is invalid. Check the token and try again.",
+  },
+  EXPIRED_MPT_PAT_ERROR: {
+    key: "ExpiredMptPatError",
+    message: "Your authentication token has expired. Create a new token.",
+  },
 };

sdk/playwrighttesting/microsoft-playwright-testing/src/common/types.ts

@@ -6,6 +6,7 @@ import { ServiceAuth, ServiceOS } from "./constants";
 import type { TokenCredential } from "@azure/identity";
 
 export type JwtPayload = {
+  aid?: string;
   iss?: string;
   sub?: string;
   aud?: string[] | string;

sdk/playwrighttesting/microsoft-playwright-testing/src/core/playwrightService.ts

@@ -19,6 +19,7 @@ import {
   validateMptPAT,
   validatePlaywrightVersion,
   validateServiceUrl,
+  exitWithFailureMessage,
 } from "../utils/utils";
 
 /**
@@ -74,7 +75,7 @@ const getServiceConfig = (
   const globalFunctions: any = {};
   if (options?.serviceAuthType === ServiceAuth.ACCESS_TOKEN) {
     // mpt PAT requested and set by the customer, no need to setup entra lifecycle handlers
-    validateMptPAT();
+    validateMptPAT(exitWithFailureMessage);
   } else {
     globalFunctions.globalSetup = require.resolve("./global/playwright-service-global-setup");
     globalFunctions.globalTeardown = require.resolve("./global/playwright-service-global-teardown");

sdk/playwrighttesting/microsoft-playwright-testing/src/reporter/mptReporter.ts

@@ -29,6 +29,7 @@ import { ServiceClient } from "../utils/serviceClient";
 import { StorageClient } from "../utils/storageClient";
 import { MPTReporterConfig } from "../common/types";
 import { ServiceErrorMessageConstants } from "../common/messages";
+import { validateMptPAT, populateValuesFromServiceUrl } from "../utils/utils";
 
 /**
  * @public
@@ -100,7 +101,13 @@ class MPTReporter implements Reporter {
   private _isInformationMessagePresent = (key: string): boolean => {
     return this.processedErrorMessageKeys.includes(key);
   };
-
+  private _reporterFailureHandler = (error: { key: string; message: string }): void => {
+    if (!this._isInformationMessagePresent(error.key)) {
+      this._addKeyToInformationMessage(error.key);
+      this._addInformationalMessage(error.message);
+    }
+    this.isTokenValid = false;
+  };
   /**
    * @public
    *
@@ -359,10 +366,10 @@ class MPTReporter implements Reporter {
     }
     reporterLogger.info(`Reporting url - ${process.env["PLAYWRIGHT_SERVICE_REPORTING_URL"]}`);
     if (this.envVariables.accessToken === undefined || this.envVariables.accessToken === "") {
-      process.stdout.write(`\n${ServiceErrorMessageConstants.NO_AUTH_ERROR}`);
+      process.stdout.write(`\n${ServiceErrorMessageConstants.NO_AUTH_ERROR.message}`);
       this.isTokenValid = false;
     } else if (ReporterUtils.hasAudienceClaim(this.envVariables.accessToken)) {
-      const result = ReporterUtils.populateValuesFromServiceUrl();
+      const result = populateValuesFromServiceUrl();
       this.envVariables.region = result!.region;
       this.envVariables.accountId = result!.accountId;
       const entraTokenDetails: EntraTokenDetails = ReporterUtils.getTokenDetails<EntraTokenDetails>(
@@ -376,6 +383,7 @@ class MPTReporter implements Reporter {
         this.envVariables.accessToken,
         TokenType.MPT,
       );
+      validateMptPAT(this._reporterFailureHandler);
       this.envVariables.accountId = mptTokenDetails.aid;
       this.envVariables.userId = mptTokenDetails.oid;
       this.envVariables.userName = mptTokenDetails.userName;

sdk/playwrighttesting/microsoft-playwright-testing/src/utils/reporterUtils.ts

@@ -354,7 +354,6 @@ class ReporterUtils {
       return 0;
     }
   }
-
   public redactAccessToken(info: string | undefined): string {
     if (!info || ReporterUtils.isNullOrEmpty(this.envVariables.accessToken)) {
       return "";
@@ -362,27 +361,6 @@ class ReporterUtils {
     const accessTokenRegex = new RegExp(this.envVariables.accessToken, "g");
     return info.replace(accessTokenRegex, Constants.DEFAULT_REDACTED_MESSAGE);
   }
-
-  public static populateValuesFromServiceUrl(): {
-    region: string;
-    accountId: string;
-  } | null {
-    // Service URL format: wss://<region>.api.playwright.microsoft.com/accounts/<workspace-id>/browsers
-    const url = process.env["PLAYWRIGHT_SERVICE_URL"]!;
-    if (!ReporterUtils.isNullOrEmpty(url)) {
-      const parts = url.split("/");
-
-      if (parts.length > 2) {
-        const subdomainParts = parts[2]!.split(".");
-        const region = subdomainParts.length > 0 ? subdomainParts[0] : null;
-        const accountId = parts[4];
-
-        return { region: region!, accountId: accountId! };
-      }
-    }
-    return null;
-  }
-
   public static getRegionFromAccountID(accountId: string): string | undefined {
     if (accountId.includes("_")) {
       return accountId.split("_")[0]!;

sdk/playwrighttesting/microsoft-playwright-testing/src/utils/utils.ts

@@ -17,18 +17,34 @@ import { CIInfoProvider } from "./cIInfoProvider";
 import { getPackageManager } from "./packageManager";
 import { execSync } from "child_process";
 
-export const exitWithFailureMessage = (message: string): never => {
+export const exitWithFailureMessage = (error: { key: string; message: string }): never => {
   console.log();
-  console.error(message);
+  console.error(error.message);
+  // eslint-disable-next-line n/no-process-exit
   process.exit(1);
 };
-
 export const base64UrlDecode = (base64Url: string): string => {
   const base64 = base64Url.replace(/-/g, "+").replace(/_/g, "/");
   const buffer = Buffer.from(base64, "base64");
   return buffer.toString("utf-8");
 };
 
+export const populateValuesFromServiceUrl = (): { region: string; accountId: string } | null => {
+  // Service URL format: wss://<region>.api.playwright.microsoft.com/accounts/<workspace-id>/browsers
+  const url = process.env["PLAYWRIGHT_SERVICE_URL"]!;
+  if (!ReporterUtils.isNullOrEmpty(url)) {
+    const parts = url.split("/");
+
+    if (parts.length > 2) {
+      const subdomainParts = parts[2]!.split(".");
+      const region = subdomainParts.length > 0 ? subdomainParts[0] : null;
+      const accountId = parts[4];
+
+      return { region: region!, accountId: accountId! };
+    }
+  }
+  return null;
+};
 export const parseJwt = <T = JwtPayload>(token: string): T => {
   const parts = token.split(".");
   if (parts.length !== 3) {
@@ -63,18 +79,24 @@ export const validateServiceUrl = (): void => {
   }
 };
 
-export const validateMptPAT = (): void => {
+export const validateMptPAT = (
+  validationFailureCallback: (error: { key: string; message: string }) => void,
+): void => {
   try {
     const accessToken = getAccessToken();
+    const result = populateValuesFromServiceUrl();
     if (!accessToken) {
-      exitWithFailureMessage(ServiceErrorMessageConstants.NO_AUTH_ERROR);
+      validationFailureCallback(ServiceErrorMessageConstants.NO_AUTH_ERROR);
     }
     const claims = parseJwt<JwtPayload>(accessToken!);
     if (!claims.exp) {
-      exitWithFailureMessage(ServiceErrorMessageConstants.INVALID_MPT_PAT_ERROR);
+      validationFailureCallback(ServiceErrorMessageConstants.INVALID_MPT_PAT_ERROR);
     }
     if (Date.now() >= claims.exp! * 1000) {
-      exitWithFailureMessage(ServiceErrorMessageConstants.EXPIRED_MPT_PAT_ERROR);
+      validationFailureCallback(ServiceErrorMessageConstants.EXPIRED_MPT_PAT_ERROR);
+    }
+    if (result!.accountId !== claims!.aid) {
+      validationFailureCallback(ServiceErrorMessageConstants.WORKSPACE_MISMATCH_ERROR);
     }
   } catch (err) {
     coreLogger.error(err);
@@ -88,7 +110,7 @@ export const fetchOrValidateAccessToken = async (credential?: TokenCredential):
     await entraIdAccessToken.fetchEntraIdAccessToken();
   }
   if (!getAccessToken()) {
-    throw new Error(ServiceErrorMessageConstants.NO_AUTH_ERROR);
+    throw new Error(ServiceErrorMessageConstants.NO_AUTH_ERROR.message);
   }
   return getAccessToken()!;
 };

sdk/playwrighttesting/microsoft-playwright-testing/test/core/playwrightService.spec.ts

@@ -44,8 +44,8 @@ describe("getServiceConfig", () => {
     const consoleErrorSpy = sandbox.stub(console, "error");
     sandbox.stub(process, "exit").throws(new Error());
     expect(() => getServiceConfig(samplePlaywrightConfigInput)).to.throw();
-    expect(consoleErrorSpy.calledWith(ServiceErrorMessageConstants.NO_SERVICE_URL_ERROR)).to.be
-      .true;
+    expect(consoleErrorSpy.calledWith(ServiceErrorMessageConstants.NO_SERVICE_URL_ERROR.message)).to
+      .be.true;
   });
 
   it("should set customer config global setup and teardown scripts in the config if passed", () => {
@@ -114,6 +114,7 @@ describe("getServiceConfig", () => {
   });
 
   it("should not set service global setup and teardown for mpt pat authentication if pat is set", () => {
+    const processExitStub = sandbox.stub(process, "exit");
     sandbox.stub(utils, "parseJwt").returns({ exp: Date.now() / 1000 + 10000 });
     const { getServiceConfig } = require("../../src/core/playwrightService");
     process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN] = "token";
@@ -122,6 +123,7 @@ describe("getServiceConfig", () => {
     });
     expect(config.globalSetup).to.be.undefined;
     expect(config.globalTeardown).to.be.undefined;
+    processExitStub.restore();
   });
 
   it("should return service config with service connect options", () => {
@@ -214,7 +216,7 @@ describe("getConnectOptions", () => {
     delete process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN];
     const { getConnectOptions } = require("../../src/core/playwrightService");
     await expect(getConnectOptions()).to.be.rejectedWith(
-      ServiceErrorMessageConstants.NO_AUTH_ERROR,
+      ServiceErrorMessageConstants.NO_AUTH_ERROR.message,
     );
   });
 

sdk/playwrighttesting/microsoft-playwright-testing/test/utils/utils.spec.ts

@@ -18,6 +18,7 @@ import {
   exitWithFailureMessage,
   fetchOrValidateAccessToken,
   emitReportingUrl,
+  populateValuesFromServiceUrl,
 } from "../../src/utils/utils";
 import * as EntraIdAccessTokenModule from "../../src/common/entraIdAccessToken";
 import sinon from "sinon";
@@ -110,7 +111,7 @@ describe("Service Utils", () => {
       throw new Error();
     });
 
-    expect(() => validateMptPAT()).to.throw();
+    expect(() => validateMptPAT(exitWithFailureMessage)).to.throw();
     expect(exitStub.calledWith(1)).to.be.true;
 
     delete process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN];
@@ -122,7 +123,7 @@ describe("Service Utils", () => {
       throw new Error();
     });
 
-    expect(() => validateMptPAT()).to.throw();
+    expect(() => validateMptPAT(exitWithFailureMessage)).to.throw();
     expect(exitStub.calledWith(1)).to.be.true;
 
     delete process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN];
@@ -135,7 +136,7 @@ describe("Service Utils", () => {
       throw new Error();
     });
 
-    expect(() => validateMptPAT()).to.throw();
+    expect(() => validateMptPAT(exitWithFailureMessage)).to.throw();
     expect(exitStub.calledWith(1)).to.be.true;
 
     delete process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN];
@@ -148,21 +149,61 @@ describe("Service Utils", () => {
       throw new Error();
     });
 
-    expect(() => validateMptPAT()).to.throw();
+    expect(() => validateMptPAT(exitWithFailureMessage)).to.throw();
     expect(exitStub.calledWith(1)).to.be.true;
 
     delete process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN];
   });
 
   it("should be no-op if MPT PAT is valid", () => {
+    const processExitStub = sandbox.stub(process, "exit");
     process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN] = "test";
     sandbox.stub(utils, "parseJwt").returns({ exp: Date.now() / 1000 + 10 });
 
-    expect(() => validateMptPAT()).not.to.throw();
-
+    expect(() => validateMptPAT(exitWithFailureMessage)).not.to.throw();
+    processExitStub.restore();
     delete process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN];
   });
 
+  it("Should exit with an error message if the MPT PAT and service URL are from different workspaces", () => {
+    process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN] = "test";
+    sandbox
+      .stub(utils, "parseJwt")
+      .returns({ aid: "eastasia_c24330dd-9249-4ae8-9ba9-b5766060427c" });
+    sandbox
+      .stub(utils, "populateValuesFromServiceUrl")
+      .returns({ region: "", accountId: "eastasia_8bda26b5-300f-4f4f-810d-eae055e4a69b" });
+    const exitStub = sandbox.stub(process, "exit").callsFake(() => {
+      throw new Error();
+    });
+    expect(() => validateMptPAT(exitWithFailureMessage)).to.throw();
+    expect(exitStub.calledWith(1)).to.be.true;
+    delete process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN];
+  });
+
+  it("should be no-op if the MPT PAT and service URL are from same workspaces", () => {
+    const processExitStub = sandbox.stub(process, "exit");
+    process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN] = "test";
+    sandbox
+      .stub(utils, "parseJwt")
+      .returns({ aid: "eastasia_8bda26b5-300f-4f4f-810d-eae055e4a69b" });
+    sandbox
+      .stub(utils, "populateValuesFromServiceUrl")
+      .returns({ region: "", accountId: "eastasia_8bda26b5-300f-4f4f-810d-eae055e4a69b" });
+
+    expect(() => validateMptPAT(exitWithFailureMessage)).not.to.throw();
+    processExitStub.restore();
+    delete process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN];
+  });
+  it("should not exit the process if workspace URL is mismatched", () => {
+    const exitStub = sandbox.stub(process, "exit");
+    process.env["PLAYWRIGHT_SERVICE_URL"] =
+      "wss://eastus.api.playwright.microsoft.com/accounts/wrong-id/browsers";
+    const result = populateValuesFromServiceUrl();
+    expect(result).to.deep.equal({ region: "eastus", accountId: "wrong-id" });
+    expect(exitStub.notCalled).to.be.true;
+    delete process.env["PLAYWRIGHT_SERVICE_URL"];
+  });
   it("should return entra access token (not close to expiry)", async () => {
     const tokenMock = "test";
     process.env[ServiceEnvironmentVariable.PLAYWRIGHT_SERVICE_ACCESS_TOKEN] = tokenMock;
@@ -267,7 +308,7 @@ describe("Service Utils", () => {
     });
     const consoleErrorSpy = sandbox.stub(console, "error");
 
-    exitWithFailureMessage("error message");
+    exitWithFailureMessage({ key: "error", message: "error message" });
 
     expect(exitStub.called).to.be.true;
     expect(consoleErrorSpy.calledWith("error message")).to.be.true;
@@ -414,4 +455,28 @@ describe("Service Utils", () => {
     expect(version).to.equal(mockVersion);
     expect(process.env[InternalEnvironmentVariables.MPT_PLAYWRIGHT_VERSION]).to.equal(mockVersion);
   });
+
+  it("should return region and accountId from a valid service URL", () => {
+    process.env["PLAYWRIGHT_SERVICE_URL"] =
+      "wss://eastus.api.playwright.microsoft.com/accounts/1234/browsers";
+
+    const result = populateValuesFromServiceUrl();
+    expect(result).to.deep.equal({ region: "eastus", accountId: "1234" });
+
+    delete process.env["PLAYWRIGHT_SERVICE_URL"];
+  });
+
+  it("should return null for an invalid service URL", () => {
+    process.env["PLAYWRIGHT_SERVICE_URL"] = "invalid-url";
+
+    const result = populateValuesFromServiceUrl();
+    expect(result).to.be.null;
+
+    delete process.env["PLAYWRIGHT_SERVICE_URL"];
+  });
+
+  it("should return null if PLAYWRIGHT_SERVICE_URL is not set", () => {
+    const result = populateValuesFromServiceUrl();
+    expect(result).to.be.null;
+  });
 });