chore: expose expect error details on TestError (#33183)

docs/src/test-reporter-api/class-testerror.md

@@ -4,18 +4,54 @@
 
 Information about an error thrown during test execution.
 
+## property: TestError.expected
+* since: v1.49
+- type: ?<[string]>
+
+Expected value formatted as a human-readable string.
+
+## property: TestError.locator
+* since: v1.49
+- type: ?<[string]>
+
+Receiver's locator.
+
+## property: TestError.log
+* since: v1.49
+- type: ?<[Array]<[string]>>
+
+Call log.
+
+## property: TestError.matcherName
+* since: v1.49
+- type: ?<[string]>
+
+Expect matcher name.
+
 ## property: TestError.message
 * since: v1.10
 - type: ?<[string]>
 
 Error message. Set when [Error] (or its subclass) has been thrown.
 
+## property: TestError.received
+* since: v1.49
+- type: ?<[string]>
+
+Received value formatted as a human-readable string.
+
 ## property: TestError.stack
 * since: v1.10
 - type: ?<[string]>
 
 Error stack. Set when [Error] (or its subclass) has been thrown.
 
+## property: TestError.timeout
+* since: v1.49
+- type: ?<[int]>
+
+Timeout in milliseconds, if the error was caused by a timeout.
+
 ## property: TestError.value
 * since: v1.10
 - type: ?<[string]>

packages/playwright/src/matchers/matcherHint.ts

@@ -39,6 +39,10 @@ export type MatcherResult<E, A> = {
   actual?: A;
   log?: string[];
   timeout?: number;
+  locator?: string;
+  printedReceived?: string;
+  printedExpected?: string;
+  printedDiff?: string;
 };
 
 export class ExpectError extends Error {

packages/playwright/src/matchers/toBeTruthy.ts

@@ -39,22 +39,41 @@ export async function toBeTruthy(
   };
 
   const timeout = options.timeout ?? this.timeout;
-  const { matches, log, timedOut, received } = await query(!!this.isNot, timeout);
+  const { matches: pass, log, timedOut, received } = await query(!!this.isNot, timeout);
+  if (pass === !this.isNot) {
+    return {
+      name: matcherName,
+      message: () => '',
+      pass,
+      expected
+    };
+  }
+
   const notFound = received === kNoElementsFoundError ? received : undefined;
-  const actual = matches ? expected : unexpected;
+  const actual = pass ? expected : unexpected;
+  let printedReceived: string | undefined;
+  let printedExpected: string | undefined;
+  if (pass) {
+    printedExpected = `Expected: not ${expected}`;
+    printedReceived = `Received: ${notFound ? kNoElementsFoundError : expected}`;
+  } else {
+    printedExpected = `Expected: ${expected}`;
+    printedReceived = `Received: ${notFound ? kNoElementsFoundError : unexpected}`;
+  }
   const message = () => {
     const header = matcherHint(this, receiver, matcherName, 'locator', arg, matcherOptions, timedOut ? timeout : undefined);
     const logText = callLogText(log);
-    return matches ? `${header}Expected: not ${expected}\nReceived: ${notFound ? kNoElementsFoundError : expected}${logText}` :
-      `${header}Expected: ${expected}\nReceived: ${notFound ? kNoElementsFoundError : unexpected}${logText}`;
+    return `${header}${printedExpected}\n${printedReceived}${logText}`;
   };
   return {
     message,
-    pass: matches,
+    pass,
     actual,
     name: matcherName,
     expected,
     log,
     timeout: timedOut ? timeout : undefined,
+    ...(printedReceived ? { printedReceived } : {}),
+    ...(printedExpected ? { printedExpected } : {}),
   };
 }

packages/playwright/src/matchers/toEqual.ts

@@ -44,22 +44,35 @@ export async function toEqual<T>(
   const timeout = options.timeout ?? this.timeout;
 
   const { matches: pass, received, log, timedOut } = await query(!!this.isNot, timeout);
+  if (pass === !this.isNot) {
+    return {
+      name: matcherName,
+      message: () => '',
+      pass,
+      expected
+    };
+  }
 
-  const message = pass
-    ? () =>
-      matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined) +
-      `Expected: not ${this.utils.printExpected(expected)}\n` +
-      `Received: ${this.utils.printReceived(received)}` + callLogText(log)
-    : () =>
-      matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined) +
-      this.utils.printDiffOrStringify(
-          expected,
-          received,
-          EXPECTED_LABEL,
-          RECEIVED_LABEL,
-          false,
-      ) + callLogText(log);
-
+  let printedReceived: string | undefined;
+  let printedExpected: string | undefined;
+  let printedDiff: string | undefined;
+  if (pass) {
+    printedExpected = `Expected: not ${this.utils.printExpected(expected)}`;
+    printedReceived = `Received: ${this.utils.printReceived(received)}`;
+  } else {
+    printedDiff = this.utils.printDiffOrStringify(
+        expected,
+        received,
+        EXPECTED_LABEL,
+        RECEIVED_LABEL,
+        false,
+    );
+  }
+  const message = () => {
+    const header = matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined);
+    const details = printedDiff || `${printedExpected}\n${printedReceived}`;
+    return `${header}${details}${callLogText(log)}`;
+  };
   // Passing the actual and expected objects so that a custom reporter
   // could access them, for example in order to display a custom visual diff,
   // or create a different error message
@@ -70,5 +83,8 @@ export async function toEqual<T>(
     pass,
     log,
     timeout: timedOut ? timeout : undefined,
+    ...(printedReceived ? { printedReceived } : {}),
+    ...(printedExpected ? { printedExpected } : {}),
+    ...(printedDiff ? { printedDiff } : {}),
   };
 }

packages/playwright/src/matchers/toMatchSnapshot.ts

@@ -194,6 +194,10 @@ class SnapshotHelper {
       pass,
       message: () => message,
       log,
+      // eslint-disable-next-line @typescript-eslint/no-base-to-string
+      ...(this.locator ? { locator: this.locator.toString() } : {}),
+      printedExpected: this.expectedPath,
+      printedReceived: this.actualPath,
     };
     return Object.fromEntries(Object.entries(unfiltered).filter(([_, v]) => v !== undefined)) as ImageMatcherResult;
   }

packages/playwright/src/matchers/toMatchText.ts

@@ -58,29 +58,56 @@ export async function toMatchText(
   const timeout = options.timeout ?? this.timeout;
 
   const { matches: pass, received, log, timedOut } = await query(!!this.isNot, timeout);
+  if (pass === !this.isNot) {
+    return {
+      name: matcherName,
+      message: () => '',
+      pass,
+      expected
+    };
+  }
+
   const stringSubstring = options.matchSubstring ? 'substring' : 'string';
   const receivedString = received || '';
   const messagePrefix = matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined);
   const notFound = received === kNoElementsFoundError;
-  const message = () => {
-    if (pass) {
-      if (typeof expected === 'string') {
-        if (notFound)
-          return messagePrefix + `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}\nReceived: ${received}` + callLogText(log);
-        const printedReceived = printReceivedStringContainExpectedSubstring(receivedString, receivedString.indexOf(expected), expected.length);
-        return messagePrefix + `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}\nReceived string: ${printedReceived}` + callLogText(log);
+
+  let printedReceived: string | undefined;
+  let printedExpected: string | undefined;
+  let printedDiff: string | undefined;
+  if (pass) {
+    if (typeof expected === 'string') {
+      if (notFound) {
+        printedExpected = `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}`;
+        printedReceived = `Received: ${received}`;
       } else {
-        if (notFound)
-          return messagePrefix + `Expected pattern: not ${this.utils.printExpected(expected)}\nReceived: ${received}` + callLogText(log);
-        const printedReceived = printReceivedStringContainExpectedResult(receivedString, typeof expected.exec === 'function' ? expected.exec(receivedString) : null);
-        return messagePrefix + `Expected pattern: not ${this.utils.printExpected(expected)}\nReceived string: ${printedReceived}` + callLogText(log);
+        printedExpected = `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}`;
+        const formattedReceived = printReceivedStringContainExpectedSubstring(receivedString, receivedString.indexOf(expected), expected.length);
+        printedReceived = `Received string: ${formattedReceived}`;
       }
     } else {
-      const labelExpected = `Expected ${typeof expected === 'string' ? stringSubstring : 'pattern'}`;
-      if (notFound)
-        return messagePrefix + `${labelExpected}: ${this.utils.printExpected(expected)}\nReceived: ${received}` + callLogText(log);
-      return messagePrefix + this.utils.printDiffOrStringify(expected, receivedString, labelExpected, 'Received string', false) + callLogText(log);
+      if (notFound) {
+        printedExpected = `Expected pattern: not ${this.utils.printExpected(expected)}`;
+        printedReceived = `Received: ${received}`;
+      } else {
+        printedExpected = `Expected pattern: not ${this.utils.printExpected(expected)}`;
+        const formattedReceived = printReceivedStringContainExpectedResult(receivedString, typeof expected.exec === 'function' ? expected.exec(receivedString) : null);
+        printedReceived = `Received string: ${formattedReceived}`;
+      }
     }
+  } else {
+    const labelExpected = `Expected ${typeof expected === 'string' ? stringSubstring : 'pattern'}`;
+    if (notFound) {
+      printedExpected = `${labelExpected}: ${this.utils.printExpected(expected)}`;
+      printedReceived = `Received: ${received}`;
+    } else {
+      printedDiff = this.utils.printDiffOrStringify(expected, receivedString, labelExpected, 'Received string', false);
+    }
+  }
+
+  const message = () => {
+    const resultDetails = printedDiff ? printedDiff : printedExpected + '\n' + printedReceived;
+    return messagePrefix + resultDetails + callLogText(log);
   };
 
   return {
@@ -91,5 +118,10 @@ export async function toMatchText(
     actual: received,
     log,
     timeout: timedOut ? timeout : undefined,
+    // eslint-disable-next-line @typescript-eslint/no-base-to-string
+    locator: receiver.toString(),
+    ...(printedReceived ? { printedReceived } : {}),
+    ...(printedExpected ? { printedExpected } : {}),
+    ...(printedDiff ? { printedDiff } : {}),
   };
 }

packages/playwright/src/reporters/base.ts

@@ -32,6 +32,13 @@ type Annotation = {
 type ErrorDetails = {
   message: string;
   location?: Location;
+  timeout?: number;
+  matcherName?: string;
+  locator?: string;
+  expected?: string;
+  received?: string;
+  log?: string[];
+  snippet?: string;
 };
 
 type TestSummary = {
@@ -383,6 +390,13 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI
     errorDetails.push({
       message: indent(formattedError.message, initialIndent),
       location: formattedError.location,
+      timeout: error.timeout,
+      matcherName: error.matcherName,
+      locator: error.locator,
+      expected: error.expected,
+      received: error.received,
+      log: error.log,
+      snippet: error.snippet,
     });
   }
   return errorDetails;

packages/playwright/src/worker/testInfo.ts

@@ -24,10 +24,11 @@ import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutMana
 import type { RunnableDescription } from './timeoutManager';
 import type { Annotation, FullConfigInternal, FullProjectInternal } from '../common/config';
 import type { FullConfig, Location } from '../../types/testReporter';
-import { debugTest, filteredStackTrace, formatLocation, getContainedPath, normalizeAndSaveAttachment, serializeError, trimLongString, windowsFilesystemFriendlyLength } from '../util';
+import { debugTest, filteredStackTrace, formatLocation, getContainedPath, normalizeAndSaveAttachment, trimLongString, windowsFilesystemFriendlyLength } from '../util';
 import { TestTracing } from './testTracing';
 import type { Attachment } from './testTracing';
 import type { StackFrame } from '@protocol/channels';
+import { serializeWorkerError } from './util';
 
 export interface TestStepInternal {
   complete(result: { error?: Error | unknown, attachments?: Attachment[] }): void;
@@ -272,7 +273,7 @@ export class TestInfoImpl implements TestInfo {
         if (result.error) {
           if (typeof result.error === 'object' && !(result.error as any)?.[stepSymbol])
             (result.error as any)[stepSymbol] = step;
-          const error = serializeError(result.error);
+          const error = serializeWorkerError(result.error);
           if (data.boxedStack)
             error.stack = `${error.message}\n${stringifyStackFrames(data.boxedStack).join('\n')}`;
           step.error = error;
@@ -330,7 +331,7 @@ export class TestInfoImpl implements TestInfo {
   _failWithError(error: Error | unknown) {
     if (this.status === 'passed' || this.status === 'skipped')
       this.status = error instanceof TimeoutManagerError ? 'timedOut' : 'failed';
-    const serialized = serializeError(error);
+    const serialized = serializeWorkerError(error);
     const step: TestStepInternal | undefined = typeof error === 'object' ? (error as any)?.[stepSymbol] : undefined;
     if (step && step.boxedStack)
       serialized.stack = `${(error as Error).name}: ${(error as Error).message}\n${stringifyStackFrames(step.boxedStack).join('\n')}`;

packages/playwright/src/worker/util.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { TestError } from '../../types/testReporter';
+import type { TestInfoError } from '../../types/test';
+import type { MatcherResult } from '../matchers/matcherHint';
+import { serializeError } from '../util';
+
+
+type MatcherResultDetails = Pick<TestError, 'timeout'|'matcherName'|'locator'|'expected'|'received'|'log'>;
+
+export function serializeWorkerError(error: Error | any): TestInfoError & MatcherResultDetails {
+  return {
+    ...serializeError(error),
+    ...serializeExpectDetails(error),
+  };
+}
+
+function serializeExpectDetails(e: Error): MatcherResultDetails {
+  const matcherResult = (e as any).matcherResult as MatcherResult<unknown, unknown>;
+  if (!matcherResult)
+    return {};
+  return {
+    timeout: matcherResult.timeout,
+    matcherName: matcherResult.name,
+    locator: matcherResult.locator,
+    expected: matcherResult.printedExpected,
+    received: matcherResult.printedReceived,
+    log: matcherResult.log,
+  };
+}
+

packages/playwright/src/worker/workerMain.ts

@@ -15,7 +15,7 @@
  */
 
 import { colors } from 'playwright-core/lib/utilsBundle';
-import { debugTest, relativeFilePath, serializeError } from '../util';
+import { debugTest, relativeFilePath } from '../util';
 import { type TestBeginPayload, type TestEndPayload, type RunPayload, type DonePayload, type WorkerInitParams, type TeardownErrorsPayload, stdioChunkToParams } from '../common/ipc';
 import { setCurrentTestInfo, setIsWorkerProcess } from '../common/globals';
 import { deserializeConfig } from '../common/configLoader';
@@ -32,6 +32,7 @@ import type { TestInfoError } from '../../types/test';
 import type { Location } from '../../types/testReporter';
 import { inheritFixtureNames } from '../common/fixtures';
 import { type TimeSlot } from './timeoutManager';
+import { serializeWorkerError } from './util';
 
 export class WorkerMain extends ProcessRunner {
   private _params: WorkerInitParams;
@@ -112,7 +113,7 @@ export class WorkerMain extends ProcessRunner {
       await fakeTestInfo._runAsStage({ title: 'worker cleanup', runnable }, () => gracefullyCloseAll()).catch(() => {});
       this._fatalErrors.push(...fakeTestInfo.errors);
     } catch (e) {
-      this._fatalErrors.push(serializeError(e));
+      this._fatalErrors.push(serializeWorkerError(e));
     }
 
     if (this._fatalErrors.length) {
@@ -153,7 +154,7 @@ export class WorkerMain extends ProcessRunner {
     // No current test - fatal error.
     if (!this._currentTest) {
       if (!this._fatalErrors.length)
-        this._fatalErrors.push(serializeError(error));
+        this._fatalErrors.push(serializeWorkerError(error));
       void this._stop();
       return;
     }
@@ -224,7 +225,7 @@ export class WorkerMain extends ProcessRunner {
       // In theory, we should run above code without any errors.
       // However, in the case we screwed up, or loadTestFile failed in the worker
       // but not in the runner, let's do a fatal error.
-      this._fatalErrors.push(serializeError(e));
+      this._fatalErrors.push(serializeWorkerError(e));
       void this._stop();
     } finally {
       const donePayload: DonePayload = {

packages/playwright/types/testReporter.d.ts

@@ -554,16 +554,41 @@ export interface TestCase {
  * Information about an error thrown during test execution.
  */
 export interface TestError {
+  /**
+   * Expected value formatted as a human-readable string.
+   */
+  expected?: string;
+
   /**
    * Error location in the source code.
    */
   location?: Location;
 
+  /**
+   * Receiver's locator.
+   */
+  locator?: string;
+
+  /**
+   * Call log.
+   */
+  log?: Array<string>;
+
+  /**
+   * Expect matcher name.
+   */
+  matcherName?: string;
+
   /**
    * Error message. Set when [Error] (or its subclass) has been thrown.
    */
   message?: string;
 
+  /**
+   * Received value formatted as a human-readable string.
+   */
+  received?: string;
+
   /**
    * Source code snippet with highlighted error.
    */
@@ -574,6 +599,11 @@ export interface TestError {
    */
   stack?: string;
 
+  /**
+   * Timeout in milliseconds, if the error was caused by a timeout.
+   */
+  timeout?: number;
+
   /**
    * The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown.
    */

tests/page/expect-matcher-result.spec.ts

@@ -24,12 +24,16 @@ test('toMatchText-based assertions should have matcher result', async ({ page })
   {
     const e = await expect(locator).toHaveText(/Text2/, { timeout: 1 }).catch(e => e);
     e.matcherResult.message = stripAnsi(e.matcherResult.message);
+    e.matcherResult.printedDiff = stripAnsi(e.matcherResult.printedDiff);
     expect.soft(e.matcherResult).toEqual({
       actual: 'Text content',
       expected: /Text2/,
       message: expect.stringContaining(`Timed out 1ms waiting for expect(locator).toHaveText(expected)`),
       name: 'toHaveText',
       pass: false,
+      locator: `locator('#node')`,
+      printedDiff: `Expected pattern: /Text2/
+Received string:  \"Text content\"`,
       log: expect.any(Array),
       timeout: 1,
     });
@@ -46,12 +50,17 @@ Call log`);
   {
     const e = await expect(locator).not.toHaveText(/Text/, { timeout: 1 }).catch(e => e);
     e.matcherResult.message = stripAnsi(e.matcherResult.message);
+    e.matcherResult.printedExpected = stripAnsi(e.matcherResult.printedExpected);
+    e.matcherResult.printedReceived = stripAnsi(e.matcherResult.printedReceived);
     expect.soft(e.matcherResult).toEqual({
       actual: 'Text content',
       expected: /Text/,
       message: expect.stringContaining(`Timed out 1ms waiting for expect(locator).not.toHaveText(expected)`),
       name: 'toHaveText',
       pass: true,
+      locator: `locator('#node')`,
+      printedExpected: 'Expected pattern: not /Text/',
+      printedReceived: `Received string: \"Text content\"`,
       log: expect.any(Array),
       timeout: 1,
     });
@@ -79,6 +88,8 @@ test('toBeTruthy-based assertions should have matcher result', async ({ page })
       name: 'toBeVisible',
       pass: false,
       log: expect.any(Array),
+      printedExpected: 'Expected: visible',
+      printedReceived: 'Received: <element(s) not found>',
       timeout: 1,
     });
 
@@ -101,6 +112,8 @@ Call log`);
       name: 'toBeVisible',
       pass: true,
       log: expect.any(Array),
+      printedExpected: 'Expected: not visible',
+      printedReceived: 'Received: visible',
       timeout: 1,
     });
 
@@ -120,6 +133,7 @@ test('toEqual-based assertions should have matcher result', async ({ page }) =>
   {
     const e = await expect(page.locator('#node2')).toHaveCount(1, { timeout: 1 }).catch(e => e);
     e.matcherResult.message = stripAnsi(e.matcherResult.message);
+    e.matcherResult.printedDiff = stripAnsi(e.matcherResult.printedDiff);
     expect.soft(e.matcherResult).toEqual({
       actual: 0,
       expected: 1,
@@ -127,6 +141,8 @@ test('toEqual-based assertions should have matcher result', async ({ page }) =>
       name: 'toHaveCount',
       pass: false,
       log: expect.any(Array),
+      printedDiff: `Expected: 1
+Received: 0`,
       timeout: 1,
     });
 
@@ -141,6 +157,8 @@ Call log`);
   {
     const e = await expect(page.locator('#node')).not.toHaveCount(1, { timeout: 1 }).catch(e => e);
     e.matcherResult.message = stripAnsi(e.matcherResult.message);
+    e.matcherResult.printedExpected = stripAnsi(e.matcherResult.printedExpected);
+    e.matcherResult.printedReceived = stripAnsi(e.matcherResult.printedReceived);
     expect.soft(e.matcherResult).toEqual({
       actual: 1,
       expected: 1,
@@ -148,6 +166,8 @@ Call log`);
       name: 'toHaveCount',
       pass: true,
       log: expect.any(Array),
+      printedExpected: `Expected: not 1`,
+      printedReceived: `Received: 1`,
       timeout: 1,
     });
 
@@ -177,6 +197,8 @@ test('toBeChecked({ checked: false }) should have expected: false', async ({ pag
       name: 'toBeChecked',
       pass: false,
       log: expect.any(Array),
+      printedExpected: 'Expected: checked',
+      printedReceived: 'Received: unchecked',
       timeout: 1,
     });
 
@@ -199,6 +221,8 @@ Call log`);
       name: 'toBeChecked',
       pass: true,
       log: expect.any(Array),
+      printedExpected: 'Expected: not checked',
+      printedReceived: 'Received: checked',
       timeout: 1,
     });
 
@@ -221,6 +245,8 @@ Call log`);
       name: 'toBeChecked',
       pass: false,
       log: expect.any(Array),
+      printedExpected: 'Expected: unchecked',
+      printedReceived: 'Received: checked',
       timeout: 1,
     });
 
@@ -243,6 +269,8 @@ Call log`);
       name: 'toBeChecked',
       pass: true,
       log: expect.any(Array),
+      printedExpected: 'Expected: not unchecked',
+      printedReceived: 'Received: unchecked',
       timeout: 1,
     });
 
@@ -271,6 +299,8 @@ test('toHaveScreenshot should populate matcherResult', async ({ page, server, is
     name: 'toHaveScreenshot',
     pass: false,
     log: expect.any(Array),
+    printedExpected: expect.stringContaining('screenshot-sanity-'),
+    printedReceived: expect.stringContaining('screenshot-sanity-actual'),
   });
 
   expect.soft(stripAnsi(e.toString())).toContain(`Error: Screenshot comparison failed: