diff --git a/docs/src/test-reporter-api/class-testerror.md b/docs/src/test-reporter-api/class-testerror.md index 7a872c63fc..8e76d6f595 100644 --- a/docs/src/test-reporter-api/class-testerror.md +++ b/docs/src/test-reporter-api/class-testerror.md @@ -4,18 +4,54 @@ Information about an error thrown during test execution. +## property: TestError.expected +* since: v1.49 +- type: ?<[string]> + +Expected value formatted as a human-readable string. + +## property: TestError.locator +* since: v1.49 +- type: ?<[string]> + +Receiver's locator. + +## property: TestError.log +* since: v1.49 +- type: ?<[Array]<[string]>> + +Call log. + +## property: TestError.matcherName +* since: v1.49 +- type: ?<[string]> + +Expect matcher name. + ## property: TestError.message * since: v1.10 - type: ?<[string]> Error message. Set when [Error] (or its subclass) has been thrown. +## property: TestError.received +* since: v1.49 +- type: ?<[string]> + +Received value formatted as a human-readable string. + ## property: TestError.stack * since: v1.10 - type: ?<[string]> Error stack. Set when [Error] (or its subclass) has been thrown. +## property: TestError.timeout +* since: v1.49 +- type: ?<[int]> + +Timeout in milliseconds, if the error was caused by a timeout. + ## property: TestError.value * since: v1.10 - type: ?<[string]> diff --git a/packages/playwright/src/matchers/matcherHint.ts b/packages/playwright/src/matchers/matcherHint.ts index 8a78932c68..5ffc745263 100644 --- a/packages/playwright/src/matchers/matcherHint.ts +++ b/packages/playwright/src/matchers/matcherHint.ts @@ -39,6 +39,10 @@ export type MatcherResult = { actual?: A; log?: string[]; timeout?: number; + locator?: string; + printedReceived?: string; + printedExpected?: string; + printedDiff?: string; }; export class ExpectError extends Error { diff --git a/packages/playwright/src/matchers/toBeTruthy.ts b/packages/playwright/src/matchers/toBeTruthy.ts index 0941ab7a63..8902a14eea 100644 --- a/packages/playwright/src/matchers/toBeTruthy.ts +++ b/packages/playwright/src/matchers/toBeTruthy.ts @@ -39,22 +39,41 @@ export async function toBeTruthy( }; const timeout = options.timeout ?? this.timeout; - const { matches, log, timedOut, received } = await query(!!this.isNot, timeout); + const { matches: pass, log, timedOut, received } = await query(!!this.isNot, timeout); + if (pass === !this.isNot) { + return { + name: matcherName, + message: () => '', + pass, + expected + }; + } + const notFound = received === kNoElementsFoundError ? received : undefined; - const actual = matches ? expected : unexpected; + const actual = pass ? expected : unexpected; + let printedReceived: string | undefined; + let printedExpected: string | undefined; + if (pass) { + printedExpected = `Expected: not ${expected}`; + printedReceived = `Received: ${notFound ? kNoElementsFoundError : expected}`; + } else { + printedExpected = `Expected: ${expected}`; + printedReceived = `Received: ${notFound ? kNoElementsFoundError : unexpected}`; + } const message = () => { const header = matcherHint(this, receiver, matcherName, 'locator', arg, matcherOptions, timedOut ? timeout : undefined); const logText = callLogText(log); - return matches ? `${header}Expected: not ${expected}\nReceived: ${notFound ? kNoElementsFoundError : expected}${logText}` : - `${header}Expected: ${expected}\nReceived: ${notFound ? kNoElementsFoundError : unexpected}${logText}`; + return `${header}${printedExpected}\n${printedReceived}${logText}`; }; return { message, - pass: matches, + pass, actual, name: matcherName, expected, log, timeout: timedOut ? timeout : undefined, + ...(printedReceived ? { printedReceived } : {}), + ...(printedExpected ? { printedExpected } : {}), }; } diff --git a/packages/playwright/src/matchers/toEqual.ts b/packages/playwright/src/matchers/toEqual.ts index 29d3fd4866..f75caf87f5 100644 --- a/packages/playwright/src/matchers/toEqual.ts +++ b/packages/playwright/src/matchers/toEqual.ts @@ -44,22 +44,35 @@ export async function toEqual( const timeout = options.timeout ?? this.timeout; const { matches: pass, received, log, timedOut } = await query(!!this.isNot, timeout); + if (pass === !this.isNot) { + return { + name: matcherName, + message: () => '', + pass, + expected + }; + } - const message = pass - ? () => - matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined) + - `Expected: not ${this.utils.printExpected(expected)}\n` + - `Received: ${this.utils.printReceived(received)}` + callLogText(log) - : () => - matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined) + - this.utils.printDiffOrStringify( - expected, - received, - EXPECTED_LABEL, - RECEIVED_LABEL, - false, - ) + callLogText(log); - + let printedReceived: string | undefined; + let printedExpected: string | undefined; + let printedDiff: string | undefined; + if (pass) { + printedExpected = `Expected: not ${this.utils.printExpected(expected)}`; + printedReceived = `Received: ${this.utils.printReceived(received)}`; + } else { + printedDiff = this.utils.printDiffOrStringify( + expected, + received, + EXPECTED_LABEL, + RECEIVED_LABEL, + false, + ); + } + const message = () => { + const header = matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined); + const details = printedDiff || `${printedExpected}\n${printedReceived}`; + return `${header}${details}${callLogText(log)}`; + }; // Passing the actual and expected objects so that a custom reporter // could access them, for example in order to display a custom visual diff, // or create a different error message @@ -70,5 +83,8 @@ export async function toEqual( pass, log, timeout: timedOut ? timeout : undefined, + ...(printedReceived ? { printedReceived } : {}), + ...(printedExpected ? { printedExpected } : {}), + ...(printedDiff ? { printedDiff } : {}), }; } diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts index b3fa3f556e..86504062d3 100644 --- a/packages/playwright/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchSnapshot.ts @@ -194,6 +194,10 @@ class SnapshotHelper { pass, message: () => message, log, + // eslint-disable-next-line @typescript-eslint/no-base-to-string + ...(this.locator ? { locator: this.locator.toString() } : {}), + printedExpected: this.expectedPath, + printedReceived: this.actualPath, }; return Object.fromEntries(Object.entries(unfiltered).filter(([_, v]) => v !== undefined)) as ImageMatcherResult; } diff --git a/packages/playwright/src/matchers/toMatchText.ts b/packages/playwright/src/matchers/toMatchText.ts index ebac8f8028..76ed48af1e 100644 --- a/packages/playwright/src/matchers/toMatchText.ts +++ b/packages/playwright/src/matchers/toMatchText.ts @@ -58,29 +58,56 @@ export async function toMatchText( const timeout = options.timeout ?? this.timeout; const { matches: pass, received, log, timedOut } = await query(!!this.isNot, timeout); + if (pass === !this.isNot) { + return { + name: matcherName, + message: () => '', + pass, + expected + }; + } + const stringSubstring = options.matchSubstring ? 'substring' : 'string'; const receivedString = received || ''; const messagePrefix = matcherHint(this, receiver, matcherName, 'locator', undefined, matcherOptions, timedOut ? timeout : undefined); const notFound = received === kNoElementsFoundError; - const message = () => { - if (pass) { - if (typeof expected === 'string') { - if (notFound) - return messagePrefix + `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}\nReceived: ${received}` + callLogText(log); - const printedReceived = printReceivedStringContainExpectedSubstring(receivedString, receivedString.indexOf(expected), expected.length); - return messagePrefix + `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}\nReceived string: ${printedReceived}` + callLogText(log); + + let printedReceived: string | undefined; + let printedExpected: string | undefined; + let printedDiff: string | undefined; + if (pass) { + if (typeof expected === 'string') { + if (notFound) { + printedExpected = `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}`; + printedReceived = `Received: ${received}`; } else { - if (notFound) - return messagePrefix + `Expected pattern: not ${this.utils.printExpected(expected)}\nReceived: ${received}` + callLogText(log); - const printedReceived = printReceivedStringContainExpectedResult(receivedString, typeof expected.exec === 'function' ? expected.exec(receivedString) : null); - return messagePrefix + `Expected pattern: not ${this.utils.printExpected(expected)}\nReceived string: ${printedReceived}` + callLogText(log); + printedExpected = `Expected ${stringSubstring}: not ${this.utils.printExpected(expected)}`; + const formattedReceived = printReceivedStringContainExpectedSubstring(receivedString, receivedString.indexOf(expected), expected.length); + printedReceived = `Received string: ${formattedReceived}`; } } else { - const labelExpected = `Expected ${typeof expected === 'string' ? stringSubstring : 'pattern'}`; - if (notFound) - return messagePrefix + `${labelExpected}: ${this.utils.printExpected(expected)}\nReceived: ${received}` + callLogText(log); - return messagePrefix + this.utils.printDiffOrStringify(expected, receivedString, labelExpected, 'Received string', false) + callLogText(log); + if (notFound) { + printedExpected = `Expected pattern: not ${this.utils.printExpected(expected)}`; + printedReceived = `Received: ${received}`; + } else { + printedExpected = `Expected pattern: not ${this.utils.printExpected(expected)}`; + const formattedReceived = printReceivedStringContainExpectedResult(receivedString, typeof expected.exec === 'function' ? expected.exec(receivedString) : null); + printedReceived = `Received string: ${formattedReceived}`; + } } + } else { + const labelExpected = `Expected ${typeof expected === 'string' ? stringSubstring : 'pattern'}`; + if (notFound) { + printedExpected = `${labelExpected}: ${this.utils.printExpected(expected)}`; + printedReceived = `Received: ${received}`; + } else { + printedDiff = this.utils.printDiffOrStringify(expected, receivedString, labelExpected, 'Received string', false); + } + } + + const message = () => { + const resultDetails = printedDiff ? printedDiff : printedExpected + '\n' + printedReceived; + return messagePrefix + resultDetails + callLogText(log); }; return { @@ -91,5 +118,10 @@ export async function toMatchText( actual: received, log, timeout: timedOut ? timeout : undefined, + // eslint-disable-next-line @typescript-eslint/no-base-to-string + locator: receiver.toString(), + ...(printedReceived ? { printedReceived } : {}), + ...(printedExpected ? { printedExpected } : {}), + ...(printedDiff ? { printedDiff } : {}), }; } diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 4249429a36..138820baee 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -32,6 +32,13 @@ type Annotation = { type ErrorDetails = { message: string; location?: Location; + timeout?: number; + matcherName?: string; + locator?: string; + expected?: string; + received?: string; + log?: string[]; + snippet?: string; }; type TestSummary = { @@ -383,6 +390,13 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI errorDetails.push({ message: indent(formattedError.message, initialIndent), location: formattedError.location, + timeout: error.timeout, + matcherName: error.matcherName, + locator: error.locator, + expected: error.expected, + received: error.received, + log: error.log, + snippet: error.snippet, }); } return errorDetails; diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 378b32524f..e41f1a9a52 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -24,10 +24,11 @@ import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutMana import type { RunnableDescription } from './timeoutManager'; import type { Annotation, FullConfigInternal, FullProjectInternal } from '../common/config'; import type { FullConfig, Location } from '../../types/testReporter'; -import { debugTest, filteredStackTrace, formatLocation, getContainedPath, normalizeAndSaveAttachment, serializeError, trimLongString, windowsFilesystemFriendlyLength } from '../util'; +import { debugTest, filteredStackTrace, formatLocation, getContainedPath, normalizeAndSaveAttachment, trimLongString, windowsFilesystemFriendlyLength } from '../util'; import { TestTracing } from './testTracing'; import type { Attachment } from './testTracing'; import type { StackFrame } from '@protocol/channels'; +import { serializeWorkerError } from './util'; export interface TestStepInternal { complete(result: { error?: Error | unknown, attachments?: Attachment[] }): void; @@ -272,7 +273,7 @@ export class TestInfoImpl implements TestInfo { if (result.error) { if (typeof result.error === 'object' && !(result.error as any)?.[stepSymbol]) (result.error as any)[stepSymbol] = step; - const error = serializeError(result.error); + const error = serializeWorkerError(result.error); if (data.boxedStack) error.stack = `${error.message}\n${stringifyStackFrames(data.boxedStack).join('\n')}`; step.error = error; @@ -330,7 +331,7 @@ export class TestInfoImpl implements TestInfo { _failWithError(error: Error | unknown) { if (this.status === 'passed' || this.status === 'skipped') this.status = error instanceof TimeoutManagerError ? 'timedOut' : 'failed'; - const serialized = serializeError(error); + const serialized = serializeWorkerError(error); const step: TestStepInternal | undefined = typeof error === 'object' ? (error as any)?.[stepSymbol] : undefined; if (step && step.boxedStack) serialized.stack = `${(error as Error).name}: ${(error as Error).message}\n${stringifyStackFrames(step.boxedStack).join('\n')}`; diff --git a/packages/playwright/src/worker/util.ts b/packages/playwright/src/worker/util.ts new file mode 100644 index 0000000000..d24d337191 --- /dev/null +++ b/packages/playwright/src/worker/util.ts @@ -0,0 +1,45 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { TestError } from '../../types/testReporter'; +import type { TestInfoError } from '../../types/test'; +import type { MatcherResult } from '../matchers/matcherHint'; +import { serializeError } from '../util'; + + +type MatcherResultDetails = Pick; + +export function serializeWorkerError(error: Error | any): TestInfoError & MatcherResultDetails { + return { + ...serializeError(error), + ...serializeExpectDetails(error), + }; +} + +function serializeExpectDetails(e: Error): MatcherResultDetails { + const matcherResult = (e as any).matcherResult as MatcherResult; + if (!matcherResult) + return {}; + return { + timeout: matcherResult.timeout, + matcherName: matcherResult.name, + locator: matcherResult.locator, + expected: matcherResult.printedExpected, + received: matcherResult.printedReceived, + log: matcherResult.log, + }; +} + diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts index f180f3d08b..5680c3ddb3 100644 --- a/packages/playwright/src/worker/workerMain.ts +++ b/packages/playwright/src/worker/workerMain.ts @@ -15,7 +15,7 @@ */ import { colors } from 'playwright-core/lib/utilsBundle'; -import { debugTest, relativeFilePath, serializeError } from '../util'; +import { debugTest, relativeFilePath } from '../util'; import { type TestBeginPayload, type TestEndPayload, type RunPayload, type DonePayload, type WorkerInitParams, type TeardownErrorsPayload, stdioChunkToParams } from '../common/ipc'; import { setCurrentTestInfo, setIsWorkerProcess } from '../common/globals'; import { deserializeConfig } from '../common/configLoader'; @@ -32,6 +32,7 @@ import type { TestInfoError } from '../../types/test'; import type { Location } from '../../types/testReporter'; import { inheritFixtureNames } from '../common/fixtures'; import { type TimeSlot } from './timeoutManager'; +import { serializeWorkerError } from './util'; export class WorkerMain extends ProcessRunner { private _params: WorkerInitParams; @@ -112,7 +113,7 @@ export class WorkerMain extends ProcessRunner { await fakeTestInfo._runAsStage({ title: 'worker cleanup', runnable }, () => gracefullyCloseAll()).catch(() => {}); this._fatalErrors.push(...fakeTestInfo.errors); } catch (e) { - this._fatalErrors.push(serializeError(e)); + this._fatalErrors.push(serializeWorkerError(e)); } if (this._fatalErrors.length) { @@ -153,7 +154,7 @@ export class WorkerMain extends ProcessRunner { // No current test - fatal error. if (!this._currentTest) { if (!this._fatalErrors.length) - this._fatalErrors.push(serializeError(error)); + this._fatalErrors.push(serializeWorkerError(error)); void this._stop(); return; } @@ -224,7 +225,7 @@ export class WorkerMain extends ProcessRunner { // In theory, we should run above code without any errors. // However, in the case we screwed up, or loadTestFile failed in the worker // but not in the runner, let's do a fatal error. - this._fatalErrors.push(serializeError(e)); + this._fatalErrors.push(serializeWorkerError(e)); void this._stop(); } finally { const donePayload: DonePayload = { diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index a9d1f020ae..5663f92f98 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -554,16 +554,41 @@ export interface TestCase { * Information about an error thrown during test execution. */ export interface TestError { + /** + * Expected value formatted as a human-readable string. + */ + expected?: string; + /** * Error location in the source code. */ location?: Location; + /** + * Receiver's locator. + */ + locator?: string; + + /** + * Call log. + */ + log?: Array; + + /** + * Expect matcher name. + */ + matcherName?: string; + /** * Error message. Set when [Error] (or its subclass) has been thrown. */ message?: string; + /** + * Received value formatted as a human-readable string. + */ + received?: string; + /** * Source code snippet with highlighted error. */ @@ -574,6 +599,11 @@ export interface TestError { */ stack?: string; + /** + * Timeout in milliseconds, if the error was caused by a timeout. + */ + timeout?: number; + /** * The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown. */ diff --git a/tests/page/expect-matcher-result.spec.ts b/tests/page/expect-matcher-result.spec.ts index 8f8a83bc83..7767ecf5f6 100644 --- a/tests/page/expect-matcher-result.spec.ts +++ b/tests/page/expect-matcher-result.spec.ts @@ -24,12 +24,16 @@ test('toMatchText-based assertions should have matcher result', async ({ page }) { const e = await expect(locator).toHaveText(/Text2/, { timeout: 1 }).catch(e => e); e.matcherResult.message = stripAnsi(e.matcherResult.message); + e.matcherResult.printedDiff = stripAnsi(e.matcherResult.printedDiff); expect.soft(e.matcherResult).toEqual({ actual: 'Text content', expected: /Text2/, message: expect.stringContaining(`Timed out 1ms waiting for expect(locator).toHaveText(expected)`), name: 'toHaveText', pass: false, + locator: `locator('#node')`, + printedDiff: `Expected pattern: /Text2/ +Received string: \"Text content\"`, log: expect.any(Array), timeout: 1, }); @@ -46,12 +50,17 @@ Call log`); { const e = await expect(locator).not.toHaveText(/Text/, { timeout: 1 }).catch(e => e); e.matcherResult.message = stripAnsi(e.matcherResult.message); + e.matcherResult.printedExpected = stripAnsi(e.matcherResult.printedExpected); + e.matcherResult.printedReceived = stripAnsi(e.matcherResult.printedReceived); expect.soft(e.matcherResult).toEqual({ actual: 'Text content', expected: /Text/, message: expect.stringContaining(`Timed out 1ms waiting for expect(locator).not.toHaveText(expected)`), name: 'toHaveText', pass: true, + locator: `locator('#node')`, + printedExpected: 'Expected pattern: not /Text/', + printedReceived: `Received string: \"Text content\"`, log: expect.any(Array), timeout: 1, }); @@ -79,6 +88,8 @@ test('toBeTruthy-based assertions should have matcher result', async ({ page }) name: 'toBeVisible', pass: false, log: expect.any(Array), + printedExpected: 'Expected: visible', + printedReceived: 'Received: ', timeout: 1, }); @@ -101,6 +112,8 @@ Call log`); name: 'toBeVisible', pass: true, log: expect.any(Array), + printedExpected: 'Expected: not visible', + printedReceived: 'Received: visible', timeout: 1, }); @@ -120,6 +133,7 @@ test('toEqual-based assertions should have matcher result', async ({ page }) => { const e = await expect(page.locator('#node2')).toHaveCount(1, { timeout: 1 }).catch(e => e); e.matcherResult.message = stripAnsi(e.matcherResult.message); + e.matcherResult.printedDiff = stripAnsi(e.matcherResult.printedDiff); expect.soft(e.matcherResult).toEqual({ actual: 0, expected: 1, @@ -127,6 +141,8 @@ test('toEqual-based assertions should have matcher result', async ({ page }) => name: 'toHaveCount', pass: false, log: expect.any(Array), + printedDiff: `Expected: 1 +Received: 0`, timeout: 1, }); @@ -141,6 +157,8 @@ Call log`); { const e = await expect(page.locator('#node')).not.toHaveCount(1, { timeout: 1 }).catch(e => e); e.matcherResult.message = stripAnsi(e.matcherResult.message); + e.matcherResult.printedExpected = stripAnsi(e.matcherResult.printedExpected); + e.matcherResult.printedReceived = stripAnsi(e.matcherResult.printedReceived); expect.soft(e.matcherResult).toEqual({ actual: 1, expected: 1, @@ -148,6 +166,8 @@ Call log`); name: 'toHaveCount', pass: true, log: expect.any(Array), + printedExpected: `Expected: not 1`, + printedReceived: `Received: 1`, timeout: 1, }); @@ -177,6 +197,8 @@ test('toBeChecked({ checked: false }) should have expected: false', async ({ pag name: 'toBeChecked', pass: false, log: expect.any(Array), + printedExpected: 'Expected: checked', + printedReceived: 'Received: unchecked', timeout: 1, }); @@ -199,6 +221,8 @@ Call log`); name: 'toBeChecked', pass: true, log: expect.any(Array), + printedExpected: 'Expected: not checked', + printedReceived: 'Received: checked', timeout: 1, }); @@ -221,6 +245,8 @@ Call log`); name: 'toBeChecked', pass: false, log: expect.any(Array), + printedExpected: 'Expected: unchecked', + printedReceived: 'Received: checked', timeout: 1, }); @@ -243,6 +269,8 @@ Call log`); name: 'toBeChecked', pass: true, log: expect.any(Array), + printedExpected: 'Expected: not unchecked', + printedReceived: 'Received: unchecked', timeout: 1, }); @@ -271,6 +299,8 @@ test('toHaveScreenshot should populate matcherResult', async ({ page, server, is name: 'toHaveScreenshot', pass: false, log: expect.any(Array), + printedExpected: expect.stringContaining('screenshot-sanity-'), + printedReceived: expect.stringContaining('screenshot-sanity-actual'), }); expect.soft(stripAnsi(e.toString())).toContain(`Error: Screenshot comparison failed: