chore: allow aria snapshot rebaselines (#33256)

packages/playwright-core/src/server/injected/ariaSnapshot.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import { escapeWithQuotes } from '@isomorphic/stringUtils';
 import * as roleUtils from './roleUtils';
 import { getElementComputedStyle } from './domUtils';
 import type { AriaRole } from './roleUtils';
@@ -184,7 +183,7 @@ function matchesText(text: string | undefined, template: RegExp | string | undef
 export function matchesAriaTree(rootElement: Element, template: AriaTemplateNode): { matches: boolean, received: string } {
   const root = generateAriaTree(rootElement);
   const matches = matchesNodeDeep(root, template);
-  return { matches, received: renderAriaTree(root, { noText: true }) };
+  return { matches, received: renderAriaTree(root) };
 }
 
 function matchesNode(node: AriaNode | string, template: AriaTemplateNode | RegExp | string, depth: number): boolean {
@@ -252,17 +251,16 @@ function matchesNodeDeep(root: AriaNode, template: AriaTemplateNode): boolean {
   return !!results.length;
 }
 
-export function renderAriaTree(ariaNode: AriaNode, options?: { noText?: boolean }): string {
+export function renderAriaTree(ariaNode: AriaNode): string {
   const lines: string[] = [];
   const visit = (ariaNode: AriaNode | string, indent: string) => {
     if (typeof ariaNode === 'string') {
-      if (!options?.noText)
+      lines.push(indent + '- text: ' + quoteYamlString(ariaNode));
-        lines.push(indent + '- text: ' + quoteYamlString(ariaNode));
       return;
     }
     let line = `${indent}- ${ariaNode.role}`;
     if (ariaNode.name)
-      line += ` ${escapeWithQuotes(ariaNode.name, '"')}`;
+      line += ` ${quoteYamlString(ariaNode.name)}`;
 
     if (ariaNode.checked === 'mixed')
       line += ` [checked=mixed]`;
@@ -281,9 +279,16 @@ export function renderAriaTree(ariaNode: AriaNode, options?: { noText?: boolean
     if (ariaNode.selected === true)
       line += ` [selected]`;
 
-    lines.push(line + (ariaNode.children.length ? ':' : ''));
+    if (!ariaNode.children.length) {
-    for (const child of ariaNode.children || [])
+      lines.push(line);
-      visit(child, indent + '  ');
+    } else if (ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string') {
+      line += ': ' + quoteYamlString(ariaNode.children[0]);
+      lines.push(line);
+    } else {
+      lines.push(line + ':');
+      for (const child of ariaNode.children || [])
+        visit(child, indent + '  ');
+    }
   };
 
   if (ariaNode.role === 'fragment') {

packages/playwright/src/common/ipc.ts

@@ -106,6 +106,7 @@ export type StepEndPayload = {
   stepId: string;
   wallTime: number;  // milliseconds since unix epoch
   error?: TestInfoErrorImpl;
+  suggestedRebaseline?: string;
 };
 
 export type TestEntry = {

packages/playwright/src/matchers/expect.ts

@@ -61,7 +61,7 @@ import {
 } from '../common/expectBundle';
 import { zones } from 'playwright-core/lib/utils';
 import { TestInfoImpl } from '../worker/testInfo';
-import { ExpectError, isExpectError } from './matcherHint';
+import { ExpectError, isJestError } from './matcherHint';
 import { toMatchAriaSnapshot } from './toMatchAriaSnapshot';
 
 // #region
@@ -323,8 +323,13 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler<any> {
 
       const step = testInfo._addStep(stepInfo);
 
-      const reportStepError = (jestError: Error | unknown) => {
+      const reportStepError = (e: Error | unknown) => {
-        const error = isExpectError(jestError) ? new ExpectError(jestError, customMessage, stackFrames) : jestError;
+        const jestError = isJestError(e) ? e : null;
+        const error = jestError ? new ExpectError(jestError, customMessage, stackFrames) : e;
+        if (jestError?.matcherResult.suggestedRebaseline) {
+          step.complete({ suggestedRebaseline: jestError?.matcherResult.suggestedRebaseline });
+          return;
+        }
         step.complete({ error });
         if (this._info.isSoft)
           testInfo._failWithError(error);

packages/playwright/src/matchers/matcherHint.ts

@@ -43,6 +43,7 @@ export type MatcherResult<E, A> = {
   printedReceived?: string;
   printedExpected?: string;
   printedDiff?: string;
+  suggestedRebaseline?: string;
 };
 
 export type MatcherResultProperty = Omit<MatcherResult<unknown, unknown>, 'message'> & {
@@ -69,6 +70,6 @@ export class ExpectError extends Error {
   }
 }
 
-export function isExpectError(e: unknown): e is ExpectError {
+export function isJestError(e: unknown): e is JestError {
   return e instanceof Error && 'matcherResult' in e;
 }

packages/playwright/src/matchers/toMatchAriaSnapshot.ts

@@ -22,6 +22,7 @@ import { colors } from 'playwright-core/lib/utilsBundle';
 import { EXPECTED_COLOR } from '../common/expectBundle';
 import { callLogText } from '../util';
 import { printReceivedStringContainExpectedSubstring } from './expect';
+import { currentTestInfo } from '../common/globals';
 
 export async function toMatchAriaSnapshot(
   this: ExpectMatcherState,
@@ -31,6 +32,15 @@ export async function toMatchAriaSnapshot(
 ): Promise<MatcherResult<string | RegExp, string>> {
   const matcherName = 'toMatchAriaSnapshot';
 
+  const testInfo = currentTestInfo();
+  if (!testInfo)
+    throw new Error(`toMatchSnapshot() must be called during the test`);
+
+  if (testInfo._projectInternal.ignoreSnapshots)
+    return { pass: !this.isNot, message: () => '', name: 'toMatchSnapshot', expected };
+
+  const updateSnapshots = testInfo.config.updateSnapshots;
+
   const matcherOptions = {
     isNot: this.isNot,
     promise: this.promise,
@@ -65,6 +75,12 @@ export async function toMatchAriaSnapshot(
     }
   };
 
+  let suggestedRebaseline: string | undefined;
+  if (!this.isNot && pass === this.isNot) {
+    if (updateSnapshots === 'all' || (updateSnapshots === 'missing' && !expected.trim()))
+      suggestedRebaseline = `toMatchAriaSnapshot(\`\n${unshift(received, '${indent}  ')}\n\${indent}\`)`;
+  }
+
   return {
     name: matcherName,
     expected,
@@ -72,6 +88,7 @@ export async function toMatchAriaSnapshot(
     pass,
     actual: received,
     log,
+    suggestedRebaseline,
     timeout: timedOut ? timeout : undefined,
   };
 }
@@ -80,7 +97,7 @@ function escapePrivateUsePoints(str: string) {
   return str.replace(/[\uE000-\uF8FF]/g, char => `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`);
 }
 
-function unshift(snapshot: string): string {
+function unshift(snapshot: string, indent: string = ''): string {
   const lines = snapshot.split('\n');
   let whitespacePrefixLength = 100;
   for (const line of lines) {
@@ -91,5 +108,5 @@ function unshift(snapshot: string): string {
       whitespacePrefixLength = match[1].length;
     break;
   }
-  return lines.filter(t => t.trim()).map(line => line.substring(whitespacePrefixLength)).join('\n');
+  return lines.filter(t => t.trim()).map(line => indent + line.substring(whitespacePrefixLength)).join('\n');
 }

packages/playwright/src/runner/dispatcher.ts

@@ -27,6 +27,7 @@ import type { FullConfigInternal } from '../common/config';
 import type { ReporterV2 } from '../reporters/reporterV2';
 import type { FailureTracker } from './failureTracker';
 import { colors } from 'playwright-core/lib/utilsBundle';
+import { addSuggestedRebaseline } from './rebase';
 
 export type EnvByProjectId = Map<string, Record<string, string | undefined>>;
 
@@ -341,6 +342,8 @@ class JobDispatcher {
     step.duration = params.wallTime - step.startTime.getTime();
     if (params.error)
       step.error = params.error;
+    if (params.suggestedRebaseline)
+      addSuggestedRebaseline(step.location!, params.suggestedRebaseline);
     steps.delete(params.stepId);
     this._reporter.onStepEnd?.(test, result, step);
   }

packages/playwright/src/runner/rebase.ts

@@ -0,0 +1,95 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+import fs from 'fs';
+import type { T } from '../transform/babelBundle';
+import { types, traverse, parse } from '../transform/babelBundle';
+import { MultiMap } from 'playwright-core/lib/utils';
+import { generateUnifiedDiff } from 'playwright-core/lib/utils';
+import type { FullConfigInternal } from '../common/config';
+import { filterProjects } from './projectUtils';
+const t: typeof T = types;
+
+type Location = {
+  file: string;
+  line: number;
+  column: number;
+};
+
+type Replacement = {
+  // Points to the call expression.
+  location: Location;
+  code: string;
+};
+
+const suggestedRebaselines = new MultiMap<string, Replacement>();
+
+export function addSuggestedRebaseline(location: Location, suggestedRebaseline: string) {
+  suggestedRebaselines.set(location.file, { location, code: suggestedRebaseline });
+}
+
+export async function applySuggestedRebaselines(config: FullConfigInternal) {
+  if (config.config.updateSnapshots !== 'all' && config.config.updateSnapshots !== 'missing')
+    return;
+  const [project] = filterProjects(config.projects, config.cliProjectFilter);
+  if (!project)
+    return;
+
+  for (const fileName of suggestedRebaselines.keys()) {
+    const source = await fs.promises.readFile(fileName, 'utf8');
+    const lines = source.split('\n');
+    const replacements = suggestedRebaselines.get(fileName);
+    const fileNode = parse(source, { sourceType: 'module' });
+    const ranges: { start: number, end: number, oldText: string, newText: string }[] = [];
+
+    traverse(fileNode, {
+      CallExpression: path => {
+        const node = path.node;
+        if (node.arguments.length !== 1)
+          return;
+        if (!t.isMemberExpression(node.callee))
+          return;
+        const argument = node.arguments[0];
+        if (!t.isStringLiteral(argument) && !t.isTemplateLiteral(argument))
+          return;
+
+        const matcher = node.callee.property;
+        for (const replacement of replacements) {
+          // In Babel, rows are 1-based, columns are 0-based.
+          if (matcher.loc!.start.line !== replacement.location.line)
+            continue;
+          if (matcher.loc!.start.column + 1 !== replacement.location.column)
+            continue;
+          const indent = lines[matcher.loc!.start.line - 1].match(/^\s*/)![0];
+          const newText = replacement.code.replace(/\$\{indent\}/g, indent);
+          ranges.push({ start: matcher.start!, end: node.end!, oldText: source.substring(matcher.start!, node.end!), newText });
+        }
+      }
+    });
+
+    ranges.sort((a, b) => b.start - a.start);
+    let result = source;
+    for (const range of ranges)
+      result = result.substring(0, range.start) + range.newText + result.substring(range.end);
+
+    const relativeName = path.relative(process.cwd(), fileName);
+
+    const patchFile = path.join(project.project.outputDir, 'rebaselines.patch');
+    await fs.promises.mkdir(path.dirname(patchFile), { recursive: true });
+    await fs.promises.writeFile(patchFile, generateUnifiedDiff(source, result, relativeName));
+  }
+}

packages/playwright/src/runner/runner.ts

@@ -24,6 +24,7 @@ import type { FullConfigInternal } from '../common/config';
 import { affectedTestFiles } from '../transform/compilationCache';
 import { InternalReporter } from '../reporters/internalReporter';
 import { LastRunReporter } from './lastRun';
+import { applySuggestedRebaselines } from './rebase';
 
 type ProjectConfigWithFiles = {
   name: string;
@@ -88,6 +89,8 @@ export class Runner {
     ];
     const status = await runTasks(new TestRun(config, reporter), tasks, config.config.globalTimeout);
 
+    await applySuggestedRebaselines(config);
+
     // Calling process.exit() might truncate large stdout/stderr output.
     // See https://github.com/nodejs/node/issues/6456.
     // See https://github.com/nodejs/node/issues/12921

packages/playwright/src/worker/testInfo.ts

@@ -31,7 +31,7 @@ import type { StackFrame } from '@protocol/channels';
 import { testInfoError } from './util';
 
 export interface TestStepInternal {
-  complete(result: { error?: Error | unknown, attachments?: Attachment[] }): void;
+  complete(result: { error?: Error | unknown, attachments?: Attachment[], suggestedRebaseline?: string }): void;
   stepId: string;
   title: string;
   category: 'hook' | 'fixture' | 'test.step' | 'expect' | 'attach' | string;
@@ -297,6 +297,7 @@ export class TestInfoImpl implements TestInfo {
           stepId,
           wallTime: step.endWallTime,
           error: step.error,
+          suggestedRebaseline: result.suggestedRebaseline,
         };
         this._onStepEnd(payload);
         const errorForTrace = step.error ? { name: '', message: step.error.message || '', stack: step.error.stack } : undefined;

tests/page/page-aria-snapshot.spec.ts

@@ -64,10 +64,8 @@ it('should snapshot list with accessible name', async ({ page }) => {
   `);
   await checkAndMatchSnapshot(page.locator('body'), `
     - list "my list":
-      - listitem:
+      - listitem: "one"
-        - text: "one"
+      - listitem: "two"
-      - listitem:
-        - text: "two"
   `);
 });
 
@@ -107,8 +105,7 @@ it('should snapshot details visibility', async ({ page }) => {
   `);
 
   await checkAndMatchSnapshot(page.locator('body'), `
-    - group:
+    - group: "Summary"
-      - text: "Summary"
   `);
 });
 
@@ -151,8 +148,7 @@ it('should snapshot integration', async ({ page }) => {
     - text: "Open source projects and samples from Microsoft"
     - list:
       - listitem:
-        - group:
+        - group: "Verified"
-          - text: "Verified"
       - listitem:
         - link "Sponsor"
   `);
@@ -168,12 +164,10 @@ it('should support multiline text', async ({ page }) => {
   `);
 
   await checkAndMatchSnapshot(page.locator('body'), `
-    - paragraph:
+    - paragraph: "Line 1 Line 2 Line 3"
-      - text: "Line 1 Line 2 Line 3"
   `);
   await expect(page.locator('body')).toMatchAriaSnapshot(`
-    - paragraph:
+    - paragraph: |
-      - text: |
           Line 1
           Line 2
           Line 3
@@ -388,8 +382,7 @@ it('should include pseudo codepoints', async ({ page, server }) => {
   `);
 
   await checkAndMatchSnapshot(page.locator('body'), `
-    - paragraph:
+    - paragraph: "\ueab2hello"
-      - text: "\ueab2hello"
   `);
 });
 
@@ -403,7 +396,6 @@ it('check aria-hidden text', async ({ page, server }) => {
   `);
 
   await checkAndMatchSnapshot(page.locator('body'), `
-    - paragraph:
+    - paragraph: "hello"
-      - text: "hello"
   `);
 });

tests/page/to-match-aria-snapshot.spec.ts

@@ -43,8 +43,8 @@ test('should match list with accessible name', async ({ page }) => {
   `);
   await expect(page.locator('body')).toMatchAriaSnapshot(`
     - list "my list":
-      - listitem: one
+      - listitem: "one"
-      - listitem: two
+      - listitem: "two"
   `);
 });
 
@@ -90,7 +90,7 @@ test('should allow text nodes', async ({ page }) => {
 
   await expect(page.locator('body')).toMatchAriaSnapshot(`
     - heading "Microsoft"
-    - text: Open source projects and samples from Microsoft
+    - text: "Open source projects and samples from Microsoft"
   `);
 });
 
@@ -103,7 +103,7 @@ test('details visibility', async ({ page }) => {
   `);
 
   await expect(page.locator('body')).toMatchAriaSnapshot(`
-    - group: Summary
+    - group: "Summary"
   `);
 });
 

tests/playwright-test/ui-mode-test-run.spec.ts

@@ -65,19 +65,19 @@ test('should run visible', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-error] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/}
+          - treeitem ${/\[icon-check\] passes/}
-          - treeitem ${/\[icon-error\] fails \d+ms/} [selected]:
+          - treeitem ${/\[icon-error\] fails/} [selected]:
             - button "Run"
             - button "Show source"
             - button "Watch"
           - treeitem "[icon-error] suite"
       - treeitem "[icon-error] b.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/}
+          - treeitem ${/\[icon-check\] passes/}
-          - treeitem ${/\[icon-error\] fails \d+ms/}
+          - treeitem ${/\[icon-error\] fails/}
       - treeitem "[icon-check] c.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/}
+          - treeitem ${/\[icon-check\] passes/}
           - treeitem "[icon-circle-slash] skipped"
   `);
 
@@ -125,7 +125,7 @@ test('should run on hover', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-circle-outline] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/}:
+          - treeitem ${/\[icon-check\] passes/}:
             - button "Run"
             - button "Show source"
             - button "Watch"
@@ -185,7 +185,7 @@ test('should run on Enter', async ({ runUITest }) => {
       - treeitem "[icon-error] a.test.ts" [expanded]:
         - group:
           - treeitem "[icon-circle-outline] passes"
-          - treeitem ${/\[icon-error\] fails \d+ms/} [selected]:
+          - treeitem ${/\[icon-error\] fails/} [selected]:
             - button "Run"
             - button "Show source"
             - button "Watch"
@@ -225,19 +225,19 @@ test('should run by project', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-error] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/}
+          - treeitem ${/\[icon-check\] passes/}
-          - treeitem ${/\[icon-error\] fails \d+ms/} [selected]:
+          - treeitem ${/\[icon-error\] fails/} [selected]:
             - button "Run"
             - button "Show source"
             - button "Watch"
           - treeitem "[icon-error] suite"
       - treeitem "[icon-error] b.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/}
+          - treeitem ${/\[icon-check\] passes/}
-          - treeitem ${/\[icon-error\] fails \d+ms/}
+          - treeitem ${/\[icon-error\] fails/}
       - treeitem "[icon-check] c.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/}
+          - treeitem ${/\[icon-check\] passes/}
           - treeitem "[icon-circle-slash] skipped"
   `);
 
@@ -299,14 +299,14 @@ test('should run by project', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-error] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-circle-outline\] passes \d+ms/} [expanded] [selected]:
+          - treeitem ${/\[icon-circle-outline\] passes/} [expanded] [selected]:
             - button "Run"
             - button "Show source"
             - button "Watch"
             - group:
-              - treeitem ${/\[icon-check\] foo \d+ms/}
+              - treeitem ${/\[icon-check\] foo/}
               - treeitem ${/\[icon-circle-outline\] bar/}
-          - treeitem ${/\[icon-error\] fails \d+ms/}
+          - treeitem ${/\[icon-error\] fails/}
   `);
 
   await expect(page.getByText('Projects: foo bar')).toBeVisible();
@@ -333,17 +333,17 @@ test('should run by project', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-error] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/} [expanded]:
+          - treeitem ${/\[icon-check\] passes/} [expanded]:
             - group:
-              - treeitem ${/\[icon-check\] foo \d+ms/}
+              - treeitem ${/\[icon-check\] foo/}
-              - treeitem ${/\[icon-check\] bar \d+ms/}
+              - treeitem ${/\[icon-check\] bar/}
-          - treeitem ${/\[icon-error\] fails \d+ms/} [expanded]:
+          - treeitem ${/\[icon-error\] fails/} [expanded]:
             - group:
-              - treeitem ${/\[icon-error\] foo \d+ms/} [selected]:
+              - treeitem ${/\[icon-error\] foo/} [selected]:
                 - button "Run"
                 - button "Show source"
                 - button "Watch"
-              - treeitem ${/\[icon-error\] bar \d+ms/}
+              - treeitem ${/\[icon-error\] bar/}
           - treeitem ${/\[icon-error\] suite/}
       - treeitem "[icon-error] b.test.ts" [expanded]:
         - group:
@@ -385,7 +385,7 @@ test('should stop', async ({ runUITest }) => {
       - treeitem "[icon-loading] a.test.ts" [expanded]:
         - group:
           - treeitem "[icon-circle-slash] test 0"
-          - treeitem ${/\[icon-check\] test 1 \d+ms/}
+          - treeitem ${/\[icon-check\] test 1/}
           - treeitem ${/\[icon-loading\] test 2/}
           - treeitem ${/\[icon-clock\] test 3/}
   `);
@@ -408,7 +408,7 @@ test('should stop', async ({ runUITest }) => {
       - treeitem "[icon-circle-outline] a.test.ts" [expanded]:
         - group:
           - treeitem "[icon-circle-slash] test 0"
-          - treeitem ${/\[icon-check\] test 1 \d+ms/}
+          - treeitem ${/\[icon-check\] test 1/}
           - treeitem ${/\[icon-circle-outline\] test 2/}
           - treeitem ${/\[icon-circle-outline\] test 3/}
   `);
@@ -478,19 +478,19 @@ test('should show time', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-error] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/}
+          - treeitem ${/\[icon-check\] passes \d+m?s/}
-          - treeitem ${/\[icon-error\] fails \d+ms/} [selected]:
+          - treeitem ${/\[icon-error\] fails \d+m?s/} [selected]:
             - button "Run"
             - button "Show source"
             - button "Watch"
           - treeitem "[icon-error] suite"
       - treeitem "[icon-error] b.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/}
+          - treeitem ${/\[icon-check\] passes \d+m?s/}
-          - treeitem ${/\[icon-error\] fails \d+ms/}
+          - treeitem ${/\[icon-error\] fails \d+m?s/}
       - treeitem "[icon-check] c.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] passes \d+ms/}
+          - treeitem ${/\[icon-check\] passes \d+m?s/}
           - treeitem "[icon-circle-slash] skipped"
   `);
 
@@ -522,7 +522,7 @@ test('should show test.fail as passing', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-check] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] should fail \d+ms/}
+          - treeitem ${/\[icon-check\] should fail \d+m?s/}
   `);
 
   await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)');
@@ -558,7 +558,7 @@ test('should ignore repeatEach', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-check] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] should pass \d+ms/}
+          - treeitem ${/\[icon-check\] should pass/}
   `);
 
   await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)');
@@ -593,7 +593,7 @@ test('should remove output folder before test run', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-check] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] should pass \d+ms/}
+          - treeitem ${/\[icon-check\] should pass/}
   `);
 
   await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)');
@@ -608,7 +608,7 @@ test('should remove output folder before test run', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-check] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] should pass \d+ms/}
+          - treeitem ${/\[icon-check\] should pass/}
   `);
 
   await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)');
@@ -656,7 +656,7 @@ test('should show proper total when using deps', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-circle-outline] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] run @setup setup \d+ms/} [selected]:
+          - treeitem ${/\[icon-check\] run @setup setup/} [selected]:
             - button "Run"
             - button "Show source"
             - button "Watch"
@@ -676,8 +676,8 @@ test('should show proper total when using deps', async ({ runUITest }) => {
     - tree:
       - treeitem "[icon-check] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] run @setup setup \d+ms/}
+          - treeitem ${/\[icon-check\] run @setup setup/}
-          - treeitem ${/\[icon-check\] run @chromium chromium \d+ms/} [selected]:
+          - treeitem ${/\[icon-check\] run @chromium chromium/} [selected]:
             - button "Run"
             - button "Show source"
             - button "Watch"
@@ -746,7 +746,7 @@ test('should respect --tsconfig option', {
     - tree:
       - treeitem "[icon-check] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] test \d+ms/}
+          - treeitem ${/\[icon-check\] test/}
   `);
 
   await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)');
@@ -775,7 +775,7 @@ test('should respect --ignore-snapshots option', {
     - tree:
       - treeitem "[icon-check] a.test.ts" [expanded]:
         - group:
-          - treeitem ${/\[icon-check\] snapshot \d+ms/}
+          - treeitem ${/\[icon-check\] snapshot/}
   `);
 });
 

tests/playwright-test/update-aria-snapshot.spec.ts

@@ -0,0 +1,48 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import { test, expect } from './playwright-test-fixtures';
+
+test('should update snapshot with the update-snapshots flag', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test', async ({ page }) => {
+        await page.setContent(\`<h1>hello</h1>\`);
+        await expect(page.locator('body')).toMatchAriaSnapshot(\`
+          - heading "world"
+        \`);
+      });
+    `
+  }, { 'update-snapshots': true });
+
+  expect(result.exitCode).toBe(0);
+  const patchPath = testInfo.outputPath('test-results/rebaselines.patch');
+  const data = fs.readFileSync(patchPath, 'utf-8');
+  expect(data).toBe(`--- a/a.spec.ts
++++ b/a.spec.ts
+@@ -3,7 +3,7 @@
+       test('test', async ({ page }) => {
+         await page.setContent(\`<h1>hello</h1>\`);
+         await expect(page.locator('body')).toMatchAriaSnapshot(\`
+-          - heading "world"
++          - heading "hello" [level=1]
+         \`);
+       });
+     
+`);
+});