feat: markdown report (#23819)

Simple markdown report generator. Can be used when posting updates to
GitHub PRs:

<img width="632" alt="image"
src="https://github.com/microsoft/playwright/assets/9798949/ea7e4bfa-477a-4cad-9cc3-00b1f8a314b1">

packages/playwright-test/src/common/config.ts

@@ -259,7 +259,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] |
   return reporters;
 }
 
-export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob'] as const;
+export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'markdown'] as const;
 export type BuiltInReporter = typeof builtInReporters[number];
 
 export type ContextReuseMode = 'none' | 'force' | 'when-possible';

packages/playwright-test/src/reporters/markdown.ts

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import type { FullResult, TestCase } from '../../types/testReporter';
+import { BaseReporter, formatTestTitle } from './base';
+
+type MarkdownReporterOptions = {
+  configDir: string,
+  outputFile?: string;
+};
+
+
+class MarkdownReporter extends BaseReporter {
+  private _options: MarkdownReporterOptions;
+
+  constructor(options: MarkdownReporterOptions) {
+    super();
+    this._options = options;
+  }
+
+  printsToStdio() {
+    return false;
+  }
+
+  override async onEnd(result: FullResult) {
+    await super.onEnd(result);
+    const summary = this.generateSummary();
+    const lines: string[] = [];
+    lines.push(`:x: <b>failed: ${summary.unexpected.length}</b>`);
+    this._printTestList(summary.unexpected, lines);
+    if (summary.flaky.length) {
+      lines.push(`:warning: <b>flaky: ${summary.flaky.length}</b>`);
+      this._printTestList(summary.flaky, lines);
+    }
+    if (summary.interrupted.length) {
+      lines.push(`:warning: <b>interrupted: ${summary.interrupted.length}</b>`);
+      this._printTestList(summary.interrupted, lines);
+    }
+    if (summary.skipped) {
+      lines.push(`:ballot_box_with_check: <b>skipped: ${summary.skipped}</b>`);
+      lines.push(``);
+    }
+    lines.push(`:white_check_mark: <b>passed: ${summary.expected}</b>`);
+    lines.push(``);
+
+    const reportFile = path.resolve(this._options.configDir, this._options.outputFile || 'report.md');
+    await fs.promises.mkdir(path.dirname(reportFile), { recursive: true });
+    await fs.promises.writeFile(reportFile, lines.join('\n'));
+  }
+
+  private _printTestList(tests: TestCase[], lines: string[]) {
+    for (const test of tests)
+      lines.push(` - ${formatTestTitle(this.config, test)}`);
+    lines.push(``);
+  }
+}
+
+
+export default MarkdownReporter;
+

packages/playwright-test/src/runner/reporters.ts

@@ -25,6 +25,7 @@ import JSONReporter from '../reporters/json';
 import JUnitReporter from '../reporters/junit';
 import LineReporter from '../reporters/line';
 import ListReporter from '../reporters/list';
+import MarkdownReporter from '../reporters/markdown';
 import type { Suite } from '../common/test';
 import type { BuiltInReporter, FullConfigInternal } from '../common/config';
 import { loadReporter } from './loadUtils';
@@ -42,6 +43,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' |
     null: EmptyReporter,
     html: mode === 'ui' ? LineReporter : HtmlReporter,
     blob: BlobReporter,
+    markdown: MarkdownReporter,
   };
   const reporters: Reporter[] = [];
   descriptions ??= config.config.reporter;

tests/playwright-test/reporter-markdown.spec.ts

@@ -0,0 +1,101 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import { expect, test } from './playwright-test-fixtures';
+
+test('simple report', async ({ runInlineTest }) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        retries: 1,
+        reporter: 'markdown',
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 1', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('failing 1', async ({}) => {
+        expect(1).toBe(2);
+      });
+      test('flaky 1', async ({}) => {
+        expect(test.info().retry).toBe(1);
+      });
+      test.skip('skipped 1', async ({}) => {});
+    `,
+    'b.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 2', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('failing 2', async ({}) => {
+        expect(1).toBe(2);
+      });
+      test.skip('skipped 2', async ({}) => {});
+    `,
+    'c.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 3', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('flaky 2', async ({}) => {
+        expect(test.info().retry).toBe(1);
+      });
+      test.skip('skipped 3', async ({}) => {});
+    `
+  };
+  const { exitCode } = await runInlineTest(files);
+  expect(exitCode).toBe(1);
+  const reportFile = await fs.promises.readFile(test.info().outputPath('report.md'));
+  expect(reportFile.toString()).toBe(`:x: <b>failed: 2</b>
+ - a.test.js:6:11 › failing 1
+ - b.test.js:6:11 › failing 2
+
+:warning: <b>flaky: 2</b>
+ - a.test.js:9:11 › flaky 1
+ - c.test.js:6:11 › flaky 2
+
+:ballot_box_with_check: <b>skipped: 3</b>
+
+:white_check_mark: <b>passed: 3</b>
+`);
+});
+
+test('custom report file', async ({ runInlineTest }) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: [['markdown', { outputFile: 'my-report.md' }]],
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 1', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+    `,
+  };
+
+  const { exitCode } = await runInlineTest(files);
+  expect(exitCode).toBe(0);
+  const reportFile = await fs.promises.readFile(test.info().outputPath('my-report.md'));
+  expect(reportFile.toString()).toBe(`:x: <b>failed: 0</b>
+
+:white_check_mark: <b>passed: 1</b>
+`);
+});