From a93cf767a1f053d1ca8cfb59f8e21401090087b3 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 7 Feb 2023 22:21:50 -0800 Subject: [PATCH] feat: html reporter sharded option (#20737) Make sharded report feature an opt-in: ```ts { reporter: [['html', { sharded: true }]] }; ``` #10437 --- docs/src/test-reporters-js.md | 31 +++++++- .../playwright-test/src/reporters/html.ts | 10 ++- packages/playwright-test/types/test.d.ts | 2 +- tests/playwright-test/reporter-html.spec.ts | 76 ++++++++++++++++--- utils/generate_types/overrides-test.d.ts | 2 +- 5 files changed, 104 insertions(+), 17 deletions(-) diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 237e9e8447..e684e2af00 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -310,8 +310,37 @@ Or if there is a custom folder name: npx playwright show-report my-report ``` -> The `html` reporter currently does not support merging reports generated across multiple [`--shards`](./test-parallel.md#shard-tests-between-multiple-machines) into a single report. See [this](https://github.com/microsoft/playwright/issues/10437) issue for available third party solutions. +#### Sharded report +When running tests on [multiple shards](./test-parallel.md#shard-tests-between-multiple-machines), the `html` reporter can automatically show test results from all shards in one page when configured with `sharded: true`. + +```js tab=js-js +// playwright.config.js +// @ts-check + +const { defineConfig } = require('@playwright/test'); + +module.exports = defineConfig({ + reporter: [['html', { sharded: true }]], +}); +``` + +```js tab=js-ts +// playwright.config.ts +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: [['html', { sharded: true }]], +}); +``` + +You can use sharded html report combined with a file hosting that allows serving html files. + +In your CI recipe, after running tests in each shard, upload all files from `playwright-report` directory to the **same location**. After that you can open `index.html` from the uploaded location directly in the browser. + +:::note +The `html` report for each shard consists of `index.html` and a data file named like `report-003-of-100.zip`. It's ok to overwrite `index.html` with one another when copying sharded reports to a single directory. +::: ### JSON reporter diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index 63327531aa..cd5626d849 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -43,6 +43,7 @@ type HtmlReportOpenOption = 'always' | 'never' | 'on-failure'; type HtmlReporterOptions = { outputFolder?: string, open?: HtmlReportOpenOption, + sharded?: boolean, host?: string, port?: number, }; @@ -53,6 +54,7 @@ class HtmlReporter implements Reporter { private _montonicStartTime: number = 0; private _options: HtmlReporterOptions; private _outputFolder!: string; + private _sharded!: boolean; private _open: string | undefined; private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined; @@ -67,8 +69,9 @@ class HtmlReporter implements Reporter { onBegin(config: FullConfig, suite: Suite) { this._montonicStartTime = monotonicTime(); this.config = config as FullConfigInternal; - const { outputFolder, open } = this._resolveOptions(); + const { outputFolder, open, sharded } = this._resolveOptions(); this._outputFolder = outputFolder; + this._sharded = sharded; this._open = open; const reportedWarnings = new Set(); for (const project of config.projects) { @@ -89,19 +92,20 @@ class HtmlReporter implements Reporter { this.suite = suite; } - _resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption } { + _resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption, sharded: boolean } { let { outputFolder } = this._options; if (outputFolder) outputFolder = path.resolve(this.config._internal.configDir, outputFolder); return { outputFolder: reportFolderFromEnv() ?? outputFolder ?? defaultReportFolder(this.config._internal.configDir), open: process.env.PW_TEST_HTML_REPORT_OPEN as any || this._options.open || 'on-failure', + sharded: !!this._options.sharded }; } async onEnd() { const duration = monotonicTime() - this._montonicStartTime; - const shard = this.config.shard; + const shard = this._sharded ? this.config.shard : null; const projectSuites = this.suite.suites; const reports = projectSuites.map(suite => { const rawReporter = new RawReporter(); diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index e47502bf0d..45528e5491 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -25,7 +25,7 @@ export type ReporterDescription = ['github'] | ['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] | ['json'] | ['json', { outputFile?: string }] | - ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure' }] | + ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure', sharded?: boolean }] | ['null'] | [string] | [string, any]; diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index a9dabf1af2..53b4441cca 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -979,7 +979,11 @@ test.describe('report location', () => { test('should shard report', async ({ runInlineTest, showReport, page }, testInfo) => { const totalShards = 3; - const testFiles = {}; + const testFiles = { + 'playwright.config.ts': ` + module.exports = { reporter: [['html', { sharded: true }]] }; + `, + }; for (let i = 0; i < totalShards; i++) { testFiles[`a-${i}.spec.ts`] = ` const { test } = pwt; @@ -995,7 +999,7 @@ test('should shard report', async ({ runInlineTest, showReport, page }, testInfo for (let i = 1; i <= totalShards; i++) { const result = await runInlineTest(testFiles, - { 'reporter': 'dot,html', 'retries': 1, 'shard': `${i}/${totalShards}` }, + { 'retries': 1, 'shard': `${i}/${totalShards}` }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { usesCustomReporters: true }); @@ -1024,14 +1028,19 @@ test('should shard report', async ({ runInlineTest, showReport, page }, testInfo await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(totalShards); }); -test('should pad report numbers with zeros', async ({ runInlineTest, showReport, page }, testInfo) => { - const result = await runInlineTest({ - 'a.test.js': ` - const { test } = pwt; - test('passes', async ({}) => {}); +test('should pad report numbers with zeros', async ({ runInlineTest }, testInfo) => { + const testFiles = { + 'playwright.config.ts': ` + module.exports = { reporter: [['html', { sharded: true }]] }; `, - }, { reporter: 'dot,html', shard: '3/100' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }); - + }; + for (let i = 0; i < 100; i++) { + testFiles[`a-${i}.spec.ts`] = ` + const { test } = pwt; + test('passes', async ({}) => { }); + `; + } + const result = await runInlineTest(testFiles, { shard: '3/100' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { usesCustomReporters: true }); expect(result.exitCode).toBe(0); const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`)); expect(new Set(files)).toEqual(new Set([ @@ -1043,7 +1052,11 @@ test('should pad report numbers with zeros', async ({ runInlineTest, showReport, test('should show report with missing shards', async ({ runInlineTest, showReport, page }, testInfo) => { const totalShards = 15; - const testFiles = {}; + const testFiles = { + 'playwright.config.ts': ` + module.exports = { reporter: [['html', { sharded: true }]] }; + `, + }; for (let i = 0; i < totalShards; i++) { testFiles[`a-${String(i).padStart(2, '0')}.spec.ts`] = ` const { test } = pwt; @@ -1060,7 +1073,7 @@ test('should show report with missing shards', async ({ runInlineTest, showRepor // Run tests in 2 out of 15 shards. for (const i of [10, 13]) { const result = await runInlineTest(testFiles, - { 'reporter': 'dot,html', 'retries': 1, 'shard': `${i}/${totalShards}` }, + { 'retries': 1, 'shard': `${i}/${totalShards}` }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { usesCustomReporters: true }); @@ -1090,3 +1103,44 @@ test('should show report with missing shards', async ({ runInlineTest, showRepor await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(2); await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(2); }); + + +test('should produce single file report when shard: false', async ({ runInlineTest, showReport, page }, testInfo) => { + const totalShards = 5; + + const testFiles = {}; + for (let i = 0; i < totalShards; i++) { + testFiles[`a-${String(i).padStart(2, '0')}.spec.ts`] = ` + const { test } = pwt; + test('passes', async ({}) => { expect(2).toBe(2); }); + test('fails', async ({}) => { expect(1).toBe(2); }); + test('skipped', async ({}) => { test.skip('Does not work') }); + test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); }); + `; + } + + // Run single shard. + const currentShard = 3; + const result = await runInlineTest(testFiles, + { 'reporter': 'dot,html', 'retries': 1, 'shard': `${currentShard}/${totalShards}` }, + { PW_TEST_HTML_REPORT_OPEN: 'never' }, + { usesCustomReporters: true }); + + + expect(result.exitCode).toBe(1); + const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`)); + expect(files).toEqual(['index.html']); + + await showReport(); + + await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('4'); + await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('1'); + await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('1'); + await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('1'); + await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('1'); + + await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(1); + await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(1); + await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(1); + await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(1); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index f1a009153a..2fb1e121d0 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -24,7 +24,7 @@ export type ReporterDescription = ['github'] | ['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] | ['json'] | ['json', { outputFile?: string }] | - ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure' }] | + ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure', sharded?: boolean }] | ['null'] | [string] | [string, any];