diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 479f08ce3a..2868efad27 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -719,3 +719,16 @@ const config: PlaywrightTestConfig = { }; export default config; ``` + +## property: TestConfig.attachments +- type: <[Array]<[Object]>> + - `name` <[string]> Attachment name. + - `contentType` <[string]> Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. + - `path` <[void]|[string]> Optional path on the filesystem to the attached file. + - `body` <[void]|[Buffer]> Optional attachment body used instead of a file. + +:::note +This does not include test-level attachments. See [`method: TestInfo.attach`] and [`property: TestInfo.attachments`] for working with test-level attachments. +::: + +The list of files or buffers attached for the overall Playwright Test run. Some reporters show attachments. diff --git a/packages/html-reporter/src/icons.tsx b/packages/html-reporter/src/icons.tsx index e4c6135bf3..a5ebcb0898 100644 --- a/packages/html-reporter/src/icons.tsx +++ b/packages/html-reporter/src/icons.tsx @@ -69,3 +69,19 @@ export const clock = () => { export const blank = () => { return ; }; + +export const externalLink = () => { + return ; +}; + +export const calendar = () => { + return ; +}; + +export const person = () => { + return ; +}; + +export const commit = () => { + return ; +}; diff --git a/packages/html-reporter/src/index.tsx b/packages/html-reporter/src/index.tsx index adc36d50c8..f227b58809 100644 --- a/packages/html-reporter/src/index.tsx +++ b/packages/html-reporter/src/index.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { HTMLReport } from '@playwright/test/src/reporters/html'; +import type { HTMLReport, TestAttachment } from '@playwright/test/src/reporters/html'; import type zip from '@zip.js/zip.js'; import * as React from 'react'; import * as ReactDOM from 'react-dom'; @@ -22,6 +22,59 @@ import './colors.css'; import { LoadedReport } from './loadedReport'; import { ReportView } from './reportView'; +export type Metadata = Partial<{ + 'generatedAt': number; + 'revision.id': string; + 'revision.author': string; + 'revision.email': string; + 'revision.subject': string; + 'revision.timestamp': number; + 'revision.link': string; + 'revision.localPendingChanges': boolean; + 'ci.link': string; +}>; + +const extractMetadata = (attachments: TestAttachment[]): Metadata | undefined => { + // The last plugin to register for a given key will take precedence + attachments = [...attachments]; + attachments.reverse(); + const field = (name: string) => attachments.find(({ name: n }) => n === name)?.body; + const fieldAsJSON = (name: string) => { + const raw = field(name); + if (raw !== undefined) + return JSON.parse(raw); + }; + const fieldAsNumber = (name: string) => { + const v = fieldAsJSON(name); + if (v !== undefined && typeof v !== 'number') + throw new Error(`Invalid value for field '${name}'. Expected type 'number', but got ${typeof v}.`); + + return v; + }; + const fieldAsBool = (name: string) => { + const v = fieldAsJSON(name); + if (v !== undefined && typeof v !== 'boolean') + throw new Error(`Invalid value for field '${name}'. Expected type 'boolean', but got ${typeof v}.`); + + return v; + }; + + const out = { + 'generatedAt': fieldAsNumber('generatedAt'), + 'revision.id': field('revision.id'), + 'revision.author': field('revision.author'), + 'revision.email': field('revision.email'), + 'revision.subject': field('revision.subject'), + 'revision.timestamp': fieldAsNumber('revision.timestamp'), + 'revision.link': field('revision.link'), + 'revision.localPendingChanges': fieldAsBool('revision.localPendingChanges'), + 'ci.link': field('ci.link'), + }; + + if (Object.entries(out).filter(([_, v]) => v !== undefined).length) + return out; +}; + const zipjs = (self as any).zip; const ReportLoader: React.FC = () => { @@ -41,13 +94,14 @@ window.onload = () => { class ZipReport implements LoadedReport { private _entries = new Map(); - private _json!: HTMLReport; + private _json!: HTMLReport & { metadata?: Metadata }; async load() { const zipReader = new zipjs.ZipReader(new zipjs.Data64URIReader(window.playwrightReportBase64), { useWebWorkers: false }) as zip.ZipReader; for (const entry of await zipReader.getEntries()) this._entries.set(entry.filename, entry); this._json = await this.entry('report.json') as HTMLReport; + this._json.metadata = extractMetadata(this._json.attachments); } json(): HTMLReport { diff --git a/packages/html-reporter/src/loadedReport.ts b/packages/html-reporter/src/loadedReport.ts index 7891d46815..ae5b930979 100644 --- a/packages/html-reporter/src/loadedReport.ts +++ b/packages/html-reporter/src/loadedReport.ts @@ -15,8 +15,9 @@ */ import { HTMLReport } from '@playwright/test/src/reporters/html'; +import { Metadata } from '.'; export interface LoadedReport { - json(): HTMLReport; + json(): HTMLReport & { metadata?: Metadata }; entry(name: string): Promise; } diff --git a/packages/html-reporter/src/reportView.css b/packages/html-reporter/src/reportView.css index 7937d4c8fd..7b987046fd 100644 --- a/packages/html-reporter/src/reportView.css +++ b/packages/html-reporter/src/reportView.css @@ -24,7 +24,6 @@ html, body { body { overflow: auto; - max-width: 1024px; margin: 0 auto; width: 100%; } @@ -33,8 +32,58 @@ body { border-top: 1px solid var(--color-border-default); } +.htmlreport { + gap: 24px; + + display: flex; + flex-direction: row; + justify-content: center; +} + +.htmlreport header { + width: 300px; + order: 2; +} + +.htmlreport main { + max-width: 1024px; + width: 100%; + order: 1; +} + +.metadata-view a { + color: var(--color-accent-fg); + text-decoration: none; +} + +.metadata-view a:hover { + text-decoration: underline; +} + +.metadata-view h1 { + font-size: 16px; + font-weight: 600; + margin-bottom: 16px; + margin-top: 0; + line-height: 1.5; +} + @media only screen and (max-width: 600px) { .report { padding: 0 !important; } } + +@media only screen and (max-width: 900px) { + .htmlreport { + flex-direction: column; + } + + .htmlreport header { + order: 1; + } + + .htmlreport main { + order: 2; + } +} diff --git a/packages/html-reporter/src/reportView.tsx b/packages/html-reporter/src/reportView.tsx index f7bc5d079b..cc7977b028 100644 --- a/packages/html-reporter/src/reportView.tsx +++ b/packages/html-reporter/src/reportView.tsx @@ -26,6 +26,8 @@ import './reportView.css'; import { TestCaseView } from './testCaseView'; import { TestFilesView } from './testFilesView'; import './theme.css'; +import * as icons from './icons'; +import { Metadata } from '.'; declare global { interface Window { @@ -43,8 +45,10 @@ export const ReportView: React.FC<{ const filter = React.useMemo(() => Filter.parse(filterText), [filterText]); return
- {report?.json() && } - {<> + + {report?.json().metadata && } +
+ {report?.json() && } @@ -54,10 +58,75 @@ export const ReportView: React.FC<{ {!!report && } - } +
; }; +const MetadataView: React.FC = metadata => { + return ( +
+

{metadata['revision.subject'] || 'Playwright Test Report'}

+ {metadata['revision.id'] && + {metadata['revision.id'].slice(0, 7)}} + href={metadata['revision.link']} + icon='commit' + /> + } + {(metadata['revision.author'] || metadata['revision.email']) && + {metadata['revision.author']}
{metadata['revision.email']} + : (metadata['revision.author'] || metadata['revision.email']) + )!} + icon='person' + /> + } + {metadata['revision.timestamp'] && + + {Intl.DateTimeFormat(undefined, { dateStyle: 'full' }).format(metadata['revision.timestamp'])} +
+ {Intl.DateTimeFormat(undefined, { timeStyle: 'long' }).format(metadata['revision.timestamp'])} + + } + icon='calendar' + /> + } + {metadata['ci.link'] && + + } + {metadata['revision.localPendingChanges'] && +

This report was generated with uncommitted changes.

+ } + {metadata['generatedAt'] && +

Report generated on {Intl.DateTimeFormat(undefined, { dateStyle: 'full', timeStyle: 'long' }).format(metadata['generatedAt'])}

+ } +
+ ); +}; + +const MetadatViewItem: React.FC<{ content: JSX.Element | string; icon: keyof typeof icons, href?: string, testId?: string }> = ({ content, icon, href, testId }) => { + return ( +
+
+ {icons[icon]()} +
+
+ {href ? {content} : content} +
+
+ ); +}; + const TestCaseViewLoader: React.FC<{ report: LoadedReport, }> = ({ report }) => { diff --git a/packages/playwright-test/package.json b/packages/playwright-test/package.json index 0e1b12487b..ea39b13176 100644 --- a/packages/playwright-test/package.json +++ b/packages/playwright-test/package.json @@ -18,6 +18,7 @@ "./lib/cli": "./lib/cli.js", "./lib/experimentalLoader": "./lib/experimentalLoader.js", "./lib/mount": "./lib/mount.js", + "./lib/ci": "./lib/ci.js", "./reporter": "./reporter.js" }, "bin": { diff --git a/packages/playwright-test/src/ci.ts b/packages/playwright-test/src/ci.ts new file mode 100644 index 0000000000..fb0e10ca1b --- /dev/null +++ b/packages/playwright-test/src/ci.ts @@ -0,0 +1,94 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { spawnAsync } from 'playwright-core/lib/utils/utils'; + +const GIT_OPERATIONS_TIMEOUT_MS = 1500; +const kContentTypePlainText = 'text/plain'; +const kContentTypeJSON = 'application/json'; +export interface Attachment { + name: string; + contentType: string; + path?: string; + body?: Buffer; +} + +export const gitStatusFromCLI = async (gitDir: string): Promise => { + const execGit = async (args: string[]) => { + const { code, stdout } = await spawnAsync('git', args, { stdio: 'pipe', cwd: gitDir, timeout: GIT_OPERATIONS_TIMEOUT_MS }); + if (!!code) + throw new Error('Exited with non-zero code.'); + + return stdout.trim(); + }; + + await execGit(['--help']).catch(() => { throw new Error('git --help failed; is git installed?');}); + const [ status, sha, subject, authorName, authorEmail, rawTimestamp ] = await Promise.all([ + execGit(['status', '--porcelain=v1']), + execGit(['rev-parse', 'HEAD']), + execGit(['show', '-s', '--format=%s', 'HEAD']), + execGit(['show', '-s', '--format=%an', 'HEAD']), + execGit(['show', '-s', '--format=%ae', 'HEAD']), + execGit(['show', '-s', '--format=%ct', 'HEAD']), + ]).catch(() => { throw new Error('one or more git commands failed');}); + + let timestamp: number = Number.parseInt(rawTimestamp, 10); + timestamp = Number.isInteger(timestamp) ? timestamp * 1000 : 0; + + return [ + { name: 'revision.id', body: Buffer.from(sha), contentType: kContentTypePlainText }, + { name: 'revision.author', body: Buffer.from(authorName), contentType: kContentTypePlainText }, + { name: 'revision.email', body: Buffer.from(authorEmail), contentType: kContentTypePlainText }, + { name: 'revision.subject', body: Buffer.from(subject), contentType: kContentTypePlainText }, + { name: 'revision.timestamp', body: Buffer.from(JSON.stringify(timestamp)), contentType: kContentTypeJSON }, + { name: 'revision.localPendingChanges', body: Buffer.from(!!status + ''), contentType: kContentTypePlainText }, + ]; +}; + +export const githubEnv = async (): Promise => { + const out: Attachment[] = []; + if (process.env.GITHUB_SERVER_URL && process.env.GITHUB_REPOSITORY && process.env.GITHUB_SHA) + out.push({ name: 'revision.link', body: Buffer.from(`${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/commit/${process.env.GITHUB_SHA}`), contentType: kContentTypePlainText }); + + if (process.env.GITHUB_SERVER_URL && process.env.GITHUB_REPOSITORY && process.env.GITHUB_RUN_ID) + out.push({ name: 'ci.link', body: Buffer.from(`${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`), contentType: kContentTypePlainText }); + + return out; +}; + +export const gitlabEnv = async (): Promise => { + // GitLab: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html + const out: Attachment[] = []; + if (process.env.CI_PROJECT_URL && process.env.CI_COMMIT_SHA) + out.push({ name: 'revision.link', body: Buffer.from(`${process.env.CI_PROJECT_URL}/-/commit/${process.env.CI_COMMIT_SHA}`), contentType: kContentTypePlainText }); + + if (process.env.CI_JOB_URL) + out.push({ name: 'ci.link', body: Buffer.from(process.env.CI_JOB_URL), contentType: kContentTypePlainText }); + + return out; +}; + +export const jenkinsEnv = async (): Promise => { + // Jenkins: https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#using-environment-variables + const out: Attachment[] = []; + if (process.env.BUILD_URL) + out.push({ name: 'ci.link', body: Buffer.from(process.env.BUILD_URL), contentType: kContentTypePlainText }); + + return out; +}; + +export const generationTimestamp = async (): Promise => { + return [{ name: 'generatedAt', body: Buffer.from(JSON.stringify(Date.now())), contentType: kContentTypeJSON }]; +}; diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 3b10f9b2ca..e8265f415b 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -456,6 +456,7 @@ const baseFullConfig: FullConfig = { version: require('../package.json').version, workers: 1, webServer: null, + attachments: [], }; function resolveReporters(reporters: Config['reporter'], rootDir: string): ReporterDescription[]|undefined { diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index 0edf1917bc..353473e4b7 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -22,7 +22,7 @@ import { Transform, TransformCallback } from 'stream'; import { FullConfig, Suite, Reporter } from '../../types/testReporter'; import { HttpServer } from 'playwright-core/lib/utils/httpServer'; import { calculateSha1, removeFolders } from 'playwright-core/lib/utils/utils'; -import RawReporter, { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from './raw'; +import RawReporter, { JsonAttachment, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from './raw'; import assert from 'assert'; import yazl from 'yazl'; import { stripAnsiEscapes } from './base'; @@ -44,6 +44,7 @@ export type Location = { }; export type HTMLReport = { + attachments: TestAttachment[]; files: TestFileSummary[]; stats: Stats; projectNames: string[]; @@ -145,7 +146,7 @@ class HtmlReporter implements Reporter { const reportFolder = htmlReportFolder(this._outputFolder); await removeFolders([reportFolder]); const builder = new HtmlBuilder(reportFolder); - const { ok, singleTestId } = await builder.build(reports); + const { ok, singleTestId } = await builder.build(new RawReporter().generateAttachments(this.config), reports); if (process.env.CI) return; @@ -228,7 +229,7 @@ class HtmlBuilder { this._dataZipFile = new yazl.ZipFile(); } - async build(rawReports: JsonReport[]): Promise<{ ok: boolean, singleTestId: string | undefined }> { + async build(testReportAttachments: JsonAttachment[], rawReports: JsonReport[]): Promise<{ ok: boolean, singleTestId: string | undefined }> { const data = new Map(); for (const projectJson of rawReports) { @@ -284,6 +285,7 @@ class HtmlBuilder { this._addDataFile(fileId + '.json', testFile); } const htmlReport: HTMLReport = { + attachments: this._serializeAttachments(testReportAttachments), files: [...data.values()].map(e => e.testFileSummary), projectNames: rawReports.map(r => r.project.name), stats: [...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) @@ -377,8 +379,84 @@ class HtmlBuilder { }; } - private _createTestResult(result: JsonTestResult): TestResult { + private _serializeAttachments(attachments: JsonAttachment[]) { let lastAttachment: TestAttachment | undefined; + return attachments.map(a => { + if (a.name === 'trace') + this._hasTraces = true; + + if ((a.name === 'stdout' || a.name === 'stderr') && a.contentType === 'text/plain') { + if (lastAttachment && + lastAttachment.name === a.name && + lastAttachment.contentType === a.contentType) { + lastAttachment.body += stripAnsiEscapes(a.body as string); + return null; + } + a.body = stripAnsiEscapes(a.body as string); + lastAttachment = a as TestAttachment; + return a; + } + + if (a.path) { + let fileName = a.path; + try { + const buffer = fs.readFileSync(a.path); + const sha1 = calculateSha1(buffer) + path.extname(a.path); + fileName = 'data/' + sha1; + fs.mkdirSync(path.join(this._reportFolder, 'data'), { recursive: true }); + fs.writeFileSync(path.join(this._reportFolder, 'data', sha1), buffer); + } catch (e) { + return { + name: `Missing attachment "${a.name}"`, + contentType: kMissingContentType, + body: `Attachment file ${fileName} is missing`, + }; + } + return { + name: a.name, + contentType: a.contentType, + path: fileName, + body: a.body, + }; + } + + if (a.body instanceof Buffer) { + if (isTextContentType(a.contentType)) { + // Content type is like this: "text/html; charset=UTF-8" + const charset = a.contentType.match(/charset=(.*)/)?.[1]; + try { + const body = a.body.toString(charset as any || 'utf-8'); + return { + name: a.name, + contentType: a.contentType, + body, + }; + } catch (e) { + // Invalid encoding, fall through and save to file. + } + } + + fs.mkdirSync(path.join(this._reportFolder, 'data'), { recursive: true }); + const sha1 = calculateSha1(a.body) + '.dat'; + fs.writeFileSync(path.join(this._reportFolder, 'data', sha1), a.body); + return { + name: a.name, + contentType: a.contentType, + path: 'data/' + sha1, + body: a.body, + }; + } + + // string + return { + name: a.name, + contentType: a.contentType, + body: a.body, + }; + }).filter(Boolean) as TestAttachment[]; + } + + private _createTestResult(result: JsonTestResult): TestResult { return { duration: result.duration, startTime: result.startTime, @@ -386,79 +464,7 @@ class HtmlBuilder { steps: result.steps.map(s => this._createTestStep(s)), errors: result.errors, status: result.status, - attachments: result.attachments.map(a => { - if (a.name === 'trace') - this._hasTraces = true; - - if ((a.name === 'stdout' || a.name === 'stderr') && a.contentType === 'text/plain') { - if (lastAttachment && - lastAttachment.name === a.name && - lastAttachment.contentType === a.contentType) { - lastAttachment.body += stripAnsiEscapes(a.body as string); - return null; - } - a.body = stripAnsiEscapes(a.body as string); - lastAttachment = a as TestAttachment; - return a; - } - - if (a.path) { - let fileName = a.path; - try { - const buffer = fs.readFileSync(a.path); - const sha1 = calculateSha1(buffer) + path.extname(a.path); - fileName = 'data/' + sha1; - fs.mkdirSync(path.join(this._reportFolder, 'data'), { recursive: true }); - fs.writeFileSync(path.join(this._reportFolder, 'data', sha1), buffer); - } catch (e) { - return { - name: `Missing attachment "${a.name}"`, - contentType: kMissingContentType, - body: `Attachment file ${fileName} is missing`, - }; - } - return { - name: a.name, - contentType: a.contentType, - path: fileName, - body: a.body, - }; - } - - if (a.body instanceof Buffer) { - if (isTextContentType(a.contentType)) { - // Content type is like this: "text/html; charset=UTF-8" - const charset = a.contentType.match(/charset=(.*)/)?.[1]; - try { - const body = a.body.toString(charset as any || 'utf-8'); - return { - name: a.name, - contentType: a.contentType, - body, - }; - } catch (e) { - // Invalid encoding, fall through and save to file. - } - } - - fs.mkdirSync(path.join(this._reportFolder, 'data'), { recursive: true }); - const sha1 = calculateSha1(a.body) + '.dat'; - fs.writeFileSync(path.join(this._reportFolder, 'data', sha1), a.body); - return { - name: a.name, - contentType: a.contentType, - path: 'data/' + sha1, - body: a.body, - }; - } - - // string - return { - name: a.name, - contentType: a.contentType, - body: a.body, - }; - }).filter(Boolean) as TestAttachment[] + attachments: this._serializeAttachments(result.attachments), }; } diff --git a/packages/playwright-test/src/reporters/json.ts b/packages/playwright-test/src/reporters/json.ts index e0370b529d..051285284b 100644 --- a/packages/playwright-test/src/reporters/json.ts +++ b/packages/playwright-test/src/reporters/json.ts @@ -20,7 +20,13 @@ import { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResul import { prepareErrorStack } from './base'; export interface JSONReport { - config: Omit & { + config: Omit & { + attachments: { + name: string; + path?: string; + body?: string; + contentType: string; + }[]; projects: { outputDir: string, repeatEach: number, @@ -121,6 +127,12 @@ class JSONReporter implements Reporter { return { config: { ...this.config, + attachments: this.config.attachments.map(a => ({ + name: a.name, + contentType: a.contentType, + path: a.path, + body: a.body?.toString('base64') + })), rootDir: toPosixPath(this.config.rootDir), projects: this.config.projects.map(project => { return { diff --git a/packages/playwright-test/src/reporters/raw.ts b/packages/playwright-test/src/reporters/raw.ts index 0a48354112..2f71acb2af 100644 --- a/packages/playwright-test/src/reporters/raw.ts +++ b/packages/playwright-test/src/reporters/raw.ts @@ -34,7 +34,7 @@ export type JsonReport = { suites: JsonSuite[], }; -export type JsonConfig = Omit; +export type JsonConfig = Omit; export type JsonProject = { metadata: any, @@ -133,6 +133,10 @@ class RawReporter { } } + generateAttachments(config: FullConfig): JsonAttachment[] { + return this._createAttachments(config.attachments); + } + generateProjectReport(config: FullConfig, suite: Suite): JsonReport { this.config = config; const project = suite.project(); @@ -223,7 +227,7 @@ class RawReporter { duration: result.duration, status: result.status, errors: formatResultFailure(this.config, test, result, '', true).map(error => error.message), - attachments: this._createAttachments(result), + attachments: this._createAttachments(result.attachments, result), steps: dedupeSteps(result.steps.map(step => this._serializeStep(test, step))) }; } @@ -245,17 +249,17 @@ class RawReporter { return result; } - private _createAttachments(result: TestResult): JsonAttachment[] { - const attachments: JsonAttachment[] = []; - for (const attachment of result.attachments) { + private _createAttachments(attachments: TestResult['attachments'], ioStreams?: Pick): JsonAttachment[] { + const out: JsonAttachment[] = []; + for (const attachment of attachments) { if (attachment.body) { - attachments.push({ + out.push({ name: attachment.name, contentType: attachment.contentType, body: attachment.body }); } else if (attachment.path) { - attachments.push({ + out.push({ name: attachment.name, contentType: attachment.contentType, path: attachment.path @@ -263,11 +267,14 @@ class RawReporter { } } - for (const chunk of result.stdout) - attachments.push(this._stdioAttachment(chunk, 'stdout')); - for (const chunk of result.stderr) - attachments.push(this._stdioAttachment(chunk, 'stderr')); - return attachments; + if (ioStreams) { + for (const chunk of ioStreams.stdout) + out.push(this._stdioAttachment(chunk, 'stdout')); + for (const chunk of ioStreams.stderr) + out.push(this._stdioAttachment(chunk, 'stderr')); + } + + return out; } private _stdioAttachment(chunk: Buffer | string, type: 'stdout' | 'stderr'): JsonAttachment { diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 00e5d40b1c..6fa1249794 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -1268,6 +1268,15 @@ export interface FullConfig { * */ webServer: WebServerConfig | null; + /** + * > NOTE: This does not include test-level attachments. See + * [testInfo.attach(name[, options])](https://playwright.dev/docs/api/class-testinfo#test-info-attach) and + * [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments) for working with test-level + * attachments. + * + * The list of files or buffers attached for the overall Playwright Test run. Some reporters show attachments. + */ + attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; } export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped'; diff --git a/tests/config/android.config.ts b/tests/config/android.config.ts index f7b3108fb0..c2f0a19dce 100644 --- a/tests/config/android.config.ts +++ b/tests/config/android.config.ts @@ -26,6 +26,7 @@ process.env.PWPAGE_IMPL = 'android'; const outputDir = path.join(__dirname, '..', '..', 'test-results'); const testDir = path.join(__dirname, '..'); const config: Config = { + globalSetup: path.join(__dirname, './globalSetup'), testDir, outputDir, timeout: 120000, diff --git a/tests/config/default.playwright.config.ts b/tests/config/default.playwright.config.ts index 3d8bbf79a0..d44b2d05b8 100644 --- a/tests/config/default.playwright.config.ts +++ b/tests/config/default.playwright.config.ts @@ -44,6 +44,7 @@ const trace = !!process.env.PWTEST_TRACE; const outputDir = path.join(__dirname, '..', '..', 'test-results'); const testDir = path.join(__dirname, '..'); const config: Config = { + globalSetup: path.join(__dirname, './globalSetup'), testDir, outputDir, expect: { diff --git a/tests/config/electron.config.ts b/tests/config/electron.config.ts index 335286556e..1ae360c339 100644 --- a/tests/config/electron.config.ts +++ b/tests/config/electron.config.ts @@ -26,6 +26,7 @@ process.env.PWPAGE_IMPL = 'electron'; const outputDir = path.join(__dirname, '..', '..', 'test-results'); const testDir = path.join(__dirname, '..'); const config: Config = { + globalSetup: path.join(__dirname, './globalSetup'), testDir, outputDir, timeout: 30000, diff --git a/tests/config/globalSetup.ts b/tests/config/globalSetup.ts new file mode 100644 index 0000000000..b148fefa5d --- /dev/null +++ b/tests/config/globalSetup.ts @@ -0,0 +1,33 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { FullConfig } from '@playwright/test'; + +// We're dogfooding this, so the …/lib/… import is acceptable +import * as ci from '@playwright/test/lib/ci'; + +async function globalSetup(config: FullConfig) { + config.attachments = [ + ...await ci.generationTimestamp(), + ...await ci.gitStatusFromCLI(config.rootDir).catch(() => []), + ...await ci.githubEnv(), + // In the future, we would add some additional plugins like: + // ...await ci.azurePipelinePlugin(), + // (and these would likley all get bundled into one call and controlled with one config instead + // of manually manipulating the attachments array) + ]; +} + +export default globalSetup; diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index cddf33fcb4..30b2cc9612 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -120,7 +120,17 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b ...process.env, PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile, PWTEST_CACHE_DIR: cacheDir, + // BEGIN: Reserved CI CI: undefined, + BUILD_URL: undefined, + CI_COMMIT_SHA: undefined, + CI_JOB_URL: undefined, + CI_PROJECT_URL: undefined, + GITHUB_REPOSITORY: undefined, + GITHUB_RUN_ID: undefined, + GITHUB_SERVER_URL: undefined, + GITHUB_SHA: undefined, + // END: Reserved CI PW_TEST_HTML_REPORT_OPEN: undefined, PLAYWRIGHT_DOCKER: undefined, PW_GRID: undefined, @@ -199,7 +209,7 @@ type RunOptions = { }; type Fixtures = { writeFiles: (files: Files) => Promise; - runInlineTest: (files: Files, params?: Params, env?: Env, options?: RunOptions) => Promise; + runInlineTest: (files: Files, params?: Params, env?: Env, options?: RunOptions, beforeRunPlaywrightTest?: ({ baseDir }: { baseDir: string }) => Promise) => Promise; runTSC: (files: Files) => Promise; }; @@ -212,8 +222,10 @@ export const test = base }, runInlineTest: async ({ childProcess }, use, testInfo: TestInfo) => { - await use(async (files: Files, params: Params = {}, env: Env = {}, options: RunOptions = {}) => { + await use(async (files: Files, params: Params = {}, env: Env = {}, options: RunOptions = {}, beforeRunPlaywrightTest?: ({ baseDir: string }) => Promise) => { const baseDir = await writeFiles(testInfo, files); + if (beforeRunPlaywrightTest) + await beforeRunPlaywrightTest({ baseDir }); return await runPlaywrightTest(childProcess, baseDir, params, env, options); }); }, diff --git a/tests/playwright-test/reporter-attachment.spec.ts b/tests/playwright-test/reporter-attachment.spec.ts index bfc12005e8..9f9c6e9757 100644 --- a/tests/playwright-test/reporter-attachment.spec.ts +++ b/tests/playwright-test/reporter-attachment.spec.ts @@ -186,3 +186,34 @@ test(`testInfo.attach allow empty buffer body`, async ({ runInlineTest }) => { expect(result.failed).toBe(1); expect(stripAnsi(result.output)).toMatch(/^.*attachment #1: name \(text\/plain\).*\n.*\n.*------/gm); }); + +test(`TestConfig.attachments works`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'globalSetup.ts': ` + import { FullConfig } from '@playwright/test'; + + async function globalSetup(config: FullConfig) { + config.attachments = [{ contentType: 'text/plain', body: Buffer.from('example data'), name: 'my-attachment.txt' }]; + }; + + export default globalSetup; + `, + 'playwright.config.ts': ` + import path from 'path'; + const config = { + globalSetup: path.join(__dirname, './globalSetup'), + } + + export default config; + `, + 'example.spec.ts': ` + const { test } = pwt; + test('sample', async ({}) => { expect(2).toBe(2); }); + `, + }, { reporter: 'json' }); + + expect(result.exitCode).toBe(0); + expect(result.report.config.attachments).toHaveLength(1); + expect(result.report.config.attachments[0].name).toBe('my-attachment.txt'); + expect(Buffer.from(result.report.config.attachments[0].body, 'base64').toString()).toBe('example data'); +}); diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index b9420f9d5d..e7cc5e312e 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -17,6 +17,7 @@ import { test as baseTest, expect, createImage } from './playwright-test-fixtures'; import { HttpServer } from '../../packages/playwright-core/lib/utils/httpServer'; import { startHtmlReportServer } from '../../packages/playwright-test/lib/reporters/html'; +import { spawnAsync } from 'playwright-core/lib/utils/utils'; const test = baseTest.extend<{ showReport: () => Promise }>({ showReport: async ({ page }, use, testInfo) => { @@ -65,6 +66,8 @@ test('should generate report', async ({ runInlineTest, showReport, page }) => { await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toBeVisible(); await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toBeVisible(); await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toBeVisible(); + + await expect(page.locator('.metadata-view')).not.toBeVisible(); }); test('should not throw when attachment is missing', async ({ runInlineTest, page, showReport }, testInfo) => { @@ -624,3 +627,64 @@ test('open tests from required file', async ({ runInlineTest, showReport, page } /expect\.toBe/, ]); }); + +test('should include metadata', async ({ runInlineTest, showReport, page }) => { + const beforeRunPlaywrightTest = async ({ baseDir }: { baseDir: string }) => { + const execGit = async (args: string[]) => { + const { code, stdout, stderr } = await spawnAsync('git', args, { stdio: 'pipe', cwd: baseDir }); + if (!!code) + throw new Error(`Non-zero exit of:\n$ git ${args.join(' ')}\nConsole:\nstdout:\n${stdout}\n\nstderr:\n${stderr}\n\n`); + return; + }; + + await execGit(['init']); + await execGit(['config', '--local', 'user.email', 'shakespeare@example.local']); + await execGit(['config', '--local', 'user.name', 'William']); + await execGit(['add', '*.ts']); + await execGit(['commit', '-m', 'awesome commit message']); + }; + + const result = await runInlineTest({ + 'uncommitted.txt': `uncommitted file`, + 'globalSetup.ts': ` + import * as ci from '@playwright/test/lib/ci'; + import { FullConfig } from '@playwright/test'; + + async function globalSetup(config: FullConfig) { + config.attachments = [ + ...await ci.generationTimestamp(), + ...await ci.gitStatusFromCLI(config.rootDir).catch(() => []), + ...await ci.githubEnv(), + ]; + }; + + export default globalSetup; + `, + 'playwright.config.ts': ` + import path from 'path'; + const config = { + globalSetup: path.join(__dirname, './globalSetup'), + } + + export default config; + `, + 'example.spec.ts': ` + const { test } = pwt; + test('sample', async ({}) => { expect(2).toBe(2); }); + `, + }, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never', GITHUB_REPOSITORY: 'microsoft/playwright-example-for-test', GITHUB_RUN_ID: 'example-run-id', GITHUB_SERVER_URL: 'https://playwright.dev', GITHUB_SHA: 'example-sha' }, undefined, beforeRunPlaywrightTest); + + await showReport(); + + expect(result.exitCode).toBe(0); + const metadata = page.locator('.metadata-view'); + await expect.soft(metadata.locator('data-test-id=revision.id')).toContainText(/^[a-f\d]{7}$/i); + await expect.soft(metadata.locator('data-test-id=revision.id >> a')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/commit/example-sha'); + await expect.soft(metadata.locator('data-test-id=revision.timestamp')).toContainText(/AM|PM/); + await expect.soft(metadata).toContainText('awesome commit message'); + await expect.soft(metadata).toContainText('William'); + await expect.soft(metadata).toContainText('shakespeare@example.local'); + await expect.soft(metadata.locator('text=CI/CD Logs')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/actions/runs/example-run-id'); + await expect.soft(metadata).toContainText('uncommitted changes'); + await expect.soft(metadata.locator('text=Report generated on')).toContainText(/AM|PM/); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index fefc8003d0..7ef6a7c42c 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -211,6 +211,7 @@ export interface FullConfig { updateSnapshots: UpdateSnapshots; workers: number; webServer: WebServerConfig | null; + attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; } export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';