feat(test runner): separate interrupted status from skipped (#16124)

docs/src/test-api/class-testinfo.md

@@ -162,7 +162,7 @@ Errors thrown during test execution, if any.
 
 ## property: TestInfo.expectedStatus
 * since: v1.10
-- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">>
+- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped"|"interrupted">>
 
 Expected status for the currently running test. This is usually `'passed'`, except for a few cases:
 * `'skipped'` for skipped tests, e.g. with [`method: Test.skip#2`];
@@ -461,7 +461,7 @@ Suffix used to differentiate snapshots between multiple test configurations. For
 
 ## property: TestInfo.status
 * since: v1.10
-- type: ?<[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">>
+- type: ?<[TestStatus]<"passed"|"failed"|"timedOut"|"skipped"|"interrupted">>
 
 Actual status for the currently running test. Available after the test has finished in [`method: Test.afterEach`] hook and fixtures.
 

docs/src/test-reporter-api/class-testcase.md

@@ -18,7 +18,7 @@ Learn more about [test annotations](../test-annotations.md).
 
 ## property: TestCase.expectedStatus
 * since: v1.10
-- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">>
+- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped"|"interrupted">>
 
 Expected test status.
 * Tests marked as [`method: Test.skip#1`] or [`method: Test.fixme#1`] are expected to be `'skipped'`.

docs/src/test-reporter-api/class-testresult.md

@@ -49,7 +49,7 @@ Start time of this particular test run.
 
 ## property: TestResult.status
 * since: v1.10
-- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">>
+- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped"|"interrupted">>
 
 The status of this test result. See also [`property: TestCase.expectedStatus`].
 

packages/html-reporter/src/statusIcon.tsx

@@ -18,7 +18,7 @@ import * as icons from './icons';
 import './colors.css';
 import './common.css';
 
-export function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expected' | 'unexpected' | 'flaky'): JSX.Element {
+export function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expected' | 'unexpected' | 'flaky' | 'interrupted'): JSX.Element {
   switch (status) {
     case 'failed':
     case 'unexpected':
@@ -31,6 +31,7 @@ export function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed'
     case 'flaky':
       return icons.warning();
     case 'skipped':
+    case 'interrupted':
       return icons.blank();
   }
 }

packages/playwright-test/src/reporters/base.ts

@@ -38,7 +38,7 @@ type ErrorDetails = {
 type TestSummary = {
   skipped: number;
   expected: number;
-  skippedWithError: TestCase[];
+  interrupted: TestCase[];
   unexpected: TestCase[];
   flaky: TestCase[];
   failuresToPrint: TestCase[];
@@ -131,7 +131,7 @@ export class BaseReporter implements Reporter  {
     return fileDurations.filter(([,duration]) => duration > threshold).slice(0, count);
   }
 
-  protected generateSummaryMessage({ skipped, expected, unexpected, flaky, fatalErrors }: TestSummary) {
+  protected generateSummaryMessage({ skipped, expected, interrupted, unexpected, flaky, fatalErrors }: TestSummary) {
     const tokens: string[] = [];
     if (fatalErrors.length)
       tokens.push(colors.red(`  ${fatalErrors.length} fatal ${fatalErrors.length === 1 ? 'error' : 'errors'}`));
@@ -140,6 +140,11 @@ export class BaseReporter implements Reporter  {
       for (const test of unexpected)
         tokens.push(colors.red(formatTestHeader(this.config, test, '    ')));
     }
+    if (interrupted.length) {
+      tokens.push(colors.red(`  ${interrupted.length} interrupted`));
+      for (const test of interrupted)
+        tokens.push(colors.red(formatTestHeader(this.config, test, '    ')));
+    }
     if (flaky.length) {
       tokens.push(colors.yellow(`  ${flaky.length} flaky`));
       for (const test of flaky)
@@ -158,16 +163,21 @@ export class BaseReporter implements Reporter  {
   protected generateSummary(): TestSummary {
     let skipped = 0;
     let expected = 0;
-    const skippedWithError: TestCase[] = [];
+    const interrupted: TestCase[] = [];
+    const interruptedToPrint: TestCase[] = [];
     const unexpected: TestCase[] = [];
     const flaky: TestCase[] = [];
 
     this.suite.allTests().forEach(test => {
       switch (test.outcome()) {
         case 'skipped': {
-          ++skipped;
-          if (test.results.some(result => !!result.error))
-            skippedWithError.push(test);
+          if (test.results.some(result => result.status === 'interrupted')) {
+            if (test.results.some(result => !!result.error))
+              interruptedToPrint.push(test);
+            interrupted.push(test);
+          } else {
+            ++skipped;
+          }
           break;
         }
         case 'expected': ++expected; break;
@@ -176,11 +186,11 @@ export class BaseReporter implements Reporter  {
       }
     });
 
-    const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError];
+    const failuresToPrint = [...unexpected, ...flaky, ...interruptedToPrint];
     return {
       skipped,
       expected,
-      skippedWithError,
+      interrupted,
       unexpected,
       flaky,
       failuresToPrint,
@@ -313,6 +323,11 @@ export function formatResultFailure(config: FullConfig, test: TestCase, result:
       message: indent(colors.red(`Expected to fail, but passed.`), initialIndent),
     });
   }
+  if (result.status === 'interrupted') {
+    errorDetails.push({
+      message: indent(colors.red(`Test was interrupted.`), initialIndent),
+    });
+  }
 
   for (const error of result.errors) {
     const formattedError = formatError(config, error, highlightCode, test.location.file);

packages/playwright-test/src/reporters/html.ts

@@ -104,7 +104,7 @@ export type TestResult = {
   steps: TestStep[];
   errors: string[];
   attachments: TestAttachment[];
-  status: 'passed' | 'failed' | 'timedOut' | 'skipped';
+  status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
 };
 
 export type TestStep = {

packages/playwright-test/src/reporters/line.ts

@@ -79,7 +79,7 @@ class LineReporter extends BaseReporter {
 
   override onTestEnd(test: TestCase, result: TestResult) {
     super.onTestEnd(test, result);
-    if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
+    if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected' || result.status === 'interrupted')) {
       if (!process.env.PW_TEST_DEBUG_REPORTERS)
         process.stdout.write(`\u001B[1A\u001B[2K`);
       console.log(formatFailure(this.config, test, {

packages/playwright-test/src/test.ts

@@ -159,7 +159,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
   }
 
   outcome(): 'skipped' | 'expected' | 'unexpected' | 'flaky' {
-    const nonSkipped = this.results.filter(result => result.status !== 'skipped');
+    const nonSkipped = this.results.filter(result => result.status !== 'skipped' && result.status !== 'interrupted');
     if (!nonSkipped.length)
       return 'skipped';
     if (nonSkipped.every(result => result.status === this.expectedStatus))

packages/playwright-test/src/workerRunner.ts

@@ -74,9 +74,8 @@ export class WorkerRunner extends EventEmitter {
       // Interrupt current action.
       this._currentTest?._timeoutManager.interrupt();
 
-      // TODO: mark test as 'interrupted' instead.
       if (this._currentTest && this._currentTest.status === 'passed')
-        this._currentTest.status = 'skipped';
+        this._currentTest.status = 'interrupted';
     }
     return this._runFinished;
   }

packages/playwright-test/types/test.d.ts

@@ -1294,7 +1294,7 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
   webServer: TestConfigWebServer | null;
 }
 
-export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
+export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
 
 /**
  * `WorkerInfo` contains information about the worker that is running tests. It is available to
@@ -1506,7 +1506,7 @@ export interface TestInfo {
    * ```
    *
    */
-  expectedStatus: "passed"|"failed"|"timedOut"|"skipped";
+  expectedStatus: "passed"|"failed"|"timedOut"|"skipped"|"interrupted";
 
   /**
    * Marks the currently running test as "should fail". Playwright Test runs this test and ensures that it is actually
@@ -1714,7 +1714,7 @@ export interface TestInfo {
    * ```
    *
    */
-  status?: "passed"|"failed"|"timedOut"|"skipped";
+  status?: "passed"|"failed"|"timedOut"|"skipped"|"interrupted";
 
   /**
    * Output written to `process.stderr` or `console.error` during the test execution.

tests/playwright-test/basic.spec.ts

@@ -424,7 +424,7 @@ test('should not reuse worker after unhandled rejection in test.fail', async ({
   }, { workers: 1 });
   expect(result.exitCode).toBe(1);
   expect(result.failed).toBe(1);
-  expect(result.skipped).toBe(1);
+  expect(result.interrupted).toBe(1);
   expect(result.output).toContain(`Error: Oh my!`);
   expect(result.output).not.toContain(`Did not teardown test scope`);
 });

tests/playwright-test/playwright-test-fixtures.ts

@@ -37,6 +37,7 @@ export type RunResult = {
   failed: number,
   flaky: number,
   skipped: number,
+  interrupted: number,
   report: JSONReport,
   results: any[],
 };
@@ -156,7 +157,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
   testProcess.onOutput = () => {
     if (options.sendSIGINTAfter && !didSendSigint && countTimes(testProcess.output, '%%SEND-SIGINT%%') >= options.sendSIGINTAfter) {
       didSendSigint = true;
-      process.kill(testProcess.process.pid, 'SIGINT');
+      process.kill(testProcess.process.pid!, 'SIGINT');
     }
   };
   const { exitCode } = await testProcess.exited;
@@ -176,6 +177,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
   const failed = summary(/(\d+) failed/g);
   const flaky = summary(/(\d+) flaky/g);
   const skipped = summary(/(\d+) skipped/g);
+  const interrupted = summary(/(\d+) interrupted/g);
   let report;
   try {
     report = JSON.parse(fs.readFileSync(reportFile).toString());
@@ -205,6 +207,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
     failed,
     flaky,
     skipped,
+    interrupted,
     report,
     results,
   };

tests/playwright-test/playwright.expect.text.spec.ts

@@ -657,7 +657,7 @@ test('should print expected/received on Ctrl+C', async ({ runInlineTest }) => {
   }, { workers: 1 }, {}, { sendSIGINTAfter: 1 });
   expect(result.exitCode).toBe(130);
   expect(result.passed).toBe(0);
-  expect(result.skipped).toBe(1);
+  expect(result.interrupted).toBe(1);
   expect(stripAnsi(result.output)).toContain('Expected string: "Text 2"');
   expect(stripAnsi(result.output)).toContain('Received string: "Text content"');
 });

tests/playwright-test/playwright.spec.ts

@@ -450,7 +450,7 @@ test('should report click error on sigint', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(130);
   expect(result.passed).toBe(0);
   expect(result.failed).toBe(0);
-  expect(result.skipped).toBe(1);
+  expect(result.interrupted).toBe(1);
   expect(stripAnsi(result.output)).toContain(`8 |         const promise = page.click('text=Missing');`);
 });
 

tests/playwright-test/playwright.unhandled.spec.ts

@@ -29,7 +29,7 @@ test('should lead in uncaughtException when page.route raises', async ({ runInli
       });
     `,
   }, { workers: 1 });
-  expect(result.skipped).toBe(1);
+  expect(result.interrupted).toBe(1);
   expect(result.output).toContain('foobar');
 });
 
@@ -46,7 +46,7 @@ test('should lead in unhandledRejection when page.route raises', async ({ runInl
       });
     `,
   }, { workers: 1 });
-  expect(result.skipped).toBe(1);
+  expect(result.interrupted).toBe(1);
   expect(result.output).toContain('foobar');
 });
 
@@ -63,7 +63,7 @@ test('should lead in uncaughtException when context.route raises', async ({ runI
       });
     `,
   }, { workers: 1 });
-  expect(result.skipped).toBe(1);
+  expect(result.interrupted).toBe(1);
   expect(result.output).toContain('foobar');
 });
 
@@ -80,6 +80,6 @@ test('should lead in unhandledRejection when context.route raises', async ({ run
       });
     `,
   }, { workers: 1 });
-  expect(result.skipped).toBe(1);
+  expect(result.interrupted).toBe(1);
   expect(result.output).toContain('foobar');
 });

tests/playwright-test/runner.spec.ts

@@ -112,7 +112,8 @@ test('sigint should stop workers', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(130);
   expect(result.passed).toBe(0);
   expect(result.failed).toBe(0);
-  expect(result.skipped).toBe(4);
+  expect(result.skipped).toBe(2);
+  expect(result.interrupted).toBe(2);
   expect(result.output).toContain('%%SEND-SIGINT%%1');
   expect(result.output).toContain('%%SEND-SIGINT%%2');
   expect(result.output).not.toContain('%%skipped1');
@@ -177,7 +178,7 @@ test('worker interrupt should report errors', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(130);
   expect(result.passed).toBe(0);
   expect(result.failed).toBe(0);
-  expect(result.skipped).toBe(1);
+  expect(result.interrupted).toBe(1);
   expect(result.output).toContain('%%SEND-SIGINT%%');
   expect(result.output).toContain('Error: INTERRUPT');
 });

utils/generate_types/overrides-test.d.ts

@@ -95,7 +95,7 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
   // [internal] !!! DO NOT ADD TO THIS !!! See prior note.
 }
 
-export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
+export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
 
 export interface WorkerInfo {
   config: FullConfig;