feat(test-runner): introduce steps (#7952)

2021-08-02 17:17:20 -07:00 · 2021-08-02 17:17:20 -07:00 · 5803035c1b
--- a/.gitignore
+++ b/.gitignore
@ -16,4 +16,4 @@ drivers/
 .gradle/
 nohup.out
 .trace
-.tmp
+.tmp
--- a/docs/src/test-reporter-api/class-reporter.md
+++ b/docs/src/test-reporter-api/class-reporter.md
@ -138,6 +138,10 @@ Output chunk.

 Test that was running. Note that output may happen when to test is running, in which case this will be [void].

+### param: Reporter.onStdErr.result
+- `result` <[void]|[TestResult]>
+
+Result of the test run, this object gets populated while the test runs.


 ## method: Reporter.onStdOut
@ -154,7 +158,48 @@ Output chunk.

 Test that was running. Note that output may happen when to test is running, in which case this will be [void].

+### param: Reporter.onStdOut.result
+- `result` <[void]|[TestResult]>

+Result of the test run, this object gets populated while the test runs.
+
+## method: Reporter.onStepBegin
+
+Called when a test step started in the worker process.
+
+### param: Reporter.onStepBegin.test
+- `test` <[TestCase]>
+
+Test that has been started.
+
+### param: Reporter.onStepBegin.result
+- `result` <[TestResult]>
+
+Result of the test run, this object gets populated while the test runs.
+
+### param: Reporter.onStepBegin.step
+- `result` <[TestStep]>
+
+Test step instance.
+
+## method: Reporter.onStepEnd
+
+Called when a test step finished in the worker process.
+
+### param: Reporter.onStepEnd.test
+- `test` <[TestCase]>
+
+Test that has been finished.
+
+### param: Reporter.onStepEnd.result
+- `result` <[TestResult]>
+
+Result of the test run.
+
+### param: Reporter.onStepEnd.step
+- `result` <[TestStep]>
+
+Test step instance.

 ## method: Reporter.onTestBegin

@ -165,6 +210,10 @@ Called after a test has been started in the worker process.

 Test that has been started.

+### param: Reporter.onTestBegin.result
+- `result` <[TestResult]>
+
+Result of the test run, this object gets populated while the test runs.


 ## method: Reporter.onTestEnd
--- a/docs/src/test-reporter-api/class-testresult.md
+++ b/docs/src/test-reporter-api/class-testresult.md
@ -49,6 +49,11 @@ Anything written to the standard error during the test run.

 Anything written to the standard output during the test run.

+## property: TestResult.steps
+- type: <[Array]<[TestStep]>>
+
+List of steps inside this test run.
+
 ## property: TestResult.workerIndex
 - type: <[int]>

--- a/docs/src/test-reporter-api/class-teststep.md
+++ b/docs/src/test-reporter-api/class-teststep.md
@ -0,0 +1,32 @@
+# class: TestStep
+* langs: js
+
+Represents a step in the [TestRun].
+
+## property: TestStep.category
+- type: <[string]>
+
+Step category to differentiate steps with different origin and verbosity. Built-in categories are:
+* `hook` for fixtures and hooks initialization and teardown
+* `expect` for expect calls
+* `pw:api` for Playwright API calls.
+
+## property: TestStep.duration
+- type: <[float]>
+
+Running time in milliseconds.
+
+## property: TestStep.error
+- type: <[void]|[TestError]>
+
+An error thrown during the step execution, if any.
+
+## property: TestStep.startTime
+- type: <[Date]>
+
+Start time of this particular test step.
+
+## property: TestStep.title
+- type: <[string]>
+
+User-friendly test step title.
--- a/src/client/channelOwner.ts
+++ b/src/client/channelOwner.ts
@ -23,8 +23,6 @@ import { isUnderTest } from '../utils/utils';
 import type { Connection } from './connection';
 import type { ClientSideInstrumentation, Logger } from './types';

-let lastCallSeq = 0;
-
 export abstract class ChannelOwner<T extends channels.Channel = channels.Channel, Initializer = {}> extends EventEmitter {
  protected _connection: Connection;
  private _parent: ChannelOwner | undefined;
@ -97,19 +95,19 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
    const stackTrace = captureStackTrace();
    const { apiName, frameTexts } = stackTrace;
    const channel = this._createChannel({}, stackTrace);
-    const seq = ++lastCallSeq;
+    let csiCallback: ((e?: Error) => void) | undefined;
    try {
      logApiCall(logger, `=> ${apiName} started`);
-      this._csi?.onApiCall({ phase: 'begin', seq, apiName, frames: stackTrace.frames });
+      csiCallback = this._csi?.onApiCall(apiName);
      const result = await func(channel as any, stackTrace);
-      this._csi?.onApiCall({ phase: 'end', seq });
+      csiCallback?.();
      logApiCall(logger, `<= ${apiName} succeeded`);
      return result;
    } catch (e) {
      const innerError = ((process.env.PWDEBUGIMPL || isUnderTest()) && e.stack) ? '\n<inner error>\n' + e.stack : '';
      e.message = apiName + ': ' + e.message;
      e.stack = e.message + '\n' + frameTexts.join('\n') + innerError;
-      this._csi?.onApiCall({ phase: 'end', seq, error: e.stack });
+      csiCallback?.(e);
      logApiCall(logger, `<= ${apiName} failed`);
      throw e;
    }
--- a/src/client/types.ts
+++ b/src/client/types.ts
@ -24,7 +24,7 @@ export interface Logger {
 }

 export interface ClientSideInstrumentation {
-  onApiCall(data: { phase: 'begin' | 'end', seq: number, apiName?: string, frames?: channels.StackFrame[], error?: string }): void;
+  onApiCall(name: string): (error?: Error) => void;
 }

 import { Size } from '../common/types';
--- a/src/test/dispatcher.ts
+++ b/src/test/dispatcher.ts
@ -17,8 +17,8 @@
 import child_process from 'child_process';
 import path from 'path';
 import { EventEmitter } from 'events';
-import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, ProgressPayload } from './ipc';
-import type { TestResult, Reporter } from '../../types/testReporter';
+import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc';
+import type { TestResult, Reporter, TestStep } from '../../types/testReporter';
 import { TestCase } from './test';
 import { Loader } from './loader';

@ -35,7 +35,7 @@ export class Dispatcher {
  private _freeWorkers: Worker[] = [];
  private _workerClaimers: (() => void)[] = [];

-  private _testById = new Map<string, { test: TestCase, result: TestResult }>();
+  private _testById = new Map<string, { test: TestCase, result: TestResult, steps: Map<string, TestStep> }>();
  private _queue: TestGroup[] = [];
  private _stopCallback = () => {};
  readonly _loader: Loader;
@ -51,7 +51,8 @@ export class Dispatcher {
    for (const group of testGroups) {
      for (const test of group.tests) {
        const result = test._appendTestResult();
-        this._testById.set(test._id, { test, result });
+        // When changing this line, change the one in retry too.
+        this._testById.set(test._id, { test, result, steps: new Map() });
      }
    }
  }
@ -136,7 +137,7 @@ export class Dispatcher {
            break;
          // There might be a single test that has started but has not finished yet.
          if (test._id !== lastStartedTestId)
-            this._reporter.onTestBegin?.(test);
+            this._reporter.onTestBegin?.(test, result);
          result.error = params.fatalError;
          result.status = first ? 'failed' : 'skipped';
          this._reportTestEnd(test, result);
@ -155,6 +156,7 @@ export class Dispatcher {
        const pair = this._testById.get(testId)!;
        if (!this._isStopped && pair.test.expectedStatus === 'passed' && pair.test.results.length < pair.test.retries + 1) {
          pair.result = pair.test._appendTestResult();
+          pair.steps = new Map();
          remaining.unshift(pair.test);
        }
      }
@ -215,7 +217,7 @@ export class Dispatcher {
      const { test, result: testRun  } = this._testById.get(params.testId)!;
      testRun.workerIndex = params.workerIndex;
      testRun.startTime = new Date(params.startWallTime);
-      this._reporter.onTestBegin?.(test);
+      this._reporter.onTestBegin?.(test, testRun);
    });
    worker.on('testEnd', (params: TestEndPayload) => {
      if (this._hasReachedMaxFailures())
@ -235,23 +237,40 @@ export class Dispatcher {
      test.timeout = params.timeout;
      this._reportTestEnd(test, result);
    });
-    worker.on('progress', (params: ProgressPayload) => {
-      const { test } = this._testById.get(params.testId)!;
-      (this._reporter as any)._onTestProgress?.(test, params.name, params.data);
+    worker.on('stepBegin', (params: StepBeginPayload) => {
+      const { test, result, steps } = this._testById.get(params.testId)!;
+      const step: TestStep = {
+        title: params.title,
+        category: params.category,
+        startTime: new Date(params.wallTime),
+        duration: 0,
+      };
+      steps.set(params.stepId, step);
+      result.steps.push(step);
+      this._reporter.onStepBegin?.(test, result, step);
+    });
+    worker.on('stepEnd', (params: StepEndPayload) => {
+      const { test, result, steps } = this._testById.get(params.testId)!;
+      const step = steps.get(params.stepId)!;
+      step.duration = params.wallTime - step.startTime.getTime();
+      if (params.error)
+        step.error = params.error;
+      steps.delete(params.stepId);
+      this._reporter.onStepEnd?.(test, result, step);
    });
    worker.on('stdOut', (params: TestOutputPayload) => {
      const chunk = chunkFromParams(params);
      const pair = params.testId ? this._testById.get(params.testId) : undefined;
      if (pair)
        pair.result.stdout.push(chunk);
-      this._reporter.onStdOut?.(chunk, pair ? pair.test : undefined);
+      this._reporter.onStdOut?.(chunk, pair?.test, pair?.result);
    });
    worker.on('stdErr', (params: TestOutputPayload) => {
      const chunk = chunkFromParams(params);
      const pair = params.testId ? this._testById.get(params.testId) : undefined;
      if (pair)
        pair.result.stderr.push(chunk);
-      this._reporter.onStdErr?.(chunk, pair ? pair.test : undefined);
+      this._reporter.onStdErr?.(chunk, pair?.test, pair?.result);
    });
    worker.on('teardownError', ({error}) => {
      this._hasWorkerErrors = true;
--- a/src/test/expect.ts
+++ b/src/test/expect.ts
@ -39,7 +39,7 @@ import {
  toHaveValue
 } from './matchers/matchers';
 import { toMatchSnapshot } from './matchers/toMatchSnapshot';
-import type { Expect } from './types';
+import type { Expect, TestStatus } from './types';
 import matchers from 'expect/build/matchers';
 import { currentTestInfo } from './globals';

@ -70,41 +70,37 @@ const customMatchers = {
  toMatchSnapshot,
 };

-let lastExpectSeq = 0;
-
 function wrap(matcherName: string, matcher: any) {
  return function(this: any, ...args: any[]) {
    const testInfo = currentTestInfo();
    if (!testInfo)
      return matcher.call(this, ...args);

-    const seq = ++lastExpectSeq;
-    testInfo._progress('expect', { phase: 'begin', seq, matcherName });
-    const endPayload: any = { phase: 'end', seq };
-    let isAsync = false;
+    const infix = this.isNot ? '.not' : '';
+    const completeStep = testInfo._addStep('expect', `expect${infix}.${matcherName}`);
+
+    const reportStepEnd = (result: any) => {
+      status = result.pass !== this.isNot ? 'passed' : 'failed';
+      let error: Error | undefined;
+      if (status === 'failed')
+        error = new Error(result.message());
+      completeStep?.(error);
+      return result;
+    };
+
+    const reportStepError = (error: Error) => {
+      completeStep?.(error);
+      throw error;
+    };
+
+    let status: TestStatus = 'passed';
    try {
      const result = matcher.call(this, ...args);
-      endPayload.pass = result.pass;
-      if (this.isNot)
-        endPayload.isNot = this.isNot;
-      if (result.pass === this.isNot && result.message)
-        endPayload.message = result.message();
-      if (result instanceof Promise) {
-        isAsync = true;
-        return result.catch(e => {
-          endPayload.error = e.stack;
-          throw e;
-        }).finally(() => {
-          testInfo._progress('expect', endPayload);
-        });
-      }
-      return result;
+      if (result instanceof Promise)
+        return result.then(reportStepEnd).catch(reportStepError);
+      return reportStepEnd(result);
    } catch (e) {
-      endPayload.error = e.stack;
-      throw e;
-    } finally {
-      if (!isAsync)
-        testInfo._progress('expect', endPayload);
+      reportStepError(e);
    }
  };
 }
--- a/src/test/index.ts
+++ b/src/test/index.ts
@ -185,7 +185,9 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
      };
      const context = await browser.newContext(combinedOptions);
      (context as any)._csi = {
-        onApiCall: (data: any) => (testInfo as any)._progress('pw:api', data),
+        onApiCall: (name: string) => {
+          return (testInfo as any)._addStep('pw:api', name);
+        },
      };
      context.setDefaultTimeout(actionTimeout || 0);
      context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0);
--- a/src/test/ipc.ts
+++ b/src/test/ipc.ts
@ -46,10 +46,19 @@ export type TestEndPayload = {
  attachments: { name: string, path?: string, body?: string, contentType: string }[];
 };

-export type ProgressPayload = {
+export type StepBeginPayload = {
  testId: string;
-  name: string;
-  data: any;
+  stepId: string;
+  title: string;
+  category: string;
+  wallTime: number;  // milliseconds since unix epoch
+};
+
+export type StepEndPayload = {
+  testId: string;
+  stepId: string;
+  wallTime: number;  // milliseconds since unix epoch
+  error?: TestError;
 };

 export type TestEntry = {
--- a/src/test/reporters/multiplexer.ts
+++ b/src/test/reporters/multiplexer.ts
@ -14,7 +14,7 @@
 * limitations under the License.
 */

-import { FullConfig, Suite, TestCase, TestError, TestResult, Reporter, FullResult } from '../../../types/testReporter';
+import { FullConfig, Suite, TestCase, TestError, TestResult, Reporter, FullResult, TestStep } from '../../../types/testReporter';

 export class Multiplexer implements Reporter {
  private _reporters: Reporter[];
@ -28,19 +28,19 @@ export class Multiplexer implements Reporter {
      reporter.onBegin?.(config, suite);
  }

-  onTestBegin(test: TestCase) {
+  onTestBegin(test: TestCase, result: TestResult) {
    for (const reporter of this._reporters)
-      reporter.onTestBegin?.(test);
+      reporter.onTestBegin?.(test, result);
  }

-  onStdOut(chunk: string | Buffer, test?: TestCase) {
+  onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
    for (const reporter of this._reporters)
-      reporter.onStdOut?.(chunk, test);
+      reporter.onStdOut?.(chunk, test, result);
  }

-  onStdErr(chunk: string | Buffer, test?: TestCase) {
+  onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
    for (const reporter of this._reporters)
-      reporter.onStdErr?.(chunk, test);
+      reporter.onStdErr?.(chunk, test, result);
  }

  onTestEnd(test: TestCase, result: TestResult) {
@ -58,8 +58,13 @@ export class Multiplexer implements Reporter {
      reporter.onError?.(error);
  }

-  _onTestProgress(test: TestCase, name: string, data: any) {
+  onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
    for (const reporter of this._reporters)
-      (reporter as any)._onTestProgress?.(test, name, data);
+      (reporter as any).onStepBegin?.(test, result, step);
+  }
+
+  onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
+    for (const reporter of this._reporters)
+      (reporter as any).onStepEnd?.(test, result, step);
  }
 }
--- a/src/test/test.ts
+++ b/src/test/test.ts
@ -173,6 +173,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
      stderr: [],
      attachments: [],
      status: 'skipped',
+      steps: []
    };
    this.results.push(result);
    return result;
--- a/src/test/types.ts
+++ b/src/test/types.ts
@ -14,7 +14,7 @@
 * limitations under the License.
 */

-import type { Fixtures, TestInfo } from '../../types/test';
+import type { Fixtures, TestError, TestInfo } from '../../types/test';
 import type { Location } from '../../types/testReporter';
 export * from '../../types/test';
 export { Location } from '../../types/testReporter';
@ -25,7 +25,9 @@ export type FixturesWithLocation = {
 };
 export type Annotations = { type: string, description?: string }[];

+export type CompleteStepCallback = (error?: TestError) => void;
+
 export interface TestInfoImpl extends TestInfo {
  _testFinished: Promise<void>;
-  _progress: (name: string, params: any) => void;
+  _addStep: (category: string, title: string) => CompleteStepCallback;
 }
--- a/src/test/worker.ts
+++ b/src/test/worker.ts
@ -74,7 +74,7 @@ process.on('message', async message => {
    workerIndex = initParams.workerIndex;
    startProfiling();
    workerRunner = new WorkerRunner(initParams);
-    for (const event of ['testBegin', 'testEnd', 'done', 'progress'])
+    for (const event of ['testBegin', 'testEnd', 'stepBegin', 'stepEnd', 'done'])
      workerRunner.on(event, sendMessageToParent.bind(null, event));
    return;
  }
--- a/src/test/workerRunner.ts
+++ b/src/test/workerRunner.ts
@ -20,11 +20,11 @@ import rimraf from 'rimraf';
 import util from 'util';
 import { EventEmitter } from 'events';
 import { monotonicTime, DeadlineRunner, raceAgainstDeadline, serializeError } from './util';
-import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams } from './ipc';
+import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc';
 import { setCurrentTestInfo } from './globals';
 import { Loader } from './loader';
 import { Modifier, Suite, TestCase } from './test';
-import { Annotations, TestError, TestInfo, TestInfoImpl, WorkerInfo } from './types';
+import { Annotations, CompleteStepCallback, TestError, TestInfo, TestInfoImpl, WorkerInfo } from './types';
 import { ProjectImpl } from './project';
 import { FixturePool, FixtureRunner } from './fixtures';

@ -221,6 +221,7 @@ export class WorkerRunner extends EventEmitter {
    })();

    let testFinishedCallback = () => {};
+    let lastStepId = 0;
    const testInfo: TestInfoImpl = {
      ...this._workerInfo,
      title: test.title,
@ -267,7 +268,26 @@ export class WorkerRunner extends EventEmitter {
          deadlineRunner.setDeadline(deadline());
      },
      _testFinished: new Promise(f => testFinishedCallback = f),
-      _progress: (name, data) => this.emit('progress', { testId, name, data }),
+      _addStep: (category: string, title: string) => {
+        const stepId = `${category}@${++lastStepId}`;
+        const payload: StepBeginPayload = {
+          testId,
+          stepId,
+          category,
+          title,
+          wallTime: Date.now()
+        };
+        this.emit('stepBegin', payload);
+        return (error?: TestError) => {
+          const payload: StepEndPayload = {
+            testId,
+            stepId,
+            wallTime: Date.now(),
+            error
+          };
+          this.emit('stepEnd', payload);
+        };
+      },
    };

    // Inherit test.setTimeout() from parent suites.
@ -361,7 +381,8 @@ export class WorkerRunner extends EventEmitter {
    setCurrentTestInfo(currentTest ? currentTest.testInfo : null);
  }

-  private async _runTestWithBeforeHooks(test: TestCase, testInfo: TestInfo) {
+  private async _runTestWithBeforeHooks(test: TestCase, testInfo: TestInfoImpl) {
+    let completeStep: CompleteStepCallback | undefined;
    try {
      const beforeEachModifiers: Modifier[] = [];
      for (let s = test.parent; s; s = s.parent) {
@ -375,6 +396,7 @@ export class WorkerRunner extends EventEmitter {
        const result = await this._fixtureRunner.resolveParametersAndRunHookOrTest(modifier.fn, 'test', testInfo);
        testInfo[modifier.type](!!result, modifier.description!);
      }
+      completeStep = testInfo._addStep('hook', 'Before Hooks');
      await this._runHooks(test.parent!, 'beforeEach', testInfo);
    } catch (error) {
      if (error instanceof SkipError) {
@ -386,6 +408,7 @@ export class WorkerRunner extends EventEmitter {
      }
      // Continue running afterEach hooks even after the failure.
    }
+    completeStep?.(testInfo.error);

    // Do not run the test when beforeEach hook fails.
    if (this._isStopped || testInfo.status === 'failed' || testInfo.status === 'skipped')
@ -409,8 +432,11 @@ export class WorkerRunner extends EventEmitter {
    }
  }

-  private async _runAfterHooks(test: TestCase, testInfo: TestInfo) {
+  private async _runAfterHooks(test: TestCase, testInfo: TestInfoImpl) {
+    let completeStep: CompleteStepCallback | undefined;
+    let teardownError: TestError | undefined;
    try {
+      completeStep = testInfo._addStep('hook', 'After Hooks');
      await this._runHooks(test.parent!, 'afterEach', testInfo);
    } catch (error) {
      if (!(error instanceof SkipError)) {
@ -428,9 +454,12 @@ export class WorkerRunner extends EventEmitter {
      if (testInfo.status === 'passed')
        testInfo.status = 'failed';
      // Do not overwrite test failure error.
-      if (!('error' in  testInfo))
+      if (!('error' in  testInfo)) {
        testInfo.error = serializeError(error);
+        teardownError = testInfo.error;
+      }
    }
+    completeStep?.(teardownError);
  }

  private async _runHooks(suite: Suite, type: 'beforeEach' | 'afterEach', testInfo: TestInfo) {
--- a/tests/config/browserTest.ts
+++ b/tests/config/browserTest.ts
@ -135,7 +135,7 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
    await run(contextOptions);
  },

-  contextFactory: async ({ browser, contextOptions }, run) => {
+  contextFactory: async ({ browser, contextOptions }, run, testInfo) => {
    const contexts: BrowserContext[] = [];
    await run(async options => {
      const context = await browser.newContext({ ...contextOptions, ...options });
--- a/tests/playwright-test/reporter.spec.ts
+++ b/tests/playwright-test/reporter.spec.ts
@ -159,13 +159,16 @@ test('should load reporter from node_modules', async ({ runInlineTest }) => {
  ]);
 });

-test('should report expect progress', async ({ runInlineTest }) => {
+test('should report expect steps', async ({ runInlineTest }) => {
  const expectReporterJS = `
    class Reporter {
-      _onTestProgress(test, name, data) {
-        if (data.frames)
-          data.frames = [];
-        console.log('%%%%', name, JSON.stringify(data));
+      onStepBegin(test, result, step) {
+        const copy = { ...step, startTime: undefined, duration: undefined };
+        console.log('%%%% begin', JSON.stringify(copy));
+      }
+      onStepEnd(test, result, step) {
+        const copy = { ...step, startTime: undefined, duration: undefined };
+        console.log('%%%% end', JSON.stringify(copy));
      }
    }
    module.exports = Reporter;
@ -195,32 +198,45 @@ test('should report expect progress', async ({ runInlineTest }) => {

  expect(result.exitCode).toBe(1);
  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
-    `%% expect {\"phase\":\"begin\",\"seq\":1,\"matcherName\":\"toBeTruthy\"}`,
-    `%% expect {\"phase\":\"end\",\"seq\":1,\"pass\":true}`,
-    `%% expect {\"phase\":\"begin\",\"seq\":2,\"matcherName\":\"toBeTruthy\"}`,
-    `%% expect {\"phase\":\"end\",\"seq\":2,\"pass\":false,\"message\":\"\\u001b[2mexpect(\\u001b[22m\\u001b[31mreceived\\u001b[39m\\u001b[2m).\\u001b[22mtoBeTruthy\\u001b[2m()\\u001b[22m\\n\\nReceived: \\u001b[31mfalse\\u001b[39m\"}`,
-
-    `%% expect {\"phase\":\"begin\",\"seq\":1,\"matcherName\":\"toBeTruthy\"}`,
-    `%% expect {\"phase\":\"end\",\"seq\":1,\"pass\":false,\"isNot\":true}`,
-
-    `%% pw:api {\"phase\":\"begin\",\"seq\":3,\"apiName\":\"browserContext.newPage\",\"frames\":[]}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":3}`,
-    `%% expect {\"phase\":\"begin\",\"seq\":2,\"matcherName\":\"toHaveTitle\"}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":4,\"apiName\":\"page.title\",\"frames\":[]}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":4}`,
-    `%% expect {\"phase\":\"end\",\"seq\":2,\"isNot\":true}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":5,\"apiName\":\"browserContext.close\",\"frames\":[]}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":5}`,
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
+    `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
+    `%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
+    `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{}}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`,
+    `%% end {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`,
+    `%% begin {\"title\":\"page.title\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"page.title\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
  ]);
 });

-test('should report log progress', async ({ runInlineTest }) => {
+test('should report api steps', async ({ runInlineTest }) => {
  const expectReporterJS = `
    class Reporter {
-      _onTestProgress(test, name, data) {
-        if (data.frames)
-          data.frames = [];
-        console.log('%%%%', name, JSON.stringify(data));
+      onStepBegin(test, result, step) {
+        const copy = { ...step, startTime: undefined, duration: undefined };
+        console.log('%%%% begin', JSON.stringify(copy));
+      }
+      onStepEnd(test, result, step) {
+        const copy = { ...step, startTime: undefined, duration: undefined };
+        console.log('%%%% end', JSON.stringify(copy));
      }
    }
    module.exports = Reporter;
@ -244,13 +260,17 @@ test('should report log progress', async ({ runInlineTest }) => {

  expect(result.exitCode).toBe(0);
  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
-    `%% pw:api {\"phase\":\"begin\",\"seq\":3,\"apiName\":\"browserContext.newPage\",\"frames\":[]}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":3}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":4,\"apiName\":\"page.setContent\",\"frames\":[]}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":4}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":5,\"apiName\":\"page.click\",\"frames\":[]}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":5}`,
-    `%% pw:api {\"phase\":\"begin\",\"seq\":6,\"apiName\":\"browserContext.close\",\"frames\":[]}`,
-    `%% pw:api {\"phase\":\"end\",\"seq\":6}`,
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
  ]);
 });
--- a/types/testReporter.d.ts
+++ b/types/testReporter.d.ts
@ -213,6 +213,39 @@ export interface TestResult {
   * Anything written to the standard error during the test run.
   */
  stderr: (string | Buffer)[];
+  /**
+   * List of steps inside this test run.
+   */
+  steps: TestStep[];
+}
+
+/**
+ * Represents a step in the [TestRun].
+ */
+export interface TestStep {
+  /**
+   * User-friendly test step title.
+   */
+  title: string;
+  /**
+   * Step category to differentiate steps with different origin and verbosity. Built-in categories are:
+   * - `hook` for fixtures and hooks initialization and teardown
+   * - `expect` for expect calls
+   * - `pw:api` for Playwright API calls.
+   */
+  category: string,
+  /**
+   * Start time of this particular test step.
+   */
+  startTime: Date;
+  /**
+   * Running time in milliseconds.
+   */
+  duration: number;
+  /**
+   * An error thrown during the step execution, if any.
+   */
+  error?: TestError;
 }

 /**
@ -321,26 +354,43 @@ export interface Reporter {
  /**
   * Called after a test has been started in the worker process.
   * @param test Test that has been started.
+   * @param result Result of the test run, this object gets populated while the test runs.
   */
-  onTestBegin?(test: TestCase): void;
+  onTestBegin?(test: TestCase, result: TestResult): void;
  /**
   * Called when something has been written to the standard output in the worker process.
   * @param chunk Output chunk.
   * @param test Test that was running. Note that output may happen when to test is running, in which case this will be [void].
+   * @param result Result of the test run, this object gets populated while the test runs.
   */
-  onStdOut?(chunk: string | Buffer, test?: TestCase): void;
+  onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
  /**
   * Called when something has been written to the standard error in the worker process.
   * @param chunk Output chunk.
   * @param test Test that was running. Note that output may happen when to test is running, in which case this will be [void].
+   * @param result Result of the test run, this object gets populated while the test runs.
   */
-  onStdErr?(chunk: string | Buffer, test?: TestCase): void;
+  onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
  /**
   * Called after a test has been finished in the worker process.
   * @param test Test that has been finished.
   * @param result Result of the test run.
   */
  onTestEnd?(test: TestCase, result: TestResult): void;
+  /**
+   * Called when a test step started in the worker process.
+   * @param test Test that has been started.
+   * @param result Result of the test run, this object gets populated while the test runs.
+   * @param result Test step instance.
+   */
+  onStepBegin?(test: TestCase, result: TestResult, step: TestStep): void;
+  /**
+   * Called when a test step finished in the worker process.
+   * @param test Test that has been finished.
+   * @param result Result of the test run.
+   * @param result Test step instance.
+   */
+  onStepEnd?(test: TestCase, result: TestResult, step: TestStep): void;
  /**
   * Called on some global error, for example unhandled exception in the worker process.
   * @param error The error.
--- a/utils/generate_types/overrides-testReporter.d.ts
+++ b/utils/generate_types/overrides-testReporter.d.ts
@ -55,6 +55,15 @@ export interface TestResult {
  attachments: { name: string, path?: string, body?: Buffer, contentType: string }[];
  stdout: (string | Buffer)[];
  stderr: (string | Buffer)[];
+  steps: TestStep[];
+}
+
+export interface TestStep {
+  title: string;
+  category: string,
+  startTime: Date;
+  duration: number;
+  error?: TestError;
 }

 /**
@ -73,10 +82,12 @@ export interface FullResult {

 export interface Reporter {
  onBegin?(config: FullConfig, suite: Suite): void;
-  onTestBegin?(test: TestCase): void;
-  onStdOut?(chunk: string | Buffer, test?: TestCase): void;
-  onStdErr?(chunk: string | Buffer, test?: TestCase): void;
+  onTestBegin?(test: TestCase, result: TestResult): void;
+  onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
+  onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
  onTestEnd?(test: TestCase, result: TestResult): void;
+  onStepBegin?(test: TestCase, result: TestResult, step: TestStep): void;
+  onStepEnd?(test: TestCase, result: TestResult, step: TestStep): void;
  onError?(error: TestError): void;
  onEnd?(result: FullResult): void | Promise<void>;
 }