feat: group filter (#17646)

2022-09-28 18:45:01 -07:00 · 2022-09-28 18:45:01 -07:00 · 9f17ee6871
--- a/docs/src/test-api/class-testconfig.md
+++ b/docs/src/test-api/class-testconfig.md
@ -236,6 +236,10 @@ Filter to only run tests with a title **not** matching one of the patterns. This
 * since: v1.27
 - type: ?<[Object]<[string],[Array]<[string]|[Array]<[string]|[Object]>>>>
  - `project` <[string]|[Array]<[string]>> Project name(s).
+  - `grep` ?<[RegExp]|[Array]<[RegExp]>> Filter to only run tests with a title matching one of the patterns.
+  - `grepInvert` ?<[RegExp]|[Array]<[RegExp]>> Filter to only run tests with a title **not** matching one of the patterns.
+  - `testMatch` ?<[string]|[RegExp]|[Array]<[string]|[RegExp]>> Only the files matching one of these patterns are executed as test files. Matching is performed against the absolute file path. Strings are treated as glob patterns.
+  - `testIgnore` ?<[string]|[RegExp]|[Array]<[string]|[RegExp]>> Files matching one of these patterns are not executed as test files. Matching is performed against the absolute file path. Strings are treated as glob patterns.

 Project groups that control project execution order.

--- a/packages/playwright-test/src/loader.ts
+++ b/packages/playwright-test/src/loader.ts
@ -515,7 +515,7 @@ function validateConfig(file: string, config: Config) {
          throw errorWithFile(file, `config.grepInvert[${index}] must be a RegExp`);
      });
    } else if (!isRegExp(config.grepInvert)) {
-      throw errorWithFile(file, `config.grep must be a RegExp`);
+      throw errorWithFile(file, `config.grepInvert must be a RegExp`);
    }
  }

@ -656,40 +656,73 @@ function validateProjectGroups(file: string, config: Config) {
      if (!projectNames.has(projectName))
        throw errorWithFile(file, `config.groups.${groupName} refers to an unknown project '${projectName}'`);
    }
-    for (const step of group) {
+    group.forEach((step, stepIndex) => {
      if (isString(step)) {
        validateProjectReference(step);
      } else if (Array.isArray(step)) {
        const parallelProjectNames = new Set();
-        for (const item of step) {
+        step.forEach((item, itemIndex) => {
          let projectName;
          if (isString(item)) {
            validateProjectReference(item);
            projectName = item;
          } else if (isObject(item)) {
-            const project = (item as any).project;
+            const project = item.project;
            if (isString(project)) {
              validateProjectReference(project);
            } else if (Array.isArray(project)) {
-              project.forEach(name => {
+              project.forEach((name, projectIndex) => {
                if (!isString(name))
-                  throw errorWithFile(file, `config.groups.${groupName}[*].project contains non string value.`);
+                  throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].project[${projectIndex}] contains non string value.`);
                validateProjectReference(name);
              });
            }
            projectName = project;
+            if ('grep' in item) {
+              if (Array.isArray(item.grep)) {
+                item.grep.forEach((item, grepIndex) => {
+                  if (!isRegExp(item))
+                    throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].grep[${grepIndex}] must be a RegExp`);
+                });
+              } else if (!isRegExp(item.grep)) {
+                throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].grep must be a RegExp`);
+              }
+            }
+            if ('grepInvert' in item) {
+              if (Array.isArray(item.grepInvert)) {
+                item.grepInvert.forEach((item, index) => {
+                  if (!isRegExp(item))
+                    throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].grepInvert[${index}] must be a RegExp`);
+                });
+              } else if (!isRegExp(item.grepInvert)) {
+                throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].grepInvert must be a RegExp`);
+              }
+            }
+            for (const prop of ['testIgnore', 'testMatch'] as const) {
+              if (prop in item) {
+                const value = item[prop];
+                if (Array.isArray(value)) {
+                  value.forEach((item, index) => {
+                    if (typeof item !== 'string' && !isRegExp(item))
+                      throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].${prop}[${index}] must be a string or a RegExp`);
+                  });
+                } else if (typeof value !== 'string' && !isRegExp(value)) {
+                  throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].${prop} must be a string or a RegExp`);
+                }
+              }
+            }
          } else {
-            throw errorWithFile(file, `config.groups.${groupName} unexpected group entry ${JSON.stringify(step, null, 2)}`);
+            throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}] unexpected group entry ${JSON.stringify(step, null, 2)}`);
          }
          // We can relax this later.
          if (parallelProjectNames.has(projectName))
-            throw errorWithFile(file, `config.groups.${groupName} group mentions project '${projectName}' twice in one parallel group`);
+            throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}] group mentions project '${projectName}' twice in one parallel group`);
          parallelProjectNames.add(projectName);
-        }
+        });
      } else {
-        throw errorWithFile(file, `config.groups.${groupName} unexpected group entry ${JSON.stringify(step, null, 2)}`);
+        throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}] unexpected group entry ${JSON.stringify(step, null, 2)}`);
      }
-    }
+    });
  }
 }

--- a/packages/playwright-test/src/runner.ts
+++ b/packages/playwright-test/src/runner.ts
@ -21,7 +21,7 @@ import * as path from 'path';
 import { promisify } from 'util';
 import type { TestGroup } from './dispatcher';
 import { Dispatcher } from './dispatcher';
-import type { TestFileFilter } from './util';
+import type { Matcher, TestFileFilter } from './util';
 import { createFileMatcher, createTitleMatcher, serializeError } from './util';
 import type { TestCase } from './test';
 import { Suite } from './test';
@ -56,9 +56,10 @@ export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.j

 // Project group is a sequence of run phases.
 type RunPhase = {
-  testFileFilters: TestFileFilter[];
-  projectFilter?: string[];
-};
+  projectName: string;
+  testFileMatcher: Matcher;
+  testTitleMatcher: Matcher;
+}[];

 type RunOptions = {
  listOnly?: boolean;
@ -220,7 +221,13 @@ export class Runner {
  }

  async listTestFiles(configFile: string, projectNames: string[] | undefined): Promise<any> {
-    const filesByProject = await this._collectFiles([], projectNames);
+    const projects = projectNames ?? this._loader.fullConfig().projects.map(p => p.name);
+    const phase: RunPhase = projects.map(projectName => ({
+      projectName,
+      testFileMatcher: () => true,
+      testTitleMatcher: () => true,
+    }));
+    const filesByProject = await this._collectFiles(phase);
    const report: any = {
      projects: []
    };
@ -235,7 +242,6 @@ export class Runner {
    return report;
  }

-
  private _collectRunPhases(options: RunOptions) {
    const config = this._loader.fullConfig();

@ -258,64 +264,82 @@ export class Runner {
      if (!group)
        throw new Error(`Cannot find project group '${projectGroup}' in the config`);
      for (const entry of group) {
-        const projectFilter: string[] = [];
-        const testFileFilters: TestFileFilter[] = [];
        if (isString(entry)) {
-          projectFilter.push(entry);
+          phases.push([{
+            projectName: entry,
+            testFileMatcher: () => true,
+            testTitleMatcher: () => true,
+          }]);
        } else {
+          const phase: RunPhase = [];
+          phases.push(phase);
          for (const p of entry) {
-            if (isString(p))
-              projectFilter.push(p);
-            else if (isString(p.project))
-              projectFilter.push(p.project);
-            else
-              projectFilter.push(...p.project);
+            if (isString(p)) {
+              phase.push({
+                projectName: p,
+                testFileMatcher: () => true,
+                testTitleMatcher: () => true,
+              });
+            } else {
+              const testMatch = p.testMatch ? createFileMatcher(p.testMatch) : () => true;
+              const testIgnore = p.testIgnore ? createFileMatcher(p.testIgnore) : () => false;
+              const grep = p.grep ? createTitleMatcher(p.grep) : () => true;
+              const grepInvert = p.grepInvert ? createTitleMatcher(p.grepInvert) : () => false;
+              const projects = isString(p.project) ? [p.project] : p.project;
+              phase.push(...projects.map(projectName => ({
+                projectName,
+                testFileMatcher: (file: string) => !testIgnore(file) && testMatch(file),
+                testTitleMatcher: (title: string) => !grepInvert(title) && grep(title),
+              })));
+            }
          }
        }
-        // TODO: filter per project set.
-        phases.push({
-          testFileFilters,
-          projectFilter
-        });
      }
    } else {
-      phases.push({
-        projectFilter: options.projectFilter,
-        testFileFilters: options.testFileFilters || [],
-      });
+      phases.push(this._runPhaseFromOptions(options));
    }
    return phases;
  }

-  private async _collectFiles(testFileFilters: TestFileFilter[], projectNames?: string[]): Promise<Map<FullProjectInternal, string[]>> {
-    const testFileFilter = testFileFilters.length ? createFileMatcher(testFileFilters.map(e => e.re || e.exact || '')) : () => true;
-    let projectsToFind: Set<string> | undefined;
-    let unknownProjects: Map<string, string> | undefined;
-    if (projectNames) {
-      projectsToFind = new Set();
-      unknownProjects = new Map();
-      projectNames.forEach(n => {
-        const name = n.toLocaleLowerCase();
-        projectsToFind!.add(name);
-        unknownProjects!.set(name, n);
-      });
-    }
+  private _runPhaseFromOptions(options: RunOptions): RunPhase {
+    const testFileMatcher = fileMatcherFrom(options.testFileFilters);
+    const testTitleMatcher = () => true;
+    const projects = options.projectFilter ?? this._loader.fullConfig().projects.map(p => p.name);
+    return projects.map(projectName => ({
+      projectName,
+      testFileMatcher,
+      testTitleMatcher
+    }));
+  }
+
+  private _collectProjects(projectNames: string[]): FullProjectInternal[] {
+    const projectsToFind = new Set<string>();
+    const unknownProjects = new Map<string, string>();
+    projectNames.forEach(n => {
+      const name = n.toLocaleLowerCase();
+      projectsToFind.add(name);
+      unknownProjects.set(name, n);
+    });
    const fullConfig = this._loader.fullConfig();
    const projects = fullConfig.projects.filter(project => {
-      if (!projectsToFind)
-        return true;
      const name = project.name.toLocaleLowerCase();
-      unknownProjects!.delete(name);
+      unknownProjects.delete(name);
      return projectsToFind.has(name);
    });
-    if (unknownProjects && unknownProjects.size) {
+    if (unknownProjects.size) {
      const names = fullConfig.projects.map(p => p.name).filter(name => !!name);
      if (!names.length)
        throw new Error(`No named projects are specified in the configuration file`);
      const unknownProjectNames = Array.from(unknownProjects.values()).map(n => `"${n}"`).join(', ');
      throw new Error(`Project(s) ${unknownProjectNames} not found. Available named projects: ${names.map(name => `"${name}"`).join(', ')}`);
    }
+    return projects;
+  }

+  private async _collectFiles(runPhase: RunPhase): Promise<Map<FullProjectInternal, string[]>> {
+    const projectNames = runPhase.map(p => p.projectName);
+    const projects = this._collectProjects(projectNames);
+    const projectToGroupEntry = new Map(runPhase.map(p => [p.projectName.toLocaleLowerCase(), p]));
    const files = new Map<FullProjectInternal, string[]>();
    for (const project of projects) {
      const allFiles = await collectFiles(project.testDir, project._respectGitIgnore);
@ -323,6 +347,7 @@ export class Runner {
      const testIgnore = createFileMatcher(project.testIgnore);
      const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
      const testFileExtension = (file: string) => extensions.includes(path.extname(file));
+      const testFileFilter = projectToGroupEntry.get(project.name.toLocaleLowerCase())!.testFileMatcher;
      const testFiles = allFiles.filter(file => !testIgnore(file) && testMatch(file) && testFileFilter(file) && testFileExtension(file));
      files.set(project, testFiles);
    }
@ -338,9 +363,9 @@ export class Runner {
    const rootSuite = new Suite('', 'root');
    const runPhases = this._collectRunPhases(options);
    assert(runPhases.length > 0);
-    for (const { projectFilter, testFileFilters } of runPhases) {
+    for (const phase of runPhases) {
      // TODO: do not collect files for each project multiple times.
-      const filesByProject = await this._collectFiles(testFileFilters, projectFilter);
+      const filesByProject = await this._collectFiles(phase);

      const allTestFiles = new Set<string>();
      for (const files of filesByProject.values())
@ -363,7 +388,8 @@ export class Runner {

      // 3. Filter tests to respect line/column filter.
      // TODO: figure out how this is supposed to work with groups.
-      filterByFocusedLine(preprocessRoot, testFileFilters);
+      if (options.testFileFilters?.length)
+        filterByFocusedLine(preprocessRoot, options.testFileFilters);

      // 4. Complain about only.
      if (config.forbidOnly) {
@ -385,6 +411,7 @@ export class Runner {
      for (const [project, files] of filesByProject) {
        const grepMatcher = createTitleMatcher(project.grep);
        const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null;
+        const groupTitleMatcher = phase.find(p => p.projectName.toLocaleLowerCase() === project.name.toLocaleLowerCase())!.testTitleMatcher;
        const projectSuite = new Suite(project.name, 'project');
        projectSuite._projectConfig = project;
        if (project._fullyParallel)
@ -399,7 +426,7 @@ export class Runner {
              const grepTitle = test.titlePath().join(' ');
              if (grepInvertMatcher?.(grepTitle))
                return false;
-              return grepMatcher(grepTitle);
+              return grepMatcher(grepTitle) && groupTitleMatcher(grepTitle);
            });
            if (builtSuite)
              projectSuite._addSuite(builtSuite);
@ -546,8 +573,7 @@ export class Runner {

    const runAndWatch = async () => {
      // 5. Collect all files.
-      const testFileFilters = options.testFileFilters || [];
-      const filesByProject = await this._collectFiles(testFileFilters, options.projectFilter);
+      const filesByProject = await this._collectFiles(this._runPhaseFromOptions(options));

      const allTestFiles = new Set<string>();
      for (const files of filesByProject.values())
@ -579,12 +605,10 @@ export class Runner {
    if (progress.canceled)
      return;

-    const testFileFilters: TestFileFilter[] = [...testFiles].map(f => ({
-      exact: f,
-      line: null,
-      column: null,
-    }));
-    const filesByProject = await this._collectFiles(testFileFilters, options.projectFilter);
+    const fileMatcher = createFileMatcher([...testFiles]);
+    const phase = this._runPhaseFromOptions(options);
+    phase.forEach(p => p.testFileMatcher = fileMatcher);
+    const filesByProject = await this._collectFiles(phase);

    if (progress.canceled)
      return;
@ -1023,6 +1047,12 @@ class ListModeReporter implements Reporter {
  }
 }

+function fileMatcherFrom(testFileFilters?: TestFileFilter[]): Matcher {
+  if (testFileFilters?.length)
+    return createFileMatcher(testFileFilters.map(e => e.re || e.exact || ''));
+  return () => true;
+}
+
 function createForbidOnlyError(config: FullConfigInternal, onlyTestsAndSuites: (TestCase | Suite)[]): TestError {
  const errorMessage = [
    '=====================================',
--- a/packages/playwright-test/src/util.ts
+++ b/packages/playwright-test/src/util.ts
@ -115,7 +115,7 @@ export function createFileMatcher(patterns: string | RegExp | (string | RegExp)[
    if (isRegExp(pattern)) {
      reList.push(pattern);
    } else {
-      if (!pattern.startsWith('**/') && !pattern.startsWith('**/'))
+      if (!pattern.startsWith('**/'))
        filePatterns.push('**/' + pattern);
      else
        filePatterns.push(pattern);
--- a/packages/playwright-test/types/test.d.ts
+++ b/packages/playwright-test/types/test.d.ts
@ -695,6 +695,28 @@ interface TestConfig {
     * Project name(s).
     */
    project: string|Array<string>;
+
+    /**
+     * Filter to only run tests with a title matching one of the patterns.
+     */
+    grep?: RegExp|Array<RegExp>;
+
+    /**
+     * Filter to only run tests with a title **not** matching one of the patterns.
+     */
+    grepInvert?: RegExp|Array<RegExp>;
+
+    /**
+     * Only the files matching one of these patterns are executed as test files. Matching is performed against the absolute
+     * file path. Strings are treated as glob patterns.
+     */
+    testMatch?: string|RegExp|Array<string|RegExp>;
+
+    /**
+     * Files matching one of these patterns are not executed as test files. Matching is performed against the absolute file
+     * path. Strings are treated as glob patterns.
+     */
+    testIgnore?: string|RegExp|Array<string|RegExp>;
  }>>; };

  /**
--- a/tests/playwright-test/config.spec.ts
+++ b/tests/playwright-test/config.spec.ts
@ -502,7 +502,103 @@ test('should throw when group has duplicate project references', async ({ runInl
  });

  expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`config.groups.default group mentions project 'a' twice in one parallel group`);
+  expect(result.output).toContain(`config.groups.default[0][1] group mentions project 'a' twice in one parallel group`);
+});
+
+test('should throw when group grep has invalid type', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          projects: [
+            { name: 'a' },
+          ],
+          groups: {
+            default: [
+              [{ project: 'a', grep: 2022 }]
+            ]
+          }
+        };
+    `,
+    'a.test.ts': `
+        const { test } = pwt;
+        test('pass', async () => {});
+      `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`config.groups.default[0][0].grep must be a RegExp`);
+});
+
+test('should throw when group grepInvert has invalid type', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          projects: [
+            { name: 'a' },
+          ],
+          groups: {
+            default: [
+              [{ project: 'a', grepInvert: [{}] }]
+            ]
+          }
+        };
+    `,
+    'a.test.ts': `
+        const { test } = pwt;
+        test('pass', async () => {});
+      `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`config.groups.default[0][0].grepInvert[0] must be a RegExp`);
+});
+
+test('should throw when group testMatch has invalid type', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          projects: [
+            { name: 'a' },
+          ],
+          groups: {
+            all: [
+              [{ project: 'a', testMatch: [{}] }]
+            ]
+          }
+        };
+    `,
+    'a.test.ts': `
+        const { test } = pwt;
+        test('pass', async () => {});
+      `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`config.groups.all[0][0].testMatch[0] must be a string or a RegEx`);
+});
+
+test('should throw when group testIgnore has invalid type', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          projects: [
+            { name: 'a' },
+          ],
+          groups: {
+            all: [
+              [{ project: 'a', testIgnore: [2022] }]
+            ]
+          }
+        };
+    `,
+    'a.test.ts': `
+        const { test } = pwt;
+        test('pass', async () => {});
+      `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`config.groups.all[0][0].testIgnore[0] must be a string or a RegEx`);
 });

 test('should throw when group has unknown project reference', async ({ runInlineTest }) => {
--- a/tests/playwright-test/groups.spec.ts
+++ b/tests/playwright-test/groups.spec.ts
@ -13,13 +13,13 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-import type { PlaywrightTestConfig, TestInfo } from '@playwright/test';
+import type { PlaywrightTestConfig, TestInfo, PlaywrightTestProject } from '@playwright/test';
 import path from 'path';
 import { test, expect } from './playwright-test-fixtures';

-function createConfigWithProjects(names: string[], testInfo: TestInfo, groups: PlaywrightTestConfig['groups']): Record<string, string> {
+function createConfigWithProjects(names: string[], testInfo: TestInfo, groups: PlaywrightTestConfig['groups'], projectTemplates?: { [name: string]: PlaywrightTestProject }): Record<string, string> {
  const config: PlaywrightTestConfig = {
-    projects: names.map(name => ({ name, testDir: testInfo.outputPath(name) })),
+    projects: names.map(name => ({ ...projectTemplates?.[name], name, testDir: testInfo.outputPath(name) })),
    groups
  };
  const files = {};
@ -30,10 +30,16 @@ function createConfigWithProjects(names: string[], testInfo: TestInfo, groups: P
        await new Promise(f => setTimeout(f, 100));
      });`;
  }
+  function replacer(key, value) {
+    if (value instanceof RegExp)
+      return `RegExp(${value.toString()})`;
+    else
+      return value;
+  }
  files['playwright.config.ts'] = `
    import * as path from 'path';
-    module.exports = ${JSON.stringify(config)};
-    `;
+    module.exports = ${JSON.stringify(config, replacer, 2)};
+    `.replace(/"RegExp\((.*)\)"/g, '$1');
  return files;
 }

@ -43,6 +49,12 @@ function formatTimeline(timeline: Timeline) {
  return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n');
 }

+function projectNames(timeline: Timeline) {
+  const projectNames = Array.from(new Set(timeline.map(({ titlePath }) => titlePath[1])).keys());
+  projectNames.sort();
+  return projectNames;
+}
+
 function expectRunBefore(timeline: Timeline, before: string[], after: string[]) {
  const begin = new Map<string, number>();
  const end = new Map<string, number>();
@ -258,3 +270,97 @@ test('should throw when unknown --group is passed', async ({ runGroups }, testIn
  expect(exitCode).toBe(1);
  expect(output).toContain(`Cannot find project group 'bar' in the config`);
 });
+
+test('should support testMatch and testIgnore', async ({ runGroups }, testInfo) => {
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      [
+        { project: ['a', 'b'], testMatch: ['**/a.spec.ts'] },
+        { project: ['c', 'd'], testMatch: [/.*c.spec.ts/, '**/*d*'] },
+        { project: ['e'], testIgnore: [/.*e.spec.ts/] },
+        { project: ['f'], testMatch: /does not match/ },
+      ],
+    ]
+  });
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(3);
+  expect(projectNames(timeline)).toEqual(['a', 'c', 'd']);
+});
+
+test('should support grep and grepInvert', async ({ runGroups }, testInfo) => {
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      [
+        { project: ['a', 'b'], grep: /.*a test/ },
+        { project: ['c', 'd'], grepInvert: [/.*c test/] },
+        { project: ['e', 'f'], grep: /.*(e|f) test/, grepInvert: [/.*f test/] },
+      ],
+    ]
+  });
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(3);
+  expect(projectNames(timeline)).toEqual(['a', 'd', 'e']);
+});
+
+test('should intercect gpoup and project level grep and grepInvert', async ({ runGroups }, testInfo) => {
+  const projectTemplates = {
+    'a': {
+      grep: /a test/,
+      grepInvert: [/no test/],
+    },
+    'b': {
+      grep: /.*b te.*/,
+      grepInvert: [/.*a test/],
+    },
+    'c': {
+      grepInvert: [/.*test/],
+    },
+    'd': {
+      grep: [/.*unkwnown test/],
+    },
+  };
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      [
+        { project: ['a', 'b', 'c', 'd', 'e'], grep: /.*(b|c|d|e) test/, grepInvert: /.*d test/ },
+      ],
+    ]
+  }, projectTemplates);
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(2);
+  expect(projectNames(timeline)).toEqual(['b', 'e']);
+});
+
+test('should intercect gpoup and project level testMatch and testIgnore', async ({ runGroups }, testInfo) => {
+  const projectTemplates = {
+    'a': {
+      testMatch: /.*a.spec.ts/,
+      testIgnore: [/no test/],
+    },
+    'b': {
+      testMatch: '**/b.spec.ts',
+      testIgnore: [/.*a.spec.ts/],
+    },
+    'c': {
+      testIgnore: [/.*no-match.spec.ts/],
+    },
+    'd': {
+      testMatch: [/.*unkwnown/],
+    },
+  };
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      [
+        { project: ['a', 'b', 'c', 'd'], testMatch: /.*(b|c|d).spec.ts/, testIgnore: /.*c.spec.ts/ },
+        { project: ['c', 'd', 'e', 'f'], testIgnore: /.*[^ef].spec.ts/ },
+      ],
+    ]
+  }, projectTemplates);
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(3);
+  expect(projectNames(timeline)).toEqual(['b', 'e', 'f']);
+});
--- a/utils/doclint/api_parser.js
+++ b/utils/doclint/api_parser.js
@ -254,7 +254,7 @@ function parseVariable(line) {
    if (depth === 0)
      return { name, type: remainder.substring(1, i), text: remainder.substring(i + 2), optional, experimental };
  }
-  throw new Error('Should not be reached');
+  throw new Error('Should not be reached, line: ' + line);
 }

 /**