Add filters to control autoclassification in JSON output from config (#679)

Fixes #665

docs/configuration.rst

@@ -112,3 +112,26 @@ logging, while setting it to ``2`` enables trace logging.
 .. _TOML: http://github.com/toml-lang/toml
 .. _cachy: https://github.com/sdispater/cachy
 .. _cachy's configuration format: https://cachy.readthedocs.io/en/latest/configuration.html
+
+autoclassification
+``````````````````
+
+Mozci controls which push classification results can be automatically processed by third-party tools (like Treeherder), using a feature flag and a set of filters for test names.
+
+The feature can be fully disabled by setting `enabled` to `false`.
+
+.. code-block:: toml
+
+    [mozci.autoclassification]
+    enabled = true
+    test-suite-names = [
+      "test-linux64-*/opt-mochitest-*",
+      "*wpt*",
+    ]
+
+
+Each value in the list of `test-suite-names` support [fnmatch](https://docs.python.org/3/library/fnmatch.html#fnmatch.fnmatch) syntax to allow glob-like syntax (using `*` for wildcard and `?` for single characters).
+
+The configuration above will enable autoclassification for tests matching `test-linux64-*/opt-mochitest-*` or `*wpt*`
+
+Finally, the JSON classification output is extended to have an `autoclassify` boolean flag on each failure details payload, to check if this specific result should be processed.

mozci/configuration.py

@@ -109,6 +109,10 @@ class Configuration(Mapping):
     ) or os.environ.get("TASKCLUSTER_SECRET")
     DEFAULTS = {
         "merge": {
+            "autoclassification": {
+                "enabled": False,
+                "test-suite-names": [],
+            },
             "cache": {"retention": 1440},
         },  # minutes
         "replace": {
@@ -149,6 +153,10 @@ class Configuration(Mapping):
         self.cache = CustomCacheManager(self._config["cache"])
         self.locked = True
 
+        # Check auto classification settings
+        assert isinstance(self._config["autoclassification"]["enabled"], bool)
+        assert isinstance(self._config["autoclassification"]["test-suite-names"], list)
+
     def __len__(self):
         return len(self._config)
 

mozci/console/commands/push.py

@@ -25,7 +25,7 @@ from mozci.push import (
     make_push_objects,
     retrigger,
 )
-from mozci.task import Task, TestTask
+from mozci.task import Task, TestTask, is_autoclassifiable
 from mozci.util.taskcluster import (
     COMMUNITY_TASKCLUSTER_ROOT_URL,
     get_taskcluster_options,
@@ -199,33 +199,31 @@ class ClassifyCommand(Command):
                 self.line("-" * 50)
 
             if output:
+
+                def _serialize_regressions(regressions):
+                    return {
+                        group: [
+                            {
+                                "task_id": task.id,
+                                "label": task.label,
+                                "autoclassify": is_autoclassifiable(task),
+                            }
+                            for task in failing_tasks
+                        ]
+                        for group, failing_tasks in regressions.items()
+                    }
+
                 to_save = {
                     "push": {
                         "id": push.push_uuid,
                         "classification": classification.name,
                     },
                     "failures": {
-                        "real": {
-                            group: [
-                                {"task_id": task.id, "label": task.label}
-                                for task in failing_tasks
-                            ]
-                            for group, failing_tasks in regressions.real.items()
-                        },
-                        "intermittent": {
-                            group: [
-                                {"task_id": task.id, "label": task.label}
-                                for task in failing_tasks
-                            ]
-                            for group, failing_tasks in regressions.intermittent.items()
-                        },
-                        "unknown": {
-                            group: [
-                                {"task_id": task.id, "label": task.label}
-                                for task in failing_tasks
-                            ]
-                            for group, failing_tasks in regressions.unknown.items()
-                        },
+                        "real": _serialize_regressions(regressions.real),
+                        "intermittent": _serialize_regressions(
+                            regressions.intermittent
+                        ),
+                        "unknown": _serialize_regressions(regressions.unknown),
                     },
                 }
 

mozci/task.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import collections
+import fnmatch
 import os
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
@@ -15,7 +16,7 @@ import requests
 import taskcluster
 from loguru import logger
 
-from mozci import data
+from mozci import config, data
 from mozci.errors import ArtifactNotFound, TaskNotFound
 from mozci.util.memoize import memoized_property
 from mozci.util.taskcluster import (
@@ -123,6 +124,21 @@ def is_bad_group(task_id: str, group: str) -> bool:
     return bad_group
 
 
+def is_autoclassifiable(task: Task) -> bool:
+    """Check a task is enabled for auto-classification
+    by applying glob patterns from configuration
+    """
+    assert task.label, "Missing task label"
+
+    if not config["autoclassification"]["enabled"]:
+        return False
+
+    return any(
+        fnmatch.fnmatch(task.label, pattern)
+        for pattern in config["autoclassification"]["test-suite-names"]
+    )
+
+
 # Transform WPT group names to a full relative path in mozilla-central.
 def wpt_workaround(group: str) -> str:
     # No need to transform empty groups (also, they will be filtered out

tests/test_task.py

@@ -5,8 +5,9 @@ import re
 
 import pytest
 
+from mozci import config
 from mozci.errors import ArtifactNotFound, TaskNotFound
-from mozci.task import GroupResult, GroupSummary, Task
+from mozci.task import GroupResult, GroupSummary, Task, is_autoclassifiable
 from mozci.util.taskcluster import get_artifact_url, get_index_url
 
 GR_2 = GroupResult(group="group2", ok=True, duration=42)
@@ -546,3 +547,49 @@ def test_GroupSummary_is_cross_config_failure(group_summary, expected_result):
 )
 def test_GroupSummary_is_config_consistent_failure(group_summary, expected_result):
     assert group_summary.is_config_consistent_failure(2) == expected_result
+
+
+@pytest.mark.parametrize(
+    "enabled, filters, result",
+    [
+        # Disabled feature
+        (False, [], False),
+        (False, ["*"], False),
+        (False, ["test-macosx*/opt-*"], False),
+        (False, ["test-macosx1015-64/opt-xpcshell-e10s-something"], False),
+        # Enabled feature
+        (True, [], False),
+        (True, ["*"], True),
+        (True, ["test-macosx*/opt-*"], True),
+        (True, ["test-macosx1015-64/opt-xpcshell-e10s-something"], True),
+        # Multiple filters
+        (True, ["*linux*/*", "test-mac*/*"], True),
+        (True, ["*linux*/*", "*/opt-xpcshell-e10s-*"], True),
+        (True, ["whatever/*", "test-macosx1015-64/opt-*-e10s-*"], True),
+        # Invalid filters
+        (
+            True,
+            [
+                "test-macosx1015-64/another-*",
+                "*-MacOsX-*",
+                "test-macosx1234*/*",
+                "*/*-suffix",
+            ],
+            False,
+        ),
+        # Support both wildcard and single character replacement
+        (True, ["test-macosx1015-?4/opt-*"], True),
+    ],
+)
+def test_autoclassify(enabled, filters, result):
+    """Check autoclassification filtering algorithm"""
+
+    # Update configuration
+    config._config["autoclassification"]["enabled"] = enabled
+    config._config["autoclassification"]["test-suite-names"] = filters
+
+    # Configure task with static label
+    task = Task.create(
+        id="someId", label="test-macosx1015-64/opt-xpcshell-e10s-something"
+    )
+    assert is_autoclassifiable(task) is result