Bug 1886193 - Use 'build' optimization strategy for new Android builds, r=geckoview-reviewers,taskgraph-reviewers,jcristau,tthibaud

By setting these to the "build" optimization, it means that bugbug will apply its
machine learning algorithm on autoland and with `mach try auto`.

But in addition, this also sets up some new SCHEDULES rules. A new mutually
exclusive SCHEDULES group around the "application" is created. Namely it can be
one of:

- firefox
- fenix
- focus-android

Files under `/browser` are tagged with the "firefox" component, while files
under `mobile/android` will be tagged with both "fenix" and "focus-android"
components. Files under the `mobile/android/fenix` or
`mobile/android/focus-android` dirs are further restricted to their respective
component. Note that any files that can impact Gecko should not be tagged with
any of these components.

On the task side, all of the prior Firefox builds and tests are tagged with the
"firefox" component, while all of the new Android builds and tests are tagged
with "fenix", "focus-android" or both (depending on the task).

The upshot of all this, is that patches that *only* modify files under
`/mobile/android` will not run any of the Firefox builds and tests. Conversely
patches that *only* modify files under `/browser` will not run any of the
Android builds and tests.

Differential Revision: https://phabricator.services.mozilla.com/D205127

browser/moz.build

@@ -63,7 +63,7 @@ FINAL_TARGET_FILES.defaults += ["app/permissions"]
 
 with Files("**"):
     BUG_COMPONENT = ("Firefox", "General")
-    SCHEDULES.exclusive = ["linux", "macosx", "windows"]
+    SCHEDULES.exclusive = ["linux", "macosx", "windows", "firefox"]
 
 with Files("docs/**"):
     SCHEDULES.exclusive = ["docs"]

mobile/android/moz.build

@@ -6,16 +6,18 @@
 
 with Files("**"):
     BUG_COMPONENT = ("GeckoView", "General")
-    SCHEDULES.exclusive = ["android"]
+    SCHEDULES.exclusive = ["android", "fenix", "focus-android"]
 
 with Files("geckoview_example/**"):
     BUG_COMPONENT = ("GeckoView", "GeckoViewExample")
 
 with Files("fenix/**"):
     BUG_COMPONENT = ("Fenix", "General")
+    SCHEDULES.exclusive = ["android", "fenix"]
 
 with Files("focus-android/**"):
     BUG_COMPONENT = ("Focus", "General")
+    SCHEDULES.exclusive = ["android", "focus-android"]
 
 with Files("android-components/**"):
     BUG_COMPONENT = ("Fenix", "General")

mobile/ios/moz.build

@@ -8,3 +8,4 @@ CONFIGURE_SUBST_FILES += ["installer/Makefile"]
 
 with Files("**"):
     BUG_COMPONENT = ("Core", "General")
+    SCHEDULES.exclusive = ["ios"]

python/mozbuild/mozbuild/schedules.py

@@ -43,6 +43,10 @@ EXCLUSIVE_COMPONENTS = [
     "macosx",
     "windows",
     "ios",
+    # application
+    "firefox",
+    "fenix",
+    "focus-android",
     # broad test harness categories
     "awsy",
     "condprofile",

taskcluster/ci/build-apk/kind.yml

@@ -6,6 +6,7 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - android_taskgraph.transforms.build_android_app:transforms
+    - gecko_taskgraph.transforms.build_schedules:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 

taskcluster/ci/build-bundle/kind.yml

@@ -6,6 +6,7 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - android_taskgraph.transforms.build_android_app:transforms
+    - gecko_taskgraph.transforms.build_schedules:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 

taskcluster/ci/build-components/kind.yml

@@ -6,6 +6,7 @@ loader: android_taskgraph.loader.build_config:components_loader
 
 transforms:
     - android_taskgraph.transforms.build_components:transforms
+    - gecko_taskgraph.transforms.build_schedules:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 

taskcluster/ci/build/kind.yml

@@ -12,6 +12,7 @@ kind-dependencies:
 transforms:
     - gecko_taskgraph.transforms.build:transforms
     - gecko_taskgraph.transforms.build_attrs:transforms
+    - gecko_taskgraph.transforms.build_schedules:transforms
     - gecko_taskgraph.transforms.build_lints:transforms
     - gecko_taskgraph.transforms.release_notifications:transforms
     - gecko_taskgraph.transforms.job:transforms

taskcluster/ci/test-apk/kind.yml

@@ -6,6 +6,8 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - gecko_taskgraph.transforms.test_apk:transforms
+    # While not a build, the build optimization works well for these tasks.
+    - gecko_taskgraph.transforms.build_schedules:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 

taskcluster/gecko_taskgraph/test/conftest.py

@@ -17,6 +17,7 @@ from taskgraph.parameters import Parameters
 from gecko_taskgraph import GECKO
 from gecko_taskgraph.actions import render_actions_json
 from gecko_taskgraph.util.templates import merge
+from taskgraph.transforms.base import TransformConfig
 
 
 @pytest.fixture
@@ -151,6 +152,20 @@ class FakeOptimization(OptimizationStrategy):
         return False
 
 
+class FakeTransformConfig:
+    kind = "fake-kind"
+    path = "/root/ci/fake-kind"
+    config = {}
+    params = FakeParameters()
+    kind_dependencies_tasks = {}
+    graph_config = {}
+    write_artifacts = False
+
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+
 @pytest.fixture
 def maketgg(monkeypatch):
     def inner(target_tasks=None, kinds=[("_fake", [])], params=None):
@@ -195,12 +210,16 @@ def maketgg(monkeypatch):
 @pytest.fixture
 def run_transform():
     graph_config = fake_load_graph_config("/root")
-    kind = FakeKind.create("fake", {}, graph_config)
+    config = FakeTransformConfig(graph_config=graph_config)
+
+    def inner(xform, tasks, **extra_config):
+        if extra_config:
+            for k, v in extra_config.items():
+                setattr(config, k, v)
 
-    def inner(xform, tasks):
         if isinstance(tasks, dict):
             tasks = [tasks]
-        return xform(kind.config, tasks)
+        return xform(config, tasks)
 
     return inner
 

taskcluster/gecko_taskgraph/test/python.toml

@@ -17,6 +17,8 @@ subsuite = "taskgraph"
 
 ["test_taskcluster_yml.py"]
 
+["test_transforms_build_schedules.py"]
+
 ["test_transforms_job.py"]
 
 ["test_transforms_test.py"]

taskcluster/gecko_taskgraph/test/test_transforms_build_schedules.py

@@ -0,0 +1,56 @@
+import pytest
+from mozunit import main
+
+from gecko_taskgraph.transforms.build_schedules import set_build_schedules_optimization
+
+
+@pytest.mark.parametrize(
+    "kind,task,expected",
+    (
+        pytest.param("build", {"when": "foo"}, None, id="no-op"),
+        pytest.param(
+            "build",
+            {"attributes": {"build_platform": "linux64/opt"}},
+            {"build": ["linux", "firefox"]},
+            id="build",
+        ),
+        pytest.param(
+            "build-components",
+            {},
+            {"build": ["android", "fenix", "focus-android"]},
+            id="build-components",
+        ),
+        pytest.param(
+            "build-bundle",
+            {"name": "build-bundle-fenix"},
+            {"build": ["android", "fenix"]},
+            id="build-bundle-fenix",
+        ),
+        pytest.param(
+            "build-apk",
+            {"name": "fenix"},
+            {"build": ["android", "fenix"]},
+            id="build-apk-fenix",
+        ),
+        pytest.param(
+            "build-apk",
+            {"name": "build-apk-focus"},
+            {"build": ["android", "focus-android"]},
+            id="build-apk-focus",
+        ),
+        pytest.param(
+            "build-apk",
+            {"name": "build-apk-klar"},
+            {"build": ["android", "focus-android"]},
+            id="build-apk-klar",
+        ),
+    ),
+)
+def test_build_schedules(run_transform, kind, task, expected):
+    tasks = list(run_transform(set_build_schedules_optimization, [task], kind=kind))
+    assert len(tasks) == 1
+    assert tasks[0].get("optimization") == expected
+
+
+if __name__ == "__main__":
+    main()

taskcluster/gecko_taskgraph/transforms/build_attrs.py

@@ -4,8 +4,6 @@
 
 from taskgraph.transforms.base import TransformSequence
 
-from gecko_taskgraph.util.platforms import platform_family
-
 transforms = TransformSequence()
 
 
@@ -34,17 +32,3 @@ def set_build_attributes(config, jobs):
         )
 
         yield job
-
-
-@transforms.add
-def set_schedules_optimization(config, jobs):
-    """Set the `skip-unless-affected` optimization based on the build platform."""
-    for job in jobs:
-        # don't add skip-unless-schedules if there's already a when defined
-        if "when" in job:
-            yield job
-            continue
-
-        build_platform = job["attributes"]["build_platform"]
-        job.setdefault("optimization", {"build": [platform_family(build_platform)]})
-        yield job

taskcluster/gecko_taskgraph/transforms/build_schedules.py

@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from taskgraph.transforms.base import TransformSequence
+
+from gecko_taskgraph.util.platforms import platform_family
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_build_schedules_optimization(config, tasks):
+    """Set the `build` optimization based on the build platform."""
+    for task in tasks:
+        # don't add an optimization if there's already one defined
+        if "when" in task or "optimization" in task:
+            yield task
+            continue
+
+        schedules = []
+        if config.kind == "build":
+            family = platform_family(task["attributes"]["build_platform"])
+            schedules = [family]
+
+            if "android" not in family:
+                # These are not GeckoView builds, so are associated with Firefox.
+                schedules.append("firefox")
+
+        elif config.kind == "build-components":
+            # These are Android components builds and can only impact Fenix or Focus.
+            schedules = ["android", "fenix", "focus-android"]
+
+        elif config.kind in ("build-apk", "build-bundle", "test-apk"):
+            # These are APK builds for Fenix or Focus
+            schedules = ["android"]
+
+            if "fenix" in task["name"]:
+                schedules.append("fenix")
+            elif "focus" in task["name"] or "klar" in task["name"]:
+                schedules.append("focus-android")
+
+        task["optimization"] = {"build": schedules}
+        yield task

taskcluster/gecko_taskgraph/transforms/test/other.py

@@ -1110,6 +1110,7 @@ def set_schedules_components(config, tasks):
 
         schedules.add(category)
         schedules.add(platform_family(task["build-platform"]))
+        schedules.add("firefox")
 
         task["schedules-component"] = sorted(schedules)
         yield task