Bug 1883499 - Update task schedule optimizations for firefox-android. r=aryx

Simplify scheduling of firefox-android tasks:
 - remove the gradle_optimization transform, which added project-specific skip-unless-changed optimizations
 - remove skip-unless-changed from task definitions, which were only in support of gradle_optimization
 - add skip-unless-backstop for firebase-based tests, to try to reduce runs of these expensive tasks on autoland.

The overall intent is for most firefox-android tasks to behave like geckoview tasks do today, with the exception of firebase-based tests (ui-test-apk, etc) which should be less likely to run on non-backstop pushes on autoland.

Differential Revision: https://phabricator.services.mozilla.com/D204204

taskcluster/android_taskgraph/build_config.py

@@ -53,31 +53,6 @@ def _read_build_config(root_dir):
         return yaml.safe_load(f)
 
 
-@memoize
-def get_upstream_deps_for_all_gradle_projects():
-    all_deps = {}
-    for root_dir in (ANDROID_COMPONENTS_DIR, FOCUS_DIR, FENIX_DIR):
-        build_config = _read_build_config(root_dir)
-        new_deps = {
-            project: project_config["upstream_dependencies"]
-            for project, project_config in build_config["projects"].items()
-        }
-
-        app_config = new_deps.pop("app", None)
-        if app_config:
-            if root_dir == FOCUS_DIR:
-                gradle_project = "focus"
-            elif root_dir == FENIX_DIR:
-                gradle_project = "fenix"
-            else:
-                raise NotImplementedError(f"Unsupported root_dir {root_dir}")
-            new_deps[gradle_project] = app_config
-
-        all_deps.update(new_deps)
-
-    return all_deps
-
-
 def get_apk_based_projects():
     return [
         {

taskcluster/android_taskgraph/gradle.py

@@ -1,13 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-def get_gradle_project(task):
-    attributes = task.get("attributes", {})
-    gradle_project = attributes.get("component")
-    if not gradle_project:
-        shipping_product = attributes.get("shipping-product", "")
-        if shipping_product:
-            return shipping_product
-    return gradle_project

taskcluster/android_taskgraph/transforms/gradle_optimization.py

@@ -1,87 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from taskgraph.transforms.base import TransformSequence
-
-from ..build_config import get_path, get_upstream_deps_for_all_gradle_projects
-from ..gradle import get_gradle_project
-
-transforms = TransformSequence()
-
-
-@transforms.add
-def add_components_optimization(config, tasks):
-    for task in tasks:
-        attributes = task.get("attributes", {})
-        # TODO bug 1806454 - Use a single attribute instead of 2. This is a historical
-        # discrepancy where A-C are labeled by build-types but APKs by release ones.
-        #
-        # The monorepo migration made this discrepancy more obvious compared to when
-        # A-C and APKs lived in different repos
-        build_type = attributes.get("build-type", "")
-        release_type = attributes.get("release-type", "")
-
-        # We want to optimize away tasks on all android-components and APKs as long as
-        # these tasks are not labeled nightly, beta, or release.
-        #
-        # Any change that impacts all a-c (e.g. a change in the a-c gradle config)
-        # should also trigger APK builds and tests.
-        if all(
-            type_ not in ("nightly", "beta", "release")
-            for type_ in (build_type, release_type)
-        ):
-            optimization = task.setdefault("optimization", {})
-            skip_unless_changed = optimization.setdefault("skip-unless-changed", [])
-            skip_unless_changed.extend(
-                [
-                    "mobile/android/android-components/build.gradle",
-                    "mobile/android/android-components/settings.gradle",
-                    "mobile/android/android-components/buildSrc.*",
-                    "mobile/android/android-components/gradle.properties",
-                    "mobile/android/android-components/gradle/wrapper/gradle-wrapper.properties",
-                    "mobile/android/android-components/plugins/dependencies/**",
-                ]
-            )
-
-        yield task
-
-
-@transforms.add
-def extend_optimization_if_one_already_exists(config, tasks):
-    deps_per_component = get_upstream_deps_for_all_gradle_projects()
-
-    for task in tasks:
-        optimization = task.get("optimization")
-        if optimization:
-            skip_unless_changed = optimization["skip-unless-changed"]
-            skip_unless_changed.append(f"{config.path}/**")
-
-            gradle_project = get_gradle_project(task)
-            # TODO Remove this special case when ui-test.sh is able to accept "browser-engine-gecko"
-            if gradle_project == "browser":
-                gradle_project = "browser-engine-gecko"
-
-            if gradle_project:
-                dependencies = deps_per_component[gradle_project]
-                gradle_project_and_deps = [gradle_project] + dependencies
-
-                skip_unless_changed.extend(
-                    sorted(
-                        [
-                            _get_path(gradle_project)
-                            for gradle_project in gradle_project_and_deps
-                        ]
-                    )
-                )
-
-        yield task
-
-
-def _get_path(gradle_project):
-    if gradle_project == "focus":
-        return "mobile/android/focus-android/**"
-    elif gradle_project == "fenix":
-        return "mobile/android/fenix/**"
-    else:
-        return f"mobile/android/android-components/{get_path(gradle_project)}/**"

taskcluster/ci/build-apk/kind.yml

@@ -6,7 +6,6 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - android_taskgraph.transforms.build_android_app:transforms
-    - android_taskgraph.transforms.gradle_optimization:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 

taskcluster/ci/build-bundle/kind.yml

@@ -6,7 +6,6 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - android_taskgraph.transforms.build_android_app:transforms
-    - android_taskgraph.transforms.gradle_optimization:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 

taskcluster/ci/build-components/kind.yml

@@ -6,7 +6,6 @@ loader: android_taskgraph.loader.build_config:components_loader
 
 transforms:
     - android_taskgraph.transforms.build_components:transforms
-    - android_taskgraph.transforms.gradle_optimization:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 

taskcluster/ci/build-samples-browser/kind.yml

@@ -6,7 +6,6 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - android_taskgraph.transforms.build_components:transforms
-    - android_taskgraph.transforms.gradle_optimization:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 

taskcluster/ci/test-apk/kind.yml

@@ -6,7 +6,6 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - gecko_taskgraph.transforms.test_apk:transforms
-    - android_taskgraph.transforms.gradle_optimization:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 
@@ -27,8 +26,6 @@ job-defaults:
             - linux64-jdk
         build-fat-aar:
             - target.maven.tar.xz
-    optimization:
-        skip-unless-changed: []  # Paths are dynamically added by transforms
     run:
         using: gradlew
         use-caches: false

taskcluster/ci/test-components/kind.yml

@@ -6,7 +6,6 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - android_taskgraph.transforms.ui_tests:transforms
-    - android_taskgraph.transforms.gradle_optimization:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 
@@ -58,9 +57,7 @@ job-defaults:
             GOOGLE_PROJECT: moz-android-components-230120
         max-run-time: 2400
     optimization:
-        skip-unless-changed:
+        skip-unless-backstop: null
-            - mobile/android/android-components/automation/taskcluster/androidTest/ui-test.sh
-            # More paths are dynamically added by transforms
 
 
 jobs:

taskcluster/ci/ui-test-apk/kind.yml

@@ -6,7 +6,6 @@ loader: gecko_taskgraph.loader.transform:loader
 
 transforms:
     - gecko_taskgraph.transforms.test_apk:transforms
-    - android_taskgraph.transforms.gradle_optimization:transforms
     - gecko_taskgraph.transforms.job:transforms
     - gecko_taskgraph.transforms.task:transforms
 
@@ -23,7 +22,7 @@ job-defaults:
         toolchain:
             - android-sdk-linux
     optimization:
-        skip-unless-changed: []  # Paths are dynamically added by transforms
+        skip-unless-backstop: null
     worker-type: b-linux-gcp
     worker:
         docker-image: {in-tree: android-ui-tests}