Bug 1771269 - Build local toolchains on integration+release rather than trunk. r=taskgraph-reviewers,ahal

In rare cases (but which do happen), some change lands on release
branches that invalidates the local toolchain hashes. In those cases,
the local toolchains are currently not produced unless something else
depends on them, which leaves bootstrapping broken on those branches.

Because setting the right set of `run-on-projects` is manual and
error-prone, it's simpler to simply leave it to the taskgraph code
to set it properly for tasks with the `local-toolchain` attribute.

Differential Revision: https://phabricator.services.mozilla.com/D147473

taskcluster/ci/toolchain/android.yml

@@ -22,7 +22,6 @@ linux64-android-avd-x86_64-repack:
     description: "Android AVD (Linux) repack toolchain build"
     treeherder:
         symbol: TL(avd-x86_64-linux)
-    run-on-projects: [trunk]
     run:
         script: repack-android-avd-linux.sh
         arguments:
@@ -42,7 +41,6 @@ linux64-android-avd-arm-repack:
     description: "Android AVD (Linux) repack toolchain build"
     treeherder:
         symbol: TL(avd-arm-linux)
-    run-on-projects: [trunk]
     run:
         script: repack-android-avd-linux.sh
         arguments:
@@ -62,7 +60,6 @@ linux64-android-avd-arm64-repack:
     description: "Android AVD (Linux) repack toolchain build"
     treeherder:
         symbol: TL(avd-arm64-linux)
-    run-on-projects: [trunk]
     run:
         script: repack-android-avd-linux.sh
         arguments:

taskcluster/ci/toolchain/cbindgen.yml

@@ -12,7 +12,6 @@ job-defaults:
     run:
         script: build-cbindgen.sh
         toolchain-artifact: public/build/cbindgen.tar.zst
-    run-on-projects: [trunk]
     fetches:
         fetch:
             # If you update this, make sure to update the minimum version in

taskcluster/ci/toolchain/clang-tidy.yml

@@ -36,8 +36,6 @@ linux64-clang-tidy:
             - 'build/build-clang/clang-tidy-linux64.json'
         resources:
             - 'build/build-clang/clang-tidy-linux64.json'
-    run-on-projects:
-        - trunk
     fetches:
         toolchain:
             - linux64-clang-14
@@ -55,8 +53,6 @@ macosx64-clang-tidy:
             - 'build/build-clang/clang-tidy-macosx64.json'
         resources:
             - 'build/build-clang/clang-tidy-macosx64.json'
-    run-on-projects:
-        - trunk
     fetches:
         toolchain:
             - linux64-cctools-port
@@ -78,8 +74,6 @@ macosx64-arch64-clang-tidy:
             - 'build/build-clang/clang-tidy-macosx64.json'
         resources:
             - 'build/build-clang/clang-tidy-macosx64.json'
-    run-on-projects:
-        - trunk
     fetches:
         toolchain:
             - linux64-cctools-port
@@ -102,8 +96,6 @@ win64-clang-tidy:
             - 'build/build-clang/clang-tidy-win64.json'
         resources:
             - 'build/build-clang/clang-tidy-win64.json'
-    run-on-projects:
-        - trunk
     fetches:
         fetch:
             - cmake

taskcluster/ci/toolchain/clang.yml

@@ -77,7 +77,6 @@ linux64-clang-14-stage1:
     description: "Clang 14 toolchain build"
     treeherder:
         symbol: TL(clang-14-stage1)
-    run-on-projects: [trunk]
     run:
         using: toolchain-script
         script: build-clang.sh
@@ -158,7 +157,6 @@ linux64-clang-14:
     worker-type: b-linux
     worker:
         max-run-time: 600
-    run-on-projects: [trunk]
     run:
         script: repack-clang.sh
         toolchain-alias:
@@ -220,8 +218,6 @@ macosx64-clang-14:
     worker-type: b-linux
     worker:
         max-run-time: 600
-    run-on-projects:
-        - trunk
     run:
         script: repack-clang.sh
         toolchain-alias:
@@ -286,8 +282,6 @@ macosx64-aarch64-clang-14:
     worker-type: b-linux
     worker:
         max-run-time: 600
-    run-on-projects:
-        - trunk
     run:
         script: repack-clang.sh
         toolchain-alias: macosx64-aarch64-clang
@@ -373,8 +367,6 @@ win64-clang-14:
     worker-type: b-linux
     worker:
         max-run-time: 600
-    run-on-projects:
-        - trunk
     run:
         script: repack-clang.sh
         toolchain-alias: win64-clang

taskcluster/ci/toolchain/dist-toolchains.yml

@@ -7,8 +7,6 @@ job-defaults:
     attributes:
         local-toolchain: true
     worker-type: b-linux
-    run-on-projects:
-        - trunk
     run:
         using: toolchain-script
         script: build-dist-toolchains.sh

taskcluster/ci/toolchain/dump-syms.yml

@@ -8,7 +8,6 @@ job-defaults:
     description: "dump_syms toolchain build"
     worker:
         max-run-time: 1800
-    run-on-projects: [trunk]
     run:
         script: build-dump-syms.sh
     fetches:

taskcluster/ci/toolchain/fix-stacks.yml

@@ -12,8 +12,6 @@ job-defaults:
     run:
         script: build-fix-stacks.sh
         toolchain-artifact: public/build/fix-stacks.tar.zst
-    run-on-projects:
-        - trunk
     fetches:
         fetch:
             - fix-stacks

taskcluster/ci/toolchain/minidump-stackwalk.yml

@@ -12,8 +12,6 @@ job-defaults:
     run:
         script: build-minidump-stackwalk.sh
         toolchain-artifact: public/build/minidump-stackwalk.tar.zst
-    run-on-projects:
-        - trunk
     fetches:
         fetch:
             - rust-minidump

taskcluster/ci/toolchain/misc.yml

@@ -204,7 +204,6 @@ linux64-wine:
         tier: 1
     worker:
         max-run-time: 10800
-    run-on-projects: [trunk]
     run:
         script: build-wine.sh
         toolchain-artifact: public/build/wine.tar.zst
@@ -239,7 +238,6 @@ win64-pdbstr:
             - name: project/gecko/pdbstr/pdbstr.tar.zst
               path: pdbstr.tar.zst
               type: file
-    run-on-projects: [trunk]
     run:
         script: exfiltrate-pdbstr.sh
         arguments: ['x64']
@@ -251,7 +249,6 @@ linux64-winchecksec:
         local-toolchain: true
     treeherder:
         symbol: TL(winchecksec)
-    run-on-projects: [trunk]
     run:
         script: build-winchecksec.sh
         arguments: ['x86_64-unknown-linux-gnu']
@@ -267,7 +264,6 @@ win64-winchecksec:
         local-toolchain: true
     treeherder:
         symbol: TW64(winchecksec)
-    run-on-projects: [trunk]
     run:
         script: build-winchecksec.sh
         arguments: ['x86_64-pc-windows-msvc']
@@ -301,7 +297,6 @@ win64-mozmake:
     treeherder:
         symbol: TW64(mozmake)
     worker-type: b-win2012
-    run-on-projects: [trunk]
     run:
         script: build-mozmake.sh
         toolchain-artifact: public/build/mozmake.tar.zst
@@ -318,7 +313,6 @@ nsis:
         local-toolchain: true
     treeherder:
         symbol: TL(nsis)
-    run-on-projects: [trunk]
     run:
         script: build-nsis.sh
         toolchain-artifact: public/build/nsis.tar.zst

taskcluster/ci/toolchain/nasm.yml

@@ -6,7 +6,6 @@ job-defaults:
     worker-type: b-linux
     worker:
         max-run-time: 1800
-    run-on-projects: [trunk]
     run:
         toolchain-artifact: public/build/nasm.tar.zst
 

taskcluster/ci/toolchain/node.yml

@@ -9,7 +9,6 @@ job-defaults:
     worker-type: b-linux
     worker:
         max-run-time: 1800
-    run-on-projects: [trunk]
     run:
         script: repack-node.sh
         toolchain-artifact: public/build/node.tar.zst

taskcluster/ci/toolchain/pkgconf.yml

@@ -9,7 +9,6 @@ job-defaults:
     worker:
         max-run-time: 1800
     worker-type: b-linux
-    run-on-projects: [trunk]
     run:
         script: build-pkgconf.sh
         toolchain-artifact: public/build/pkgconf.tar.zst

taskcluster/ci/toolchain/sccache.yml

@@ -8,7 +8,6 @@ job-defaults:
         local-toolchain: true
     treeherder:
         symbol: TL(sccache)
-    run-on-projects: [trunk]
     run:
         using: toolchain-script
         script: build-sccache.sh

taskcluster/ci/toolchain/sysroot.yml

@@ -15,7 +15,6 @@ sysroot-i686-linux-gnu:
     description: "Sysroot for linux32 builds"
     attributes:
         local-toolchain: true
-    run-on-projects: [trunk]
     worker:
         env:
             PACKAGES_TASKS: {task-reference: '<gcc-7>'}
@@ -32,7 +31,6 @@ sysroot-x86_64-linux-gnu:
     description: "Sysroot for linux64 builds"
     attributes:
         local-toolchain: true
-    run-on-projects: [trunk]
     worker:
         env:
             PACKAGES_TASKS: {task-reference: '<gcc-7>'}
@@ -49,7 +47,6 @@ sysroot-x86_64-linux-gnu-x11:
     description: "Sysroot for linux64 builds (X11-only)"
     attributes:
         local-toolchain: true
-    run-on-projects: [trunk]
     worker:
         env:
             PACKAGES_TASKS: {task-reference: '<gcc-7> <gtk3>'}
@@ -67,7 +64,6 @@ sysroot-wasm32-wasi-clang-14:
     description: "Sysroot for wasi"
     attributes:
         local-toolchain: true
-    run-on-projects: [trunk]
     treeherder:
         symbol: TL(sysroot-wasi-14)
     run:
@@ -101,7 +97,6 @@ sysroot-aarch64-linux-gnu:
     description: "Sysroot for linux64 aarch64 builds"
     attributes:
         local-toolchain: true
-    run-on-projects: [trunk]
     treeherder:
         symbol: TL(sysroot-aarch64)
     run:

taskcluster/gecko_taskgraph/transforms/job/toolchain.py

@@ -167,6 +167,16 @@ def common_toolchain(config, job, taskdesc, is_docker):
             "digest-data": digest_data,
         }
 
+    # Toolchains that are used for local development need to be built on a
+    # level-3 branch to be installable via `mach bootstrap`.
+    if taskdesc["attributes"].get("local-toolchain"):
+        if taskdesc.get("run-on-projects"):
+            raise Exception(
+                "Toolchain {} used for local developement must not have"
+                " run-on-projects set".format(taskdesc["label"])
+            )
+        taskdesc["run-on-projects"] = ["integration", "release"]
+
     run["using"] = "run-task"
     if is_docker:
         gecko_path = "workspace/build/src"

taskcluster/gecko_taskgraph/util/verify.py

@@ -431,24 +431,3 @@ def verify_run_known_projects(task, taskgraph, scratch_pad, graph_config, parame
                 "Task '{}' has an invalid run-on-projects value: "
                 "{}".format(task.label, invalid_projects)
             )
-
-
-@verifications.add("full_task_graph")
-def verify_local_toolchains(task, taskgraph, scratch_pad, graph_config, parameters):
-    """
-    Toolchains that are used for local development need to be built on a
-    level-3 branch to installable via `mach bootstrap`. We ensure here that all
-    such tasks run on at least trunk projects, even if they aren't pulled in as
-    a dependency of other tasks in the graph.
-
-    There is code in `mach artifact toolchain` that verifies that anything
-    installed via `mach bootstrap` has the attribute set.
-    """
-    if task and task.attributes.get("local-toolchain"):
-        run_on_projects = task.attributes.get("run_on_projects", [])
-        if not any(alias in run_on_projects for alias in ["all", "trunk"]):
-            raise Exception(
-                "Toolchain {} used for local development is not built on trunk. {}".format(
-                    task.label, run_on_projects
-                )
-            )