Bug 1245969 - Add tasks that perform a bootstrap on various bare Linux environments. r=taskgraph-reviewers,firefox-build-system-reviewers,jlorenzo,ahochheiden

Differential Revision: https://phabricator.services.mozilla.com/D162974

taskcluster/ci/bootstrap/kind.yml

@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: gecko_taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - toolchain
+
+transforms:
+    - gecko_taskgraph.transforms.bootstrap:transforms
+    - gecko_taskgraph.transforms.task:transforms
+
+jobs:
+    debian10:
+        image: debian:buster
+        pre-commands:
+            - apt-get update
+            - apt-get install -y --no-install-recommends curl mercurial python3-pip
+    debian11:
+        image: debian:bullseye
+        pre-commands:
+            - apt-get update
+            - apt-get install -y --no-install-recommends curl mercurial python3-pip
+    debian12:
+        image: debian:bookworm
+        pre-commands:
+            - apt-get update
+            - apt-get install -y --no-install-recommends curl mercurial python3-pip
+    ubuntu2204:
+        image: ubuntu:jammy
+        pre-commands:
+            - apt-get update
+            - apt-get install -y --no-install-recommends curl mercurial python3-pip
+    ubuntu2210:
+        image: ubuntu:kinetic
+        pre-commands:
+            - apt-get update
+            - apt-get install -y --no-install-recommends curl mercurial python3-pip
+    fedora36:
+        image: fedora:36
+        pre-commands:
+            - dnf install -y mercurial python3-pip
+    fedora37:
+        image: fedora:37
+        pre-commands:
+            - dnf install -y mercurial python3-pip
+            # Work around https://bz.mercurial-scm.org/show_bug.cgi?id=6761
+            - mkdir -p /etc/mercurial/hgrc.d
+            - (echo "[devel]"; echo discovery.randomize=false) > /etc/mercurial/hgrc.d/workaround.rc
+    rockylinux8:
+        image: rockylinux:8
+        pre-commands:
+            - dnf install -y mercurial python3-pip
+    opensuse-leap:
+        image: opensuse/leap
+        pre-commands:
+            - zypper install -y mercurial python3-pip gzip
+    opensuse-tumbleweed:
+        image: opensuse/tumbleweed
+        pre-commands:
+            - zypper install -y mercurial python3-pip
+    gentoo:
+        image: gentoo/stage3
+        pre-commands:
+            - emerge --sync
+            - emerge mercurial dev-python/pip
+    archlinux:
+        image: archlinux
+        pre-commands:
+            - pacman -Sy --noconfirm mercurial python-pip
+    voidlinux:
+        image: voidlinux/voidlinux
+        pre-commands:
+            - xbps-install -Suyv xbps
+            - xbps-install -Suyv curl mercurial python3-pip
+            # Work around https://bz.mercurial-scm.org/show_bug.cgi?id=6761
+            - mkdir -p /etc/mercurial/hgrc.d
+            - (echo "[devel]"; echo discovery.randomize=false) > /etc/mercurial/hgrc.d/workaround.rc

taskcluster/ci/config.yml

@@ -205,6 +205,7 @@ treeherder:
         'l10n': 'Localization checks'
         'fxrec': 'Desktop startup recorder (fxrecord)'
         'wc': 'webcompat'
+        'Boot': 'Bootstrap'
 
 index:
     products:

taskcluster/docs/kinds.rst

@@ -53,6 +53,11 @@ This kind performs an artifact build: one based on precompiled binaries
 discovered via the TaskCluster index.  This task verifies that such builds
 continue to work correctly.
 
+bootstrap
+---------
+
+This kind performs a standalone bootstrap of a Firefox build from various system environments.
+
 hazard
 ------
 

taskcluster/gecko_taskgraph/transforms/bootstrap.py

@@ -0,0 +1,104 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from voluptuous import Any, Optional, Required
+
+transforms = TransformSequence()
+
+bootstrap_schema = Schema(
+    {
+        # Name of the bootstrap task.
+        Required("name"): str,
+        # Name of the docker image. Ideally, we'd also have tasks for mac and windows,
+        # but we unfortunately don't have workers barebones enough for such testing
+        # to be satisfactory.
+        Required("image"): Any(str, {"in-tree": str}),
+        # Initialization commands.
+        Required("pre-commands"): [str],
+        # relative path (from config.path) to the file task was defined in
+        Optional("job-from"): str,
+    }
+)
+
+
+transforms.add_validate(bootstrap_schema)
+
+
+@transforms.add
+def bootstrap_tasks(config, tasks):
+    for task in tasks:
+        name = task.pop("name")
+        image = task.pop("image")
+        pre_commands = task.pop("pre-commands")
+
+        head_repo = config.params["head_repository"]
+        head_rev = config.params["head_rev"]
+
+        # Get all the non macos/windows local toolchains (the only ones bootstrap can use),
+        # and use them as dependencies for the tasks we create, so that they don't start
+        # before any potential toolchain task that would be triggered on the same push
+        # (which would lead to bootstrap failing).
+        dependencies = {
+            name: name
+            for name, task in config.kind_dependencies_tasks.items()
+            if task.attributes.get("local-toolchain")
+            and not name.startswith(("toolchain-macos", "toolchain-win"))
+        }
+        # We don't test the artifacts variants, or js, because they are essentially subsets.
+        # Mobile and browser are different enough to warrant testing them separately.
+        for app in ("browser", "mobile_android"):
+            commands = pre_commands + [
+                # MOZ_AUTOMATION changes the behavior, and we want something closer to user
+                # machines.
+                "unset MOZ_AUTOMATION",
+                f"curl -O {head_repo}/raw-file/{head_rev}/python/mozboot/bin/bootstrap.py",
+                f"python3 bootstrap.py --no-interactive --application-choice {app}",
+                "cd mozilla-unified",
+                # After bootstrap, configure should go through without its own auto-bootstrap.
+                "./mach configure --disable-bootstrap",
+                # Then a build should go through too.
+                "./mach build",
+            ]
+
+            taskdesc = {
+                "label": f"{config.kind}-{name}-{app}",
+                "description": f"Bootstrap {app} build on {name}",
+                "always-target": True,
+                "scopes": [],
+                "treeherder": {
+                    "symbol": f"Boot({name})",
+                    "platform": {
+                        "browser": "linux64/opt",
+                        "mobile_android": "android-5-0-armv7/opt",
+                    }[app],
+                    "kind": "other",
+                    "tier": 2,
+                },
+                "worker-type": "b-linux-gcp",
+                "worker": {
+                    "implementation": "docker-worker",
+                    "docker-image": image,
+                    "os": "linux",
+                    "env": {
+                        "GECKO_HEAD_REPOSITORY": head_repo,
+                        "GECKO_HEAD_REV": head_rev,
+                        "MACH_NO_TERMINAL_FOOTER": "1",
+                        "MOZ_SCM_LEVEL": config.params["level"],
+                    },
+                    "command": ["sh", "-c", "-x", "-e", " && ".join(commands)],
+                    "max-run-time": 7200,
+                },
+                "dependencies": dependencies,
+                "optimization": {
+                    "skip-unless-changed": [
+                        "python/mozboot/bin/bootstrap.py",
+                        "python/mozboot/mozboot/**",
+                    ],
+                },
+            }
+
+            yield taskdesc