Bug 1886927 - Avoid getting hostutils.manifest off hg.mozilla.org. r=jmaher

This introduces a toolchain task that just re-exposes the hostutils from tooltool, which means adding that task as dependency of the relevant tests gets it automatically. We can then get the hostutils from MOZ_FETCHES_DIR directly. Differential Revision: https://phabricator.services.mozilla.com/D205424
2024-03-22 20:34:15 +00:00 · 2024-03-22 20:34:15 +00:00 · 6393a05402
--- a/taskcluster/ci/generate-profile/kind.yml
+++ b/taskcluster/ci/generate-profile/kind.yml
@ -123,6 +123,7 @@ jobs:
            toolchain:
                - linux64-clang
                - linux64-minidump-stackwalk
+                - linux64-hostutils
                - android-sdk-linux
                - android-system-image-x86_64-linux
                - android-avd-x86_64-linux
@ -164,6 +165,7 @@ jobs:
            toolchain:
                - linux64-clang
                - linux64-minidump-stackwalk
+                - linux64-hostutils
                - android-sdk-linux
                - android-system-image-x86_64-linux
                - android-avd-x86_64-linux
@ -206,6 +208,7 @@ jobs:
            toolchain:
                - linux64-clang
                - linux64-minidump-stackwalk
+                - linux64-hostutils
                - android-sdk-linux
                - android31-system-image-x86_64-linux
                - android31-avd-x86_64-linux
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@ -84,6 +84,11 @@ job-defaults:
                    - android-emulator-linux
                    - linux64-minidump-stackwalk
                    - linux64-fix-stacks
+                    - linux64-hostutils
+                android-hw.*:
+                    - linux64-minidump-stackwalk
+                    - linux64-fix-stacks
+                    - linux64-hostutils
                default:
                    - linux64-minidump-stackwalk
                    - linux64-fix-stacks
--- a/taskcluster/ci/test/misc.yml
+++ b/taskcluster/ci/test/misc.yml
@ -128,6 +128,7 @@ test-verify:
                    - linux64-node
                    - linux64-minidump-stackwalk
                    - linux64-fix-stacks
+                    - linux64-hostutils

 test-verify-gpu:
    description: "Extra verification of tests modified on this push on gpu instances"
@ -229,6 +230,7 @@ test-coverage:
                    - linux64-node
                    - linux64-minidump-stackwalk
                    - linux64-fix-stacks
+                    - linux64-hostutils

 test-coverage-gpu:
    description: "Per-test coverage for tests that require gpu instances"
--- a/taskcluster/ci/test/mochitest.yml
+++ b/taskcluster/ci/test/mochitest.yml
@ -177,6 +177,7 @@ mochitest-plain:
                    - android-emulator-linux
                    - linux64-node
                    - linux64-minidump-stackwalk
+                    - linux64-hostutils

 mochitest-a11y:
    description: "Mochitest a11y run"
@ -545,6 +546,7 @@ mochitest-devtools-chrome:
                    - android-emulator-linux
                    - linux64-node
                    - linux64-minidump-stackwalk
+                    - linux64-hostutils


 mochitest-devtools-chrome-failures:
--- a/taskcluster/ci/test/variants.yml
+++ b/taskcluster/ci/test/variants.yml
@ -77,6 +77,7 @@ aab:
                - linux64-minidump-stackwalk
                - linux64-fix-stacks
                - linux64-jdk
+                - linux64-hostutils
    replace:
        target: "geckoview-test_runner.aab"

--- a/taskcluster/ci/test/web-platform.yml
+++ b/taskcluster/ci/test/web-platform.yml
@ -415,6 +415,7 @@ web-platform-tests-wdspec:
                    - linux64-geckodriver
                    - linux64-minidump-stackwalk
                    - linux64-fix-stacks
+                    - linux64-hostutils
                linux.*64.*:
                    - linux64-geckodriver
                    - linux64-minidump-stackwalk
--- a/taskcluster/ci/test/xpcshell.yml
+++ b/taskcluster/ci/test/xpcshell.yml
@ -148,6 +148,7 @@ xpcshell:
                    - android-emulator-linux
                    - linux64-node
                    - linux64-minidump-stackwalk
+                    - linux64-hostutils

 xpcshell-failures:
    description: "xpcshell specific failures test run"
--- a/taskcluster/ci/toolchain/hostutils.yml
+++ b/taskcluster/ci/toolchain/hostutils.yml
@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+    description: "hostutils toolchain build"
+    worker-type: b-linux-gcp
+    worker:
+        max-run-time: 1800
+    run:
+        using: toolchain-script
+        script: get-hostutils.sh
+        toolchain-artifact: public/build/hostutils.tar.zst
+
+linux64-hostutils:
+    treeherder:
+        symbol: TL(hostutils)
+    worker:
+        env:
+            TOOLTOOL_MANIFEST: testing/config/tooltool-manifests/linux64/hostutils.manifest
+    run:
+        resources:
+            - testing/config/tooltool-manifests/linux64/hostutils.manifest
--- a/taskcluster/ci/toolchain/kind.yml
+++ b/taskcluster/ci/toolchain/kind.yml
@ -36,6 +36,7 @@ jobs-from:
    - geckodriver.yml
    - gn.yml
    - grcov.yml
+    - hostutils.yml
    - libunwind.yml
    - llvm-symbolizer.yml
    - macos-sdk.yml
--- a/taskcluster/scripts/misc/get-hostutils.sh
+++ b/taskcluster/scripts/misc/get-hostutils.sh
@ -0,0 +1,14 @@
+#!/bin/sh
+
+artifact=$(basename "$TOOLCHAIN_ARTIFACT")
+project=${artifact%.tar.*}
+
+cd $GECKO_PATH
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+cd $MOZ_FETCHES_DIR
+mv host-utils-* $project
+tar -acvf $artifact $project
+mkdir -p $UPLOAD_DIR
+mv $artifact $UPLOAD_DIR
--- a/testing/mozharness/mozharness/mozilla/testing/android.py
+++ b/testing/mozharness/mozharness/mozilla/testing/android.py
@ -116,21 +116,6 @@ class AndroidMixin(object):
        url = "%s/raw-file/%s/%s" % (repo, revision, path)
        return url

-    def _tooltool_fetch(self, url, dir):
-        c = self.config
-        manifest_path = self.download_file(
-            url, file_name="releng.manifest", parent_dir=dir
-        )
-        if not os.path.exists(manifest_path):
-            self.fatal(
-                "Could not retrieve manifest needed to retrieve "
-                "artifacts from %s" % manifest_path
-            )
-        # from TooltoolMixin, included in TestingMixin
-        self.tooltool_fetch(
-            manifest_path, output_dir=dir, cache=c.get("tooltool_cache", None)
-        )
-
    def _launch_emulator(self):
        env = self.query_env()

@ -507,25 +492,6 @@ class AndroidMixin(object):
        if reset_dir:
            del os.environ["MOZ_UPLOAD_DIR"]

-    def download_hostutils(self, xre_dir):
-        """
-        Download and install hostutils from tooltool.
-        """
-        xre_path = None
-        self.rmtree(xre_dir)
-        self.mkdir_p(xre_dir)
-        if self.config["hostutils_manifest_path"]:
-            url = self._get_repo_url(self.config["hostutils_manifest_path"])
-            self._tooltool_fetch(url, xre_dir)
-            for p in glob.glob(os.path.join(xre_dir, "host-utils-*")):
-                if os.path.isdir(p) and os.path.isfile(os.path.join(p, "xpcshell")):
-                    xre_path = p
-            if not xre_path:
-                self.fatal("xre path not found in %s" % xre_dir)
-        else:
-            self.fatal("configure hostutils_manifest_path!")
-        return xre_path
-
    def query_package_name(self):
        if self.app_name is None:
            # For convenience, assume geckoview.test/geckoview_example when install
--- a/testing/mozharness/scripts/android_emulator_pgo.py
+++ b/testing/mozharness/scripts/android_emulator_pgo.py
@ -100,9 +100,9 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin):
        dirs = {}

        dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_src_dir"], "testing")
-        dirs["abs_xre_dir"] = os.path.join(abs_dirs["abs_work_dir"], "hostutils")
        dirs["abs_blob_upload_dir"] = "/builds/worker/artifacts/blobber_upload_dir"
        work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"]
+        dirs["abs_xre_dir"] = os.path.join(work_dir, "hostutils")
        dirs["abs_sdk_dir"] = os.path.join(work_dir, "android-sdk-linux")
        dirs["abs_avds_dir"] = os.path.join(work_dir, "android-device")
        dirs["abs_bundletool_path"] = os.path.join(work_dir, "bundletool.jar")
@ -134,7 +134,7 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin):
        Download host utilities
        """
        dirs = self.query_abs_dirs()
-        self.xre_path = self.download_hostutils(dirs["abs_xre_dir"])
+        self.xre_path = dirs["abs_xre_dir"]

    def install(self):
        """
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@ -209,7 +209,6 @@ class AndroidEmulatorTest(
        dirs["abs_test_bin_dir"] = os.path.join(
            abs_dirs["abs_work_dir"], "tests", "bin"
        )
-        dirs["abs_xre_dir"] = os.path.join(abs_dirs["abs_work_dir"], "hostutils")
        dirs["abs_modules_dir"] = os.path.join(dirs["abs_test_install_dir"], "modules")
        dirs["abs_blob_upload_dir"] = os.path.join(
            abs_dirs["abs_work_dir"], "blobber_upload_dir"
@ -222,6 +221,7 @@ class AndroidEmulatorTest(
            dirs["abs_test_install_dir"], "xpcshell"
        )
        work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"]
+        dirs["abs_xre_dir"] = os.path.join(work_dir, "hostutils")
        dirs["abs_sdk_dir"] = os.path.join(work_dir, "android-sdk-linux")
        dirs["abs_avds_dir"] = os.path.join(work_dir, "android-device")
        dirs["abs_bundletool_path"] = os.path.join(work_dir, "bundletool.jar")
@ -460,7 +460,7 @@ class AndroidEmulatorTest(
            suite_categories=self._query_suite_categories()
        )
        dirs = self.query_abs_dirs()
-        self.xre_path = self.download_hostutils(dirs["abs_xre_dir"])
+        self.xre_path = dirs["abs_xre_dir"]

    def install(self):
        """
--- a/testing/mozharness/scripts/android_hardware_unittest.py
+++ b/testing/mozharness/scripts/android_hardware_unittest.py
@ -178,7 +178,6 @@ class AndroidHardwareTest(
        dirs["abs_test_bin_dir"] = os.path.join(
            abs_dirs["abs_work_dir"], "tests", "bin"
        )
-        dirs["abs_xre_dir"] = os.path.join(abs_dirs["abs_work_dir"], "hostutils")
        dirs["abs_modules_dir"] = os.path.join(dirs["abs_test_install_dir"], "modules")
        dirs["abs_blob_upload_dir"] = os.path.join(
            abs_dirs["abs_work_dir"], "blobber_upload_dir"
@ -190,6 +189,8 @@ class AndroidHardwareTest(
        dirs["abs_xpcshell_dir"] = os.path.join(
            dirs["abs_test_install_dir"], "xpcshell"
        )
+        work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"]
+        dirs["abs_xre_dir"] = os.path.join(work_dir, "hostutils")

        for key in dirs.keys():
            if key not in abs_dirs:
@ -390,7 +391,7 @@ class AndroidHardwareTest(
            suite_categories=self._query_suite_categories()
        )
        dirs = self.query_abs_dirs()
-        self.xre_path = self.download_hostutils(dirs["abs_xre_dir"])
+        self.xre_path = dirs["abs_xre_dir"]

    def install(self):
        """
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@ -281,10 +281,10 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidM
        dirs["abs_test_extensions_dir"] = os.path.join(
            dirs["abs_test_install_dir"], "extensions"
        )
+        work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"]
        if self.is_android:
-            dirs["abs_xre_dir"] = os.path.join(abs_dirs["abs_work_dir"], "hostutils")
+            dirs["abs_xre_dir"] = os.path.join(work_dir, "hostutils")
        if self.is_emulator:
-            work_dir = os.environ.get("MOZ_FETCHES_DIR") or abs_dirs["abs_work_dir"]
            dirs["abs_sdk_dir"] = os.path.join(work_dir, "android-sdk-linux")
            dirs["abs_avds_dir"] = os.path.join(work_dir, "android-device")
            dirs["abs_bundletool_path"] = os.path.join(work_dir, "bundletool.jar")
@ -543,7 +543,7 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidM
        )
        dirs = self.query_abs_dirs()
        if self.is_android:
-            self.xre_path = self.download_hostutils(dirs["abs_xre_dir"])
+            self.xre_path = dirs["abs_xre_dir"]
        # Make sure that the logging directory exists
        if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
            self.fatal("Could not create blobber upload directory")