Bug 1748966 - Speed up bootstrap. r=firefox-build-system-reviewers,andi

Currently, the bootstrap code from configure gets bootstrapped task info
from the taskgraph, and then invokes `mach artifact toolchain` which...
does the same again. That cumulatively wastes a noticeable amount of
time, especially on Windows.

So to avoid repeating this work that the configure side already did,
we do a little more on the configure side (resolution from index to
task-id), and just give a (task-id, artifact path) pair to `mach
artifact toolchain`. The added code to `mach artifact toolchain` is
actually ironically very similar to the code that was removed in
bug 1687594.

And now that mach bootstrap uses the configure code, it benefits from
this change as well.

Differential Revision: https://phabricator.services.mozilla.com/D135311

build/moz.configure/bootstrap.configure

@@ -96,7 +96,7 @@ def bootstrap_toolchain_tasks(host):
     # amount of data to what we use, so that trace logs can be more useful.
     tasks = {
         k: {
-            "index": t.optimization["index-search"][0],
+            "index": t.optimization["index-search"],
             "artifact": t.attributes["toolchain-artifact"],
         }
         for k, t in tasks.items()
@@ -124,6 +124,7 @@ def bootstrap_path(path, **kwargs):
     @imports("subprocess")
     @imports("sys")
     @imports(_from="mozbuild.util", _import="ensureParentDir")
+    @imports(_from="importlib", _import="import_module")
     @imports(_from="__builtin__", _import="open")
     @imports(_from="__builtin__", _import="Exception")
     def bootstrap_path(bootstrap, toolchains_base_dir, tasks, build_env, path):
@@ -144,8 +145,8 @@ def bootstrap_path(path, **kwargs):
             if not task:
                 return False
             task_index = task["index"]
-            log.debug("Resolved %s to %s", label, task_index)
-            task_index = task_index.split(".")[-1]
+            log.debug("Resolved %s to %s", label, task_index[0])
+            task_index = task_index[0].split(".")[-1]
             artifact = task["artifact"]
             # `mach artifact toolchain` doesn't support authentication for
             # private artifacts.
@@ -161,6 +162,18 @@ def bootstrap_path(path, **kwargs):
             if index == task_index and exists:
                 log.debug("%s is up-to-date", label)
                 return True
+            # Manually import with import_module so that we can gracefully disable bootstrap
+            # when e.g. building from a js standalone tarball, that doesn't contain the
+            # taskgraph code. In those cases, `mach artifact toolchain --from-build` would
+            # also fail.
+            try:
+                IndexSearch = import_module(
+                    "gecko_taskgraph.optimize.strategies"
+                ).IndexSearch
+            except Exception:
+                log.debug("Cannot bootstrap %s: missing taskgraph module", label)
+                return False
+            task_id = IndexSearch().should_replace_task(task, {}, None, task["index"])
             log.info(
                 "%s bootstrapped toolchain in %s",
                 "Updating" if exists else "Installing",
@@ -174,8 +187,8 @@ def bootstrap_path(path, **kwargs):
                     "--log-no-times",
                     "artifact",
                     "toolchain",
-                    "--from-build",
-                    label,
+                    "--from-task",
+                    f"{task_id}:{artifact}",
                 ],
                 cwd=toolchains_base_dir,
                 check=True,

python/mozbuild/mozbuild/artifact_commands.py

@@ -220,6 +220,12 @@ def artifact_clear_cache(command_context, tree=None, job=None, verbose=False):
     help="Download toolchains resulting from the given build(s); "
     "BUILD is a name of a toolchain task, e.g. linux64-clang",
 )
+@CommandArgument(
+    "--from-task",
+    metavar="TASK_ID:ARTIFACT",
+    nargs="+",
+    help="Download toolchain artifact from a given task.",
+)
 @CommandArgument(
     "--tooltool-manifest",
     metavar="MANIFEST",
@@ -248,6 +254,7 @@ def artifact_toolchain(
     cache_dir=None,
     skip_cache=False,
     from_build=(),
+    from_task=(),
     tooltool_manifest=None,
     no_unpack=False,
     retry=0,
@@ -461,6 +468,20 @@ def artifact_toolchain(
             record = ArtifactRecord(task_id, artifact_name)
             records[record.filename] = record
 
+    # Handle the list of files of the form task_id:path from --from-task.
+    for f in from_task:
+        task_id, colon, name = f.partition(":")
+        if not colon:
+            self.log(
+                logging.ERROR,
+                "artifact",
+                {},
+                "Expected an argument of the form task_id:path",
+            )
+            return 1
+        record = ArtifactRecord(task_id, name)
+        records[record.filename] = record
+
     for record in six.itervalues(records):
         command_context.log(
             logging.INFO,
@@ -558,6 +579,8 @@ def artifact_toolchain(
 
     if not downloaded:
         command_context.log(logging.ERROR, "artifact", {}, "Nothing to download")
+        if from_task:
+            return 1
 
     if artifacts:
         ensureParentDir(artifact_manifest)