Bug 1677593 - Add support for building rust from source. r=glandium

This adds a linux64-rust-dev toolchain and a git fetch of rust-lang/rust
that it can source from (stable tag for 1.47).

There are some issues with cross-compiling, so for now the toolchain only
builds a host build, although a lot of the machinery for cross compiling is
there for anyone brave/desperate enough to get it working.

Also note some changes were made to Rust's config.toml between 1.47 and 1.50,
so some version detection may need to be added in the future.

There is experimental support for providing patches via a new --patch flag.

Additionally, I documented the existence of the "bors-" mode.

Differential Revision: https://phabricator.services.mozilla.com/D97497

taskcluster/ci/fetch/toolchains.yml

@@ -431,6 +431,13 @@ clang-11:
         repo: https://github.com/llvm/llvm-project
         revision: 176249bd6732a8044d457092ed932768724a6f06
 
+rust-1.47.0-dev:
+    description: Rust 1.47.0-dev source code
+    fetch:
+        type: git
+        repo: https://github.com/rust-lang/rust/
+        revision: 18bf6b4f01a6feaf7259ba7cdae58031af1b7b39
+
 wasi-sdk:
     description: wasi-sdk source code
     fetch:

taskcluster/ci/toolchain/rust.yml

@@ -54,6 +54,26 @@ linux64-rust-nightly:
             '--target', 'i686-unknown-linux-gnu',
         ]
 
+linux64-rust-dev:
+    description: "build rust from source"
+    worker-type: b-linux-large
+    treeherder:
+        symbol: TL(rust-dev)
+    run:
+        arguments: [
+            '--channel', 'dev',
+            '--host', 'x86_64-unknown-linux-gnu',
+            '--target', 'x86_64-unknown-linux-gnu',
+            '--compiler-builtins-hack',
+        ]
+    fetches:
+        fetch:
+            - rust-1.47.0-dev
+        toolchain:
+            - linux64-clang
+            - linux64-binutils
+
+
 linux64-rust-cross-1.47:
     description: "rust repack with macos and windows cross support"
     treeherder:

taskcluster/docker/toolchain-build/Dockerfile

@@ -33,6 +33,7 @@ RUN apt-get update && \
       libucl-dev \
       ninja-build \
       p7zip-full \
+      pkg-config \
       procps \
       python-pip \
       python-setuptools \

taskcluster/scripts/misc/repack_rust.py

@@ -21,6 +21,7 @@ import subprocess
 from contextlib import contextmanager
 import tarfile
 import tempfile
+import textwrap
 
 import requests
 import pytoml as toml
@@ -29,7 +30,7 @@ import zstandard
 
 
 def log(msg):
-    print("repack: %s" % msg)
+    print("repack: %s" % msg, flush=True)
 
 
 def fetch_file(url):
@@ -348,11 +349,125 @@ def fetch_manifest(channel="stable", host=None, targets=()):
     return manifest
 
 
-def repack(
-    host, targets, channel="stable", cargo_channel=None, compiler_builtins_hack=False
-):
-    log("Repacking rust for %s supporting %s..." % (host, targets))
+def patch_src(patch, module):
+    log("Patching Rust src... {} with {}".format(module, patch))
+    patch = os.path.realpath(patch)
+    subprocess.check_call(["patch", "-d", module, "-p1", "-i", patch, "--fuzz=0", "-s"])
 
+
+def build_src(install_dir, host, targets, patch):
+    install_dir = os.path.abspath(install_dir)
+    fetches = os.environ["MOZ_FETCHES_DIR"]
+    rust_dir = os.path.join(fetches, "rust")
+    patch_dir = os.path.join(os.environ["GECKO_PATH"], "build", "build-rust")
+
+    # Clear and remake any previous install directory.
+    try:
+        shutil.rmtree(install_dir)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    os.makedirs(install_dir)
+
+    # Patch the src (see the --patch flag's description for details)
+    if patch:
+        for p in patch:
+            module, colon, file = p.partition(":")
+            if not colon:
+                module, file = "", p
+            patch_file = os.path.join(patch_dir, file)
+            patch_module = os.path.join(rust_dir, module)
+            patch_src(patch_file, patch_module)
+
+    log("Building Rust...")
+
+    # Rust builds are configured primarily through a config.toml file.
+    #
+    # `sysconfdir` is overloaded to be relative instead of absolute.
+    # This is the default of `install.sh`, but for whatever reason
+    # `x.py install` has its own default of `/etc` which we don't want.
+    #
+    # `missing-tools` is set so `rustfmt` is allowed to fail. This means
+    # we can "succeed" at building Rust while failing to build, say, Cargo.
+    # Ideally the build system would have better granularity:
+    # https://github.com/rust-lang/rust/issues/79249
+    base_config = textwrap.dedent(
+        """
+        [build]
+        extended = true
+        tools = ["analysis", "cargo", "rustfmt", "src"]
+
+        [install]
+        prefix = "{prefix}"
+        sysconfdir = "etc"
+
+        [dist]
+        missing-tools = true
+
+        """.format(
+            prefix=install_dir
+        )
+    )
+
+    # Rust requires these to be specified per-target
+    target_config = textwrap.dedent(
+        """
+        [target.{target}]
+        cc = "clang"
+        cxx = "clang++"
+        linker = "clang"
+
+        """
+    )
+
+    final_config = base_config
+    for target in sorted(set(targets) | set([host])):
+        final_config = final_config + target_config.format(target=target)
+
+    with open(os.path.join(rust_dir, "config.toml"), "w") as file:
+        file.write(final_config)
+
+    # Setup the env so compilers and toolchains are visible
+    binutils = os.path.join(fetches, "binutils", "bin")
+    clang = os.path.join(fetches, "clang")
+    clang_bin = os.path.join(clang, "bin")
+    clang_lib = os.path.join(clang, "lib")
+
+    env = os.environ.copy()
+    env.update(
+        {
+            "PATH": os.pathsep.join((binutils, clang_bin, os.environ["PATH"])),
+            "LD_LIBRARY_PATH": clang_lib,
+        }
+    )
+
+    # x.py install does everything we need for us.
+    # If you're running into issues, consider using `-vv` to debug it.
+    command = ["python3", "x.py", "install", "-v", "--host", host]
+    for target in targets:
+        command.extend(["--target", target])
+
+    subprocess.check_call(command, stderr=subprocess.STDOUT, env=env, cwd=rust_dir)
+
+
+def repack(
+    host,
+    targets,
+    channel="stable",
+    cargo_channel=None,
+    compiler_builtins_hack=False,
+    patch=None,
+):
+    install_dir = "rustc"
+    if channel == "dev":
+        build_src(install_dir, host, targets, patch)
+    else:
+        if patch:
+            raise ValueError(
+                'Patch specified, but channel "%s" is not "dev"!'
+                "\nPatches are only for building from source." % channel
+            )
+        log("Repacking rust for %s supporting %s..." % (host, targets))
         manifest = fetch_manifest(channel, host, targets)
         log("Using manifest for rust %s as of %s." % (channel, manifest["date"]))
         if cargo_channel == channel:
@@ -372,7 +487,7 @@ def repack(
         rustfmt = fetch_optional(manifest, "rustfmt-preview", host)
 
         log("Installing packages...")
-    install_dir = "rustc"
+
         # Clear any previous install directory.
         try:
             shutil.rmtree(install_dir)
@@ -544,9 +659,20 @@ def args():
         "--channel",
         help="Release channel to use:"
         " 1.xx.y, beta-yyyy-mm-dd,"
-        " or nightly-yyyy-mm-dd.",
+        " nightly-yyyy-mm-dd,"
+        " bors-$rev (grab a build from rust's CI),"
+        " or 1.xx.y-dev (build from source).",
         required=True,
     )
+    parser.add_argument(
+        "--patch",
+        nargs="+",
+        help="apply the given patch file to a dev build."
+        " Patch files should be placed in /build/rust-build."
+        " Patches can be prefixed with `module-path:` to specify they"
+        " apply to that git submodule in the Rust source."
+        " e.g. `src/llvm-project:mypatch.diff` patches rust's llvm.",
+    )
     parser.add_argument(
         "--cargo-channel",
         help="Release channel version to use for cargo."