Bug 1575760 - Make `mach vendor rust` create a .cargo/config and check it in the tree. r=nalexander

Maybe back when .cargo/config.in was added, the directory indicated for
vendored crates needed to be absolute. That is at least not the case
with the current supported versions of rust.

The current setup has a few caveats:
- .cargo/config.in has shown to become stale (it currently contains
  multiple unused entries)
- non-gecko build tasks have to generate a .cargo/config on their own if
  they want to use vendored crates
- in turn, non-gecko build tasks that don't, may unknowingly get their
  dependencies from crates.io (see the recent attempt at moving
  geckodriver builds to a separate task).

By checking in a .cargo/config file, we can alleviate the last two, but
that comes at the price of `cargo update` not wanting to act when
.cargo/config exists, because of the source replacement configuration.

But rust vendor gently generates a suitable configuration on its own, so
we can use that to generate a .cargo/config automatically. Which
addresses the first caveat of the current setup. That leaves us with
`cargo update` not working out of the box, but that just requires people
running it to manually remove .cargo/config first. Which is arguably
what rust wants you to do in the first place. It's kind of incidental
that we started with a .cargo/config.in rather than .cargo/config.

Now, while a simple .cargo/config works, that's not enough for the case
where the objdir doesn't live inside the source directory. In that case
cargo looks for the configuration from the objdir, and fails to find it.
So we still need a .cargo/config.in, which we generate with a little
trick.

Differential Revision: https://phabricator.services.mozilla.com/D43012

--HG--
rename : .cargo/config.in => .cargo/config
extra : moz-landing-system : lando

.cargo/.gitignore

@@ -1 +0,0 @@
-config

.cargo/config

@@ -0,0 +1,41 @@
+# This file contains vendoring instructions for cargo.
+# It was generated by `mach vendor rust`.
+# Please do not edit.
+
+[source."https://github.com/hsivonen/packed_simd"]
+branch = "rust_1_32"
+git = "https://github.com/hsivonen/packed_simd"
+replace-with = "vendored-sources"
+
+[source."https://github.com/froydnj/winapi-rs"]
+branch = "aarch64"
+git = "https://github.com/froydnj/winapi-rs"
+replace-with = "vendored-sources"
+
+[source."https://github.com/alexcrichton/mio-named-pipes"]
+branch = "master"
+git = "https://github.com/alexcrichton/mio-named-pipes"
+replace-with = "vendored-sources"
+
+[source."https://github.com/NikVolf/tokio-named-pipes"]
+branch = "stable"
+git = "https://github.com/NikVolf/tokio-named-pipes"
+replace-with = "vendored-sources"
+
+[source."https://github.com/CraneStation/Cranelift"]
+git = "https://github.com/CraneStation/Cranelift"
+replace-with = "vendored-sources"
+rev = "164f91a1f473e582e18e48d056c51787d9a1c24d"
+
+[source.crates-io]
+replace-with = "vendored-sources"
+
+# Take advantage of the fact that cargo will treat lines starting with #
+# as comments to add preprocessing directives for when this file is included
+# from .cargo/config.in.
+#define REPLACE_NAME vendored-sources
+#define VENDORED_DIRECTORY third_party/rust
+#ifndef top_srcdir
+[source.vendored-sources]
+directory = "third_party/rust"
+#endif

.cargo/config.in

@@ -1,45 +1,10 @@
-# Note: if you add more configure substitutions here with required values
-# you will also need to fix the sed commands in:
-# taskcluster/scripts/builder/build-sm-mozjs-crate.sh
-# taskcluster/scripts/builder/build-sm-rust-bindings.sh
+# Please do not edit this file.
 
-[source.crates-io]
-registry = 'https://github.com/rust-lang/crates.io-index'
-replace-with = 'vendored-sources'
+# Note: this file is only really needed when objdir is not a subdirectory of
+# the top source directory.
 
-[source."https://github.com/servo/serde"]
-git = "https://github.com/servo/serde"
-branch = "deserialize_from_enums10"
-replace-with = "vendored-sources"
+#include config
+#filter substitution
 
-[source."https://github.com/retep998/winapi-rs"]
-git = "https://github.com/froydnj/winapi-rs"
-branch = "aarch64"
-replace-with = "vendored-sources"
-
-[source."https://github.com/rust-lang-nursery/packed_simd"]
-git = "https://github.com/hsivonen/packed_simd"
-branch = "rust_1_32"
-replace-with = "vendored-sources"
-
-[source."https://github.com/CraneStation/Cranelift"]
-git = "https://github.com/CraneStation/Cranelift"
-rev = "164f91a1f473e582e18e48d056c51787d9a1c24d"
-replace-with = "vendored-sources"
-
-[source."https://github.com/ChunMinChang/coreaudio-sys"]
-git = "https://github.com/ChunMinChang/coreaudio-sys"
-branch = "gecko-build"
-replace-with = "vendored-sources"
-
-[source."https://github.com/alexcrichton/mio-named-pipes"]
-git = "https://github.com/alexcrichton/mio-named-pipes"
-replace-with = "vendored-sources"
-
-[source."https://github.com/NikVolf/tokio-named-pipes"]
-git = "https://github.com/NikVolf/tokio-named-pipes"
-branch = "stable"
-replace-with = "vendored-sources"
-
-[source.vendored-sources]
-directory = '@top_srcdir@/third_party/rust'
+[source.@REPLACE_NAME@]
+directory = "@top_srcdir@/@VENDORED_DIRECTORY@"

js/src/make-source-package.sh

@@ -129,6 +129,7 @@ case $cmd in
 
     ${MKDIR} -p ${tgtpath}/.cargo
     cp -pPR \
+        ${TOPSRCDIR}/.cargo/config \
         ${TOPSRCDIR}/.cargo/config.in \
         ${tgtpath}/.cargo/
 

js/src/wasm/cranelift/Cargo.toml

@@ -14,7 +14,7 @@ name = "baldrdash"
 # you should update the following files:
 # - $TOP_LEVEL/Cargo.toml, look for the revision (rev) hashes of both cranelift
 # dependencies (codegen and wasm).
-# - $TOP_LEVEL/.cargo/config.in, look for the revision (rev) field of the
+# - $TOP_LEVEL/.cargo/config, look for the revision (rev) field of the
 # Cranelift source.
 cranelift-codegen = { version = "0.40", default-features = false }
 cranelift-wasm = "0.40"

python/mozbuild/mozbuild/vendor_rust.py

@@ -11,8 +11,11 @@ import os
 import re
 import subprocess
 import sys
+from collections import OrderedDict
 from distutils.version import LooseVersion
+from itertools import dropwhile
 
+import pytoml
 import mozpack.path as mozpath
 from mozbuild.base import (
     BuildEnvironmentNotFoundException,
@@ -20,6 +23,24 @@ from mozbuild.base import (
 )
 
 
+CARGO_CONFIG_TEMPLATE = '''\
+# This file contains vendoring instructions for cargo.
+# It was generated by `mach vendor rust`.
+# Please do not edit.
+
+{config}
+
+# Take advantage of the fact that cargo will treat lines starting with #
+# as comments to add preprocessing directives for when this file is included
+# from .cargo/config.in.
+#define REPLACE_NAME {replace_name}
+#define VENDORED_DIRECTORY {directory}
+#ifndef top_srcdir
+{replace}
+#endif
+'''
+
+
 class VendorRust(MozbuildObject):
     def get_cargo_path(self):
         try:
@@ -317,12 +338,83 @@ license file's hash.
         relative_vendor_dir = 'third_party/rust'
         vendor_dir = mozpath.join(self.topsrcdir, relative_vendor_dir)
 
+        cargo_config = os.path.join(self.topsrcdir, '.cargo', 'config')
+        # First, remove .cargo/config
+        try:
+            os.remove(cargo_config)
+        except Exception:
+            pass
+
         # We use check_call instead of mozprocess to ensure errors are displayed.
         # We do an |update -p| here to regenerate the Cargo.lock file with minimal
         # changes. See bug 1324462
         subprocess.check_call([cargo, 'update', '-p', 'gkrust'], cwd=self.topsrcdir)
 
-        subprocess.check_call([cargo, 'vendor', '--quiet', vendor_dir], cwd=self.topsrcdir)
+        output = subprocess.check_output([cargo, 'vendor', vendor_dir],
+                                         stderr=subprocess.STDOUT,
+                                         cwd=self.topsrcdir)
+
+        # Get the snippet of configuration that cargo vendor outputs, and
+        # update .cargo/config with it.
+        # XXX(bug 1576765): Hopefully do something better after
+        # https://github.com/rust-lang/cargo/issues/7280 is addressed.
+        config = '\n'.join(dropwhile(lambda l: not l.startswith('['),
+                                     output.splitlines()))
+
+        # The config is toml, parse it as such.
+        config = pytoml.loads(config)
+
+        # For each replace-with, extract their configuration and update the
+        # corresponding directory to be relative to topsrcdir.
+        replaces = {
+            v['replace-with']
+            for v in config['source'].values()
+            if 'replace-with' in v
+        }
+
+        # We only really expect one replace-with
+        if len(replaces) != 1:
+            self.log(
+                logging.ERROR, 'vendor_failed', {},
+                '''cargo vendor didn't output a unique replace-with. Found: %s.''' % replaces)
+            sys.exit(1)
+
+        replace_name = replaces.pop()
+        replace = config['source'].pop(replace_name)
+        replace['directory'] = mozpath.relpath(
+            mozpath.normsep(os.path.normcase(replace['directory'])),
+            mozpath.normsep(os.path.normcase(self.topsrcdir)),
+        )
+
+        # Introduce some determinism for the output.
+        def recursive_sort(obj):
+            if isinstance(obj, dict):
+                return OrderedDict(sorted(
+                    (k, recursive_sort(v)) for k, v in obj.items()))
+            if isinstance(obj, list):
+                return [recursive_sort(o) for o in obj]
+            return obj
+
+        config = recursive_sort(config)
+
+        # Normalize pytoml output:
+        # - removing empty lines
+        # - remove empty [section]
+        def toml_dump(data):
+            dump = pytoml.dumps(data)
+            if isinstance(data, dict):
+                for k, v in data.items():
+                    if all(isinstance(v2, dict) for v2 in v.values()):
+                        dump = dump.replace('[%s]' % k, '')
+            return dump.strip()
+
+        with open(cargo_config, 'w') as fh:
+            fh.write(CARGO_CONFIG_TEMPLATE.format(
+                config=toml_dump(config),
+                replace_name=replace_name,
+                directory=replace['directory'],
+                replace=toml_dump({'source': {replace_name: replace}}),
+            ))
 
         if not self._check_licenses(vendor_dir):
             self.log(

taskcluster/scripts/builder/build-sm-mozjs-crate.sh

@@ -4,11 +4,6 @@ set -xe
 
 source $(dirname $0)/sm-tooltool-config.sh
 
-# Ensure that we have a .config/cargo that points us to our vendored crates
-# rather than to crates.io.
-cd "$SRCDIR/.cargo"
-sed -e "s|@top_srcdir@|$SRCDIR|" -e 's|@[^@]*@||g' < config.in > config
-
 cd "$SRCDIR/js/src"
 
 export PATH="$PATH:$MOZ_FETCHES_DIR/cargo/bin:$MOZ_FETCHES_DIR/rustc/bin"

taskcluster/scripts/builder/build-sm-rust-bindings.sh

@@ -4,11 +4,6 @@ set -xe
 
 source $(dirname $0)/sm-tooltool-config.sh
 
-# Ensure that we have a .config/cargo that points us to our vendored crates
-# rather than to crates.io.
-cd "$SRCDIR/.cargo"
-sed -e "s|@top_srcdir@|$SRCDIR|" -e 's|@[^@]*@||g' < config.in > config
-
 cd "$SRCDIR/js/rust"
 
 export LD_LIBRARY_PATH="$MOZ_FETCHES_DIR/gcc/lib64"