Bug 1524662 - Add --enable-path-remapping producing compiled objects with generic paths. r=glandium

For "c" (i.e., gcc, clang, and clang-cl), this configures the
`-f{debug,macro}-path-prefix` flags.  We'd prefer to use
`-ffile-path-prefix`, but it seems that `clang-cl` does not recognize
that flag.

For "rust" (i.e., rustc/cargo), this configures `--remap-path-prefix`.

This is one step toward getting `sccache` hits across source and
object directories.

Differential Revision: https://phabricator.services.mozilla.com/D113065

build/moz.configure/flags.configure

@@ -61,3 +61,47 @@ def new_pass_manager_flags(enabled, compiler, host, target, pgo, enable_fuzzing,
 
 
 set_config("MOZ_NEW_PASS_MANAGER_FLAGS", new_pass_manager_flags)
+
+
+# Try to make builds more reproducible and allow sharing built artifacts across
+# source and object directories by using -ffile-prefix-map and friends.  To
+# "unwind" the prefix maps, use:
+#
+# (gdb) set substitute-path /topsrcdir/ $topsrcdir/
+#
+# (lldb) settings set target.source-map /topobjdir/ $topobjdir/
+#
+# See, for example, https://lldb.llvm.org/use/map.html.
+@depends(
+    path_remapping,
+    path_remappings,
+    c_compiler,
+)
+@imports(_from="os", _import="sep")
+def file_prefix_map_flags(path_remapping, path_remappings, compiler):
+    if "c" not in path_remapping:
+        return []
+
+    if (compiler.type == "gcc" and compiler.version < "8.1") or (
+        compiler.type in ("clang", "clang-cl") and compiler.version < "10.0.0"
+    ):
+        die(
+            f"Compiler of type {compiler.type} and version {compiler.version} "
+            "does not support --enable-path-remapping."
+        )
+
+    flags = []
+    for old, new in path_remappings:
+        # We would prefer to use just -ffile-prefix-map, but clang-cl doesn't
+        # seem to recognize it.
+        for flag in ("-fdebug-prefix-map", "-fmacro-prefix-map"):
+            flag = f"{flag}={old}={new}"
+            if compiler.type in ("gcc", "clang"):
+                flags.append(flag)
+            elif compiler.type == "clang-cl":
+                flags.extend(["-Xclang", flag])
+
+    return flags
+
+
+set_config("MOZ_FILE_PREFIX_MAP_FLAGS", file_prefix_map_flags)

build/moz.configure/rust.configure

@@ -608,8 +608,18 @@ set_config("CARGO_PROFILE_DEV_OPT_LEVEL", rustc_opt_level)
     target,
     "--enable-debug-symbols",
     "--enable-frame-pointers",
+    path_remapping,
+    path_remappings,
 )
-def rust_compile_flags(opt_level, debug_rust, target, debug_symbols, frame_pointers):
+def rust_compile_flags(
+    opt_level,
+    debug_rust,
+    target,
+    debug_symbols,
+    frame_pointers,
+    path_remapping,
+    path_remappings,
+):
     # Cargo currently supports only two interesting profiles for building:
     # development and release. Those map (roughly) to --enable-debug and
     # --disable-debug in Gecko, respectively.
@@ -649,6 +659,13 @@ def rust_compile_flags(opt_level, debug_rust, target, debug_symbols, frame_point
     for opt in opts:
         flags.extend(["-C", opt])
 
+    if "rust" in path_remapping:
+        # rustc has supported --remap-path-prefix since version 1.26, well
+        # before our required minimum Rust version, so there's no need to
+        # feature-detect or gate on versions.
+        for old, new in path_remappings:
+            flags.append(f"--remap-path-prefix={old}={new}")
+
     return flags
 
 

build/moz.configure/toolchain.configure

@@ -2693,3 +2693,78 @@ check_prog("NM", nm_names, paths=clang_search_path, when=target_is_linux)
 option("--enable-cpp-rtti", help="Enable C++ RTTI")
 
 add_old_configure_assignment("_MOZ_USE_RTTI", "1", when="--enable-cpp-rtti")
+
+
+option(
+    "--enable-path-remapping",
+    nargs="*",
+    choices=("c", "rust"),
+    help="Enable remapping source and object paths in compiled outputs.",
+)
+
+
+@depends("--enable-path-remapping")
+def path_remapping(value):
+    if len(value):
+        return value
+    if bool(value):
+        return ["c", "rust"]
+    return []
+
+
+@depends(
+    target,
+    check_build_environment,
+    sysroot_path,
+    macos_sdk,
+    windows_sdk_dir,
+    vc_path,
+    when="--enable-path-remapping",
+)
+def path_remappings(
+    target, build_env, sysroot_path, macos_sdk, windows_sdk_dir, vc_path
+):
+    win = target.kernel == "WINNT"
+
+    # The prefix maps are processed in the order they're specified on the
+    # command line.  Therefore, to accommodate object directories in the source
+    # directory, it's important that we map the topobjdir before the topsrcdir,
+    # 'cuz we might have /src/obj/=/o/ and /src/=/s/.  The various other
+    # directories might be subdirectories of topsrcdir as well, so they come
+    # earlier still.
+
+    path_remappings = []
+
+    # We will have only one sysroot or SDK, so all can have the same mnemonic: K
+    # for "kit" (since S is taken for "source").  See
+    # https://blog.llvm.org/2019/11/deterministic-builds-with-clang-and-lld.html
+    # for how to use the Windows `subst` command to map these in debuggers and
+    # IDEs.
+    if sysroot_path:
+        path_remappings.append((sysroot_path, "k:/" if win else "/sysroot/"))
+    if macos_sdk:
+        path_remappings.append((macos_sdk, "k:/" if win else "/sysroot/"))
+    if windows_sdk_dir:
+        path_remappings.append((windows_sdk_dir, "k:/" if win else "/windows_sdk/"))
+    if vc_path:
+        path_remappings.append((vc_path, "v:/" if win else "/vc/"))
+
+    path_remappings += [
+        (build_env.topobjdir, "o:/" if win else "/topobjdir/"),
+        (build_env.topsrcdir, "s:/" if win else "/topsrcdir/"),
+    ]
+
+    path_remappings = [
+        (normsep(old).rstrip("/") + "/", new) for old, new in path_remappings
+    ]
+
+    # It is tempting to sort these, but we want the order to be the same across
+    # machines so that we can share cache hits.  Therefore we reject bad
+    # configurations rather than trying to make the configuration good.
+    for i in range(len(path_remappings) - 1):
+        p = path_remappings[i][0]
+        for q, _ in path_remappings[i + 1 :]:
+            if q.startswith(p):
+                die(f"Cannot remap paths because {p} is an ancestor of {q}")
+
+    return path_remappings

python/mozbuild/mozbuild/frontend/context.py

@@ -19,10 +19,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 import operator
 import os
 
-from collections import (
-    Counter,
-    OrderedDict,
-)
+from collections import Counter, OrderedDict
 from mozbuild.util import (
     HierarchicalStringList,
     ImmutableStrictOrderingOnAppendList,
@@ -40,10 +37,7 @@ from mozbuild.util import (
 
 from .. import schedules
 
-from ..testing import (
-    read_manifestparser_manifest,
-    read_reftest_manifest,
-)
+from ..testing import read_manifestparser_manifest, read_reftest_manifest
 
 import mozpack.path as mozpath
 from types import FunctionType
@@ -670,6 +664,11 @@ class CompileFlags(TargetCompileFlags):
                 context.config.substs.get("MOZ_NEW_PASS_MANAGER_FLAGS"),
                 ("CXXFLAGS", "CFLAGS"),
             ),
+            (
+                "FILE_PREFIX_MAP",
+                context.config.substs.get("MOZ_FILE_PREFIX_MAP_FLAGS"),
+                ("CXXFLAGS", "CFLAGS"),
+            ),
         )
 
         TargetCompileFlags.__init__(self, context)
@@ -713,21 +712,13 @@ class WasmFlags(TargetCompileFlags):
                 ("WASM_CXXFLAGS", "WASM_CFLAGS"),
             ),
             ("RTL", None, ("WASM_CXXFLAGS", "WASM_CFLAGS")),
-            (
-                "DEBUG",
-                self._debug_flags(),
-                ("WASM_CFLAGS", "WASM_CXXFLAGS"),
-            ),
+            ("DEBUG", self._debug_flags(), ("WASM_CFLAGS", "WASM_CXXFLAGS")),
             (
                 "CLANG_PLUGIN",
                 context.config.substs.get("CLANG_PLUGIN_FLAGS"),
                 ("WASM_CFLAGS", "WASM_CXXFLAGS"),
             ),
-            (
-                "OPTIMIZE",
-                self._optimize_flags(),
-                ("WASM_CFLAGS", "WASM_CXXFLAGS"),
-            ),
+            ("OPTIMIZE", self._optimize_flags(), ("WASM_CFLAGS", "WASM_CXXFLAGS")),
             (
                 "FRAMEPTR",
                 context.config.substs.get("MOZ_FRAMEPTR_FLAGS"),
@@ -754,6 +745,11 @@ class WasmFlags(TargetCompileFlags):
                 context.config.substs.get("MOZ_NEW_PASS_MANAGER_FLAGS"),
                 ("WASM_CFLAGS", "WASM_CXXFLAGS"),
             ),
+            (
+                "FILE_PREFIX_MAP",
+                context.config.substs.get("MOZ_FILE_PREFIX_MAP_FLAGS"),
+                ("WASM_CFLAGS", "WASM_CXXFLAGS"),
+            ),
         )
 
         TargetCompileFlags.__init__(self, context)
@@ -1173,12 +1169,7 @@ SchedulingComponents = ContextDerivedTypedRecord(
 )
 
 GeneratedFilesList = StrictOrderingOnAppendListWithFlagsFactory(
-    {
-        "script": six.text_type,
-        "inputs": list,
-        "force": bool,
-        "flags": list,
-    }
+    {"script": six.text_type, "inputs": list, "force": bool, "flags": list}
 )
 
 
@@ -1379,9 +1370,7 @@ class Files(SubContext):
 # Arbitrary arguments can be passed to the class constructor. The first
 # argument is always the parent context. It is up to each class to perform
 # argument validation.
-SUBCONTEXTS = [
-    Files,
-]
+SUBCONTEXTS = [Files]
 
 for cls in SUBCONTEXTS:
     if not issubclass(cls, SubContext):