From 5f86b697afdd1a3dc5d0015267539d06c11cb740 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Fri, 12 Feb 2021 20:31:49 +0000 Subject: [PATCH] Bug 1692137 - Allow local linux builds to use a bootstrapped sysroot. r=firefox-build-system-reviewers,andi,mhentges Bug 1690930 added sysroots that can be bootstrapped. With this change, we allow --enable-bootstrap=install to pull the right sysroot for the configured target, and --enable-bootstrap to update it if it was already there. Differential Revision: https://phabricator.services.mozilla.com/D104797 --- build/autoconf/pkg.m4 | 7 ++ build/moz.configure/bindgen.configure | 6 +- build/moz.configure/pkg.configure | 65 ++++++++++--- build/moz.configure/toolchain.configure | 72 ++++++++++++-- config/makefiles/rust.mk | 6 ++ moz.configure | 95 +++++++++++++------ .../test/configure/test_checks_configure.py | 2 + 7 files changed, 202 insertions(+), 51 deletions(-) diff --git a/build/autoconf/pkg.m4 b/build/autoconf/pkg.m4 index ca8ef7e5a24b..e16d1f86df12 100644 --- a/build/autoconf/pkg.m4 +++ b/build/autoconf/pkg.m4 @@ -20,6 +20,13 @@ AC_DEFUN([PKG_CHECK_MODULES], if $PKG_CONFIG --atleast-pkgconfig-version $PKG_CONFIG_MIN_VERSION; then AC_MSG_CHECKING(for $2) + if test -n "$PKG_CONFIG_SYSROOT_DIR"; then + export PKG_CONFIG_SYSROOT_DIR + fi + if test -n "$PKG_CONFIG_LIBDIR"; then + export PKG_CONFIG_LIBDIR + fi + if $PKG_CONFIG --exists "$2" ; then AC_MSG_RESULT(yes) succeeded=yes diff --git a/build/moz.configure/bindgen.configure b/build/moz.configure/bindgen.configure index e5e49d8df820..f3e90a29655f 100644 --- a/build/moz.configure/bindgen.configure +++ b/build/moz.configure/bindgen.configure @@ -114,11 +114,11 @@ option( cxx_compiler, clang_search_path, target, - macos_sdk, + target_sysroot_flags, ) @checking("for clang for bindgen", lambda x: x.path if x else "not found") def bindgen_clang_compiler( - clang_path, c_compiler, cxx_compiler, clang_search_path, target, macos_sdk + clang_path, c_compiler, cxx_compiler, clang_search_path, target, sysroot_flags ): # When the target compiler is clang, use that, including flags. if cxx_compiler.type == "clang": @@ -155,7 +155,7 @@ def bindgen_clang_compiler( # Hack before bug 1617793: if the compiler is clang-cl, hack the target if cxx_compiler.type == "clang-cl": target = split_triplet("%s-pc-windows-msvc" % target.raw_cpu, allow_msvc=True) - flags = prepare_flags(target, macos_sdk) + flags = list(sysroot_flags or ()) info = check_compiler([clang_path] + flags, "C++", target) return namespace( path=clang_path, diff --git a/build/moz.configure/pkg.configure b/build/moz.configure/pkg.configure index 20d90f17fdd7..7ea253b80ac9 100644 --- a/build/moz.configure/pkg.configure +++ b/build/moz.configure/pkg.configure @@ -19,6 +19,41 @@ def pkg_config_version(pkg_config): return Version(check_cmd_output(pkg_config, "--version").rstrip()) +@depends(sysroot_path, multiarch_dir, when=pkg_config) +@imports(_from="os", _import="environ") +def pkg_config_vars(sysroot_path, multiarch_dir): + if sysroot_path: + return namespace( + PKG_CONFIG_SYSROOT_DIR=sysroot_path, + PKG_CONFIG_LIBDIR=":".join( + os.path.join(sysroot_path, d) + for d in ( + "usr/lib/pkgconfig", + "usr/lib/{}/pkgconfig".format(multiarch_dir), + "usr/share/pkgconfig", + ) + ), + ) + + +@depends(pkg_config_vars) +@imports(_from="os", _import="environ") +def pkg_config_env(vars): + if vars: + env = dict(environ) + env["PKG_CONFIG_SYSROOT_DIR"] = vars.PKG_CONFIG_SYSROOT_DIR + env["PKG_CONFIG_LIBDIR"] = vars.PKG_CONFIG_LIBDIR + return env + + +set_config("PKG_CONFIG_SYSROOT_DIR", pkg_config_vars.PKG_CONFIG_SYSROOT_DIR) +set_config("PKG_CONFIG_LIBDIR", pkg_config_vars.PKG_CONFIG_LIBDIR) +add_old_configure_assignment( + "PKG_CONFIG_SYSROOT_DIR", pkg_config_vars.PKG_CONFIG_SYSROOT_DIR +) +add_old_configure_assignment("PKG_CONFIG_LIBDIR", pkg_config_vars.PKG_CONFIG_LIBDIR) + + # Locates the given module using pkg-config. # - `var` determines the name of variables to set when the package is found. # _CFLAGS and _LIBS are set with corresponding values. @@ -31,8 +66,6 @@ def pkg_config_version(pkg_config): # will not result in an error or logged message, and any error message # will be returned to the caller. # Returns `True` when the package description is fulfilled. - - @template def pkg_check_modules(var, package_desc, when=always, allow_missing=False, config=True): if isinstance(package_desc, (tuple, list)): @@ -59,15 +92,25 @@ def pkg_check_modules(var, package_desc, when=always, allow_missing=False, confi min_version, ) - @depends(pkg_config, package_desc, allow_missing, when=when_and_compile_environment) + @depends( + pkg_config, + pkg_config_env, + package_desc, + allow_missing, + when=when_and_compile_environment, + ) @imports("sys") @imports(_from="mozbuild.configure.util", _import="LineIO") - def package(pkg_config, package_desc, allow_missing): + def package(pkg_config, env, package_desc, allow_missing): # package_desc may start as a depends function, so we can't use # @checking here. log.info("checking for %s... " % package_desc) retcode, stdout, stderr = get_cmd_output( - pkg_config, "--errors-to-stdout", "--print-errors", package_desc + pkg_config, + "--errors-to-stdout", + "--print-errors", + package_desc, + env=env, ) if retcode == 0: log.info("yes") @@ -79,16 +122,16 @@ def pkg_check_modules(var, package_desc, when=always, allow_missing=False, confi if not allow_missing: sys.exit(1) - @depends(pkg_config, package_desc, when=package) + @depends(pkg_config, pkg_config_env, package_desc, when=package) @checking("%s_CFLAGS" % var, callback=lambda t: " ".join(t)) - def pkg_cflags(pkg_config, package_desc): - flags = check_cmd_output(pkg_config, "--cflags", package_desc) + def pkg_cflags(pkg_config, env, package_desc): + flags = check_cmd_output(pkg_config, "--cflags", package_desc, env=env) return tuple(flags.split()) - @depends(pkg_config, package_desc, when=package) + @depends(pkg_config, pkg_config_env, package_desc, when=package) @checking("%s_LIBS" % var, callback=lambda t: " ".join(t)) - def pkg_libs(pkg_config, package_desc): - libs = check_cmd_output(pkg_config, "--libs", package_desc) + def pkg_libs(pkg_config, env, package_desc): + libs = check_cmd_output(pkg_config, "--libs", package_desc, env=env) # Remove evil flags like -Wl,--export-dynamic return tuple(libs.replace("-Wl,--export-dynamic", "").split()) diff --git a/build/moz.configure/toolchain.configure b/build/moz.configure/toolchain.configure index 11c0771e9b53..7fffeebd0872 100755 --- a/build/moz.configure/toolchain.configure +++ b/build/moz.configure/toolchain.configure @@ -953,10 +953,46 @@ def provided_program(env_var, when=None): return provided -def prepare_flags(host_or_target, macos_sdk): - if macos_sdk and host_or_target.os == "OSX": - return ["-isysroot", macos_sdk] - return [] +# We'll re-evaluate later, but for now, don't use the sysroot automatically +# even if it exists, unless --enable-bootstrap was passed, or when building +# on automation. +@depends( + bootstrap_path("sysroot", context=target, when=target_is_linux), + bootstrap, + "MOZ_AUTOMATION", +) +def sysroot_path(path, bootstrap, automation): + if bootstrap or automation: + if path: + log.info("Using sysroot in %s", path) + return path + + +@template +def sysroot_flags(host_or_target): + @depends( + host_or_target, macos_sdk, sysroot_path if host_or_target is target else never + ) + def sysroot_flags(host_or_target, macos_sdk, sysroot_path): + if macos_sdk and host_or_target.os == "OSX": + return ["-isysroot", macos_sdk] + if sysroot_path: + return ["--sysroot", sysroot_path] + return [] + + return sysroot_flags + + +host_sysroot_flags = sysroot_flags(host) +target_sysroot_flags = sysroot_flags(target) + + +@depends(target, when=sysroot_path) +def multiarch_dir(target): + if target.cpu == "x86": + # Turn e.g. i686-linux-gnu into i386-linux-gnu + return target.toolchain.replace(target.raw_cpu, "i386") + return target.toolchain def minimum_gcc_version(): @@ -994,6 +1030,11 @@ def compiler( target: "target", }[host_or_target] + sysroot_flags = { + host: host_sysroot_flags, + target: target_sysroot_flags, + }[host_or_target] + var = { ("C", target): "CC", ("C++", target): "CXX", @@ -1028,14 +1069,16 @@ def compiler( paths=clang_search_path, ) - @depends(compiler, provided_compiler, compiler_wrapper, host_or_target, macos_sdk) + @depends( + compiler, provided_compiler, compiler_wrapper, host_or_target, sysroot_flags + ) @checking("whether %s can be used" % what, lambda x: bool(x)) @imports(_from="mozbuild.shellutil", _import="quote") def valid_compiler( - compiler, provided_compiler, compiler_wrapper, host_or_target, macos_sdk + compiler, provided_compiler, compiler_wrapper, host_or_target, sysroot_flags ): wrapper = list(compiler_wrapper or ()) - flags = prepare_flags(host_or_target, macos_sdk) + flags = list(sysroot_flags or ()) if provided_compiler: provided_wrapper = list(provided_compiler.wrapper) # When doing a subconfigure, the compiler is set by old-configure @@ -2375,8 +2418,8 @@ def select_linker( set_config("LINKER_KIND", select_linker.KIND) -@depends_if(select_linker, macos_sdk) -def linker_ldflags(linker, macos_sdk): +@depends_if(select_linker, macos_sdk, sysroot_path, multiarch_dir) +def linker_ldflags(linker, macos_sdk, sysroot_path, multiarch_dir): flags = list((linker and linker.LINKER_FLAG) or []) if macos_sdk: if linker and linker.KIND == "ld64": @@ -2384,6 +2427,17 @@ def linker_ldflags(linker, macos_sdk): else: flags.append("-Wl,--sysroot=%s" % macos_sdk) + if sysroot_path and multiarch_dir: + # Non-Debian-patched binutils linkers (both BFD and gold) don't lookup + # in multi-arch directories. + flags.append( + "-Wl,-rpath-link,%s" % os.path.join(sysroot_path, "lib", multiarch_dir) + ) + flags.append( + "-Wl,-rpath-link,%s" + % os.path.join(sysroot_path, "usr", "lib", multiarch_dir) + ) + return flags diff --git a/config/makefiles/rust.mk b/config/makefiles/rust.mk index 3c7934b78650..b77e0c49ac54 100644 --- a/config/makefiles/rust.mk +++ b/config/makefiles/rust.mk @@ -191,6 +191,12 @@ export LIBCLANG_PATH=$(MOZ_LIBCLANG_PATH) export CLANG_PATH=$(MOZ_CLANG_PATH) export PKG_CONFIG export PKG_CONFIG_ALLOW_CROSS=1 +ifneq (,$(PKG_CONFIG_SYSROOT_DIR)) +export PKG_CONFIG_SYSROOT_DIR +endif +ifneq (,$(PKG_CONFIG_LIBDIR)) +export PKG_CONFIG_LIBDIR +endif export RUST_BACKTRACE=full export MOZ_TOPOBJDIR=$(topobjdir) export PYTHON3 diff --git a/moz.configure b/moz.configure index a7c7892daa08..9e5c2a4dd453 100755 --- a/moz.configure +++ b/moz.configure @@ -264,33 +264,49 @@ def bootstrap(value): return True -@depends(host, when=bootstrap) -@imports("os") -@imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions") -@imports(_from="__builtin__", _import="Exception") -def bootstrap_toolchain_tasks(host): - prefix = { - ("x86_64", "GNU", "Linux"): "linux64", - ("x86_64", "OSX", "Darwin"): "macosx64", - ("x86_64", "WINNT", "WINNT"): "win64", - }.get((host.cpu, host.os, host.kernel)) - if prefix: - try: - return namespace(prefix=prefix, tasks=toolchain_task_definitions()) - except Exception: - return None +@template +def bootstrap_toolchain_tasks(host_or_target): + @depends(host_or_target, when=bootstrap) + @imports("os") + @imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions") + @imports(_from="__builtin__", _import="Exception") + def bootstrap_toolchain_tasks(host_or_target): + prefix = { + ("x86", "GNU", "Linux"): "linux32", + ("x86_64", "GNU", "Linux"): "linux64", + ("aarch64", "GNU", "Linux"): "linux64-aarch64", + ("x86_64", "OSX", "Darwin"): "macosx64", + ("x86_64", "WINNT", "WINNT"): "win64", + }.get((host_or_target.cpu, host_or_target.os, host_or_target.kernel)) + if prefix: + try: + return namespace(prefix=prefix, tasks=toolchain_task_definitions()) + except Exception: + return None + + return bootstrap_toolchain_tasks + + +host_bootstrap_toolchain_tasks = bootstrap_toolchain_tasks(host) +target_bootstrap_toolchain_tasks = bootstrap_toolchain_tasks(target) @template -def bootstrap_search_path(*path_parts, **kwargs): +def bootstrap_path(*path_parts, **kwargs): when = kwargs.pop("when", None) + context = kwargs.pop("context", host) if kwargs: - configure_error("bootstrap_search_path only takes `when` as keyword argument") + configure_error( + "bootstrap_path only takes `when` or `context` as keyword arguments" + ) + + bootstrap_toolchain_tasks = { + host: host_bootstrap_toolchain_tasks, + target: target_bootstrap_toolchain_tasks, + }[context] @depends( bootstrap, - bootstrap_search_path_order, - original_path, toolchains_base_dir, bootstrap_toolchain_tasks, shell, @@ -302,25 +318,25 @@ def bootstrap_search_path(*path_parts, **kwargs): @imports(_from="mozbuild.util", _import="ensureParentDir") @imports(_from="__builtin__", _import="open") @imports(_from="__builtin__", _import="Exception") - def bootstrap_search_path( - bootstrap, order, original_path, toolchains_base_dir, tasks, shell, build_env - ): + def bootstrap_path(bootstrap, toolchains_base_dir, tasks, shell, build_env): def try_bootstrap(exists): + if not tasks: + return False label = "toolchain-{}-{}".format( tasks.prefix, path_parts[0].replace("_", "-") ) task = tasks.tasks.get(label) if not task: - return + return False task_index = task.optimization.get("index-search") if not task_index: - return + return False task_index = task_index[0].split(".")[-1] artifact = task.attributes["toolchain-artifact"] # `mach artifact toolchain` doesn't support authentication for # private artifacts. if not artifact.startswith("public/"): - return + return False index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0]) try: with open(index_file) as fh: @@ -328,7 +344,7 @@ def bootstrap_search_path(*path_parts, **kwargs): except Exception: index = None if index == task_index and exists: - return + return True log.info( "%s bootstrapped toolchain in %s", "Updating" if exists else "Installing", @@ -350,15 +366,38 @@ def bootstrap_search_path(*path_parts, **kwargs): ensureParentDir(index_file) with open(index_file, "w") as fh: fh.write(task_index) + return True path = os.path.join(toolchains_base_dir, *path_parts) exists = os.path.exists(path) - if bootstrap and tasks and (exists or bootstrap == "install"): + if bootstrap and (exists or bootstrap == "install"): try: - try_bootstrap(exists) + if not try_bootstrap(exists): + # If there aren't toolchain artifacts to use for this build, + # don't return a path. + return None except Exception as e: log.error("%s", e) die("If you can't fix the above, retry with --disable-bootstrap.") + # We re-test whether the path exists because it may have been created by + # try_bootstrap. Automation will not have gone through the bootstrap + # process, but we want to return the path if it exists. + if os.path.exists(path): + return path + + return bootstrap_path + + +@template +def bootstrap_search_path(*path_parts, **kwargs): + @depends( + bootstrap_path(*path_parts, **kwargs), + bootstrap_search_path_order, + original_path, + ) + def bootstrap_search_path(path, order, original_path): + if not path: + return original_path if order == "prepend": return [path] + original_path return original_path + [path] diff --git a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py index ac8c9287c803..3d5a9ef62553 100644 --- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py @@ -792,6 +792,8 @@ and/or set $JAVA_HOME. option('--disable-compile-environment', help='compile env') compile_environment = depends(when='--enable-compile-environment')(lambda: True) toolchain_prefix = depends(when=True)(lambda: None) + multiarch_dir = depends(when=True)(lambda: None) + sysroot_path = depends(when=True)(lambda: None) include('%(topsrcdir)s/build/moz.configure/util.configure') include('%(topsrcdir)s/build/moz.configure/checks.configure') include('%(topsrcdir)s/build/moz.configure/pkg.configure')