From 6366f29369638392b846cfd6246d4c958523e8e4 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 16 Feb 2022 08:24:40 +0000 Subject: [PATCH] Bug 1755415 - Use compiler-rt artifacts for clang PGO. r=firefox-build-system-reviewers,mhentges Instead of building compiler-rt alongside clang, we reuse the clang repack logic to put compiler-rts in place when building clang stage2 (which requires the profiling runtime). The repack logic is unified to handle all platforms as well, and now that compiler-rt is not built in clang, we also need to copy everything that is not lib/* from the compiler-rt artifacts, so we now also ensure that they're not diverging between those compiler-rt artifacts (except for darwin libs, which need to be unified into universal libraries). And as a bonus, we can also cross-build the mac clang toolchains with the stage1 clang and the clang runtime for the target platform. Differential Revision: https://phabricator.services.mozilla.com/D138748 --- build/build-clang/build-clang.py | 11 +---- taskcluster/ci/toolchain/clang.yml | 22 ++++++---- taskcluster/scripts/misc/build-clang.sh | 3 ++ .../scripts/misc/repack-clang-linux-cross.sh | 18 -------- .../misc/repack-clang-linux-macosx-cross.sh | 30 ------------- .../misc/repack-clang-linux-win-cross.sh | 21 ---------- taskcluster/scripts/misc/repack-clang.sh | 42 +++++++++++++++++++ 7 files changed, 59 insertions(+), 88 deletions(-) delete mode 100755 taskcluster/scripts/misc/repack-clang-linux-cross.sh delete mode 100755 taskcluster/scripts/misc/repack-clang-linux-macosx-cross.sh delete mode 100755 taskcluster/scripts/misc/repack-clang-linux-win-cross.sh create mode 100755 taskcluster/scripts/misc/repack-clang.sh diff --git a/build/build-clang/build-clang.py b/build/build-clang/build-clang.py index a2271d0f9e28..dd1a18ef9232 100755 --- a/build/build-clang/build-clang.py +++ b/build/build-clang/build-clang.py @@ -242,7 +242,6 @@ def build_one_stage( # libc++ doesn't build with MSVC because of the use of #include_next. if is_final_stage and os.path.basename(cc[0]).lower() != "cl.exe": cmake_args += [ - "-DLLVM_FORCE_BUILD_RUNTIME=ON", "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"), # libc++abi has conflicting definitions between the shared and static # library on Windows because of the import library for the dll having @@ -252,13 +251,9 @@ def build_one_stage( ] if not is_final_stage: cmake_args += [ - "-DLLVM_ENABLE_PROJECTS=clang;compiler-rt", + "-DLLVM_ENABLE_PROJECTS=clang", "-DLLVM_INCLUDE_TESTS=OFF", "-DLLVM_TOOL_LLI_BUILD=OFF", - "-DCOMPILER_RT_BUILD_SANITIZERS=OFF", - "-DCOMPILER_RT_BUILD_XRAY=OFF", - "-DCOMPILER_RT_BUILD_MEMPROF=OFF", - "-DCOMPILER_RT_BUILD_LIBFUZZER=OFF", ] # There is no libxml2 on Windows except if we build one ourselves. @@ -512,7 +507,6 @@ def main(): extra_source_dir = source_dir + "/clang-tools-extra" clang_source_dir = source_dir + "/clang" lld_source_dir = source_dir + "/lld" - compiler_rt_source_dir = source_dir + "/compiler-rt" libcxx_source_dir = source_dir + "/libcxx" libcxxabi_source_dir = source_dir + "/libcxxabi" @@ -647,13 +641,10 @@ def main(): for p in config.get("patches", []): patch(p, source_dir) - compiler_rt_source_link = llvm_source_dir + "/projects/compiler-rt" - symlinks = [ (clang_source_dir, llvm_source_dir + "/tools/clang"), (extra_source_dir, llvm_source_dir + "/tools/clang/tools/extra"), (lld_source_dir, llvm_source_dir + "/tools/lld"), - (compiler_rt_source_dir, compiler_rt_source_link), (libcxx_source_dir, llvm_source_dir + "/projects/libcxx"), (libcxxabi_source_dir, llvm_source_dir + "/projects/libcxxabi"), (source_dir + "/cmake", llvm_source_dir + "/projects/cmake"), diff --git a/taskcluster/ci/toolchain/clang.yml b/taskcluster/ci/toolchain/clang.yml index b4ebbb58d497..b20f2c1460ca 100644 --- a/taskcluster/ci/toolchain/clang.yml +++ b/taskcluster/ci/toolchain/clang.yml @@ -149,6 +149,7 @@ linux64-clang-13-profile: - clang-13 toolchain: - linux64-clang-13-stage1 + - linux64-x64-compiler-rt-13 - linux64-toolchain-sysroot linux64-clang-13: @@ -189,7 +190,7 @@ linux64-clang-13-android-cross: symbol: TL(clang13-android) run: using: toolchain-script - script: repack-clang-linux-cross.sh + script: repack-clang.sh toolchain-alias: linux64-clang-android-cross toolchain-artifact: public/build/clang.tar.zst fetches: @@ -208,7 +209,7 @@ linux64-clang-13-aarch64-cross: worker: max-run-time: 600 run: - script: repack-clang-linux-cross.sh + script: repack-clang.sh toolchain-alias: linux64-clang-aarch64-cross toolchain-artifact: public/build/clang.tar.zst fetches: @@ -224,7 +225,7 @@ linux64-clang-13-macosx-cross: worker: max-run-time: 600 run: - script: repack-clang-linux-macosx-cross.sh + script: repack-clang.sh toolchain-alias: linux64-clang-macosx-cross toolchain-artifact: public/build/clang.tar.zst fetches: @@ -242,7 +243,7 @@ linux64-clang-13-win-cross: worker: max-run-time: 600 run: - script: repack-clang-linux-win-cross.sh + script: repack-clang.sh toolchain-alias: linux64-clang-win-cross toolchain-artifact: public/build/clang.tar.zst fetches: @@ -276,9 +277,10 @@ macosx64-clang-13-raw: - clang-13 toolchain: - linux64-cctools-port - - linux64-clang-13-macosx-cross + - linux64-clang-13-stage1 - linux64-clang-13-profile - macosx64-sdk-11.0 + - macosx64-x64-compiler-rt-13 - wasm32-wasi-compiler-rt-13 macosx64-clang-13: @@ -293,7 +295,7 @@ macosx64-clang-13: run-on-projects: - trunk run: - script: repack-clang-linux-macosx-cross.sh + script: repack-clang.sh toolchain-alias: - macosx64-clang - macosx64-clang-toolchain @@ -332,9 +334,10 @@ macosx64-aarch64-clang-13-raw: - clang-13 toolchain: - linux64-cctools-port - - linux64-clang-13-macosx-cross + - linux64-clang-13-stage1 - linux64-clang-13-profile - macosx64-sdk-11.0 + - macosx64-aarch64-compiler-rt-13 - wasm32-wasi-compiler-rt-13 macosx64-aarch64-clang-13: @@ -349,7 +352,7 @@ macosx64-aarch64-clang-13: run-on-projects: - trunk run: - script: repack-clang-linux-macosx-cross.sh + script: repack-clang.sh toolchain-alias: macosx64-aarch64-clang toolchain-artifact: public/build/clang.tar.zst fetches: @@ -391,6 +394,7 @@ win64-clang-13-raw: toolchain: - win64-clang-13-stage1 - wasm32-wasi-compiler-rt-13 + - win64-compiler-rt-13 win64-clang-13: description: "Clang-cl 13 toolchain build" @@ -404,7 +408,7 @@ win64-clang-13: run-on-projects: - trunk run: - script: repack-clang-linux-win-cross.sh + script: repack-clang.sh toolchain-alias: win64-clang toolchain-artifact: public/build/clang.tar.zst fetches: diff --git a/taskcluster/scripts/misc/build-clang.sh b/taskcluster/scripts/misc/build-clang.sh index 2c25196d9645..d8e5ee6aa174 100755 --- a/taskcluster/scripts/misc/build-clang.sh +++ b/taskcluster/scripts/misc/build-clang.sh @@ -16,6 +16,9 @@ if [ -d "$MOZ_FETCHES_DIR/binutils/bin" ]; then export PATH="$MOZ_FETCHES_DIR/binutils/bin:$PATH" fi +# Make the installed compiler-rt(s) available to clang. +UPLOAD_DIR= taskcluster/scripts/misc/repack-clang.sh + case "$CONFIGS" in *macosx64*) # these variables are used in build-clang.py diff --git a/taskcluster/scripts/misc/repack-clang-linux-cross.sh b/taskcluster/scripts/misc/repack-clang-linux-cross.sh deleted file mode 100755 index 74b98b201262..000000000000 --- a/taskcluster/scripts/misc/repack-clang-linux-cross.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -x -e -v - -# This script is for building clang for cross Linux targets on a Linux host. - -cd $MOZ_FETCHES_DIR - -# We have a native linux64 toolchain in $MOZ_FETCHES_DIR/clang -# We have some linux compiler-rts in $MOZ_FETCHES_DIR/compiler-rt* -clang_lib=$(echo $PWD/clang/lib/clang/*/lib) -find compiler-rt-*/lib/linux -type f | while read f; do - cp -n $f $clang_lib/linux -done - -tar -caf clang.tar.zst clang - -mkdir -p $UPLOAD_DIR -mv clang.tar.zst $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/repack-clang-linux-macosx-cross.sh b/taskcluster/scripts/misc/repack-clang-linux-macosx-cross.sh deleted file mode 100755 index 345271f13029..000000000000 --- a/taskcluster/scripts/misc/repack-clang-linux-macosx-cross.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -x -e -v - -# This script is for building clang for macOS targets on a Linux host, -# including native macOS Compiler-RT libraries. - -cd $MOZ_FETCHES_DIR - -# We have a native linux64 toolchain in $MOZ_FETCHES_DIR/clang -# We have a native aarch64 macos compiler-rt in $MOZ_FETCHES_DIR/compiler-rt-aarch64-apple-darwin -# We have a native x86_64 macos compiler-rt in $MOZ_FETCHES_DIR/compiler-rt-x86_64-apple-darwin -clang_lib=$(echo clang/lib/clang/*/lib) -mkdir -p $clang_lib/darwin -find compiler-rt-*/lib/darwin -type f -printf '%f\n' | sort -u | while read f; do - f=lib/darwin/$f - if [ -f compiler-rt-aarch64-apple-darwin/$f -a -f compiler-rt-x86_64-apple-darwin/$f ]; then - # For compiler-rt files that exist on both ends, merge them - $MOZ_FETCHES_DIR/cctools/bin/lipo -create compiler-rt-{aarch64,x86_64}-apple-darwin/$f -output $clang_lib/darwin/${f##*/} - elif [ -f compiler-rt-aarch64-apple-darwin/$f ]; then - # For compiler-rt files that exist on either end, copy the existing one - cp compiler-rt-aarch64-apple-darwin/$f $clang_lib/darwin - else - cp compiler-rt-x86_64-apple-darwin/$f $clang_lib/darwin - fi -done - -tar caf clang.tar.zst clang - -mkdir -p $UPLOAD_DIR -mv clang.tar.zst $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/repack-clang-linux-win-cross.sh b/taskcluster/scripts/misc/repack-clang-linux-win-cross.sh deleted file mode 100755 index f471363bd2f7..000000000000 --- a/taskcluster/scripts/misc/repack-clang-linux-win-cross.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -x -e -v - -# This script is for building clang for windows targets on a Linux host, -# including native Windows Compiler-RT libraries. - -cd $MOZ_FETCHES_DIR - -# We have a native linux64 toolchain in $MOZ_FETCHES_DIR/clang -# We have a native x86 windows compiler-rt in $MOZ_FETCHES_DIR/compiler-rt-i686-pc-windows-msvc -# We have a native x86_64 windows compiler-rt in $MOZ_FETCHES_DIR/compiler-rt-x86_64-pc-windows-msvc -clang_lib=$(echo clang/lib/clang/*/lib) -mkdir -p $clang_lib/windows -cp compiler-rt-i686-pc-windows-msvc/lib/windows/* $clang_lib/windows -cp compiler-rt-x86_64-pc-windows-msvc/lib/windows/* $clang_lib/windows - -tar caf clang.tar.zst clang - -# Put a tarball in the artifacts dir -mkdir -p $UPLOAD_DIR -cp clang.tar.zst $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/repack-clang.sh b/taskcluster/scripts/misc/repack-clang.sh new file mode 100755 index 000000000000..6331aca5ad43 --- /dev/null +++ b/taskcluster/scripts/misc/repack-clang.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -x -e -v + +shopt -s nullglob + +# This script is for repacking clang for cross targets on a Linux host. + +cd $MOZ_FETCHES_DIR + +# We have a clang toolchain in $MOZ_FETCHES_DIR/clang +# We have some compiler-rts in $MOZ_FETCHES_DIR/compiler-rt* +# We copy everything from the compiler-rts into clang/lib/clang/$version +clang_dir=$(echo clang/lib/clang/*/include) +clang_dir=${clang_dir%/include} +[ -n "$clang_dir" ] && for c in compiler-rt*; do + find $c -mindepth 1 -type d | while read d; do + mkdir -p "$clang_dir/${d#compiler-rt-*/}" + find $d -mindepth 1 -maxdepth 1 -not -type d | while read f; do + target_file="$clang_dir/${f#compiler-rt-*/}" + case $d in + compiler-rt-*/lib/darwin) + if [ -f "$target_file" ]; then + # Unify overlapping files for darwin/ + $MOZ_FETCHES_DIR/cctools/bin/lipo -create "$f" "$target_file" -output "$target_file.new" + mv "$target_file.new" "$target_file" + continue + fi + ;; + esac + if [ -f "$target_file" ] && ! diff -q "$f" "$target_file" 2>/dev/null; then + echo "Cannot copy $f because it is already in ${target_file%/*}" >&2 && exit 1 + fi + cp "$f" "$target_file" + done + done +done + +if [ -n "$UPLOAD_DIR" ]; then + tar caf clang.tar.zst clang + mkdir -p $UPLOAD_DIR + mv clang.tar.zst $UPLOAD_DIR +fi