Bug 1755415 - Use compiler-rt artifacts for clang PGO. r=firefox-build-system-reviewers,mhentges

Instead of building compiler-rt alongside clang, we reuse the clang
repack logic to put compiler-rts in place when building clang stage2
(which requires the profiling runtime).

The repack logic is unified to handle all platforms as well, and now
that compiler-rt is not built in clang, we also need to copy everything
that is not lib/* from the compiler-rt artifacts, so we now also ensure
that they're not diverging between those compiler-rt artifacts (except
for darwin libs, which need to be unified into universal libraries).

And as a bonus, we can also cross-build the mac clang toolchains with
the stage1 clang and the clang runtime for the target platform.

Differential Revision: https://phabricator.services.mozilla.com/D138748
This commit is contained in:
Mike Hommey 2022-02-16 08:24:40 +00:00
Родитель 722d7cd010
Коммит 6366f29369
7 изменённых файлов: 59 добавлений и 88 удалений

Просмотреть файл

@ -242,7 +242,6 @@ def build_one_stage(
# libc++ doesn't build with MSVC because of the use of #include_next. # libc++ doesn't build with MSVC because of the use of #include_next.
if is_final_stage and os.path.basename(cc[0]).lower() != "cl.exe": if is_final_stage and os.path.basename(cc[0]).lower() != "cl.exe":
cmake_args += [ cmake_args += [
"-DLLVM_FORCE_BUILD_RUNTIME=ON",
"-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"), "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"),
# libc++abi has conflicting definitions between the shared and static # libc++abi has conflicting definitions between the shared and static
# library on Windows because of the import library for the dll having # library on Windows because of the import library for the dll having
@ -252,13 +251,9 @@ def build_one_stage(
] ]
if not is_final_stage: if not is_final_stage:
cmake_args += [ cmake_args += [
"-DLLVM_ENABLE_PROJECTS=clang;compiler-rt", "-DLLVM_ENABLE_PROJECTS=clang",
"-DLLVM_INCLUDE_TESTS=OFF", "-DLLVM_INCLUDE_TESTS=OFF",
"-DLLVM_TOOL_LLI_BUILD=OFF", "-DLLVM_TOOL_LLI_BUILD=OFF",
"-DCOMPILER_RT_BUILD_SANITIZERS=OFF",
"-DCOMPILER_RT_BUILD_XRAY=OFF",
"-DCOMPILER_RT_BUILD_MEMPROF=OFF",
"-DCOMPILER_RT_BUILD_LIBFUZZER=OFF",
] ]
# There is no libxml2 on Windows except if we build one ourselves. # There is no libxml2 on Windows except if we build one ourselves.
@ -512,7 +507,6 @@ def main():
extra_source_dir = source_dir + "/clang-tools-extra" extra_source_dir = source_dir + "/clang-tools-extra"
clang_source_dir = source_dir + "/clang" clang_source_dir = source_dir + "/clang"
lld_source_dir = source_dir + "/lld" lld_source_dir = source_dir + "/lld"
compiler_rt_source_dir = source_dir + "/compiler-rt"
libcxx_source_dir = source_dir + "/libcxx" libcxx_source_dir = source_dir + "/libcxx"
libcxxabi_source_dir = source_dir + "/libcxxabi" libcxxabi_source_dir = source_dir + "/libcxxabi"
@ -647,13 +641,10 @@ def main():
for p in config.get("patches", []): for p in config.get("patches", []):
patch(p, source_dir) patch(p, source_dir)
compiler_rt_source_link = llvm_source_dir + "/projects/compiler-rt"
symlinks = [ symlinks = [
(clang_source_dir, llvm_source_dir + "/tools/clang"), (clang_source_dir, llvm_source_dir + "/tools/clang"),
(extra_source_dir, llvm_source_dir + "/tools/clang/tools/extra"), (extra_source_dir, llvm_source_dir + "/tools/clang/tools/extra"),
(lld_source_dir, llvm_source_dir + "/tools/lld"), (lld_source_dir, llvm_source_dir + "/tools/lld"),
(compiler_rt_source_dir, compiler_rt_source_link),
(libcxx_source_dir, llvm_source_dir + "/projects/libcxx"), (libcxx_source_dir, llvm_source_dir + "/projects/libcxx"),
(libcxxabi_source_dir, llvm_source_dir + "/projects/libcxxabi"), (libcxxabi_source_dir, llvm_source_dir + "/projects/libcxxabi"),
(source_dir + "/cmake", llvm_source_dir + "/projects/cmake"), (source_dir + "/cmake", llvm_source_dir + "/projects/cmake"),

Просмотреть файл

@ -149,6 +149,7 @@ linux64-clang-13-profile:
- clang-13 - clang-13
toolchain: toolchain:
- linux64-clang-13-stage1 - linux64-clang-13-stage1
- linux64-x64-compiler-rt-13
- linux64-toolchain-sysroot - linux64-toolchain-sysroot
linux64-clang-13: linux64-clang-13:
@ -189,7 +190,7 @@ linux64-clang-13-android-cross:
symbol: TL(clang13-android) symbol: TL(clang13-android)
run: run:
using: toolchain-script using: toolchain-script
script: repack-clang-linux-cross.sh script: repack-clang.sh
toolchain-alias: linux64-clang-android-cross toolchain-alias: linux64-clang-android-cross
toolchain-artifact: public/build/clang.tar.zst toolchain-artifact: public/build/clang.tar.zst
fetches: fetches:
@ -208,7 +209,7 @@ linux64-clang-13-aarch64-cross:
worker: worker:
max-run-time: 600 max-run-time: 600
run: run:
script: repack-clang-linux-cross.sh script: repack-clang.sh
toolchain-alias: linux64-clang-aarch64-cross toolchain-alias: linux64-clang-aarch64-cross
toolchain-artifact: public/build/clang.tar.zst toolchain-artifact: public/build/clang.tar.zst
fetches: fetches:
@ -224,7 +225,7 @@ linux64-clang-13-macosx-cross:
worker: worker:
max-run-time: 600 max-run-time: 600
run: run:
script: repack-clang-linux-macosx-cross.sh script: repack-clang.sh
toolchain-alias: linux64-clang-macosx-cross toolchain-alias: linux64-clang-macosx-cross
toolchain-artifact: public/build/clang.tar.zst toolchain-artifact: public/build/clang.tar.zst
fetches: fetches:
@ -242,7 +243,7 @@ linux64-clang-13-win-cross:
worker: worker:
max-run-time: 600 max-run-time: 600
run: run:
script: repack-clang-linux-win-cross.sh script: repack-clang.sh
toolchain-alias: linux64-clang-win-cross toolchain-alias: linux64-clang-win-cross
toolchain-artifact: public/build/clang.tar.zst toolchain-artifact: public/build/clang.tar.zst
fetches: fetches:
@ -276,9 +277,10 @@ macosx64-clang-13-raw:
- clang-13 - clang-13
toolchain: toolchain:
- linux64-cctools-port - linux64-cctools-port
- linux64-clang-13-macosx-cross - linux64-clang-13-stage1
- linux64-clang-13-profile - linux64-clang-13-profile
- macosx64-sdk-11.0 - macosx64-sdk-11.0
- macosx64-x64-compiler-rt-13
- wasm32-wasi-compiler-rt-13 - wasm32-wasi-compiler-rt-13
macosx64-clang-13: macosx64-clang-13:
@ -293,7 +295,7 @@ macosx64-clang-13:
run-on-projects: run-on-projects:
- trunk - trunk
run: run:
script: repack-clang-linux-macosx-cross.sh script: repack-clang.sh
toolchain-alias: toolchain-alias:
- macosx64-clang - macosx64-clang
- macosx64-clang-toolchain - macosx64-clang-toolchain
@ -332,9 +334,10 @@ macosx64-aarch64-clang-13-raw:
- clang-13 - clang-13
toolchain: toolchain:
- linux64-cctools-port - linux64-cctools-port
- linux64-clang-13-macosx-cross - linux64-clang-13-stage1
- linux64-clang-13-profile - linux64-clang-13-profile
- macosx64-sdk-11.0 - macosx64-sdk-11.0
- macosx64-aarch64-compiler-rt-13
- wasm32-wasi-compiler-rt-13 - wasm32-wasi-compiler-rt-13
macosx64-aarch64-clang-13: macosx64-aarch64-clang-13:
@ -349,7 +352,7 @@ macosx64-aarch64-clang-13:
run-on-projects: run-on-projects:
- trunk - trunk
run: run:
script: repack-clang-linux-macosx-cross.sh script: repack-clang.sh
toolchain-alias: macosx64-aarch64-clang toolchain-alias: macosx64-aarch64-clang
toolchain-artifact: public/build/clang.tar.zst toolchain-artifact: public/build/clang.tar.zst
fetches: fetches:
@ -391,6 +394,7 @@ win64-clang-13-raw:
toolchain: toolchain:
- win64-clang-13-stage1 - win64-clang-13-stage1
- wasm32-wasi-compiler-rt-13 - wasm32-wasi-compiler-rt-13
- win64-compiler-rt-13
win64-clang-13: win64-clang-13:
description: "Clang-cl 13 toolchain build" description: "Clang-cl 13 toolchain build"
@ -404,7 +408,7 @@ win64-clang-13:
run-on-projects: run-on-projects:
- trunk - trunk
run: run:
script: repack-clang-linux-win-cross.sh script: repack-clang.sh
toolchain-alias: win64-clang toolchain-alias: win64-clang
toolchain-artifact: public/build/clang.tar.zst toolchain-artifact: public/build/clang.tar.zst
fetches: fetches:

Просмотреть файл

@ -16,6 +16,9 @@ if [ -d "$MOZ_FETCHES_DIR/binutils/bin" ]; then
export PATH="$MOZ_FETCHES_DIR/binutils/bin:$PATH" export PATH="$MOZ_FETCHES_DIR/binutils/bin:$PATH"
fi fi
# Make the installed compiler-rt(s) available to clang.
UPLOAD_DIR= taskcluster/scripts/misc/repack-clang.sh
case "$CONFIGS" in case "$CONFIGS" in
*macosx64*) *macosx64*)
# these variables are used in build-clang.py # these variables are used in build-clang.py

Просмотреть файл

@ -1,18 +0,0 @@
#!/bin/bash
set -x -e -v
# This script is for building clang for cross Linux targets on a Linux host.
cd $MOZ_FETCHES_DIR
# We have a native linux64 toolchain in $MOZ_FETCHES_DIR/clang
# We have some linux compiler-rts in $MOZ_FETCHES_DIR/compiler-rt*
clang_lib=$(echo $PWD/clang/lib/clang/*/lib)
find compiler-rt-*/lib/linux -type f | while read f; do
cp -n $f $clang_lib/linux
done
tar -caf clang.tar.zst clang
mkdir -p $UPLOAD_DIR
mv clang.tar.zst $UPLOAD_DIR

Просмотреть файл

@ -1,30 +0,0 @@
#!/bin/bash
set -x -e -v
# This script is for building clang for macOS targets on a Linux host,
# including native macOS Compiler-RT libraries.
cd $MOZ_FETCHES_DIR
# We have a native linux64 toolchain in $MOZ_FETCHES_DIR/clang
# We have a native aarch64 macos compiler-rt in $MOZ_FETCHES_DIR/compiler-rt-aarch64-apple-darwin
# We have a native x86_64 macos compiler-rt in $MOZ_FETCHES_DIR/compiler-rt-x86_64-apple-darwin
clang_lib=$(echo clang/lib/clang/*/lib)
mkdir -p $clang_lib/darwin
find compiler-rt-*/lib/darwin -type f -printf '%f\n' | sort -u | while read f; do
f=lib/darwin/$f
if [ -f compiler-rt-aarch64-apple-darwin/$f -a -f compiler-rt-x86_64-apple-darwin/$f ]; then
# For compiler-rt files that exist on both ends, merge them
$MOZ_FETCHES_DIR/cctools/bin/lipo -create compiler-rt-{aarch64,x86_64}-apple-darwin/$f -output $clang_lib/darwin/${f##*/}
elif [ -f compiler-rt-aarch64-apple-darwin/$f ]; then
# For compiler-rt files that exist on either end, copy the existing one
cp compiler-rt-aarch64-apple-darwin/$f $clang_lib/darwin
else
cp compiler-rt-x86_64-apple-darwin/$f $clang_lib/darwin
fi
done
tar caf clang.tar.zst clang
mkdir -p $UPLOAD_DIR
mv clang.tar.zst $UPLOAD_DIR

Просмотреть файл

@ -1,21 +0,0 @@
#!/bin/bash
set -x -e -v
# This script is for building clang for windows targets on a Linux host,
# including native Windows Compiler-RT libraries.
cd $MOZ_FETCHES_DIR
# We have a native linux64 toolchain in $MOZ_FETCHES_DIR/clang
# We have a native x86 windows compiler-rt in $MOZ_FETCHES_DIR/compiler-rt-i686-pc-windows-msvc
# We have a native x86_64 windows compiler-rt in $MOZ_FETCHES_DIR/compiler-rt-x86_64-pc-windows-msvc
clang_lib=$(echo clang/lib/clang/*/lib)
mkdir -p $clang_lib/windows
cp compiler-rt-i686-pc-windows-msvc/lib/windows/* $clang_lib/windows
cp compiler-rt-x86_64-pc-windows-msvc/lib/windows/* $clang_lib/windows
tar caf clang.tar.zst clang
# Put a tarball in the artifacts dir
mkdir -p $UPLOAD_DIR
cp clang.tar.zst $UPLOAD_DIR

Просмотреть файл

@ -0,0 +1,42 @@
#!/bin/bash
set -x -e -v
shopt -s nullglob
# This script is for repacking clang for cross targets on a Linux host.
cd $MOZ_FETCHES_DIR
# We have a clang toolchain in $MOZ_FETCHES_DIR/clang
# We have some compiler-rts in $MOZ_FETCHES_DIR/compiler-rt*
# We copy everything from the compiler-rts into clang/lib/clang/$version
clang_dir=$(echo clang/lib/clang/*/include)
clang_dir=${clang_dir%/include}
[ -n "$clang_dir" ] && for c in compiler-rt*; do
find $c -mindepth 1 -type d | while read d; do
mkdir -p "$clang_dir/${d#compiler-rt-*/}"
find $d -mindepth 1 -maxdepth 1 -not -type d | while read f; do
target_file="$clang_dir/${f#compiler-rt-*/}"
case $d in
compiler-rt-*/lib/darwin)
if [ -f "$target_file" ]; then
# Unify overlapping files for darwin/
$MOZ_FETCHES_DIR/cctools/bin/lipo -create "$f" "$target_file" -output "$target_file.new"
mv "$target_file.new" "$target_file"
continue
fi
;;
esac
if [ -f "$target_file" ] && ! diff -q "$f" "$target_file" 2>/dev/null; then
echo "Cannot copy $f because it is already in ${target_file%/*}" >&2 && exit 1
fi
cp "$f" "$target_file"
done
done
done
if [ -n "$UPLOAD_DIR" ]; then
tar caf clang.tar.zst clang
mkdir -p $UPLOAD_DIR
mv clang.tar.zst $UPLOAD_DIR
fi