From 3138b4cffd4a97784c51562835703fc3d26c8a7c Mon Sep 17 00:00:00 2001 From: Chris Manchester Date: Thu, 12 Dec 2019 19:31:17 +0000 Subject: [PATCH] Bug 1528374 - Add macOS pgo builds to the taskgraph. r=firefox-build-system-reviewers,mshal,tomprince Differential Revision: https://phabricator.services.mozilla.com/D20409 --HG-- extra : moz-landing-system : lando --- .../mozconfigs/macosx64/profile-generate | 10 +++++ build/macosx/mozconfig.common | 10 +++++ taskcluster/ci/build/macosx.yml | 40 +++++++++++++++++++ taskcluster/ci/generate-profile/kind.yml | 27 +++++++++++++ taskcluster/ci/instrumented-build/kind.yml | 35 ++++++++++++++++ .../scripts/misc/run-profileserver-macos.sh | 20 ++++++++++ 6 files changed, 142 insertions(+) create mode 100644 browser/config/mozconfigs/macosx64/profile-generate create mode 100755 taskcluster/scripts/misc/run-profileserver-macos.sh diff --git a/browser/config/mozconfigs/macosx64/profile-generate b/browser/config/mozconfigs/macosx64/profile-generate new file mode 100644 index 000000000000..58a72a2ff1cb --- /dev/null +++ b/browser/config/mozconfigs/macosx64/profile-generate @@ -0,0 +1,10 @@ +. $topsrcdir/browser/config/mozconfigs/macosx64/nightly + +mk_add_options "export MOZ_AUTOMATION_PACKAGE_TESTS=0" +mk_add_options "export MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0" +mk_add_options "export MOZ_AUTOMATION_L10N_CHECK=0" + +unset MOZ_LTO + +ac_add_options --enable-profile-generate=cross +ac_add_options --disable-tests diff --git a/build/macosx/mozconfig.common b/build/macosx/mozconfig.common index 487981fa1740..40f7edd4fbce 100644 --- a/build/macosx/mozconfig.common +++ b/build/macosx/mozconfig.common @@ -4,4 +4,14 @@ else . $topsrcdir/build/macosx/local-mozconfig.common fi +if [ -z "$USE_ARTIFACT" ]; then + if [ -n "$MOZ_PGO_PROFILE_USE" ]; then + export LLVM_PROFDATA="$MOZ_FETCHES_DIR/clang/bin/llvm-profdata" + export MOZ_LTO=cross + ac_add_options --enable-profile-use=cross + ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log + ac_add_options --with-pgo-profile-path=/builds/worker/fetches/merged.profdata + fi +fi + . $topsrcdir/build/mozconfig.stylo diff --git a/taskcluster/ci/build/macosx.yml b/taskcluster/ci/build/macosx.yml index d4257a13646e..27e10f026369 100644 --- a/taskcluster/ci/build/macosx.yml +++ b/taskcluster/ci/build/macosx.yml @@ -85,6 +85,46 @@ macosx64/opt: - linux64-nasm - linux64-node +macosx64/pgo: + description: "MacOS X x64 Cross-compile PGO" + use-pgo: true + run-on-projects: ['trunk', 'try'] + index: + product: firefox + job-name: macosx64-pgo + attributes: + enable-full-crashsymbols: true + treeherder: + platform: osx-cross/pgo + symbol: Bpgo(B) + worker-type: b-linux + worker: + max-run-time: 7200 + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: internal + fetches: + toolchain: + - linux64-binutils + - linux64-cctools-port + - linux64-clang-macosx-cross + - linux64-hfsplus + - linux64-libdmg + - linux64-llvm-dsymutil + - linux64-rust-macos + - linux64-rust-size + - linux64-cbindgen + - linux64-nasm + - linux64-node + macosx64-asan-fuzzing/opt: description: "MacOS X x64 Cross-compile Fuzzing ASAN" index: diff --git a/taskcluster/ci/generate-profile/kind.yml b/taskcluster/ci/generate-profile/kind.yml index 6d692173851c..03aea849f8f5 100644 --- a/taskcluster/ci/generate-profile/kind.yml +++ b/taskcluster/ci/generate-profile/kind.yml @@ -173,3 +173,30 @@ jobs: toolchain: - win64-clang-cl - win32-minidump-stackwalk + + macosx64/pgo: + description: "macosx64 Profile Generation" + shipping-phase: build + shipping-product: firefox + index: + product: firefox + job-name: macosx64-profile + treeherder: + platform: macosx64/pgo + worker-type: t-osx-1014-pgo + worker: + max-run-time: 1200 + env: + SHELL: "/bin/bash" + artifacts: + - type: directory + name: public/build + path: artifacts/ + run: + using: run-task + cwd: '{checkout}' + command: ./taskcluster/scripts/misc/run-profileserver-macos.sh + fetches: + toolchain: + - macosx64-clang + - macosx64-minidump-stackwalk diff --git a/taskcluster/ci/instrumented-build/kind.yml b/taskcluster/ci/instrumented-build/kind.yml index 497d5eb21409..860b7c3e11df 100644 --- a/taskcluster/ci/instrumented-build/kind.yml +++ b/taskcluster/ci/instrumented-build/kind.yml @@ -83,6 +83,41 @@ jobs: - linux64-nasm - linux64-node + macosx64/pgo: + description: "MacOS X x64 Cross-compile Instrumented" + shipping-phase: build + shipping-product: firefox + treeherder: + platform: osx-cross/pgo + worker-type: b-linux + worker: + max-run-time: 7200 + env: + PERFHERDER_EXTRA_OPTIONS: instrumented + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest" + run: + using: mozharness + actions: [get-secrets, build] + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + mozconfig-variant: profile-generate + tooltool-downloads: internal + fetches: + toolchain: + - linux64-cctools-port + - linux64-clang-macosx-cross + - linux64-hfsplus + - linux64-libdmg + - linux64-llvm-dsymutil + - linux64-rust-macos + - linux64-rust-size + - linux64-cbindgen + - linux64-nasm + - linux64-node + android-api-16/pgo: description: "Android 4.0 api-16+ PGO instrumented" index: diff --git a/taskcluster/scripts/misc/run-profileserver-macos.sh b/taskcluster/scripts/misc/run-profileserver-macos.sh new file mode 100755 index 000000000000..61873a273a44 --- /dev/null +++ b/taskcluster/scripts/misc/run-profileserver-macos.sh @@ -0,0 +1,20 @@ +#! /bin/bash -vex +set -x -e + +#### +# Taskcluster friendly wrapper for running the profileserver on macOS +#### + +export UPLOAD_PATH=../../artifacts +mkdir -p $UPLOAD_PATH + +export JARLOG_FILE="en-US.log" + +export LLVM_PROFDATA=$MOZ_FETCHES_DIR/clang/bin/llvm-profdata + +set -v + +./mach python python/mozbuild/mozbuild/action/install.py $MOZ_FETCHES_DIR/target.dmg $MOZ_FETCHES_DIR +./mach python build/pgo/profileserver.py --binary $MOZ_FETCHES_DIR/*.app/Contents/MacOS/firefox + +tar -Jcvf $UPLOAD_PATH/profdata.tar.xz merged.profdata en-US.log