Bug 1528374 - Add macOS pgo builds to the taskgraph. r=firefox-build-system-reviewers,mshal,tomprince

Differential Revision: https://phabricator.services.mozilla.com/D20409

--HG--
extra : moz-landing-system : lando

browser/config/mozconfigs/macosx64/profile-generate

@@ -0,0 +1,10 @@
+. $topsrcdir/browser/config/mozconfigs/macosx64/nightly
+
+mk_add_options "export MOZ_AUTOMATION_PACKAGE_TESTS=0"
+mk_add_options "export MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0"
+mk_add_options "export MOZ_AUTOMATION_L10N_CHECK=0"
+
+unset MOZ_LTO
+
+ac_add_options --enable-profile-generate=cross
+ac_add_options --disable-tests

build/macosx/mozconfig.common

@@ -4,4 +4,14 @@ else
   . $topsrcdir/build/macosx/local-mozconfig.common
 fi
 
+if [ -z "$USE_ARTIFACT" ]; then
+    if [ -n "$MOZ_PGO_PROFILE_USE" ]; then
+        export LLVM_PROFDATA="$MOZ_FETCHES_DIR/clang/bin/llvm-profdata"
+        export MOZ_LTO=cross
+        ac_add_options --enable-profile-use=cross
+        ac_add_options --with-pgo-jarlog=/builds/worker/fetches/en-US.log
+        ac_add_options --with-pgo-profile-path=/builds/worker/fetches/merged.profdata
+    fi
+fi
+
 . $topsrcdir/build/mozconfig.stylo

taskcluster/ci/build/macosx.yml

@@ -85,6 +85,46 @@ macosx64/opt:
             - linux64-nasm
             - linux64-node
 
+macosx64/pgo:
+    description: "MacOS X x64 Cross-compile PGO"
+    use-pgo: true
+    run-on-projects: ['trunk', 'try']
+    index:
+        product: firefox
+        job-name: macosx64-pgo
+    attributes:
+        enable-full-crashsymbols: true
+    treeherder:
+        platform: osx-cross/pgo
+        symbol: Bpgo(B)
+    worker-type: b-linux
+    worker:
+        max-run-time: 7200
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
+    run:
+        using: mozharness
+        actions: [get-secrets, build]
+        config:
+            - builds/releng_base_firefox.py
+            - builds/releng_base_mac_64_cross_builds.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
+        tooltool-downloads: internal
+    fetches:
+        toolchain:
+            - linux64-binutils
+            - linux64-cctools-port
+            - linux64-clang-macosx-cross
+            - linux64-hfsplus
+            - linux64-libdmg
+            - linux64-llvm-dsymutil
+            - linux64-rust-macos
+            - linux64-rust-size
+            - linux64-cbindgen
+            - linux64-nasm
+            - linux64-node
+
 macosx64-asan-fuzzing/opt:
     description: "MacOS X x64 Cross-compile Fuzzing ASAN"
     index:

taskcluster/ci/generate-profile/kind.yml

@@ -173,3 +173,30 @@ jobs:
             toolchain:
                 - win64-clang-cl
                 - win32-minidump-stackwalk
+
+    macosx64/pgo:
+        description: "macosx64 Profile Generation"
+        shipping-phase: build
+        shipping-product: firefox
+        index:
+            product: firefox
+            job-name: macosx64-profile
+        treeherder:
+            platform: macosx64/pgo
+        worker-type: t-osx-1014-pgo
+        worker:
+            max-run-time: 1200
+            env:
+                SHELL: "/bin/bash"
+            artifacts:
+                - type: directory
+                  name: public/build
+                  path: artifacts/
+        run:
+            using: run-task
+            cwd: '{checkout}'
+            command: ./taskcluster/scripts/misc/run-profileserver-macos.sh
+        fetches:
+            toolchain:
+                - macosx64-clang
+                - macosx64-minidump-stackwalk

taskcluster/ci/instrumented-build/kind.yml

@@ -83,6 +83,41 @@ jobs:
                 - linux64-nasm
                 - linux64-node
 
+    macosx64/pgo:
+        description: "MacOS X x64 Cross-compile Instrumented"
+        shipping-phase: build
+        shipping-product: firefox
+        treeherder:
+            platform: osx-cross/pgo
+        worker-type: b-linux
+        worker:
+            max-run-time: 7200
+            env:
+                PERFHERDER_EXTRA_OPTIONS: instrumented
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
+        run:
+            using: mozharness
+            actions: [get-secrets, build]
+            config:
+                - builds/releng_base_firefox.py
+                - builds/releng_base_mac_64_cross_builds.py
+            script: "mozharness/scripts/fx_desktop_build.py"
+            secrets: true
+            mozconfig-variant: profile-generate
+            tooltool-downloads: internal
+        fetches:
+            toolchain:
+                - linux64-cctools-port
+                - linux64-clang-macosx-cross
+                - linux64-hfsplus
+                - linux64-libdmg
+                - linux64-llvm-dsymutil
+                - linux64-rust-macos
+                - linux64-rust-size
+                - linux64-cbindgen
+                - linux64-nasm
+                - linux64-node
+
     android-api-16/pgo:
         description: "Android 4.0 api-16+ PGO instrumented"
         index:

taskcluster/scripts/misc/run-profileserver-macos.sh

@@ -0,0 +1,20 @@
+#! /bin/bash -vex
+set -x -e
+
+####
+# Taskcluster friendly wrapper for running the profileserver on macOS
+####
+
+export UPLOAD_PATH=../../artifacts
+mkdir -p $UPLOAD_PATH
+
+export JARLOG_FILE="en-US.log"
+
+export LLVM_PROFDATA=$MOZ_FETCHES_DIR/clang/bin/llvm-profdata
+
+set -v
+
+./mach python python/mozbuild/mozbuild/action/install.py $MOZ_FETCHES_DIR/target.dmg $MOZ_FETCHES_DIR
+./mach python build/pgo/profileserver.py --binary $MOZ_FETCHES_DIR/*.app/Contents/MacOS/firefox
+
+tar -Jcvf $UPLOAD_PATH/profdata.tar.xz merged.profdata en-US.log