From b06582c791d6bf6fca0c7c0a39a9f6bdedec8058 Mon Sep 17 00:00:00 2001 From: Agi Sferro Date: Fri, 16 Jul 2021 17:49:54 +0000 Subject: [PATCH] Bug 1718341 - Run instrumented build on x86_64. r=aklotz,glandium The ARM emulator images have very poor support and haven't been updated for a long time. Normally x86_64 images need KVM acceleration which is not available on build machines (see Bug 1545497). We can work around this by starting the emulator with the command line |--no-accel|. Differential Revision: https://phabricator.services.mozilla.com/D119223 --- .../profile-generate | 2 +- .../android-x86_64/profile-generate | 6 +++ taskcluster/ci/build/android.yml | 8 ++- taskcluster/ci/generate-profile/kind.yml | 49 ++++++++++++++++-- taskcluster/ci/instrumented-build/kind.yml | 51 +++++++++++++++++-- .../android-x86_64-profile-generation.py | 50 ++++++++++++++++++ .../64_x86_64_profile_generate.py | 8 +++ ...generate.py => 64_x86_profile_generate.py} | 4 +- .../mozharness/mozilla/building/buildbase.py | 3 +- 9 files changed, 165 insertions(+), 16 deletions(-) rename mobile/android/config/mozconfigs/{android-arm => android-x86}/profile-generate (70%) create mode 100644 mobile/android/config/mozconfigs/android-x86_64/profile-generate create mode 100644 testing/mozharness/configs/android/android-x86_64-profile-generation.py create mode 100644 testing/mozharness/configs/builds/releng_sub_android_configs/64_x86_64_profile_generate.py rename testing/mozharness/configs/builds/releng_sub_android_configs/{64_arm_profile_generate.py => 64_x86_profile_generate.py} (63%) diff --git a/mobile/android/config/mozconfigs/android-arm/profile-generate b/mobile/android/config/mozconfigs/android-x86/profile-generate similarity index 70% rename from mobile/android/config/mozconfigs/android-arm/profile-generate rename to mobile/android/config/mozconfigs/android-x86/profile-generate index 418b30df5375..2ba606187d39 100644 --- a/mobile/android/config/mozconfigs/android-arm/profile-generate +++ b/mobile/android/config/mozconfigs/android-x86/profile-generate @@ -1,4 +1,4 @@ -. "$topsrcdir/mobile/android/config/mozconfigs/android-arm/nightly" +. "$topsrcdir/mobile/android/config/mozconfigs/android-x86/nightly" mk_add_options "export MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0" diff --git a/mobile/android/config/mozconfigs/android-x86_64/profile-generate b/mobile/android/config/mozconfigs/android-x86_64/profile-generate new file mode 100644 index 000000000000..0bd9464da6ec --- /dev/null +++ b/mobile/android/config/mozconfigs/android-x86_64/profile-generate @@ -0,0 +1,6 @@ +. "$topsrcdir/mobile/android/config/mozconfigs/android-x86_64/nightly" + +mk_add_options "export MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0" + +ac_add_options --enable-profile-generate=cross +ac_add_options --disable-tests diff --git a/taskcluster/ci/build/android.yml b/taskcluster/ci/build/android.yml index f3f4b47d3042..faaa32cddcb6 100644 --- a/taskcluster/ci/build/android.yml +++ b/taskcluster/ci/build/android.yml @@ -117,6 +117,7 @@ android-x86-fuzzing/debug: android-x86-shippable/opt: description: "Android 4.2 x86 Shippable" + use-pgo: true attributes: enable-full-crashsymbols: true shippable: true @@ -171,7 +172,7 @@ android-arm/opt: android-arm-shippable/opt: description: "Android 4.0 Arm Shippable" - use-pgo: true + use-pgo: android-x86-shippable/opt attributes: enable-full-crashsymbols: true shippable: true @@ -198,6 +199,7 @@ android-arm-shippable/opt: fetches: toolchain: - linux64-clang + - android-gradle-dependencies android-aarch64/opt: description: "Android 5.0 AArch64 Opt" @@ -250,7 +252,7 @@ android-aarch64/debug: android-aarch64-shippable/opt: description: "Android 5.0 AArch64 Shippable" - use-pgo: android-arm-shippable/opt + use-pgo: android-x86_64-shippable/opt attributes: enable-full-crashsymbols: true shippable: true @@ -277,6 +279,7 @@ android-aarch64-shippable/opt: fetches: toolchain: - linux64-clang + - android-gradle-dependencies android-x86_64/opt: description: "Android 5.0 x86-64 Opt" @@ -305,6 +308,7 @@ android-x86_64/opt: android-x86_64-shippable/opt: description: "Android 5.0 x86-64 Shippable" + use-pgo: true attributes: enable-full-crashsymbols: true shippable: true diff --git a/taskcluster/ci/generate-profile/kind.yml b/taskcluster/ci/generate-profile/kind.yml index cb98e77c4532..97b3d3b93d29 100644 --- a/taskcluster/ci/generate-profile/kind.yml +++ b/taskcluster/ci/generate-profile/kind.yml @@ -85,15 +85,15 @@ jobs: - linux64-clang - linux64-minidump-stackwalk - android-arm-shippable/opt: - description: "Android 4.0 Arm Profile Generation" + android-x86-shippable/opt: + description: "Android 4.0 x86 Profile Generation" shipping-phase: build shipping-product: fennec index: product: mobile - job-name: android-arm-profile + job-name: android-x86-profile treeherder: - platform: android-4-0-armv7/pgo + platform: android-4-0-x86/pgo worker-type: t-linux-xlarge-pgo worker: chain-of-trust: true @@ -115,12 +115,51 @@ jobs: options: [installer-path=/builds/worker/fetches/geckoview-androidTest.apk] config: - android/android_common.py - - android/androidarm.py + - android/android-x86_64-profile-generation.py - android/android_pgo.py fetches: toolchain: - linux64-clang - linux64-minidump-stackwalk + - android-sdk-linux + + android-x86_64-shippable/opt: + description: "Android 5.0 x86_64 Profile Generation" + shipping-phase: build + shipping-product: fennec + index: + product: mobile + job-name: android-x86_64-profile + treeherder: + platform: android-5-0-x86_64/pgo + worker-type: t-linux-xlarge-pgo + worker: + chain-of-trust: true + max-run-time: 2700 + docker-image: {in-tree: ubuntu1804-test} + env: + WORKING_DIR: "/builds/worker" + MOZHARNESS_PATH: "/builds/worker/checkouts/gecko/testing/mozharness" + artifacts: + - type: directory + name: public/build + path: /builds/worker/artifacts/ + run: + using: mozharness + need-xvfb: true + job-script: taskcluster/scripts/tester/test-linux.sh + script: android_emulator_pgo.py + tooltool-downloads: internal + options: [installer-path=/builds/worker/fetches/geckoview-androidTest.apk] + config: + - android/android_common.py + - android/android-x86_64-profile-generation.py + - android/android_pgo.py + fetches: + toolchain: + - linux64-clang + - linux64-minidump-stackwalk + - android-sdk-linux win32-shippable/opt: description: "Win32 Profile Generation" diff --git a/taskcluster/ci/instrumented-build/kind.yml b/taskcluster/ci/instrumented-build/kind.yml index 3a26275c6e14..22af5ea767cd 100644 --- a/taskcluster/ci/instrumented-build/kind.yml +++ b/taskcluster/ci/instrumented-build/kind.yml @@ -137,14 +137,14 @@ jobs: - sysroot-x86_64-linux-gnu - wasi-sysroot - android-arm-shippable/opt: - description: "Android 4.0 Arm PGO instrumented" + android-x86-shippable/opt: + description: "Android 4.0 x86 PGO instrumented" index: product: mobile - job-name: instrumented-android-arm + job-name: instrumented-android-x86 shipping-product: fennec treeherder: - platform: android-4-0-armv7/pgo + platform: android-4-0-x86/pgo worker-type: b-linux worker: docker-image: {in-tree: android-build} @@ -164,7 +164,48 @@ jobs: run: config: - builds/releng_base_android_64_builds.py - custom-build-variant-cfg: arm-profile-generate + custom-build-variant-cfg: x86-profile-generate + tooltool-downloads: internal + fetches: + toolchain: + - android-gradle-dependencies + - android-ndk-linux + - android-sdk-linux + - linux64-clang-android-cross + - linux64-rust-android + - linux64-rust-size + - linux64-cbindgen + - linux64-nasm + - linux64-node + + android-x86_64-shippable/opt: + description: "Android 5.0 x86_64 PGO instrumented" + index: + product: mobile + job-name: instrumented-android-x86_64 + shipping-product: fennec + treeherder: + platform: android-5-0-x86_64/pgo + worker-type: b-linux + worker: + docker-image: {in-tree: android-build} + env: + GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline" + TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest" + artifacts: + - name: public/build + path: /builds/worker/artifacts/ + type: directory + - name: public/build/geckoview-androidTest.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk + type: file + - name: public/build/geckoview_example.apk + path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk + type: file + run: + config: + - builds/releng_base_android_64_builds.py + custom-build-variant-cfg: x86_64-profile-generate tooltool-downloads: internal fetches: toolchain: diff --git a/testing/mozharness/configs/android/android-x86_64-profile-generation.py b/testing/mozharness/configs/android/android-x86_64-profile-generation.py new file mode 100644 index 000000000000..72a328051c4a --- /dev/null +++ b/testing/mozharness/configs/android/android-x86_64-profile-generation.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# mozharness configuration for Android x86/x86_64 7.0 unit tests +# +# This configuration should be combined with suite definitions and other +# mozharness configuration from android_common.py, or similar. + +config = { + "emulator_avd_name": "mozemulator-x86_64", + "emulator_process_name": "qemu-system-x86_64", + "emulator_extra_args": [ + "-gpu", + "on", + "-skip-adb-auth", + "-verbose", + "-show-kernel", + "-ranchu", + "-selinux", + "permissive", + "-memory", + "3072", + "-cores", + "4", + "-skin", + "800x1280", + # Build machines cannot use KVM (because it requires a privileged docker + # container) so we run the profile generation with -no-accel which disables + # hardware acceleration. + "-no-accel", + "-no-snapstorage", + "-no-snapshot", + # Disables first-run dialogs + "-prop", + "ro.test_harness=true", + ], + "exes": { + "adb": "%(abs_sdk_dir)s/platform-tools/adb", + }, + "env": { + "DISPLAY": ":0.0", + "PATH": "%(PATH)s:%(abs_sdk_dir)s/emulator:%(abs_sdk_dir)s/tools:%(abs_sdk_dir)s/tools/bin:%(abs_sdk_dir)s/platform-tools", + # "LIBGL_DEBUG": "verbose" + }, + "bogomips_minimum": 3000, + "android_version": 24, + "is_fennec": False, + "is_emulator": True, +} diff --git a/testing/mozharness/configs/builds/releng_sub_android_configs/64_x86_64_profile_generate.py b/testing/mozharness/configs/builds/releng_sub_android_configs/64_x86_64_profile_generate.py new file mode 100644 index 000000000000..935802172378 --- /dev/null +++ b/testing/mozharness/configs/builds/releng_sub_android_configs/64_x86_64_profile_generate.py @@ -0,0 +1,8 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +config = { + "stage_platform": "android-x86_64", + "src_mozconfig": "mobile/android/config/mozconfigs/android-x86_64/profile-generate", +} diff --git a/testing/mozharness/configs/builds/releng_sub_android_configs/64_arm_profile_generate.py b/testing/mozharness/configs/builds/releng_sub_android_configs/64_x86_profile_generate.py similarity index 63% rename from testing/mozharness/configs/builds/releng_sub_android_configs/64_arm_profile_generate.py rename to testing/mozharness/configs/builds/releng_sub_android_configs/64_x86_profile_generate.py index 3d99ac472af9..08ce1529c382 100644 --- a/testing/mozharness/configs/builds/releng_sub_android_configs/64_arm_profile_generate.py +++ b/testing/mozharness/configs/builds/releng_sub_android_configs/64_x86_profile_generate.py @@ -3,6 +3,6 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. config = { - "stage_platform": "android-arm", - "src_mozconfig": "mobile/android/config/mozconfigs/android-arm/profile-generate", + "stage_platform": "android-x86", + "src_mozconfig": "mobile/android/config/mozconfigs/android-x86/profile-generate", } diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index 2f21dbb23be5..f0affa9dfb10 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -301,7 +301,6 @@ class BuildOptionParser(object): "arm-debug-ccov": path_base + "%s_arm_debug_ccov.py", "arm-debug-searchfox": path_base + "%s_arm_debug_searchfox.py", "arm-gradle": path_base + "%s_arm_gradle.py", - "arm-profile-generate": path_base + "%s_arm_profile_generate.py", "rusttests": path_base + "%s_rusttests.py", "rusttests-debug": path_base + "%s_rusttests_debug.py", "x86": path_base + "%s_x86.py", @@ -309,6 +308,7 @@ class BuildOptionParser(object): "x86-beta-debug": path_base + "%s_x86_beta_debug.py", "x86-debug": path_base + "%s_x86_debug.py", "x86-fuzzing-debug": path_base + "%s_x86_fuzzing_debug.py", + "x86-profile-generate": path_base + "%s_x86_profile_generate.py", "x86_64": path_base + "%s_x86_64.py", "x86_64-beta": path_base + "%s_x86_64_beta.py", "x86_64-beta-debug": path_base + "%s_x86_64_beta_debug.py", @@ -316,6 +316,7 @@ class BuildOptionParser(object): "x86_64-debug-isolated-process": path_base + "%s_x86_64_debug_isolated_process.py", "x86_64-fuzzing-asan": path_base + "%s_x86_64_fuzzing_asan.py", + "x86_64-profile-generate": path_base + "%s_x86_64_profile_generate.py", "arm-partner-sample1": path_base + "%s_arm_partner_sample1.py", "aarch64": path_base + "%s_aarch64.py", "aarch64-beta": path_base + "%s_aarch64_beta.py",