Bug 1718341 - Run instrumented build on x86_64. r=aklotz,glandium

The ARM emulator images have very poor support and haven't been updated for a
long time.

Normally x86_64 images need KVM acceleration which is not available on build
machines (see Bug 1545497). We can work around this by starting the emulator
with the command line |--no-accel|.

Differential Revision: https://phabricator.services.mozilla.com/D119223

mobile/android/config/mozconfigs/android-x86/profile-generate

@@ -1,4 +1,4 @@
-. "$topsrcdir/mobile/android/config/mozconfigs/android-arm/nightly"
+. "$topsrcdir/mobile/android/config/mozconfigs/android-x86/nightly"
 
 mk_add_options "export MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0"
 

mobile/android/config/mozconfigs/android-x86_64/profile-generate

@@ -0,0 +1,6 @@
+. "$topsrcdir/mobile/android/config/mozconfigs/android-x86_64/nightly"
+
+mk_add_options "export MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0"
+
+ac_add_options --enable-profile-generate=cross
+ac_add_options --disable-tests

taskcluster/ci/build/android.yml

@@ -117,6 +117,7 @@ android-x86-fuzzing/debug:
 
 android-x86-shippable/opt:
     description: "Android 4.2 x86 Shippable"
+    use-pgo: true
     attributes:
         enable-full-crashsymbols: true
         shippable: true
@@ -171,7 +172,7 @@ android-arm/opt:
 
 android-arm-shippable/opt:
     description: "Android 4.0 Arm Shippable"
-    use-pgo: true
+    use-pgo: android-x86-shippable/opt
     attributes:
         enable-full-crashsymbols: true
         shippable: true
@@ -198,6 +199,7 @@ android-arm-shippable/opt:
     fetches:
         toolchain:
             - linux64-clang
+            - android-gradle-dependencies
 
 android-aarch64/opt:
     description: "Android 5.0 AArch64 Opt"
@@ -250,7 +252,7 @@ android-aarch64/debug:
 
 android-aarch64-shippable/opt:
     description: "Android 5.0 AArch64 Shippable"
-    use-pgo: android-arm-shippable/opt
+    use-pgo: android-x86_64-shippable/opt
     attributes:
         enable-full-crashsymbols: true
         shippable: true
@@ -277,6 +279,7 @@ android-aarch64-shippable/opt:
     fetches:
         toolchain:
             - linux64-clang
+            - android-gradle-dependencies
 
 android-x86_64/opt:
     description: "Android 5.0 x86-64 Opt"
@@ -305,6 +308,7 @@ android-x86_64/opt:
 
 android-x86_64-shippable/opt:
     description: "Android 5.0 x86-64 Shippable"
+    use-pgo: true
     attributes:
         enable-full-crashsymbols: true
         shippable: true

taskcluster/ci/generate-profile/kind.yml

@@ -85,15 +85,15 @@ jobs:
                 - linux64-clang
                 - linux64-minidump-stackwalk
 
-    android-arm-shippable/opt:
-        description: "Android 4.0 Arm Profile Generation"
+    android-x86-shippable/opt:
+        description: "Android 4.0 x86 Profile Generation"
         shipping-phase: build
         shipping-product: fennec
         index:
             product: mobile
-            job-name: android-arm-profile
+            job-name: android-x86-profile
         treeherder:
-            platform: android-4-0-armv7/pgo
+            platform: android-4-0-x86/pgo
         worker-type: t-linux-xlarge-pgo
         worker:
             chain-of-trust: true
@@ -115,12 +115,51 @@ jobs:
             options: [installer-path=/builds/worker/fetches/geckoview-androidTest.apk]
             config:
                 - android/android_common.py
-                - android/androidarm.py
+                - android/android-x86_64-profile-generation.py
                 - android/android_pgo.py
         fetches:
             toolchain:
                 - linux64-clang
                 - linux64-minidump-stackwalk
+                - android-sdk-linux
+
+    android-x86_64-shippable/opt:
+        description: "Android 5.0 x86_64 Profile Generation"
+        shipping-phase: build
+        shipping-product: fennec
+        index:
+            product: mobile
+            job-name: android-x86_64-profile
+        treeherder:
+            platform: android-5-0-x86_64/pgo
+        worker-type: t-linux-xlarge-pgo
+        worker:
+            chain-of-trust: true
+            max-run-time: 2700
+            docker-image: {in-tree: ubuntu1804-test}
+            env:
+                WORKING_DIR: "/builds/worker"
+                MOZHARNESS_PATH: "/builds/worker/checkouts/gecko/testing/mozharness"
+            artifacts:
+                - type: directory
+                  name: public/build
+                  path: /builds/worker/artifacts/
+        run:
+            using: mozharness
+            need-xvfb: true
+            job-script: taskcluster/scripts/tester/test-linux.sh
+            script: android_emulator_pgo.py
+            tooltool-downloads: internal
+            options: [installer-path=/builds/worker/fetches/geckoview-androidTest.apk]
+            config:
+                - android/android_common.py
+                - android/android-x86_64-profile-generation.py
+                - android/android_pgo.py
+        fetches:
+            toolchain:
+                - linux64-clang
+                - linux64-minidump-stackwalk
+                - android-sdk-linux
 
     win32-shippable/opt:
         description: "Win32 Profile Generation"

taskcluster/ci/instrumented-build/kind.yml

@@ -137,14 +137,14 @@ jobs:
                 - sysroot-x86_64-linux-gnu
                 - wasi-sysroot
 
-    android-arm-shippable/opt:
-        description: "Android 4.0 Arm PGO instrumented"
+    android-x86-shippable/opt:
+        description: "Android 4.0 x86 PGO instrumented"
         index:
             product: mobile
-            job-name: instrumented-android-arm
+            job-name: instrumented-android-x86
         shipping-product: fennec
         treeherder:
-            platform: android-4-0-armv7/pgo
+            platform: android-4-0-x86/pgo
         worker-type: b-linux
         worker:
             docker-image: {in-tree: android-build}
@@ -164,7 +164,48 @@ jobs:
         run:
             config:
                 - builds/releng_base_android_64_builds.py
-            custom-build-variant-cfg: arm-profile-generate
+            custom-build-variant-cfg: x86-profile-generate
+            tooltool-downloads: internal
+        fetches:
+            toolchain:
+                - android-gradle-dependencies
+                - android-ndk-linux
+                - android-sdk-linux
+                - linux64-clang-android-cross
+                - linux64-rust-android
+                - linux64-rust-size
+                - linux64-cbindgen
+                - linux64-nasm
+                - linux64-node
+
+    android-x86_64-shippable/opt:
+        description: "Android 5.0 x86_64 PGO instrumented"
+        index:
+            product: mobile
+            job-name: instrumented-android-x86_64
+        shipping-product: fennec
+        treeherder:
+            platform: android-5-0-x86_64/pgo
+        worker-type: b-linux
+        worker:
+            docker-image: {in-tree: android-build}
+            env:
+                GRADLE_USER_HOME: "/builds/worker/checkouts/gecko/mobile/android/gradle/dotgradle-offline"
+                TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android/releng.manifest"
+            artifacts:
+                - name: public/build
+                  path: /builds/worker/artifacts/
+                  type: directory
+                - name: public/build/geckoview-androidTest.apk
+                  path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview/outputs/apk/androidTest/withGeckoBinaries/debug/geckoview-withGeckoBinaries-debug-androidTest.apk
+                  type: file
+                - name: public/build/geckoview_example.apk
+                  path: /builds/worker/workspace/obj-build/gradle/build/mobile/android/geckoview_example/outputs/apk/withGeckoBinaries/debug/geckoview_example-withGeckoBinaries-debug.apk
+                  type: file
+        run:
+            config:
+                - builds/releng_base_android_64_builds.py
+            custom-build-variant-cfg: x86_64-profile-generate
             tooltool-downloads: internal
         fetches:
             toolchain:

testing/mozharness/configs/android/android-x86_64-profile-generation.py

@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# mozharness configuration for Android x86/x86_64 7.0 unit tests
+#
+# This configuration should be combined with suite definitions and other
+# mozharness configuration from android_common.py, or similar.
+
+config = {
+    "emulator_avd_name": "mozemulator-x86_64",
+    "emulator_process_name": "qemu-system-x86_64",
+    "emulator_extra_args": [
+        "-gpu",
+        "on",
+        "-skip-adb-auth",
+        "-verbose",
+        "-show-kernel",
+        "-ranchu",
+        "-selinux",
+        "permissive",
+        "-memory",
+        "3072",
+        "-cores",
+        "4",
+        "-skin",
+        "800x1280",
+        # Build machines cannot use KVM (because it requires a privileged docker
+        # container) so we run the profile generation with -no-accel which disables
+        # hardware acceleration.
+        "-no-accel",
+        "-no-snapstorage",
+        "-no-snapshot",
+        # Disables first-run dialogs
+        "-prop",
+        "ro.test_harness=true",
+    ],
+    "exes": {
+        "adb": "%(abs_sdk_dir)s/platform-tools/adb",
+    },
+    "env": {
+        "DISPLAY": ":0.0",
+        "PATH": "%(PATH)s:%(abs_sdk_dir)s/emulator:%(abs_sdk_dir)s/tools:%(abs_sdk_dir)s/tools/bin:%(abs_sdk_dir)s/platform-tools",
+        # "LIBGL_DEBUG": "verbose"
+    },
+    "bogomips_minimum": 3000,
+    "android_version": 24,
+    "is_fennec": False,
+    "is_emulator": True,
+}

testing/mozharness/configs/builds/releng_sub_android_configs/64_x86_64_profile_generate.py

@@ -0,0 +1,8 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+config = {
+    "stage_platform": "android-x86_64",
+    "src_mozconfig": "mobile/android/config/mozconfigs/android-x86_64/profile-generate",
+}

testing/mozharness/configs/builds/releng_sub_android_configs/64_x86_profile_generate.py

@@ -3,6 +3,6 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 config = {
-    "stage_platform": "android-arm",
-    "src_mozconfig": "mobile/android/config/mozconfigs/android-arm/profile-generate",
+    "stage_platform": "android-x86",
+    "src_mozconfig": "mobile/android/config/mozconfigs/android-x86/profile-generate",
 }

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -301,7 +301,6 @@ class BuildOptionParser(object):
         "arm-debug-ccov": path_base + "%s_arm_debug_ccov.py",
         "arm-debug-searchfox": path_base + "%s_arm_debug_searchfox.py",
         "arm-gradle": path_base + "%s_arm_gradle.py",
-        "arm-profile-generate": path_base + "%s_arm_profile_generate.py",
         "rusttests": path_base + "%s_rusttests.py",
         "rusttests-debug": path_base + "%s_rusttests_debug.py",
         "x86": path_base + "%s_x86.py",
@@ -309,6 +308,7 @@ class BuildOptionParser(object):
         "x86-beta-debug": path_base + "%s_x86_beta_debug.py",
         "x86-debug": path_base + "%s_x86_debug.py",
         "x86-fuzzing-debug": path_base + "%s_x86_fuzzing_debug.py",
+        "x86-profile-generate": path_base + "%s_x86_profile_generate.py",
         "x86_64": path_base + "%s_x86_64.py",
         "x86_64-beta": path_base + "%s_x86_64_beta.py",
         "x86_64-beta-debug": path_base + "%s_x86_64_beta_debug.py",
@@ -316,6 +316,7 @@ class BuildOptionParser(object):
         "x86_64-debug-isolated-process": path_base
         + "%s_x86_64_debug_isolated_process.py",
         "x86_64-fuzzing-asan": path_base + "%s_x86_64_fuzzing_asan.py",
+        "x86_64-profile-generate": path_base + "%s_x86_64_profile_generate.py",
         "arm-partner-sample1": path_base + "%s_arm_partner_sample1.py",
         "aarch64": path_base + "%s_aarch64.py",
         "aarch64-beta": path_base + "%s_aarch64_beta.py",