diff --git a/layout/style/test/mochitest.ini b/layout/style/test/mochitest.ini index 458963b792cb..04a2c6e6387c 100644 --- a/layout/style/test/mochitest.ini +++ b/layout/style/test/mochitest.ini @@ -248,7 +248,7 @@ support-files = file_font_loading_api_vframe.html # This test checks font loading state. When loaded second time, fonts may be # loaded synchronously, causing this test to fail in test-verify task. -skip-if = verify +skip-if = verify || toolkit == 'android' # Bug 1455824 [test_garbage_at_end_of_declarations.html] [test_grid_container_shorthands.html] [test_grid_item_shorthands.html] diff --git a/python/mozboot/mozboot/android-avds/x86_64.json b/python/mozboot/mozboot/android-avds/x86_64.json new file mode 100644 index 000000000000..200486385e32 --- /dev/null +++ b/python/mozboot/mozboot/android-avds/x86_64.json @@ -0,0 +1,9 @@ +{ + "emulator_package": "system-images;android-24;default;x86_64", + "emulator_avd_name": "mozemulator-x86_64", + "emulator_extra_config": { + "hw.lcd.density": "320", + "disk.dataPartition.size": "4000MB", + "sdcard.size": "600M" + } +} diff --git a/python/mozboot/mozboot/android-system-images-packages.txt b/python/mozboot/mozboot/android-system-images-packages.txt new file mode 100644 index 000000000000..86069f7680ef --- /dev/null +++ b/python/mozboot/mozboot/android-system-images-packages.txt @@ -0,0 +1 @@ +emulator diff --git a/python/mozboot/mozboot/android.py b/python/mozboot/mozboot/android.py index 29b834527d3a..578cd4870767 100644 --- a/python/mozboot/mozboot/android.py +++ b/python/mozboot/mozboot/android.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, print_function, unicode_literals import errno +import json import os import stat import subprocess @@ -184,7 +185,13 @@ def get_paths(os_name): "ANDROID_NDK_HOME", os.path.join(mozbuild_path, "android-ndk-{0}".format(NDK_VERSION)), ) - return (mozbuild_path, sdk_path, ndk_path) + avd_path = os.environ.get( + "ANDROID_AVD_HOME", os.path.join(mozbuild_path, "android-device", "avd") + ) + emulator_path = os.environ.get( + "ANDROID_EMULATOR_HOME", os.path.join(mozbuild_path, "android-device") + ) + return (mozbuild_path, sdk_path, ndk_path, avd_path, emulator_path) def sdkmanager_tool(sdk_path): @@ -193,6 +200,12 @@ def sdkmanager_tool(sdk_path): return os.path.join(sdk_path, "tools", "bin", sdkmanager) +def avdmanager_tool(sdk_path): + # sys.platform is win32 even if Python/Win64. + sdkmanager = "avdmanager.bat" if sys.platform.startswith("win") else "avdmanager" + return os.path.join(sdk_path, "tools", "bin", sdkmanager) + + def ensure_dir(dir): """Ensures the given directory exists""" if dir and not os.path.exists(dir): @@ -207,7 +220,9 @@ def ensure_android( os_name, artifact_mode=False, ndk_only=False, + system_images_only=False, emulator_only=False, + avd_manifest_path=None, no_interactive=False, ): """ @@ -222,12 +237,10 @@ def ensure_android( # save them a lengthy download), or they may have already # completed the download. We unpack to # ~/.mozbuild/{android-sdk-$OS_NAME, android-ndk-$VER}. - mozbuild_path, sdk_path, ndk_path = get_paths(os_name) + mozbuild_path, sdk_path, ndk_path, avd_path, emulator_path = get_paths(os_name) os_tag = "darwin" if os_name == "macosx" else os_name - sdk_url = ( - "https://dl.google.com/android/repository/sdk-tools-{0}-4333796.zip".format( - os_tag - ) + sdk_url = "https://dl.google.com/android/repository/sdk-tools-{0}-4333796.zip".format( + os_tag ) ndk_url = android_ndk_url(os_name) @@ -246,14 +259,32 @@ def ensure_android( if ndk_only: return + avd_manifest = None + if avd_manifest_path is not None: + with open(avd_manifest_path) as f: + avd_manifest = json.load(f) + # We expect the |sdkmanager| tool to be at # ~/.mozbuild/android-sdk-$OS_NAME/tools/bin/sdkmanager. ensure_android_packages( sdkmanager_tool=sdkmanager_tool(sdk_path), emulator_only=emulator_only, + system_images_only=system_images_only, + avd_manifest=avd_manifest, no_interactive=no_interactive, ) + if emulator_only or system_images_only: + return + + ensure_android_avd( + avdmanager_tool=avdmanager_tool(sdk_path), + avd_path=avd_path, + sdk_path=sdk_path, + no_interactive=no_interactive, + avd_manifest=avd_manifest, + ) + def ensure_android_sdk_and_ndk( mozbuild_path, @@ -311,27 +342,79 @@ def ensure_android_sdk_and_ndk( ) -def get_packages_to_install(packages_file_name): +def get_packages_to_install(packages_file_content, avd_manifest): + packages = [] + packages += map(lambda package: package.strip(), packages_file_content) + if avd_manifest is not None: + packages += [avd_manifest["emulator_package"]] + return packages + + +def ensure_android_avd( + avdmanager_tool, avd_path, sdk_path, no_interactive=False, avd_manifest=None +): """ - sdkmanager version 26.1.1 (current) and some versions below have a bug that makes - the following command fail: - args = [sdkmanager_tool, '--package_file={0}'.format(package_file_name)] + Use the given sdkmanager tool (like 'sdkmanager') to install required + Android packages. + """ + if avd_manifest is None: + return + + ensure_dir(avd_path) + # The AVD needs this folder to boot, so make sure it exists here. + ensure_dir(os.path.join(sdk_path, "platforms")) + + avd_name = avd_manifest["emulator_avd_name"] + args = [ + avdmanager_tool, + "--verbose", + "create", + "avd", + "--force", + "--name", + avd_name, + "--package", + avd_manifest["emulator_package"], + ] + + if not no_interactive: subprocess.check_call(args) - The error is in the sdkmanager, where the --package_file param isn't recognized. - The error is being tracked here https://issuetracker.google.com/issues/66465833 - Meanwhile, this workaround achives installing all required Android packages by reading - them out of the same file that --package_file would have used, and passing them as strings. - So from here: https://developer.android.com/studio/command-line/sdkmanager - Instead of: - sdkmanager --package_file=package_file [options] - We're doing: - sdkmanager "platform-tools" "platforms;android-26" - """ - with open(packages_file_name) as package_file: - return map(lambda package: package.strip(), package_file.readlines()) + return + + # Flush outputs before running sdkmanager. + sys.stdout.flush() + env = os.environ.copy() + env["ANDROID_AVD_HOME"] = avd_path + proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env) + proc.communicate("no\n".encode("UTF-8")) + + retcode = proc.poll() + if retcode: + cmd = args[0] + e = subprocess.CalledProcessError(retcode, cmd) + raise e + + config_file_name = os.path.join(avd_path, avd_name + ".avd", "config.ini") + + print("Writing config at %s" % config_file_name) + + if os.path.isfile(config_file_name): + with open(config_file_name, "a") as config: + for key, value in avd_manifest["emulator_extra_config"].items(): + config.write("%s=%s\n" % (key, value)) + else: + raise NotImplementedError( + "Could not find config file at %s, something went wrong" % config_file_name + ) -def ensure_android_packages(sdkmanager_tool, emulator_only=False, no_interactive=False): +def ensure_android_packages( + sdkmanager_tool, + emulator_only=False, + system_images_only=False, + avd_manifest=None, + no_interactive=False, +): """ Use the given sdkmanager tool (like 'sdkmanager') to install required Android packages. @@ -339,18 +422,24 @@ def ensure_android_packages(sdkmanager_tool, emulator_only=False, no_interactive # This tries to install all the required Android packages. The user # may be prompted to agree to the Android license. - if emulator_only: - package_file_name = os.path.abspath( - os.path.join(os.path.dirname(__file__), "android-emulator-packages.txt") - ) + if system_images_only: + packages_file_name = "android-system-images-packages.txt" + elif emulator_only: + packages_file_name = "android-emulator-packages.txt" else: - package_file_name = os.path.abspath( - os.path.join(os.path.dirname(__file__), "android-packages.txt") - ) - print(INSTALLING_ANDROID_PACKAGES % open(package_file_name, "rt").read()) + packages_file_name = "android-packages.txt" + + packages_file_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), packages_file_name) + ) + with open(packages_file_path) as packages_file: + packages_file_content = packages_file.readlines() + + packages = get_packages_to_install(packages_file_content, avd_manifest) + print(INSTALLING_ANDROID_PACKAGES % "\n".join(packages)) args = [sdkmanager_tool] - args.extend(get_packages_to_install(package_file_name)) + args.extend(packages) if not no_interactive: subprocess.check_call(args) @@ -373,7 +462,7 @@ def ensure_android_packages(sdkmanager_tool, emulator_only=False, no_interactive def generate_mozconfig(os_name, artifact_mode=False): - moz_state_dir, sdk_path, ndk_path = get_paths(os_name) + moz_state_dir, sdk_path, ndk_path, avd_path, emulator_path = get_paths(os_name) extra_lines = [] if extra_lines: @@ -387,6 +476,7 @@ def generate_mozconfig(os_name, artifact_mode=False): kwargs = dict( sdk_path=sdk_path, ndk_path=ndk_path, + avd_path=avd_path, moz_state_dir=moz_state_dir, extra_lines="\n".join(extra_lines), ) @@ -428,6 +518,12 @@ def main(argv): action="store_true", help="If true, install only the Android NDK (and not the Android SDK).", ) + parser.add_option( + "--system-images-only", + dest="system_images_only", + action="store_true", + help="If true, install only the system images for the AVDs.", + ) parser.add_option( "--no-interactive", dest="no_interactive", @@ -440,6 +536,11 @@ def main(argv): action="store_true", help="If true, install only the Android emulator (and not the SDK or NDK).", ) + parser.add_option( + "--avd-manifest", + dest="avd_manifest_path", + help="If present, generate AVD from the manifest pointed by this argument.", + ) options, _ = parser.parse_args(argv) @@ -466,7 +567,9 @@ def main(argv): os_name, artifact_mode=options.artifact_mode, ndk_only=options.ndk_only, + system_images_only=options.system_images_only, emulator_only=options.emulator_only, + avd_manifest_path=options.avd_manifest_path, no_interactive=options.no_interactive, ) mozconfig = generate_mozconfig(os_name, options.artifact_mode) diff --git a/taskcluster/ci/generate-profile/kind.yml b/taskcluster/ci/generate-profile/kind.yml index 97b3d3b93d29..ee69054d664b 100644 --- a/taskcluster/ci/generate-profile/kind.yml +++ b/taskcluster/ci/generate-profile/kind.yml @@ -122,6 +122,8 @@ jobs: - linux64-clang - linux64-minidump-stackwalk - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux android-x86_64-shippable/opt: description: "Android 5.0 x86_64 Profile Generation" @@ -160,6 +162,8 @@ jobs: - linux64-clang - linux64-minidump-stackwalk - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux win32-shippable/opt: description: "Win32 Profile Generation" diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml index 2f23ebf9f5c1..f1147a545bd4 100644 --- a/taskcluster/ci/test/kind.yml +++ b/taskcluster/ci/test/kind.yml @@ -63,6 +63,8 @@ job-defaults: - macosx64-minidump-stackwalk - macosx64-fix-stacks android-em-7.*: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux - android-emulator-linux - linux64-minidump-stackwalk - linux64-fix-stacks diff --git a/taskcluster/ci/test/misc.yml b/taskcluster/ci/test/misc.yml index ab22cf0d4989..e45e9405e8a2 100644 --- a/taskcluster/ci/test/misc.yml +++ b/taskcluster/ci/test/misc.yml @@ -133,6 +133,8 @@ test-verify: - win32-fix-stacks android-em-7.*: - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux - linux64-node - linux64-minidump-stackwalk - linux64-fix-stacks @@ -242,6 +244,8 @@ test-coverage: - win32-fix-stacks android-em-7.*: - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux - linux64-node - linux64-minidump-stackwalk - linux64-fix-stacks diff --git a/taskcluster/ci/test/xpcshell.yml b/taskcluster/ci/test/xpcshell.yml index 6f88889b1266..dba0b97210d8 100644 --- a/taskcluster/ci/test/xpcshell.yml +++ b/taskcluster/ci/test/xpcshell.yml @@ -101,6 +101,8 @@ xpcshell: - win32-minidump-stackwalk - win32-fix-stacks android-em-7.*: + - android-system-image-x86_64-linux + - android-avd-x86_64-linux - android-emulator-linux - linux64-node - linux64-minidump-stackwalk diff --git a/taskcluster/ci/toolchain/android.yml b/taskcluster/ci/toolchain/android.yml index 770efd8f8bc1..7638af544550 100644 --- a/taskcluster/ci/toolchain/android.yml +++ b/taskcluster/ci/toolchain/android.yml @@ -8,6 +8,42 @@ job-defaults: docker-image: {in-tree: android-build} max-run-time: 1800 +linux64-android-avd-x86_64-repack: + description: "Android AVD (Linux) repack toolchain build" + treeherder: + symbol: TL(avd-x86_64-linux) + worker: + artifacts: + - name: project/gecko/android-avd + path: /builds/worker/project/gecko/android-avd/ + type: directory + run: + script: repack-android-avd-linux.sh + arguments: + - 'python/mozboot/mozboot/android-avds/x86_64.json' + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: project/gecko/android-avd/android-avd-linux.tar.zst + toolchain-alias: android-avd-x86_64-linux + +linux64-android-system-image-x86_64-repack: + description: "Android System Images (Linux) repack toolchain build" + treeherder: + symbol: TL(x86_64-avd-img-linux) + worker: + artifacts: + - name: project/gecko/android-system-images + path: /builds/worker/project/gecko/android-system-images/ + type: directory + run: + script: repack-android-system-images-linux.sh + arguments: + - 'python/mozboot/mozboot/android-avds/x86_64.json' + resources: + - 'python/mozboot/**/*android*' + toolchain-artifact: project/gecko/android-system-images/android-system-images-linux.tar.zst + toolchain-alias: android-system-image-x86_64-linux + linux64-android-sdk-linux-repack: description: "Android SDK (Linux) repack toolchain build" treeherder: diff --git a/taskcluster/ci/webrender/kind.yml b/taskcluster/ci/webrender/kind.yml index 5b5768929642..2f8b1cdf1cbf 100644 --- a/taskcluster/ci/webrender/kind.yml +++ b/taskcluster/ci/webrender/kind.yml @@ -271,6 +271,8 @@ jobs: - android-gradle-dependencies - android-ndk-linux - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux - linux64-rust-android - wrench-deps treeherder: @@ -311,6 +313,8 @@ jobs: - android-gradle-dependencies - android-ndk-linux - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux - linux64-rust-android - wrench-deps treeherder: @@ -347,6 +351,8 @@ jobs: - 'wrench-debug.apk' toolchain: - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux run: using: run-task tooltool-downloads: internal @@ -389,6 +395,8 @@ jobs: - 'wrench-release.apk' toolchain: - android-sdk-linux + - android-system-image-x86_64-linux + - android-avd-x86_64-linux run: using: run-task tooltool-downloads: internal diff --git a/taskcluster/scripts/misc/repack-android-avd-linux.sh b/taskcluster/scripts/misc/repack-android-avd-linux.sh new file mode 100755 index 000000000000..4b80c8a4df08 --- /dev/null +++ b/taskcluster/scripts/misc/repack-android-avd-linux.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -x -e -v + +# This script is for fetching and repacking the Android SDK (for +# Linux), the tools required to produce Android packages. + +UPLOAD_DIR=$HOME/project/gecko/android-avd +AVD_JSON_CONFIG="$1" + +mkdir -p $HOME/artifacts $UPLOAD_DIR + +# Populate /builds/worker/.mozbuild/android-device +cd $GECKO_PATH +./mach python python/mozboot/mozboot/android.py --artifact-mode --avd-manifest="$AVD_JSON_CONFIG" --no-interactive + +tar cavf $UPLOAD_DIR/android-avd-linux.tar.zst -C /builds/worker/.mozbuild android-device + +ls -al $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/repack-android-system-images-linux.sh b/taskcluster/scripts/misc/repack-android-system-images-linux.sh new file mode 100755 index 000000000000..d1707bc2ace4 --- /dev/null +++ b/taskcluster/scripts/misc/repack-android-system-images-linux.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -x -e -v + +# This script is for fetching and repacking the Android SDK (for +# Linux), the tools required to produce Android packages. + +AVD_JSON_CONFIG="$1" +UPLOAD_DIR=$HOME/project/gecko/android-system-images + +mkdir -p $HOME/artifacts $UPLOAD_DIR + +# Populate /builds/worker/.mozbuild/android-sdk-linux. +cd $GECKO_PATH +./mach python python/mozboot/mozboot/android.py --artifact-mode --system-images-only --avd-manifest="$AVD_JSON_CONFIG" --no-interactive + +# It's nice to have the build logs include the state of the world upon +# completion. +/builds/worker/.mozbuild/android-sdk-linux/tools/bin/sdkmanager --list + +tar cavf $UPLOAD_DIR/android-system-images-linux.tar.zst -C /builds/worker/.mozbuild android-sdk-linux/system-images + +ls -al $UPLOAD_DIR diff --git a/testing/config/tooltool-manifests/androidarm_4_3/mach-emulator.manifest b/testing/config/tooltool-manifests/androidarm_4_3/mach-emulator.manifest deleted file mode 100644 index 7cd4e4ef0e27..000000000000 --- a/testing/config/tooltool-manifests/androidarm_4_3/mach-emulator.manifest +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "size": 136624500, - "visibility": "public", - "digest": "1fcebe172773704aef5f30a9056b68a8b77d42de26270443dfa0f80d629f89b63e4bf47b23e8641f1aec66db38e1c4ff2fd10bf0fe836438cb476a7bd74de36d", - "algorithm": "sha512", - "filename": "AVDs-armv7a-android-4.3.1_r1-build-2019-01-22.tar.gz", - "unpack": true - } -] diff --git a/testing/config/tooltool-manifests/androidarm_4_3/releng.manifest b/testing/config/tooltool-manifests/androidarm_4_3/releng.manifest deleted file mode 100644 index f223a6821948..000000000000 --- a/testing/config/tooltool-manifests/androidarm_4_3/releng.manifest +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "algorithm": "sha512", - "visibility": "public", - "filename": "AVDs-armv7a-android-4.3.1_r1-build-2016-08-02_larger_disk.tar.gz", - "unpack": true, - "digest": "03e2812cfe9cd733a9094900e36a7fa2e67d948e392ec09b84810134ce0662deaf73f784f5d4ba141d19550c6ad5ca19b7e2ba13b0a51f3540c4648e0be499d5", - "size": 130163149 - } -] \ No newline at end of file diff --git a/testing/config/tooltool-manifests/androidarm_7_0/mach-emulator.manifest b/testing/config/tooltool-manifests/androidarm_7_0/mach-emulator.manifest deleted file mode 100644 index d42632a5aa6d..000000000000 --- a/testing/config/tooltool-manifests/androidarm_7_0/mach-emulator.manifest +++ /dev/null @@ -1,10 +0,0 @@ -[ -{ -"size": 408178881, -"visibility": "public", -"digest": "7d6a138946e4a45d846bc35f362567871cbb473b19baf2c49dd04cd7c35f0e5299cb98a8995be0f0b54a1b4c241110562a7b863225839a5a5a9d75ee0138ba03", -"algorithm": "sha512", -"filename": "AVDs-arm64v8a-android-7.0-build-2018-06-07.tar.gz", -"unpack": true -} -] diff --git a/testing/config/tooltool-manifests/androidx86_7_0/mach-emulator.manifest b/testing/config/tooltool-manifests/androidx86_7_0/mach-emulator.manifest deleted file mode 100644 index 49f46c60ffe1..000000000000 --- a/testing/config/tooltool-manifests/androidx86_7_0/mach-emulator.manifest +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "algorithm": "sha512", - "visibility": "public", - "filename": "AVDs-x86-android-7.0-build-2019-04-23.tar.gz", - "unpack": true, - "digest": "3cc03789aabfc692c76e5ae4ebefa7a5628f386df3c9778af2485a49b2401d4ad66301be6c3d116ff7d3ee747e00ce6332381216f55a7253b6b5b600d059baa2", - "size": 445250935 - } -] diff --git a/testing/config/tooltool-manifests/androidx86_7_0/releng.manifest b/testing/config/tooltool-manifests/androidx86_7_0/releng.manifest deleted file mode 100644 index 49f46c60ffe1..000000000000 --- a/testing/config/tooltool-manifests/androidx86_7_0/releng.manifest +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "algorithm": "sha512", - "visibility": "public", - "filename": "AVDs-x86-android-7.0-build-2019-04-23.tar.gz", - "unpack": true, - "digest": "3cc03789aabfc692c76e5ae4ebefa7a5628f386df3c9778af2485a49b2401d4ad66301be6c3d116ff7d3ee747e00ce6332381216f55a7253b6b5b600d059baa2", - "size": 445250935 - } -] diff --git a/testing/mozbase/mozrunner/mozrunner/devices/android_device.py b/testing/mozbase/mozrunner/mozrunner/devices/android_device.py index 69fe7a612b50..94d3ed688619 100644 --- a/testing/mozbase/mozrunner/mozrunner/devices/android_device.py +++ b/testing/mozbase/mozrunner/mozrunner/devices/android_device.py @@ -19,7 +19,6 @@ import time from distutils.spawn import find_executable from enum import Enum -from mozbuild.base import MozbuildObject from mozdevice import ADBHost, ADBDeviceFactory from six.moves import input, urllib @@ -84,16 +83,9 @@ class AvdInfo(object): Simple class to contain an AVD description. """ - def __init__( - self, description, name, tooltool_manifest, toolchain_job, extra_args, x86 - ): - assert not (tooltool_manifest and toolchain_job), ( - "%s: specify manifest or toolchain job, not both" % description - ) + def __init__(self, description, name, extra_args, x86): self.description = description self.name = name - self.tooltool_manifest = tooltool_manifest - self.toolchain_job = toolchain_job self.extra_args = extra_args self.x86 = x86 @@ -114,10 +106,8 @@ AVD_DICT = { False, ), "x86_64": AvdInfo( - "Android 7.0 x86/x86_64", - "mozemulator-x86-7.0", - "testing/config/tooltool-manifests/androidx86_7_0/mach-emulator.manifest", - None, + "Android x86_64", + "mozemulator-x86_64", [ "-skip-adb-auth", "-verbose", @@ -129,6 +119,12 @@ AVD_DICT = { "3072", "-cores", "4", + "-skin", + "800x1280", + "-prop", + "ro.test_harness=true", + "-no-snapstorage", + "-no-snapshot", ], True, ), @@ -283,8 +279,8 @@ def verify_android_device( ).strip() if response.lower().startswith("y") or response == "": if not emulator.check_avd(): - _log_info("Fetching AVD...") - emulator.update_avd() + _log_info("Android AVD not found, please run |mach bootstrap|") + return _log_info( "Starting emulator running %s..." % emulator.get_avd_description() ) @@ -531,7 +527,7 @@ class AndroidEmulator(object): emulator = AndroidEmulator() if not emulator.is_running() and emulator.is_available(): if not emulator.check_avd(): - emulator.update_avd() + print("Android Emulator AVD not found, please run |mach bootstrap|") emulator.start() emulator.wait_for_start() emulator.wait() @@ -581,56 +577,18 @@ class AndroidEmulator(object): found = True return found - def check_avd(self, force=False): + def check_avd(self): """ Determine if the AVD is already installed locally. - (This is usually used to determine if update_avd() is likely - to require a download.) Returns True if the AVD is installed. """ avd = os.path.join(EMULATOR_HOME_DIR, "avd", self.avd_info.name + ".avd") - if force and os.path.exists(avd): - shutil.rmtree(avd) if os.path.exists(avd): _log_debug("AVD found at %s" % avd) return True return False - def update_avd(self, force=False): - """ - If required, update the AVD via tooltool. - - If the AVD directory is not found, or "force" is requested, - download the tooltool manifest associated with the AVD and then - invoke tooltool.py on the manifest. tooltool.py will download the - required archive (unless already present in the local tooltool - cache) and install the AVD. - """ - avd = os.path.join(EMULATOR_HOME_DIR, "avd", self.avd_info.name + ".avd") - ini_file = os.path.join(EMULATOR_HOME_DIR, "avd", self.avd_info.name + ".ini") - if force and os.path.exists(avd): - shutil.rmtree(avd) - if force: - for f in glob.glob(os.path.join(EMULATOR_HOME_DIR, "AVD*.checksum")): - os.remove(f) - if not os.path.exists(avd): - if os.path.exists(ini_file): - os.remove(ini_file) - if self.avd_info.tooltool_manifest: - path = self.avd_info.tooltool_manifest - _get_tooltool_manifest( - self.substs, path, EMULATOR_HOME_DIR, "releng.manifest" - ) - _tooltool_fetch(self.substs) - elif self.avd_info.toolchain_job: - _install_toolchain_artifact(self.avd_info.toolchain_job) - else: - raise Exception( - "either a tooltool manifest or a toolchain job is required" - ) - self._update_avd_paths() - def start(self, gpu_arg=None): """ Launch the emulator. @@ -645,6 +603,7 @@ class AndroidEmulator(object): auth_file.close() env = os.environ + env["ANDROID_EMULATOR_HOME"] = EMULATOR_HOME_DIR env["ANDROID_AVD_HOME"] = os.path.join(EMULATOR_HOME_DIR, "avd") command = [self.emulator_path, "-avd", self.avd_info.name] override = os.environ.get("MOZ_EMULATOR_COMMAND_ARGS") @@ -1012,34 +971,6 @@ def _tooltool_fetch(substs): _log_warning(str(e)) -def _install_toolchain_artifact(toolchain_job, no_unpack=False): - build_obj = MozbuildObject.from_environment() - mach_binary = os.path.join(build_obj.topsrcdir, "mach") - mach_binary = os.path.abspath(mach_binary) - if not os.path.exists(mach_binary): - raise ValueError("mach not found at %s" % mach_binary) - - # If Python can't figure out what its own executable is, there's little - # chance we're going to be able to execute mach on its own, particularly - # on Windows. - if not sys.executable: - raise ValueError("cannot determine path to Python executable") - - cmd = [ - sys.executable, - mach_binary, - "artifact", - "toolchain", - "--from-build", - toolchain_job, - ] - - if no_unpack: - cmd += ["--no-unpack"] - - subprocess.check_call(cmd, cwd=EMULATOR_HOME_DIR) - - def _get_host_platform(): plat = None if "darwin" in str(sys.platform).lower(): diff --git a/testing/mozharness/configs/android/android-x86_64.py b/testing/mozharness/configs/android/android-x86_64.py index 3bca064e1bf6..676c93ccfcf3 100644 --- a/testing/mozharness/configs/android/android-x86_64.py +++ b/testing/mozharness/configs/android/android-x86_64.py @@ -8,10 +8,29 @@ # mozharness configuration from android_common.py, or similar. config = { - "emulator_package": "system-images;android-24;default;x86_64", "emulator_avd_name": "mozemulator-x86_64", "emulator_process_name": "qemu-system-x86_64", - "emulator_extra_args": "-gpu on -skip-adb-auth -verbose -show-kernel -ranchu -selinux permissive -memory 3072 -cores 4", + "emulator_extra_args": [ + "-gpu", + "on", + "-skip-adb-auth", + "-verbose", + "-show-kernel", + "-ranchu", + "-selinux", + "permissive", + "-memory", + "3072", + "-cores", + "4", + "-skin", + "800x1280", + "-no-snapstorage", + "-no-snapshot", + # Skips first-run dialogs + "-prop", + "ro.test_harness=true", + ], "exes": { "adb": "%(abs_sdk_dir)s/platform-tools/adb", }, diff --git a/testing/mozharness/configs/android/android_common.py b/testing/mozharness/configs/android/android_common.py index fcd3836ed5fb..ac3dd2f2dab5 100644 --- a/testing/mozharness/configs/android/android_common.py +++ b/testing/mozharness/configs/android/android_common.py @@ -44,7 +44,6 @@ def WebglSuite(name): config = { "default_actions": [ "clobber", - "setup-avds", "download-and-extract", "create-virtualenv", "start-emulator", diff --git a/testing/mozharness/configs/android/android_pgo.py b/testing/mozharness/configs/android/android_pgo.py index 7c9d91fd1c5a..a118ea674097 100644 --- a/testing/mozharness/configs/android/android_pgo.py +++ b/testing/mozharness/configs/android/android_pgo.py @@ -9,7 +9,6 @@ config = { "default_actions": [ - "setup-avds", "download", "create-virtualenv", "start-emulator", diff --git a/testing/mozharness/configs/android/androidarm.py b/testing/mozharness/configs/android/androidarm.py deleted file mode 100644 index ba2e1f19a632..000000000000 --- a/testing/mozharness/configs/android/androidarm.py +++ /dev/null @@ -1,39 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# mozharness configuration for Android 4.3 unit tests -# -# This configuration should be combined with suite definitions and other -# mozharness configuration from android_common.py, or similar. - -config = { - "deprecated_sdk_path": True, - "tooltool_manifest_path": "testing/config/tooltool-manifests/androidarm_4_3/releng.manifest", - "emulator_manifest": """ - [ - { - "algorithm": "sha512", - "visibility": "internal", - "filename": "android-sdk_r24.0.2a-linux.tar.gz", - "unpack": true, - "digest": "9b7d4a6fcb33d80884c68e9099a3e11963a79ec0a380a5a9e1a093e630f960d0a5083392c8804121c3ad27ee8ba29ca8df785d19d5a7fdc89458c4e51ada5120", - "size": 38591399 - } - ]""", - "emulator_avd_name": "test-1", - "emulator_process_name": "emulator64-arm", - "emulator_extra_args": "-show-kernel -debug init,console,gles,memcheck,adbserver,adbclient,adb,avd_config,socket", - "exes": { - "adb": "%(abs_work_dir)s/android-sdk-linux/platform-tools/adb", - }, - "env": { - "DISPLAY": ":0.0", - "PATH": "%(PATH)s:%(abs_work_dir)s/android-sdk-linux/tools:%(abs_work_dir)s/android-sdk-linux/platform-tools", - }, - "bogomips_minimum": 250, - # in support of test-verify - "android_version": 18, - "is_fennec": True, - "is_emulator": True, -} diff --git a/testing/mozharness/mozharness/mozilla/testing/android.py b/testing/mozharness/mozharness/mozilla/testing/android.py index 42edf10f23d4..39bc3216857c 100644 --- a/testing/mozharness/mozharness/mozilla/testing/android.py +++ b/testing/mozharness/mozharness/mozilla/testing/android.py @@ -21,6 +21,16 @@ from mozharness.mozilla.automation import TBPL_RETRY, EXIT_STATUS_DICT from mozharness.base.script import PreScriptAction, PostScriptAction +def ensure_dir(dir): + """Ensures the given directory exists""" + if dir and not os.path.exists(dir): + try: + os.makedirs(dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise + + class AndroidMixin(object): """ Mixin class used by Android test scripts. @@ -136,6 +146,7 @@ class AndroidMixin(object): except Exception: self.warning("failed to remove %s" % AUTH_FILE) + env["ANDROID_EMULATOR_HOME"] = avd_home_dir avd_path = os.path.join(avd_home_dir, "avd") if os.path.exists(avd_path): env["ANDROID_AVD_HOME"] = avd_path @@ -149,15 +160,39 @@ class AndroidMixin(object): sdk_path = self.abs_dirs["abs_sdk_dir"] if os.path.exists(sdk_path): env["ANDROID_SDK_HOME"] = sdk_path + env["ANDROID_SDK_ROOT"] = sdk_path self.info("Found sdk at %s" % sdk_path) else: self.warning("Android sdk missing? Not found at %s" % sdk_path) - if self.use_gles3: - # enable EGL 3.0 in advancedFeatures.ini - AF_FILE = os.path.join(sdk_path, "advancedFeatures.ini") - with open(AF_FILE, "w") as f: + avd_config_path = os.path.join( + avd_path, "%s.ini" % self.config["emulator_avd_name"] + ) + avd_folder = os.path.join(avd_path, "%s.avd" % self.config["emulator_avd_name"]) + if os.path.isfile(avd_config_path): + # The ini file points to the absolute path to the emulator folder, + # which might be different, so we need to update it. + old_config = "" + with open(avd_config_path, "r") as config_file: + old_config = config_file.readlines() + self.info("Old Config: %s" % old_config) + with open(avd_config_path, "w") as config_file: + for line in old_config: + if line.startswith("path="): + config_file.write("path=%s\n" % avd_folder) + self.info("Updating path from: %s" % line) + else: + config_file.write("%s\n" % line) + else: + self.warning("Could not find config path at %s" % avd_config_path) + + # enable EGL 3.0 in advancedFeatures.ini + AF_FILE = os.path.join(avd_home_dir, "advancedFeatures.ini") + with open(AF_FILE, "w") as f: + if self.use_gles3: f.write("GLESDynamicVersion=on\n") + else: + f.write("GLESDynamicVersion=off\n") # extra diagnostics for kvm acceleration emu = self.config.get("emulator_process_name") @@ -169,9 +204,10 @@ class AndroidMixin(object): except Exception as e: self.warning("Extra kvm diagnostics failed: %s" % str(e)) + self.info("emulator env: %s" % str(env)) command = ["emulator", "-avd", self.config["emulator_avd_name"]] if "emulator_extra_args" in self.config: - command += self.config["emulator_extra_args"].split() + command += self.config["emulator_extra_args"] dir = self.query_abs_dirs()["abs_blob_upload_dir"] tmp_file = tempfile.NamedTemporaryFile( @@ -400,9 +436,7 @@ class AndroidMixin(object): import mozdevice try: - out = self.device.get_prop("sys.boot_completed", timeout=30) - if out.strip() == "1": - return True + return self.device.is_device_ready(timeout=30) except (ValueError, mozdevice.ADBError, mozdevice.ADBTimeoutError): pass return False @@ -579,44 +613,6 @@ class AndroidMixin(object): # Script actions - def setup_avds(self): - """ - If tooltool cache mechanism is enabled, the cached version is used by - the fetch command. If the manifest includes an "unpack" field, tooltool - will unpack all compressed archives mentioned in the manifest. - """ - if not self.is_emulator: - return - - c = self.config - dirs = self.query_abs_dirs() - self.mkdir_p(dirs["abs_work_dir"]) - self.mkdir_p(dirs["abs_blob_upload_dir"]) - - # Always start with a clean AVD: AVD includes Android images - # which can be stateful. - self.rmtree(dirs["abs_avds_dir"]) - self.mkdir_p(dirs["abs_avds_dir"]) - if "avd_url" in c: - # Intended for experimental setups to evaluate an avd prior to - # tooltool deployment. - url = c["avd_url"] - self.download_unpack(url, dirs["abs_avds_dir"]) - else: - url = self._get_repo_url(c["tooltool_manifest_path"]) - self._tooltool_fetch(url, dirs["abs_avds_dir"]) - - avd_home_dir = self.abs_dirs["abs_avds_dir"] - if avd_home_dir != "/home/cltbld/.android": - # Modify the downloaded avds to point to the right directory. - cmd = [ - "bash", - "-c", - 'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini' - % (avd_home_dir, os.path.join(avd_home_dir, "avd")), - ] - subprocess.check_call(cmd) - def start_emulator(self): """ Starts the emulator @@ -624,20 +620,10 @@ class AndroidMixin(object): if not self.is_emulator: return - if "emulator_url" in self.config or "emulator_manifest" in self.config: - dirs = self.query_abs_dirs() - if self.config.get("emulator_url"): - self.download_unpack(self.config["emulator_url"], dirs["abs_work_dir"]) - elif self.config.get("emulator_manifest"): - manifest_path = self.create_tooltool_manifest( - self.config["emulator_manifest"] - ) - dirs = self.query_abs_dirs() - cache = self.config.get("tooltool_cache", None) - if self.tooltool_fetch( - manifest_path, output_dir=dirs["abs_work_dir"], cache=cache - ): - self.fatal("Unable to download emulator via tooltool!") + dirs = self.query_abs_dirs() + ensure_dir(dirs["abs_work_dir"]) + ensure_dir(dirs["abs_blob_upload_dir"]) + if not os.path.isfile(self.adb_path): self.fatal("The adb binary '%s' is not a valid file!" % self.adb_path) self.kill_processes("xpcshell") diff --git a/testing/mozharness/scripts/android_emulator_pgo.py b/testing/mozharness/scripts/android_emulator_pgo.py index 3aae63c16101..9b9a4aac9cad 100644 --- a/testing/mozharness/scripts/android_emulator_pgo.py +++ b/testing/mozharness/scripts/android_emulator_pgo.py @@ -72,7 +72,6 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin): super(AndroidProfileRun, self).__init__( config_options=self.config_options, all_actions=[ - "setup-avds", "download", "create-virtualenv", "start-emulator", @@ -103,7 +102,17 @@ class AndroidProfileRun(TestingMixin, BaseScript, MozbaseMixin, AndroidMixin): dirs["abs_test_install_dir"] = os.path.join(abs_dirs["abs_src_dir"], "testing") dirs["abs_xre_dir"] = os.path.join(abs_dirs["abs_work_dir"], "hostutils") dirs["abs_blob_upload_dir"] = "/builds/worker/artifacts/blobber_upload_dir" - dirs["abs_avds_dir"] = os.path.join(abs_dirs["abs_work_dir"], ".android") + fetches_dir = os.environ.get("MOZ_FETCHES_DIR") + if fetches_dir: + dirs["abs_sdk_dir"] = os.path.join(fetches_dir, "android-sdk-linux") + dirs["abs_avds_dir"] = os.path.join(fetches_dir, "android-device") + else: + dirs["abs_sdk_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "android-sdk-linux" + ) + dirs["abs_avds_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "android-device" + ) for key in dirs.keys(): if key not in abs_dirs: diff --git a/testing/mozharness/scripts/android_emulator_unittest.py b/testing/mozharness/scripts/android_emulator_unittest.py index 310887b8605c..ad1646e58981 100644 --- a/testing/mozharness/scripts/android_emulator_unittest.py +++ b/testing/mozharness/scripts/android_emulator_unittest.py @@ -144,7 +144,6 @@ class AndroidEmulatorTest( config_options=self.config_options, all_actions=[ "clobber", - "setup-avds", "download-and-extract", "create-virtualenv", "start-emulator", @@ -204,14 +203,17 @@ class AndroidEmulatorTest( dirs["abs_xpcshell_dir"] = os.path.join( dirs["abs_test_install_dir"], "xpcshell" ) - dirs["abs_avds_dir"] = os.path.join(abs_dirs["abs_work_dir"], ".android") fetches_dir = os.environ.get("MOZ_FETCHES_DIR") if fetches_dir: dirs["abs_sdk_dir"] = os.path.join(fetches_dir, "android-sdk-linux") + dirs["abs_avds_dir"] = os.path.join(fetches_dir, "android-device") else: dirs["abs_sdk_dir"] = os.path.join( abs_dirs["abs_work_dir"], "android-sdk-linux" ) + dirs["abs_avds_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "android-device" + ) for key in dirs.keys(): if key not in abs_dirs: diff --git a/testing/mozharness/scripts/web_platform_tests.py b/testing/mozharness/scripts/web_platform_tests.py index 70b6af8bdc98..cd09991ea8fe 100755 --- a/testing/mozharness/scripts/web_platform_tests.py +++ b/testing/mozharness/scripts/web_platform_tests.py @@ -173,7 +173,6 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidM config_options=self.config_options, all_actions=[ "clobber", - "setup-avds", "download-and-extract", "download-and-process-manifest", "create-virtualenv", @@ -233,14 +232,17 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidM if self.is_android: dirs["abs_xre_dir"] = os.path.join(abs_dirs["abs_work_dir"], "hostutils") if self.is_emulator: - dirs["abs_avds_dir"] = os.path.join(abs_dirs["abs_work_dir"], ".android") fetches_dir = os.environ.get("MOZ_FETCHES_DIR") if fetches_dir: dirs["abs_sdk_dir"] = os.path.join(fetches_dir, "android-sdk-linux") + dirs["abs_avds_dir"] = os.path.join(fetches_dir, "android-device") else: dirs["abs_sdk_dir"] = os.path.join( abs_dirs["abs_work_dir"], "android-sdk-linux" ) + dirs["abs_avds_dir"] = os.path.join( + abs_dirs["abs_work_dir"], "android-device" + ) if self.config["enable_webrender"]: # AndroidMixin uses this when launching the emulator. We only want # GLES3 if we're running WebRender diff --git a/testing/web-platform/tests/tools/wpt/android.py b/testing/web-platform/tests/tools/wpt/android.py index 8ded6e1a606d..ab6847480ee2 100644 --- a/testing/web-platform/tests/tools/wpt/android.py +++ b/testing/web-platform/tests/tools/wpt/android.py @@ -87,7 +87,6 @@ def install_android_packages(logger, sdk_path, no_prompt=False): if not os.path.exists(sdk_manager_path): raise OSError("Can't find sdkmanager at %s" % sdk_manager_path) - #TODO: Not sure what's really needed here packages = ["platform-tools", "build-tools;30.0.2", "platforms;android-30", @@ -115,10 +114,6 @@ def get_emulator(sdk_path): substs = {"top_srcdir": wpt_root, "TARGET_CPU": "x86"} emulator = android_device.AndroidEmulator("*", substs=substs) emulator.emulator_path = os.path.join(sdk_path, "emulator", "emulator") - emulator.avd_info.tooltool_manifest = os.path.join(wpt_root, - "tools", - "wpt", - "mach-emulator.manifest") return emulator @@ -134,7 +129,6 @@ def install(logger, reinstall=False, no_prompt=False): os.environ["ANDROID_SDK_ROOT"] = dest emulator = get_emulator(dest) - emulator.update_avd() return emulator @@ -148,7 +142,8 @@ def start(logger, emulator=None, reinstall=False): emulator = get_emulator(sdk_path) if not emulator.check_avd(): - emulator.update_avd() + logger.critical("Android AVD not found, please run |mach bootstrap|") + raise NotImplementedError emulator.start() emulator.wait_for_start() diff --git a/testing/web-platform/tests/tools/wpt/mach-emulator.manifest b/testing/web-platform/tests/tools/wpt/mach-emulator.manifest deleted file mode 100644 index 49f46c60ffe1..000000000000 --- a/testing/web-platform/tests/tools/wpt/mach-emulator.manifest +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "algorithm": "sha512", - "visibility": "public", - "filename": "AVDs-x86-android-7.0-build-2019-04-23.tar.gz", - "unpack": true, - "digest": "3cc03789aabfc692c76e5ae4ebefa7a5628f386df3c9778af2485a49b2401d4ad66301be6c3d116ff7d3ee747e00ce6332381216f55a7253b6b5b600d059baa2", - "size": 445250935 - } -]