gecko-dev/testing/mochitest/runjunit.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import os
import posixpath
import re
import shutil
import sys
import tempfile
import traceback

import mozcrash
import mozinfo
import mozlog
import moznetwork
from mozdevice import ADBAndroid
from mozprofile import Profile, DEFAULT_PORTS
from mozprofile.permissions import ServerLocations
from runtests import MochitestDesktop, update_mozinfo

here = os.path.abspath(os.path.dirname(__file__))

try:
    from mozbuild.base import (
        MozbuildObject,
        MachCommandConditions as conditions,
    )
    build_obj = MozbuildObject.from_environment(cwd=here)
except ImportError:
    build_obj = None
    conditions = None


class JUnitTestRunner(MochitestDesktop):
    """
       A test harness to run geckoview junit tests on a remote device.
    """

    def __init__(self, log, options):
        self.log = log
        verbose = False
        if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug':
            verbose = True
        self.device = ADBAndroid(adb=options.adbPath or 'adb',
                                 device=options.deviceSerial,
                                 test_root=options.remoteTestRoot,
                                 verbose=verbose)
        self.options = options
        self.log.debug("options=%s" % vars(options))
        update_mozinfo()
        self.remote_profile = posixpath.join(self.device.test_root, 'junit-profile')
        self.server_init()

        self.cleanup()
        self.device.clear_logcat()
        self.build_profile()
        self.startServers(
            self.options,
            debuggerInfo=None,
            ignoreSSLTunnelExts=True)
        self.log.debug("Servers started")

    def server_init(self):
        """
           Additional initialization required to satisfy MochitestDesktop.startServers
        """
        self._locations = None
        self.server = None
        self.wsserver = None
        self.websocketProcessBridge = None
        self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get('debug') else 90
        if self.options.remoteWebServer is None:
            if os.name != "nt":
                self.options.remoteWebServer = moznetwork.get_ip()
            else:
                raise Exception("--remote-webserver must be specified")
        self.options.webServer = self.options.remoteWebServer
        self.options.webSocketPort = '9988'
        self.options.httpdPath = None
        self.options.keep_open = False
        self.options.pidFile = ""
        self.options.subsuite = None
        self.options.xrePath = None
        if build_obj and 'MOZ_HOST_BIN' in os.environ:
            self.options.xrePath = os.environ['MOZ_HOST_BIN']
            if not self.options.utilityPath:
                self.options.utilityPath = self.options.xrePath
        if not self.options.xrePath:
            self.options.xrePath = self.options.utilityPath
        if build_obj:
            self.options.certPath = os.path.join(build_obj.topsrcdir,
                                                 'build', 'pgo', 'certs')

    def build_profile(self):
        """
           Create a local profile with test prefs and proxy definitions and
           push it to the remote device.
        """

        self.profile = Profile(locations=self.locations, proxy=self.proxy(self.options))
        self.options.profilePath = self.profile.profile

        # Set preferences
        self.merge_base_profiles(self.options)
        self.profile.set_preferences(self.extraPrefs(self.options.extraPrefs))

        if self.fillCertificateDB(self.options):
            self.log.error("Certificate integration failed")

        self.device.mkdir(self.remote_profile, parents=True)
        self.device.push(self.profile.profile, self.remote_profile)
        self.log.debug("profile %s -> %s" %
                       (str(self.profile.profile), str(self.remote_profile)))

    def cleanup(self):
        try:
            self.stopServers()
            self.log.debug("Servers stopped")
            self.device.stop_application(self.options.app)
            self.device.rm(self.remote_profile, force=True, recursive=True)
            if hasattr(self, 'profile'):
                del self.profile
        except Exception:
            traceback.print_exc()
            self.log.info("Caught and ignored an exception during cleanup")

    def build_command_line(self, test_filters):
        """
           Construct and return the 'am instrument' command line.
        """
        cmd = "am instrument -w -r"
        # profile location
        cmd = cmd + " -e args '-profile %s'" % self.remote_profile
        # multi-process
        e10s = 'true' if self.options.e10s else 'false'
        cmd = cmd + " -e use_multiprocess %s" % e10s
        # chunks (shards)
        shards = self.options.totalChunks
        shard = self.options.thisChunk
        if shards is not None and shard is not None:
            shard -= 1  # shard index is 0 based
            cmd = cmd + " -e numShards %d -e shardIndex %d" % (shards, shard)
        # test filters: limit run to specific test(s)
        for f in test_filters:
            # filter can be class-name or 'class-name#method-name' (single test)
            cmd = cmd + " -e class %s" % f
        # environment
        env = {}
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["XPCOM_DEBUG_BREAK"] = "stack"
        env["DISABLE_UNSAFE_CPOW_WARNINGS"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["MOZ_IN_AUTOMATION"] = "1"
        env["R_LOG_VERBOSE"] = "1"
        env["R_LOG_LEVEL"] = "6"
        env["R_LOG_DESTINATION"] = "stderr"
        for (env_count, (env_key, env_val)) in enumerate(env.iteritems()):
            cmd = cmd + " -e env%d %s=%s" % (env_count, env_key, env_val)
        # runner
        cmd = cmd + " %s/%s" % (self.options.app, self.options.runner)
        return cmd

    @property
    def locations(self):
        if self._locations is not None:
            return self._locations
        locations_file = os.path.join(here, 'server-locations.txt')
        self._locations = ServerLocations(locations_file)
        return self._locations

    def run_tests(self, test_filters=None):
        """
           Run the tests.
        """
        if not self.device.is_app_installed(self.options.app):
            raise Exception("%s is not installed" %
                            self.options.app)
        if self.device.process_exist(self.options.app):
            raise Exception("%s already running before starting tests" %
                            self.options.app)

        self.test_started = False
        self.pass_count = 0
        self.fail_count = 0
        self.todo_count = 0
        self.class_name = ""
        self.test_name = ""
        self.current_full_name = ""

        def callback(line):
            # Output callback: Parse the raw junit log messages, translating into
            # treeherder-friendly test start/pass/fail messages.

            self.log.process_output(self.options.app, str(line))
            # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something"
            match = re.match(r'INSTRUMENTATION_STATUS:\s*class=(.*)', line)
            if match:
                self.class_name = match.group(1)
            # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something"
            match = re.match(r'INSTRUMENTATION_STATUS:\s*test=(.*)', line)
            if match:
                self.test_name = match.group(1)
            # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..."
            match = re.match(r'INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)', line)
            if match:
                status = match.group(1)
                full_name = "%s.%s" % (self.class_name, self.test_name)
                if full_name == self.current_full_name:
                    if status == '0':
                        message = ''
                        status = 'PASS'
                        expected = 'PASS'
                        self.pass_count += 1
                    elif status == '-3':  # ignored (skipped)
                        message = ''
                        status = 'SKIP'
                        expected = 'SKIP'
                        self.todo_count += 1
                    elif status == '-4':  # known fail
                        message = ''
                        status = 'FAIL'
                        expected = 'FAIL'
                        self.todo_count += 1
                    else:
                        message = 'status %s' % status
                        status = 'FAIL'
                        expected = 'PASS'
                        self.fail_count += 1
                    self.log.test_end(full_name, status, expected, message)
                    self.test_started = False
                else:
                    if self.test_started:
                        # next test started without reporting previous status
                        self.fail_count += 1
                        status = 'FAIL'
                        expected = 'PASS'
                        self.log.test_end(self.current_full_name, status, expected,
                                          "missing test completion status")
                    self.log.test_start(full_name)
                    self.test_started = True
                    self.current_full_name = full_name

        # Ideally all test names should be reported to suite_start, but these test
        # names are not known in advance.
        self.log.suite_start(["geckoview-junit"])
        try:
            cmd = self.build_command_line(test_filters)
            self.log.info("launching %s" % cmd)
            self.device.shell(cmd, timeout=self.options.max_time, stdout_callback=callback)
            self.log.info("Passed: %d" % self.pass_count)
            self.log.info("Failed: %d" % self.fail_count)
            self.log.info("Todo: %d" % self.todo_count)
        finally:
            self.log.suite_end()

        if self.check_for_crashes():
            self.fail_count = 1

        return 1 if self.fail_count else 0

    def check_for_crashes(self):
        logcat = self.device.get_logcat()
        if logcat:
            if mozcrash.check_for_java_exception(logcat, self.current_full_name):
                return True
        symbols_path = self.options.symbolsPath
        try:
            dump_dir = tempfile.mkdtemp()
            remote_dir = posixpath.join(self.remote_profile, 'minidumps')
            if not self.device.is_dir(remote_dir):
                # If crash reporting is enabled (MOZ_CRASHREPORTER=1), the
                # minidumps directory is automatically created when the app
                # (first) starts, so its lack of presence is a hint that
                # something went wrong.
                print "Automation Error: No crash directory (%s) found on remote device" % \
                    remote_dir
                return True
            self.device.pull(remote_dir, dump_dir)
            crashed = mozcrash.log_crashes(self.log, dump_dir, symbols_path,
                                           test=self.current_full_name)
        finally:
            try:
                shutil.rmtree(dump_dir)
            except Exception:
                self.log.warning("unable to remove directory: %s" % dump_dir)
        return crashed


class JunitArgumentParser(argparse.ArgumentParser):
    """
       An argument parser for geckoview-junit.
    """
    def __init__(self, **kwargs):
        super(JunitArgumentParser, self).__init__(**kwargs)

        self.add_argument("--appname",
                          action="store",
                          type=str,
                          dest="app",
                          default="org.mozilla.geckoview.test",
                          help="Test package name.")
        self.add_argument("--adbpath",
                          action="store",
                          type=str,
                          dest="adbPath",
                          default=None,
                          help="Path to adb executable.")
        self.add_argument("--deviceSerial",
                          action="store",
                          type=str,
                          dest="deviceSerial",
                          help="adb serial number of remote device.")
        self.add_argument("--remoteTestRoot",
                          action="store",
                          type=str,
                          dest="remoteTestRoot",
                          help="Remote directory to use as test root "
                               "(eg. /mnt/sdcard/tests or /data/local/tests).")
        self.add_argument("--disable-e10s",
                          action="store_false",
                          dest="e10s",
                          default=True,
                          help="Disable multiprocess mode in test app.")
        self.add_argument("--max-time",
                          action="store",
                          type=str,
                          dest="max_time",
                          default="2400",
                          help="Max time in seconds to wait for tests (default 2400s).")
        self.add_argument("--runner",
                          action="store",
                          type=str,
                          dest="runner",
                          default="android.support.test.runner.AndroidJUnitRunner",
                          help="Test runner name.")
        self.add_argument("--symbols-path",
                          action="store",
                          type=str,
                          dest="symbolsPath",
                          default=None,
                          help="Path to directory containing breakpad symbols, "
                               "or the URL of a zip file containing symbols.")
        self.add_argument("--utility-path",
                          action="store",
                          type=str,
                          dest="utilityPath",
                          default=None,
                          help="Path to directory containing host utility programs.")
        self.add_argument("--total-chunks",
                          action="store",
                          type=int,
                          dest="totalChunks",
                          default=None,
                          help="Total number of chunks to split tests into.")
        self.add_argument("--this-chunk",
                          action="store",
                          type=int,
                          dest="thisChunk",
                          default=None,
                          help="If running tests by chunks, the chunk number to run.")
        # Additional options for server.
        self.add_argument("--certificate-path",
                          action="store",
                          type=str,
                          dest="certPath",
                          default=None,
                          help="Path to directory containing certificate store."),
        self.add_argument("--http-port",
                          action="store",
                          type=str,
                          dest="httpPort",
                          default=DEFAULT_PORTS['http'],
                          help="Port of the web server for http traffic.")
        self.add_argument("--remote-webserver",
                          action="store",
                          type=str,
                          dest="remoteWebServer",
                          help="IP address of the webserver.")
        self.add_argument("--ssl-port",
                          action="store",
                          type=str,
                          dest="sslPort",
                          default=DEFAULT_PORTS['https'],
                          help="Port of the web server for https traffic.")
        # Remaining arguments are test filters.
        self.add_argument("test_filters",
                          nargs="*",
                          help="Test filter(s): class and/or method names of test(s) to run.")

        mozlog.commandline.add_logging_group(self)


def run_test_harness(parser, options):
    if hasattr(options, 'log'):
        log = options.log
    else:
        log = mozlog.commandline.setup_logging("runjunit", options,
                                               {"tbpl": sys.stdout})
    runner = JUnitTestRunner(log, options)
    result = -1
    try:
        result = runner.run_tests(options.test_filters)
    except KeyboardInterrupt:
        log.info("runjunit.py | Received keyboard interrupt")
        result = -1
    except Exception:
        traceback.print_exc()
        log.error(
            "runjunit.py | Received unexpected exception while running tests")
        result = 1
    finally:
        runner.cleanup()
    return result


def main(args=sys.argv[1:]):
    parser = JunitArgumentParser()
    options = parser.parse_args()
    return run_test_harness(parser, options)


if __name__ == "__main__":
    sys.exit(main())