Switch monkey test to platform mode.
BUG=663127 Review-Url: https://codereview.chromium.org/2505713002 Cr-Original-Commit-Position: refs/heads/master@{#432695} Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src Cr-Mirrored-Commit: a7fb522b96eadb5e710f8252d95ad242f2858788
This commit is contained in:
mikecase
Commit bot
Родитель
7fb1aa5fc8
Коммит
12779f6b6e
|
|
@ -5,6 +5,7 @@
|
|||
from pylib.gtest import gtest_test_instance
|
||||
from pylib.instrumentation import instrumentation_test_instance
|
||||
from pylib.junit import junit_test_instance
|
||||
from pylib.monkey import monkey_test_instance
|
||||
from pylib.perf import perf_test_instance
|
||||
from pylib.utils import isolator
|
||||
|
||||
|
|
@ -19,6 +20,8 @@ def CreateTestInstance(args, error_func):
|
|||
args, isolator.Isolator(), error_func)
|
||||
elif args.command == 'junit':
|
||||
return junit_test_instance.JunitTestInstance(args, error_func)
|
||||
elif args.command == 'monkey':
|
||||
return monkey_test_instance.MonkeyTestInstance(args, error_func)
|
||||
elif args.command == 'perf':
|
||||
return perf_test_instance.PerfTestInstance(args, error_func)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,9 +5,11 @@
|
|||
from pylib.gtest import gtest_test_instance
|
||||
from pylib.instrumentation import instrumentation_test_instance
|
||||
from pylib.junit import junit_test_instance
|
||||
from pylib.monkey import monkey_test_instance
|
||||
from pylib.local.device import local_device_environment
|
||||
from pylib.local.device import local_device_gtest_run
|
||||
from pylib.local.device import local_device_instrumentation_test_run
|
||||
from pylib.local.device import local_device_monkey_test_run
|
||||
from pylib.local.device import local_device_perf_test_run
|
||||
from pylib.local.machine import local_machine_environment
|
||||
from pylib.local.machine import local_machine_junit_test_run
|
||||
|
|
@ -33,6 +35,9 @@ def CreateTestRun(args, env, test_instance, error_func):
|
|||
instrumentation_test_instance.InstrumentationTestInstance):
|
||||
return (local_device_instrumentation_test_run
|
||||
.LocalDeviceInstrumentationTestRun(env, test_instance))
|
||||
if isinstance(test_instance, monkey_test_instance.MonkeyTestInstance):
|
||||
return (local_device_monkey_test_run
|
||||
.LocalDeviceMonkeyTestRun(env, test_instance))
|
||||
if isinstance(test_instance,
|
||||
perf_test_instance.PerfTestInstance):
|
||||
return _CreatePerfTestRun(args, env, test_instance)
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ class LocalDeviceInstrumentationTestRun(
|
|||
super(LocalDeviceInstrumentationTestRun, self).__init__(env, test_instance)
|
||||
self._flag_changers = {}
|
||||
|
||||
#override
|
||||
def TestPackage(self):
|
||||
return self._test_instance.suite
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,126 @@
|
|||
# Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
import logging
|
||||
|
||||
from devil.android import device_errors
|
||||
from devil.android.sdk import intent
|
||||
from pylib import constants
|
||||
from pylib.base import base_test_result
|
||||
from pylib.local.device import local_device_test_run
|
||||
|
||||
|
||||
_CHROME_PACKAGE = constants.PACKAGE_INFO['chrome'].package
|
||||
|
||||
class LocalDeviceMonkeyTestRun(local_device_test_run.LocalDeviceTestRun):
|
||||
def __init__(self, env, test_instance):
|
||||
super(LocalDeviceMonkeyTestRun, self).__init__(env, test_instance)
|
||||
|
||||
def TestPackage(self):
|
||||
return 'monkey'
|
||||
|
||||
#override
|
||||
def SetUp(self):
|
||||
pass
|
||||
|
||||
#override
|
||||
def _RunTest(self, device, test):
|
||||
device.ClearApplicationState(self._test_instance.package)
|
||||
|
||||
# Chrome crashes are not always caught by Monkey test runner.
|
||||
# Launch Chrome and verify Chrome has the same PID before and after
|
||||
# the test.
|
||||
device.StartActivity(
|
||||
intent.Intent(package=self._test_instance.package,
|
||||
activity=self._test_instance.activity,
|
||||
action='android.intent.action.MAIN'),
|
||||
blocking=True, force_stop=True)
|
||||
before_pids = device.GetPids(self._test_instance.package)
|
||||
|
||||
output = ''
|
||||
if before_pids:
|
||||
if len(before_pids.get(self._test_instance.package, [])) > 1:
|
||||
raise Exception(
|
||||
'At most one instance of process %s expected but found pids: '
|
||||
'%s' % (self._test_instance.package, before_pids))
|
||||
output = '\n'.join(self._LaunchMonkeyTest(device))
|
||||
after_pids = device.GetPids(self._test_instance.package)
|
||||
|
||||
crashed = True
|
||||
if not self._test_instance.package in before_pids:
|
||||
logging.error('Failed to start the process.')
|
||||
elif not self._test_instance.package in after_pids:
|
||||
logging.error('Process %s has died.',
|
||||
before_pids[self._test_instance.package])
|
||||
elif (before_pids[self._test_instance.package] !=
|
||||
after_pids[self._test_instance.package]):
|
||||
logging.error('Detected process restart %s -> %s',
|
||||
before_pids[self._test_instance.package],
|
||||
after_pids[self._test_instance.package])
|
||||
else:
|
||||
crashed = False
|
||||
|
||||
success_pattern = 'Events injected: %d' % self._test_instance.event_count
|
||||
if success_pattern in output and not crashed:
|
||||
result = base_test_result.BaseTestResult(
|
||||
test, base_test_result.ResultType.PASS, log=output)
|
||||
else:
|
||||
result = base_test_result.BaseTestResult(
|
||||
test, base_test_result.ResultType.FAIL, log=output)
|
||||
if 'chrome' in self._test_instance.package:
|
||||
logging.warning('Starting MinidumpUploadService...')
|
||||
# TODO(jbudorick): Update this after upstreaming.
|
||||
minidump_intent = intent.Intent(
|
||||
action='%s.crash.ACTION_FIND_ALL' % _CHROME_PACKAGE,
|
||||
package=self._test_instance.package,
|
||||
activity='%s.crash.MinidumpUploadService' % _CHROME_PACKAGE)
|
||||
try:
|
||||
device.RunShellCommand(
|
||||
['am', 'startservice'] + minidump_intent.am_args,
|
||||
as_root=True, check_return=True)
|
||||
except device_errors.CommandFailedError:
|
||||
logging.exception('Failed to start MinidumpUploadService')
|
||||
|
||||
return result
|
||||
|
||||
#override
|
||||
def TearDown(self):
|
||||
pass
|
||||
|
||||
#override
|
||||
def _CreateShards(self, tests):
|
||||
return tests
|
||||
|
||||
#override
|
||||
def _ShouldShard(self):
|
||||
# TODO(mikecase): Run Monkey test concurrently on each attached device.
|
||||
return False
|
||||
|
||||
#override
|
||||
def _GetTests(self):
|
||||
return ['MonkeyTest']
|
||||
|
||||
def _LaunchMonkeyTest(self, device):
|
||||
try:
|
||||
cmd = ['monkey',
|
||||
'-p', self._test_instance.package,
|
||||
'--throttle', str(self._test_instance.throttle),
|
||||
'-s', str(self._test_instance.seed),
|
||||
'--monitor-native-crashes',
|
||||
'--kill-process-after-error']
|
||||
for category in self._test_instance.categories:
|
||||
cmd.extend(['-c', category])
|
||||
for _ in range(self._test_instance.verbose_count):
|
||||
cmd.append('-v')
|
||||
cmd.append(str(self._test_instance.event_count))
|
||||
return device.RunShellCommand(
|
||||
cmd, timeout=self._test_instance.timeout)
|
||||
finally:
|
||||
try:
|
||||
# Kill the monkey test process on the device. If you manually
|
||||
# interupt the test run, this will prevent the monkey test from
|
||||
# continuing to run.
|
||||
device.KillAll('com.android.commands.monkey')
|
||||
except device_errors.CommandFailedError:
|
||||
pass
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
# Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
import random
|
||||
|
||||
from pylib import constants
|
||||
from pylib.base import test_instance
|
||||
|
||||
|
||||
_SINGLE_EVENT_TIMEOUT = 100 # Milliseconds
|
||||
|
||||
class MonkeyTestInstance(test_instance.TestInstance):
|
||||
|
||||
def __init__(self, args, _):
|
||||
super(MonkeyTestInstance, self).__init__()
|
||||
|
||||
self._categories = args.categories
|
||||
self._event_count = args.event_count
|
||||
self._seed = args.seed or random.randint(1, 100)
|
||||
self._throttle = args.throttle
|
||||
self._verbose_count = args.verbose_count
|
||||
|
||||
self._package = constants.PACKAGE_INFO[args.browser].package
|
||||
self._activity = constants.PACKAGE_INFO[args.browser].activity
|
||||
|
||||
self._timeout_ms = (self.event_count *
|
||||
(self.throttle + _SINGLE_EVENT_TIMEOUT))
|
||||
|
||||
#override
|
||||
def TestType(self):
|
||||
return 'monkey'
|
||||
|
||||
#override
|
||||
def SetUp(self):
|
||||
pass
|
||||
|
||||
#override
|
||||
def TearDown(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def activity(self):
|
||||
return self._activity
|
||||
|
||||
@property
|
||||
def categories(self):
|
||||
return self._categories
|
||||
|
||||
@property
|
||||
def event_count(self):
|
||||
return self._event_count
|
||||
|
||||
@property
|
||||
def package(self):
|
||||
return self._package
|
||||
|
||||
@property
|
||||
def seed(self):
|
||||
return self._seed
|
||||
|
||||
@property
|
||||
def throttle(self):
|
||||
return self._throttle
|
||||
|
||||
@property
|
||||
def timeout(self):
|
||||
return self._timeout_ms
|
||||
|
||||
@property
|
||||
def verbose_count(self):
|
||||
return self._verbose_count
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# Copyright 2013 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""Generates test runner factory and tests for monkey tests."""
|
||||
|
||||
from pylib.monkey import test_runner
|
||||
|
||||
|
||||
def Setup(test_options):
|
||||
"""Create and return the test runner factory and tests.
|
||||
|
||||
Args:
|
||||
test_options: A MonkeyOptions object.
|
||||
|
||||
Returns:
|
||||
A tuple of (TestRunnerFactory, tests).
|
||||
"""
|
||||
# Token to replicate across devices as the "test". The TestRunner does all of
|
||||
# the work to run the test.
|
||||
tests = ['MonkeyTest']
|
||||
|
||||
def TestRunnerFactory(device, shard_index):
|
||||
return test_runner.TestRunner(
|
||||
test_options, device, shard_index)
|
||||
|
||||
return (TestRunnerFactory, tests)
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
# Copyright 2013 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""Defines the MonkeyOptions named tuple."""
|
||||
|
||||
import collections
|
||||
|
||||
MonkeyOptions = collections.namedtuple('MonkeyOptions', [
|
||||
'verbose_count',
|
||||
'package',
|
||||
'event_count',
|
||||
'category',
|
||||
'throttle',
|
||||
'seed',
|
||||
'extra_args'])
|
||||
|
|
@ -1,110 +0,0 @@
|
|||
# Copyright 2013 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""Runs a monkey test on a single device."""
|
||||
|
||||
import logging
|
||||
import random
|
||||
|
||||
from devil.android import device_errors
|
||||
from devil.android.sdk import intent
|
||||
from pylib import constants
|
||||
from pylib.base import base_test_result
|
||||
from pylib.base import base_test_runner
|
||||
|
||||
_CHROME_PACKAGE = constants.PACKAGE_INFO['chrome'].package
|
||||
|
||||
class TestRunner(base_test_runner.BaseTestRunner):
|
||||
"""A TestRunner instance runs a monkey test on a single device."""
|
||||
|
||||
def __init__(self, test_options, device, _):
|
||||
super(TestRunner, self).__init__(device, None)
|
||||
self._options = test_options
|
||||
self._package = constants.PACKAGE_INFO[self._options.package].package
|
||||
self._activity = constants.PACKAGE_INFO[self._options.package].activity
|
||||
|
||||
def _LaunchMonkeyTest(self):
|
||||
"""Runs monkey test for a given package.
|
||||
|
||||
Returns:
|
||||
Output from the monkey command on the device.
|
||||
"""
|
||||
|
||||
timeout_ms = self._options.event_count * self._options.throttle * 1.5
|
||||
|
||||
cmd = ['monkey',
|
||||
'-p %s' % self._package,
|
||||
' '.join(['-c %s' % c for c in self._options.category]),
|
||||
'--throttle %d' % self._options.throttle,
|
||||
'-s %d' % (self._options.seed or random.randint(1, 100)),
|
||||
'-v ' * self._options.verbose_count,
|
||||
'--monitor-native-crashes',
|
||||
'--kill-process-after-error',
|
||||
self._options.extra_args,
|
||||
'%d' % self._options.event_count]
|
||||
return self.device.RunShellCommand(' '.join(cmd), timeout=timeout_ms)
|
||||
|
||||
def RunTest(self, test_name):
|
||||
"""Run a Monkey test on the device.
|
||||
|
||||
Args:
|
||||
test_name: String to use for logging the test result.
|
||||
|
||||
Returns:
|
||||
A tuple of (TestRunResults, retry).
|
||||
"""
|
||||
self.device.StartActivity(
|
||||
intent.Intent(package=self._package, activity=self._activity,
|
||||
action='android.intent.action.MAIN'),
|
||||
blocking=True, force_stop=True)
|
||||
|
||||
# Chrome crashes are not always caught by Monkey test runner.
|
||||
# Verify Chrome has the same PID before and after the test.
|
||||
before_pids = self.device.GetPids(self._package)
|
||||
|
||||
# Run the test.
|
||||
output = ''
|
||||
if before_pids:
|
||||
if len(before_pids.get(self._package, [])) > 1:
|
||||
raise Exception(
|
||||
'At most one instance of process %s expected but found pids: '
|
||||
'%s' % (self._package, before_pids))
|
||||
output = '\n'.join(self._LaunchMonkeyTest())
|
||||
after_pids = self.device.GetPids(self._package)
|
||||
|
||||
crashed = True
|
||||
if not self._package in before_pids:
|
||||
logging.error('Failed to start the process.')
|
||||
elif not self._package in after_pids:
|
||||
logging.error('Process %s has died.', before_pids[self._package])
|
||||
elif before_pids[self._package] != after_pids[self._package]:
|
||||
logging.error('Detected process restart %s -> %s',
|
||||
before_pids[self._package], after_pids[self._package])
|
||||
else:
|
||||
crashed = False
|
||||
|
||||
results = base_test_result.TestRunResults()
|
||||
success_pattern = 'Events injected: %d' % self._options.event_count
|
||||
if success_pattern in output and not crashed:
|
||||
result = base_test_result.BaseTestResult(
|
||||
test_name, base_test_result.ResultType.PASS, log=output)
|
||||
else:
|
||||
result = base_test_result.BaseTestResult(
|
||||
test_name, base_test_result.ResultType.FAIL, log=output)
|
||||
if 'chrome' in self._options.package:
|
||||
logging.warning('Starting MinidumpUploadService...')
|
||||
# TODO(jbudorick): Update this after upstreaming.
|
||||
minidump_intent = intent.Intent(
|
||||
action='%s.crash.ACTION_FIND_ALL' % _CHROME_PACKAGE,
|
||||
package=self._package,
|
||||
activity='%s.crash.MinidumpUploadService' % _CHROME_PACKAGE)
|
||||
try:
|
||||
self.device.RunShellCommand(
|
||||
['am', 'startservice'] + minidump_intent.am_args,
|
||||
as_root=True, check_return=True)
|
||||
except device_errors.CommandFailedError:
|
||||
logging.exception('Failed to start MinidumpUploadService')
|
||||
|
||||
results.AddResult(result)
|
||||
return results, False
|
||||
|
|
@ -36,8 +36,6 @@ from pylib.constants import host_paths
|
|||
from pylib.linker import setup as linker_setup
|
||||
from pylib.junit import setup as junit_setup
|
||||
from pylib.junit import test_dispatcher as junit_dispatcher
|
||||
from pylib.monkey import setup as monkey_setup
|
||||
from pylib.monkey import test_options as monkey_test_options
|
||||
from pylib.results import json_results
|
||||
from pylib.results import report_results
|
||||
|
||||
|
|
@ -426,53 +424,32 @@ def AddMonkeyTestOptions(parser):
|
|||
|
||||
group = parser.add_argument_group('Monkey Test Options')
|
||||
group.add_argument(
|
||||
'--package', required=True, choices=constants.PACKAGE_INFO.keys(),
|
||||
metavar='PACKAGE', help='Package under test.')
|
||||
'--browser', required=True, choices=constants.PACKAGE_INFO.keys(),
|
||||
metavar='BROWSER', help='Browser under test.')
|
||||
group.add_argument(
|
||||
'--event-count', default=10000, type=int,
|
||||
help='Number of events to generate (default: %(default)s).')
|
||||
group.add_argument(
|
||||
'--category', default='',
|
||||
help='A list of allowed categories.')
|
||||
'--category', nargs='*', dest='categories', default=[],
|
||||
help='A list of allowed categories. Monkey will only visit activities '
|
||||
'that are listed with one of the specified categories.')
|
||||
group.add_argument(
|
||||
'--throttle', default=100, type=int,
|
||||
help='Delay between events (ms) (default: %(default)s). ')
|
||||
group.add_argument(
|
||||
'--seed', type=int,
|
||||
help=('Seed value for pseudo-random generator. Same seed value generates '
|
||||
'the same sequence of events. Seed is randomized by default.'))
|
||||
help='Seed value for pseudo-random generator. Same seed value generates '
|
||||
'the same sequence of events. Seed is randomized by default.')
|
||||
group.add_argument(
|
||||
'--extra-args', default='',
|
||||
help=('String of other args to pass to the command verbatim.'))
|
||||
|
||||
'--repeat', dest='repeat', type=int, default=0,
|
||||
help='Number of times to repeat the specified set of tests.')
|
||||
group.add_argument(
|
||||
'--break-on-failure', '--break_on_failure',
|
||||
dest='break_on_failure', action='store_true',
|
||||
help='Whether to break on failure.')
|
||||
AddCommonOptions(parser)
|
||||
AddDeviceOptions(parser)
|
||||
|
||||
def ProcessMonkeyTestOptions(args):
|
||||
"""Processes all monkey test options.
|
||||
|
||||
Args:
|
||||
args: argparse.Namespace object.
|
||||
|
||||
Returns:
|
||||
A MonkeyOptions named tuple which contains all options relevant to
|
||||
monkey tests.
|
||||
"""
|
||||
# TODO(jbudorick): Handle this directly in argparse with nargs='+'
|
||||
category = args.category
|
||||
if category:
|
||||
category = args.category.split(',')
|
||||
|
||||
# TODO(jbudorick): Get rid of MonkeyOptions.
|
||||
return monkey_test_options.MonkeyOptions(
|
||||
args.verbose_count,
|
||||
args.package,
|
||||
args.event_count,
|
||||
category,
|
||||
args.throttle,
|
||||
args.seed,
|
||||
args.extra_args)
|
||||
|
||||
|
||||
def AddPerfTestOptions(parser):
|
||||
"""Adds perf test options to |parser|."""
|
||||
|
|
@ -608,27 +585,6 @@ def _RunJUnitTests(args):
|
|||
return exit_code
|
||||
|
||||
|
||||
def _RunMonkeyTests(args, devices):
|
||||
"""Subcommand of RunTestsCommands which runs monkey tests."""
|
||||
monkey_options = ProcessMonkeyTestOptions(args)
|
||||
|
||||
runner_factory, tests = monkey_setup.Setup(monkey_options)
|
||||
|
||||
results, exit_code = test_dispatcher.RunTests(
|
||||
tests, runner_factory, devices, shard=False, test_timeout=None,
|
||||
num_retries=args.num_retries)
|
||||
|
||||
report_results.LogFull(
|
||||
results=results,
|
||||
test_type='Monkey',
|
||||
test_package='Monkey')
|
||||
|
||||
if args.json_results_file:
|
||||
json_results.GenerateJsonResultsFile([results], args.json_results_file)
|
||||
|
||||
return exit_code
|
||||
|
||||
|
||||
def _RunPythonTests(args):
|
||||
"""Subcommand of RunTestsCommand which runs python unit tests."""
|
||||
suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
|
||||
|
|
@ -678,7 +634,7 @@ def _GetAttachedDevices(blacklist_file, test_device, enable_cache, num_retries):
|
|||
return sorted(attached_devices)
|
||||
|
||||
|
||||
_DEFAULT_PLATFORM_MODE_TESTS = ['gtest', 'instrumentation', 'perf']
|
||||
_DEFAULT_PLATFORM_MODE_TESTS = ['gtest', 'instrumentation', 'monkey', 'perf']
|
||||
|
||||
|
||||
def RunTestsCommand(args): # pylint: disable=too-many-return-statements
|
||||
|
|
@ -718,8 +674,6 @@ def RunTestsCommand(args): # pylint: disable=too-many-return-statements
|
|||
return _RunLinkerTests(args, get_devices())
|
||||
elif command == 'junit':
|
||||
return _RunJUnitTests(args)
|
||||
elif command == 'monkey':
|
||||
return _RunMonkeyTests(args, get_devices())
|
||||
elif command == 'python':
|
||||
return _RunPythonTests(args)
|
||||
else:
|
||||
|
|
@ -731,6 +685,7 @@ _SUPPORTED_IN_PLATFORM_MODE = [
|
|||
'gtest',
|
||||
'instrumentation',
|
||||
'junit',
|
||||
'monkey',
|
||||
'perf',
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ pylib/local/device/__init__.py
|
|||
pylib/local/device/local_device_environment.py
|
||||
pylib/local/device/local_device_gtest_run.py
|
||||
pylib/local/device/local_device_instrumentation_test_run.py
|
||||
pylib/local/device/local_device_monkey_test_run.py
|
||||
pylib/local/device/local_device_perf_test_run.py
|
||||
pylib/local/device/local_device_test_run.py
|
||||
pylib/local/local_test_server_spawner.py
|
||||
|
|
@ -113,9 +114,7 @@ pylib/local/machine/__init__.py
|
|||
pylib/local/machine/local_machine_environment.py
|
||||
pylib/local/machine/local_machine_junit_test_run.py
|
||||
pylib/monkey/__init__.py
|
||||
pylib/monkey/setup.py
|
||||
pylib/monkey/test_options.py
|
||||
pylib/monkey/test_runner.py
|
||||
pylib/monkey/monkey_test_instance.py
|
||||
pylib/perf/__init__.py
|
||||
pylib/perf/perf_test_instance.py
|
||||
pylib/results/__init__.py
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче