From 40a653819fae62f716e57fce2901ec0a248cabb3 Mon Sep 17 00:00:00 2001 From: "gkanwar@chromium.org" Date: Thu, 8 Aug 2013 01:08:14 +0000 Subject: [PATCH] Converts monkey test to its own test type NOTRY=True BUG=223650, 263479 Review URL: https://chromiumcodereview.appspot.com/22617003 git-svn-id: http://src.chromium.org/svn/trunk/src/build@216343 4ff67af0-8c30-449e-8e8b-ad334ec8d88c --- android/pylib/monkey/__init__.py | 0 android/pylib/monkey/setup.py | 27 +++++ android/pylib/monkey/test_options.py | 18 +++ android/pylib/monkey/test_runner.py | 77 ++++++++++++ android/run_monkey_test.py | 170 --------------------------- android/test_runner.py | 87 +++++++++++++- 6 files changed, 205 insertions(+), 174 deletions(-) create mode 100644 android/pylib/monkey/__init__.py create mode 100644 android/pylib/monkey/setup.py create mode 100644 android/pylib/monkey/test_options.py create mode 100644 android/pylib/monkey/test_runner.py delete mode 100755 android/run_monkey_test.py diff --git a/android/pylib/monkey/__init__.py b/android/pylib/monkey/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/android/pylib/monkey/setup.py b/android/pylib/monkey/setup.py new file mode 100644 index 000000000..5735d2e95 --- /dev/null +++ b/android/pylib/monkey/setup.py @@ -0,0 +1,27 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Generates test runner factory and tests for monkey tests.""" + +import test_runner + + +def Setup(test_options): + """Create and return the test runner factory and tests. + + Args: + test_options: A MonkeyOptions object. + + Returns: + A tuple of (TestRunnerFactory, tests). + """ + # Token to replicate across devices as the "test". The TestRunner does all of + # the work to run the test. + tests = ['MonkeyTest'] + + def TestRunnerFactory(device, shard_index): + return test_runner.TestRunner( + test_options, device, shard_index) + + return (TestRunnerFactory, tests) diff --git a/android/pylib/monkey/test_options.py b/android/pylib/monkey/test_options.py new file mode 100644 index 000000000..6b095f3ec --- /dev/null +++ b/android/pylib/monkey/test_options.py @@ -0,0 +1,18 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Defines the MonkeyOptions named tuple.""" + +import collections + +MonkeyOptions = collections.namedtuple('MonkeyOptions', [ + 'build_type', + 'verbose_count', + 'package_name', + 'activity_name', + 'event_count', + 'category', + 'throttle', + 'seed', + 'extra_args']) diff --git a/android/pylib/monkey/test_runner.py b/android/pylib/monkey/test_runner.py new file mode 100644 index 000000000..99bc2e681 --- /dev/null +++ b/android/pylib/monkey/test_runner.py @@ -0,0 +1,77 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Runs a monkey test on a single device.""" + +import random + +from pylib.base import base_test_result +from pylib.base import base_test_runner + + +class TestRunner(base_test_runner.BaseTestRunner): + """A TestRunner instance runs a monkey test on a single device.""" + + def __init__(self, test_options, device, shard_index): + super(TestRunner, self).__init__(device, None, test_options.build_type) + self.options = test_options + + def _LaunchMonkeyTest(self): + """Runs monkey test for a given package. + + Returns: + Output from the monkey command on the device. + """ + + timeout_ms = self.options.event_count * self.options.throttle * 1.5 + + cmd = ['monkey', + '-p %s' % self.options.package_name, + ' '.join(['-c %s' % c for c in self.options.category]), + '--throttle %d' % self.options.throttle, + '-s %d' % (self.options.seed or random.randint(1, 100)), + '-v ' * self.options.verbose_count, + '--monitor-native-crashes', + '--kill-process-after-error', + self.options.extra_args, + '%d' % self.options.event_count] + return self.adb.RunShellCommand(' '.join(cmd), timeout_time=timeout_ms) + + def RunTest(self, test_name): + """Run a Monkey test on the device. + + Args: + test_name: String to use for logging the test result. + + Returns: + A tuple of (TestRunResults, retry). + """ + self.adb.StartActivity(self.options.package_name, + self.options.activity_name, + wait_for_completion=True, + action='android.intent.action.MAIN', + force_stop=True) + + # Chrome crashes are not always caught by Monkey test runner. + # Verify Chrome has the same PID before and after the test. + before_pids = self.adb.ExtractPid(self.options.package_name) + + # Run the test. + output = '' + if before_pids: + output = '\n'.join(self._LaunchMonkeyTest()) + after_pids = self.adb.ExtractPid(self.options.package_name) + + crashed = (not before_pids or not after_pids + or after_pids[0] != before_pids[0]) + + results = base_test_result.TestRunResults() + if 'Monkey finished' in output and not crashed: + result = base_test_result.BaseTestResult( + test_name, base_test_result.ResultType.PASS, log=output) + else: + result = base_test_result.BaseTestResult( + test_name, base_test_result.ResultType.FAIL, log=output) + results.AddResult(result) + return results, False diff --git a/android/run_monkey_test.py b/android/run_monkey_test.py deleted file mode 100755 index 4957ac6ec..000000000 --- a/android/run_monkey_test.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Runs the Monkey tests on one or more devices.""" -import logging -import optparse -import random -import sys - -from pylib.base import base_test_result -from pylib.base import test_dispatcher -from pylib.host_driven import test_case -from pylib.host_driven import test_runner -from pylib.utils import report_results -from pylib.utils import test_options_parser - - -class MonkeyTest(test_case.HostDrivenTestCase): - def __init__(self, test_name, package_name, activity_name, category, seed, - throttle, event_count, verbosity, extra_args): - """Create a MonkeyTest object. - - Args: - test_name: Name of the method to run for this test object. - package_name: Allowed package. - activity_name: Name of the activity to start. - category: A list of allowed categories. - seed: Seed value for pseduo-random generator. Same seed value - generates the same sequence of events. Seed is randomized by default. - throttle: Delay between events (ms). - event_count: Number of events to generate. - verbosity: Verbosity level [0-3]. - extra_args: A string of other args to pass to the command verbatim. - """ - super(MonkeyTest, self).__init__(test_name) - self.package_name = package_name - self.activity_name = activity_name - self.category = category - self.seed = seed or random.randint(1, 100) - self.throttle = throttle - self.event_count = event_count - self.verbosity = verbosity - self.extra_args = extra_args - - def testMonkey(self): - # Launch and wait for Chrome to launch. - self.adb.StartActivity(self.package_name, - self.activity_name, - wait_for_completion=True, - action='android.intent.action.MAIN', - force_stop=True) - - # Chrome crashes are not always caught by Monkey test runner. - # Verify Chrome has the same PID before and after the test. - before_pids = self.adb.ExtractPid(self.package_name) - - # Run the test. - output = '' - if before_pids: - output = '\n'.join(self._LaunchMonkeyTest()) - after_pids = self.adb.ExtractPid(self.package_name) - - crashed = (not before_pids or not after_pids - or after_pids[0] != before_pids[0]) - - results = base_test_result.TestRunResults() - if 'Monkey finished' in output and not crashed: - result = base_test_result.BaseTestResult( - self.tagged_name, base_test_result.ResultType.PASS, log=output) - else: - result = base_test_result.BaseTestResult( - self.tagged_name, base_test_result.ResultType.FAIL, log=output) - results.AddResult(result) - return results - - def _LaunchMonkeyTest(self): - """Runs monkey test for a given package. - - Returns: - Output from the monkey command on the device. - """ - - timeout_ms = self.event_count * self.throttle * 1.5 - - cmd = ['monkey', - '-p %s' % self.package_name, - ' '.join(['-c %s' % c for c in self.category]), - '--throttle %d' % self.throttle, - '-s %d' % self.seed, - '-v ' * self.verbosity, - '--monitor-native-crashes', - '--kill-process-after-error', - self.extra_args, - '%d' % self.event_count] - return self.adb.RunShellCommand(' '.join(cmd), timeout_time=timeout_ms) - - -def RunMonkeyTests(options): - """Runs the Monkey tests, replicating it if there multiple devices.""" - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - - # Actually run the tests. - logging.debug('Running monkey tests.') - available_tests = [ - MonkeyTest('testMonkey', options.package_name, options.activity_name, - category=options.category, seed=options.seed, - throttle=options.throttle, event_count=options.event_count, - verbosity=options.verbosity, extra_args=options.extra_args)] - - def TestRunnerFactory(device, shard_index): - return test_runner.HostDrivenTestRunner( - device, shard_index, '', options.build_type, False, False) - - results, exit_code = test_dispatcher.RunTests( - available_tests, TestRunnerFactory, False, None, shard=False, - build_type=options.build_type, num_retries=0) - - report_results.LogFull( - results=results, - test_type='Monkey', - test_package='Monkey', - build_type=options.build_type) - - return exit_code - - -def main(): - desc = 'Run the Monkey tests on 1 or more devices.' - parser = optparse.OptionParser(description=desc) - test_options_parser.AddBuildTypeOption(parser) - parser.add_option('--package-name', help='Allowed package.') - parser.add_option('--activity-name', - default='com.google.android.apps.chrome.Main', - help='Name of the activity to start [default: %default].') - parser.add_option('--category', default='', - help='A list of allowed categories [default: %default].') - parser.add_option('--throttle', default=100, type='int', - help='Delay between events (ms) [default: %default]. ') - parser.add_option('--seed', type='int', - help=('Seed value for pseudo-random generator. Same seed ' - 'value generates the same sequence of events. Seed ' - 'is randomized by default.')) - parser.add_option('--event-count', default=10000, type='int', - help='Number of events to generate [default: %default].') - parser.add_option('--verbosity', default=1, type='int', - help='Verbosity level [0-3] [default: %default].') - parser.add_option('--extra-args', default='', - help=('String of other args to pass to the command verbatim' - ' [default: "%default"].')) - (options, args) = parser.parse_args() - - if args: - parser.print_help(sys.stderr) - parser.error('Unknown arguments: %s' % args) - - if not options.package_name: - parser.print_help(sys.stderr) - parser.error('Missing package name') - - if options.category: - options.category = options.category.split(',') - - RunMonkeyTests(options) - - -if __name__ == '__main__': - main() diff --git a/android/test_runner.py b/android/test_runner.py index b8de366a4..865572081 100755 --- a/android/test_runner.py +++ b/android/test_runner.py @@ -26,6 +26,8 @@ from pylib.gtest import test_options as gtest_test_options from pylib.host_driven import setup as host_driven_setup from pylib.instrumentation import setup as instrumentation_setup from pylib.instrumentation import test_options as instrumentation_test_options +from pylib.monkey import setup as monkey_setup +from pylib.monkey import test_options as monkey_test_options from pylib.uiautomator import setup as uiautomator_setup from pylib.uiautomator import test_options as uiautomator_test_options from pylib.utils import report_results @@ -315,7 +317,7 @@ def ProcessUIAutomatorOptions(options, error_func): Returns: A UIAutomatorOptions named tuple which contains all options relevant to - instrumentation tests. + uiautomator tests. """ ProcessJavaTestOptions(options, error_func) @@ -353,6 +355,63 @@ def ProcessUIAutomatorOptions(options, error_func): options.package_name) +def AddMonkeyTestOptions(option_parser): + """Adds monkey test options to |option_parser|.""" + option_parser.add_option('--package-name', help='Allowed package.') + option_parser.add_option( + '--activity-name', default='com.google.android.apps.chrome.Main', + help='Name of the activity to start [default: %default].') + option_parser.add_option( + '--event-count', default=10000, type='int', + help='Number of events to generate [default: %default].') + option_parser.add_option( + '--category', default='', + help='A list of allowed categories [default: %default].') + option_parser.add_option( + '--throttle', default=100, type='int', + help='Delay between events (ms) [default: %default]. ') + option_parser.add_option( + '--seed', type='int', + help=('Seed value for pseudo-random generator. Same seed value generates ' + 'the same sequence of events. Seed is randomized by default.')) + option_parser.add_option( + '--extra-args', default='', + help=('String of other args to pass to the command verbatim ' + '[default: "%default"].')) + + AddCommonOptions(option_parser) + + +def ProcessMonkeyTestOptions(options, error_func): + """Processes all monkey test options. + + Args: + options: optparse.Options object. + error_func: Function to call with the error message in case of an error. + + Returns: + A MonkeyOptions named tuple which contains all options relevant to + monkey tests. + """ + if not options.package_name: + error_func('Package name is required.') + + category = options.category + if category: + category = options.category.split(',') + + return monkey_test_options.MonkeyOptions( + options.build_type, + options.verbose_count, + options.package_name, + options.activity_name, + options.event_count, + category, + options.throttle, + options.seed, + options.extra_args) + + def _RunGTests(options, error_func): """Subcommand of RunTestsCommands which runs gtests.""" ProcessGTestOptions(options) @@ -450,9 +509,6 @@ def _RunUIAutomatorTests(options, error_func): """Subcommand of RunTestsCommands which runs uiautomator tests.""" uiautomator_options = ProcessUIAutomatorOptions(options, error_func) - results = base_test_result.TestRunResults() - exit_code = 0 - runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) results, exit_code = test_dispatcher.RunTests( @@ -473,6 +529,25 @@ def _RunUIAutomatorTests(options, error_func): return exit_code +def _RunMonkeyTests(options, error_func): + """Subcommand of RunTestsCommands which runs monkey tests.""" + monkey_options = ProcessMonkeyTestOptions(options, error_func) + + runner_factory, tests = monkey_setup.Setup(monkey_options) + + results, exit_code = test_dispatcher.RunTests( + tests, runner_factory, False, None, shard=False) + + report_results.LogFull( + results=results, + test_type='Monkey', + test_package='Monkey', + build_type=options.build_type) + + return exit_code + + + def RunTestsCommand(command, options, args, option_parser): """Checks test type and dispatches to the appropriate function. @@ -504,6 +579,8 @@ def RunTestsCommand(command, options, args, option_parser): return _RunInstrumentationTests(options, option_parser.error) elif command == 'uiautomator': return _RunUIAutomatorTests(options, option_parser.error) + elif command == 'monkey': + return _RunMonkeyTests(options, option_parser.error) else: raise Exception('Unknown test type.') @@ -560,6 +637,8 @@ VALID_COMMANDS = { AddInstrumentationTestOptions, RunTestsCommand), 'uiautomator': CommandFunctionTuple( AddUIAutomatorTestOptions, RunTestsCommand), + 'monkey': CommandFunctionTuple( + AddMonkeyTestOptions, RunTestsCommand), 'help': CommandFunctionTuple(lambda option_parser: None, HelpCommand) }