From ad7b94108f17ffd96e8b26fe60034241dc4c12f1 Mon Sep 17 00:00:00 2001 From: "nileshagrawal@chromium.org" Date: Sat, 13 Apr 2013 02:39:54 +0000 Subject: [PATCH] Android: Make number of test retries configurable. BUG=230113 This also reduces the default number of retries from 3 to 2. So we will now run a test 3 times before giving up (instead of 4). Review URL: https://chromiumcodereview.appspot.com/13956018 git-svn-id: http://src.chromium.org/svn/trunk/src/build@194078 4ff67af0-8c30-449e-8e8b-ad334ec8d88c --- android/pylib/base/shard.py | 22 ++++++++++++++-------- android/pylib/browsertests/dispatch.py | 3 ++- android/pylib/gtest/dispatch.py | 3 ++- android/pylib/instrumentation/dispatch.py | 3 ++- android/pylib/utils/test_options_parser.py | 7 ++++--- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/android/pylib/base/shard.py b/android/pylib/base/shard.py index 61820eff0..e95355010 100644 --- a/android/pylib/base/shard.py +++ b/android/pylib/base/shard.py @@ -116,7 +116,8 @@ class _TestCollection(object): yield r -def _RunTestsFromQueue(runner, test_collection, out_results, watcher): +def _RunTestsFromQueue(runner, test_collection, out_results, watcher, + num_retries): """Runs tests from the test_collection until empty using the given runner. Adds TestRunResults objects to the out_results list and may add tests to the @@ -127,6 +128,7 @@ def _RunTestsFromQueue(runner, test_collection, out_results, watcher): test_collection: A _TestCollection from which to get _Test objects to run. out_results: A list to add TestRunResults to. watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout. + num_retries: Number of retries for a test. """ for test in test_collection: watcher.Reset() @@ -138,7 +140,7 @@ def _RunTestsFromQueue(runner, test_collection, out_results, watcher): raise android_commands.errors.DeviceUnresponsiveError(msg) result, retry = runner.RunTest(test.test) test.tries += 1 - if retry and test.tries <= 3: + if retry and test.tries <= num_retries: # Retry non-passing results, only record passing results. pass_results = base_test_result.TestRunResults() pass_results.AddResults(result.GetPass()) @@ -187,12 +189,13 @@ def _SetUp(runner_factory, device, out_runners, threadsafe_counter): logging.warning('Failed to create shard for %s: [%s]', device, e) -def _RunAllTests(runners, tests, timeout=None): +def _RunAllTests(runners, tests, num_retries, timeout=None): """Run all tests using the given TestRunners. Args: runners: a list of TestRunner objects. tests: a list of Tests to run using the given TestRunners. + num_retries: number of retries for a test. timeout: watchdog timeout in seconds, defaults to the default timeout. Returns: @@ -204,9 +207,10 @@ def _RunAllTests(runners, tests, timeout=None): results = [] watcher = watchdog_timer.WatchdogTimer(timeout) workers = reraiser_thread.ReraiserThreadGroup( - [reraiser_thread.ReraiserThread(_RunTestsFromQueue, - [r, tests_collection, results, watcher], - name=r.device[-4:]) + [reraiser_thread.ReraiserThread( + _RunTestsFromQueue, + [r, tests_collection, results, watcher, num_retries], + name=r.device[-4:]) for r in runners]) workers.StartAll() workers.JoinAll(watcher) @@ -259,7 +263,8 @@ def _TearDownRunners(runners, timeout=None): def ShardAndRunTests(runner_factory, devices, tests, build_type='Debug', test_timeout=DEFAULT_TIMEOUT, - setup_timeout=DEFAULT_TIMEOUT): + setup_timeout=DEFAULT_TIMEOUT, + num_retries=2): """Run all tests on attached devices, retrying tests that don't pass. Args: @@ -272,6 +277,7 @@ def ShardAndRunTests(runner_factory, devices, tests, build_type='Debug', default timeout. setup_timeout: watchdog timeout in seconds for creating and cleaning up test runners, defaults to the default timeout. + num_retries: number of retries for a test. Returns: A base_test_result.TestRunResults object. @@ -280,7 +286,7 @@ def ShardAndRunTests(runner_factory, devices, tests, build_type='Debug', forwarder.Forwarder.KillHost(build_type) runners = _CreateRunners(runner_factory, devices, setup_timeout) try: - return _RunAllTests(runners, tests, test_timeout) + return _RunAllTests(runners, tests, num_retries, test_timeout) finally: try: _TearDownRunners(runners, setup_timeout) diff --git a/android/pylib/browsertests/dispatch.py b/android/pylib/browsertests/dispatch.py index de6411a98..c598d6447 100644 --- a/android/pylib/browsertests/dispatch.py +++ b/android/pylib/browsertests/dispatch.py @@ -68,7 +68,8 @@ def Dispatch(options): test_results = shard.ShardAndRunTests(RunnerFactory, attached_devices, all_tests, options.build_type, setup_timeout=setup_timeout, - test_timeout=None) + test_timeout=None, + num_retries=options.num_retries) report_results.LogFull( results=test_results, test_type='Unit test', diff --git a/android/pylib/gtest/dispatch.py b/android/pylib/gtest/dispatch.py index 8fc985427..ca36345c7 100644 --- a/android/pylib/gtest/dispatch.py +++ b/android/pylib/gtest/dispatch.py @@ -165,7 +165,8 @@ def _RunATestSuite(options, suite_name): # Run tests. test_results = shard.ShardAndRunTests(RunnerFactory, attached_devices, tests, - options.build_type, test_timeout=None) + options.build_type, test_timeout=None, + num_retries=options.num_retries) report_results.LogFull( results=test_results, diff --git a/android/pylib/instrumentation/dispatch.py b/android/pylib/instrumentation/dispatch.py index ff4e367ab..f9ba21793 100644 --- a/android/pylib/instrumentation/dispatch.py +++ b/android/pylib/instrumentation/dispatch.py @@ -55,4 +55,5 @@ def Dispatch(options): options, device, shard_index, test_pkg, []) return shard.ShardAndRunTests(TestRunnerFactory, attached_devices, tests, - options.build_type) + options.build_type, + num_retries=options.num_retries) diff --git a/android/pylib/utils/test_options_parser.py b/android/pylib/utils/test_options_parser.py index 4ddc32f00..b00cfb09b 100644 --- a/android/pylib/utils/test_options_parser.py +++ b/android/pylib/utils/test_options_parser.py @@ -64,6 +64,10 @@ def AddTestRunnerOptions(option_parser, default_timeout=60): option_parser.add_option('-c', dest='cleanup_test_files', help='Cleanup test files on the device after run', action='store_true') + option_parser.add_option('--num_retries', dest='num_retries', type='int', + default=2, + help='Number of retries for a test before ' + 'giving up.') option_parser.add_option('-v', '--verbose', dest='verbose_count', @@ -117,9 +121,6 @@ def AddGTestOptions(option_parser): help='Use Xvfb around tests (ignored if not Linux).') option_parser.add_option('--webkit', action='store_true', help='Run the tests from a WebKit checkout.') - option_parser.add_option('--repeat', dest='repeat', type='int', - default=2, - help='Repeat count on test timeout.') option_parser.add_option('--exit_code', action='store_true', help='If set, the exit code will be total number ' 'of failures.')