Android: Make number of test retries configurable.

BUG=230113

This also reduces the default number of retries from 3 to 2.
So we will now run a test 3 times before giving up (instead of 4).

Review URL: https://chromiumcodereview.appspot.com/13956018

git-svn-id: http://src.chromium.org/svn/trunk/src/build@194078 4ff67af0-8c30-449e-8e8b-ad334ec8d88c

android/pylib/base/shard.py

@@ -116,7 +116,8 @@ class _TestCollection(object):
       yield r
 
 
-def _RunTestsFromQueue(runner, test_collection, out_results, watcher):
+def _RunTestsFromQueue(runner, test_collection, out_results, watcher,
+    num_retries):
   """Runs tests from the test_collection until empty using the given runner.
 
   Adds TestRunResults objects to the out_results list and may add tests to the
@@ -127,6 +128,7 @@ def _RunTestsFromQueue(runner, test_collection, out_results, watcher):
     test_collection: A _TestCollection from which to get _Test objects to run.
     out_results: A list to add TestRunResults to.
     watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout.
+    num_retries: Number of retries for a test.
   """
   for test in test_collection:
     watcher.Reset()
@@ -138,7 +140,7 @@ def _RunTestsFromQueue(runner, test_collection, out_results, watcher):
         raise android_commands.errors.DeviceUnresponsiveError(msg)
       result, retry = runner.RunTest(test.test)
       test.tries += 1
-      if retry and test.tries <= 3:
+      if retry and test.tries <= num_retries:
         # Retry non-passing results, only record passing results.
         pass_results = base_test_result.TestRunResults()
         pass_results.AddResults(result.GetPass())
@@ -187,12 +189,13 @@ def _SetUp(runner_factory, device, out_runners, threadsafe_counter):
     logging.warning('Failed to create shard for %s: [%s]', device, e)
 
 
-def _RunAllTests(runners, tests, timeout=None):
+def _RunAllTests(runners, tests, num_retries, timeout=None):
   """Run all tests using the given TestRunners.
 
   Args:
     runners: a list of TestRunner objects.
     tests: a list of Tests to run using the given TestRunners.
+    num_retries: number of retries for a test.
     timeout: watchdog timeout in seconds, defaults to the default timeout.
 
   Returns:
@@ -204,9 +207,10 @@ def _RunAllTests(runners, tests, timeout=None):
   results = []
   watcher = watchdog_timer.WatchdogTimer(timeout)
   workers = reraiser_thread.ReraiserThreadGroup(
-      [reraiser_thread.ReraiserThread(_RunTestsFromQueue,
-                                      [r, tests_collection, results, watcher],
-                                      name=r.device[-4:])
+      [reraiser_thread.ReraiserThread(
+          _RunTestsFromQueue,
+          [r, tests_collection, results, watcher, num_retries],
+          name=r.device[-4:])
        for r in runners])
   workers.StartAll()
   workers.JoinAll(watcher)
@@ -259,7 +263,8 @@ def _TearDownRunners(runners, timeout=None):
 
 def ShardAndRunTests(runner_factory, devices, tests, build_type='Debug',
                      test_timeout=DEFAULT_TIMEOUT,
-                     setup_timeout=DEFAULT_TIMEOUT):
+                     setup_timeout=DEFAULT_TIMEOUT,
+                     num_retries=2):
   """Run all tests on attached devices, retrying tests that don't pass.
 
   Args:
@@ -272,6 +277,7 @@ def ShardAndRunTests(runner_factory, devices, tests, build_type='Debug',
       default timeout.
     setup_timeout: watchdog timeout in seconds for creating and cleaning up
       test runners, defaults to the default timeout.
+    num_retries: number of retries for a test.
 
   Returns:
     A base_test_result.TestRunResults object.
@@ -280,7 +286,7 @@ def ShardAndRunTests(runner_factory, devices, tests, build_type='Debug',
   forwarder.Forwarder.KillHost(build_type)
   runners = _CreateRunners(runner_factory, devices, setup_timeout)
   try:
-    return _RunAllTests(runners, tests, test_timeout)
+    return _RunAllTests(runners, tests, num_retries, test_timeout)
   finally:
     try:
       _TearDownRunners(runners, setup_timeout)

android/pylib/browsertests/dispatch.py

@@ -68,7 +68,8 @@ def Dispatch(options):
   test_results = shard.ShardAndRunTests(RunnerFactory, attached_devices,
                                         all_tests, options.build_type,
                                         setup_timeout=setup_timeout,
-                                        test_timeout=None)
+                                        test_timeout=None,
+                                        num_retries=options.num_retries)
   report_results.LogFull(
       results=test_results,
       test_type='Unit test',

android/pylib/gtest/dispatch.py

@@ -165,7 +165,8 @@ def _RunATestSuite(options, suite_name):
 
   # Run tests.
   test_results = shard.ShardAndRunTests(RunnerFactory, attached_devices, tests,
-                                        options.build_type, test_timeout=None)
+                                        options.build_type, test_timeout=None,
+                                        num_retries=options.num_retries)
 
   report_results.LogFull(
       results=test_results,

android/pylib/instrumentation/dispatch.py

@@ -55,4 +55,5 @@ def Dispatch(options):
         options, device, shard_index, test_pkg, [])
 
   return shard.ShardAndRunTests(TestRunnerFactory, attached_devices, tests,
-                                options.build_type)
+                                options.build_type,
+                                num_retries=options.num_retries)

android/pylib/utils/test_options_parser.py

@@ -64,6 +64,10 @@ def AddTestRunnerOptions(option_parser, default_timeout=60):
   option_parser.add_option('-c', dest='cleanup_test_files',
                            help='Cleanup test files on the device after run',
                            action='store_true')
+  option_parser.add_option('--num_retries', dest='num_retries', type='int',
+                           default=2,
+                           help='Number of retries for a test before '
+                           'giving up.')
   option_parser.add_option('-v',
                            '--verbose',
                            dest='verbose_count',
@@ -117,9 +121,6 @@ def AddGTestOptions(option_parser):
                            help='Use Xvfb around tests (ignored if not Linux).')
   option_parser.add_option('--webkit', action='store_true',
                            help='Run the tests from a WebKit checkout.')
-  option_parser.add_option('--repeat', dest='repeat', type='int',
-                           default=2,
-                           help='Repeat count on test timeout.')
   option_parser.add_option('--exit_code', action='store_true',
                            help='If set, the exit code will be total number '
                            'of failures.')