Added tombstones in instrumentation tests results.

If an instrumentation test crashes, tombstones were not captured.
In this cl, tombstones are captured and becomes part of the result.

This is a reland of a previous cl, http://crrev.com/2201833002#ps200001.
The reason for reverting the previous cl is that:
it breaks the stack_tool_for_tombstones step for many bots in
https://build.chromium.org/p/chromium.android.
In this reland, the edge case where there is no tombstone is fixed.

BUG=631213

Committed: https://crrev.com/d7dd97f2828447fb9af45ea119e703500b167cb8
Cr-Commit-Position: refs/heads/master@{#410929}

patch from issue 2201833002 at patchset 200001 (http://crrev.com/2201833002#ps200001)

Review-Url: https://codereview.chromium.org/2237553003
Cr-Original-Commit-Position: refs/heads/master@{#412089}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 02ff9ee5eb08ccdff4d4da042a53e2346a352130

android/pylib/instrumentation/instrumentation_test_instance.py

@@ -393,6 +393,9 @@ class InstrumentationTestInstance(test_instance.TestInstance):
     self._coverage_directory = None
     self._initializeTestCoverageAttributes(args)
 
+    self._store_tombstones = False
+    self._initializeTombstonesAttributes(args)
+
   def _initializeApkAttributes(self, args, error_func):
     if args.apk_under_test:
       apk_under_test_path = args.apk_under_test
@@ -537,6 +540,9 @@ class InstrumentationTestInstance(test_instance.TestInstance):
   def _initializeTestCoverageAttributes(self, args):
     self._coverage_directory = args.coverage_dir
 
+  def _initializeTombstonesAttributes(self, args):
+    self._store_tombstones = args.store_tombstones
+
   @property
   def additional_apks(self):
     return self._additional_apks
@@ -577,6 +583,10 @@ class InstrumentationTestInstance(test_instance.TestInstance):
   def screenshot_dir(self):
     return self._screenshot_dir
 
+  @property
+  def store_tombstones(self):
+    return self._store_tombstones
+
   @property
   def suite(self):
     return self._suite

android/pylib/instrumentation/test_result.py

@@ -28,3 +28,7 @@ class InstrumentationTestResult(base_test_result.BaseTestResult):
       self._class_name = full_name
       self._test_name = full_name
     self._start_date = start_date
+    self._tombstones = None
+
+  def SetTombstones(self, tombstones):
+    self._tombstones = tombstones

android/pylib/local/device/local_device_instrumentation_test_run.py

@@ -15,7 +15,7 @@ from pylib import valgrind_tools
 from pylib.base import base_test_result
 from pylib.local.device import local_device_environment
 from pylib.local.device import local_device_test_run
-
+import tombstones
 
 TIMEOUT_ANNOTATIONS = [
   ('Manual', 10 * 60 * 60),
@@ -143,6 +143,8 @@ class LocalDeviceInstrumentationTestRun(
       else:
         for step in steps:
           step()
+      if self._test_instance.store_tombstones:
+        tombstones.ClearAllTombstones(dev)
 
     self._env.parallel_devices.pMap(
         individual_device_set_up,
@@ -329,6 +331,15 @@ class LocalDeviceInstrumentationTestRun(
           self._test_instance.coverage_directory)
       device.RunShellCommand('rm -f %s' % os.path.join(coverage_directory,
           '*'))
+    if self._test_instance.store_tombstones:
+      for result in results:
+        if result.GetType() == base_test_result.ResultType.CRASH:
+          resolved_tombstones = tombstones.ResolveTombstones(
+              device,
+              resolve_all_tombstones=True,
+              include_stack_symbols=False,
+              wipe_tombstones=True)
+          result.SetTombstones('\n'.join(resolved_tombstones))
     return results
 
   #override

android/test_runner.py

@@ -427,6 +427,9 @@ def AddInstrumentationTestOptions(parser):
                      help='Causes the render tests to not fail when a check'
                           'fails or the golden image is missing but to render'
                           'the view and carry on.')
+  group.add_argument('--store-tombstones', dest='store_tombstones',
+                     action='store_true',
+                     help='Add tombstones in results if crash.')
 
   AddCommonOptions(parser)
   AddDeviceOptions(parser)

android/test_runner.pydeps

@@ -144,3 +144,4 @@ pylib/utils/repo_utils.py
 pylib/utils/test_environment.py
 pylib/valgrind_tools.py
 test_runner.py
+tombstones.py

android/tombstones.py

@@ -159,23 +159,27 @@ def _ResolveTombstones(jobs, tombstones):
   """
   if not tombstones:
     logging.warning('No tombstones to resolve.')
-    return
+    return []
   if len(tombstones) == 1:
     data = [_ResolveTombstone(tombstones[0])]
   else:
     pool = multiprocessing.Pool(processes=jobs)
     data = pool.map(_ResolveTombstone, tombstones)
+  resolved_tombstones = []
   for tombstone in data:
-    for line in tombstone:
-      logging.info(line)
+    resolved_tombstones.extend(tombstone)
+  return resolved_tombstones
 
-
-def _GetTombstonesForDevice(device, args):
+def _GetTombstonesForDevice(device, resolve_all_tombstones,
+                            include_stack_symbols,
+                            wipe_tombstones):
   """Returns a list of tombstones on a given device.
 
   Args:
     device: An instance of DeviceUtils.
-    args: command line arguments
+    resolve_all_tombstone: Whether to resolve every tombstone.
+    include_stack_symbols: Whether to include symbols for stack data.
+    wipe_tombstones: Whether to wipe tombstones.
   """
   ret = []
   all_tombstones = list(_ListTombstones(device))
@@ -187,7 +191,7 @@ def _GetTombstonesForDevice(device, args):
   all_tombstones.sort(cmp=lambda a, b: cmp(b[1], a[1]))
 
   # Only resolve the most recent unless --all-tombstones given.
-  tombstones = all_tombstones if args.all_tombstones else [all_tombstones[0]]
+  tombstones = all_tombstones if resolve_all_tombstones else [all_tombstones[0]]
 
   device_now = _GetDeviceDateTime(device)
   try:
@@ -197,7 +201,7 @@ def _GetTombstonesForDevice(device, args):
                'device_now': device_now,
                'time': tombstone_time,
                'file': tombstone_file,
-               'stack': args.stack,
+               'stack': include_stack_symbols,
                'data': _GetTombstoneData(device, tombstone_file)}]
   except device_errors.CommandFailedError:
     for entry in device.StatDirectory(
@@ -206,12 +210,41 @@ def _GetTombstonesForDevice(device, args):
     raise
 
   # Erase all the tombstones if desired.
-  if args.wipe_tombstones:
+  if wipe_tombstones:
     for tombstone_file, _ in all_tombstones:
       _EraseTombstone(device, tombstone_file)
 
   return ret
 
+def ClearAllTombstones(device):
+  """Clear all tombstones in the device.
+
+  Args:
+    device: An instance of DeviceUtils.
+  """
+  all_tombstones = list(_ListTombstones(device))
+  if not all_tombstones:
+    logging.warning('No tombstones to clear.')
+
+  for tombstone_file, _ in all_tombstones:
+    _EraseTombstone(device, tombstone_file)
+
+def ResolveTombstones(device, resolve_all_tombstones, include_stack_symbols,
+                      wipe_tombstones, jobs=4):
+  """Resolve tombstones in the device.
+
+  Args:
+    device: An instance of DeviceUtils.
+    resolve_all_tombstone: Whether to resolve every tombstone.
+    include_stack_symbols: Whether to include symbols for stack data.
+    wipe_tombstones: Whether to wipe tombstones.
+    jobs: Number of jobs to use when processing multiple crash stacks.
+  """
+  return _ResolveTombstones(jobs,
+                            _GetTombstonesForDevice(device,
+                                                    resolve_all_tombstones,
+                                                    include_stack_symbols,
+                                                    wipe_tombstones))
 
 def main():
   custom_handler = logging.StreamHandler(sys.stdout)
@@ -260,12 +293,12 @@ def main():
   # This must be done serially because strptime can hit a race condition if
   # used for the first time in a multithreaded environment.
   # http://bugs.python.org/issue7980
-  tombstones = []
   for device in devices:
-    tombstones += _GetTombstonesForDevice(device, args)
-
-  _ResolveTombstones(args.jobs, tombstones)
-
+    resolved_tombstones = ResolveTombstones(
+        device, args.all_tombstones,
+        args.stack, args.wipe_tombstones, args.jobs)
+    for line in resolved_tombstones:
+      logging.info(line)
 
 if __name__ == '__main__':
   sys.exit(main())