Bug 1341466 - Stop Marionette test run when Android emulator is unresponsive; r=automatedtester

The clean-up code in FennecInstance now counts and logs consecutive DMErrors.
The Marionette clean-up code then throws an UnresponsiveInstanceException
if we hit consecutive errors more than twice, which interrupts the
test run entirely. (Previously, MarionetteTestRunner would just keep running
tests despite consecutive clean-up failures, and eventually it would time out.)

This change allows us to take advantage of the retry mechanism in the
mozharness script that runs all Android tests: it sets the job status to "retry"
if it finds DMError in the test log after the run_tests step is done.

MozReview-Commit-ID: J36XuFVK1aK

--HG--
extra : rebase_source : 99914fd768522fe45160f171966a5efc65c59cc2

testing/marionette/client/marionette_driver/errors.py

@@ -167,6 +167,10 @@ class UnsupportedOperationException(MarionetteException):
     status = "unsupported operation"
 
 
+class UnresponsiveInstanceException(Exception):
+    pass
+
+
 es_ = [e for e in locals().values() if type(e) == type and issubclass(e, MarionetteException)]
 by_string = {e.status: e for e in es_}
 

testing/marionette/client/marionette_driver/geckoinstance.py

@@ -6,11 +6,13 @@ import os
 import sys
 import tempfile
 import time
+import traceback
 
 from copy import deepcopy
 
 import mozversion
 
+from mozdevice import DMError
 from mozprofile import Profile
 from mozrunner import Runner, FennecEmulatorRunner
 
@@ -140,6 +142,8 @@ class GeckoInstance(object):
         self._gecko_log = None
         self.verbose = verbose
         self.headless = headless
+        # keep track of errors to decide whether instance is unresponsive
+        self.unresponsive_count = 0
 
     @property
     def gecko_log(self):
@@ -402,8 +406,13 @@ class FennecInstance(GeckoInstance):
         """
         super(FennecInstance, self).close(clean)
         if clean and self.runner and self.runner.device.connected:
-            self.runner.device.dm.remove_forward(
-                "tcp:{}".format(self.marionette_port))
+            try:
+                self.runner.device.dm.remove_forward(
+                    "tcp:{}".format(self.marionette_port))
+                self.unresponsive_count = 0
+            except DMError:
+                self.unresponsive_count += 1
+                traceback.print_exception(*sys.exc_info())
 
 
 class DesktopInstance(GeckoInstance):

testing/marionette/client/marionette_driver/marionette.py

@@ -653,6 +653,9 @@ class Marionette(object):
         if self.instance:
             # stop application and, if applicable, stop emulator
             self.instance.close(clean=True)
+            if self.instance.unresponsive_count >= 3:
+                raise errors.UnresponsiveInstanceException(
+                    "Application clean-up has failed >2 consecutive times.")
 
     def __del__(self):
         self.cleanup()

testing/marionette/harness/marionette_harness/marionette_test/testcases.py

@@ -20,6 +20,7 @@ from unittest.case import (
 
 from marionette_driver.errors import (
     TimeoutException,
+    UnresponsiveInstanceException
 )
 from mozlog import get_default_logger
 
@@ -137,7 +138,7 @@ class CommonTestCase(unittest.TestCase):
                     self.setUp()
             except SkipTest as e:
                 self._addSkip(result, str(e))
-            except KeyboardInterrupt:
+            except (KeyboardInterrupt, UnresponsiveInstanceException) as e:
                 raise
             except _ExpectedFailure as e:
                 expected_failure(result, e.exc_info)
@@ -157,7 +158,7 @@ class CommonTestCase(unittest.TestCase):
                 except self.failureException:
                     self._enter_pm()
                     result.addFailure(self, sys.exc_info())
-                except KeyboardInterrupt:
+                except (KeyboardInterrupt, UnresponsiveInstanceException) as e:
                     raise
                 except _ExpectedFailure as e:
                     expected_failure(result, e.exc_info)
@@ -185,7 +186,7 @@ class CommonTestCase(unittest.TestCase):
                             raise _ExpectedFailure(sys.exc_info())
                     else:
                         self.tearDown()
-                except KeyboardInterrupt:
+                except (KeyboardInterrupt, UnresponsiveInstanceException) as e:
                     raise
                 except _ExpectedFailure as e:
                     expected_failure(result, e.exc_info)