Bug 1403616 - Defer logging of Marionette IOError to after post-test checks. r=gbrown

If Firefox crashes while mochitests or reftests are running, Marionette
will trigger an IOError exception which currently gets logged immediately,
and causes no post-test checks to be performed. This results in missing
crash and leak checks, and an unclear failure message on Treeherder.

With this change only the IOError from Marionette gets deferred until
all post-test checks are done. This fixes the failure messages, and will
put PROCESS_CRASH or leak log output first.

MozReview-Commit-ID: JCYP5LlPE1m

--HG--
extra : rebase_source : a4a9455402b01db8ef1dbafccc7a726d2927ec03

layout/tools/reftest/runreftest.py

@@ -709,6 +709,9 @@ class RefTest(object):
                      outputTimeout=timeout)
         proc = runner.process_handler
 
+        # Used to defer a possible IOError exception from Marionette
+        marionette_exception = None
+
         if self.use_marionette:
             marionette_args = {
                 'socket_timeout': options.marionette_socket_timeout,
@@ -720,16 +723,24 @@ class RefTest(object):
                 marionette_args['host'] = host
                 marionette_args['port'] = int(port)
 
-            marionette = Marionette(**marionette_args)
-            marionette.start_session()
+            try:
+                marionette = Marionette(**marionette_args)
+                marionette.start_session()
 
-            addons = Addons(marionette)
-            if options.specialPowersExtensionPath:
-                addons.install(options.specialPowersExtensionPath, temp=True)
+                addons = Addons(marionette)
+                if options.specialPowersExtensionPath:
+                    addons.install(options.specialPowersExtensionPath, temp=True)
 
-            addons.install(options.reftestExtensionPath, temp=True)
+                addons.install(options.reftestExtensionPath, temp=True)
 
-            marionette.delete_session()
+                marionette.delete_session()
+            except IOError:
+                # Any IOError as thrown by Marionette means that something is
+                # wrong with the process, like a crash or the socket is no
+                # longer open. We defer raising this specific error so that
+                # post-test checks for leaks and crashes are performed and
+                # reported first.
+                marionette_exception = sys.exc_info()
 
         status = runner.wait()
         runner.process_handler = None
@@ -744,10 +755,15 @@ class RefTest(object):
 
         crashed = mozcrash.log_crashes(self.log, os.path.join(profile.profile, 'minidumps'),
                                        symbolsPath, test=self.lastTestSeen)
-
-        runner.cleanup()
         if not status and crashed:
             status = 1
+
+        runner.cleanup()
+
+        if marionette_exception is not None:
+            exc, value, tb = marionette_exception
+            raise exc, value, tb
+
         return status, self.lastTestSeen
 
     def runSerialTests(self, manifests, options, cmdargs=None):

testing/mochitest/runtests.py

@@ -2066,6 +2066,9 @@ toolbar#nav-bar {
         # copy env so we don't munge the caller's environment
         env = env.copy()
 
+        # Used to defer a possible IOError exception from Marionette
+        marionette_exception = None
+
         # make sure we clean up after ourselves.
         try:
             # set process log environment variable
@@ -2150,25 +2153,34 @@ toolbar#nav-bar {
             self.log.process_start(gecko_id)
             self.message_logger.gecko_id = gecko_id
 
-            # start marionette and kick off the tests
-            marionette_args = marionette_args or {}
-            self.marionette = Marionette(**marionette_args)
-            self.marionette.start_session()
+            try:
+                # start marionette and kick off the tests
+                marionette_args = marionette_args or {}
+                self.marionette = Marionette(**marionette_args)
+                self.marionette.start_session()
 
-            # install specialpowers and mochikit addons
-            addons = Addons(self.marionette)
+                # install specialpowers and mochikit addons
+                addons = Addons(self.marionette)
 
-            addons.install(create_zip(
-                os.path.join(here, 'extensions', 'specialpowers')
-            ))
-            addons.install(create_zip(self.mochijar))
+                addons.install(create_zip(
+                    os.path.join(here, 'extensions', 'specialpowers')
+                ))
+                addons.install(create_zip(self.mochijar))
 
-            self.execute_start_script()
+                self.execute_start_script()
 
-            # an open marionette session interacts badly with mochitest,
-            # delete it until we figure out why.
-            self.marionette.delete_session()
-            del self.marionette
+                # an open marionette session interacts badly with mochitest,
+                # delete it until we figure out why.
+                self.marionette.delete_session()
+                del self.marionette
+
+            except IOError:
+                # Any IOError as thrown by Marionette means that something is
+                # wrong with the process, like a crash or the socket is no
+                # longer open. We defer raising this specific error so that
+                # post-test checks for leaks and crashes are performed and
+                # reported first.
+                marionette_exception = sys.exc_info()
 
             # wait until app is finished
             # XXX copy functionality from
@@ -2218,6 +2230,10 @@ toolbar#nav-bar {
                 os.remove(processLog)
             self.urlOpts = []
 
+        if marionette_exception is not None:
+            exc, value, tb = marionette_exception
+            raise exc, value, tb
+
         return status, self.lastTestSeen
 
     def initializeLooping(self, options):