Bug 1605256 - Add exception type to message. r=snorp

This commit does two things:

- It exposes the error that caused the test to fail in our logs, so it would
  appear like so:

```
TEST-UNEXPECTED-FAIL | org.mozilla.geckoview.test.ExtensionActionTest.setIconSvg[#pageAction] | java.lang.AssertionError: We shouldn't get here
```

- It catches exceptions that are raised before any test code is run and outputs
  a different message (since those failures do not depend on the test at all),
  like so:

```
ERROR runjunit.py | The previous test failed because of an error in the test harness | org.mozilla.geckoview.test.rule.TestHarnessException: java.lang.RuntimeException: We shouldn't get here
```

This will hopefully help with spurious intermittents and understand when a
failure changes.

Differential Revision: https://phabricator.services.mozilla.com/D58022

--HG--
extra : moz-landing-system : lando

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java

@@ -1253,37 +1253,48 @@ public class GeckoSessionTestRule implements TestRule {
     @Override
     public Statement apply(final Statement base, final Description description) {
         return new Statement() {
+            private TestServer mServer;
+
+            private void initTest() {
+                try {
+                    mServer.start(TEST_PORT);
+
+                    RuntimeCreator.setPortDelegate(mPortDelegate);
+
+                    getRuntime();
+
+                    long timeout = env.getDefaultTimeoutMillis() + System.currentTimeMillis();
+                    while (!GeckoThread.isStateAtLeast(GeckoThread.State.PROFILE_READY)) {
+                        if (System.currentTimeMillis() > timeout) {
+                            throw new TimeoutException("Could not startup runtime after "
+                                    + env.getDefaultTimeoutMillis() + ".ms");
+                        }
+                        Log.e(LOGTAG, "GeckoThread not ready, sleeping 1000ms.");
+                        try {
+                            Thread.sleep(1000);
+                        } catch (InterruptedException ex) {
+                        }
+                    }
+
+                    Log.e(LOGTAG, "====");
+                    Log.e(LOGTAG, "before prepareStatement " + description);
+                    prepareStatement(description);
+                    Log.e(LOGTAG, "after prepareStatement");
+                } catch (final Throwable t) {
+                    // Any error here is not related to a specific test
+                    throw new TestHarnessException(t);
+                }
+            }
+
             @Override
             public void evaluate() throws Throwable {
                 final AtomicReference<Throwable> exceptionRef = new AtomicReference<>();
 
-                TestServer server = new TestServer(InstrumentationRegistry.getTargetContext());
+                mServer = new TestServer(InstrumentationRegistry.getTargetContext());
 
                 mInstrumentation.runOnMainSync(() -> {
                     try {
-                        server.start(TEST_PORT);
-
-                        RuntimeCreator.setPortDelegate(mPortDelegate);
-
-                        getRuntime();
-
-                        long timeout = env.getDefaultTimeoutMillis() + System.currentTimeMillis();
-                        while (!GeckoThread.isStateAtLeast(GeckoThread.State.PROFILE_READY)) {
-                            if (System.currentTimeMillis() > timeout) {
-                                throw new TimeoutException("Could not startup runtime after "
-                                        + env.getDefaultTimeoutMillis() + ".ms");
-                            }
-                            Log.e(LOGTAG, "GeckoThread not ready, sleeping 1000ms.");
-                            try {
-                                Thread.sleep(1000);
-                            } catch (InterruptedException ex) {
-                            }
-                        }
-
-                        Log.e(LOGTAG, "====");
-                        Log.e(LOGTAG, "before prepareStatement " + description);
-                        prepareStatement(description);
-                        Log.e(LOGTAG, "after prepareStatement");
+                        initTest();
                         base.evaluate();
                         Log.e(LOGTAG, "after evaluate");
                         performTestEndCheck();
@@ -1294,7 +1305,7 @@ public class GeckoSessionTestRule implements TestRule {
                         exceptionRef.set(t);
                     } finally {
                         try {
-                            server.stop();
+                            mServer.stop();
                             cleanupStatement();
                         } catch (Throwable t) {
                             exceptionRef.compareAndSet(null, t);

mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/TestHarnessException.java

@@ -0,0 +1,10 @@
+package org.mozilla.geckoview.test.rule;
+
+/**
+ * Exception thrown when an error occurs in the test harness itself and not in a specific test
+ */
+public class TestHarnessException extends RuntimeException {
+    public TestHarnessException(final Throwable cause) {
+        super(cause);
+    }
+}

testing/mochitest/runjunit.py

@@ -35,6 +35,10 @@ except ImportError:
     conditions = None
 
 
+class JavaTestHarnessException(Exception):
+    pass
+
+
 class JUnitTestRunner(MochitestDesktop):
     """
        A test harness to run geckoview junit tests on a remote device.
@@ -225,6 +229,13 @@ class JUnitTestRunner(MochitestDesktop):
             match = re.match(r'INSTRUMENTATION_STATUS:\s*test=(.*)', line)
             if match:
                 self.test_name = match.group(1)
+            match = re.match(r'INSTRUMENTATION_STATUS:\s*stack=(.*)', line)
+            if match:
+                self.exception_message = match.group(1)
+            if "org.mozilla.geckoview.test.rule.TestHarnessException" in self.exception_message:
+                # This is actually a problem in the test harness itself
+                raise JavaTestHarnessException(self.exception_message)
+
             # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..."
             match = re.match(r'INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)', line)
             if match:
@@ -247,7 +258,10 @@ class JUnitTestRunner(MochitestDesktop):
                         expected = 'FAIL'
                         self.todo_count += 1
                     else:
-                        message = 'status %s' % status
+                        if self.exception_message:
+                            message = self.exception_message
+                        else:
+                            message = 'status %s' % status
                         status = 'FAIL'
                         expected = 'PASS'
                         self.fail_count += 1
@@ -273,6 +287,7 @@ class JUnitTestRunner(MochitestDesktop):
             cmd = self.build_command_line(test_filters)
             while self.need_more_runs():
                 self.class_name = ""
+                self.exception_message = ""
                 self.test_name = ""
                 self.current_full_name = ""
                 self.runs += 1
@@ -469,6 +484,9 @@ def run_test_harness(parser, options):
     except KeyboardInterrupt:
         log.info("runjunit.py | Received keyboard interrupt")
         result = -1
+    except JavaTestHarnessException as e:
+        log.error("runjunit.py | The previous test failed because of an error "
+                  "in the test harness | %s" % (str(e)))
     except Exception as e:
         traceback.print_exc()
         log.error(