Bug 991348 - Support taking screenshots on all test failures in mochitests; r=jmaher

testing/mochitest/mach_commands.py

@@ -184,7 +184,7 @@ class MochitestRunner(MozbuildObject):
         return mochitest.run_remote_mochitests(parser, options)
 
     def run_desktop_test(self, context, suite=None, test_paths=None, debugger=None,
-        debugger_args=None, slowscript=False, shuffle=False, keep_open=False,
+        debugger_args=None, slowscript=False, screenshot_on_fail=False, shuffle=False, keep_open=False,
         rerun_failures=False, no_autorun=False, repeat=0, run_until_failure=False,
         slow=False, chunk_by_dir=0, total_chunks=None, this_chunk=None,
         jsdebugger=False, debug_on_failure=False, start_at=None, end_at=None,
@@ -293,6 +293,7 @@ class MochitestRunner(MozbuildObject):
         options.autorun = not no_autorun
         options.closeWhenDone = not keep_open
         options.slowscript = slowscript
+        options.screenshotOnFail = screenshot_on_fail
         options.shuffle = shuffle
         options.consoleLevel = 'INFO'
         options.repeat = repeat
@@ -411,6 +412,10 @@ def MochitestCommand(func):
         help='Do not set the JS_DISABLE_SLOW_SCRIPT_SIGNALS env variable; when not set, recoverable but misleading SIGSEGV instances may occur in Ion/Odin JIT code')
     func = slowscript(func)
 
+    screenshot_on_fail = CommandArgument('--screenshot-on-fail', action='store_true',
+        help='Take screenshots on all test failures. Set $MOZ_UPLOAD_DIR to a directory for storing the screenshots.')
+    func = screenshot_on_fail(func)
+
     shuffle = CommandArgument('--shuffle', action='store_true',
         help='Shuffle execution order.')
     func = shuffle(func)

testing/mochitest/mochitest_options.py

@@ -397,6 +397,12 @@ class MochitestOptions(optparse.OptionParser):
                    "when not set, recoverable but misleading SIGSEGV instances "
                    "may occur in Ion/Odin JIT code."
         }],
+        [["--screenshot-on-fail"],
+         { "action": "store_true",
+           "default": False,
+           "dest": "screenshotOnFail",
+           "help": "Take screenshots on all test failures. Set $MOZ_UPLOAD_DIR to a directory for storing the screenshots."
+        }],
         [["--quiet"],
          { "action": "store_true",
            "default": False,

testing/mochitest/runtests.py

@@ -890,7 +890,8 @@ class Mochitest(MochitestUtilsMixin):
              timeout=-1,
              onLaunch=None,
              webapprtChrome=False,
-             hide_subtests=False):
+             hide_subtests=False,
+             screenshotOnFail=False):
     """
     Run the app, log the duration it took to execute, return the status code.
     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
@@ -980,6 +981,7 @@ class Mochitest(MochitestUtilsMixin):
                                          utilityPath=utilityPath,
                                          symbolsPath=symbolsPath,
                                          dump_screen_on_timeout=not debuggerInfo,
+                                         dump_screen_on_fail=screenshotOnFail,
                                          hide_subtests=hide_subtests,
                                          shutdownLeaks=shutdownLeaks,
         )
@@ -1151,6 +1153,7 @@ class Mochitest(MochitestUtilsMixin):
                                options.app,
                                profile=self.profile,
                                extraArgs=options.browserArgs,
+                               screenshotOnFail=options.screenshotOnFail,
                                utilityPath=options.utilityPath,
                                xrePath=options.xrePath,
                                certPath=options.certPath,
@@ -1212,7 +1215,7 @@ class Mochitest(MochitestUtilsMixin):
 
   class OutputHandler(object):
     """line output handler for mozrunner"""
-    def __init__(self, harness, utilityPath, symbolsPath=None, dump_screen_on_timeout=True,
+    def __init__(self, harness, utilityPath, symbolsPath=None, dump_screen_on_timeout=True, dump_screen_on_fail=False,
                  hide_subtests=False, shutdownLeaks=None):
       """
       harness -- harness instance
@@ -1224,6 +1227,7 @@ class Mochitest(MochitestUtilsMixin):
       self.utilityPath = utilityPath
       self.symbolsPath = symbolsPath
       self.dump_screen_on_timeout = dump_screen_on_timeout
+      self.dump_screen_on_fail = dump_screen_on_fail
       self.hide_subtests = hide_subtests
       self.shutdownLeaks = shutdownLeaks
 
@@ -1249,6 +1253,7 @@ class Mochitest(MochitestUtilsMixin):
       return [self.fix_stack,
               self.format,
               self.dumpScreenOnTimeout,
+              self.dumpScreenOnFail,
               self.metro_subprocess_id,
               self.trackShutdownLeaks,
               self.check_test_failure,
@@ -1321,7 +1326,13 @@ class Mochitest(MochitestUtilsMixin):
       return line.rstrip().decode("UTF-8", "ignore")
 
     def dumpScreenOnTimeout(self, line):
-      if self.dump_screen_on_timeout and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:
+      if not self.dump_screen_on_fail and self.dump_screen_on_timeout and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:
+        self.log_output_buffer()
+        self.harness.dumpScreen(self.utilityPath)
+      return line
+
+    def dumpScreenOnFail(self, line):
+      if self.dump_screen_on_fail and "TEST-UNEXPECTED-FAIL" in line:
         self.log_output_buffer()
         self.harness.dumpScreen(self.utilityPath)
       return line