Bug 1438975 - Add tbpl-like output for failing refests to the mach formatter r=ahal

This allows the output to be used in the reftest anaylzer directly.

Differential Revision: https://phabricator.services.mozilla.com/D6819

--HG--
extra : moz-landing-system : lando

testing/mozbase/mozlog/mozlog/commandline.py

@@ -60,6 +60,11 @@ def buffer_handler_wrapper(handler, buffer_limit):
     return handlers.BufferHandler(handler, buffer_limit)
 
 
+def screenshot_wrapper(formatter, enable_screenshot):
+    formatter.enable_screenshot = enable_screenshot
+    return formatter
+
+
 def valgrind_handler_wrapper(handler):
     return handlers.ValgrindHandler(handler)
 
@@ -96,6 +101,9 @@ fmt_options = {
     'buffer': (buffer_handler_wrapper,
                "If specified, enables message buffering at the given buffer size limit.",
                ["mach", "tbpl"], "store"),
+    'screenshot': (screenshot_wrapper,
+                   "Enable logging reftest-analyzer compatible screenshot data.",
+                   ["mach"], "store_true"),
 }
 
 

testing/mozbase/mozlog/mozlog/formatters/machformatter.py

@@ -11,6 +11,7 @@ from mozterm import Terminal
 
 from . import base
 from .process import strstatus
+from .tbplformatter import TbplFormatter
 from ..handlers import SummaryHandler
 import six
 from functools import reduce
@@ -26,7 +27,7 @@ class MachFormatter(base.BaseFormatter):
 
     def __init__(self, start_time=None, write_interval=False, write_times=True,
                  terminal=None, disable_colors=False, summary_on_shutdown=False,
-                 verbose=False, **kwargs):
+                 verbose=False, enable_screenshot=False, **kwargs):
         super(MachFormatter, self).__init__(**kwargs)
 
         if start_time is None:
@@ -41,6 +42,8 @@ class MachFormatter(base.BaseFormatter):
         self.term = Terminal(disable_styling=disable_colors)
         self.verbose = verbose
         self._known_pids = set()
+        self.tbpl_formatter = None
+        self.enable_screenshot = enable_screenshot
 
         self.summary = SummaryHandler()
         self.summary_on_shutdown = summary_on_shutdown
@@ -183,6 +186,8 @@ class MachFormatter(base.BaseFormatter):
             parent_unexpected = False
             expected_str = ""
 
+        has_screenshots = "reftest_screenshots" in data.get("extra", {})
+
         test = self._get_test_id(data)
 
         # Reset the counts to 0
@@ -215,7 +220,13 @@ class MachFormatter(base.BaseFormatter):
             color = self.term.red
 
         action = color(data['action'].upper())
-        return "%s: %s" % (action, rv)
+        rv = "%s: %s" % (action, rv)
+        if has_screenshots and self.enable_screenshot:
+            if self.tbpl_formatter is None:
+                self.tbpl_formatter = TbplFormatter()
+            # Create TBPL-like output that can be pasted into the reftest analyser
+            rv = "\n".join((rv, self.tbpl_formatter.test_end(data)))
+        return rv
 
     def valgrind_error(self, data):
         rv = " " + data['primary'] + "\n"

testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py

@@ -213,6 +213,19 @@ class TbplFormatter(BaseFormatter):
             time = data["time"] - start_time
             duration_msg = "took %ims" % time
 
+        screenshot_msg = ""
+        extra = data.get("extra", {})
+        if "reftest_screenshots" in extra:
+            screenshots = extra["reftest_screenshots"]
+            if len(screenshots) == 3:
+                screenshot_msg = ("\nREFTEST   IMAGE 1 (TEST): data:image/png;base64,%s\n"
+                                  "REFTEST   IMAGE 2 (REFERENCE): data:image/png;base64,%s") % (
+                                      screenshots[0]["screenshot"],
+                                      screenshots[2]["screenshot"])
+            elif len(screenshots) == 1:
+                screenshot_msg = ("\nREFTEST   IMAGE: data:image/png;base64,%s" %
+                                  screenshots[0]["screenshot"])
+
         if "expected" in data:
             message = data.get("message", "")
             if not message:
@@ -222,17 +235,7 @@ class TbplFormatter(BaseFormatter):
             if message and message[-1] == "\n":
                 message = message[:-1]
 
-            extra = data.get("extra", {})
-            if "reftest_screenshots" in extra:
-                screenshots = extra["reftest_screenshots"]
-                if len(screenshots) == 3:
-                    message += ("\nREFTEST   IMAGE 1 (TEST): data:image/png;base64,%s\n"
-                                "REFTEST   IMAGE 2 (REFERENCE): data:image/png;base64,%s") % (
-                                    screenshots[0]["screenshot"],
-                                    screenshots[2]["screenshot"])
-                elif len(screenshots) == 1:
-                    message += "\nREFTEST   IMAGE: data:image/png;base64,%s" \
-                               % screenshots[0]["screenshot"]
+            message += screenshot_msg
 
             failure_line = "TEST-UNEXPECTED-%s | %s | %s\n" % (
                 data["status"], test_id, message)
@@ -249,6 +252,8 @@ class TbplFormatter(BaseFormatter):
         if duration_msg:
             sections.append(duration_msg)
         rv.append(' | '.join(sections) + '\n')
+        if screenshot_msg:
+            rv.append(screenshot_msg[1:])
         return "".join(rv)
 
     def suite_end(self, data):

testing/web-platform/mach_commands.py

@@ -56,6 +56,9 @@ class WebPlatformTestsRunnerSetup(MozbuildObject):
             if kwargs["host_cert_path"] is None:
                 kwargs["host_cert_path"] = os.path.join(cert_root, "web-platform.test.pem")
 
+        if kwargs["log_mach_screenshot"] is None:
+            kwargs["log_mach_screenshot"] = True
+
         kwargs["capture_stdio"] = True
 
         return kwargs