Bug 1403983 - Fix reftest regression where screenshots aren't being output and add a test, r=jmaher

MozReview-Commit-ID: 8ZqmukDtWpO

--HG--
extra : rebase_source : 86544bda6e79e5e35444b4083e76d2cf8642c408

layout/tools/reftest/output.py

@@ -52,23 +52,13 @@ class ReftestFormatter(TbplFormatter):
         return status_msg
 
     def test_status(self, data):
+        extra = data.get('extra', {})
         test = data['test']
 
         status_msg = self._format_status(data)
         output_text = "%s | %s | %s" % (status_msg, test, data.get("subtest", "unknown test"))
         if data.get('message'):
             output_text += " | %s" % data['message']
-        return output_text + "\n"
-
-    def test_end(self, data):
-        extra = data.get('extra', {})
-        status = data['status']
-        test = data['test']
-
-        output_text = ""
-        if status != "OK":
-            status_msg = self._format_status(data)
-            output_text = "%s | %s | %s" % (status_msg, test, data.get("message", ""))
 
         if "reftest_screenshots" in extra:
             screenshots = extra["reftest_screenshots"]
@@ -82,6 +72,17 @@ class ReftestFormatter(TbplFormatter):
             elif len(screenshots) == 1:
                 output_text += "\nREFTEST   IMAGE: data:image/png;base64,%s" % image_1
 
+        return output_text + "\n"
+
+    def test_end(self, data):
+        status = data['status']
+        test = data['test']
+
+        output_text = ""
+        if status != "OK":
+            status_msg = self._format_status(data)
+            output_text = "%s | %s | %s" % (status_msg, test, data.get("message", ""))
+
         if output_text:
             output_text += "\nREFTEST "
         output_text += "TEST-END | %s" % test

layout/tools/reftest/selftest/test_reftest_output.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
+from cStringIO import StringIO
 from functools import partial
 
 import mozunit
@@ -34,21 +35,32 @@ def test_output_pass(runtests):
 
 
 def test_output_fail(runtests):
+    formatter = pytest.importorskip('output').ReftestFormatter()
+
     status, lines = runtests('reftest-fail.list')
     assert status == 0
 
-    tbpl_status, log_level = get_mozharness_status(lines, status)
+    buf = StringIO()
+    tbpl_status, log_level = get_mozharness_status(
+        lines, status, formatter=formatter, buf=buf)
+
     assert tbpl_status == TBPL_WARNING
     assert log_level == WARNING
 
     test_status = filter_action('test_status', lines)
     assert len(test_status) == 3
     assert all(t['status'] == 'FAIL' for t in test_status)
+    assert all('reftest_screenshots' in t['extra'] for t in test_status)
 
     test_end = filter_action('test_end', lines)
     assert len(test_end) == 3
     assert all(t['status'] == 'OK' for t in test_end)
 
+    # ensure screenshots were printed
+    formatted = buf.getvalue()
+    assert 'REFTEST   IMAGE 1' in formatted
+    assert 'REFTEST   IMAGE 2' in formatted
+
 
 @pytest.mark.skip_mozinfo("!crashreporter")
 def test_output_crash(runtests):

testing/mozbase/moztest/moztest/selftest/output.py

@@ -20,7 +20,7 @@ from mozharness.mozilla.structuredlog import StructuredOutputParser
 from mozharness.mozilla.testing.errors import HarnessErrorList
 
 
-def get_mozharness_status(suite, lines, status):
+def get_mozharness_status(suite, lines, status, formatter=None, buf=None):
     """Given list of log lines, determine what the mozharness status would be."""
     parser = StructuredOutputParser(
         config={'log_level': INFO},
@@ -29,15 +29,18 @@ def get_mozharness_status(suite, lines, status):
         suite_category=suite,
     )
 
+    if formatter:
+        parser.formatter = formatter
+
     # Processing the log with mozharness will re-print all the output to stdout
     # Since this exact same output has already been printed by the actual test
     # run, temporarily redirect stdout to devnull.
-    with open(os.devnull, 'w') as fh:
-        orig = sys.stdout
-        sys.stdout = fh
-        for line in lines:
-            parser.parse_single_line(json.dumps(line))
-        sys.stdout = orig
+    buf = buf or open(os.devnull, 'w')
+    orig = sys.stdout
+    sys.stdout = buf
+    for line in lines:
+        parser.parse_single_line(json.dumps(line))
+    sys.stdout = orig
     return parser.evaluate_parser(status)