From cc7b5832aa423adbc60dfef1ea2b4e5add54ffae Mon Sep 17 00:00:00 2001 From: James Graham Date: Tue, 25 Sep 2018 19:42:46 +0000 Subject: [PATCH] Bug 1438975 - Add tbpl-like output for failing refests to the mach formatter r=ahal This allows the output to be used in the reftest anaylzer directly. Differential Revision: https://phabricator.services.mozilla.com/D6819 --HG-- extra : moz-landing-system : lando --- testing/mozbase/mozlog/mozlog/commandline.py | 8 ++++++ .../mozlog/mozlog/formatters/machformatter.py | 15 +++++++++-- .../mozlog/mozlog/formatters/tbplformatter.py | 27 +++++++++++-------- testing/web-platform/mach_commands.py | 3 +++ 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/testing/mozbase/mozlog/mozlog/commandline.py b/testing/mozbase/mozlog/mozlog/commandline.py index 25a40e0396df..4064211bf1c7 100644 --- a/testing/mozbase/mozlog/mozlog/commandline.py +++ b/testing/mozbase/mozlog/mozlog/commandline.py @@ -60,6 +60,11 @@ def buffer_handler_wrapper(handler, buffer_limit): return handlers.BufferHandler(handler, buffer_limit) +def screenshot_wrapper(formatter, enable_screenshot): + formatter.enable_screenshot = enable_screenshot + return formatter + + def valgrind_handler_wrapper(handler): return handlers.ValgrindHandler(handler) @@ -96,6 +101,9 @@ fmt_options = { 'buffer': (buffer_handler_wrapper, "If specified, enables message buffering at the given buffer size limit.", ["mach", "tbpl"], "store"), + 'screenshot': (screenshot_wrapper, + "Enable logging reftest-analyzer compatible screenshot data.", + ["mach"], "store_true"), } diff --git a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py index e88e5327f375..b42c0b3f21b4 100644 --- a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py +++ b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py @@ -11,6 +11,7 @@ from mozterm import Terminal from . import base from .process import strstatus +from .tbplformatter import TbplFormatter from ..handlers import SummaryHandler import six from functools import reduce @@ -26,7 +27,7 @@ class MachFormatter(base.BaseFormatter): def __init__(self, start_time=None, write_interval=False, write_times=True, terminal=None, disable_colors=False, summary_on_shutdown=False, - verbose=False, **kwargs): + verbose=False, enable_screenshot=False, **kwargs): super(MachFormatter, self).__init__(**kwargs) if start_time is None: @@ -41,6 +42,8 @@ class MachFormatter(base.BaseFormatter): self.term = Terminal(disable_styling=disable_colors) self.verbose = verbose self._known_pids = set() + self.tbpl_formatter = None + self.enable_screenshot = enable_screenshot self.summary = SummaryHandler() self.summary_on_shutdown = summary_on_shutdown @@ -183,6 +186,8 @@ class MachFormatter(base.BaseFormatter): parent_unexpected = False expected_str = "" + has_screenshots = "reftest_screenshots" in data.get("extra", {}) + test = self._get_test_id(data) # Reset the counts to 0 @@ -215,7 +220,13 @@ class MachFormatter(base.BaseFormatter): color = self.term.red action = color(data['action'].upper()) - return "%s: %s" % (action, rv) + rv = "%s: %s" % (action, rv) + if has_screenshots and self.enable_screenshot: + if self.tbpl_formatter is None: + self.tbpl_formatter = TbplFormatter() + # Create TBPL-like output that can be pasted into the reftest analyser + rv = "\n".join((rv, self.tbpl_formatter.test_end(data))) + return rv def valgrind_error(self, data): rv = " " + data['primary'] + "\n" diff --git a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py index 1e41a80215c2..e4409ca97d13 100644 --- a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py +++ b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py @@ -213,6 +213,19 @@ class TbplFormatter(BaseFormatter): time = data["time"] - start_time duration_msg = "took %ims" % time + screenshot_msg = "" + extra = data.get("extra", {}) + if "reftest_screenshots" in extra: + screenshots = extra["reftest_screenshots"] + if len(screenshots) == 3: + screenshot_msg = ("\nREFTEST IMAGE 1 (TEST): data:image/png;base64,%s\n" + "REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,%s") % ( + screenshots[0]["screenshot"], + screenshots[2]["screenshot"]) + elif len(screenshots) == 1: + screenshot_msg = ("\nREFTEST IMAGE: data:image/png;base64,%s" % + screenshots[0]["screenshot"]) + if "expected" in data: message = data.get("message", "") if not message: @@ -222,17 +235,7 @@ class TbplFormatter(BaseFormatter): if message and message[-1] == "\n": message = message[:-1] - extra = data.get("extra", {}) - if "reftest_screenshots" in extra: - screenshots = extra["reftest_screenshots"] - if len(screenshots) == 3: - message += ("\nREFTEST IMAGE 1 (TEST): data:image/png;base64,%s\n" - "REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,%s") % ( - screenshots[0]["screenshot"], - screenshots[2]["screenshot"]) - elif len(screenshots) == 1: - message += "\nREFTEST IMAGE: data:image/png;base64,%s" \ - % screenshots[0]["screenshot"] + message += screenshot_msg failure_line = "TEST-UNEXPECTED-%s | %s | %s\n" % ( data["status"], test_id, message) @@ -249,6 +252,8 @@ class TbplFormatter(BaseFormatter): if duration_msg: sections.append(duration_msg) rv.append(' | '.join(sections) + '\n') + if screenshot_msg: + rv.append(screenshot_msg[1:]) return "".join(rv) def suite_end(self, data): diff --git a/testing/web-platform/mach_commands.py b/testing/web-platform/mach_commands.py index 32bd91a57608..f46a421d8a29 100644 --- a/testing/web-platform/mach_commands.py +++ b/testing/web-platform/mach_commands.py @@ -56,6 +56,9 @@ class WebPlatformTestsRunnerSetup(MozbuildObject): if kwargs["host_cert_path"] is None: kwargs["host_cert_path"] = os.path.join(cert_root, "web-platform.test.pem") + if kwargs["log_mach_screenshot"] is None: + kwargs["log_mach_screenshot"] = True + kwargs["capture_stdio"] = True return kwargs