From 69dbf231e622f68ce57331da422d59c4caae8b4d Mon Sep 17 00:00:00 2001 From: KS Date: Thu, 29 Feb 2024 02:46:34 +0000 Subject: [PATCH] Bug 1861731 - Add a method to capture screenshots of failing browsertime tests. r=perftest-reviewers,sparky This patch uses mozscreenshot to take a screenshot of the test right up to the moment of failure. This should be helpful in diagnosing failures in CI like benchmark tests (which don't output videos like pageload tests) as well as pageload tests if they fail before visual metrics can capture a video/screenshot to output. Differential Revision: https://phabricator.services.mozilla.com/D202634 --- .../mozharness/mozilla/testing/raptor.py | 15 ++++++++++++++ testing/raptor/mach_commands.py | 2 ++ testing/raptor/raptor/browsertime/base.py | 20 ++++++++++++++++++- testing/raptor/raptor/cmdline.py | 7 +++++++ testing/raptor/raptor/perftest.py | 2 ++ testing/raptor/raptor/raptor.py | 1 + 6 files changed, 46 insertions(+), 1 deletion(-) diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py index 8636ae81a4e2..93b7c037a3b2 100644 --- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -619,6 +619,15 @@ class Raptor( ), }, ], + [ + ["--screenshot-on-failure"], + { + "action": "store_true", + "dest": "screenshot_on_failure", + "default": False, + "help": "Take a screenshot when the test fails.", + }, + ], ] + testing_config_options + copy.deepcopy(code_coverage_config_options) @@ -743,6 +752,7 @@ class Raptor( self.browser_cycles = self.config.get("browser_cycles") self.clean = self.config.get("clean") self.page_timeout = self.config.get("page_timeout", None) + self.screenshot_on_failure = self.config.get("screenshot_on_failure") for (arg,), details in Raptor.browsertime_options: # Allow overriding defaults on the `./mach raptor-test ...` command-line. @@ -1069,6 +1079,11 @@ class Raptor( options.extend( [f"--post-startup-delay={self.config['post_startup_delay']}"] ) + if ( + self.config.get("screenshot_on_failure", False) + or os.environ.get("MOZ_AUTOMATION", None) is not None + ): + options.extend(["--screenshot-on-failure"]) for (arg,), details in Raptor.browsertime_options: # Allow overriding defaults on the `./mach raptor-test ...` command-line diff --git a/testing/raptor/mach_commands.py b/testing/raptor/mach_commands.py index 1a900c2f333f..14ffb1caf204 100644 --- a/testing/raptor/mach_commands.py +++ b/testing/raptor/mach_commands.py @@ -68,6 +68,7 @@ class RaptorRunner(MozbuildObject): self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"] self.browsertime_node = kwargs["browsertime_node"] self.clean = kwargs["clean"] + self.screenshot_on_failure = kwargs["screenshot_on_failure"] if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS: self.binary_path = None @@ -122,6 +123,7 @@ class RaptorRunner(MozbuildObject): "browsertime_node": self.browsertime_node, "mozbuild_path": get_state_dir(), "clean": self.clean, + "screenshot_on_failure": self.screenshot_on_failure, } sys.path.insert(0, os.path.join(self.topsrcdir, "tools", "browsertime")) diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py index f0e9946ecf17..57ce01721be8 100644 --- a/testing/raptor/raptor/browsertime/base.py +++ b/testing/raptor/raptor/browsertime/base.py @@ -19,7 +19,7 @@ from benchmark import Benchmark from cmdline import CHROME_ANDROID_APPS from logger.logger import RaptorLogger from manifestparser.util import evaluate_list_from_string -from perftest import GECKO_PROFILER_APPS, TRACE_APPS, Perftest +from perftest import FIREFOX_APPS, GECKO_PROFILER_APPS, TRACE_APPS, Perftest from results import BrowsertimeResultsHandler from utils import bool_from_str @@ -825,6 +825,21 @@ class Browsertime(Perftest): os.killpg(proc.pid, signal.SIGKILL) proc.wait() + def get_failure_screenshot(self): + if ( + self.config.get("screenshot_on_failure") + and self.config["app"] in FIREFOX_APPS + ): + from mozscreenshot import dump_screen + + obj_dir = os.environ.get("MOZ_DEVELOPER_OBJ_DIR", None) + if obj_dir is None: + build_dir = pathlib.Path(os.environ.get("MOZ_UPLOAD_DIR")).parent + utility_path = pathlib.Path(build_dir, "tests", "bin") + else: + utility_path = os.path.join(obj_dir, "dist", "bin") + dump_screen(utility_path, LOG) + def run_extra_profiler_run( self, test, timeout, proc_timeout, output_timeout, line_handler, env ): @@ -1016,16 +1031,19 @@ class Browsertime(Perftest): ) if self.output_timed_out: + self.get_failure_screenshot() raise Exception( f"Browsertime process timed out after waiting {output_timeout} seconds " "for output" ) if self.timed_out: + self.get_failure_screenshot() raise Exception( f"Browsertime process timed out after {proc_timeout} seconds" ) if self.browsertime_failure: + self.get_failure_screenshot() raise Exception(self.browsertime_failure) # We've run the main browsertime process, now we need to run the diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index 37533142aed5..9dad1daf2ebc 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -520,6 +520,13 @@ def create_parser(mach_interface=False): type=str, help="Repository branch that should be used for a particular benchmark test.", ) + add_arg( + "--screenshot-on-failure", + action="store_true", + dest="screenshot_on_failure", + default=False, + help="Take a screenshot when the test fails.", + ) add_logging_group(parser) return parser diff --git a/testing/raptor/raptor/perftest.py b/testing/raptor/raptor/perftest.py index da536d5c2968..6e21b6d1148d 100644 --- a/testing/raptor/raptor/perftest.py +++ b/testing/raptor/raptor/perftest.py @@ -110,6 +110,7 @@ class Perftest(object): benchmark_revision=None, benchmark_branch=None, clean=False, + screenshot_on_failure=False, **kwargs ): self._remote_test_root = None @@ -156,6 +157,7 @@ class Perftest(object): "benchmark_revision": benchmark_revision, "benchmark_branch": benchmark_branch, "clean": clean, + "screenshot_on_failure": screenshot_on_failure, } self.firefox_android_apps = FIREFOX_ANDROID_APPS diff --git a/testing/raptor/raptor/raptor.py b/testing/raptor/raptor/raptor.py index 9390b530b883..8b9cacda253a 100644 --- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -124,6 +124,7 @@ def main(args=sys.argv[1:]): benchmark_branch=args.benchmark_branch, page_timeout=args.page_timeout, clean=args.clean, + screenshot_on_failure=args.screenshot_on_failure, ) except Exception: traceback.print_exc()