diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py index 8636ae81a4e2..93b7c037a3b2 100644 --- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -619,6 +619,15 @@ class Raptor( ), }, ], + [ + ["--screenshot-on-failure"], + { + "action": "store_true", + "dest": "screenshot_on_failure", + "default": False, + "help": "Take a screenshot when the test fails.", + }, + ], ] + testing_config_options + copy.deepcopy(code_coverage_config_options) @@ -743,6 +752,7 @@ class Raptor( self.browser_cycles = self.config.get("browser_cycles") self.clean = self.config.get("clean") self.page_timeout = self.config.get("page_timeout", None) + self.screenshot_on_failure = self.config.get("screenshot_on_failure") for (arg,), details in Raptor.browsertime_options: # Allow overriding defaults on the `./mach raptor-test ...` command-line. @@ -1069,6 +1079,11 @@ class Raptor( options.extend( [f"--post-startup-delay={self.config['post_startup_delay']}"] ) + if ( + self.config.get("screenshot_on_failure", False) + or os.environ.get("MOZ_AUTOMATION", None) is not None + ): + options.extend(["--screenshot-on-failure"]) for (arg,), details in Raptor.browsertime_options: # Allow overriding defaults on the `./mach raptor-test ...` command-line diff --git a/testing/raptor/mach_commands.py b/testing/raptor/mach_commands.py index 1a900c2f333f..14ffb1caf204 100644 --- a/testing/raptor/mach_commands.py +++ b/testing/raptor/mach_commands.py @@ -68,6 +68,7 @@ class RaptorRunner(MozbuildObject): self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"] self.browsertime_node = kwargs["browsertime_node"] self.clean = kwargs["clean"] + self.screenshot_on_failure = kwargs["screenshot_on_failure"] if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS: self.binary_path = None @@ -122,6 +123,7 @@ class RaptorRunner(MozbuildObject): "browsertime_node": self.browsertime_node, "mozbuild_path": get_state_dir(), "clean": self.clean, + "screenshot_on_failure": self.screenshot_on_failure, } sys.path.insert(0, os.path.join(self.topsrcdir, "tools", "browsertime")) diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py index f0e9946ecf17..57ce01721be8 100644 --- a/testing/raptor/raptor/browsertime/base.py +++ b/testing/raptor/raptor/browsertime/base.py @@ -19,7 +19,7 @@ from benchmark import Benchmark from cmdline import CHROME_ANDROID_APPS from logger.logger import RaptorLogger from manifestparser.util import evaluate_list_from_string -from perftest import GECKO_PROFILER_APPS, TRACE_APPS, Perftest +from perftest import FIREFOX_APPS, GECKO_PROFILER_APPS, TRACE_APPS, Perftest from results import BrowsertimeResultsHandler from utils import bool_from_str @@ -825,6 +825,21 @@ class Browsertime(Perftest): os.killpg(proc.pid, signal.SIGKILL) proc.wait() + def get_failure_screenshot(self): + if ( + self.config.get("screenshot_on_failure") + and self.config["app"] in FIREFOX_APPS + ): + from mozscreenshot import dump_screen + + obj_dir = os.environ.get("MOZ_DEVELOPER_OBJ_DIR", None) + if obj_dir is None: + build_dir = pathlib.Path(os.environ.get("MOZ_UPLOAD_DIR")).parent + utility_path = pathlib.Path(build_dir, "tests", "bin") + else: + utility_path = os.path.join(obj_dir, "dist", "bin") + dump_screen(utility_path, LOG) + def run_extra_profiler_run( self, test, timeout, proc_timeout, output_timeout, line_handler, env ): @@ -1016,16 +1031,19 @@ class Browsertime(Perftest): ) if self.output_timed_out: + self.get_failure_screenshot() raise Exception( f"Browsertime process timed out after waiting {output_timeout} seconds " "for output" ) if self.timed_out: + self.get_failure_screenshot() raise Exception( f"Browsertime process timed out after {proc_timeout} seconds" ) if self.browsertime_failure: + self.get_failure_screenshot() raise Exception(self.browsertime_failure) # We've run the main browsertime process, now we need to run the diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index 37533142aed5..9dad1daf2ebc 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -520,6 +520,13 @@ def create_parser(mach_interface=False): type=str, help="Repository branch that should be used for a particular benchmark test.", ) + add_arg( + "--screenshot-on-failure", + action="store_true", + dest="screenshot_on_failure", + default=False, + help="Take a screenshot when the test fails.", + ) add_logging_group(parser) return parser diff --git a/testing/raptor/raptor/perftest.py b/testing/raptor/raptor/perftest.py index da536d5c2968..6e21b6d1148d 100644 --- a/testing/raptor/raptor/perftest.py +++ b/testing/raptor/raptor/perftest.py @@ -110,6 +110,7 @@ class Perftest(object): benchmark_revision=None, benchmark_branch=None, clean=False, + screenshot_on_failure=False, **kwargs ): self._remote_test_root = None @@ -156,6 +157,7 @@ class Perftest(object): "benchmark_revision": benchmark_revision, "benchmark_branch": benchmark_branch, "clean": clean, + "screenshot_on_failure": screenshot_on_failure, } self.firefox_android_apps = FIREFOX_ANDROID_APPS diff --git a/testing/raptor/raptor/raptor.py b/testing/raptor/raptor/raptor.py index 9390b530b883..8b9cacda253a 100644 --- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -124,6 +124,7 @@ def main(args=sys.argv[1:]): benchmark_branch=args.benchmark_branch, page_timeout=args.page_timeout, clean=args.clean, + screenshot_on_failure=args.screenshot_on_failure, ) except Exception: traceback.print_exc()