Bug 1861731 - Add a method to capture screenshots of failing browsertime tests. r=perftest-reviewers,sparky

This patch uses mozscreenshot to take a screenshot of the test right up
to the moment of failure. This should be helpful in diagnosing failures
in CI like benchmark tests (which don't output videos like pageload
tests) as well as pageload tests if they fail before visual metrics can
capture a video/screenshot to output.

Differential Revision: https://phabricator.services.mozilla.com/D202634

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -619,6 +619,15 @@ class Raptor(
                     ),
                 },
             ],
+            [
+                ["--screenshot-on-failure"],
+                {
+                    "action": "store_true",
+                    "dest": "screenshot_on_failure",
+                    "default": False,
+                    "help": "Take a screenshot when the test fails.",
+                },
+            ],
         ]
         + testing_config_options
         + copy.deepcopy(code_coverage_config_options)
@@ -743,6 +752,7 @@ class Raptor(
         self.browser_cycles = self.config.get("browser_cycles")
         self.clean = self.config.get("clean")
         self.page_timeout = self.config.get("page_timeout", None)
+        self.screenshot_on_failure = self.config.get("screenshot_on_failure")
 
         for (arg,), details in Raptor.browsertime_options:
             # Allow overriding defaults on the `./mach raptor-test ...` command-line.
@@ -1069,6 +1079,11 @@ class Raptor(
             options.extend(
                 [f"--post-startup-delay={self.config['post_startup_delay']}"]
             )
+        if (
+            self.config.get("screenshot_on_failure", False)
+            or os.environ.get("MOZ_AUTOMATION", None) is not None
+        ):
+            options.extend(["--screenshot-on-failure"])
 
         for (arg,), details in Raptor.browsertime_options:
             # Allow overriding defaults on the `./mach raptor-test ...` command-line

testing/raptor/mach_commands.py

@@ -68,6 +68,7 @@ class RaptorRunner(MozbuildObject):
         self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"]
         self.browsertime_node = kwargs["browsertime_node"]
         self.clean = kwargs["clean"]
+        self.screenshot_on_failure = kwargs["screenshot_on_failure"]
 
         if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS:
             self.binary_path = None
@@ -122,6 +123,7 @@ class RaptorRunner(MozbuildObject):
             "browsertime_node": self.browsertime_node,
             "mozbuild_path": get_state_dir(),
             "clean": self.clean,
+            "screenshot_on_failure": self.screenshot_on_failure,
         }
 
         sys.path.insert(0, os.path.join(self.topsrcdir, "tools", "browsertime"))

testing/raptor/raptor/browsertime/base.py

@@ -19,7 +19,7 @@ from benchmark import Benchmark
 from cmdline import CHROME_ANDROID_APPS
 from logger.logger import RaptorLogger
 from manifestparser.util import evaluate_list_from_string
-from perftest import GECKO_PROFILER_APPS, TRACE_APPS, Perftest
+from perftest import FIREFOX_APPS, GECKO_PROFILER_APPS, TRACE_APPS, Perftest
 from results import BrowsertimeResultsHandler
 from utils import bool_from_str
 
@@ -825,6 +825,21 @@ class Browsertime(Perftest):
             os.killpg(proc.pid, signal.SIGKILL)
         proc.wait()
 
+    def get_failure_screenshot(self):
+        if (
+            self.config.get("screenshot_on_failure")
+            and self.config["app"] in FIREFOX_APPS
+        ):
+            from mozscreenshot import dump_screen
+
+            obj_dir = os.environ.get("MOZ_DEVELOPER_OBJ_DIR", None)
+            if obj_dir is None:
+                build_dir = pathlib.Path(os.environ.get("MOZ_UPLOAD_DIR")).parent
+                utility_path = pathlib.Path(build_dir, "tests", "bin")
+            else:
+                utility_path = os.path.join(obj_dir, "dist", "bin")
+            dump_screen(utility_path, LOG)
+
     def run_extra_profiler_run(
         self, test, timeout, proc_timeout, output_timeout, line_handler, env
     ):
@@ -1016,16 +1031,19 @@ class Browsertime(Perftest):
             )
 
             if self.output_timed_out:
+                self.get_failure_screenshot()
                 raise Exception(
                     f"Browsertime process timed out after waiting {output_timeout} seconds "
                     "for output"
                 )
             if self.timed_out:
+                self.get_failure_screenshot()
                 raise Exception(
                     f"Browsertime process timed out after {proc_timeout} seconds"
                 )
 
             if self.browsertime_failure:
+                self.get_failure_screenshot()
                 raise Exception(self.browsertime_failure)
 
             # We've run the main browsertime process, now we need to run the

testing/raptor/raptor/cmdline.py

@@ -520,6 +520,13 @@ def create_parser(mach_interface=False):
         type=str,
         help="Repository branch that should be used for a particular benchmark test.",
     )
+    add_arg(
+        "--screenshot-on-failure",
+        action="store_true",
+        dest="screenshot_on_failure",
+        default=False,
+        help="Take a screenshot when the test fails.",
+    )
 
     add_logging_group(parser)
     return parser

testing/raptor/raptor/perftest.py

@@ -110,6 +110,7 @@ class Perftest(object):
         benchmark_revision=None,
         benchmark_branch=None,
         clean=False,
+        screenshot_on_failure=False,
         **kwargs
     ):
         self._remote_test_root = None
@@ -156,6 +157,7 @@ class Perftest(object):
             "benchmark_revision": benchmark_revision,
             "benchmark_branch": benchmark_branch,
             "clean": clean,
+            "screenshot_on_failure": screenshot_on_failure,
         }
 
         self.firefox_android_apps = FIREFOX_ANDROID_APPS

testing/raptor/raptor/raptor.py

@@ -124,6 +124,7 @@ def main(args=sys.argv[1:]):
             benchmark_branch=args.benchmark_branch,
             page_timeout=args.page_timeout,
             clean=args.clean,
+            screenshot_on_failure=args.screenshot_on_failure,
         )
     except Exception:
         traceback.print_exc()