Bug 1624288 - Log error if Raptor web extension cannot be loaded r=perftest-reviewers,whimboo,sparky

Differential Revision: https://phabricator.services.mozilla.com/D68990

--HG--
extra : moz-landing-system : lando

testing/raptor/raptor/control_server.py

@@ -32,6 +32,7 @@ here = os.path.abspath(os.path.dirname(__file__))
 def MakeCustomHandlerClass(
     results_handler,
     error_handler,
+    startup_handler,
     shutdown_browser,
     handle_gecko_profile,
     background_app,
@@ -118,6 +119,7 @@ def MakeCustomHandlerClass(
         def __init__(self, *args, **kwargs):
             self.results_handler = results_handler
             self.error_handler = error_handler
+            self.startup_handler = startup_handler
             self.shutdown_browser = shutdown_browser
             self.handle_gecko_profile = handle_gecko_profile
             self.background_app = background_app
@@ -234,6 +236,9 @@ def MakeCustomHandlerClass(
                 )
             elif data["type"] == "webext_status":
                 LOG.info("received " + data["type"] + ": " + str(data["data"]))
+            elif data["type"] == "webext_loaded":
+                LOG.info("received " + data["type"] + ": raptor runner.js is loaded!")
+                self.startup_handler(True)
             elif data["type"] == "wait-set":
                 LOG.info("received " + data["type"] + ": " + str(data["data"]))
                 MyHandler.wait_after_messages[str(data["data"])] = True
@@ -316,6 +321,7 @@ class RaptorControlServer:
         self.gecko_profile_dir = None
         self.debug_mode = debug_mode
         self.user_profile = None
+        self.is_webextension_loaded = False
 
     def start(self):
         config_dir = os.path.join(here, "tests")
@@ -332,10 +338,11 @@ class RaptorControlServer:
         handler_class = MakeCustomHandlerClass(
             self.results_handler,
             self.error_handler,
+            self.startup_handler,
             self.shutdown_browser,
             self.handle_gecko_profile,
             self.background_app,
-            self.foreground_app,
+            self.foreground_app
         )
 
         httpd = server_class(server_address, handler_class)
@@ -349,6 +356,9 @@ class RaptorControlServer:
     def error_handler(self, error, stack):
         self._runtime_error = {"error": error, "stack": stack}
 
+    def startup_handler(self, value):
+        self.is_webextension_loaded = value
+
     def shutdown_browser(self):
         # if debug-mode enabled, leave the browser running - require manual shutdown
         # this way the browser console remains open, so we can review the logs etc.

testing/raptor/raptor/webextension/base.py

@@ -116,6 +116,10 @@ class WebExtension(Perftest):
                 elapsed_time += 1
                 if elapsed_time > (timeout) - 5:  # stop 5 seconds early
                     self.control_server.wait_for_quit()
+
+                    if not self.control_server.is_webextension_loaded:
+                        raise RuntimeError("Connection to Raptor webextension failed!")
+
                     raise RuntimeError(
                         "Test failed to finish. "
                         "Application timed out after {} seconds".format(timeout)
@@ -193,6 +197,8 @@ class WebExtension(Perftest):
         except AttributeError:
             self.webext_id = None
 
+        self.control_server.startup_handler(False)
+
     def remove_raptor_webext(self):
         # remove the raptor webext; as it must be reloaded with each subtest anyway
         if not self.raptor_webext:

testing/raptor/webext/raptor/runner.js

@@ -774,7 +774,7 @@ async function init() {
 
   ext = isGecko ? browser : chrome;
 
-  await postToControlServer("status", "raptor runner.js is loaded!");
+  await postToControlServer("loaded");
   await postToControlServer("status", `test name is: ${testName}`);
   await postToControlServer("status", `test settings url is: ${settingsURL}`);