Bug 1619512 - Crashes of Firefox for Raptor jobs aren't correctly reported for classification r=perftest-reviewers,octavian_negru,sparky,AlexandruIonescu

Differential Revision: https://phabricator.services.mozilla.com/D65144

--HG--
extra : moz-landing-system : lando

testing/raptor/raptor/perftest.py

@@ -181,6 +181,9 @@ either Raptor or browsertime."""
         LOG.info("main raptor init, config is: %s" % str(self.config))
         self.build_browser_profile()
 
+        # Crashes counter
+        self.crashes = 0
+
     @property
     def is_localhost(self):
         return self.config.get("host") in ("localhost", "127.0.0.1")
@@ -324,8 +327,11 @@ either Raptor or browsertime."""
                 try:
                     self.run_test(test, timeout=int(test.get("page_timeout")))
                 except RuntimeError as e:
-                    LOG.critical("Tests failed to finish! Application timed out.")
-                    LOG.error(e)
+                    # Check for crashes before showing the timeout error.
+                    self.check_for_crashes()
+                    if self.crashes == 0:
+                        LOG.critical("Tests failed to finish! Application timed out.")
+                        LOG.error(e)
                     os.sys.exit(1)
                 finally:
                     self.run_test_teardown(test)

testing/raptor/raptor/webextension/android.py

@@ -391,7 +391,7 @@ class WebExtensionAndroid(PerftestAndroid, WebExtension):
             if not self.device.is_dir(remote_dir):
                 return
             self.device.pull(remote_dir, dump_dir)
-            mozcrash.log_crashes(LOG, dump_dir, self.config["symbols_path"])
+            self.crashes += mozcrash.log_crashes(LOG, dump_dir, self.config["symbols_path"])
         finally:
             try:
                 shutil.rmtree(dump_dir)

testing/raptor/raptor/webextension/desktop.py

@@ -204,6 +204,8 @@ class WebExtensionDesktop(PerftestDesktop, WebExtension):
         except NotImplementedError:  # not implemented for Chrome
             pass
 
+        self.crashes += self.runner.crashed
+
     def clean_up(self):
         self.runner.stop()