Bug 1641351 - raise on node error in BT r=acreskey

We forgot to check that the node command succeeded.
This patch fixes it.

Differential Revision: https://phabricator.services.mozilla.com/D77148

python/mozperftest/mozperftest/browser/browsertime/runner.py

@@ -45,6 +45,10 @@ def extract_browser_name(args):
     return res[0][-1]
 
 
+class NodeException(Exception):
+    pass
+
+
 class BrowsertimeRunner(NodeRunner):
     """Runs a browsertime test.
     """
@@ -331,7 +335,9 @@ class BrowsertimeRunner(NodeRunner):
         extra = self.extra_default_args(args=args)
         command = [str(self.browsertime_js)] + extra + args
         self.info("Running browsertime with this command %s" % " ".join(command))
-        self.node(command)
+        exit_code = self.node(command)
+        if exit_code != 0:
+            raise NodeException(exit_code)
 
         metadata.add_result(
             {

python/mozperftest/mozperftest/tests/support.py

@@ -47,6 +47,7 @@ def get_running_env(**kwargs):
     mach_cmd.topobjdir = config.topobjdir
     mach_cmd._mach_context = MagicMock()
     mach_cmd._mach_context.state_dir = tempfile.mkdtemp()
+    mach_cmd.run_process.return_value = 0
 
     mach_args = {
         "flavor": "script",

python/mozperftest/mozperftest/tests/test_browsertime.py

@@ -11,7 +11,11 @@ import pytest
 from mozperftest.tests.support import get_running_env, EXAMPLE_TEST
 from mozperftest.environment import BROWSER
 from mozperftest.browser.browsertime import add_options
-from mozperftest.browser.browsertime.runner import matches, extract_browser_name
+from mozperftest.browser.browsertime.runner import (
+    NodeException,
+    matches,
+    extract_browser_name,
+)
 from mozperftest.utils import silence, temporary_env
 
 
@@ -66,6 +70,32 @@ def test_browser(*mocked):
     assert results[0]["name"] == "Example"
 
 
+@mock.patch("mozperftest.browser.browsertime.runner.install_package")
+@mock.patch(
+    "mozperftest.browser.noderunner.NodeRunner.verify_node_install", new=lambda x: True
+)
+@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
+@mock.patch(
+    "mozperftest.browser.browsertime.runner.BrowsertimeRunner._setup_node_packages",
+    new=lambda x, y: None,
+)
+def test_browser_failed(*mocked):
+    mach_cmd, metadata, env = get_running_env(
+        android=True,
+        android_app_name="something",
+        browsertime_geckodriver="GECKODRIVER",
+        browsertime_iterations=1,
+        browsertime_extra_options="one=1,two=2",
+    )
+    # set the return value to 1 to simulate a node failure
+    mach_cmd.run_process.return_value = 1
+    browser = env.layers[BROWSER]
+    env.set_arg("tests", [EXAMPLE_TEST])
+
+    with browser as b, silence(), pytest.raises(NodeException):
+        b(metadata)
+
+
 @mock.patch("mozperftest.browser.browsertime.runner.install_package")
 @mock.patch(
     "mozperftest.browser.noderunner.NodeRunner.verify_node_install", new=lambda x: True