Bug 1623965 - Improve error message when binary not found while testing. r=glandium,remote-protocol-reviewers,marionette-reviewers,webdriver-reviewers,perftest-reviewers,Bebe,whimboo

Differential Revision: https://phabricator.services.mozilla.com/D67726

build/pgo/genpgocert.py

@@ -16,7 +16,7 @@ import subprocess
 import sys
 import distutils
 
-from mozbuild.base import MozbuildObject
+from mozbuild.base import MozbuildObject, BinaryNotFoundException
 from mozfile import NamedTemporaryFile, TemporaryDirectory
 from mozprofile.permissions import ServerLocations
 
@@ -89,8 +89,12 @@ def writeCertspecForServerLocations(fd):
 
 
 def constructCertDatabase(build, srcDir):
-    certutil = build.get_binary_path(what="certutil")
-    pk12util = build.get_binary_path(what="pk12util")
+    try:
+        certutil = build.get_binary_path(what="certutil")
+        pk12util = build.get_binary_path(what="pk12util")
+    except BinaryNotFoundException as e:
+        print('{}\n\n{}\n'.format(e, e.help()))
+        return 1
     openssl = distutils.spawn.find_executable("openssl")
     pycert = os.path.join(build.topsrcdir, "security", "manager", "ssl", "tests",
                           "unit", "pycert.py")

build/pgo/profileserver.py

@@ -11,7 +11,7 @@ import glob
 import subprocess
 
 import mozcrash
-from mozbuild.base import MozbuildObject
+from mozbuild.base import MozbuildObject, BinaryNotFoundException
 from mozfile import TemporaryDirectory
 from mozhttpd import MozHttpd
 from mozprofile import FirefoxProfile, Preferences
@@ -63,7 +63,11 @@ if __name__ == '__main__':
 
     binary = runner_args.get('binary')
     if not binary:
-        binary = build.get_binary_path(where="staged-package")
+        try:
+            binary = build.get_binary_path(where="staged-package")
+        except BinaryNotFoundException as e:
+            print('{}\n\n{}\n'.format(e, e.help()))
+            sys.exit(1)
     binary = os.path.normpath(os.path.abspath(binary))
 
     path_mappings = {

build/valgrind/mach_commands.py

@@ -17,6 +17,7 @@ from mach.decorators import (
 from mozbuild.base import (
     MachCommandBase,
     MachCommandConditions as conditions,
+    BinaryNotFoundException,
 )
 
 
@@ -155,6 +156,7 @@ class MachCommands(MachCommandBase):
 
             exitcode = None
             timeout = 1800
+            binary_not_found_exception = None
             try:
                 runner = FirefoxRunner(profile=profile,
                                        binary=self.get_binary_path(),
@@ -163,7 +165,8 @@ class MachCommands(MachCommandBase):
                                        process_args=kp_kwargs)
                 runner.start(debug_args=valgrind_args)
                 exitcode = runner.wait(timeout=timeout)
-
+            except BinaryNotFoundException as e:
+                binary_not_found_exception = e
             finally:
                 errs = outputHandler.error_count
                 supps = outputHandler.suppression_count
@@ -182,7 +185,15 @@ class MachCommands(MachCommandBase):
                     status = 1  # turns the TBPL job orange
                     # We've already printed details of the errors.
 
-                if exitcode is None:
+                if binary_not_found_exception:
+                    status = 2  # turns the TBPL job red
+                    self.log(logging.ERROR, 'valgrind-fail-errors',
+                             {'error': str(binary_not_found_exception)},
+                             'TEST-UNEXPECTED-FAIL | valgrind-test | {error}')
+                    self.log(logging.INFO, 'valgrind-fail-errors',
+                             {'help': binary_not_found_exception.help()},
+                             '{help}')
+                elif exitcode is None:
                     status = 2  # turns the TBPL job red
                     self.log(logging.ERROR, 'valgrind-fail-timeout',
                              {'timeout': timeout},

devtools/shared/css/generated/mach_commands.py

@@ -10,6 +10,7 @@ This information is used to generate the properties-db.js file.
 
 from __future__ import absolute_import, print_function
 import json
+import logging
 import os
 import runpy
 import sys
@@ -19,6 +20,7 @@ from mozbuild import shellutil
 from mozbuild.base import (
     MozbuildObject,
     MachCommandBase,
+    BinaryNotFoundException,
 )
 from mach.decorators import (
     CommandProvider,
@@ -43,6 +45,8 @@ class MachCommands(MachCommandBase):
 
         print("Re-generating the css properties database...")
         db = self.get_properties_db_from_xpcshell()
+        if not db:
+            return 1
 
         self.output_template({
             'preferences': stringify(db['preferences']),
@@ -58,7 +62,17 @@ class MachCommands(MachCommandBase):
             'devtools/shared/css/generated/generate-properties-db.js')
         gre_path = resolve_path(self.topobjdir, 'dist/bin')
         browser_path = resolve_path(self.topobjdir, 'dist/bin/browser')
-        xpcshell_path = build.get_binary_path(what='xpcshell')
+        try:
+            xpcshell_path = build.get_binary_path(what='xpcshell')
+        except BinaryNotFoundException as e:
+            self.log(logging.ERROR, 'devtools-css-db',
+                     {'error': str(e)},
+                     'ERROR: {error}')
+            self.log(logging.INFO, 'devtools-css-db',
+                     {'help': e.help()},
+                     '{help}')
+            return None
+
         print(browser_path)
 
         sub_env = dict(os.environ)

layout/tools/reftest/reftestcommandline.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function
 
 import argparse
 import os
+import sys
 from collections import OrderedDict
 from urlparse import urlparse
 import mozinfo
@@ -408,10 +409,17 @@ class DesktopArgumentsParser(ReftestArgumentsParser):
             self.error("No test files specified.")
 
         if options.app is None:
-            bin_dir = (self.build_obj.get_binary_path() if
-                       self.build_obj and self.build_obj.substs[
-                           'MOZ_BUILD_APP'] != 'mobile/android'
-                       else None)
+            if self.build_obj and self.build_obj.substs[
+                    'MOZ_BUILD_APP'] != 'mobile/android':
+                from mozbuild.base import BinaryNotFoundException
+
+                try:
+                    bin_dir = self.build_obj.get_binary_path()
+                except BinaryNotFoundException as e:
+                    print('{}\n\n{}\n'.format(e, e.help()), file=sys.stderr)
+                    sys.exit(1)
+            else:
+                bin_dir = None
 
             if bin_dir:
                 options.app = bin_dir

python/mozbuild/mozbuild/base.py

@@ -84,6 +84,19 @@ class ObjdirMismatchException(BadEnvironmentException):
         return "Objdir mismatch: %s != %s" % (self.objdir1, self.objdir2)
 
 
+class BinaryNotFoundException(Exception):
+    """Raised when the binary is not found in the expected location."""
+
+    def __init__(self, path):
+        self.path = path
+
+    def __str__(self):
+        return 'Binary expected at {} does not exist.'.format(self.path)
+
+    def help(self):
+        return 'It looks like your program isn\'t built. You can run |./mach build| to build it.'
+
+
 class MozbuildObject(ProcessExecutionMixin):
     """Base class providing basic functionality useful to many modules.
 
@@ -556,7 +569,7 @@ class MozbuildObject(ProcessExecutionMixin):
         path = os.path.join(stem, leaf)
 
         if validate_exists and not os.path.exists(path):
-            raise Exception('Binary expected at %s does not exist.' % path)
+            raise BinaryNotFoundException(path)
 
         return path
 

python/mozbuild/mozbuild/mach_commands.py

@@ -28,6 +28,7 @@ from mach.decorators import (
 )
 
 from mozbuild.base import (
+    BinaryNotFoundException,
     BuildEnvironmentNotFoundException,
     MachCommandBase,
     MachCommandConditions as conditions,
@@ -955,10 +956,13 @@ class RunProgram(MachCommandBase):
     def _run_jsshell(self, params, debug, debugger, debugger_args):
         try:
             binpath = self.get_binary_path('app')
-        except Exception as e:
-            print("It looks like your program isn't built.",
-                  "You can run |mach build| to build it.")
-            print(e)
+        except BinaryNotFoundException as e:
+            self.log(logging.ERROR, 'run',
+                     {'error': str(e)},
+                     'ERROR: {error}')
+            self.log(logging.INFO, 'run',
+                     {'help': e.help()},
+                     '{help}')
             return 1
 
         args = [binpath]
@@ -1011,10 +1015,13 @@ class RunProgram(MachCommandBase):
 
         try:
             binpath = self.get_binary_path('app')
-        except Exception as e:
-            print("It looks like your program isn't built.",
-                  "You can run |mach build| to build it.")
-            print(e)
+        except BinaryNotFoundException as e:
+            self.log(logging.ERROR, 'run',
+                     {'error': str(e)},
+                     'ERROR: {error}')
+            self.log(logging.INFO, 'run',
+                     {'help': e.help()},
+                     '{help}')
             return 1
 
         args = []
@@ -1413,7 +1420,17 @@ class WebRTCGTestCommands(GTestCommands):
                      'split as the Bourne shell would.')
     def gtest(self, gtest_filter, debug, debugger,
               debugger_args):
-        app_path = self.get_binary_path('webrtc-gtest')
+        try:
+            app_path = self.get_binary_path('webrtc-gtest')
+        except BinaryNotFoundException as e:
+            self.log(logging.ERROR, 'webrtc-gtest',
+                     {'error': str(e)},
+                     'ERROR: {error}')
+            self.log(logging.INFO, 'webrtc-gtest',
+                     {'help': e.help()},
+                     '{help}')
+            return 1
+
         args = [app_path]
 
         if debug or debugger or debugger_args:

remote/mach_commands.py

@@ -30,6 +30,7 @@ from mach.decorators import (
 from mozbuild.base import (
     MachCommandBase,
     MozbuildObject,
+    BinaryNotFoundException,
 )
 from mozbuild import nodeutil
 import mozlog
@@ -496,6 +497,10 @@ class PuppeteerTest(MachCommandBase):
         puppeteer = self._spawn(PuppeteerRunner)
         try:
             return puppeteer.run_test(logger, *tests, **params)
+        except BinaryNotFoundException as e:
+            logger.error(e)
+            logger.info(e.help())
+            exit(1)
         except Exception as e:
             exit(EX_SOFTWARE, e)
 

testing/awsy/mach_commands.py

@@ -12,6 +12,7 @@ import sys
 from mozbuild.base import (
     MachCommandBase,
     MachCommandConditions as conditions,
+    BinaryNotFoundException,
 )
 
 from mach.decorators import (
@@ -273,5 +274,14 @@ class MachCommands(MachCommandBase):
             del kwargs['test_objects']
 
         if not kwargs.get('binary') and conditions.is_firefox(self):
-            kwargs['binary'] = self.get_binary_path('app')
+            try:
+                kwargs['binary'] = self.get_binary_path('app')
+            except BinaryNotFoundException as e:
+                self.log(logging.ERROR, 'awsy',
+                         {'error': str(e)},
+                         'ERROR: {error}')
+                self.log(logging.INFO, 'awsy',
+                         {'help': e.help()},
+                         '{help}')
+                return 1
         return self.run_awsy(tests, **kwargs)

testing/condprofile/mach_commands.py

@@ -1,12 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+import logging
 import sys
 import os
 import tempfile
 
 from mach.decorators import CommandArgument, CommandProvider, Command
-from mozbuild.base import MachCommandBase
+from mozbuild.base import MachCommandBase, BinaryNotFoundException
 
 requirements = os.path.join(os.path.dirname(__file__), "requirements", "base.txt")
 
@@ -91,7 +92,17 @@ class CondprofileCommandProvider(MachCommandBase):
         self._init()
 
         if kw["firefox"] is None:
-            kw["firefox"] = self.get_binary_path()
+            try:
+                kw["firefox"] = self.get_binary_path()
+            except BinaryNotFoundException as e:
+                self.log(logging.ERROR, 'run-condprofile',
+                         {'error': str(e)},
+                         'ERROR: {error}')
+                self.log(logging.INFO, 'run-condprofile',
+                         {'help': e.help()},
+                         '{help}')
+                return 1
+
         from condprof.runner import run
 
         run(**kw)

testing/firefox-ui/mach_commands.py

@@ -4,12 +4,14 @@
 
 from __future__ import absolute_import, unicode_literals
 
+import logging
 import os
 import sys
 
 from mozbuild.base import (
     MachCommandBase,
     MachCommandConditions as conditions,
+    BinaryNotFoundException,
 )
 
 from mach.decorators import (
@@ -90,6 +92,16 @@ class MachCommands(MachCommandBase):
              parser=setup_argument_parser_functional,
              )
     def run_firefox_ui_functional(self, **kwargs):
-        kwargs['binary'] = kwargs['binary'] or self.get_binary_path('app')
+        try:
+            kwargs['binary'] = kwargs['binary'] or self.get_binary_path('app')
+        except BinaryNotFoundException as e:
+            self.log(logging.ERROR, 'firefox-ui-functional',
+                     {'error': str(e)},
+                     'ERROR: {error}')
+            self.log(logging.INFO, 'firefox-ui-functional',
+                     {'help': e.help()},
+                     '{help}')
+            return 1
+
         return run_firefox_ui_test(testtype='functional',
                                    topsrcdir=self.topsrcdir, **kwargs)

testing/geckodriver/mach_commands.py

@@ -14,7 +14,7 @@ from mach.decorators import (
     CommandProvider,
 )
 
-from mozbuild.base import MachCommandBase
+from mozbuild.base import MachCommandBase, BinaryNotFoundException
 
 
 @CommandProvider
@@ -41,12 +41,15 @@ class GeckoDriver(MachCommandBase):
     def run(self, binary, params, debug, debugger, debugger_args):
         try:
             binpath = self.get_binary_path("geckodriver")
-        except Exception as e:
-            print("It looks like geckodriver isn't built. "
-                  "Add ac_add_options --enable-geckodriver to your "
-                  "mozconfig ",
-                  "and run |mach build| to build it.")
-            print(e)
+        except BinaryNotFoundException as e:
+            self.log(logging.ERROR, 'geckodriver',
+                     {'error': str(e)},
+                     'ERROR: {error}')
+            self.log(logging.INFO, 'geckodriver', {},
+                     "It looks like geckodriver isn't built. "
+                     "Add ac_add_options --enable-geckodriver to your "
+                     "mozconfig "
+                     "and run |./mach build| to build it.")
             return 1
 
         args = [binpath]
@@ -55,7 +58,16 @@ class GeckoDriver(MachCommandBase):
             args.extend(params)
 
         if binary is None:
-            binary = self.get_binary_path("app")
+            try:
+                binary = self.get_binary_path("app")
+            except BinaryNotFoundException as e:
+                self.log(logging.ERROR, 'geckodriver',
+                         {'error': str(e)},
+                         'ERROR: {error}')
+                self.log(logging.INFO, 'geckodriver',
+                         {'help': e.help()},
+                         '{help}')
+                return 1
 
         args.extend(["--binary", binary])
 

testing/marionette/mach_commands.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import functools
+import logging
 import os
 import sys
 
@@ -19,6 +20,7 @@ from mach.decorators import (
 from mozbuild.base import (
     MachCommandBase,
     MachCommandConditions as conditions,
+    BinaryNotFoundException,
 )
 
 SUPPORTED_APPS = ['firefox', 'android', 'thunderbird']
@@ -88,6 +90,15 @@ class MarionetteTest(MachCommandBase):
 
         if not kwargs.get("binary") and \
                 (conditions.is_firefox(self) or conditions.is_thunderbird(self)):
-            kwargs["binary"] = self.get_binary_path("app")
+            try:
+                kwargs["binary"] = self.get_binary_path("app")
+            except BinaryNotFoundException as e:
+                self.log(logging.ERROR, 'marionette-test',
+                         {'error': str(e)},
+                         'ERROR: {error}')
+                self.log(logging.INFO, 'marionette-test',
+                         {'help': e.help()},
+                         '{help}')
+                return 1
 
         return run_marionette(tests, topsrcdir=self.topsrcdir, **kwargs)

testing/mochitest/mochitest_options.py

@@ -11,6 +11,7 @@ from urlparse import urlparse
 import json
 import os
 import tempfile
+import sys
 
 from mozprofile import DEFAULT_PORTS
 import mozinfo
@@ -632,7 +633,12 @@ class MochitestArguments(ArgumentContainer):
         if parser.app != 'android':
             if options.app is None:
                 if build_obj:
-                    options.app = build_obj.get_binary_path()
+                    from mozbuild.base import BinaryNotFoundException
+                    try:
+                        options.app = build_obj.get_binary_path()
+                    except BinaryNotFoundException as e:
+                        print('{}\n\n{}\n'.format(e, e.help()))
+                        sys.exit(1)
                 else:
                     parser.error(
                         "could not find the application path, --appname must be specified")

testing/raptor/mach_commands.py

@@ -9,6 +9,7 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
+import logging
 import os
 import shutil
 import socket
@@ -18,7 +19,11 @@ import sys
 import mozfile
 from mach.decorators import Command, CommandProvider
 from mozboot.util import get_state_dir
-from mozbuild.base import MachCommandBase, MozbuildObject
+from mozbuild.base import (
+    MachCommandBase,
+    MozbuildObject,
+    BinaryNotFoundException,
+)
 from mozbuild.base import MachCommandConditions as Conditions
 from raptor.power import enable_charging, disable_charging
 
@@ -241,6 +246,14 @@ class MachRaptor(MachCommandBase):
                 device = ADBAndroid(verbose=True)
                 disable_charging(device)
             return raptor.run_test(sys.argv[2:], kwargs)
+        except BinaryNotFoundException as e:
+            self.log(logging.ERROR, 'raptor',
+                     {'error': str(e)},
+                     'ERROR: {error}')
+            self.log(logging.INFO, 'raptor',
+                     {'help': e.help()},
+                     '{help}')
+            return 1
         except Exception as e:
             print(repr(e))
             return 1

testing/talos/mach_commands.py

@@ -6,12 +6,17 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import logging
 import os
 import sys
 import json
 import socket
 
-from mozbuild.base import MozbuildObject, MachCommandBase
+from mozbuild.base import (
+    MozbuildObject,
+    MachCommandBase,
+    BinaryNotFoundException,
+)
 from mach.decorators import CommandProvider, Command
 
 HERE = os.path.dirname(os.path.realpath(__file__))
@@ -26,7 +31,17 @@ class TalosRunner(MozbuildObject):
         3. Run mozharness
         """
 
-        self.init_variables(talos_args)
+        try:
+            self.init_variables(talos_args)
+        except BinaryNotFoundException as e:
+            self.log(logging.ERROR, 'talos',
+                     {'error': str(e)},
+                     'ERROR: {error}')
+            self.log(logging.INFO, 'raptor',
+                     {'help': e.help()},
+                     '{help}')
+            return 1
+
         self.make_config()
         self.write_config()
         self.make_args()

testing/web-platform/mach_commands_base.py

@@ -27,13 +27,19 @@ class WebPlatformTestsRunner(object):
 
     def run(self, logger, **kwargs):
         from wptrunner import wptrunner
+        from mozbuild.base import BinaryNotFoundException
 
         if kwargs["manifest_update"] is not False:
             self.update_manifest(logger)
         kwargs["manifest_update"] = False
 
         if kwargs["product"] in ["firefox", None]:
-            kwargs = self.setup.kwargs_firefox(kwargs)
+            try:
+                kwargs = self.setup.kwargs_firefox(kwargs)
+            except BinaryNotFoundException as e:
+                logger.error(e)
+                logger.info(e.help())
+                return 1
         elif kwargs["product"] == "firefox_android":
             from wptrunner import wptcommandline
             kwargs = wptcommandline.check_args(self.setup.kwargs_common(kwargs))

testing/xpcshell/mach_commands.py

@@ -7,6 +7,7 @@
 from __future__ import absolute_import, unicode_literals, print_function
 
 import errno
+import logging
 import os
 import sys
 
@@ -16,6 +17,7 @@ from mozbuild.base import (
     MachCommandBase,
     MozbuildObject,
     MachCommandConditions as conditions,
+    BinaryNotFoundException,
 )
 
 from mach.decorators import (
@@ -80,7 +82,16 @@ class XPCShellRunner(MozbuildObject):
             kwargs["verbose"] = True
 
         if kwargs["xpcshell"] is None:
-            kwargs["xpcshell"] = self.get_binary_path('xpcshell')
+            try:
+                kwargs["xpcshell"] = self.get_binary_path('xpcshell')
+            except BinaryNotFoundException as e:
+                self.log(logging.ERROR, 'xpcshell-test',
+                         {'error': str(e)},
+                         'ERROR: {error}')
+                self.log(logging.INFO, 'xpcshell-test',
+                         {'help': e.help()},
+                         '{help}')
+                return 1
 
         if kwargs["mozInfo"] is None:
             kwargs["mozInfo"] = os.path.join(self.topobjdir, 'mozinfo.json')

toolkit/components/telemetry/tests/marionette/mach_commands.py

@@ -4,12 +4,17 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
+import logging
 import os
 import sys
 
 from mach.decorators import CommandProvider, Command
 
-from mozbuild.base import MachCommandBase, MachCommandConditions as conditions
+from mozbuild.base import (
+    MachCommandBase,
+    MachCommandConditions as conditions,
+    BinaryNotFoundException,
+)
 
 
 def create_parser_tests():
@@ -75,7 +80,16 @@ class TelemetryTest(MachCommandBase):
                 tests.append(obj["file_relpath"])
             del kwargs["test_objects"]
         if not kwargs.get("binary") and conditions.is_firefox(self):
-            kwargs["binary"] = self.get_binary_path("app")
+            try:
+                kwargs["binary"] = self.get_binary_path("app")
+            except BinaryNotFoundException as e:
+                self.log(logging.ERROR, 'telemetry-tests-client',
+                         {'error': str(e)},
+                         'ERROR: {error}')
+                self.log(logging.INFO, 'telemetry-tests-client',
+                         {'help': e.help()},
+                         '{help}')
+                return 1
         if not kwargs.get("server_root"):
             kwargs["server_root"] = "toolkit/components/telemetry/tests/marionette/harness/www"
         return run_telemetry(tests, topsrcdir=self.topsrcdir, **kwargs)

tools/browsertime/mach_commands.py

@@ -42,7 +42,7 @@ import contextlib
 
 from six import StringIO
 from mach.decorators import CommandArgument, CommandProvider, Command
-from mozbuild.base import MachCommandBase
+from mozbuild.base import MachCommandBase, BinaryNotFoundException
 from mozbuild.util import mkdir
 import mozpack.path as mozpath
 
@@ -503,9 +503,16 @@ class MachBrowsertime(MachCommandBase):
             if not specifies_binaryPath:
                 try:
                     extra_args.extend(('--firefox.binaryPath', self.get_binary_path()))
-                except Exception:
-                    print('Please run |./mach build| '
-                          'or specify a Firefox binary with --firefox.binaryPath.')
+                except BinaryNotFoundException as e:
+                    self.log(logging.ERROR,
+                             'browsertime',
+                             {'error': str(e)},
+                             'ERROR: {error}')
+                    self.log(logging.INFO,
+                             'browsertime',
+                             {},
+                             'Please run |./mach build| '
+                             'or specify a Firefox binary with --firefox.binaryPath.')
                     return 1
 
         if extra_args: