Bug 1750632: `./mach lint` should bootstrap `clang-format` r=ahal

Tie into `code_analysis`'s `get_clang_tools()` functionality to
intelligently bootstrap clang if it either doesn't exist or is
out-of-date.

This required exposing `command_context` to the linting logic, as it's
needed to call `artifact_toolchain(...)`.

Note that this means that the standalone `runcli.py` file won't be able
to support bootstrapping `clang-format`, or other linters that lean on
`command_context` in the future.

Finally, `substs.get("HOST_BIN_SUFFIX")` was replaced with a
windows-specific `binary += ".exe"`, because not all contexts where
the tests are run will have access to populated `substs` data.
Note that this worked before without the extension because it was
only used for starting a process, in which context Windows automatically
tries all `PATHEXT` options. Since we're now doing an `isfile()` check
(to enable more intelligent failure cases when `clang-format` doesn't
exist), we need the path to be fully correct.

Differential Revision: https://phabricator.services.mozilla.com/D137335

python/mozbuild/mozbuild/backend/mach_commands.py

@@ -219,7 +219,7 @@ def setup_vscode(command_context, vscode_cmd):
             {},
             "Unable to locate clangd in {}.".format(clang_tidy_bin),
         )
-        rc = _get_clang_tools(command_context, clang_tools_path)
+        rc = get_clang_tools(command_context, clang_tools_path)
 
         if rc != 0:
             return rc
@@ -336,7 +336,7 @@ def setup_vscode(command_context, vscode_cmd):
     return 0
 
 
-def _get_clang_tools(command_context, clang_tools_path):
+def get_clang_tools(command_context, clang_tools_path):
 
     import shutil
 

python/mozbuild/mozbuild/code_analysis/mach_commands.py

@@ -328,7 +328,7 @@ def check(
     command_context.activate_virtualenv()
     command_context.log_manager.enable_unstructured()
 
-    rc, clang_paths = _get_clang_tools(command_context, verbose=verbose)
+    rc, clang_paths = get_clang_tools(command_context, verbose=verbose)
     if rc != 0:
         return rc
 
@@ -691,7 +691,7 @@ def autotest(
         # Ensure that clang-tidy is present
         rc = not os.path.exists(clang_paths._clang_tidy_path)
     else:
-        rc, clang_paths = _get_clang_tools(
+        rc, clang_paths = get_clang_tools(
             command_context, force=force_download, verbose=verbose
         )
 
@@ -1030,7 +1030,7 @@ def install(
     verbose=False,
 ):
     command_context._set_log_level(verbose)
-    rc, _ = _get_clang_tools(
+    rc, _ = get_clang_tools(
         command_context,
         force=force,
         skip_cache=skip_cache,
@@ -1047,7 +1047,7 @@ def install(
 )
 def clear_cache(command_context, verbose=False):
     command_context._set_log_level(verbose)
-    rc, _ = _get_clang_tools(
+    rc, _ = get_clang_tools(
         command_context,
         force=True,
         download_if_needed=True,
@@ -1070,7 +1070,7 @@ def clear_cache(command_context, verbose=False):
 )
 def print_checks(command_context, verbose=False):
     command_context._set_log_level(verbose)
-    rc, clang_paths = _get_clang_tools(command_context, verbose=verbose)
+    rc, clang_paths = get_clang_tools(command_context, verbose=verbose)
 
     if rc != 0:
         return rc
@@ -1247,7 +1247,7 @@ def clang_format(
         if not _is_version_eligible(command_context, clang_paths):
             return 1
     else:
-        rc, clang_paths = _get_clang_tools(command_context, verbose=verbose)
+        rc, clang_paths = get_clang_tools(command_context, verbose=verbose)
         if rc != 0:
             return rc
 
@@ -1561,7 +1561,7 @@ def _do_clang_tools_exist(clang_paths):
     )
 
 
-def _get_clang_tools(
+def get_clang_tools(
     command_context,
     force=False,
     skip_cache=False,
@@ -1586,7 +1586,7 @@ def _get_clang_tools(
         # The directory exists, perhaps it's corrupted?  Delete it
         # and start from scratch.
         shutil.rmtree(clang_paths._clang_tools_path)
-        return _get_clang_tools(
+        return get_clang_tools(
             command_context,
             force=force,
             skip_cache=skip_cache,

python/mozlint/mozlint/cli.py

@@ -322,6 +322,7 @@ def run(
     list_linters=False,
     num_procs=None,
     virtualenv_manager=None,
+    setupargs=None,
     **lintargs
 ):
     from mozlint import LintRoller, formatters
@@ -346,7 +347,7 @@ def run(
         )
         return 0
 
-    lint = LintRoller(**lintargs)
+    lint = LintRoller(setupargs=setupargs or {}, **lintargs)
     linters_info = find_linters(lintargs["config_paths"], linters)
 
     result = None

python/mozlint/mozlint/roller.py

@@ -164,7 +164,7 @@ class LintRoller(object):
         50  # set a max size to prevent command lines that are too long on Windows
     )
 
-    def __init__(self, root, exclude=None, **lintargs):
+    def __init__(self, root, exclude=None, setupargs=None, **lintargs):
         self.parse = Parser(root)
         try:
             self.vcs = get_repository_object(root)
@@ -174,6 +174,7 @@ class LintRoller(object):
         self.linters = []
         self.lintargs = lintargs
         self.lintargs["root"] = root
+        self._setupargs = setupargs or {}
 
         # result state
         self.result = ResultSummary(root)
@@ -216,6 +217,7 @@ class LintRoller(object):
 
             try:
                 setupargs = copy.deepcopy(self.lintargs)
+                setupargs.update(self._setupargs)
                 setupargs["name"] = linter["name"]
                 setupargs["log"] = logging.LoggerAdapter(
                     self.log, {"lintname": linter["name"]}
@@ -223,7 +225,9 @@ class LintRoller(object):
                 if virtualenv_manager is not None:
                     setupargs["virtualenv_manager"] = virtualenv_manager
                 start_time = time.time()
-                res = findobject(linter["setup"])(**setupargs)
+                res = findobject(linter["setup"])(
+                    **setupargs,
+                )
                 self.log.debug(
                     f"setup for {linter['name']} finished in "
                     f"{round(time.time() - start_time, 2)} seconds"

tools/lint/clang-format.yml

@@ -9,3 +9,4 @@ clang-format:
     type: external
     payload: clang-format:lint
     code_review_warnings: false
+    setup: clang-format:setup

tools/lint/clang-format/__init__.py

@@ -5,22 +5,31 @@
 import os
 import signal
 import re
+import sys
 
-from buildconfig import substs
 from mozboot.util import get_tools_dir
 from mozlint import result
 from mozlint.pathutils import expand_exclusions
 from mozprocess import ProcessHandler
 
 CLANG_FORMAT_NOT_FOUND = """
-Could not find clang-format! Install clang-format with:
-
-    $ ./mach bootstrap
-
-And make sure that it is in the PATH
+Could not find clang-format! It should've been installed automatically - \
+please report a bug here:
+https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox%20Build%20System&component=Lint%20and%20Formatting
 """.strip()
 
 
+def setup(root, mach_command_context, **lintargs):
+    if get_clang_format_binary():
+        return 0
+
+    from mozbuild.code_analysis.mach_commands import get_clang_tools
+
+    rc, _ = get_clang_tools(mach_command_context)
+    if rc:
+        return 1
+
+
 def parse_issues(config, output, paths, log):
 
     diff_line = re.compile("^(.*):(.*):(.*): warning: .*;(.*);(.*)")
@@ -81,7 +90,15 @@ def get_clang_format_binary():
 
     clang_tools_path = os.path.join(get_tools_dir(), "clang-tools")
     bin_path = os.path.join(clang_tools_path, "clang-tidy", "bin")
-    return os.path.join(bin_path, "clang-format" + substs.get("HOST_BIN_SUFFIX", ""))
+    binary = os.path.join(bin_path, "clang-format")
+
+    if sys.platform.startswith("win"):
+        binary += ".exe"
+
+    if not os.path.isfile(binary):
+        return None
+
+    return binary
 
 
 def is_ignored_path(ignored_dir_re, topsrcdir, f):

tools/lint/mach_commands.py

@@ -96,7 +96,10 @@ def lint(command_context, *runargs, **lintargs):
         parser.GLOBAL_SUPPORT_FILES.append(
             os.path.join(command_context.topsrcdir, path)
         )
-    return cli.run(*runargs, **lintargs)
+    setupargs = {
+        "mach_command_context": command_context,
+    }
+    return cli.run(*runargs, setupargs=setupargs, **lintargs)
 
 
 @Command(

tools/lint/test/conftest.py

@@ -93,6 +93,11 @@ def run_setup(config):
     if "setup" not in config:
         return
 
+    if config["name"] == "clang-format":
+        # Skip the setup for the clang-format linter, as it requires a Mach context
+        # (which we may not have if pytest is invoked directly).
+        return
+
     log = logging.LoggerAdapter(
         logger, {"lintname": config.get("name"), "pid": os.getpid()}
     )