Bug 1656740 - Integrate `clangd` in `vscode` for C++ language support. r=froydnj

In order to have a cross platform ide for C++ language support we've added `clangd`
extenssion and artifact part of `vscode` suite.
To generate the configuration you simply run:
`./mach ide vscode `.

Differential Revision: https://phabricator.services.mozilla.com/D85416

.vscode/extensions.json

@@ -8,8 +8,8 @@
     "dbaeumer.vscode-eslint",
     // Prettier support.
     "esbenp.prettier-vscode",
-    // C/C++ language support.
-    "ms-vscode.cpptools",
+    // C/C++ language support with clangd
+    "llvm-vs-code-extensions.vscode-clangd",
     // Rust language support.
     "rust-lang.rust",
     // Provides support for rust-analyzer: novel LSP server for the Rust programming language.

docs/contributing/editor.rst

@@ -16,14 +16,21 @@ Visual Studio Code
 
 For general information on using VS Code, see their
 `home page <https://code.visualstudio.com/>`__,
-`repo <https://github.com/Microsoft/vscode/>`__ and
-`guide to working with C++ <https://code.visualstudio.com/docs/languages/cpp>`__.
+`repo <https://github.com/Microsoft/vscode/>`__.
 
-For IntelliSense to work properly, a
-:ref:`compilation database <CompileDB back-end / compileflags>` as described
-below is required. When it is present when you open the mozilla source code
-folder, it will be automatically detected and Visual Studio Code will ask you
-if it should use it, which you should confirm.
+For C++ support we offer an out of the box configuration based on
+`clangd <https://clangd.llvm.org>`__. This covers code completion, compile errors,
+go-to-definition and more.
+
+In order to build the configuration for `VS Code` simply run from
+the terminal:
+
+`./mach ide vscode`
+
+If `VS Code` is already open with a previous configuration generated, please make sure to
+restart `VS Code` otherwise the new configuration will not be used, and the `compile_commands.json`
+needed by `clangd` server will not be refreshed. This is a known `bug <https://github.com/clangd/vscode-clangd/issues/42>`__
+in `clangd-vscode` extension
 
 VS Code provides number of extensions for JavaScript, Rust, etc.
 

python/mozbuild/mozbuild/backend/__init__.py

@@ -5,6 +5,7 @@
 from __future__ import absolute_import, print_function
 
 backends = {
+    'Clangd': 'mozbuild.backend.clangd',
     'ChromeMap': 'mozbuild.codecoverage.chrome_map',
     'CompileDB': 'mozbuild.compilation.database',
     'CppEclipse': 'mozbuild.backend.cpp_eclipse',

python/mozbuild/mozbuild/backend/clangd.py

@@ -0,0 +1,47 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides a backend for `clangd` in order to have support for
+# code completion, compile errors, go-to-definition and more.
+# It is based on `database.py` with the difference that we don't generate
+# an unified `compile_commands.json` but we generate a per file basis `command` in
+# `objdir/clangd/compile_commands.json`
+
+from __future__ import absolute_import, print_function
+
+import os
+
+from mozbuild.compilation.database import CompileDBBackend
+
+import mozpack.path as mozpath
+
+
+class ClangdBackend(CompileDBBackend):
+    """
+    Configuration that generates the backend for clangd, it is used with `clangd`
+    extension for vscode
+    """
+
+    def _init(self):
+        CompileDBBackend._init(self)
+
+    def _build_cmd(self, cmd, filename, unified):
+        cmd = list(cmd)
+
+        cmd.append(filename)
+
+        return cmd
+
+    def _outputfile_path(self):
+        clangd_cc_path = os.path.join(self.environment.topobjdir, "clangd")
+
+        if not os.path.exists(clangd_cc_path):
+            os.mkdir(clangd_cc_path)
+
+        # Output the database (a JSON file) to objdir/clangd/compile_commands.json
+        return mozpath.join(clangd_cc_path, "compile_commands.json")
+
+    def _process_unified_sources(self, obj):
+        for f in list(sorted(obj.files)):
+            self._build_db_line(obj.objdir, obj.relsrcdir, obj.config, f, obj.canonical_suffix)

python/mozbuild/mozbuild/backend/mach_commands.py

@@ -5,10 +5,13 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
+import logging
 import os
 import subprocess
 
 from mozbuild.base import MachCommandBase
+from mozbuild.build_commands import Build
+
 from mozfile import which
 from mach.decorators import (
     CommandArgument,
@@ -16,51 +19,278 @@ from mach.decorators import (
     Command,
 )
 
+import mozpack.path as mozpath
+
 
 @CommandProvider
 class MachCommands(MachCommandBase):
-    @Command('ide', category='devenv',
-             description='Generate a project and launch an IDE.')
-    @CommandArgument('ide', choices=['eclipse', 'visualstudio'])
-    @CommandArgument('args', nargs=argparse.REMAINDER)
+    @Command("ide", category="devenv", description="Generate a project and launch an IDE.")
+    @CommandArgument("ide", choices=["eclipse", "visualstudio", "vscode"])
+    @CommandArgument("args", nargs=argparse.REMAINDER)
     def eclipse(self, ide, args):
-        if ide == 'eclipse':
-            backend = 'CppEclipse'
-        elif ide == 'visualstudio':
-            backend = 'VisualStudio'
+        if ide == "eclipse":
+            backend = "CppEclipse"
+        elif ide == "visualstudio":
+            backend = "VisualStudio"
+        elif ide == "vscode":
+            backend = "Clangd"
 
-        if ide == 'eclipse' and not which('eclipse'):
-            print('Eclipse CDT 8.4 or later must be installed in your PATH.')
-            print('Download: http://www.eclipse.org/cdt/downloads.php')
+        if ide == "eclipse" and not which("eclipse"):
+            self.log(
+                logging.ERROR,
+                "ide",
+                {},
+                "Eclipse CDT 8.4 or later must be installed in your PATH.",
+            )
+            self.log(
+                logging.ERROR, "ide", {}, "Download: http://www.eclipse.org/cdt/downloads.php"
+            )
             return 1
 
-        # Here we refresh the whole build. 'build export' is sufficient here and is probably more
-        # correct but it's also nice having a single target to get a fully built and indexed
-        # project (gives a easy target to use before go out to lunch).
-        res = self._mach_context.commands.dispatch('build', self._mach_context)
-        if res != 0:
-            return 1
+        if ide == "vscode":
+            # Verify if platform has VSCode installed
+            if not self.found_vscode_path():
+                self.log(logging.ERROR, "ide", {}, "VSCode cannot be found, abording!")
+                return 1
+
+            # Create the Build environment to configure the tree
+            builder = Build(self._mach_context)
+
+            rc = builder.configure()
+            if rc != 0:
+                return rc
+
+            # First install what we can through install manifests.
+            rc = builder._run_make(
+                directory=self.topobjdir, target="pre-export", line_handler=None
+            )
+            if rc != 0:
+                return rc
+
+            # Then build the rest of the build dependencies by running the full
+            # export target, because we can't do anything better.
+            for target in ("export", "pre-compile"):
+                rc = builder._run_make(directory=self.topobjdir, target=target, line_handler=None)
+                if rc != 0:
+                    return rc
+        else:
+            # Here we refresh the whole build. 'build export' is sufficient here and is
+            # probably more correct but it's also nice having a single target to get a fully
+            # built and indexed project (gives a easy target to use before go out to lunch).
+            res = self._mach_context.commands.dispatch("build", self._mach_context)
+            if res != 0:
+                return 1
 
         # Generate or refresh the IDE backend.
         python = self.virtualenv_manager.python_path
-        config_status = os.path.join(self.topobjdir, 'config.status')
-        args = [python, config_status, '--backend=%s' % backend]
+        config_status = os.path.join(self.topobjdir, "config.status")
+        args = [python, config_status, "--backend=%s" % backend]
         res = self._run_command_in_objdir(args=args, pass_thru=True, ensure_exit_code=False)
         if res != 0:
             return 1
 
-        if ide == 'eclipse':
+        if ide == "eclipse":
             eclipse_workspace_dir = self.get_eclipse_workspace_path()
-            subprocess.check_call(['eclipse', '-data', eclipse_workspace_dir])
-        elif ide == 'visualstudio':
+            subprocess.check_call(["eclipse", "-data", eclipse_workspace_dir])
+        elif ide == "visualstudio":
             visual_studio_workspace_dir = self.get_visualstudio_workspace_path()
-            subprocess.check_call(
-                ['explorer.exe', visual_studio_workspace_dir]
-            )
+            subprocess.check_call(["explorer.exe", visual_studio_workspace_dir])
+        elif ide == "vscode":
+            return self.setup_vscode()
 
     def get_eclipse_workspace_path(self):
         from mozbuild.backend.cpp_eclipse import CppEclipseBackend
+
         return CppEclipseBackend.get_workspace_path(self.topsrcdir, self.topobjdir)
 
     def get_visualstudio_workspace_path(self):
-        return os.path.join(self.topobjdir, 'msvc', 'mozilla.sln')
+        return os.path.join(self.topobjdir, "msvc", "mozilla.sln")
+
+    def found_vscode_path(self):
+
+        if "linux" in self.platform[0]:
+            self.vscode_path = "/usr/bin/code"
+        elif "macos" in self.platform[0]:
+            self.vscode_path = "/usr/local/bin/code"
+        elif "win64" in self.platform[0]:
+            from pathlib import Path
+
+            self.vscode_path = mozpath.join(
+                str(Path.home()), "AppData", "Local", "Programs", "Microsoft VS Code", "Code.exe",
+            )
+
+        # Path found
+        if os.path.exists(self.vscode_path):
+            return True
+
+        for _ in range(5):
+            self.vscode_path = input(
+                "Could not find the VSCode binary. Please provide the full path to it:\n"
+            )
+            if os.path.exists(self.vscode_path):
+                return True
+
+        # Path cannot be found
+        return False
+
+    def setup_vscode(self):
+        vscode_settings = mozpath.join(self.topsrcdir, ".vscode", "settings.json")
+
+        clangd_cc_path = mozpath.join(self.topobjdir, "clangd")
+
+        # Verify if the required files are present
+        clang_tools_path = mozpath.join(self._mach_context.state_dir, "clang-tools")
+        clang_tidy_bin = mozpath.join(clang_tools_path, "clang-tidy", "bin")
+
+        clangd_path = mozpath.join(
+            clang_tidy_bin, "clangd" + self.config_environment.substs.get("BIN_SUFFIX", ""),
+        )
+
+        if not os.path.exists(clangd_path):
+            self.log(
+                logging.ERROR, "ide", {}, "Unable to locate clangd in {}.".format(clang_tidy_bin)
+            )
+            rc = self._get_clang_tools(clang_tools_path)
+
+            if rc != 0:
+                return rc
+
+        import multiprocessing
+        import json
+
+        clangd_json = json.loads(
+            """
+        {
+            "clangd.path": "%s",
+            "clangd.arguments": [
+                "--compile-commands-dir",
+                "%s",
+                "-j",
+                "%s",
+                "--limit-results",
+                "0",
+                "--completion-style",
+                "detailed",
+                "--background-index",
+                "--all-scopes-completion",
+                "--log",
+                "error",
+                "--pch-storage",
+                "memory"
+            ]
+        }
+        """
+            % (clangd_path, clangd_cc_path, multiprocessing.cpu_count(),)
+        )
+
+        # Create an empty settings dictionary
+        settings = {}
+
+        # Modify the .vscode/settings.json configuration file
+        if os.path.exists(vscode_settings):
+            # If exists prompt for a configuration change
+            choice = prompt_bool(
+                "Configuration for {settings} must change. "
+                "Do you want to proceed?".format(settings=vscode_settings)
+            )
+            if not choice:
+                return 1
+
+            # Read the original vscode settings
+            with open(vscode_settings) as fh:
+                try:
+                    settings = json.load(fh)
+                    print(
+                        "The following modifications will occur:\nOriginal:\n{orig}\n"
+                        "New:\n{new}".format(
+                            orig=json.dumps(
+                                {
+                                    key: settings[key] if key in settings else ""
+                                    for key in ["clangd.path", "clangd.arguments"]
+                                },
+                                indent=4,
+                            ),
+                            new=json.dumps(clangd_json, indent=4),
+                        )
+                    )
+
+                except ValueError:
+                    # Decoding has failed, work with an empty dict
+                    settings = {}
+
+        # Write our own Configuration
+        settings["clangd.path"] = clangd_json["clangd.path"]
+        settings["clangd.arguments"] = clangd_json["clangd.arguments"]
+
+        with open(vscode_settings, "w") as fh:
+            fh.write(json.dumps(settings, indent=4))
+
+        # Open vscode with new configuration
+        rc = subprocess.call([self.vscode_path, self.topsrcdir])
+
+        if rc != 0:
+            self.log(
+                logging.ERROR,
+                "ide",
+                {},
+                "Unable to open VS Code. Please open VS Code manually and load "
+                "directory: {}".format(self.topsrcdir),
+            )
+            return rc
+
+        return 0
+
+    def _get_clang_tools(self, clang_tools_path):
+
+        import shutil
+
+        if os.path.isdir(clang_tools_path):
+            shutil.rmtree(clang_tools_path)
+
+        # Create base directory where we store clang binary
+        os.mkdir(clang_tools_path)
+
+        from mozbuild.artifact_commands import PackageFrontend
+
+        self._artifact_manager = PackageFrontend(self._mach_context)
+
+        job, _ = self.platform
+
+        if job is None:
+            self.log(
+                logging.ERROR,
+                "ide",
+                {},
+                "The current platform isn't supported. "
+                "Currently only the following platforms are "
+                "supported: win32/win64, linux64 and macosx64.",
+            )
+            return 1
+
+        job += "-clang-tidy"
+
+        # We want to unpack data in the clang-tidy mozbuild folder
+        currentWorkingDir = os.getcwd()
+        os.chdir(clang_tools_path)
+        rc = self._artifact_manager.artifact_toolchain(
+            verbose=False, from_build=[job], no_unpack=False, retry=0
+        )
+        # Change back the cwd
+        os.chdir(currentWorkingDir)
+
+        return rc
+
+
+def prompt_bool(prompt, limit=5):
+    """ Prompts the user with prompt and requires a boolean value. """
+    from distutils.util import strtobool
+
+    for _ in range(limit):
+        try:
+            return strtobool(input(prompt + " [Y/N]\n"))
+        except ValueError:
+            print(
+                "ERROR! Please enter a valid option! Please use any of the following:"
+                " Y, N, True, False, 1, 0"
+            )
+    return False

python/mozbuild/mozbuild/compilation/database.py

@@ -42,6 +42,15 @@ class CompileDBBackend(CommonBackend):
         self._local_flags = defaultdict(dict)
         self._per_source_flags = defaultdict(list)
 
+    def _build_cmd(self, cmd, filename, unified):
+        cmd = list(cmd)
+        if unified is None:
+            cmd.append(filename)
+        else:
+            cmd.append(unified)
+
+        return cmd
+
     def consume_object(self, obj):
         # Those are difficult directories, that will be handled later.
         if obj.relsrcdir in (
@@ -86,11 +95,7 @@ class CompileDBBackend(CommonBackend):
 
         for (directory, filename, unified), cmd in self._db.items():
             env = self._envs[directory]
-            cmd = list(cmd)
-            if unified is None:
-                cmd.append(filename)
-            else:
-                cmd.append(unified)
+            cmd = self._build_cmd(cmd, filename, unified)
             variables = {
                 'DIST': mozpath.join(env.topobjdir, 'dist'),
                 'DEPTH': env.topobjdir,
@@ -136,11 +141,14 @@ class CompileDBBackend(CommonBackend):
             })
 
         import json
-        # Output the database (a JSON file) to objdir/compile_commands.json
-        outputfile = os.path.join(self.environment.topobjdir, 'compile_commands.json')
+        outputfile = self._outputfile_path()
         with self._write_file(outputfile) as jsonout:
             json.dump(db, jsonout, indent=0)
 
+    def _outputfile_path(self):
+        # Output the database (a JSON file) to objdir/compile_commands.json
+        return os.path.join(self.environment.topobjdir, 'compile_commands.json')
+
     def _process_unified_sources(self, obj):
         if not obj.have_unified_mapping:
             for f in list(sorted(obj.files)):