Bug 1613823 - Serialize cargo invocations in the build system, prioritizing gecko. r=froydnj

Differential Revision: https://phabricator.services.mozilla.com/D61961

--HG--
extra : moz-landing-system : lando

build/templates.mozbuild

@@ -62,7 +62,7 @@ def DisableCompilerWarnings():
     COMPILE_FLAGS['WARNINGS_CFLAGS'] = []
 
 @template
-def RustLibrary(name, features=None, output_category=None):
+def RustLibrary(name, features=None, output_category=None, is_gkrust=False):
     '''Template for Rust libraries.'''
     Library(name)
 
@@ -81,6 +81,9 @@ def RustLibrary(name, features=None, output_category=None):
     if output_category:
         RUST_LIBRARY_OUTPUT_CATEGORY = output_category
 
+    if is_gkrust:
+        IS_GKRUST = True
+
 
 @template
 def SharedLibrary(name, output_category=None):

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -79,6 +79,7 @@ from ..util import (
     ensureParentDir,
     FileAvoidWrite,
     OrderedDefaultDict,
+    pairwise,
 )
 from ..makeutil import Makefile
 from mozbuild.shellutil import quote as shell_quote
@@ -406,6 +407,8 @@ class RecursiveMakeBackend(MakeBackend):
         self._traversal = RecursiveMakeTraversal()
         self._compile_graph = OrderedDefaultDict(set)
         self._rust_targets = set()
+        self._rust_lib_targets = set()
+        self._gkrust_target = None
 
         self._no_skip = {
             'export': set(),
@@ -638,6 +641,9 @@ class RecursiveMakeBackend(MakeBackend):
             build_target = self._build_target_for_obj(obj)
             self._compile_graph[build_target]
             self._rust_targets.add(build_target)
+            self._rust_lib_targets.add(build_target)
+            if obj.is_gkrust:
+                self._gkrust_target = build_target
 
         elif isinstance(obj, SharedLibrary):
             self._process_shared_library(obj, backend_file)
@@ -776,9 +782,21 @@ class RecursiveMakeBackend(MakeBackend):
             # on other directories in the tree, so putting them first here will
             # start them earlier in the build.
             rust_roots = [r for r in roots if r in self._rust_targets]
+            rust_libs = [r for r in roots if r in self._rust_lib_targets]
             if category == 'compile' and rust_roots:
                 rust_rule = root_deps_mk.create_rule(['recurse_rust'])
                 rust_rule.add_dependencies(rust_roots)
+                # Ensure our cargo invocations are serialized, and gecko comes
+                # first. Cargo will lock on the build output directory anyway,
+                # so trying to run things in parallel is not useful. Dependencies
+                # for gecko are especially expensive to build and parallelize
+                # poorly, so prioritizing these will save some idle time in full
+                # builds.
+                for prior_target, target in pairwise(
+                        sorted([t for t in rust_libs],
+                               key=lambda t: t != self._gkrust_target)):
+                    r = root_deps_mk.create_rule([target])
+                    r.add_dependencies([prior_target])
 
             rule.add_dependencies(chain(rust_roots, roots))
             for target, deps in sorted(graph.items()):

python/mozbuild/mozbuild/frontend/context.py

@@ -1335,6 +1335,15 @@ VARIABLES = {
         """
         ),
 
+    'IS_GKRUST': (
+        bool,
+        bool,
+        """Whether the current library defined by this moz.build is gkrust.
+
+        Indicates whether the current library contains rust for libxul.
+        """
+        ),
+
     'RUST_LIBRARY_FEATURES': (
         List,
         list,
@@ -2399,6 +2408,7 @@ TEMPLATE_VARIABLES = {
     'HOST_LIBRARY_NAME',
     'HOST_SIMPLE_PROGRAMS',
     'IS_FRAMEWORK',
+    'IS_GKRUST',
     'LIBRARY_NAME',
     'PROGRAM',
     'SIMPLE_PROGRAMS',
@@ -2831,6 +2841,18 @@ DEPRECATION_HINTS = {
             IS_FRAMEWORK = True
         ''',
 
+
+    'IS_GKRUST': '''
+        Please use
+
+            RustLibrary('gkrust', ... is_gkrust=True)
+
+        instead of
+
+            RustLibrary('gkrust') [ or LIBRARY_NAME = 'gkrust' ]
+            IS_GKRUST = True
+        ''',
+
     'TOOL_DIRS': 'Please use the DIRS variable instead.',
 
     'TEST_TOOL_DIRS': 'Please use the TEST_DIRS variable instead.',

python/mozbuild/mozbuild/frontend/data.py

@@ -705,10 +705,12 @@ class BaseRustLibrary(object):
         'features',
         'target_dir',
         'output_category',
+        'is_gkrust',
     )
 
     def init(self, context, basename, cargo_file, crate_type, dependencies,
-             features, target_dir):
+             features, target_dir, is_gkrust):
+        self.is_gkrust = is_gkrust
         self.cargo_file = cargo_file
         self.crate_type = crate_type
         # We need to adjust our naming here because cargo replaces '-' in
@@ -745,13 +747,14 @@ class RustLibrary(StaticLibrary, BaseRustLibrary):
     __slots__ = BaseRustLibrary.slots
 
     def __init__(self, context, basename, cargo_file, crate_type, dependencies,
-                 features, target_dir, link_into=None):
+                 features, target_dir, is_gkrust=False, link_into=None):
         StaticLibrary.__init__(self, context, basename, link_into=link_into,
                                # A rust library is a real static library ; make
                                # it known to the build system.
                                no_expand_lib=True)
         BaseRustLibrary.init(self, context, basename, cargo_file,
-                             crate_type, dependencies, features, target_dir)
+                             crate_type, dependencies, features, target_dir,
+                             is_gkrust)
 
 
 class SharedLibrary(Library):
@@ -880,10 +883,11 @@ class HostRustLibrary(HostLibrary, BaseRustLibrary):
     no_expand_lib = True
 
     def __init__(self, context, basename, cargo_file, crate_type, dependencies,
-                 features, target_dir):
+                 features, target_dir, is_gkrust):
         HostLibrary.__init__(self, context, basename)
         BaseRustLibrary.init(self, context, basename, cargo_file,
-                             crate_type, dependencies, features, target_dir)
+                             crate_type, dependencies, features, target_dir,
+                             is_gkrust)
 
 
 class TestManifest(ContextDerived):

python/mozbuild/mozbuild/frontend/emitter.py

@@ -544,7 +544,8 @@ class TreeMetadataEmitter(LoggingMixin):
                         description, dep_crate_name, crate_name),
                     context)
 
-    def _rust_library(self, context, libname, static_args, cls=RustLibrary):
+    def _rust_library(self, context, libname, static_args, is_gkrust=False,
+                      cls=RustLibrary):
         # We need to note any Rust library for linking purposes.
         config, cargo_file = self._parse_cargo_file(context)
         crate_name = config['package']['name']
@@ -585,7 +586,7 @@ class TreeMetadataEmitter(LoggingMixin):
                 context)
 
         return cls(context, libname, cargo_file, crate_type, dependencies,
-                   features, cargo_target_dir, **static_args)
+                   features, cargo_target_dir, is_gkrust, **static_args)
 
     def _handle_gn_dirs(self, context):
         for target_dir in context.get('GN_DIRS', []):
@@ -848,7 +849,8 @@ class TreeMetadataEmitter(LoggingMixin):
             if static_lib:
                 is_rust_library = context.get('IS_RUST_LIBRARY')
                 if is_rust_library:
-                    lib = self._rust_library(context, libname, static_args)
+                    lib = self._rust_library(context, libname, static_args,
+                                             is_gkrust=bool(context.get('IS_GKRUST')))
                 else:
                     lib = StaticLibrary(context, libname, **static_args)
                 self._libs[libname].append(lib)

toolkit/library/rust/moz.build

@@ -6,7 +6,7 @@
 
 include('gkrust-features.mozbuild')
 
-RustLibrary('gkrust', gkrust_features)
+RustLibrary('gkrust', gkrust_features, is_gkrust=True)
 
 # Target directory doesn't matter a lot here, since we can't share panic=abort
 # compilation artifacts with gkrust.