Bug 1047267 - Allow to reference libraries from third-party build systems in USE_LIBS. r=gps

build/autoconf/hooks.m4

@@ -15,6 +15,7 @@ define([_MOZ_AC_INIT_PREPARE], defn([AC_INIT_PREPARE]))
 define([AC_INIT_PREPARE],
 [_MOZ_AC_INIT_PREPARE($1)
 MOZ_CONFIG_LOG_TRAP
+> subconfigures
 ])
 
 define([AC_OUTPUT_SUBDIRS],
@@ -28,6 +29,11 @@ define([AC_OUTPUT_SUBDIRS],
     objdir=$moz_config_dir
     ;;
   esac
+  dnl Because config.status, storing AC_SUBSTs, is written before any
+  dnl subconfigure runs, we need to use a file. Moreover, some subconfigures
+  dnl are started from a subshell, and variable modifications from a subshell
+  dnl wouldn't be preserved.
+  echo $objdir >> subconfigures
 
   dumpenv="true | "
   case "$host" in

build/docs/defining-binaries.rst

@@ -68,7 +68,7 @@ Note that currently, the build system may not create an actual library for
 static libraries. It is an implementation detail that shouldn't need to be
 worried about.
 
-As a special rule, USE_LIBS is allowed to contain references to shared
+As a special rule, ``USE_LIBS`` is allowed to contain references to shared
 libraries. In such cases, programs and shared libraries linking this static
 library will inherit those shared library dependencies.
 
@@ -185,6 +185,27 @@ Like ``USE_LIBS``, this variable applies to static and shared libraries, as
 well as programs.
 
 
+Libraries from third party build system
+=======================================
+
+Some libraries in the tree are not built by the moz.build-governed build
+system, and there is no ``LIBRARY_NAME`` corresponding to them.
+
+However, ``USE_LIBS`` allows to reference such libraries by giving a full
+path (like when disambiguating identical ``LIBRARY_NAME``). The same naming
+rules apply as other uses of ``USE_LIBS``, so only the library name without
+prefix and suffix shall be given.
+
+   USE_LIBS += [
+       '/path/from/topsrcdir/to/third-party/bar',
+       '../relative/third-party/baz',
+   ]
+
+Note that ``/path/from/topsrcdir/to/third-party`` and
+``../relative/third-party/baz`` must lead under a subconfigured directory (a
+directory with an AC_OUTPUT_SUBDIRS in configure.in), or ``security/nss``.
+
+
 Building both static and shared libraries
 =========================================
 

config/rules.mk

@@ -565,7 +565,7 @@ everything::
 	$(MAKE) clean
 	$(MAKE) all
 
-STATIC_LIBS_DEPS := $(addsuffix .$(LIBS_DESC_SUFFIX),$(STATIC_LIBS))
+STATIC_LIBS_DEPS := $(wildcard $(addsuffix .$(LIBS_DESC_SUFFIX),$(STATIC_LIBS)))
 
 # Dependencies which, if modified, should cause everything to rebuild
 GLOBAL_DEPS += Makefile $(DEPTH)/config/autoconf.mk $(topsrcdir)/config/config.mk

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -33,6 +33,7 @@ from ..frontend.data import (
     Defines,
     DirectoryTraversal,
     Exports,
+    ExternalLibrary,
     GeneratedInclude,
     HostLibrary,
     HostProgram,
@@ -1209,8 +1210,9 @@ class RecursiveMakeBackend(CommonBackend):
             self._compile_graph[build_target].add('build/stlport/target')
 
         for lib in obj.linked_libraries:
-            self._compile_graph[build_target].add(
-                self._build_target_for_obj(lib))
+            if not isinstance(lib, ExternalLibrary):
+                self._compile_graph[build_target].add(
+                    self._build_target_for_obj(lib))
             relpath = pretty_relpath(lib)
             if isinstance(obj, Library):
                 if isinstance(lib, StaticLibrary):

python/mozbuild/mozbuild/frontend/data.py

@@ -497,6 +497,20 @@ class SharedLibrary(Library):
             self.soname = self.lib_name
 
 
+class ExternalLibrary(object):
+    """Empty mixin for libraries built by an external build system."""
+
+
+class ExternalStaticLibrary(StaticLibrary, ExternalLibrary):
+    """Sandbox container for static libraries built by an external build
+    system."""
+
+
+class ExternalSharedLibrary(SharedLibrary, ExternalLibrary):
+    """Sandbox container for shared libraries built by an external build
+    system."""
+
+
 class HostLibrary(BaseLibrary):
     """Sandbox container object for a host library"""
     KIND = 'host'

python/mozbuild/mozbuild/frontend/emitter.py

@@ -13,7 +13,10 @@ import time
 
 from collections import OrderedDict
 from mach.mixin.logging import LoggingMixin
-from mozbuild.util import OrderedDefaultDict
+from mozbuild.util import (
+    memoize,
+    OrderedDefaultDict,
+)
 
 import mozpack.path as mozpath
 import manifestparser
@@ -29,6 +32,8 @@ from .data import (
     GeneratedInclude,
     GeneratedWebIDLFile,
     ExampleWebIDLInterface,
+    ExternalStaticLibrary,
+    ExternalSharedLibrary,
     HeaderFileSubstitution,
     HostLibrary,
     HostProgram,
@@ -64,6 +69,7 @@ from .reader import (
 )
 
 from .gyp_reader import GypSandbox
+from .sandbox import GlobalNamespace
 
 
 class TreeMetadataEmitter(LoggingMixin):
@@ -95,6 +101,17 @@ class TreeMetadataEmitter(LoggingMixin):
         self._linkage = []
         self._static_linking_shared = set()
 
+        # Keep track of external paths (third party build systems), starting
+        # from what we run a subconfigure in. We'll eliminate some directories
+        # as we traverse them with moz.build (e.g. js/src).
+        subconfigures = os.path.join(self.config.topobjdir, 'subconfigures')
+        paths = []
+        if os.path.exists(subconfigures):
+            paths = open(subconfigures).read().splitlines()
+        self._external_paths = set(mozpath.normsep(d) for d in paths)
+        # Add security/nss manually, since it doesn't have a subconfigure.
+        self._external_paths.add('security/nss')
+
     def emit(self, output):
         """Convert the BuildReader output into data structures.
 
@@ -253,6 +270,16 @@ class TreeMetadataEmitter(LoggingMixin):
                         mozpath.join(obj.objdir, dir))
                 dir = mozpath.relpath(dir, obj.topobjdir)
                 candidates = [l for l in candidates if l.relobjdir == dir]
+                if not candidates:
+                    # If the given directory is under one of the external
+                    # (third party) paths, use a fake library reference to
+                    # there.
+                    for d in self._external_paths:
+                        if dir.startswith('%s/' % d):
+                            candidates = [self._get_external_library(dir, name,
+                                force_static)]
+                            break
+
                 if not candidates:
                     raise SandboxValidationError(
                         '%s contains "%s", but there is no "%s" %s in %s.'
@@ -318,6 +345,27 @@ class TreeMetadataEmitter(LoggingMixin):
         for lib in sandbox.get(variable.replace('USE', 'OS'), []):
             obj.link_system_library(lib)
 
+    @memoize
+    def _get_external_library(self, dir, name, force_static):
+        # Create ExternalStaticLibrary or ExternalSharedLibrary object with a
+        # mock sandbox more or less truthful about where the external library
+        # is.
+        sandbox = GlobalNamespace()
+        sandbox.config = self.config
+        sandbox.main_path = dir
+        sandbox.all_paths = set([dir])
+        with sandbox.allow_all_writes() as s:
+            s['TOPSRCDIR'] = self.config.topsrcdir
+            s['TOPOBJDIR'] = self.config.topobjdir
+            s['RELATIVEDIR'] = dir
+            s['SRCDIR'] = mozpath.join(self.config.topsrcdir, dir)
+            s['OBJDIR'] = mozpath.join(self.config.topobjdir, dir)
+
+        if force_static:
+            return ExternalStaticLibrary(sandbox, name)
+        else:
+            return ExternalSharedLibrary(sandbox, name)
+
     def emit_from_sandbox(self, sandbox):
         """Convert a MozbuildSandbox to tree metadata objects.
 
@@ -945,6 +993,10 @@ class TreeMetadataEmitter(LoggingMixin):
         o.test_dirs = sandbox.get('TEST_DIRS', [])
         o.affected_tiers = sandbox.get_affected_tiers()
 
+        # Some paths have a subconfigure, yet also have a moz.build. Those
+        # shouldn't end up in self._external_paths.
+        self._external_paths -= { o.relobjdir }
+
         if 'TIERS' in sandbox:
             for tier in sandbox['TIERS']:
                 o.tier_dirs[tier] = sandbox['TIERS'][tier]['regular'] + \