bug 1295937 - Improvements to gyp_reader to handle NSS gyp files. r=glandium

This patch contains a number of changes to the gyp_reader code:
* Add three new flags to GYP_DIRS:
** no_chromium, to skip forcing the includes/etc needed for chromium gyp files
** no_unified, to force building all sources without unification
** action_overrides, to pass scripts used when mapping gyp actions to moz.build GENERATED_FILES
* Handle the flags mentioned above in read_from_gyp
* Handle actions in gyp targets by mapping them to GENERATED_FILES, using scripts specified in the action_overrides flag. We don't try to handle the generic action case, we require special-casing for each action.
* Handle a subset of copies in gyp targets by mapping them to EXPORTS, just enough to handle the use of them for NSS exports.
* Handle shared_library and executable gyp targets
* Handle gyp target dependencies/libraries as USE_LIBS/OS_LIBS
* Handle generated source files
* Handle .def files in sources by mapping them to SYMBOLS_FILE
* Special-case some include_dirs:
** Map `<(PRODUCT_DIR)/dist/` to $DIST/include (to handle include paths for NSS exports)
** Map include_dirs starting with topobjdir to objdir-relative paths, to handle passing the NSPR include path to NSS
* split /build/gyp.mozbuild into two parts, with gyp_base.mozbuild containing generic bits, and gyp.mozbuild containing chromium-specific bits


MozReview-Commit-ID: FbDmlqDjRp4

--HG--
extra : rebase_source : d3fb470c589f92d8c956b9ecd550fb8df79ff5bc

build/gyp.mozbuild

@@ -4,7 +4,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-gyp_vars = {
+include('gyp_base.mozbuild')
+
+gyp_vars.update({
     'lsan': 0,
     'asan': 0,
     'build_with_mozilla': 1,
@@ -69,16 +71,9 @@ gyp_vars = {
     # linker to throw away uneeded bits.
     'include_isac': 1,
     'include_pcm16b': 1,
-}
+})
 
-os = CONFIG['OS_TARGET']
-
-if os == 'WINNT':
-    gyp_vars.update(
-        MSVS_VERSION=CONFIG['_MSVS_VERSION'],
-        MSVS_OS_BITS=64 if CONFIG['HAVE_64BIT_BUILD'] else 32,
-    )
-elif os == 'Android':
+if os == 'Android':
     if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'gonk':
         gyp_vars['build_with_gonk'] = 1
         gyp_vars['moz_widget_toolkit_gonk'] = 1
@@ -92,28 +87,6 @@ elif os == 'Android':
             android_toolchain=CONFIG.get('ANDROID_TOOLCHAIN', ''),
         )
 
-flavors = {
-    'WINNT': 'win',
-    'Android': 'linux' if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'gonk' else 'android',
-    'Linux': 'linux',
-    'Darwin': 'mac' if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'cocoa' else 'ios',
-    'SunOS': 'solaris',
-    'GNU/kFreeBSD': 'freebsd',
-    'DragonFly': 'dragonfly',
-    'FreeBSD': 'freebsd',
-    'NetBSD': 'netbsd',
-    'OpenBSD': 'openbsd',
-}
-gyp_vars['OS'] = flavors.get(os)
-
-arches = {
-    'x86_64': 'x64',
-    'x86': 'ia32',
-    'aarch64': 'arm64',
-}
-
-gyp_vars['target_arch'] = arches.get(CONFIG['CPU_ARCH'], CONFIG['CPU_ARCH'])
-
 if CONFIG['ARM_ARCH']:
     if int(CONFIG['ARM_ARCH']) < 7:
         gyp_vars['armv7'] = 0

build/gyp_base.mozbuild

@@ -0,0 +1,38 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+gyp_vars = {}
+
+os = CONFIG['OS_TARGET']
+
+if os == 'WINNT':
+    gyp_vars.update(
+        MSVS_VERSION=CONFIG['_MSVS_VERSION'],
+        MSVS_OS_BITS=64 if CONFIG['HAVE_64BIT_BUILD'] else 32,
+    )
+
+flavors = {
+    'WINNT': 'win',
+    'Android': 'linux' if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'gonk' else 'android',
+    'Linux': 'linux',
+    'Darwin': 'mac' if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'cocoa' else 'ios',
+    'SunOS': 'solaris',
+    'GNU/kFreeBSD': 'freebsd',
+    'DragonFly': 'dragonfly',
+    'FreeBSD': 'freebsd',
+    'NetBSD': 'netbsd',
+    'OpenBSD': 'openbsd',
+}
+gyp_vars['OS'] = flavors.get(os)
+
+arches = {
+    'x86_64': 'x64',
+    'x86': 'ia32',
+    'aarch64': 'arm64',
+}
+
+gyp_vars['host_arch'] = arches.get(CONFIG['HOST_CPU_ARCH'], CONFIG['HOST_CPU_ARCH'])
+gyp_vars['target_arch'] = arches.get(CONFIG['CPU_ARCH'], CONFIG['CPU_ARCH'])

build/moz.configure/init.configure

@@ -520,6 +520,7 @@ def host_variables(host):
         HOST_OS_ARCH=os_arch,
     )
 
+set_config('HOST_CPU_ARCH', delayed_getattr(host, 'cpu'))
 set_config('HOST_OS_ARCH', delayed_getattr(host_variables, 'HOST_OS_ARCH'))
 add_old_configure_assignment('HOST_OS_ARCH',
                              delayed_getattr(host_variables, 'HOST_OS_ARCH'))

python/mozbuild/mozbuild/frontend/context.py

@@ -1624,7 +1624,10 @@ VARIABLES = {
             'variables': dict,
             'input': unicode,
             'sandbox_vars': dict,
+            'no_chromium': bool,
+            'no_unified': bool,
             'non_unified_sources': StrictOrderingOnAppendList,
+            'action_overrides': dict,
         }), list,
         """Defines a list of object directories handled by gyp configurations.
 
@@ -1639,9 +1642,15 @@ VARIABLES = {
             - sandbox_vars, a dictionary containing variables and values to
               pass to the mozbuild processor on top of those derived from gyp
               configuration.
+            - no_chromium, a boolean which if set to True disables some
+              special handling that emulates gyp_chromium.
+            - no_unified, a boolean which if set to True disables source
+              file unification entirely.
             - non_unified_sources, a list containing sources files, relative to
               the current moz.build, that should be excluded from source file
               unification.
+            - action_overrides, a dict of action_name to values of the `script`
+              attribute to use for GENERATED_FILES for the specified action.
 
         Typical use looks like:
             GYP_DIRS += ['foo', 'bar']

python/mozbuild/mozbuild/frontend/gyp_reader.py

@@ -5,6 +5,7 @@
 from __future__ import absolute_import, unicode_literals
 
 import gyp
+import gyp.msvs_emulation
 import sys
 import os
 import types
@@ -12,6 +13,7 @@ import mozpack.path as mozpath
 from mozpack.files import FileFinder
 from .sandbox import alphabetical_sorted
 from .context import (
+    ObjDirPath,
     SourcePath,
     TemplateContext,
     VARIABLES,
@@ -38,13 +40,20 @@ chrome_src = mozpath.abspath(mozpath.join(mozpath.dirname(gyp.__file__),
     '../../../..'))
 script_dir = mozpath.join(chrome_src, 'build')
 
+
+def encode(value):
+    if isinstance(value, unicode):
+        return value.encode('utf-8')
+    return value
+
+
 # Default variables gyp uses when evaluating gyp files.
 generator_default_variables = {
 }
-for dirname in ['INTERMEDIATE_DIR', 'SHARED_INTERMEDIATE_DIR', 'PRODUCT_DIR',
-                'LIB_DIR', 'SHARED_LIB_DIR']:
+for dirname in [b'INTERMEDIATE_DIR', b'SHARED_INTERMEDIATE_DIR', b'PRODUCT_DIR',
+                b'LIB_DIR', b'SHARED_LIB_DIR']:
   # Some gyp steps fail if these are empty(!).
-  generator_default_variables[dirname] = b'dir'
+  generator_default_variables[dirname] = b'$' + dirname
 
 for unused in ['RULE_INPUT_PATH', 'RULE_INPUT_ROOT', 'RULE_INPUT_NAME',
                'RULE_INPUT_DIRNAME', 'RULE_INPUT_EXT',
@@ -68,13 +77,37 @@ class GypContext(TemplateContext):
             allowed_variables=VARIABLES, config=config)
 
 
-def encode(value):
-    if isinstance(value, unicode):
-        return value.encode('utf-8')
-    return value
+def handle_actions(actions, context, action_overrides):
+  idir = '$INTERMEDIATE_DIR/'
+  for action in actions:
+    name = action['action_name']
+    if name not in action_overrides:
+      raise RuntimeError('GYP action %s not listed in action_overrides' % name)
+    outputs = action['outputs']
+    if len(outputs) > 1:
+      raise NotImplementedError('GYP actions with more than one output not supported: %s' % name)
+    output = outputs[0]
+    if not output.startswith(idir):
+      raise NotImplementedError('GYP actions outputting to somewhere other than <(INTERMEDIATE_DIR) not supported: %s' % output)
+    output = output[len(idir):]
+    context['GENERATED_FILES'] += [output]
+    g = context['GENERATED_FILES'][output]
+    g.script = action_overrides[name]
+    g.inputs = action['inputs']
 
+def handle_copies(copies, context):
+  dist = '$PRODUCT_DIR/dist/'
+  for copy in copies:
+    dest = copy['destination']
+    if not dest.startswith(dist):
+      raise NotImplementedError('GYP copies to somewhere other than <(PRODUCT_DIR)/dist not supported: %s' % dest)
+    dest_paths = dest[len(dist):].split('/')
+    exports = context['EXPORTS']
+    while dest_paths:
+      exports = getattr(exports, dest_paths.pop(0))
+    exports += sorted(copy['files'], key=lambda x: x.lower())
 
-def read_from_gyp(config, path, output, vars, non_unified_sources = set()):
+def read_from_gyp(config, path, output, vars, no_chromium, no_unified, action_overrides, non_unified_sources = set()):
     """Read a gyp configuration and emits GypContexts for the backend to
     process.
 
@@ -84,10 +117,17 @@ def read_from_gyp(config, path, output, vars, non_unified_sources = set()):
     processor.
     """
 
+    is_win = config.substs['OS_TARGET'] == 'WINNT'
+    is_msvc = bool(config.substs['_MSC_VER'])
     # gyp expects plain str instead of unicode. The frontend code gives us
     # unicode strings, so convert them.
     path = encode(path)
     str_vars = dict((name, encode(value)) for name, value in vars.items())
+    if is_msvc:
+        # This isn't actually used anywhere in this generator, but it's needed
+        # to override the registry detection of VC++ in gyp.
+        os.environ['GYP_MSVS_OVERRIDE_PATH'] = 'fake_path'
+        os.environ['GYP_MSVS_VERSION'] = config.substs['MSVS_VERSION']
 
     params = {
         b'parallel': False,
@@ -96,18 +136,23 @@ def read_from_gyp(config, path, output, vars, non_unified_sources = set()):
         b'root_targets': None,
     }
 
-    # Files that gyp_chromium always includes
-    includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
-    finder = FileFinder(chrome_src, find_executables=False)
-    includes.extend(encode(mozpath.join(chrome_src, name))
-        for name, _ in finder.find('*/supplement.gypi'))
+    if no_chromium:
+      includes = []
+      depth = mozpath.dirname(path)
+    else:
+      depth = chrome_src
+      # Files that gyp_chromium always includes
+      includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
+      finder = FileFinder(chrome_src, find_executables=False)
+      includes.extend(encode(mozpath.join(chrome_src, name))
+          for name, _ in finder.find('*/supplement.gypi'))
 
     # Read the given gyp file and its dependencies.
     generator, flat_list, targets, data = \
         gyp.Load([path], format=b'mozbuild',
             default_variables=str_vars,
             includes=includes,
-            depth=encode(chrome_src),
+            depth=encode(depth),
             params=params)
 
     # Process all targets from the given gyp files and its dependencies.
@@ -146,60 +191,134 @@ def read_from_gyp(config, path, output, vars, non_unified_sources = set()):
                                'in %s' % (c, target_name, build_file))
         target_conf = spec['configurations'][c]
 
+        if 'actions' in spec:
+          handle_actions(spec['actions'], context, action_overrides)
+        if 'copies' in spec:
+          handle_copies(spec['copies'], context)
+
+        use_libs = []
+        libs = []
+        def add_deps(s):
+            for t in s.get('dependencies', []) + s.get('dependencies_original', []):
+                ty = targets[t]['type']
+                if ty in ('static_library', 'shared_library'):
+                    use_libs.append(targets[t]['target_name'])
+                # Manually expand out transitive dependencies--
+                # gyp won't do this for static libs or none targets.
+                if ty in ('static_library', 'none'):
+                    add_deps(targets[t])
+            libs.extend(spec.get('libraries', []))
+        #XXX: this sucks, but webrtc breaks with this right now because
+        # it builds a library called 'gtest' and we just get lucky
+        # that it isn't in USE_LIBS by that name anywhere.
+        if no_chromium:
+            add_deps(spec)
+
+        os_libs = []
+        for l in libs:
+          if l.startswith('-'):
+              os_libs.append(l)
+          elif l.endswith('.lib'):
+              os_libs.append(l[:-4])
+          elif l:
+            # For library names passed in from moz.build.
+            use_libs.append(os.path.basename(l))
+
         if spec['type'] == 'none':
+          if not ('actions' in spec or 'copies' in spec):
             continue
-        elif spec['type'] == 'static_library':
+        elif spec['type'] in ('static_library', 'shared_library', 'executable'):
             # Remove leading 'lib' from the target_name if any, and use as
             # library name.
             name = spec['target_name']
-            if name.startswith('lib'):
-                name = name[3:]
-            # The context expects an unicode string.
-            context['LIBRARY_NAME'] = name.decode('utf-8')
+            if spec['type'] in ('static_library', 'shared_library'):
+                if name.startswith('lib'):
+                    name = name[3:]
+                # The context expects an unicode string.
+                context['LIBRARY_NAME'] = name.decode('utf-8')
+            else:
+                context['PROGRAM'] = name.decode('utf-8')
+            if spec['type'] == 'shared_library':
+                context['FORCE_SHARED_LIB'] = True
+            elif spec['type'] == 'static_library' and spec.get('variables', {}).get('no_expand_libs', '0') == '1':
+                # PSM links a NSS static library, but our folded libnss
+                # doesn't actually export everything that all of the
+                # objects within would need, so that one library
+                # should be built as a real static library.
+                context['NO_EXPAND_LIBS'] = True
+            if use_libs:
+                context['USE_LIBS'] = sorted(use_libs, key=lambda s: s.lower())
+            if os_libs:
+                context['OS_LIBS'] = os_libs
             # gyp files contain headers and asm sources in sources lists.
             sources = []
             unified_sources = []
             extensions = set()
+            use_defines_in_asflags = False
             for f in spec.get('sources', []):
                 ext = mozpath.splitext(f)[-1]
                 extensions.add(ext)
-                s = SourcePath(context, f)
+                if f.startswith('$INTERMEDIATE_DIR/'):
+                  s = ObjDirPath(context, f.replace('$INTERMEDIATE_DIR/', '!'))
+                else:
+                  s = SourcePath(context, f)
                 if ext == '.h':
                     continue
-                if ext != '.S' and s not in non_unified_sources:
+                if ext == '.def':
+                    context['SYMBOLS_FILE'] = s
+                elif ext != '.S' and not no_unified and s not in non_unified_sources:
                     unified_sources.append(s)
                 else:
                     sources.append(s)
+                # The Mozilla build system doesn't use DEFINES for building
+                # ASFILES.
+                if ext == '.s':
+                    use_defines_in_asflags = True
 
             # The context expects alphabetical order when adding sources
             context['SOURCES'] = alphabetical_sorted(sources)
             context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
 
-            for define in target_conf.get('defines', []):
+            defines = target_conf.get('defines', [])
+            if is_msvc and no_chromium:
+                msvs_settings = gyp.msvs_emulation.MsvsSettings(spec, {})
+                defines.extend(msvs_settings.GetComputedDefines(c))
+            for define in defines:
                 if '=' in define:
                     name, value = define.split('=', 1)
                     context['DEFINES'][name] = value
                 else:
                     context['DEFINES'][define] = True
 
+            product_dir_dist = '$PRODUCT_DIR/dist/'
             for include in target_conf.get('include_dirs', []):
-                # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
-                #
-                # NB: gyp files sometimes have actual absolute paths (e.g.
-                # /usr/include32) and sometimes paths that moz.build considers
-                # absolute, i.e. starting from topsrcdir. There's no good way
-                # to tell them apart here, and the actual absolute paths are
-                # likely bogus. In any event, actual absolute paths will be
-                # filtered out by trying to find them in topsrcdir.
-                if include.startswith('/'):
-                    resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:]))
+                if include.startswith(product_dir_dist):
+                    # special-case includes of <(PRODUCT_DIR)/dist/ to match
+                    # handle_copies above. This is used for NSS' exports.
+                    include = '!/dist/include/' + include[len(product_dir_dist):]
+                elif include.startswith(config.topobjdir):
+                    # NSPR_INCLUDE_DIR gets passed into the NSS build this way.
+                    include = '!/' + mozpath.relpath(include, config.topobjdir)
                 else:
-                    resolved = mozpath.abspath(mozpath.join(mozpath.dirname(build_file), include))
-                if not os.path.exists(resolved):
-                    continue
+                  # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
+                  #
+                  # NB: gyp files sometimes have actual absolute paths (e.g.
+                  # /usr/include32) and sometimes paths that moz.build considers
+                  # absolute, i.e. starting from topsrcdir. There's no good way
+                  # to tell them apart here, and the actual absolute paths are
+                  # likely bogus. In any event, actual absolute paths will be
+                  # filtered out by trying to find them in topsrcdir.
+                  if include.startswith('/'):
+                      resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:]))
+                  else:
+                      resolved = mozpath.abspath(mozpath.join(mozpath.dirname(build_file), include))
+                  if not os.path.exists(resolved):
+                      continue
                 context['LOCAL_INCLUDES'] += [include]
 
             context['ASFLAGS'] = target_conf.get('asflags_mozilla', [])
+            if use_defines_in_asflags and defines:
+                context['ASFLAGS'] += ['-D' + d for d in defines]
             flags = target_conf.get('cflags_mozilla', [])
             if flags:
                 suffix_map = {
@@ -227,22 +346,23 @@ def read_from_gyp(config, path, output, vars, non_unified_sources = set()):
                         else:
                             context[var].extend(f)
         else:
-            # Ignore other types than static_library because we don't have
+            # Ignore other types because we don't have
             # anything using them, and we're not testing them. They can be
             # added when that becomes necessary.
             raise NotImplementedError('Unsupported gyp target type: %s' % spec['type'])
 
-        # Add some features to all contexts. Put here in case LOCAL_INCLUDES
-        # order matters.
-        context['LOCAL_INCLUDES'] += [
-            '!/ipc/ipdl/_ipdlheaders',
-            '/ipc/chromium/src',
-            '/ipc/glue',
-        ]
-        # These get set via VC project file settings for normal GYP builds.
-        if config.substs['OS_TARGET'] == 'WINNT':
-            context['DEFINES']['UNICODE'] = True
-            context['DEFINES']['_UNICODE'] = True
+        if not no_chromium:
+          # Add some features to all contexts. Put here in case LOCAL_INCLUDES
+          # order matters.
+          context['LOCAL_INCLUDES'] += [
+              '!/ipc/ipdl/_ipdlheaders',
+              '/ipc/chromium/src',
+              '/ipc/glue',
+          ]
+          # These get set via VC project file settings for normal GYP builds.
+          if is_win:
+              context['DEFINES']['UNICODE'] = True
+              context['DEFINES']['_UNICODE'] = True
         context['DISABLE_STL_WRAPPING'] = True
 
         yield context

python/mozbuild/mozbuild/frontend/reader.py

@@ -1157,12 +1157,18 @@ class BuildReader(object):
                     raise SandboxValidationError('Cannot find %s.' % source,
                         context)
                 non_unified_sources.add(source)
+            action_overrides = {}
+            for action, script in gyp_dir.action_overrides.iteritems():
+                action_overrides[action] = SourcePath(context, script)
             time_start = time.time()
             for gyp_context in read_from_gyp(context.config,
                                              mozpath.join(curdir, gyp_dir.input),
                                              mozpath.join(context.objdir,
                                                           target_dir),
                                              gyp_dir.variables,
+                                             gyp_dir.no_chromium,
+                                             gyp_dir.no_unified,
+                                             action_overrides,
                                              non_unified_sources = non_unified_sources):
                 gyp_context.update(gyp_dir.sandbox_vars)
                 gyp_contexts.append(gyp_context)