Bug 1646936 - Generate a single metadata file in the objdir, and feed it to cbindgen. r=glandium

Differential Revision: https://phabricator.services.mozilla.com/D80360

build/RunCbindgen.py

@@ -21,23 +21,15 @@ def _get_crate_name(crate_path):
 
 
 CARGO_LOCK = mozpath.join(buildconfig.topsrcdir, "Cargo.lock")
+CARGO_TOML = mozpath.join(buildconfig.topsrcdir, "Cargo.toml")
 
 
-def _generate(output, cbindgen_crate_path, metadata_crate_path,
-              in_tree_dependencies):
+def _run_process(args):
     env = os.environ.copy()
     env['CARGO'] = str(buildconfig.substs['CARGO'])
     env['RUSTC'] = str(buildconfig.substs['RUSTC'])
 
-    p = subprocess.Popen([
-        buildconfig.substs['CBINDGEN'],
-        metadata_crate_path,
-        "--lockfile",
-        CARGO_LOCK,
-        "--crate",
-        _get_crate_name(cbindgen_crate_path),
-        "--cpp-compat"
-    ], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    p = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     stdout, stderr = p.communicate()
     stdout = six.ensure_text(stdout)
@@ -45,7 +37,45 @@ def _generate(output, cbindgen_crate_path, metadata_crate_path,
     if p.returncode != 0:
         print(stdout)
         print(stderr)
-        return p.returncode
+    return (stdout, p.returncode)
+
+
+def generate_metadata(output, cargo_config):
+    stdout, returncode = _run_process([
+        buildconfig.substs['CARGO'],
+        "metadata",
+        "--all-features",
+        "--format-version",
+        "1",
+        "--manifest-path",
+        CARGO_TOML
+    ])
+
+    if returncode != 0:
+        return returncode
+
+    output.write(stdout)
+
+    # This is not quite accurate, but cbindgen only cares about a subset of the
+    # data which, when changed, causes these files to change.
+    return set([CARGO_LOCK, CARGO_TOML])
+
+
+def generate(output, metadata_path, cbindgen_crate_path, *in_tree_dependencies):
+    stdout, returncode = _run_process([
+        buildconfig.substs['CBINDGEN'],
+        buildconfig.topsrcdir,
+        "--lockfile",
+        CARGO_LOCK,
+        "--crate",
+        _get_crate_name(cbindgen_crate_path),
+        "--metadata",
+        metadata_path,
+        "--cpp-compat"
+    ])
+
+    if returncode != 0:
+        return returncode
 
     output.write(stdout)
 
@@ -59,22 +89,3 @@ def _generate(output, cbindgen_crate_path, metadata_crate_path,
                     deps.add(mozpath.join(path, file))
 
     return deps
-
-
-def generate(output, cbindgen_crate_path, *in_tree_dependencies):
-    metadata_crate_path = mozpath.join(buildconfig.topsrcdir,
-                                       "toolkit", "library", "rust")
-    return _generate(output, cbindgen_crate_path, metadata_crate_path,
-                     in_tree_dependencies)
-
-
-# Use the binding's crate directory instead of toolkit/library/rust as
-# the metadata crate directory.
-#
-# This is necessary for the bindings inside SpiderMonkey, given that
-# SpiderMonkey tarball doesn't contain toolkit/library/rust and its
-# dependencies.
-def generate_with_same_crate(output, cbindgen_crate_path,
-                             *in_tree_dependencies):
-    return _generate(output, cbindgen_crate_path, cbindgen_crate_path,
-                     in_tree_dependencies)

build/moz.build

@@ -95,6 +95,7 @@ if CONFIG['MOZ_APP_BASENAME']:
     GeneratedFile('application.ini.h', script='appini_header.py',
                   inputs=['!application.ini'])
 
+
 # Put a .lldbinit in the bin directory and the objdir, to be picked up
 # automatically by LLDB when we debug executables using either of those two
 # directories as the current working directory.  The .lldbinit file will

build/moz.configure/bindgen.configure

@@ -21,7 +21,7 @@ option(env='CBINDGEN', nargs=1, when=cbindgen_is_needed,
 def check_cbindgen_version(cbindgen, fatal=False):
     log.debug("trying cbindgen: %s" % cbindgen)
 
-    cbindgen_min_version = Version('0.14.1')
+    cbindgen_min_version = Version('0.14.3')
 
     # cbindgen x.y.z
     version = Version(check_cmd_output(cbindgen, '--version').strip().split(" ")[1])

build/templates.mozbuild

@@ -201,6 +201,15 @@ def GeneratedFile(name, *names, **kwargs):
     generated_file.flags = flags
     generated_file.force = force
 
+@template
+def CbindgenHeader(name, inputs):
+    """Add one GENERATED_FILES by running RunCbindgen.py"""
+
+    inputs = ['!/config/cbindgen-metadata.json'] + inputs
+    GeneratedFile(name, script='/build/RunCbindgen.py',
+                  entry_point='generate', inputs=inputs)
+
+
 include('gecko_templates.mozbuild')
 include('test_templates.mozbuild')
 

config/moz.build

@@ -79,3 +79,10 @@ if CONFIG['WRAP_SYSTEM_INCLUDES']:
     GeneratedFile(*outputs, script='make-system-wrappers.py',
                   entry_point='gen_wrappers',
                   flags = [TOPOBJDIR + output_dir] + stl_headers + system_headers)
+
+if CONFIG['COMPILE_ENVIRONMENT'] and CONFIG['CBINDGEN']:
+    GeneratedFile(
+        'cbindgen-metadata.json',
+        script='/build/RunCbindgen.py',
+        entry_point='generate_metadata',
+        inputs=['!/.cargo/config'])

config/recurse.mk

@@ -228,7 +228,7 @@ $(addprefix build/unix/stdc++compat/,target host) build/clang-plugin/host: confi
 # prior to Make running. So we also set it as a dependency of pre-export, which
 # ensures it exists before recursing the rust targets and the export targets
 # that run cbindgen, tricking Make into keeping them early.
-$(rust_targets) gfx/webrender_bindings/export layout/style/export xpcom/base/export: $(DEPTH)/.cargo/config
+$(rust_targets): $(DEPTH)/.cargo/config
 ifndef TEST_MOZBUILD
 pre-export:: $(DEPTH)/.cargo/config
 endif

dom/media/moz.build

@@ -383,17 +383,11 @@ if CONFIG['COMPILE_ENVIRONMENT']:
         '!audioipc_server_ffi_generated.h',
     ]
 
-    GeneratedFile('audioipc_client_ffi_generated.h',
-                  script='/build/RunCbindgen.py', entry_point='generate',
-                  inputs=[
-                      '/media/audioipc/client',
-                  ])
+    CbindgenHeader('audioipc_client_ffi_generated.h',
+                   inputs=['/media/audioipc/client'])
 
-    GeneratedFile('audioipc_server_ffi_generated.h',
-                  script='/build/RunCbindgen.py', entry_point='generate',
-                  inputs=[
-                      '/media/audioipc/server',
-                  ])
+    CbindgenHeader('audioipc_server_ffi_generated.h',
+                   inputs=['/media/audioipc/server'])
 
 include('/ipc/chromium/chromium-config.mozbuild')
 

gfx/webrender_bindings/moz.build

@@ -97,13 +97,12 @@ if CONFIG['COMPILE_ENVIRONMENT']:
         '!webrender_ffi_generated.h',
     ]
 
-    GeneratedFile('webrender_ffi_generated.h',
-                  script='/build/RunCbindgen.py', entry_point='generate',
-                  inputs=[
-                      '/gfx/webrender_bindings',
-                      '/gfx/wr/webrender',
-                      '/gfx/wr/webrender_api',
-                  ])
+    CbindgenHeader('webrender_ffi_generated.h',
+                   inputs=[
+                       '/gfx/webrender_bindings',
+                       '/gfx/wr/webrender',
+                       '/gfx/wr/webrender_api',
+                   ])
 
 
 include('/ipc/chromium/chromium-config.mozbuild')

gfx/wgpu_bindings/moz.build

@@ -12,12 +12,7 @@ UNIFIED_SOURCES += [
 ]
 
 if CONFIG['COMPILE_ENVIRONMENT']:
-    GeneratedFile('wgpu_ffi_generated.h',
-                  script='/build/RunCbindgen.py',
-                  entry_point='generate',
-                  inputs=[
-                    '/gfx/wgpu_bindings',
-                  ])
+    CbindgenHeader('wgpu_ffi_generated.h', inputs=['/gfx/wgpu_bindings'])
 
     EXPORTS.mozilla.webgpu.ffi += [
         '!wgpu_ffi_generated.h',

intl/hyphenation/glue/moz.build

@@ -22,7 +22,4 @@ if CONFIG['CC_TYPE'] in ('clang', 'gcc'):
     CXXFLAGS += ['-Wno-error=shadow']
 
 if CONFIG['COMPILE_ENVIRONMENT']:
-    GeneratedFile('mapped_hyph.h',
-                  script='/build/RunCbindgen.py',
-                  entry_point='generate',
-                  inputs=['/third_party/rust/mapped_hyph'])
+    CbindgenHeader('mapped_hyph.h', inputs=['/third_party/rust/mapped_hyph'])

intl/l10n/moz.build

@@ -37,10 +37,8 @@ LOCAL_INCLUDES += [
 ]
 
 if CONFIG['COMPILE_ENVIRONMENT']:
-    GeneratedFile('fluent_ffi_generated.h',
-                  script='/build/RunCbindgen.py',
-                  entry_point='generate',
-                  inputs=['/intl/l10n/rust/fluent-ffi'])
+    CbindgenHeader('fluent_ffi_generated.h',
+                   inputs=['/intl/l10n/rust/fluent-ffi'])
 
     EXPORTS.mozilla.intl += [
         '!fluent_ffi_generated.h',

intl/locale/moz.build

@@ -83,14 +83,10 @@ if CONFIG['ENABLE_TESTS']:
     DIRS += ['tests/gtest']
 
 if CONFIG['COMPILE_ENVIRONMENT']:
-    GeneratedFile('fluent_langneg_ffi_generated.h',
-                  script='/build/RunCbindgen.py',
-                  entry_point='generate',
-                  inputs=['/intl/locale/rust/fluent-langneg-ffi'])
-    GeneratedFile('unic_langid_ffi_generated.h',
-                  script='/build/RunCbindgen.py',
-                  entry_point='generate',
-                  inputs=['/intl/locale/rust/unic-langid-ffi'])
+    CbindgenHeader('fluent_langneg_ffi_generated.h',
+                   inputs=['/intl/locale/rust/fluent-langneg-ffi'])
+    CbindgenHeader('unic_langid_ffi_generated.h',
+                   inputs=['/intl/locale/rust/unic-langid-ffi'])
 
     EXPORTS.mozilla.intl += [
         '!fluent_langneg_ffi_generated.h',

js/src/frontend/moz.build

@@ -21,12 +21,7 @@ GeneratedFile('ReservedWordsGenerated.h', script='GenerateReservedWords.py',
               inputs=['ReservedWords.h'])
 
 if CONFIG['JS_ENABLE_SMOOSH']:
-    GeneratedFile('smoosh_generated.h',
-                  script='/build/RunCbindgen.py',
-                  entry_point='generate_with_same_crate',
-                  inputs=[
-                      '/js/src/frontend/smoosh',
-                  ])
+    CbindgenHeader('smoosh_generated.h', inputs=['/js/src/frontend/smoosh'])
 
 UNIFIED_SOURCES += [
     'AbstractScopePtr.cpp',

layout/style/moz.build

@@ -299,9 +299,8 @@ if CONFIG['COMPILE_ENVIRONMENT']:
                   script='GenerateCSSPropsGenerated.py',
                   entry_point='generate',
                   inputs=['!ServoCSSPropList.py'])
-    GeneratedFile('ServoStyleConsts.h', script='/build/RunCbindgen.py',
-                  entry_point='generate',
-                  inputs = ['/servo/ports/geckolib', '/servo/components/style'])
+    CbindgenHeader('ServoStyleConsts.h',
+                   inputs=['/servo/ports/geckolib', '/servo/components/style'])
 
     CONFIGURE_SUBST_FILES += [
         'bindgen.toml',

media/mp4parse-rust/moz.build

@@ -9,9 +9,8 @@ EXPORTS += [
 ]
 
 if CONFIG['COMPILE_ENVIRONMENT']:
-    GeneratedFile('mp4parse_ffi_generated.h',
-                  script='/build/RunCbindgen.py', entry_point='generate',
-                  inputs=['/third_party/rust/mp4parse_capi'])
+    CbindgenHeader('mp4parse_ffi_generated.h',
+                   inputs=['/third_party/rust/mp4parse_capi'])
 
     EXPORTS += [ # Should this be namespaced? EXPORTS.mozilla.media
         '!mp4parse_ffi_generated.h',

netwerk/base/mozurl/moz.build

@@ -15,10 +15,7 @@ SOURCES += [
 FINAL_LIBRARY = 'xul'
 
 if CONFIG['COMPILE_ENVIRONMENT']:
-    GeneratedFile('MozURL_ffi.h',
-                  script='/build/RunCbindgen.py',
-                  entry_point='generate',
-                  inputs=['/netwerk/base/mozurl'])
+    CbindgenHeader('MozURL_ffi.h', inputs=['/netwerk/base/mozurl'])
 
     EXPORTS.mozilla.net += [
         '!MozURL_ffi.h',

netwerk/base/rust-helper/moz.build

@@ -5,10 +5,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 if CONFIG['COMPILE_ENVIRONMENT']:
-    GeneratedFile('rust_helper.h',
-                  script='/build/RunCbindgen.py',
-                  entry_point='generate',
-                  inputs=['/netwerk/base/rust-helper'])
+    CbindgenHeader('rust_helper.h', inputs=['/netwerk/base/rust-helper'])
 
     EXPORTS.mozilla.net += [
         '!rust_helper.h',

netwerk/socket/neqo_glue/moz.build

@@ -14,10 +14,8 @@ LOCAL_INCLUDES += [
 ]
 
 if CONFIG['COMPILE_ENVIRONMENT']:
-    GeneratedFile('neqo_glue_ffi_generated.h',
-                  script='/build/RunCbindgen.py',
-                  entry_point='generate',
-                  inputs=['/netwerk/socket/neqo_glue'])
+    CbindgenHeader('neqo_glue_ffi_generated.h',
+                   inputs=['/netwerk/socket/neqo_glue'])
 
     EXPORTS.mozilla.net += [
         '!neqo_glue_ffi_generated.h',

taskcluster/ci/fetch/toolchains.yml

@@ -280,12 +280,12 @@ wine-5.0:
             sig-url: "{url}.sign"
             key-path: build/unix/build-gcc/DA23579A74D4AD9AF9D3F945CEFAC8EAAF17519D.key
 
-cbindgen-0.14.1:
+cbindgen-0.14.3:
     description: cbindgen source code
     fetch:
         type: git
         repo: https://github.com/eqrion/cbindgen
-        revision: 0761b9bbe48d01ded1bbec45bbeea5544b3b1002
+        revision: cc2876f709808a52d132a5f84e53ca1964eb92db
 
 cctools-port:
     description: cctools-port source code

taskcluster/ci/toolchain/cbindgen.yml

@@ -17,7 +17,7 @@ job-defaults:
         fetch:
             # If you update this, make sure to update the minimum version in
             # build/moz.configure/bindgen.configure as well.
-            - cbindgen-0.14.1
+            - cbindgen-0.14.3
 
 linux64-cbindgen:
     treeherder:

xpcom/base/moz.build

@@ -220,9 +220,8 @@ if CONFIG['COMPILE_ENVIRONMENT']:
         '!gk_rust_utils_ffi_generated.h',
     ]
 
-    GeneratedFile('gk_rust_utils_ffi_generated.h',
-                  script='/build/RunCbindgen.py',
-                  entry_point='generate', inputs=['/xpcom/rust/gkrust_utils'])
+    CbindgenHeader('gk_rust_utils_ffi_generated.h',
+                   inputs=['/xpcom/rust/gkrust_utils'])
 
 include('/ipc/chromium/chromium-config.mozbuild')