Bug 1402012 - Update buildconfig.py to use PartialConfigEnvironment; r=glandium

By using the PartialConfigEnvironment, the clients of buildconfig will
depend on config.statusd/ files instead of config.status directly.
Clients can access substs and defines using buildconfig.substs['FOO'] or
buildconfig.defines['BAR'], and then collect file-level dependencies for
make using buildconfig.get_dependencies(). All GENERATED_FILES rules
already make use of this because file_generate.py automatically includes
these dependencies (along with all python modules loaded).

As a result of this commit, re-running configure will no longer cause
the world to be rebuilt. Although config.status is updated, no build
steps use config.status directly and instead depend on values in
config.statusd/, which are written with FileAvoidWrite. Since those
files are not official targets according to the make backend, make won't
try to continually rebuild the backend when those files are out of date.
And since they are FileAvoidWrite, make will only re-run dependent steps
if the actual configure value has changed.

As a result of using JSON to load data from the config.statusd
directory, substs can be unicode (instead of a bare string type).
generate_certdata.py converts the subst manually to a string so the
value can be exported to the environment without issue on Windows.

Additionally, patching the buildconfig.substs dict no longer works, so
the unit-symbolstore.py test was modified to patch the underlying
buildconfig.substs._dict instead.

The other files that needed to be modified make use of all the defines
for the preprocessor. Those that are used during 'mach build' now use
buildconfig.defines['ALLDEFINES'], which maps to a special
FileAvoidWrite file generated for the PartialConfigEnvironment.

MozReview-Commit-ID: 2pJ4s3TVeS8

--HG--
extra : rebase_source : d6bb0208483f9f043e7be1b36907ca13243985f8

build/buildconfig.py

@@ -5,16 +5,15 @@
 import os
 import sys
 from mozbuild.base import MozbuildObject
+from mozbuild.backend.configenvironment import PartialConfigEnvironment
 
 config = MozbuildObject.from_environment()
+partial_config = PartialConfigEnvironment(config.topobjdir)
 
-for var in ('topsrcdir', 'topobjdir', 'defines', 'non_global_defines',
-            'substs'):
+for var in ('topsrcdir', 'topobjdir'):
     value = getattr(config, var)
     setattr(sys.modules[__name__], var, value)
 
-substs = dict(substs)
-
-for var in os.environ:
-    if var not in ('CPP', 'CXXCPP', 'SHELL') and var in substs:
-        substs[var] = os.environ[var]
+for var in ('defines', 'substs', 'get_dependencies'):
+    value = getattr(partial_config, var)
+    setattr(sys.modules[__name__], var, value)

js/src/builtin/embedjs.py

@@ -132,8 +132,7 @@ def messages(jsmsg):
 
 def get_config_defines(buildconfig):
   # Collect defines equivalent to ACDEFINES and add MOZ_DEBUG_DEFINES.
-  env = {key: value for key, value in buildconfig.defines.iteritems()
-         if key not in buildconfig.non_global_defines}
+  env = buildconfig.defines['ALLDEFINES']
   for define in buildconfig.substs['MOZ_DEBUG_DEFINES']:
     env[define] = 1
   return env

mobile/android/base/generate_build_config.py

@@ -33,8 +33,8 @@ from mozbuild.android_version_code import android_version_code
 
 def _defines():
     CONFIG = defaultdict(lambda: None)
-    CONFIG.update(buildconfig.substs)
-    DEFINES = dict(buildconfig.defines)
+    CONFIG.update(buildconfig.substs.iteritems())
+    DEFINES = dict(buildconfig.defines.iteritems())
 
     for var in ('MOZ_ANDROID_ACTIVITY_STREAM'
                 'MOZ_ANDROID_ANR_REPORTER',

python/mozbuild/mozbuild/action/file_generate.py

@@ -85,6 +85,10 @@ def main(argv):
                 # the script.
                 deps |= set(iter_modules_in_path(buildconfig.topsrcdir,
                                                  buildconfig.topobjdir))
+                # Add dependencies on any buildconfig items that were accessed
+                # by the script.
+                deps |= set(buildconfig.get_dependencies())
+
                 mk = Makefile()
                 mk.create_rule([args.output_file]).add_dependencies(deps)
                 with FileAvoidWrite(args.dep_file) as dep_file:

python/mozbuild/mozbuild/action/generate_symbols_file.py

@@ -22,7 +22,7 @@ def generate_symbols_file(output, *args):
     input = os.path.abspath(args.input)
 
     pp = Preprocessor()
-    pp.context.update(buildconfig.defines)
+    pp.context.update(buildconfig.defines['ALLDEFINES'])
     if args.D:
         pp.context.update(args.D)
     for undefine in args.U:

security/generate_certdata.py

@@ -9,7 +9,7 @@ import subprocess
 
 def main(output, *inputs):
     env=dict(os.environ)
-    env['PERL'] = buildconfig.substs['PERL']
+    env['PERL'] = str(buildconfig.substs['PERL'])
     output.write(subprocess.check_output([buildconfig.substs['PYTHON'],
                  inputs[0], inputs[2]], env=env))
     return None

toolkit/crashreporter/tools/unit-symbolstore.py

@@ -150,7 +150,7 @@ class TestCopyDebug(HelperMixin, unittest.TestCase):
         d.Process(os.path.join(self.test_dir, add_extension(["foo"])[0]))
         self.assertEqual(1, len(copied))
 
-    @patch.dict('buildconfig.substs', {'MAKECAB': 'makecab'})
+    @patch.dict('buildconfig.substs._dict', {'MAKECAB': 'makecab'})
     def test_copy_debug_copies_binaries(self):
         """
         Test that CopyDebug copies binaries as well on Windows.

toolkit/mozapps/installer/find-dupes.py

@@ -102,7 +102,7 @@ def main():
     allowed_dupes = []
     for filename in args.dupes_files:
         pp = Preprocessor()
-        pp.context.update(buildconfig.defines)
+        pp.context.update(buildconfig.defines['ALLDEFINES'])
         if args.D:
             pp.context.update(args.D)
         for undefine in args.U:

toolkit/mozapps/installer/packager.py

@@ -226,7 +226,7 @@ def main():
                         help='Extra files not to be considered as resources')
     args = parser.parse_args()
 
-    defines = dict(buildconfig.defines)
+    defines = dict(buildconfig.defines['ALLDEFINES'])
     if args.ignore_errors:
         errors.ignore_errors()