Bug 846523 - Relax Unicode encoding requirements in moz.build files; r=ted

This should be only until bug 844509 is addressed.

python/mozbuild/mozbuild/frontend/reader.py

@@ -118,6 +118,8 @@ class MozbuildSandbox(Sandbox):
         """
         Sandbox.__init__(self, allowed_variables=VARIABLES)
 
+        self._log = logging.getLogger(__name__)
+
         self.config = config
 
         topobjdir = os.path.abspath(config.topobjdir)
@@ -172,7 +174,14 @@ class MozbuildSandbox(Sandbox):
             substs = {}
             for k, v in config.substs.items():
                 if not isinstance(v, text_type):
-                    v = v.decode('utf-8', 'strict')
+                    try:
+                        v = v.decode('utf-8')
+                    except UnicodeDecodeError:
+                        log(self._log, logging.INFO, 'lossy_encoding',
+                            {'variable': k},
+                            'Lossy Unicode encoding for {variable}. See bug 844509.')
+
+                        v = v.decode('utf-8', 'replace')
 
                 substs[k] = v
 

python/mozbuild/mozbuild/test/frontend/test_sandbox.py

@@ -300,6 +300,17 @@ add_tier_dir('t1', 'bat', static=True)
             'test.py')
         self.assertEqual(sandbox['CONFIGURE_SUBST_FILES'], ['foo', 'bar'])
 
+    def test_invalid_utf8_substs(self):
+        """Ensure invalid UTF-8 in substs is converted with an error."""
+
+        config = MockConfig()
+        # This is really mbcs. It's a bunch of invalid UTF-8.
+        config.substs['BAD_UTF8'] = b'\x83\x81\x83\x82\x3A'
+
+        sandbox = MozbuildSandbox(config, '/foo/moz.build')
+
+        self.assertEqual(sandbox['CONFIG']['BAD_UTF8'],
+            u'\ufffd\ufffd\ufffd\ufffd:')
 
 if __name__ == '__main__':
     main()