From e46038ecc7702948204989fc0ce2c5f501305987 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Thu, 14 Mar 2013 12:42:03 -0700 Subject: [PATCH] Bug 846523 - Relax Unicode encoding requirements in moz.build files; r=ted This should be only until bug 844509 is addressed. --- python/mozbuild/mozbuild/frontend/reader.py | 11 ++++++++++- .../mozbuild/mozbuild/test/frontend/test_sandbox.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbuild/frontend/reader.py index c145e4db7762..1c39b5470bb4 100644 --- a/python/mozbuild/mozbuild/frontend/reader.py +++ b/python/mozbuild/mozbuild/frontend/reader.py @@ -118,6 +118,8 @@ class MozbuildSandbox(Sandbox): """ Sandbox.__init__(self, allowed_variables=VARIABLES) + self._log = logging.getLogger(__name__) + self.config = config topobjdir = os.path.abspath(config.topobjdir) @@ -172,7 +174,14 @@ class MozbuildSandbox(Sandbox): substs = {} for k, v in config.substs.items(): if not isinstance(v, text_type): - v = v.decode('utf-8', 'strict') + try: + v = v.decode('utf-8') + except UnicodeDecodeError: + log(self._log, logging.INFO, 'lossy_encoding', + {'variable': k}, + 'Lossy Unicode encoding for {variable}. See bug 844509.') + + v = v.decode('utf-8', 'replace') substs[k] = v diff --git a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py index 2e147624a613..f5bee86a822c 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py +++ b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py @@ -300,6 +300,17 @@ add_tier_dir('t1', 'bat', static=True) 'test.py') self.assertEqual(sandbox['CONFIGURE_SUBST_FILES'], ['foo', 'bar']) + def test_invalid_utf8_substs(self): + """Ensure invalid UTF-8 in substs is converted with an error.""" + + config = MockConfig() + # This is really mbcs. It's a bunch of invalid UTF-8. + config.substs['BAD_UTF8'] = b'\x83\x81\x83\x82\x3A' + + sandbox = MozbuildSandbox(config, '/foo/moz.build') + + self.assertEqual(sandbox['CONFIG']['BAD_UTF8'], + u'\ufffd\ufffd\ufffd\ufffd:') if __name__ == '__main__': main()