Bug 1274090 - Attempt to convert str objects containing non-ascii characters in Python configure rather than failing outright. r=glandium,gps

MozReview-Commit-ID: CxFqFXS7Dh9

--HG--
extra : rebase_source : cd5136461756248335245d433b579dd06d266c97

python/mozbuild/mozbuild/configure/__init__.py

@@ -25,6 +25,7 @@ from mozbuild.configure.options import (
 from mozbuild.configure.help import HelpFormatter
 from mozbuild.configure.util import (
     ConfigureOutputHandler,
+    getpreferredencoding,
     LineIO,
 )
 from mozbuild.util import (
@@ -157,8 +158,24 @@ class ConfigureSandbox(dict):
             def queue_debug():
                 yield
 
+        # Some callers will manage to log a bytestring with characters in it
+        # that can't be converted to ascii. Make our log methods robust to this
+        # by detecting the encoding that a producer is likely to have used.
+        encoding = getpreferredencoding()
+        def wrapped_log_method(logger, key):
+            method = getattr(logger, key)
+            if not encoding:
+                return method
+            def wrapped(*args, **kwargs):
+                out_args = [
+                    arg.decode(encoding) if isinstance(arg, str) else arg
+                    for arg in args
+                ]
+                return method(*out_args, **kwargs)
+            return wrapped
+
         log_namespace = {
-            k: getattr(logger, k)
+            k: wrapped_log_method(logger, k)
             for k in ('debug', 'info', 'warning', 'error')
         }
         log_namespace['queue_debug'] = queue_debug

python/mozbuild/mozbuild/configure/util.py

@@ -14,6 +14,20 @@ from collections import deque
 from contextlib import contextmanager
 from distutils.version import LooseVersion
 
+def getpreferredencoding():
+    # locale._parse_localename makes locale.getpreferredencoding
+    # return None when LC_ALL is C, instead of e.g. 'US-ASCII' or
+    # 'ANSI_X3.4-1968' when it uses nl_langinfo.
+    encoding = None
+    try:
+        encoding = locale.getpreferredencoding()
+    except ValueError:
+        # On english OSX, LC_ALL is UTF-8 (not en-US.UTF-8), and
+        # that throws off locale._parse_localename, which ends up
+        # being used on e.g. homebrew python.
+        if os.environ.get('LC_ALL', '').upper() == 'UTF-8':
+            encoding = 'utf-8'
+    return encoding
 
 class Version(LooseVersion):
     '''A simple subclass of distutils.version.LooseVersion.
@@ -70,19 +84,7 @@ class ConfigureOutputHandler(logging.Handler):
                 isatty = True
 
             if not isatty:
-                encoding = None
-                try:
-                    encoding = locale.getpreferredencoding()
-                except ValueError:
-                    # On english OSX, LC_ALL is UTF-8 (not en-US.UTF-8), and
-                    # that throws off locale._parse_localename, which ends up
-                    # being used on e.g. homebrew python.
-                    if os.environ.get('LC_ALL', '').upper() == 'UTF-8':
-                        encoding = 'utf-8'
-
-                # locale._parse_localename makes locale.getpreferredencoding
-                # return None when LC_ALL is C, instead of e.g. 'US-ASCII' or
-                # 'ANSI_X3.4-1968' when it uses nl_langinfo.
+                encoding = getpreferredencoding()
                 if encoding:
                     return codecs.getwriter(encoding)(fh)
             return fh
@@ -195,8 +197,11 @@ class LineIO(object):
     def __init__(self, callback):
         self._callback = callback
         self._buf = ''
+        self._encoding = getpreferredencoding()
 
     def write(self, buf):
+        if self._encoding and isinstance(buf, str):
+            buf = buf.decode(self._encoding)
         lines = buf.splitlines()
         if not lines:
             return

python/mozbuild/mozbuild/test/configure/data/subprocess.configure

@@ -0,0 +1,23 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+@depends('--help')
+@imports('codecs')
+@imports(_from='mozbuild.configure.util', _import='getpreferredencoding')
+@imports('os')
+@imports(_from='__builtin__', _import='open')
+def dies_when_logging(_):
+    test_file = 'test.txt'
+    quote_char = "'"
+    if getpreferredencoding().lower() == 'utf-8':
+        quote_char = '\u00B4'.encode('utf-8')
+    try:
+        with open(test_file, 'w+') as fh:
+            fh.write(quote_char)
+        out = check_cmd_output('cat', 'test.txt')
+        log.info(out)
+    finally:
+        os.remove(test_file)

python/mozbuild/mozbuild/test/configure/test_util.py

@@ -18,9 +18,15 @@ from mozpack import path as mozpath
 
 from mozbuild.configure.util import (
     ConfigureOutputHandler,
+    getpreferredencoding,
     LineIO,
     Version,
 )
+
+from mozbuild.configure import (
+    ConfigureSandbox,
+)
+
 from mozbuild.util import exec_
 
 from buildconfig import topsrcdir
@@ -411,6 +417,30 @@ class TestLineIO(unittest.TestCase):
         self.assertEqual(lines, ['a', 'b', 'c'])
 
 
+class TestLogSubprocessOutput(unittest.TestCase):
+
+    def test_non_ascii_subprocess_output(self):
+        out = StringIO()
+        sandbox = ConfigureSandbox({}, {}, [], out, out)
+
+        sandbox.include_file(mozpath.join(topsrcdir, 'build',
+                             'moz.configure', 'util.configure'))
+        sandbox.include_file(mozpath.join(topsrcdir, 'python', 'mozbuild',
+                                          'mozbuild', 'test', 'configure',
+                                          'data', 'subprocess.configure'))
+        status = 0
+        try:
+            sandbox.run()
+        except SystemExit as e:
+            status = e.code
+
+        self.assertEquals(status, 0)
+        quote_char = "'"
+        if getpreferredencoding().lower() == 'utf-8':
+            quote_char = '\u00B4'.encode('utf-8')
+        self.assertEquals(out.getvalue().strip(), quote_char)
+
+
 class TestVersion(unittest.TestCase):
     def test_version_simple(self):
         v = Version('1')