Bug 1607284 - mozbuild/configure/util.py supports Python 3 r=ahal,mars

Differential Revision: https://phabricator.services.mozilla.com/D58834

--HG--
extra : moz-landing-system : lando

build/moz.configure/util.configure

@@ -23,22 +23,21 @@ def configure_error(message):
 # A wrapper to obtain a process' output and return code.
 # Returns a tuple (retcode, stdout, stderr).
 @imports('os')
+@imports('six')
 @imports('subprocess')
 @imports(_from='mozbuild.shellutil', _import='quote')
 @imports(_from='mozbuild.util', _import='system_encoding')
 def get_cmd_output(*args, **kwargs):
     log.debug('Executing: `%s`', quote(*args))
-    proc = subprocess.Popen(args, stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE,
-                            # On Python 2 on Windows, close_fds prevents the
-                            # process from inheriting stdout/stderr.
-                            # Elsewhere, it simply prevents it from inheriting
-                            # extra file descriptors, which is what we want.
-                            close_fds=os.name != 'nt',
-                            **kwargs)
+    proc = subprocess.Popen(
+        args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+        # On Python 2 on Windows, close_fds prevents the process from inheriting
+        # stdout/stderr. Elsewhere, it simply prevents it from inheriting extra
+        # file descriptors, which is what we want.
+        close_fds=os.name != 'nt', universal_newlines=True, **kwargs)
     stdout, stderr = proc.communicate()
-    stdout = stdout.decode(system_encoding, 'replace')
-    stderr = stderr.decode(system_encoding, 'replace')
+    stdout = six.ensure_text(stdout, encoding=system_encoding, errors='replace')
+    stderr = six.ensure_text(stderr, encoding=system_encoding, errors='replace')
     return proc.wait(), stdout, stderr
 
 

python/mozbuild/mozbuild/base.py

@@ -280,7 +280,7 @@ class MozbuildObject(ProcessExecutionMixin):
         # the environment variable, which has an impact on autodetection (when
         # path is MozconfigLoader.AUTODETECT), and memoization wouldn't account
         # for it without the explicit (unused) argument.
-        out = six.BytesIO()
+        out = six.StringIO()
         env = os.environ
         if path and path != MozconfigLoader.AUTODETECT:
             env = dict(env)

python/mozbuild/mozbuild/configure/__init__.py

@@ -373,13 +373,10 @@ class ConfigureSandbox(dict):
 
         def wrapped_log_method(logger, key):
             method = getattr(logger, key)
-            if not encoding:
-                return method
-
             def wrapped(*args, **kwargs):
                 out_args = [
-                    arg.decode(encoding) if isinstance(arg, six.binary_type) else arg
-                    for arg in args
+                    six.ensure_text(arg, encoding=encoding or 'utf-8')
+                    if isinstance(arg, six.binary_type) else arg for arg in args
                 ]
                 return method(*out_args, **kwargs)
             return wrapped
@@ -917,6 +914,8 @@ class ConfigureSandbox(dict):
         # restricted mode". We also make open() look more like python 3's,
         # decoding to unicode strings unless the mode says otherwise.
         if what == '__builtin__.open':
+            if six.PY3:
+                return open
             def wrapped_open(name, mode=None, buffering=None):
                 args = (name,)
                 kwargs = {}

python/mozbuild/mozbuild/configure/util.py

@@ -83,6 +83,8 @@ class ConfigureOutputHandler(logging.Handler):
         # Python has this feature where it sets the encoding of pipes to
         # ascii, which blatantly fails when trying to print out non-ascii.
         def fix_encoding(fh):
+            if six.PY3:
+                return fh
             try:
                 isatty = fh.isatty()
             except AttributeError:
@@ -128,7 +130,7 @@ class ConfigureOutputHandler(logging.Handler):
         try:
             if record.levelno == logging.INFO:
                 stream = self._stdout
-                msg = record.getMessage()
+                msg = six.ensure_text(record.getMessage())
                 if (self._stdout_waiting == self.INTERRUPTED and
                         self._same_output):
                     msg = ' ... %s' % msg
@@ -207,8 +209,7 @@ class LineIO(object):
         self._errors = errors
 
     def write(self, buf):
-        if self._encoding and isinstance(buf, str):
-            buf = buf.decode(self._encoding, self._errors)
+        buf = six.ensure_text(buf, encoding=self._encoding or 'utf-8')
         lines = buf.splitlines()
         if not lines:
             return

python/mozbuild/mozbuild/shellutil.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function
+from __future__ import absolute_import, print_function, unicode_literals
 
 import re
 

python/mozbuild/mozbuild/test/configure/common.py

@@ -7,11 +7,12 @@ from __future__ import absolute_import, print_function, unicode_literals
 import copy
 import errno
 import os
+import six
 import subprocess
 import sys
 import tempfile
 import unittest
-from StringIO import StringIO
+from six import StringIO
 
 from mozbuild.configure import ConfigureSandbox
 from mozbuild.util import ReadOnlyNamespace
@@ -78,10 +79,10 @@ class ConfigureTestSandbox(ConfigureSandbox):
         self._search_path = environ.get('PATH', '').split(os.pathsep)
 
         self._subprocess_paths = {
-            mozpath.abspath(k): v for k, v in paths.iteritems() if v
+            mozpath.abspath(k): v for k, v in six.iteritems(paths) if v
         }
 
-        paths = paths.keys()
+        paths = list(paths)
 
         environ = copy.copy(environ)
         if 'CONFIG_SHELL' not in environ:

python/mozbuild/mozbuild/test/configure/data/subprocess.configure

@@ -13,9 +13,9 @@ def dies_when_logging(_):
     test_file = 'test.txt'
     quote_char = "'"
     if getpreferredencoding().lower() == 'utf-8':
-        quote_char = '\u00B4'.encode('utf-8')
+        quote_char = '\u00B4'
     try:
-        with open(test_file, 'w+b') as fh:
+        with open(test_file, 'w+t') as fh:
             fh.write(quote_char)
         out = check_cmd_output('cat', 'test.txt')
         log.info(out)

python/mozbuild/mozbuild/test/configure/test_util.py

@@ -9,9 +9,10 @@ import os
 import tempfile
 import textwrap
 import unittest
+import six
 import sys
 
-from StringIO import StringIO
+from six import StringIO
 
 from mozunit import main
 from mozpack import path as mozpath
@@ -437,8 +438,8 @@ class TestLogSubprocessOutput(unittest.TestCase):
         self.assertEquals(status, 0)
         quote_char = "'"
         if getpreferredencoding().lower() == 'utf-8':
-            quote_char = '\u00B4'.encode('utf-8')
-        self.assertEquals(out.getvalue().strip(), quote_char)
+            quote_char = '\u00B4'
+        self.assertEquals(six.ensure_text(out.getvalue().strip()), quote_char)
 
 
 class TestVersion(unittest.TestCase):

python/mozbuild/mozbuild/test/python.ini

@@ -1,6 +1,7 @@
 [DEFAULT]
 subsuite = mozbuild
 
+[configure/test_util.py]
 [controller/test_ccachestats.py]
 [controller/test_clobber.py]
 [test_artifact_cache.py]

python/mozbuild/mozbuild/test/python2.ini

@@ -28,7 +28,6 @@ skip-if = (os == "win")
 [configure/test_toolchain_configure.py]
 [configure/test_toolchain_helpers.py]
 [configure/test_toolkit_moz_configure.py]
-[configure/test_util.py]
 [frontend/test_context.py]
 [frontend/test_emitter.py]
 [frontend/test_namespaces.py]