Bug 1613263 - Use io.open() rather than open() in mozbuild/preprocessor.py r=firefox-build-system-reviewers,mshal

Differential Revision: https://phabricator.services.mozilla.com/D61647

--HG--
extra : moz-landing-system : lando

config/mozunit/mozunit/mozunit.py

@@ -4,7 +4,12 @@
 
 from __future__ import absolute_import
 import inspect
+import io
 import os
+from six import (
+    BytesIO,
+    StringIO,
+)
 import sys
 import unittest
 from unittest import TextTestRunner as _TestRunner, TestResult as _TestResult
@@ -24,8 +29,6 @@ except ImportError:
     build = MozbuildObject.from_environment(cwd=here)
     topsrcdir = build.topsrcdir
 
-StringIO = six.StringIO
-
 '''Helper to make python unit tests report the way that the Mozilla
 unit test infrastructure expects tests to report.
 
@@ -113,21 +116,30 @@ class MozTestRunner(_TestRunner):
         return result
 
 
-class MockedFile(StringIO):
-    def __init__(self, context, filename, content=''):
-        self.context = context
-        self.name = filename
-        StringIO.__init__(self, content)
+def _mocked_file(cls):
+    '''Create a mocked file class that inherits from the given class.
+    '''
+    class MockedFile(cls):
+        def __init__(self, context, filename, content):
+            self.context = context
+            self.name = filename
+            cls.__init__(self, content)
 
-    def close(self):
-        self.context.files[self.name] = self.getvalue()
-        StringIO.close(self)
+        def close(self):
+            self.context.files[self.name] = self.getvalue()
+            cls.close(self)
 
-    def __enter__(self):
-        return self
+        def __enter__(self):
+            return self
 
-    def __exit__(self, type, value, traceback):
-        self.close()
+        def __exit__(self, type, value, traceback):
+            self.close()
+
+    return MockedFile
+
+
+MockedStringFile = _mocked_file(StringIO)
+MockedBytesFile = _mocked_file(BytesIO)
 
 
 def normcase(path):
@@ -142,6 +154,60 @@ def normcase(path):
     return path
 
 
+class _MockBaseOpen(object):
+    '''Callable that acts like the open() function; see MockedOpen for more
+    info.
+    '''
+    def __init__(self, open, files):
+        self.open = open
+        self.files = files
+
+    def __call__(self, name, mode='r', buffering=None, encoding=None):
+        # open() can be called with an integer "name" (i.e. a file descriptor).
+        # We don't generally do this in our codebase, but internal Python
+        # libraries sometimes do and we want to handle that cleanly.
+        if isinstance(name, int):
+            return self.open(name, mode=mode, buffering=buffering,
+                             encoding=encoding)
+        # buffering is ignored.
+        absname = normcase(os.path.abspath(name))
+        if 'w' in mode:
+            file = self._mocked_file(absname, mode)
+        elif absname in self.files:
+            content = self.files[absname]
+            if content is None:
+                raise IOError(2, 'No such file or directory')
+            file = self._mocked_file(absname, mode, content)
+        elif 'a' in mode:
+            read_mode = 'rb' if 'b' in mode else 'r'
+            file = self._mocked_file(
+                absname, mode, self.open(name, read_mode).read())
+        else:
+            file = self.open(name, mode)
+        if 'a' in mode:
+            file.seek(0, os.SEEK_END)
+        return file
+
+    def _mocked_file(self, name, mode, content=None):
+        raise NotImplementedError('subclass must implement')
+
+
+class _MockPy2Open(_MockBaseOpen):
+    def _mocked_file(self, name, mode, content=None):
+        content = six.ensure_binary(content or b'')
+        return MockedBytesFile(self, name, content)
+
+
+class _MockOpen(_MockBaseOpen):
+    def _mocked_file(self, name, mode, content=None):
+        if 'b' in mode:
+            content = six.ensure_binary(content or b'')
+            return MockedBytesFile(self, name, content)
+        else:
+            content = six.ensure_text(content or u'')
+            return MockedStringFile(self, name, content)
+
+
 class MockedOpen(object):
     '''
     Context manager diverting the open builtin such that opening files
@@ -170,31 +236,16 @@ class MockedOpen(object):
         for name, content in files.items():
             self.files[normcase(os.path.abspath(name))] = content
 
-    def __call__(self, name, mode='r', buffering=None, encoding=None):
-        # buffering is ignored.
-        absname = normcase(os.path.abspath(name))
-        if 'w' in mode:
-            file = MockedFile(self, absname)
-        elif absname in self.files:
-            content = self.files[absname]
-            if content is None:
-                raise IOError(2, 'No such file or directory')
-            file = MockedFile(self, absname, content)
-        elif 'a' in mode:
-            file = MockedFile(self, absname, self.open(name, 'r').read())
-        else:
-            file = self.open(name, mode)
-        if 'a' in mode:
-            file.seek(0, os.SEEK_END)
-        return file
-
     def __enter__(self):
         import six.moves.builtins
         self.open = six.moves.builtins.open
+        self.io_open = io.open
         self._orig_path_exists = os.path.exists
         self._orig_path_isdir = os.path.isdir
         self._orig_path_isfile = os.path.isfile
-        six.moves.builtins.open = self
+        builtin_cls = _MockPy2Open if six.PY2 else _MockOpen
+        six.moves.builtins.open = builtin_cls(self.open, self.files)
+        io.open = _MockOpen(self.io_open, self.files)
         os.path.exists = self._wrapped_exists
         os.path.isdir = self._wrapped_isdir
         os.path.isfile = self._wrapped_isfile
@@ -202,6 +253,7 @@ class MockedOpen(object):
     def __exit__(self, type, value, traceback):
         import six.moves.builtins
         six.moves.builtins.open = self.open
+        io.open = self.io_open
         os.path.exists = self._orig_path_exists
         os.path.isdir = self._orig_path_isdir
         os.path.isfile = self._orig_path_isfile

python/mozbuild/mozbuild/jar.py

@@ -10,16 +10,17 @@ See the documentation for jar.mn on MDC for further details on the format.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import sys
-import os
 import errno
+import io
+import logging
+import os
 import re
 import six
-import logging
-from time import localtime
-from MozZipFile import ZipFile
 from six import BytesIO
+import sys
+from time import localtime
 
+from MozZipFile import ZipFile
 from mozbuild.preprocessor import Preprocessor
 from mozbuild.action.buildlist import addEntriesToListFile
 from mozbuild.util import ensure_bytes
@@ -467,8 +468,8 @@ class JarMaker(object):
         self._seen_output.add(out)
 
         if e.preprocess:
-            outf = outHelper.getOutput(out)
-            inf = open(realsrc)
+            outf = outHelper.getOutput(out, mode='w')
+            inf = io.open(realsrc, encoding='utf-8')
             pp = self.pp.clone()
             if src[-4:] == '.css':
                 pp.setMarker('%')
@@ -507,7 +508,7 @@ class JarMaker(object):
             except Exception:
                 return localtime(0)
 
-        def getOutput(self, name):
+        def getOutput(self, name, mode='wb'):
             return ZipEntry(name, self.jarfile)
 
     class OutputHelper_flat(object):
@@ -522,7 +523,7 @@ class JarMaker(object):
         def getDestModTime(self, aPath):
             return getModTime(os.path.join(self.basepath, aPath))
 
-        def getOutput(self, name):
+        def getOutput(self, name, mode='wb'):
             out = self.ensureDirFor(name)
 
             # remove previous link or file
@@ -531,7 +532,10 @@ class JarMaker(object):
             except OSError as e:
                 if e.errno != errno.ENOENT:
                     raise
-            return open(out, 'wb')
+            if 'b' in mode:
+                return io.open(out, mode)
+            else:
+                return io.open(out, mode, encoding='utf-8', newline='\n')
 
         def ensureDirFor(self, name):
             out = os.path.join(self.basepath, name)

python/mozbuild/mozbuild/preprocessor.py

@@ -24,12 +24,14 @@ value :
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import sys
+import errno
+import io
+from optparse import OptionParser
 import os
 import re
 import six
-from optparse import OptionParser
-import errno
+import sys
+
 from mozbuild.makeutil import Makefile
 
 # hack around win32 mangling our line endings
@@ -492,7 +494,7 @@ class Preprocessor:
                 except OSError as error:
                     if error.errno != errno.EEXIST:
                         raise
-            return open(path, 'wb')
+            return io.open(path, 'w', encoding='utf-8', newline='\n')
 
         p = self.getCommandLineParser()
         options, args = p.parse_args(args=args)
@@ -514,7 +516,7 @@ class Preprocessor:
 
         if args:
             for f in args:
-                with open(f, 'rU') as input:
+                with io.open(f, 'rU', encoding='utf-8') as input:
                     self.processFile(input=input, output=out)
             if depfile:
                 mk = Makefile()
@@ -804,7 +806,7 @@ class Preprocessor:
                     args = self.applyFilters(args)
                 if not os.path.isabs(args):
                     args = os.path.join(self.curdir, args)
-                args = open(args, 'rU')
+                args = io.open(args, 'rU', encoding='utf-8')
             except Preprocessor.Error:
                 raise
             except Exception:
@@ -859,7 +861,7 @@ def preprocess(includes=[sys.stdin], defines={},
     pp = Preprocessor(defines=defines,
                       marker=marker)
     for f in includes:
-        with open(f, 'rU') as input:
+        with io.open(f, 'rU', encoding='utf-8') as input:
             pp.processFile(input=input, output=output)
     return pp.includes
 

python/mozbuild/mozbuild/test/backend/test_build.py

@@ -151,11 +151,8 @@ class TestBuild(unittest.TestCase):
         test_path = os.sep.join(('$SRCDIR', 'python', 'mozbuild', 'mozbuild',
                                  'test', 'backend', 'data', 'build')) + os.sep
 
-        # We want unicode instances out of the files, because having plain str
-        # makes assertEqual diff output in case of error extra verbose because
-        # of the difference in type.
         result = {
-            p: f.open().read().decode('utf-8')
+            p: f.open(mode='r').read()
             for p, f in FileFinder(mozpath.join(config.topobjdir, 'dist'))
         }
         self.assertTrue(len(result))