From e52b331537d90ed48a5aa89cb4bcb4f238f22e7f Mon Sep 17 00:00:00 2001 From: Ricky Stewart Date: Wed, 25 Mar 2020 19:40:06 +0000 Subject: [PATCH] Bug 1623982 - Fix performance of indented_repr in Python 3 r=ahal Differential Revision: https://phabricator.services.mozilla.com/D67688 --HG-- extra : moz-landing-system : lando --- configure.py | 5 ++-- python/mozbuild/mozbuild/test/test_util.py | 23 ++++++++++----- python/mozbuild/mozbuild/util.py | 34 +++++----------------- 3 files changed, 25 insertions(+), 37 deletions(-) diff --git a/configure.py b/configure.py index 3b8f99422785..a310a35a9e0c 100644 --- a/configure.py +++ b/configure.py @@ -29,7 +29,7 @@ from mozbuild.configure import ( from mozbuild.pythonutil import iter_modules_in_path from mozbuild.backend.configenvironment import PartialConfigEnvironment from mozbuild.util import ( - indented_repr, + write_indented_repr, ) import mozpack.path as mozpath import six @@ -120,7 +120,8 @@ def config_status(config): from __future__ import unicode_literals ''') % {'python': config['PYTHON']}) for k, v in six.iteritems(sanitized_config): - fh.write('%s = %s\n' % (k, indented_repr(v))) + fh.write('%s = ' % k) + write_indented_repr(fh, v) fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'non_global_defines', 'substs', 'mozconfig']") diff --git a/python/mozbuild/mozbuild/test/test_util.py b/python/mozbuild/mozbuild/test/test_util.py index 3b00a1d5fb23..69d5601a1adc 100644 --- a/python/mozbuild/mozbuild/test/test_util.py +++ b/python/mozbuild/mozbuild/test/test_util.py @@ -5,8 +5,9 @@ from __future__ import absolute_import, print_function, unicode_literals -import itertools import hashlib +import io +import itertools import os import unittest import six @@ -21,11 +22,11 @@ from mozbuild.util import ( expand_variables, group_unified_files, hash_file, - indented_repr, memoize, memoized_property, pair, resolve_target_to_make, + write_indented_repr, MozbuildDeletionError, HierarchicalStringList, EnumString, @@ -819,7 +820,7 @@ class TestEnumString(unittest.TestCase): class TestIndentedRepr(unittest.TestCase): @unittest.skipUnless(six.PY2, 'requires Python 2') - def test_indented_repr_py2(self): + def test_write_indented_repr_py2(self): data = textwrap.dedent(r''' { 'a': 1, @@ -840,14 +841,17 @@ class TestIndentedRepr(unittest.TestCase): 'pile_of_poo': '💩', 'special_chars': '\\\'"\x08\n\t', 'with_accents': 'éàñ', - }''').lstrip() + } + ''').lstrip() obj = eval(data) + buf = io.StringIO() + write_indented_repr(buf, obj) - self.assertEqual(indented_repr(obj), data) + self.assertEqual(buf.getvalue(), data) @unittest.skipUnless(six.PY3, 'requires Python 3') - def test_indented_repr(self): + def test_write_indented_repr(self): data = textwrap.dedent(r''' { b'c': 'xyz', 'a': 1, @@ -858,11 +862,14 @@ class TestIndentedRepr(unittest.TestCase): 'pile_of_bytes': b'\xf0\x9f\x92\xa9', 'pile_of_poo': '💩', 'special_chars': '\\\'"\x08\n\t', - 'with_accents': 'éàñ'}''').lstrip() + 'with_accents': 'éàñ'} + ''').lstrip() obj = eval(data) + buf = six.StringIO() + write_indented_repr(buf, obj) - self.assertEqual(indented_repr(obj), data) + self.assertEqual(buf.getvalue(), data) if __name__ == '__main__': diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py index 0547cd591cec..044cf645c1f1 100644 --- a/python/mozbuild/mozbuild/util.py +++ b/python/mozbuild/mozbuild/util.py @@ -20,7 +20,6 @@ import os import pprint import re import stat -import subprocess import sys import time from collections import ( @@ -1297,28 +1296,6 @@ def _escape_char(c): return six.text_type(c.encode('unicode_escape')) -# The default PrettyPrinter has some issues with UTF-8, so we need to override -# some stuff here. -class _PrettyPrinter(pprint.PrettyPrinter): - def format(self, object, context, maxlevels, level): - if not (isinstance(object, six.text_type) or - isinstance(object, six.binary_type)): - return super(_PrettyPrinter, self).format( - object, context, maxlevels, level) - # This is super hacky and weird, but the output of 'repr' actually - # varies based on the default I/O encoding of the process, which isn't - # necessarily utf-8. Instead we open a new shell and ask what the repr - # WOULD be assuming the default encoding is utf-8. If you can come up - # with a better way of doing this without simply re-implementing the - # logic of "repr", please replace this. - env = dict(os.environ) - env['PYTHONIOENCODING'] = 'utf-8' - ret = six.ensure_text(subprocess.check_output( - [sys.executable], input='print(repr(%s))' % repr(object), - universal_newlines=True, env=env, encoding='utf-8')).strip() - return (ret, True, False) - - if six.PY2: # Delete when we get rid of Python 2. # Mapping table between raw characters below \x80 and their escaped # counterpart, when they differ @@ -1333,14 +1310,16 @@ if six.PY2: # Delete when we get rid of Python 2. '([' + ''.join(_INDENTED_REPR_TABLE.values()) + ']+)') -def indented_repr(o, indent=4): - '''Similar to repr(), but returns an indented representation of the object +def write_indented_repr(f, o, indent=4): + '''Write an indented representation (similar to repr()) of the object to the + given file `f`. One notable difference with repr is that the returned representation assumes `from __future__ import unicode_literals`. ''' if six.PY3: - return _PrettyPrinter(indent=indent).pformat(o) + pprint.pprint(o, stream=f, indent=indent) + return # Delete everything below when we get rid of Python 2. one_indent = ' ' * indent @@ -1382,7 +1361,8 @@ def indented_repr(o, indent=4): yield ']' else: yield repr(o) - return ''.join(recurse_indented_repr(o, 0)) + result = ''.join(recurse_indented_repr(o, 0)) + '\n' + f.write(result) def patch_main():