Improve support for Python3 in Mac, iOS build

- Decode the result of subprocess.Popen() so that the stdout/stderr result is str, not bytes. - Use dict.items() instead of dict.iteritems(). - If the return of map() needs to be a list, use list comprehension. - Use range() instead of xrange(). - Tuple parameter unpacking is removed on Python3, refactor its usage. - struct.pack() doesn't accept str, convert arguments as bytes. - plistlib has several API changes. Replace removed. functions (readPlistFromString). Still have some warnings like this: "The readPlist function is deprecated, use load() instead" - _GetOutputNoError() in tweak_info_plist.py has no usage. Removed. No intended behavior change. Bug: 941669 Change-Id: Ibc204fad8b56e864a083046559133403dcf99fe3 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2392289 Reviewed-by: Robert Sesek <rsesek@chromium.org> Commit-Queue: Robert Sesek <rsesek@chromium.org> Cr-Commit-Position: refs/heads/master@{#805790} GitOrigin-RevId: 1fdde535bbe5fb06abdb35a7aadf5e5903ab3913
2020-09-10 16:38:39 +00:00 · 2020-09-10 16:38:39 +00:00 · 820101cf3c
--- a/apple/tweak_info_plist.py
+++ b/apple/tweak_info_plist.py
@ -43,22 +43,8 @@ def _GetOutput(args):
  """Runs a subprocess and waits for termination. Returns (stdout, returncode)
  of the process. stderr is attached to the parent."""
  proc = subprocess.Popen(args, stdout=subprocess.PIPE)
-  (stdout, stderr) = proc.communicate()
-  return (stdout, proc.returncode)
-
-
-def _GetOutputNoError(args):
-  """Similar to _GetOutput() but ignores stderr. If there's an error launching
-  the child (like file not found), the exception will be caught and (None, 1)
-  will be returned to mimic quiet failure."""
-  try:
-    proc = subprocess.Popen(args,
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE)
-  except OSError:
-    return (None, 1)
-  (stdout, stderr) = proc.communicate()
-  return (stdout, proc.returncode)
+  stdout, _ = proc.communicate()
+  return stdout.decode('UTF-8'), proc.returncode


 def _RemoveKeys(plist, *keys):
@ -194,9 +180,9 @@ def _TagSuffixes():
  components_len = len(components)
  combinations = 1 << components_len
  tag_suffixes = []
-  for combination in xrange(0, combinations):
+  for combination in range(0, combinations):
    tag_suffix = ''
-    for component_index in xrange(0, components_len):
+    for component_index in range(0, components_len):
      if combination & (1 << component_index):
        tag_suffix += '-' + components[component_index]
    tag_suffixes.append(tag_suffix)
--- a/config/ios/codesign.py
+++ b/config/ios/codesign.py
@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

+from __future__ import print_function
+
 import argparse
 import codecs
 import datetime
@ -15,6 +17,11 @@ import subprocess
 import sys
 import tempfile

+if sys.version_info.major < 3:
+  basestring_compat = basestring
+else:
+  basestring_compat = str
+

 def GetProvisioningProfilesDir():
  """Returns the location of the installed mobile provisioning profiles.
@ -27,6 +34,21 @@ def GetProvisioningProfilesDir():
      os.environ['HOME'], 'Library', 'MobileDevice', 'Provisioning Profiles')


+def ReadPlistFromString(plist_bytes):
+  """Parse property list from given |plist_bytes|.
+
+    Args:
+      plist_bytes: contents of property list to load. Must be bytes in python 3.
+
+    Returns:
+      The contents of property list as a python object.
+    """
+  if sys.version_info.major == 2:
+    return plistlib.readPlistFromString(plist_bytes)
+  else:
+    return plistlib.loads(plist_bytes)
+
+
 def LoadPlistFile(plist_path):
  """Loads property list file at |plist_path|.

@ -36,8 +58,9 @@ def LoadPlistFile(plist_path):
  Returns:
    The content of the property list file as a python object.
  """
-  return plistlib.readPlistFromString(subprocess.check_output([
-      'xcrun', 'plutil', '-convert', 'xml1', '-o', '-', plist_path]))
+  return ReadPlistFromString(
+      subprocess.check_output(
+          ['xcrun', 'plutil', '-convert', 'xml1', '-o', '-', plist_path]))


 class Bundle(object):
@ -74,7 +97,7 @@ class Bundle(object):
      error message. The dictionary will be empty if there are no errors.
    """
    errors = {}
-    for key, expected_value in expected_mappings.iteritems():
+    for key, expected_value in expected_mappings.items():
      if key in self._data:
        value = self._data[key]
        if value != expected_value:
@ -88,9 +111,11 @@ class ProvisioningProfile(object):
  def __init__(self, provisioning_profile_path):
    """Initializes the ProvisioningProfile with data from profile file."""
    self._path = provisioning_profile_path
-    self._data = plistlib.readPlistFromString(subprocess.check_output([
-        'xcrun', 'security', 'cms', '-D', '-u', 'certUsageAnyCA',
-        '-i', provisioning_profile_path]))
+    self._data = ReadPlistFromString(
+        subprocess.check_output([
+            'xcrun', 'security', 'cms', '-D', '-u', 'certUsageAnyCA', '-i',
+            provisioning_profile_path
+        ]))

  @property
  def path(self):
@ -155,13 +180,13 @@ class Entitlements(object):
    self._data = self._ExpandVariables(self._data, substitutions)

  def _ExpandVariables(self, data, substitutions):
-    if isinstance(data, str):
-      for key, substitution in substitutions.iteritems():
+    if isinstance(data, basestring_compat):
+      for key, substitution in substitutions.items():
        data = data.replace('$(%s)' % (key,), substitution)
      return data

    if isinstance(data, dict):
-      for key, value in data.iteritems():
+      for key, value in data.items():
        data[key] = self._ExpandVariables(value, substitutions)
      return data

@ -172,7 +197,7 @@ class Entitlements(object):
    return data

  def LoadDefaults(self, defaults):
-    for key, value in defaults.iteritems():
+    for key, value in defaults.items():
      if key not in self._data:
        self._data[key] = value

--- a/config/ios/compile_xcassets.py
+++ b/config/ios/compile_xcassets.py
@ -32,16 +32,18 @@ NOTICE_SECTION = 'com.apple.actool.compilation-results'
 # Regular expressions matching spurious messages from actool that should be
 # ignored (as they are bogus). Generally a bug should be filed with Apple
 # when adding a pattern here.
-SPURIOUS_PATTERNS = map(re.compile, [
-    # crbug.com/770634, likely a bug in Xcode 9.1 beta, remove once build
-    # requires a version of Xcode with a fix.
-    r'\[\]\[ipad\]\[76x76\]\[\]\[\]\[1x\]\[\]\[\]: notice: \(null\)',
+SPURIOUS_PATTERNS = [
+    re.compile(v) for v in [
+        # crbug.com/770634, likely a bug in Xcode 9.1 beta, remove once build
+        # requires a version of Xcode with a fix.
+        r'\[\]\[ipad\]\[76x76\]\[\]\[\]\[1x\]\[\]\[\]: notice: \(null\)',

-    # crbug.com/770634, likely a bug in Xcode 9.2 beta, remove once build
-    # requires a version of Xcode with a fix.
-    r'\[\]\[ipad\]\[76x76\]\[\]\[\]\[1x\]\[\]\[\]: notice: 76x76@1x app icons'
-    ' only apply to iPad apps targeting releases of iOS prior to 10.0.',
-])
+        # crbug.com/770634, likely a bug in Xcode 9.2 beta, remove once build
+        # requires a version of Xcode with a fix.
+        r'\[\]\[ipad\]\[76x76\]\[\]\[\]\[1x\]\[\]\[\]: notice: 76x76@1x app icons'
+        ' only apply to iPad apps targeting releases of iOS prior to 10.0.',
+    ]
+]

 # Map special type of asset catalog to the corresponding command-line
 # parameter that need to be passed to actool.
@ -193,7 +195,7 @@ def CompileAssetCatalog(output, platform, product_type, min_deployment_target,
    stdout, _ = process.communicate()

    # Filter the output to remove all garbarge and to fix the paths.
-    stdout = FilterCompilerOutput(stdout, relative_paths)
+    stdout = FilterCompilerOutput(stdout.decode('UTF-8'), relative_paths)

    if process.returncode or stdout:
      sys.stderr.write(stdout)
--- a/config/ios/strip_arm64e.py
+++ b/config/ios/strip_arm64e.py
@ -18,9 +18,9 @@ def check_output(command):
  if process.returncode:
    sys.stderr.write('error: command failed with retcode %d: %s\n\n' %
                     (process.returncode, ' '.join(map(repr, command))))
-    sys.stderr.write(errs)
+    sys.stderr.write(errs.decode('UTF-8', errors='ignore'))
    sys.exit(process.returncode)
-  return outs
+  return outs.decode('UTF-8')


 def check_call(command):
--- a/config/ios/write_framework_hmap.py
+++ b/config/ios/write_framework_hmap.py
@ -50,7 +50,7 @@ def WriteHmap(output_name, filelist):
  count = len(filelist)
  capacity = NextGreaterPowerOf2(count)
  strings_offset = 24 + (12 * capacity)
-  max_value_length = len(max(filelist.items(), key=lambda (k,v):len(v))[1])
+  max_value_length = len(max(filelist.values(), key=lambda v: len(v)))

  out = open(output_name, 'wb')
  out.write(struct.pack('<LHHLLLL', magic, version, _reserved, strings_offset,
@ -86,14 +86,17 @@ def WriteHmap(output_name, filelist):
  for bucket in buckets:
    if bucket is not None:
      (file, path) = bucket
-      out.write(struct.pack('<%ds' % len(file), file))
-      out.write(struct.pack('<s', '\0'))
      base = os.path.dirname(path) + os.sep
-      out.write(struct.pack('<%ds' % len(base), base))
-      out.write(struct.pack('<s', '\0'))
      path = os.path.basename(path)
+      file = file.encode('UTF-8')
+      base = base.encode('UTF-8')
+      path = path.encode('UTF-8')
+      out.write(struct.pack('<%ds' % len(file), file))
+      out.write(struct.pack('<s', b'\0'))
+      out.write(struct.pack('<%ds' % len(base), base))
+      out.write(struct.pack('<s', b'\0'))
      out.write(struct.pack('<%ds' % len(path), path))
-      out.write(struct.pack('<s', '\0'))
+      out.write(struct.pack('<s', b'\0'))


 if __name__ == '__main__':
--- a/config/mac/package_framework.py
+++ b/config/mac/package_framework.py
@ -23,7 +23,7 @@ def Main():
  # Foo.framework/Versions/Current symlink to it.
  if args.version:
    try:
-      os.makedirs(os.path.join(args.framework, VERSIONS, args.version), 0755)
+      os.makedirs(os.path.join(args.framework, VERSIONS, args.version), 0o755)
    except OSError as e:
      if e.errno != errno.EEXIST:
        raise e
--- a/config/mac/plist_util.py
+++ b/config/mac/plist_util.py
@ -12,6 +12,10 @@ import sys
 import tempfile
 import shlex

+if sys.version_info.major < 3:
+  basestring_compat = basestring
+else:
+  basestring_compat = str

 # Xcode substitutes variables like ${PRODUCT_NAME} or $(PRODUCT_NAME) when
 # compiling Info.plist. It also supports supports modifiers like :identifier
@ -80,10 +84,10 @@ def Interpolate(value, substitutions):
    substitution.
  """
  if isinstance(value, dict):
-    return {k: Interpolate(v, substitutions) for k, v in value.iteritems()}
+    return {k: Interpolate(v, substitutions) for k, v in value.items()}
  if isinstance(value, list):
    return [Interpolate(v, substitutions) for v in value]
-  if isinstance(value, str):
+  if isinstance(value, basestring_compat):
    return InterpolateString(value, substitutions)
  return value

@ -93,7 +97,7 @@ def LoadPList(path):
  fd, name = tempfile.mkstemp()
  try:
    subprocess.check_call(['plutil', '-convert', 'xml1', '-o', name, path])
-    with os.fdopen(fd, 'r') as f:
+    with os.fdopen(fd, 'rb') as f:
      return plistlib.readPlist(f)
  finally:
    os.unlink(name)
@ -109,7 +113,7 @@ def SavePList(path, format, data):
    # it does exist.
    if os.path.exists(path):
      os.unlink(path)
-    with os.fdopen(fd, 'w') as f:
+    with os.fdopen(fd, 'wb') as f:
      plistlib.writePlist(data, f)
    subprocess.check_call(['plutil', '-convert', format, '-o', path, name])
  finally:
@ -134,7 +138,7 @@ def MergePList(plist1, plist2):
    are concatenated.
  """
  result = plist1.copy()
-  for key, value in plist2.iteritems():
+  for key, value in plist2.items():
    if isinstance(value, dict):
      old_value = result.get(key)
      if isinstance(old_value, dict):
--- a/config/mac/sdk_info.py
+++ b/config/mac/sdk_info.py
@ -13,6 +13,11 @@ import re
 import subprocess
 import sys

+if sys.version_info.major < 3:
+  basestring_compat = basestring
+else:
+  basestring_compat = str
+
 # src directory
 ROOT_SRC_DIR = os.path.dirname(
    os.path.dirname(
@ -61,7 +66,8 @@ def FillXcodeVersion(settings, developer_dir):
    settings['xcode_build'] = version_plist['ProductBuildVersion']
    return

-  lines = subprocess.check_output(['xcodebuild', '-version']).splitlines()
+  lines = subprocess.check_output(['xcodebuild',
+                                   '-version']).decode('UTF-8').splitlines()
  settings['xcode_version'] = FormatVersion(lines[0].split()[-1])
  settings['xcode_version_int'] = int(settings['xcode_version'], 10)
  settings['xcode_build'] = lines[-1].split()[-1]
@ -69,8 +75,8 @@ def FillXcodeVersion(settings, developer_dir):

 def FillMachineOSBuild(settings):
  """Fills OS build number into |settings|."""
-  machine_os_build = subprocess.check_output(['sw_vers', '-buildVersion'],
-                                             universal_newlines=True).strip()
+  machine_os_build = subprocess.check_output(['sw_vers', '-buildVersion'
+                                              ]).decode('UTF-8').strip()
  settings['machine_os_build'] = machine_os_build

  # The reported build number is made up from the kernel major version number,
@ -91,14 +97,17 @@ def FillMachineOSBuild(settings):

 def FillSDKPathAndVersion(settings, platform, xcode_version):
  """Fills the SDK path and version for |platform| into |settings|."""
-  settings['sdk_path'] = subprocess.check_output([
-      'xcrun', '-sdk', platform, '--show-sdk-path']).strip()
-  settings['sdk_version'] = subprocess.check_output([
-      'xcrun', '-sdk', platform, '--show-sdk-version']).strip()
-  settings['sdk_platform_path'] = subprocess.check_output([
-      'xcrun', '-sdk', platform, '--show-sdk-platform-path']).strip()
+  settings['sdk_path'] = subprocess.check_output(
+      ['xcrun', '-sdk', platform, '--show-sdk-path']).decode('UTF-8').strip()
+  settings['sdk_version'] = subprocess.check_output(
+      ['xcrun', '-sdk', platform,
+       '--show-sdk-version']).decode('UTF-8').strip()
+  settings['sdk_platform_path'] = subprocess.check_output(
+      ['xcrun', '-sdk', platform,
+       '--show-sdk-platform-path']).decode('UTF-8').strip()
  settings['sdk_build'] = subprocess.check_output(
-      ['xcrun', '-sdk', platform, '--show-sdk-build-version']).strip()
+      ['xcrun', '-sdk', platform,
+       '--show-sdk-build-version']).decode('UTF-8').strip()


 def CreateXcodeSymlinkAt(src, dst):
@ -157,6 +166,6 @@ if __name__ == '__main__':
    value = settings[key]
    if args.create_symlink_at and '_path' in key:
      value = CreateXcodeSymlinkAt(value, args.create_symlink_at)
-    if isinstance(value, str):
+    if isinstance(value, basestring_compat):
      value = '"%s"' % value
    print('%s=%s' % (key, value))
--- a/mac/find_sdk.py
+++ b/mac/find_sdk.py
@ -65,7 +65,7 @@ def main():
    print(out, file=sys.stderr)
    print(err, file=sys.stderr)
    raise Exception('Error %d running xcode-select' % job.returncode)
-  dev_dir = out.rstrip()
+  dev_dir = out.decode('UTF-8').rstrip()
  sdk_dir = os.path.join(
      dev_dir, 'Platforms/MacOSX.platform/Developer/SDKs')

--- a/toolchain/mac/filter_libtool.py
+++ b/toolchain/mac/filter_libtool.py
@ -12,14 +12,16 @@ import sys
 # This script executes libool and filters out logspam lines like:
 #    '/path/to/libtool: file: foo.o has no symbols'

-BLACKLIST_PATTERNS = map(re.compile, [
-    r'^.*libtool: (?:for architecture: \S* )?file: .* has no symbols$',
-    r'^.*libtool: warning for library: .* the table of contents is empty '
+BLACKLIST_PATTERNS = [
+    re.compile(v) for v in [
+        r'^.*libtool: (?:for architecture: \S* )?file: .* has no symbols$',
+        r'^.*libtool: warning for library: .* the table of contents is empty '
        r'\(no object file members in the library define global symbols\)$',
-    r'^.*libtool: warning same member name \(\S*\) in output file used for '
+        r'^.*libtool: warning same member name \(\S*\) in output file used for '
        r'input files: \S* and: \S* \(due to use of basename, truncation, '
        r'blank padding or duplicate input files\)$',
-])
+    ]
+]


 def IsBlacklistedLine(line):
@ -34,7 +36,7 @@ def Main(cmd_list):
  env = os.environ.copy()
  libtoolout = subprocess.Popen(cmd_list, stderr=subprocess.PIPE, env=env)
  _, err = libtoolout.communicate()
-  for line in err.splitlines():
+  for line in err.decode('UTF-8').splitlines():
    if not IsBlacklistedLine(line):
      print(line, file=sys.stderr)
  return libtoolout.returncode