Improve support for Python3 in Mac, iOS build

- Decode the result of subprocess.Popen() so that the stdout/stderr
  result is str, not bytes.
- Use dict.items() instead of dict.iteritems().
- If the return of map() needs to be a list, use list comprehension.
- Use range() instead of xrange().
- Tuple parameter unpacking is removed on Python3, refactor its usage.
- struct.pack() doesn't accept str, convert arguments as bytes.
- plistlib has several API changes. Replace removed.
  functions (readPlistFromString). Still have some warnings like this:
  "The readPlist function is deprecated, use load() instead"
- _GetOutputNoError() in tweak_info_plist.py has no usage. Removed.

No intended behavior change.

Bug: 941669
Change-Id: Ibc204fad8b56e864a083046559133403dcf99fe3
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2392289
Reviewed-by: Robert Sesek <rsesek@chromium.org>
Commit-Queue: Robert Sesek <rsesek@chromium.org>
Cr-Commit-Position: refs/heads/master@{#805790}
GitOrigin-RevId: 1fdde535bbe5fb06abdb35a7aadf5e5903ab3913

apple/tweak_info_plist.py

@@ -43,22 +43,8 @@ def _GetOutput(args):
   """Runs a subprocess and waits for termination. Returns (stdout, returncode)
   of the process. stderr is attached to the parent."""
   proc = subprocess.Popen(args, stdout=subprocess.PIPE)
-  (stdout, stderr) = proc.communicate()
+  stdout, _ = proc.communicate()
-  return (stdout, proc.returncode)
+  return stdout.decode('UTF-8'), proc.returncode
-
-
-def _GetOutputNoError(args):
-  """Similar to _GetOutput() but ignores stderr. If there's an error launching
-  the child (like file not found), the exception will be caught and (None, 1)
-  will be returned to mimic quiet failure."""
-  try:
-    proc = subprocess.Popen(args,
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE)
-  except OSError:
-    return (None, 1)
-  (stdout, stderr) = proc.communicate()
-  return (stdout, proc.returncode)
 
 
 def _RemoveKeys(plist, *keys):
@@ -194,9 +180,9 @@ def _TagSuffixes():
   components_len = len(components)
   combinations = 1 << components_len
   tag_suffixes = []
-  for combination in xrange(0, combinations):
+  for combination in range(0, combinations):
     tag_suffix = ''
-    for component_index in xrange(0, components_len):
+    for component_index in range(0, components_len):
       if combination & (1 << component_index):
         tag_suffix += '-' + components[component_index]
     tag_suffixes.append(tag_suffix)

config/ios/codesign.py

@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+from __future__ import print_function
+
 import argparse
 import codecs
 import datetime
@@ -15,6 +17,11 @@ import subprocess
 import sys
 import tempfile
 
+if sys.version_info.major < 3:
+  basestring_compat = basestring
+else:
+  basestring_compat = str
+
 
 def GetProvisioningProfilesDir():
   """Returns the location of the installed mobile provisioning profiles.
@@ -27,6 +34,21 @@ def GetProvisioningProfilesDir():
       os.environ['HOME'], 'Library', 'MobileDevice', 'Provisioning Profiles')
 
 
+def ReadPlistFromString(plist_bytes):
+  """Parse property list from given |plist_bytes|.
+
+    Args:
+      plist_bytes: contents of property list to load. Must be bytes in python 3.
+
+    Returns:
+      The contents of property list as a python object.
+    """
+  if sys.version_info.major == 2:
+    return plistlib.readPlistFromString(plist_bytes)
+  else:
+    return plistlib.loads(plist_bytes)
+
+
 def LoadPlistFile(plist_path):
   """Loads property list file at |plist_path|.
 
@@ -36,8 +58,9 @@ def LoadPlistFile(plist_path):
   Returns:
     The content of the property list file as a python object.
   """
-  return plistlib.readPlistFromString(subprocess.check_output([
+  return ReadPlistFromString(
-      'xcrun', 'plutil', '-convert', 'xml1', '-o', '-', plist_path]))
+      subprocess.check_output(
+          ['xcrun', 'plutil', '-convert', 'xml1', '-o', '-', plist_path]))
 
 
 class Bundle(object):
@@ -74,7 +97,7 @@ class Bundle(object):
       error message. The dictionary will be empty if there are no errors.
     """
     errors = {}
-    for key, expected_value in expected_mappings.iteritems():
+    for key, expected_value in expected_mappings.items():
       if key in self._data:
         value = self._data[key]
         if value != expected_value:
@@ -88,9 +111,11 @@ class ProvisioningProfile(object):
   def __init__(self, provisioning_profile_path):
     """Initializes the ProvisioningProfile with data from profile file."""
     self._path = provisioning_profile_path
-    self._data = plistlib.readPlistFromString(subprocess.check_output([
+    self._data = ReadPlistFromString(
-        'xcrun', 'security', 'cms', '-D', '-u', 'certUsageAnyCA',
+        subprocess.check_output([
-        '-i', provisioning_profile_path]))
+            'xcrun', 'security', 'cms', '-D', '-u', 'certUsageAnyCA', '-i',
+            provisioning_profile_path
+        ]))
 
   @property
   def path(self):
@@ -155,13 +180,13 @@ class Entitlements(object):
     self._data = self._ExpandVariables(self._data, substitutions)
 
   def _ExpandVariables(self, data, substitutions):
-    if isinstance(data, str):
+    if isinstance(data, basestring_compat):
-      for key, substitution in substitutions.iteritems():
+      for key, substitution in substitutions.items():
         data = data.replace('$(%s)' % (key,), substitution)
       return data
 
     if isinstance(data, dict):
-      for key, value in data.iteritems():
+      for key, value in data.items():
         data[key] = self._ExpandVariables(value, substitutions)
       return data
 
@@ -172,7 +197,7 @@ class Entitlements(object):
     return data
 
   def LoadDefaults(self, defaults):
-    for key, value in defaults.iteritems():
+    for key, value in defaults.items():
       if key not in self._data:
         self._data[key] = value
 

config/ios/compile_xcassets.py

@@ -32,16 +32,18 @@ NOTICE_SECTION = 'com.apple.actool.compilation-results'
 # Regular expressions matching spurious messages from actool that should be
 # ignored (as they are bogus). Generally a bug should be filed with Apple
 # when adding a pattern here.
-SPURIOUS_PATTERNS = map(re.compile, [
+SPURIOUS_PATTERNS = [
-    # crbug.com/770634, likely a bug in Xcode 9.1 beta, remove once build
+    re.compile(v) for v in [
-    # requires a version of Xcode with a fix.
+        # crbug.com/770634, likely a bug in Xcode 9.1 beta, remove once build
-    r'\[\]\[ipad\]\[76x76\]\[\]\[\]\[1x\]\[\]\[\]: notice: \(null\)',
+        # requires a version of Xcode with a fix.
+        r'\[\]\[ipad\]\[76x76\]\[\]\[\]\[1x\]\[\]\[\]: notice: \(null\)',
 
-    # crbug.com/770634, likely a bug in Xcode 9.2 beta, remove once build
+        # crbug.com/770634, likely a bug in Xcode 9.2 beta, remove once build
-    # requires a version of Xcode with a fix.
+        # requires a version of Xcode with a fix.
-    r'\[\]\[ipad\]\[76x76\]\[\]\[\]\[1x\]\[\]\[\]: notice: 76x76@1x app icons'
+        r'\[\]\[ipad\]\[76x76\]\[\]\[\]\[1x\]\[\]\[\]: notice: 76x76@1x app icons'
-    ' only apply to iPad apps targeting releases of iOS prior to 10.0.',
+        ' only apply to iPad apps targeting releases of iOS prior to 10.0.',
-])
+    ]
+]
 
 # Map special type of asset catalog to the corresponding command-line
 # parameter that need to be passed to actool.
@@ -193,7 +195,7 @@ def CompileAssetCatalog(output, platform, product_type, min_deployment_target,
     stdout, _ = process.communicate()
 
     # Filter the output to remove all garbarge and to fix the paths.
-    stdout = FilterCompilerOutput(stdout, relative_paths)
+    stdout = FilterCompilerOutput(stdout.decode('UTF-8'), relative_paths)
 
     if process.returncode or stdout:
       sys.stderr.write(stdout)

config/ios/strip_arm64e.py

@@ -18,9 +18,9 @@ def check_output(command):
   if process.returncode:
     sys.stderr.write('error: command failed with retcode %d: %s\n\n' %
                      (process.returncode, ' '.join(map(repr, command))))
-    sys.stderr.write(errs)
+    sys.stderr.write(errs.decode('UTF-8', errors='ignore'))
     sys.exit(process.returncode)
-  return outs
+  return outs.decode('UTF-8')
 
 
 def check_call(command):

config/ios/write_framework_hmap.py

@@ -50,7 +50,7 @@ def WriteHmap(output_name, filelist):
   count = len(filelist)
   capacity = NextGreaterPowerOf2(count)
   strings_offset = 24 + (12 * capacity)
-  max_value_length = len(max(filelist.items(), key=lambda (k,v):len(v))[1])
+  max_value_length = len(max(filelist.values(), key=lambda v: len(v)))
 
   out = open(output_name, 'wb')
   out.write(struct.pack('<LHHLLLL', magic, version, _reserved, strings_offset,
@@ -86,14 +86,17 @@ def WriteHmap(output_name, filelist):
   for bucket in buckets:
     if bucket is not None:
       (file, path) = bucket
-      out.write(struct.pack('<%ds' % len(file), file))
-      out.write(struct.pack('<s', '\0'))
       base = os.path.dirname(path) + os.sep
-      out.write(struct.pack('<%ds' % len(base), base))
-      out.write(struct.pack('<s', '\0'))
       path = os.path.basename(path)
+      file = file.encode('UTF-8')
+      base = base.encode('UTF-8')
+      path = path.encode('UTF-8')
+      out.write(struct.pack('<%ds' % len(file), file))
+      out.write(struct.pack('<s', b'\0'))
+      out.write(struct.pack('<%ds' % len(base), base))
+      out.write(struct.pack('<s', b'\0'))
       out.write(struct.pack('<%ds' % len(path), path))
-      out.write(struct.pack('<s', '\0'))
+      out.write(struct.pack('<s', b'\0'))
 
 
 if __name__ == '__main__':

config/mac/package_framework.py

@@ -23,7 +23,7 @@ def Main():
   # Foo.framework/Versions/Current symlink to it.
   if args.version:
     try:
-      os.makedirs(os.path.join(args.framework, VERSIONS, args.version), 0755)
+      os.makedirs(os.path.join(args.framework, VERSIONS, args.version), 0o755)
     except OSError as e:
       if e.errno != errno.EEXIST:
         raise e

config/mac/plist_util.py

@@ -12,6 +12,10 @@ import sys
 import tempfile
 import shlex
 
+if sys.version_info.major < 3:
+  basestring_compat = basestring
+else:
+  basestring_compat = str
 
 # Xcode substitutes variables like ${PRODUCT_NAME} or $(PRODUCT_NAME) when
 # compiling Info.plist. It also supports supports modifiers like :identifier
@@ -80,10 +84,10 @@ def Interpolate(value, substitutions):
     substitution.
   """
   if isinstance(value, dict):
-    return {k: Interpolate(v, substitutions) for k, v in value.iteritems()}
+    return {k: Interpolate(v, substitutions) for k, v in value.items()}
   if isinstance(value, list):
     return [Interpolate(v, substitutions) for v in value]
-  if isinstance(value, str):
+  if isinstance(value, basestring_compat):
     return InterpolateString(value, substitutions)
   return value
 
@@ -93,7 +97,7 @@ def LoadPList(path):
   fd, name = tempfile.mkstemp()
   try:
     subprocess.check_call(['plutil', '-convert', 'xml1', '-o', name, path])
-    with os.fdopen(fd, 'r') as f:
+    with os.fdopen(fd, 'rb') as f:
       return plistlib.readPlist(f)
   finally:
     os.unlink(name)
@@ -109,7 +113,7 @@ def SavePList(path, format, data):
     # it does exist.
     if os.path.exists(path):
       os.unlink(path)
-    with os.fdopen(fd, 'w') as f:
+    with os.fdopen(fd, 'wb') as f:
       plistlib.writePlist(data, f)
     subprocess.check_call(['plutil', '-convert', format, '-o', path, name])
   finally:
@@ -134,7 +138,7 @@ def MergePList(plist1, plist2):
     are concatenated.
   """
   result = plist1.copy()
-  for key, value in plist2.iteritems():
+  for key, value in plist2.items():
     if isinstance(value, dict):
       old_value = result.get(key)
       if isinstance(old_value, dict):

config/mac/sdk_info.py

@@ -13,6 +13,11 @@ import re
 import subprocess
 import sys
 
+if sys.version_info.major < 3:
+  basestring_compat = basestring
+else:
+  basestring_compat = str
+
 # src directory
 ROOT_SRC_DIR = os.path.dirname(
     os.path.dirname(
@@ -61,7 +66,8 @@ def FillXcodeVersion(settings, developer_dir):
     settings['xcode_build'] = version_plist['ProductBuildVersion']
     return
 
-  lines = subprocess.check_output(['xcodebuild', '-version']).splitlines()
+  lines = subprocess.check_output(['xcodebuild',
+                                   '-version']).decode('UTF-8').splitlines()
   settings['xcode_version'] = FormatVersion(lines[0].split()[-1])
   settings['xcode_version_int'] = int(settings['xcode_version'], 10)
   settings['xcode_build'] = lines[-1].split()[-1]
@@ -69,8 +75,8 @@ def FillXcodeVersion(settings, developer_dir):
 
 def FillMachineOSBuild(settings):
   """Fills OS build number into |settings|."""
-  machine_os_build = subprocess.check_output(['sw_vers', '-buildVersion'],
+  machine_os_build = subprocess.check_output(['sw_vers', '-buildVersion'
-                                             universal_newlines=True).strip()
+                                              ]).decode('UTF-8').strip()
   settings['machine_os_build'] = machine_os_build
 
   # The reported build number is made up from the kernel major version number,
@@ -91,14 +97,17 @@ def FillMachineOSBuild(settings):
 
 def FillSDKPathAndVersion(settings, platform, xcode_version):
   """Fills the SDK path and version for |platform| into |settings|."""
-  settings['sdk_path'] = subprocess.check_output([
+  settings['sdk_path'] = subprocess.check_output(
-      'xcrun', '-sdk', platform, '--show-sdk-path']).strip()
+      ['xcrun', '-sdk', platform, '--show-sdk-path']).decode('UTF-8').strip()
-  settings['sdk_version'] = subprocess.check_output([
+  settings['sdk_version'] = subprocess.check_output(
-      'xcrun', '-sdk', platform, '--show-sdk-version']).strip()
+      ['xcrun', '-sdk', platform,
-  settings['sdk_platform_path'] = subprocess.check_output([
+       '--show-sdk-version']).decode('UTF-8').strip()
-      'xcrun', '-sdk', platform, '--show-sdk-platform-path']).strip()
+  settings['sdk_platform_path'] = subprocess.check_output(
+      ['xcrun', '-sdk', platform,
+       '--show-sdk-platform-path']).decode('UTF-8').strip()
   settings['sdk_build'] = subprocess.check_output(
-      ['xcrun', '-sdk', platform, '--show-sdk-build-version']).strip()
+      ['xcrun', '-sdk', platform,
+       '--show-sdk-build-version']).decode('UTF-8').strip()
 
 
 def CreateXcodeSymlinkAt(src, dst):
@@ -157,6 +166,6 @@ if __name__ == '__main__':
     value = settings[key]
     if args.create_symlink_at and '_path' in key:
       value = CreateXcodeSymlinkAt(value, args.create_symlink_at)
-    if isinstance(value, str):
+    if isinstance(value, basestring_compat):
       value = '"%s"' % value
     print('%s=%s' % (key, value))

mac/find_sdk.py

@@ -65,7 +65,7 @@ def main():
     print(out, file=sys.stderr)
     print(err, file=sys.stderr)
     raise Exception('Error %d running xcode-select' % job.returncode)
-  dev_dir = out.rstrip()
+  dev_dir = out.decode('UTF-8').rstrip()
   sdk_dir = os.path.join(
       dev_dir, 'Platforms/MacOSX.platform/Developer/SDKs')
 

toolchain/mac/filter_libtool.py

@@ -12,14 +12,16 @@ import sys
 # This script executes libool and filters out logspam lines like:
 #    '/path/to/libtool: file: foo.o has no symbols'
 
-BLACKLIST_PATTERNS = map(re.compile, [
+BLACKLIST_PATTERNS = [
-    r'^.*libtool: (?:for architecture: \S* )?file: .* has no symbols$',
+    re.compile(v) for v in [
-    r'^.*libtool: warning for library: .* the table of contents is empty '
+        r'^.*libtool: (?:for architecture: \S* )?file: .* has no symbols$',
+        r'^.*libtool: warning for library: .* the table of contents is empty '
         r'\(no object file members in the library define global symbols\)$',
-    r'^.*libtool: warning same member name \(\S*\) in output file used for '
+        r'^.*libtool: warning same member name \(\S*\) in output file used for '
         r'input files: \S* and: \S* \(due to use of basename, truncation, '
         r'blank padding or duplicate input files\)$',
-])
+    ]
+]
 
 
 def IsBlacklistedLine(line):
@@ -34,7 +36,7 @@ def Main(cmd_list):
   env = os.environ.copy()
   libtoolout = subprocess.Popen(cmd_list, stderr=subprocess.PIPE, env=env)
   _, err = libtoolout.communicate()
-  for line in err.splitlines():
+  for line in err.decode('UTF-8').splitlines():
     if not IsBlacklistedLine(line):
       print(line, file=sys.stderr)
   return libtoolout.returncode