diff --git a/python/mozbuild/mozpack/files.py b/python/mozbuild/mozpack/files.py index 632ba5090659..cc0866653f8e 100644 --- a/python/mozbuild/mozpack/files.py +++ b/python/mozbuild/mozpack/files.py @@ -75,6 +75,14 @@ else: raise TypeError('mismatched path types!') +# Helper function; ensures we always open files with the correct encoding when +# opening them in text mode. +def _open(path, mode='r'): + if six.PY3 and 'b' not in mode: + return open(path, mode, encoding='utf-8') + return open(path, mode) + + class Dest(object): ''' Helper interface for BaseFile.copy. The interface works as follows: @@ -96,13 +104,13 @@ class Dest(object): def read(self, length=-1, mode='rb'): if self.mode != 'r': - self.file = open(self.path, mode) + self.file = _open(self.path, mode) self.mode = 'r' return self.file.read(length) def write(self, data, mode='wb'): if self.mode != 'w': - self.file = open(self.path, mode) + self.file = _open(self.path, mode) self.mode = 'w' if 'b' in mode: to_write = six.ensure_binary(data) @@ -245,7 +253,7 @@ class BaseFile(object): a custom file-like object. ''' assert self.path is not None - return open(self.path, mode=mode) + return _open(self.path, mode=mode) def read(self): raise NotImplementedError('BaseFile.read() not implemented. Bug 1170329.') @@ -293,7 +301,7 @@ class File(BaseFile): def read(self): '''Return the contents of the file.''' - with open(self.path, 'rb') as fh: + with _open(self.path, 'rb') as fh: return fh.read() def size(self): @@ -547,8 +555,8 @@ class PreprocessedFile(BaseFile): pp = Preprocessor(defines=self.defines, marker=self.marker) pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings) - with open(self.path, 'rU') as input: - with open(os.devnull, 'w') as output: + with _open(self.path, 'rU') as input: + with _open(os.devnull, 'w') as output: pp.processFile(input=input, output=output) # This always yields at least self.path. @@ -579,7 +587,7 @@ class PreprocessedFile(BaseFile): # dependencies from that file to our list. if self.depfile and os.path.exists(self.depfile): target = mozpath.normpath(dest.name) - with open(self.depfile, 'rt') as fileobj: + with _open(self.depfile, 'rt') as fileobj: for rule in makeutil.read_dep_makefile(fileobj): if target in rule.targets(): pp_deps.update(rule.dependencies()) @@ -603,7 +611,7 @@ class PreprocessedFile(BaseFile): pp = Preprocessor(defines=self.defines, marker=self.marker) pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings) - with open(self.path, 'rU') as input: + with _open(self.path, 'rU') as input: pp.processFile(input=input, output=dest, depfile=deps_out) dest.close() diff --git a/python/mozbuild/mozpack/packager/unpack.py b/python/mozbuild/mozpack/packager/unpack.py index 0ad36c6f3708..6f8c1b628c11 100644 --- a/python/mozbuild/mozpack/packager/unpack.py +++ b/python/mozbuild/mozpack/packager/unpack.py @@ -145,7 +145,7 @@ class UnpackFinder(BaseFinder): jar = JarReader(fileobj=file.open()) self.compressed = max(self.compressed, jar.compression) if jar.last_preloaded: - jarlog = jar.entries.keys() + jarlog = list(jar.entries.keys()) self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1] return jar diff --git a/toolkit/mozapps/installer/find-dupes.py b/toolkit/mozapps/installer/find-dupes.py index 0de76cb2697e..6beb25040f70 100644 --- a/toolkit/mozapps/installer/find-dupes.py +++ b/toolkit/mozapps/installer/find-dupes.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import print_function +from __future__ import absolute_import, unicode_literals, print_function import sys import hashlib @@ -14,7 +14,8 @@ from mozbuild.util import DefinesAction from mozpack.packager.unpack import UnpackFinder from mozpack.files import DeflatedFile from collections import OrderedDict -from StringIO import StringIO +import six +from six import StringIO import argparse import buildconfig @@ -52,9 +53,10 @@ def find_dupes(source, allowed_dupes, bail=True): for p, f in UnpackFinder(source): md5 = hashlib.md5() content_size = 0 - for buf in iter(functools.partial(f.open().read, md5_chunk_size), b''): - md5.update(buf) - content_size += len(buf) + for buf in iter(functools.partial(f.open('rb').read, md5_chunk_size), + b''): + md5.update(six.ensure_binary(buf)) + content_size += len(six.ensure_binary(buf)) m = md5.digest() if m not in md5s: if isinstance(f, DeflatedFile): @@ -67,7 +69,7 @@ def find_dupes(source, allowed_dupes, bail=True): total_compressed = 0 num_dupes = 0 unexpected_dupes = [] - for m, (size, compressed, paths) in sorted(md5s.iteritems(), + for m, (size, compressed, paths) in sorted(six.iteritems(md5s), key=lambda x: x[1][1]): if len(paths) > 1: _compressed = ' (%d compressed)' % compressed if compressed != size else '' diff --git a/toolkit/mozapps/installer/informulate.py b/toolkit/mozapps/installer/informulate.py index 3203a97e0698..1c8ddb0fdf95 100644 --- a/toolkit/mozapps/installer/informulate.py +++ b/toolkit/mozapps/installer/informulate.py @@ -47,11 +47,11 @@ def main(): 'moz_pkg_platform': args.pkg_platform, }) - with open(args.output_json, 'wb') as f: + with open(args.output_json, 'wt') as f: json.dump(all_key_value_pairs, f, indent=2, sort_keys=True) f.write('\n') - with open(args.buildhub_json, 'wb') as f: + with open(args.buildhub_json, 'wt') as f: build_time = datetime.datetime.strptime(build_id, '%Y%m%d%H%M%S') s = buildconfig.substs record = { @@ -100,7 +100,7 @@ def main(): json.dump(record, f, indent=2, sort_keys=True) f.write('\n') - with open(args.output_txt, 'wb') as f: + with open(args.output_txt, 'wt') as f: f.write('buildID={}\n'.format(build_id)) diff --git a/toolkit/mozapps/installer/packager.mk b/toolkit/mozapps/installer/packager.mk index b02b6c4e51d7..ef72740e8cad 100644 --- a/toolkit/mozapps/installer/packager.mk +++ b/toolkit/mozapps/installer/packager.mk @@ -23,7 +23,7 @@ export USE_ELF_HACK ELF_HACK_FLAGS stage-package: multilocale.txt locale-manifest.in $(MOZ_PKG_MANIFEST) $(MOZ_PKG_MANIFEST_DEPS) NO_PKG_FILES="$(NO_PKG_FILES)" \ - $(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/packager.py $(DEFINES) $(ACDEFINES) \ + $(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/packager.py $(DEFINES) $(ACDEFINES) \ --format $(MOZ_PACKAGER_FORMAT) \ $(addprefix --removals ,$(MOZ_PKG_REMOVALS)) \ $(if $(filter-out 0,$(MOZ_PKG_FATAL_WARNINGS)),,--ignore-errors) \ @@ -37,7 +37,7 @@ stage-package: multilocale.txt locale-manifest.in $(MOZ_PKG_MANIFEST) $(MOZ_PKG_ $(MOZ_PKG_MANIFEST) '$(DIST)' '$(DIST)'/$(MOZ_PKG_DIR)$(if $(MOZ_PKG_MANIFEST),,$(_BINPATH)) \ $(if $(filter omni,$(MOZ_PACKAGER_FORMAT)),$(if $(NON_OMNIJAR_FILES),--non-resource $(NON_OMNIJAR_FILES))) ifdef RUN_FIND_DUPES - $(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/find-dupes.py $(DEFINES) $(ACDEFINES) $(MOZ_PKG_DUPEFLAGS) $(DIST)/$(MOZ_PKG_DIR) + $(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/find-dupes.py $(DEFINES) $(ACDEFINES) $(MOZ_PKG_DUPEFLAGS) $(DIST)/$(MOZ_PKG_DIR) endif # RUN_FIND_DUPES ifndef MOZ_IS_COMM_TOPDIR ifdef RUN_MOZHARNESS_ZIP @@ -71,7 +71,7 @@ ifdef MOZ_CODE_COVERAGE $(topsrcdir)/mach build-backend -b ChromeMap @echo 'Packaging code coverage data...' $(RM) $(CODE_COVERAGE_ARCHIVE_BASENAME).zip - $(PYTHON) -mmozbuild.codecoverage.packager \ + $(PYTHON3) -mmozbuild.codecoverage.packager \ --output-file='$(DIST)/$(PKG_PATH)$(CODE_COVERAGE_ARCHIVE_BASENAME).zip' endif ifdef ENABLE_MOZSEARCH_PLUGIN @@ -94,7 +94,7 @@ endif ifeq (Darwin, $(OS_ARCH)) ifdef MOZ_ASAN @echo "Rewriting ASan runtime dylib paths for all binaries in $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH) ..." - $(PYTHON) $(MOZILLA_DIR)/build/unix/rewrite_asan_dylib.py $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH) + $(PYTHON3) $(MOZILLA_DIR)/build/unix/rewrite_asan_dylib.py $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH) endif # MOZ_ASAN endif # Darwin @@ -113,7 +113,7 @@ endif endif ifdef MOZ_AUTOMATION cp $(DEPTH)/mozinfo.json $(MOZ_MOZINFO_FILE) - $(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/informulate.py \ + $(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/informulate.py \ $(MOZ_BUILDINFO_FILE) $(MOZ_BUILDHUB_JSON) $(MOZ_BUILDID_INFO_TXT_FILE) \ $(MOZ_PKG_PLATFORM) \ $(if $(or $(filter-out mobile/android,$(MOZ_BUILD_APP)),$(MOZ_ANDROID_WITH_FENNEC)), \ @@ -148,16 +148,16 @@ endif ln -s $(installdir)/$(MOZ_APP_NAME) $(DESTDIR)$(bindir) upload: - $(PYTHON) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(UPLOAD_FILES) + $(PYTHON3) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(UPLOAD_FILES) mkdir -p `dirname $(CHECKSUM_FILE)` - @$(PYTHON) $(MOZILLA_DIR)/build/checksums.py \ + @$(PYTHON3) $(MOZILLA_DIR)/build/checksums.py \ -o $(CHECKSUM_FILE) \ $(CHECKSUM_ALGORITHM_PARAM) \ $(UPLOAD_PATH) @echo 'CHECKSUM FILE START' @cat $(CHECKSUM_FILE) @echo 'CHECKSUM FILE END' - $(PYTHON) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(CHECKSUM_FILES) + $(PYTHON3) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(CHECKSUM_FILES) # source-package creates a source tarball from the files in MOZ_PKG_SRCDIR, # which is either set to a clean checkout or defaults to $topsrcdir diff --git a/toolkit/mozapps/installer/packager.py b/toolkit/mozapps/installer/packager.py index 0ab32173355b..a82f4f705744 100644 --- a/toolkit/mozapps/installer/packager.py +++ b/toolkit/mozapps/installer/packager.py @@ -2,6 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, unicode_literals, print_function + from mozpack.packager.formats import ( FlatFormatter, JarFormatter, @@ -30,7 +32,8 @@ import buildconfig from argparse import ArgumentParser from createprecomplete import generate_precomplete import os -from StringIO import StringIO +import six +from six import StringIO import subprocess import mozinfo @@ -82,13 +85,7 @@ class ToolLauncher(object): for e in extra_env: env[e] = extra_env[e] - # Work around a bug in Python 2.7.2 and lower where unicode types in - # environment variables aren't handled by subprocess. - for k, v in env.items(): - if isinstance(v, unicode): - env[k] = v.encode('utf-8') - - print >>errors.out, 'Executing', ' '.join(cmd) + print('Executing %s' % ' '.join(cmd), file=errors.out) errors.out.flush() return subprocess.call(cmd, env=env) @@ -103,7 +100,7 @@ class LibSignFile(File): File class for shlibsign signatures. ''' def copy(self, dest, skip_if_older=True): - assert isinstance(dest, basestring) + assert isinstance(dest, six.string_types) # os.path.getmtime returns a result in seconds with precision up to the # microsecond. But microsecond is too precise because shutil.copystat # only copies milliseconds, and seconds is not enough precision. @@ -121,15 +118,16 @@ class RemovedFiles(GeneratedFile): ''' def __init__(self, copier): self.copier = copier - GeneratedFile.__init__(self, '') + GeneratedFile.__init__(self, b'') - def handle_line(self, str): - f = str.strip() + def handle_line(self, f): + f = f.strip() if not f: return if self.copier.contains(f): errors.error('Removal of packaged file(s): %s' % f) - self.content += f + '\n' + ensure = six.ensure_binary if 'b' in self._mode else six.ensure_text + self.content += ensure(f) + ensure('\n') def split_define(define): @@ -303,7 +301,10 @@ def main(): # If a pdb file is present and we were instructed to copy it, include it. # Run on all OSes to capture MinGW builds if buildconfig.substs.get('MOZ_COPY_PDBS'): - for p, f in copier: + # We want to mutate the copier while we're iterating through it, so copy + # the items to a list first. + copier_items = [(p, f) for p, f in copier] + for p, f in copier_items: if isinstance(f, ExecutableFile): pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb' if os.path.exists(pdbname):