From b23acd28e8d7edb79dafdb3e261e171eeadd944e Mon Sep 17 00:00:00 2001 From: Ciure Andrei Date: Wed, 1 Jan 2020 13:00:37 +0200 Subject: [PATCH] Backed out changeset fe3a47b458ad (bug 1605264) for causing build bustages on central a=backout --- build/unix/rewrite_asan_dylib.py | 7 ++---- python/mozbuild/mozpack/files.py | 24 ++++++------------ python/mozbuild/mozpack/packager/unpack.py | 2 +- toolkit/mozapps/installer/find-dupes.py | 14 +++++------ toolkit/mozapps/installer/informulate.py | 6 ++--- toolkit/mozapps/installer/packager.mk | 16 ++++++------ toolkit/mozapps/installer/packager.py | 29 +++++++++++----------- 7 files changed, 42 insertions(+), 56 deletions(-) diff --git a/build/unix/rewrite_asan_dylib.py b/build/unix/rewrite_asan_dylib.py index ada15fa690b2..ee76936ccdca 100644 --- a/build/unix/rewrite_asan_dylib.py +++ b/build/unix/rewrite_asan_dylib.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals - import re import sys import os @@ -22,8 +20,7 @@ DYLIB_NAME = 'libclang_rt.asan_osx_dynamic.dylib' def resolve_rpath(filename): - otoolOut = subprocess.check_output([substs['OTOOL'], '-l', filename], - universal_newlines=True) + otoolOut = subprocess.check_output([substs['OTOOL'], '-l', filename]) currentCmd = None # The lines we need to find look like this: @@ -65,7 +62,7 @@ def scan_directory(path): try: otoolOut = subprocess.check_output( - [substs['OTOOL'], '-L', filename], universal_newlines=True) + [substs['OTOOL'], '-L', filename]) except Exception: # Errors are expected on non-mach executables, ignore them and continue continue diff --git a/python/mozbuild/mozpack/files.py b/python/mozbuild/mozpack/files.py index cc0866653f8e..632ba5090659 100644 --- a/python/mozbuild/mozpack/files.py +++ b/python/mozbuild/mozpack/files.py @@ -75,14 +75,6 @@ else: raise TypeError('mismatched path types!') -# Helper function; ensures we always open files with the correct encoding when -# opening them in text mode. -def _open(path, mode='r'): - if six.PY3 and 'b' not in mode: - return open(path, mode, encoding='utf-8') - return open(path, mode) - - class Dest(object): ''' Helper interface for BaseFile.copy. The interface works as follows: @@ -104,13 +96,13 @@ class Dest(object): def read(self, length=-1, mode='rb'): if self.mode != 'r': - self.file = _open(self.path, mode) + self.file = open(self.path, mode) self.mode = 'r' return self.file.read(length) def write(self, data, mode='wb'): if self.mode != 'w': - self.file = _open(self.path, mode) + self.file = open(self.path, mode) self.mode = 'w' if 'b' in mode: to_write = six.ensure_binary(data) @@ -253,7 +245,7 @@ class BaseFile(object): a custom file-like object. ''' assert self.path is not None - return _open(self.path, mode=mode) + return open(self.path, mode=mode) def read(self): raise NotImplementedError('BaseFile.read() not implemented. Bug 1170329.') @@ -301,7 +293,7 @@ class File(BaseFile): def read(self): '''Return the contents of the file.''' - with _open(self.path, 'rb') as fh: + with open(self.path, 'rb') as fh: return fh.read() def size(self): @@ -555,8 +547,8 @@ class PreprocessedFile(BaseFile): pp = Preprocessor(defines=self.defines, marker=self.marker) pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings) - with _open(self.path, 'rU') as input: - with _open(os.devnull, 'w') as output: + with open(self.path, 'rU') as input: + with open(os.devnull, 'w') as output: pp.processFile(input=input, output=output) # This always yields at least self.path. @@ -587,7 +579,7 @@ class PreprocessedFile(BaseFile): # dependencies from that file to our list. if self.depfile and os.path.exists(self.depfile): target = mozpath.normpath(dest.name) - with _open(self.depfile, 'rt') as fileobj: + with open(self.depfile, 'rt') as fileobj: for rule in makeutil.read_dep_makefile(fileobj): if target in rule.targets(): pp_deps.update(rule.dependencies()) @@ -611,7 +603,7 @@ class PreprocessedFile(BaseFile): pp = Preprocessor(defines=self.defines, marker=self.marker) pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings) - with _open(self.path, 'rU') as input: + with open(self.path, 'rU') as input: pp.processFile(input=input, output=dest, depfile=deps_out) dest.close() diff --git a/python/mozbuild/mozpack/packager/unpack.py b/python/mozbuild/mozpack/packager/unpack.py index 6f8c1b628c11..0ad36c6f3708 100644 --- a/python/mozbuild/mozpack/packager/unpack.py +++ b/python/mozbuild/mozpack/packager/unpack.py @@ -145,7 +145,7 @@ class UnpackFinder(BaseFinder): jar = JarReader(fileobj=file.open()) self.compressed = max(self.compressed, jar.compression) if jar.last_preloaded: - jarlog = list(jar.entries.keys()) + jarlog = jar.entries.keys() self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1] return jar diff --git a/toolkit/mozapps/installer/find-dupes.py b/toolkit/mozapps/installer/find-dupes.py index 6beb25040f70..0de76cb2697e 100644 --- a/toolkit/mozapps/installer/find-dupes.py +++ b/toolkit/mozapps/installer/find-dupes.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import print_function import sys import hashlib @@ -14,8 +14,7 @@ from mozbuild.util import DefinesAction from mozpack.packager.unpack import UnpackFinder from mozpack.files import DeflatedFile from collections import OrderedDict -import six -from six import StringIO +from StringIO import StringIO import argparse import buildconfig @@ -53,10 +52,9 @@ def find_dupes(source, allowed_dupes, bail=True): for p, f in UnpackFinder(source): md5 = hashlib.md5() content_size = 0 - for buf in iter(functools.partial(f.open('rb').read, md5_chunk_size), - b''): - md5.update(six.ensure_binary(buf)) - content_size += len(six.ensure_binary(buf)) + for buf in iter(functools.partial(f.open().read, md5_chunk_size), b''): + md5.update(buf) + content_size += len(buf) m = md5.digest() if m not in md5s: if isinstance(f, DeflatedFile): @@ -69,7 +67,7 @@ def find_dupes(source, allowed_dupes, bail=True): total_compressed = 0 num_dupes = 0 unexpected_dupes = [] - for m, (size, compressed, paths) in sorted(six.iteritems(md5s), + for m, (size, compressed, paths) in sorted(md5s.iteritems(), key=lambda x: x[1][1]): if len(paths) > 1: _compressed = ' (%d compressed)' % compressed if compressed != size else '' diff --git a/toolkit/mozapps/installer/informulate.py b/toolkit/mozapps/installer/informulate.py index 1c8ddb0fdf95..3203a97e0698 100644 --- a/toolkit/mozapps/installer/informulate.py +++ b/toolkit/mozapps/installer/informulate.py @@ -47,11 +47,11 @@ def main(): 'moz_pkg_platform': args.pkg_platform, }) - with open(args.output_json, 'wt') as f: + with open(args.output_json, 'wb') as f: json.dump(all_key_value_pairs, f, indent=2, sort_keys=True) f.write('\n') - with open(args.buildhub_json, 'wt') as f: + with open(args.buildhub_json, 'wb') as f: build_time = datetime.datetime.strptime(build_id, '%Y%m%d%H%M%S') s = buildconfig.substs record = { @@ -100,7 +100,7 @@ def main(): json.dump(record, f, indent=2, sort_keys=True) f.write('\n') - with open(args.output_txt, 'wt') as f: + with open(args.output_txt, 'wb') as f: f.write('buildID={}\n'.format(build_id)) diff --git a/toolkit/mozapps/installer/packager.mk b/toolkit/mozapps/installer/packager.mk index ef72740e8cad..b02b6c4e51d7 100644 --- a/toolkit/mozapps/installer/packager.mk +++ b/toolkit/mozapps/installer/packager.mk @@ -23,7 +23,7 @@ export USE_ELF_HACK ELF_HACK_FLAGS stage-package: multilocale.txt locale-manifest.in $(MOZ_PKG_MANIFEST) $(MOZ_PKG_MANIFEST_DEPS) NO_PKG_FILES="$(NO_PKG_FILES)" \ - $(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/packager.py $(DEFINES) $(ACDEFINES) \ + $(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/packager.py $(DEFINES) $(ACDEFINES) \ --format $(MOZ_PACKAGER_FORMAT) \ $(addprefix --removals ,$(MOZ_PKG_REMOVALS)) \ $(if $(filter-out 0,$(MOZ_PKG_FATAL_WARNINGS)),,--ignore-errors) \ @@ -37,7 +37,7 @@ stage-package: multilocale.txt locale-manifest.in $(MOZ_PKG_MANIFEST) $(MOZ_PKG_ $(MOZ_PKG_MANIFEST) '$(DIST)' '$(DIST)'/$(MOZ_PKG_DIR)$(if $(MOZ_PKG_MANIFEST),,$(_BINPATH)) \ $(if $(filter omni,$(MOZ_PACKAGER_FORMAT)),$(if $(NON_OMNIJAR_FILES),--non-resource $(NON_OMNIJAR_FILES))) ifdef RUN_FIND_DUPES - $(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/find-dupes.py $(DEFINES) $(ACDEFINES) $(MOZ_PKG_DUPEFLAGS) $(DIST)/$(MOZ_PKG_DIR) + $(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/find-dupes.py $(DEFINES) $(ACDEFINES) $(MOZ_PKG_DUPEFLAGS) $(DIST)/$(MOZ_PKG_DIR) endif # RUN_FIND_DUPES ifndef MOZ_IS_COMM_TOPDIR ifdef RUN_MOZHARNESS_ZIP @@ -71,7 +71,7 @@ ifdef MOZ_CODE_COVERAGE $(topsrcdir)/mach build-backend -b ChromeMap @echo 'Packaging code coverage data...' $(RM) $(CODE_COVERAGE_ARCHIVE_BASENAME).zip - $(PYTHON3) -mmozbuild.codecoverage.packager \ + $(PYTHON) -mmozbuild.codecoverage.packager \ --output-file='$(DIST)/$(PKG_PATH)$(CODE_COVERAGE_ARCHIVE_BASENAME).zip' endif ifdef ENABLE_MOZSEARCH_PLUGIN @@ -94,7 +94,7 @@ endif ifeq (Darwin, $(OS_ARCH)) ifdef MOZ_ASAN @echo "Rewriting ASan runtime dylib paths for all binaries in $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH) ..." - $(PYTHON3) $(MOZILLA_DIR)/build/unix/rewrite_asan_dylib.py $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH) + $(PYTHON) $(MOZILLA_DIR)/build/unix/rewrite_asan_dylib.py $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH) endif # MOZ_ASAN endif # Darwin @@ -113,7 +113,7 @@ endif endif ifdef MOZ_AUTOMATION cp $(DEPTH)/mozinfo.json $(MOZ_MOZINFO_FILE) - $(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/informulate.py \ + $(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/informulate.py \ $(MOZ_BUILDINFO_FILE) $(MOZ_BUILDHUB_JSON) $(MOZ_BUILDID_INFO_TXT_FILE) \ $(MOZ_PKG_PLATFORM) \ $(if $(or $(filter-out mobile/android,$(MOZ_BUILD_APP)),$(MOZ_ANDROID_WITH_FENNEC)), \ @@ -148,16 +148,16 @@ endif ln -s $(installdir)/$(MOZ_APP_NAME) $(DESTDIR)$(bindir) upload: - $(PYTHON3) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(UPLOAD_FILES) + $(PYTHON) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(UPLOAD_FILES) mkdir -p `dirname $(CHECKSUM_FILE)` - @$(PYTHON3) $(MOZILLA_DIR)/build/checksums.py \ + @$(PYTHON) $(MOZILLA_DIR)/build/checksums.py \ -o $(CHECKSUM_FILE) \ $(CHECKSUM_ALGORITHM_PARAM) \ $(UPLOAD_PATH) @echo 'CHECKSUM FILE START' @cat $(CHECKSUM_FILE) @echo 'CHECKSUM FILE END' - $(PYTHON3) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(CHECKSUM_FILES) + $(PYTHON) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(CHECKSUM_FILES) # source-package creates a source tarball from the files in MOZ_PKG_SRCDIR, # which is either set to a clean checkout or defaults to $topsrcdir diff --git a/toolkit/mozapps/installer/packager.py b/toolkit/mozapps/installer/packager.py index a82f4f705744..0ab32173355b 100644 --- a/toolkit/mozapps/installer/packager.py +++ b/toolkit/mozapps/installer/packager.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals, print_function - from mozpack.packager.formats import ( FlatFormatter, JarFormatter, @@ -32,8 +30,7 @@ import buildconfig from argparse import ArgumentParser from createprecomplete import generate_precomplete import os -import six -from six import StringIO +from StringIO import StringIO import subprocess import mozinfo @@ -85,7 +82,13 @@ class ToolLauncher(object): for e in extra_env: env[e] = extra_env[e] - print('Executing %s' % ' '.join(cmd), file=errors.out) + # Work around a bug in Python 2.7.2 and lower where unicode types in + # environment variables aren't handled by subprocess. + for k, v in env.items(): + if isinstance(v, unicode): + env[k] = v.encode('utf-8') + + print >>errors.out, 'Executing', ' '.join(cmd) errors.out.flush() return subprocess.call(cmd, env=env) @@ -100,7 +103,7 @@ class LibSignFile(File): File class for shlibsign signatures. ''' def copy(self, dest, skip_if_older=True): - assert isinstance(dest, six.string_types) + assert isinstance(dest, basestring) # os.path.getmtime returns a result in seconds with precision up to the # microsecond. But microsecond is too precise because shutil.copystat # only copies milliseconds, and seconds is not enough precision. @@ -118,16 +121,15 @@ class RemovedFiles(GeneratedFile): ''' def __init__(self, copier): self.copier = copier - GeneratedFile.__init__(self, b'') + GeneratedFile.__init__(self, '') - def handle_line(self, f): - f = f.strip() + def handle_line(self, str): + f = str.strip() if not f: return if self.copier.contains(f): errors.error('Removal of packaged file(s): %s' % f) - ensure = six.ensure_binary if 'b' in self._mode else six.ensure_text - self.content += ensure(f) + ensure('\n') + self.content += f + '\n' def split_define(define): @@ -301,10 +303,7 @@ def main(): # If a pdb file is present and we were instructed to copy it, include it. # Run on all OSes to capture MinGW builds if buildconfig.substs.get('MOZ_COPY_PDBS'): - # We want to mutate the copier while we're iterating through it, so copy - # the items to a list first. - copier_items = [(p, f) for p, f in copier] - for p, f in copier_items: + for p, f in copier: if isinstance(f, ExecutableFile): pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb' if os.path.exists(pdbname):