зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1605264 - "./mach package" should use Python 3 r=firefox-build-system-reviewers,mshal
Differential Revision: https://phabricator.services.mozilla.com/D57908 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Ricky Stewart
Родитель
adee218e3f
Коммит
024cc29999
|
|
@ -75,6 +75,14 @@ else:
|
|||
raise TypeError('mismatched path types!')
|
||||
|
||||
|
||||
# Helper function; ensures we always open files with the correct encoding when
|
||||
# opening them in text mode.
|
||||
def _open(path, mode='r'):
|
||||
if six.PY3 and 'b' not in mode:
|
||||
return open(path, mode, encoding='utf-8')
|
||||
return open(path, mode)
|
||||
|
||||
|
||||
class Dest(object):
|
||||
'''
|
||||
Helper interface for BaseFile.copy. The interface works as follows:
|
||||
|
|
@ -96,13 +104,13 @@ class Dest(object):
|
|||
|
||||
def read(self, length=-1, mode='rb'):
|
||||
if self.mode != 'r':
|
||||
self.file = open(self.path, mode)
|
||||
self.file = _open(self.path, mode)
|
||||
self.mode = 'r'
|
||||
return self.file.read(length)
|
||||
|
||||
def write(self, data, mode='wb'):
|
||||
if self.mode != 'w':
|
||||
self.file = open(self.path, mode)
|
||||
self.file = _open(self.path, mode)
|
||||
self.mode = 'w'
|
||||
if 'b' in mode:
|
||||
to_write = six.ensure_binary(data)
|
||||
|
|
@ -245,7 +253,7 @@ class BaseFile(object):
|
|||
a custom file-like object.
|
||||
'''
|
||||
assert self.path is not None
|
||||
return open(self.path, mode=mode)
|
||||
return _open(self.path, mode=mode)
|
||||
|
||||
def read(self):
|
||||
raise NotImplementedError('BaseFile.read() not implemented. Bug 1170329.')
|
||||
|
|
@ -293,7 +301,7 @@ class File(BaseFile):
|
|||
|
||||
def read(self):
|
||||
'''Return the contents of the file.'''
|
||||
with open(self.path, 'rb') as fh:
|
||||
with _open(self.path, 'rb') as fh:
|
||||
return fh.read()
|
||||
|
||||
def size(self):
|
||||
|
|
@ -547,8 +555,8 @@ class PreprocessedFile(BaseFile):
|
|||
pp = Preprocessor(defines=self.defines, marker=self.marker)
|
||||
pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings)
|
||||
|
||||
with open(self.path, 'rU') as input:
|
||||
with open(os.devnull, 'w') as output:
|
||||
with _open(self.path, 'rU') as input:
|
||||
with _open(os.devnull, 'w') as output:
|
||||
pp.processFile(input=input, output=output)
|
||||
|
||||
# This always yields at least self.path.
|
||||
|
|
@ -579,7 +587,7 @@ class PreprocessedFile(BaseFile):
|
|||
# dependencies from that file to our list.
|
||||
if self.depfile and os.path.exists(self.depfile):
|
||||
target = mozpath.normpath(dest.name)
|
||||
with open(self.depfile, 'rt') as fileobj:
|
||||
with _open(self.depfile, 'rt') as fileobj:
|
||||
for rule in makeutil.read_dep_makefile(fileobj):
|
||||
if target in rule.targets():
|
||||
pp_deps.update(rule.dependencies())
|
||||
|
|
@ -603,7 +611,7 @@ class PreprocessedFile(BaseFile):
|
|||
pp = Preprocessor(defines=self.defines, marker=self.marker)
|
||||
pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings)
|
||||
|
||||
with open(self.path, 'rU') as input:
|
||||
with _open(self.path, 'rU') as input:
|
||||
pp.processFile(input=input, output=dest, depfile=deps_out)
|
||||
|
||||
dest.close()
|
||||
|
|
|
|||
|
|
@ -145,7 +145,7 @@ class UnpackFinder(BaseFinder):
|
|||
jar = JarReader(fileobj=file.open())
|
||||
self.compressed = max(self.compressed, jar.compression)
|
||||
if jar.last_preloaded:
|
||||
jarlog = jar.entries.keys()
|
||||
jarlog = list(jar.entries.keys())
|
||||
self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1]
|
||||
return jar
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import, unicode_literals, print_function
|
||||
|
||||
import sys
|
||||
import hashlib
|
||||
|
|
@ -14,7 +14,8 @@ from mozbuild.util import DefinesAction
|
|||
from mozpack.packager.unpack import UnpackFinder
|
||||
from mozpack.files import DeflatedFile
|
||||
from collections import OrderedDict
|
||||
from StringIO import StringIO
|
||||
import six
|
||||
from six import StringIO
|
||||
import argparse
|
||||
import buildconfig
|
||||
|
||||
|
|
@ -52,9 +53,10 @@ def find_dupes(source, allowed_dupes, bail=True):
|
|||
for p, f in UnpackFinder(source):
|
||||
md5 = hashlib.md5()
|
||||
content_size = 0
|
||||
for buf in iter(functools.partial(f.open().read, md5_chunk_size), b''):
|
||||
md5.update(buf)
|
||||
content_size += len(buf)
|
||||
for buf in iter(functools.partial(f.open('rb').read, md5_chunk_size),
|
||||
b''):
|
||||
md5.update(six.ensure_binary(buf))
|
||||
content_size += len(six.ensure_binary(buf))
|
||||
m = md5.digest()
|
||||
if m not in md5s:
|
||||
if isinstance(f, DeflatedFile):
|
||||
|
|
@ -67,7 +69,7 @@ def find_dupes(source, allowed_dupes, bail=True):
|
|||
total_compressed = 0
|
||||
num_dupes = 0
|
||||
unexpected_dupes = []
|
||||
for m, (size, compressed, paths) in sorted(md5s.iteritems(),
|
||||
for m, (size, compressed, paths) in sorted(six.iteritems(md5s),
|
||||
key=lambda x: x[1][1]):
|
||||
if len(paths) > 1:
|
||||
_compressed = ' (%d compressed)' % compressed if compressed != size else ''
|
||||
|
|
|
|||
|
|
@ -47,11 +47,11 @@ def main():
|
|||
'moz_pkg_platform': args.pkg_platform,
|
||||
})
|
||||
|
||||
with open(args.output_json, 'wb') as f:
|
||||
with open(args.output_json, 'wt') as f:
|
||||
json.dump(all_key_value_pairs, f, indent=2, sort_keys=True)
|
||||
f.write('\n')
|
||||
|
||||
with open(args.buildhub_json, 'wb') as f:
|
||||
with open(args.buildhub_json, 'wt') as f:
|
||||
build_time = datetime.datetime.strptime(build_id, '%Y%m%d%H%M%S')
|
||||
s = buildconfig.substs
|
||||
record = {
|
||||
|
|
@ -100,7 +100,7 @@ def main():
|
|||
json.dump(record, f, indent=2, sort_keys=True)
|
||||
f.write('\n')
|
||||
|
||||
with open(args.output_txt, 'wb') as f:
|
||||
with open(args.output_txt, 'wt') as f:
|
||||
f.write('buildID={}\n'.format(build_id))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ export USE_ELF_HACK ELF_HACK_FLAGS
|
|||
|
||||
stage-package: multilocale.txt locale-manifest.in $(MOZ_PKG_MANIFEST) $(MOZ_PKG_MANIFEST_DEPS)
|
||||
NO_PKG_FILES="$(NO_PKG_FILES)" \
|
||||
$(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/packager.py $(DEFINES) $(ACDEFINES) \
|
||||
$(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/packager.py $(DEFINES) $(ACDEFINES) \
|
||||
--format $(MOZ_PACKAGER_FORMAT) \
|
||||
$(addprefix --removals ,$(MOZ_PKG_REMOVALS)) \
|
||||
$(if $(filter-out 0,$(MOZ_PKG_FATAL_WARNINGS)),,--ignore-errors) \
|
||||
|
|
@ -37,7 +37,7 @@ stage-package: multilocale.txt locale-manifest.in $(MOZ_PKG_MANIFEST) $(MOZ_PKG_
|
|||
$(MOZ_PKG_MANIFEST) '$(DIST)' '$(DIST)'/$(MOZ_PKG_DIR)$(if $(MOZ_PKG_MANIFEST),,$(_BINPATH)) \
|
||||
$(if $(filter omni,$(MOZ_PACKAGER_FORMAT)),$(if $(NON_OMNIJAR_FILES),--non-resource $(NON_OMNIJAR_FILES)))
|
||||
ifdef RUN_FIND_DUPES
|
||||
$(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/find-dupes.py $(DEFINES) $(ACDEFINES) $(MOZ_PKG_DUPEFLAGS) $(DIST)/$(MOZ_PKG_DIR)
|
||||
$(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/find-dupes.py $(DEFINES) $(ACDEFINES) $(MOZ_PKG_DUPEFLAGS) $(DIST)/$(MOZ_PKG_DIR)
|
||||
endif # RUN_FIND_DUPES
|
||||
ifndef MOZ_IS_COMM_TOPDIR
|
||||
ifdef RUN_MOZHARNESS_ZIP
|
||||
|
|
@ -71,7 +71,7 @@ ifdef MOZ_CODE_COVERAGE
|
|||
$(topsrcdir)/mach build-backend -b ChromeMap
|
||||
@echo 'Packaging code coverage data...'
|
||||
$(RM) $(CODE_COVERAGE_ARCHIVE_BASENAME).zip
|
||||
$(PYTHON) -mmozbuild.codecoverage.packager \
|
||||
$(PYTHON3) -mmozbuild.codecoverage.packager \
|
||||
--output-file='$(DIST)/$(PKG_PATH)$(CODE_COVERAGE_ARCHIVE_BASENAME).zip'
|
||||
endif
|
||||
ifdef ENABLE_MOZSEARCH_PLUGIN
|
||||
|
|
@ -94,7 +94,7 @@ endif
|
|||
ifeq (Darwin, $(OS_ARCH))
|
||||
ifdef MOZ_ASAN
|
||||
@echo "Rewriting ASan runtime dylib paths for all binaries in $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH) ..."
|
||||
$(PYTHON) $(MOZILLA_DIR)/build/unix/rewrite_asan_dylib.py $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH)
|
||||
$(PYTHON3) $(MOZILLA_DIR)/build/unix/rewrite_asan_dylib.py $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH)
|
||||
endif # MOZ_ASAN
|
||||
endif # Darwin
|
||||
|
||||
|
|
@ -113,7 +113,7 @@ endif
|
|||
endif
|
||||
ifdef MOZ_AUTOMATION
|
||||
cp $(DEPTH)/mozinfo.json $(MOZ_MOZINFO_FILE)
|
||||
$(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/informulate.py \
|
||||
$(PYTHON3) $(MOZILLA_DIR)/toolkit/mozapps/installer/informulate.py \
|
||||
$(MOZ_BUILDINFO_FILE) $(MOZ_BUILDHUB_JSON) $(MOZ_BUILDID_INFO_TXT_FILE) \
|
||||
$(MOZ_PKG_PLATFORM) \
|
||||
$(if $(or $(filter-out mobile/android,$(MOZ_BUILD_APP)),$(MOZ_ANDROID_WITH_FENNEC)), \
|
||||
|
|
@ -148,16 +148,16 @@ endif
|
|||
ln -s $(installdir)/$(MOZ_APP_NAME) $(DESTDIR)$(bindir)
|
||||
|
||||
upload:
|
||||
$(PYTHON) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(UPLOAD_FILES)
|
||||
$(PYTHON3) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(UPLOAD_FILES)
|
||||
mkdir -p `dirname $(CHECKSUM_FILE)`
|
||||
@$(PYTHON) $(MOZILLA_DIR)/build/checksums.py \
|
||||
@$(PYTHON3) $(MOZILLA_DIR)/build/checksums.py \
|
||||
-o $(CHECKSUM_FILE) \
|
||||
$(CHECKSUM_ALGORITHM_PARAM) \
|
||||
$(UPLOAD_PATH)
|
||||
@echo 'CHECKSUM FILE START'
|
||||
@cat $(CHECKSUM_FILE)
|
||||
@echo 'CHECKSUM FILE END'
|
||||
$(PYTHON) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(CHECKSUM_FILES)
|
||||
$(PYTHON3) -u $(MOZILLA_DIR)/build/upload.py --base-path $(DIST) $(CHECKSUM_FILES)
|
||||
|
||||
# source-package creates a source tarball from the files in MOZ_PKG_SRCDIR,
|
||||
# which is either set to a clean checkout or defaults to $topsrcdir
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, unicode_literals, print_function
|
||||
|
||||
from mozpack.packager.formats import (
|
||||
FlatFormatter,
|
||||
JarFormatter,
|
||||
|
|
@ -30,7 +32,8 @@ import buildconfig
|
|||
from argparse import ArgumentParser
|
||||
from createprecomplete import generate_precomplete
|
||||
import os
|
||||
from StringIO import StringIO
|
||||
import six
|
||||
from six import StringIO
|
||||
import subprocess
|
||||
import mozinfo
|
||||
|
||||
|
|
@ -82,13 +85,7 @@ class ToolLauncher(object):
|
|||
for e in extra_env:
|
||||
env[e] = extra_env[e]
|
||||
|
||||
# Work around a bug in Python 2.7.2 and lower where unicode types in
|
||||
# environment variables aren't handled by subprocess.
|
||||
for k, v in env.items():
|
||||
if isinstance(v, unicode):
|
||||
env[k] = v.encode('utf-8')
|
||||
|
||||
print >>errors.out, 'Executing', ' '.join(cmd)
|
||||
print('Executing %s' % ' '.join(cmd), file=errors.out)
|
||||
errors.out.flush()
|
||||
return subprocess.call(cmd, env=env)
|
||||
|
||||
|
|
@ -103,7 +100,7 @@ class LibSignFile(File):
|
|||
File class for shlibsign signatures.
|
||||
'''
|
||||
def copy(self, dest, skip_if_older=True):
|
||||
assert isinstance(dest, basestring)
|
||||
assert isinstance(dest, six.string_types)
|
||||
# os.path.getmtime returns a result in seconds with precision up to the
|
||||
# microsecond. But microsecond is too precise because shutil.copystat
|
||||
# only copies milliseconds, and seconds is not enough precision.
|
||||
|
|
@ -121,15 +118,16 @@ class RemovedFiles(GeneratedFile):
|
|||
'''
|
||||
def __init__(self, copier):
|
||||
self.copier = copier
|
||||
GeneratedFile.__init__(self, '')
|
||||
GeneratedFile.__init__(self, b'')
|
||||
|
||||
def handle_line(self, str):
|
||||
f = str.strip()
|
||||
def handle_line(self, f):
|
||||
f = f.strip()
|
||||
if not f:
|
||||
return
|
||||
if self.copier.contains(f):
|
||||
errors.error('Removal of packaged file(s): %s' % f)
|
||||
self.content += f + '\n'
|
||||
ensure = six.ensure_binary if 'b' in self._mode else six.ensure_text
|
||||
self.content += ensure(f) + ensure('\n')
|
||||
|
||||
|
||||
def split_define(define):
|
||||
|
|
@ -303,7 +301,10 @@ def main():
|
|||
# If a pdb file is present and we were instructed to copy it, include it.
|
||||
# Run on all OSes to capture MinGW builds
|
||||
if buildconfig.substs.get('MOZ_COPY_PDBS'):
|
||||
for p, f in copier:
|
||||
# We want to mutate the copier while we're iterating through it, so copy
|
||||
# the items to a list first.
|
||||
copier_items = [(p, f) for p, f in copier]
|
||||
for p, f in copier_items:
|
||||
if isinstance(f, ExecutableFile):
|
||||
pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb'
|
||||
if os.path.exists(pdbname):
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче