зеркало из https://github.com/mozilla/gecko-dev.git
Backed out 2 changesets (bug 1359965) for likely breaking tc nightlies a=backout
Backed out changeset a0e257e346cc (bug 1359965) Backed out changeset ae8bce278626 (bug 1359965) MozReview-Commit-ID: 9rGpv7CFofi
This commit is contained in:
Wes Kocher
Родитель
0fea6cd28c
Коммит
afb7b41b84
|
|
@ -20,7 +20,6 @@ from manifestparser import TestManifest
|
|||
from reftest import ReftestManifest
|
||||
|
||||
from mozbuild.util import ensureParentDir
|
||||
from mozpack.archive import create_tar_gz_from_files
|
||||
from mozpack.copier import FileRegistry
|
||||
from mozpack.files import ExistingFile, FileFinder
|
||||
from mozpack.manifests import InstallManifest
|
||||
|
|
@ -615,31 +614,22 @@ def main(argv):
|
|||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
out_file = args.outputfile
|
||||
if not out_file.endswith(('.tar.gz', '.zip')):
|
||||
raise Exception('expected tar.gz or zip output file')
|
||||
if not args.outputfile.endswith('.zip'):
|
||||
raise Exception('expected zip output file')
|
||||
|
||||
file_count = 0
|
||||
t_start = time.time()
|
||||
ensureParentDir(out_file)
|
||||
res = find_files(args.archive)
|
||||
with open(out_file, 'wb') as fh:
|
||||
ensureParentDir(args.outputfile)
|
||||
with open(args.outputfile, 'wb') as fh:
|
||||
# Experimentation revealed that level 5 is significantly faster and has
|
||||
# marginally larger sizes than higher values and is the sweet spot
|
||||
# for optimal compression. Read the detailed commit message that
|
||||
# introduced this for raw numbers.
|
||||
if out_file.endswith('.tar.gz'):
|
||||
files = dict(res)
|
||||
create_tar_gz_from_files(fh, files, compresslevel=5)
|
||||
file_count = len(files)
|
||||
elif out_file.endswith('.zip'):
|
||||
with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
|
||||
for p, f in res:
|
||||
writer.add(p.encode('utf-8'), f.read(), mode=f.mode,
|
||||
skip_duplicates=True)
|
||||
file_count += 1
|
||||
else:
|
||||
raise Exception('unhandled file extension: %s' % out_file)
|
||||
with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
|
||||
res = find_files(args.archive)
|
||||
for p, f in res:
|
||||
writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True)
|
||||
file_count += 1
|
||||
|
||||
duration = time.time() - t_start
|
||||
zip_size = os.path.getsize(args.outputfile)
|
||||
|
|
|
|||
|
|
@ -6,13 +6,9 @@ from __future__ import absolute_import
|
|||
|
||||
import bz2
|
||||
import gzip
|
||||
import io
|
||||
import stat
|
||||
import tarfile
|
||||
|
||||
from .files import (
|
||||
BaseFile,
|
||||
)
|
||||
|
||||
# 2016-01-01T00:00:00+0000
|
||||
DEFAULT_MTIME = 1451606400
|
||||
|
|
@ -22,26 +18,22 @@ def create_tar_from_files(fp, files):
|
|||
"""Create a tar file deterministically.
|
||||
|
||||
Receives a dict mapping names of files in the archive to local filesystem
|
||||
paths or ``mozpack.files.BaseFile`` instances.
|
||||
paths.
|
||||
|
||||
The files will be archived and written to the passed file handle opened
|
||||
for writing.
|
||||
|
||||
Only regular files can be written.
|
||||
|
||||
FUTURE accept mozpack.files classes for writing
|
||||
FUTURE accept a filename argument (or create APIs to write files)
|
||||
"""
|
||||
with tarfile.open(name='', mode='w', fileobj=fp, dereference=True) as tf:
|
||||
for archive_path, f in sorted(files.items()):
|
||||
if isinstance(f, BaseFile):
|
||||
ti = tarfile.TarInfo(archive_path)
|
||||
ti.mode = f.mode or 0644
|
||||
ti.type = tarfile.REGTYPE
|
||||
else:
|
||||
ti = tf.gettarinfo(f, archive_path)
|
||||
for archive_path, fs_path in sorted(files.items()):
|
||||
ti = tf.gettarinfo(fs_path, archive_path)
|
||||
|
||||
if not ti.isreg():
|
||||
raise ValueError('not a regular file: %s' % f)
|
||||
raise ValueError('not a regular file: %s' % fs_path)
|
||||
|
||||
# Disallow setuid and setgid bits. This is an arbitrary restriction.
|
||||
# However, since we set uid/gid to root:root, setuid and setgid
|
||||
|
|
@ -49,7 +41,7 @@ def create_tar_from_files(fp, files):
|
|||
# uncompressed as root.
|
||||
if ti.mode & (stat.S_ISUID | stat.S_ISGID):
|
||||
raise ValueError('cannot add file with setuid or setgid set: '
|
||||
'%s' % f)
|
||||
'%s' % fs_path)
|
||||
|
||||
# Set uid, gid, username, and group as deterministic values.
|
||||
ti.uid = 0
|
||||
|
|
@ -60,14 +52,8 @@ def create_tar_from_files(fp, files):
|
|||
# Set mtime to a constant value.
|
||||
ti.mtime = DEFAULT_MTIME
|
||||
|
||||
if isinstance(f, BaseFile):
|
||||
ti.size = f.size()
|
||||
# tarfile wants to pass a size argument to read(). So just
|
||||
# wrap/buffer in a proper file object interface.
|
||||
tf.addfile(ti, f.open())
|
||||
else:
|
||||
with open(f, 'rb') as fh:
|
||||
tf.addfile(ti, fh)
|
||||
with open(fs_path, 'rb') as fh:
|
||||
tf.addfile(ti, fh)
|
||||
|
||||
|
||||
def create_tar_gz_from_files(fp, files, filename=None, compresslevel=9):
|
||||
|
|
|
|||
|
|
@ -219,14 +219,6 @@ class BaseFile(object):
|
|||
def read(self):
|
||||
raise NotImplementedError('BaseFile.read() not implemented. Bug 1170329.')
|
||||
|
||||
def size(self):
|
||||
"""Returns size of the entry.
|
||||
|
||||
Derived classes are highly encouraged to override this with a more
|
||||
optimal implementation.
|
||||
"""
|
||||
return len(self.read())
|
||||
|
||||
@property
|
||||
def mode(self):
|
||||
'''
|
||||
|
|
@ -258,9 +250,6 @@ class File(BaseFile):
|
|||
with open(self.path, 'rb') as fh:
|
||||
return fh.read()
|
||||
|
||||
def size(self):
|
||||
return os.stat(self.path).st_size
|
||||
|
||||
|
||||
class ExecutableFile(File):
|
||||
'''
|
||||
|
|
@ -508,12 +497,6 @@ class GeneratedFile(BaseFile):
|
|||
def open(self):
|
||||
return BytesIO(self.content)
|
||||
|
||||
def read(self):
|
||||
return self.content
|
||||
|
||||
def size(self):
|
||||
return len(self.content)
|
||||
|
||||
|
||||
class DeflatedFile(BaseFile):
|
||||
'''
|
||||
|
|
|
|||
|
|
@ -18,9 +18,6 @@ from mozpack.archive import (
|
|||
create_tar_gz_from_files,
|
||||
create_tar_bz2_from_files,
|
||||
)
|
||||
from mozpack.files import (
|
||||
GeneratedFile,
|
||||
)
|
||||
|
||||
from mozunit import main
|
||||
|
||||
|
|
@ -44,22 +41,19 @@ class TestArchive(unittest.TestCase):
|
|||
def _create_files(self, root):
|
||||
files = {}
|
||||
for i in range(10):
|
||||
p = os.path.join(root, b'file%02d' % i)
|
||||
p = os.path.join(root, b'file%d' % i)
|
||||
with open(p, 'wb') as fh:
|
||||
fh.write(b'file%02d' % i)
|
||||
fh.write(b'file%d' % i)
|
||||
# Need to set permissions or umask may influence testing.
|
||||
os.chmod(p, MODE_STANDARD)
|
||||
files[b'file%02d' % i] = p
|
||||
|
||||
for i in range(10):
|
||||
files[b'file%02d' % (i + 10)] = GeneratedFile('file%02d' % (i + 10))
|
||||
files[b'file%d' % i] = p
|
||||
|
||||
return files
|
||||
|
||||
def _verify_basic_tarfile(self, tf):
|
||||
self.assertEqual(len(tf.getmembers()), 20)
|
||||
self.assertEqual(len(tf.getmembers()), 10)
|
||||
|
||||
names = ['file%02d' % i for i in range(20)]
|
||||
names = ['file%d' % i for i in range(10)]
|
||||
self.assertEqual(tf.getnames(), names)
|
||||
|
||||
for ti in tf.getmembers():
|
||||
|
|
@ -112,7 +106,7 @@ class TestArchive(unittest.TestCase):
|
|||
create_tar_from_files(fh, files)
|
||||
|
||||
# Output should be deterministic.
|
||||
self.assertEqual(file_hash(tp), '01cd314e277f060e98c7de6c8ea57f96b3a2065c')
|
||||
self.assertEqual(file_hash(tp), 'cd16cee6f13391abd94dfa435d2633b61ed727f1')
|
||||
|
||||
with tarfile.open(tp, 'r') as tf:
|
||||
self._verify_basic_tarfile(tf)
|
||||
|
|
@ -150,7 +144,7 @@ class TestArchive(unittest.TestCase):
|
|||
with open(gp, 'wb') as fh:
|
||||
create_tar_gz_from_files(fh, files)
|
||||
|
||||
self.assertEqual(file_hash(gp), '7c4da5adc5088cdf00911d5daf9a67b15de714b7')
|
||||
self.assertEqual(file_hash(gp), 'acb602239c1aeb625da5e69336775609516d60f5')
|
||||
|
||||
with tarfile.open(gp, 'r:gz') as tf:
|
||||
self._verify_basic_tarfile(tf)
|
||||
|
|
@ -167,7 +161,7 @@ class TestArchive(unittest.TestCase):
|
|||
with open(gp, 'wb') as fh:
|
||||
create_tar_gz_from_files(fh, files, filename='foobar', compresslevel=1)
|
||||
|
||||
self.assertEqual(file_hash(gp), '1cc8b96f0262350977c2e9d61f40a1fa76f35c52')
|
||||
self.assertEqual(file_hash(gp), 'fd099f96480cc1100f37baa8e89a6b820dbbcbd3')
|
||||
|
||||
with tarfile.open(gp, 'r:gz') as tf:
|
||||
self._verify_basic_tarfile(tf)
|
||||
|
|
@ -184,7 +178,7 @@ class TestArchive(unittest.TestCase):
|
|||
with open(bp, 'wb') as fh:
|
||||
create_tar_bz2_from_files(fh, files)
|
||||
|
||||
self.assertEqual(file_hash(bp), 'eb5096d2fbb71df7b3d690001a6f2e82a5aad6a7')
|
||||
self.assertEqual(file_hash(bp), '1827ad00dfe7acf857b7a1c95ce100361e3f6eea')
|
||||
|
||||
with tarfile.open(bp, 'r:bz2') as tf:
|
||||
self._verify_basic_tarfile(tf)
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
|
|||
"target.awsy.tests.zip",
|
||||
"target.test_packages.json",
|
||||
"target.txt",
|
||||
"target.web-platform.tests.tar.zip",
|
||||
"target.web-platform.tests.zip",
|
||||
"target.xpcshell.tests.zip",
|
||||
"target_info.txt",
|
||||
"target.jsshell.zip",
|
||||
|
|
@ -64,7 +64,7 @@ _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
|
|||
"en-US/target.awsy.tests.zip",
|
||||
"en-US/target.test_packages.json",
|
||||
"en-US/target.txt",
|
||||
"en-US/target.web-platform.tests.tar.gz",
|
||||
"en-US/target.web-platform.tests.zip",
|
||||
"en-US/target.xpcshell.tests.zip",
|
||||
"en-US/target_info.txt",
|
||||
"en-US/bouncer.apk",
|
||||
|
|
@ -84,7 +84,7 @@ _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI = [
|
|||
"target.awsy.tests.zip",
|
||||
"target.test_packages.json",
|
||||
"target.txt",
|
||||
"target.web-platform.tests.tar.gz",
|
||||
"target.web-platform.tests.zip",
|
||||
"target.xpcshell.tests.zip",
|
||||
"target_info.txt",
|
||||
"bouncer.apk",
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
|
|||
"target.awsy.tests.zip",
|
||||
"target.test_packages.json",
|
||||
"target.txt",
|
||||
"target.web-platform.tests.tar.gz",
|
||||
"target.web-platform.tests.zip",
|
||||
"target.xpcshell.tests.zip",
|
||||
"target_info.txt",
|
||||
"target.jsshell.zip",
|
||||
|
|
|
|||
|
|
@ -119,26 +119,23 @@ ifdef COMPILE_ENVIRONMENT
|
|||
stage-all: stage-cppunittests
|
||||
endif
|
||||
|
||||
TEST_PKGS_ZIP := \
|
||||
TEST_PKGS := \
|
||||
common \
|
||||
cppunittest \
|
||||
mochitest \
|
||||
reftest \
|
||||
talos \
|
||||
awsy \
|
||||
xpcshell \
|
||||
$(NULL)
|
||||
|
||||
TEST_PKGS_TARGZ := \
|
||||
web-platform \
|
||||
xpcshell \
|
||||
$(NULL)
|
||||
|
||||
ifdef LINK_GTEST_DURING_COMPILE
|
||||
stage-all: stage-gtest
|
||||
TEST_PKGS_ZIP += gtest
|
||||
TEST_PKGS += gtest
|
||||
endif
|
||||
|
||||
PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.$(2)'
|
||||
PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.zip'
|
||||
|
||||
test-packages-manifest:
|
||||
@rm -f $(MOZ_TEST_PACKAGES_FILE)
|
||||
|
|
@ -146,9 +143,8 @@ test-packages-manifest:
|
|||
$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
|
||||
--jsshell $(JSSHELL_NAME) \
|
||||
--dest-file '$(MOZ_TEST_PACKAGES_FILE)' \
|
||||
$(call PKG_ARG,common,zip) \
|
||||
$(foreach pkg,$(TEST_PKGS_ZIP),$(call PKG_ARG,$(pkg),zip)) \
|
||||
$(foreach pkg,$(TEST_PKGS_TARGZ),$(call PKG_ARG,$(pkg),tar.gz))
|
||||
$(call PKG_ARG,common) \
|
||||
$(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
|
||||
|
||||
package-tests-prepare-dest:
|
||||
@rm -f '$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)'
|
||||
|
|
@ -158,12 +154,11 @@ define package_archive
|
|||
package-tests-$(1): stage-all package-tests-prepare-dest
|
||||
$$(call py_action,test_archive, \
|
||||
$(1) \
|
||||
'$$(abspath $$(DIST))/$$(PKG_PATH)/$$(PKG_BASENAME).$(1).tests.$(2)')
|
||||
'$$(abspath $$(DIST))/$$(PKG_PATH)/$$(PKG_BASENAME).$(1).tests.zip')
|
||||
package-tests: package-tests-$(1)
|
||||
endef
|
||||
|
||||
$(foreach name,$(TEST_PKGS_ZIP),$(eval $(call package_archive,$(name),zip)))
|
||||
$(foreach name,$(TEST_PKGS_TARGZ),$(eval $(call package_archive,$(name),tar.gz)))
|
||||
$(foreach name,$(TEST_PKGS),$(eval $(call package_archive,$(name))))
|
||||
|
||||
ifeq ($(MOZ_BUILD_APP),mobile/android)
|
||||
stage-all: stage-android
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ CPP_TEST_PACKAGE = $(PKG_BASENAME).cppunittest.tests.zip
|
|||
XPC_TEST_PACKAGE = $(PKG_BASENAME).xpcshell.tests.zip
|
||||
MOCHITEST_PACKAGE = $(PKG_BASENAME).mochitest.tests.zip
|
||||
REFTEST_PACKAGE = $(PKG_BASENAME).reftest.tests.zip
|
||||
WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.tar.gz
|
||||
WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.zip
|
||||
TALOS_PACKAGE = $(PKG_BASENAME).talos.tests.zip
|
||||
AWSY_PACKAGE = $(PKG_BASENAME).awsy.tests.zip
|
||||
GTEST_PACKAGE = $(PKG_BASENAME).gtest.tests.zip
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче