зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1283919 - Improve test package archiver for collecting files from directories referenced by a root manifest r=gps
MozReview-Commit-ID: EuGK3OS8XLj --HG-- extra : rebase_source : 1a547d02d0da68fac3ca5401df36589357f0bbbe
This commit is contained in:
Родитель
035755489b
Коммит
c2f0ef7c09
|
@ -16,6 +16,9 @@ import os
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
from manifestparser import TestManifest
|
||||||
|
from reftest import ReftestManifest
|
||||||
|
|
||||||
from mozbuild.util import ensureParentDir
|
from mozbuild.util import ensureParentDir
|
||||||
from mozpack.files import FileFinder
|
from mozpack.files import FileFinder
|
||||||
from mozpack.mozjar import JarWriter
|
from mozpack.mozjar import JarWriter
|
||||||
|
@ -315,6 +318,15 @@ ARCHIVE_FILES = {
|
||||||
'pattern': 'mozinfo.json',
|
'pattern': 'mozinfo.json',
|
||||||
'dest': 'reftest',
|
'dest': 'reftest',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
'source': buildconfig.topsrcdir,
|
||||||
|
'base': '',
|
||||||
|
'manifests': [
|
||||||
|
'layout/reftests/reftest.list',
|
||||||
|
'testing/crashtest/crashtests.list',
|
||||||
|
],
|
||||||
|
'dest': 'reftest/tests',
|
||||||
|
}
|
||||||
],
|
],
|
||||||
'talos': [
|
'talos': [
|
||||||
{
|
{
|
||||||
|
@ -414,15 +426,28 @@ for k, v in ARCHIVE_FILES.items():
|
||||||
def find_files(archive):
|
def find_files(archive):
|
||||||
for entry in ARCHIVE_FILES[archive]:
|
for entry in ARCHIVE_FILES[archive]:
|
||||||
source = entry['source']
|
source = entry['source']
|
||||||
|
dest = entry.get('dest')
|
||||||
base = entry.get('base', '')
|
base = entry.get('base', '')
|
||||||
|
|
||||||
pattern = entry.get('pattern')
|
pattern = entry.get('pattern')
|
||||||
patterns = entry.get('patterns', [])
|
patterns = entry.get('patterns', [])
|
||||||
if pattern:
|
if pattern:
|
||||||
patterns.append(pattern)
|
patterns.append(pattern)
|
||||||
dest = entry.get('dest')
|
|
||||||
|
manifest = entry.get('manifest')
|
||||||
|
manifests = entry.get('manifests', [])
|
||||||
|
if manifest:
|
||||||
|
manifests.append(manifest)
|
||||||
|
if manifests:
|
||||||
|
dirs = find_manifest_dirs(buildconfig.topsrcdir, manifests)
|
||||||
|
patterns.extend({'{}/**'.format(d) for d in dirs})
|
||||||
|
|
||||||
ignore = list(entry.get('ignore', []))
|
ignore = list(entry.get('ignore', []))
|
||||||
ignore.append('**/.mkdir.done')
|
ignore.extend([
|
||||||
ignore.append('**/*.pyc')
|
'**/.flake8',
|
||||||
|
'**/.mkdir.done',
|
||||||
|
'**/*.pyc',
|
||||||
|
])
|
||||||
|
|
||||||
common_kwargs = {
|
common_kwargs = {
|
||||||
'find_executables': False,
|
'find_executables': False,
|
||||||
|
@ -439,14 +464,29 @@ def find_files(archive):
|
||||||
yield p, f
|
yield p, f
|
||||||
|
|
||||||
|
|
||||||
def find_reftest_dirs(topsrcdir, manifests):
|
def find_manifest_dirs(topsrcdir, manifests):
|
||||||
from reftest import ReftestManifest
|
"""Routine to retrieve directories specified in a manifest, relative to topsrcdir.
|
||||||
|
|
||||||
|
It does not recurse into manifests, as we currently have no need for that.
|
||||||
|
"""
|
||||||
dirs = set()
|
dirs = set()
|
||||||
|
|
||||||
for p in manifests:
|
for p in manifests:
|
||||||
m = ReftestManifest()
|
p = os.path.join(topsrcdir, p)
|
||||||
m.load(os.path.join(topsrcdir, p))
|
|
||||||
dirs |= m.dirs
|
if p.endswith('.ini'):
|
||||||
|
test_manifest = TestManifest()
|
||||||
|
test_manifest.read(p)
|
||||||
|
dirs |= set([os.path.dirname(m) for m in test_manifest.manifests()])
|
||||||
|
|
||||||
|
elif p.endswith('.list'):
|
||||||
|
m = ReftestManifest()
|
||||||
|
m.load(p)
|
||||||
|
dirs |= m.dirs
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise Exception('"{}" is not a supported manifest format.'.format(
|
||||||
|
os.path.splitext(p)[1]))
|
||||||
|
|
||||||
dirs = {mozpath.normpath(d[len(topsrcdir):]).lstrip('/') for d in dirs}
|
dirs = {mozpath.normpath(d[len(topsrcdir):]).lstrip('/') for d in dirs}
|
||||||
|
|
||||||
|
@ -467,27 +507,6 @@ def find_reftest_dirs(topsrcdir, manifests):
|
||||||
return sorted(seen)
|
return sorted(seen)
|
||||||
|
|
||||||
|
|
||||||
def insert_reftest_entries(entries):
|
|
||||||
"""Reftests have their own mechanism for defining tests and locations.
|
|
||||||
|
|
||||||
This function is called when processing the reftest archive to process
|
|
||||||
reftest test manifests and insert the results into the existing list of
|
|
||||||
archive entries.
|
|
||||||
"""
|
|
||||||
manifests = (
|
|
||||||
'layout/reftests/reftest.list',
|
|
||||||
'testing/crashtest/crashtests.list',
|
|
||||||
)
|
|
||||||
|
|
||||||
for base in find_reftest_dirs(buildconfig.topsrcdir, manifests):
|
|
||||||
entries.append({
|
|
||||||
'source': buildconfig.topsrcdir,
|
|
||||||
'base': '',
|
|
||||||
'pattern': '%s/**' % base,
|
|
||||||
'dest': 'reftest/tests',
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv):
|
def main(argv):
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description='Produce test archives')
|
description='Produce test archives')
|
||||||
|
@ -499,11 +518,6 @@ def main(argv):
|
||||||
if not args.outputfile.endswith('.zip'):
|
if not args.outputfile.endswith('.zip'):
|
||||||
raise Exception('expected zip output file')
|
raise Exception('expected zip output file')
|
||||||
|
|
||||||
# Adjust reftest entries only if processing reftests (because it is
|
|
||||||
# unnecessary overhead otherwise).
|
|
||||||
if args.archive == 'reftest':
|
|
||||||
insert_reftest_entries(ARCHIVE_FILES['reftest'])
|
|
||||||
|
|
||||||
file_count = 0
|
file_count = 0
|
||||||
t_start = time.time()
|
t_start = time.time()
|
||||||
ensureParentDir(args.outputfile)
|
ensureParentDir(args.outputfile)
|
||||||
|
@ -515,8 +529,8 @@ def main(argv):
|
||||||
with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
|
with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
|
||||||
res = find_files(args.archive)
|
res = find_files(args.archive)
|
||||||
for p, f in res:
|
for p, f in res:
|
||||||
|
writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True)
|
||||||
file_count += 1
|
file_count += 1
|
||||||
writer.add(p.encode('utf-8'), f.read(), mode=f.mode)
|
|
||||||
|
|
||||||
duration = time.time() - t_start
|
duration = time.time() - t_start
|
||||||
zip_size = os.path.getsize(args.outputfile)
|
zip_size = os.path.getsize(args.outputfile)
|
||||||
|
|
|
@ -570,7 +570,7 @@ class JarWriter(object):
|
||||||
self._data.write(end.serialize())
|
self._data.write(end.serialize())
|
||||||
self._data.close()
|
self._data.close()
|
||||||
|
|
||||||
def add(self, name, data, compress=None, mode=None):
|
def add(self, name, data, compress=None, mode=None, skip_duplicates=False):
|
||||||
'''
|
'''
|
||||||
Add a new member to the jar archive, with the given name and the given
|
Add a new member to the jar archive, with the given name and the given
|
||||||
data.
|
data.
|
||||||
|
@ -582,13 +582,15 @@ class JarWriter(object):
|
||||||
than the uncompressed size.
|
than the uncompressed size.
|
||||||
The mode option gives the unix permissions that should be stored
|
The mode option gives the unix permissions that should be stored
|
||||||
for the jar entry.
|
for the jar entry.
|
||||||
|
If a duplicated member is found skip_duplicates will prevent raising
|
||||||
|
an exception if set to True.
|
||||||
The given data may be a buffer, a file-like instance, a Deflater or a
|
The given data may be a buffer, a file-like instance, a Deflater or a
|
||||||
JarFileReader instance. The latter two allow to avoid uncompressing
|
JarFileReader instance. The latter two allow to avoid uncompressing
|
||||||
data to recompress it.
|
data to recompress it.
|
||||||
'''
|
'''
|
||||||
name = mozpath.normsep(name)
|
name = mozpath.normsep(name)
|
||||||
|
|
||||||
if name in self._contents:
|
if name in self._contents and not skip_duplicates:
|
||||||
raise JarWriterError("File %s already in JarWriter" % name)
|
raise JarWriterError("File %s already in JarWriter" % name)
|
||||||
if compress is None:
|
if compress is None:
|
||||||
compress = self._compress
|
compress = self._compress
|
||||||
|
|
Загрузка…
Ссылка в новой задаче