зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1312520 - Store and process manifest-level defaults in the build system separately from individual tests. r=gps
MozReview-Commit-ID: 1dSMAaOqToJ
This commit is contained in:
Chris Manchester
Родитель
8792602f28
Коммит
0d0d70f0f2
|
|
@ -175,9 +175,9 @@ class TestManager(object):
|
|||
self.tests_by_path = defaultdict(list)
|
||||
self.installs_by_path = defaultdict(list)
|
||||
self.deferred_installs = set()
|
||||
self.manifest_default_support_files = {}
|
||||
self.manifest_defaults = {}
|
||||
|
||||
def add(self, t, flavor, topsrcdir, default_supp_files):
|
||||
def add(self, t, flavor, topsrcdir):
|
||||
t = dict(t)
|
||||
t['flavor'] = flavor
|
||||
|
||||
|
|
@ -188,19 +188,14 @@ class TestManager(object):
|
|||
t['file_relpath'] = key
|
||||
t['dir_relpath'] = mozpath.dirname(key)
|
||||
|
||||
# Support files are propagated from the default section to individual
|
||||
# tests by the manifest parser, but we end up storing a lot of
|
||||
# redundant data due to the huge number of support files.
|
||||
# So if we have support files that are the same as the manifest default
|
||||
# we track that separately, per-manifest instead of per-test, to save
|
||||
# space.
|
||||
supp_files = t.get('support-files')
|
||||
if supp_files and supp_files == default_supp_files:
|
||||
self.manifest_default_support_files[t['manifest']] = default_supp_files
|
||||
del t['support-files']
|
||||
|
||||
self.tests_by_path[key].append(t)
|
||||
|
||||
def add_defaults(self, manifest):
|
||||
if not hasattr(manifest, 'manifest_defaults'):
|
||||
return
|
||||
for sub_manifest, defaults in manifest.manifest_defaults.items():
|
||||
self.manifest_defaults[sub_manifest] = defaults
|
||||
|
||||
def add_installs(self, obj, topsrcdir):
|
||||
for src, (dest, _) in obj.installs.iteritems():
|
||||
key = src[len(topsrcdir)+1:]
|
||||
|
|
@ -236,8 +231,8 @@ class CommonBackend(BuildBackend):
|
|||
|
||||
if isinstance(obj, TestManifest):
|
||||
for test in obj.tests:
|
||||
self._test_manager.add(test, obj.flavor, obj.topsrcdir,
|
||||
obj.default_support_files)
|
||||
self._test_manager.add(test, obj.flavor, obj.topsrcdir)
|
||||
self._test_manager.add_defaults(obj.manifest)
|
||||
self._test_manager.add_installs(obj, obj.topsrcdir)
|
||||
|
||||
elif isinstance(obj, XPIDLFile):
|
||||
|
|
@ -376,8 +371,10 @@ class CommonBackend(BuildBackend):
|
|||
# Write out a machine-readable file describing every test.
|
||||
topobjdir = self.environment.topobjdir
|
||||
with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh:
|
||||
json.dump([self._test_manager.tests_by_path,
|
||||
self._test_manager.manifest_default_support_files], fh)
|
||||
json.dump(self._test_manager.tests_by_path, fh)
|
||||
|
||||
with self._write_file(mozpath.join(topobjdir, 'test-defaults.json')) as fh:
|
||||
json.dump(self._test_manager.manifest_defaults, fh)
|
||||
|
||||
path = mozpath.join(self.environment.topobjdir, 'test-installs.json')
|
||||
with self._write_file(path) as fh:
|
||||
|
|
|
|||
|
|
@ -642,11 +642,6 @@ class TestManifest(ContextDerived):
|
|||
# If this manifest is a duplicate of another one, this is the
|
||||
# manifestparser.TestManifest of the other one.
|
||||
'dupe_manifest',
|
||||
|
||||
# The support files appearing in the DEFAULT section of this
|
||||
# manifest. This enables a space optimization in all-tests.json,
|
||||
# see the comment in mozbuild/backend/common.py.
|
||||
'default_support_files',
|
||||
)
|
||||
|
||||
def __init__(self, context, path, manifest, flavor=None,
|
||||
|
|
@ -668,7 +663,6 @@ class TestManifest(ContextDerived):
|
|||
self.tests = []
|
||||
self.external_installs = set()
|
||||
self.deferred_installs = set()
|
||||
self.default_support_files = None
|
||||
|
||||
|
||||
class LocalInclude(ContextDerived):
|
||||
|
|
|
|||
|
|
@ -1200,18 +1200,16 @@ class TreeMetadataEmitter(LoggingMixin):
|
|||
install_prefix = mozpath.join(install_root, install_subdir)
|
||||
|
||||
try:
|
||||
defaults = mpmanifest.manifest_defaults[os.path.normpath(path)]
|
||||
if not mpmanifest.tests:
|
||||
raise SandboxValidationError('Empty test manifest: %s'
|
||||
% path, context)
|
||||
|
||||
defaults = mpmanifest.manifest_defaults[os.path.normpath(path)]
|
||||
obj = TestManifest(context, path, mpmanifest, flavor=flavor,
|
||||
install_prefix=install_prefix,
|
||||
relpath=mozpath.join(manifest_reldir, mozpath.basename(path)),
|
||||
dupe_manifest='dupe-manifest' in defaults)
|
||||
|
||||
obj.default_support_files = defaults.get('support-files')
|
||||
|
||||
filtered = mpmanifest.tests
|
||||
|
||||
# Jetpack add-on tests are expected to be generated during the
|
||||
|
|
@ -1263,6 +1261,9 @@ class TreeMetadataEmitter(LoggingMixin):
|
|||
|
||||
process_support_files(test)
|
||||
|
||||
for path, m_defaults in mpmanifest.manifest_defaults.items():
|
||||
process_support_files(m_defaults)
|
||||
|
||||
# We also copy manifests into the output directory,
|
||||
# including manifests from [include:foo] directives.
|
||||
for mpath in mpmanifest.manifests():
|
||||
|
|
@ -1281,8 +1282,8 @@ class TreeMetadataEmitter(LoggingMixin):
|
|||
del obj.installs[mozpath.join(manifest_dir, f)]
|
||||
except KeyError:
|
||||
raise SandboxValidationError('Error processing test '
|
||||
'manifest %s: entry in generated-files not present '
|
||||
'elsewhere in manifest: %s' % (path, f), context)
|
||||
'manifest %s: entry in generated-files not present '
|
||||
'elsewhere in manifest: %s' % (path, f), context)
|
||||
|
||||
yield obj
|
||||
except (AssertionError, Exception):
|
||||
|
|
|
|||
|
|
@ -528,7 +528,7 @@ class TestRecursiveMakeBackend(BackendTester):
|
|||
self.assertTrue(os.path.exists(all_tests_path))
|
||||
|
||||
with open(all_tests_path, 'rt') as fh:
|
||||
o, _ = json.load(fh)
|
||||
o = json.load(fh)
|
||||
|
||||
self.assertIn('xpcshell.js', o)
|
||||
self.assertIn('dir1/test_bar.js', o)
|
||||
|
|
@ -854,8 +854,7 @@ class TestRecursiveMakeBackend(BackendTester):
|
|||
self.assertTrue(os.path.exists(all_tests_path))
|
||||
|
||||
with open(all_tests_path, 'rt') as fh:
|
||||
o, _ = json.load(fh)
|
||||
|
||||
o = json.load(fh)
|
||||
self.assertIn('mochitest.js', o)
|
||||
self.assertIn('not_packaged.java', o)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
# http://creativecommons.org/publicdomain/zero/1.0/
|
||||
|
||||
[DEFAULT]
|
||||
support-files = foo.js bar.js
|
||||
support-files = bar.js foo.js bar.js
|
||||
|
||||
[test_baz.js]
|
||||
support-files = bar.js
|
||||
|
|
@ -22,7 +22,7 @@ from mozbuild.testing import (
|
|||
|
||||
|
||||
ALL_TESTS_JSON = b'''
|
||||
[{
|
||||
{
|
||||
"accessible/tests/mochitest/actions/test_anchors.html": [
|
||||
{
|
||||
"dir_relpath": "accessible/tests/mochitest/actions",
|
||||
|
|
@ -154,9 +154,11 @@ ALL_TESTS_JSON = b'''
|
|||
"tags": "devtools"
|
||||
}
|
||||
]
|
||||
}, {
|
||||
"/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": "\\ndata/**\\nxpcshell_updater.ini"
|
||||
}]'''.strip()
|
||||
}'''.strip()
|
||||
|
||||
TEST_DEFAULTS = b'''{
|
||||
"/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\\ndata/**\\nxpcshell_updater.ini"}
|
||||
}'''
|
||||
|
||||
|
||||
class Base(unittest.TestCase):
|
||||
|
|
@ -170,12 +172,17 @@ class Base(unittest.TestCase):
|
|||
self._temp_files = []
|
||||
|
||||
def _get_test_metadata(self):
|
||||
f = NamedTemporaryFile()
|
||||
f.write(ALL_TESTS_JSON)
|
||||
f.flush()
|
||||
self._temp_files.append(f)
|
||||
all_tests = NamedTemporaryFile()
|
||||
all_tests.write(ALL_TESTS_JSON)
|
||||
all_tests.flush()
|
||||
self._temp_files.append(all_tests)
|
||||
|
||||
return TestMetadata(filename=f.name)
|
||||
test_defaults = NamedTemporaryFile()
|
||||
test_defaults.write(TEST_DEFAULTS)
|
||||
test_defaults.flush()
|
||||
self._temp_files.append(test_defaults)
|
||||
|
||||
return TestMetadata(all_tests.name, test_defaults=test_defaults.name)
|
||||
|
||||
|
||||
class TestTestMetadata(Base):
|
||||
|
|
@ -246,6 +253,8 @@ class TestTestResolver(Base):
|
|||
|
||||
with open(os.path.join(topobjdir, 'all-tests.json'), 'wt') as fh:
|
||||
fh.write(ALL_TESTS_JSON)
|
||||
with open(os.path.join(topobjdir, 'test-defaults.json'), 'wt') as fh:
|
||||
fh.write(TEST_DEFAULTS)
|
||||
|
||||
o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir)
|
||||
|
||||
|
|
|
|||
|
|
@ -51,30 +51,29 @@ class TestMetadata(object):
|
|||
configuration.
|
||||
"""
|
||||
|
||||
def __init__(self, filename=None):
|
||||
def __init__(self, all_tests, test_defaults=None):
|
||||
self._tests_by_path = OrderedDefaultDict(list)
|
||||
self._tests_by_flavor = defaultdict(set)
|
||||
self._test_dirs = set()
|
||||
|
||||
if filename:
|
||||
with open(filename, 'rt') as fh:
|
||||
test_data, manifest_support_files = json.load(fh)
|
||||
|
||||
for path, tests in test_data.items():
|
||||
for metadata in tests:
|
||||
# Many tests inherit their "support-files" from a manifest
|
||||
# level default, so we store these separately to save
|
||||
# disk space, and propagate them to each test when
|
||||
# de-serializing here.
|
||||
manifest = metadata['manifest']
|
||||
support_files = manifest_support_files.get(manifest)
|
||||
if support_files and 'support-files' not in metadata:
|
||||
metadata['support-files'] = support_files
|
||||
self._tests_by_path[path].append(metadata)
|
||||
self._test_dirs.add(os.path.dirname(path))
|
||||
|
||||
flavor = metadata.get('flavor')
|
||||
self._tests_by_flavor[flavor].add(path)
|
||||
with open(all_tests, 'rt') as fh:
|
||||
test_data = json.load(fh)
|
||||
defaults = None
|
||||
if test_defaults:
|
||||
with open(test_defaults, 'rt') as fh:
|
||||
defaults = json.load(fh)
|
||||
for path, tests in test_data.items():
|
||||
for metadata in tests:
|
||||
if defaults:
|
||||
manifest = metadata['manifest']
|
||||
manifest_defaults = defaults.get(manifest)
|
||||
if manifest_defaults:
|
||||
metadata = manifestparser.combine_fields(manifest_defaults,
|
||||
metadata)
|
||||
self._tests_by_path[path].append(metadata)
|
||||
self._test_dirs.add(os.path.dirname(path))
|
||||
flavor = metadata.get('flavor')
|
||||
self._tests_by_flavor[flavor].add(path)
|
||||
|
||||
def tests_with_flavor(self, flavor):
|
||||
"""Obtain all tests having the specified flavor.
|
||||
|
|
@ -183,8 +182,11 @@ class TestResolver(MozbuildObject):
|
|||
self._run_make(target='run-tests-deps', pass_thru=True,
|
||||
print_directory=False)
|
||||
|
||||
self._tests = TestMetadata(filename=os.path.join(self.topobjdir,
|
||||
'all-tests.json'))
|
||||
self._tests = TestMetadata(os.path.join(self.topobjdir,
|
||||
'all-tests.json'),
|
||||
test_defaults=os.path.join(self.topobjdir,
|
||||
'test-defaults.json'))
|
||||
|
||||
self._test_rewrites = {
|
||||
'a11y': os.path.join(self.topobjdir, '_tests', 'testing',
|
||||
'mochitest', 'a11y'),
|
||||
|
|
@ -503,7 +505,8 @@ def read_manifestparser_manifest(context, manifest_path):
|
|||
path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
|
||||
return manifestparser.TestManifest(manifests=[path], strict=True,
|
||||
rootdir=context.config.topsrcdir,
|
||||
finder=context._finder)
|
||||
finder=context._finder,
|
||||
handle_defaults=False)
|
||||
|
||||
def read_reftest_manifest(context, manifest_path):
|
||||
import reftest
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче