diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py index 19197427022a..cc8e74be800a 100644 --- a/python/mozbuild/mozbuild/backend/common.py +++ b/python/mozbuild/mozbuild/backend/common.py @@ -175,9 +175,9 @@ class TestManager(object): self.tests_by_path = defaultdict(list) self.installs_by_path = defaultdict(list) self.deferred_installs = set() - self.manifest_default_support_files = {} + self.manifest_defaults = {} - def add(self, t, flavor, topsrcdir, default_supp_files): + def add(self, t, flavor, topsrcdir): t = dict(t) t['flavor'] = flavor @@ -188,19 +188,14 @@ class TestManager(object): t['file_relpath'] = key t['dir_relpath'] = mozpath.dirname(key) - # Support files are propagated from the default section to individual - # tests by the manifest parser, but we end up storing a lot of - # redundant data due to the huge number of support files. - # So if we have support files that are the same as the manifest default - # we track that separately, per-manifest instead of per-test, to save - # space. - supp_files = t.get('support-files') - if supp_files and supp_files == default_supp_files: - self.manifest_default_support_files[t['manifest']] = default_supp_files - del t['support-files'] - self.tests_by_path[key].append(t) + def add_defaults(self, manifest): + if not hasattr(manifest, 'manifest_defaults'): + return + for sub_manifest, defaults in manifest.manifest_defaults.items(): + self.manifest_defaults[sub_manifest] = defaults + def add_installs(self, obj, topsrcdir): for src, (dest, _) in obj.installs.iteritems(): key = src[len(topsrcdir)+1:] @@ -236,8 +231,8 @@ class CommonBackend(BuildBackend): if isinstance(obj, TestManifest): for test in obj.tests: - self._test_manager.add(test, obj.flavor, obj.topsrcdir, - obj.default_support_files) + self._test_manager.add(test, obj.flavor, obj.topsrcdir) + self._test_manager.add_defaults(obj.manifest) self._test_manager.add_installs(obj, obj.topsrcdir) elif isinstance(obj, XPIDLFile): @@ -376,8 +371,10 @@ class CommonBackend(BuildBackend): # Write out a machine-readable file describing every test. topobjdir = self.environment.topobjdir with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh: - json.dump([self._test_manager.tests_by_path, - self._test_manager.manifest_default_support_files], fh) + json.dump(self._test_manager.tests_by_path, fh) + + with self._write_file(mozpath.join(topobjdir, 'test-defaults.json')) as fh: + json.dump(self._test_manager.manifest_defaults, fh) path = mozpath.join(self.environment.topobjdir, 'test-installs.json') with self._write_file(path) as fh: diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py index e26f0851c641..fdf8cca17221 100644 --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -642,11 +642,6 @@ class TestManifest(ContextDerived): # If this manifest is a duplicate of another one, this is the # manifestparser.TestManifest of the other one. 'dupe_manifest', - - # The support files appearing in the DEFAULT section of this - # manifest. This enables a space optimization in all-tests.json, - # see the comment in mozbuild/backend/common.py. - 'default_support_files', ) def __init__(self, context, path, manifest, flavor=None, @@ -668,7 +663,6 @@ class TestManifest(ContextDerived): self.tests = [] self.external_installs = set() self.deferred_installs = set() - self.default_support_files = None class LocalInclude(ContextDerived): diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index 0dd1da9ac292..52f571867200 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -1200,18 +1200,16 @@ class TreeMetadataEmitter(LoggingMixin): install_prefix = mozpath.join(install_root, install_subdir) try: - defaults = mpmanifest.manifest_defaults[os.path.normpath(path)] if not mpmanifest.tests: raise SandboxValidationError('Empty test manifest: %s' % path, context) + defaults = mpmanifest.manifest_defaults[os.path.normpath(path)] obj = TestManifest(context, path, mpmanifest, flavor=flavor, install_prefix=install_prefix, relpath=mozpath.join(manifest_reldir, mozpath.basename(path)), dupe_manifest='dupe-manifest' in defaults) - obj.default_support_files = defaults.get('support-files') - filtered = mpmanifest.tests # Jetpack add-on tests are expected to be generated during the @@ -1263,6 +1261,9 @@ class TreeMetadataEmitter(LoggingMixin): process_support_files(test) + for path, m_defaults in mpmanifest.manifest_defaults.items(): + process_support_files(m_defaults) + # We also copy manifests into the output directory, # including manifests from [include:foo] directives. for mpath in mpmanifest.manifests(): @@ -1281,8 +1282,8 @@ class TreeMetadataEmitter(LoggingMixin): del obj.installs[mozpath.join(manifest_dir, f)] except KeyError: raise SandboxValidationError('Error processing test ' - 'manifest %s: entry in generated-files not present ' - 'elsewhere in manifest: %s' % (path, f), context) + 'manifest %s: entry in generated-files not present ' + 'elsewhere in manifest: %s' % (path, f), context) yield obj except (AssertionError, Exception): diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index 683cd5275d97..df4fa958d3eb 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -528,7 +528,7 @@ class TestRecursiveMakeBackend(BackendTester): self.assertTrue(os.path.exists(all_tests_path)) with open(all_tests_path, 'rt') as fh: - o, _ = json.load(fh) + o = json.load(fh) self.assertIn('xpcshell.js', o) self.assertIn('dir1/test_bar.js', o) @@ -854,8 +854,7 @@ class TestRecursiveMakeBackend(BackendTester): self.assertTrue(os.path.exists(all_tests_path)) with open(all_tests_path, 'rt') as fh: - o, _ = json.load(fh) - + o = json.load(fh) self.assertIn('mochitest.js', o) self.assertIn('not_packaged.java', o) diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/mochitest.ini index 237f5c383097..2f1fc406a05b 100644 --- a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/mochitest.ini +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/mochitest.ini @@ -2,7 +2,6 @@ # http://creativecommons.org/publicdomain/zero/1.0/ [DEFAULT] -support-files = foo.js bar.js +support-files = bar.js foo.js bar.js [test_baz.js] -support-files = bar.js \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/test_testing.py b/python/mozbuild/mozbuild/test/test_testing.py index 76c769e5f5ab..853109413482 100644 --- a/python/mozbuild/mozbuild/test/test_testing.py +++ b/python/mozbuild/mozbuild/test/test_testing.py @@ -22,7 +22,7 @@ from mozbuild.testing import ( ALL_TESTS_JSON = b''' -[{ +{ "accessible/tests/mochitest/actions/test_anchors.html": [ { "dir_relpath": "accessible/tests/mochitest/actions", @@ -154,9 +154,11 @@ ALL_TESTS_JSON = b''' "tags": "devtools" } ] -}, { - "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": "\\ndata/**\\nxpcshell_updater.ini" -}]'''.strip() +}'''.strip() + +TEST_DEFAULTS = b'''{ + "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\\ndata/**\\nxpcshell_updater.ini"} +}''' class Base(unittest.TestCase): @@ -170,12 +172,17 @@ class Base(unittest.TestCase): self._temp_files = [] def _get_test_metadata(self): - f = NamedTemporaryFile() - f.write(ALL_TESTS_JSON) - f.flush() - self._temp_files.append(f) + all_tests = NamedTemporaryFile() + all_tests.write(ALL_TESTS_JSON) + all_tests.flush() + self._temp_files.append(all_tests) - return TestMetadata(filename=f.name) + test_defaults = NamedTemporaryFile() + test_defaults.write(TEST_DEFAULTS) + test_defaults.flush() + self._temp_files.append(test_defaults) + + return TestMetadata(all_tests.name, test_defaults=test_defaults.name) class TestTestMetadata(Base): @@ -246,6 +253,8 @@ class TestTestResolver(Base): with open(os.path.join(topobjdir, 'all-tests.json'), 'wt') as fh: fh.write(ALL_TESTS_JSON) + with open(os.path.join(topobjdir, 'test-defaults.json'), 'wt') as fh: + fh.write(TEST_DEFAULTS) o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir) diff --git a/python/mozbuild/mozbuild/testing.py b/python/mozbuild/mozbuild/testing.py index 099fcbc43966..ac891d4b9c98 100644 --- a/python/mozbuild/mozbuild/testing.py +++ b/python/mozbuild/mozbuild/testing.py @@ -51,30 +51,29 @@ class TestMetadata(object): configuration. """ - def __init__(self, filename=None): + def __init__(self, all_tests, test_defaults=None): self._tests_by_path = OrderedDefaultDict(list) self._tests_by_flavor = defaultdict(set) self._test_dirs = set() - if filename: - with open(filename, 'rt') as fh: - test_data, manifest_support_files = json.load(fh) - - for path, tests in test_data.items(): - for metadata in tests: - # Many tests inherit their "support-files" from a manifest - # level default, so we store these separately to save - # disk space, and propagate them to each test when - # de-serializing here. - manifest = metadata['manifest'] - support_files = manifest_support_files.get(manifest) - if support_files and 'support-files' not in metadata: - metadata['support-files'] = support_files - self._tests_by_path[path].append(metadata) - self._test_dirs.add(os.path.dirname(path)) - - flavor = metadata.get('flavor') - self._tests_by_flavor[flavor].add(path) + with open(all_tests, 'rt') as fh: + test_data = json.load(fh) + defaults = None + if test_defaults: + with open(test_defaults, 'rt') as fh: + defaults = json.load(fh) + for path, tests in test_data.items(): + for metadata in tests: + if defaults: + manifest = metadata['manifest'] + manifest_defaults = defaults.get(manifest) + if manifest_defaults: + metadata = manifestparser.combine_fields(manifest_defaults, + metadata) + self._tests_by_path[path].append(metadata) + self._test_dirs.add(os.path.dirname(path)) + flavor = metadata.get('flavor') + self._tests_by_flavor[flavor].add(path) def tests_with_flavor(self, flavor): """Obtain all tests having the specified flavor. @@ -183,8 +182,11 @@ class TestResolver(MozbuildObject): self._run_make(target='run-tests-deps', pass_thru=True, print_directory=False) - self._tests = TestMetadata(filename=os.path.join(self.topobjdir, - 'all-tests.json')) + self._tests = TestMetadata(os.path.join(self.topobjdir, + 'all-tests.json'), + test_defaults=os.path.join(self.topobjdir, + 'test-defaults.json')) + self._test_rewrites = { 'a11y': os.path.join(self.topobjdir, '_tests', 'testing', 'mochitest', 'a11y'), @@ -503,7 +505,8 @@ def read_manifestparser_manifest(context, manifest_path): path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path)) return manifestparser.TestManifest(manifests=[path], strict=True, rootdir=context.config.topsrcdir, - finder=context._finder) + finder=context._finder, + handle_defaults=False) def read_reftest_manifest(context, manifest_path): import reftest