Bug 1331899 - wptrunner changes to support v4 manifests, r=Ms2ger

The manifest changes came with some trivial API changes, mostly
removing unused arguments.

MozReview-Commit-ID: G9RZNds4MIE

testing/web-platform/harness/wptrunner/metadata.py

@@ -20,6 +20,7 @@ import wptmanifest
 import wpttest
 from vcs import git
 manifest = None  # Module that will be imported relative to test_root
+manifestitem = None
 
 logger = structuredlog.StructuredLogger("web-platform-tests")
 
@@ -66,8 +67,8 @@ def update_expected(test_paths, serve_root, log_file_names,
 
 
 def do_delayed_imports(serve_root):
-    global manifest
-    from manifest import manifest
+    global manifest, manifestitem
+    from manifest import manifest, item as manifestitem
 
 
 def files_in_repo(repo_root):
@@ -112,7 +113,7 @@ def unexpected_changes(manifests, change_data, files_changed):
 
     rv = []
 
-    return [fn for fn, tests in root_manifest if fn in files_changed and change_data.get(fn) != "M"]
+    return [fn for _, fn, _ in root_manifest if fn in files_changed and change_data.get(fn) != "M"]
 
 # For each testrun
 # Load all files and scan for the suite_start entry
@@ -295,9 +296,13 @@ def create_test_tree(metadata_path, test_manifest, property_order=None,
                      boolean_properties=None):
     expected_map = {}
     id_test_map = {}
-    exclude_types = frozenset(["stub", "helper", "manual"])
-    include_types = set(manifest.item_types) - exclude_types
-    for test_path, tests in test_manifest.itertypes(*include_types):
+    exclude_types = frozenset(["stub", "helper", "manual", "support", "conformancechecker"])
+    all_types = [item.item_type for item in manifestitem.__dict__.itervalues()
+                 if type(item) == type and
+                 issubclass(item, manifestitem.ManifestItem) and
+                 item.item_type is not None]
+    include_types = set(all_types) - exclude_types
+    for _, test_path, tests in test_manifest.itertypes(*include_types):
         expected_data = load_expected(test_manifest, metadata_path, test_path, tests,
                                       property_order=property_order,
                                       boolean_properties=boolean_properties)

testing/web-platform/harness/wptrunner/testloader.py

@@ -46,10 +46,10 @@ class Unchunked(TestChunker):
 class HashChunker(TestChunker):
     def __call__(self, manifest):
         chunk_index = self.chunk_number - 1
-        for test_path, tests in manifest:
+        for test_type, test_path, tests in manifest:
             h = int(hashlib.md5(test_path).hexdigest(), 16)
             if h % self.total_chunks == chunk_index:
-                yield test_path, tests
+                yield test_type, test_path, tests
 
 
 class DirectoryHashChunker(TestChunker):
@@ -60,10 +60,10 @@ class DirectoryHashChunker(TestChunker):
     """
     def __call__(self, manifest):
         chunk_index = self.chunk_number - 1
-        for test_path, tests in manifest:
+        for test_type, test_path, tests in manifest:
             h = int(hashlib.md5(os.path.dirname(test_path)).hexdigest(), 16)
             if h % self.total_chunks == chunk_index:
-                yield test_path, tests
+                yield test_type, test_path, tests
 
 
 class EqualTimeChunker(TestChunker):
@@ -85,7 +85,7 @@ class EqualTimeChunker(TestChunker):
         by_dir = OrderedDict()
         total_time = 0
 
-        for i, (test_path, tests) in enumerate(manifest_items):
+        for i, (test_type, test_path, tests) in enumerate(manifest_items):
             test_dir = tuple(os.path.split(test_path)[0].split(os.path.sep)[:3])
 
             if not test_dir in by_dir:
@@ -96,7 +96,7 @@ class EqualTimeChunker(TestChunker):
                        "long" else wpttest.LONG_TIMEOUT for test in tests)
             data.time += time
             total_time += time
-            data.tests.append((test_path, tests))
+            data.tests.append((test_type, test_path, tests))
 
         return by_dir, total_time
 
@@ -352,14 +352,14 @@ class TestFilter(object):
                 self.manifest.add_exclude(test_manifests, item)
 
     def __call__(self, manifest_iter):
-        for test_path, tests in manifest_iter:
+        for test_type, test_path, tests in manifest_iter:
             include_tests = set()
             for test in tests:
                 if self.manifest.include(test):
                     include_tests.add(test)
 
             if include_tests:
-                yield test_path, include_tests
+                yield test_type, test_path, include_tests
 
 class TagFilter(object):
     def __init__(self, tags):
@@ -406,14 +406,14 @@ class ManifestLoader(object):
                 pass
 
         if not json_data:
-            manifest_file = manifest.Manifest(None, url_base)
+            manifest_file = manifest.Manifest(url_base)
         else:
             try:
                 manifest_file = manifest.Manifest.from_json(tests_path, json_data)
             except manifest.ManifestVersionMismatch:
-                manifest_file = manifest.Manifest(None, url_base)
+                manifest_file = manifest.Manifest(url_base)
 
-            manifest_update.update(tests_path, url_base, manifest_file)
+            manifest_update.update(tests_path, manifest_file, True)
 
         manifest.write(manifest_file, manifest_path)
 
@@ -522,14 +522,14 @@ class TestLoader(object):
         if self.chunker is not None:
             manifest_items = self.chunker(manifest_items)
 
-        for test_path, tests in manifest_items:
+        for test_type, test_path, tests in manifest_items:
             manifest_file = iter(tests).next().manifest
             metadata_path = self.manifests[manifest_file]["metadata_path"]
             inherit_metadata, test_metadata = self.load_metadata(manifest_file, metadata_path, test_path)
 
             for test in iterfilter(self.meta_filters,
                                    self.iter_wpttest(inherit_metadata, test_metadata, tests)):
-                yield test_path, test.test_type, test
+                yield test_path, test_type, test
 
     def iter_wpttest(self, inherit_metadata, test_metadata, tests):
         for manifest_test in tests:

testing/web-platform/harness/wptrunner/update/sync.py

@@ -124,14 +124,12 @@ class GetSyncTargetCommit(Step):
 class LoadManifest(Step):
     """Load the test manifest"""
 
-    provides = ["manifest_path", "test_manifest", "old_manifest"]
+    provides = ["manifest_path", "test_manifest"]
 
     def create(self, state):
         from manifest import manifest
         state.manifest_path = os.path.join(state.metadata_path, "MANIFEST.json")
-        # Conservatively always rebuild the manifest when doing a sync
-        state.old_manifest = manifest.load(state.tests_path, state.manifest_path)
-        state.test_manifest = manifest.Manifest(None, "/")
+        state.test_manifest = manifest.Manifest("/")
 
 
 class UpdateManifest(Step):
@@ -139,7 +137,7 @@ class UpdateManifest(Step):
 
     def create(self, state):
         from manifest import manifest, update
-        update.update(state.sync["path"], "/", state.test_manifest)
+        update.update(state.sync["path"], state.test_manifest)
         manifest.write(state.test_manifest, state.manifest_path)