Bug 1283919 - Improve test package archiver for collecting files from directories referenced by a root manifest r=gps

MozReview-Commit-ID: EuGK3OS8XLj

--HG--
extra : rebase_source : 1a547d02d0da68fac3ca5401df36589357f0bbbe

python/mozbuild/mozbuild/action/test_archive.py

@@ -16,6 +16,9 @@ import os
 import sys
 import time
 
+from manifestparser import TestManifest
+from reftest import ReftestManifest
+
 from mozbuild.util import ensureParentDir
 from mozpack.files import FileFinder
 from mozpack.mozjar import JarWriter
@@ -315,6 +318,15 @@ ARCHIVE_FILES = {
             'pattern': 'mozinfo.json',
             'dest': 'reftest',
         },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': '',
+            'manifests': [
+                'layout/reftests/reftest.list',
+                'testing/crashtest/crashtests.list',
+            ],
+            'dest': 'reftest/tests',
+        }
     ],
     'talos': [
         {
@@ -414,15 +426,28 @@ for k, v in ARCHIVE_FILES.items():
 def find_files(archive):
     for entry in ARCHIVE_FILES[archive]:
         source = entry['source']
+        dest = entry.get('dest')
         base = entry.get('base', '')
+
         pattern = entry.get('pattern')
         patterns = entry.get('patterns', [])
         if pattern:
             patterns.append(pattern)
-        dest = entry.get('dest')
+
+        manifest = entry.get('manifest')
+        manifests = entry.get('manifests', [])
+        if manifest:
+            manifests.append(manifest)
+        if manifests:
+            dirs = find_manifest_dirs(buildconfig.topsrcdir, manifests)
+            patterns.extend({'{}/**'.format(d) for d in dirs})
+
         ignore = list(entry.get('ignore', []))
-        ignore.append('**/.mkdir.done')
+        ignore.extend([
-        ignore.append('**/*.pyc')
+            '**/.flake8',
+            '**/.mkdir.done',
+            '**/*.pyc',
+        ])
 
         common_kwargs = {
             'find_executables': False,
@@ -439,15 +464,30 @@ def find_files(archive):
                 yield p, f
 
 
-def find_reftest_dirs(topsrcdir, manifests):
+def find_manifest_dirs(topsrcdir, manifests):
-    from reftest import ReftestManifest
+    """Routine to retrieve directories specified in a manifest, relative to topsrcdir.
 
+    It does not recurse into manifests, as we currently have no need for that.
+    """
     dirs = set()
+
     for p in manifests:
+        p = os.path.join(topsrcdir, p)
+
+        if p.endswith('.ini'):
+            test_manifest = TestManifest()
+            test_manifest.read(p)
+            dirs |= set([os.path.dirname(m) for m in test_manifest.manifests()])
+
+        elif p.endswith('.list'):
             m = ReftestManifest()
-        m.load(os.path.join(topsrcdir, p))
+            m.load(p)
             dirs |= m.dirs
 
+        else:
+            raise Exception('"{}" is not a supported manifest format.'.format(
+                os.path.splitext(p)[1]))
+
     dirs = {mozpath.normpath(d[len(topsrcdir):]).lstrip('/') for d in dirs}
 
     # Filter out children captured by parent directories because duplicates
@@ -467,27 +507,6 @@ def find_reftest_dirs(topsrcdir, manifests):
     return sorted(seen)
 
 
-def insert_reftest_entries(entries):
-    """Reftests have their own mechanism for defining tests and locations.
-
-    This function is called when processing the reftest archive to process
-    reftest test manifests and insert the results into the existing list of
-    archive entries.
-    """
-    manifests = (
-        'layout/reftests/reftest.list',
-        'testing/crashtest/crashtests.list',
-    )
-
-    for base in find_reftest_dirs(buildconfig.topsrcdir, manifests):
-        entries.append({
-            'source': buildconfig.topsrcdir,
-            'base': '',
-            'pattern': '%s/**' % base,
-            'dest': 'reftest/tests',
-        })
-
-
 def main(argv):
     parser = argparse.ArgumentParser(
         description='Produce test archives')
@@ -499,11 +518,6 @@ def main(argv):
     if not args.outputfile.endswith('.zip'):
         raise Exception('expected zip output file')
 
-    # Adjust reftest entries only if processing reftests (because it is
-    # unnecessary overhead otherwise).
-    if args.archive == 'reftest':
-        insert_reftest_entries(ARCHIVE_FILES['reftest'])
-
     file_count = 0
     t_start = time.time()
     ensureParentDir(args.outputfile)
@@ -515,8 +529,8 @@ def main(argv):
         with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
             res = find_files(args.archive)
             for p, f in res:
+                writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True)
                 file_count += 1
-                writer.add(p.encode('utf-8'), f.read(), mode=f.mode)
 
     duration = time.time() - t_start
     zip_size = os.path.getsize(args.outputfile)

python/mozbuild/mozpack/mozjar.py

@@ -570,7 +570,7 @@ class JarWriter(object):
         self._data.write(end.serialize())
         self._data.close()
 
-    def add(self, name, data, compress=None, mode=None):
+    def add(self, name, data, compress=None, mode=None, skip_duplicates=False):
         '''
         Add a new member to the jar archive, with the given name and the given
         data.
@@ -582,13 +582,15 @@ class JarWriter(object):
         than the uncompressed size.
         The mode option gives the unix permissions that should be stored
         for the jar entry.
+        If a duplicated member is found skip_duplicates will prevent raising
+        an exception if set to True.
         The given data may be a buffer, a file-like instance, a Deflater or a
         JarFileReader instance. The latter two allow to avoid uncompressing
         data to recompress it.
         '''
         name = mozpath.normsep(name)
 
-        if name in self._contents:
+        if name in self._contents and not skip_duplicates:
             raise JarWriterError("File %s already in JarWriter" % name)
         if compress is None:
             compress = self._compress