Backed out 2 changesets (bug 1359965) for likely breaking tc nightlies a=backout

Backed out changeset a0e257e346cc (bug 1359965)
Backed out changeset ae8bce278626 (bug 1359965)

MozReview-Commit-ID: 9rGpv7CFofi

python/mozbuild/mozbuild/action/test_archive.py

@@ -20,7 +20,6 @@ from manifestparser import TestManifest
 from reftest import ReftestManifest
 
 from mozbuild.util import ensureParentDir
-from mozpack.archive import create_tar_gz_from_files
 from mozpack.copier import FileRegistry
 from mozpack.files import ExistingFile, FileFinder
 from mozpack.manifests import InstallManifest
@@ -615,31 +614,22 @@ def main(argv):
 
     args = parser.parse_args(argv)
 
-    out_file = args.outputfile
+    if not args.outputfile.endswith('.zip'):
-    if not out_file.endswith(('.tar.gz', '.zip')):
+        raise Exception('expected zip output file')
-        raise Exception('expected tar.gz or zip output file')
 
     file_count = 0
     t_start = time.time()
-    ensureParentDir(out_file)
+    ensureParentDir(args.outputfile)
-    res = find_files(args.archive)
+    with open(args.outputfile, 'wb') as fh:
-    with open(out_file, 'wb') as fh:
         # Experimentation revealed that level 5 is significantly faster and has
         # marginally larger sizes than higher values and is the sweet spot
         # for optimal compression. Read the detailed commit message that
         # introduced this for raw numbers.
-        if out_file.endswith('.tar.gz'):
+        with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
-            files = dict(res)
+            res = find_files(args.archive)
-            create_tar_gz_from_files(fh, files, compresslevel=5)
+            for p, f in res:
-            file_count = len(files)
+                writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True)
-        elif out_file.endswith('.zip'):
+                file_count += 1
-            with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
-                for p, f in res:
-                    writer.add(p.encode('utf-8'), f.read(), mode=f.mode,
-                               skip_duplicates=True)
-                    file_count += 1
-        else:
-            raise Exception('unhandled file extension: %s' % out_file)
 
     duration = time.time() - t_start
     zip_size = os.path.getsize(args.outputfile)

python/mozbuild/mozpack/archive.py

@@ -6,13 +6,9 @@ from __future__ import absolute_import
 
 import bz2
 import gzip
-import io
 import stat
 import tarfile
 
-from .files import (
-    BaseFile,
-)
 
 # 2016-01-01T00:00:00+0000
 DEFAULT_MTIME = 1451606400
@@ -22,26 +18,22 @@ def create_tar_from_files(fp, files):
     """Create a tar file deterministically.
 
     Receives a dict mapping names of files in the archive to local filesystem
-    paths or ``mozpack.files.BaseFile`` instances.
+    paths.
 
     The files will be archived and written to the passed file handle opened
     for writing.
 
     Only regular files can be written.
 
+    FUTURE accept mozpack.files classes for writing
     FUTURE accept a filename argument (or create APIs to write files)
     """
     with tarfile.open(name='', mode='w', fileobj=fp, dereference=True) as tf:
-        for archive_path, f in sorted(files.items()):
+        for archive_path, fs_path in sorted(files.items()):
-            if isinstance(f, BaseFile):
+            ti = tf.gettarinfo(fs_path, archive_path)
-                ti = tarfile.TarInfo(archive_path)
-                ti.mode = f.mode or 0644
-                ti.type = tarfile.REGTYPE
-            else:
-                ti = tf.gettarinfo(f, archive_path)
 
             if not ti.isreg():
-                raise ValueError('not a regular file: %s' % f)
+                raise ValueError('not a regular file: %s' % fs_path)
 
             # Disallow setuid and setgid bits. This is an arbitrary restriction.
             # However, since we set uid/gid to root:root, setuid and setgid
@@ -49,7 +41,7 @@ def create_tar_from_files(fp, files):
             # uncompressed as root.
             if ti.mode & (stat.S_ISUID | stat.S_ISGID):
                 raise ValueError('cannot add file with setuid or setgid set: '
-                                 '%s' % f)
+                                 '%s' % fs_path)
 
             # Set uid, gid, username, and group as deterministic values.
             ti.uid = 0
@@ -60,14 +52,8 @@ def create_tar_from_files(fp, files):
             # Set mtime to a constant value.
             ti.mtime = DEFAULT_MTIME
 
-            if isinstance(f, BaseFile):
+            with open(fs_path, 'rb') as fh:
-                ti.size = f.size()
+                tf.addfile(ti, fh)
-                # tarfile wants to pass a size argument to read(). So just
-                # wrap/buffer in a proper file object interface.
-                tf.addfile(ti, f.open())
-            else:
-                with open(f, 'rb') as fh:
-                    tf.addfile(ti, fh)
 
 
 def create_tar_gz_from_files(fp, files, filename=None, compresslevel=9):

python/mozbuild/mozpack/files.py

@@ -219,14 +219,6 @@ class BaseFile(object):
     def read(self):
         raise NotImplementedError('BaseFile.read() not implemented. Bug 1170329.')
 
-    def size(self):
-        """Returns size of the entry.
-
-        Derived classes are highly encouraged to override this with a more
-        optimal implementation.
-        """
-        return len(self.read())
-
     @property
     def mode(self):
         '''
@@ -258,9 +250,6 @@ class File(BaseFile):
         with open(self.path, 'rb') as fh:
             return fh.read()
 
-    def size(self):
-        return os.stat(self.path).st_size
-
 
 class ExecutableFile(File):
     '''
@@ -508,12 +497,6 @@ class GeneratedFile(BaseFile):
     def open(self):
         return BytesIO(self.content)
 
-    def read(self):
-        return self.content
-
-    def size(self):
-        return len(self.content)
-
 
 class DeflatedFile(BaseFile):
     '''

python/mozbuild/mozpack/test/test_archive.py

@@ -18,9 +18,6 @@ from mozpack.archive import (
     create_tar_gz_from_files,
     create_tar_bz2_from_files,
 )
-from mozpack.files import (
-    GeneratedFile,
-)
 
 from mozunit import main
 
@@ -44,22 +41,19 @@ class TestArchive(unittest.TestCase):
     def _create_files(self, root):
         files = {}
         for i in range(10):
-            p = os.path.join(root, b'file%02d' % i)
+            p = os.path.join(root, b'file%d' % i)
             with open(p, 'wb') as fh:
-                fh.write(b'file%02d' % i)
+                fh.write(b'file%d' % i)
             # Need to set permissions or umask may influence testing.
             os.chmod(p, MODE_STANDARD)
-            files[b'file%02d' % i] = p
+            files[b'file%d' % i] = p
-
-        for i in range(10):
-            files[b'file%02d' % (i + 10)] = GeneratedFile('file%02d' % (i + 10))
 
         return files
 
     def _verify_basic_tarfile(self, tf):
-        self.assertEqual(len(tf.getmembers()), 20)
+        self.assertEqual(len(tf.getmembers()), 10)
 
-        names = ['file%02d' % i for i in range(20)]
+        names = ['file%d' % i for i in range(10)]
         self.assertEqual(tf.getnames(), names)
 
         for ti in tf.getmembers():
@@ -112,7 +106,7 @@ class TestArchive(unittest.TestCase):
                 create_tar_from_files(fh, files)
 
             # Output should be deterministic.
-            self.assertEqual(file_hash(tp), '01cd314e277f060e98c7de6c8ea57f96b3a2065c')
+            self.assertEqual(file_hash(tp), 'cd16cee6f13391abd94dfa435d2633b61ed727f1')
 
             with tarfile.open(tp, 'r') as tf:
                 self._verify_basic_tarfile(tf)
@@ -150,7 +144,7 @@ class TestArchive(unittest.TestCase):
             with open(gp, 'wb') as fh:
                 create_tar_gz_from_files(fh, files)
 
-            self.assertEqual(file_hash(gp), '7c4da5adc5088cdf00911d5daf9a67b15de714b7')
+            self.assertEqual(file_hash(gp), 'acb602239c1aeb625da5e69336775609516d60f5')
 
             with tarfile.open(gp, 'r:gz') as tf:
                 self._verify_basic_tarfile(tf)
@@ -167,7 +161,7 @@ class TestArchive(unittest.TestCase):
             with open(gp, 'wb') as fh:
                 create_tar_gz_from_files(fh, files, filename='foobar', compresslevel=1)
 
-            self.assertEqual(file_hash(gp), '1cc8b96f0262350977c2e9d61f40a1fa76f35c52')
+            self.assertEqual(file_hash(gp), 'fd099f96480cc1100f37baa8e89a6b820dbbcbd3')
 
             with tarfile.open(gp, 'r:gz') as tf:
                 self._verify_basic_tarfile(tf)
@@ -184,7 +178,7 @@ class TestArchive(unittest.TestCase):
             with open(bp, 'wb') as fh:
                 create_tar_bz2_from_files(fh, files)
 
-            self.assertEqual(file_hash(bp), 'eb5096d2fbb71df7b3d690001a6f2e82a5aad6a7')
+            self.assertEqual(file_hash(bp), '1827ad00dfe7acf857b7a1c95ce100361e3f6eea')
 
             with tarfile.open(bp, 'r:bz2') as tf:
                 self._verify_basic_tarfile(tf)

taskcluster/taskgraph/transforms/beetmover.py

@@ -33,7 +33,7 @@ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
     "target.awsy.tests.zip",
     "target.test_packages.json",
     "target.txt",
-    "target.web-platform.tests.tar.zip",
+    "target.web-platform.tests.zip",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "target.jsshell.zip",
@@ -64,7 +64,7 @@ _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
     "en-US/target.awsy.tests.zip",
     "en-US/target.test_packages.json",
     "en-US/target.txt",
-    "en-US/target.web-platform.tests.tar.gz",
+    "en-US/target.web-platform.tests.zip",
     "en-US/target.xpcshell.tests.zip",
     "en-US/target_info.txt",
     "en-US/bouncer.apk",
@@ -84,7 +84,7 @@ _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI = [
     "target.awsy.tests.zip",
     "target.test_packages.json",
     "target.txt",
-    "target.web-platform.tests.tar.gz",
+    "target.web-platform.tests.zip",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "bouncer.apk",

taskcluster/taskgraph/transforms/beetmover_repackage.py

@@ -35,7 +35,7 @@ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
     "target.awsy.tests.zip",
     "target.test_packages.json",
     "target.txt",
-    "target.web-platform.tests.tar.gz",
+    "target.web-platform.tests.zip",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "target.jsshell.zip",

testing/testsuite-targets.mk

@@ -119,26 +119,23 @@ ifdef COMPILE_ENVIRONMENT
 stage-all: stage-cppunittests
 endif
 
-TEST_PKGS_ZIP := \
+TEST_PKGS := \
   common \
   cppunittest \
   mochitest \
   reftest \
   talos \
   awsy \
-  xpcshell \
-  $(NULL)
-
-TEST_PKGS_TARGZ := \
   web-platform \
+  xpcshell \
   $(NULL)
 
 ifdef LINK_GTEST_DURING_COMPILE
 stage-all: stage-gtest
-TEST_PKGS_ZIP += gtest
+TEST_PKGS += gtest
 endif
 
-PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.$(2)'
+PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.zip'
 
 test-packages-manifest:
 	@rm -f $(MOZ_TEST_PACKAGES_FILE)
@@ -146,9 +143,8 @@ test-packages-manifest:
 	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
       --jsshell $(JSSHELL_NAME) \
       --dest-file '$(MOZ_TEST_PACKAGES_FILE)' \
-      $(call PKG_ARG,common,zip) \
+      $(call PKG_ARG,common) \
-      $(foreach pkg,$(TEST_PKGS_ZIP),$(call PKG_ARG,$(pkg),zip)) \
+      $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
-      $(foreach pkg,$(TEST_PKGS_TARGZ),$(call PKG_ARG,$(pkg),tar.gz))
 
 package-tests-prepare-dest:
 	@rm -f '$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)'
@@ -158,12 +154,11 @@ define package_archive
 package-tests-$(1): stage-all package-tests-prepare-dest
 	$$(call py_action,test_archive, \
 		$(1) \
-		'$$(abspath $$(DIST))/$$(PKG_PATH)/$$(PKG_BASENAME).$(1).tests.$(2)')
+		'$$(abspath $$(DIST))/$$(PKG_PATH)/$$(PKG_BASENAME).$(1).tests.zip')
 package-tests: package-tests-$(1)
 endef
 
-$(foreach name,$(TEST_PKGS_ZIP),$(eval $(call package_archive,$(name),zip)))
+$(foreach name,$(TEST_PKGS),$(eval $(call package_archive,$(name))))
-$(foreach name,$(TEST_PKGS_TARGZ),$(eval $(call package_archive,$(name),tar.gz)))
 
 ifeq ($(MOZ_BUILD_APP),mobile/android)
 stage-all: stage-android

toolkit/mozapps/installer/package-name.mk

@@ -92,7 +92,7 @@ CPP_TEST_PACKAGE = $(PKG_BASENAME).cppunittest.tests.zip
 XPC_TEST_PACKAGE = $(PKG_BASENAME).xpcshell.tests.zip
 MOCHITEST_PACKAGE = $(PKG_BASENAME).mochitest.tests.zip
 REFTEST_PACKAGE = $(PKG_BASENAME).reftest.tests.zip
-WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.tar.gz
+WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.zip
 TALOS_PACKAGE = $(PKG_BASENAME).talos.tests.zip
 AWSY_PACKAGE = $(PKG_BASENAME).awsy.tests.zip
 GTEST_PACKAGE = $(PKG_BASENAME).gtest.tests.zip