Bug 1359965 - Support and generate tar.gz WPT archive; r=glandium

Several years ago there was a single zip file for all test files. Clients
would only extract the files they needed. Thus, zip was a reasonable
archive format because it allowed direct access to members without
having to decompress the entirety of the stream.

We have since split up that monolithic archive into separate,
domain-specific archives. e.g. 1 archive for mochitests and one
for xpcshell tests. This drastically cut down on network I/O
required on testers because they only fetched archives/data that
was relevant. It also enabled parallel generation of test archives,
we shaved dozens of seconds off builds due to compression being
a long pole.

Despite the architectural changes to test archive management, we
still used zip files. This is not ideal because we no longer access
specific files in test archives and thus don't care about single/partial
member access performance.

This commit implements support for generating tar.gz test archives.
And it switches the web-platform archive to a tar.gz file.

The performance implications for archive generation are significant:

before: 48,321,250 bytes; 6.05s
after:  31,844,267 bytes; 4.57s

The size is reduced because we have a single compression context
so data from 1 file can benefit compression in a subsequent file.
CPU usage is reduced because the compressor has to work less with
1 context than it does with N. While I didn't measure it, decompression
performance should also be improved for the same reasons. And of course
network I/O will be reduced.

mozharness consumers use a generic method for handling unarchiving.
This method automagically handles multiple file extensions. So as long
as downstream consumers aren't hard coding ".zip" this change should
"just work."

MozReview-Commit-ID: LQa5MIHLsms

--HG--
extra : rebase_source : 100092c2f2ff609362a724fff60f46dd6e49c94e
extra : intermediate-source : d10f5ccd882b965fcad39914f7c3c930d1301a41
extra : source : a0e257e346ccf3c1db332ec5903241f4eeb9a7ee

python/mozbuild/mozbuild/action/test_archive.py

@@ -20,6 +20,7 @@ from manifestparser import TestManifest
 from reftest import ReftestManifest
 
 from mozbuild.util import ensureParentDir
+from mozpack.archive import create_tar_gz_from_files
 from mozpack.copier import FileRegistry
 from mozpack.files import ExistingFile, FileFinder
 from mozpack.manifests import InstallManifest
@@ -614,22 +615,31 @@ def main(argv):
 
     args = parser.parse_args(argv)
 
-    if not args.outputfile.endswith('.zip'):
-        raise Exception('expected zip output file')
+    out_file = args.outputfile
+    if not out_file.endswith(('.tar.gz', '.zip')):
+        raise Exception('expected tar.gz or zip output file')
 
     file_count = 0
     t_start = time.time()
-    ensureParentDir(args.outputfile)
-    with open(args.outputfile, 'wb') as fh:
+    ensureParentDir(out_file)
+    res = find_files(args.archive)
+    with open(out_file, 'wb') as fh:
         # Experimentation revealed that level 5 is significantly faster and has
         # marginally larger sizes than higher values and is the sweet spot
         # for optimal compression. Read the detailed commit message that
         # introduced this for raw numbers.
-        with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
-            res = find_files(args.archive)
-            for p, f in res:
-                writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True)
-                file_count += 1
+        if out_file.endswith('.tar.gz'):
+            files = dict(res)
+            create_tar_gz_from_files(fh, files, compresslevel=5)
+            file_count = len(files)
+        elif out_file.endswith('.zip'):
+            with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
+                for p, f in res:
+                    writer.add(p.encode('utf-8'), f.read(), mode=f.mode,
+                               skip_duplicates=True)
+                    file_count += 1
+        else:
+            raise Exception('unhandled file extension: %s' % out_file)
 
     duration = time.time() - t_start
     zip_size = os.path.getsize(args.outputfile)

taskcluster/taskgraph/transforms/beetmover.py

@@ -32,7 +32,7 @@ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
     "target.awsy.tests.zip",
     "target.test_packages.json",
     "target.txt",
-    "target.web-platform.tests.zip",
+    "target.web-platform.tests.tar.gz",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "target.jsshell.zip",
@@ -79,7 +79,7 @@ _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
     "en-US/target.awsy.tests.zip",
     "en-US/target.test_packages.json",
     "en-US/target.txt",
-    "en-US/target.web-platform.tests.zip",
+    "en-US/target.web-platform.tests.tar.gz",
     "en-US/target.xpcshell.tests.zip",
     "en-US/target_info.txt",
     "en-US/bouncer.apk",
@@ -103,7 +103,7 @@ _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI = [
     "target.awsy.tests.zip",
     "target.test_packages.json",
     "target.txt",
-    "target.web-platform.tests.zip",
+    "target.web-platform.tests.tar.gz",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "bouncer.apk",

taskcluster/taskgraph/transforms/beetmover_repackage.py

@@ -34,7 +34,7 @@ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
     "target.awsy.tests.zip",
     "target.test_packages.json",
     "target.txt",
-    "target.web-platform.tests.zip",
+    "target.web-platform.tests.tar.gz",
     "target.xpcshell.tests.zip",
     "target_info.txt",
     "target.jsshell.zip",

testing/testsuite-targets.mk

@@ -119,23 +119,26 @@ ifdef COMPILE_ENVIRONMENT
 stage-all: stage-cppunittests
 endif
 
-TEST_PKGS := \
+TEST_PKGS_ZIP := \
   common \
   cppunittest \
   mochitest \
   reftest \
   talos \
   awsy \
-  web-platform \
   xpcshell \
   $(NULL)
 
+TEST_PKGS_TARGZ := \
+  web-platform \
+  $(NULL)
+
 ifdef LINK_GTEST_DURING_COMPILE
 stage-all: stage-gtest
-TEST_PKGS += gtest
+TEST_PKGS_ZIP += gtest
 endif
 
-PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.zip'
+PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.$(2)'
 
 test-packages-manifest:
 	@rm -f $(MOZ_TEST_PACKAGES_FILE)
@@ -143,8 +146,9 @@ test-packages-manifest:
 	$(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
       --jsshell $(JSSHELL_NAME) \
       --dest-file '$(MOZ_TEST_PACKAGES_FILE)' \
-      $(call PKG_ARG,common) \
-      $(foreach pkg,$(TEST_PKGS),$(call PKG_ARG,$(pkg)))
+      $(call PKG_ARG,common,zip) \
+      $(foreach pkg,$(TEST_PKGS_ZIP),$(call PKG_ARG,$(pkg),zip)) \
+      $(foreach pkg,$(TEST_PKGS_TARGZ),$(call PKG_ARG,$(pkg),tar.gz))
 
 package-tests-prepare-dest:
 	@rm -f '$(DIST)/$(PKG_PATH)$(TEST_PACKAGE)'
@@ -154,11 +158,12 @@ define package_archive
 package-tests-$(1): stage-all package-tests-prepare-dest
 	$$(call py_action,test_archive, \
 		$(1) \
-		'$$(abspath $$(DIST))/$$(PKG_PATH)/$$(PKG_BASENAME).$(1).tests.zip')
+		'$$(abspath $$(DIST))/$$(PKG_PATH)/$$(PKG_BASENAME).$(1).tests.$(2)')
 package-tests: package-tests-$(1)
 endef
 
-$(foreach name,$(TEST_PKGS),$(eval $(call package_archive,$(name))))
+$(foreach name,$(TEST_PKGS_ZIP),$(eval $(call package_archive,$(name),zip)))
+$(foreach name,$(TEST_PKGS_TARGZ),$(eval $(call package_archive,$(name),tar.gz)))
 
 ifeq ($(MOZ_BUILD_APP),mobile/android)
 stage-all: stage-android

toolkit/mozapps/installer/package-name.mk

@@ -92,7 +92,7 @@ CPP_TEST_PACKAGE = $(PKG_BASENAME).cppunittest.tests.zip
 XPC_TEST_PACKAGE = $(PKG_BASENAME).xpcshell.tests.zip
 MOCHITEST_PACKAGE = $(PKG_BASENAME).mochitest.tests.zip
 REFTEST_PACKAGE = $(PKG_BASENAME).reftest.tests.zip
-WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.zip
+WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.tar.gz
 TALOS_PACKAGE = $(PKG_BASENAME).talos.tests.zip
 AWSY_PACKAGE = $(PKG_BASENAME).awsy.tests.zip
 GTEST_PACKAGE = $(PKG_BASENAME).gtest.tests.zip