Bug 1484012: [fetch-content] Add an option to not unpack downloaded artifacts; r=gps

Differential Revision: https://phabricator.services.mozilla.com/D3554

--HG--
extra : rebase_source : 58bba31bd921d29d4a40ad8d9ba09c4c7ac1f8dc

taskcluster/scripts/misc/fetch-content

@@ -218,7 +218,7 @@ def extract_archive(path, dest_dir, typ):
     log('%s extracted in %.3fs' % (path, time.time() - t0))
 
 
-def fetch_and_extract(url, dest_dir, sha256=None, size=None):
+def fetch_and_extract(url, dest_dir, extract=True, sha256=None, size=None):
     """Fetch a URL and extract it to a destination path.
 
     If the downloaded URL is an archive, it is extracted automatically
@@ -231,6 +231,9 @@ def fetch_and_extract(url, dest_dir, sha256=None, size=None):
 
     download_to_path(url, dest_path, sha256=sha256, size=size)
 
+    if not extract:
+        return
+
     typ = archive_type(dest_path)
     if typ:
         extract_archive(dest_path, dest_dir, typ)
@@ -290,7 +293,7 @@ def command_task_artifacts(args):
         extdir.mkdir(parents=True, exist_ok=True)
         url = ARTIFACT_URL.format(task=fetch['task'],
                                   artifact=fetch['artifact'])
-        downloads.append((url, extdir))
+        downloads.append((url, extdir, fetch['extract']))
 
     fetch_urls(downloads)
 

taskcluster/taskgraph/transforms/job/__init__.py

@@ -84,6 +84,7 @@ job_description_schema = Schema({
         basestring: [basestring, {
             Required('artifact'): basestring,
             Optional('dest'): basestring,
+            Optional('extract'): bool,
         }],
     },
 
@@ -189,6 +190,7 @@ def use_fetches(config, jobs):
                     job_fetches.append({
                         'artifact': path,
                         'task': '<{dep}>'.format(dep=dep),
+                        'extract': True,
                     })
 
             else:
@@ -200,13 +202,16 @@ def use_fetches(config, jobs):
                     if isinstance(artifact, basestring):
                         path = artifact
                         dest = None
+                        extract = True
                     else:
                         path = artifact['artifact']
                         dest = artifact.get('dest')
+                        extract = artifact.get('extract', True)
 
                     fetch = {
                         'artifact': '{prefix}/{path}'.format(prefix=prefix, path=path),
                         'task': '<{dep}>'.format(dep=kind),
+                        'extract': extract,
                     }
                     if dest is not None:
                         fetch['dest'] = dest

testing/mozharness/mozharness/mozilla/fetches.py

@@ -56,5 +56,6 @@ class FetchesMixin(object):
 
             path = self.download_file(url, parent_dir=extdir)
 
-            mozfile.extract(path, extdir)
-            os.remove(path)
+            if fetch['extract']:
+                mozfile.extract(path, extdir)
+                os.remove(path)