Backed out 2 changesets (bug 1258539) for mozharness failures a=backout

Backed out changeset 8322ffecd9d9 (bug 1258539)
Backed out changeset cc2996a53b71 (bug 1258539)

testing/mozharness/mozharness/base/script.py

@@ -15,11 +15,8 @@ import codecs
 from contextlib import contextmanager
 import datetime
 import errno
-import fnmatch
-import functools
 import gzip
 import inspect
-import itertools
 import os
 import platform
 import pprint
@@ -28,11 +25,9 @@ import shutil
 import socket
 import subprocess
 import sys
-import tarfile
 import time
 import traceback
 import urllib2
-import zipfile
 import httplib
 import urlparse
 import hashlib
@@ -52,10 +47,10 @@ except ImportError:
 
 from mozprocess import ProcessHandler
 from mozharness.base.config import BaseConfig
+from mozharness.base.errors import ZipErrorList
 from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \
     LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL
 
-
 def platform_name():
     pm = PlatformMixin()
 
@@ -458,28 +453,39 @@ class ScriptMixin(PlatformMixin):
             **retry_args
         )
 
-    def download_unpack(self, url, extract_to, extract_dirs=None,
-                        error_level=FATAL):
-        """Generic method to download and extract a compressed file.
+    def download_unzip(self, url, parent_dir, target_unzip_dirs=None, halt_on_failure=True):
+        """Generic method to download and extract a zip file.
 
         The downloaded file will always be saved to the working directory and is not getting
         deleted after extracting.
 
         Args:
             url (str): URL where the file to be downloaded is located.
-            extract_to (str): directory where the downloaded file will
+            parent_dir (str): directory where the downloaded file will
                               be extracted to.
-            extract_dirs (list, optional): directories inside the archive to extract.
-                                           Defaults to `None`.
-            error_level (str, optional): log level to use in case an error occurs.
-                                         Defaults to `FATAL`.
+            target_unzip_dirs (list, optional): directories inside the zip file to extract.
+                                                Defaults to `None`.
+            halt_on_failure (bool, optional): whether or not to redefine the
+                                              log level as `FATAL` on errors. Defaults to True.
 
         """
         dirs = self.query_abs_dirs()
-        archive = self.download_file(url, parent_dir=dirs['abs_work_dir'],
-                                     error_level=error_level)
-        self.unpack(archive, extract_to, extract_dirs=extract_dirs,
-                    error_level=error_level)
+        zipfile = self.download_file(url, parent_dir=dirs['abs_work_dir'],
+                                     error_level=FATAL)
+
+        command = self.query_exe('unzip', return_type='list')
+        # Always overwrite to not get an input in a hidden pipe if files already exist
+        command.extend(['-q', '-o', zipfile, '-d', parent_dir])
+        if target_unzip_dirs:
+            command.extend(target_unzip_dirs)
+        # TODO error_list: http://www.info-zip.org/mans/unzip.html#DIAGNOSTICS
+        # unzip return code 11 is 'no matching files were found'
+        self.run_command(command,
+                         error_list=ZipErrorList,
+                         halt_on_failure=halt_on_failure,
+                         fatal_exit_code=3,
+                         success_codes=[0, 11],
+                         )
 
     def load_json_url(self, url, error_level=None, *args, **kwargs):
         """ Returns a json object from a url (it retries). """
@@ -1099,7 +1105,7 @@ class ScriptMixin(PlatformMixin):
             output_timeout (int): amount of seconds to wait for output before
               the process is killed.
             fatal_exit_code (int, optional): call `self.fatal` if the return value
-              of the command is not in `success_codes`. Defaults to 2.
+              of the command is not on in `success_codes`. Defaults to 2.
             error_level (str, optional): log level name to use on error. Defaults
               to `ERROR`.
             **kwargs: Arbitrary keyword arguments.
@@ -1391,69 +1397,26 @@ class ScriptMixin(PlatformMixin):
                 self.log(msg, error_level=error_level)
         os.utime(file_name, times)
 
-    def unpack(self, filename, extract_to, extract_dirs=None,
-               error_level=ERROR, fatal_exit_code=2, verbose=False):
-        """The method allows to extract a file regardless of its extension.
+    def unpack(self, filename, extract_to):
+        '''
+        This method allows us to extract a file regardless of its extension
 
         Args:
             filename (str): filename of the compressed file.
             extract_to (str): where to extract the compressed file.
-            extract_dirs (list, optional): directories inside the archive file to extract.
-                                           Defaults to `None`.
-            fatal_exit_code (int, optional): call `self.fatal` if the return value
-              of the command is not in `success_codes`. Defaults to 2.
-            verbose (bool, optional): whether or not extracted content should be displayed.
-                                      Defaults to False.
-
-        Raises:
-            IOError: on `filename` file not found.
-
-        """
-        def _filter_entries(namelist):
-            """Filter entries of the archive based on the specified list of to extract dirs."""
-            filter_partial = functools.partial(fnmatch.filter, namelist)
-            for entry in itertools.chain(*map(filter_partial, extract_dirs or ['*'])):
-                yield entry
-
-        if not os.path.isfile(filename):
-            raise IOError('Could not find file to extract: %s' % filename)
-
-        if zipfile.is_zipfile(filename):
-            try:
-                self.info('Using ZipFile to extract {} to {}'.format(filename, extract_to))
-                with zipfile.ZipFile(filename) as bundle:
-                    for entry in _filter_entries(bundle.namelist()):
-                        if verbose:
-                            self.info(' %s' % entry)
-                        bundle.extract(entry, path=extract_to)
-
-                        # ZipFile doesn't preserve permissions during extraction:
-                        # http://bugs.python.org/issue15795
-                        fname = os.path.realpath(os.path.join(extract_to, entry))
-                        mode = bundle.getinfo(entry).external_attr >> 16 & 0x1FF
-                        # Only set permissions if attributes are available. Otherwise all
-                        # permissions will be removed eg. on Windows.
-                        if mode:
-                            os.chmod(fname, mode)
-            except zipfile.BadZipfile as e:
-                self.log('%s (%s)' % (e.message, filename),
-                         level=error_level, exit_code=fatal_exit_code)
-
-        # Bug 1211882 - is_tarfile cannot be trusted for dmg files
-        elif tarfile.is_tarfile(filename) and not filename.lower().endswith('.dmg'):
-            try:
-                self.info('Using TarFile to extract {} to {}'.format(filename, extract_to))
-                with tarfile.open(filename) as bundle:
-                    for entry in _filter_entries(bundle.getnames()):
-                        if verbose:
-                            self.info(' %s' % entry)
-                        bundle.extract(entry, path=extract_to)
-            except tarfile.TarError as e:
-                self.log('%s (%s)' % (e.message, filename),
-                         level=error_level, exit_code=fatal_exit_code)
+        '''
+        # XXX: Make sure that filename has a extension of one of our supported file formats
+        m = re.search('\.tar\.(bz2|gz)$', filename)
+        if m:
+            command = self.query_exe('tar', return_type='list')
+            tar_cmd = "jxfv"
+            if m.group(1) == "gz":
+                tar_cmd = "zxfv"
+            command.extend([tar_cmd, filename, "-C", extract_to])
+            self.run_command(command, halt_on_failure=True)
         else:
-            self.log('No extraction method found for: %s' % filename,
-                     level=error_level, exit_code=fatal_exit_code)
+            # XXX implement
+            pass
 
 
 def PreScriptRun(func):

testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py

@@ -163,14 +163,15 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
         harness, puppeteer, and tests from and how to set them up.
 
         """
-        extract_dirs = ['config/*',
-                        'external-media-tests/*',
-                        'marionette/*',
-                        'mozbase/*',
-                        'puppeteer/*',
-                        'tools/wptserve/*',
-                        ]
-        super(FirefoxMediaTestsBase, self).download_and_extract(extract_dirs=extract_dirs)
+        target_unzip_dirs = ['config/*',
+                             'external-media-tests/*',
+                             'marionette/*',
+                             'mozbase/*',
+                             'puppeteer/*',
+                             'tools/wptserve/*',
+                             ]
+        super(FirefoxMediaTestsBase, self).download_and_extract(
+                target_unzip_dirs=target_unzip_dirs)
 
     def query_abs_dirs(self):
         if self.abs_dirs:

testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py

@@ -153,14 +153,14 @@ class FirefoxUITests(TestingMixin, VCSToolsScript):
         harness, puppeteer, and tests from and how to set them up.
 
         """
-        extract_dirs = ['config/*',
-                        'firefox-ui/*',
-                        'marionette/*',
-                        'mozbase/*',
-                        'puppeteer/*',
-                        'tools/wptserve/*',
-                        ]
-        super(FirefoxUITests, self).download_and_extract(extract_dirs=extract_dirs)
+        target_unzip_dirs = ['config/*',
+                             'firefox-ui/*',
+                             'marionette/*',
+                             'mozbase/*',
+                             'puppeteer/*',
+                             'tools/wptserve/*',
+                             ]
+        super(FirefoxUITests, self).download_and_extract(target_unzip_dirs=target_unzip_dirs)
 
     def query_abs_dirs(self):
         if self.abs_dirs:
@@ -299,6 +299,53 @@ class FirefoxUITests(TestingMixin, VCSToolsScript):
             env=self.query_env(),
         )
 
+    def download_unzip(self, url, parent_dir, target_unzip_dirs=None, halt_on_failure=True):
+        """Overwritten method from BaseScript until bug 1258539 is fixed.
+
+        The downloaded file will always be saved to the working directory and is not getting
+        deleted after extracting.
+
+        Args:
+            url (str): URL where the file to be downloaded is located.
+            parent_dir (str): directory where the downloaded file will
+                              be extracted to.
+            target_unzip_dirs (list, optional): directories inside the zip file to extract.
+                                                Defaults to `None`.
+            halt_on_failure (bool, optional): whether or not to redefine the
+                                              log level as `FATAL` on errors. Defaults to True.
+
+        """
+        import fnmatch
+        import itertools
+        import functools
+        import zipfile
+
+        def _filter_entries(namelist):
+            """Filter entries of the archive based on the specified list of extract_dirs."""
+            filter_partial = functools.partial(fnmatch.filter, namelist)
+            for entry in itertools.chain(*map(filter_partial, target_unzip_dirs or ['*'])):
+                yield entry
+
+        dirs = self.query_abs_dirs()
+        zip = self.download_file(url, parent_dir=dirs['abs_work_dir'],
+                                 error_level=FATAL)
+
+        try:
+            self.info('Using ZipFile to extract {0} to {1}'.format(zip, parent_dir))
+            with zipfile.ZipFile(zip) as bundle:
+                for entry in _filter_entries(bundle.namelist()):
+                    bundle.extract(entry, path=parent_dir)
+
+                    # ZipFile doesn't preserve permissions: http://bugs.python.org/issue15795
+                    fname = os.path.realpath(os.path.join(parent_dir, entry))
+                    mode = bundle.getinfo(entry).external_attr >> 16 & 0x1FF
+                    # Only set permissions if attributes are available.
+                    if mode:
+                        os.chmod(fname, mode)
+        except zipfile.BadZipfile as e:
+            self.log('{0} ({1})'.format(e.message, zip),
+                     level=FATAL, exit_code=2)
+
 
 class FirefoxUIFunctionalTests(FirefoxUITests):
 

testing/mozharness/mozharness/mozilla/testing/talos.py

@@ -287,13 +287,13 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
         if self.query_pagesets_url():
             self.info("Downloading pageset...")
             src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
-            self.download_unpack(self.pagesets_url, src_talos_pageset)
+            self.download_unzip(self.pagesets_url, src_talos_pageset)
 
     # Action methods. {{{1
     # clobber defined in BaseScript
     # read_buildbot_config defined in BuildbotMixin
 
-    def download_and_extract(self, extract_dirs=None, suite_categories=None):
+    def download_and_extract(self, target_unzip_dirs=None, suite_categories=None):
         return super(Talos, self).download_and_extract(
             suite_categories=['common', 'talos']
         )

testing/mozharness/mozharness/mozilla/testing/testbase.py

@@ -401,6 +401,17 @@ You can set this by:
         if message:
             self.fatal(message + "Can't run download-and-extract... exiting")
 
+        if self.config.get("developer_mode") and self._is_darwin():
+            # Bug 1066700 only affects Mac users that try to run mozharness locally
+            version = self._query_binary_version(
+                    regex=re.compile("UnZip\ (\d+\.\d+)\ .*", re.MULTILINE),
+                    cmd=[self.query_exe('unzip'), '-v']
+            )
+            if not version >= 6:
+                self.fatal("We require a more recent version of unzip to unpack our tests.zip files.\n"
+                        "You are currently using version %s. Please update to at least 6.0.\n"
+                        "You can visit http://www.info-zip.org/UnZip.html" % version)
+
     def _read_packages_manifest(self):
         dirs = self.query_abs_dirs()
         source = self.download_file(self.test_packages_url,
@@ -418,7 +429,7 @@ You can set this by:
                   pprint.pformat(package_requirements))
         return package_requirements
 
-    def _download_test_packages(self, suite_categories, extract_dirs):
+    def _download_test_packages(self, suite_categories, target_unzip_dirs):
         # Some platforms define more suite categories/names than others.
         # This is a difference in the convention of the configs more than
         # to how these tests are run, so we pave over these differences here.
@@ -454,21 +465,21 @@ You can set this by:
                       (target_packages, category))
             for file_name in target_packages:
                 target_dir = test_install_dir
-                unpack_dirs = extract_dirs
+                unzip_dirs = target_unzip_dirs
                 if "jsshell-" in file_name or file_name == "target.jsshell.zip":
                     self.info("Special-casing the jsshell zip file")
-                    unpack_dirs = None
+                    unzip_dirs = None
                     target_dir = dirs['abs_test_bin_dir']
                 url = self.query_build_dir_url(file_name)
-                self.download_unpack(url, target_dir,
-                                     extract_dirs=unpack_dirs)
+                self.download_unzip(url, target_dir,
+                                     target_unzip_dirs=unzip_dirs)
 
-    def _download_test_zip(self, extract_dirs=None):
+    def _download_test_zip(self, target_unzip_dirs=None):
         dirs = self.query_abs_dirs()
         test_install_dir = dirs.get('abs_test_install_dir',
                                     os.path.join(dirs['abs_work_dir'], 'tests'))
-        self.download_unpack(self.test_url, test_install_dir,
-                             extract_dirs=extract_dirs)
+        self.download_unzip(self.test_url, test_install_dir,
+                             target_unzip_dirs=target_unzip_dirs)
 
     def structured_output(self, suite_category):
         """Defines whether structured logging is in use in this configuration. This
@@ -515,9 +526,9 @@ You can set this by:
 
         self.set_buildbot_property("symbols_url", self.symbols_url,
                                    write_to_file=True)
-        self.download_unpack(self.symbols_url, self.symbols_path)
+        self.download_unzip(self.symbols_url, self.symbols_path)
 
-    def download_and_extract(self, extract_dirs=None, suite_categories=None):
+    def download_and_extract(self, target_unzip_dirs=None, suite_categories=None):
         """
         download and extract test zip / download installer
         """
@@ -540,7 +551,7 @@ You can set this by:
                            ' package data at "%s" will be ignored.' %
                            (self.config.get('test_url'), self.test_packages_url))
 
-            self._download_test_zip(extract_dirs)
+            self._download_test_zip(target_unzip_dirs)
         else:
             if not self.test_packages_url:
                 # The caller intends to download harness specific packages, but doesn't know
@@ -550,7 +561,7 @@ You can set this by:
                 self.test_packages_url = self.query_prefixed_build_dir_url('.test_packages.json')
 
             suite_categories = suite_categories or ['common']
-            self._download_test_packages(suite_categories, extract_dirs)
+            self._download_test_packages(suite_categories, target_unzip_dirs)
 
         self._download_installer()
         if self.config.get('download_symbols'):

testing/mozharness/mozharness/mozilla/testing/unittest.py

@@ -235,7 +235,7 @@ class EmulatorMixin(object):
         dirs = self.query_abs_dirs()
         self.mkdir_p(dirs['abs_emulator_dir'])
         if self.config.get('emulator_url'):
-            self.download_unpack(self.config['emulator_url'], dirs['abs_emulator_dir'])
+            self.download_unzip(self.config['emulator_url'], dirs['abs_emulator_dir'])
         elif self.config.get('emulator_manifest'):
             manifest_path = self.create_tooltool_manifest(self.config['emulator_manifest'])
             do_unzip = True

testing/mozharness/scripts/b2g_emulator_unittest.py

@@ -211,7 +211,8 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
                          error_list=TarErrorList,
                          halt_on_failure=True, fatal_exit_code=3)
 
-        self.download_unpack(self.config['xre_url'], dirs['abs_xre_dir'])
+        self.download_unzip(self.config['xre_url'],
+                             dirs['abs_xre_dir'])
 
         if self.config.get('busybox_url'):
             self.download_file(self.config['busybox_url'],

testing/mozharness/scripts/desktop_unittest.py

@@ -479,20 +479,20 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
         """
         c = self.config
 
-        extract_dirs = None
+        target_unzip_dirs = None
         if c['specific_tests_zip_dirs']:
-            extract_dirs = list(c['minimum_tests_zip_dirs'])
+            target_unzip_dirs = list(c['minimum_tests_zip_dirs'])
             for category in c['specific_tests_zip_dirs'].keys():
                 if c['run_all_suites'] or self._query_specified_suites(category) \
                         or 'run-tests' not in self.actions:
-                    extract_dirs.extend(c['specific_tests_zip_dirs'][category])
+                    target_unzip_dirs.extend(c['specific_tests_zip_dirs'][category])
 
         if c.get('run_all_suites'):
             target_categories = SUITE_CATEGORIES
         else:
             target_categories = [cat for cat in SUITE_CATEGORIES
                                  if self._query_specified_suites(cat) is not None]
-        super(DesktopUnittest, self).download_and_extract(extract_dirs=extract_dirs,
+        super(DesktopUnittest, self).download_and_extract(target_unzip_dirs=target_unzip_dirs,
                                                           suite_categories=target_categories)
 
     def stage_files(self):

testing/mozharness/scripts/web_platform_tests.py

@@ -166,12 +166,12 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin):
 
     def download_and_extract(self):
         super(WebPlatformTest, self).download_and_extract(
-            extract_dirs=["bin/*",
-                          "config/*",
-                          "mozbase/*",
-                          "marionette/*",
-                          "tools/wptserve/*",
-                          "web-platform/*"],
+            target_unzip_dirs=["bin/*",
+                               "config/*",
+                               "mozbase/*",
+                               "marionette/*",
+                               "tools/wptserve/*",
+                               "web-platform/*"],
             suite_categories=["web-platform"])
 
     def run_tests(self):

testing/mozharness/test/helper_files/archives/reference/bin/script.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-echo Hello world!

testing/mozharness/test/helper_files/archives/reference/lorem.txt

@@ -1 +0,0 @@
-Lorem ipsum dolor sit amet.

testing/mozharness/test/helper_files/create_archives.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-# Script to auto-generate the different archive types under the archives directory.
-
-cd archives
-
-rm archive.*
-
-tar cf archive.tar -C reference .
-gzip -fk archive.tar >archive.tar.gz
-bzip2 -fk archive.tar >archive.tar.bz2
-cd reference && zip ../archive.zip -r * && cd ..

testing/mozharness/test/test_base_script.py

@@ -2,8 +2,6 @@ import gc
 import mock
 import os
 import re
-import shutil
-import tempfile
 import types
 import unittest
 PYWIN32 = False
@@ -21,9 +19,6 @@ from mozharness.base.log import DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL, IG
 import mozharness.base.script as script
 from mozharness.base.config import parse_config_file
 
-
-here = os.path.dirname(os.path.abspath(__file__))
-
 test_string = '''foo
 bar
 baz'''
@@ -36,12 +31,10 @@ class CleanupObj(script.ScriptMixin, log.LogMixin):
         self.config = {'log_level': ERROR}
 
 
-def cleanup(files=None):
-    files = files or []
-    files.extend(('test_logs', 'test_dir', 'tmpfile_stdout', 'tmpfile_stderr'))
+def cleanup():
     gc.collect()
     c = CleanupObj()
-    for f in files:
+    for f in ('test_logs', 'test_dir', 'tmpfile_stdout', 'tmpfile_stderr'):
         c.rmtree(f)
 
 
@@ -63,13 +56,12 @@ class TestScript(unittest.TestCase):
     def setUp(self):
         cleanup()
         self.s = None
-        self.tmpdir = tempfile.mkdtemp(suffix='.mozharness')
 
     def tearDown(self):
         # Close the logfile handles, or windows can't remove the logs
         if hasattr(self, 's') and isinstance(self.s, object):
             del(self.s)
-        cleanup([self.tmpdir])
+        cleanup()
 
     # test _dump_config_hierarchy() when --dump-config-hierarchy is passed
     def test_dump_config_hierarchy_valid_files_len(self):
@@ -259,38 +251,6 @@ class TestScript(unittest.TestCase):
         self.assertTrue(error_logsize > 0,
                         msg="error list not working properly")
 
-    def test_unpack(self):
-        self.s = get_debug_script_obj()
-
-        archives_path = os.path.join(here, 'helper_files', 'archives')
-
-        # Test basic decompression
-        for archive in ('archive.tar', 'archive.tar.bz2', 'archive.tar.gz', 'archive.zip'):
-            self.s.unpack(os.path.join(archives_path, archive), self.tmpdir)
-            self.assertIn('script.sh', os.listdir(os.path.join(self.tmpdir, 'bin')))
-            self.assertIn('lorem.txt', os.listdir(self.tmpdir))
-            shutil.rmtree(self.tmpdir)
-
-        # Test permissions for extracted entries from zip archive
-        self.s.unpack(os.path.join(archives_path, 'archive.zip'), self.tmpdir)
-        file_stats = os.stat(os.path.join(self.tmpdir, 'bin', 'script.sh'))
-        orig_fstats = os.stat(os.path.join(archives_path, 'reference', 'bin', 'script.sh'))
-        self.assertEqual(file_stats.st_mode, orig_fstats.st_mode)
-        shutil.rmtree(self.tmpdir)
-
-        # Test extract specific dirs only
-        self.s.unpack(os.path.join(archives_path, 'archive.zip'), self.tmpdir,
-                      extract_dirs=['bin/*'])
-        self.assertIn('bin', os.listdir(self.tmpdir))
-        self.assertNotIn('lorem.txt', os.listdir(self.tmpdir))
-        shutil.rmtree(self.tmpdir)
-
-        # Test for invalid filenames (Windows only)
-        if PYWIN32:
-            with self.assertRaises(IOError):
-                self.s.unpack(os.path.join(archives_path, 'archive_invalid_filename.zip'),
-                              self.tmpdir)
-
 
 # TestHelperFunctions {{{1
 class TestHelperFunctions(unittest.TestCase):