Makes sure compatibility tests are run on all tiers (bug 657840)

apps/devhub/tasks.py

@@ -50,7 +50,7 @@ def file_validator(file_id, **kw):
     return FileValidation.from_json(file, result)
 
 
-def run_validator(file_path, for_appversions=None):
+def run_validator(file_path, for_appversions=None, test_all_tiers=False):
     """A pre-configured wrapper around the addon validator.
 
     *file_path*
@@ -61,6 +61,11 @@ def run_validator(file_path, for_appversions=None):
         for. The key is an application GUID and its value is a list of
         versions.
 
+    *test_all_tiers=False*
+        When False (default) the validator will not continue if it
+        encounters fatal errors.  When True, all tests in all tiers are run.
+        See bug 615426 for discussion on this default.
+
     To validate the addon for compatibility with Firefox 5 and 6,
     you'd pass in::
 
@@ -85,9 +90,9 @@ def run_validator(file_path, for_appversions=None):
     return validate(file_path,
                     for_appversions=for_appversions,
                     format='json',
-                    # This flag says to stop testing after one tier fails.
-                    # bug 615426
-                    determined=False,
+                    # When False, this flag says to stop testing after one
+                    # tier fails.
+                    determined=test_all_tiers,
                     approved_applications=apps,
                     spidermonkey=settings.SPIDERMONKEY)
 

apps/zadmin/tasks.py

@@ -60,7 +60,8 @@ def bulk_validate_file(result_id, **kw):
                  % (res.file, file_base, res.id))
         target = res.validation_job.target_version
         ver = {target.application.guid: [target.version]}
-        validation = run_validator(res.file.file_path, for_appversions=ver)
+        validation = run_validator(res.file.file_path, for_appversions=ver,
+                                   test_all_tiers=True)
     except:
         task_error = sys.exc_info()
         log.error(task_error[1])

apps/zadmin/tests/test_views.py

@@ -24,6 +24,7 @@ from versions.models import Version
 from zadmin.forms import NotifyForm
 from zadmin.models import ValidationJob, ValidationResult, EmailPreviewTopic
 from zadmin.views import completed_versions_dirty
+from zadmin import tasks
 
 
 class TestFlagged(test_utils.TestCase):
@@ -206,6 +207,16 @@ class TestBulkValidation(BulkValidationTest):
             eq_(AppVersion.objects.get(pk=id).version, ver)
         assert not empty, "Unexpected: %r" % data
 
+    @mock.patch('zadmin.tasks.run_validator')
+    def test_validate_all_tiers(self, run_validator):
+        run_validator.return_value = json.dumps(dict(errors=0, warnings=0,
+                                                     notices=0))
+        self.job = self.create_job(completed=datetime.now())
+        res = self.create_result(self.job, self.create_file(), **{})
+        tasks.bulk_validate_file(res.id)
+        assert run_validator.called
+        eq_(run_validator.call_args[1]['test_all_tiers'], True)
+
 
 class TestBulkUpdate(BulkValidationTest):