Bug 1465300 - Exit test coverage chunk after all suites are checked for baseline tests. r=jmaher

This patch fixes baseline test addition. It allows all baselines to run in all test suites. Previously, only the test suite which had too many tests executed in it, was searched to the end for any baselines. Now, we continue to all other test suites before we finish.

MozReview-Commit-ID: FWMAsHb22CO

--HG--
extra : rebase_source : 7dbcdb3977bd351367ff9fa7892e41c2b86a2dce

testing/mozharness/scripts/desktop_unittest.py

@@ -400,11 +400,11 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
                     base_cmd.append('--e10s')
 
             # Ignore chunking if we have user specified test paths
-            if not (self.verify_enabled or self.per_test_coverage):
-                if os.environ.get('MOZHARNESS_TEST_PATHS'):
-                    base_cmd.extend(os.environ['MOZHARNESS_TEST_PATHS'].split(':'))
-                elif c.get('total_chunks') and c.get('this_chunk'):
-                    base_cmd.extend(['--total-chunks', c['total_chunks'],
+            if not (self.verify_enabled or self.per_test_coverage):
+                if os.environ.get('MOZHARNESS_TEST_PATHS'):
+                    base_cmd.extend(os.environ['MOZHARNESS_TEST_PATHS'].split(':'))
+                elif c.get('total_chunks') and c.get('this_chunk'):
+                    base_cmd.extend(['--total-chunks', c['total_chunks'],
                                      '--this-chunk', c['this_chunk']])
 
             if c['no_random']:
@@ -770,10 +770,14 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
         max_per_test_time = timedelta(minutes=60)
         max_per_test_tests = 10
         executed_tests = 0
+        executed_too_many_tests = False
 
         if suites:
             self.info('#### Running %s suites' % suite_category)
             for suite in suites:
+                if executed_too_many_tests and not self.per_test_coverage:
+                    return False
+
                 abs_base_cmd = self._query_abs_base_cmd(suite_category, suite)
                 cmd = abs_base_cmd[:]
                 replace_dict = {
@@ -849,7 +853,6 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
                 cmd_timeout = self.get_timeout_for_category(suite_category)
 
                 summary = {}
-                executed_too_many_tests = False
                 for per_test_args in self.query_args(suite):
                     # Make sure baseline code coverage tests are never
                     # skipped and that having them run has no influence
@@ -923,8 +926,8 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
                         self.log("The %s suite: %s ran with return status: %s" %
                                  (suite_category, suite, tbpl_status), level=log_level)
 
-                if executed_too_many_tests:
-                    return False
+            if executed_too_many_tests:
+                return False
         else:
             self.debug('There were no suites to run for %s' % suite_category)
         return True

testing/mozharness/scripts/web_platform_tests.py

@@ -319,6 +319,7 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin):
         max_per_test_time = timedelta(minutes=60)
         max_per_test_tests = 10
         executed_tests = 0
+        executed_too_many_tests = False
 
         if self.per_test_coverage or self.verify_enabled:
             suites = self.query_per_test_category_suites(None, None)
@@ -332,11 +333,13 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin):
             test_types = self.config.get("test_type", [])
             suites = [None]
         for suite in suites:
+            if executed_too_many_tests and not self.per_test_coverage:
+                continue
+
             if suite:
                 test_types = [suite]
 
             summary = {}
-            executed_too_many_tests = False
             for per_test_args in self.query_args(suite):
                 # Make sure baseline code coverage tests are never
                 # skipped and that having them run has no influence