Bug 1462434 - Prevent baseline coverage tests from being skipped. r=jmaher

This patch prevents baseline coverage tests from being skipped when too many tests are being run.

MozReview-Commit-ID: JVTOYZAXbwf

--HG--
extra : rebase_source : dedd6a323445f030b60180805c6c6adf5d10771b

testing/mozharness/scripts/desktop_unittest.py

@@ -850,25 +850,36 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
                 cmd_timeout = self.get_timeout_for_category(suite_category)
 
                 summary = None
+                executed_too_many_tests = False
                 for per_test_args in self.query_args(suite):
-                    if (datetime.now() - self.start_time) > max_per_test_time:
-                        # Running tests has run out of time. That is okay! Stop running
-                        # them so that a task timeout is not triggered, and so that
-                        # (partial) results are made available in a timely manner.
-                        self.info("TinderboxPrint: Running tests took too long: Not all tests "
-                                  "were executed.<br/>")
-                        # Signal per-test time exceeded, to break out of suites and
-                        # suite categories loops also.
-                        return False
-                    if executed_tests >= max_per_test_tests:
-                        # When changesets are merged between trees or many tests are
-                        # otherwise updated at once, there probably is not enough time
-                        # to run all tests, and attempting to do so may cause other
-                        # problems, such as generating too much log output.
-                        self.info("TinderboxPrint: Too many modified tests: Not all tests "
-                                  "were executed.<br/>")
-                        return False
-                    executed_tests = executed_tests + 1
+                    # Make sure baseline code coverage tests are never
+                    # skipped and that having them run has no influence
+                    # on the max number of actual tests that are to be run.
+                    is_baseline_test = 'baselinecoverage' in per_test_args[-1] \
+                                       if self.per_test_coverage else False
+                    if executed_too_many_tests and not is_baseline_test:
+                        continue
+
+                    if not is_baseline_test:
+                        if (datetime.now() - self.start_time) > max_per_test_time:
+                            # Running tests has run out of time. That is okay! Stop running
+                            # them so that a task timeout is not triggered, and so that
+                            # (partial) results are made available in a timely manner.
+                            self.info("TinderboxPrint: Running tests took too long: Not all tests "
+                                      "were executed.<br/>")
+                            # Signal per-test time exceeded, to break out of suites and
+                            # suite categories loops also.
+                            return False
+                        if executed_tests >= max_per_test_tests:
+                            # When changesets are merged between trees or many tests are
+                            # otherwise updated at once, there probably is not enough time
+                            # to run all tests, and attempting to do so may cause other
+                            # problems, such as generating too much log output.
+                            self.info("TinderboxPrint: Too many modified tests: Not all tests "
+                                      "were executed.<br/>")
+                            executed_too_many_tests = True
+
+                        executed_tests = executed_tests + 1
 
                     final_cmd = copy.copy(cmd)
                     final_cmd.extend(per_test_args)
@@ -912,6 +923,9 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin,
                     else:
                         self.log("The %s suite: %s ran with return status: %s" %
                                  (suite_category, suite, tbpl_status), level=log_level)
+
+                if executed_too_many_tests:
+                    return False
         else:
             self.debug('There were no suites to run for %s' % suite_category)
         return True

testing/mozharness/scripts/web_platform_tests.py

@@ -336,23 +336,34 @@ class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin):
                 test_types = [suite]
 
             summary = None
+            executed_too_many_tests = False
             for per_test_args in self.query_args(suite):
-                if (datetime.now() - start_time) > max_per_test_time:
-                    # Running tests has run out of time. That is okay! Stop running
-                    # them so that a task timeout is not triggered, and so that
-                    # (partial) results are made available in a timely manner.
-                    self.info("TinderboxPrint: Running tests took too long: Not all tests "
-                              "were executed.<br/>")
-                    return
-                if executed_tests >= max_per_test_tests:
-                    # When changesets are merged between trees or many tests are
-                    # otherwise updated at once, there probably is not enough time
-                    # to run all tests, and attempting to do so may cause other
-                    # problems, such as generating too much log output.
-                    self.info("TinderboxPrint: Too many modified tests: Not all tests "
-                              "were executed.<br/>")
-                    return
-                executed_tests = executed_tests + 1
+                # Make sure baseline code coverage tests are never
+                # skipped and that having them run has no influence
+                # on the max number of actual tests that are to be run.
+                is_baseline_test = 'baselinecoverage' in per_test_args[-1] \
+                                   if self.per_test_coverage else False
+                if executed_too_many_tests and not is_baseline_test:
+                    continue
+
+                if not is_baseline_test:
+                    if (datetime.now() - start_time) > max_per_test_time:
+                        # Running tests has run out of time. That is okay! Stop running
+                        # them so that a task timeout is not triggered, and so that
+                        # (partial) results are made available in a timely manner.
+                        self.info("TinderboxPrint: Running tests took too long: Not all tests "
+                                  "were executed.<br/>")
+                        return
+                    if executed_tests >= max_per_test_tests:
+                        # When changesets are merged between trees or many tests are
+                        # otherwise updated at once, there probably is not enough time
+                        # to run all tests, and attempting to do so may cause other
+                        # problems, such as generating too much log output.
+                        self.info("TinderboxPrint: Too many modified tests: Not all tests "
+                                  "were executed.<br/>")
+                        executed_too_many_tests = True
+
+                    executed_tests = executed_tests + 1
 
                 cmd = self._query_cmd(test_types)
                 cmd.extend(per_test_args)