Bug 1453056 - test-verify should have the ability to run in chunks depending on the incoming tests. r=gbrown

taskcluster/taskgraph/transforms/tests.py

@@ -23,6 +23,8 @@ from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import resolve_keyed_by, OptimizationSchema
 from taskgraph.util.treeherder import split_symbol, join_symbol
 from taskgraph.util.platforms import platform_family
+from taskgraph import files_changed
+from mozpack.path import match as mozpackmatch
 from taskgraph.util.schema import (
     validate_schema,
     optionally_keyed_by,
@@ -40,6 +42,7 @@ from voluptuous import (
 
 import copy
 import logging
+import math
 
 # default worker types keyed by instance-size
 LINUX_WORKER_TYPES = {
@@ -673,7 +676,7 @@ def handle_suite_category(config, tests):
 
         script = test['mozharness']['script']
         category_arg = None
-        if suite == 'test-verify' or suite == 'test-coverage':
+        if suite.startswith('test-verify') or suite.startswith('test-coverage'):
             pass
         elif script == 'android_emulator_unittest.py':
             category_arg = '--test-suite'
@@ -773,6 +776,18 @@ def split_chunks(config, tests):
     them and assigning 'this-chunk' appropriately and updating the treeherder
     symbol."""
     for test in tests:
+        if test['suite'].startswith('test-verify'):
+            test['chunks'] = perfile_number_of_chunks(config, test['test-name'])
+            if test['chunks'] == 0:
+                continue
+            # limit the number of chunks we run for test-verify mode because
+            # test-verify is comprehensive and takes a lot of time, if we have
+            # >30 tests changed, this is probably an import of external tests,
+            # or a patch renaming/moving files in bulk
+            maximum_number_verify_chunks = 3
+            if test['chunks'] > maximum_number_verify_chunks:
+                test['chunks'] = maximum_number_verify_chunks
+
         if test['chunks'] == 1:
             test['this-chunk'] = 1
             yield test
@@ -791,6 +806,31 @@ def split_chunks(config, tests):
             yield chunked
 
 
+def perfile_number_of_chunks(config, type):
+    # A rough estimate of how many chunks we need based on simple rules
+    # for determining what a test file is.
+
+    # TODO: Make this flexible based on coverage vs verify || test type
+    tests_per_chunk = 10.0
+
+    if type.startswith('test-verify-wpt'):
+        file_patterns = ['testing/web-platform/tests/**']
+    elif type.startswith('test-verify'):
+        file_patterns = ['**/test_*', '**/browser_*', '**/crashtest*/**',
+                          'js/src/test/test/', 'js/src/test/non262/', 'js/src/test/test262/']
+
+    changed_files = files_changed.get_changed_files(config.params.get('head_repository'),
+                                                    config.params.get('head_rev'))
+    test_count = 0
+    for pattern in file_patterns:
+        for path in changed_files:
+            if mozpackmatch(path, pattern):
+                test_count += 1
+
+    chunks = test_count/tests_per_chunk
+    return int(math.ceil(chunks))
+
+
 @transforms.add
 def allow_software_gl_layers(config, tests):
     """

testing/mozharness/mozharness/mozilla/testing/per_test_base.py

@@ -6,6 +6,7 @@
 # ***** END LICENSE BLOCK *****
 
 import argparse
+import math
 import os
 import posixpath
 import re
@@ -189,6 +190,32 @@ class SingleTestMixin(object):
         else:
             self._find_misc_tests(dirs, changed_files)
 
+        # per test mode run specific tests from any given test suite
+        # _find_*_tests organizes tests to run into suites so we can
+        # run each suite at a time
+
+        # chunk files
+        total_tests = sum([len(self.suites[x]) for x in self.suites])
+
+        files_per_chunk = total_tests / float(self.config.get('total_chunks', 1))
+        files_per_chunk = int(math.ceil(files_per_chunk))
+
+        chunk_number = int(self.config.get('this_chunk', 1))
+        suites = {}
+        start = (chunk_number - 1) * files_per_chunk
+        end = (chunk_number * files_per_chunk)
+        current = -1
+        for suite in self.suites:
+            for test in self.suites[suite]:
+                current += 1
+                if current >= start and current < end:
+                    if suite not in suites:
+                        suites[suite] = []
+                    suites[suite].append(test)
+            if current >= end:
+                break
+
+        self.suites = suites
         self.tests_downloaded = True
 
     def query_args(self, suite):

testing/mozharness/scripts/android_emulator_unittest.py

@@ -465,7 +465,7 @@ class AndroidEmulatorTest(TestingMixin, EmulatorMixin, BaseScript, MozbaseMixin,
 
         if user_paths:
             cmd.extend(user_paths.split(':'))
-        else:
+        elif not self.verify_enabled:
             if self.this_chunk is not None:
                 cmd.extend(['--this-chunk', self.this_chunk])
             if self.total_chunks is not None:

testing/mozharness/scripts/desktop_unittest.py

@@ -400,7 +400,7 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
             # Ignore chunking if we have user specified test paths
             if os.environ.get('MOZHARNESS_TEST_PATHS'):
                 base_cmd.extend(os.environ['MOZHARNESS_TEST_PATHS'].split(':'))
-            elif c.get('total_chunks') and c.get('this_chunk'):
+            elif c.get('total_chunks') and c.get('this_chunk') and not self.verify_enabled:
                 base_cmd.extend(['--total-chunks', c['total_chunks'],
                                  '--this-chunk', c['this_chunk']])
 

testing/mozharness/scripts/web_platform_tests.py

@@ -219,7 +219,7 @@ class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin, CodeCovera
             paths = [os.path.join(dirs["abs_wpttest_dir"], os.path.relpath(p, prefix))
                      for p in paths if p.startswith(prefix)]
             cmd.extend(paths)
-        else:
+        elif not self.verify_enabled:
             for opt in ["total_chunks", "this_chunk"]:
                 val = c.get(opt)
                 if val: