Bug 1455872 - Add taskcluster configs for raptor on OSX; r=jmaher

MozReview-Commit-ID: I67InZTEziy

--HG--
extra : rebase_source : b18672eca2d3922367d3e4e50ff4b451a06e479e

python/mozbuild/mozbuild/schedules.py

@@ -50,6 +50,7 @@ EXCLUSIVE_COMPONENTS = [
     'gtest',
     'marionette',
     'mochitest',
+    'raptor',
     'reftest',
     'robocop',
     'talos',

taskcluster/ci/config.yml

@@ -15,6 +15,8 @@ treeherder:
         'M-V': 'Mochitests on Valgrind'
         'R': 'Reftests'
         'R-e10s': 'Reftests with e10s'
+        'Rap': 'Raptor performance tests on Firefox'
+        'Rap-e10s': 'Raptor performance tests on Firefox with e10s'
         'T': 'Talos performance tests'
         'Tsd': 'Talos performance tests with Stylo disabled'
         'Tss': 'Talos performance tests with Stylo sequential'

taskcluster/ci/test/kind.yml

@@ -26,6 +26,7 @@ jobs-from:
     - marionette.yml
     - misc.yml
     - mochitest.yml
+    - raptor.yml
     - reftest.yml
     - talos.yml
     - web-platform.yml

taskcluster/ci/test/raptor.yml

@@ -0,0 +1,40 @@
+job-defaults:
+    max-run-time: 1800
+    suite: raptor
+    tier:
+        by-test-platform:
+            windows10-64-ccov/.*: 3
+            linux64-ccov/.*: 3
+            default: 3
+    virtualization:
+        by-test-platform:
+            windows10-64-ccov/.*: virtual
+            default: hardware
+    mozharness:
+        script: raptor_script.py
+        no-read-buildbot-config: true
+        config:
+            by-test-platform:
+                macosx.*:
+                    - raptor/mac_config.py
+                windows.*:
+                    - raptor/windows_config.py
+                windows10-64-ccov/debug:
+                    - raptor/windows_vm_config.py
+                linux64-ccov/opt:
+                    - raptor/linux64_config_taskcluster.py
+                default:
+                    - raptor/linux_config.py
+
+raptor-firefox-tp6:
+    description: "Raptor Firefox tp6"
+    try-name: raptor-firefox-tp6
+    treeherder-symbol: Rap(tp6)
+    run-on-projects:
+        by-test-platform:
+            .*-qr/.*: ['try']
+            default: ['try']
+    max-run-time: 1200
+    mozharness:
+        extra-options:
+            - --test=raptor-firefox-tp6

taskcluster/ci/test/test-platforms.yml

@@ -242,6 +242,7 @@ macosx64/opt:
         - macosx64-tests
         - desktop-screenshot-capture
         - awsy
+        - macosx64-raptor
 
 macosx64-nightly/opt:
     build-platform: macosx64-nightly/opt

taskcluster/ci/test/test-sets.yml

@@ -310,6 +310,9 @@ macosx64-qr-tests:
     - reftest
     - reftest-fonts
 
+macosx64-raptor:
+    - raptor-firefox-tp6
+
 linux32-tests:
     - cppunit
     - crashtest

taskcluster/docs/attributes.rst

@@ -101,6 +101,11 @@ talos_try_name
 
 This is the name used to refer to a talos job via try syntax.
 
+raptor_try_name
+===============
+
+This is the name used to refer to a raptor job via try syntax.
+
 job_try_name
 ============
 

taskcluster/taskgraph/target_tasks.py

@@ -70,7 +70,8 @@ def filter_beta_release_tasks(task, parameters, ignore_kinds=None, allow_l10n=Fa
             'win32', 'win64',
             ):
         if task.attributes['build_type'] == 'opt' and \
-           task.attributes.get('unittest_suite') != 'talos':
+           task.attributes.get('unittest_suite') != 'talos' and \
+           task.attributes.get('unittest_suite') != 'raptor':
             return False
 
     # skip l10n, beetmover, balrog
@@ -129,6 +130,11 @@ def _try_option_syntax(full_task_graph, parameters, graph_config):
             task.attributes['task_duplicates'] = options.talos_trigger_tests
             task.attributes['profile'] = options.profile
 
+        # If the developer wants test raptor jobs to be rebuilt N times we add that value here
+        if options.raptor_trigger_tests > 1 and task.attributes.get('unittest_suite') == 'raptor':
+            task.attributes['task_duplicates'] = options.raptor_trigger_tests
+            task.attributes['profile'] = options.profile
+
         task.attributes.update(attributes)
 
     # Add notifications here as well
@@ -195,6 +201,9 @@ def target_tasks_ash(full_task_graph, parameters, graph_config):
             # don't run talos on ash
             if task.attributes.get('unittest_suite') == 'talos':
                 return False
+            # don't run raptor on ash
+            if task.attributes.get('unittest_suite') == 'raptor':
+                return False
         # don't upload symbols
         if task.attributes['kind'] == 'upload-symbols':
             return False

taskcluster/taskgraph/transforms/tests.py

@@ -488,6 +488,25 @@ def setup_talos(config, tests):
         yield test
 
 
+@transforms.add
+def setup_raptor(config, tests):
+    """Add options that are specific to raptor jobs (identified by suite=raptor)"""
+    for test in tests:
+        if test['suite'] != 'raptor':
+            yield test
+            continue
+
+        extra_options = test.setdefault('mozharness', {}).setdefault('extra-options', [])
+
+        # Per https://bugzilla.mozilla.org/show_bug.cgi?id=1357753#c3, branch
+        # name is only required for try
+        if config.params.is_try():
+            extra_options.append('--branch-name')
+            extra_options.append('try')
+
+        yield test
+
+
 @transforms.add
 def handle_artifact_prefix(config, tests):
     """Handle translating `artifact_prefix` appropriately"""
@@ -719,6 +738,10 @@ def enable_code_coverage(config, tests):
                 test['mozharness']['extra-options'].append('--no-upload-results')
                 test['mozharness']['extra-options'].append('--add-option')
                 test['mozharness']['extra-options'].append('--tptimeout,15000')
+            if 'raptor' in test['test-name']:
+                test['max-run-time'] = 1800
+                if 'linux' in test['build-platform']:
+                    test['docker-image'] = {"in-tree": "desktop1604-test"}
         elif test['build-platform'] == 'linux64-jsdcov/opt':
             # Ensure we don't run on inbound/autoland/beta, but if the test is try only, ignore it
             if 'mozilla-central' in test['run-on-projects'] or \
@@ -757,7 +780,7 @@ def split_e10s(config, tests):
             if group != '?':
                 group += '-e10s'
             test['treeherder-symbol'] = join_symbol(group, symbol)
-            if test['suite'] == 'talos':
+            if test['suite'] == 'talos' or test['suite'] == 'raptor':
                 for i, option in enumerate(test['mozharness']['extra-options']):
                     if option.startswith('--suite='):
                         test['mozharness']['extra-options'][i] += '-e10s'
@@ -809,38 +832,38 @@ def perfile_number_of_chunks(config, type):
     # TODO: Make this flexible based on coverage vs verify || test type
     tests_per_chunk = 10.0
 
-    if type.startswith('test-verify-wpt'):
-        file_patterns = ['testing/web-platform/tests/**']
-    elif type.startswith('test-verify-gpu'):
-        file_patterns = ['**/*webgl*/**/test_*',
-                         '**/dom/canvas/**/test_*',
-                         '**/gfx/tests/**/test_*',
+    if type.startswith('test-verify-wpt'):
+        file_patterns = ['testing/web-platform/tests/**']
+    elif type.startswith('test-verify-gpu'):
+        file_patterns = ['**/*webgl*/**/test_*',
+                         '**/dom/canvas/**/test_*',
+                         '**/gfx/tests/**/test_*',
                          '**/devtools/canvasdebugger/**/browser_*',
                          '**/reftest*/**']
-    elif type.startswith('test-verify'):
-        file_patterns = ['**/test_*',
-                         '**/browser_*',
-                         '**/crashtest*/**',
-                         'js/src/test/test/',
-                         'js/src/test/non262/',
-                         'js/src/test/test262/']
+    elif type.startswith('test-verify'):
+        file_patterns = ['**/test_*',
+                         '**/browser_*',
+                         '**/crashtest*/**',
+                         'js/src/test/test/',
+                         'js/src/test/non262/',
+                         'js/src/test/test262/']
 
     changed_files = files_changed.get_changed_files(config.params.get('head_repository'),
                                                     config.params.get('head_rev'))
     test_count = 0
-    for pattern in file_patterns:
-        for path in changed_files:
-            if mozpackmatch(path, pattern):
-                gpu = False
-                if type == 'test-verify-e10s':
-                    # file_patterns for test-verify will pick up some gpu tests, lets ignore
-                    # in the case of reftest, we will not have any in the regular case
-                    gpu_dirs = ['dom/canvas', 'gfx/tests', 'devtools/canvasdebugger', 'webgl']
-                    for gdir in gpu_dirs:
-                        if len(path.split(gdir)) > 1:
-                            gpu = True
-
-                if not gpu:
+    for pattern in file_patterns:
+        for path in changed_files:
+            if mozpackmatch(path, pattern):
+                gpu = False
+                if type == 'test-verify-e10s':
+                    # file_patterns for test-verify will pick up some gpu tests, lets ignore
+                    # in the case of reftest, we will not have any in the regular case
+                    gpu_dirs = ['dom/canvas', 'gfx/tests', 'devtools/canvasdebugger', 'webgl']
+                    for gdir in gpu_dirs:
+                        if len(path.split(gdir)) > 1:
+                            gpu = True
+
+                if not gpu:
                     test_count += 1
 
     chunks = test_count/tests_per_chunk
@@ -997,6 +1020,8 @@ def make_job_description(config, tests):
         try_name = test['try-name']
         if test['suite'] == 'talos':
             attr_try_name = 'talos_try_name'
+        elif test['suite'] == 'raptor':
+            attr_try_name = 'raptor_try_name'
         else:
             attr_try_name = 'unittest_try_name'
 

taskcluster/taskgraph/try_option_syntax.py

@@ -201,6 +201,7 @@ def parse_message(message):
     parser.add_argument('-u', '--unittests', nargs='?',
                         dest='unittests', const='all', default='all')
     parser.add_argument('-t', '--talos', nargs='?', dest='talos', const='all', default='none')
+    parser.add_argument('-r', '--raptor', nargs='?', dest='raptor', const='all', default='none')
     parser.add_argument('-i', '--interactive',
                         dest='interactive', action='store_true', default=False)
     parser.add_argument('-e', '--all-emails',
@@ -210,6 +211,8 @@ def parse_message(message):
     parser.add_argument('-j', '--job', dest='jobs', action='append')
     parser.add_argument('--rebuild-talos', dest='talos_trigger_tests', action='store',
                         type=int, default=1)
+    parser.add_argument('--rebuild-raptor', dest='raptor_trigger_tests', action='store',
+                        type=int, default=1)
     parser.add_argument('--setenv', dest='env', action='append')
     parser.add_argument('--geckoProfile', dest='profile', action='store_true')
     parser.add_argument('--tag', dest='tag', action='store', default=None)
@@ -263,10 +266,12 @@ class TryOptionSyntax(object):
         self.platforms = []
         self.unittests = []
         self.talos = []
+        self.raptor = []
         self.trigger_tests = 0
         self.interactive = False
         self.notifications = None
         self.talos_trigger_tests = 0
+        self.raptor_trigger_tests = 0
         self.env = []
         self.profile = False
         self.tag = None
@@ -281,10 +286,12 @@ class TryOptionSyntax(object):
         self.unittests = self.parse_test_option(
             "unittest_try_name", options['unittests'], full_task_graph)
         self.talos = self.parse_test_option("talos_try_name", options['talos'], full_task_graph)
+        self.raptor = self.parse_test_option("raptor_try_name", options['raptor'], full_task_graph)
         self.trigger_tests = options['trigger_tests']
         self.interactive = options['interactive']
         self.notifications = options['notifications']
         self.talos_trigger_tests = options['talos_trigger_tests']
+        self.raptor_trigger_tests = options['raptor_trigger_tests']
         self.env = options['env']
         self.profile = options['profile']
         self.tag = options['tag']
@@ -583,7 +590,8 @@ class TryOptionSyntax(object):
             return check_run_on_projects()
         elif attr('kind') == 'test':
             return match_test(self.unittests, 'unittest_try_name') \
-                 or match_test(self.talos, 'talos_try_name')
+                 or match_test(self.talos, 'talos_try_name') \
+                 or match_test(self.raptor, 'raptor_try_name')
         elif attr('kind') in BUILD_KINDS:
             if attr('build_type') not in self.build_types:
                 return False
@@ -608,11 +616,13 @@ class TryOptionSyntax(object):
             "platforms: " + none_for_all(self.platforms),
             "unittests: " + none_for_all(self.unittests),
             "talos: " + none_for_all(self.talos),
+            "raptor" + none_for_all(self.raptor),
             "jobs: " + none_for_all(self.jobs),
             "trigger_tests: " + str(self.trigger_tests),
             "interactive: " + str(self.interactive),
             "notifications: " + str(self.notifications),
             "talos_trigger_tests: " + str(self.talos_trigger_tests),
+            "raptor_trigger_tests: " + str(self.raptor_trigger_tests),
             "env: " + str(self.env),
             "profile: " + str(self.profile),
             "tag: " + str(self.tag),