Bug 1363714: Split tests tasks between TC and BB. r=dustin

In order to migrate from buildbot to taskcluster gradually, we read an
external json file to tells the percentage of jobs that must run in
Taskcluster.

MozReview-Commit-ID: JQhvsXKpcLz

--HG--
extra : rebase_source : 0c1a83b31831b788fb374adcb9fd181f97121239

taskcluster/taskgraph/transforms/tests.py

@@ -35,6 +35,8 @@ from voluptuous import (
 
 import copy
 import logging
+import requests
+from collections import defaultdict
 
 WORKER_TYPE = {
     # default worker types keyed by instance-size
@@ -394,21 +396,19 @@ def set_treeherder_machine_platform(config, tests):
 def set_worker_implementation(config, tests):
     """Set the worker implementation based on the test platform."""
     for test in tests:
-        if test['test-platform'].startswith('macosx'):
-            # see if '-g' appears in try syntax
-            if config.config['args'].generic_worker:
-                test['worker-implementation'] = 'generic-worker'
-            # see if '-w' appears in try syntax
-            elif config.config['args'].taskcluster_worker:
+        test_platform = test['test-platform']
+        if test_platform.startswith('macosx'):
+            if config.config['args'].taskcluster_worker:
                 test['worker-implementation'] = 'native-engine'
             else:
-                test['worker-implementation'] = 'buildbot-bridge'
+                test['worker-implementation'] = 'generic-worker'
         elif test.get('suite', '') == 'talos':
             test['worker-implementation'] = 'buildbot-bridge'
-        elif test['test-platform'].startswith('win'):
+        elif test_platform.startswith('win'):
             test['worker-implementation'] = 'generic-worker'
         else:
             test['worker-implementation'] = 'docker-worker'
+
         yield test
 
 
@@ -667,6 +667,33 @@ def parallel_stylo_tests(config, tests):
             yield test
 
 
+@transforms.add
+def allocate_to_bbb(config, tests):
+    """Make the load balancing between taskcluster and buildbot"""
+    j = get_load_balacing_settings()
+
+    tests_set = defaultdict(list)
+    for test in tests:
+        tests_set[test['test-platform']].append(test)
+
+    # Make the load balancing between taskcluster and buildbot
+    for test_platform, t in tests_set.iteritems():
+        # We sort the list to make the order of the tasks deterministic
+        t.sort(key=lambda x: (x['test-name'], x.get('this_chunk', 1)))
+        # The json file tells the percentage of tasks that run on
+        # taskcluster. The logic here is inverted, as tasks have been
+        # previously assigned to taskcluster. Therefore we assign the
+        # 1-p tasks to buildbot-bridge.
+        n = j.get(test_platform, 1.0)
+        if not (test_platform.startswith('mac')
+                and config.config['args'].taskcluster_worker):
+            for i in range(int(n * len(t)), len(t)):
+                t[i]['worker-implementation'] = 'buildbot-bridge'
+
+        for y in t:
+            yield y
+
+
 @transforms.add
 def make_job_description(config, tests):
     """Convert *test* descriptions to *job* descriptions (input to
@@ -767,3 +794,11 @@ def normpath(path):
 def get_firefox_version():
     with open('browser/config/version.txt', 'r') as f:
         return f.readline().strip()
+
+
+def get_load_balacing_settings():
+    url = "https://s3.amazonaws.com/taskcluster-graph-scheduling/tests-load.json"
+    try:
+        return requests.get(url).json()
+    except Exception:
+        return {}

taskcluster/taskgraph/try_option_syntax.py

@@ -243,9 +243,6 @@ def parse_message(message):
     # this temporary option to be able to push jobs to tc-worker.
     parser.add_argument('-w', '--taskcluster-worker',
                         dest='taskcluster_worker', action='store_true', default=False)
-    # Similarly, an extra flag for enabling os x jobs in generic-worker
-    parser.add_argument('-g', '--generic-worker',
-                        dest='generic_worker', action='store_true', default=False)
 
     # In order to run test jobs multiple times
     parser.add_argument('--rebuild', dest='trigger_tests', type=int, default=1)