Bug 1568277 - [ci] Add an experimental SETA optimize strategy and task to run it r=tomprince

These "shadow scheduler" tasks will generate artifacts per-push on autoland.
Basically, given the scheduling algorithms defined in
TASKGRAPH_OPTIMIZE_STRATEGIES, which tasks *would* have been scheduled on this
push.

This will allow us to download the artifacts and run comparisons against the
baseline to see whether things like code coverage or machine learning are
making the situation better or worse.

Differential Revision: https://phabricator.services.mozilla.com/D40427

--HG--
extra : moz-landing-system : lando

taskcluster/ci/config.yml

@@ -116,6 +116,7 @@ treeherder:
         'Gd': 'Geckodriver'
         'clang': 'Clang Tidy & Format'
         'coverity': 'Coverity Static Analysis'
+        'SS': 'Shadow scheduler'
 
 index:
     products:

taskcluster/ci/source-test/kind.yml

@@ -27,6 +27,7 @@ jobs-from:
     - node.yml
     - python.yml
     - remote.yml
+    - shadow-scheduler.yml
     - webidl.yml
     - wpt-manifest.yml
     - wpt-metadata.yml

taskcluster/ci/source-test/shadow-scheduler.yml

@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+    platform: gecko-decision/opt
+    worker-type: t-linux-xlarge
+    worker:
+        docker-image: {in-tree: "lint"}
+        max-run-time: 3600
+        artifacts:
+            - type: file
+              name: public/shadow-scheduler/optimized_tasks.list
+              path: /builds/worker/optimized_tasks.list
+    treeherder:
+        kind: other
+        tier: 3
+    require-decision-task-id: true
+    run-on-projects: ['autoland']
+    run:
+        using: mach
+        mach: taskgraph optimized -p task-id=$DECISION_TASK_ID --output-file /builds/worker/optimized_tasks.list
+        sparse-profile: taskgraph
+
+seta_10_120:
+    description: Runs the seta_10_120 experimental optimization strategy
+    treeherder:
+        symbol: SS(seta_10_120)
+    worker:
+        env:
+            TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:seta_10_120

taskcluster/taskgraph/optimize/__init__.py

@@ -310,3 +310,11 @@ import_sibling_modules()
 register_strategy('test', args=('skip-unless-schedules', 'seta'))(Either)
 register_strategy('test-inclusive', args=('skip-unless-schedules',))(Alias)
 register_strategy('test-try', args=('skip-unless-schedules',))(Alias)
+
+
+# Experimental strategy that replaces the default SETA with a version that runs
+# all tasks every 10th push or 2 hours.
+seta_10_120 = {
+    'seta': Alias('seta_10_120'),
+    'test': Either('skip-unless-schedules', 'seta_10_120'),
+}

taskcluster/taskgraph/optimize/seta.py

@@ -263,10 +263,13 @@ class SETA(object):
 is_low_value_task = SETA().is_low_value_task
 
 
-@register_strategy('seta')
+@register_strategy('seta', args=(5, 60))
+@register_strategy('seta_10_120', args=(10, 120))
 class SkipLowValue(OptimizationStrategy):
-    push_interval = 5
-    time_interval = 60
+
+    def __init__(self, push_interval, time_interval):
+        self.push_interval = push_interval
+        self.time_interval = time_interval
 
     def should_remove_task(self, task, params, _):
         label = task.label

taskcluster/taskgraph/transforms/source_test.py

@@ -54,6 +54,8 @@ source_test_description_schema = Schema({
         {'by-platform': {basestring: job_description_schema['worker']}},
     ),
     Optional('python-version'): [int],
+    # If true, the DECISION_TASK_ID env will be populated.
+    Optional('require-decision-task-id'): bool,
 })
 
 transforms = TransformSequence()
@@ -213,3 +215,19 @@ def handle_shell(config, jobs):
 
         del job['shell']
         yield job
+
+
+@transforms.add
+def add_decision_task_id_to_env(config, jobs):
+    """
+    Creates the `DECISION_TASK_ID` environment variable in tasks that set the
+    `require-decision-task-id` config.
+    """
+    for job in jobs:
+        if not job.pop('require-decision-task-id', False):
+            yield job
+            continue
+
+        env = job['worker'].setdefault('env', {})
+        env['DECISION_TASK_ID'] = os.environ.get('TASK_ID', '')
+        yield job