From 9b4853268fd81593c2ade7605aab66a9154131b9 Mon Sep 17 00:00:00 2001
From: Andrew Halberstadt <ahalberstadt@mozilla.com>
Date: Thu, 15 Aug 2019 19:36:35 +0000
Subject: [PATCH] Bug 1568277 - [ci] Add an experimental SETA optimize strategy
 and task to run it r=tomprince

These "shadow scheduler" tasks will generate artifacts per-push on autoland.
Basically, given the scheduling algorithms defined in
TASKGRAPH_OPTIMIZE_STRATEGIES, which tasks *would* have been scheduled on this
push.

This will allow us to download the artifacts and run comparisons against the
baseline to see whether things like code coverage or machine learning are
making the situation better or worse.

Differential Revision: https://phabricator.services.mozilla.com/D40427

--HG--
extra : moz-landing-system : lando
---
 taskcluster/ci/config.yml                     |  1 +
 taskcluster/ci/source-test/kind.yml           |  1 +
 .../ci/source-test/shadow-scheduler.yml       | 31 +++++++++++++++++++
 taskcluster/taskgraph/optimize/__init__.py    |  8 +++++
 taskcluster/taskgraph/optimize/seta.py        |  9 ++++--
 .../taskgraph/transforms/source_test.py       | 18 +++++++++++
 6 files changed, 65 insertions(+), 3 deletions(-)
 create mode 100644 taskcluster/ci/source-test/shadow-scheduler.yml

diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml
index 58cb67f0846c..21d04de0ef9c 100755
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -115,6 +115,7 @@ treeherder:
         'Gd': 'Geckodriver'
         'clang': 'Clang Tidy & Format'
         'coverity': 'Coverity Static Analysis'
+        'SS': 'Shadow scheduler'
 
 index:
     products:
diff --git a/taskcluster/ci/source-test/kind.yml b/taskcluster/ci/source-test/kind.yml
index b2f464287e9f..d93c85b56910 100644
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -26,6 +26,7 @@ jobs-from:
     - mozlint-android.yml
     - node.yml
     - python.yml
+    - shadow-scheduler.yml
     - webidl.yml
     - wpt-metadata.yml
     - wpt-manifest.yml
diff --git a/taskcluster/ci/source-test/shadow-scheduler.yml b/taskcluster/ci/source-test/shadow-scheduler.yml
new file mode 100644
index 000000000000..a9083bf93fe8
--- /dev/null
+++ b/taskcluster/ci/source-test/shadow-scheduler.yml
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+    platform: gecko-decision/opt
+    worker-type: t-linux-xlarge
+    worker:
+        docker-image: {in-tree: "lint"}
+        max-run-time: 3600
+        artifacts:
+            - type: file
+              name: public/shadow-scheduler/optimized_tasks.list
+              path: /builds/worker/optimized_tasks.list
+    treeherder:
+        kind: other
+        tier: 3
+    require-decision-task-id: true
+    run-on-projects: ['autoland']
+    run:
+        using: mach
+        mach: taskgraph optimized -p task-id=$DECISION_TASK_ID --output-file /builds/worker/optimized_tasks.list
+        sparse-profile: taskgraph
+
+seta_10_120:
+    description: Runs the seta_10_120 experimental optimization strategy
+    treeherder:
+        symbol: SS(seta_10_120)
+    worker:
+        env:
+            TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:seta_10_120
diff --git a/taskcluster/taskgraph/optimize/__init__.py b/taskcluster/taskgraph/optimize/__init__.py
index 701354a5b3e0..793c8223c3df 100644
--- a/taskcluster/taskgraph/optimize/__init__.py
+++ b/taskcluster/taskgraph/optimize/__init__.py
@@ -310,3 +310,11 @@ import_sibling_modules()
 register_strategy('test', args=('skip-unless-schedules', 'seta'))(Either)
 register_strategy('test-inclusive', args=('skip-unless-schedules',))(Alias)
 register_strategy('test-try', args=('skip-unless-schedules',))(Alias)
+
+
+# Experimental strategy that replaces the default SETA with a version that runs
+# all tasks every 10th push or 2 hours.
+seta_10_120 = {
+    'seta': Alias('seta_10_120'),
+    'test': Either('skip-unless-schedules', 'seta_10_120'),
+}
diff --git a/taskcluster/taskgraph/optimize/seta.py b/taskcluster/taskgraph/optimize/seta.py
index 87ded3e455ac..b3344343dcd2 100644
--- a/taskcluster/taskgraph/optimize/seta.py
+++ b/taskcluster/taskgraph/optimize/seta.py
@@ -263,10 +263,13 @@ class SETA(object):
 is_low_value_task = SETA().is_low_value_task
 
 
-@register_strategy('seta')
+@register_strategy('seta', args=(5, 60))
+@register_strategy('seta_10_120', args=(10, 120))
 class SkipLowValue(OptimizationStrategy):
-    push_interval = 5
-    time_interval = 60
+
+    def __init__(self, push_interval, time_interval):
+        self.push_interval = push_interval
+        self.time_interval = time_interval
 
     def should_remove_task(self, task, params, _):
         label = task.label
diff --git a/taskcluster/taskgraph/transforms/source_test.py b/taskcluster/taskgraph/transforms/source_test.py
index fdd2cd845bbd..606f6226e8a0 100644
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -54,6 +54,8 @@ source_test_description_schema = Schema({
         {'by-platform': {basestring: job_description_schema['worker']}},
     ),
     Optional('python-version'): [int],
+    # If true, the DECISION_TASK_ID env will be populated.
+    Optional('require-decision-task-id'): bool,
 })
 
 transforms = TransformSequence()
@@ -213,3 +215,19 @@ def handle_shell(config, jobs):
 
         del job['shell']
         yield job
+
+
+@transforms.add
+def add_decision_task_id_to_env(config, jobs):
+    """
+    Creates the `DECISION_TASK_ID` environment variable in tasks that set the
+    `require-decision-task-id` config.
+    """
+    for job in jobs:
+        if not job.pop('require-decision-task-id', False):
+            yield job
+            continue
+
+        env = job['worker'].setdefault('env', {})
+        env['DECISION_TASK_ID'] = os.environ.get('TASK_ID', '')
+        yield job