Bug 1387135 - Add ability to apply templates to task definitions via try_task_config.json, r=dustin

This provides a mechanism to modify the behaviour of tasks from a try push. The try_task_config.json
looks something like:

{
  "tasks": ["build-linux64/opt", "test-linux64/opt-mochitest-e10s-1"],
  "templates": {
    "artifact": {"enabled": 1}
  }
}

This tells taskgraph to apply the 'artifact' template to all tasks. Templates are JSONe based
.yml files that live under taskcluster/taskgraph/templates. Taskgraph will render every template
against every task definition. The templates themselves can then use JSONe condition statements to
filter out which tasks they should or shouldn't apply to.

MozReview-Commit-ID: J8HVZzOt4mX

--HG--
extra : rebase_source : 95a78bc56d3f90ff1b34aabd84ed92aff1e3d954

taskcluster/docs/how-tos.rst

@@ -278,7 +278,8 @@ example, the ``try_task_config.json`` file might look like:
 
 .. parsed-literal::
 
-    [
+    {
+      "tasks": [
         "test-windows10-64/opt-web-platform-tests-12",
         "test-windows7-32/opt-reftest-1",
         "test-windows7-32/opt-reftest-2",
@@ -286,10 +287,12 @@ example, the ``try_task_config.json`` file might look like:
         "build-linux64/debug",
         "source-test-mozlint-eslint"
       ]
+    }
 
 Very simply, this will run any task label that gets passed in as well as their
 dependencies. While it is possible to manually commit this file and push to
-try, it is mainly meant to be a generation target for various trychooser tools.
+try, it is mainly meant to be a generation target for various `tryselect`_
+choosers.
 
 A list of all possible task labels can be obtained by running:
 
@@ -303,3 +306,58 @@ obtained with:
 .. parsed-literal::
 
     $ ./mach taskgraph target
+
+Modifying Task Behavior on Try
+``````````````````````````````
+
+It's possible to alter the definition of a task with templates. Templates are
+`JSON-e`_ files that live in the `taskgraph module`_. Templates can be specified
+from the ``try_task_config.json`` like this:
+
+.. parsed-literal::
+
+    {
+      "tasks": [...],
+      "templates": {
+        artifact: {"enabled": 1}
+      }
+    }
+
+Each key in the templates object denotes a new template to apply, and the value
+denotes extra context to use while rendering. When specified, a template will
+be applied to every task no matter what. If the template should only be applied
+to certain kinds of tasks, this needs to be specified in the template itself
+using JSON-e `condition statements`_.
+
+The context available to the JSON-e render aims to match that of ``actions``.
+It looks like this:
+
+.. parsed-literal::
+
+    {
+      "task": {
+        "payload": {
+          "env": { ... },
+          ...
+        }
+        "extra": {
+          "treeherder": { ... },
+          ...
+        },
+        "tags": { "kind": "<kind>", ... },
+        ...
+      },
+      "input": {
+        "enabled": 1,
+        ...
+      },
+      "taskId": "<task id>"
+    }
+
+See the `existing templates`_ for examples.
+
+.. _tryselect: https://dxr.mozilla.org/mozilla-central/source/tools/tryselect
+.. _JSON-e: https://taskcluster.github.io/json-e/
+.. _taskgraph module: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates
+.. _condition statements: https://taskcluster.github.io/json-e/#%60$if%60%20-%20%60then%60%20-%20%60else%60
+.. _existing templates: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates

taskcluster/docs/parameters.rst

@@ -93,6 +93,10 @@ syntax or reading a project-specific configuration file).
     apply. This is usually defined internally, as filters are typically
     global.
 
+``target_task_labels``
+    List of task labels to select. Labels not listed will be filtered out.
+    Enabled on try only.
+
 ``target_tasks_method``
     The method to use to determine the target task set.  This is the suffix of
     one of the functions in ``taskcluster/taskgraph/target_tasks.py``.
@@ -102,3 +106,12 @@ syntax or reading a project-specific configuration file).
 
 ``include_nightly``
     If true, then nightly tasks are eligible for optimization.
+
+Morphed Set
+-----------
+
+``morph_templates``
+    Dict of JSON-e templates to apply to each task, keyed by template name.
+    Values are extra context that will be available to the template under the
+    ``input.<template>`` key. Available templates live in
+    ``taskcluster/taskgraph/templates``. Enabled on try only.

taskcluster/taskgraph/decision.py

@@ -166,6 +166,8 @@ def get_decision_parameters(options):
         'check_servo',
         'target_tasks_method',
     ]
+    parameters['target_task_labels'] = []
+    parameters['morph_templates'] = {}
 
     # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
     # case, fake it
@@ -187,6 +189,15 @@ def get_decision_parameters(options):
                        "for this project".format(project, __file__))
         parameters.update(PER_PROJECT_PARAMETERS['default'])
 
+    # morph_templates and target_task_labels are only used on try, so don't
+    # bother loading them elsewhere
+    task_config_file = os.path.join(GECKO, 'try_task_config.json')
+    if project == 'try' and os.path.isfile(task_config_file):
+        with open(task_config_file, 'r') as fh:
+            task_config = json.load(fh)
+        parameters['morph_templates'] = task_config.get('templates', {})
+        parameters['target_task_labels'] = task_config.get('tasks')
+
     # `target_tasks_method` has higher precedence than `project` parameters
     if options.get('target_tasks_method'):
         parameters['target_tasks_method'] = options['target_tasks_method']

taskcluster/taskgraph/generator.py

@@ -280,7 +280,8 @@ class TaskGraphGenerator(object):
 
         yield 'optimized_task_graph', optimized_task_graph
 
-        morphed_task_graph, label_to_taskid = morph(optimized_task_graph, label_to_taskid)
+        morphed_task_graph, label_to_taskid = morph(
+            optimized_task_graph, label_to_taskid, self.parameters)
 
         yield 'label_to_taskid', label_to_taskid
         yield 'morphed_task_graph', morphed_task_graph

taskcluster/taskgraph/morph.py

@@ -19,13 +19,17 @@ the graph.
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
+import os
 import re
 
+import jsone
+import yaml
 from slugid import nice as slugid
 from .task import Task
 from .graph import Graph
 from .taskgraph import TaskGraph
 
+here = os.path.abspath(os.path.dirname(__file__))
 logger = logging.getLogger(__name__)
 MAX_ROUTES = 10
 
@@ -241,11 +245,45 @@ def add_s3_uploader_task(taskgraph, label_to_taskid):
     return taskgraph, label_to_taskid
 
 
-def morph(taskgraph, label_to_taskid):
+class apply_jsone_templates(object):
+    """Apply a set of JSON-e templates to each task's `task` attribute.
+
+    :param templates: A dict with the template name as the key, and extra context
+                      to use (in addition to task.to_json()) as the value.
+    """
+    template_dir = os.path.join(here, 'templates')
+
+    def __init__(self, templates):
+        self.templates = templates
+
+    def __call__(self, taskgraph, label_to_taskid):
+        if not self.templates:
+            return taskgraph, label_to_taskid
+
+        for task in taskgraph.tasks.itervalues():
+            for template in sorted(self.templates):
+                context = {
+                    'task': task.task,
+                    'taskGroup': None,
+                    'taskId': task.task_id,
+                    'kind': task.kind,
+                    'input': self.templates[template],
+                }
+
+                template_path = os.path.join(self.template_dir, template + '.yml')
+                with open(template_path) as f:
+                    template = yaml.load(f)
+                task.task = jsone.render(template, context)
+
+        return taskgraph, label_to_taskid
+
+
+def morph(taskgraph, label_to_taskid, parameters):
     """Apply all morphs"""
     morphs = [
         add_index_tasks,
         add_s3_uploader_task,
+        apply_jsone_templates(parameters.get('morph_templates')),
     ]
     for m in morphs:
         taskgraph, label_to_taskid = m(taskgraph, label_to_taskid)

taskcluster/taskgraph/parameters.py

@@ -21,15 +21,22 @@ PARAMETER_NAMES = set([
     'include_nightly',
     'level',
     'message',
+    'morph_templates',
     'moz_build_date',
     'optimize_target_tasks',
     'owner',
     'project',
     'pushdate',
     'pushlog_id',
+    'target_task_labels',
     'target_tasks_method',
 ])
 
+TRY_ONLY_PARAMETERS = set([
+    'morph_templates',
+    'target_task_labels',
+])
+
 
 class Parameters(ReadOnlyDict):
     """An immutable dictionary with nicer KeyError messages on failure"""
@@ -37,7 +44,7 @@ class Parameters(ReadOnlyDict):
         names = set(self)
         msg = []
 
-        missing = PARAMETER_NAMES - names
+        missing = PARAMETER_NAMES - TRY_ONLY_PARAMETERS - names
         if missing:
             msg.append("missing parameters: " + ", ".join(missing))
 

taskcluster/taskgraph/target_tasks.py

@@ -7,11 +7,11 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
-import json
 
 from taskgraph import try_option_syntax
 from taskgraph.util.attributes import match_run_on_projects
 
+here = os.path.abspath(os.path.dirname(__file__))
 _target_task_methods = {}
 
 
@@ -53,20 +53,11 @@ def standard_filter(task, parameters):
 
 
 def _try_task_config(full_task_graph, parameters):
-    task_config_file = os.path.join(os.getcwd(), 'try_task_config.json')
-
-    if not os.path.isfile(task_config_file):
+    if not parameters.get('target_task_labels'):
         return []
 
-    with open(task_config_file, 'r') as fh:
-        task_config = json.load(fh)
-
-    target_task_labels = []
-    for task in full_task_graph.tasks.itervalues():
-        if task.label in task_config:
-            target_task_labels.append(task.label)
-
-    return target_task_labels
+    return [t.label for t in full_task_graph.tasks.itervalues()
+            if t.label in parameters['target_task_labels']]
 
 
 def _try_option_syntax(full_task_graph, parameters):

taskcluster/taskgraph/test/test_target_tasks.py

@@ -4,7 +4,6 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import os
 import unittest
 
 from taskgraph import target_tasks
@@ -76,33 +75,32 @@ class TestTargetTasks(unittest.TestCase):
         tg = TaskGraph(tasks, graph)
 
         method = target_tasks.get_method('try_tasks')
-        config = os.path.join(os.getcwd(), 'try_task_config.json')
+        params = {
+            'message': '',
+            'target_task_labels': [],
+        }
 
         orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax
         try:
             try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax
 
             # no try specifier
-            self.assertEqual(method(tg, {'message': ''}), ['b'])
+            self.assertEqual(method(tg, params), ['b'])
 
             # try syntax only
-            self.assertEqual(method(tg, {'message': 'try: me'}), ['b'])
+            params['message'] = 'try: me'
+            self.assertEqual(method(tg, params), ['b'])
 
             # try task config only
-            with open(config, 'w') as fh:
-                fh.write('["c"]')
-            self.assertEqual(method(tg, {'message': ''}), ['c'])
-
-            with open(config, 'w') as fh:
-                fh.write('{"c": {}}')
-            self.assertEqual(method(tg, {'message': ''}), ['c'])
+            params['message'] = ''
+            params['target_task_labels'] = ['c']
+            self.assertEqual(method(tg, params), ['c'])
 
             # both syntax and config
-            self.assertEqual(set(method(tg, {'message': 'try: me'})), set(['b', 'c']))
+            params['message'] = 'try: me'
+            self.assertEqual(set(method(tg, params)), set(['b', 'c']))
         finally:
             try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
-            if os.path.isfile(config):
-                os.remove(config)
 
 
 if __name__ == '__main__':