зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1387135 - Add ability to apply templates to task definitions via try_task_config.json, r=dustin
This provides a mechanism to modify the behaviour of tasks from a try push. The try_task_config.json looks something like: { "tasks": ["build-linux64/opt", "test-linux64/opt-mochitest-e10s-1"], "templates": { "artifact": {"enabled": 1} } } This tells taskgraph to apply the 'artifact' template to all tasks. Templates are JSONe based .yml files that live under taskcluster/taskgraph/templates. Taskgraph will render every template against every task definition. The templates themselves can then use JSONe condition statements to filter out which tasks they should or shouldn't apply to. MozReview-Commit-ID: J8HVZzOt4mX --HG-- extra : rebase_source : 95a78bc56d3f90ff1b34aabd84ed92aff1e3d954
This commit is contained in:
Родитель
e95b5d48f4
Коммит
d042e4c525
|
@ -278,18 +278,21 @@ example, the ``try_task_config.json`` file might look like:
|
|||
|
||||
.. parsed-literal::
|
||||
|
||||
[
|
||||
"test-windows10-64/opt-web-platform-tests-12",
|
||||
"test-windows7-32/opt-reftest-1",
|
||||
"test-windows7-32/opt-reftest-2",
|
||||
"test-windows7-32/opt-reftest-3",
|
||||
"build-linux64/debug",
|
||||
"source-test-mozlint-eslint"
|
||||
]
|
||||
{
|
||||
"tasks": [
|
||||
"test-windows10-64/opt-web-platform-tests-12",
|
||||
"test-windows7-32/opt-reftest-1",
|
||||
"test-windows7-32/opt-reftest-2",
|
||||
"test-windows7-32/opt-reftest-3",
|
||||
"build-linux64/debug",
|
||||
"source-test-mozlint-eslint"
|
||||
]
|
||||
}
|
||||
|
||||
Very simply, this will run any task label that gets passed in as well as their
|
||||
dependencies. While it is possible to manually commit this file and push to
|
||||
try, it is mainly meant to be a generation target for various trychooser tools.
|
||||
try, it is mainly meant to be a generation target for various `tryselect`_
|
||||
choosers.
|
||||
|
||||
A list of all possible task labels can be obtained by running:
|
||||
|
||||
|
@ -303,3 +306,58 @@ obtained with:
|
|||
.. parsed-literal::
|
||||
|
||||
$ ./mach taskgraph target
|
||||
|
||||
Modifying Task Behavior on Try
|
||||
``````````````````````````````
|
||||
|
||||
It's possible to alter the definition of a task with templates. Templates are
|
||||
`JSON-e`_ files that live in the `taskgraph module`_. Templates can be specified
|
||||
from the ``try_task_config.json`` like this:
|
||||
|
||||
.. parsed-literal::
|
||||
|
||||
{
|
||||
"tasks": [...],
|
||||
"templates": {
|
||||
artifact: {"enabled": 1}
|
||||
}
|
||||
}
|
||||
|
||||
Each key in the templates object denotes a new template to apply, and the value
|
||||
denotes extra context to use while rendering. When specified, a template will
|
||||
be applied to every task no matter what. If the template should only be applied
|
||||
to certain kinds of tasks, this needs to be specified in the template itself
|
||||
using JSON-e `condition statements`_.
|
||||
|
||||
The context available to the JSON-e render aims to match that of ``actions``.
|
||||
It looks like this:
|
||||
|
||||
.. parsed-literal::
|
||||
|
||||
{
|
||||
"task": {
|
||||
"payload": {
|
||||
"env": { ... },
|
||||
...
|
||||
}
|
||||
"extra": {
|
||||
"treeherder": { ... },
|
||||
...
|
||||
},
|
||||
"tags": { "kind": "<kind>", ... },
|
||||
...
|
||||
},
|
||||
"input": {
|
||||
"enabled": 1,
|
||||
...
|
||||
},
|
||||
"taskId": "<task id>"
|
||||
}
|
||||
|
||||
See the `existing templates`_ for examples.
|
||||
|
||||
.. _tryselect: https://dxr.mozilla.org/mozilla-central/source/tools/tryselect
|
||||
.. _JSON-e: https://taskcluster.github.io/json-e/
|
||||
.. _taskgraph module: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates
|
||||
.. _condition statements: https://taskcluster.github.io/json-e/#%60$if%60%20-%20%60then%60%20-%20%60else%60
|
||||
.. _existing templates: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates
|
||||
|
|
|
@ -93,12 +93,25 @@ syntax or reading a project-specific configuration file).
|
|||
apply. This is usually defined internally, as filters are typically
|
||||
global.
|
||||
|
||||
``target_task_labels``
|
||||
List of task labels to select. Labels not listed will be filtered out.
|
||||
Enabled on try only.
|
||||
|
||||
``target_tasks_method``
|
||||
The method to use to determine the target task set. This is the suffix of
|
||||
one of the functions in ``taskcluster/taskgraph/target_tasks.py``.
|
||||
|
||||
``optimize_target_tasks``
|
||||
If true, then target tasks are eligible for optimization.
|
||||
If true, then target tasks are eligible for optimization.
|
||||
|
||||
``include_nightly``
|
||||
If true, then nightly tasks are eligible for optimization.
|
||||
If true, then nightly tasks are eligible for optimization.
|
||||
|
||||
Morphed Set
|
||||
-----------
|
||||
|
||||
``morph_templates``
|
||||
Dict of JSON-e templates to apply to each task, keyed by template name.
|
||||
Values are extra context that will be available to the template under the
|
||||
``input.<template>`` key. Available templates live in
|
||||
``taskcluster/taskgraph/templates``. Enabled on try only.
|
||||
|
|
|
@ -166,6 +166,8 @@ def get_decision_parameters(options):
|
|||
'check_servo',
|
||||
'target_tasks_method',
|
||||
]
|
||||
parameters['target_task_labels'] = []
|
||||
parameters['morph_templates'] = {}
|
||||
|
||||
# owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
|
||||
# case, fake it
|
||||
|
@ -187,6 +189,15 @@ def get_decision_parameters(options):
|
|||
"for this project".format(project, __file__))
|
||||
parameters.update(PER_PROJECT_PARAMETERS['default'])
|
||||
|
||||
# morph_templates and target_task_labels are only used on try, so don't
|
||||
# bother loading them elsewhere
|
||||
task_config_file = os.path.join(GECKO, 'try_task_config.json')
|
||||
if project == 'try' and os.path.isfile(task_config_file):
|
||||
with open(task_config_file, 'r') as fh:
|
||||
task_config = json.load(fh)
|
||||
parameters['morph_templates'] = task_config.get('templates', {})
|
||||
parameters['target_task_labels'] = task_config.get('tasks')
|
||||
|
||||
# `target_tasks_method` has higher precedence than `project` parameters
|
||||
if options.get('target_tasks_method'):
|
||||
parameters['target_tasks_method'] = options['target_tasks_method']
|
||||
|
|
|
@ -280,7 +280,8 @@ class TaskGraphGenerator(object):
|
|||
|
||||
yield 'optimized_task_graph', optimized_task_graph
|
||||
|
||||
morphed_task_graph, label_to_taskid = morph(optimized_task_graph, label_to_taskid)
|
||||
morphed_task_graph, label_to_taskid = morph(
|
||||
optimized_task_graph, label_to_taskid, self.parameters)
|
||||
|
||||
yield 'label_to_taskid', label_to_taskid
|
||||
yield 'morphed_task_graph', morphed_task_graph
|
||||
|
|
|
@ -19,13 +19,17 @@ the graph.
|
|||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
import jsone
|
||||
import yaml
|
||||
from slugid import nice as slugid
|
||||
from .task import Task
|
||||
from .graph import Graph
|
||||
from .taskgraph import TaskGraph
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
logger = logging.getLogger(__name__)
|
||||
MAX_ROUTES = 10
|
||||
|
||||
|
@ -241,11 +245,45 @@ def add_s3_uploader_task(taskgraph, label_to_taskid):
|
|||
return taskgraph, label_to_taskid
|
||||
|
||||
|
||||
def morph(taskgraph, label_to_taskid):
|
||||
class apply_jsone_templates(object):
|
||||
"""Apply a set of JSON-e templates to each task's `task` attribute.
|
||||
|
||||
:param templates: A dict with the template name as the key, and extra context
|
||||
to use (in addition to task.to_json()) as the value.
|
||||
"""
|
||||
template_dir = os.path.join(here, 'templates')
|
||||
|
||||
def __init__(self, templates):
|
||||
self.templates = templates
|
||||
|
||||
def __call__(self, taskgraph, label_to_taskid):
|
||||
if not self.templates:
|
||||
return taskgraph, label_to_taskid
|
||||
|
||||
for task in taskgraph.tasks.itervalues():
|
||||
for template in sorted(self.templates):
|
||||
context = {
|
||||
'task': task.task,
|
||||
'taskGroup': None,
|
||||
'taskId': task.task_id,
|
||||
'kind': task.kind,
|
||||
'input': self.templates[template],
|
||||
}
|
||||
|
||||
template_path = os.path.join(self.template_dir, template + '.yml')
|
||||
with open(template_path) as f:
|
||||
template = yaml.load(f)
|
||||
task.task = jsone.render(template, context)
|
||||
|
||||
return taskgraph, label_to_taskid
|
||||
|
||||
|
||||
def morph(taskgraph, label_to_taskid, parameters):
|
||||
"""Apply all morphs"""
|
||||
morphs = [
|
||||
add_index_tasks,
|
||||
add_s3_uploader_task,
|
||||
apply_jsone_templates(parameters.get('morph_templates')),
|
||||
]
|
||||
for m in morphs:
|
||||
taskgraph, label_to_taskid = m(taskgraph, label_to_taskid)
|
||||
|
|
|
@ -21,15 +21,22 @@ PARAMETER_NAMES = set([
|
|||
'include_nightly',
|
||||
'level',
|
||||
'message',
|
||||
'morph_templates',
|
||||
'moz_build_date',
|
||||
'optimize_target_tasks',
|
||||
'owner',
|
||||
'project',
|
||||
'pushdate',
|
||||
'pushlog_id',
|
||||
'target_task_labels',
|
||||
'target_tasks_method',
|
||||
])
|
||||
|
||||
TRY_ONLY_PARAMETERS = set([
|
||||
'morph_templates',
|
||||
'target_task_labels',
|
||||
])
|
||||
|
||||
|
||||
class Parameters(ReadOnlyDict):
|
||||
"""An immutable dictionary with nicer KeyError messages on failure"""
|
||||
|
@ -37,7 +44,7 @@ class Parameters(ReadOnlyDict):
|
|||
names = set(self)
|
||||
msg = []
|
||||
|
||||
missing = PARAMETER_NAMES - names
|
||||
missing = PARAMETER_NAMES - TRY_ONLY_PARAMETERS - names
|
||||
if missing:
|
||||
msg.append("missing parameters: " + ", ".join(missing))
|
||||
|
||||
|
|
|
@ -7,11 +7,11 @@
|
|||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import json
|
||||
|
||||
from taskgraph import try_option_syntax
|
||||
from taskgraph.util.attributes import match_run_on_projects
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
_target_task_methods = {}
|
||||
|
||||
|
||||
|
@ -53,20 +53,11 @@ def standard_filter(task, parameters):
|
|||
|
||||
|
||||
def _try_task_config(full_task_graph, parameters):
|
||||
task_config_file = os.path.join(os.getcwd(), 'try_task_config.json')
|
||||
|
||||
if not os.path.isfile(task_config_file):
|
||||
if not parameters.get('target_task_labels'):
|
||||
return []
|
||||
|
||||
with open(task_config_file, 'r') as fh:
|
||||
task_config = json.load(fh)
|
||||
|
||||
target_task_labels = []
|
||||
for task in full_task_graph.tasks.itervalues():
|
||||
if task.label in task_config:
|
||||
target_task_labels.append(task.label)
|
||||
|
||||
return target_task_labels
|
||||
return [t.label for t in full_task_graph.tasks.itervalues()
|
||||
if t.label in parameters['target_task_labels']]
|
||||
|
||||
|
||||
def _try_option_syntax(full_task_graph, parameters):
|
||||
|
|
|
@ -4,7 +4,6 @@
|
|||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from taskgraph import target_tasks
|
||||
|
@ -76,33 +75,32 @@ class TestTargetTasks(unittest.TestCase):
|
|||
tg = TaskGraph(tasks, graph)
|
||||
|
||||
method = target_tasks.get_method('try_tasks')
|
||||
config = os.path.join(os.getcwd(), 'try_task_config.json')
|
||||
params = {
|
||||
'message': '',
|
||||
'target_task_labels': [],
|
||||
}
|
||||
|
||||
orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax
|
||||
try:
|
||||
try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax
|
||||
|
||||
# no try specifier
|
||||
self.assertEqual(method(tg, {'message': ''}), ['b'])
|
||||
self.assertEqual(method(tg, params), ['b'])
|
||||
|
||||
# try syntax only
|
||||
self.assertEqual(method(tg, {'message': 'try: me'}), ['b'])
|
||||
params['message'] = 'try: me'
|
||||
self.assertEqual(method(tg, params), ['b'])
|
||||
|
||||
# try task config only
|
||||
with open(config, 'w') as fh:
|
||||
fh.write('["c"]')
|
||||
self.assertEqual(method(tg, {'message': ''}), ['c'])
|
||||
|
||||
with open(config, 'w') as fh:
|
||||
fh.write('{"c": {}}')
|
||||
self.assertEqual(method(tg, {'message': ''}), ['c'])
|
||||
params['message'] = ''
|
||||
params['target_task_labels'] = ['c']
|
||||
self.assertEqual(method(tg, params), ['c'])
|
||||
|
||||
# both syntax and config
|
||||
self.assertEqual(set(method(tg, {'message': 'try: me'})), set(['b', 'c']))
|
||||
params['message'] = 'try: me'
|
||||
self.assertEqual(set(method(tg, params)), set(['b', 'c']))
|
||||
finally:
|
||||
try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
|
||||
if os.path.isfile(config):
|
||||
os.remove(config)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
Загрузка…
Ссылка в новой задаче