Bug 1332506 - Treeherder actions defined in-tree. r=dustin

Framework for defining actions in-tree that can be displayed
and triggered from Treeherder.

MozReview-Commit-ID: 3rvwgy2i4xu

--HG--
extra : rebase_source : beca394f4337aae4ab149e4db810352f57ec4988
This commit is contained in:
Jonas Finnemann Jensen 2017-01-31 15:34:05 -08:00
Родитель f9108552c0
Коммит 24c8b38973
7 изменённых файлов: 585 добавлений и 0 удалений

Просмотреть файл

@ -0,0 +1,10 @@
from registry import (
register_task_action, register_callback_action, render_actions_json, trigger_action_callback,
)
__all__ = [
'register_task_action',
'register_callback_action',
'render_actions_json',
'trigger_action_callback',
]

Просмотреть файл

@ -0,0 +1,30 @@
from .registry import register_callback_action
@register_callback_action(
title='Say Hello',
symbol='hw',
description="""
Simple **proof-of-concept** action that prints a hello action.
""",
order=10000, # Put this at the very bottom/end of any menu (default)
context=[{}], # Applies to any task
available=lambda parameters: True, # available regardless decision parameters (default)
schema={
'type': 'string',
'maxLength': 255,
'default': 'World',
'title': 'Target Name',
'description': """
A name wish to say hello to...
This should normally be **your name**.
But you can also use the default value `'World'`.
""".strip(),
},
)
def hello_world_action(parameters, input, task_group_id, task_id, task):
print "This message was triggered from context-menu of taskId: {}".format(task_id)
print ""
print "Hello {}".format(input)
print "--- Action is now executed"

Просмотреть файл

@ -0,0 +1,305 @@
import json
import os
import inspect
from types import FunctionType
from collections import namedtuple
from taskgraph.util.docker import docker_image
from taskgraph.parameters import Parameters
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
actions = []
callbacks = {}
Action = namedtuple('Action', [
'title', 'description', 'order', 'context', 'schema', 'task_template_builder',
])
def is_json(data):
""" Return ``True``, if ``data`` is a JSON serializable data structure. """
try:
json.dumps(data)
except ValueError:
return False
return True
def register_task_action(title, description, order, context, schema):
"""
Register an action task that can be triggered from supporting
user interfaces, such as Treeherder.
Most actions will create intermediate action tasks that call back into
in-tree python code. To write such an action please use
:func:`register_callback_action`.
This function is to be used a decorator for a function that returns a task
template, see :doc:`specification <action-spec>` for details on the
templating features. The decorated function will be given decision task
parameters, which can be embedded in the task template that is returned.
Parameters
----------
title : str
A human readable title for the action to be used as label on a button
or text on a link for triggering the action.
description : str
A human readable description of the action in **markdown**.
This will be display as tooltip and in dialog window when the action
is triggered. This is a good place to describe how to use the action.
order : int
Order of the action in menus, this is relative to the ``order`` of
other actions declared.
context : list of dict
List of tag-sets specifying which tasks the action is can take as input.
If no tag-sets is specified as input the action is related to the
entire task-group, and won't be triggered with a given task.
Otherwise, if ``context = [{'k': 'b', 'p': 'l'}, {'k': 't'}]`` will only
be displayed in the context menu for tasks that has
``task.tags.k == 'b' && task.tags.p = 'l'`` or ``task.tags.k = 't'``.
Esentially, this allows filtering on ``task.tags``.
schema : dict
JSON schema specifying input accepted by the action.
This is optional and can be left ``null`` if no input is taken.
Returns
-------
function
To be used as decorator for the function that builds the task template.
The decorated function will be given decision parameters and may return
``None`` instead of a task template, if the action is disabled.
"""
assert isinstance(title, basestring), 'title must be a string'
assert isinstance(description, basestring), 'description must be a string'
assert isinstance(order, int), 'order must be an integer'
assert is_json(schema), 'schema must be a JSON compatible object'
mem = {"registered": False} # workaround nonlocal missing in 2.x
def register_task_template_builder(task_template_builder):
assert not mem['registered'], 'register_task_action must be used as decorator'
actions.append(Action(
title.strip(), description.strip(), order, context, schema, task_template_builder,
))
mem['registered'] = True
return register_task_template_builder
def register_callback_action(title, symbol, description, order=10000, context=[],
available=lambda parameters: True, schema=None):
"""
Register an action callback that can be triggered from supporting
user interfaces, such as Treeherder.
This function is to be used as a decorator for a callback that takes
parameters as follows:
``parameters``:
Decision task parameters, see ``taskgraph.parameters.Parameters``.
``input``:
Input matching specified JSON schema, ``None`` if no ``schema``
parameter is given to ``register_callback_action``.
``task_group_id``:
The id of the task-group this was triggered for.
``task_id`` and `task``:
task identifier and task definition for task the action was triggered
for, ``None`` if no ``context`` parameters was given to
``register_callback_action``.
Parameters
----------
title : str
A human readable title for the action to be used as label on a button
or text on a link for triggering the action.
symbol : str
Treeherder symbol for the action callback, this is the symbol that the
task calling your callback will be displayed as. This is usually 1-3
letters abbreviating the action title.
description : str
A human readable description of the action in **markdown**.
This will be display as tooltip and in dialog window when the action
is triggered. This is a good place to describe how to use the action.
order : int
Order of the action in menus, this is relative to the ``order`` of
other actions declared.
context : list of dict
List of tag-sets specifying which tasks the action is can take as input.
If no tag-sets is specified as input the action is related to the
entire task-group, and won't be triggered with a given task.
Otherwise, if ``context = [{'k': 'b', 'p': 'l'}, {'k': 't'}]`` will only
be displayed in the context menu for tasks that has
``task.tags.k == 'b' && task.tags.p = 'l'`` or ``task.tags.k = 't'``.
Esentially, this allows filtering on ``task.tags``.
available : function
An optional function that given decision parameters decides if the
action is available. Defaults to a function that always returns ``True``.
schema : dict
JSON schema specifying input accepted by the action.
This is optional and can be left ``null`` if no input is taken.
Returns
-------
function
To be used as decorator for the callback function.
"""
mem = {"registered": False} # workaround nonlocal missing in 2.x
def register_callback(cb):
assert isinstance(cb, FunctionType), 'callback must be a function'
assert isinstance(symbol, basestring), 'symbol must be a string'
assert 1 <= len(symbol) <= 25, 'symbol must be between 1 and 25 characters'
assert not mem['registered'], 'register_callback_action must be used as decorator'
assert cb.__name__ not in callbacks, 'callback name {} is not unique'.format(cb.__name__)
source_path = os.path.relpath(inspect.stack()[1][1], GECKO)
@register_task_action(title, description, order, context, schema)
def build_callback_action_task(parameters):
if not available(parameters):
return None
return {
'created': {'$fromNow': ''},
'deadline': {'$fromNow': '12 hours'},
'expires': {'$fromNow': '14 days'},
'metadata': {
'owner': 'mozilla-taskcluster-maintenance@mozilla.com',
'source': '{}raw-file/{}/{}'.format(
parameters['head_repository'], parameters['head_rev'], source_path,
),
'name': 'Action: {}'.format(title),
'description': 'Task executing callback for action.\n\n---\n' + description,
},
'workerType': 'gecko-decision',
'provisionerId': 'aws-provisioner-v1',
'scopes': [
'assume:repo:hg.mozilla.org/projects/{}:*'.format(parameters['project']),
],
'tags': {
'createdForUser': parameters['owner'],
'kind': 'action-callback',
},
'routes': [
'tc-treeherder.v2.{}.{}.{}'.format(
parameters['project'], parameters['head_rev'], parameters['pushlog_id']),
'tc-treeherder-stage.v2.{}.{}.{}'.format(
parameters['project'], parameters['head_rev'], parameters['pushlog_id']),
],
'payload': {
'env': {
'GECKO_BASE_REPOSITORY': 'https://hg.mozilla.org/mozilla-unified',
'GECKO_HEAD_REPOSITORY': parameters['head_repository'],
'GECKO_HEAD_REF': parameters['head_ref'],
'GECKO_HEAD_REV': parameters['head_rev'],
'HG_STORE_PATH': '/home/worker/checkouts/hg-store',
'ACTION_TASK_GROUP_ID': {'$eval': 'taskGroupId'},
'ACTION_TASK_ID': {'$dumps': {'$eval': 'taskId'}},
'ACTION_TASK': {'$dumps': {'$eval': 'task'}},
'ACTION_INPUT': {'$dumps': {'$eval': 'input'}},
'ACTION_CALLBACK': cb.__name__,
'ACTION_PARAMETERS': {'$dumps': {'$eval': 'parameters'}},
},
'cache': {
'level-{}-checkouts'.format(parameters['level']):
'/home/worker/checkouts',
},
'features': {
'taskclusterProxy': True,
'chainOfTrust': True,
},
'image': docker_image('decision'),
'maxRunTime': 1800,
'command': [
'/home/worker/bin/run-task', '--vcs-checkout=/home/worker/checkouts/gecko',
'--', 'bash', '-cx',
"""\
cd /home/worker/checkouts/gecko &&
ln -s /home/worker/artifacts artifacts &&
./mach --log-no-times taskgraph action-callback """ + ' '.join([
"--pushlog-id='{}'".format(parameters['pushlog_id']),
"--pushdate='{}'".format(parameters['pushdate']),
"--project='{}'".format(parameters['project']),
"--message='{}'".format(parameters['message'].replace("'", "'\\''")),
"--owner='{}'".format(parameters['owner']),
"--level='{}'".format(parameters['level']),
"--base-repository='https://hg.mozilla.org/mozilla-central'",
"--head-repository='{}'".format(parameters['head_repository']),
"--head-ref='{}'".format(parameters['head_ref']),
"--head-rev='{}'".format(parameters['head_rev']),
"--revision-hash='{}'\n".format(parameters['head_rev']),
]),
],
},
'extra': {
'treeherder': {
'groupName': 'action-callback',
'groupSymbol': 'AC',
'symbol': symbol,
},
},
}
mem['registered'] = True
callbacks[cb.__name__] = cb
return register_callback
def render_actions_json(parameters):
"""
Render JSON object for the ``public/actions.json`` artifact.
Parameters
----------
parameters : taskgraph.parameters.Parameters
Decision task parameters.
Returns
-------
dict
JSON object representation of the ``public/actions.json`` artifact.
"""
global actions
assert isinstance(parameters, Parameters), 'requires instance of Parameters'
result = []
for action in sorted(actions, key=lambda action: action.order):
task = action.task_template_builder(parameters)
if task:
assert is_json(task), 'task must be a JSON compatible object'
result.append({
'title': action.title,
'description': action.description,
'context': action.context,
'schema': action.schema,
'task': task,
})
return {
'version': 1,
'variables': {
'parameters': dict(**parameters),
},
'actions': result,
}
def trigger_action_callback():
"""
Trigger action callback using arguments from environment variables.
"""
global callbacks
task_group_id = os.environ.get('ACTION_TASK_GROUP_ID', None)
task_id = json.loads(os.environ.get('ACTION_TASK_ID', 'null'))
task = json.loads(os.environ.get('ACTION_TASK', 'null'))
input = json.loads(os.environ.get('ACTION_INPUT', 'null'))
callback = os.environ.get('ACTION_CALLBACK', None)
parameters = json.loads(os.environ.get('ACTION_PARAMETERS', 'null'))
cb = callbacks.get(callback, None)
if not cb:
raise Exception('Unknown callback: {}'.format(callback))
cb(Parameters(**parameters), input, task_group_id, task_id, task)
# Load all modules from this folder, relying on the side-effects of register_
# functions to populate the action registry.
for f in os.listdir(os.path.dirname(__file__)):
if f.endswith('.py') and f not in ('__init__.py', 'registry.py'):
__import__('actions.' + f[:-3])

Просмотреть файл

@ -0,0 +1,229 @@
Writing Treeherder Actions in-tree
==================================
This document shows how to define an action in-tree such that it shows up in
supported user interfaces like Treeherder. For details on interface between
in-tree logic and external user interfaces, see
:doc:`the specification for actions.json <action-spec>`.
Creating a Callback Action
--------------------------
A *callback action* is an action that calls back into in-tree logic. That is
you register the action with title, description, context, input schema and a
python callback. When the action is triggered in a user interface,
input matching the schema is collected, passed to a new task which then calls
your python callback, enabling it to do pretty much anything it wants to.
To create a new action you must create a file
``/taskcluster/actions/my-action.py``, that at minimum contains::
from registry import register_callback_action
@register_callback_action(
title='Say Hello',
symbol='hw', # Show the callback task in treeherder as 'hw'
description="Simple **proof-of-concept** callback action",
order=10000, # Order in which it should appear relative to other actions
)
def hello_world_action(parameters, input, task_group_id, task_id, task):
# parameters is an instance of taskgraph.parameters.Parameters
# it carries decision task parameters from the original decision task.
# input, task_id, and task should all be None
print "Hello was triggered from taskGroupId: " + taskGroupId
The example above defines an action that is available in the context-menu for
the entire task-group (result-set or push in Treeherder terminology). To create
an action that shows up in the context menu for a task we must specify the
``context`` parameter.
Setting the Action Context
--------------------------
The context parameter should be a list of tag-sets, such as
``context=[{"platform": "linux"}]``, which will make the task show up in the
context-menu for any task with ``task.tags.platform = 'linux'``. Below is
some examples of context parameters and the resulting conditions on
``task.tags`` (tags used below are just illustrative).
``context=[{"platform": "linux"}]``:
Requires ``task.tags.platform = 'linux'``.
``context=[{"kind": "test", "platform": "linux"}]``:
Requires ``task.tags.platform = 'linux'`` **and** ``task.tags.kind = 'test'``.
``context=[{"kind": "test"}, {"platform": "linux"}]``:
Requires ``task.tags.platform = 'linux'`` **or** ``task.tags.kind = 'test'``.
``context=[{}]``:
Requires nothing and the action will show up in the context menu for all tasks.
``context=[]``:
Is the same as not setting the context parameter, which will make the action
show up in the context menu for the task-group.
(ie. the action is not specific to some task)
The example action below will be shown in the context-menu for tasks with
``task.tags.platform = 'linux'``::
from registry import register_callback_action
@register_callback_action(
title='Retrigger',
symbol='re-c', # Show the callback task in treeherder as 're-c'
description="Create a clone of the task",
order=1,
context=[{'platform': 'linux'}]
)
def retrigger_action(parameters, input, task_group_id, task_id, task):
# input will be None
print "Retriggering: {}".format(task_id)
print "task definition: {}".format(task)
When the ``context`` parameter is set, the ``task_id`` and ``task`` parameters
will provided to the callback. In this case the ``task_id`` and ``task``
parameters will be the ``taskId`` and *task definition* of the task from whose
context-menu the action was triggered.
Typically, the ``context`` parameter is used for actions that operates on
tasks, such as retriggering, running a specific test case, creating a loaner,
bisection, etc. You can think of the context as a place the action should
appear, but it's also very much a form of input the action can use.
Specifying an Input Schema
--------------------------
In call examples so far the ``input`` parameter for the callback have been
``None``, to make an action that takes input you must specify an input schema.
This is done by passing a JSON schema as the ``schema`` parameter.
When designing a schema for the input it is important to exploit as many of the
JSON schema validation features as reasonably possible. Furthermore, it is
*strongly* encouraged that the ``title`` and ``description`` properties in
JSON schemas is used to provide a detailed explanation of what the input
value will do. Authors can reasonably expect JSON schema ``description``
properties to be rendered as markdown before being presented.
The example below illustrates how to specify an input schema. Notice that while
this example doesn't specify a ``context`` it is perfectly legal to specify
both ``input`` and ``context``::
from registry import register_callback_action
@register_callback_action(
title='Run All Tasks',
symbol='ra-c', # Show the callback task in treeherder as 'ra-c'
description="**Run all tasks** that have been _optimized_ away.",
order=1,
input={
'title': 'Action Options',
'description': 'Options for how you wish to run all tasks',
'properties': {
'priority': {
'title': 'priority'
'description': 'Priority that should be given to the tasks',
'type': 'string',
'enum': ['low', 'normal', 'high'],
'default': 'low',
},
'runTalos': {
'title': 'Run Talos'
'description': 'Do you wish to also include talos tasks?',
'type': 'boolean',
'default': 'false',
}
},
'required': ['priority', 'runTalos'],
'additionalProperties': False,
},
)
def retrigger_action(parameters, input, task_group_id, task_id, task):
print "Create all pruned tasks with priority: {}".format(input['priority'])
if input['runTalos']:
print "Also running talos jobs..."
When the ``schema`` parameter is given the callback will always be called with
an ``input`` parameter that satisfies the previously given JSON schema.
It is encouraged to set ``additionalProperties: false``, as well as specifying
all properties as ``required`` in the JSON schema. Furthermore, it's good
practice to provide ``default`` values for properties, user interface generators
will often take advantage of such properties.
Once you have specified input and context as applicable for your action you can
do pretty much anything you want from within your callback. Whether you want
to create one or more tasks or run a specific piece of code like a test.
Conditional Availability
------------------------
The decision parameters ``taskgraph.parameters.Parameters`` passed to
the callback is also available when the decision task generates the list of
actions to be displayed in the user interface. When registering an action
callback the ``availability`` parameter can be used to specify a lambda function
that given the decision parameters determines if the action should be available.
The feature is illustrated below::
from registry import register_callback_action
@register_callback_action(
title='Say Hello',
symbol='hw', # Show the callback task in treeherder as 'hw'
description="Simple **proof-of-concept** callback action",
order=2,
# Define an action that is only included if this is a push to try
available=lambda parameters: parameters.get('project', None) == 'try',
)
def try_only_action(parameters, input, task_group_id, task_id, task):
print "My try-only action"
Properties of ``parameters`` is documented in the
:doc:`parameters section <parameters>`. You can also checkout the
``parameters.yml`` artifact created by decisions tasks.
Skipping the Action Callback
----------------------------
It is possible to define an action that doesn't take a callback, instead you'll
then have to provide a task template. For details on how the task template
language works refer to :doc:`the specification for actions.json <action-spec>`,
the example below illustrates how to create such an action::
from registry import register_task_action
@register_task_action(
title='Retrigger',
description="Create a clone of the task",
order=1,
context=[{'platform': 'linux'}],
input={
'title': 'priority'
'description': 'Priority that should be given to the tasks',
'type': 'string',
'enum': ['low', 'normal', 'high'],
'default': 'low',
},
def task_template_builder(parameters):
# The task template builder may return None to signal that the action
# isn't available.
if parameters.get('project', None) != 'try':
return None
return {
'created': {'$fromNow': ''},
'deadline': {'$fromNow': '1 hour'},
'expires': {'$fromNow': '14 days'},
'provisionerId': '...',
'workerType': '...',
'priority': '${input}',
'payload': {
'command': '...',
'env': {
'TASK_DEFINITION': {'$json': {'eval': 'task'}}
},
...
},
# It's now your responsibility to include treeherder routes, as well
# additional metadata for treeherder in task.extra.treeherder.
...
},
)
This kind of actions is rarely useful due to the limited expressiveness of the
template language. For further details on the template see
:doc:`the specification for actions.json <action-spec>`. Obviously, this kind of
action can be slightly more responsive because it doesn't create an intermediate
task in-order to trigger a python callback inside the tree.

Просмотреть файл

@ -28,5 +28,6 @@ check out the :doc:`how-to section <how-tos>`.
docker-images
cron
how-tos
in-tree-actions
action-spec
reference

Просмотреть файл

@ -302,6 +302,12 @@ class MachCommands(MachCommandBase):
traceback.print_exc()
sys.exit(1)
@SubCommand('taskgraph', 'action-callback',
description='Run action callback used by action tasks')
def action_callback(self, **options):
import actions
actions.trigger_action_callback()
def setup_logging(self, quiet=False, verbose=True):
"""
Set up Python logging for all loggers, sending results to stderr (so

Просмотреть файл

@ -17,6 +17,7 @@ from .generator import TaskGraphGenerator
from .create import create_tasks
from .parameters import Parameters
from .taskgraph import TaskGraph
from actions import render_actions_json
from taskgraph.util.templates import Templates
from taskgraph.util.time import (
@ -87,6 +88,9 @@ def taskgraph_decision(options):
# write out the yml file for action tasks
write_artifact('action.yml', get_action_yml(parameters))
# write out the public/actions.json file
write_artifact('actions.json', render_actions_json(parameters))
# write out the full graph for reference
full_task_json = tgg.full_task_graph.to_json()
write_artifact('full-task-graph.json', full_task_json)