From 06a6c31865fe24dfe24b2b786b16d0e6531a76de Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Thu, 25 Oct 2018 18:39:22 +0000 Subject: [PATCH] Bug 1497575: [staging-release] Document new `try_task_config.json` format; r=ahal Differential Revision: https://phabricator.services.mozilla.com/D9680 --HG-- extra : moz-landing-system : lando --- taskcluster/docs/try.rst | 49 +++++++++++++++++++++++++++---- taskcluster/taskgraph/decision.py | 26 ++++++++++++++-- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/taskcluster/docs/try.rst b/taskcluster/docs/try.rst index a6fd41772a7a..8e6ed3fd7b67 100644 --- a/taskcluster/docs/try.rst +++ b/taskcluster/docs/try.rst @@ -36,10 +36,10 @@ Try Task Config ::::::::::::::: The second, more modern method specifies exactly the tasks to run. That list -of tasks is usually generated locally with some `local tool `_ and -attached to the commit pushed to the try repository. This gives finer control -over exactly what runs and enables growth of an ecosystem of tooling -appropriate to varied circumstances. +of tasks is usually generated locally with some :doc:`local tool ` +and attached to the commit pushed to the try repository. This gives +finer control over exactly what runs and enables growth of an +ecosystem of tooling appropriate to varied circumstances. Implementation ,,,,,,,,,,,,,, @@ -52,6 +52,7 @@ at the root of the source dir. The JSON object in this file contains a .. parsed-literal:: { + "version": 1, "tasks": [ "test-windows10-64/opt-web-platform-tests-12", "test-windows7-32/opt-reftest-1", @@ -64,7 +65,7 @@ at the root of the source dir. The JSON object in this file contains a Very simply, this will run any task label that gets passed in as well as their dependencies. While it is possible to manually commit this file and push to -try, it is mainly meant to be a generation target for various `tryselect`_ +try, it is mainly meant to be a generation target for various :doc:`tryselect ` choosers. For example: .. parsed-literal:: @@ -94,6 +95,7 @@ from the ``try_task_config.json`` like this: .. parsed-literal:: { + "version": 1, "tasks": [...], "templates": { artifact: {"enabled": 1} @@ -134,9 +136,44 @@ parameter is None and no tasks are selected to run. The resulting push will only have a decision task, but one with an "add jobs" action that can be used to add the desired jobs to the try push. -.. _tryselect: https://dxr.mozilla.org/mozilla-central/source/tools/tryselect + +Complex Configuration +::::::::::::::::::::: + +If you need more control over the build configuration, +(:doc:`staging releases `, for example), +you can directly specify :doc:`parameters ` +to override from the ``try_task_config.json`` like this: + +.. parsed-literal:: + + { + "version": 2, + "parameters": { + "optimize_target_tasks": true, + "release_type": "beta", + "target_tasks_method": "staging_release_builds" + } + } + +This format can express a superset of the version 1 format, as the +version one configuration is equivalent to the following version 2 +config. + +.. parsed-literal:: + + { + "version": 2, + "parameters": { + "try_task_config": {...}, + "try_mode": "try_task_config", + } + } + .. _JSON-e: https://taskcluster.github.io/json-e/ .. _taskgraph module: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates .. _condition statements: https://taskcluster.github.io/json-e/#%60$if%60%20-%20%60then%60%20-%20%60else%60 .. _existing templates: https://dxr.mozilla.org/mozilla-central/source/taskcluster/taskgraph/templates .. _SCM Level: https://www.mozilla.org/en-US/about/governance/policies/commit/access-policy/ + + diff --git a/taskcluster/taskgraph/decision.py b/taskcluster/taskgraph/decision.py index f0063242f32f..5d81a3fc0b7d 100644 --- a/taskcluster/taskgraph/decision.py +++ b/taskcluster/taskgraph/decision.py @@ -18,9 +18,11 @@ from .parameters import Parameters, get_version, get_app_version from .taskgraph import TaskGraph from .try_option_syntax import parse_message from .actions import render_actions_json -from taskgraph.util.partials import populate_release_history -from taskgraph.util.yaml import load_yaml +from .util.partials import populate_release_history +from .util.yaml import load_yaml +from .util.schema import validate_schema, Schema +from voluptuous import Required, Optional logger = logging.getLogger(__name__) @@ -106,6 +108,16 @@ PER_PROJECT_PARAMETERS = { } } +try_task_config_schema = Schema({ + Required('tasks'): [basestring], + Optional('templates'): {basestring: object}, +}) + + +try_task_config_schema_v2 = Schema({ + Optional('parameters'): {basestring: object}, +}) + def full_task_graph_to_runnable_jobs(full_task_json): runnable_jobs = {} @@ -278,11 +290,19 @@ def set_try_config(parameters, task_config_file): logger.info("using try tasks from {}".format(task_config_file)) with open(task_config_file, 'r') as fh: task_config = json.load(fh) - task_config_version = task_config.get('version', 1) + task_config_version = task_config.pop('version', 1) if task_config_version == 1: + validate_schema( + try_task_config_schema, task_config, + "Invalid v1 `try_task_config.json`.", + ) parameters['try_mode'] = 'try_task_config' parameters['try_task_config'] = task_config elif task_config_version == 2: + validate_schema( + try_task_config_schema_v2, task_config, + "Invalid v1 `try_task_config.json`.", + ) parameters.update(task_config['parameters']) return else: