From 02106e80c93612251c413913ba0badec3bc99863 Mon Sep 17 00:00:00 2001 From: Justin Wood Date: Wed, 21 Dec 2016 11:21:06 -0500 Subject: [PATCH] Bug 1322041 - Add signing support for all nightlies. r=dustin Involved work which landed on the date project branch with: * Bug 1277579, by kmoir@mozilla.com (https://hg.mozilla.org/projects/date/rev/afd3823c852b, https://hg.mozilla.org/projects/date/rev/345f83708453, https://hg.mozilla.org/projects/date/rev/8a3ed233af86, https://hg.mozilla.org/projects/date/rev/0d737cf8e743, https://hg.mozilla.org/projects/date/rev/e42b3d606002, https://hg.mozilla.org/projects/date/rev/c3a160ac642b) * Bug 1305096, by kmoir@mozilla.com (https://hg.mozilla.org/projects/date/rev/2287a2568038) * Bug 1277579, by dustin@mozilla.com (https://hg.mozilla.org/projects/date/rev/79ceb7fa0589) * Bug 1306166, by kmoir@mozilla.com (https://hg.mozilla.org/projects/date/rev/d3fd1966095d, https://hg.mozilla.org/projects/date/rev/2f52061c02e6, https://hg.mozilla.org/projects/date/rev/a9f52549a3c3, https://hg.mozilla.org/projects/date/rev/f69efa90945a) * Bug 1314847, by Callek@gmail.com (https://hg.mozilla.org/projects/date/rev/4a1231655fbb, https://hg.mozilla.org/projects/date/rev/3cd3a0d32f43) * Bug 1312000, by Callek@gmail.com (https://hg.mozilla.org/projects/date/rev/79a2f66ff5c0) * Bug 1312500, by Callek@gmail.com (https://hg.mozilla.org/projects/date/rev/6ae07fa4b011) * Bug 1316214, by Callek@gmail.com (https://hg.mozilla.org/projects/date/rev/edae37481cab, https://hg.mozilla.org/projects/date/rev/62bd3371e954) * Bug 1319189, by Callek@gmail.com (https://hg.mozilla.org/projects/date/rev/4c33f8ccecf5) * Bug 1319546, by kmoir@mozilla.com (https://hg.mozilla.org/projects/date/rev/70a23d243d2c) * No Bug, by asasaki@mozilla.com (https://hg.mozilla.org/projects/date/rev/5d8ba3560ae9) * No Bug, by kmoir@mozilla.com (https://hg.mozilla.org/projects/date/rev/37d9733a7174) MozReview-Commit-ID: K1uOY4HOWPX --HG-- extra : rebase_source : 567392d5d5ddb5ee638c53221a6e545e7b5f1805 --- .../ci/build-signing/android-signing.yml | 26 ---- taskcluster/ci/build-signing/kind.yml | 8 +- taskcluster/docs/transforms.rst | 15 +++ taskcluster/taskgraph/task/signing.py | 70 +++-------- .../taskgraph/transforms/build_signing.py | 54 ++++++++ taskcluster/taskgraph/transforms/signing.py | 118 ++++++++++++++++++ taskcluster/taskgraph/transforms/task.py | 31 +++++ 7 files changed, 239 insertions(+), 83 deletions(-) delete mode 100644 taskcluster/ci/build-signing/android-signing.yml create mode 100644 taskcluster/taskgraph/transforms/build_signing.py create mode 100644 taskcluster/taskgraph/transforms/signing.py diff --git a/taskcluster/ci/build-signing/android-signing.yml b/taskcluster/ci/build-signing/android-signing.yml deleted file mode 100644 index 2dd64260d953..000000000000 --- a/taskcluster/ci/build-signing/android-signing.yml +++ /dev/null @@ -1,26 +0,0 @@ -signing-nightly-fennec: - task: - provisionerId: "scriptworker-prov-v1" - workerType: "signing-linux-v1" - scopes: - - "project:releng:signing:cert:dep-signing" - - "project:releng:signing:format:jar" - created: - relative-datestamp: "0 seconds" - deadline: - relative-datestamp: "24 hours" - payload: - unsignedArtifacts: [] - maxRunTime: 600 - metadata: - name: "Signing Scriptworker Task" - description: "Sign Android Build Tasks" - owner: "jlund@mozilla.com" - source: "https://tools.taskcluster.net/task-creator/" - attributes: - nightly: true - unsigned-task: - label: "build-android-api-15-nightly/opt" - artifacts: - - "public/build/target.apk" - - "public/build/en-US/target.apk" diff --git a/taskcluster/ci/build-signing/kind.yml b/taskcluster/ci/build-signing/kind.yml index d2ca9886e0a8..0cdaefef0034 100644 --- a/taskcluster/ci/build-signing/kind.yml +++ b/taskcluster/ci/build-signing/kind.yml @@ -2,10 +2,12 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -implementation: 'taskgraph.task.signing:SigningTask' +implementation: taskgraph.task.signing:SigningTask -jobs-from: - - android-signing.yml +transforms: + - taskgraph.transforms.build_signing:transforms + - taskgraph.transforms.signing:transforms + - taskgraph.transforms.task:transforms kind-dependencies: - build diff --git a/taskcluster/docs/transforms.rst b/taskcluster/docs/transforms.rst index 8368faa5a13a..e68f80703935 100644 --- a/taskcluster/docs/transforms.rst +++ b/taskcluster/docs/transforms.rst @@ -186,6 +186,21 @@ The ``task.py`` file also contains a dictionary mapping treeherder groups to group names using an internal list of group names. Feel free to add additional groups to this list as necessary. +Signing Descriptions +-------------------- + +Signing kinds are passed a single dependent job (from its kind dependency) to act +on. + +The transforms in ``taskcluster/taskgraph/transforms/signing.py`` implement +this common functionality. They expect a "signing description", and produce a +task definition. The schema for a signing description is defined at the top of +``signing.py``, with copious comments. + +In particular you define a set of upstream artifact urls (that point at the dependent +task) and can optionally provide a dependent name (defaults to build) for use in +task-reference. You also need to provide the signing formats to use. + More Detail ----------- diff --git a/taskcluster/taskgraph/task/signing.py b/taskcluster/taskgraph/task/signing.py index a2a9ae3d666a..53d4e584fd94 100644 --- a/taskcluster/taskgraph/task/signing.py +++ b/taskcluster/taskgraph/task/signing.py @@ -4,61 +4,23 @@ from __future__ import absolute_import, print_function, unicode_literals -import logging -import os - -from . import base -from taskgraph.util.templates import Templates +from . import transform -logger = logging.getLogger(__name__) -GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..')) -ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' -INDEX_URL = 'https://index.taskcluster.net/v1/task/{}' - - -class SigningTask(base.Task): - - def __init__(self, kind, name, task, attributes): - self.unsigned_artifact_label = task['unsigned-task']['label'] - super(SigningTask, self).__init__(kind, name, task=task['task'], - attributes=attributes) +class SigningTask(transform.TransformTask): + """ + A task implementing a signing job. These depend on nightly build jobs and + sign the artifacts after a build has completed. + """ @classmethod - def load_tasks(cls, kind, path, config, params, loaded_tasks): - root = os.path.abspath(path) - - tasks = [] - for filename in config.get('jobs-from', []): - templates = Templates(root) - jobs = templates.load(filename, {}) - - for name, job in jobs.iteritems(): - for artifact in job['unsigned-task']['artifacts']: - url = ARTIFACT_URL.format('<{}>'.format('unsigned-artifact'), artifact) - job['task']['payload']['unsignedArtifacts'].append({ - 'task-reference': url - }) - attributes = job.setdefault('attributes', {}) - attributes.update({'kind': 'signing'}) - tasks.append(cls(kind, name, job, attributes=attributes)) - - return tasks - - def get_dependencies(self, taskgraph): - return [(self.unsigned_artifact_label, 'unsigned-artifact')] - - def optimize(self, params): - return False, None - - @classmethod - def from_json(cls, task_dict): - unsigned_task_label = task_dict['dependencies']['unsigned-artifact'] - task_dict['unsigned-task'] = { - 'label': unsigned_task_label - } - signing_task = cls(kind='build-signing', - name=task_dict['label'], - attributes=task_dict['attributes'], - task=task_dict) - return signing_task + def get_inputs(cls, kind, path, config, params, loaded_tasks): + if (config.get('kind-dependencies', []) != ["build"]): + raise Exception("Signing kinds must depend on builds") + for task in loaded_tasks: + if task.kind not in config.get('kind-dependencies'): + continue + if not task.attributes.get('nightly'): + continue + signing_task = {'dependent-task': task} + yield signing_task diff --git a/taskcluster/taskgraph/transforms/build_signing.py b/taskcluster/taskgraph/transforms/build_signing.py new file mode 100644 index 000000000000..c4c79a3de02a --- /dev/null +++ b/taskcluster/taskgraph/transforms/build_signing.py @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the signing task into an actual task description. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def make_signing_description(config, jobs): + for job in jobs: + dep_job = job['dependent-task'] + + if 'android' in dep_job.attributes.get('build_platform'): + job_specs = [ + { + 'artifacts': ['public/build/target.apk', + 'public/build/en-US/target.apk'], + 'format': 'jar', + }, + ] + else: + job_specs = [ + { + 'artifacts': ['public/build/target.tar.bz2', + 'public/build/target.checksums'], + 'format': 'gpg', + }, { + 'artifacts': ['public/build/update/target.complete.mar'], + 'format': 'mar', + } + ] + upstream_artifacts = [] + for spec in job_specs: + fmt = spec["format"] + upstream_artifacts.append({ + "taskId": {"task-reference": ""}, + "taskType": "build", + "paths": spec["artifacts"], + "formats": [fmt] + }) + + job['upstream-artifacts'] = upstream_artifacts + + label = dep_job.label.replace("build-", "signing-") + job['label'] = label + + yield job diff --git a/taskcluster/taskgraph/transforms/signing.py b/taskcluster/taskgraph/transforms/signing.py new file mode 100644 index 000000000000..89f05dfb581a --- /dev/null +++ b/taskcluster/taskgraph/transforms/signing.py @@ -0,0 +1,118 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the signing task into an actual task description. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import ( + validate_schema, + TransformSequence +) +from taskgraph.transforms.task import task_description_schema +from voluptuous import Schema, Any, Required, Optional + + +ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/{}' + + +# Voluptuous uses marker objects as dictionary *keys*, but they are not +# comparable, so we cast all of the keys back to regular strings +task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} + +transforms = TransformSequence() + +# shortcut for a string where task references are allowed +taskref_or_string = Any( + basestring, + {Required('task-reference'): basestring}) + +signing_description_schema = Schema({ + # the dependant task (object) for this signing job, used to inform signing. + Required('dependent-task'): object, + + # Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py + # because this is passed directly into the signingscript worker + Required('upstream-artifacts'): [{ + # taskId of the task with the artifact + Required('taskId'): taskref_or_string, + + # type of signing task (for CoT) + Required('taskType'): basestring, + + # Paths to the artifacts to sign + Required('paths'): [basestring], + + # Signing formats to use on each of the paths + Required('formats'): [basestring], + }], + + # depname is used in taskref's to identify the taskID of the unsigned things + Required('depname', default='build'): basestring, + + # unique label to describe this signing task, defaults to {dep.label}-signing + Optional('label'): basestring, + + # treeherder is allowed here to override any defaults we use for signing. See + # taskcluster/taskgraph/transforms/task.py for the schema details, and the + # below transforms for defaults of various values. + Optional('treeherder'): task_description_schema['treeherder'], +}) + + +@transforms.add +def validate(config, jobs): + for job in jobs: + label = job.get('dependent-task', object).__dict__.get('label', '?no-label?') + yield validate_schema( + signing_description_schema, job, + "In signing ({!r} kind) task for {!r}:".format(config.kind, label)) + + +@transforms.add +def make_task_description(config, jobs): + for job in jobs: + dep_job = job['dependent-task'] + + signing_format_scopes = [] + formats = set([]) + for artifacts in job['upstream-artifacts']: + for f in artifacts['formats']: + formats.update(f) # Add each format only once + for format in formats: + signing_format_scopes.append("project:releng:signing:format:{}".format(format)) + + treeherder = job.get('treeherder', {}) + treeherder.setdefault('symbol', 'tc(Ns)') + dep_th_platform = dep_job.task.get('extra', {}).get( + 'treeherder', {}).get('machine', {}).get('platform', '') + treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) + treeherder.setdefault('tier', 2) + treeherder.setdefault('kind', 'build') + + label = job.get('label', "{}-signing".format(dep_job.label)) + + attributes = { + 'nightly': dep_job.attributes.get('nightly', False), + 'build_platform': dep_job.attributes.get('build_platform'), + 'build_type': dep_job.attributes.get('build_type'), + } + + task = { + 'label': label, + 'description': "{} Signing".format( + dep_job.task["metadata"]["description"]), + 'worker-type': "scriptworker-prov-v1/signing-linux-v1", + 'worker': {'implementation': 'scriptworker-signing', + 'upstream-artifacts': job['upstream-artifacts'], + 'max-run-time': 3600}, + 'scopes': ["project:releng:signing:cert:nightly-signing"] + signing_format_scopes, + 'dependencies': {job['depname']: dep_job.label}, + 'attributes': attributes, + 'run-on-projects': dep_job.attributes.get('run_on_projects'), + 'treeherder': treeherder, + } + + yield task diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py index d8710afaf609..f7213965c0f7 100644 --- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -22,6 +22,7 @@ from voluptuous import Schema, Any, Required, Optional, Extra from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR + # shortcut for a string where task references are allowed taskref_or_string = Any( basestring, @@ -264,6 +265,26 @@ task_description_schema = Schema({ # type=directory) Required('name'): basestring, }], + }, { + Required('implementation'): 'scriptworker-signing', + + # the maximum time to spend signing, in seconds + Required('max-run-time', default=600): int, + + # list of artifact URLs for the artifacts that should be signed + Required('upstream-artifacts'): [{ + # taskId of the task with the artifact + Required('taskId'): taskref_or_string, + + # type of signing task (for CoT) + Required('taskType'): basestring, + + # Paths to the artifacts to sign + Required('paths'): [basestring], + + # Signing formats to use on each of the paths + Required('formats'): [basestring], + }], }), # The "when" section contains descriptions of the circumstances @@ -452,6 +473,16 @@ def build_generic_worker_payload(config, task, task_def): raise Exception("retry-exit-status not supported in generic-worker") +@payload_builder('scriptworker-signing') +def build_scriptworker_signing_payload(config, task, task_def): + worker = task['worker'] + + task_def['payload'] = { + 'maxRunTime': worker['max-run-time'], + 'upstreamArtifacts': worker['upstream-artifacts'] + } + + @payload_builder('macosx-engine') def build_macosx_engine_payload(config, task, task_def): worker = task['worker']