зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1322041 - Add signing support for all nightlies. r=dustin
Involved work which landed on the date project branch with: * Bug 1277579, by kmoir@mozilla.com (https://hg.mozilla.org/projects/date/rev/afd3823c852b, https://hg.mozilla.org/projects/date/rev/345f83708453, https://hg.mozilla.org/projects/date/rev/8a3ed233af86, https://hg.mozilla.org/projects/date/rev/0d737cf8e743, https://hg.mozilla.org/projects/date/rev/e42b3d606002, https://hg.mozilla.org/projects/date/rev/c3a160ac642b) * Bug 1305096, by kmoir@mozilla.com (https://hg.mozilla.org/projects/date/rev/2287a2568038) * Bug 1277579, by dustin@mozilla.com (https://hg.mozilla.org/projects/date/rev/79ceb7fa0589) * Bug 1306166, by kmoir@mozilla.com (https://hg.mozilla.org/projects/date/rev/d3fd1966095d, https://hg.mozilla.org/projects/date/rev/2f52061c02e6, https://hg.mozilla.org/projects/date/rev/a9f52549a3c3, https://hg.mozilla.org/projects/date/rev/f69efa90945a) * Bug 1314847, by Callek@gmail.com (https://hg.mozilla.org/projects/date/rev/4a1231655fbb, https://hg.mozilla.org/projects/date/rev/3cd3a0d32f43) * Bug 1312000, by Callek@gmail.com (https://hg.mozilla.org/projects/date/rev/79a2f66ff5c0) * Bug 1312500, by Callek@gmail.com (https://hg.mozilla.org/projects/date/rev/6ae07fa4b011) * Bug 1316214, by Callek@gmail.com (https://hg.mozilla.org/projects/date/rev/edae37481cab, https://hg.mozilla.org/projects/date/rev/62bd3371e954) * Bug 1319189, by Callek@gmail.com (https://hg.mozilla.org/projects/date/rev/4c33f8ccecf5) * Bug 1319546, by kmoir@mozilla.com (https://hg.mozilla.org/projects/date/rev/70a23d243d2c) * No Bug, by asasaki@mozilla.com (https://hg.mozilla.org/projects/date/rev/5d8ba3560ae9) * No Bug, by kmoir@mozilla.com (https://hg.mozilla.org/projects/date/rev/37d9733a7174) MozReview-Commit-ID: K1uOY4HOWPX --HG-- extra : rebase_source : 567392d5d5ddb5ee638c53221a6e545e7b5f1805
This commit is contained in:
Justin Wood
Родитель
a330ac85ae
Коммит
02106e80c9
|
|
@ -1,26 +0,0 @@
|
|||
signing-nightly-fennec:
|
||||
task:
|
||||
provisionerId: "scriptworker-prov-v1"
|
||||
workerType: "signing-linux-v1"
|
||||
scopes:
|
||||
- "project:releng:signing:cert:dep-signing"
|
||||
- "project:releng:signing:format:jar"
|
||||
created:
|
||||
relative-datestamp: "0 seconds"
|
||||
deadline:
|
||||
relative-datestamp: "24 hours"
|
||||
payload:
|
||||
unsignedArtifacts: []
|
||||
maxRunTime: 600
|
||||
metadata:
|
||||
name: "Signing Scriptworker Task"
|
||||
description: "Sign Android Build Tasks"
|
||||
owner: "jlund@mozilla.com"
|
||||
source: "https://tools.taskcluster.net/task-creator/"
|
||||
attributes:
|
||||
nightly: true
|
||||
unsigned-task:
|
||||
label: "build-android-api-15-nightly/opt"
|
||||
artifacts:
|
||||
- "public/build/target.apk"
|
||||
- "public/build/en-US/target.apk"
|
||||
|
|
@ -2,10 +2,12 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
implementation: 'taskgraph.task.signing:SigningTask'
|
||||
implementation: taskgraph.task.signing:SigningTask
|
||||
|
||||
jobs-from:
|
||||
- android-signing.yml
|
||||
transforms:
|
||||
- taskgraph.transforms.build_signing:transforms
|
||||
- taskgraph.transforms.signing:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
- build
|
||||
|
|
|
|||
|
|
@ -186,6 +186,21 @@ The ``task.py`` file also contains a dictionary mapping treeherder groups to
|
|||
group names using an internal list of group names. Feel free to add additional
|
||||
groups to this list as necessary.
|
||||
|
||||
Signing Descriptions
|
||||
--------------------
|
||||
|
||||
Signing kinds are passed a single dependent job (from its kind dependency) to act
|
||||
on.
|
||||
|
||||
The transforms in ``taskcluster/taskgraph/transforms/signing.py`` implement
|
||||
this common functionality. They expect a "signing description", and produce a
|
||||
task definition. The schema for a signing description is defined at the top of
|
||||
``signing.py``, with copious comments.
|
||||
|
||||
In particular you define a set of upstream artifact urls (that point at the dependent
|
||||
task) and can optionally provide a dependent name (defaults to build) for use in
|
||||
task-reference. You also need to provide the signing formats to use.
|
||||
|
||||
More Detail
|
||||
-----------
|
||||
|
||||
|
|
|
|||
|
|
@ -4,61 +4,23 @@
|
|||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from . import base
|
||||
from taskgraph.util.templates import Templates
|
||||
from . import transform
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
|
||||
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
|
||||
INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
|
||||
|
||||
|
||||
class SigningTask(base.Task):
|
||||
|
||||
def __init__(self, kind, name, task, attributes):
|
||||
self.unsigned_artifact_label = task['unsigned-task']['label']
|
||||
super(SigningTask, self).__init__(kind, name, task=task['task'],
|
||||
attributes=attributes)
|
||||
class SigningTask(transform.TransformTask):
|
||||
"""
|
||||
A task implementing a signing job. These depend on nightly build jobs and
|
||||
sign the artifacts after a build has completed.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def load_tasks(cls, kind, path, config, params, loaded_tasks):
|
||||
root = os.path.abspath(path)
|
||||
|
||||
tasks = []
|
||||
for filename in config.get('jobs-from', []):
|
||||
templates = Templates(root)
|
||||
jobs = templates.load(filename, {})
|
||||
|
||||
for name, job in jobs.iteritems():
|
||||
for artifact in job['unsigned-task']['artifacts']:
|
||||
url = ARTIFACT_URL.format('<{}>'.format('unsigned-artifact'), artifact)
|
||||
job['task']['payload']['unsignedArtifacts'].append({
|
||||
'task-reference': url
|
||||
})
|
||||
attributes = job.setdefault('attributes', {})
|
||||
attributes.update({'kind': 'signing'})
|
||||
tasks.append(cls(kind, name, job, attributes=attributes))
|
||||
|
||||
return tasks
|
||||
|
||||
def get_dependencies(self, taskgraph):
|
||||
return [(self.unsigned_artifact_label, 'unsigned-artifact')]
|
||||
|
||||
def optimize(self, params):
|
||||
return False, None
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, task_dict):
|
||||
unsigned_task_label = task_dict['dependencies']['unsigned-artifact']
|
||||
task_dict['unsigned-task'] = {
|
||||
'label': unsigned_task_label
|
||||
}
|
||||
signing_task = cls(kind='build-signing',
|
||||
name=task_dict['label'],
|
||||
attributes=task_dict['attributes'],
|
||||
task=task_dict)
|
||||
return signing_task
|
||||
def get_inputs(cls, kind, path, config, params, loaded_tasks):
|
||||
if (config.get('kind-dependencies', []) != ["build"]):
|
||||
raise Exception("Signing kinds must depend on builds")
|
||||
for task in loaded_tasks:
|
||||
if task.kind not in config.get('kind-dependencies'):
|
||||
continue
|
||||
if not task.attributes.get('nightly'):
|
||||
continue
|
||||
signing_task = {'dependent-task': task}
|
||||
yield signing_task
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Transform the signing task into an actual task description.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def make_signing_description(config, jobs):
|
||||
for job in jobs:
|
||||
dep_job = job['dependent-task']
|
||||
|
||||
if 'android' in dep_job.attributes.get('build_platform'):
|
||||
job_specs = [
|
||||
{
|
||||
'artifacts': ['public/build/target.apk',
|
||||
'public/build/en-US/target.apk'],
|
||||
'format': 'jar',
|
||||
},
|
||||
]
|
||||
else:
|
||||
job_specs = [
|
||||
{
|
||||
'artifacts': ['public/build/target.tar.bz2',
|
||||
'public/build/target.checksums'],
|
||||
'format': 'gpg',
|
||||
}, {
|
||||
'artifacts': ['public/build/update/target.complete.mar'],
|
||||
'format': 'mar',
|
||||
}
|
||||
]
|
||||
upstream_artifacts = []
|
||||
for spec in job_specs:
|
||||
fmt = spec["format"]
|
||||
upstream_artifacts.append({
|
||||
"taskId": {"task-reference": "<build>"},
|
||||
"taskType": "build",
|
||||
"paths": spec["artifacts"],
|
||||
"formats": [fmt]
|
||||
})
|
||||
|
||||
job['upstream-artifacts'] = upstream_artifacts
|
||||
|
||||
label = dep_job.label.replace("build-", "signing-")
|
||||
job['label'] = label
|
||||
|
||||
yield job
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Transform the signing task into an actual task description.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import (
|
||||
validate_schema,
|
||||
TransformSequence
|
||||
)
|
||||
from taskgraph.transforms.task import task_description_schema
|
||||
from voluptuous import Schema, Any, Required, Optional
|
||||
|
||||
|
||||
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/<{}>/artifacts/{}'
|
||||
|
||||
|
||||
# Voluptuous uses marker objects as dictionary *keys*, but they are not
|
||||
# comparable, so we cast all of the keys back to regular strings
|
||||
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
# shortcut for a string where task references are allowed
|
||||
taskref_or_string = Any(
|
||||
basestring,
|
||||
{Required('task-reference'): basestring})
|
||||
|
||||
signing_description_schema = Schema({
|
||||
# the dependant task (object) for this signing job, used to inform signing.
|
||||
Required('dependent-task'): object,
|
||||
|
||||
# Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py
|
||||
# because this is passed directly into the signingscript worker
|
||||
Required('upstream-artifacts'): [{
|
||||
# taskId of the task with the artifact
|
||||
Required('taskId'): taskref_or_string,
|
||||
|
||||
# type of signing task (for CoT)
|
||||
Required('taskType'): basestring,
|
||||
|
||||
# Paths to the artifacts to sign
|
||||
Required('paths'): [basestring],
|
||||
|
||||
# Signing formats to use on each of the paths
|
||||
Required('formats'): [basestring],
|
||||
}],
|
||||
|
||||
# depname is used in taskref's to identify the taskID of the unsigned things
|
||||
Required('depname', default='build'): basestring,
|
||||
|
||||
# unique label to describe this signing task, defaults to {dep.label}-signing
|
||||
Optional('label'): basestring,
|
||||
|
||||
# treeherder is allowed here to override any defaults we use for signing. See
|
||||
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
|
||||
# below transforms for defaults of various values.
|
||||
Optional('treeherder'): task_description_schema['treeherder'],
|
||||
})
|
||||
|
||||
|
||||
@transforms.add
|
||||
def validate(config, jobs):
|
||||
for job in jobs:
|
||||
label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
|
||||
yield validate_schema(
|
||||
signing_description_schema, job,
|
||||
"In signing ({!r} kind) task for {!r}:".format(config.kind, label))
|
||||
|
||||
|
||||
@transforms.add
|
||||
def make_task_description(config, jobs):
|
||||
for job in jobs:
|
||||
dep_job = job['dependent-task']
|
||||
|
||||
signing_format_scopes = []
|
||||
formats = set([])
|
||||
for artifacts in job['upstream-artifacts']:
|
||||
for f in artifacts['formats']:
|
||||
formats.update(f) # Add each format only once
|
||||
for format in formats:
|
||||
signing_format_scopes.append("project:releng:signing:format:{}".format(format))
|
||||
|
||||
treeherder = job.get('treeherder', {})
|
||||
treeherder.setdefault('symbol', 'tc(Ns)')
|
||||
dep_th_platform = dep_job.task.get('extra', {}).get(
|
||||
'treeherder', {}).get('machine', {}).get('platform', '')
|
||||
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
|
||||
treeherder.setdefault('tier', 2)
|
||||
treeherder.setdefault('kind', 'build')
|
||||
|
||||
label = job.get('label', "{}-signing".format(dep_job.label))
|
||||
|
||||
attributes = {
|
||||
'nightly': dep_job.attributes.get('nightly', False),
|
||||
'build_platform': dep_job.attributes.get('build_platform'),
|
||||
'build_type': dep_job.attributes.get('build_type'),
|
||||
}
|
||||
|
||||
task = {
|
||||
'label': label,
|
||||
'description': "{} Signing".format(
|
||||
dep_job.task["metadata"]["description"]),
|
||||
'worker-type': "scriptworker-prov-v1/signing-linux-v1",
|
||||
'worker': {'implementation': 'scriptworker-signing',
|
||||
'upstream-artifacts': job['upstream-artifacts'],
|
||||
'max-run-time': 3600},
|
||||
'scopes': ["project:releng:signing:cert:nightly-signing"] + signing_format_scopes,
|
||||
'dependencies': {job['depname']: dep_job.label},
|
||||
'attributes': attributes,
|
||||
'run-on-projects': dep_job.attributes.get('run_on_projects'),
|
||||
'treeherder': treeherder,
|
||||
}
|
||||
|
||||
yield task
|
||||
|
|
@ -22,6 +22,7 @@ from voluptuous import Schema, Any, Required, Optional, Extra
|
|||
|
||||
from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
|
||||
|
||||
|
||||
# shortcut for a string where task references are allowed
|
||||
taskref_or_string = Any(
|
||||
basestring,
|
||||
|
|
@ -264,6 +265,26 @@ task_description_schema = Schema({
|
|||
# type=directory)
|
||||
Required('name'): basestring,
|
||||
}],
|
||||
}, {
|
||||
Required('implementation'): 'scriptworker-signing',
|
||||
|
||||
# the maximum time to spend signing, in seconds
|
||||
Required('max-run-time', default=600): int,
|
||||
|
||||
# list of artifact URLs for the artifacts that should be signed
|
||||
Required('upstream-artifacts'): [{
|
||||
# taskId of the task with the artifact
|
||||
Required('taskId'): taskref_or_string,
|
||||
|
||||
# type of signing task (for CoT)
|
||||
Required('taskType'): basestring,
|
||||
|
||||
# Paths to the artifacts to sign
|
||||
Required('paths'): [basestring],
|
||||
|
||||
# Signing formats to use on each of the paths
|
||||
Required('formats'): [basestring],
|
||||
}],
|
||||
}),
|
||||
|
||||
# The "when" section contains descriptions of the circumstances
|
||||
|
|
@ -452,6 +473,16 @@ def build_generic_worker_payload(config, task, task_def):
|
|||
raise Exception("retry-exit-status not supported in generic-worker")
|
||||
|
||||
|
||||
@payload_builder('scriptworker-signing')
|
||||
def build_scriptworker_signing_payload(config, task, task_def):
|
||||
worker = task['worker']
|
||||
|
||||
task_def['payload'] = {
|
||||
'maxRunTime': worker['max-run-time'],
|
||||
'upstreamArtifacts': worker['upstream-artifacts']
|
||||
}
|
||||
|
||||
|
||||
@payload_builder('macosx-engine')
|
||||
def build_macosx_engine_payload(config, task, task_def):
|
||||
worker = task['worker']
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче