Bug 1501878 - required_signoffs attribute. r=tomprince

If a task has `required_signoffs` that match a release's `required_signoffs`, we should defer running that task until we have a matching `signoff_url`.

- add filter_out_missing_signoffs
- add transforms changes to inherit upstream `required_signoffs`
- add `mar-signing` `required_signoffs` to the `partials-signing`, `mar-signing`, and `mar-signing-l10n` kinds

Differential Revision: https://phabricator.services.mozilla.com/D11734

--HG--
extra : moz-landing-system : lando

taskcluster/ci/mar-signing-l10n/kind.yml

@@ -29,3 +29,5 @@ job-template:
     shipping-phase: promote
     treeherder-group: ms
     description-suffix: 'mar signing'
+    required_signoffs:
+        - mar-signing

taskcluster/ci/mar-signing/kind.yml

@@ -31,3 +31,5 @@ job-template:
     shipping-phase: promote
     treeherder-group: ms
     description-suffix: 'mar signing'
+    required_signoffs:
+        - mar-signing

taskcluster/ci/partials-signing/kind.yml

@@ -16,3 +16,5 @@ job-template:
   shipping-phase: promote
   treeherder-group: ps
   description-suffix: 'partial signing'
+  required_signoffs:
+    - mar-signing

taskcluster/docs/attributes.rst

@@ -266,3 +266,10 @@ cache_type
 Some tasks generate artifacts that are cached between pushes. This is the type of cache that is
 used for the this task. See :py:mod:`taskgraph.util.cached_task`.
 
+required_signoffs
+=================
+A list of release signoffs that this kind requires, should the release also
+require these signoffs. For example, ``mar-signing`` signoffs may be required
+by some releases in the future; for any releases that require ``mar-signing``
+signoffs, the kinds that also require that signoff are marked with this
+attribute.

taskcluster/taskgraph/loader/multi_dep.py

@@ -9,6 +9,7 @@ import copy
 from voluptuous import Required
 
 from ..task import Task
+from ..util.attributes import sorted_unique_list
 from ..util.schema import Schema
 
 schema = Schema({
@@ -66,6 +67,11 @@ def loader(kind, path, config, params, loaded_tasks):
         )
         if product:
             job.setdefault('shipping-product', product)
+        job.setdefault('attributes', {})['required_signoffs'] = sorted_unique_list(
+            *[task.attributes.get('required_signoffs', [])
+              for task in dep_tasks.values()
+              ]
+        )
 
         yield job
 

taskcluster/taskgraph/target_tasks.py

@@ -85,6 +85,14 @@ def filter_release_tasks(task, parameters):
     return True
 
 
+def filter_out_missing_signoffs(task, parameters):
+    for signoff in parameters['required_signoffs']:
+        if signoff not in parameters['signoff_urls'] and \
+           signoff in task.attributes.get('required_signoffs', []):
+            return False
+    return True
+
+
 def standard_filter(task, parameters):
     return all(
         filter_func(task, parameters) for filter_func in
@@ -320,6 +328,9 @@ def target_tasks_promote_desktop(full_task_graph, parameters, graph_config):
             if 'secondary' in task.kind:
                 return False
 
+        if not filter_out_missing_signoffs(task, parameters):
+            return False
+
         if task.attributes.get('shipping_phase') == 'promote':
             return True
 
@@ -335,6 +346,8 @@ def target_tasks_push_desktop(full_task_graph, parameters, graph_config):
     )
 
     def filter(task):
+        if not filter_out_missing_signoffs(task, parameters):
+            return False
         # Include promotion tasks; these will be optimized out
         if task.label in filtered_for_candidates:
             return True
@@ -363,6 +376,8 @@ def target_tasks_ship_desktop(full_task_graph, parameters, graph_config):
         )
 
     def filter(task):
+        if not filter_out_missing_signoffs(task, parameters):
+            return False
         # Include promotion tasks; these will be optimized out
         if task.label in filtered_for_candidates:
             return True

taskcluster/taskgraph/transforms/balrog_submit.py

@@ -35,6 +35,8 @@ balrog_description_schema = schema.extend({
     # below transforms for defaults of various values.
     Optional('treeherder'): task_description_schema['treeherder'],
 
+    Optional('attributes'): task_description_schema['attributes'],
+
     # Shipping product / phase
     Optional('shipping-product'): task_description_schema['shipping-product'],
     Optional('shipping-phase'): task_description_schema['shipping-phase'],

taskcluster/taskgraph/transforms/beetmover_repackage.py

@@ -163,6 +163,8 @@ beetmover_description_schema = schema.extend({
     # below transforms for defaults of various values.
     Optional('treeherder'): task_description_schema['treeherder'],
 
+    Optional('attributes'): task_description_schema['attributes'],
+
     # locale is passed only for l10n beetmoving
     Optional('locale'): basestring,
     Required('shipping-phase'): task_description_schema['shipping-phase'],
@@ -226,6 +228,7 @@ def make_task_description(config, jobs):
             dependencies["repackage-signing"] = upstream_deps[repackage_signing_name]
 
         attributes = copy_attributes_from_dependent_job(dep_job)
+        attributes.update(job.get('attributes', {}))
         if job.get('locale'):
             attributes['locale'] = job['locale']
 

taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py

@@ -36,6 +36,7 @@ def make_beetmover_description(config, jobs):
             'label': job['label'],
             'primary-dependency': dep_job,
             'dependent-tasks': job['dependent-tasks'],
+            'attributes': job['attributes'],
             'treeherder': treeherder,
             'locale': locale,
             'shipping-phase': job['shipping-phase'],

taskcluster/taskgraph/transforms/mar_signing.py

@@ -9,7 +9,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 import os
 
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.attributes import copy_attributes_from_dependent_job, sorted_unique_list
 from taskgraph.util.scriptworker import (
     get_signing_cert_scope_per_platform,
     get_worker_type_for_scope,
@@ -110,6 +110,10 @@ def make_task_description(config, jobs):
         dependencies.update(signing_dependencies)
 
         attributes = copy_attributes_from_dependent_job(dep_job)
+        attributes['required_signoffs'] = sorted_unique_list(
+            attributes.get('required_signoffs', []),
+            job.pop('required_signoffs')
+        )
         attributes['shipping_phase'] = job['shipping-phase']
         if locale:
             attributes['locale'] = locale

taskcluster/taskgraph/transforms/per_platform_dummy.py

@@ -8,6 +8,7 @@ Transform the repackage task into an actual task description.
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
 
 transforms = TransformSequence()
 
@@ -22,9 +23,9 @@ def one_task_per_product_and_platform(config, jobs):
         product = dep_task.attributes.get("shipping_product")
         platform = dep_task.attributes.get("build_platform")
         if (product, platform) not in unique_products_and_platforms:
-            job.setdefault("attributes", {})
-            job["attributes"]["shipping_product"] = product
-            job["attributes"]["build_platform"] = platform
+            attributes = copy_attributes_from_dependent_job(dep_task)
+            attributes.update(job.get('attributes', {}))
+            job['attributes'] = attributes
             job["name"] = "{}-{}".format(product, platform)
             yield job
             unique_products_and_platforms.add((product, platform))

taskcluster/taskgraph/transforms/release_deps.py

@@ -24,6 +24,7 @@ def add_dependencies(config, jobs):
         if product is None:
             continue
 
+        required_signoffs = set(job.setdefault('attributes', {}).get('required_signoffs', []))
         for dep_task in config.kind_dependencies_tasks:
             # Weed out unwanted tasks.
             # XXX we have run-on-projects which specifies the on-push behavior;
@@ -47,7 +48,10 @@ def add_dependencies(config, jobs):
             if dep_task.task.get('shipping-product') == product or \
                     dep_task.attributes.get('shipping_product') == product:
                 dependencies[dep_task.label] = dep_task.label
+                required_signoffs.update(dep_task.attributes.get('required_signoffs', []))
 
         job.setdefault('dependencies', {}).update(dependencies)
+        if required_signoffs:
+            job['attributes']['required_signoffs'] = sorted(required_signoffs)
 
         yield job

taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py

@@ -52,6 +52,8 @@ release_generate_checksums_beetmover_schema = schema.extend({
 
     Optional('shipping-phase'): task_description_schema['shipping-phase'],
     Optional('shipping-product'): task_description_schema['shipping-product'],
+
+    Optional('attributes'): task_description_schema['attributes'],
 })
 
 transforms = TransformSequence()

taskcluster/taskgraph/transforms/release_snap_push.py

@@ -30,6 +30,7 @@ push_snap_description_schema = Schema({
     Required('shipping-phase'): task_description_schema['shipping-phase'],
     Required('shipping-product'): task_description_schema['shipping-product'],
     Optional('extra'): task_description_schema['extra'],
+    Optional('attributes'): task_description_schema['attributes'],
 })
 
 transforms = TransformSequence()

taskcluster/taskgraph/util/attributes.py

@@ -36,6 +36,7 @@ _OPTIONAL_ATTRIBUTES = (
     'l10n_chunk',
     'locale',
     'nightly',
+    'required_signoffs',
     'signed',
     'shipping_phase',
     'shipping_product',
@@ -131,3 +132,9 @@ def copy_attributes_from_dependent_job(dep_job):
     })
 
     return attributes
+
+
+def sorted_unique_list(*args):
+    """Join one or more lists, and return a sorted list of unique members"""
+    combined = set().union(*args)
+    return sorted(combined)