bug 1415391 - add `shipping_phase` and `shipping_product` attributes. r=dustin

These two new attributes will help us determine which tasks belong in which release promotion graphs. In the future, we'll specify these for all shippable builds as well, and we can reduce the usage of the `product` keys. However, sometimes we need `product` to differ from `shipping-product` (e.g., `mobile` vs `fennec`; in this case we mean `stage_product` vs `shipping_product`), so I haven't yet touched those key/value pairs.

MozReview-Commit-ID: LEuf4CS277Q

--HG--
extra : rebase_source : 4971883500658ace209bec97a639a54c94292b50

taskcluster/ci/beetmover-cdns/kind.yml

@@ -19,11 +19,13 @@ job-defaults:
          mozilla-beta: scriptworker-prov-v1/beetmoverworker-v1
          default: scriptworker-prov-v1/beetmoverworker-dev
    run-on-projects: []
+   shipping-phase: publish
 
 jobs:
    fennec-push-to-cdns:
       name: fennec_push_to_cdns
       product: fennec
+      shipping-product: fennec
       routes:
         - index.releases.v1.{branch}.latest.fennec.latest.beetmover_cdns
         - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.beetmover_cdns

taskcluster/ci/push-apk-breakpoint/kind.yml

@@ -17,6 +17,8 @@ jobs:
         attributes:
             build_platform: android-nightly
             nightly: true
+        shipping-phase: ship
+        shipping-product: fennec
         worker-type: # see transforms
         worker:
             implementation: push-apk-breakpoint

taskcluster/ci/push-apk/kind.yml

@@ -18,6 +18,8 @@ jobs:
         attributes:
             build_platform: android-nightly
             nightly: true
+        shipping-phase: ship
+        shipping-product: fennec
         worker-type: scriptworker-prov-v1/pushapk-v1
         worker:
             upstream-artifacts: # see transforms

taskcluster/ci/release-bouncer-aliases/kind.yml

@@ -12,12 +12,16 @@ transforms:
 kind-dependencies:
    - release-uptake-monitoring
 
+job-defaults:
+   shipping-phase: ship
+
 jobs:
    fennec:
       name: fennec_release_bouncer_aliases
       description: Update bouncer aliases job
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
+      shipping-product: fennec
       run:
          using: buildbot
          product: fennec

taskcluster/ci/release-bouncer-sub/kind.yml

@@ -8,6 +8,8 @@ transforms:
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
+job-defaults:
+   shipping-phase: promote
 
 jobs:
    fennec:
@@ -15,6 +17,7 @@ jobs:
       description: release bouncer submission job
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
+      shipping-product: fennec
       run:
          using: buildbot
          product: fennec

taskcluster/ci/release-mark-as-shipped/kind.yml

@@ -14,6 +14,8 @@ kind-dependencies:
    - release-bouncer-aliases
    - release-version-bump
 
+job-defaults:
+   shipping-phase: ship
 
 jobs:
    fennec:
@@ -21,6 +23,7 @@ jobs:
       description: mark release as shipped in Ship-It
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
+      shipping-product: fennec
       run:
          using: buildbot
          product: fennec

taskcluster/ci/release-notify-promote/kind.yml

@@ -12,11 +12,15 @@ kind-dependencies:
   - beetmover-checksums
   - release-bouncer-sub
 
+job-defaults:
+    shipping-phase: promote
+
 jobs:
    fennec:
       name: notify-release-drivers-promote
       description: Sends email to release-drivers telling release was promoted.
       run-on-projects: []
+      shipping-product: fennec
       worker-type: aws-provisioner-v1/gecko-{level}-b-linux
       worker:
          implementation: docker-worker

taskcluster/ci/release-notify-publish/kind.yml

@@ -13,11 +13,15 @@ kind-dependencies:
    - release-bouncer-aliases
    - release-version-bump
 
+job-defaults:
+   shipping-phase: ship
+
 jobs:
    fennec:
       name: notify-release-drivers-publish
       description: Sends email to release-drivers telling release was published.
       run-on-projects: []
+      shipping-product: fennec
       worker-type: aws-provisioner-v1/gecko-{level}-b-linux
       worker:
          implementation: docker-worker

taskcluster/ci/release-uptake-monitoring/kind.yml

@@ -12,12 +12,16 @@ transforms:
 kind-dependencies:
    - beetmover-cdns
 
+job-defaults:
+   shipping-phase: publish
+
 jobs:
    fennec:
       name: fennec_release_uptake_monitoring
       description: Uptake monitoring job
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
+      shipping-product: fennec
       run:
          using: buildbot
          product: fennec

taskcluster/ci/release-version-bump/kind.yml

@@ -12,12 +12,16 @@ transforms:
 kind-dependencies:
    - beetmover-cdns
 
+job-defaults:
+   shipping-phase: ship
+
 jobs:
    fennec:
       name: fennec-version-bump
       description: Release Promotion version bump
       worker-type: buildbot-bridge/buildbot-bridge
       run-on-projects: []
+      shipping-product: fennec
       run:
          using: buildbot
          product: fennec

taskcluster/docs/attributes.rst

@@ -186,3 +186,11 @@ optimization even if the ``optimize_target_tasks`` parameter is False.
 This is meant to be used for tasks which a developer would almost always want to
 run. Typically these tasks will be short running and have a high risk of causing
 a backout. For example ``lint`` or ``python-unittest`` tasks.
+
+shipping_product
+================
+For release promotion jobs, this is the product we are shipping.
+
+shipping_phase
+==============
+For release promotion jobs, this is the shipping phase (promote, publish, ship).

taskcluster/taskgraph/target_tasks.py

@@ -334,6 +334,10 @@ def target_tasks_mozilla_beta_desktop_promotion(full_task_graph, parameters, gra
         if task.label in beta_tasks:
             return True
 
+        if task.attributes.get('shipping_product') == 'firefox' and \
+                task.attributes.get('shipping_phase') == 'promote':
+            return True
+
         # TODO: partner repacks
         # TODO: source task
         # TODO: funsize, all but balrog submission
@@ -360,6 +364,9 @@ def target_tasks_publish_firefox(full_task_graph, parameters, graph_config):
         # Include promotion tasks; these will be optimized out
         if task.label in filtered_for_candidates:
             return True
+        if task.attributes.get('shipping_product') == 'firefox' and \
+                task.attributes.get('shipping_phase') in ('publish', 'ship'):
+            return True
         # TODO: add beetmover push-to-releases
         # TODO: tagging / version bumping
         # TODO: publish to balrog
@@ -390,14 +397,9 @@ def target_tasks_candidates_fennec(full_task_graph, parameters, graph_config):
             if task.kind not in ('balrog', 'push-apk', 'push-apk-breakpoint'):
                 if task.attributes.get('nightly'):
                     return True
-        if task.task['payload'].get('properties', {}).get('product') == 'fennec':
-            if task.kind in ('release-bouncer-sub',
-                             ):
-                return True
-        if task.task.get('extra', {}).get('product') == 'fennec':
-            if task.kind in ('release-notify-promote',
-                             ):
-                return True
+        if task.attributes.get('shipping_product') == 'fennec' and \
+                task.attributes.get('shipping_phase') == 'promote':
+            return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(full_task_graph[l])]
 
@@ -414,23 +416,8 @@ def target_tasks_publish_fennec(full_task_graph, parameters, graph_config):
         # Include candidates build tasks; these will be optimized out
         if task.label in filtered_for_candidates:
             return True
-        if task.task['payload'].get('properties', {}).get('product') == 'fennec':
-            if task.kind in ('release-mark-as-shipped',
-                             'release-bouncer-aliases',
-                             'release-uptake-monitoring',
-                             'release-version-bump',
-                             ):
-                return True
-        if task.task.get('extra', {}).get('product') == 'fennec':
-            if task.kind in ('release-notify-publish',
-                             ):
-                return True
-
-        if task.task['payload'].get('product') == 'fennec':
-            if task.kind in ('beetmover-cdns', ):
-                return True
-
-        if task.kind in ('push-apk', 'push-apk-breakpoint'):
+        if task.attributes.get('shipping_product') == 'fennec' and \
+                task.attributes.get('shipping_phase') in ('ship', 'publish'):
             return True
 
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(full_task_graph[l])]

taskcluster/taskgraph/transforms/beetmover_cdns.py

@@ -41,6 +41,8 @@ beetmover_cdns_description_schema = Schema({
     Optional('dependencies'): {basestring: taskref_or_string},
     Optional('index'): {basestring: basestring},
     Optional('routes'): [basestring],
+    Required('shipping-phase'): task_description_schema['shipping-phase'],
+    Required('shipping-product'): task_description_schema['shipping-product'],
 })
 
 

taskcluster/taskgraph/transforms/job/__init__.py

@@ -60,6 +60,8 @@ job_description_schema = Schema({
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
+    Optional('shipping-phase'): task_description_schema['shipping-phase'],
+    Optional('shipping-product'): task_description_schema['shipping-product'],
     Optional('coalesce'): task_description_schema['coalesce'],
     Optional('always-target'): task_description_schema['always-target'],
     Exclusive('optimization', 'optimization'): task_description_schema['optimization'],

taskcluster/taskgraph/transforms/push_apk.py

@@ -10,6 +10,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 import functools
 
 from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.task import task_description_schema
 from taskgraph.util.schema import Schema
 from taskgraph.util.scriptworker import get_push_apk_scope, get_push_apk_track, \
     get_push_apk_dry_run_option, get_push_apk_rollout_percentage
@@ -21,6 +22,11 @@ from voluptuous import Optional, Required
 
 transforms = TransformSequence()
 
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+
 push_apk_description_schema = Schema({
     # the dependent task (object) for this beetmover job, used to inform beetmover.
     Required('dependent-tasks'): object,
@@ -35,6 +41,8 @@ push_apk_description_schema = Schema({
     Required('worker'): object,
     Required('scopes'): None,
     Required('deadline-after'): basestring,
+    Required('shipping-phase'): task_description_schema['shipping-phase'],
+    Required('shipping-product'): task_description_schema['shipping-product'],
     Optional('extra'): object,
 })
 

taskcluster/taskgraph/transforms/push_apk_breakpoint.py

@@ -10,6 +10,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 import functools
 
 from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.task import task_description_schema
 from taskgraph.util.schema import Schema
 from taskgraph.util.scriptworker import get_push_apk_breakpoint_worker_type
 from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
@@ -19,6 +20,10 @@ from voluptuous import Required
 
 transforms = TransformSequence()
 
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
 push_apk_breakpoint_description_schema = Schema({
     # the dependent task (object) for this beetmover job, used to inform beetmover.
     Required('dependent-tasks'): object,
@@ -32,6 +37,8 @@ push_apk_breakpoint_description_schema = Schema({
     Required('treeherder'): object,
     Required('run-on-projects'): list,
     Required('deadline-after'): basestring,
+    Required('shipping-phase'): task_description_schema['shipping-phase'],
+    Required('shipping-product'): task_description_schema['shipping-product'],
 })
 
 validate_jobs_schema_transform = functools.partial(

taskcluster/taskgraph/transforms/release_deps.py

@@ -14,8 +14,9 @@ transforms = TransformSequence()
 
 def _get_product(job_or_task):
     # Find the product.
-    # XXX officially support a product attribute that is consistently set.
-    product = job_or_task.get('product')
+    # XXX Once shipping-product is set for nightly builds as well, we can get
+    # rid of this function.
+    product = job_or_task.get('shipping-product', job_or_task.get('product'))
     if 'payload' in job_or_task:
         product = product or job_or_task['payload'].get(
             'product',

taskcluster/taskgraph/transforms/task.py

@@ -171,6 +171,24 @@ task_description_schema = Schema({
     # See the attributes documentation for details.
     Optional('run-on-projects'): [basestring],
 
+    # The `shipping_phase` attribute, defaulting to None. This specifies the
+    # release promotion phase that this task belongs to.
+    Required('shipping-phase', default=None): Any(
+        None,
+        'promote',
+        'publish',
+        'ship',
+    ),
+
+    # The `shipping_product` attribute, defaulting to None. This specifies the
+    # release promotion product that this task belongs to.
+    Required('shipping-product', default=None): Any(
+        None,
+        'devedition',
+        'fennec',
+        'firefox',
+    ),
+
     # Coalescing provides the facility for tasks to be superseded by the same
     # task in a subsequent commit, if the current task backlog reaches an
     # explicit threshold. Both age and size thresholds need to be met in order
@@ -1309,6 +1327,8 @@ def build_task(config, tasks):
         attributes = task.get('attributes', {})
         attributes['run_on_projects'] = task.get('run-on-projects', ['all'])
         attributes['always_target'] = task['always-target']
+        attributes['shipping_phase'] = task['shipping-phase']
+        attributes['shipping_product'] = task['shipping-product']
 
         # Set MOZ_AUTOMATION on all jobs.
         if task['worker']['implementation'] in (