bug 1412690 - add task dependencies. r=rail

This patch adds the `release_deps` transform, which adds every kind-dependency task that has the same product as a task dependency (with some exceptions). This patch made it clear that we need a standard way of defining product. MozReview-Commit-ID: 4xOJRQSCTgF --HG-- extra : rebase_source : 4b98b040ac5d8777394cff6f3249ab354fff641b
2017-10-29 17:40:50 -07:00 · 2017-10-29 17:40:50 -07:00 · d16d2d20cb
--- a/taskcluster/ci/beetmover-cdns/kind.yml
+++ b/taskcluster/ci/beetmover-cdns/kind.yml
@ -5,9 +5,13 @@
 loader: taskgraph.loader.transform:loader

 transforms:
+   - taskgraph.transforms.release_deps:transforms
   - taskgraph.transforms.beetmover_cdns:transforms
   - taskgraph.transforms.task:transforms

+kind-dependencies:
+   - beetmover-checksums
+
 job-defaults:
   worker-type:
      by-project:
@ -25,7 +29,3 @@ jobs:
            - index.releases.v1.{branch}.latest.fennec.latest.beetmover_cdns
            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.beetmover_cdns
      treeherder-platform: Android/opt
-
-# We probably want a dummy task after the beetmover checksums, or the publish notify task?
-# kind-dependencies:
-#   - beetmover-checksums
--- a/taskcluster/ci/push-apk/kind.yml
+++ b/taskcluster/ci/push-apk/kind.yml
@ -32,3 +32,5 @@ jobs:
            kind: other
        run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
        deadline-after: 5 days
+        extra:
+            product: fennec
--- a/taskcluster/ci/release-bouncer-aliases/kind.yml
+++ b/taskcluster/ci/release-bouncer-aliases/kind.yml
@ -5,9 +5,13 @@
 loader: taskgraph.loader.transform:loader

 transforms:
+   - taskgraph.transforms.release_deps:transforms
   - taskgraph.transforms.job:transforms
   - taskgraph.transforms.task:transforms

+kind-dependencies:
+   - release-uptake-monitoring
+
 jobs:
   fennec:
      name: fennec_release_bouncer_aliases
--- a/taskcluster/ci/release-mark-as-shipped/kind.yml
+++ b/taskcluster/ci/release-mark-as-shipped/kind.yml
@ -5,12 +5,15 @@
 loader: taskgraph.loader.transform:loader

 transforms:
+   - taskgraph.transforms.release_deps:transforms
   - taskgraph.transforms.job:transforms
   - taskgraph.transforms.task:transforms

-# TODO:
-# kind-dependencies:
-#   - release-push-to-cdns
+kind-dependencies:
+   - push-apk
+   - release-bouncer-aliases
+   - release-version-bump
+

 jobs:
   fennec:
@ -51,3 +54,5 @@ jobs:
                  #- "{task[tags][createdForUser]}"
               default:
                  - "release-drivers"
+      extra:
+         product: fennec
--- a/taskcluster/ci/release-notify-promote/kind.yml
+++ b/taskcluster/ci/release-notify-promote/kind.yml
@ -5,10 +5,12 @@
 loader: taskgraph.loader.transform:loader

 transforms:
+   - taskgraph.transforms.release_deps:transforms
   - taskgraph.transforms.task:transforms

 kind-dependencies:
  - beetmover-checksums
+  - release-bouncer-sub

 jobs:
   fennec:
@ -37,3 +39,5 @@ jobs:
                     #- "{task[tags][createdForUser]}"
                  default:
                     - "release-drivers"
+      extra:
+         product: fennec
--- a/taskcluster/ci/release-notify-publish/kind.yml
+++ b/taskcluster/ci/release-notify-publish/kind.yml
@ -5,10 +5,13 @@
 loader: taskgraph.loader.transform:loader

 transforms:
+   - taskgraph.transforms.release_deps:transforms
   - taskgraph.transforms.task:transforms

 kind-dependencies:
   - push-apk
+   - release-bouncer-aliases
+   - release-version-bump

 jobs:
   fennec:
@ -37,3 +40,5 @@ jobs:
                     #- "{task[tags][createdForUser]}"
                  default:
                     - "release-drivers"
+      extra:
+         product: fennec
--- a/taskcluster/ci/release-uptake-monitoring/kind.yml
+++ b/taskcluster/ci/release-uptake-monitoring/kind.yml
@ -5,12 +5,12 @@
 loader: taskgraph.loader.transform:loader

 transforms:
+   - taskgraph.transforms.release_deps:transforms
   - taskgraph.transforms.job:transforms
   - taskgraph.transforms.task:transforms

-# TODO:
-# kind-dependencies:
-#   - release-push-to-cdns
+kind-dependencies:
+   - beetmover-cdns

 jobs:
   fennec:
--- a/taskcluster/ci/release-version-bump/kind.yml
+++ b/taskcluster/ci/release-version-bump/kind.yml
@ -5,12 +5,12 @@
 loader: taskgraph.loader.transform:loader

 transforms:
+   - taskgraph.transforms.release_deps:transforms
   - taskgraph.transforms.job:transforms
   - taskgraph.transforms.task:transforms

-# TODO:
-# kind-dependencies:
-#   - release-push-to-cdns (aka. push to releases)
+kind-dependencies:
+   - beetmover-cdns

 jobs:
   fennec:
--- a/taskcluster/taskgraph/actions/release_promotion.py
+++ b/taskcluster/taskgraph/actions/release_promotion.py
@ -36,6 +36,7 @@ RELEASE_PROMOTION_CONFIG = {
            'release-bouncer-sub', 'beetmover', 'beetmover-checksums',
            'beetmover-l10n', 'beetmover-repackage',
            'beetmover-repackage-signing', "checksums-signing",
+            'release-notify-promote',
        ],
        'do_not_optimize': [],
    },
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@ -389,10 +389,12 @@ def target_tasks_candidates_fennec(full_task_graph, parameters):
            if task.kind not in ('balrog', 'push-apk', 'push-apk-breakpoint'):
                if task.attributes.get('nightly'):
                    return True
-        if task.task['payload'].get('properties', {}).get('product') == 'fennec' or \
-           task.label.endswith('-fennec'):
+        if task.task['payload'].get('properties', {}).get('product') == 'fennec':
            if task.kind in ('release-bouncer-sub',
-                             'release-notify-promote',
+                             ):
+                return True
+        if task.task.get('extra', {}).get('product') == 'fennec':
+            if task.kind in ('release-notify-promote',
                             ):
                return True

@ -409,14 +411,15 @@ def target_tasks_publish_fennec(full_task_graph, parameters):
        # Include candidates build tasks; these will be optimized out
        if task.label in filtered_for_candidates:
            return True
-        # TODO: Include [beetmover] fennec mozilla-beta push to releases
-        if task.task['payload'].get('properties', {}).get('product') == 'fennec' or \
-           task.label.endswith('-fennec'):
+        if task.task['payload'].get('properties', {}).get('product') == 'fennec':
            if task.kind in ('release-mark-as-shipped',
                             'release-bouncer-aliases',
                             'release-uptake-monitoring',
                             'release-version-bump',
-                             'release-notify-publish',
+                             ):
+                return True
+        if task.task.get('extra', {}).get('product') == 'fennec':
+            if task.kind in ('release-notify-publish',
                             ):
                return True

--- a/taskcluster/taskgraph/transforms/beetmover_cdns.py
+++ b/taskcluster/taskgraph/transforms/beetmover_cdns.py
@ -39,6 +39,7 @@ beetmover_cdns_description_schema = Schema({
        job_description_schema['worker-type'],
        {'by-project': {basestring: job_description_schema['worker-type']}},
    ),
+    Optional('dependencies'): {basestring: taskref_or_string},
 })


@ -70,16 +71,13 @@ def make_beetmover_cdns_description(config, jobs):
        bucket_scope = get_beetmover_bucket_scope(config)
        action_scope = get_beetmover_action_scope(config)

-        # TODO
-        dependencies = {}
-
        task = {
            'label': label,
            'description': description,
            'worker-type': 'scriptworker-prov-v1/beetmoverworker-dev',
            'scopes': [bucket_scope, action_scope],
            'product': job['product'],
-            'dependencies': dependencies,
+            'dependencies': job['dependencies'],
            'attributes': job.get('attributes', {}),
            'run-on-projects': job.get('run-on-projects'),
            'treeherder': treeherder,
--- a/taskcluster/taskgraph/transforms/beetmover_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py
@ -59,15 +59,25 @@ def make_beetmover_checksums_description(config, jobs):
        treeherder.setdefault('kind', 'build')

        label = job['label']
+        build_platform = attributes.get('build_platform')
+
        description = (
            "Beetmover submission of checksums for locale '{locale}' for build '"
            "{build_platform}/{build_type}'".format(
                locale=attributes.get('locale', 'en-US'),
-                build_platform=attributes.get('build_platform'),
+                build_platform=build_platform,
                build_type=attributes.get('build_type')
            )
        )

+        extra = {}
+        if build_platform.startswith("android"):
+            extra['product'] = 'fennec'
+        elif 'devedition' in build_platform:
+            extra['product'] = 'devedition'
+        else:
+            extra['product'] = 'firefox'
+
        dependent_kind = str(dep_job.kind)
        dependencies = {dependent_kind: dep_job.label}
        for k, v in dep_job.dependencies.items():
@ -92,6 +102,7 @@ def make_beetmover_checksums_description(config, jobs):
            'attributes': attributes,
            'run-on-projects': dep_job.attributes.get('run_on_projects'),
            'treeherder': treeherder,
+            'extra': extra,
        }

        yield task
--- a/taskcluster/taskgraph/transforms/push_apk.py
+++ b/taskcluster/taskgraph/transforms/push_apk.py
@ -16,7 +16,7 @@ from taskgraph.util.scriptworker import get_push_apk_scope, get_push_apk_track,
 from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies

-from voluptuous import Required
+from voluptuous import Optional, Required


 transforms = TransformSequence()
@ -35,6 +35,7 @@ push_apk_description_schema = Schema({
    Required('worker'): object,
    Required('scopes'): None,
    Required('deadline-after'): basestring,
+    Optional('extra'): object,
 })

 validate_jobs_schema_transform = functools.partial(