Bug 1442545: [partner-repack] Beetmove the partner repacks; r=Callek

Differential Revision: https://phabricator.services.mozilla.com/D992

--HG--
extra : rebase_source : 66571663a4f3455a3f5b9d5afa616c48a35e46ad
extra : source : f292fad571a41e45706cd75a0b0afde7feff58e0
extra : histedit_source : 304e44cef85dfe8514748db3fad484a09cbca727

taskcluster/ci/config.yml

@@ -113,8 +113,10 @@ scriptworker:
         'scriptworker-prov-v1/beetmoverworker-v1':
             - 'project:releng:beetmover:bucket:release'
             - 'project:releng:beetmover:bucket:nightly'
+            - 'project:releng:beetmover:bucket:partner'
         'scriptworker-prov-v1/beetmoverworker-dev':
             - 'project:releng:beetmover:bucket:dep'
+            - 'project:releng:beetmover:bucket:dep-partner'
         'scriptworker-prov-v1/balrogworker-v1':
             - 'project:releng:balrog:server:nightly'
             - 'project:releng:balrog:server:aurora'

taskcluster/ci/release-eme-free-repack-beetmover/kind.yml

@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.beetmover_repackage_partner:transforms
+   - taskgraph.transforms.release_notifications:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - release-eme-free-repack-repackage  # Mac
+   - release-eme-free-repack-repackage-signing  # Windows
+
+only-for-build-platforms:
+   - macosx64-nightly/opt
+   - win32-nightly/opt
+   - win64-nightly/opt
+
+job-template:
+   shipping-phase: promote
+   partner-bucket-scope:
+      by-project:
+         mozilla-beta: beetmover:bucket:partner
+         mozilla-release: beetmover:bucket:partner
+         default: beetmover:bucket:dep-partner
+   partner-private-path: null
+   partner-public-path: "{platform}-EME-free/{locale}"

taskcluster/ci/release-partner-repack-beetmover/kind.yml

@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.name_sanity:transforms
+   - taskgraph.transforms.beetmover_repackage_partner:transforms
+   - taskgraph.transforms.release_notifications:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - release-partner-repack-chunking-dummy  # Linux
+   - release-partner-repack-repackage  # Mac
+   - release-partner-repack-repackage-signing  # Windows
+
+only-for-build-platforms:
+   - linux-nightly/opt
+   - linux64-nightly/opt
+   - macosx64-nightly/opt
+   - win32-nightly/opt
+   - win64-nightly/opt
+
+job-template:
+   shipping-phase: promote
+   partner-bucket-scope:
+      by-project:
+         mozilla-beta: beetmover:bucket:partner
+         mozilla-release: beetmover:bucket:partner
+         default: beetmover:bucket:dep-partner
+   partner-public-path: "partner-repacks/{partner}/{subpartner}/v{release_partner_build_number}/{platform}/{locale}"
+   partner-private-path: "{partner}/{version}-{build_number}/{subpartner}/{platform}/{locale}"

taskcluster/ci/release-partner-repack-chunking-dummy/kind.yml

@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+   - taskgraph.transforms.chunk_partners:transforms
+   - taskgraph.transforms.name_sanity:transforms
+   # This transform sets build_platform to the same thing as the upstream task.
+   # We'd do it here, except single_dep doesn't pay attention to any
+   # per platform things that we set.
+   - taskgraph.transforms.copy_attributes_from_dependent_task:transforms
+   # This transform is needed because task.py doesn't allow "dependent-task" to be
+   # set, but the single_dep loader sets it (and we need it for chunk_partners,
+   # name_sanity, and copy_build_platform_from_dependent_task to work).
+   - taskgraph.transforms.strip_dependent_task:transforms
+   - taskgraph.transforms.release_deps:transforms
+   - taskgraph.transforms.release_notifications:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+   - release-partner-repack
+
+only-for-build-platforms:
+   - linux-nightly/opt
+   - linux64-nightly/opt
+
+job-template:
+   shipping-phase: promote
+   shipping-product: firefox
+   name: release-partner-repack-chunking-dummy
+   description: Dummy task to deal with fanning out Linux partner repacks
+   run-on-projects: []
+   worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+   worker:
+      implementation: docker-worker
+      os: linux
+      docker-image: "ubuntu:16.10"
+      max-run-time: 600
+      command:
+         - /bin/bash
+         - -c
+         - echo "Dummy task"

taskcluster/docs/kinds.rst

@@ -354,6 +354,10 @@ release-partner-repack
 ----------------------
 Generates customized versions of releases for partners.
 
+release-partner-repack-chunking-dummy
+----------------------
+Chunks the partner repacks by locale.
+
 release-partner-repack-signing
 ------------------------------
 Internal signing of partner repacks.
@@ -362,6 +366,10 @@ release-partner-repack-repackage
 ------------------------------
 Repackaging of partner repacks.
 
+release-partner-repack-beetmover
+------------------------------
+Moves the partner repacks to S3 buckets.
+
 release-eme-free-repack
 ----------------------
 Generates customized versions of releases for eme-free repacks.
@@ -374,6 +382,10 @@ release-eme-free-repack-repackage
 ------------------------------
 Repackaging of eme-free repacks.
 
+
+release-eme-free-repack-beetmover
+------------------------------
+Moves the eme-free repacks to S3 buckets.
 repackage
 ---------
 Repackage tasks take a signed output and package them up into something suitable

taskcluster/taskgraph/transforms/beetmover_repackage_partner.py

@@ -0,0 +1,304 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.partners import (
+    check_if_partners_enabled,
+    get_ftp_platform,
+    get_partner_config_by_kind,
+)
+from taskgraph.util.schema import (
+    Schema,
+    optionally_keyed_by,
+    resolve_keyed_by,
+    validate_schema,
+)
+from taskgraph.util.scriptworker import (
+    add_scope_prefix,
+    get_beetmover_bucket_scope,
+    get_worker_type_for_scope,
+)
+from taskgraph.util.taskcluster import get_artifact_prefix
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Any, Required, Optional
+
+from copy import deepcopy
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+beetmover_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-task'): object,
+
+    # depname is used in taskref's to identify the taskID of the unsigned things
+    Required('depname', default='build'): basestring,
+
+    # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
+    Optional('label'): basestring,
+
+    Required('partner-bucket-scope'): optionally_keyed_by('project', basestring),
+    Required('partner-public-path'): Any(None, basestring),
+    Required('partner-private-path'): Any(None, basestring),
+
+    Optional('extra'): object,
+    Required('shipping-phase'): task_description_schema['shipping-phase'],
+    Optional('shipping-product'): task_description_schema['shipping-product'],
+})
+
+transforms.add(check_if_partners_enabled)
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        validate_schema(
+            beetmover_description_schema, job,
+            "In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
+        yield job
+
+
+@transforms.add
+def skip_for_indirect_dependencies(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        build_platform = dep_job.attributes.get("build_platform")
+        if not build_platform:
+            raise Exception("Cannot find build platform!")
+
+        # Partner and EME free beetmover tasks have multiple upstreams defined
+        # because some platforms don't run some parts of the sign -> repack ->
+        # repack sign chain. We only want to run beetmover for the last part of
+        # that chain that runs for any given platform.
+        # For Linux, it is the eme-free/partner repack build tasks.
+        # For Mac, it is repackage.
+        # For Windows, it is repackage-signing.
+        if "win" in build_platform:
+            if "repackage" not in dep_job.label:
+                continue
+            elif "signing" not in dep_job.label:
+                continue
+        if "macosx" in build_platform:
+            if "repackage" not in dep_job.label:
+                continue
+
+        yield job
+
+
+@transforms.add
+def resolve_keys(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job, 'partner-bucket-scope', item_name=job['label'], project=config.params['project']
+        )
+        yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        repack_id = dep_job.task.get('extra', {}).get('repack_id')
+        if not repack_id:
+            raise Exception("Cannot find repack id!")
+
+        attributes = dep_job.attributes
+        build_platform = attributes.get("build_platform")
+        if not build_platform:
+            raise Exception("Cannot find build platform!")
+
+        label = dep_job.label.replace("repackage-signing-", "beetmover-")
+        label = label.replace("repackage-", "beetmover-")
+        label = label.replace("chunking-dummy-", "beetmover-")
+        description = (
+            "Beetmover submission for repack_id '{repack_id}' for build '"
+            "{build_platform}/{build_type}'".format(
+                repack_id=repack_id,
+                build_platform=build_platform,
+                build_type=attributes.get('build_type')
+            )
+        )
+
+        dependencies = {}
+
+        base_label = "release-partner-repack"
+        if "eme" in config.kind:
+            base_label = "release-eme-free-repack"
+        dependencies["build"] = "{}-{}".format(base_label, build_platform)
+        if "macosx" in build_platform or "win" in build_platform:
+            dependencies["repackage"] = "{}-repackage-{}-{}".format(
+                base_label, build_platform, repack_id.replace('/', '-')
+            )
+        if "win" in build_platform:
+            dependencies["repackage-signing"] = "{}-repackage-signing-{}-{}".format(
+                base_label, build_platform, repack_id.replace('/', '-')
+            )
+
+        attributes = copy_attributes_from_dependent_job(dep_job)
+
+        task = {
+            'label': label,
+            'description': description,
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'shipping-phase': job['shipping-phase'],
+            'shipping-product': job.get('shipping-product'),
+            'partner-private-path': job['partner-private-path'],
+            'partner-public-path': job['partner-public-path'],
+            'partner-bucket-scope': job['partner-bucket-scope'],
+            'extra': {
+                'repack_id': repack_id,
+            },
+        }
+        yield task
+
+
+def populate_scopes_and_worker_type(config, job, bucket_scope, partner_public=False):
+    action_scope = add_scope_prefix(config, 'beetmover:action:push-to-partner')
+
+    task = deepcopy(job)
+    task['scopes'] = [bucket_scope, action_scope]
+    task['worker-type'] = get_worker_type_for_scope(config, bucket_scope)
+    task['partner_public'] = partner_public
+    if partner_public:
+        task['label'] = "{}-public".format(task['label'])
+    return task
+
+
+@transforms.add
+def split_public_and_private(config, jobs):
+    public_bucket_scope = get_beetmover_bucket_scope(config)
+    partner_config = get_partner_config_by_kind(config, config.kind)
+
+    for job in jobs:
+        partner_bucket_scope = add_scope_prefix(config, job['partner-bucket-scope'])
+        partner, subpartner, _ = job['extra']['repack_id'].split('/')
+
+        # public
+        if partner_config[partner][subpartner].get('upload_to_candidates'):
+            yield populate_scopes_and_worker_type(
+                config, job, public_bucket_scope, partner_public=True
+            )
+        # private
+        yield populate_scopes_and_worker_type(
+            config, job, partner_bucket_scope, partner_public=False
+        )
+
+
+def generate_upstream_artifacts(job, build_task_ref, repackage_task_ref,
+                                repackage_signing_task_ref, platform, repack_id,
+                                partner_path):
+
+    upstream_artifacts = []
+    artifact_prefix = get_artifact_prefix(job)
+
+    if "linux" in platform:
+        upstream_artifacts.append({
+            "taskId": {"task-reference": build_task_ref},
+            "taskType": "build",
+            "paths": ["{}/{}/target.tar.bz2".format(artifact_prefix, repack_id)],
+            "locale": partner_path,
+        })
+    elif "macosx" in platform:
+        upstream_artifacts.append({
+            "taskId": {"task-reference": repackage_task_ref},
+            "taskType": "repackage",
+            "paths": ["{}/{}/target.dmg".format(artifact_prefix, repack_id)],
+            "locale": partner_path,
+        })
+    elif "win" in platform:
+        upstream_artifacts.append({
+            "taskId": {"task-reference": repackage_signing_task_ref},
+            "taskType": "repackage",
+            "paths": ["{}/{}/target.installer.exe".format(artifact_prefix, repack_id)],
+            "locale": partner_path,
+        })
+
+    if not upstream_artifacts:
+        raise Exception("Couldn't find any upstream artifacts.")
+
+    return upstream_artifacts
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        platform = job["attributes"]["build_platform"]
+        repack_id = job["extra"]["repack_id"]
+        partner, subpartner, locale = job['extra']['repack_id'].split('/')
+        build_task = None
+        repackage_task = None
+        repackage_signing_task = None
+
+        for dependency in job["dependencies"].keys():
+            if 'repackage-signing' in dependency:
+                repackage_signing_task = dependency
+            elif 'repackage' in dependency:
+                repackage_task = dependency
+            else:
+                build_task = "build"
+
+        build_task_ref = "<" + str(build_task) + ">"
+        repackage_task_ref = "<" + str(repackage_task) + ">"
+        repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">"
+
+        # generate the partner path; we'll send this to beetmover as the "locale"
+        ftp_platform = get_ftp_platform(platform)
+        repl_dict = {
+            "build_number": config.params['build_number'],
+            "locale": locale,
+            "partner": partner,
+            "platform": ftp_platform,
+            "release_partner_build_number": config.params['release_partner_build_number'],
+            "subpartner": subpartner,
+            "version": config.params['version'],
+        }
+        partner_public = job['partner_public']
+        if partner_public:
+            partner_path_key = 'partner-public-path'
+        else:
+            partner_path_key = 'partner-private-path'
+        # Kinds can set these to None
+        if not job[partner_path_key]:
+            continue
+        partner_path = job[partner_path_key].format(**repl_dict)
+        del(job['partner_public'])
+        del(job['partner-private-path'])
+        del(job['partner-public-path'])
+        del(job['partner-bucket-scope'])
+
+        worker = {
+            'implementation': 'beetmover',
+            'release-properties': craft_release_properties(config, job),
+            'upstream-artifacts': generate_upstream_artifacts(
+                job, build_task_ref, repackage_task_ref,
+                repackage_signing_task_ref, platform, repack_id,
+                partner_path
+            ),
+            'partner-public': partner_public,
+        }
+        job["worker"] = worker
+
+        yield job

taskcluster/taskgraph/transforms/copy_attributes_from_dependent_task.py

@@ -0,0 +1,22 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the repackage task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def copy_attributes(config, jobs):
+    for job in jobs:
+        job.setdefault('attributes', {})
+        job['attributes'].update(copy_attributes_from_dependent_job(job['dependent-task']))
+
+        yield job

taskcluster/taskgraph/transforms/strip_dependent_task.py

@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+FIXME
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def strip_dependent_task(config, jobs):
+    for job in jobs:
+        del job['dependent-task']
+        yield job