Bug 1424482 - beetmover: Get rid of balrog_props in favor of task payload r=mtabara

MozReview-Commit-ID: 5eRwwQ10BQD
2018-02-15 14:46:19 +01:00 · 2018-02-15 14:46:19 +01:00 · 69fb87824b
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@ -22,7 +22,6 @@ from voluptuous import Any, Required, Optional
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
-    "balrog_props.json",
    "target.common.tests.zip",
    "target.cppunittest.tests.zip",
    "target.crashreporter-symbols.zip",
@ -55,7 +54,6 @@ _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
    "target.langpack.xpi",
-    "balrog_props.json",
 ]
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
@ -92,7 +90,6 @@ _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI = [
-    "balrog_props.json",
    "target.common.tests.zip",
    "target.cppunittest.tests.zip",
    "target.json",
@ -189,7 +186,7 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
    'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
    'android-x86-old-id-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
    'android-aarch64-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
-    'android-api-16-nightly-l10n': ["balrog_props.json"],
+    'android-api-16-nightly-l10n': [],
    'android-api-16-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
    'android-api-16-old-id-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
    'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
@ -447,6 +444,29 @@ def generate_upstream_artifacts(signing_task_ref, build_task_ref, platform,
    return upstream_artifacts


+def craft_release_properties(config, job):
+    params = config.params
+    build_platform = job['attributes']['build_platform']
+    build_platform = build_platform.replace('-nightly', '')
+    if 'fennec-source' in build_platform:
+        # XXX This case is hardcoded to match the current implementation in beetmover
+        build_platform = 'android-api-16'
+    else:
+        build_platform = build_platform.replace('-source', '')
+
+    app_name = 'Fennec' if 'android' in job['label'] or 'fennec' in job['label'] else 'Firefox'
+
+    return {
+        # XXX Even DevEdition is called Firefox
+        'app-name': app_name,
+        'app-version': str(params['app_version']),
+        'branch': params['project'],
+        'build-id': str(params['moz_build_date']),
+        'hash-type': 'sha512',
+        'platform': build_platform,
+    }
+
+
@transforms.add
 def make_task_worker(config, jobs):
    for job in jobs:
@ -467,12 +487,15 @@ def make_task_worker(config, jobs):

        signing_task_ref = "<" + str(signing_task) + ">"
        build_task_ref = "<" + str(build_task) + ">"
-        upstream_artifacts = generate_upstream_artifacts(
-            signing_task_ref, build_task_ref, platform, locale
-        )

-        worker = {'implementation': 'beetmover',
-                  'upstream-artifacts': upstream_artifacts}
+        worker = {
+            'implementation': 'beetmover',
+            'release-properties': craft_release_properties(config, job),
+            'upstream-artifacts': generate_upstream_artifacts(
+                signing_task_ref, build_task_ref, platform, locale
+            )
+        }
+
        if locale:
            worker["locale"] = locale
        job["worker"] = worker
--- a/taskcluster/taskgraph/transforms/beetmover_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py
@ -8,6 +8,7 @@ Transform the checksums signing task into an actual task description.
 from __future__ import absolute_import, print_function, unicode_literals

 from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
@ -132,11 +133,6 @@ def generate_upstream_artifacts(refs, platform, locale=None):
        "taskType": "signing",
        "paths": common_paths,
        "locale": locale or "en-US",
-    }, {
-        "taskId": {"task-reference": refs["beetmover"]},
-        "taskType": "beetmover",
-        "paths": ["public/balrog_props.json"],
-        "locale": locale or "en-US",
    }]

    return upstream_artifacts
@ -165,11 +161,14 @@ def make_beetmover_checksums_worker(config, jobs):
            raise NotImplementedError(
                "Beetmover checksums must have a beetmover and signing dependency!")

-        upstream_artifacts = generate_upstream_artifacts(refs,
-                                                         platform, locale)
+        worker = {
+            'implementation': 'beetmover',
+            'release-properties': craft_release_properties(config, job),
+            'upstream-artifacts': generate_upstream_artifacts(
+                refs, platform, locale
+            ),
+        }

-        worker = {'implementation': 'beetmover',
-                  'upstream-artifacts': upstream_artifacts}
        if locale:
            worker["locale"] = locale
        job["worker"] = worker
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@ -8,6 +8,7 @@ Transform the beetmover task into an actual task description.
 from __future__ import absolute_import, print_function, unicode_literals

 from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.partials import (get_balrog_platform_name,
                                     get_partials_artifacts,
@ -37,7 +38,6 @@ _WINDOWS_BUILD_PLATFORMS = [
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
-    "balrog_props.json",
    "target.common.tests.zip",
    "target.cppunittest.tests.zip",
    "target.crashreporter-symbols.zip",
@ -63,7 +63,6 @@ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
 # See example in bug 1348286
 _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
    "target.langpack.xpi",
-    "balrog_props.json",
 ]

 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
@ -371,13 +370,15 @@ def make_task_worker(config, jobs):
        build_signing_task_ref = "<" + str(build_signing_task) + ">"
        repackage_task_ref = "<" + str(repackage_task) + ">"
        repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">"
-        upstream_artifacts = generate_upstream_artifacts(
-            build_task_ref, build_signing_task_ref, repackage_task_ref,
-            repackage_signing_task_ref, platform, locale
-        )

-        worker = {'implementation': 'beetmover',
-                  'upstream-artifacts': upstream_artifacts}
+        worker = {
+            'implementation': 'beetmover',
+            'release-properties': craft_release_properties(config, job),
+            'upstream-artifacts': generate_upstream_artifacts(
+                build_task_ref, build_signing_task_ref, repackage_task_ref,
+                repackage_signing_task_ref, platform, locale
+            ),
+        }
        if locale:
            worker["locale"] = locale
        job["worker"] = worker
--- a/taskcluster/taskgraph/transforms/beetmover_source.py
+++ b/taskcluster/taskgraph/transforms/beetmover_source.py
@ -12,37 +12,19 @@ transforms = TransformSequence()


@transforms.add
-def tweak_beetmover_source_dependencies_and_upstream_artifacts(config, jobs):
+def remove_build_dependency_in_beetmover_source(config, jobs):
    for job in jobs:
-        # HACK1: instead of grabbing SOURCE file from `release-source` task, we
-        # instead take it along with SOURCE.asc directly from the
-        # `release-source-signing`.
-        #
-        # HACK2: This way, we can just overwrite the `build`
-        # dependency, which at this point still is `release-source` task, with
-        # the actual Nightly en-US linux64 build which contains the
-        # `balrog_props` file we're interested in.
-        #
-        # XXX: this hack should go away by either:
-        # * rewriting beetmover transforms to allow more flexibility in deps
-        # * ditch balrog_props in beetmover and rely on in-tree task payload
+        # XXX: We delete the build dependency because, unlike the other beetmover
+        # tasks, source doesn't depend on any build task at all. This hack should
+        # go away when we rewrite beetmover transforms to allow more flexibility in deps
+        del job['dependencies']['build']

-        if job['attributes']['shipping_product'] == 'firefox':
-            job['dependencies']['build'] = u'build-linux64-nightly/opt'
-        elif job['attributes']['shipping_product'] == 'fennec':
-            job['dependencies']['build'] = u'build-android-api-16-nightly/opt'
-        elif job['attributes']['shipping_product'] == 'devedition':
-            job['dependencies']['build'] = u'build-linux64-devedition-nightly/opt'
-        else:
-            raise NotImplemented(
-                "Unknown shipping_product {} for beetmover_source!".format(
-                    job['attributes']['shipping_product']
-                )
-            )
-        upstream_artifacts = job['worker']['upstream-artifacts']
-        for artifact in upstream_artifacts:
-            if artifact['taskType'] == 'build':
-                artifact['paths'].append(u'public/build/balrog_props.json')
-                break
+        all_upstream_artifacts = job['worker']['upstream-artifacts']
+        upstream_artifacts_without_build = [
+            upstream_artifact
+            for upstream_artifact in all_upstream_artifacts
+            if upstream_artifact['taskId']['task-reference'] != '<build>'
+        ]
+        job['worker']['upstream-artifacts'] = upstream_artifacts_without_build

        yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@ -506,6 +506,15 @@ task_description_schema = Schema({
        # locale key, if this is a locale beetmover job
        Optional('locale'): basestring,

+        Required('release-properties'): {
+            'app-name': basestring,
+            'app-version': basestring,
+            'branch': basestring,
+            'build-id': basestring,
+            'hash-type': basestring,
+            'platform': basestring,
+        },
+
        # list of artifact URLs for the artifacts that should be beetmoved
        Required('upstream-artifacts'): [{
            # taskId of the task with the artifact
@ -1018,11 +1027,20 @@ def build_binary_transparency_payload(config, task, task_def):
 def build_beetmover_payload(config, task, task_def):
    worker = task['worker']
    release_config = get_release_config(config)
+    release_properties = worker['release-properties']

    task_def['payload'] = {
        'maxRunTime': worker['max-run-time'],
+        'releaseProperties': {
+            'appName': release_properties['app-name'],
+            'appVersion': release_properties['app-version'],
+            'branch': release_properties['branch'],
+            'buildid': release_properties['build-id'],
+            'hashType': release_properties['hash-type'],
+            'platform': release_properties['platform'],
+        },
        'upload_date': config.params['build_date'],
-        'upstreamArtifacts':  worker['upstream-artifacts']
+        'upstreamArtifacts':  worker['upstream-artifacts'],
    }
    if worker.get('locale'):
        task_def['payload']['locale'] = worker['locale']