Bug 1481121: [release] Use existing release-artifact metadata in beetmover_repackage; r=Callek

Differential Revision: https://phabricator.services.mozilla.com/D3817

--HG--
extra : rebase_source : 1137564eb7bc32406b15059cc89e91b1f2a978c1
extra : histedit_source : b7ba4f2be262c10e86c5635a959425c9585678c9

taskcluster/ci/beetmover-repackage/kind.yml

@@ -15,6 +15,8 @@ kind-dependencies:
    - repackage-signing-l10n
    - partials-signing
 
+fake-multi-dep: true
+
 only-for-build-platforms:
    - linux-nightly/opt
    - linux64-nightly/opt

taskcluster/taskgraph/loader/single_dep.py

@@ -29,6 +29,8 @@ def loader(kind, path, config, params, loaded_tasks):
     only_attributes = config.get('only-for-attributes')
     job_template = config.get('job-template')
 
+    include_parents = config.get('fake-multi-dep')
+
     for task in loaded_tasks:
         if task.kind not in config.get('kind-dependencies', []):
             continue
@@ -51,6 +53,9 @@ def loader(kind, path, config, params, loaded_tasks):
                 continue
 
         job = {'dependent-task': task}
+        if include_parents:
+            job['grandparent-tasks'] = _get_grandparent_tasks(task.dependencies, loaded_tasks)
+
         if job_template:
             job.update(copy.deepcopy(job_template))
 
@@ -62,3 +67,15 @@ def loader(kind, path, config, params, loaded_tasks):
             job.setdefault('shipping-product', product)
 
         yield job
+
+
+def _get_grandparent_tasks(dependencies, loaded_tasks):
+    parent_tasks = {}
+    for task in loaded_tasks:
+        if task.label in dependencies.values():
+            for name, label in dependencies.items():
+                if label == task.label:
+                    parent_tasks[name] = task
+    if set(parent_tasks.keys()) != set(dependencies.keys()):
+        raise Exception("Missing parent tasks.")
+    return parent_tasks

taskcluster/taskgraph/transforms/beetmover_repackage.py

@@ -118,29 +118,23 @@ UPSTREAM_ARTIFACT_SIGNED_PATHS = {
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
-UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
-    r'^macosx64(|-devedition)-nightly(|-l10n)$': ['target.dmg'],
-}
+UPSTREAM_ARTIFACT_REPACKAGE_PATHS = [
+    'target.dmg',
+]
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
-UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS = {
-    r'^(linux(|64)|macosx64)(|-devedition|-asan-reporter)-nightly(|-l10n)$':
-        ['target.complete.mar'],
-    r'^win64(|-devedition|-asan-reporter)-nightly(|-l10n)$':
-        ['target.complete.mar', 'target.installer.exe'],
-    r'^win32(|-devedition)-nightly(|-l10n)$': [
-        'target.complete.mar',
-        'target.installer.exe',
-        'target.stub-installer.exe'
-        ],
-}
+UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS = [
+    'target.complete.mar',
+    'target.bz2.complete.mar',
+    'target.installer.exe',
+    'target.stub-installer.exe',
+]
 
 # Compile every regex once at import time
 for dict_ in (
     UPSTREAM_ARTIFACT_UNSIGNED_PATHS, UPSTREAM_ARTIFACT_SIGNED_PATHS,
-    UPSTREAM_ARTIFACT_REPACKAGE_PATHS, UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS,
 ):
     for uncompiled_regex, value in dict_.iteritems():
         compiled_regex = re.compile(uncompiled_regex)
@@ -162,6 +156,8 @@ beetmover_description_schema = Schema({
     # the dependent task (object) for this beetmover job, used to inform beetmover.
     Required('dependent-task'): object,
 
+    Required('grandparent-tasks'): object,
+
     # depname is used in taskref's to identify the taskID of the unsigned things
     Required('depname', default='build'): basestring,
 
@@ -217,28 +213,27 @@ def make_task_description(config, jobs):
         dependent_kind = dep_job.kind
         if dependent_kind == 'repackage-signing-l10n':
             dependent_kind = "repackage-signing"
-        dependencies = {dependent_kind: dep_job.label}
+        dependencies = {dependent_kind: dep_job}
 
         signing_name = "build-signing"
         if job.get('locale'):
             signing_name = "nightly-l10n-signing"
-        dependencies['signing'] = dep_job.dependencies[signing_name]
+        dependencies['signing'] = job['grandparent-tasks'][signing_name]
 
         build_name = "build"
         if job.get('locale'):
             build_name = "unsigned-repack"
-        dependencies["build"] = dep_job.dependencies[build_name]
+        dependencies["build"] = job['grandparent-tasks'][build_name]
 
         # repackage-l10n actually uses the repackage depname here
-        dependencies["repackage"] = dep_job.dependencies["repackage"]
+        dependencies["repackage"] = job['grandparent-tasks']["repackage"]
 
         # If this isn't a direct dependency, it won't be in there.
-        if 'repackage-signing' not in dependencies and \
-                'repackage-signing-l10n' not in dependencies:
+        if 'repackage-signing' not in dependencies:
             repackage_signing_name = "repackage-signing"
             if job.get('locale'):
                 repackage_signing_name = "repackage-signing-l10n"
-            dependencies["repackage-signing"] = dep_job.dependencies[repackage_signing_name]
+            dependencies["repackage-signing"] = job['grandparent-tasks'][repackage_signing_name]
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         if job.get('locale'):
@@ -264,7 +259,7 @@ def make_task_description(config, jobs):
         yield task
 
 
-def generate_upstream_artifacts(job, platform, locale=None, project=None):
+def generate_upstream_artifacts(job, dependencies, platform, locale=None, project=None):
 
     build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
     build_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
@@ -278,11 +273,9 @@ def generate_upstream_artifacts(job, platform, locale=None, project=None):
 
     upstream_artifacts = []
 
-    for task_type, cot_type, mapping in [
-        ("build", "build", build_mapping),
-        ("signing", "signing", build_signing_mapping),
-        ('repackage', 'repackage', repackage_mapping),
-        ('repackage-signing', 'repackage', repackage_signing_mapping),
+    for task_type, mapping in [
+        ("build", build_mapping),
+        ("signing", build_signing_mapping),
     ]:
         platform_was_previously_matched_by_regex = None
         for platform_regex, paths in mapping.iteritems():
@@ -294,26 +287,42 @@ def generate_upstream_artifacts(job, platform, locale=None, project=None):
                 if paths:
                     usable_paths = paths[:]
 
-                    use_stub = job["attributes"].get('stub-installer')
-                    if not use_stub:
-                        if 'target.stub-installer.exe' in usable_paths:
-                            usable_paths.remove('target.stub-installer.exe')
                     if 'target.langpack.xpi' in usable_paths and \
                             not project == "mozilla-central":
                         # XXX This is only beetmoved for m-c nightlies.
                         # we should determine this better
                         usable_paths.remove('target.langpack.xpi')
+
                         if not len(usable_paths):
                             # We may have removed our only path.
                             continue
+
                     upstream_artifacts.append({
                         "taskId": {"task-reference": "<{}>".format(task_type)},
-                        "taskType": cot_type,
-                        "paths": ["{}/{}".format(artifact_prefix, path)
-                                  for path in usable_paths],
+                        "taskType": task_type,
+                        "paths": ["{}/{}".format(artifact_prefix, path) for path in usable_paths],
                         "locale": locale or "en-US",
                     })
 
+    for task_type, cot_type, paths in [
+        ('repackage', 'repackage', repackage_mapping),
+        ('repackage-signing', 'repackage', repackage_signing_mapping),
+    ]:
+        paths = ["{}/{}".format(artifact_prefix, path) for path in paths]
+        paths = [
+            path for path in paths
+            if path in dependencies[task_type].release_artifacts]
+
+        if not paths:
+            continue
+
+        upstream_artifacts.append({
+            "taskId": {"task-reference": "<{}>".format(task_type)},
+            "taskType": cot_type,
+            "paths": paths,
+            "locale": locale or "en-US",
+        })
+
     return upstream_artifacts
 
 
@@ -371,7 +380,7 @@ def make_task_worker(config, jobs):
         platform = job["attributes"]["build_platform"]
 
         upstream_artifacts = generate_upstream_artifacts(
-            job, platform, locale,
+            job, job['dependencies'], platform, locale,
             project=config.params['project']
         )
 
@@ -443,3 +452,13 @@ def make_partials_artifacts(config, jobs):
         job['extra']['partials'] = extra
 
         yield job
+
+
+@transforms.add
+def convert_deps(config, jobs):
+    for job in jobs:
+        job['dependencies'] = {
+            name: dep_job.label
+            for name, dep_job in job['dependencies'].items()
+        }
+        yield job

taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py

@@ -35,6 +35,7 @@ def make_beetmover_description(config, jobs):
         beet_description = {
             'label': job['label'],
             'dependent-task': dep_job,
+            'grandparent-tasks': job['grandparent-tasks'],
             'treeherder': treeherder,
             'locale': locale,
             'shipping-phase': job.get('shipping-phase'),