Backed out 2 changesets (bug 1607857) for causing lint failure at /builds/worker/checkouts/gecko/taskcluster/taskgraph/transforms/mar_signing.py

CLOSED TREE

Backed out changeset ec6201138386 (bug 1607857)
Backed out changeset a6d10498ee87 (bug 1607857)

taskcluster/docker/funsize-update-generator/runme.sh

@@ -53,6 +53,10 @@ else
   export MBSDIFF_HOOK=
 fi
 
+if [ -n "$FILENAME_TEMPLATE" ]; then
+    EXTRA_PARAMS="--filename-template $FILENAME_TEMPLATE $EXTRA_PARAMS"
+fi
+
 # EXTRA_PARAMS is optional
 # shellcheck disable=SC2086
 pipenv run /home/worker/bin/funsize.py \

taskcluster/docker/funsize-update-generator/scripts/funsize.py

@@ -46,6 +46,9 @@ STAGING_URL_PREFIXES = (
     "https://ftp.stage.mozaws.net/",
 )
 
+DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \
+                            "{locale}-{from_buildid}-{to_buildid}.partial.mar"
+
 BCJ_OPTIONS = {
     'x86': ['--x86'],
     'x86_64': ['--x86'],
@@ -215,6 +218,7 @@ async def generate_partial(work_env, from_dir, to_dir, dest_mar, mar_data,
     env["MOZ_PRODUCT_VERSION"] = mar_data['version']
     env["MAR_CHANNEL_ID"] = mar_data["MAR_CHANNEL_ID"]
     env['BRANCH'] = mar_data['branch']
+    env['PLATFORM'] = mar_data['platform']
     if use_old_format:
         env['MAR_OLD_FORMAT'] = '1'
     elif 'MAR_OLD_FORMAT' in env:
@@ -288,7 +292,7 @@ def verify_allowed_url(mar, allowed_url_prefixes):
         ))
 
 
-async def manage_partial(partial_def, artifacts_dir,
+async def manage_partial(partial_def, filename_template, artifacts_dir,
                          allowed_url_prefixes, signing_certs, arch=None):
     """Manage the creation of partial mars based on payload."""
 
@@ -355,6 +359,7 @@ async def manage_partial(partial_def, artifacts_dir,
                                section="Build", option="SourceStamp"),
         "from_mar": partial_def["from_mar"],
         "to_mar": partial_def["to_mar"],
+        "platform": partial_def["platform"],
         "locale": partial_def["locale"],
     }
 
@@ -370,7 +375,11 @@ async def manage_partial(partial_def, artifacts_dir,
     # if branch not set explicitly use repo-name
     mar_data['branch'] = partial_def.get('branch', mar_data['repo'].rstrip('/').split('/')[-1])
 
-    mar_name = partial_def['dest_mar']
+    if 'dest_mar' in partial_def:
+        mar_name = partial_def['dest_mar']
+    else:
+        # default to formatted name if not specified
+        mar_name = filename_template.format(**mar_data)
 
     mar_data['mar'] = mar_name
     dest_mar = os.path.join(work_env.workdir, mar_name)
@@ -410,6 +419,7 @@ async def async_main(args, signing_certs):
                                            ),
                                            kwargs=dict(
                                                partial_def=definition,
+                                               filename_template=args.filename_template,
                                                artifacts_dir=args.artifacts_dir,
                                                allowed_url_prefixes=allowed_url_prefixes,
                                                signing_certs=signing_certs,
@@ -432,6 +442,8 @@ def main():
                         default=strtobool(
                             os.environ.get('FUNSIZE_ALLOW_STAGING_PREFIXES', "false")),
                         help="Allow files from staging buckets.")
+    parser.add_argument("--filename-template",
+                        default=DEFAULT_FILENAME_TEMPLATE)
     parser.add_argument("-q", "--quiet", dest="log_level",
                         action="store_const", const=logging.WARNING,
                         default=logging.DEBUG)

taskcluster/taskgraph/transforms/beetmover_repackage.py

@@ -21,7 +21,7 @@ from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
                                          get_beetmover_action_scope,
                                          get_worker_type_for_scope)
 from taskgraph.util.taskcluster import get_artifact_prefix
-from taskgraph.util.treeherder import replace_group, inherit_treeherder_from_dep
+from taskgraph.util.treeherder import replace_group
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
@@ -61,7 +61,7 @@ def make_task_description(config, jobs):
         dep_job = job['primary-dependency']
         attributes = dep_job.attributes
 
-        treeherder = inherit_treeherder_from_dep(job, dep_job)
+        treeherder = job.get('treeherder', {})
         upstream_symbol = dep_job.task['extra']['treeherder']['symbol']
         if 'build' in job['dependent-tasks']:
             upstream_symbol = job['dependent-tasks']['build'].task['extra']['treeherder']['symbol']
@@ -69,6 +69,12 @@ def make_task_description(config, jobs):
             'symbol',
             replace_group(upstream_symbol, 'BMR')
         )
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 1)
+        treeherder.setdefault('kind', 'build')
         label = job['label']
         description = (
             "Beetmover submission for locale '{locale}' for build '"

taskcluster/taskgraph/transforms/mar_signing.py

@@ -15,7 +15,7 @@ from taskgraph.util.scriptworker import (
 )
 from taskgraph.util.partials import get_balrog_platform_name, get_partials_artifacts_from_params
 from taskgraph.util.taskcluster import get_artifact_prefix
-from taskgraph.util.treeherder import join_symbol, inherit_treeherder_from_dep
+from taskgraph.util.treeherder import join_symbol
 
 import logging
 logger = logging.getLogger(__name__)
@@ -96,12 +96,18 @@ def make_task_description(config, jobs):
         dep_job = job['primary-dependency']
         locale = dep_job.attributes.get('locale')
 
-        treeherder = inherit_treeherder_from_dep(job, dep_job)
+        treeherder = job.get('treeherder', {})
         treeherder.setdefault(
             'symbol', join_symbol(job.get('treeherder-group', 'ms'), locale or 'N')
         )
 
         label = job.get('label', "{}-{}".format(config.kind, dep_job.label))
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('kind', 'build')
+        treeherder.setdefault('tier', 1)
 
         dependencies = {dep_job.kind: dep_job.label}
         signing_dependencies = dep_job.dependencies
@@ -118,13 +124,14 @@ def make_task_description(config, jobs):
         if locale:
             attributes['locale'] = locale
 
-        build_platform = attributes.get('build_platform')
+        balrog_platform = get_balrog_platform_name(dep_th_platform)
         if config.kind == 'partials-signing':
             upstream_artifacts = generate_partials_artifacts(
-                dep_job, config.params['release_history'], build_platform, locale)
+                dep_job, config.params['release_history'], balrog_platform, locale)
         else:
             upstream_artifacts = generate_complete_artifacts(dep_job, config.kind)
 
+        build_platform = dep_job.attributes.get('build_platform')
         is_nightly = job.get(
             'nightly',  # First check current job
             dep_job.attributes.get(

taskcluster/taskgraph/transforms/partials.py

@@ -8,10 +8,9 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.partials import get_builds
+from taskgraph.util.partials import get_balrog_platform_name, get_builds
 from taskgraph.util.platforms import architecture
 from taskgraph.util.taskcluster import get_artifact_prefix
-from taskgraph.util.treeherder import inherit_treeherder_from_dep
 
 import logging
 logger = logging.getLogger(__name__)
@@ -46,10 +45,17 @@ def make_task_description(config, jobs):
     for job in jobs:
         dep_job = job['primary-dependency']
 
-        treeherder = inherit_treeherder_from_dep(job, dep_job)
+        treeherder = job.get('treeherder', {})
         treeherder.setdefault('symbol', 'p(N)')
 
         label = job.get('label', "partials-{}".format(dep_job.label))
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('kind', 'build')
+        treeherder.setdefault('tier', 1)
 
         dependencies = {dep_job.kind: dep_job.label}
 
@@ -62,8 +68,7 @@ def make_task_description(config, jobs):
 
         build_locale = locale or 'en-US'
 
-        build_platform = attributes['build_platform']
-        builds = get_builds(config.params['release_history'], build_platform,
+        builds = get_builds(config.params['release_history'], attributes["build_platform"],
                             build_locale)
 
         # If the list is empty there's no available history for this platform
@@ -85,6 +90,7 @@ def make_task_description(config, jobs):
                 'locale': build_locale,
                 'from_mar': builds[build]['mar_url'],
                 'to_mar': {'artifact-reference': artifact_path},
+                'platform': get_balrog_platform_name(dep_th_platform),
                 'branch': config.params['project'],
                 'update_number': update_number,
                 'dest_mar': build,
@@ -111,7 +117,7 @@ def make_task_description(config, jobs):
             'env': {
                 'SHA1_SIGNING_CERT': 'nightly_sha1',
                 'SHA384_SIGNING_CERT': 'nightly_sha384',
-                'EXTRA_PARAMS': '--arch={}'.format(architecture(build_platform)),
+                'EXTRA_PARAMS': '--arch={}'.format(architecture(attributes['build_platform'])),
                 'MAR_CHANNEL_ID': attributes['mar-channel-id']
             }
         }
@@ -134,7 +140,7 @@ def make_task_description(config, jobs):
 
         # We only want caching on linux/windows due to bug 1436977
         if int(level) == 3 \
-                and any([build_platform.startswith(prefix) for prefix in ['linux', 'win']]):
+                and any([platform in dep_th_platform for platform in ['linux', 'windows']]):
             task['scopes'].append(
                 'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/')