Bug 1442545: [partner-repack] Add parameters needed for doing partner repacks; r=Callek

Differential Revision: https://phabricator.services.mozilla.com/D977

--HG--
extra : rebase_source : 0f2ef4edf6093d644c82c6b0475ba7796ebaa90b
extra : histedit_source : b446b09b30a0f709f4c80eaf69ec40952562ce6d

taskcluster/docs/parameters.rst

@@ -159,6 +159,21 @@ Release Promotion
 ``release_eta``
    The time and date when a release is scheduled to live. This value is passed to Balrog.
 
+``release_enable_partners``
+   Boolean which controls repacking vanilla Firefox builds for partners.
+
+``release_partners``
+   List of partners to repack. A null value defaults to all.
+
+``release_partner_config``
+   Configuration for partner repacks.
+
+``release_partner_build_number``
+   The build number for partner repacks. We sometimes have multiple partner build numbers per release build number; this parameter lets us bump them independently. Defaults to 1.
+
+``release_enable_emefree``
+   Boolean which controls repacking vanilla Firefox builds into EME-free builds.
+
 Comm Push Information
 ---------------------
 

taskcluster/taskgraph/actions/release_promotion.py

@@ -15,6 +15,7 @@ from .util import (find_decision_task, find_existing_tasks_from_previous_kinds,
                    find_hg_revision_pushlog_id)
 from taskgraph.util.taskcluster import get_artifact
 from taskgraph.util.partials import populate_release_history
+from taskgraph.util.partners import fix_partner_config
 from taskgraph.taskgraph import TaskGraph
 from taskgraph.decision import taskgraph_decision
 from taskgraph.parameters import Parameters
@@ -56,6 +57,18 @@ RELEASE_PROMOTION_CONFIG = {
         'product': 'firefox',
         'release_type': 'rc',
     },
+    'promote_firefox_partners': {
+        'target_tasks_method': 'promote_firefox',
+        'product': 'firefox',
+        'rebuild_kinds': [
+            'release-partner-repack',
+            'release-partner-beetmover',
+            'release-partner-repack-chunking-dummy',
+            'release-partner-repackage-signing',
+            'release-partner-repackage',
+            'release-partner-signing',
+        ],
+    },
     'promote_devedition': {
         'target_tasks_method': 'promote_devedition',
         'product': 'devedition',
@@ -88,6 +101,9 @@ PARTIAL_UPDATES_FLAVORS = (
     'ship_devedition',
 )
 
+PARTNER_BRANCHES = ('mozilla-beta', 'mozilla-release', 'maple', 'birch', 'jamun')
+EMEFREE_BRANCHES = ('mozilla-beta', 'mozilla-release', 'maple', 'birch', 'jamun')
+
 
 def is_release_promotion_available(parameters):
     return parameters['project'] in RELEASE_PROMOTION_PROJECTS
@@ -203,11 +219,42 @@ def is_release_promotion_available(parameters):
                     'additionalProperties': False,
                 }
             },
-
             'release_eta': {
                 'type': 'string',
                 'default': '',
             },
+            'release_enable_partners': {
+                'type': 'boolean',
+                'default': False,
+                'description': ('Toggle for creating partner repacks'),
+            },
+            'release_partner_build_number': {
+                'type': 'integer',
+                'default': 1,
+                'minimum': 1,
+                'description': ('The partner build number. This translates to, e.g. '
+                                '`v1` in the path. We generally only have to '
+                                'bump this on off-cycle partner rebuilds.'),
+            },
+            'release_partners': {
+                'type': 'array',
+                'description': ('A list of partners to repack, or if null or empty then use '
+                                'the current full set'),
+                'items': {
+                    'type': 'string',
+                }
+            },
+            'release_partner_config': {
+                'type': 'object',
+                'description': ('Partner configuration to use for partner repacks.'),
+                'properties': {},
+                'additionalProperties': True,
+            },
+            'release_enable_emefree': {
+                'type': 'boolean',
+                'default': False,
+                'description': ('Toggle for creating EME-free repacks'),
+            },
         },
         "required": ['release_promotion_flavor', 'build_number'],
     }
@@ -253,6 +300,14 @@ def release_promotion_action(parameters, graph_config, input, task_group_id, tas
     do_not_optimize = input.get(
         'do_not_optimize', promotion_config.get('do_not_optimize', [])
     )
+    release_enable_partners = input.get(
+        'release_enable_partners',
+        parameters['project'] in PARTNER_BRANCHES and product in ('firefox',)
+    )
+    release_enable_emefree = input.get(
+        'release_enable_emefree',
+        parameters['project'] in EMEFREE_BRANCHES and product in ('firefox',)
+    )
 
     # make parameters read-write
     parameters = dict(parameters)
@@ -287,6 +342,12 @@ def release_promotion_action(parameters, graph_config, input, task_group_id, tas
     parameters['release_history'] = release_history
     parameters['release_type'] = promotion_config.get('release_type', '')
     parameters['release_eta'] = input.get('release_eta', '')
+    parameters['release_enable_partners'] = release_enable_partners
+    parameters['release_partners'] = input.get('release_partners')
+    if input.get('release_partner_config'):
+        parameters['release_partner_config'] = fix_partner_config(input['release_partner_config'])
+    parameters['release_enable_emefree'] = release_enable_emefree
+
     if input['version']:
         parameters['version'] = input['version']
 

taskcluster/taskgraph/decision.py

@@ -198,6 +198,11 @@ def get_decision_parameters(options):
     parameters['next_version'] = None
     parameters['release_type'] = ''
     parameters['release_eta'] = ''
+    parameters['release_enable_partners'] = False
+    parameters['release_partners'] = []
+    parameters['release_partner_config'] = {}
+    parameters['release_partner_build_number'] = 1
+    parameters['release_enable_emefree'] = False
 
     # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
     # case, fake it

taskcluster/taskgraph/parameters.py

@@ -60,8 +60,13 @@ PARAMETERS = {
     'project': 'mozilla-central',
     'pushdate': lambda: int(time.time()),
     'pushlog_id': '0',
+    'release_enable_emefree': False,
+    'release_enable_partners': False,
     'release_eta': '',
     'release_history': {},
+    'release_partners': None,
+    'release_partner_config': None,
+    'release_partner_build_number': 1,
     'release_type': '',
     'target_tasks_method': 'default',
     'try_mode': None,

taskcluster/taskgraph/util/partners.py

@@ -0,0 +1,103 @@
+from __future__ import absolute_import, print_function, unicode_literals
+
+from copy import deepcopy
+import json
+import os
+
+LOCALES_FILE = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
+    'browser', 'locales', 'l10n-changesets.json'
+)
+
+
+def check_if_partners_enabled(config, tasks):
+    if (
+        config.params['release_enable_partners'] and
+        config.kind.startswith('release-partner-repack')
+    ) or (
+        config.params['release_enable_emefree'] and
+        config.kind.startswith('release-eme-free-repack')
+    ):
+        for task in tasks:
+            yield task
+
+
+def get_partner_config_by_kind(config, kind):
+    """ Retrieve partner data starting from the manifest url, which points to a repository
+    containing a default.xml that is intended to be drive the Google tool 'repo'. It
+    descends into each partner repo to lookup and parse the repack.cfg file(s).
+
+    Supports caching data by kind to avoid repeated requests, relying on the related kinds for
+    partner repacking, signing, repackage, repackage signing all having the same kind prefix.
+    """
+    partner_subset = config.params['release_partners']
+    partner_configs = config.params['release_partner_config'] or {}
+
+    # TODO eme-free should be a partner; we shouldn't care about per-kind
+    for k in partner_configs:
+        if kind.startswith(k):
+            kind_config = partner_configs[k]
+            break
+    else:
+        return {}
+    # if we're only interested in a subset of partners we remove the rest
+    if isinstance(partner_subset, (list, tuple)):
+        # TODO - should be fatal to have an unknown partner in partner_subset
+        for partner in kind_config.keys():
+            if partner not in partner_subset:
+                del(kind_config[partner])
+
+    return kind_config
+
+
+def _fix_subpartner_locales(orig_config, all_locales):
+    subpartner_config = deepcopy(orig_config)
+    # Get an ordered list of subpartner locales that is a subset of all_locales
+    subpartner_config['locales'] = sorted(list(
+        set(orig_config['locales']) & set(all_locales)
+    ))
+    return subpartner_config
+
+
+def fix_partner_config(orig_config):
+    pc = {}
+    with open(LOCALES_FILE, 'r') as fh:
+        all_locales = json.load(fh).keys()
+    # l10n-changesets.json doesn't include en-US, but the repack list does
+    if 'en-US' not in all_locales:
+        all_locales.append('en-US')
+    for kind, kind_config in orig_config.iteritems():
+        for partner, partner_config in kind_config.iteritems():
+            for subpartner, subpartner_config in partner_config.iteritems():
+                # get rid of empty subpartner configs
+                if not subpartner_config:
+                    continue
+                # Make sure our locale list is a subset of all_locales
+                pc.setdefault(kind, {}).setdefault(partner, {})[subpartner] = \
+                    _fix_subpartner_locales(subpartner_config, all_locales)
+    return pc
+
+
+# seems likely this exists elsewhere already
+def get_ftp_platform(platform):
+    if platform.startswith('win32'):
+        return 'win32'
+    elif platform.startswith('win64'):
+        return 'win64'
+    elif platform.startswith('macosx'):
+        return 'mac'
+    elif platform.startswith('linux-'):
+        return 'linux-i686'
+    elif platform.startswith('linux64'):
+        return 'linux-x86_64'
+    else:
+        raise ValueError('Unimplemented platform %s'.format(platform))
+
+
+# Ugh
+def locales_per_build_platform(build_platform, locales):
+    if build_platform.startswith('mac'):
+        exclude = ['ja']
+    else:
+        exclude = ['ja-JP-mac']
+    return [locale for locale in locales if locale not in exclude]