bug 1412690 - add fennec release bouncer sub. r=rail

This patch adds the first releasetask as a new kind. To support this, we
added a new `release-promotion` flag in the buildbot job. If this is
set, we use the new `bb_release_worker` function; otherwise we fall back
to the `bb_ci_worker` function (this is the old behavior, factored out
into a separate function).

We also added `build_number` and `release_promotion` attributes in the
task definition.  Finally, `build_number` now defaults to 1, allowing us
to create the task graph locally without forcing us to set
`BUILD_NUMBER` in the environment.

MozReview-Commit-ID: 8vNMHJemqAG

--HG--
extra : rebase_source : c8816d3e7b2f5358ea51ee8d5fb12585e67e9853

taskcluster/ci/release-bouncer-sub/kind.yml

@@ -0,0 +1,25 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+
+jobs:
+   fennec:
+      name: fennec_release_bouncer_sub
+      description: release bouncer submission job
+      worker-type: buildbot-bridge/buildbot-bridge
+      run-on-projects: []
+      run:
+         using: buildbot
+         product: fennec
+         buildername: release-{branch}-fennec_bncr_sub
+         release-promotion: true
+         routes:
+            - index.releases.v1.{branch}.latest.fennec.latest.bouncer_submitter
+            - index.releases.v1.{branch}.{revision}.fennec.{underscore_version}.build{build_number}.bouncer_submitter

taskcluster/docs/kinds.rst

@@ -154,9 +154,6 @@ The task definition used to create the image-building tasks is given in
 ``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML
 Template <yaml-templates>`.
 
-android-stuff
--------------
-
 balrog
 ------
 
@@ -217,13 +214,16 @@ PushApk publishes Android packages onto Google Play Store. Jobs of this kind tak
 all the signed multi-locales (aka "multi") APKs for a given release and upload them
 all at once. They also depend on the breakpoint.
 
+release-bouncer-sub
+-------------------
+Submits bouncer updates for releases.
+
 repackage
 ---------
 Repackage tasks take a signed output and package them up into something suitable
 for shipping to our users. For example, on OSX we return a tarball as the signed output
 and this task would package that up as an Apple Disk Image (.dmg)
 
-
 repackage-l10n
 --------------
 Repackage-L10n is a ```Repackage``` task split up to be suitable for use after l10n repacks.

taskcluster/taskgraph/transforms/job/buildbot.py

@@ -9,9 +9,11 @@ Support for running jobs via buildbot.
 
 from __future__ import absolute_import, print_function, unicode_literals
 import slugid
+from urlparse import urlparse
 
 from taskgraph.util.schema import Schema
-from voluptuous import Required, Any
+from taskgraph.util.scriptworker import get_release_config
+from voluptuous import Optional, Required, Any
 
 from taskgraph.transforms.job import run_job_using
 
@@ -23,10 +25,53 @@ buildbot_run_schema = Schema({
     Required('buildername'): basestring,
 
     # the product to use
-    Required('product'): Any('firefox', 'mobile', 'devedition', 'thunderbird'),
+    Required('product'): Any('firefox', 'mobile', 'fennec', 'devedition', 'thunderbird'),
+
+    Optional('release-promotion'): bool,
+    Optional('routes'): [basestring],
 })
 
 
+def bb_release_worker(config, worker, run):
+    # props
+    release_props = get_release_config(config, force=True)
+    repo_path = urlparse(config.params['head_repository']).path.lstrip('/')
+    revision = config.params['head_rev']
+    branch = config.params['project']
+    buildername = worker['buildername']
+    underscore_version = release_props['version'].replace('.', '_')
+    release_props.update({
+        'release_promotion': True,
+        'repo_path': repo_path,
+        'revision': revision,
+        'script_repo_revision': revision,
+    })
+    worker['properties'].update(release_props)
+    # scopes
+    worker['scopes'] = [
+        "project:releng:buildbot-bridge:builder-name:{}".format(buildername)
+    ]
+    # routes
+    if run.get('routes'):
+        worker['routes'] = []
+        repl_dict = {
+            'branch': branch,
+            'build_number': str(release_props['build_number']),
+            'revision': revision,
+            'underscore_version': underscore_version,
+        }
+        for route in run['routes']:
+            route = route.format(**repl_dict)
+            worker['routes'].append(route)
+
+
+def bb_ci_worker(config, worker):
+    worker['properties'].update({
+        'who': config.params['owner'],
+        'upload_to_task_id': slugid.nice(),
+    })
+
+
 @run_job_using('buildbot-bridge', 'buildbot', schema=buildbot_run_schema)
 def mozharness_on_buildbot_bridge(config, job, taskdesc):
     run = job['run']
@@ -35,17 +80,21 @@ def mozharness_on_buildbot_bridge(config, job, taskdesc):
     product = run['product']
 
     buildername = run['buildername'].format(branch=branch)
+    revision = config.params['head_rev']
 
     worker.update({
         'buildername': buildername,
         'sourcestamp': {
             'branch': branch,
             'repository': config.params['head_repository'],
-            'revision': config.params['head_rev'],
+            'revision': revision,
         },
         'properties': {
             'product': product,
-            'who': config.params['owner'],
-            'upload_to_task_id': slugid.nice(),
-        }
+        },
     })
+
+    if run.get('release-promotion'):
+        bb_release_worker(config, worker, run)
+    else:
+        bb_ci_worker(config, worker)

taskcluster/taskgraph/transforms/task.py

@@ -375,8 +375,12 @@ task_description_schema = Schema({
         },
         Required('properties'): {
             'product': basestring,
+            Optional('build_number'): int,
+            Optional('release_promotion'): bool,
             Extra: taskref_or_string,  # additional properties are allowed
         },
+        Optional('scopes'): [basestring],
+        Optional('routes'): [basestring],
     }, {
         Required('implementation'): 'native-engine',
         Required('os'): Any('macosx', 'linux'),
@@ -1003,6 +1007,8 @@ def build_buildbot_bridge_payload(config, task, task_def):
         'sourcestamp': worker['sourcestamp'],
         'properties': worker['properties'],
     }
+    task_def['scopes'].extend(worker.get('scopes', []))
+    task_def['routes'].extend(worker.get('routes', []))
 
 
 transforms = TransformSequence()

taskcluster/taskgraph/util/schema.py

@@ -144,6 +144,9 @@ def resolve_keyed_by(item, field, item_name, **extra_values):
 WHITELISTED_SCHEMA_IDENTIFIERS = [
     # upstream-artifacts are handed directly to scriptWorker, which expects interCaps
     lambda path: "[u'upstream-artifacts']" in path,
+    # bbb release promotion properties
+    lambda path: path.endswith("[u'build_number']"),
+    lambda path: path.endswith("[u'release_promotion']"),
 ]
 
 

taskcluster/taskgraph/util/scriptworker.py

@@ -396,7 +396,7 @@ get_push_apk_rollout_percentage = functools.partial(
 
 
 # release_config {{{1
-def get_release_config(config):
+def get_release_config(config, force=False):
     """Get the build number and version for a release task.
 
     Currently only applies to beetmover tasks.
@@ -413,8 +413,8 @@ def get_release_config(config):
             update `task.payload`.
     """
     release_config = {}
-    if config.params['target_tasks_method'] in BEETMOVER_RELEASE_TARGET_TASKS:
-        build_number = str(os.environ.get("BUILD_NUMBER", ""))
+    if force or config.params['target_tasks_method'] in BEETMOVER_RELEASE_TARGET_TASKS:
+        build_number = str(os.environ.get("BUILD_NUMBER", 1))
         if not build_number.isdigit():
             raise ValueError("Release graphs must specify `BUILD_NUMBER` in the environment!")
         release_config['build_number'] = int(build_number)