Bug 1286075: add upload-symbols kind; r=ted.mielczarek

MozReview-Commit-ID: C3Se6gdSPri

--HG--
extra : rebase_source : 593030907b5edcdbb0f8863f4226a73b50d32b51

taskcluster/ci/upload-symbols/job-template.yml

@@ -0,0 +1,19 @@
+label: # see transforms
+description: Upload Symbols
+dependencies: # see transforms
+expires-after: 7 days
+deadline-after: 24 hours
+run-on-projects:
+    - try
+worker-type: aws-provisioner-v1/symbol-upload
+worker:
+    implementation: docker-worker
+    max-run-time: 600
+    command: ["/bin/bash", "bin/upload.sh"]
+    docker-image: taskclusterprivate/upload_symbols:0.0.3
+    env:
+        GECKO_HEAD_REPOSITORY: # see transforms
+        GECKO_HEAD_REV: # see transforms
+        ARTIFACT_TASKID: {"task-reference": "<build>"}
+scopes:
+    - docker-worker:image:taskclusterprivate/upload_symbols:0.0.3

taskcluster/ci/upload-symbols/kind.yml

@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.post_build:PostBuildTask
+
+transforms:
+   - taskgraph.transforms.upload_symbols:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+    - build
+
+job-template: job-template.yml
+
+only-for-build-platforms:
+    - linux64/opt
+    - linux64/debug
+    - android-api-15/opt

taskcluster/docs/kinds.rst

@@ -38,6 +38,12 @@ correctness.  This can include linting, Python unit tests, source-code
 analysis, or measurement work -- basically anything that does not require a
 build.
 
+upload-symbols
+--------------
+
+Upload-symbols tasks run after builds and upload the symbols files generated by
+build tasks to Socorro for later use in crash analysis.
+
 Tests
 -----
 

taskcluster/taskgraph/task/post_build.py

@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+import logging
+
+from . import transform
+from ..util.yaml import load_yaml
+
+logger = logging.getLogger(__name__)
+
+
+class PostBuildTask(transform.TransformTask):
+    """
+    A task implementing a post-build job.  These depend on jobs and perform
+    various followup tasks after a build has completed.
+
+    The `only-for-build-platforms` kind configuration, if specified, will limit
+    the build platforms for which a post-build task will be created.
+
+    The `job-template' kind configuration points to a yaml file which will
+    be used to create the input to the transforms.  It will have added to it
+    keys `build-label`, the label for the build task, and `build-platform`, its
+    platform.
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+        if config.get('kind-dependencies', []) != ["build"]:
+            raise Exception("PostBuildTask kinds must depend on builds")
+
+        only_platforms = config.get('only-for-build-platforms')
+        prototype = load_yaml(path, config.get('job-template'))
+
+        for task in loaded_tasks:
+            if task.kind != 'build':
+                continue
+
+            build_platform = task.attributes.get('build_platform')
+            build_type = task.attributes.get('build_type')
+            if not build_platform or not build_type:
+                continue
+            platform = "{}/{}".format(build_platform, build_type)
+            if only_platforms and platform not in only_platforms:
+                continue
+
+            post_task = copy.deepcopy(prototype)
+            post_task['build-label'] = task.label
+            post_task['build-platform'] = platform
+            yield post_task

taskcluster/taskgraph/transforms/upload_symbols.py

@@ -0,0 +1,36 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the upload-symbols task description template,
+  taskcluster/ci/upload-symbols/job-template.yml
+into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def fill_template(config, tasks):
+    for task in tasks:
+        # Fill out the dynamic fields in the task description
+        task['label'] = task['build-label'] + '-upload-symbols'
+        task['dependencies'] = {'build': task['build-label']}
+        task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
+        task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev']
+
+        build_platform, build_type = task['build-platform'].split('/')
+        attributes = task.setdefault('attributes', {})
+        attributes['build_platform'] = build_platform
+        attributes['build_type'] = build_type
+
+        # clear out the stuff that's not part of a task description
+        del task['build-label']
+        del task['build-platform']
+
+        yield task

taskcluster/taskgraph/try_option_syntax.py

@@ -27,6 +27,7 @@ BUILD_KINDS = set([
     'artifact-build',
     'hazard',
     'l10n',
+    'upload-symbols',
 ])
 
 # anything in this list is governed by -j