Bug 1231320: pull from secrets API in TaskCluster r=garndt,mshal,pmoore

This adds a mozharness action, only run in TaskCluster, to fetch secrets from
the TaskCluster secrets API via the TaskCluster Proxy.

It requires that the SCM level of the build be passed in with the --scm-level
argument, defaulting to 1 (try)

MozReview-Commit-ID: C3rvOPF6Bm1

--HG--
extra : rebase_source : 603d9d30ba34484ceabdf405cb560f91b5bc2fc2

testing/mozharness/configs/builds/releng_base_android_64_builds.py

@@ -39,6 +39,10 @@ config = {
         ('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
         ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
     ],
+    'secret_files': [
+        {'filename': '/builds/mozilla-fennec-geoloc-api.key',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-fennec-geoloc-api.key'},
+    ],
     'enable_ccache': True,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox',

testing/mozharness/configs/builds/releng_base_linux_32_builds.py

@@ -46,6 +46,16 @@ config = {
         ('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
         ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
     ],
+    'secret_files': [
+        {'filename': '/builds/gapi.data',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data'},
+        {'filename': '/builds/mozilla-desktop-geoloc-api.key',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key'},
+        {'filename': '/builds/adjust-sdk.token',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token'},
+        {'filename': '/builds/adjust-sdk-beta.token',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token'},
+    ],
     'enable_ccache': True,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox',

testing/mozharness/configs/builds/releng_base_linux_64_builds.py

@@ -46,6 +46,16 @@ config = {
         ('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
         ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
     ],
+    'secret_files': [
+        {'filename': '/builds/gapi.data',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data'},
+        {'filename': '/builds/mozilla-desktop-geoloc-api.key',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key'},
+        {'filename': '/builds/adjust-sdk.token',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token'},
+        {'filename': '/builds/adjust-sdk-beta.token',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token'},
+    ],
     'enable_ccache': True,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox',

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -43,6 +43,7 @@ from mozharness.mozilla.buildbot import (
 )
 from mozharness.mozilla.purge import PurgeMixin
 from mozharness.mozilla.mock import MockMixin
+from mozharness.mozilla.secrets import SecretsMixin
 from mozharness.mozilla.signing import SigningMixin
 from mozharness.mozilla.mock import ERROR_MSGS as MOCK_ERROR_MSGS
 from mozharness.mozilla.testing.errors import TinderBoxPrintRe
@@ -545,6 +546,14 @@ BUILD_BASE_CONFIG_OPTIONS = [
                 " %s for possibilites" % (
                     BuildOptionParser.branch_cfg_file,
                 )}],
+    [['--scm-level'], {
+        "action": "store",
+        "type": "int",
+        "dest": "scm_level",
+        "default": 1,
+        "help": "This sets the SCM level for the branch being built."
+                " See https://www.mozilla.org/en-US/about/"
+                "governance/policies/commit/access-policy/"}],
     [['--enable-pgo'], {
         "action": "store_true",
         "dest": "pgo_build",
@@ -578,7 +587,7 @@ def generate_build_UID():
 
 class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
                   SigningMixin, VirtualenvMixin, MercurialScript,
-                  InfluxRecordingMixin):
+                  InfluxRecordingMixin, SecretsMixin):
     def __init__(self, **kwargs):
         # objdir is referenced in _query_abs_dirs() so let's make sure we
         # have that attribute before calling BaseScript.__init__

testing/mozharness/mozharness/mozilla/secrets.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""Support for fetching secrets from the secrets API
+"""
+
+import os
+import mozharness
+import urllib2
+import json
+from mozharness.base.log import ERROR
+
+
+class SecretsMixin(object):
+
+    def get_secrets(self):
+        """
+        Get the secrets specified by the `secret_files` configuration.  This is
+        a list of dictionaries, one for each secret.  The `secret_name` key
+        names the key in the TaskCluster secrets API to fetch (see
+        http://docs.taskcluster.net/services/secrets/).  It can contain
+        %-substitutions based on the `subst` dictionary below.
+
+        Since secrets must be JSON objects, the `content` property of the
+        secret is used as the value to be written to disk.
+
+        The `filename` key in the dictionary gives the filename to which the
+        secret should be written.
+        """
+        secret_files = self.config.get('secret_files', [])
+
+        subst = {
+            'scm-level': self.config.get('scm-level', 1),
+        }
+
+        for sf in secret_files:
+            filename = sf['filename']
+            secret_name = sf['secret_name'] % subst
+            self.info("fetching {} from secret {}".format(filename, secret_name))
+
+            # fetch from http://taskcluster, which points to the taskcluster proxy
+            # within a taskcluster task.  Outside of that environment, do not
+            # use this action.
+            url = "http://taskcluster/secrets/v1/secret/" + secret_name
+            res = urllib2.urlopen(url)
+            if res.getcode() != 200:
+                self.fatal("Error fetching from secrets API:" + res.read())
+
+            secret = json.load(res)['secret']['content']
+
+            open(filename, "w").write(filename)

testing/mozharness/scripts/fx_desktop_build.py

@@ -28,6 +28,7 @@ class FxDesktopBuild(BuildScript, object):
         buildscript_kwargs = {
             'config_options': BUILD_BASE_CONFIG_OPTIONS,
             'all_actions': [
+                'get-secrets',
                 'clobber',
                 'clone-tools',
                 'checkout-sources',

testing/taskcluster/scripts/builder/build-linux.sh

@@ -20,6 +20,7 @@ echo "running as" $(id)
 : MH_CUSTOM_BUILD_VARIANT_CFG   ${MH_CUSTOM_BUILD_VARIANT_CFG}
 : MH_BRANCH                     ${MH_BRANCH:=mozilla-central}
 : MH_BUILD_POOL                 ${MH_BUILD_POOL:=staging}
+: MOZ_SCM_LEVEL                 ${MOZ_SCM_LEVEL:=1}
 
 : WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
 
@@ -118,22 +119,18 @@ for cfg in $MOZHARNESS_CONFIG; do
   config_cmds="${config_cmds} --config ${cfg}"
 done
 
-# Mozharness would ordinarily do the checkouts itself, but they are disabled
-# here (--no-checkout-sources, --no-clone-tools) as the checkout is performed above.
+# Mozharness would ordinarily do a whole mess of buildbot-specific steps, but those
+# are overridden by this list of steps.  The get-secrets step is unique to TC tasks
+# and not run in Buildbot
+steps="--get-secrets --build --check-test"
 
 python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \
   $debug_flag \
   $custom_build_variant_cfg_flag \
   --disable-mock \
-  --no-setup-mock \
-  --no-checkout-sources \
-  --no-clone-tools \
-  --no-clobber \
-  --no-update \
-  --no-upload-files \
-  --no-sendchange \
+  $steps \
   --log-level=debug \
+  --scm-level=$MOZ_SCM_LEVEL \
   --work-dir=$WORKSPACE/build \
-  --no-action=generate-build-stats \
   --branch=${MH_BRANCH} \
   --build-pool=${MH_BUILD_POOL}

testing/taskcluster/tasks/build.yml

@@ -49,6 +49,7 @@ task:
       TOOLTOOL_REPO: 'https://git.mozilla.org/build/tooltool.git'
       TOOLTOOL_REV: 'master'
       MOZ_BUILD_DATE: '{{pushdate}}'
+      MOZ_SCM_LEVEL: '{{level}}'
 
   extra:
     build_product: '{{build_product}}'

testing/taskcluster/tasks/builds/firefox_base.yml

@@ -3,6 +3,8 @@ $inherits:
   variables:
     build_product: 'firefox'
 task:
+  scopes:
+    - "secrets:get:project/releng/gecko/build/level-{{level}}/*"
   extra:
     locations:
       mozharness: 'public/build/mozharness.zip'
@@ -12,3 +14,5 @@ task:
       type: 'task-image'
       path: 'public/image.tar'
       taskId: '{{#task_id_for_image}}desktop-build{{/task_id_for_image}}'
+    features:
+      taskclusterProxy: true