Bug 1529921: Use secrets from taskcluster for windows builds; r=aki

Differential Revision: https://phabricator.services.mozilla.com/D20849 --HG-- extra : moz-landing-system : lando
2019-02-22 22:01:44 +00:00 · 2019-02-22 22:01:44 +00:00 · 8abfdcb46d
--- a/browser/config/mozconfigs/win32/common-opt
+++ b/browser/config/mozconfigs/win32/common-opt
@ -6,14 +6,9 @@

 ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}

-if [ -f /c/builds/gapi.data ]; then
-  _gapi_keyfile=c:/builds/gapi.data
-else
-  _gapi_keyfile=e:/builds/gapi.data
-fi
-ac_add_options --with-google-api-keyfile=${_gapi_keyfile}
+ac_add_options --with-google-api-keyfile=z:/build/gapi.data

-ac_add_options --with-mozilla-api-keyfile=c:/builds/mozilla-desktop-geoloc-api.key
+ac_add_options --with-mozilla-api-keyfile=z:/build/mozilla-desktop-geoloc-api.key

 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
--- a/browser/config/mozconfigs/win64-aarch64/common-opt
+++ b/browser/config/mozconfigs/win64-aarch64/common-opt
@ -6,14 +6,9 @@

 ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}

-if [ -f /c/builds/gapi.data ]; then
-  _gapi_keyfile=c:/builds/gapi.data
-else
-  _gapi_keyfile=e:/builds/gapi.data
-fi
-ac_add_options --with-google-api-keyfile=${_gapi_keyfile}
+ac_add_options --with-google-api-keyfile=z:/build/gapi.data

-ac_add_options --with-mozilla-api-keyfile=c:/builds/mozilla-desktop-geoloc-api.key
+ac_add_options --with-mozilla-api-keyfile=z:/build/mozilla-desktop-geoloc-api.key

 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
--- a/browser/config/mozconfigs/win64/common-opt
+++ b/browser/config/mozconfigs/win64/common-opt
@ -6,14 +6,9 @@

 ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}

-if [ -f /c/builds/gapi.data ]; then
-  _gapi_keyfile=c:/builds/gapi.data
-else
-  _gapi_keyfile=e:/builds/gapi.data
-fi
-ac_add_options --with-google-api-keyfile=${_gapi_keyfile}
+ac_add_options --with-google-api-keyfile=z:/build/gapi.data

-ac_add_options --with-mozilla-api-keyfile=c:/builds/mozilla-desktop-geoloc-api.key
+ac_add_options --with-mozilla-api-keyfile=z:/build/mozilla-desktop-geoloc-api.key

 # Needed to enable breakpad in application.ini
 export MOZILLA_OFFICIAL=1
--- a/browser/config/mozconfigs/win64/nightly-asan-reporter
+++ b/browser/config/mozconfigs/win64/nightly-asan-reporter
@ -1,13 +1,8 @@
 MOZ_AUTOMATION_L10N_CHECK=0

 ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}
-if [ -f /c/builds/gapi.data ]; then
-  _gapi_keyfile=c:/builds/gapi.data
-else
-  _gapi_keyfile=e:/builds/gapi.data
-fi
-ac_add_options --with-google-api-keyfile=${_gapi_keyfile}
-ac_add_options --with-mozilla-api-keyfile=c:/builds/mozilla-desktop-geoloc-api.key
+ac_add_options --with-google-api-keyfile=z:/build/gapi.data
+ac_add_options --with-mozilla-api-keyfile=z:/build/mozilla-desktop-geoloc-api.key

 . "$topsrcdir/build/mozconfig.win-common"
 . "$topsrcdir/browser/config/mozconfigs/common"
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@ -22,6 +22,7 @@ win32/debug:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -56,6 +57,7 @@ win32/opt:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -91,6 +93,7 @@ win32/pgo:
    run:
        options: [enable-pgo, append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -125,6 +128,7 @@ win64/debug:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -160,6 +164,7 @@ win64-fuzzing/debug:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -192,6 +197,7 @@ win64-plain/debug:
            PERFHERDER_EXTRA_OPTIONS: plain
            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
    run:
+        actions: [build, check-test]
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
        config:
@ -229,6 +235,7 @@ win64/opt:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -311,6 +318,7 @@ win32-nightly/opt:
        actions: [build, check-test]
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -347,9 +355,9 @@ win64-nightly/opt:
        env:
            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
    run:
-        actions: [build, check-test]
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -384,6 +392,7 @@ win64/pgo:
    run:
        options: [enable-pgo, append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -416,6 +425,7 @@ win32-add-on-devel/opt:
    run:
        options: [append-env-variables-from-configs]
        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -450,6 +460,7 @@ win64-add-on-devel/opt:
    run:
        options: [append-env-variables-from-configs]
        script: "mozharness/scripts/fx_desktop_build.py"
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -484,6 +495,7 @@ win64-noopt/debug:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -517,6 +529,7 @@ win32-noopt/debug:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -549,9 +562,10 @@ win32-rusttests/opt:
            PERFHERDER_EXTRA_OPTIONS: rusttests
            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
    run:
-        actions: [build]
+        actions: [get-secrets, build]
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -587,9 +601,10 @@ win32-rusttests/debug:
            PERFHERDER_EXTRA_OPTIONS: rusttests
            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
    run:
-        actions: [build]
+        actions: [get-secrets, build]
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -625,8 +640,10 @@ win64-rusttests/opt:
            PERFHERDER_EXTRA_OPTIONS: rusttests
            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
    run:
+        actions: [get-secrets, build]
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -658,8 +675,10 @@ win64-rusttests/debug:
            PERFHERDER_EXTRA_OPTIONS: rusttests
            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
    run:
+        actions: [get-secrets, build]
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -696,6 +715,7 @@ win64-ccov/debug:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -733,6 +753,7 @@ win64-asan/debug:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -767,6 +788,7 @@ win64-asan/opt:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -808,6 +830,7 @@ win64-asan-reporter-nightly/opt:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -843,6 +866,7 @@ win64-asan-fuzzing/opt:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -889,6 +913,7 @@ win32-devedition-nightly/opt:
            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
    run:
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        options: [enable-pgo, append-env-variables-from-configs]
        config:
            - builds/releng_base_firefox.py
@ -930,6 +955,7 @@ win64-devedition-nightly/opt:
    run:
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -968,6 +994,7 @@ win64-aarch64/debug:
        actions: [get-secrets, build]
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -1004,6 +1031,7 @@ win64-aarch64/opt:
        actions: [get-secrets, build]
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
@ -1044,6 +1072,7 @@ win64-aarch64-nightly/opt:
        actions: [get-secrets, build]
        options: [append-env-variables-from-configs]
        script: mozharness/scripts/fx_desktop_build.py
+        secrets: true
        config:
            - builds/releng_base_firefox.py
            - builds/taskcluster_base_windows.py
--- a/taskcluster/ci/searchfox/kind.yml
+++ b/taskcluster/ci/searchfox/kind.yml
@ -104,6 +104,7 @@ jobs:
                MOZSEARCH_PLATFORM: "windows"
        run:
            using: mozharness
+            actions: [build]
            use-caches: false
            options: [append-env-variables-from-configs]
            script: mozharness/scripts/fx_desktop_build.py
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@ -191,7 +191,7 @@ def generic_worker_hg_commands(base_repo, head_repo, head_rev, path,
    return [' '.join(args), ' '.join(logging_args)]


-def docker_worker_setup_secrets(config, job, taskdesc):
+def setup_secrets(config, job, taskdesc):
    """Set up access to secrets via taskcluster-proxy.  The value of
    run['secrets'] should be a boolean or a list of secret names that
    can be accessed."""
--- a/taskcluster/taskgraph/transforms/job/hazard.py
+++ b/taskcluster/taskgraph/transforms/job/hazard.py
@ -13,7 +13,7 @@ from voluptuous import Required, Optional, Any
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
    docker_worker_add_workspace_cache,
-    docker_worker_setup_secrets,
+    setup_secrets,
    docker_worker_add_artifacts,
    docker_worker_add_tooltool,
    support_vcs_checkout,
@ -50,7 +50,7 @@ def docker_worker_hazard(config, job, taskdesc):
    docker_worker_add_artifacts(config, job, taskdesc)
    docker_worker_add_workspace_cache(config, job, taskdesc)
    docker_worker_add_tooltool(config, job, taskdesc)
-    docker_worker_setup_secrets(config, job, taskdesc)
+    setup_secrets(config, job, taskdesc)
    support_vcs_checkout(config, job, taskdesc)

    env = worker['env']
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@ -20,7 +20,7 @@ from voluptuous.validators import Match
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
    docker_worker_add_workspace_cache,
-    docker_worker_setup_secrets,
+    setup_secrets,
    docker_worker_add_artifacts,
    docker_worker_add_tooltool,
    generic_worker_add_artifacts,
@ -218,7 +218,7 @@ def mozharness_on_docker_worker_setup(config, job, taskdesc):
    # Retry if mozharness returns TBPL_RETRY
    worker['retry-exit-status'] = [4]

-    docker_worker_setup_secrets(config, job, taskdesc)
+    setup_secrets(config, job, taskdesc)

    command = [
        '{workdir}/bin/run-task'.format(**run),
@ -247,8 +247,7 @@ def mozharness_on_generic_worker(config, job, taskdesc):

    # fail if invalid run options are included
    invalid = []
-    for prop in ['tooltool-downloads',
-                 'secrets', 'taskcluster-proxy', 'need-xvfb']:
+    for prop in ['tooltool-downloads', 'taskcluster-proxy', 'need-xvfb']:
        if prop in run and run[prop]:
            invalid.append(prop)
    if not run.get('keep-artifacts', True):
@ -259,6 +258,8 @@ def mozharness_on_generic_worker(config, job, taskdesc):

    worker = taskdesc['worker']

+    setup_secrets(config, job, taskdesc)
+
    taskdesc['worker'].setdefault('artifacts', []).append({
        'name': 'public/logs',
        'path': 'logs',
--- a/testing/mozharness/configs/builds/taskcluster_base_windows.py
+++ b/testing/mozharness/configs/builds/taskcluster_base_windows.py
@ -2,6 +2,7 @@ import os

 config = {
    'default_actions': [
+        'get-secrets',
        'build',
        'check-test',
    ],
@ -25,4 +26,12 @@ config = {
    'upload_env': {
        'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
    },
+    'secret_files': [
+        {'filename': 'z:/build/gapi.data',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data',
+         'min_scm_level': 1},
+        {'filename': 'z:/build/mozilla-desktop-geoloc-api.key',
+         'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key',
+         'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
+    ],
 }