Bug 1405413 - Migrate Gradle dependency fetching to toolchain dependencies. r=dustin

MozReview-Commit-ID: 6OOD4mhICG1 --HG-- rename : taskcluster/scripts/builder/build-android-dependencies.sh => taskcluster/scripts/misc/android-gradle-dependencies.sh rename : taskcluster/scripts/builder/build-android-dependencies/after.sh => taskcluster/scripts/misc/android-gradle-dependencies/after.sh rename : taskcluster/scripts/builder/build-android-dependencies/before.sh => taskcluster/scripts/misc/android-gradle-dependencies/before.sh rename : taskcluster/scripts/builder/build-android-dependencies/nexus.xml => taskcluster/scripts/misc/android-gradle-dependencies/nexus.xml extra : rebase_source : 8c476377ded6b7177b3ba45f648431a08468948d
2017-10-13 12:59:04 -07:00 · 2017-10-13 12:59:04 -07:00 · e83bd0bc4e
--- a/mobile/android/config/mozconfigs/android-api-16-frontend/nightly
+++ b/mobile/android/config/mozconfigs/android-api-16-frontend/nightly
@ -14,8 +14,8 @@ NO_NDK=1

 . "$topsrcdir/mobile/android/config/mozconfigs/common"

-ac_add_options --with-gradle="$topsrcdir/gradle-dist/bin/gradle"
-export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/jcenter","file://$topsrcdir/google"
+ac_add_options --with-gradle="$topsrcdir/android-gradle-dependencies/gradle-dist/bin/gradle"
+export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/android-gradle-dependencies/jcenter","file://$topsrcdir/android-gradle-dependencies/google"

 unset HOST_CC
 unset HOST_CXX
--- a/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
+++ b/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
@ -53,3 +53,12 @@ export MOZ_ANDROID_POCKET=1
 . "$topsrcdir/mobile/android/config/mozconfigs/common.override"

 # End ../android-api-16-frontend/nightly.
+
+# Disable Keyfile Loading (and checks) since dependency fetching doesn't need these keys.
+# This overrides the settings in the common android mozconfig
+ac_add_options --without-mozilla-api-keyfile
+ac_add_options --without-google-api-keyfile
+# We need dummy Keyfiles in order to enable features we care about.
+ac_add_options --with-adjust-sdk-keyfile="$topsrcdir/mobile/android/base/adjust-sdk-sandbox.token"
+ac_add_options --with-leanplum-sdk-keyfile="$topsrcdir/mobile/android/base/leanplum-sdk-sandbox.token"
+ac_add_options --with-pocket-api-keyfile="$topsrcdir/mobile/android/base/pocket-api-sandbox.token"
--- a/mobile/android/config/mozconfigs/android-api-16-gradle/nightly
+++ b/mobile/android/config/mozconfigs/android-api-16-gradle/nightly
@ -2,7 +2,7 @@

 . "$topsrcdir/mobile/android/config/mozconfigs/android-api-16/nightly"

-ac_add_options --with-gradle="$topsrcdir/gradle-dist/bin/gradle"
-export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/jcenter","file://$topsrcdir/google"
+ac_add_options --with-gradle="$topsrcdir/android-gradle-dependencies/gradle-dist/bin/gradle"
+export GRADLE_MAVEN_REPOSITORIES="file://$topsrcdir/android-gradle-dependencies/jcenter","file://$topsrcdir/android-gradle-dependencies/google"

 . "$topsrcdir/mobile/android/config/mozconfigs/common.override"
--- a/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
@ -1,34 +1,2 @@
 [
-  {
-    "size": 37130176,
-    "visibility": "internal",
-    "digest": "254eecc06eea407c098417c1dbedf89f59883cdde7e173b0473fa5b34719c967830afa11189c3a9f69dc3f3b9e5fb43469434340e1f8feb378609af965783b48",
-    "algorithm": "sha512",
-    "filename": "java_home.tar.xz",
-    "unpack": true
-  },
-  {
-    "size": 7797928,
-    "visibility": "public",
-    "digest": "7bb5c7b3586179a7805ae60b025b7856cb8dbed66da1b7962bc446d94ea6fa45ed55ebfe065d2d82690c68bf71bdab613e97f53da79b85e3185955461f751256",
-    "algorithm": "sha512",
-    "filename": "google.tar.xz",
-    "unpack": true
-  },
-  {
-    "size": 52693504,
-    "visibility": "public",
-    "digest": "d8c0d91ff4039dd8a22e34634c0f72b21b1c272d60987f20a709fbfd919ba62eab4c3bddbaf96af1c6caf85320b4bf0676589071fe3b31724c51782a55da83e0",
-    "algorithm": "sha512",
-    "filename": "jcenter.tar.xz",
-    "unpack": true
-  },
-  {
-    "algorithm": "sha512",
-    "visibility": "public",
-    "filename": "gradle-dist.tar.xz",
-    "unpack": true,
-    "digest": "e3cfe7f8259ad97722243d4e873d5a05c014bfc24d637427f89d804bf5073290229c778ea303142cf06c2dc79e0492f23521f57d3a73825f55b8db587317646f",
-    "size": 51753660
-  }
 ]
--- a/mobile/android/config/tooltool-manifests/android-x86/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-x86/releng.manifest
@ -15,37 +15,5 @@
    "digest": "d68dd7d31b0153095ecf5cde5837fb1f95dc6e3f799d496fb764f7afeb9c6095c332467177c3aa54d3749b1901e0d6fa84c42162526e764e8a9d2196a0189861",
    "algorithm": "sha512",
    "filename": "jsshell.tar.xz"
-  },
-  {
-    "size": 7797928,
-    "visibility": "public",
-    "digest": "7bb5c7b3586179a7805ae60b025b7856cb8dbed66da1b7962bc446d94ea6fa45ed55ebfe065d2d82690c68bf71bdab613e97f53da79b85e3185955461f751256",
-    "algorithm": "sha512",
-    "filename": "google.tar.xz",
-    "unpack": true
-  },
-  {
-    "size": 52693504,
-    "visibility": "public",
-    "digest": "d8c0d91ff4039dd8a22e34634c0f72b21b1c272d60987f20a709fbfd919ba62eab4c3bddbaf96af1c6caf85320b4bf0676589071fe3b31724c51782a55da83e0",
-    "algorithm": "sha512",
-    "filename": "jcenter.tar.xz",
-    "unpack": true
-  },
-  {
-    "algorithm": "sha512",
-    "visibility": "public",
-    "filename": "gradle-dist.tar.xz",
-    "unpack": true,
-    "digest": "e3cfe7f8259ad97722243d4e873d5a05c014bfc24d637427f89d804bf5073290229c778ea303142cf06c2dc79e0492f23521f57d3a73825f55b8db587317646f",
-    "size": 51753660
-  },
-  {
-    "size": 37130176,
-    "visibility": "internal",
-    "digest": "254eecc06eea407c098417c1dbedf89f59883cdde7e173b0473fa5b34719c967830afa11189c3a9f69dc3f3b9e5fb43469434340e1f8feb378609af965783b48",
-    "algorithm": "sha512",
-    "filename": "java_home.tar.xz",
-    "unpack": true
  }
 ]
--- a/mobile/android/config/tooltool-manifests/android/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android/releng.manifest
@ -15,37 +15,5 @@
    "digest": "d68dd7d31b0153095ecf5cde5837fb1f95dc6e3f799d496fb764f7afeb9c6095c332467177c3aa54d3749b1901e0d6fa84c42162526e764e8a9d2196a0189861",
    "algorithm": "sha512",
    "filename": "jsshell.tar.xz"
-  },
-  {
-    "size": 37130176,
-    "visibility": "internal",
-    "digest": "254eecc06eea407c098417c1dbedf89f59883cdde7e173b0473fa5b34719c967830afa11189c3a9f69dc3f3b9e5fb43469434340e1f8feb378609af965783b48",
-    "algorithm": "sha512",
-    "filename": "java_home.tar.xz",
-    "unpack": true
-  },
-  {
-    "size": 7797928,
-    "visibility": "public",
-    "digest": "7bb5c7b3586179a7805ae60b025b7856cb8dbed66da1b7962bc446d94ea6fa45ed55ebfe065d2d82690c68bf71bdab613e97f53da79b85e3185955461f751256",
-    "algorithm": "sha512",
-    "filename": "google.tar.xz",
-    "unpack": true
-  },
-  {
-    "size": 52693504,
-    "visibility": "public",
-    "digest": "d8c0d91ff4039dd8a22e34634c0f72b21b1c272d60987f20a709fbfd919ba62eab4c3bddbaf96af1c6caf85320b4bf0676589071fe3b31724c51782a55da83e0",
-    "algorithm": "sha512",
-    "filename": "jcenter.tar.xz",
-    "unpack": true
-  },
-  {
-    "algorithm": "sha512",
-    "visibility": "public",
-    "filename": "gradle-dist.tar.xz",
-    "unpack": true,
-    "digest": "e3cfe7f8259ad97722243d4e873d5a05c014bfc24d637427f89d804bf5073290229c778ea303142cf06c2dc79e0492f23521f57d3a73825f55b8db587317646f",
-    "size": 51753660
  }
 ]
--- a/taskcluster/ci/build/android-stuff.yml
+++ b/taskcluster/ci/build/android-stuff.yml
@ -1,48 +1,3 @@
-android-dependencies/opt:
-    description: "Android armv7 api-16+ gradle dependencies"
-    index:
-        product: mobile
-        job-name: android-dependencies
-    treeherder:
-        platform: android-4-0-armv7-api16/opt
-        kind: build
-        tier: 2
-        symbol: tc(Deps)
-    worker-type: aws-provisioner-v1/gecko-{level}-b-android
-    worker:
-        docker-image: {in-tree: android-build}
-        env:
-            GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online"
-            PERFHERDER_EXTRA_OPTIONS: android-dependencies
-            TOOLTOOL_MANIFEST: "mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest"
-            WORKSPACE: "/builds/worker/workspace"
-        artifacts:
-          - name: public/build
-            path: /builds/worker/artifacts/
-            type: directory
-        max-run-time: 36000
-    run:
-        using: mozharness
-        actions: [get-secrets build multi-l10n update]
-        config:
-            - builds/releng_base_android_64_builds.py
-            - disable_signing.py
-            - platform_supports_post_upload_to_latest.py
-        script: "mozharness/scripts/fx_desktop_build.py"
-        secrets: true
-        custom-build-variant-cfg: api-16-gradle-dependencies
-        tooltool-downloads: internal
-        job-script: taskcluster/scripts/builder/build-android-dependencies.sh
-    toolchains:
-        - android-sdk-linux
-        - proguard-jar
-    optimization:
-        skip-unless-changed:
-          - "mobile/android/config/**"
-          - "testing/mozharness/configs/builds/releng_sub_android_configs/*gradle_dependencies.py"
-          - "**/*.gradle"
-          - "taskcluster/docker/android-build/**"
-
 android-test/opt:
    description: "Android armv7 unit tests"
    index:
@ -80,6 +35,7 @@ android-test/opt:
        custom-build-variant-cfg: android-test
        tooltool-downloads: internal
    toolchains:
+        - android-gradle-dependencies
        - android-sdk-linux
        - proguard-jar
    optimization:
@ -141,6 +97,7 @@ android-lint/opt:
        custom-build-variant-cfg: android-lint
        tooltool-downloads: internal
    toolchains:
+        - android-gradle-dependencies
        - android-sdk-linux
        - proguard-jar
    optimization:
@ -196,6 +153,7 @@ android-checkstyle/opt:
        custom-build-variant-cfg: android-checkstyle
        tooltool-downloads: internal
    toolchains:
+        - android-gradle-dependencies
        - android-sdk-linux
        - proguard-jar
    optimization:
@ -253,6 +211,7 @@ android-findbugs/opt:
        custom-build-variant-cfg: android-findbugs
        tooltool-downloads: internal
    toolchains:
+        - android-gradle-dependencies
        - android-sdk-linux
        - proguard-jar
    optimization:
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@ -335,6 +335,7 @@ android-api-16-gradle/opt:
        custom-build-variant-cfg: api-16-gradle
        tooltool-downloads: internal
    toolchains:
+        - android-gradle-dependencies
        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@ -269,6 +269,38 @@ linux64-android-sdk-linux-repack:
        toolchain-artifact: project/gecko/android-sdk/android-sdk-linux.tar.xz
        toolchain-alias: android-sdk-linux

+linux64-android-gradle-dependencies:
+    description: "Android Gradle dependencies toolchain task"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(gradle-dependencies)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: android-build}
+        env:
+            GRADLE_USER_HOME: "/builds/worker/workspace/build/src/mobile/android/gradle/dotgradle-online"
+        max-run-time: 36000
+    run:
+        using: toolchain-script
+        script: android-gradle-dependencies.sh
+        sparse-profile: null
+
+        tc-vcs: false
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+            - 'taskcluster/scripts/misc/android-gradle-dependencies/**'
+            - '**/*.gradle'
+            - 'mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/**'
+            - 'mobile/android/config/mozconfigs/common*'
+        toolchain-artifact: public/build/android-gradle-dependencies.tar.xz
+        toolchain-alias: android-gradle-dependencies
+    toolchains:
+        # Aliases aren't allowed for toolchains depending on toolchains.
+        - linux64-android-sdk-linux-repack
+        - linux64-proguard-jar-repack
+
 linux64-rust-1.19:
    description: "rust repack"
    treeherder:
--- a/taskcluster/scripts/builder/build-android-dependencies.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies.sh
@ -1,15 +0,0 @@
-#!/bin/bash -vex
-
-set -x -e
-
-echo "running as" $(id)
-
-: WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
-
-set -v
-
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/before.sh
-
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-linux.sh
-
-. $WORKSPACE/build/src/taskcluster/scripts/builder/build-android-dependencies/after.sh
--- a/taskcluster/scripts/misc/android-gradle-dependencies.sh
+++ b/taskcluster/scripts/misc/android-gradle-dependencies.sh
@ -0,0 +1,22 @@
+#!/bin/bash -vex
+
+set -x -e
+
+echo "running as" $(id)
+
+: WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
+
+set -v
+
+cd $WORKSPACE/build/src
+
+# Download toolchain artifacts.
+. taskcluster/scripts/misc/tooltool-download.sh
+
+. taskcluster/scripts/misc/android-gradle-dependencies/before.sh
+
+export MOZCONFIG=mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
+./mach build
+./mach android gradle-dependencies
+
+. taskcluster/scripts/misc/android-gradle-dependencies/after.sh
--- a/taskcluster/scripts/builder/build-android-dependencies/after.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies/after.sh
@ -11,25 +11,20 @@ set -v

 # Package everything up.
 pushd $WORKSPACE
+mkdir -p android-gradle-dependencies /builds/worker/artifacts

-cp -R ${NEXUS_WORK}/storage/jcenter jcenter
-tar cJf jcenter.tar.xz jcenter
-
-cp -R ${NEXUS_WORK}/storage/google google
-tar cJf google.tar.xz google
+cp -R ${NEXUS_WORK}/storage/jcenter android-gradle-dependencies
+cp -R ${NEXUS_WORK}/storage/google android-gradle-dependencies

 # The Gradle wrapper will have downloaded and verified the hash of exactly one
 # Gradle distribution.  It will be located in $GRADLE_USER_HOME, like
 # ~/.gradle/wrapper/dists/gradle-2.7-all/$PROJECT_HASH/gradle-2.7-all.zip.  We
 # want to remove the version from the internal directory for use via tooltool in
 # a mozconfig.
-cp $GRADLE_USER_HOME/wrapper/dists/gradle-${GRADLE_VERSION}-all/*/gradle-${GRADLE_VERSION}-all.zip gradle-${GRADLE_VERSION}-all.zip
+cp ${GRADLE_USER_HOME}/wrapper/dists/gradle-${GRADLE_VERSION}-all/*/gradle-${GRADLE_VERSION}-all.zip gradle-${GRADLE_VERSION}-all.zip
 unzip -q gradle-${GRADLE_VERSION}-all.zip
-mv gradle-${GRADLE_VERSION} gradle-dist
-tar cJf gradle-dist.tar.xz gradle-dist
+mv gradle-${GRADLE_VERSION} android-gradle-dependencies/gradle-dist
+
+tar cf - android-gradle-dependencies | xz > /builds/worker/artifacts/android-gradle-dependencies.tar.xz

-mkdir -p /builds/worker/artifacts
-mv jcenter.tar.xz /builds/worker/artifacts
-mv google.tar.xz /builds/worker/artifacts
-mv gradle-dist.tar.xz /builds/worker/artifacts
 popd
--- a/taskcluster/scripts/builder/build-android-dependencies/before.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies/before.sh
@ -9,7 +9,7 @@ echo "running as" $(id)
 set -v

 mkdir -p ${NEXUS_WORK}/conf
-cp /builds/worker/workspace/build/src/taskcluster/scripts/builder/build-android-dependencies/nexus.xml ${NEXUS_WORK}/conf/nexus.xml
+cp /builds/worker/workspace/build/src/taskcluster/scripts/misc/android-gradle-dependencies/nexus.xml ${NEXUS_WORK}/conf/nexus.xml

 RUN_AS_USER=worker /opt/sonatype/nexus/bin/nexus restart

--- a/taskcluster/scripts/builder/build-android-dependencies/nexus.xml
+++ b/taskcluster/scripts/builder/build-android-dependencies/nexus.xml