Bug 1405412 - Migrate Android SDK to android-sdk-linux toolchain task. r=dustin

The only tricky piece here is that the resulting toolchain archive is private, and uses a newly allocated Task Cluster scope (queue:get-artifact:project/gecko/android-sdk/*) to restrict access to the archive. All SCM levels (1, 2, 3) have been given the new scope: see https://tools.taskcluster.net/auth/roles/moz-tree:level:1 and friends. MozReview-Commit-ID: CcDqDOHODpe --HG-- extra : rebase_source : 81dbb065f2a3c4e7733e964be66adb1733db52c6
2017-10-03 11:45:27 -07:00 · 2017-10-03 11:45:27 -07:00 · daf88e8cc6
--- a/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
+++ b/mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
@ -53,7 +53,3 @@ export MOZ_ANDROID_POCKET=1
 . "$topsrcdir/mobile/android/config/mozconfigs/common.override"

 # End ../android-api-16-frontend/nightly.
-
-# Populated after checking out the sources and before building the
-# tree as part of the dependencies task bin/ scripts.
-ac_add_options --with-android-sdk="/builds/worker/.mozbuild/android-sdk-linux"
--- a/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
@ -1,12 +1,4 @@
 [
-  {
-    "size": 735632692,
-    "visibility": "internal",
-    "digest": "dc4329803b4f941d52d61a32c054e209890c2dd793ac2cbbeadb15e4bd23104f7ba99c113472326c7751dbe99e00238208432a05183d9a01bed13f38297b3b3c",
-    "algorithm": "sha512",
-    "filename": "android-sdk-linux.tar.xz",
-    "unpack": true
-  },
  {
    "size": 37130176,
    "visibility": "internal",
--- a/mobile/android/config/tooltool-manifests/android-x86/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-x86/releng.manifest
@ -8,14 +8,6 @@
    "filename": "android-ndk.tar.bz2",
    "unpack": true
  },
-  {
-    "size": 735632692,
-    "visibility": "internal",
-    "digest": "dc4329803b4f941d52d61a32c054e209890c2dd793ac2cbbeadb15e4bd23104f7ba99c113472326c7751dbe99e00238208432a05183d9a01bed13f38297b3b3c",
-    "algorithm": "sha512",
-    "filename": "android-sdk-linux.tar.xz",
-    "unpack": true
-  },
  {
    "size": 6856444,
    "visibility": "public",
--- a/mobile/android/config/tooltool-manifests/android/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android/releng.manifest
@ -8,14 +8,6 @@
    "filename": "android-ndk.tar.bz2",
    "unpack": true
  },
-  {
-    "size": 735632692,
-    "visibility": "internal",
-    "digest": "dc4329803b4f941d52d61a32c054e209890c2dd793ac2cbbeadb15e4bd23104f7ba99c113472326c7751dbe99e00238208432a05183d9a01bed13f38297b3b3c",
-    "algorithm": "sha512",
-    "filename": "android-sdk-linux.tar.xz",
-    "unpack": true
-  },
  {
    "size": 6856444,
    "visibility": "public",
--- a/taskcluster/ci/build/android-stuff.yml
+++ b/taskcluster/ci/build/android-stuff.yml
@ -20,12 +20,6 @@ android-dependencies/opt:
          - name: public/build
            path: /builds/worker/artifacts/
            type: directory
-          - name: private/android-sdk
-            path: /builds/worker/private/android-sdk
-            type: directory
-          - name: private/java_home
-            path: /builds/worker/private/java_home
-            type: directory
        max-run-time: 36000
    run:
        using: mozharness
@ -40,6 +34,7 @@ android-dependencies/opt:
        tooltool-downloads: internal
        job-script: taskcluster/scripts/builder/build-android-dependencies.sh
    toolchains:
+        - android-sdk-linux
        - proguard-jar
    optimization:
        skip-unless-changed:
@ -85,6 +80,7 @@ android-test/opt:
        custom-build-variant-cfg: android-test
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - proguard-jar
    optimization:
        skip-unless-changed:
@ -145,6 +141,7 @@ android-lint/opt:
        custom-build-variant-cfg: android-lint
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - proguard-jar
    optimization:
        skip-unless-changed:
@ -199,6 +196,7 @@ android-checkstyle/opt:
        custom-build-variant-cfg: android-checkstyle
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - proguard-jar
    optimization:
        skip-unless-changed:
@ -255,6 +253,7 @@ android-findbugs/opt:
        custom-build-variant-cfg: android-findbugs
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - proguard-jar
    optimization:
        skip-unless-changed:
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@ -24,6 +24,7 @@ android-api-16/debug:
        custom-build-variant-cfg: api-16-debug
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -54,6 +55,7 @@ android-x86/opt:
        custom-build-variant-cfg: x86
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -88,6 +90,7 @@ android-x86-nightly/opt:
        custom-build-variant-cfg: x86
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -118,6 +121,7 @@ android-api-16/opt:
        custom-build-variant-cfg: api-16
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -152,6 +156,7 @@ android-api-16-nightly/opt:
        custom-build-variant-cfg: api-16
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -183,6 +188,7 @@ android-x86-old-id/opt:
        tooltool-downloads: internal
    run-on-projects: [ 'mozilla-central' ]
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -218,6 +224,7 @@ android-x86-old-id-nightly/opt:
        tooltool-downloads: internal
    run-on-projects: [ 'mozilla-central' ]
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -249,6 +256,7 @@ android-api-16-old-id/opt:
        tooltool-downloads: internal
    run-on-projects: [ 'mozilla-central' ]
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -284,6 +292,7 @@ android-api-16-old-id-nightly/opt:
        tooltool-downloads: internal
    run-on-projects: [ 'mozilla-central' ]
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -326,6 +335,7 @@ android-api-16-gradle/opt:
        custom-build-variant-cfg: api-16-gradle
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -356,6 +366,7 @@ android-aarch64/opt:
        custom-build-variant-cfg: aarch64
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
@ -390,6 +401,7 @@ android-aarch64-nightly/opt:
        custom-build-variant-cfg: aarch64
        tooltool-downloads: internal
    toolchains:
+        - android-sdk-linux
        - linux64-rust-android
        - linux64-sccache
        - proguard-jar
--- a/taskcluster/ci/l10n/kind.yml
+++ b/taskcluster/ci/l10n/kind.yml
@ -50,6 +50,7 @@ job-template:
      by-build-platform:
         default: []
         android-api-16-l10n:
+            - android-sdk-linux
            - proguard-jar
   tooltool:
      by-build-platform:
--- a/taskcluster/ci/nightly-l10n/kind.yml
+++ b/taskcluster/ci/nightly-l10n/kind.yml
@ -53,6 +53,7 @@ job-template:
      by-build-platform:
         default: []
         android-api-16-nightly:
+            - android-sdk-linux
            - proguard-jar
   tooltool:
      by-build-platform:
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@ -224,6 +224,30 @@ linux64-proguard-jar-repack:
        toolchain-artifact: public/build/proguard-jar.tar.xz
        toolchain-alias: proguard-jar

+linux64-android-sdk-linux-repack:
+    description: "Android SDK (Linux) repack toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(android-sdk-linux)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: android-build}
+        max-run-time: 36000
+        artifacts:
+          - name: project/gecko/android-sdk
+            path: /builds/worker/project/gecko/android-sdk/
+            type: directory
+    run:
+        using: toolchain-script
+        script: repack-android-sdk-linux.sh
+        tc-vcs: false
+        resources:
+            - 'python/mozboot/**/*android*'
+        toolchain-artifact: project/gecko/android-sdk/android-sdk-linux.tar.xz
+        toolchain-alias: android-sdk-linux
+
 linux64-rust-1.19:
    description: "rust repack"
    treeherder:
--- a/taskcluster/scripts/builder/build-android-dependencies/after.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies/after.sh
@ -12,13 +12,6 @@ set -v
 # Package everything up.
 pushd $WORKSPACE

-cp -R /builds/worker/.mozbuild/android-sdk-linux android-sdk-linux
-tar cJf android-sdk-linux.tar.xz android-sdk-linux
-
-# We can't redistribute the Android SDK publicly.
-mkdir -p /builds/worker/private/android-sdk
-mv android-sdk-linux.tar.xz /builds/worker/private/android-sdk
-
 cp -R ${NEXUS_WORK}/storage/jcenter jcenter
 tar cJf jcenter.tar.xz jcenter

--- a/taskcluster/scripts/builder/build-android-dependencies/before.sh
+++ b/taskcluster/scripts/builder/build-android-dependencies/before.sh
@ -26,6 +26,3 @@ curl --fail --silent --location http://localhost:8081/nexus/service/local/status

 # It's helpful when debugging to see the repository configurations.
 curl http://localhost:8081/nexus/service/local/repositories || true
-
-# Populate /builds/worker/.mozbuild/android-sdk-linux.
-python2.7 /builds/worker/workspace/build/src/python/mozboot/mozboot/android.py --artifact-mode --no-interactive
--- a/taskcluster/scripts/misc/repack-android-sdk-linux.sh
+++ b/taskcluster/scripts/misc/repack-android-sdk-linux.sh
@ -0,0 +1,18 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for fetching and repacking the Android SDK (for
+# Linux), the tools required to produce Android packages.
+
+WORKSPACE=$HOME/workspace
+UPLOAD_DIR=$HOME/project/gecko/android-sdk
+
+mkdir -p $HOME/artifacts $UPLOAD_DIR
+
+# Populate /builds/worker/.mozbuild/android-sdk-linux.
+cd /builds/worker/workspace/build/src
+./mach python python/mozboot/mozboot/android.py --artifact-mode --no-interactive
+
+tar cf - -C /builds/worker/.mozbuild android-sdk-linux | xz > $UPLOAD_DIR/android-sdk-linux.tar.xz
+
+ls -al $UPLOAD_DIR