From c00716af37341edade9c4dbcb4b16dd05d646feb Mon Sep 17 00:00:00 2001
From: Rob Thijssen <rthijssen@mozilla.com>
Date: Thu, 2 May 2019 12:56:20 +0000
Subject: [PATCH] Bug 1543026 - enable sccache on gcp r=wcosta

this change comprises the in-tree changes required to make use of sccache in gcp.
specifically:
- a gcp metadata lookup for availability-zone is added to mozconfig, enabling a build to determine its regional gcp sccache bucket
- the sccache cargo build command is modified to include the gcs feature when the environment contains gcs configuration

note that further changes are required on infra to support sccache use. the required changes already [exist](https://github.com/mozilla-releng/OpenCloudConfig/commit/1d515dc) and are enabled for gcp windows infra, including:
- a json credential file on the build instance filesystem, containing credentials valid for the appropriate scm level bucket for the gcp region
- an `SCCACHE_GCS_KEY_PATH` env variable containing the path to the json credential file
- an `SCCACHE_GCS_RW_MODE` env variable containg the text `READ_WRITE`
- sccache buckets must exist for each region and scm levels 1 & 3
- credentials for scm level 1 buckets **must not** be valid for scm level 3 buckets

on gcp systems which do not contain credential files and the above mentioned env variables (eg gecko-[1-3]-b-linux), sccache should fail gracefully without breaking builds.

Differential Revision: https://phabricator.services.mozilla.com/D29622

--HG--
extra : moz-landing-system : lando
---
 build/mozconfig.cache                     | 25 ++++++++++++++++++++---
 taskcluster/scripts/misc/build-sccache.sh | 12 +++++++++--
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/build/mozconfig.cache b/build/mozconfig.cache
index 281dc6f7ea7f..1c14eee8b2f7 100644
--- a/build/mozconfig.cache
+++ b/build/mozconfig.cache
@@ -14,15 +14,30 @@ if test -z "$bucket" -a -z "$SCCACHE_DISABLE" -a -z "$MOZ_PGO"; then
             # stick an extra character on to make the already-convoluted logic
             # here simpler.
             availability_zone="${TASKCLUSTER_WORKER_GROUP}x"
+        elif [ -n "${SCCACHE_GCS_KEY_PATH}" ]; then
+            # gcp availability_zone is of the form <region>-<letter> where region is e.g. us-west2, and az is us-west2-a
+            gcp_zone=$(wget -T 1 -t 1 -q -O - http://169.254.169.254/computeMetadata/v1beta1/instance/zone || true)
+            availability_zone=${gcp_zone##*/}
         else
             # timeout after 1 second, and don't retry (failure indicates instance is not in ec2 or network issue)
-            # availability_zone is of the form <region><letter> where region is e.g. us-west-2, and az is us-west-2a
+            # ec2 availability_zone is of the form <region><letter> where region is e.g. us-west-2, and az is us-west-2a
             availability_zone=$(wget -T 1 -t 1 -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone || true)
         fi
         if test -z "$availability_zone" -o "$availability_zone" = "not-ec2"; then
             availability_zone=not-ec2
+        elif [ -n "${SCCACHE_GCS_KEY_PATH}" ]; then
+            # gcp region is az with last two letters trimmed
+            region=${availability_zone::${#availability_zone}-2}
+            case "${GECKO_HEAD_REPOSITORY}" in
+            *hg.mozilla.org/try*)
+                bucket=taskcluster-level-1-sccache-${region}
+                ;;
+            *hg.mozilla.org/integration/autoland*|*hg.mozilla.org/integration/mozilla-inbound*)
+                bucket=taskcluster-level-3-sccache-${region}
+                ;;
+            esac
         else
-            # region is az with last letter trimmed
+            # ec2 region is az with last letter trimmed
             region=${availability_zone%?}
             # set S3 bucket according to tree (level)
             case "${GECKO_HEAD_REPOSITORY}" in
@@ -38,7 +53,11 @@ if test -z "$bucket" -a -z "$SCCACHE_DISABLE" -a -z "$MOZ_PGO"; then
 fi
 
 if test -n "$bucket"; then
-    mk_add_options "export SCCACHE_BUCKET=$bucket"
+    if [ -n "${SCCACHE_GCS_KEY_PATH}" ]; then
+        mk_add_options "export SCCACHE_GCS_BUCKET=$bucket"
+    else
+        mk_add_options "export SCCACHE_BUCKET=$bucket"
+    fi
     export CCACHE="$topsrcdir/sccache2/sccache"
     export SCCACHE_VERBOSE_STATS=1
     mk_add_options MOZBUILD_MANAGE_SCCACHE_DAEMON=${topsrcdir}/sccache2/sccache
diff --git a/taskcluster/scripts/misc/build-sccache.sh b/taskcluster/scripts/misc/build-sccache.sh
index 08482c0c921f..cb4364745513 100755
--- a/taskcluster/scripts/misc/build-sccache.sh
+++ b/taskcluster/scripts/misc/build-sccache.sh
@@ -66,10 +66,18 @@ EOF
 
     # We don't need to set OPENSSL_STATIC here, because we only have static
     # libraries in the directory we are passing.
-    env "OPENSSL_DIR=$OPENSSL_BUILD_DIRECTORY" cargo build --features "all dist-server" --verbose --release
+    if [ -n "${SCCACHE_GCS_KEY_PATH}" ]; then
+        env "OPENSSL_DIR=$OPENSSL_BUILD_DIRECTORY" cargo build --features "all dist-server gcs" --verbose --release
+    else
+        env "OPENSSL_DIR=$OPENSSL_BUILD_DIRECTORY" cargo build --features "all dist-server" --verbose --release
+    fi
     ;;
 MINGW*)
-    cargo build --verbose --release
+    if [ -n "${SCCACHE_GCS_KEY_PATH}" ]; then
+        cargo build --verbose --release --features=gcs
+    else
+        cargo build --verbose --release
+    fi
     ;;
 esac