From 5d5f326c0571ad563b3fd0d7fb746ad5462581fc Mon Sep 17 00:00:00 2001
From: Wander Lairson Costa <wcosta@mozilla.com>
Date: Tue, 6 Dec 2016 15:27:40 -1000
Subject: [PATCH] Bug 1266624: Apply exponential backoff for mozharness
 download. r=dustin

MozReview-Commit-ID: Ip4MjVJFwT6

--HG--
extra : rebase_source : 62a0c16da8804df0dfcb8d194b6bbc0f1d74b8f8
---
 taskcluster/scripts/tester/test-macosx.sh | 38 +++++++++++++++++++----
 taskcluster/scripts/tester/test-ubuntu.sh | 38 ++++++++++++++++++-----
 2 files changed, 63 insertions(+), 13 deletions(-)

diff --git a/taskcluster/scripts/tester/test-macosx.sh b/taskcluster/scripts/tester/test-macosx.sh
index 81c2cb6e1715..a5c0086c1949 100644
--- a/taskcluster/scripts/tester/test-macosx.sh
+++ b/taskcluster/scripts/tester/test-macosx.sh
@@ -25,12 +25,38 @@ if [[ -z ${MOZHARNESS_URL} ]]; then fail "MOZHARNESS_URL is not set"; fi
 if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
 if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi
 
-# Unzip the mozharness ZIP file created by the build task
-if ! curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
-    fail "failed to download mozharness zip"
-fi
-rm -rf mozharness
-unzip -q mozharness.zip
+# Download mozharness with exponential backoff
+# curl already applies exponential backoff, but not for all
+# failed cases, apparently, as we keep getting failed downloads
+# with 404 code.
+download_mozharness() {
+    local max_attempts=10
+    local timeout=1
+    local attempt=0
+
+    echo "Downloading mozharness"
+
+    while [[ $attempt < $max_attempts ]]; do
+        if curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
+            rm -rf mozharness
+            if unzip -q mozharness.zip; then
+                break
+            else
+                echo "error unzipping mozharness.zip" >&2
+            fi
+        else
+            echo "failed to download mozharness zip" >&2
+        fi
+        echo "Download failed, retrying in $timeout seconds..." >&2
+        sleep $timeout
+        timeout=$((timeout*2))
+        attempt=$((attempt+1))
+    done
+
+    fail "Failed to download and unzip mozharness"
+}
+
+download_mozharness
 rm mozharness.zip
 
 # For telemetry purposes, the build process wants information about the
diff --git a/taskcluster/scripts/tester/test-ubuntu.sh b/taskcluster/scripts/tester/test-ubuntu.sh
index 3bbafef31dc2..0c2ccc702198 100644
--- a/taskcluster/scripts/tester/test-ubuntu.sh
+++ b/taskcluster/scripts/tester/test-ubuntu.sh
@@ -69,15 +69,39 @@ cleanup() {
 }
 trap cleanup EXIT INT
 
+# Download mozharness with exponential backoff
+# curl already applies exponential backoff, but not for all
+# failed cases, apparently, as we keep getting failed downloads
+# with 404 code.
+download_mozharness() {
+    local max_attempts=10
+    local timeout=1
+    local attempt=0
+
+    echo "Downloading mozharness"
+
+    while [[ $attempt < $max_attempts ]]; do
+        if curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
+            rm -rf mozharness
+            if unzip -q mozharness.zip; then
+                return 0
+            fi
+            echo "error unzipping mozharness.zip" >&2
+        else
+            echo "failed to download mozharness zip" >&2
+        fi
+        echo "Download failed, retrying in $timeout seconds..." >&2
+        sleep $timeout
+        timeout=$((timeout*2))
+        attempt=$((attempt+1))
+    done
+
+    fail "Failed to download and unzip mozharness"
+}
+
 # Download mozharness if we're told to.
 if [ ${MOZHARNESS_URL} ]; then
-    if ! curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
-        fail "failed to download mozharness zip"
-    fi
-    rm -rf mozharness
-    if !  unzip -q mozharness.zip; then
-        fail "error unzipping mozharness.zip"
-    fi
+    download_mozharness
     rm mozharness.zip
 
     if ! [ -d mozharness ]; then