CI improvements (#708)

- updated az-cli to 2.0.7 (resolved parallelization issue github.com/Azure/azure-cli/issues/3255)
- added workaround for occasional hanging in 'docker network create --driver overlay'
- added retries and logs to cluster-tests/dcos/test.sh

Dockerfile

@@ -2,7 +2,7 @@ FROM buildpack-deps:xenial
 
 ENV GO_VERSION 1.8
 ENV KUBECTL_VERSION 1.6.0
-ENV AZURE_CLI_VERSION 2.0.3
+ENV AZURE_CLI_VERSION 2.0.7
 
 RUN apt-get update \
     && apt-get -y upgrade \

test/acs-engine-test/main.go

@@ -202,7 +202,7 @@ func isValidEnv() bool {
 }
 
 func (m *TestManager) runStep(name, step string, env []string, timeout time.Duration) (string, error) {
-	// work around az-cli parallelization issue https://github.com/Azure/azure-cli/issues/3255
+	// prevent ARM throttling
 	m.lock.Lock()
 	go func() {
 		time.Sleep(2 * time.Second)

test/cluster-tests/dcos/test.sh

@@ -10,7 +10,7 @@ done
 DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 ####################################################
 
-set -e
+# do not use 'set -e'
 # -o pipefail
 set -x
 
@@ -24,20 +24,38 @@ function teardown {
   ${remote_exec} ./dcos marathon app remove /web
 }
 
-${remote_exec} curl -O https://downloads.dcos.io/binaries/cli/linux/x86-64/dcos-1.8/dcos
-${remote_exec} chmod a+x ./dcos
-${remote_exec} ./dcos config set core.dcos_url http://localhost:80
+log "Downloading dcos"
+${remote_exec} curl -O https://downloads.dcos.io/binaries/cli/linux/x86-64/dcos-1.8/dcos || (log "Failed to download dcos"; exit 1)
+log "Setting dcos permissions"
+${remote_exec} chmod a+x ./dcos || (log "Failed to chmod dcos"; exit 1)
+log "Configuring dcos"
+${remote_exec} ./dcos config set core.dcos_url http://localhost:80 || (log "Failed to configure dcos"; exit 1)
 
-${remote_cp} "${DIR}/marathon.json" azureuser@${INSTANCE_NAME}.${LOCATION}.cloudapp.azure.com:marathon.json
+log "Copying marathon.json"
+${remote_cp} "${DIR}/marathon.json" azureuser@${INSTANCE_NAME}.${LOCATION}.cloudapp.azure.com:marathon.json || (log "Failed to copy marathon.json"; exit 1)
 
 # feed agentFQDN to marathon.json
-${remote_exec} sed -i "s/{agentFQDN}/${agentFQDN}/g" marathon.json
-${remote_exec} ./dcos marathon app add marathon.json
+log "Configuring marathon.json"
+${remote_exec} sed -i "s/{agentFQDN}/${agentFQDN}/g" marathon.json || (log "Failed to configure marathon.json"; exit 1)
+
+log "Adding marathon app"
+count=5
+while (( $count > 0 )); do
+  log "  ... counting down $count"
+  ${remote_exec} ./dcos marathon app add marathon.json
+  retval=$?
+  if [[ "$retval" == "0" ]]; then break; fi
+  sleep 5; count=$((count-1))
+done
+if [[ "$retval" != "0" ]]; then
+  log "gave up waiting for marathon to be added"
+  exit -1
+fi
 
 # only need to teardown if app added successfully
 trap teardown EXIT
 
-log "Validating app"
+log "Validating marathon app"
 count=0
 while [[ ${count} -lt 25 ]]; do
   count=$((count+1))
@@ -51,20 +69,20 @@ while [[ ${count} -lt 25 ]]; do
 done
 
 if [[ "${running}" != "3" ]]; then
-  log "Validation failed"
+  log "marathon validation failed"
   ${remote_exec} ./dcos marathon app show /web
   exit 1
 fi
 
 # install marathon-lb
-${remote_exec} ./dcos package install marathon-lb --yes
+${remote_exec} ./dcos package install marathon-lb --yes || (log "Failed to install marathon-lb"; exit 1)
 
 # curl simpleweb through external haproxy
 log "Checking Service"
 count=10
 while (( $count > 0 )); do
   log "  ... counting down $count"
-  [[ $(curl -sI --max-time 60 "http://${agentFQDN}" |head -n1 |cut -d$' ' -f2) -eq "200" ]] && echo "Successfully hitting simpleweb through external haproxy http://${agentFQDN}" && break
+  [[ $(curl -sI --max-time 60 "http://${agentFQDN}" |head -n1 |cut -d$' ' -f2) -eq "200" ]] && log "Successfully hitting simpleweb through external haproxy http://${agentFQDN}" && break
   if [[ "${count}" -le 0 ]]; then
     log "failed to get expected response from nginx through the loadbalancer"
     exit 1