diff --git a/Gopkg.lock b/Gopkg.lock index b9ae05da7..ed78b9344 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -54,14 +54,6 @@ revision = "7be49c3f43696c7ca8c9a76d6836703d83aea404" version = "v1.1.1" -[[projects]] - digest = "1:d6613cccd218e927314d7f6a7a6e8169473b07db7e07810341cacd3bdca920ba" - name = "github.com/alexcesaro/statsd" - packages = ["."] - pruneopts = "NUT" - revision = "7fea3f0d2fab1ad973e641e51dba45443a311a90" - version = "v2.0.0" - [[projects]] digest = "1:aba270497eb2d49f5cba6f4162d524b9a1195a24cbce8be20bf56a0051f47deb" name = "github.com/blang/semver" @@ -861,7 +853,6 @@ "github.com/Azure/go-autorest/autorest/date", "github.com/Azure/go-autorest/autorest/to", "github.com/Jeffail/gabs", - "github.com/alexcesaro/statsd", "github.com/blang/semver", "github.com/davecgh/go-spew/spew", "github.com/fatih/structs", diff --git a/Makefile b/Makefile index a6c4a3b71..f120def6a 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,6 @@ generate-azure-constants: .PHONY: build build: validate-dependencies generate $(GO) build $(GOFLAGS) -ldflags '$(LDFLAGS)' -o $(BINDIR)/$(PROJECT)$(EXTENSION) $(REPO_PATH) - $(GO) build $(GOFLAGS) -o $(BINDIR)/aks-engine-test$(EXTENSION) $(REPO_PATH)/test/aks-engine-test build-binary: generate go build $(GOFLAGS) -v -ldflags "$(LDFLAGS)" -o $(BINARY_DEST_DIR)/aks-engine . @@ -138,7 +137,7 @@ ginkgoBuild: generate ginkgo build test/e2e/kubernetes test: generate ginkgoBuild - ginkgo -skipPackage test/e2e/dcos,test/e2e/kubernetes -failFast -r . + ginkgo -skipPackage test/e2e/kubernetes -failFast -r . .PHONY: test-style test-style: validate-go validate-shell validate-copyright-headers diff --git a/test/acse-conf/acse-feature-validation.json b/test/acse-conf/acse-feature-validation.json deleted file mode 100644 index bae96266a..000000000 --- a/test/acse-conf/acse-feature-validation.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "windows/kubernetes-hybrid.json", - "location": "eastus" - }, - { - "cluster_definition": "windows/kubernetes.json", - "location": "eastus" - }, - { - "cluster_definition": "dcos-releases/dcos1.9.json", - "location": "eastus" - }, - { - "cluster_definition": "disks-managed/dcos-preAttachedDisks-vmas.json", - "location": "eastus" - }, - { - "cluster_definition": "disks-managed/dcos-vmss.json", - "location": "eastus" - }, - { - "cluster_definition": "disks-managed/kubernetes-preAttachedDisks-vmas.json", - "location": "eastus" - }, - { - "cluster_definition": "disks-managed/kubernetes-vmas.json", - "location": "eastus" - }, - { - "cluster_definition": "disks-managed/swarm-preAttachedDisks-vmss.json", - "location": "southcentralus" - }, - { - "cluster_definition": "disks-managed/swarmmode-vmas.json", - "location": "southcentralus" - }, - { - "cluster_definition": "disks-managed/swarmmode-vmss.json", - "location": "southcentralus" - }, - { - "cluster_definition": "disks-storageaccount/dcos.json", - "location": "westcentralus" - }, - { - "cluster_definition": "disks-storageaccount/kubernetes.json", - "location": "westcentralus" - }, - { - "cluster_definition": "disks-storageaccount/swarmmode.json", - "location": "southcentralus" - }, - { - "cluster_definition": "networkpolicy/kubernetes-calico.json", - "location": "southcentralus" - }, - { - "cluster_definition": "kubernetes-config/kubernetes-clustersubnet.json", - "location": "westus2" - }, - { - "cluster_definition": "v20170131/swarmmode.json", - "location": "westus2" - }, - { - "cluster_definition": "vnet/dcosvnet.json", - "location": "westus2" - }, - { - "cluster_definition": "vnet/kubernetesvnet.json", - "location": "westus2" - }, - { - "cluster_definition": "vnet/swarmmodevnet.json", - "location": "westus2" - } - ] -} diff --git a/test/acse-conf/acse-pr-non-k8s.json b/test/acse-conf/acse-pr-non-k8s.json deleted file mode 100644 index 1c5e5b59b..000000000 --- a/test/acse-conf/acse-pr-non-k8s.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "swarm.json", - "location": "westus2" - }, - { - "cluster_definition": "swarmmode.json", - "location": "westcentralus" - } - ] -} diff --git a/test/acse-conf/acse-regression-dcos.json b/test/acse-conf/acse-regression-dcos.json deleted file mode 100644 index 13b6ea121..000000000 --- a/test/acse-conf/acse-regression-dcos.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "dcos-releases/dcos1.9.json", - "category": "version" - }, - { - "cluster_definition": "disks-managed/dcos-preAttachedDisks-vmas.json", - "category": "managed-disk" - }, - { - "cluster_definition": "disks-managed/dcos-vmss.json", - "category": "managed-disk" - }, - { - "cluster_definition": "disks-storageaccount/dcos.json", - "category": "managed-disk" - }, - { - "cluster_definition": "vnet/dcosvnet.json", - "category": "network" - } - ] - } \ No newline at end of file diff --git a/test/acse-conf/acse-regression.json b/test/acse-conf/acse-regression.json deleted file mode 100644 index 2945d0d7a..000000000 --- a/test/acse-conf/acse-regression.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "dcos-releases/dcos1.9.json", - "category": "version" - }, - { - "cluster_definition": "disks-managed/dcos-preAttachedDisks-vmas.json", - "category": "managed-disk" - }, - { - "cluster_definition": "disks-managed/dcos-vmss.json", - "category": "managed-disk" - }, - { - "cluster_definition": "disks-managed/swarm-preAttachedDisks-vmss.json", - "category": "managed-disk" - }, - { - "cluster_definition": "disks-managed/swarmmode-vmas.json", - "category": "managed-disk" - }, - { - "cluster_definition": "disks-managed/swarmmode-vmss.json", - "category": "managed-disk" - }, - { - "cluster_definition": "disks-storageaccount/dcos.json", - "category": "managed-disk" - }, - { - "cluster_definition": "disks-storageaccount/swarmmode.json", - "category": "sa-disk" - }, - { - "cluster_definition": "keyvaultcerts/swarmmode.json", - "category": "keyvault", - "location": "westus" - }, - { - "cluster_definition": "networkpolicy/kubernetes-calico.json", - "category": "network" - }, - { - "cluster_definition": "v20170131/swarmmode.json", - "category": "version" - }, - { - "cluster_definition": "vnet/dcosvnet.json", - "category": "network" - }, - { - "cluster_definition": "vnet/kubernetesvnet.json", - "category": "network" - }, - { - "cluster_definition": "vnet/swarmmodevnet.json", - "category": "network" - } - ] -} \ No newline at end of file diff --git a/test/acse-conf/acse-scale-vnet.json b/test/acse-conf/acse-scale-vnet.json deleted file mode 100644 index 10cd50755..000000000 --- a/test/acse-conf/acse-scale-vnet.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "azure-cni/k8s-vnet-scaleup.json" - }, - { - "cluster_definition": "azure-cni/k8s-vnet-scaledown.json" - } - ] -} - diff --git a/test/acse-conf/acse-scale.json b/test/acse-conf/acse-scale.json deleted file mode 100644 index 1be659474..000000000 --- a/test/acse-conf/acse-scale.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "azure-cni/k8s-scaleup.json" - }, - { - "cluster_definition": "azure-cni/k8s-scaledown.json" - } - ] -} - diff --git a/test/acse-conf/acse-svc-canary-dcos.json b/test/acse-conf/acse-svc-canary-dcos.json deleted file mode 100644 index c6a8deb5a..000000000 --- a/test/acse-conf/acse-svc-canary-dcos.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "dcos.json", - "category": "canary", - "location": "eastus2euap" - } - ] -} diff --git a/test/acse-conf/acse-svc-canary-k8s-win.json b/test/acse-conf/acse-svc-canary-k8s-win.json deleted file mode 100644 index 02c77d5a6..000000000 --- a/test/acse-conf/acse-svc-canary-k8s-win.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "windows/kubernetes.json", - "category": "canary", - "location": "eastus2euap" - } - ] -} diff --git a/test/acse-conf/acse-svc-canary-k8s.json b/test/acse-conf/acse-svc-canary-k8s.json deleted file mode 100644 index e341fa001..000000000 --- a/test/acse-conf/acse-svc-canary-k8s.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "kubernetes.json", - "category": "canary", - "location": "eastus2euap" - } - ] -} diff --git a/test/acse-conf/acse-svc-canary-swarmmode.json b/test/acse-conf/acse-svc-canary-swarmmode.json deleted file mode 100644 index 938304240..000000000 --- a/test/acse-conf/acse-svc-canary-swarmmode.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "swarmmode.json", - "category": "canary", - "location": "eastus2euap" - } - ] -} diff --git a/test/acse-conf/acse-svc-dcos.json b/test/acse-conf/acse-svc-dcos.json deleted file mode 100644 index 6b3cdd827..000000000 --- a/test/acse-conf/acse-svc-dcos.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "canadacentral" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "canadaeast" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "centralindia" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "westindia" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "southindia" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "centralus" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "eastus2" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "eastus" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "westcentralus" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "westus2" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "westus" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "southeastasia" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "koreacentral" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "japaneast" - }, - { - "cluster_definition": "dcos-D2.json", - "category": "service-availability", - "location": "japanwest" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "northeurope" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "westeurope" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "uksouth" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "australiaeast" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "australiasoutheast" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "koreasouth" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "northcentralus" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "southcentralus" - }, - { - "cluster_definition": "dcos.json", - "category": "service-availability", - "location": "ukwest" - } - ] -} diff --git a/test/acse-conf/acse-svc-k8s-win.json b/test/acse-conf/acse-svc-k8s-win.json deleted file mode 100644 index a9cff67ef..000000000 --- a/test/acse-conf/acse-svc-k8s-win.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "canadacentral" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "canadaeast" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "centralindia" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "westindia" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "southindia" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "centralus" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "eastus2" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "eastus" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "westcentralus" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "westus2" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "westus" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "southeastasia" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "koreacentral" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "japaneast" - }, - { - "cluster_definition": "windows/kubernetes-D2.json", - "category": "service-availability", - "location": "japanwest" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "northeurope" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "westeurope" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "uksouth" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "australiaeast" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "australiasoutheast" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "koreasouth" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "northcentralus" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "southcentralus" - }, - { - "cluster_definition": "windows/kubernetes.json", - "category": "service-availability", - "location": "ukwest" - } - ] -} diff --git a/test/acse-conf/acse-svc-k8s.json b/test/acse-conf/acse-svc-k8s.json deleted file mode 100644 index ed05e681c..000000000 --- a/test/acse-conf/acse-svc-k8s.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "canadacentral" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "canadaeast" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "centralindia" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "westindia" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "southindia" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "centralus" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "eastus2" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "eastus" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "westcentralus" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "westus2" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "westus" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "southeastasia" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "koreacentral" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "japaneast" - }, - { - "cluster_definition": "kubernetes-D2.json", - "category": "service-availability", - "location": "japanwest" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "northeurope" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "westeurope" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "uksouth" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "australiaeast" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "australiasoutheast" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "koreasouth" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "northcentralus" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "southcentralus" - }, - { - "cluster_definition": "kubernetes.json", - "category": "service-availability", - "location": "ukwest" - } - ] -} diff --git a/test/acse-conf/acse-svc-swarmmode.json b/test/acse-conf/acse-svc-swarmmode.json deleted file mode 100644 index 9c2685b5c..000000000 --- a/test/acse-conf/acse-svc-swarmmode.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "canadacentral" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "canadaeast" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "centralindia" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "westindia" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "southindia" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "centralus" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "eastus2" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "eastus" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "westcentralus" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "westus2" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "westus" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "southeastasia" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "koreacentral" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "japaneast" - }, - { - "cluster_definition": "swarmmode-D2.json", - "category": "service-availability", - "location": "japanwest" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "northeurope" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "westeurope" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "uksouth" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "australiaeast" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "australiasoutheast" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "koreasouth" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "northcentralus" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "southcentralus" - }, - { - "cluster_definition": "swarmmode.json", - "category": "service-availability", - "location": "ukwest" - } - ] -} diff --git a/test/acse-conf/acse-upgrade-lnx.json b/test/acse-conf/acse-upgrade-lnx.json deleted file mode 100644 index 467c0fc91..000000000 --- a/test/acse-conf/acse-upgrade-lnx.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "k8s-upgrade/v1.7.7.json", - "location": "eastus" - }, - { - "cluster_definition": "k8s-upgrade/v1.7.9.json", - "location": "centralus" - }, - { - "cluster_definition": "k8s-upgrade/v1.8.4.json", - "location": "westus2" - } - ] -} - diff --git a/test/acse-conf/acse-upgrade-win.json b/test/acse-conf/acse-upgrade-win.json deleted file mode 100644 index 752b29743..000000000 --- a/test/acse-conf/acse-upgrade-win.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "deployments": [ - { - "cluster_definition": "k8s-upgrade/v1.7.9-win.json", - "location": "eastus" - }, - { - "cluster_definition": "k8s-upgrade/v1.7.9-hybrid.json", - "location": "westus" - } - ] -} diff --git a/test/acsengine-ci.groovy b/test/acsengine-ci.groovy deleted file mode 100644 index 2b9085932..000000000 --- a/test/acsengine-ci.groovy +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env groovy - -node("slave") { - withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'AZURE_CLI_SPN_AKS_TEST', - passwordVariable: 'SPN_PASSWORD', usernameVariable: 'SPN_USER']]) { - timestamps { - wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) { - env.GOPATH="${WORKSPACE}" - env.PATH="${env.PATH}:${env.GOPATH}/bin" - def clone_dir = "${env.GOPATH}/src/github.com/Azure/aks-engine" - env.HOME=clone_dir - def success = true - Integer timeoutInMinutes = TEST_TIMEOUT.toInteger() - - dir(clone_dir) { - def img = null - try { - stage('Init') { - deleteDir() - checkout scm - img = docker.build('aks-engine-test', '--pull .') - } - } - catch(exc) { - echo "Exception ${exc}" - success = false - } - img.inside("-u root:root") { - String errorMsg = "" - def log_dir = pwd()+"/_logs" - try { - stage('Test') { - if(success) { - // Create log directory - sh("mkdir -p ${log_dir}") - // Create template, deploy and test - env.SERVICE_PRINCIPAL_CLIENT_ID="${SPN_USER}" - env.SERVICE_PRINCIPAL_CLIENT_SECRET="${SPN_PASSWORD}" - env.TENANT_ID="${TENANT_ID}" - env.SUBSCRIPTION_ID="${SUBSCRIPTION_ID}" - env.LOCATION = "${LOCATION}" - env.LOGFILE = "${log_dir}/${LOCATION}.log" - env.CLEANUP = "${CLEANUP}" - - env.INSTANCE_NAME = "test-acs-ci-${ORCHESTRATOR}-${env.LOCATION}-${env.BUILD_NUM}" - env.INSTANCE_NAME_PREFIX = "test-acs-ci" - env.ORCHESTRATOR = "${ORCHESTRATOR}" - env.CLUSTER_DEFINITION="examples/${ORCHESTRATOR}.json" - env.CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID="${CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID}" - env.CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET="${CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET}" - - script="test/cluster-tests/${ORCHESTRATOR}/test.sh" - def exists = fileExists script - - if (exists) { - env.VALIDATE = script - } else { - echo 'Skip validation' - } - timeout(time: timeoutInMinutes, unit: 'MINUTES') { - sh('./test/deploy.sh') - } - } - } - } - catch(exc) { - echo "Exception ${exc}" - success = false - errorMsg = "Please run \"make ci\" for verification" - } - - archiveArtifacts(allowEmptyArchive: true, artifacts: "${log_dir}/**/*.log") - - // Allow for future removal from the host - sh("chmod -R a+rwx ${WORKSPACE}") - - if(!success) { - currentBuild.result = "FAILURE" - String to = "${SEND_TO}".trim() - if(errorMsg != "") { - if(to != "") { - to += ";" - } - to += emailextrecipients([[$class: 'CulpritsRecipientProvider']]) - } - if(to != "") { - def url = "${env.BUILD_URL}\n\n" - for(String addr : to.tokenize('[ \t\n;,]+')) { - if(!addr.endsWith("@microsoft.com")) { - url = "" - } - } - gitCommit = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - emailext( - to: to, - subject: "[AKS Engine is BROKEN] ${env.JOB_NAME} #${env.BUILD_NUM}", - body: "Commit: ${gitCommit}\n\n${url}${errorMsg}" - ) - } - } - } - } - } - } - } -} diff --git a/test/acsengine-features.groovy b/test/acsengine-features.groovy deleted file mode 100644 index fa93bbb9f..000000000 --- a/test/acsengine-features.groovy +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env groovy - -node("slave") { - withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'AZURE_CLI_SPN_AKS_TEST', - passwordVariable: 'SPN_PASSWORD', usernameVariable: 'SPN_USER']]) { - timestamps { - wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) { - env.GOPATH="${WORKSPACE}" - env.PATH="${env.PATH}:${env.GOPATH}/bin" - def clone_dir = "${env.GOPATH}/src/github.com/Azure/aks-engine" - env.HOME=clone_dir - String sendTo = "${SEND_TO}".trim() - Integer timeoutInMinutes = STAGE_TIMEOUT.toInteger() - def autoclean="${AUTOCLEAN}" - - dir(clone_dir) { - def img = null - stage('Init') { - deleteDir() - checkout scm - img = docker.build('aks-engine-test', '--pull .') - } - - img.inside("-u root:root") { - def success = true - def junit_dir = "_junit" - def prefix = "" - try { - stage('Setup') { - // Set up Azure - sh("az login --service-principal -u ${SPN_USER} -p ${SPN_PASSWORD} --tenant ${TENANT_ID}") - sh("az account set --subscription ${SUBSCRIPTION_ID}") - - env.SERVICE_PRINCIPAL_CLIENT_ID="${SPN_USER}" - env.SERVICE_PRINCIPAL_CLIENT_SECRET="${SPN_PASSWORD}" - env.TENANT_ID="${TENANT_ID}" - env.SUBSCRIPTION_ID="${SUBSCRIPTION_ID}" - env.CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID="${CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID}" - env.CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET="${CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET}" - - // First check to see if var exists in context, then check for true-ness - // In Groovy, null and empty strings are false... - if(getBinding().hasVariable("CUSTOM_HYPERKUBE_SPEC") && CUSTOM_HYPERKUBE_SPEC) { - env.CUSTOM_HYPERKUBE_SPEC="${CUSTOM_HYPERKUBE_SPEC}" - } - - sh("printf 'acs-features-test%x' \$(date '+%s') > INSTANCE_NAME_PREFIX") - prefix = readFile('INSTANCE_NAME_PREFIX').trim() - // Create report directory - sh("mkdir -p ${junit_dir}") - // Build and test aks-engine - sh('make ci') - } - def pairs = "${SCENARIOS_LOCATIONS}".tokenize('|') - for(i = 0; i < pairs.size(); i++) { - def pair = pairs[i].tokenize('[ \t\n]+') - if(pair.size() != 2) { - echo "Skipping '"+pairs[i]+"'" - continue - } - def subdir = pair[0] - def names = sh(returnStdout: true, script: "cd examples; ls ${subdir}/*.json").split("\\r?\\n") - env.LOCATION = pair[1] - for(j = 0; j< names.size(); j++) { - def name = names[j].trim() - env.CLUSTER_DEFINITION = pwd()+"/examples/${name}" - env.INSTANCE_NAME = "${prefix}-${i}-${j}" - env.RESOURCE_GROUP = "test-acs-${subdir}-${env.LOCATION}-${env.BUILD_NUM}-${i}-${j}" - env.DEPLOYMENT_NAME = "${env.RESOURCE_GROUP}" - env.ORCHESTRATOR = sh(returnStdout: true, script: './test/step.sh get_orchestrator_type').trim() - env.LOGFILE = pwd()+"/${junit_dir}/${name}.log" - env.CLEANUP = "y" - // Generate and deploy template, validate deployments - try { - stage(name) { - def scripts = ["generate_template.sh", "deploy_template.sh"] - if(env.ORCHESTRATOR == "dcos" || env.ORCHESTRATOR == "swarmmode" || env.ORCHESTRATOR == "kubernetes") { - scripts += "validate_deployment.sh" - } - for(k = 0; k < scripts.size(); k++) { - def script = scripts[k] - def test = "${name}.${script}" - sh("mkdir -p ${junit_dir}/${test}") - sh("cp ./test/shunit/${script} ${junit_dir}/${test}/t.sh") - timeout(time: timeoutInMinutes, unit: 'MINUTES') { - sh("cd ${junit_dir}; shunit.sh -t ${test} > ${test}/junit.xml") - } - sh("grep 'failures=\"0\"' ${junit_dir}/${test}/junit.xml") - } - } - } - catch(exc) { - env.CLEANUP = autoclean - echo "Exception in [${name}] : ${exc}" - } - // Clean up - try { - sh('./test/step.sh cleanup') - } - catch(exc) { - echo "Exception ${exc}" - } - } // for (j = 0; j INSTANCE_NAME") - env.INSTANCE_NAME = readFile('INSTANCE_NAME').trim() - env.CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID="${CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID}" - env.CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET="${CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET}" - timeout(time: timeoutInMinutes, unit: 'MINUTES') { - sh('./test/step.sh generate_template') - } - } - - for (i = 0; i ${test}/junit.xml") - } - sh("grep 'failures=\"0\"' ${junit_dir}/${test}/junit.xml") - } - } - catch(exc) { - env.CLEANUP = autoclean - echo "Exception in [deploy ${canonicalName}/${env.LOCATION}] : ${exc}" - ok = false - } - // Scale-up cycle - try { - def counts = "${AGENT_POOL_SIZES}".tokenize('[ \t\n]+') - for (i = 0; i ${test}/junit.xml") - } - sh("grep 'failures=\"0\"' ${junit_dir}/${test}/junit.xml") - } - } - } - } - catch(exc) { - env.CLEANUP = autoclean - echo "Exception in [scale ${canonicalName}/${env.LOCATION}] : ${exc}" - ok = false - } - - // Validate deployment - try { - stage("${env.LOCATION} validate") { - if(ok) { - env.EXPECTED_NODE_COUNT = sh(returnStdout: true, script: './test/step.sh get_node_count').trim() - env.EXPECTED_ORCHESTRATOR_VERSION = sh(returnStdout: true, script: './test/step.sh get_orchestrator_release').trim() - def test = "validate-${env.LOCATION}" - sh("mkdir -p ${junit_dir}/${test}") - sh("cp ./test/shunit/validate_deployment.sh ${junit_dir}/${test}/t.sh") - timeout(time: timeoutInMinutes, unit: 'MINUTES') { - sh("cd ${junit_dir}; shunit.sh -t ${test} > ${test}/junit.xml") - } - sh("grep 'failures=\"0\"' ${junit_dir}/${test}/junit.xml") - } - else { - echo "Skipped verification for ${env.RESOURCE_GROUP}" - } - } - } - catch(exc) { - env.CLEANUP = autoclean - echo "Exception in [validate ${canonicalName}/${env.LOCATION}] : ${exc}" - } - // Clean up - try { - sh('./test/step.sh cleanup') - } - catch(exc) { - echo "Exception ${exc}" - } - } // for (i = 0; i INSTANCE_NAME") - env.INSTANCE_NAME = readFile('INSTANCE_NAME').trim() - env.CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID="${CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID}" - env.CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET="${CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET}" - timeout(time: timeoutInMinutes, unit: 'MINUTES') { - sh('./test/step.sh generate_template') - } - } - - for (i = 0; i ${test}/junit.xml") - } - sh("grep 'failures=\"0\"' ${junit_dir}/${test}/junit.xml") - } - } - catch(exc) { - env.CLEANUP = autoclean - echo "Exception in [deploy ${canonicalName}/${env.LOCATION}] : ${exc}" - ok = false - } - // Verify deployment - try { - stage("${env.LOCATION} validate") { - if(ok) { - def test = "validate-${env.LOCATION}" - sh("mkdir -p ${junit_dir}/${test}") - sh("cp ./test/shunit/validate_deployment.sh ${junit_dir}/${test}/t.sh") - timeout(time: timeoutInMinutes, unit: 'MINUTES') { - sh("cd ${junit_dir}; shunit.sh -t ${test} > ${test}/junit.xml") - } - sh("grep 'failures=\"0\"' ${junit_dir}/${test}/junit.xml") - } - else { - echo "Skipped verification for ${env.RESOURCE_GROUP}" - } - } - } - catch(exc) { - env.CLEANUP = autoclean - echo "Exception in [validate ${canonicalName}/${env.LOCATION}] : ${exc}" - } - // Clean up - try { - sh('./test/step.sh cleanup') - } - catch(exc) { - echo "Exception ${exc}" - } - } // for (i = 0; i - - Options: - -c : JSON file containing a list of deployment configurations. - Refer to aks-engine/test/aks-engine-test/aks-engine-test.json for examples - -d - -e -` - -var ( - logDir string - orchestratorRe *regexp.Regexp - enableMetrics bool - saName string - saKey string - sa promote.StorageAccount - subID string - rgPrefix string - orchestrator string -) - -func init() { - orchestratorRe = regexp.MustCompile(`"orchestratorType": "(\S+)"`) -} - -// ErrorStat represents an error status that will be reported -type ErrorStat struct { - errorInfo *report.ErrorInfo - testCategory string - count int64 -} - -// TestManager is object that contains test runner functions -type TestManager struct { - config *config.TestConfig - Manager *report.Manager - lock sync.Mutex - wg sync.WaitGroup - rootDir string - regions []string -} - -// Run begins the test run process -func (m *TestManager) Run() error { - fmt.Printf("Randomizing regional tests against the following regions: %s\n", m.regions) - n := len(m.config.Deployments) - if n == 0 { - return nil - } - - sa = promote.StorageAccount{ - Name: saName, - Key: saKey, - } - - // determine timeout - timeoutMin, err := strconv.Atoi(os.Getenv("STAGE_TIMEOUT_MIN")) - if err != nil { - return errors.Wrap(err, "Error [Atoi STAGE_TIMEOUT_MIN]") - } - timeout := time.Minute * time.Duration(timeoutMin) - - usePromoteToFailure := os.Getenv("PROMOTE_TO_FAILURE") == "true" - promoteToFailureTestSuffix := os.Getenv("PROMOTE_TO_FAILURE_TEST_SUFFIX") - - var retries int - if usePromoteToFailure { - fmt.Println("Using promote to failure to determine pass/fail") - } else { - // determine number of retries - retries, err = strconv.Atoi(os.Getenv("NUM_OF_RETRIES")) - if err != nil { - // Set default retries if not set - retries = 1 - } - fmt.Printf("Will allow %d retries to determine pass/fail\n", retries) - } - - // login to Azure - if _, _, err = m.runStep("init", stepInitAzure, os.Environ(), timeout); err != nil { - return err - } - - // return values for tests - success := make([]bool, n) - rand.Seed(time.Now().UnixNano()) - - m.wg.Add(n) - for index, dep := range m.config.Deployments { - go func(index int, dep config.Deployment) { - defer m.wg.Done() - var promToFailInfo promote.DigitalSignalFilter - resMap := make(map[string]*ErrorStat) - if usePromoteToFailure { - testName := strings.Replace(dep.ClusterDefinition, "/", "-", -1) - if promoteToFailureTestSuffix != "" { - testName += fmt.Sprintf("-%s", promoteToFailureTestSuffix) - } - if dep.Location != "" { - testName += fmt.Sprintf("-%s", dep.Location) - } - - errorInfo := m.testRun(dep, index, 0, timeout) - var failureStr string - if errorInfo != nil { - if errorStat, ok := resMap[errorInfo.ErrName]; !ok { - resMap[errorInfo.ErrName] = &ErrorStat{errorInfo: errorInfo, testCategory: dep.TestCategory, count: 1} - } else { - errorStat.count++ - } - - // For RecordTestRun - success[index] = false - failureStr = errorInfo.ErrName - // RecordTestRun QoS - sendRecordTestRun(sa, success[index], dep.Location, testName, dep.TestCategory, failureStr) - - // RunPromoteToFailure - if isPromoteToFailureStep(errorInfo.Step) { - promToFailInfo = promote.DigitalSignalFilter{ - TestName: testName, - TestType: metricsNS, - FailureStr: failureStr, - FailureCount: 1, - } - - var result bool - result, err = promote.RunPromoteToFailure(sa, promToFailInfo) - if err != nil { - fmt.Printf("Got error from RunPromoteToFailure: %#v\n", err) - } - if result { - success[index] = false - } else { - success[index] = true - } - } - - } else { - success[index] = true - failureStr = "" - // RecordTestRun QoS - sendRecordTestRun(sa, success[index], dep.Location, testName, dep.TestCategory, failureStr) - - // RunPromoteToFailure - promToFailInfo = promote.DigitalSignalFilter{ - TestName: testName, - TestType: metricsNS, - FailureStr: failureStr, - FailureCount: 0, - } - - promote.RunPromoteToFailure(sa, promToFailInfo) - - } - - if success[index] { - fmt.Printf("Promote to Fail passed: SUCCESS [%s]\n", testName) - } else { - fmt.Printf("Promote to Fail did not pass: ERROR [%s]\n", testName) - } - - } else { - for attempt := 0; attempt < retries; attempt++ { - errorInfo := m.testRun(dep, index, attempt, timeout) - // do not retry if successful - if errorInfo == nil { - success[index] = true - break - } - if errorStat, ok := resMap[errorInfo.ErrName]; !ok { - resMap[errorInfo.ErrName] = &ErrorStat{errorInfo: errorInfo, testCategory: dep.TestCategory, count: 1} - } else { - errorStat.count++ - } - } - } - - sendErrorMetrics(resMap, usePromoteToFailure) - }(index, dep) - } - m.wg.Wait() - //create reports - if err = m.Manager.CreateTestReport(fmt.Sprintf("%s/%s", logDir, testReport)); err != nil { - fmt.Printf("Failed to create %s: %v\n", testReport, err) - } - if err = m.Manager.CreateCombinedReport(fmt.Sprintf("%s/%s", logDir, combinedReport), testReport); err != nil { - fmt.Printf("Failed to create %s: %v\n", combinedReport, err) - } - // fail the test on error - for _, ok := range success { - if !ok { - return errors.New("Test failed") - } - } - return nil -} - -func (m *TestManager) testRun(d config.Deployment, index, attempt int, timeout time.Duration) *report.ErrorInfo { - subID = os.Getenv("SUBSCRIPTION_ID") - - rgPrefix = os.Getenv("RESOURCE_GROUP_PREFIX") - if rgPrefix == "" { - rgPrefix = "y" - fmt.Printf("RESOURCE_GROUP_PREFIX is not set. Using default '%s'\n", rgPrefix) - } - // Randomize region if no location was configured - if d.Location == "" { - randomIndex := rand.Intn(len(m.regions)) - d.Location = m.regions[randomIndex] - } - testName := strings.TrimSuffix(d.ClusterDefinition, filepath.Ext(d.ClusterDefinition)) - instanceName := fmt.Sprintf("acse-%d-%s-%s-%d-%d", rand.Intn(0x0ffffff), d.Location, os.Getenv("BUILD_NUM"), index, attempt) - resourceGroup := fmt.Sprintf("%s-%s-%s-%s-%d-%d", rgPrefix, strings.Replace(testName, "/", "-", -1), d.Location, os.Getenv("BUILD_NUM"), index, attempt) - logFile := fmt.Sprintf("%s/%s.log", logDir, resourceGroup) - validateLogFile := fmt.Sprintf("%s/validate-%s.log", logDir, resourceGroup) - - // determine orchestrator - env := os.Environ() - env = append(env, fmt.Sprintf("CLUSTER_DEFINITION=examples/%s", d.ClusterDefinition)) - cmd := exec.Command("test/step.sh", "get_orchestrator_type") - cmd.Env = env - out, err := cmd.Output() - if err != nil { - wrileLog(logFile, "Error [getOrchestrator %s] : %v", d.ClusterDefinition, err) - return report.NewErrorInfo(testName, "pretest", "OrchestratorTypeParsingError", "PreRun", d.Location) - } - orchestrator = strings.TrimSpace(string(out)) - - // update environment - env = append(env, fmt.Sprintf("LOCATION=%s", d.Location)) - env = append(env, fmt.Sprintf("ORCHESTRATOR=%s", orchestrator)) - env = append(env, fmt.Sprintf("INSTANCE_NAME=%s", instanceName)) - env = append(env, fmt.Sprintf("DEPLOYMENT_NAME=%s", instanceName)) - env = append(env, fmt.Sprintf("RESOURCE_GROUP=%s", resourceGroup)) - - // add scenario-specific environment variables - envFile := fmt.Sprintf("examples/%s.env", d.ClusterDefinition) - if _, err = os.Stat(envFile); err == nil { - var envHandle *os.File - envHandle, err = os.Open(envFile) - if err != nil { - wrileLog(logFile, "Error [open %s] : %v", envFile, err) - return report.NewErrorInfo(testName, "pretest", "FileAccessError", "PreRun", d.Location) - } - defer envHandle.Close() - - fileScanner := bufio.NewScanner(envHandle) - for fileScanner.Scan() { - str := strings.TrimSpace(fileScanner.Text()) - if match, _ := regexp.MatchString(`^\S+=\S+$`, str); match { - env = append(env, str) - } - } - } - - var errorInfo *report.ErrorInfo - steps := []string{stepCreateRG, stepPredeploy, stepGenerateTemplate, stepDeployTemplate, stepPostDeploy} - - // determine validation script - if !d.SkipValidation { - validate := fmt.Sprintf("test/cluster-tests/%s/test.sh", orchestrator) - if _, err = os.Stat(fmt.Sprintf("%s/%s", m.rootDir, validate)); err == nil { - env = append(env, fmt.Sprintf("VALIDATE=%s", validate)) - steps = append(steps, stepValidate) - } - } - for _, step := range steps { - txt, duration, err := m.runStep(resourceGroup, step, env, timeout) - if err != nil { - errorInfo = m.Manager.Process(txt, step, testName, d.Location) - sendDurationMetrics(step, d.Location, duration, errorInfo.ErrName) - wrileLog(logFile, "Error [%s:%s] %v\nOutput: %s", step, resourceGroup, err, txt) - // check AUTOCLEAN flag: if set to 'n', don't remove deployment - if os.Getenv("AUTOCLEAN") == "n" { - env = append(env, "CLEANUP=n") - } - break - } - sendDurationMetrics(step, d.Location, duration, report.ErrSuccess) - wrileLog(logFile, txt) - if step == stepGenerateTemplate { - // set up extra environment variables available after template generation - validateLogFile = fmt.Sprintf("%s/validate-%s.log", logDir, resourceGroup) - env = append(env, fmt.Sprintf("LOGFILE=%s", validateLogFile)) - - cmd := exec.Command("test/step.sh", "get_orchestrator_version") - cmd.Env = env - out, err := cmd.Output() - if err != nil { - wrileLog(logFile, "Error [%s:%s] %v", "get_orchestrator_version", resourceGroup, err) - errorInfo = report.NewErrorInfo(testName, step, "OrchestratorVersionParsingError", "PreRun", d.Location) - break - } - env = append(env, fmt.Sprintf("EXPECTED_ORCHESTRATOR_VERSION=%s", strings.TrimSpace(string(out)))) - - cmd = exec.Command("test/step.sh", "get_node_count") - cmd.Env = env - out, err = cmd.Output() - if err != nil { - wrileLog(logFile, "Error [%s:%s] %v", "get_node_count", resourceGroup, err) - errorInfo = report.NewErrorInfo(testName, step, "NodeCountParsingError", "PreRun", d.Location) - break - } - nodesCount := strings.Split(strings.TrimSpace(string(out)), ":") - if len(nodesCount) != 3 { - wrileLog(logFile, "get_node_count: unexpected output '%s'", string(out)) - errorInfo = report.NewErrorInfo(testName, step, "NodeCountParsingError", "PreRun", d.Location) - break - } - env = append(env, fmt.Sprintf("EXPECTED_NODE_COUNT=%s", nodesCount[0])) - env = append(env, fmt.Sprintf("EXPECTED_LINUX_AGENTS=%s", nodesCount[1])) - env = append(env, fmt.Sprintf("EXPECTED_WINDOWS_AGENTS=%s", nodesCount[2])) - } - } - // clean up - if txt, _, err := m.runStep(resourceGroup, stepCleanup, env, timeout); err != nil { - wrileLog(logFile, "Error: %v\nOutput: %s", err, txt) - } - if errorInfo == nil { - // do not keep logs for successful test - for _, fname := range []string{logFile, validateLogFile} { - if _, err := os.Stat(fname); !os.IsNotExist(err) { - if err = os.Remove(fname); err != nil { - fmt.Printf("Failed to remove %s : %v\n", fname, err) - } - } - } - } - return errorInfo -} - -func isPromoteToFailureStep(step string) bool { - switch step { - case stepDeployTemplate: - return true - case stepValidate: - return true - case stepPostDeploy: - return true - default: - return false - } -} - -func isValidEnv() bool { - valid := true - envVars := []string{ - "SERVICE_PRINCIPAL_CLIENT_ID", - "SERVICE_PRINCIPAL_CLIENT_SECRET", - "TENANT_ID", - "SUBSCRIPTION_ID", - "CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID", - "CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET", - "STAGE_TIMEOUT_MIN"} - - for _, envVar := range envVars { - if os.Getenv(envVar) == "" { - fmt.Printf("Must specify environment variable %s\n", envVar) - valid = false - } - } - return valid -} - -func (m *TestManager) runStep(name, step string, env []string, timeout time.Duration) (string, time.Duration, error) { - // prevent ARM throttling - m.lock.Lock() - go func() { - time.Sleep(2 * time.Second) - m.lock.Unlock() - }() - start := time.Now() - cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s %s", script, step)) - cmd.Dir = m.rootDir - cmd.Env = env - - var out bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &out - - if err := cmd.Start(); err != nil { - return "", time.Since(start), err - } - timer := time.AfterFunc(timeout, func() { - cmd.Process.Kill() - }) - err := cmd.Wait() - timer.Stop() - - now := time.Now().Format("15:04:05") - if err != nil { - fmt.Printf("ERROR [%s] [%s %s]\n", now, step, name) - return out.String(), time.Since(start), err - } - fmt.Printf("SUCCESS [%s] [%s %s]\n", now, step, name) - return out.String(), time.Since(start), nil -} - -func wrileLog(fname string, format string, args ...interface{}) { - str := fmt.Sprintf(format, args...) - - f, err := os.OpenFile(fname, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) - if err != nil { - fmt.Printf("Error [OpenFile %s] : %v\n", fname, err) - return - } - defer f.Close() - - if _, err = f.Write([]byte(str)); err != nil { - fmt.Printf("Error [Write %s] : %v\n", fname, err) - } -} - -func sendErrorMetrics(resMap map[string]*ErrorStat, usePromoteToFailure bool) { - if !enableMetrics { - return - } - for _, errorStat := range resMap { - var severity string - if usePromoteToFailure || errorStat.count > 1 { - severity = "Critical" - } else { - severity = "Intermittent" - } - category := errorStat.testCategory - if len(category) == 0 { - category = "generic" - } - // add metrics - dims := map[string]string{ - "TestName": errorStat.errorInfo.TestName, - "TestCategory": category, - "Location": errorStat.errorInfo.Location, - "Error": errorStat.errorInfo.ErrName, - "Class": errorStat.errorInfo.ErrClass, - "Severity": severity, - } - err := metrics.AddMetric(metricsEndpoint, metricsNS, metricError, errorStat.count, dims) - if err != nil { - fmt.Printf("Failed to send metric: %v\n", err) - } - } -} - -func sendDurationMetrics(step, location string, duration time.Duration, errorName string) { - if !enableMetrics { - return - } - var metricName string - - switch step { - case stepDeployTemplate: - metricName = metricDeploymentDuration - case stepValidate: - metricName = metricValidationDuration - default: - return - } - - durationSec := int64(duration / time.Second) - // add metrics - dims := map[string]string{ - "Location": location, - "Error": errorName, - } - err := metrics.AddMetric(metricsEndpoint, metricsNS, metricName, durationSec, dims) - if err != nil { - fmt.Printf("Failed to send metric: %v\n", err) - } -} - -func sendRecordTestRun(sa promote.StorageAccount, success bool, location, testName, testtype, failureStr string) { - - testRecordQoS := promote.TestRunQos{ - TimeStampUTC: time.Now(), - TestName: testName, - TestType: metricsNS, - SubscriptionID: subID, - ResourceGroup: rgPrefix, - Region: location, - Orchestrator: testtype, - Success: success, - FailureStr: failureStr, - } - promote.RecordTestRun(sa, testRecordQoS) -} - -func mainInternal() error { - var configFile string - var rootDir string - var logErrorFile string - var err error - flag.StringVar(&configFile, "c", "", "deployment configurations") - flag.StringVar(&rootDir, "d", "", "aks-engine root directory") - flag.StringVar(&logErrorFile, "e", "", "logError config file") - flag.StringVar(&saName, "j", "", "SA Name") - flag.StringVar(&saKey, "k", "", "SA Key") - flag.Usage = func() { - fmt.Println(usage) - } - flag.Parse() - - testManager := TestManager{} - - // validate environment - if !isValidEnv() { - return errors.New("environment is not set") - } - // get test configuration - if configFile == "" { - return errors.New("test configuration is not provided") - } - testManager.config, err = config.GetTestConfig(configFile) - if err != nil { - return err - } - // get Jenkins build number - buildNum, err := strconv.Atoi(os.Getenv("BUILD_NUM")) - if err != nil { - fmt.Println("Warning: BUILD_NUM is not set or invalid. Assuming 0") - buildNum = 0 - } - // set environment variable ENABLE_METRICS=y to enable sending the metrics (disabled by default) - if os.Getenv("ENABLE_METRICS") == "y" { - enableMetrics = true - } - // initialize report manager - testManager.Manager = report.New(os.Getenv("JOB_BASE_NAME"), buildNum, len(testManager.config.Deployments), logErrorFile) - // check root directory - if rootDir == "" { - return errors.New("aks-engine root directory is not provided") - } - testManager.rootDir = rootDir - if _, err = os.Stat(fmt.Sprintf("%s/%s", rootDir, script)); err != nil { - return err - } - // make logs directory - logDir = fmt.Sprintf("%s/_logs", rootDir) - os.RemoveAll(logDir) - if err = os.Mkdir(logDir, os.FileMode(0755)); err != nil { - return err - } - // set regions - regions := []string{} - for _, region := range helpers.GetAzureLocations() { - switch region { - case "eastus2euap": // initial deploy region for all RPs, known to be less stable - case "japanwest": // no D2V2 support - case "chinaeast": // private cloud - case "chinanorth": // private cloud - case "chinaeast2": // private cloud - case "chinanorth2": // private cloud - case "germanycentral": // Germany cloud - case "germanynortheast": // Germany cloud - case "usgovvirginia": // US Gov cloud - case "usgoviowa": // US Gov cloud - case "usgovarizona": // US Gov cloud - case "usgovtexas": // US Gov cloud - case "koreacentral": // TODO make sure our versions of azure-cli support this cloud - case "centraluseuap": // TODO determine why this region is flaky - case "brazilsouth": // canary region - case "francecentral": // not supported by sub - default: - regions = append(regions, region) - } - } - testManager.regions = regions - // seed random number generator - rand.Seed(time.Now().Unix()) - // run tests - return testManager.Run() -} - -func main() { - if err := mainInternal(); err != nil { - fmt.Printf("Error: %v\n", err) - os.Exit(1) - } - os.Exit(0) -} diff --git a/test/aks-engine-test/metrics/metrics.go b/test/aks-engine-test/metrics/metrics.go deleted file mode 100644 index f5e1981ef..000000000 --- a/test/aks-engine-test/metrics/metrics.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -package metrics - -import ( - "fmt" - - "github.com/Azure/aks-engine/pkg/helpers" - "github.com/alexcesaro/statsd" -) - -type mdmBucket struct { - Namespace string `json:"Namespace"` - Metric string `json:"Metric"` - Dims map[string]string `json:"Dims"` -} - -// AddMetric adds the defined metric to a list of metrics to send to MDM -func AddMetric(endpoint, namespace, metric string, count int64, dims map[string]string) error { - bucket := mdmBucket{ - Namespace: namespace, - Metric: metric, - Dims: dims} - data, err := helpers.JSONMarshal(bucket, false) - if err != nil { - return err - } - client, err := statsd.New( - statsd.Address(endpoint), - statsd.Network("udp"), - statsd.ErrorHandler( - func(err error) { - fmt.Println(err.Error()) - })) - if err != nil { - return err - } - defer client.Close() - client.Count(string(data), count) - return nil -} diff --git a/test/aks-engine-test/metrics/metrics_test.go b/test/aks-engine-test/metrics/metrics_test.go deleted file mode 100644 index b95470529..000000000 --- a/test/aks-engine-test/metrics/metrics_test.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -package metrics - -import ( - "errors" - "net" - "sync" - "testing" - "time" -) - -type testSession struct { - endpoint string - conn *net.UDPConn - timer *time.Timer - wg sync.WaitGroup - err error -} - -func newSession() *testSession { - sess := &testSession{ - endpoint: ":12345", - timer: time.NewTimer(time.Second), - } - sess.timer.Stop() - return sess -} - -func (s *testSession) start() error { - addr, err := net.ResolveUDPAddr("udp", s.endpoint) - if err != nil { - return err - } - s.conn, err = net.ListenUDP("udp", addr) - if err != nil { - return err - } - s.wg.Add(1) - go s.run() - return nil -} - -func (s *testSession) stop() error { - // allow up to 2 sec to complete session - s.timer.Reset(2 * time.Second) - s.wg.Wait() - s.conn.Close() - return s.err -} - -func (s *testSession) run() { - defer s.wg.Done() - buffer := make([]byte, 1024) - for { - select { - case <-s.timer.C: - s.err = errors.New("No metrics message. Exiting by timeout") - return - default: - - n, err := s.conn.Read(buffer) - if err != nil { - s.err = err - return - } - if n > 0 { - s.timer.Stop() - return - } - } - } -} - -func TestMetric(t *testing.T) { - sess := newSession() - if err := sess.start(); err != nil { - t.Fatal(err) - } - - dims := map[string]string{ - "test": "myTest", - "location": "myLocation", - "error": "myError", - "errClass": "myErrorClass", - } - - if err := AddMetric(sess.endpoint, "metricsNS", "metricName", 1, dims); err != nil { - t.Fatal(err) - } - - if err := sess.stop(); err != nil { - t.Fatal(err) - } -} diff --git a/test/aks-engine-test/promote/promote.go b/test/aks-engine-test/promote/promote.go deleted file mode 100644 index a5d0ea8a5..000000000 --- a/test/aks-engine-test/promote/promote.go +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -package promote - -import ( - "fmt" - "strings" - "time" - - "github.com/Azure/azure-sdk-for-go/storage" -) - -const ( - // ACSRPTests are the rp tests - ACSRPTests string = "ACSRpTests" - // AKSEngineTests are the AKS Engine tests - AKSEngineTests string = "AKSEngineTests" - // SimDemTests are the SimDem tests - SimDemTests string = "SimDemTests" -) - -// StorageAccount is how we connect to storage -type StorageAccount struct { - Name string - Key string -} - -const ( - // Mesos is the string constant for MESOS orchestrator type - Mesos string = "Mesos" - // DCOS is the string constant for DCOS orchestrator type and defaults to DCOS188 - DCOS string = "DCOS" - // Swarm is the string constant for the Swarm orchestrator type - Swarm string = "Swarm" - // Kubernetes is the string constant for the Kubernetes orchestrator type - Kubernetes string = "Kubernetes" - // SwarmMode is the string constant for the Swarm Mode orchestrator type - SwarmMode string = "SwarmMode" - // RecordTestRunTableName for storing RecordTestRun - RecordTestRunTableName string = "RecordTestRun" - // PromoteToFailureTableName for storing PromoteToFailure - PromoteToFailureTableName string = "PromoteToFailure" -) - -// TestRunQos structure -type TestRunQos struct { - TimeStampUTC time.Time // partition key - TestName string // row key - TestType string - SubscriptionID string - ResourceGroup string - Region string - Orchestrator string - Success bool - FailureStr string - // DeploymentDurationInSeconds int -} - -//DigitalSignalFilter structure -type DigitalSignalFilter struct { - TestName string // partition key - TestType string // row key - FailureStr string - FailureCount float64 -} - -// RecordTestRun procedure pushes all test result data to RecordTestRun Table -func RecordTestRun(sa StorageAccount, testRunQos TestRunQos) { - // fmt.Printf("record test run Qos to '%s': %v\n", sa.Name, testRunQos) - - // get Azure Storage Client - var err error - var azureStoreClient storage.Client - if azureStoreClient, err = storage.NewBasicClient(sa.Name, sa.Key); err != nil { - fmt.Printf("FAIL to create azure storage basic client, Error: %s\n", err.Error()) - return - } - - // From storageClient, get Table Service Client - tsc := azureStoreClient.GetTableService() - table1 := tsc.GetTableReference(RecordTestRunTableName) - - // Create Table if it does not exist - if err = table1.Create(30, storage.FullMetadata, nil); err != nil && !strings.Contains(err.Error(), "The table specified already exists") { - fmt.Printf("Failed to create table: %s, Error: %s\n", RecordTestRunTableName, err.Error()) - return - } - // fmt.Printf("Table : %s is created\n", RecordTestRunTableName) - - t := testRunQos.TimeStampUTC.Format("2006-01-02 15:04:05") - - // Insert Entity Entry into Table - entity := table1.GetEntityReference(t, testRunQos.TestName) - - props := map[string]interface{}{ - "TestType": testRunQos.TestType, - "SubscriptionID": testRunQos.SubscriptionID, - "ResourceGroup": testRunQos.ResourceGroup, - "Region": testRunQos.Region, - "Orchestrator": testRunQos.Orchestrator, - "Success": testRunQos.Success, - "FailureStr": testRunQos.FailureStr, - } - - entity.Properties = props - if err = entity.Insert(storage.FullMetadata, nil); err != nil { - fmt.Printf("Could not insert entity into table, Error: %v\n", err) - return - } -} - -// RunPromoteToFailure procedure -// Returns True when Error is Promoted, Else False -func RunPromoteToFailure(sa StorageAccount, testRunPromToFail DigitalSignalFilter) (bool, error) { - - // get Azure Storage Client - var err error - var azureStoreClient storage.Client - if azureStoreClient, err = storage.NewBasicClient(sa.Name, sa.Key); err != nil { - fmt.Printf("FAIL to create azure storage basic client, Error: %s\n", err.Error()) - return false, err - } - - // From azureStoreClient, get Table Service Client - tsc := azureStoreClient.GetTableService() - table1 := tsc.GetTableReference(PromoteToFailureTableName) - - // Create Table if it does not exist - if err = table1.Create(30, storage.FullMetadata, nil); err != nil && !strings.Contains(err.Error(), "The table specified already exists") { - fmt.Printf("Failed to create table: %s, Error: %s\n", PromoteToFailureTableName, err.Error()) - return false, err - } - // 1. Get the Entity using partition key and row key - // 2. If doesnt exist, then create the new entity and exit as success - // 3. If it exists, then increment the FailureCount - // 4. If FailureCount == 3, push out faillure - - entity := table1.GetEntityReference(testRunPromToFail.TestName, testRunPromToFail.TestType) - - err = entity.Get(30, storage.FullMetadata, &storage.GetEntityOptions{ - Select: []string{"FailureStr", "FailureCount"}, - }) - - if err != nil { - if strings.Contains(err.Error(), "The specified resource does not exist") { - // Entity does not exist in Table - // Insert Entity into Table - - err = insertEntity(table1, testRunPromToFail) - if err != nil { - fmt.Printf("Error inserting entity : %v\n", err) - return false, err - } - } - return false, err - } - - existingFailureStr := entity.Properties["FailureStr"] - existingFailureCount := entity.Properties["FailureCount"] - - if existingFailureStr != testRunPromToFail.FailureStr { - // Perform Update of this entity with testRunPromToFail.FailureStr and testRunPromToFail.FailureCount - updateEntity(entity, testRunPromToFail.FailureCount, testRunPromToFail.FailureStr) - return false, nil - } - - if testRunPromToFail.FailureCount == 0 { - // Update the Entity with FailureCount 0 - // Return False - updateEntity(entity, testRunPromToFail.FailureCount, testRunPromToFail.FailureStr) - fmt.Printf("Reset Failure Count for %s to : %v\n\n", testRunPromToFail.TestName, testRunPromToFail.FailureCount) - return false, nil - } - - fmt.Printf("Existing Failure Count for %s : %v\n\n", testRunPromToFail.TestName, existingFailureCount) - newFailureCount := existingFailureCount.(float64) + testRunPromToFail.FailureCount - fmt.Printf("Incremented Failure Count for %s to : %v\n\n", testRunPromToFail.TestName, newFailureCount) - updateEntity(entity, newFailureCount, testRunPromToFail.FailureStr) - - if newFailureCount >= 3 { - return true, nil - } - - return false, nil -} - -func insertEntity(table *storage.Table, testRunPromToFail DigitalSignalFilter) error { - // Insert Entity Entry into Table - entity := table.GetEntityReference(testRunPromToFail.TestName, testRunPromToFail.TestType) - - props := map[string]interface{}{ - "FailureStr": testRunPromToFail.FailureStr, - "FailureCount": testRunPromToFail.FailureCount, - } - - entity.Properties = props - if err := entity.Insert(storage.FullMetadata, nil); err != nil { - fmt.Printf("Could not insert entity into table, Error: %v\n", err) - return err - } - return nil -} - -func updateEntity(entity *storage.Entity, failureCount float64, failureStr string) { - props := map[string]interface{}{ - "FailureStr": failureStr, - "FailureCount": failureCount, - } - entity.Properties = props - if err := entity.Update(false, nil); err != nil { - fmt.Printf("Error in Updating Entity : %v\n\n", err) - } -} diff --git a/test/aks-engine-test/report/report.go b/test/aks-engine-test/report/report.go deleted file mode 100644 index 4a9ff36e0..000000000 --- a/test/aks-engine-test/report/report.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -package report - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "regexp" - "strconv" - "sync" - "time" - - "github.com/Azure/aks-engine/pkg/helpers" -) - -// ErrorInfo represents the CI error -type ErrorInfo struct { - TestName string - Step string - ErrName string - ErrClass string - Location string -} - -// ErrorStat represents the aggregate error count and region -type ErrorStat struct { - Count int `json:"count"` - Locations map[string]int `json:"locations"` -} - -// Manager represents the details about a build and errors in that build -type Manager struct { - lock sync.Mutex - JobName string `json:"job"` - BuildNum int `json:"build"` - Deployments int `json:"deployments"` - Errors int `json:"errors"` - StartTime time.Time `json:"startTime"` - Duration string `json:"duration"` - // Failure map: key=error, value=locations - Failures map[string]*ErrorStat `json:"failures"` - LogErrors logErrors `json:"-"` -} - -type logErrors struct { - LogErrors []logError `json:"Errors"` -} - -type logError struct { - Name string `json:"name"` - Class string `json:"class"` - Regex string `json:"regex"` -} - -const ( - // ErrClassDeployment represents an error during deployment - ErrClassDeployment = "Deployment" - // ErrClassValidation represents an error during validation (tests) - ErrClassValidation = "Validation" - // ErrClassAzcli represents an error with Azure CLI - ErrClassAzcli = "AzCLI" - // ErrClassNone represents absence of error - ErrClassNone = "None" - - // ErrSuccess represents a success, for some reason - ErrSuccess = "Success" - // ErrUnknown represents an unknown error - ErrUnknown = "UnspecifiedError" -) - -// New creates a new error report -func New(jobName string, buildNum int, nDeploys int, logErrorsFileName string) *Manager { - h := &Manager{} - h.JobName = jobName - h.BuildNum = buildNum - h.Deployments = nDeploys - h.Errors = 0 - h.StartTime = time.Now().UTC() - h.Failures = make(map[string]*ErrorStat) - h.LogErrors = makeErrorList(logErrorsFileName) - return h -} - -func makeErrorList(fileName string) logErrors { - dummy := logErrors{} - - if fileName != "" { - file, e := ioutil.ReadFile(fileName) - if e != nil { - // do not exit the tests - fmt.Printf("ERROR: %v\n", e) - } - json.Unmarshal(file, &dummy) - } - return dummy -} - -// Copy TBD needs definition [ToDo] -func (h *Manager) Copy() *Manager { - n := New(h.JobName, h.BuildNum, h.Deployments, "") - n.Errors = h.Errors - n.StartTime = h.StartTime - for e, f := range h.Failures { - locs := make(map[string]int) - for l, c := range f.Locations { - locs[l] = c - } - n.Failures[e] = &ErrorStat{Count: f.Count, Locations: locs} - } - return n -} - -// Process TBD needs definition -func (h *Manager) Process(txt, step, testName, location string) *ErrorInfo { - for _, logErr := range h.LogErrors.LogErrors { - if match, _ := regexp.MatchString(logErr.Regex, txt); match { - h.addFailure(logErr.Name, map[string]int{location: 1}) - return NewErrorInfo(testName, step, logErr.Name, logErr.Class, location) - } - } - h.addFailure(ErrUnknown, map[string]int{location: 1}) - return NewErrorInfo(testName, step, ErrUnknown, ErrClassNone, location) -} - -func (h *Manager) addFailure(key string, locations map[string]int) { - h.lock.Lock() - defer h.lock.Unlock() - - cnt := 0 - if failure, ok := h.Failures[key]; !ok { - locs := make(map[string]int) - for l, c := range locations { - locs[l] = c - cnt += c - } - h.Failures[key] = &ErrorStat{Count: cnt, Locations: locs} - } else { - for l, c := range locations { - cnt += c - if _, ok := failure.Locations[l]; !ok { - failure.Locations[l] = c - } else { - failure.Locations[l] += c - } - } - failure.Count += cnt - } - h.Errors += cnt -} - -// CreateTestReport TBD needs definition -func (h *Manager) CreateTestReport(filepath string) error { - h.Duration = time.Now().UTC().Sub(h.StartTime).String() - data, err := helpers.JSONMarshalIndent(h, "", " ", false) - if err != nil { - return err - } - file, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, os.FileMode(0644)) - if err != nil { - return err - } - defer file.Close() - _, err = file.Write(data) - return err -} - -// CreateCombinedReport TBD needs definition -func (h *Manager) CreateCombinedReport(filepath, testReportFname string) error { - // "COMBINED_PAST_REPORTS" is the number of recent reports in the combined report - reports, err := strconv.Atoi(os.Getenv("COMBINED_PAST_REPORTS")) - if err != nil || reports <= 0 { - return nil - } - combinedReport := h.Copy() - for i := 1; i <= reports; i++ { - data, err := ioutil.ReadFile(fmt.Sprintf("%s/%d/%s/%s", - os.Getenv("JOB_BUILD_ROOTDIR"), h.BuildNum-i, os.Getenv("JOB_BUILD_SUBDIR"), testReportFname)) - if err != nil { - break - } - testReport := &Manager{} - if err := json.Unmarshal(data, &testReport); err != nil { - break - } - combinedReport.StartTime = testReport.StartTime - combinedReport.Deployments += testReport.Deployments - - for e, f := range testReport.Failures { - combinedReport.addFailure(e, f.Locations) - } - } - return combinedReport.CreateTestReport(filepath) -} - -// NewErrorInfo TBD needs definition -func NewErrorInfo(testName, step, errName, errClass, location string) *ErrorInfo { - return &ErrorInfo{TestName: testName, Step: step, ErrName: errName, ErrClass: errClass, Location: location} -} diff --git a/test/aks-engine-test/report/report_test.go b/test/aks-engine-test/report/report_test.go deleted file mode 100644 index 848c74173..000000000 --- a/test/aks-engine-test/report/report_test.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -package report - -import ( - "encoding/json" - "io/ioutil" - "testing" -) - -func TestReportParse(t *testing.T) { - jobName := "TestJob" - buildNum := 001 - nDeploys := 4 - fileName := "../aks-engine-errors.json" - dummy := New(jobName, buildNum, nDeploys, fileName) - - txt := "Error loading command module" - step := "step" - testName := "dummyTest" - d := "westus" - _ = dummy.Process(txt, step, testName, d) - - testReport := "TestReport.json" - if err := dummy.CreateTestReport(testReport); err != nil { - t.Fatal(err) - } - - raw, err := ioutil.ReadFile(testReport) - if err != nil { - t.Fatal(err) - } - - h := &Manager{} - json.Unmarshal(raw, &h) - - if len(h.LogErrors.LogErrors) != 0 { - t.Fatalf("Expected LogErrors to be empty, instead it is of size %d", len(h.LogErrors.LogErrors)) - } -} diff --git a/test/all-chestrator.groovy b/test/all-chestrator.groovy deleted file mode 100644 index af6de2cdc..000000000 --- a/test/all-chestrator.groovy +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env groovy - -node("slave") { - withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'AZURE_CLI_SPN_AKS_TEST', - passwordVariable: 'SPN_PASSWORD', usernameVariable: 'SPN_USER']]) { - timestamps { - wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) { - def jobname = "${JOBNAME}" - def tests = [:] - - def pairs = "${ORCHESTRATOR_LOCATION}".tokenize('|') - for(i = 0; i < pairs.size(); i++) { - def pair = pairs[i].tokenize('[ \t\n]+') - if(pair.size() != 2) { - echo "Skipping '"+pairs[i]+"'" - continue - } - def orchestrator = pair[0] - def location = pair[1] - def name = "${orchestrator}-${location}" - - tests[name] = { - stage(name) { - build job: jobname, - parameters: - [[$class: 'StringParameterValue', name: 'ORCHESTRATOR', value: orchestrator], - [$class: 'StringParameterValue', name: 'LOCATION', value: location]] - } - } - } - parallel tests - } - } - } -} diff --git a/test/bootstrap/checkout-pr.sh b/test/bootstrap/checkout-pr.sh deleted file mode 100755 index 81a8914d8..000000000 --- a/test/bootstrap/checkout-pr.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -#################################################### -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -#################################################### - -set -eu -o pipefail -set -x - -git init . -git clean -dfx -git reset --hard -git config --local user.name 'AKS Bot' -git config --local user.email 'aks-bot@microsoft.com' -git fetch --tags https://github.com/${REPO_OWNER}/${REPO_NAME} master +refs/pull/${PULL_NUMBER}/head:refs/pr/${PULL_NUMBER} -git checkout -B test "${PULL_BASE_SHA}" -git merge --no-ff -m "Merge +refs/pull/${PULL_NUMBER}/head:refs/pr/${PULL_NUMBER}" "${PULL_PULL_SHA}" - -echo "----------------------------------------------------------" -env -echo "----------------------------------------------------------" diff --git a/test/cluster-tests/dcos/marathon-slave-public.json b/test/cluster-tests/dcos/marathon-slave-public.json deleted file mode 100644 index 46cc29fc4..000000000 --- a/test/cluster-tests/dcos/marathon-slave-public.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "id": "web", - "container": { - "type": "DOCKER", - "docker": { - "image": "yeasy/simple-web", - "network": "HOST", - "forcePullImage": false, - "privileged": false - } - }, - "instances": 3, - "cpus": 0.1, - "mem": 65, - "cmd": null, - "disk": 0, - "executor": null, - "fetch": null, - "constraints": null, - "acceptedResourceRoles": [ - "slave_public" - ], - "user": null, - "env": null, - "healthChecks": [{ - "protocol": "HTTP", - "path": "/", - "portIndex": 0, - "timeoutSeconds": 10, - "gracePeriodSeconds": 10, - "intervalSeconds": 2, - "maxConsecutiveFailures": 10 - }], - "portDefinitions": [ - { - "protocol": "Tcp", - "port": 0, - "name": "80" - } - ], - "labels":{ - "HAPROXY_GROUP":"external", - "HAPROXY_0_VHOST":"{agentFQDN}", - "HAPROXY_0_MODE":"http" - } -} \ No newline at end of file diff --git a/test/cluster-tests/dcos/marathon.json b/test/cluster-tests/dcos/marathon.json deleted file mode 100644 index 0b6a768ba..000000000 --- a/test/cluster-tests/dcos/marathon.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "id": "web", - "container": { - "type": "DOCKER", - "docker": { - "image": "yeasy/simple-web", - "network": "BRIDGE", - "portMappings": [ - { "hostPort": 0, "containerPort": 80, "servicePort": 10000 } - ], - "forcePullImage":true - } - }, - "instances": 3, - "cpus": 0.1, - "mem": 65, - "healthChecks": [{ - "protocol": "HTTP", - "path": "/", - "portIndex": 0, - "timeoutSeconds": 10, - "gracePeriodSeconds": 10, - "intervalSeconds": 2, - "maxConsecutiveFailures": 10 - }], - "labels":{ - "HAPROXY_GROUP":"external", - "HAPROXY_0_VHOST":"{agentFQDN}", - "HAPROXY_0_MODE":"http" - } -} \ No newline at end of file diff --git a/test/cluster-tests/dcos/test.sh b/test/cluster-tests/dcos/test.sh deleted file mode 100755 index 7f6f41639..000000000 --- a/test/cluster-tests/dcos/test.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -#################################################### -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -#################################################### - -# do not use 'set -e' -# -o pipefail -set -x - -source "$DIR/../utils.sh" - -ENV_FILE="${CLUSTER_DEFINITION}.env" -if [ -e "${ENV_FILE}" ]; then - source "${ENV_FILE}" -fi - -MARATHON_JSON="${MARATHON_JSON:-marathon.json}" - -remote_exec="ssh -i "${SSH_KEY}" -o ConnectTimeout=30 -o StrictHostKeyChecking=no azureuser@${INSTANCE_NAME}.${LOCATION}.cloudapp.azure.com -p2200" -agentFQDN="${INSTANCE_NAME}0.${LOCATION}.cloudapp.azure.com" -remote_cp="scp -i "${SSH_KEY}" -P 2200 -o StrictHostKeyChecking=no" - -function teardown { - ${remote_exec} ./dcos marathon app remove /web -} - -###### Check node count -function check_node_count() { - log "Checking node count" - count=20 - while (( $count > 0 )); do - log " ... counting down $count" - node_count=$(${remote_exec} curl -s http://localhost:1050/system/health/v1/nodes | jq '.nodes | length') - [ $? -eq 0 ] && [ ! -z "$node_count" ] && [ $node_count -eq ${EXPECTED_NODE_COUNT} ] && log "Successfully got $EXPECTED_NODE_COUNT nodes" && break - sleep 30; count=$((count-1)) - done - if (( $node_count != ${EXPECTED_NODE_COUNT} )); then - log "gave up waiting for DCOS nodes: $node_count available, ${EXPECTED_NODE_COUNT} expected" - exit 1 - fi -} - -check_node_count - -log "Downloading dcos" -${remote_exec} curl -O https://dcos-mirror.azureedge.net/binaries/cli/linux/x86-64/dcos-1.10/dcos -if [[ "$?" != "0" ]]; then log "Failed to download dcos"; exit 1; fi -log "Setting dcos permissions" -${remote_exec} chmod a+x ./dcos -if [[ "$?" != "0" ]]; then log "Failed to chmod dcos"; exit 1; fi -log "Configuring dcos" -${remote_exec} ./dcos cluster setup http://localhost:80 -if [[ "$?" != "0" ]]; then log "Failed to configure dcos"; exit 1; fi - -log "Copying marathon.json" - -${remote_cp} "${DIR}/${MARATHON_JSON}" azureuser@${INSTANCE_NAME}.${LOCATION}.cloudapp.azure.com:marathon.json -if [[ "$?" != "0" ]]; then log "Failed to copy marathon.json"; exit 1; fi - -# feed agentFQDN to marathon.json -log "Configuring marathon.json" -${remote_exec} sed -i "s/{agentFQDN}/${agentFQDN}/g" marathon.json -if [[ "$?" != "0" ]]; then log "Failed to configure marathon.json"; exit 1; fi - - -log "Adding marathon app" -count=20 -while (( $count > 0 )); do - log " ... counting down $count" - ${remote_exec} ./dcos marathon app list | grep /web - retval=$? - if [[ $retval -eq 0 ]]; then log "Marathon App successfully installed" && break; fi - ${remote_exec} ./dcos marathon app add marathon.json - retval=$? - if [[ "$retval" == "0" ]]; then break; fi - sleep 15; count=$((count-1)) -done -if [[ $retval -ne 0 ]]; then log "gave up waiting for marathon to be added"; exit 1; fi - -# only need to teardown if app added successfully -trap teardown EXIT - -log "Validating marathon app" -count=0 -while [[ ${count} -lt 25 ]]; do - count=$((count+1)) - log " ... cycle $count" - running=$(${remote_exec} ./dcos marathon app show /web | jq .tasksRunning) - if [[ "${running}" == "3" ]]; then - log "Found 3 running tasks" - break - fi - sleep ${count} -done - -if [[ "${running}" != "3" ]]; then - log "marathon validation failed" - ${remote_exec} ./dcos marathon app show /web - ${remote_exec} ./dcos marathon app list - exit 1 -fi - -# install marathon-lb -${remote_exec} ./dcos package install marathon-lb --yes -if [[ "$?" != "0" ]]; then log "Failed to install marathon-lb"; exit 1; fi - -# curl simpleweb through external haproxy -log "Checking Service" -count=20 -while true; do - log " ... counting down $count" - rc=$(curl -sI --max-time 60 "http://${agentFQDN}" | head -n1 | cut -d$' ' -f2) - [[ "$rc" -eq "200" ]] && log "Successfully hitting simpleweb through external haproxy http://${agentFQDN}" && break - if [[ "${count}" -le 1 ]]; then - log "failed to get expected response from nginx through the loadbalancer: Error $rc" - exit 1 - fi - sleep 15; count=$((count-1)) -done diff --git a/test/cluster-tests/kubernetes/k8s-utils.sh b/test/cluster-tests/kubernetes/k8s-utils.sh deleted file mode 100644 index 35ee1787a..000000000 --- a/test/cluster-tests/kubernetes/k8s-utils.sh +++ /dev/null @@ -1,183 +0,0 @@ -#!/bin/bash - -function test_linux_deployment() { - ###### Testing an nginx deployment - log "Testing deployments" - k create namespace ${namespace} - - NGINX="docker.io/library/nginx:latest" - IMAGE="${NGINX}" # default to the library image unless we're in TEST_ACR mode - if [[ "${TEST_ACR}" == "y" ]]; then - # force it to pull from ACR - IMAGE="${ACR_REGISTRY}/test/nginx:latest" - # wait for acr - wait - # TODO: how to do this without polluting user home dir? - docker login --username="${SERVICE_PRINCIPAL_CLIENT_ID}" --password="${SERVICE_PRINCIPAL_CLIENT_SECRET}" "${ACR_REGISTRY}" - docker pull "${NGINX}" - docker tag "${NGINX}" "${IMAGE}" - docker push "${IMAGE}" - fi - - k run --image="${IMAGE}" nginx --namespace=${namespace} --overrides='{ "apiVersion": "extensions/v1beta1", "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"linux"}}}}}' - count=12 - while (( $count > 0 )); do - log " ... counting down $count" - running=$(k get pods --namespace=${namespace} | grep nginx | grep Running | wc | awk '{print $1}') - if (( ${running} == 1 )); then break; fi - sleep 5; count=$((count-1)) - done - if (( ${running} != 1 )); then - log "K8S-Linux: gave up waiting for deployment" - k get all --namespace=${namespace} - exit 1 - fi - - k expose deployments/nginx --type=LoadBalancer --namespace=${namespace} --port=80 - - log "Checking Service External IP" - count=60 - external_ip="" - while (( $count > 0 )); do - log " ... counting down $count" - external_ip=$(k get svc --namespace ${namespace} nginx --template="{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}" || echo "") - [[ -n "${external_ip}" ]] && break - sleep 10; count=$((count-1)) - done - if [[ -z "${external_ip}" ]]; then - log "K8S-Linux: gave up waiting for loadbalancer to get an ingress ip" - exit 1 - fi - - log "Checking Service" - count=5 - success="n" - while (( $count > 0 )); do - log " ... counting down $count" - ret=$(curl -f --max-time 60 "http://${external_ip}" | grep 'Welcome to nginx!' || echo "curl_error") - if [[ $ret =~ .*'Welcome to nginx!'.* ]]; then - success="y" - break - fi - sleep 5; count=$((count-1)) - done - if [[ "${success}" != "y" ]]; then - log "K8S-Linux: failed to get expected response from nginx through the loadbalancer" - exit 1 - fi -} - -function test_windows_deployment() { - ###### Testing a simpleweb windows deployment - log "Testing Windows deployments" - - log "Creating simpleweb service" - k apply -f "$DIR/simpleweb-windows.yaml" - count=90 - while (( $count > 0 )); do - log " ... counting down $count" - running=$(k get pods --namespace=default | grep win-webserver | grep Running | wc | awk '{print $1}') - if (( ${running} == 1 )); then break; fi - sleep 10; count=$((count-1)) - done - if (( ${running} != 1 )); then - log "K8S-Windows: gave up waiting for deployment" - k get all --namespace=default - exit 1 - fi - - log "Checking Service External IP" - count=60 - external_ip="" - while (( $count > 0 )); do - log " ... counting down $count" - external_ip=$(k get svc --namespace default win-webserver --template="{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}" || echo "") - [[ -n "${external_ip}" ]] && break - sleep 10; count=$((count-1)) - done - if [[ -z "${external_ip}" ]]; then - log "K8S-Windows: gave up waiting for loadbalancer to get an ingress ip" - exit 1 - fi - - log "Checking Service" - count=5 - success="n" - while (( $count > 0 )); do - log " ... counting down $count" - ret=$(curl -f --max-time 60 "http://${external_ip}" | grep 'Windows Container Web Server' || echo "curl_error") - if [[ $ret =~ .*'Windows Container Web Server'.* ]]; then - success="y" - break - fi - sleep 10; count=$((count-1)) - done - if [[ "${success}" != "y" ]]; then - log "K8S-Windows: failed to get expected response from simpleweb through the loadbalancer" - exit 1 - fi - - log "Checking outbound connection" - count=10 - while (( $count > 0 )); do - log " ... counting down $count" - winpodname=$(k get pods --namespace=default | grep win-webserver | awk '{print $1}') - [[ -n "${winpodname}" ]] && break - sleep 10; count=$((count-1)) - done - if [[ -z "${winpodname}" ]]; then - log "K8S-Windows: failed to get expected pod name for simpleweb" - exit 1 - fi - - log "query DNS" - count=0 # disabled while outbound connection bug is present - success="y" # disabled while outbound connection bug is present - while (( $count > 0 )); do - log " ... counting down $count" - query=$(k exec $winpodname -- powershell nslookup www.bing.com) - if echo ${query} | grep -q "DNS request timed out" && echo ${query} | grep -q "UnKnown"; then - success="y" - break - fi - sleep 10; count=$((count-1)) - done - - # temporarily disable breaking on errors to allow the retry - set +e - log "curl external website" - count=0 # disabled while outbound connection bug is present - success="y" # disabled while outbound connection bug is present - while (( $count > 0 )); do - log " ... counting down $count" - # curl without getting status first and see the response. getting status sometimes has the problem to hang - # and it doesn't repro when running k from the node - k exec $winpodname -- powershell iwr -UseBasicParsing -TimeoutSec 60 www.bing.com - statuscode=$(k exec $winpodname -- powershell iwr -UseBasicParsing -TimeoutSec 60 www.bing.com | grep StatusCode) - if [[ ${statuscode} != "" ]] && [[ $(echo ${statuscode} | grep 200 | awk '{print $3}' | tr -d '\r') -eq "200" ]]; then - log "got 200 status code" - log "${statuscode}" - success="y" - break - fi - log "curl failed, retrying..." - ipconfig=$(k exec $winpodname -- powershell ipconfig /all) - log "$ipconfig" - # TODO: reduce sleep time when outbound connection delay is fixed - sleep 100; count=$((count-1)) - done - set -e - if [[ "${success}" != "y" ]]; then - nslookup=$(k exec $winpodname -- powershell nslookup www.bing.com) - log "$nslookup" - log "getting the last 50 events to check timeout failure" - hdr=$(k get events | head -n 1) - log "$hdr" - evt=$(k get events | tail -n 50) - log "$evt" - log "K8S-Windows: failed to get outbound internet connection inside simpleweb container" - exit 1 - else - log "outbound connection succeeded!" - fi -} diff --git a/test/cluster-tests/kubernetes/simpleweb-windows.yaml b/test/cluster-tests/kubernetes/simpleweb-windows.yaml deleted file mode 100644 index f9e5d0aa2..000000000 --- a/test/cluster-tests/kubernetes/simpleweb-windows.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: win-webserver - labels: - app: win-webserver -spec: - ports: - # the port that this service should serve on - - port: 80 - targetPort: 80 - selector: - app: win-webserver - type: LoadBalancer ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - labels: - app: win-webserver - name: win-webserver -spec: - replicas: 1 - template: - metadata: - labels: - app: win-webserver - name: win-webserver - spec: - containers: - - name: windowswebserver - image: microsoft/windowsservercore:1803 - command: - - powershell.exe - - -command - - "<#code used from https://gist.github.com/wagnerandrade/5424431#> ; $$listener = New-Object System.Net.HttpListener ; $$listener.Prefixes.Add('http://*:80/') ; $$listener.Start() ; $$callerCounts = @{} ; Write-Host('Listening at http://*:80/') ; while ($$listener.IsListening) { ;$$context = $$listener.GetContext() ;$$requestUrl = $$context.Request.Url ;$$clientIP = $$context.Request.RemoteEndPoint.Address ;$$response = $$context.Response ;Write-Host '' ;Write-Host('> {0}' -f $$requestUrl) ; ;$$count = 1 ;$$k=$$callerCounts.Get_Item($$clientIP) ;if ($$k -ne $$null) { $$count += $$k } ;$$callerCounts.Set_Item($$clientIP, $$count) ;$$ip=(Get-NetAdapter | Get-NetIpAddress); $$header='

Windows Container Web Server

' ;$$callerCountsString='' ;$$callerCounts.Keys | % { $$callerCountsString+='

IP {0} callerCount {1} ' -f $$ip[1].IPAddress,$$callerCounts.Item($$_) } ;$$footer='' ;$$content='{0}{1}{2}' -f $$header,$$callerCountsString,$$footer ;Write-Output $$content ;$$buffer = [System.Text.Encoding]::UTF8.GetBytes($$content) ;$$response.ContentLength64 = $$buffer.Length ;$$response.OutputStream.Write($$buffer, 0, $$buffer.Length) ;$$response.Close() ;$$responseStatus = $$response.StatusCode ;Write-Host('< {0}' -f $$responseStatus) } ; " - nodeSelector: - beta.kubernetes.io/os: windows \ No newline at end of file diff --git a/test/cluster-tests/kubernetes/test.sh b/test/cluster-tests/kubernetes/test.sh deleted file mode 100755 index 9e734b2ab..000000000 --- a/test/cluster-tests/kubernetes/test.sh +++ /dev/null @@ -1,219 +0,0 @@ -#!/bin/bash - -#################################################### -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -#################################################### - -# exit on errors -set -e -# exit on unbound variables -set -u -# verbose logging -set -x - -source "$DIR/../utils.sh" -source "$DIR/k8s-utils.sh" - -ENV_FILE="${CLUSTER_DEFINITION}.env" -if [ -e "${ENV_FILE}" ]; then - source "${ENV_FILE}" -fi - -EXPECTED_NODE_COUNT="${EXPECTED_NODE_COUNT:-4}" -EXPECTED_LINUX_AGENTS="${EXPECTED_LINUX_AGENTS:-3}" -EXPECTED_WINDOWS_AGENTS="${EXPECTED_WINDOWS_AGENTS:-0}" -EXPECTED_DNS="${EXPECTED_DNS:-2}" -EXPECTED_DASHBOARD="${EXPECTED_DASHBOARD:-1}" -EXPECTED_RESCHEDULER="${EXPECTED_RESCHEDULER:-0}" -EXPECTED_ORCHESTRATOR_VERSION="${EXPECTED_ORCHESTRATOR_VERSION:-}" - -KUBE_PROXY_COUNT=$((EXPECTED_NODE_COUNT-$EXPECTED_WINDOWS_AGENTS)) - -# set TEST_ACR to "y" for ACR testing -TEST_ACR="${TEST_ACR:-n}" - -namespace="namespace-${RANDOM}" -log "Running test in namespace: ${namespace}" -trap teardown EXIT - -function teardown { - k get all --all-namespaces || echo "teardown error" - k get nodes || echo "teardown error" - k get namespaces || echo "teardown error" - k delete namespaces ${namespace} || echo "teardown error" -} - -# TODO: cleanup the loops more -# TODO: the wc|awk business can just be kubectl with an output format and wc -l - -###### Deploy ACR -if [[ "${TEST_ACR}" == "y" ]]; then - ACR_NAME="${INSTANCE_NAME//[-._]/}1" - ACR_REGISTRY="${ACR_NAME}-microsoft.azurecr.io" # fix this for non-ms tenant users - if ! az acr show --resource-group "${RESOURCE_GROUP}" --name "${ACR_NAME}" ; then - az acr create --location "${LOCATION}" --resource-group "${RESOURCE_GROUP}" --name "${ACR_NAME}" & - fi -fi - -###### Check node count -function check_node_count() { - log "Checking node count" - count=120 - while (( $count > 0 )); do - log " ... counting down $count" - node_count=$(k get nodes --no-headers | grep -v NotReady | grep Ready | wc | awk '{print $1}') - if (( ${node_count} == ${EXPECTED_NODE_COUNT} )); then break; fi - sleep 5; count=$((count-1)) - done - if (( $node_count != ${EXPECTED_NODE_COUNT} )); then - log "K8S: gave up waiting for apiserver / node counts"; exit 1 - fi -} - -check_node_count - -###### Validate Kubernetes version -kubernetes_version=$(k version --short) -DASHBOARD_PORT=80 -if [[ ${kubernetes_version} == *"Server Version: v1.9."* ]]; then - DASHBOARD_PORT=443 -fi -log "Setting dashboard port to ${DASHBOARD_PORT}" - -log "Checking Kubernetes version. Expected: ${EXPECTED_ORCHESTRATOR_VERSION}" -if [ -n "${EXPECTED_ORCHESTRATOR_VERSION}" ]; then - if [[ ${kubernetes_version} != *"Server Version: v${EXPECTED_ORCHESTRATOR_VERSION}"* ]]; then - log "K8S: unexpected kubernetes version:\n${kubernetes_version}"; exit 1 - fi -fi - -###### Wait for no more container creating -log "Checking containers being created" -count=60 -while (( $count > 0 )); do - log " ... counting down $count" - creating_count=$(k get nodes --no-headers | grep 'CreatingContainer' | wc | awk '{print $1}') - if (( ${creating_count} == 0 )); then break; fi - sleep 5; count=$((count-1)) -done -if (( ${creating_count} != 0 )); then - log "K8S: gave up waiting for containers"; exit 1 -fi - -###### Check existence and status of essential pods - -# we test other essential pods (kube-dns, kube-proxy) separately -pods="heapster kube-addon-manager kube-apiserver kube-controller-manager kube-scheduler tiller" -if (( ${EXPECTED_RESCHEDULER} != 0 )); then - pods="$pods rescheduler" -fi -log "Checking $pods" - -count=60 -while (( $count > 0 )); do - for pod in $pods; do - if k get pods --all-namespaces | grep $pod | grep -q Running; then - log "... $pod is Running" - pods=${pods/$pod/} - fi - done - if [ -z "$(echo $pods | tr -d '[:space:]')" ]; then - break - fi - sleep 5; count=$((count-1)) -done - -if [ -n "$(echo $pods | tr -d '[:space:]')" ]; then - log "K8S: gave up waiting for running pods [$pods]"; exit 1 -fi - -###### Check for Kube-DNS -log "Checking Kube-DNS" -count=60 -while (( $count > 0 )); do - log " ... counting down $count" - running=$(k get pods --namespace=kube-system | grep kube-dns | grep Running | wc | awk '{print $1}') - if (( ${running} == ${EXPECTED_DNS} )); then break; fi - sleep 5; count=$((count-1)) -done -if (( ${running} != ${EXPECTED_DNS} )); then - log "K8S: gave up waiting for kube-dns"; exit 1 -fi - -###### Check for Kube-Dashboard -if (( ${EXPECTED_DASHBOARD} != 0 )); then - log "Checking Kube-Dashboard" - count=60 - while (( $count > 0 )); do - log " ... counting down $count" - running=$(k get pods --namespace=kube-system | grep kubernetes-dashboard | grep Running | wc | awk '{print $1}') - if (( ${running} == ${EXPECTED_DASHBOARD} )); then break; fi - sleep 5; count=$((count-1)) - done - if (( ${running} != ${EXPECTED_DASHBOARD} )); then - log "K8S: gave up waiting for kubernetes-dashboard"; exit 1 - fi -else - log "Expecting no dashboard" -fi - -###### Check for Kube-Proxys -log "Checking Kube-Proxys" -count=60 -while (( $count > 0 )); do - log " ... counting down $count" - running=$(k get pods --namespace=kube-system | grep kube-proxy | grep Running | wc | awk '{print $1}') - if (( ${running} == ${KUBE_PROXY_COUNT} )); then break; fi - sleep 5; count=$((count-1)) -done -if (( ${running} != ${KUBE_PROXY_COUNT} )); then - log "K8S: gave up waiting for kube-proxy"; exit 1 -fi - -if ! [ $EXPECTED_WINDOWS_AGENTS -gt 0 ] ; then - if (( ${EXPECTED_DASHBOARD} != 0 )); then - # get master public hostname - master=$(k config view | grep server | cut -f 3- -d "/" | tr -d " ") - # get dashboard port - port=$(k get svc --namespace=kube-system | grep dashboard | awk '{print $4}' | sed -n 's/^'${DASHBOARD_PORT}':\(.*\)\/TCP$/\1/p') - # get internal IPs of the nodes - ips=$(k get nodes --all-namespaces -o yaml | grep -B 1 InternalIP | grep address | awk '{print $3}') - - for ip in $ips; do - log "Probing IP address ${ip}" - count=60 - success="n" - while (( $count > 0 )); do - log " ... counting down $count" - ret=$(ssh -i "${OUTPUT}/id_rsa" -o ConnectTimeout=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "azureuser@${master}" "curl --max-time 60 http://${ip}:${port}" || echo "curl_error") - if [[ ! $ret =~ .*curl_error.* ]]; then - success="y" - break - fi - if (( $count < 2 )); then - log $ret - fi - sleep 5; count=$((count-1)) - done - if [[ "${success}" == "n" ]]; then - log "K8S: gave up verifying proxy"; exit 1 - fi - done - fi -fi - -if [ $EXPECTED_LINUX_AGENTS -gt 0 ] ; then - test_linux_deployment -fi - -if [ $EXPECTED_WINDOWS_AGENTS -gt 0 ] ; then - test_windows_deployment -fi - -check_node_count diff --git a/test/cluster-tests/swarm/test.sh b/test/cluster-tests/swarm/test.sh deleted file mode 100755 index a0ce6dce8..000000000 --- a/test/cluster-tests/swarm/test.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -#################################################### -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -#################################################### - -# do not use 'set -e' -set -x -set -u - -source "$DIR/../utils.sh" - -SSH="ssh -i ${SSH_KEY} -o ConnectTimeout=30 -o StrictHostKeyChecking=no -p2200 azureuser@${INSTANCE_NAME}.${LOCATION}.cloudapp.azure.com" - -deploy="docker -H :2375 run -d -p 80:80 yeasy/simple-web" -wait_duration=10 -total_loops=30 -while true; do - # || true is used to suppress the failure like "Error response from daemon: No elected primary cluster manager" - # it should be gone after a few retries - containerId="$($SSH $deploy 2>/dev/null )" || true - [[ ! -z $containerId ]] && [[ "$(echo $containerId | grep '[0-9a-z]\{64\}')" ]] && log "container deployed! containerId is $containerId" && break - log "Validation: Expected to get containerId. $(($total_loops*$wait_duration)) seconds remain" - sleep $wait_duration - total_loops=$((total_loops-1)) - if [ $total_loops -eq 0 ]; then - log "Swarm validation failed: timeout"; exit 1; - fi -done - -result=$($SSH curl localhost:2375/containers/json | jq "[.[].Id==\"$containerId\"] | any") - -if [ "$result" != "true" ]; then - log "Swarm validation failed: container not found"; exit 1; -fi - -log "Swarm validation completed" diff --git a/test/cluster-tests/swarmmode/test.sh b/test/cluster-tests/swarmmode/test.sh deleted file mode 100755 index 38ce4d598..000000000 --- a/test/cluster-tests/swarmmode/test.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -#################################################### -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -#################################################### - -# do not use 'set -e' -set -x -set -u - - -source "$DIR/../utils.sh" - -ssh_args="-i ${SSH_KEY} -o ConnectTimeout=30 -o StrictHostKeyChecking=no -p2200 azureuser@${INSTANCE_NAME}.${LOCATION}.cloudapp.azure.com" - -function teardown { - ssh ${ssh_args} docker service rm nginx - sleep 10 - ssh ${ssh_args} docker network rm network -} - -trap teardown EXIT - -log "Starting swarmmode deployment validation in ${LOCATION}" -sleep 30 -log "Creating network" -wait=10 -count=12 -args="${ssh_args} docker network create --driver overlay --subnet 10.0.9.0/24 --opt encrypted network" -while (( $count > 0 )); do - log " ... counting down $count" - timeout -k 60s -s KILL 60s ssh $args - retval=$? - if [[ "$retval" == "0" ]]; then break; fi - sleep $wait - count=$((count-1)) -done -if [[ "$retval" != "0" ]]; then - log "DockerCE: gave up waiting for network to be created" - exit 1 -fi - -log "Creating service" -wait=5 -count=12 -args="${ssh_args} docker service create --replicas 3 --name nginx --network network --publish 80:80 nginx" -while (( $count > 0 )); do - log " ... counting down $count" - ssh $args - retval=$? - if [[ "$retval" == "0" ]]; then break; fi - sleep $wait - count=$((count-1)) -done -if [[ "$retval" != "0" ]]; then - log "DockerCE: gave up waiting for service to be created" - exit 1 -fi - -sleep 10 -log "Testing service" -wait=5 -count=12 -while (( $count > 0 )); do - log " ... counting down $count" - curl --fail "http://${INSTANCE_NAME}0.${LOCATION}.cloudapp.azure.com:80/" - retval=$? - if [[ "$retval" == "0" ]]; then break; fi - sleep $wait - count=$((count-1)) -done -if [[ "$retval" != "0" ]]; then - log "DockerCE: gave up waiting for service to be externally reachable" - exit 1 -fi diff --git a/test/cluster-tests/utils.sh b/test/cluster-tests/utils.sh deleted file mode 100644 index fa3090161..000000000 --- a/test/cluster-tests/utils.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -function log { - local message="$1" - local caller - caller="$(caller 0)" - now=$(date +"%D %T %Z") - - if [[ -n "${LOGFILE:-}" ]]; then - echo "[${now}] [${caller}] ${message}" | tee -a ${LOGFILE} - else - echo "[${now}] [${caller}] ${message}" - fi -} diff --git a/test/common.sh b/test/common.sh deleted file mode 100755 index bfe37f3f9..000000000 --- a/test/common.sh +++ /dev/null @@ -1,219 +0,0 @@ -#!/bin/bash - -#################################################### -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -#################################################### - -ROOT="${DIR}/.." - -# see: https://github.com/stedolan/jq/issues/105 & https://github.com/stedolan/jq/wiki/FAQ#general-questions -function jqi() { filename="${1}"; jqexpr="${2}"; jq "${jqexpr}" "${filename}" > "${filename}.tmp" && mv "${filename}.tmp" "${filename}"; } - -function generate_template() { - # Check pre-requisites - [[ -n "${INSTANCE_NAME:-}" ]] || (echo "Must specify INSTANCE_NAME" >&2 && exit 1) - [[ -n "${CLUSTER_DEFINITION:-}" ]] || (echo "Must specify CLUSTER_DEFINITION" >&2 && exit 1) - [[ -n "${SERVICE_PRINCIPAL_CLIENT_ID:-}" ]] || [[ -n "${CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID:-}" ]] || (echo "Must specify SERVICE_PRINCIPAL_CLIENT_ID" >&2 && exit 1) - [[ -n "${SERVICE_PRINCIPAL_CLIENT_SECRET:-}" ]] || [[ -n "${CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET:-}" ]] || (echo "Must specify SERVICE_PRINCIPAL_CLIENT_SECRET" >&2 && exit 1) - [[ -n "${OUTPUT:-}" ]] || (echo "Must specify OUTPUT" >&2 && exit 1) - - # Set output directory - mkdir -p "${OUTPUT}" - - # Prep SSH Key - ssh-keygen -b 2048 -t rsa -f "${OUTPUT}/id_rsa" -q -N "" - ssh-keygen -y -f "${OUTPUT}/id_rsa" > "${OUTPUT}/id_rsa.pub" - SSH_KEY_DATA="$(cat "${OUTPUT}/id_rsa.pub")" - export SSH_KEY_DATA - - # Allow different credentials for cluster vs the deployment - export CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID="${CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID:-${SERVICE_PRINCIPAL_CLIENT_ID}}" - export CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET="${CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET:-${SERVICE_PRINCIPAL_CLIENT_SECRET}}" - - # Form the final cluster_definition file - export FINAL_CLUSTER_DEFINITION="${OUTPUT}/clusterdefinition.json" - cp "${CLUSTER_DEFINITION}" "${FINAL_CLUSTER_DEFINITION}" - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.masterProfile.dnsPrefix = \"${INSTANCE_NAME}\"" - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.agentPoolProfiles |= map(if .name==\"agentpublic\" then .dnsPrefix = \"${INSTANCE_NAME}0\" else . end)" - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.linuxProfile.ssh.publicKeys[0].keyData = \"${SSH_KEY_DATA}\"" - - k8sServicePrincipal=$(jq 'getpath(["properties","servicePrincipalProfile"])' ${FINAL_CLUSTER_DEFINITION}) - if [[ "${k8sServicePrincipal}" != "null" ]]; then - apiVersion=$(get_api_version) - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.servicePrincipalProfile.clientId = \"${CLUSTER_SERVICE_PRINCIPAL_CLIENT_ID}\"" - if [[ ${CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET} =~ /subscription.* ]]; then - vaultID=$(echo $CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET | awk -F"/secrets/" '{print $1}') - secretName=$(echo $CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET | awk -F"/secrets/" '{print $2}') - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.servicePrincipalProfile.keyvaultSecretRef.vaultID = \"${vaultID}\"" - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.servicePrincipalProfile.keyvaultSecretRef.secretName = \"${secretName}\"" - else - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.servicePrincipalProfile.secret = \"${CLUSTER_SERVICE_PRINCIPAL_CLIENT_SECRET}\"" - fi - fi - - secrets=$(jq 'getpath(["properties","linuxProfile","secrets"])' ${FINAL_CLUSTER_DEFINITION}) - if [[ "${secrets}" != "null" ]]; then - [[ -n "${CERT_KEYVAULT_ID:-}" ]] || (echo "Must specify CERT_KEYVAULT_ID" >&2 && exit 1) - [[ -n "${CERT_SECRET_URL:-}" ]] || (echo "Must specify CERT_SECRET_URL" >&2 && exit 1) - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.linuxProfile.secrets[0].sourceVault.id = \"${CERT_KEYVAULT_ID}\"" - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.linuxProfile.secrets[0].vaultCertificates[0].certificateUrl = \"${CERT_SECRET_URL}\"" - fi - secrets=$(jq 'getpath(["properties","windowsProfile","secrets"])' ${FINAL_CLUSTER_DEFINITION}) - if [[ "${secrets}" != "null" ]]; then - [[ -n "${CERT_KEYVAULT_ID:-}" ]] || (echo "Must specify CERT_KEYVAULT_ID" >&2 && exit 1) - [[ -n "${CERT_SECRET_URL:-}" ]] || (echo "Must specify CERT_SECRET_URL" >&2 && exit 1) - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.windowsProfile.secrets[0].sourceVault.id = \"${CERT_KEYVAULT_ID}\"" - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.windowsProfile.secrets[0].vaultCertificates[0].certificateUrl = \"${CERT_SECRET_URL}\"" - jqi "${FINAL_CLUSTER_DEFINITION}" ".properties.windowsProfile.secrets[0].vaultCertificates[0].certificateStore = \"My\"" - fi - # Generate template - "${DIR}/../bin/aks-engine" generate --output-directory "${OUTPUT}" "${FINAL_CLUSTER_DEFINITION}" --debug - - # Fill in custom hyperkube spec, if it was set - if [[ -n "${CUSTOM_HYPERKUBE_SPEC:-}" ]]; then - # TODO: plumb hyperkube into the apimodel - jqi "${OUTPUT}/azuredeploy.parameters.json" ".parameters.kubernetesHyperkubeSpec.value = \"${CUSTOM_HYPERKUBE_SPEC}\"" - fi -} - -function set_azure_account() { - # Check pre-requisites - [[ -n "${SUBSCRIPTION_ID:-}" ]] || (echo "Must specify SUBSCRIPTION_ID" >&2 && exit 1) - [[ -n "${TENANT_ID:-}" ]] || (echo "Must specify TENANT_ID" >&2 && exit 1) - [[ -n "${SERVICE_PRINCIPAL_CLIENT_ID:-}" ]] || (echo "Must specify SERVICE_PRINCIPAL_CLIENT_ID" >&2 && exit 1) - [[ -n "${SERVICE_PRINCIPAL_CLIENT_SECRET:-}" ]] || (echo "Must specify SERVICE_PRINCIPAL_CLIENT_SECRET" >&2 && exit 1) - command -v k || (echo "k must be on PATH" >&2 && exit 1) - command -v az || (echo "az must be on PATH" >&2 && exit 1) - - # Login to Azure-Cli - az login --service-principal \ - --username "${SERVICE_PRINCIPAL_CLIENT_ID}" \ - --password "${SERVICE_PRINCIPAL_CLIENT_SECRET}" \ - --tenant "${TENANT_ID}" &>/dev/null - - az account set --subscription "${SUBSCRIPTION_ID}" -} - -function create_resource_group() { - [[ -n "${LOCATION:-}" ]] || (echo "Must specify LOCATION" >&2 && exit 1) - [[ -n "${RESOURCE_GROUP:-}" ]] || (echo "Must specify RESOURCE_GROUP" >&2 && exit 1) - - # Create resource group if doesn't exist - az group show --name="${RESOURCE_GROUP}" || [ $? -eq 3 ] && echo "will create resource group ${RESOURCE_GROUP}" || exit 1 - az group create --name="${RESOURCE_GROUP}" --location="${LOCATION}" --tags "type=${RESOURCE_GROUP_TAG_TYPE:-}" "now=$(date +%s)" "job=${JOB_BASE_NAME:-}" "buildno=${BUILD_NUM:-}" - sleep 3 # TODO: investigate why this is needed (eventual consistency in ARM) -} - -function deploy_template() { - # Check pre-requisites - [[ -n "${DEPLOYMENT_NAME:-}" ]] || (echo "Must specify DEPLOYMENT_NAME" >&2 && exit 1) - [[ -n "${LOCATION:-}" ]] || (echo "Must specify LOCATION" >&2 && exit 1) - [[ -n "${RESOURCE_GROUP:-}" ]] || (echo "Must specify RESOURCE_GROUP" >&2 && exit 1) - [[ -n "${OUTPUT:-}" ]] || (echo "Must specify OUTPUT" >&2 && exit 1) - - command -v k || (echo "k must be on PATH" >&2 && exit 1) - command -v az || (echo "az must be on PATH" >&2 && exit 1) - - create_resource_group - - # Deploy the template - az group deployment create \ - --name "${DEPLOYMENT_NAME}" \ - --resource-group "${RESOURCE_GROUP}" \ - --template-file "${OUTPUT}/azuredeploy.json" \ - --parameters "@${OUTPUT}/azuredeploy.parameters.json" -} - -function scale_agent_pool() { - # Check pre-requisites - [[ -n "${AGENT_POOL_SIZE:-}" ]] || (echo "Must specify AGENT_POOL_SIZE" >&2 && exit 1) - [[ -n "${DEPLOYMENT_NAME:-}" ]] || (echo "Must specify DEPLOYMENT_NAME" >&2 && exit 1) - [[ -n "${LOCATION:-}" ]] || (echo "Must specify LOCATION" >&2 && exit 1) - [[ -n "${RESOURCE_GROUP:-}" ]] || (echo "Must specify RESOURCE_GROUP" >&2 && exit 1) - [[ -n "${OUTPUT:-}" ]] || (echo "Must specify OUTPUT" >&2 && exit 1) - - command -v az || (echo "az must be on PATH" >&2 && exit 1) - - APIMODEL="${OUTPUT}/apimodel.json" - DEPLOYMENT_PARAMS="${OUTPUT}/azuredeploy.parameters.json" - - for poolname in $(jq '.properties.agentPoolProfiles[].name' "${APIMODEL}" | tr -d '\"'); do - offset=$(jq "getpath([\"parameters\", \"${poolname}Count\", \"value\"])" ${DEPLOYMENT_PARAMS}) - echo "$poolname : offset=$offset count=$AGENT_POOL_SIZE" - jqi "${DEPLOYMENT_PARAMS}" ".${poolname}Count.value = $AGENT_POOL_SIZE" - jqi "${DEPLOYMENT_PARAMS}" ".${poolname}Offset.value = $offset" - done - - az group deployment create \ - --name "${DEPLOYMENT_NAME}" \ - --resource-group "${RESOURCE_GROUP}" \ - --template-file "${OUTPUT}/azuredeploy.json" \ - --parameters "@${OUTPUT}/azuredeploy.parameters.json" -} - -function get_node_count() { - [[ -n "${CLUSTER_DEFINITION:-}" ]] || (echo "Must specify CLUSTER_DEFINITION" >&2 && exit 1) - - count=$(jq '.properties.masterProfile.count' ${CLUSTER_DEFINITION}) - linux_agents=0 - windows_agents=0 - - nodes=$(jq -r '.properties.agentPoolProfiles[].count' ${CLUSTER_DEFINITION}) - osTypes=$(jq -r '.properties.agentPoolProfiles[].osType' ${CLUSTER_DEFINITION}) - - nArr=( $nodes ) - oArr=( $osTypes ) - indx=0 - for n in "${nArr[@]}"; do - count=$((count+n)) - if [ "${oArr[$indx]}" = "Windows" ]; then - windows_agents=$((windows_agents+n)) - else - linux_agents=$((linux_agents+n)) - fi - indx=$((indx+1)) - done - echo "${count}:${linux_agents}:${windows_agents}" -} - -function get_orchestrator_type() { - [[ -n "${CLUSTER_DEFINITION:-}" ]] || (echo "Must specify CLUSTER_DEFINITION" >&2 && exit 1) - - orchestratorType=$(jq -r 'getpath(["properties","orchestratorProfile","orchestratorType"])' ${CLUSTER_DEFINITION} | tr '[:upper:]' '[:lower:]') - - echo $orchestratorType -} - -function get_orchestrator_version() { - [[ -n "${CLUSTER_DEFINITION:-}" ]] || (echo "Must specify CLUSTER_DEFINITION" >&2 && exit 1) - - orchestratorVersion=$(jq -r 'getpath(["properties","orchestratorProfile","orchestratorVersion"])' ${CLUSTER_DEFINITION}) - if [[ "$orchestratorVersion" == "null" ]]; then - orchestratorVersion="" - fi - - echo $orchestratorVersion -} - -function get_api_version() { - [[ -n "${CLUSTER_DEFINITION:-}" ]] || (echo "Must specify CLUSTER_DEFINITION" >&2 && exit 1) - - apiVersion=$(jq -r 'getpath(["apiVersion"])' ${CLUSTER_DEFINITION}) - if [[ "$apiVersion" == "null" ]]; then - apiVersion="" - fi - - echo $apiVersion -} - -function cleanup() { - if [[ "${CLEANUP:-}" == "y" ]]; then - az group delete --no-wait --name="${RESOURCE_GROUP}" --yes || true - fi -} diff --git a/test/deploy.sh b/test/deploy.sh deleted file mode 100755 index 8a9f669bd..000000000 --- a/test/deploy.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash - -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - -############################################################################### - -set -e -set -u -set -o pipefail - -ROOT="${DIR}/.." - -# Usage: -# -# Manual user usage (Specific name): -# export INSTANCE_NAME=official-jenkins-infra -# ./scripts/deploy.sh ./examples/kubernetes.json -# -# Manual user usage (Lots of rapid fire): -# In this mode, the user can repeat the same deploy -# command blindly and get new clusters each time. -# unset INSTANCE_NAME -# vim ./test/user.env (add your stuff) -# ./scripts/deploy.sh ./examples.kubernetes.json -# sleep 1 -# ./scripts/deploy.sh ./examples.kubernetes.json -# -# Prow: -# export PULL_NUMBER=... -# export VALIDATE=