test: more resilient azure-arc-onboarding E2E (#4069)

test/e2e/cluster.sh

@@ -184,6 +184,7 @@ docker run --rm \
 -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
 -e ARC_SUBSCRIPTION_ID=${ARC_SUBSCRIPTION_ID:-$AZURE_SUBSCRIPTION_ID} \
 -e ARC_LOCATION=${ARC_LOCATION:-$LOCATION} \
+-e ARC_TENANT_ID=${ARC_TENANT_ID:-$AZURE_TENANT_ID} \
 -e LINUX_CONTAINERD_URL=${LINUX_CONTAINERD_URL} \
 -e WINDOWS_CONTAINERD_URL=${WINDOWS_CONTAINERD_URL} \
 -e VALIDATE_CPU_LOAD=${VALIDATE_CPU_LOAD} \
@@ -330,6 +331,7 @@ if [ -n "$ADD_NODE_POOL_INPUT" ]; then
     -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
     -e ARC_SUBSCRIPTION_ID=${ARC_SUBSCRIPTION_ID:-$AZURE_SUBSCRIPTION_ID} \
     -e ARC_LOCATION=${ARC_LOCATION:-$LOCATION} \
+    -e ARC_TENANT_ID=${ARC_TENANT_ID:-$AZURE_TENANT_ID} \
     ${DEV_IMAGE} make test-kubernetes || tryExit && renameResultsFile "add-node-pool"
 fi
 
@@ -447,6 +449,7 @@ if [ "${SCALE_CLUSTER}" = "true" ]; then
     -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
     -e ARC_SUBSCRIPTION_ID=${ARC_SUBSCRIPTION_ID:-$AZURE_SUBSCRIPTION_ID} \
     -e ARC_LOCATION=${ARC_LOCATION:-$LOCATION} \
+    -e ARC_TENANT_ID=${ARC_TENANT_ID:-$AZURE_TENANT_ID} \
     ${DEV_IMAGE} make test-kubernetes || tryExit && renameResultsFile "scale-down"
 fi
 
@@ -539,6 +542,7 @@ if [ "${UPGRADE_CLUSTER}" = "true" ]; then
       -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
       -e ARC_SUBSCRIPTION_ID=${ARC_SUBSCRIPTION_ID:-$AZURE_SUBSCRIPTION_ID} \
       -e ARC_LOCATION=${ARC_LOCATION:-$LOCATION} \
+      -e ARC_TENANT_ID=${ARC_TENANT_ID:-$AZURE_TENANT_ID} \
       ${DEV_IMAGE} make test-kubernetes || tryExit && renameResultsFile "upgrade"
   done
 fi
@@ -620,5 +624,6 @@ if [ "${SCALE_CLUSTER}" = "true" ]; then
     -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
     -e ARC_SUBSCRIPTION_ID=${ARC_SUBSCRIPTION_ID:-$AZURE_SUBSCRIPTION_ID} \
     -e ARC_LOCATION=${ARC_LOCATION:-$LOCATION} \
+    -e ARC_TENANT_ID=${ARC_TENANT_ID:-$AZURE_TENANT_ID} \
     ${DEV_IMAGE} make test-kubernetes || tryExit && renameResultsFile "scale-up"
 fi

test/e2e/config/config.go

@@ -59,7 +59,11 @@ type Config struct {
 	ClientID                string `envconfig:"CLIENT_ID"`
 	ClientSecret            string `envconfig:"CLIENT_SECRET"`
 	ValidateCPULoad         bool   `envconfig:"VALIDATE_CPU_LOAD" default:"false"`
-	*ArcOnboardingConfig
+	ArcClientID             string `envconfig:"ARC_CLIENT_ID" default:""`
+	ArcClientSecret         string `envconfig:"ARC_CLIENT_SECRET" default:""`
+	ArcSubscriptionID       string `envconfig:"ARC_SUBSCRIPTION_ID" default:""`
+	ArcLocation             string `envconfig:"ARC_LOCATION" default:""`
+	ArcTenantID             string `envconfig:"ARC_TENANT_ID" default:""`
 }
 
 // CustomCloudConfig holds configurations for custom cloud
@@ -86,15 +90,6 @@ type CustomCloudConfig struct {
 	KeyVaultEndpoint             string `envconfig:"KEY_VAULT_ENDPOINT"`
 }
 
-// ArcOnboardingConfig holds the azure arc onboarding addon configuration
-type ArcOnboardingConfig struct {
-	ClientID       string `envconfig:"ARC_CLIENT_ID" default:""`
-	ClientSecret   string `envconfig:"ARC_CLIENT_SECRET" default:""`
-	SubscriptionID string `envconfig:"ARC_SUBSCRIPTION_ID" default:""`
-	Location       string `envconfig:"ARC_LOCATION" default:""`
-	TenantID       string `envconfig:"TENANT_ID"`
-}
-
 const (
 	kubernetesOrchestrator = "kubernetes"
 )

test/e2e/engine/template.go

@@ -72,7 +72,11 @@ type Config struct {
 	LinuxContainerdURL             string `envconfig:"LINUX_CONTAINERD_URL"`
 	WindowsContainerdURL           string `envconfig:"WINDOWS_CONTAINERD_URL"`
 	WindowsProvisioningScriptsURL  string `envconfig:"WINDOWS_PROVISIONING_SCRIPTS_URL" default:""`
-	*ArcOnboardingConfig
+	ArcClientID                    string `envconfig:"ARC_CLIENT_ID" default:""`
+	ArcClientSecret                string `envconfig:"ARC_CLIENT_SECRET" default:""`
+	ArcSubscriptionID              string `envconfig:"ARC_SUBSCRIPTION_ID" default:""`
+	ArcLocation                    string `envconfig:"ARC_LOCATION" default:""`
+	ArcTenantID                    string `envconfig:"ARC_TENANT_ID" default:""`
 
 	ClusterDefinitionPath     string // The original template we want to use to build the cluster from.
 	ClusterDefinitionTemplate string // This is the template after we splice in the environment variables
@@ -90,15 +94,6 @@ type Engine struct {
 	ExpandedDefinition *api.ContainerService         // Holds the expanded ClusterDefinition
 }
 
-// ArcOnboardingConfig holds the azure arc onboarding addon configuration
-type ArcOnboardingConfig struct {
-	ClientID       string `envconfig:"ARC_CLIENT_ID" default:""`
-	ClientSecret   string `envconfig:"ARC_CLIENT_SECRET" default:""`
-	SubscriptionID string `envconfig:"ARC_SUBSCRIPTION_ID" default:""`
-	Location       string `envconfig:"ARC_LOCATION" default:""`
-	TenantID       string `envconfig:"TENANT_ID"`
-}
-
 // ParseConfig will return a new engine config struct taking values from env vars
 func ParseConfig(cwd, clusterDefinition, name string) (*Config, error) {
 	c := new(Config)
@@ -344,12 +339,40 @@ func Build(cfg *config.Config, masterSubnetID string, agentSubnetIDs []string, i
 				if addon.Config == nil {
 					addon.Config = make(map[string]string)
 				}
-				if cfg.ArcOnboardingConfig != nil {
+				if addon.Config["tenantID"] == "" {
-					addon.Config["tenantID"] = config.ArcOnboardingConfig.TenantID
+					if config.ArcTenantID != "" {
-					addon.Config["subscriptionID"] = config.ArcOnboardingConfig.SubscriptionID
+						addon.Config["tenantID"] = config.ArcTenantID
-					addon.Config["clientID"] = config.ArcOnboardingConfig.ClientID
+					} else {
-					addon.Config["clientSecret"] = config.ArcOnboardingConfig.ClientSecret
+						addon.Config["tenantID"] = config.TenantID
-					addon.Config["location"] = config.ArcOnboardingConfig.Location
+					}
+				}
+				if addon.Config["subscriptionID"] == "" {
+					if config.ArcSubscriptionID != "" {
+						addon.Config["subscriptionID"] = config.ArcSubscriptionID
+					} else {
+						addon.Config["subscriptionID"] = config.SubscriptionID
+					}
+				}
+				if addon.Config["clientID"] == "" {
+					if config.ArcClientID != "" {
+						addon.Config["clientID"] = config.ArcClientID
+					} else {
+						addon.Config["clientID"] = config.ClientID
+					}
+				}
+				if addon.Config["clientSecret"] == "" {
+					if config.ArcClientSecret != "" {
+						addon.Config["clientSecret"] = config.ArcClientSecret
+					} else {
+						addon.Config["clientSecret"] = config.ClientSecret
+					}
+				}
+				if addon.Config["location"] == "" {
+					if config.ArcLocation != "" {
+						addon.Config["location"] = config.ArcLocation
+					} else {
+						addon.Config["location"] = "eastus"
+					}
 				}
 				addon.Config["clusterName"] = cfg.Name
 				addon.Config["resourceGroup"] = fmt.Sprintf("%s-arc", cfg.Name) // set to config.Name once Arc is supported in all regions

test/e2e/runner.go

@@ -12,11 +12,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Azure/aks-engine/pkg/api/common"
 	"github.com/Azure/aks-engine/test/e2e/azure"
 	"github.com/Azure/aks-engine/test/e2e/config"
 	"github.com/Azure/aks-engine/test/e2e/engine"
 	"github.com/Azure/aks-engine/test/e2e/metrics"
 	"github.com/Azure/aks-engine/test/e2e/runner"
+	"github.com/Azure/go-autorest/autorest/to"
 )
 
 var (
@@ -292,9 +294,13 @@ func teardown() {
 			acct.DeleteGroup(rg, false)
 		}
 		// Delete once we reuse the cluster group for the connectedCluster resource
-		if cfg.ArcOnboardingConfig != nil {
+		for _, addon := range eng.ClusterDefinition.Properties.OrchestratorProfile.KubernetesConfig.Addons {
+			if addon.Name == common.AzureArcOnboardingAddonName && to.Bool(addon.Enabled) &&
+				addon.Config["resourceGroup"] != "" &&
+				addon.Config["location"] != "" {
 				log.Printf("Deleting Arc Group: %s\n", fmt.Sprintf("%s-arc", cfg.Name))
 				acct.DeleteGroup(fmt.Sprintf("%s-arc", cfg.Name), false)
 			}
 		}
 	}
+}

test/e2e/runner/cli_provisioner.go

@@ -456,7 +456,9 @@ func (cli *CLIProvisioner) FetchActivityLog(acct *azure.Account, logPath string)
 // https://docs.microsoft.com/en-us/azure/azure-arc/kubernetes/overview#supported-regions
 func (cli *CLIProvisioner) EnsureArcResourceGroup() error {
 	for _, addon := range cli.Engine.ClusterDefinition.Properties.OrchestratorProfile.KubernetesConfig.Addons {
-		if addon.Name == common.AzureArcOnboardingAddonName && to.Bool(addon.Enabled) {
+		if addon.Name == common.AzureArcOnboardingAddonName && to.Bool(addon.Enabled) &&
+			addon.Config["resourceGroup"] != "" &&
+			addon.Config["location"] != "" {
 			if err := cli.Account.CreateGroupWithRetry(addon.Config["resourceGroup"], addon.Config["location"], 30*time.Second, cli.Config.Timeout); err != nil {
 				return errors.Wrapf(err, "Error while trying to create Azure Arc resource group: %s", addon.Config["resourceGroup"])
 			}

test/e2e/test_cluster_configs/everything.json

@@ -68,12 +68,6 @@
 							"name": "azure-arc-onboarding",
 							"enabled": true,
 							"config": {
-								"tenantID": "",
-								"subscriptionID": "",
-								"resourceGroup": "",
-								"clusterName": "",
-								"clientID": "",
-								"clientSecret": "",
 								"location": "eastus"
 							}
 						}