From 773d5c8b87bf634c982183bbb981843ed19018a3 Mon Sep 17 00:00:00 2001 From: Bhuvaneswari Santharam <31025851+Bhuvaneswari-Santharam@users.noreply.github.com> Date: Thu, 30 May 2019 10:38:09 -0700 Subject: [PATCH] test: enable aks-engine e2e test on azurestack (#1397) * feat: AzureStack e2e changes * feat: AzureStack e2e changes * feat: AzureStack e2e changes * feat: AzureStack e2e changes * feat: AzureStack e2e changes * test: to enable aks-engine e2e test on azurestack * test: to enable aks-engine e2e test on azurestack * Update runner.go * Update kubernetes_suite_test.go * test: to enable aks-engine e2e test on azurestack * test: to enable aks-engine e2e test on azurestack --- test/e2e/config/config.go | 146 +++++++++++++++++++ test/e2e/engine/template.go | 8 +- test/e2e/kubernetes/kubernetes_suite_test.go | 10 +- test/e2e/runner.go | 17 +++ 4 files changed, 174 insertions(+), 7 deletions(-) diff --git a/test/e2e/config/config.go b/test/e2e/config/config.go index f8692fcf6..70ca820c1 100644 --- a/test/e2e/config/config.go +++ b/test/e2e/config/config.go @@ -5,6 +5,7 @@ package config import ( "bufio" + "encoding/json" "fmt" "io/ioutil" "log" @@ -15,6 +16,8 @@ import ( "strings" "time" + "github.com/Azure/aks-engine/pkg/api" + "github.com/Azure/aks-engine/pkg/api/vlabs" "github.com/Azure/aks-engine/test/e2e/kubernetes/util" "github.com/kelseyhightower/envconfig" ) @@ -41,6 +44,28 @@ type Config struct { GinkgoSkip string `envconfig:"GINKGO_SKIP"` } +// CustomCloudConfig holds configurations for custom clould +type CustomCloudConfig struct { + ServiceManagementEndpoint string `envconfig:"SERVICE_MANAGEMENT_ENDPOINT"` + ResourceManagerEndpoint string `envconfig:"RESOURCE_MANAGER_ENDPOINT"` + ActiveDirectoryEndpoint string `envconfig:"ACTIVE_DIRECTORY_ENDPOINT"` + GalleryEndpoint string `envconfig:"GALLERY_ENDPOINT"` + StorageEndpointSuffix string `envconfig:"STORAGE_ENDPOINT_SUFFIX"` + KeyVaultDNSSuffix string `envconfig:"KEY_VAULT_DNS_SUFFIX"` + GraphEndpoint string `envconfig:"GRAPH_ENDPOINT"` + ServiceManagementVMDNSSuffix string `envconfig:"SERVICE_MANAGEMENT_VM_DNS_SUFFIX"` + ResourceManagerVMDNSSuffix string `envconfig:"RESOURCE_MANAGER_VM_DNS_SUFFIX"` + IdentitySystem string `envconfig:"IDENTITY_SYSTEM"` + AuthenticationMethod string `envconfig:"AUTHENTICATION_METHOD"` + VaultID string `envconfig:"VAULT_ID"` + SecretName string `envconfig:"SECRET_NAME"` + CustomCloudClientID string `envconfig:"CUSTOM_CLOUD_CLIENT_ID"` + CustomCloudSecret string `envconfig:"CUSTOM_CLOUD_SECRET"` + APIProfile string `envconfig:"API_PROFILE"` + PortalURL string `envconfig:"PORTAL_ENDPOINT"` + TimeoutCommands bool +} + const ( kubernetesOrchestrator = "kubernetes" ) @@ -57,6 +82,15 @@ func ParseConfig() (*Config, error) { return c, nil } +// ParseCustomCloudConfig will parse needed environment variables for running the tests +func ParseCustomCloudConfig() (*CustomCloudConfig, error) { + ccc := new(CustomCloudConfig) + if err := envconfig.Process("customcloudconfig", ccc); err != nil { + return nil, err + } + return ccc, nil +} + // GetKubeConfig returns the absolute path to the kubeconfig for c.Location func (c *Config) GetKubeConfig() string { var kubeconfigPath string @@ -68,6 +102,118 @@ func (c *Config) GetKubeConfig() string { return kubeconfigPath } +// IsAzureStackCloud returns true if the cloud is AzureStack +func (c *Config) IsAzureStackCloud() bool { + clusterDefinitionFullPath := fmt.Sprintf("%s/%s", c.CurrentWorkingDir, c.ClusterDefinition) + cs := parseVlabsContainerSerice(clusterDefinitionFullPath) + return cs.Properties.IsAzureStackCloud() +} + +// UpdateCustomCloudClusterDefinition updates the cluster definition from environment variables +func (c *Config) UpdateCustomCloudClusterDefinition(ccc *CustomCloudConfig) error { + clusterDefinitionFullPath := fmt.Sprintf("%s/%s", c.CurrentWorkingDir, c.ClusterDefinition) + cs := parseVlabsContainerSerice(clusterDefinitionFullPath) + + cs.Location = c.Location + cs.Properties.CustomCloudProfile.PortalURL = ccc.PortalURL + cs.Properties.ServicePrincipalProfile.ClientID = ccc.CustomCloudClientID + cs.Properties.ServicePrincipalProfile.Secret = ccc.CustomCloudSecret + cs.Properties.CustomCloudProfile.AuthenticationMethod = ccc.AuthenticationMethod + cs.Properties.CustomCloudProfile.IdentitySystem = ccc.IdentitySystem + + if ccc.AuthenticationMethod == "client_certificate" { + cs.Properties.ServicePrincipalProfile.Secret = "" + cs.Properties.ServicePrincipalProfile.KeyvaultSecretRef = &vlabs.KeyvaultSecretRef{ + VaultID: ccc.VaultID, + SecretName: ccc.SecretName, + } + } + + csBytes, err := json.Marshal(cs) + if err != nil { + return fmt.Errorf("Error fail to marshal containerService object %p", err) + } + err = ioutil.WriteFile(clusterDefinitionFullPath, csBytes, 644) + if err != nil { + return fmt.Errorf("Error fail to write file object %p", err) + } + return nil +} + +func parseVlabsContainerSerice(clusterDefinitionFullPath string) api.VlabsARMContainerService { + + bytes, err := ioutil.ReadFile(clusterDefinitionFullPath) + if err != nil { + log.Fatalf("Error while trying to read cluster definition at (%s):%s\n", clusterDefinitionFullPath, err) + } + cs := api.VlabsARMContainerService{} + err = json.Unmarshal(bytes, &cs) + if err != nil { + log.Fatalf("Fail to unmarshal file %q , err - %q", clusterDefinitionFullPath, err) + } + return cs +} + +// SetEnvironment will set the cloud context +func (ccc *CustomCloudConfig) SetEnvironment() error { + var cmd *exec.Cmd + environmentName := fmt.Sprintf("AzureStack%v", time.Now().Unix()) + if ccc.TimeoutCommands { + cmd = exec.Command("timeout", "60", "az", "cloud", "register", + "-n", environmentName, + "--endpoint-resource-manager", ccc.ResourceManagerEndpoint, + "--suffix-storage-endpoint", ccc.StorageEndpointSuffix, + "--suffix-keyvault-dns", ccc.KeyVaultDNSSuffix, + "--endpoint-active-directory-resource-id", ccc.ServiceManagementEndpoint, + "--endpoint-active-directory", ccc.ActiveDirectoryEndpoint, + "--endpoint-active-directory-graph-resource-id", ccc.GraphEndpoint) + } else { + cmd = exec.Command("az", "cloud", "register", + "-n", environmentName, + "--endpoint-resource-manager", ccc.ResourceManagerEndpoint, + "--suffix-storage-endpoint", ccc.StorageEndpointSuffix, + "--suffix-keyvault-dns", ccc.KeyVaultDNSSuffix, + "--endpoint-active-directory-resource-id", ccc.ServiceManagementEndpoint, + "--endpoint-active-directory", ccc.ActiveDirectoryEndpoint, + "--endpoint-active-directory-graph-resource-id", ccc.GraphEndpoint) + } + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("output:%s\n", out) + return err + } + + if ccc.TimeoutCommands { + cmd = exec.Command("timeout", "60", "az", "cloud", "set", + "-n", environmentName) + + } else { + cmd = exec.Command("az", "cloud", "set", + "-n", environmentName) + } + out, err = cmd.CombinedOutput() + if err != nil { + log.Printf("output:%s\n", out) + return err + } + + if ccc.TimeoutCommands { + cmd = exec.Command("timeout", "60", "az", "cloud", "update", + "--profile", ccc.APIProfile) + + } else { + cmd = exec.Command("az", "cloud", "update", + "--profile", ccc.APIProfile) + } + out, err = cmd.CombinedOutput() + if err != nil { + log.Printf("output:%s\n", out) + return err + } + + return nil +} + // SetKubeConfig will set the KUBECONIFG env var func (c *Config) SetKubeConfig() { os.Setenv("KUBECONFIG", c.GetKubeConfig()) diff --git a/test/e2e/engine/template.go b/test/e2e/engine/template.go index 39258efec..1bef8a832 100644 --- a/test/e2e/engine/template.go +++ b/test/e2e/engine/template.go @@ -92,9 +92,11 @@ func Build(cfg *config.Config, masterSubnetID string, agentSubnetIDs []string, i prop := cs.ContainerService.Properties if config.ClientID != "" && config.ClientSecret != "" { - prop.ServicePrincipalProfile = &vlabs.ServicePrincipalProfile{ - ClientID: config.ClientID, - Secret: config.ClientSecret, + if !prop.IsAzureStackCloud() { + prop.ServicePrincipalProfile = &vlabs.ServicePrincipalProfile{ + ClientID: config.ClientID, + Secret: config.ClientSecret, + } } } diff --git a/test/e2e/kubernetes/kubernetes_suite_test.go b/test/e2e/kubernetes/kubernetes_suite_test.go index 0d6bf0c70..730439c18 100644 --- a/test/e2e/kubernetes/kubernetes_suite_test.go +++ b/test/e2e/kubernetes/kubernetes_suite_test.go @@ -4,13 +4,15 @@ package kubernetes_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "testing" + + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" ) func TestKubernetes(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Kubernetes Suite") + junitReporter := reporters.NewJUnitReporter("junit.xml") + RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes Suite", []Reporter{junitReporter}) } diff --git a/test/e2e/runner.go b/test/e2e/runner.go index 6975d8784..ed48ce9c0 100644 --- a/test/e2e/runner.go +++ b/test/e2e/runner.go @@ -20,6 +20,7 @@ import ( var ( cfg *config.Config + cccfg *config.CustomCloudConfig acct *azure.Account eng *engine.Engine rgs []string @@ -37,6 +38,22 @@ func main() { } cfg.CurrentWorkingDir = cwd + if cfg.IsAzureStackCloud() { + cccfg, err = config.ParseCustomCloudConfig() + + if err != nil { + log.Fatalf("Error while trying to parse custom cloud configuration: %s\n", err) + } + err = cfg.UpdateCustomCloudClusterDefinition(cccfg) + if err != nil { + log.Fatalf("Error while trying to update cluster definition: %s\n", cfg.ClusterDefinition) + } + cccfg.SetEnvironment() + if err != nil { + log.Fatalf("Error while trying to set environment to azure account! %s\n", err) + } + } + acct, err = azure.NewAccount() if err != nil { log.Fatalf("Error while trying to setup azure account: %s\n", err)