From dbd0fff76a94ef1b62bd5994382ecb4edc29febe Mon Sep 17 00:00:00 2001 From: Ernest Wong Date: Wed, 29 Jul 2020 19:00:37 -0700 Subject: [PATCH] test: add aks as part of pr and nightly test (#717) --- .pipelines/deploy-aks-cluster.yml | 21 +++++++++ .pipelines/deploy-aks-engine-cluster.yml | 39 ++++++++++++++++ .pipelines/e2e-tests-template.yml | 40 ++++------------ .pipelines/nightly.yml | 2 + .pipelines/pr.yml | 1 + .pipelines/soak-tests-template.yml | 18 +++++--- test/e2e/Makefile | 2 +- test/e2e/README.md | 35 +++++++------- test/e2e/backward_compat_test.go | 2 + test/e2e/block_instance_metadata_test.go | 31 ------------- test/e2e/framework/config.go | 6 ++- test/e2e/framework/helm/helm_helpers.go | 59 +++++++++++------------- 12 files changed, 135 insertions(+), 121 deletions(-) create mode 100644 .pipelines/deploy-aks-cluster.yml create mode 100644 .pipelines/deploy-aks-engine-cluster.yml diff --git a/.pipelines/deploy-aks-cluster.yml b/.pipelines/deploy-aks-cluster.yml new file mode 100644 index 00000000..01126b0f --- /dev/null +++ b/.pipelines/deploy-aks-cluster.yml @@ -0,0 +1,21 @@ +steps: + - script: | + az group create --name ${RESOURCE_GROUP} --location $(LOCATION) + + az aks create \ + --resource-group ${RESOURCE_GROUP} \ + --name ${RESOURCE_GROUP} \ + --max-pods ${MAX_PODS} \ + --service-principal $(AZURE_CLIENT_ID) \ + --client-secret $(AZURE_CLIENT_SECRET) \ + --generate-ssh-keys > /dev/null + + # store kubeconfig to ~/.kube/config + az aks get-credentials \ + --resource-group ${RESOURCE_GROUP} \ + --name ${RESOURCE_GROUP} + + # set CLUSTER_RESOURCE_GROUP for e2e test config + export CLUSTER_RESOURCE_GROUP="MC_${RESOURCE_GROUP}_${RESOURCE_GROUP}_$(LOCATION)" + echo "##vso[task.setvariable variable=CLUSTER_RESOURCE_GROUP]${CLUSTER_RESOURCE_GROUP}" + displayName: "Deploy an AKS cluster" diff --git a/.pipelines/deploy-aks-engine-cluster.yml b/.pipelines/deploy-aks-engine-cluster.yml new file mode 100644 index 00000000..ecd2a9f1 --- /dev/null +++ b/.pipelines/deploy-aks-engine-cluster.yml @@ -0,0 +1,39 @@ +steps: + - script: | + echo "Installing aks-engine..." + curl -LO https://github.com/Azure/aks-engine/releases/download/$(AKS_ENGINE_VERSION)/aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64.tar.gz + tar -zxvf aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64.tar.gz + sudo mv aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64/aks-engine /usr/local/bin/ + displayName: "Install aks-engine $(AKS_ENGINE_VERSION)" + + - script: | + # Generate SSH keypair + echo -e 'y\n' | ssh-keygen -f id_rsa -t rsa -N '' > /dev/null + export SSH_PUBLIC_KEY="$(cat id_rsa.pub)" + + envsubst < test/e2e/cluster_configs/$(CLUSTER_CONFIG).json > kubernetes.json + cat kubernetes.json + displayName: "Generate API model" + + - script: | + aks-engine deploy \ + --api-model kubernetes.json \ + --location $(LOCATION) \ + --subscription-id $(SUBSCRIPTION_ID) \ + --client-id $(AZURE_CLIENT_ID) \ + --client-secret $(AZURE_CLIENT_SECRET) + + export KUBECONFIG="$(pwd)/_output/${RESOURCE_GROUP}/kubeconfig/kubeconfig.$(LOCATION).json" + echo "##vso[task.setvariable variable=KUBECONFIG]${KUBECONFIG}" + + # set CLUSTER_RESOURCE_GROUP for e2e test config + export CLUSTER_RESOURCE_GROUP="${RESOURCE_GROUP}" + echo "##vso[task.setvariable variable=CLUSTER_RESOURCE_GROUP]${CLUSTER_RESOURCE_GROUP}" + + # Sleep for 120 seconds to wait for nodes and pods to become ready + sleep 120 + kubectl wait --for=condition=ready node --all + kubectl wait pod -n kube-system --for=condition=Ready --all + kubectl get nodes -owide + kubectl cluster-info + displayName: "aks-engine deploy" diff --git a/.pipelines/e2e-tests-template.yml b/.pipelines/e2e-tests-template.yml index 8dc19b8d..88175740 100644 --- a/.pipelines/e2e-tests-template.yml +++ b/.pipelines/e2e-tests-template.yml @@ -8,7 +8,7 @@ jobs: - ${{ each k8sRelease in parameters.k8sReleases }}: - ${{ each clusterConfig in parameters.clusterConfigs }}: - job: - displayName: ${{ format('v{0}/{1}', k8sRelease, clusterConfig) }} + displayName: ${{ format('{0}', clusterConfig) }} dependsOn: unit_tests timeoutInMinutes: 120 cancelTimeoutInMinutes: 5 @@ -28,55 +28,31 @@ jobs: - template: build-images.yml - script: | - echo "Installing aks-engine..." - curl -LO https://github.com/Azure/aks-engine/releases/download/$(AKS_ENGINE_VERSION)/aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64.tar.gz - tar -zxvf aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64.tar.gz - sudo mv aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64/aks-engine /usr/local/bin/ - echo - echo "Installing kubectl..." curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl chmod +x kubectl sudo mv kubectl /usr/local/bin/ - displayName: "Install tools" + displayName: "Install kubectl" - script: | - # Generate SSH keypair - echo -e 'y\n' | ssh-keygen -f id_rsa -t rsa -N '' > /dev/null - export SSH_PUBLIC_KEY="$(cat id_rsa.pub)" - - # Generate resource group name export RESOURCE_GROUP="aad-pod-identity-e2e-$(openssl rand -hex 6)" echo "##vso[task.setvariable variable=RESOURCE_GROUP]${RESOURCE_GROUP}" + displayName: "Generate resource group name" - envsubst < test/e2e/cluster_configs/$(CLUSTER_CONFIG).json > kubernetes.json - cat kubernetes.json - displayName: "Generate API model" + - ${{ if eq(clusterConfig, 'aks') }}: + - template: deploy-aks-cluster.yml + - ${{ if not(eq(clusterConfig, 'aks')) }}: + - template: deploy-aks-engine-cluster.yml - script: | - aks-engine deploy \ - --api-model kubernetes.json \ - --location $(LOCATION) \ - --subscription-id $(SUBSCRIPTION_ID) \ - --client-id $(AZURE_CLIENT_ID) \ - --client-secret $(AZURE_CLIENT_SECRET) - - export KUBECONFIG="$(pwd)/_output/${RESOURCE_GROUP}/kubeconfig/kubeconfig.$(LOCATION).json" - echo "##vso[task.setvariable variable=KUBECONFIG]${KUBECONFIG}" - - # Sleep for 120 seconds to wait for nodes and pods to become ready - sleep 120 kubectl wait --for=condition=ready node --all kubectl wait pod -n kube-system --for=condition=Ready --all kubectl get nodes -owide kubectl cluster-info - displayName: "aks-engine deploy" + displayName: "Check cluster's health" - script: | export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/k8s/aad-pod-identity}" - export CLUSTER_RESOURCE_GROUP="${RESOURCE_GROUP}" - export IDENTITY_RESOURCE_GROUP="pi-vmss-e2e-daily" - make e2e env: SUBSCRIPTION_ID: $(SUBSCRIPTION_ID) diff --git a/.pipelines/nightly.yml b/.pipelines/nightly.yml index 8c136f2a..7ae09e00 100644 --- a/.pipelines/nightly.yml +++ b/.pipelines/nightly.yml @@ -18,6 +18,7 @@ jobs: k8sReleases: - "1.18" clusterConfigs: + - "aks" # File names in test/e2e/cluster_configs without file extension - "vmss" - "vmas" @@ -26,3 +27,4 @@ jobs: clusterConfigs: - "pi-vmas-e2e-daily" - "pi-vmss-e2e-daily" + - "pi-aks-e2e-daily" diff --git a/.pipelines/pr.yml b/.pipelines/pr.yml index a94e46e0..c5c11bc6 100644 --- a/.pipelines/pr.yml +++ b/.pipelines/pr.yml @@ -18,6 +18,7 @@ jobs: k8sReleases: - "1.18" clusterConfigs: + - "aks" # File names in test/e2e/cluster_configs without file extension - "vmss" - "vmas" diff --git a/.pipelines/soak-tests-template.yml b/.pipelines/soak-tests-template.yml index da89c3ff..32df870e 100644 --- a/.pipelines/soak-tests-template.yml +++ b/.pipelines/soak-tests-template.yml @@ -36,10 +36,19 @@ jobs: sudo mv kubectl /usr/local/bin/ displayName: "Install tools" - - script: | - export KUBECONFIG=$(kubeconfig.secureFilePath) - echo "##vso[task.setvariable variable=KUBECONFIG]${KUBECONFIG}" + - ${{ if eq(clusterConfig, "pi-aks-e2e-daily") }}: + - script: | + az aks get-credentials \ + --resource-group $(CLUSTER_CONFIG) \ + --name $(CLUSTER_CONFIG) + displayName: "Set KUBECONFIG" + - ${{ if not(eq(clusterConfig, "pi-aks-e2e-daily")) }}: + - script: | + export KUBECONFIG=$(kubeconfig.secureFilePath) + echo "##vso[task.setvariable variable=KUBECONFIG]${KUBECONFIG}" + displayName: "Set KUBECONFIG" + - script: | kubectl wait --for=condition=ready node --all kubectl wait pod -n kube-system --for=condition=Ready --all kubectl get nodes -owide @@ -48,9 +57,6 @@ jobs: - script: | export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/k8s/aad-pod-identity}" - export CLUSTER_RESOURCE_GROUP="$(CLUSTER_CONFIG)" - export IDENTITY_RESOURCE_GROUP="$(CLUSTER_CONFIG)" - make e2e env: SUBSCRIPTION_ID: $(SUBSCRIPTION_ID) diff --git a/test/e2e/Makefile b/test/e2e/Makefile index fb995d29..b9c60674 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -6,7 +6,7 @@ GINKGO_FAIL_FAST ?= true .PHONY: run run: - cd $(TEST_E2E_DIR); go test -tags=e2e -timeout=60m -v -ginkgo.v \ + cd $(TEST_E2E_DIR); go test -tags=e2e -timeout=90m -v -ginkgo.v \ -ginkgo.focus=$(GINKGO_FOCUS) \ -ginkgo.skip=$(GINKGO_SKIP) \ -ginkgo.failFast=$(GINKGO_FAIL_FAST) diff --git a/test/e2e/README.md b/test/e2e/README.md index aa6f9ccc..6f3c2df4 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -18,23 +18,24 @@ export KEYVAULT_SECRET_NAME="test-secret" The E2E test suite extracts runtime configurations through environment variables. Below is a list of environment variables to set before running the E2E test suite. -| Variable | Description | -| ------------------------------ | -------------------------------------------------------------------------------------- | -| `SUBSCRIPTION_ID` | The Azure subscription ID. | -| `RESOURCE_GROUP` | The resource group of your Azure Kubernetes cluster. | -| `AZURE_CLIENT_ID` | The client ID of your service principal. | -| `AZURE_CLIENT_SECRET` | The client secret of your service principal. | -| `AZURE_TENANT_ID` | The Azure tenant ID. | -| `KEYVAULT_NAME` | The Azure KeyVault name. | -| `KEYVAULT_SECRET_NAME` | The name of the secret stored in the Azure KeyVault. | -| `KEYVAULT_SECRET_VERSION` | The version of the secret stored in the Azure KeyVault. | -| `MIC_VERSION` | The MIC version. | -| `NMI_VERSION` | The NMI version. | -| `IDENTITY_VALIDATOR_VERSION` | The identity validator version | -| `SYSTEM_MSI_CLUSTER` | Set to `true` if you are using an Azure cluster with system-assigned identity enabled. | -| `ENABLE_SCALE_FEATURES` | Set to `true` if you want to enable the scale features. | -| `IMMUTABLE_IDENTITY_CLIENT_ID` | The client ID of the immutable user-assigned identity created by running setup.sh. | -| `NMI_MODE` | The NMI mode (`standard`, `managed`). | +| Variable | Description | +| ------------------------------ | --------------------------------------------------------------------------------------- | +| `SUBSCRIPTION_ID` | The Azure subscription ID. | +| `RESOURCE_GROUP` | The resource group of your Azure Kubernetes cluster. | +| `AZURE_CLIENT_ID` | The client ID of your service principal. | +| `AZURE_CLIENT_SECRET` | The client secret of your service principal. | +| `AZURE_TENANT_ID` | The Azure tenant ID. | +| `KEYVAULT_NAME` | The Azure KeyVault name. | +| `KEYVAULT_SECRET_NAME` | The name of the secret stored in the Azure KeyVault. | +| `KEYVAULT_SECRET_VERSION` | The version of the secret stored in the Azure KeyVault. | +| `MIC_VERSION` | The MIC version. | +| `NMI_VERSION` | The NMI version. | +| `IDENTITY_VALIDATOR_VERSION` | The identity validator version | +| `SYSTEM_MSI_CLUSTER` | Set to `true` if you are using an Azure cluster with system-assigned identity enabled. | +| `ENABLE_SCALE_FEATURES` | Set to `true` if you want to enable the scale features. | +| `IMMUTABLE_IDENTITY_CLIENT_ID` | The client ID of the immutable user-assigned identity created by running setup.sh. | +| `NMI_MODE` | The NMI mode (`standard`, `managed`). | +| `BLOCK_INSTANCE_METADATA` | Set to `true` if you want to run test cases related to block-instance-metadata feature. | Finally, to kick off a test run: diff --git a/test/e2e/backward_compat_test.go b/test/e2e/backward_compat_test.go index d26f43d8..d07123a9 100644 --- a/test/e2e/backward_compat_test.go +++ b/test/e2e/backward_compat_test.go @@ -63,6 +63,8 @@ var _ = Describe("When upgrading AAD Pod Identity", func() { configOldVersion.MICVersion = "1.5" configOldVersion.NMIVersion = "1.5" configOldVersion.ImmutableUserMSIs = "" + configOldVersion.BlockInstanceMetadata = false + helm.Upgrade(helm.UpgradeInput{ Config: configOldVersion, }) diff --git a/test/e2e/block_instance_metadata_test.go b/test/e2e/block_instance_metadata_test.go index 64374167..5fdbf7b4 100644 --- a/test/e2e/block_instance_metadata_test.go +++ b/test/e2e/block_instance_metadata_test.go @@ -6,8 +6,6 @@ import ( "strings" "github.com/Azure/aad-pod-identity/test/e2e/framework/exec" - "github.com/Azure/aad-pod-identity/test/e2e/framework/helm" - "github.com/Azure/aad-pod-identity/test/e2e/framework/namespace" "github.com/Azure/aad-pod-identity/test/e2e/framework/pod" corev1 "k8s.io/api/core/v1" @@ -16,36 +14,7 @@ import ( ) var _ = Describe("When blocking pods from accessing Instance Metadata Service", func() { - var ( - specName = "block-instance-metadata" - ns *corev1.Namespace - ) - - BeforeEach(func() { - ns = namespace.Create(namespace.CreateInput{ - Creator: kubeClient, - Name: specName, - }) - }) - - AfterEach(func() { - namespace.Delete(namespace.DeleteInput{ - Deleter: kubeClient, - Getter: kubeClient, - Namespace: ns, - }) - }) - It("should receive a HTTP 403 response when contacting /metadata/instance endpoint", func() { - helm.Upgrade(helm.UpgradeInput{ - Config: config, - BlockInstanceMetadata: true, - }) - defer helm.Upgrade(helm.UpgradeInput{ - Config: config, - BlockInstanceMetadata: false, - }) - nmiPods := pod.List(pod.ListInput{ Lister: kubeClient, Namespace: corev1.NamespaceDefault, diff --git a/test/e2e/framework/config.go b/test/e2e/framework/config.go index 42a82d4c..c23c6e2f 100644 --- a/test/e2e/framework/config.go +++ b/test/e2e/framework/config.go @@ -27,7 +27,8 @@ type Config struct { SystemMSICluster bool `envconfig:"SYSTEM_MSI_CLUSTER" default:"false"` EnableScaleFeatures bool `envconfig:"ENABLE_SCALE_FEATURES" default:"false"` ImmutableUserMSIs string `envconfig:"IMMUTABLE_IDENTITY_CLIENT_ID"` - NmiMode string `envconfig:"NMI_MODE" default:"standard"` + NMIMode string `envconfig:"NMI_MODE" default:"standard"` + BlockInstanceMetadata bool `envconfig:"BLOCK_INSTANCE_METADATA" default:"true"` } func (c *Config) DeepCopy() *Config { @@ -49,7 +50,8 @@ func (c *Config) DeepCopy() *Config { copy.SystemMSICluster = c.SystemMSICluster copy.EnableScaleFeatures = c.EnableScaleFeatures copy.ImmutableUserMSIs = c.ImmutableUserMSIs - copy.NmiMode = c.NmiMode + copy.NMIMode = c.NMIMode + copy.BlockInstanceMetadata = c.BlockInstanceMetadata return copy } diff --git a/test/e2e/framework/helm/helm_helpers.go b/test/e2e/framework/helm/helm_helpers.go index 46d49c39..a5304564 100644 --- a/test/e2e/framework/helm/helm_helpers.go +++ b/test/e2e/framework/helm/helm_helpers.go @@ -20,10 +20,8 @@ const ( // InstallInput is the input for Install. type InstallInput struct { - Config *framework.Config - ManagedMode bool - NamespacedMode bool - BlockInstanceMetadata bool + Config *framework.Config + NamespacedMode bool } // Install installs aad-pod-identity via Helm 3. @@ -43,22 +41,8 @@ func Install(input InstallInput) { chartName, "charts/aad-pod-identity", "--wait", - fmt.Sprintf("--set=image.repository=%s", input.Config.Registry), - fmt.Sprintf("--set=mic.tag=%s", input.Config.MICVersion), - fmt.Sprintf("--set=nmi.tag=%s", input.Config.NMIVersion), }) - - if input.Config.ImmutableUserMSIs != "" { - args = append(args, fmt.Sprintf("--set=mic.immutableUserMSIs=%s", input.Config.ImmutableUserMSIs)) - } - - if input.ManagedMode { - args = append(args, fmt.Sprintf("--set=operationMode=%s", "managed")) - } - - if input.BlockInstanceMetadata { - args = append(args, fmt.Sprintf("--set=nmi.blockInstanceMetadata=%t", input.BlockInstanceMetadata)) - } + args = append(args, generateValueArgs(input.Config)...) helm(args) } @@ -75,8 +59,7 @@ func Uninstall() { // UpgradeInput is the input for Upgrade. type UpgradeInput struct { - Config *framework.Config - BlockInstanceMetadata bool + Config *framework.Config } // Upgrade upgrades aad-pod-identity via Helm 3. @@ -96,22 +79,34 @@ func Upgrade(input UpgradeInput) { chartName, "charts/aad-pod-identity", "--wait", - fmt.Sprintf("--set=image.repository=%s", input.Config.Registry), - fmt.Sprintf("--set=mic.tag=%s", input.Config.MICVersion), - fmt.Sprintf("--set=nmi.tag=%s", input.Config.NMIVersion), }) - - if input.Config.ImmutableUserMSIs != "" { - args = append(args, fmt.Sprintf("--set=mic.immutableUserMSIs=%s", input.Config.ImmutableUserMSIs)) - } - - if input.BlockInstanceMetadata { - args = append(args, fmt.Sprintf("--set=nmi.blockInstanceMetadata=%t", input.BlockInstanceMetadata)) - } + args = append(args, generateValueArgs(input.Config)...) helm(args) } +func generateValueArgs(config *framework.Config) []string { + args := []string{ + fmt.Sprintf("--set=image.repository=%s", config.Registry), + fmt.Sprintf("--set=mic.tag=%s", config.MICVersion), + fmt.Sprintf("--set=nmi.tag=%s", config.NMIVersion), + } + + if config.ImmutableUserMSIs != "" { + args = append(args, fmt.Sprintf("--set=mic.immutableUserMSIs=%s", config.ImmutableUserMSIs)) + } + + if config.NMIMode == "managed" { + args = append(args, fmt.Sprintf("--set=operationMode=%s", "managed")) + } + + if config.BlockInstanceMetadata { + args = append(args, fmt.Sprintf("--set=nmi.blockInstanceMetadata=%t", config.BlockInstanceMetadata)) + } + + return args +} + func helm(args []string) { By(fmt.Sprintf("helm %s", strings.Join(args, " ")))