test: add aks as part of pr and nightly test (#717)

2020-07-29 19:00:37 -07:00 · 2020-07-29 19:00:37 -07:00 · dbd0fff76a
--- a/.pipelines/deploy-aks-cluster.yml
+++ b/.pipelines/deploy-aks-cluster.yml
@ -0,0 +1,21 @@
+steps:
+  - script: |
+      az group create --name ${RESOURCE_GROUP} --location $(LOCATION)
+
+      az aks create \
+        --resource-group ${RESOURCE_GROUP} \
+        --name ${RESOURCE_GROUP} \
+        --max-pods ${MAX_PODS} \
+        --service-principal $(AZURE_CLIENT_ID) \
+        --client-secret $(AZURE_CLIENT_SECRET) \
+        --generate-ssh-keys > /dev/null
+
+      # store kubeconfig to ~/.kube/config
+      az aks get-credentials \
+        --resource-group ${RESOURCE_GROUP} \
+        --name ${RESOURCE_GROUP}
+
+      # set CLUSTER_RESOURCE_GROUP for e2e test config
+      export CLUSTER_RESOURCE_GROUP="MC_${RESOURCE_GROUP}_${RESOURCE_GROUP}_$(LOCATION)"
+      echo "##vso[task.setvariable variable=CLUSTER_RESOURCE_GROUP]${CLUSTER_RESOURCE_GROUP}"
+    displayName: "Deploy an AKS cluster"
--- a/.pipelines/deploy-aks-engine-cluster.yml
+++ b/.pipelines/deploy-aks-engine-cluster.yml
@ -0,0 +1,39 @@
+steps:
+  - script: |
+      echo "Installing aks-engine..."
+      curl -LO https://github.com/Azure/aks-engine/releases/download/$(AKS_ENGINE_VERSION)/aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64.tar.gz
+      tar -zxvf aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64.tar.gz
+      sudo mv aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64/aks-engine /usr/local/bin/
+    displayName: "Install aks-engine $(AKS_ENGINE_VERSION)"
+
+  - script: |
+      # Generate SSH keypair
+      echo -e 'y\n' | ssh-keygen -f id_rsa -t rsa -N '' > /dev/null
+      export SSH_PUBLIC_KEY="$(cat id_rsa.pub)"
+
+      envsubst < test/e2e/cluster_configs/$(CLUSTER_CONFIG).json > kubernetes.json
+      cat kubernetes.json
+    displayName: "Generate API model"
+
+  - script: |
+      aks-engine deploy \
+        --api-model kubernetes.json \
+        --location $(LOCATION) \
+        --subscription-id $(SUBSCRIPTION_ID) \
+        --client-id $(AZURE_CLIENT_ID) \
+        --client-secret $(AZURE_CLIENT_SECRET)
+
+      export KUBECONFIG="$(pwd)/_output/${RESOURCE_GROUP}/kubeconfig/kubeconfig.$(LOCATION).json"
+      echo "##vso[task.setvariable variable=KUBECONFIG]${KUBECONFIG}"
+
+      # set CLUSTER_RESOURCE_GROUP for e2e test config
+      export CLUSTER_RESOURCE_GROUP="${RESOURCE_GROUP}"
+      echo "##vso[task.setvariable variable=CLUSTER_RESOURCE_GROUP]${CLUSTER_RESOURCE_GROUP}"
+
+      # Sleep for 120 seconds to wait for nodes and pods to become ready
+      sleep 120
+      kubectl wait --for=condition=ready node --all
+      kubectl wait pod -n kube-system --for=condition=Ready --all
+      kubectl get nodes -owide
+      kubectl cluster-info
+    displayName: "aks-engine deploy"
--- a/.pipelines/e2e-tests-template.yml
+++ b/.pipelines/e2e-tests-template.yml
@ -8,7 +8,7 @@ jobs:
  - ${{ each k8sRelease in parameters.k8sReleases }}:
    - ${{ each clusterConfig in parameters.clusterConfigs }}:
      - job:
-        displayName: ${{ format('v{0}/{1}', k8sRelease, clusterConfig) }}
+        displayName: ${{ format('{0}', clusterConfig) }}
        dependsOn: unit_tests
        timeoutInMinutes: 120
        cancelTimeoutInMinutes: 5
@ -28,55 +28,31 @@ jobs:
          - template: build-images.yml

          - script: |
-              echo "Installing aks-engine..."
-              curl -LO https://github.com/Azure/aks-engine/releases/download/$(AKS_ENGINE_VERSION)/aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64.tar.gz
-              tar -zxvf aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64.tar.gz
-              sudo mv aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64/aks-engine /usr/local/bin/
-              echo
-
              echo "Installing kubectl..."
              curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
              chmod +x kubectl
              sudo mv kubectl /usr/local/bin/
-            displayName: "Install tools"
+            displayName: "Install kubectl"

          - script: |
-              # Generate SSH keypair
-              echo -e 'y\n' | ssh-keygen -f id_rsa -t rsa -N '' > /dev/null
-              export SSH_PUBLIC_KEY="$(cat id_rsa.pub)"
-
-              # Generate resource group name
              export RESOURCE_GROUP="aad-pod-identity-e2e-$(openssl rand -hex 6)"
              echo "##vso[task.setvariable variable=RESOURCE_GROUP]${RESOURCE_GROUP}"
+            displayName: "Generate resource group name"

-              envsubst < test/e2e/cluster_configs/$(CLUSTER_CONFIG).json > kubernetes.json
-              cat kubernetes.json
-            displayName: "Generate API model"
+          - ${{ if eq(clusterConfig, 'aks') }}:
+            - template: deploy-aks-cluster.yml
+          - ${{ if not(eq(clusterConfig, 'aks')) }}:
+            - template: deploy-aks-engine-cluster.yml

          - script: |
-              aks-engine deploy \
-                --api-model kubernetes.json \
-                --location $(LOCATION) \
-                --subscription-id $(SUBSCRIPTION_ID) \
-                --client-id $(AZURE_CLIENT_ID) \
-                --client-secret $(AZURE_CLIENT_SECRET)
-
-              export KUBECONFIG="$(pwd)/_output/${RESOURCE_GROUP}/kubeconfig/kubeconfig.$(LOCATION).json"
-              echo "##vso[task.setvariable variable=KUBECONFIG]${KUBECONFIG}"
-
-              # Sleep for 120 seconds to wait for nodes and pods to become ready
-              sleep 120
              kubectl wait --for=condition=ready node --all
              kubectl wait pod -n kube-system --for=condition=Ready --all
              kubectl get nodes -owide
              kubectl cluster-info
-            displayName: "aks-engine deploy"
+            displayName: "Check cluster's health"

          - script: |
              export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/k8s/aad-pod-identity}"
-              export CLUSTER_RESOURCE_GROUP="${RESOURCE_GROUP}"
-              export IDENTITY_RESOURCE_GROUP="pi-vmss-e2e-daily"
-
              make e2e
            env:
              SUBSCRIPTION_ID: $(SUBSCRIPTION_ID)
--- a/.pipelines/nightly.yml
+++ b/.pipelines/nightly.yml
@ -18,6 +18,7 @@ jobs:
      k8sReleases:
        - "1.18"
      clusterConfigs:
+        - "aks"
        # File names in test/e2e/cluster_configs without file extension
        - "vmss"
        - "vmas"
@ -26,3 +27,4 @@ jobs:
      clusterConfigs:
        - "pi-vmas-e2e-daily"
        - "pi-vmss-e2e-daily"
+        - "pi-aks-e2e-daily"
--- a/.pipelines/pr.yml
+++ b/.pipelines/pr.yml
@ -18,6 +18,7 @@ jobs:
      k8sReleases:
        - "1.18"
      clusterConfigs:
+        - "aks"
        # File names in test/e2e/cluster_configs without file extension
        - "vmss"
        - "vmas"
--- a/.pipelines/soak-tests-template.yml
+++ b/.pipelines/soak-tests-template.yml
@ -36,10 +36,19 @@ jobs:
            sudo mv kubectl /usr/local/bin/
          displayName: "Install tools"

-        - script: |
-            export KUBECONFIG=$(kubeconfig.secureFilePath)
-            echo "##vso[task.setvariable variable=KUBECONFIG]${KUBECONFIG}"
+        - ${{ if eq(clusterConfig, "pi-aks-e2e-daily") }}:
+          - script: |
+              az aks get-credentials \
+                --resource-group $(CLUSTER_CONFIG) \
+                --name $(CLUSTER_CONFIG)
+            displayName: "Set KUBECONFIG"
+        - ${{ if not(eq(clusterConfig, "pi-aks-e2e-daily")) }}:
+          - script: |
+              export KUBECONFIG=$(kubeconfig.secureFilePath)
+              echo "##vso[task.setvariable variable=KUBECONFIG]${KUBECONFIG}"
+            displayName: "Set KUBECONFIG"

+        - script: |
            kubectl wait --for=condition=ready node --all
            kubectl wait pod -n kube-system --for=condition=Ready --all
            kubectl get nodes -owide
@ -48,9 +57,6 @@ jobs:

        - script: |
            export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/k8s/aad-pod-identity}"
-            export CLUSTER_RESOURCE_GROUP="$(CLUSTER_CONFIG)"
-            export IDENTITY_RESOURCE_GROUP="$(CLUSTER_CONFIG)"
-
            make e2e
          env:
            SUBSCRIPTION_ID: $(SUBSCRIPTION_ID)
--- a/test/e2e/Makefile
+++ b/test/e2e/Makefile
@ -6,7 +6,7 @@ GINKGO_FAIL_FAST ?= true

 .PHONY: run
 run:
-	cd $(TEST_E2E_DIR); go test -tags=e2e -timeout=60m -v -ginkgo.v \
+	cd $(TEST_E2E_DIR); go test -tags=e2e -timeout=90m -v -ginkgo.v \
 		-ginkgo.focus=$(GINKGO_FOCUS) \
 		-ginkgo.skip=$(GINKGO_SKIP) \
 		-ginkgo.failFast=$(GINKGO_FAIL_FAST)
--- a/test/e2e/README.md
+++ b/test/e2e/README.md
@ -18,23 +18,24 @@ export KEYVAULT_SECRET_NAME="test-secret"

 The E2E test suite extracts runtime configurations through environment variables. Below is a list of environment variables to set before running the E2E test suite.

-| Variable                       | Description                                                                            |
-| ------------------------------ | -------------------------------------------------------------------------------------- |
-| `SUBSCRIPTION_ID`              | The Azure subscription ID.                                                             |
-| `RESOURCE_GROUP`               | The resource group of your Azure Kubernetes cluster.                                   |
-| `AZURE_CLIENT_ID`              | The client ID of your service principal.                                               |
-| `AZURE_CLIENT_SECRET`          | The client secret of your service principal.                                           |
-| `AZURE_TENANT_ID`              | The Azure tenant ID.                                                                   |
-| `KEYVAULT_NAME`                | The Azure KeyVault name.                                                               |
-| `KEYVAULT_SECRET_NAME`         | The name of the secret stored in the Azure KeyVault.                                   |
-| `KEYVAULT_SECRET_VERSION`      | The version of the secret stored in the Azure KeyVault.                                |
-| `MIC_VERSION`                  | The MIC version.                                                                       |
-| `NMI_VERSION`                  | The NMI version.                                                                       |
-| `IDENTITY_VALIDATOR_VERSION`   | The identity validator version                                                         |
-| `SYSTEM_MSI_CLUSTER`           | Set to `true` if you are using an Azure cluster with system-assigned identity enabled. |
-| `ENABLE_SCALE_FEATURES`        | Set to `true` if you want to enable the scale features.                                |
-| `IMMUTABLE_IDENTITY_CLIENT_ID` | The client ID of the immutable user-assigned identity created by running setup.sh.     |
-| `NMI_MODE`                     | The NMI mode (`standard`, `managed`).                                                  |
+| Variable                       | Description                                                                             |
+| ------------------------------ | --------------------------------------------------------------------------------------- |
+| `SUBSCRIPTION_ID`              | The Azure subscription ID.                                                              |
+| `RESOURCE_GROUP`               | The resource group of your Azure Kubernetes cluster.                                    |
+| `AZURE_CLIENT_ID`              | The client ID of your service principal.                                                |
+| `AZURE_CLIENT_SECRET`          | The client secret of your service principal.                                            |
+| `AZURE_TENANT_ID`              | The Azure tenant ID.                                                                    |
+| `KEYVAULT_NAME`                | The Azure KeyVault name.                                                                |
+| `KEYVAULT_SECRET_NAME`         | The name of the secret stored in the Azure KeyVault.                                    |
+| `KEYVAULT_SECRET_VERSION`      | The version of the secret stored in the Azure KeyVault.                                 |
+| `MIC_VERSION`                  | The MIC version.                                                                        |
+| `NMI_VERSION`                  | The NMI version.                                                                        |
+| `IDENTITY_VALIDATOR_VERSION`   | The identity validator version                                                          |
+| `SYSTEM_MSI_CLUSTER`           | Set to `true` if you are using an Azure cluster with system-assigned identity enabled.  |
+| `ENABLE_SCALE_FEATURES`        | Set to `true` if you want to enable the scale features.                                 |
+| `IMMUTABLE_IDENTITY_CLIENT_ID` | The client ID of the immutable user-assigned identity created by running setup.sh.      |
+| `NMI_MODE`                     | The NMI mode (`standard`, `managed`).                                                   |
+| `BLOCK_INSTANCE_METADATA`      | Set to `true` if you want to run test cases related to block-instance-metadata feature. |

 Finally, to kick off a test run:

--- a/test/e2e/backward_compat_test.go
+++ b/test/e2e/backward_compat_test.go
@ -63,6 +63,8 @@ var _ = Describe("When upgrading AAD Pod Identity", func() {
 		configOldVersion.MICVersion = "1.5"
 		configOldVersion.NMIVersion = "1.5"
 		configOldVersion.ImmutableUserMSIs = ""
+		configOldVersion.BlockInstanceMetadata = false
+
 		helm.Upgrade(helm.UpgradeInput{
 			Config: configOldVersion,
 		})
--- a/test/e2e/block_instance_metadata_test.go
+++ b/test/e2e/block_instance_metadata_test.go
@ -6,8 +6,6 @@ import (
 	"strings"

 	"github.com/Azure/aad-pod-identity/test/e2e/framework/exec"
-	"github.com/Azure/aad-pod-identity/test/e2e/framework/helm"
-	"github.com/Azure/aad-pod-identity/test/e2e/framework/namespace"
 	"github.com/Azure/aad-pod-identity/test/e2e/framework/pod"
 	corev1 "k8s.io/api/core/v1"

@ -16,36 +14,7 @@ import (
 )

 var _ = Describe("When blocking pods from accessing Instance Metadata Service", func() {
-	var (
-		specName = "block-instance-metadata"
-		ns       *corev1.Namespace
-	)
-
-	BeforeEach(func() {
-		ns = namespace.Create(namespace.CreateInput{
-			Creator: kubeClient,
-			Name:    specName,
-		})
-	})
-
-	AfterEach(func() {
-		namespace.Delete(namespace.DeleteInput{
-			Deleter:   kubeClient,
-			Getter:    kubeClient,
-			Namespace: ns,
-		})
-	})
-
 	It("should receive a HTTP 403 response when contacting /metadata/instance endpoint", func() {
-		helm.Upgrade(helm.UpgradeInput{
-			Config:                config,
-			BlockInstanceMetadata: true,
-		})
-		defer helm.Upgrade(helm.UpgradeInput{
-			Config:                config,
-			BlockInstanceMetadata: false,
-		})
-
 		nmiPods := pod.List(pod.ListInput{
 			Lister:    kubeClient,
 			Namespace: corev1.NamespaceDefault,
--- a/test/e2e/framework/config.go
+++ b/test/e2e/framework/config.go
@ -27,7 +27,8 @@ type Config struct {
 	SystemMSICluster         bool   `envconfig:"SYSTEM_MSI_CLUSTER" default:"false"`
 	EnableScaleFeatures      bool   `envconfig:"ENABLE_SCALE_FEATURES" default:"false"`
 	ImmutableUserMSIs        string `envconfig:"IMMUTABLE_IDENTITY_CLIENT_ID"`
-	NmiMode                  string `envconfig:"NMI_MODE" default:"standard"`
+	NMIMode                  string `envconfig:"NMI_MODE" default:"standard"`
+	BlockInstanceMetadata    bool   `envconfig:"BLOCK_INSTANCE_METADATA" default:"true"`
 }

 func (c *Config) DeepCopy() *Config {
@ -49,7 +50,8 @@ func (c *Config) DeepCopy() *Config {
 	copy.SystemMSICluster = c.SystemMSICluster
 	copy.EnableScaleFeatures = c.EnableScaleFeatures
 	copy.ImmutableUserMSIs = c.ImmutableUserMSIs
-	copy.NmiMode = c.NmiMode
+	copy.NMIMode = c.NMIMode
+	copy.BlockInstanceMetadata = c.BlockInstanceMetadata

 	return copy
 }
--- a/test/e2e/framework/helm/helm_helpers.go
+++ b/test/e2e/framework/helm/helm_helpers.go
@ -20,10 +20,8 @@ const (

 // InstallInput is the input for Install.
 type InstallInput struct {
-	Config                *framework.Config
-	ManagedMode           bool
-	NamespacedMode        bool
-	BlockInstanceMetadata bool
+	Config         *framework.Config
+	NamespacedMode bool
 }

 // Install installs aad-pod-identity via Helm 3.
@ -43,22 +41,8 @@ func Install(input InstallInput) {
 		chartName,
 		"charts/aad-pod-identity",
 		"--wait",
-		fmt.Sprintf("--set=image.repository=%s", input.Config.Registry),
-		fmt.Sprintf("--set=mic.tag=%s", input.Config.MICVersion),
-		fmt.Sprintf("--set=nmi.tag=%s", input.Config.NMIVersion),
 	})
-
-	if input.Config.ImmutableUserMSIs != "" {
-		args = append(args, fmt.Sprintf("--set=mic.immutableUserMSIs=%s", input.Config.ImmutableUserMSIs))
-	}
-
-	if input.ManagedMode {
-		args = append(args, fmt.Sprintf("--set=operationMode=%s", "managed"))
-	}
-
-	if input.BlockInstanceMetadata {
-		args = append(args, fmt.Sprintf("--set=nmi.blockInstanceMetadata=%t", input.BlockInstanceMetadata))
-	}
+	args = append(args, generateValueArgs(input.Config)...)

 	helm(args)
 }
@ -75,8 +59,7 @@ func Uninstall() {

 // UpgradeInput is the input for Upgrade.
 type UpgradeInput struct {
-	Config                *framework.Config
-	BlockInstanceMetadata bool
+	Config *framework.Config
 }

 // Upgrade upgrades aad-pod-identity via Helm 3.
@ -96,22 +79,34 @@ func Upgrade(input UpgradeInput) {
 		chartName,
 		"charts/aad-pod-identity",
 		"--wait",
-		fmt.Sprintf("--set=image.repository=%s", input.Config.Registry),
-		fmt.Sprintf("--set=mic.tag=%s", input.Config.MICVersion),
-		fmt.Sprintf("--set=nmi.tag=%s", input.Config.NMIVersion),
 	})
-
-	if input.Config.ImmutableUserMSIs != "" {
-		args = append(args, fmt.Sprintf("--set=mic.immutableUserMSIs=%s", input.Config.ImmutableUserMSIs))
-	}
-
-	if input.BlockInstanceMetadata {
-		args = append(args, fmt.Sprintf("--set=nmi.blockInstanceMetadata=%t", input.BlockInstanceMetadata))
-	}
+	args = append(args, generateValueArgs(input.Config)...)

 	helm(args)
 }

+func generateValueArgs(config *framework.Config) []string {
+	args := []string{
+		fmt.Sprintf("--set=image.repository=%s", config.Registry),
+		fmt.Sprintf("--set=mic.tag=%s", config.MICVersion),
+		fmt.Sprintf("--set=nmi.tag=%s", config.NMIVersion),
+	}
+
+	if config.ImmutableUserMSIs != "" {
+		args = append(args, fmt.Sprintf("--set=mic.immutableUserMSIs=%s", config.ImmutableUserMSIs))
+	}
+
+	if config.NMIMode == "managed" {
+		args = append(args, fmt.Sprintf("--set=operationMode=%s", "managed"))
+	}
+
+	if config.BlockInstanceMetadata {
+		args = append(args, fmt.Sprintf("--set=nmi.blockInstanceMetadata=%t", config.BlockInstanceMetadata))
+	}
+
+	return args
+}
+
 func helm(args []string) {
 	By(fmt.Sprintf("helm %s", strings.Join(args, " ")))