Upgrade tests (#97)

* feat: adds upgrade tests * fix: adds wait step before test Signed-off-by: Nilekh Chaudhari <1626598+nilekhc@users.noreply.github.com> * fix: updates image version * fix: updates registry name * fix: adds variables * fix: checks if pod exists * fix: updates deploy curl pod * test: adds debug * fix: adds test cleanup * fix: updates param names * chore: moves template to it's dir * fix: updates template path * fix: updates job name * fix: adds creds env var * fix: updates make target * fix: removes file * fix: removes e2e tests from nightly * fix: removes unit tests from nightly * feat: adds soak test * fix: updates job name * test: env var * test * fix: kubeconfig * test: kubeconfig * test * test * fix: kubeconfig * chore: remove commented code * fix: config * test: key * chore: removes newline * test: key * fix: installs all prerequisites * fix: test * fix: updates tests for aks-engine * test: all tests * test: all test * chore: removes tests for PR * chore: adds unit tests to nightly * fix: removes cluster info * chore: updates step name * fix: adds grace-period to pod delete * chore: adds new line * ping * chore: gofmt * chore: added comments * chore: adds newline * chore: updates target * test: template changes * test: kind setup * chore: downloads entire kubeconfig via secure file * fix: kubeconfig * chore: removes dead code * chore: removes newline * chore: changes to setup_file to run once * test: make target * test: makefile * test: removes env vars * test: make target * test: make target * test: updates script invocation * test * fix: sleep for 30s * fix: increase sleep time * fix: increase sleep time * chore: removes dead code
2021-06-03 17:18:50 -07:00 · 2021-06-03 17:18:50 -07:00 · a319efc244
--- a/.pipelines/e2e-kind-template.yml
+++ b/.pipelines/e2e-kind-template.yml
@ -1,45 +0,0 @@
-jobs:
-  - job: e2e_tests
-    timeoutInMinutes: 10
-    cancelTimeoutInMinutes: 5
-    workspace:
-      clean: all
-    variables:
-      - group: kubernetes-kms
-
-    steps:
-      - task: GoTool@0
-        inputs:
-          version: 1.16
-
-      - script: make e2e-install-prerequisites
-        displayName: "Install e2e test prerequisites"
-      - script: make e2e-setup-kind
-        displayName: "Setup kind cluster with azure kms plugin"
-        env:
-          CLIENT_ID: $(AZURE_CLIENT_ID)
-          CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
-      - script: |
-          kubectl wait --for=condition=ready node --all
-          kubectl wait pod -n kube-system --for=condition=Ready --all
-          kubectl get nodes -owide
-          kubectl cluster-info
-        displayName: "Check cluster's health"
-      - script: |
-          docker exec kms-control-plane bash -c "cat /etc/kubernetes/manifests/kubernetes-kms.yaml"
-          docker exec kms-control-plane bash -c "cat /etc/kubernetes/manifests/kube-apiserver.yaml"
-          docker exec kms-control-plane bash -c "cat /etc/kubernetes/encryption-config.yaml"
-          docker exec kms-control-plane bash -c "journalctl -u kubelet > kubelet.log && cat kubelet.log"
-
-          docker exec kms-control-plane bash -c "cd /var/log/containers ; cat *"
-          docker network ls
-        displayName: "Debug logs"
-        condition: failed()
-      - script: make e2e-test
-        displayName: "Run e2e tests"
-      - script: |
-          kubectl logs -l component=azure-kms-provider -n kube-system --tail -1
-          kubectl get pods -o wide -A
-        displayName: "Get logs"
-      - script: make e2e-delete-kind
-        displayName: "Delete cluster"
--- a/.pipelines/nightly.yml
+++ b/.pipelines/nightly.yml
@ -12,5 +12,6 @@ pool:
  vmImage: ubuntu-latest

 jobs:
-  - template: unit-tests-template.yml
-  - template: e2e-kind-template.yml
+  - template: templates/unit-tests-template.yml
+  - template: templates/soak-test-template.yml
+  - template: templates/e2e-upgrade-template.yml
--- a/.pipelines/pr.yml
+++ b/.pipelines/pr.yml
@ -15,5 +15,5 @@ pool:
  vmImage: ubuntu-latest

 jobs:
-  - template: unit-tests-template.yml
-  - template: e2e-kind-template.yml
+  - template: templates/unit-tests-template.yml
+  - template: templates/e2e-kind-template.yml
--- a/.pipelines/templates/cleanup-template.yml
+++ b/.pipelines/templates/cleanup-template.yml
@ -0,0 +1,8 @@
+steps:
+  - script: |
+      kubectl logs -l component=azure-kms-provider -n kube-system --tail -1
+      kubectl get pods -o wide -A
+    displayName: "Get logs"
+    
+  - script: make e2e-delete-kind
+    displayName: "Delete cluster"
--- a/.pipelines/templates/cluster-health-template.yml
+++ b/.pipelines/templates/cluster-health-template.yml
@ -0,0 +1,6 @@
+steps:
+  - script: |
+      kubectl wait --for=condition=ready node --all
+      kubectl wait pod -n kube-system --for=condition=Ready --all
+      kubectl get nodes -owide
+    displayName: "Check cluster health"
--- a/.pipelines/templates/e2e-kind-template.yml
+++ b/.pipelines/templates/e2e-kind-template.yml
@ -0,0 +1,46 @@
+jobs:
+  - job: e2e_tests
+    timeoutInMinutes: 10
+    cancelTimeoutInMinutes: 5
+    workspace:
+      clean: all
+    variables:
+      - name: REGISTRY_NAME
+        value: kind-registry
+      - name: REGISTRY_PORT
+        value: 5000
+      - name: KUBERNETES_VERSION
+        value: v1.21.1
+      - name: KIND_CLUSTER_NAME
+        value: kms
+      - name: KIND_NETWORK
+        value: kind
+      - group: kubernetes-kms
+
+    steps:
+      - task: GoTool@0
+        inputs:
+          version: 1.16
+
+      - script: make e2e-install-prerequisites
+        displayName: "Install e2e test prerequisites"
+
+      - script: |
+          REGISTRY_NAME=$(REGISTRY_NAME) REGISTRY_PORT=$(REGISTRY_PORT) KUBERNETES_VERSION=$(KUBERNETES_VERSION) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) KIND_NETWORK=$(KIND_NETWORK) make e2e-setup-kind
+        displayName: "Setup kind cluster with azure kms plugin"
+        env:
+          REGISTRY_NAME: $(REGISTRY_NAME)
+          REGISTRY_PORT: $(REGISTRY_PORT)
+          KUBERNETES_VERSION: $(KUBERNETES_VERSION)
+          KIND_CLUSTER_NAME: $(KIND_CLUSTER_NAME)
+          KIND_NETWORK: $(KIND_NETWORK)
+          CLIENT_ID: $(AZURE_CLIENT_ID)
+          CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
+
+      - template: cluster-health-template.yml
+      - template: kind-debug-template.yml
+
+      - script: make e2e-test
+        displayName: "Run e2e tests"
+
+      - template: cleanup-template.yml
--- a/.pipelines/templates/e2e-upgrade-template.yml
+++ b/.pipelines/templates/e2e-upgrade-template.yml
@ -0,0 +1,97 @@
+jobs:
+  - job: e2e_upgrade_tests
+    timeoutInMinutes: 10
+    cancelTimeoutInMinutes: 5
+    workspace:
+      clean: all
+    variables:
+      - name: REGISTRY_NAME
+        value: kind-registry
+      - name: REGISTRY_PORT
+        value: 5000
+      - name: KUBERNETES_VERSION
+        value: v1.21.1
+      - name: KIND_CLUSTER_NAME
+        value: kms
+      - name: KIND_NETWORK
+        value: kind
+      - group: kubernetes-kms
+
+    steps:
+      - task: GoTool@0
+        inputs:
+          version: 1.16
+
+      - script: make e2e-install-prerequisites
+        displayName: "Install e2e test prerequisites"
+
+      - script: |
+          . scripts/setup-local-registry.sh
+        displayName: "Setup local registry"
+        env:
+          REGISTRY_NAME: $(REGISTRY_NAME)
+          REGISTRY_PORT: $(REGISTRY_PORT)
+
+      - script: |
+          version=$(git tag -l --sort=v:refname | tail -n 1)
+          echo "##vso[task.setvariable variable=LATEST_KMS_VERSION]$version"
+
+          echo "Latest released kms version - $version"
+        displayName: "Get latest released version"
+      
+      - template: manifest-template.yml
+        parameters:
+          registry: mcr.microsoft.com/oss/azure/kms
+          imageName: keyvault
+          imageVersion: $(LATEST_KMS_VERSION)
+
+      - script: |
+          . scripts/setup-kind-cluster.sh &
+          . scripts/connect-registry.sh &
+          wait
+        displayName: "Setup kind cluster with azure kms plugin"
+        env:
+          REGISTRY_NAME: $(REGISTRY_NAME)
+          REGISTRY_PORT: $(REGISTRY_PORT)
+          KUBERNETES_VERSION: $(KUBERNETES_VERSION)
+          KIND_CLUSTER_NAME: $(KIND_CLUSTER_NAME)
+          KIND_NETWORK: $(KIND_NETWORK)
+
+      - template: cluster-health-template.yml
+      - template: kind-debug-template.yml
+
+      - script: make e2e-test
+        displayName: "Run e2e tests"
+
+      - script: |
+          echo "##vso[task.setvariable variable=LOCAL_IMAGE_VERSION]$(git rev-parse --short HEAD)"
+        displayName: "Update Image version"
+      
+      # This stage will upgrade kms plugin. The path (./tests/e2e/generated_manifests) is mounted in kind cluster.
+      # Any changes in the host will automatically be reflected in /etc/kubernetes/manifests mount path and that static pod is restarted with new changes.
+      # manifest-template updates these files with registry, imageName and version to desired upgrade values.
+      - template: manifest-template.yml
+        parameters:
+          registry: localhost:$(REGISTRY_PORT)
+          imageName: keyvault
+          imageVersion: e2e-$(LOCAL_IMAGE_VERSION)
+
+      - script: |
+          # wait for the kind network to exist
+          echo "waiting for upgraded kms pod to be Running"
+          for i in $(seq 1 25); do
+            image=$(kubectl get pods -n kube-system azure-kms-provider-kms-control-plane -o jsonpath="{.spec.containers[*].image}")
+            phase=$(kubectl get pods -n kube-system azure-kms-provider-kms-control-plane -o jsonpath="{.status.phase}")
+            echo "image - $image phase - $phase"
+            if [ "${image}" == "${REGISTRY}/${IMAGE_NAME}:e2e-${LOCAL_IMAGE_VERSION}" ] && [ "${phase}" == "Running" ]; then
+              break
+            else
+              sleep 5
+            fi
+          done
+        displayName: "Wait for kms upgrade"
+
+      - script: make e2e-test
+        displayName: "Run e2e tests"
+
+      - template: cleanup-template.yml
--- a/.pipelines/templates/kind-debug-template.yml
+++ b/.pipelines/templates/kind-debug-template.yml
@ -0,0 +1,10 @@
+steps:
+  - script: |
+      docker exec kms-control-plane bash -c "cat /etc/kubernetes/manifests/kubernetes-kms.yaml"
+      docker exec kms-control-plane bash -c "cat /etc/kubernetes/manifests/kube-apiserver.yaml"
+      docker exec kms-control-plane bash -c "cat /etc/kubernetes/encryption-config.yaml"
+      docker exec kms-control-plane bash -c "journalctl -u kubelet > kubelet.log && cat kubelet.log"
+      docker exec kms-control-plane bash -c "cd /var/log/containers ; cat *"
+      docker network ls
+    displayName: "Debug logs"
+    condition: failed()
--- a/.pipelines/templates/manifest-template.yml
+++ b/.pipelines/templates/manifest-template.yml
@ -0,0 +1,22 @@
+parameters:
+  - name: registry
+    type: string
+  - name: imageName
+    type: string
+  - name: imageVersion
+    type: string
+
+steps:
+  - script: |
+      export REGISTRY=${{ parameters.registry }}
+      export IMAGE_NAME=${{ parameters.imageName }}
+      export IMAGE_VERSION=${{ parameters.imageVersion }}
+
+      make e2e-generate-manifests
+
+      echo "##vso[task.setvariable variable=REGISTRY]${{ parameters.registry }}"
+      echo "##vso[task.setvariable variable=IMAGE_NAME]${{ parameters.imageName }}"
+    displayName: "Generate Manifests"
+    env:
+      CLIENT_ID: $(AZURE_CLIENT_ID)
+      CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
--- a/.pipelines/templates/scan-images-template.yml
+++ b/.pipelines/templates/scan-images-template.yml
--- a/.pipelines/templates/soak-test-template.yml
+++ b/.pipelines/templates/soak-test-template.yml
@ -0,0 +1,30 @@
+jobs:
+  - job: soak_test_aks_engine
+    timeoutInMinutes: 10
+    cancelTimeoutInMinutes: 5
+
+    workspace:
+      clean: all
+
+    variables:
+      - group: kubernetes-kms-soak-aks-engine
+
+    steps:
+      - script: make install-soak-prerequisites
+        displayName: "Install e2e soak test prerequisites"
+
+      - task: DownloadSecureFile@1
+        name: kubeconfig
+        inputs:
+          secureFile: kubeconfig
+        displayName: "Download KUBECONFIG"
+      
+      - script: |
+          export KUBECONFIG=$(kubeconfig.secureFilePath)
+          echo "##vso[task.setvariable variable=KUBECONFIG]${KUBECONFIG}"
+        displayName: "Set KUBECONFIG"
+
+      - template: cluster-health-template.yml
+
+      - script: make IS_SOAK_TEST=true e2e-test
+        displayName: "Run e2e tests"
--- a/.pipelines/templates/unit-tests-template.yml
+++ b/.pipelines/templates/unit-tests-template.yml
@ -34,4 +34,4 @@ jobs:
          sleep 2
          make integration-test
        displayName: Run integration tests
-      - template: scan-images.yml
+      - template: scan-images-template.yml
--- a/21
+++ b/21
@ -5,6 +5,8 @@ REPO_PATH="$(ORG_PATH)/$(PROJECT_NAME)"
 REGISTRY_NAME ?= upstreamk8sci
 REPO_PREFIX ?= oss/azure/kms
 REGISTRY ?= $(REGISTRY_NAME).azurecr.io/$(REPO_PREFIX)
+LOCAL_REGISTRY_NAME ?= kind-registry
+LOCAL_REGISTRY_PORT ?= 5000
 IMAGE_NAME ?= keyvault
 IMAGE_VERSION ?= v0.0.11
 IMAGE_TAG ?= $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)
@ -26,8 +28,8 @@ DOCKER_BUILDKIT = 1
 export DOCKER_BUILDKIT

 # Testing var
-KIND_VERSION ?= 0.8.1
-KUBERNETES_VERSION ?= v1.19.0
+KIND_VERSION ?= 0.11.0
+KUBERNETES_VERSION ?= v1.21.1
 BATS_VERSION ?= 1.2.1

 GO_BUILD_OPTIONS := --tags "netgo osusergo"  -ldflags "-s -X $(BUILD_VERSION_VAR)=$(IMAGE_VERSION) -X $(GIT_VAR)=$(GIT_HASH) -X $(BUILD_DATE_VAR)=$(BUILD_DATE) -extldflags '-static'"
@ -87,8 +89,19 @@ e2e-install-prerequisites:
 	# Download and install bats
 	curl -sSLO https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz && tar -zxvf v${BATS_VERSION}.tar.gz && sudo bash bats-core-${BATS_VERSION}/install.sh /usr/local

-e2e-setup-kind:
-	./scripts/ci-e2e-kind.sh
+.PHONY: install-soak-prerequisites
+install-soak-prerequisites: e2e-install-prerequisites
+	# Download and install node-shell
+	curl -LO https://github.com/kvaps/kubectl-node-shell/raw/master/kubectl-node_shell && chmod +x ./kubectl-node_shell && sudo mv ./kubectl-node_shell /usr/local/bin/kubectl-node_shell
+
+e2e-setup-kind: setup-local-registry
+	./scripts/setup-kind-cluster.sh &
+	./scripts/connect-registry.sh &
+	sleep 90s
+
+.PHONY: setup-local-registry
+setup-local-registry:
+	./scripts/setup-local-registry.sh

 e2e-generate-manifests:
 	@mkdir -p tests/e2e/generated_manifests
--- a/scripts/ci-e2e-kind.sh
+++ b/scripts/ci-e2e-kind.sh
@ -1,121 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-create_kind_cluster () {
-  # create a cluster with the local registry enabled in containerd
-  # add encryption config and the kms static pod manifest with custom image
-  cat <<EOF | kind create cluster --retain --image kindest/node:"${KUBERNETES_VERSION}" --name "${KIND_CLUSTER_NAME}" --wait 2m --config=-
-  kind: Cluster
-  apiVersion: kind.x-k8s.io/v1alpha4
-  containerdConfigPatches:
-  - |-
-    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
-      endpoint = ["http://${reg_host}:${reg_port}"]
-  nodes:
-  - role: control-plane
-    extraMounts:
-    - containerPath: /etc/kubernetes/encryption-config.yaml
-      hostPath: tests/e2e/encryption-config.yaml
-      readOnly: true
-      propagation: None
-    - containerPath: /etc/kubernetes/manifests/kubernetes-kms.yaml
-      hostPath: tests/e2e/generated_manifests/kms.yaml
-      readOnly: true
-      propagation: None
-    - containerPath: /etc/kubernetes/azure.json
-      hostPath: tests/e2e/generated_manifests/azure.json
-      readOnly: true
-      propagation: None
-    kubeadmConfigPatches:
-      - |
-        kind: ClusterConfiguration
-        apiServer:
-          extraArgs:
-            encryption-provider-config: "/etc/kubernetes/encryption-config.yaml"
-          extraVolumes:
-          - name: encryption-config
-            hostPath: "/etc/kubernetes/encryption-config.yaml"
-            mountPath: "/etc/kubernetes/encryption-config.yaml"
-            readOnly: true
-            pathType: File
-          - name: sock-path
-            hostPath: "/opt"
-            mountPath: "/opt"
-EOF
-}
-
-connect_registry () {
-  if [ "${kind_network}" != "bridge" ]; then
-    # wait for the kind network to exist
-    for i in $(seq 1 25); do
-      if docker network ls | grep "${kind_network}"; then
-        break
-      else
-        sleep 1
-      fi
-    done
-    containers=$(docker network inspect "${kind_network}" -f "{{range .Containers}}{{.Name}} {{end}}")
-    needs_connect="true"
-    for c in $containers; do
-      if [ "$c" = "${reg_name}" ]; then
-        needs_connect="false"
-      fi
-    done
-    if [ "${needs_connect}" = "true" ]; then
-      docker network connect "${kind_network}" "${reg_name}" || true
-    fi
-  fi
-}
-
-# desired cluster name; default is "kind"
-KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kms}"
-KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.19.0}"
-
-if kind get clusters | grep -q ^kms$ ; then
-  echo "cluster already exists, moving on"
-  exit 0
-fi
-
-# create registry container unless it already exists
-kind_version=$(kind version)
-kind_network='kind'
-reg_name='kind-registry'
-reg_port='5000'
-case "${kind_version}" in
-  "kind v0.7."* | "kind v0.6."* | "kind v0.5."*)
-    kind_network='bridge'
-    ;;
-esac
-
-# create registry container unless it already exists
-running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
-if [ "${running}" != 'true' ]; then
-  docker run \
-    -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \
-    registry:2
-fi
-
-reg_host="${reg_name}"
-if [ "${kind_network}" = "bridge" ]; then
-    reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
-fi
-echo "Registry Host: ${reg_host}"
-
-# Build and push kms image
-export REGISTRY=localhost:${reg_port}
-export IMAGE_NAME=keyvault
-export IMAGE_VERSION=e2e-$(git rev-parse --short HEAD)
-# push build image to local registry
-make push-image
-# generate kms plugin manifest and azure.json for testing
-make e2e-generate-manifests
-
-create_kind_cluster &
-# the registry needs to be connected to the network in parallel
-# so the image pull from local registry works. KMS plugin needs to
-# start for api-server to respond successfully to health check.
-connect_registry &
-wait
--- a/scripts/connect-registry.sh
+++ b/scripts/connect-registry.sh
@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [ "${KIND_NETWORK}" != "bridge" ]; then
+  # wait for the kind network to exist
+  for i in $(seq 1 25); do
+    if docker network ls | grep "${KIND_NETWORK}"; then
+      break
+    else
+      sleep 1
+    fi
+  done
+  containers=$(docker network inspect "${KIND_NETWORK}" -f "{{range .Containers}}{{.Name}} {{end}}")
+  needs_connect="true"
+  for c in $containers; do
+    if [ "$c" = "${REGISTRY_NAME}" ]; then
+      needs_connect="false"
+    fi
+  done
+  if [ "${needs_connect}" = "true" ]; then
+    echo "connecting ${KIND_NETWORK} network to ${REGISTRY_NAME}"
+    docker network connect "${KIND_NETWORK}" "${REGISTRY_NAME}" || true
+  fi
+fi
--- a/scripts/setup-kind-cluster.sh
+++ b/scripts/setup-kind-cluster.sh
@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# create a cluster with the local registry enabled in containerd
+# add encryption config and the kms static pod manifest with custom image
+cat <<EOF | kind create cluster --retain --image kindest/node:"${KUBERNETES_VERSION}" --name "${KIND_CLUSTER_NAME}" --wait 2m --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${REGISTRY_PORT}"]
+    endpoint = ["http://${REGISTRY_NAME}:${REGISTRY_PORT}"]
+nodes:
+- role: control-plane
+  extraMounts:
+  - containerPath: /etc/kubernetes/encryption-config.yaml
+    hostPath: tests/e2e/encryption-config.yaml
+    readOnly: true
+    propagation: None
+  - containerPath: /etc/kubernetes/manifests/kubernetes-kms.yaml
+    hostPath: tests/e2e/generated_manifests/kms.yaml
+    readOnly: true
+    propagation: None
+  - containerPath: /etc/kubernetes/azure.json
+    hostPath: tests/e2e/generated_manifests/azure.json
+    readOnly: true
+    propagation: None
+  kubeadmConfigPatches:
+    - |
+      kind: ClusterConfiguration
+      apiServer:
+        extraArgs:
+          encryption-provider-config: "/etc/kubernetes/encryption-config.yaml"
+        extraVolumes:
+        - name: encryption-config
+          hostPath: "/etc/kubernetes/encryption-config.yaml"
+          mountPath: "/etc/kubernetes/encryption-config.yaml"
+          readOnly: true
+          pathType: File
+        - name: sock-path
+          hostPath: "/opt"
+          mountPath: "/opt"
+EOF
--- a/scripts/setup-local-registry.sh
+++ b/scripts/setup-local-registry.sh
@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# create registry container unless it already exists
+running="$(docker inspect -f '{{.State.Running}}' "${REGISTRY_NAME}" 2>/dev/null || true)"
+if [ "${running}" != 'true' ]; then
+  echo "Creating local registry"
+  docker run \
+    -d --restart=always -p "${REGISTRY_PORT}:5000" --name "${REGISTRY_NAME}" \
+    registry:2
+fi
+
+# Build and push kms image
+export REGISTRY=localhost:${REGISTRY_PORT}
+export IMAGE_NAME=keyvault
+export IMAGE_VERSION=e2e-$(git rev-parse --short HEAD)
+
+# push build image to local registry
+echo "Pushing image to local registry"
+make push-image
+
+# generate manifest for local
+make e2e-generate-manifests
--- a/tests/e2e/test.bats
+++ b/tests/e2e/test.bats
@ -8,6 +8,15 @@ ETCD_CA_CERT=/etc/kubernetes/pki/etcd/ca.crt
 ETCD_CERT=/etc/kubernetes/pki/etcd/server.crt
 ETCD_KEY=/etc/kubernetes/pki/etcd/server.key

+setup_file() {
+    export IS_SOAK_TEST="${IS_SOAK_TEST}"
+    if [ ${IS_SOAK_TEST} = true ]; then
+      ETCD_CA_CERT=/etc/kubernetes/certs/ca.crt
+      ETCD_CERT=/etc/kubernetes/certs/etcdclient.crt
+      ETCD_KEY=/etc/kubernetes/certs/etcdclient.key
+    fi
+}
+
@test "azure keyvault kms plugin is running" {
    wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl -n kube-system wait --for=condition=Ready --timeout=60s pod -l component=azure-kms-provider"
 }
@ -23,10 +32,20 @@ ETCD_KEY=/etc/kubernetes/pki/etcd/server.key
 }

@test "check if secret is encrypted in etcd" {
-    local pod_name=$(kubectl get pod -n kube-system -l component=etcd -o jsonpath="{.items[0].metadata.name}")
-    run kubectl exec ${pod_name} -n kube-system -- etcdctl --cacert=${ETCD_CA_CERT} --cert=${ETCD_CERT} --key=${ETCD_KEY} get /registry/secrets/default/secret1
-    assert_match "k8s:enc:kms:v1:azurekmsprovider" "${output}"
-    assert_success
+    if [ ${IS_SOAK_TEST} = true ]; then
+      local node_name=$(kubectl get nodes -l kubernetes.azure.com/role=master -o jsonpath="{.items[0].metadata.name}")
+      run kubectl node-shell ${node_name} -- sh -c "ETCDCTL_API=3 etcdctl --cacert=${ETCD_CA_CERT} --cert=${ETCD_CERT} --key=${ETCD_KEY} get /registry/secrets/default/secret1"
+      assert_match "k8s:enc:kms:v1:azurekmsprovider" "${output}"
+      assert_success
+    else
+      local pod_name=$(kubectl get pod -n kube-system -l component=etcd -o jsonpath="{.items[0].metadata.name}")
+      run kubectl exec ${pod_name} -n kube-system -- etcdctl --cacert=${ETCD_CA_CERT} --cert=${ETCD_CERT} --key=${ETCD_KEY} get /registry/secrets/default/secret1
+      assert_match "k8s:enc:kms:v1:azurekmsprovider" "${output}"
+      assert_success
+    fi
+
+    # cleanup
+    run kubectl delete secret secret1 -n default
 }

@test "check healthz for kms plugin" {
@ -39,4 +58,7 @@ ETCD_KEY=/etc/kubernetes/pki/etcd/server.key

    result=$(kubectl exec curl -- curl http://${pod_ip}:8787/healthz -o /dev/null -w '%{http_code}\n' -s)
    [[ "${result//$'\r'}" == "200" ]]
+
+    # cleanup
+    run kubectl delete pod curl --force --grace-period 0
 }