test: add e2e tests with kind cluster (#75)

2021-02-08 17:54:31 -08:00 · 2021-02-08 17:54:31 -08:00 · 3f4a6c5c77
--- a/.gitignore
+++ b/.gitignore
@ -15,7 +15,6 @@
 setenv.sh
 kubernetes-kms
 vendor
-azure.json
 *.env

 # Vscode files
@ -26,3 +25,6 @@ azure.json

 .idea/
 _output/
+
+# e2e output
+tests/e2e/generated_manifests/*
--- a/.pipelines/e2e-kind-template.yml
+++ b/.pipelines/e2e-kind-template.yml
@ -0,0 +1,45 @@
+jobs:
+  - job: e2e_tests
+    timeoutInMinutes: 10
+    cancelTimeoutInMinutes: 5
+    workspace:
+      clean: all
+    variables:
+      - group: kubernetes-kms
+
+    steps:
+      - task: GoTool@0
+        inputs:
+          version: 1.15
+
+      - script: make e2e-install-prerequisites
+        displayName: "Install e2e test prerequisites"
+      - script: make e2e-setup-kind
+        displayName: "Setup kind cluster with azure kms plugin"
+        env:
+          CLIENT_ID: $(AZURE_CLIENT_ID)
+          CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
+      - script: |
+          kubectl wait --for=condition=ready node --all
+          kubectl wait pod -n kube-system --for=condition=Ready --all
+          kubectl get nodes -owide
+          kubectl cluster-info
+        displayName: "Check cluster's health"
+      - script: |
+          docker exec kms-control-plane bash -c "cat /etc/kubernetes/manifests/kubernetes-kms.yaml"
+          docker exec kms-control-plane bash -c "cat /etc/kubernetes/manifests/kube-apiserver.yaml"
+          docker exec kms-control-plane bash -c "cat /etc/kubernetes/encryption-config.yaml"
+          docker exec kms-control-plane bash -c "journalctl -u kubelet > kubelet.log && cat kubelet.log"
+
+          docker exec kms-control-plane bash -c "cd /var/log/containers ; cat *"
+          docker network ls
+        displayName: "Debug logs"
+        condition: failed()
+      - script: make e2e-test
+        displayName: "Run e2e tests"
+      - script: |
+          kubectl logs -l component=azure-kms-provider -n kube-system --tail -1
+          kubectl get pods -o wide -A
+        displayName: "Get logs"
+      - script: make e2e-delete-kind
+        displayName: "Delete cluster"
--- a/.pipelines/pr.yml
+++ b/.pipelines/pr.yml
@ -16,3 +16,4 @@ pool:

 jobs:
  - template: unit-tests-template.yml
+  - template: e2e-kind-template.yml
--- a/.pipelines/unit-tests-template.yml
+++ b/.pipelines/unit-tests-template.yml
@ -11,7 +11,7 @@ jobs:
      - task: GoTool@0
        inputs:
          version: 1.15
-      - script: V=1 make build
+      - script: make build
        displayName: Build
      - script: make unit-test
        displayName: Run unit tests
@ -20,7 +20,7 @@ jobs:
        displayName: Check binary version
      - script: |
          sudo mkdir /etc/kubernetes
-          echo -e '{\n    "tenantId": "'$TENANT_ID'",\n    "subscriptionId": "'$SUBSCRIPTION_ID'",\n    "aadClientId": "'$CLIENT_ID'",\n    "aadClientSecret": "'$CLIENT_SECRET'",\n    "resourceGroup": "'$KV_RESOURCE_GROUP'",\n    "location": "'$AZURE_LOCATION'",\n    "providerVaultName": "'$KV_NAME'",\n    "providerKeyName": "'$KV_KEY'",\n    "providerKeyVersion": "'$KV_KEY_VERSION'"\n}' | sudo tee --append /etc/kubernetes/azure.json  > /dev/null
+          echo -e '{\n    "tenantId": "'$TENANT_ID'",\n    "subscriptionId": "'$SUBSCRIPTION_ID'",\n    "aadClientId": "'$CLIENT_ID'",\n    "aadClientSecret": "'$CLIENT_SECRET'",\n}' | sudo tee --append /etc/kubernetes/azure.json  > /dev/null
        displayName: Setup azure.json on host
        env:
          CLIENT_ID: $(AZURE_CLIENT_ID)
--- a/57
+++ b/57
@ -21,27 +21,22 @@ GIT_HASH := $$(git rev-parse --short HEAD)
 DOCKER_BUILDKIT = 1
 export DOCKER_BUILDKIT

-ifeq ($(OS),Windows_NT)
-	GOOS_FLAG = windows
-else
-	UNAME_S := $(shell uname -s)
-	ifeq ($(UNAME_S), Linux)
-		GOOS_FLAG = linux
-	endif
-	ifeq ($(UNAME_S), Darwin)
-		GOOS_FLAG = darwin
-	endif
-endif
+# Testing var
+KIND_VERSION ?= 0.8.1
+KUBERNETES_VERSION ?= v1.19.0
+BATS_VERSION ?= 1.2.1

 GO_BUILD_OPTIONS := --tags "netgo osusergo"  -ldflags "-s -X $(BUILD_VERSION_VAR)=$(IMAGE_VERSION) -X $(GIT_VAR)=$(GIT_HASH) -X $(BUILD_DATE_VAR)=$(BUILD_DATE) -extldflags '-static'"

 .PHONY: build
-build: authors
-	@echo "Building..."
-	$Q GOOS=${GOOS_FLAG} CGO_ENABLED=${CGO_ENABLED_FLAG} go build $(GO_BUILD_OPTIONS) -o _output/kubernetes-kms ./cmd/server/
+build:
+	$Q GOOS=linux CGO_ENABLED=0 go build $(GO_BUILD_OPTIONS) -o _output/kubernetes-kms ./cmd/server/

-build-image: authors clean build
-	@echo "Building docker image..."
+.PHONY: build-darwin
+build-darwin:
+	$Q GOOS=darwin CGO_ENABLED=0 go build $(GO_BUILD_OPTIONS) -o _output/kubernetes-kms ./cmd/server/
+
+build-image: clean build
 	$Q docker build -t $(IMAGE_TAG) .

 push-image: build-image
@ -50,7 +45,6 @@ push-image: build-image
 .PHONY: clean unit-test integration-test

 clean:
-	@echo "Clean..."
 	$Q rm -rf _output/

 authors:
@ -61,13 +55,38 @@ authors:
 	$Q rm -f GITAUTHORS

 integration-test:
-	@echo "Running Integration tests..."
 	$Q sudo GOPATH=$(GOPATH) go test -v -count=1 github.com/Azure/kubernetes-kms/tests/client

 unit-test:
-	@echo "Running Unit Tests..."
 	go test -race -v -count=1 `go list ./... | grep -v client`

 .PHONY: mod
 mod:
 	@go mod tidy
+
+## --------------------------------------
+## E2E Testing
+## --------------------------------------
+e2e-install-prerequisites:
+	# Download and install kind
+	curl -L https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-linux-amd64 --output kind && chmod +x kind && sudo mv kind /usr/local/bin/
+	# Download and install kubectl
+	curl -LO https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl && chmod +x ./kubectl && sudo mv kubectl /usr/local/bin/
+	# Download and install bats
+	curl -sSLO https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz && tar -zxvf v${BATS_VERSION}.tar.gz && sudo bash bats-core-${BATS_VERSION}/install.sh /usr/local
+
+e2e-setup-kind:
+	./scripts/ci-e2e-kind.sh
+
+e2e-generate-manifests:
+	@mkdir -p tests/e2e/generated_manifests
+	envsubst < tests/e2e/azure.json > tests/e2e/generated_manifests/azure.json
+	envsubst < tests/e2e/kms.yaml > tests/e2e/generated_manifests/kms.yaml
+
+e2e-delete-kind:
+	# delete kind e2e cluster created for tests
+	kind delete cluster --name kms
+
+e2e-test:
+	# Run test suite with kind cluster
+	bats -t tests/e2e/test.bats
--- a/docs/testing.md
+++ b/docs/testing.md
@ -0,0 +1,51 @@
+# End-to-end testing for KMS Plugin for Keyvault
+
+## Prerequisites
+
+To run tests locally, following components are required:
+
+1. [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
+1. [bats](https://bats-core.readthedocs.io/en/latest/installation.html)
+1. [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation)
+
+To install the prerequisites, run the following command:
+
+```bash
+make e2e-install-prerequisites
+```
+
+The E2E test suite extracts runtime configurations through environment variables. Below is a list of environment variables to set before running the E2E test suite.
+| Variable      | Description                                                                                         |
+| ------------- | --------------------------------------------------------------------------------------------------- |
+| CLIENT_ID     | The client ID of your service principal that has `encrypt, decrypt` access to the keyvault key.     |
+| CLIENT_SECRET | The client secret of your service principal that has `encrypt, decrypt` access to the keyvault key. |
+| TENANT_ID     | The Azure tenant ID.                                                                                |
+| KEYVAULT_NAME | The Azure Keyvault name.                                                                            |
+| KEY_NAME      | The name of Keyvault key that will be used by the kms plugin.                                       |
+| KEY_VERSION   | The version of Keyvault key that will be used by the kms plugin.                                    |
+
+## Running the tests
+
+The e2e tests are run against a [kind](https://kind.sigs.k8s.io/) cluster that's created as part of the test script. The script also creates a local docker registry that's used for test images.
+
+1. Setup cluster, registry and build image:
+
+```bash
+make e2e-setup-kind
+```
+
+- This creates the local registry
+- Builds a kms plugin image with the latest changes and pushes to local registry
+- Creates a kind cluster with connectivity to local registry and kms plugin enabled with custom image
+
+1. Run the end-to-end tests:
+
+```bash
+make e2e-test
+```
+
+1. To delete the kind cluster after running tests:
+
+```bash
+make e2e-delete-kind
+```
--- a/pkg/config/azure_config.go
+++ b/pkg/config/azure_config.go
@ -14,10 +14,6 @@ type AzureConfig struct {
 	TenantID                    string `json:"tenantId" yaml:"tenantId"`
 	ClientID                    string `json:"aadClientId" yaml:"aadClientId"`
 	ClientSecret                string `json:"aadClientSecret" yaml:"aadClientSecret"`
-	SubscriptionID              string `json:"subscriptionId" yaml:"subscriptionId"`
-	ResourceGroupName           string `json:"resourceGroup" yaml:"resourceGroup"`
-	SecurityGroupName           string `json:"securityGroupName" yaml:"securityGroupName"`
-	VMType                      string `json:"vmType" yaml:"vmType"`
 	UseManagedIdentityExtension bool   `json:"useManagedIdentityExtension,omitempty" yaml:"useManagedIdentityExtension,omitempty"`
 	UserAssignedIdentityID      string `json:"userAssignedIdentityID,omitempty" yaml:"userAssignedIdentityID,omitempty"`
 	AADClientCertPath           string `json:"aadClientCertPath" yaml:"aadClientCertPath"`
--- a/pkg/plugin/keyvault.go
+++ b/pkg/plugin/keyvault.go
@ -98,6 +98,7 @@ func (kvc *keyVaultClient) Decrypt(ctx context.Context, plain []byte) ([]byte, e
 		Algorithm: kv.RSA15,
 		Value:     &value,
 	}
+
 	result, err := kvc.baseClient.Decrypt(ctx, kvc.vaultURL, kvc.keyName, kvc.keyVersion, params)
 	if err != nil {
 		return nil, fmt.Errorf("failed to decrypt, error: %+v", err)
--- a/scripts/ci-e2e-kind.sh
+++ b/scripts/ci-e2e-kind.sh
@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+create_kind_cluster () {
+  # create a cluster with the local registry enabled in containerd
+  # add encryption config and the kms static pod manifest with custom image
+  cat <<EOF | kind create cluster --retain --image kindest/node:"${KUBERNETES_VERSION}" --name "${KIND_CLUSTER_NAME}" --wait 2m --config=-
+  kind: Cluster
+  apiVersion: kind.x-k8s.io/v1alpha4
+  containerdConfigPatches:
+  - |-
+    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
+      endpoint = ["http://${reg_host}:${reg_port}"]
+  nodes:
+  - role: control-plane
+    extraMounts:
+    - containerPath: /etc/kubernetes/encryption-config.yaml
+      hostPath: tests/e2e/encryption-config.yaml
+      readOnly: true
+      propagation: None
+    - containerPath: /etc/kubernetes/manifests/kubernetes-kms.yaml
+      hostPath: tests/e2e/generated_manifests/kms.yaml
+      readOnly: true
+      propagation: None
+    - containerPath: /etc/kubernetes/azure.json
+      hostPath: tests/e2e/generated_manifests/azure.json
+      readOnly: true
+      propagation: None
+    kubeadmConfigPatches:
+      - |
+        kind: ClusterConfiguration
+        apiServer:
+          extraArgs:
+            encryption-provider-config: "/etc/kubernetes/encryption-config.yaml"
+          extraVolumes:
+          - name: encryption-config
+            hostPath: "/etc/kubernetes/encryption-config.yaml"
+            mountPath: "/etc/kubernetes/encryption-config.yaml"
+            readOnly: true
+            pathType: File
+          - name: sock-path
+            hostPath: "/opt"
+            mountPath: "/opt"
+EOF
+}
+
+connect_registry () {
+  if [ "${kind_network}" != "bridge" ]; then
+    # wait for the kind network to exist
+    for i in $(seq 1 25); do
+      if docker network ls | grep "${kind_network}"; then
+        break
+      else
+        sleep 1
+      fi
+    done
+    containers=$(docker network inspect "${kind_network}" -f "{{range .Containers}}{{.Name}} {{end}}")
+    needs_connect="true"
+    for c in $containers; do
+      if [ "$c" = "${reg_name}" ]; then
+        needs_connect="false"
+      fi
+    done
+    if [ "${needs_connect}" = "true" ]; then
+      docker network connect "${kind_network}" "${reg_name}" || true
+    fi
+  fi
+}
+
+# desired cluster name; default is "kind"
+KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kms}"
+KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.19.0}"
+
+if kind get clusters | grep -q ^kms$ ; then
+  echo "cluster already exists, moving on"
+  exit 0
+fi
+
+# create registry container unless it already exists
+kind_version=$(kind version)
+kind_network='kind'
+reg_name='kind-registry'
+reg_port='5000'
+case "${kind_version}" in
+  "kind v0.7."* | "kind v0.6."* | "kind v0.5."*)
+    kind_network='bridge'
+    ;;
+esac
+
+# create registry container unless it already exists
+running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
+if [ "${running}" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \
+    registry:2
+fi
+
+reg_host="${reg_name}"
+if [ "${kind_network}" = "bridge" ]; then
+    reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")"
+fi
+echo "Registry Host: ${reg_host}"
+
+# Build and push kms image
+export REGISTRY=localhost:${reg_port}
+export IMAGE_NAME=keyvault
+export IMAGE_VERSION=e2e-$(git rev-parse --short HEAD)
+# push build image to local registry
+make push-image
+# generate kms plugin manifest and azure.json for testing
+make e2e-generate-manifests
+
+create_kind_cluster &
+# the registry needs to be connected to the network in parallel
+# so the image pull from local registry works. KMS plugin needs to
+# start for api-server to respond successfully to health check.
+connect_registry &
+wait
--- a/tests/e2e/azure.json
+++ b/tests/e2e/azure.json
@ -0,0 +1,6 @@
+{
+    "cloud": "AzurePublicCloud",
+    "tenantId": "$TENANT_ID",
+    "aadClientId": "$CLIENT_ID",
+    "aadClientSecret": "$CLIENT_SECRET"
+}
--- a/tests/e2e/encryption-config.yaml
+++ b/tests/e2e/encryption-config.yaml
@ -0,0 +1,11 @@
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+      - secrets
+    providers:
+      - kms:
+          name: azurekmsprovider
+          endpoint: unix:///opt/azurekms.socket
+          cachesize: 1000
+      - identity: {}
--- a/tests/e2e/helpers.bash
+++ b/tests/e2e/helpers.bash
@ -0,0 +1,41 @@
+#!/bin/bash
+
+assert_success() {
+  if [[ "$status" != 0 ]]; then
+    echo "expected: 0"
+    echo "actual: $status"
+    echo "output: $output"
+    return 1
+  fi
+}
+
+assert_equal() {
+  if [[ "$1" != "$2" ]]; then
+    echo "expected: $1"
+    echo "actual: $2"
+    return 1
+  fi
+}
+
+assert_match() {
+  if [[ ! "$2" =~ $1 ]]; then
+    echo "expected: $1"
+    echo "actual: $2"
+    return 1
+  fi
+}
+
+wait_for_process() {
+  wait_time="$1"
+  sleep_time="$2"
+  cmd="$3"
+  while [ "$wait_time" -gt 0 ]; do
+    if eval "$cmd"; then
+      return 0
+    else
+      sleep "$sleep_time"
+      wait_time=$((wait_time - sleep_time))
+    fi
+  done
+  return 1
+}
--- a/tests/e2e/kms.yaml
+++ b/tests/e2e/kms.yaml
@ -0,0 +1,45 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: azure-kms-provider
+  namespace: kube-system
+  labels:
+    tier: control-plane
+    component: azure-kms-provider
+spec:
+  priorityClassName: system-node-critical
+  hostNetwork: true
+  containers:
+    - name: azure-kms-provider
+      image: ${REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION}
+      imagePullPolicy: IfNotPresent
+      args:
+        - --keyvault-name=${KEYVAULT_NAME}
+        - --key-name=${KEY_NAME}
+        - --key-version=${KEY_VERSION}
+        - -v=2
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: 4
+          memory: 2Gi
+      volumeMounts:
+        - name: etc-kubernetes
+          mountPath: /etc/kubernetes
+        - name: etc-ssl
+          mountPath: /etc/ssl
+          readOnly: true
+        - name: sock
+          mountPath: /opt
+  volumes:
+    - name: etc-kubernetes
+      hostPath:
+        path: /etc/kubernetes
+    - name: etc-ssl
+      hostPath:
+        path: /etc/ssl
+    - name: sock
+      hostPath:
+        path: /opt
--- a/tests/e2e/test.bats
+++ b/tests/e2e/test.bats
@ -0,0 +1,30 @@
+#!/usr/bin/env bats
+
+load helpers
+
+WAIT_TIME=120
+SLEEP_TIME=1
+ETCD_CA_CERT=/etc/kubernetes/pki/etcd/ca.crt
+ETCD_CERT=/etc/kubernetes/pki/etcd/server.crt
+ETCD_KEY=/etc/kubernetes/pki/etcd/server.key
+
+@test "azure keyvault kms plugin is running" {
+    wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl -n kube-system wait --for=condition=Ready --timeout=60s pod -l component=azure-kms-provider"
+}
+
+@test "creating secret resource" {
+    run kubectl create secret generic secret1 -n default --from-literal=foo=bar
+    assert_success
+}
+
+@test "read the secret resource test" {
+    result=$(kubectl get secret secret1 -o jsonpath='{.data.foo}' | base64 -d)
+    [[ "${result//$'\r'}" == "bar" ]]
+}
+
+@test "check if secret is encrypted in etcd" {
+    local pod_name=$(kubectl get pod -n kube-system -l component=etcd -o jsonpath="{.items[0].metadata.name}")
+    run kubectl exec ${pod_name} -n kube-system -- etcdctl --cacert=${ETCD_CA_CERT} --cert=${ETCD_CERT} --key=${ETCD_KEY} get /registry/secrets/default/secret1
+    assert_match "k8s:enc:kms:v1:azurekmsprovider" "${output}"
+    assert_success
+}