From 3f4a6c5c77d6f53a756fc2c7423eb0c7a14f83c8 Mon Sep 17 00:00:00 2001 From: Anish Ramasekar Date: Mon, 8 Feb 2021 17:54:31 -0800 Subject: [PATCH] test: add e2e tests with kind cluster (#75) --- .gitignore | 4 +- .pipelines/e2e-kind-template.yml | 45 +++++++++++ .pipelines/pr.yml | 1 + .pipelines/unit-tests-template.yml | 4 +- Makefile | 57 +++++++++----- docs/testing.md | 51 ++++++++++++ pkg/config/azure_config.go | 4 - pkg/plugin/keyvault.go | 1 + scripts/ci-e2e-kind.sh | 121 +++++++++++++++++++++++++++++ tests/e2e/azure.json | 6 ++ tests/e2e/encryption-config.yaml | 11 +++ tests/e2e/helpers.bash | 41 ++++++++++ tests/e2e/kms.yaml | 45 +++++++++++ tests/e2e/test.bats | 30 +++++++ 14 files changed, 395 insertions(+), 26 deletions(-) create mode 100644 .pipelines/e2e-kind-template.yml create mode 100644 docs/testing.md create mode 100755 scripts/ci-e2e-kind.sh create mode 100644 tests/e2e/azure.json create mode 100644 tests/e2e/encryption-config.yaml create mode 100644 tests/e2e/helpers.bash create mode 100644 tests/e2e/kms.yaml create mode 100644 tests/e2e/test.bats diff --git a/.gitignore b/.gitignore index 3fc248a..99d3293 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ setenv.sh kubernetes-kms vendor -azure.json *.env # Vscode files @@ -26,3 +25,6 @@ azure.json .idea/ _output/ + +# e2e output +tests/e2e/generated_manifests/* diff --git a/.pipelines/e2e-kind-template.yml b/.pipelines/e2e-kind-template.yml new file mode 100644 index 0000000..70897a5 --- /dev/null +++ b/.pipelines/e2e-kind-template.yml @@ -0,0 +1,45 @@ +jobs: + - job: e2e_tests + timeoutInMinutes: 10 + cancelTimeoutInMinutes: 5 + workspace: + clean: all + variables: + - group: kubernetes-kms + + steps: + - task: GoTool@0 + inputs: + version: 1.15 + + - script: make e2e-install-prerequisites + displayName: "Install e2e test prerequisites" + - script: make e2e-setup-kind + displayName: "Setup kind cluster with azure kms plugin" + env: + CLIENT_ID: $(AZURE_CLIENT_ID) + CLIENT_SECRET: $(AZURE_CLIENT_SECRET) + - script: | + kubectl wait --for=condition=ready node --all + kubectl wait pod -n kube-system --for=condition=Ready --all + kubectl get nodes -owide + kubectl cluster-info + displayName: "Check cluster's health" + - script: | + docker exec kms-control-plane bash -c "cat /etc/kubernetes/manifests/kubernetes-kms.yaml" + docker exec kms-control-plane bash -c "cat /etc/kubernetes/manifests/kube-apiserver.yaml" + docker exec kms-control-plane bash -c "cat /etc/kubernetes/encryption-config.yaml" + docker exec kms-control-plane bash -c "journalctl -u kubelet > kubelet.log && cat kubelet.log" + + docker exec kms-control-plane bash -c "cd /var/log/containers ; cat *" + docker network ls + displayName: "Debug logs" + condition: failed() + - script: make e2e-test + displayName: "Run e2e tests" + - script: | + kubectl logs -l component=azure-kms-provider -n kube-system --tail -1 + kubectl get pods -o wide -A + displayName: "Get logs" + - script: make e2e-delete-kind + displayName: "Delete cluster" diff --git a/.pipelines/pr.yml b/.pipelines/pr.yml index 64bf05f..dfbc0a2 100644 --- a/.pipelines/pr.yml +++ b/.pipelines/pr.yml @@ -16,3 +16,4 @@ pool: jobs: - template: unit-tests-template.yml + - template: e2e-kind-template.yml diff --git a/.pipelines/unit-tests-template.yml b/.pipelines/unit-tests-template.yml index 7eb56d5..94096c5 100644 --- a/.pipelines/unit-tests-template.yml +++ b/.pipelines/unit-tests-template.yml @@ -11,7 +11,7 @@ jobs: - task: GoTool@0 inputs: version: 1.15 - - script: V=1 make build + - script: make build displayName: Build - script: make unit-test displayName: Run unit tests @@ -20,7 +20,7 @@ jobs: displayName: Check binary version - script: | sudo mkdir /etc/kubernetes - echo -e '{\n "tenantId": "'$TENANT_ID'",\n "subscriptionId": "'$SUBSCRIPTION_ID'",\n "aadClientId": "'$CLIENT_ID'",\n "aadClientSecret": "'$CLIENT_SECRET'",\n "resourceGroup": "'$KV_RESOURCE_GROUP'",\n "location": "'$AZURE_LOCATION'",\n "providerVaultName": "'$KV_NAME'",\n "providerKeyName": "'$KV_KEY'",\n "providerKeyVersion": "'$KV_KEY_VERSION'"\n}' | sudo tee --append /etc/kubernetes/azure.json > /dev/null + echo -e '{\n "tenantId": "'$TENANT_ID'",\n "subscriptionId": "'$SUBSCRIPTION_ID'",\n "aadClientId": "'$CLIENT_ID'",\n "aadClientSecret": "'$CLIENT_SECRET'",\n}' | sudo tee --append /etc/kubernetes/azure.json > /dev/null displayName: Setup azure.json on host env: CLIENT_ID: $(AZURE_CLIENT_ID) diff --git a/Makefile b/Makefile index 6e8d2a4..112a37d 100644 --- a/Makefile +++ b/Makefile @@ -21,27 +21,22 @@ GIT_HASH := $$(git rev-parse --short HEAD) DOCKER_BUILDKIT = 1 export DOCKER_BUILDKIT -ifeq ($(OS),Windows_NT) - GOOS_FLAG = windows -else - UNAME_S := $(shell uname -s) - ifeq ($(UNAME_S), Linux) - GOOS_FLAG = linux - endif - ifeq ($(UNAME_S), Darwin) - GOOS_FLAG = darwin - endif -endif +# Testing var +KIND_VERSION ?= 0.8.1 +KUBERNETES_VERSION ?= v1.19.0 +BATS_VERSION ?= 1.2.1 GO_BUILD_OPTIONS := --tags "netgo osusergo" -ldflags "-s -X $(BUILD_VERSION_VAR)=$(IMAGE_VERSION) -X $(GIT_VAR)=$(GIT_HASH) -X $(BUILD_DATE_VAR)=$(BUILD_DATE) -extldflags '-static'" .PHONY: build -build: authors - @echo "Building..." - $Q GOOS=${GOOS_FLAG} CGO_ENABLED=${CGO_ENABLED_FLAG} go build $(GO_BUILD_OPTIONS) -o _output/kubernetes-kms ./cmd/server/ +build: + $Q GOOS=linux CGO_ENABLED=0 go build $(GO_BUILD_OPTIONS) -o _output/kubernetes-kms ./cmd/server/ -build-image: authors clean build - @echo "Building docker image..." +.PHONY: build-darwin +build-darwin: + $Q GOOS=darwin CGO_ENABLED=0 go build $(GO_BUILD_OPTIONS) -o _output/kubernetes-kms ./cmd/server/ + +build-image: clean build $Q docker build -t $(IMAGE_TAG) . push-image: build-image @@ -50,7 +45,6 @@ push-image: build-image .PHONY: clean unit-test integration-test clean: - @echo "Clean..." $Q rm -rf _output/ authors: @@ -61,13 +55,38 @@ authors: $Q rm -f GITAUTHORS integration-test: - @echo "Running Integration tests..." $Q sudo GOPATH=$(GOPATH) go test -v -count=1 github.com/Azure/kubernetes-kms/tests/client unit-test: - @echo "Running Unit Tests..." go test -race -v -count=1 `go list ./... | grep -v client` .PHONY: mod mod: @go mod tidy + +## -------------------------------------- +## E2E Testing +## -------------------------------------- +e2e-install-prerequisites: + # Download and install kind + curl -L https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-linux-amd64 --output kind && chmod +x kind && sudo mv kind /usr/local/bin/ + # Download and install kubectl + curl -LO https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl && chmod +x ./kubectl && sudo mv kubectl /usr/local/bin/ + # Download and install bats + curl -sSLO https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz && tar -zxvf v${BATS_VERSION}.tar.gz && sudo bash bats-core-${BATS_VERSION}/install.sh /usr/local + +e2e-setup-kind: + ./scripts/ci-e2e-kind.sh + +e2e-generate-manifests: + @mkdir -p tests/e2e/generated_manifests + envsubst < tests/e2e/azure.json > tests/e2e/generated_manifests/azure.json + envsubst < tests/e2e/kms.yaml > tests/e2e/generated_manifests/kms.yaml + +e2e-delete-kind: + # delete kind e2e cluster created for tests + kind delete cluster --name kms + +e2e-test: + # Run test suite with kind cluster + bats -t tests/e2e/test.bats diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..d8076f3 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,51 @@ +# End-to-end testing for KMS Plugin for Keyvault + +## Prerequisites + +To run tests locally, following components are required: + +1. [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +1. [bats](https://bats-core.readthedocs.io/en/latest/installation.html) +1. [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) + +To install the prerequisites, run the following command: + +```bash +make e2e-install-prerequisites +``` + +The E2E test suite extracts runtime configurations through environment variables. Below is a list of environment variables to set before running the E2E test suite. +| Variable | Description | +| ------------- | --------------------------------------------------------------------------------------------------- | +| CLIENT_ID | The client ID of your service principal that has `encrypt, decrypt` access to the keyvault key. | +| CLIENT_SECRET | The client secret of your service principal that has `encrypt, decrypt` access to the keyvault key. | +| TENANT_ID | The Azure tenant ID. | +| KEYVAULT_NAME | The Azure Keyvault name. | +| KEY_NAME | The name of Keyvault key that will be used by the kms plugin. | +| KEY_VERSION | The version of Keyvault key that will be used by the kms plugin. | + +## Running the tests + +The e2e tests are run against a [kind](https://kind.sigs.k8s.io/) cluster that's created as part of the test script. The script also creates a local docker registry that's used for test images. + +1. Setup cluster, registry and build image: + +```bash +make e2e-setup-kind +``` + +- This creates the local registry +- Builds a kms plugin image with the latest changes and pushes to local registry +- Creates a kind cluster with connectivity to local registry and kms plugin enabled with custom image + +1. Run the end-to-end tests: + +```bash +make e2e-test +``` + +1. To delete the kind cluster after running tests: + +```bash +make e2e-delete-kind +``` diff --git a/pkg/config/azure_config.go b/pkg/config/azure_config.go index c6b4c02..1b84553 100644 --- a/pkg/config/azure_config.go +++ b/pkg/config/azure_config.go @@ -14,10 +14,6 @@ type AzureConfig struct { TenantID string `json:"tenantId" yaml:"tenantId"` ClientID string `json:"aadClientId" yaml:"aadClientId"` ClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"` - SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"` - ResourceGroupName string `json:"resourceGroup" yaml:"resourceGroup"` - SecurityGroupName string `json:"securityGroupName" yaml:"securityGroupName"` - VMType string `json:"vmType" yaml:"vmType"` UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty" yaml:"useManagedIdentityExtension,omitempty"` UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty" yaml:"userAssignedIdentityID,omitempty"` AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"` diff --git a/pkg/plugin/keyvault.go b/pkg/plugin/keyvault.go index 91d45d6..a8dc161 100644 --- a/pkg/plugin/keyvault.go +++ b/pkg/plugin/keyvault.go @@ -98,6 +98,7 @@ func (kvc *keyVaultClient) Decrypt(ctx context.Context, plain []byte) ([]byte, e Algorithm: kv.RSA15, Value: &value, } + result, err := kvc.baseClient.Decrypt(ctx, kvc.vaultURL, kvc.keyName, kvc.keyVersion, params) if err != nil { return nil, fmt.Errorf("failed to decrypt, error: %+v", err) diff --git a/scripts/ci-e2e-kind.sh b/scripts/ci-e2e-kind.sh new file mode 100755 index 0000000..8e365d9 --- /dev/null +++ b/scripts/ci-e2e-kind.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +create_kind_cluster () { + # create a cluster with the local registry enabled in containerd + # add encryption config and the kms static pod manifest with custom image + cat </dev/null || true)" +if [ "${running}" != 'true' ]; then + docker run \ + -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \ + registry:2 +fi + +reg_host="${reg_name}" +if [ "${kind_network}" = "bridge" ]; then + reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")" +fi +echo "Registry Host: ${reg_host}" + +# Build and push kms image +export REGISTRY=localhost:${reg_port} +export IMAGE_NAME=keyvault +export IMAGE_VERSION=e2e-$(git rev-parse --short HEAD) +# push build image to local registry +make push-image +# generate kms plugin manifest and azure.json for testing +make e2e-generate-manifests + +create_kind_cluster & +# the registry needs to be connected to the network in parallel +# so the image pull from local registry works. KMS plugin needs to +# start for api-server to respond successfully to health check. +connect_registry & +wait diff --git a/tests/e2e/azure.json b/tests/e2e/azure.json new file mode 100644 index 0000000..924107b --- /dev/null +++ b/tests/e2e/azure.json @@ -0,0 +1,6 @@ +{ + "cloud": "AzurePublicCloud", + "tenantId": "$TENANT_ID", + "aadClientId": "$CLIENT_ID", + "aadClientSecret": "$CLIENT_SECRET" +} diff --git a/tests/e2e/encryption-config.yaml b/tests/e2e/encryption-config.yaml new file mode 100644 index 0000000..f493bab --- /dev/null +++ b/tests/e2e/encryption-config.yaml @@ -0,0 +1,11 @@ +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - kms: + name: azurekmsprovider + endpoint: unix:///opt/azurekms.socket + cachesize: 1000 + - identity: {} diff --git a/tests/e2e/helpers.bash b/tests/e2e/helpers.bash new file mode 100644 index 0000000..e1647ea --- /dev/null +++ b/tests/e2e/helpers.bash @@ -0,0 +1,41 @@ +#!/bin/bash + +assert_success() { + if [[ "$status" != 0 ]]; then + echo "expected: 0" + echo "actual: $status" + echo "output: $output" + return 1 + fi +} + +assert_equal() { + if [[ "$1" != "$2" ]]; then + echo "expected: $1" + echo "actual: $2" + return 1 + fi +} + +assert_match() { + if [[ ! "$2" =~ $1 ]]; then + echo "expected: $1" + echo "actual: $2" + return 1 + fi +} + +wait_for_process() { + wait_time="$1" + sleep_time="$2" + cmd="$3" + while [ "$wait_time" -gt 0 ]; do + if eval "$cmd"; then + return 0 + else + sleep "$sleep_time" + wait_time=$((wait_time - sleep_time)) + fi + done + return 1 +} diff --git a/tests/e2e/kms.yaml b/tests/e2e/kms.yaml new file mode 100644 index 0000000..0aa22da --- /dev/null +++ b/tests/e2e/kms.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Pod +metadata: + name: azure-kms-provider + namespace: kube-system + labels: + tier: control-plane + component: azure-kms-provider +spec: + priorityClassName: system-node-critical + hostNetwork: true + containers: + - name: azure-kms-provider + image: ${REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION} + imagePullPolicy: IfNotPresent + args: + - --keyvault-name=${KEYVAULT_NAME} + - --key-name=${KEY_NAME} + - --key-version=${KEY_VERSION} + - -v=2 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 4 + memory: 2Gi + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + - name: sock + mountPath: /opt + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + - name: sock + hostPath: + path: /opt diff --git a/tests/e2e/test.bats b/tests/e2e/test.bats new file mode 100644 index 0000000..7e59212 --- /dev/null +++ b/tests/e2e/test.bats @@ -0,0 +1,30 @@ +#!/usr/bin/env bats + +load helpers + +WAIT_TIME=120 +SLEEP_TIME=1 +ETCD_CA_CERT=/etc/kubernetes/pki/etcd/ca.crt +ETCD_CERT=/etc/kubernetes/pki/etcd/server.crt +ETCD_KEY=/etc/kubernetes/pki/etcd/server.key + +@test "azure keyvault kms plugin is running" { + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl -n kube-system wait --for=condition=Ready --timeout=60s pod -l component=azure-kms-provider" +} + +@test "creating secret resource" { + run kubectl create secret generic secret1 -n default --from-literal=foo=bar + assert_success +} + +@test "read the secret resource test" { + result=$(kubectl get secret secret1 -o jsonpath='{.data.foo}' | base64 -d) + [[ "${result//$'\r'}" == "bar" ]] +} + +@test "check if secret is encrypted in etcd" { + local pod_name=$(kubectl get pod -n kube-system -l component=etcd -o jsonpath="{.items[0].metadata.name}") + run kubectl exec ${pod_name} -n kube-system -- etcdctl --cacert=${ETCD_CA_CERT} --cert=${ETCD_CERT} --key=${ETCD_KEY} get /registry/secrets/default/secret1 + assert_match "k8s:enc:kms:v1:azurekmsprovider" "${output}" + assert_success +}