add etcd encryption at rest (#1973)

* add etcd encryption at rest * changed to EnableDataEncryptionAtRest *bool
2018-01-03 00:54:24 +00:00 · 2018-01-03 00:54:24 +00:00 · a30d445417
--- a/docs/clusterdefinition.md
+++ b/docs/clusterdefinition.md
@ -39,6 +39,7 @@ Here are the valid values for the orchestrator types:
 |serviceCidr|no|IP range for Service IPs, Default is "10.0.0.0/16". This range is never routed outside of a node so does not need to lie within clusterSubnet or the VNet.|
 |enableRbac|no|Enable [Kubernetes RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) (boolean - default == true) |
 |enableAggregatedAPIs|no|Enable [Kubernetes Aggregated APIs](https://kubernetes.io/docs/concepts/api-extension/apiserver-aggregation/).This is required by [Service Catalog](https://github.com/kubernetes-incubator/service-catalog/blob/master/README.md). (boolean - default == false) |
+|enableDataEncryptionAtRest|no|Enable [kuberetes data encryption at rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/).This is currently an alpha feature. (boolean - default == false) |
 |maxPods|no|The maximum number of pods per node. The minimum valid value, necessary for running kube-system pods, is 5. Default value is 30 when networkPolicy equals azure, 110 otherwise.|
 |gcHighThreshold|no|Sets the --image-gc-high-threshold value on the kublet configuration. Default is 85. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) |
 |gcLowThreshold|no|Sets the --image-gc-low-threshold value on the kublet configuration. Default is 80. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) |
--- a/examples/kubernetes-config/kubernetes-data-encryption-at-rest.json
+++ b/examples/kubernetes-config/kubernetes-data-encryption-at-rest.json
@ -0,0 +1,38 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "kubernetesConfig": {
+        "enableDataEncryptionAtRest": true
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 1,
+        "vmSize": "Standard_D2_v2",
+        "availabilityProfile": "AvailabilitySet"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}
--- a/parts/k8s/kubernetesmastercustomdata.yml
+++ b/parts/k8s/kubernetesmastercustomdata.yml
@ -86,6 +86,26 @@ write_files:
      name: localclustercontext
    current-context: localclustercontext

+{{if EnableDataEncryptionAtRest}}
+- path: "/etc/kubernetes/encryption-config.yaml"
+  permissions: "0600"
+  owner: "root"
+  content: |
+    apiVersion: v1
+    kind: Config
+    kind: EncryptionConfig
+    apiVersion: v1
+    resources:
+      - resources:
+          - secrets
+        providers:
+          - aescbc:
+              keys:
+                - name: key1
+                  secret: <etcdEncryptionSecret>
+          - identity: {}
+{{end}}
+
 MASTER_MANIFESTS_CONFIG_PLACEHOLDER

 MASTER_ADDONS_CONFIG_PLACEHOLDER
@ -232,6 +252,14 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
    sed -i "/<kubernetesEnableRbac>/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
 {{end}}

+{{if EnableDataEncryptionAtRest }}
+    ETCD_ENCRYPTION_SECRET="$(head -c 32 /dev/urandom | base64)"
+    sed -i "s|<etcdEncryptionSecret>|$ETCD_ENCRYPTION_SECRET|g" "/etc/kubernetes/encryption-config.yaml"
+    sed -i "s|<kubernetesEnableEtcdEncryption>|--experimental-encryption-provider-config=/etc/kubernetes/encryption-config.yaml|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
+{{else}}
+    sed -i "/<kubernetesEnableEtcdEncryption>/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
+{{end}}
+
 {{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}}
    # If Calico Policy enabled then update Cluster Cidr
    sed -i "s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml"
@ -392,4 +420,4 @@ runcmd:
 {{end}}
 - apt-mark unhold walinuxagent
 - touch /opt/azure/containers/runcmd.complete
-{{end}}
+{{end}}
--- a/parts/k8s/manifests/kubernetesmaster-kube-apiserver.yaml
+++ b/parts/k8s/manifests/kubernetesmaster-kube-apiserver.yaml
@ -43,6 +43,7 @@ spec:
        - "--storage-backend=<etcdApiVersion>"
        - "--v=4"
        - "<kubernetesEnableRbac>"
+        - "<kubernetesEnableEtcdEncryption>"
        - "--requestheader-allowed-names="
        - "--requestheader-extra-headers-prefix=X-Remote-Extra-"
        - "--requestheader-group-headers=X-Remote-Group"
--- a/pkg/acsengine/engine.go
+++ b/pkg/acsengine/engine.go
@ -1500,6 +1500,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 		"UseCloudControllerManager": func() bool {
 			return cs.Properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager != nil && *cs.Properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager
 		},
+		"EnableDataEncryptionAtRest": func() bool {
+			return helpers.IsTrueBoolPointer(cs.Properties.OrchestratorProfile.KubernetesConfig.EnableDataEncryptionAtRest)
+		},
 		// inspired by http://stackoverflow.com/questions/18276173/calling-a-template-with-several-pipeline-parameters/18276968#18276968
 		"dict": func(values ...interface{}) (map[string]interface{}, error) {
 			if len(values)%2 != 0 {
--- a/pkg/api/converterfromapi.go
+++ b/pkg/api/converterfromapi.go
@ -666,6 +666,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
 	vlabs.UseInstanceMetadata = api.UseInstanceMetadata
 	vlabs.EnableRbac = api.EnableRbac
 	vlabs.EnableAggregatedAPIs = api.EnableAggregatedAPIs
+	vlabs.EnableDataEncryptionAtRest = api.EnableDataEncryptionAtRest
 	vlabs.GCHighThreshold = api.GCHighThreshold
 	vlabs.GCLowThreshold = api.GCLowThreshold
 	vlabs.EtcdVersion = api.EtcdVersion
--- a/pkg/api/convertertoapi.go
+++ b/pkg/api/convertertoapi.go
@ -610,6 +610,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	api.UseInstanceMetadata = vlabs.UseInstanceMetadata
 	api.EnableRbac = vlabs.EnableRbac
 	api.EnableAggregatedAPIs = vlabs.EnableAggregatedAPIs
+	api.EnableDataEncryptionAtRest = vlabs.EnableDataEncryptionAtRest
 	api.GCHighThreshold = vlabs.GCHighThreshold
 	api.GCLowThreshold = vlabs.GCLowThreshold
 	api.EtcdVersion = vlabs.EtcdVersion
--- a/pkg/api/types.go
+++ b/pkg/api/types.go
@ -224,6 +224,7 @@ type KubernetesConfig struct {
 	GCLowThreshold               int               `json:"gclowthreshold,omitempty"`
 	EtcdVersion                  string            `json:"etcdVersion,omitempty"`
 	EtcdDiskSizeGB               string            `json:"etcdDiskSizeGB,omitempty"`
+	EnableDataEncryptionAtRest   *bool             `json:"enableDataEncryptionAtRest,omitempty"`
 	Addons                       []KubernetesAddon `json:"addons,omitempty"`
 	KubeletConfig                map[string]string `json:"kubeletConfig,omitempty"`
 	ControllerManagerConfig      map[string]string `json:"controllerManagerConfig,omitempty"`
--- a/pkg/api/vlabs/types.go
+++ b/pkg/api/vlabs/types.go
@ -242,6 +242,7 @@ type KubernetesConfig struct {
 	GCLowThreshold               int               `json:"gclowthreshold,omitempty"`
 	EtcdVersion                  string            `json:"etcdVersion,omitempty"`
 	EtcdDiskSizeGB               string            `json:"etcdDiskSizeGB,omitempty"`
+	EnableDataEncryptionAtRest   *bool             `json:"enableDataEncryptionAtRest,omitempty"`
 	Addons                       []KubernetesAddon `json:"addons,omitempty"`
 	KubeletConfig                map[string]string `json:"kubeletConfig,omitempty"`
 	ControllerManagerConfig      map[string]string `json:"controllerManagerConfig,omitempty"`
--- a/pkg/api/vlabs/validate.go
+++ b/pkg/api/vlabs/validate.go
@ -10,6 +10,7 @@ import (
 	"time"

 	"github.com/Azure/acs-engine/pkg/api/common"
+	"github.com/Azure/acs-engine/pkg/helpers"
 	"github.com/satori/uuid"
 	validator "gopkg.in/go-playground/validator.v9"
 )
@ -98,6 +99,17 @@ func (o *OrchestratorProfile) Validate(isUpdate bool) error {
 							return fmt.Errorf("enableAggregatedAPIs requires the enableRbac feature as a prerequisite")
 						}
 					}
+
+					if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableDataEncryptionAtRest) {
+						if o.OrchestratorVersion == common.KubernetesVersion1Dot5Dot7 ||
+							o.OrchestratorVersion == common.KubernetesVersion1Dot5Dot8 ||
+							o.OrchestratorVersion == common.KubernetesVersion1Dot6Dot6 ||
+							o.OrchestratorVersion == common.KubernetesVersion1Dot6Dot9 ||
+							o.OrchestratorVersion == common.KubernetesVersion1Dot6Dot11 {
+							return fmt.Errorf("enableDataEncryptionAtRest is only available in Kubernetes version %s or greater; unable to validate for Kubernetes version %s",
+								"1.7.0", o.OrchestratorVersion)
+						}
+					}
 				}
 			}