diff --git a/parts/k8s/addons/azure-cni-networkmonitor.yaml b/parts/k8s/containeraddons/azure-cni-networkmonitor.yaml similarity index 93% rename from parts/k8s/addons/azure-cni-networkmonitor.yaml rename to parts/k8s/containeraddons/azure-cni-networkmonitor.yaml index fc43c3c1f..4e95dda55 100644 --- a/parts/k8s/addons/azure-cni-networkmonitor.yaml +++ b/parts/k8s/containeraddons/azure-cni-networkmonitor.yaml @@ -28,7 +28,7 @@ spec: beta.kubernetes.io/os: linux containers: - name: azure-cnms - image: + image: {{ContainerImage "azure-cni-networkmonitor"}} imagePullPolicy: IfNotPresent securityContext: privileged: true diff --git a/parts/k8s/addons/dns-autoscaler.yaml b/parts/k8s/containeraddons/dns-autoscaler.yaml similarity index 83% rename from parts/k8s/addons/dns-autoscaler.yaml rename to parts/k8s/containeraddons/dns-autoscaler.yaml index 2f68f7522..826c864ac 100644 --- a/parts/k8s/addons/dns-autoscaler.yaml +++ b/parts/k8s/containeraddons/dns-autoscaler.yaml @@ -18,11 +18,11 @@ spec: spec: containers: - name: autoscaler - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.1 + image: {{ContainerImage "dns-autoscaler"}} resources: requests: - cpu: "20m" - memory: "10Mi" + cpu: {{ContainerCPUReqs "dns-autoscaler"}} + memory: {{ContainerMemReqs "dns-autoscaler"}} command: - /cluster-proportional-autoscaler - --namespace=kube-system diff --git a/parts/k8s/addons/ip-masq-agent.yaml b/parts/k8s/containeraddons/ip-masq-agent.yaml similarity index 82% rename from parts/k8s/addons/ip-masq-agent.yaml rename to parts/k8s/containeraddons/ip-masq-agent.yaml index 7208b8982..0c50453e4 100644 --- a/parts/k8s/addons/ip-masq-agent.yaml +++ b/parts/k8s/containeraddons/ip-masq-agent.yaml @@ -27,7 +27,7 @@ spec: effect: NoSchedule containers: - name: azure-ip-masq-agent - image: gcr.io/google-containers/ip-masq-agent-amd64:v2.0.0 + image: {{ContainerImage "ip-masq-agent"}} imagePullPolicy: IfNotPresent securityContext: privileged: true @@ -36,11 +36,11 @@ spec: mountPath: /etc/config resources: requests: - cpu: - memory: < + cpu: {{ContainerCPUReqs "ip-masq-agent"}} + memory: {{ContainerMemReqs "ip-masq-agent"}} limits: - cpu: - memory: + cpu: {{ContainerCPULimits "ip-masq-agent"}} + memory: {{ContainerMemLimits "ip-masq-agent"}} volumes: - name: azure-ip-masq-agent-config-volume configMap: diff --git a/parts/k8s/addons/kubernetesmasteraddons-aad-pod-identity-deployment.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-aad-pod-identity-deployment.yaml similarity index 100% rename from parts/k8s/addons/kubernetesmasteraddons-aad-pod-identity-deployment.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-aad-pod-identity-deployment.yaml diff --git a/parts/k8s/addons/kubernetesmasteraddons-aci-connector-deployment.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-aci-connector-deployment.yaml similarity index 84% rename from parts/k8s/addons/kubernetesmasteraddons-aci-connector-deployment.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-aci-connector-deployment.yaml index 060235aac..10d4cc50d 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-aci-connector-deployment.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-aci-connector-deployment.yaml @@ -83,7 +83,7 @@ spec: beta.kubernetes.io/os: linux containers: - name: aci-connector - image: + image: {{ContainerImage "aci-connector"}} imagePullPolicy: Always env: - name: KUBELET_PORT @@ -104,17 +104,17 @@ spec: fieldPath: status.podIP resources: requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "aci-connector"}} + memory: {{ContainerMemReqs "aci-connector"}} limits: - cpu: - memory: + cpu: {{ContainerCPULimits "aci-connector"}} + memory: {{ContainerMemLimits "aci-connector"}} volumeMounts: - name: credentials mountPath: "/etc/virtual-kubelet" readOnly: true command: ["virtual-kubelet"] - args: ["--provider", "azure", "--nodename", "" , "--os", "", "--taint", ""] + args: ["--provider", "azure", "--nodename", "{{ContainerConfig "nodeName"}}" , "--os", "{{ContainerConfig "os"}}", "--taint", "{{ContainerConfig "taint"}}"] volumes: - name: credentials secret: diff --git a/parts/k8s/addons/kubernetesmasteraddons-blobfuse-flexvolume-installer.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-blobfuse-flexvolume-installer.yaml similarity index 76% rename from parts/k8s/addons/kubernetesmasteraddons-blobfuse-flexvolume-installer.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-blobfuse-flexvolume-installer.yaml index e483bd7fe..0b4daff13 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-blobfuse-flexvolume-installer.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-blobfuse-flexvolume-installer.yaml @@ -25,15 +25,15 @@ spec: spec: containers: - name: blobfuse-flexvol-installer - image: andyzhangx/blobfuse-flexvol-installer + image: {{ContainerImage "blobfuse-flexvolume"}} imagePullPolicy: Always resources: requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "blobfuse-flexvolume"}} + memory: {{ContainerMemReqs "blobfuse-flexvolume"}} limits: - cpu: - memory: + cpu: {{ContainerCPULimits "blobfuse-flexvolume"}} + memory: {{ContainerMemLimits "blobfuse-flexvolume"}} volumeMounts: - name: volplugins mountPath: /etc/kubernetes/volumeplugins/ diff --git a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml similarity index 92% rename from parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml index bab28bb26..f2efec19f 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml @@ -153,23 +153,23 @@ spec: kubernetes.io/role: master beta.kubernetes.io/os: linux containers: - - image: + - image: {{ContainerImage "cluster-autoscaler"}} imagePullPolicy: IfNotPresent name: cluster-autoscaler resources: limits: - cpu: - memory: + cpu: {{ContainerCPULimits "cluster-autoscaler"}} + memory: {{ContainerMemLimits "cluster-autoscaler"}} requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "cluster-autoscaler"}} + memory: {{ContainerMemReqs "cluster-autoscaler"}} command: - ./cluster-autoscaler - --v=3 - --logtostderr=true - --cloud-provider=azure - --skip-nodes-with-local-storage=false - - --nodes=:: + - --nodes={{ContainerConfig "min-nodes"}}:{{ContainerConfig "max-nodes"}}: env: - name: ARM_CLOUD value: "" @@ -204,7 +204,7 @@ spec: key: VMType name: cluster-autoscaler-azure - name: ARM_USE_MANAGED_IDENTITY_EXTENSION - value: + value: volumeMounts: - mountPath: /etc/ssl/certs/ca-certificates.crt name: ssl-certs diff --git a/parts/k8s/addons/kubernetesmasteraddons-keyvault-flexvolume-installer.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-keyvault-flexvolume-installer.yaml similarity index 78% rename from parts/k8s/addons/kubernetesmasteraddons-keyvault-flexvolume-installer.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-keyvault-flexvolume-installer.yaml index b631dffc3..3f1c1b8e1 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-keyvault-flexvolume-installer.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-keyvault-flexvolume-installer.yaml @@ -26,15 +26,15 @@ spec: tolerations: containers: - name: keyvault-flexvolume - image: mcr.microsoft.com/k8s/flexvolume/keyvault-flexvolume:v0.0.5 + image: {{ContainerImage "keyvault-flexvolume"}} imagePullPolicy: IfNotPresent resources: requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "keyvault-flexvolume"}} + memory: {{ContainerMemReqs "keyvault-flexvolume"}} limits: - cpu: - memory: + cpu: {{ContainerCPULimits "keyvault-flexvolume"}} + memory: {{ContainerMemLimits "keyvault-flexvolume"}} env: - name: TARGET_DIR value: /etc/kubernetes/volumeplugins diff --git a/parts/k8s/addons/kubernetesmasteraddons-kube-rescheduler-deployment.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-kube-rescheduler-deployment.yaml similarity index 68% rename from parts/k8s/addons/kubernetesmasteraddons-kube-rescheduler-deployment.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-kube-rescheduler-deployment.yaml index 279713cb6..e449bf3ec 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-kube-rescheduler-deployment.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-kube-rescheduler-deployment.yaml @@ -22,16 +22,16 @@ spec: nodeSelector: beta.kubernetes.io/os: linux containers: - - image: + - image: {{ContainerImage "rescheduler"}} imagePullPolicy: IfNotPresent name: rescheduler resources: requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "rescheduler"}} + memory: {{ContainerMemReqs "rescheduler"}} limits: - cpu: - memory: + cpu: {{ContainerCPULimits "rescheduler"}} + memory: {{ContainerMemLimits "rescheduler"}} command: - sh - -c diff --git a/parts/k8s/addons/kubernetesmasteraddons-kubernetes-dashboard-deployment.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-kubernetes-dashboard-deployment.yaml similarity index 90% rename from parts/k8s/addons/kubernetesmasteraddons-kubernetes-dashboard-deployment.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-kubernetes-dashboard-deployment.yaml index 94abe1864..3e9c24bfa 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-kubernetes-dashboard-deployment.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-kubernetes-dashboard-deployment.yaml @@ -98,7 +98,7 @@ spec: - args: - --auto-generate-certificates - --heapster-host=http://heapster.kube-system:80 - image: + image: {{ContainerImage "kubernetes-dashboard"}} imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -113,11 +113,11 @@ spec: protocol: TCP resources: requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "kubernetes-dashboard"}} + memory: {{ContainerMemReqs "kubernetes-dashboard"}} limits: - cpu: - memory: + cpu: {{ContainerCPULimits "kubernetes-dashboard"}} + memory: {{ContainerMemLimits "kubernetes-dashboard"}} volumeMounts: - name: kubernetes-dashboard-certs mountPath: /certs diff --git a/parts/k8s/addons/kubernetesmasteraddons-metrics-server-deployment.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-metrics-server-deployment.yaml similarity index 98% rename from parts/k8s/addons/kubernetesmasteraddons-metrics-server-deployment.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-metrics-server-deployment.yaml index 5db5fe650..7f8990297 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-metrics-server-deployment.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-metrics-server-deployment.yaml @@ -122,7 +122,7 @@ spec: serviceAccountName: metrics-server containers: - name: metrics-server - image: + image: {{ContainerImage "metrics-server"}} imagePullPolicy: IfNotPresent command: - /metrics-server diff --git a/parts/k8s/addons/kubernetesmasteraddons-nvidia-device-plugin-daemonset.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-nvidia-device-plugin-daemonset.yaml similarity index 83% rename from parts/k8s/addons/kubernetesmasteraddons-nvidia-device-plugin-daemonset.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-nvidia-device-plugin-daemonset.yaml index 3ddaae12f..111a7b14e 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-nvidia-device-plugin-daemonset.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-nvidia-device-plugin-daemonset.yaml @@ -38,15 +38,15 @@ spec: operator: Equal value: "true" containers: - - image: + - image: {{ContainerImage "nvidia-device-plugin"}} name: nvidia-device-plugin-ctr resources: requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "nvidia-device-plugin"}} + memory: {{ContainerMemReqs "nvidia-device-plugin"}} limits: - cpu: - memory: + cpu: {{ContainerCPULimits "nvidia-device-plugin"}} + memory: {{ContainerMemLimits "nvidia-device-plugin"}} securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/parts/k8s/addons/kubernetesmasteraddons-omsagent-daemonset.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-omsagent-daemonset.yaml similarity index 91% rename from parts/k8s/addons/kubernetesmasteraddons-omsagent-daemonset.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-omsagent-daemonset.yaml index 7f3808fc7..230fcf10c 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-omsagent-daemonset.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-omsagent-daemonset.yaml @@ -8,8 +8,8 @@ metadata: addonmanager.kubernetes.io/mode: Reconcile type: Opaque data: - WSID: "" - KEY: "" + WSID: "{{ContainerConfig "workspaceGuid"}}" + KEY: "{{ContainerConfig "workspaceKey"}}" --- apiVersion: v1 kind: ServiceAccount @@ -112,8 +112,8 @@ spec: template: metadata: annotations: - agentVersion: - dockerProviderVersion: + agentVersion: {{ContainerConfig "omsAgentVersion"}} + dockerProviderVersion: {{ContainerConfig "dockerProviderVersion"}} labels: component: oms-agent tier: node @@ -121,15 +121,15 @@ spec: serviceAccountName: omsagent containers: - name: omsagent - image: + image: {{ContainerImage "omsagent"}} imagePullPolicy: Always resources: limits: - cpu: - memory: + cpu: {{ContainerCPULimits "omsagent"}} + memory: {{ContainerMemLimits "omsagent"}} requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "omsagent"}} + memory: {{ContainerMemReqs "omsagent"}} env: - name: NODE_IP valueFrom: @@ -214,21 +214,21 @@ spec: labels: rsName: omsagent-rs annotations: - agentVersion: - dockerProviderVersion: + agentVersion: {{ContainerConfig "omsAgentVersion"}} + dockerProviderVersion: {{ContainerConfig "dockerProviderVersion"}} spec: serviceAccountName: omsagent containers: - name: omsagent - image: + image: {{ContainerImage "omsagent"}} imagePullPolicy: IfNotPresent resources: limits: - cpu: - memory: + cpu: {{ContainerCPULimits "omsagent"}} + memory: {{ContainerMemLimits "omsagent"}} requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "omsagent"}} + memory: {{ContainerMemReqs "omsagent"}} env: - name: NODE_IP valueFrom: diff --git a/parts/k8s/addons/kubernetesmasteraddons-smb-flexvolume-installer.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-smb-flexvolume-installer.yaml similarity index 78% rename from parts/k8s/addons/kubernetesmasteraddons-smb-flexvolume-installer.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-smb-flexvolume-installer.yaml index b312a63bb..a04a6a2c3 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-smb-flexvolume-installer.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-smb-flexvolume-installer.yaml @@ -25,15 +25,15 @@ spec: spec: containers: - name: smb-flexvol-installer - image: andyzhangx/smb-flexvol-installer + image: {{ContainerImage "smb-flexvolume"}} imagePullPolicy: Always resources: requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "smb-flexvolume"}} + memory: {{ContainerMemReqs "smb-flexvolume"}} limits: - cpu: - memory: + cpu: {{ContainerCPULimits "smb-flexvolume"}} + memory: {{ContainerMemLimits "smb-flexvolume"}} volumeMounts: - name: volplugins mountPath: /etc/kubernetes/volumeplugins/ diff --git a/parts/k8s/addons/kubernetesmasteraddons-tiller-deployment.yaml b/parts/k8s/containeraddons/kubernetesmasteraddons-tiller-deployment.yaml similarity index 86% rename from parts/k8s/addons/kubernetesmasteraddons-tiller-deployment.yaml rename to parts/k8s/containeraddons/kubernetesmasteraddons-tiller-deployment.yaml index 9ef6890b1..a74683cb9 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-tiller-deployment.yaml +++ b/parts/k8s/containeraddons/kubernetesmasteraddons-tiller-deployment.yaml @@ -66,8 +66,8 @@ spec: - name: TILLER_NAMESPACE value: kube-system - name: TILLER_HISTORY_MAX - value: "" - image: + value: "{{ContainerConfig "max-history"}}" + image: {{ContainerImage "tiller"}} imagePullPolicy: IfNotPresent livenessProbe: httpGet: @@ -87,10 +87,10 @@ spec: timeoutSeconds: 1 resources: requests: - cpu: - memory: + cpu: {{ContainerCPUReqs "tiller"}} + memory: {{ContainerMemReqs "tiller"}} limits: - cpu: - memory: + cpu: {{ContainerCPULimits "tiller"}} + memory: {{ContainerMemLimits "tiller"}} nodeSelector: beta.kubernetes.io/os: linux diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index df2590d41..07a8fce0b 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -241,6 +241,8 @@ MASTER_ADDONS_CONFIG_PLACEHOLDER MASTER_CUSTOM_FILES_PLACEHOLDER +MASTER_CONTAINER_ADDONS_PLACEHOLDER + - path: /etc/default/kubelet permissions: "0644" owner: root @@ -279,9 +281,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER iptables -t nat -A PREROUTING -p tcp --dport 4443 -j REDIRECT --to-port 443 {{end}} -{{if IsAzureCNI}} - sed -i "s||{{WrapAsParameter "AzureCNINetworkMonitorImageURL"}}|g" /etc/kubernetes/addons/azure-cni-networkmonitor.yaml -{{end}} sed -i "s||{{WrapAsParameter "kubernetesAddonManagerSpec"}}|g" /etc/kubernetes/manifests/kube-addon-manager.yaml for a in "/etc/kubernetes/manifests/kube-apiserver.yaml /etc/kubernetes/manifests/kube-controller-manager.yaml /etc/kubernetes/manifests/kube-scheduler.yaml"; do sed -i "s||{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" $a @@ -302,54 +301,18 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{end}} sed -i "s||{{WrapAsParameter "kubernetesHeapsterSpec"}}|g; s||{{WrapAsParameter "kubernetesAddonResizerSpec"}}|g" /etc/kubernetes/addons/kube-heapster-deployment.yaml -{{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesDashboardSpec"}}|g; s||{{WrapAsParameter "kubernetesDashboardCPURequests"}}|g; s||{{WrapAsParameter "kubernetesDashboardMemoryRequests"}}|g; s||{{WrapAsParameter "kubernetesDashboardCPULimit"}}|g; s||{{WrapAsParameter "kubernetesDashboardMemoryLimit"}}|g" /etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsTillerEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesTillerSpec"}}|g; s||{{WrapAsParameter "kubernetesTillerCPURequests"}}|g; s||{{WrapAsParameter "kubernetesTillerMemoryRequests"}}|g; s||{{WrapAsParameter "kubernetesTillerCPULimit"}}|g; s||{{WrapAsParameter "kubernetesTillerMemoryLimit"}}|g; s||{{WrapAsParameter "kubernetesTillerMaxHistory"}}|g" /etc/kubernetes/addons/kube-tiller-deployment.yaml -{{end}} - {{if AdminGroupID }} sed -i "s||{{WrapAsParameter "aadAdminGroupId"}}|g" "/etc/kubernetes/addons/aad-default-admin-group-rbac.yaml" {{end}} -{{if .OrchestratorProfile.KubernetesConfig.IsACIConnectorEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesACIConnectorSpec"}}|g; s||{{WrapAsParameter "kubernetesACIConnectorNodeName"}}|g; s||{{WrapAsParameter "kubernetesACIConnectorOS"}}|g; s||{{WrapAsParameter "kubernetesACIConnectorTaint"}}|g; s||{{WrapAsParameter "kubernetesACIConnectorRegion"}}|g; s||{{WrapAsParameter "kubernetesACIConnectorCPURequests"}}|g; s||{{WrapAsParameter "kubernetesACIConnectorMemoryRequests"}}|g; s||{{WrapAsParameter "kubernetesACIConnectorCPULimit"}}|g; s||{{WrapAsParameter "kubernetesACIConnectorMemoryLimit"}}|g" /etc/kubernetes/addons/aci-connector-deployment.yaml -{{end}} - {{if .OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerAzureCloud"}}|g; s||{{WrapAsParameter "kubernetesClusterAutoscalerSpec"}}|g; s||{{WrapAsParameter "kubernetesClusterAutoscalerCPULimit"}}|g; s||{{WrapAsParameter "kubernetesClusterAutoscalerMemoryLimit"}}|g; s||{{WrapAsParameter "kubernetesClusterAutoscalerCPURequests"}}|g; s||{{WrapAsParameter "kubernetesClusterAutoscalerMemoryRequests"}}|g; s||{{WrapAsParameter "kubernetesClusterAutoscalerMinNodes"}}|g; s||{{WrapAsParameter "kubernetesClusterAutoscalerMaxNodes"}}|g; s||{{WrapAsParameter "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" /etc/kubernetes/addons/cluster-autoscaler-deployment.yaml + sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerAzureCloud"}}|g; s||{{WrapAsParameter "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" /etc/kubernetes/addons/cluster-autoscaler-deployment.yaml {{end}} {{if eq .OrchestratorProfile.KubernetesConfig.LoadBalancerSku "Standard"}} sed -i "s||{{WrapAsParameter "kuberneteselbsvcname"}}|g" "/etc/kubernetes/addons/elb-svc.yaml" {{end}} -{{if .OrchestratorProfile.KubernetesConfig.IsBlobfuseFlexVolumeEnabled}} - sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerCPURequests"}}|g; s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerMemoryRequests"}}|g; s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerCPULimit"}}|g; s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerMemoryLimit"}}|g" /etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsSMBFlexVolumeEnabled}} - sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerCPURequests"}}|g; s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerMemoryRequests"}}|g; s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerCPULimit"}}|g; s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerMemoryLimit"}}|g" /etc/kubernetes/addons/smb-flexvolume-installer.yaml -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsKeyVaultFlexVolumeEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerCPURequests"}}|g; s||{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests"}}|g; s||{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerCPULimit"}}|g; s||{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit"}}|g" /etc/kubernetes/addons/keyvault-flexvolume-installer.yaml -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesReschedulerSpec"}}|g; s||{{WrapAsParameter "kubernetesReschedulerCPURequests"}}|g; s||{{WrapAsParameter "kubernetesReschedulerMemoryRequests"}}|g; s||{{WrapAsParameter "kubernetesReschedulerCPULimit"}}|g; s||{{WrapAsParameter "kubernetesReschedulerMemoryLimit"}}|g" /etc/kubernetes/addons/kube-rescheduler-deployment.yaml -{{end}} - -{{if .OrchestratorProfile.IsMetricsServerEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesMetricsServerSpec"}}|g" "/etc/kubernetes/addons/kube-metrics-server-deployment.yaml" -{{end}} - -{{if IsNVIDIADevicePluginEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesNVIDIADevicePluginSpec"}}|g; s||{{WrapAsParameter "kubernetesNVIDIADevicePluginCPURequests"}}|g; s||{{WrapAsParameter "kubernetesNVIDIADevicePluginMemoryRequests"}}|g; s||{{WrapAsParameter "kubernetesNVIDIADevicePluginCPULimit"}}|g; s||{{WrapAsParameter "kubernetesNVIDIADevicePluginMemoryLimit"}}|g" /etc/kubernetes/addons/nvidia-device-plugin.yaml -{{end}} - {{if EnableDataEncryptionAtRest }} sed -i "s||\"{{WrapAsParameter "etcdEncryptionKey"}}\"|g" /etc/kubernetes/encryption-config.yaml {{end}} @@ -386,12 +349,9 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{end}} {{if HasCustomSearchDomain}} sed -i "s||{{WrapAsParameter "searchDomainName"}}|g; s||{{WrapAsParameter "searchDomainRealmUser"}}|g; s||{{WrapAsParameter "searchDomainRealmPassword"}}|g" /opt/azure/containers/setup-custom-search-domains.sh -{{end}} -{{if .OrchestratorProfile.KubernetesConfig.IsContainerMonitoringEnabled}} - sed -i "s||{{WrapAsParameter "omsAgentVersion"}}|g; s||{{WrapAsParameter "omsAgentDockerProviderVersion"}}|g; s||{{WrapAsParameter "omsAgentImage"}}|g; s||{{WrapAsParameter "omsAgentWorkspaceGuid"}}|g; s||{{WrapAsParameter "omsAgentWorkspaceKey"}}|g; s||{{WrapAsParameter "kubernetesOMSAgentCPURequests"}}|g; s||{{WrapAsParameter "kubernetesOMSAgentMemoryRequests"}}|g; s||{{WrapAsParameter "kubernetesOMSAgentCPULimit"}}|g; s||{{WrapAsParameter "kubernetesOMSAgentMemoryLimit"}}|g" /etc/kubernetes/addons/omsagent-daemonset.yaml {{end}} a=/etc/kubernetes/addons/ip-masq-agent.yaml - sed -i "s||{{WrapAsParameter "kubernetesNonMasqueradeCidr"}}|g; s||{{WrapAsParameter "kubernetesIPMasqAgentCPURequests"}}|g; s|<|{{WrapAsParameter "kubernetesIPMasqAgentMemoryRequests"}}|g; s||{{WrapAsParameter "kubernetesIPMasqAgentCPULimit"}}|g; s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryLimit"}}|g" $a + sed -i "s||{{WrapAsParameter "kubernetesNonMasqueradeCidr"}}|g" $a {{if IsAzureCNI}} sed -i "s||168.63.129.16/32|g; s||true|g" $a {{else}} diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index f97221862..2a6d0be90 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -235,38 +235,6 @@ }, "type": "string" }, -{{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}} - "kubernetesDashboardSpec": { - "metadata": { - "description": "The container spec for kubernetes-dashboard-amd64." - }, - "type": "string" - }, - "kubernetesDashboardCPURequests": { - "metadata": { - "description": "Dashboard CPU Requests." - }, - "type": "string" - }, - "kubernetesDashboardMemoryRequests": { - "metadata": { - "description": "Dashboard Memory Requests." - }, - "type": "string" - }, - "kubernetesDashboardCPULimit": { - "metadata": { - "description": "Dashboard CPU Limit." - }, - "type": "string" - }, - "kubernetesDashboardMemoryLimit": { - "metadata": { - "description": "Dashboard Memory Limit." - }, - "type": "string" - }, -{{end}} "enableAggregatedAPIs": { "metadata": { "description": "Enable aggregated API on master nodes" @@ -294,84 +262,6 @@ }, "type": "string" }, -{{if .OrchestratorProfile.IsMetricsServerEnabled}} - "kubernetesMetricsServerSpec": { - "metadata": { - "description": "The container spec for Metrics Server." - }, - "type": "string" - }, -{{end}} -{{if .IsNVIDIADevicePluginEnabled}} - "kubernetesNVIDIADevicePluginSpec": { - "metadata": { - "description": "The container spec for NVIDIA Device Plugin." - }, - "type": "string" - }, - "kubernetesNVIDIADevicePluginCPURequests": { - "metadata": { - "description": "NVIDIA Device Plugin CPU Requests" - }, - "type": "string" - }, - "kubernetesNVIDIADevicePluginMemoryRequests": { - "metadata": { - "description": "NVIDIA Device Plugin Memory Requests" - }, - "type": "string" - }, - "kubernetesNVIDIADevicePluginCPULimit": { - "metadata": { - "description": "NVIDIA Device Plugin CPU Limit" - }, - "type": "string" - }, - "kubernetesNVIDIADevicePluginMemoryLimit": { - "metadata": { - "description": "NVIDIA Device Plugin Memory Limit" - }, - "type": "string" - }, -{{end}} -{{if .OrchestratorProfile.KubernetesConfig.IsTillerEnabled}} - "kubernetesTillerSpec": { - "metadata": { - "description": "The container spec for Helm Tiller." - }, - "type": "string" - }, - "kubernetesTillerCPURequests": { - "metadata": { - "description": "Helm Tiller CPU Requests." - }, - "type": "string" - }, - "kubernetesTillerMemoryRequests": { - "metadata": { - "description": "Helm Tiller Memory Requests." - }, - "type": "string" - }, - "kubernetesTillerCPULimit": { - "metadata": { - "description": "Helm Tiller CPU Limit." - }, - "type": "string" - }, - "kubernetesTillerMemoryLimit": { - "metadata": { - "description": "Helm Tiller Memory Limit." - }, - "type": "string" - }, - "kubernetesTillerMaxHistory": { - "metadata": { - "description": "Helm Tiller Max History to Store. '0' for no limit." - }, - "type": "string" - }, -{{end}} {{if .OrchestratorProfile.KubernetesConfig.IsAADPodIdentityEnabled}} "kubernetesAADPodIdentityEnabled": { "defaultValue": false, @@ -387,62 +277,6 @@ }, "type": "bool" }, -{{if .OrchestratorProfile.KubernetesConfig.IsACIConnectorEnabled}} - "kubernetesACIConnectorSpec": { - "metadata": { - "description": "The container spec for ACI Connector." - }, - "type": "string" - }, - "kubernetesACIConnectorNodeName": { - "metadata": { - "description": "Node name for ACI Connector." - }, - "type": "string" - }, - "kubernetesACIConnectorOS": { - "metadata": { - "description": "OS for ACI Connector." - }, - "type": "string" - }, - "kubernetesACIConnectorTaint": { - "metadata": { - "description": "Taint for ACI Connector." - }, - "type": "string" - }, - "kubernetesACIConnectorRegion": { - "metadata": { - "description": "Region for ACI Connector." - }, - "type": "string" - }, - "kubernetesACIConnectorCPURequests": { - "metadata": { - "description": "ACI Connector CPU Requests" - }, - "type": "string" - }, - "kubernetesACIConnectorMemoryRequests": { - "metadata": { - "description": "ACI Connector Memory Requests" - }, - "type": "string" - }, - "kubernetesACIConnectorCPULimit": { - "metadata": { - "description": "ACI Connector CPU Limit" - }, - "type": "string" - }, - "kubernetesACIConnectorMemoryLimit": { - "metadata": { - "description": "ACI Connector Memory Limit" - }, - "type": "string" - }, -{{end}} "kubernetesClusterAutoscalerEnabled": { "metadata": { "description": "Cluster autoscaler status" @@ -450,157 +284,18 @@ "type": "bool" }, {{if .OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled}} - "kubernetesClusterAutoscalerSpec": { - "metadata": { - "description": "The container spec for the cluster autoscaler." - }, - "type": "string" - }, "kubernetesClusterAutoscalerAzureCloud": { "metadata": { "description": "Name of the Azure cloud for the cluster autoscaler." }, "type": "string" }, - "kubernetesClusterAutoscalerCPULimit": { - "metadata": { - "description": "Cluster autoscaler cpu limit" - }, - "type": "string" - }, - "kubernetesClusterAutoscalerMemoryLimit": { - "metadata": { - "description": "Cluster autoscaler memory limit" - }, - "type": "string" - }, - "kubernetesClusterAutoscalerCPURequests": { - "metadata": { - "description": "Cluster autoscaler cpu requests" - }, - "type": "string" - }, - "kubernetesClusterAutoscalerMemoryRequests": { - "metadata": { - "description": "Cluster autoscaler memory requests" - }, - "type": "string" - }, - "kubernetesClusterAutoscalerMinNodes": { - "metadata": { - "description": "Cluster autoscaler min nodes" - }, - "type": "string" - }, - "kubernetesClusterAutoscalerMaxNodes": { - "metadata": { - "description": "Cluster autoscaler max nodes" - }, - "type": "string" - }, "kubernetesClusterAutoscalerUseManagedIdentity": { "metadata": { "description": "Managed identity for the cluster autoscaler addon" }, "type": "string" }, -{{end}} - "flexVolumeDriverConfig": { - "type": "object", - "defaultValue": { - "kubernetesBlobfuseFlexVolumeInstallerCPURequests": "50m", - "kubernetesBlobfuseFlexVolumeInstallerMemoryRequests": "10Mi", - "kubernetesBlobfuseFlexVolumeInstallerCPULimit": "50m", - "kubernetesBlobfuseFlexVolumeInstallerMemoryLimit": "10Mi", - "kubernetesSMBFlexVolumeInstallerCPURequests": "50m", - "kubernetesSMBFlexVolumeInstallerMemoryRequests": "10Mi", - "kubernetesSMBFlexVolumeInstallerCPULimit": "50m", - "kubernetesSMBFlexVolumeInstallerMemoryLimit": "10Mi" - } - }, -{{if .OrchestratorProfile.KubernetesConfig.IsKeyVaultFlexVolumeEnabled}} - "kubernetesKeyVaultFlexVolumeInstallerCPURequests": { - "metadata": { - "description": "Key Vault FlexVolume Installer CPU Requests" - }, - "type": "string" - }, - "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests": { - "metadata": { - "description": "Key Vault FlexVolume Installer Memory Requests" - }, - "type": "string" - }, - "kubernetesKeyVaultFlexVolumeInstallerCPULimit": { - "metadata": { - "description": "Key Vault FlexVolume Installer CPU Limit" - }, - "type": "string" - }, - "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit": { - "metadata": { - "description": "Key Vault FlexVolume Installer Memory Limit" - }, - "type": "string" - }, -{{end}} -{{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}} - "kubernetesReschedulerSpec": { - "metadata": { - "description": "The container spec for rescheduler." - }, - "type": "string" - }, - "kubernetesReschedulerCPURequests": { - "metadata": { - "description": "Rescheduler CPU Requests." - }, - "type": "string" - }, - "kubernetesReschedulerMemoryRequests": { - "metadata": { - "description": "Rescheduler Memory Requests." - }, - "type": "string" - }, - "kubernetesReschedulerCPULimit": { - "metadata": { - "description": "Rescheduler CPU Limit." - }, - "type": "string" - }, - "kubernetesReschedulerMemoryLimit": { - "metadata": { - "description": "Rescheduler Memory Limit." - }, - "type": "string" - }, -{{end}} -{{if .OrchestratorProfile.KubernetesConfig.IsIPMasqAgentEnabled}} - "kubernetesIPMasqAgentCPURequests": { - "metadata": { - "description": "IP Masq Agent CPU Requests" - }, - "type": "string" - }, - "kubernetesIPMasqAgentMemoryRequests": { - "metadata": { - "description": "IP Masq Agent Memory Requests" - }, - "type": "string" - }, - "kubernetesIPMasqAgentCPULimit": { - "metadata": { - "description": "IP Masq Agent CPU Limit" - }, - "type": "string" - }, - "kubernetesIPMasqAgentMemoryLimit": { - "metadata": { - "description": "IP Masq Agent Memory Limit" - }, - "type": "string" - }, {{end}} "kubernetesPodInfraContainerSpec": { "metadata": { @@ -733,71 +428,6 @@ }, "type": "string" }, -{{if .OrchestratorProfile.KubernetesConfig.IsContainerMonitoringEnabled}} - "omsAgentVersion": { - "defaultValue": "", - "metadata": { - "description": "OMS agent version for Container Monitoring." - }, - "type": "string" - }, - "omsAgentDockerProviderVersion": { - "defaultValue": "", - "metadata": { - "description": "Docker provider version for Container Monitoring." - }, - "type": "string" - }, - "omsAgentImage": { - "defaultValue": "", - "metadata": { - "description": "OMS agent image for Container Monitoring." - }, - "type": "string" - }, - "omsAgentWorkspaceGuid": { - "defaultValue": "", - "metadata": { - "description": "OMS workspace guid" - }, - "type": "string" - }, - "omsAgentWorkspaceKey": { - "defaultValue": "", - "metadata": { - "description": "OMS workspace key" - }, - "type": "string" - }, - "kubernetesOMSAgentCPURequests": { - "defaultValue": "", - "metadata": { - "description": "OMS Agent CPU requests resource limit" - }, - "type": "string" - }, - "kubernetesOMSAgentMemoryRequests": { - "defaultValue": "", - "metadata": { - "description": "OMS Agent memory requests resource limit" - }, - "type": "string" - }, - "kubernetesOMSAgentCPULimit": { - "defaultValue": "", - "metadata": { - "description": "OMS Agent CPU limit resource limit" - }, - "type": "string" - }, - "kubernetesOMSAgentMemoryLimit": { - "defaultValue": "", - "metadata": { - "description": "OMS Agent memory limit resource limit" - }, - "type": "string" - }, -{{end}} {{ if not UseManagedIdentity }} "servicePrincipalClientId": { "metadata": { diff --git a/pkg/acsengine/artifacts.go b/pkg/acsengine/artifacts.go index 7167d25fe..528f47971 100644 --- a/pkg/acsengine/artifacts.go +++ b/pkg/acsengine/artifacts.go @@ -16,6 +16,104 @@ type kubernetesFeatureSetting struct { rawScript string } +func kubernetesContainerAddonSettingsInit(profile *api.Properties) map[string]kubernetesFeatureSetting { + return map[string]kubernetesFeatureSetting{ + DefaultMetricsServerAddonName: { + "kubernetesmasteraddons-metrics-server-deployment.yaml", + "kube-metrics-server-deployment.yaml", + profile.OrchestratorProfile.IsMetricsServerEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultMetricsServerAddonName), + }, + DefaultTillerAddonName: { + "kubernetesmasteraddons-tiller-deployment.yaml", + "kube-tiller-deployment.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsTillerEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultTillerAddonName), + }, + DefaultAADPodIdentityAddonName: { + "kubernetesmasteraddons-aad-pod-identity-deployment.yaml", + "aad-pod-identity-deployment.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsAADPodIdentityEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultAADPodIdentityAddonName), + }, + DefaultACIConnectorAddonName: { + "kubernetesmasteraddons-aci-connector-deployment.yaml", + "aci-connector-deployment.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsACIConnectorEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultACIConnectorAddonName), + }, + DefaultClusterAutoscalerAddonName: { + "kubernetesmasteraddons-cluster-autoscaler-deployment.yaml", + "cluster-autoscaler-deployment.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultClusterAutoscalerAddonName), + }, + DefaultBlobfuseFlexVolumeAddonName: { + "kubernetesmasteraddons-blobfuse-flexvolume-installer.yaml", + "blobfuse-flexvolume-installer.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsBlobfuseFlexVolumeEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultBlobfuseFlexVolumeAddonName), + }, + + DefaultSMBFlexVolumeAddonName: { + "kubernetesmasteraddons-smb-flexvolume-installer.yaml", + "smb-flexvolume-installer.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsSMBFlexVolumeEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultSMBFlexVolumeAddonName), + }, + DefaultKeyVaultFlexVolumeAddonName: { + "kubernetesmasteraddons-keyvault-flexvolume-installer.yaml", + "keyvault-flexvolume-installer.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsKeyVaultFlexVolumeEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultKeyVaultFlexVolumeAddonName), + }, + DefaultDashboardAddonName: { + "kubernetesmasteraddons-kubernetes-dashboard-deployment.yaml", + "kubernetes-dashboard-deployment.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsDashboardEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultDashboardAddonName), + }, + DefaultReschedulerAddonName: { + "kubernetesmasteraddons-kube-rescheduler-deployment.yaml", + "kube-rescheduler-deployment.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultReschedulerAddonName), + }, + NVIDIADevicePluginAddonName: { + "kubernetesmasteraddons-nvidia-device-plugin-daemonset.yaml", + "nvidia-device-plugin.yaml", + profile.IsNVIDIADevicePluginEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(NVIDIADevicePluginAddonName), + }, + ContainerMonitoringAddonName: { + "kubernetesmasteraddons-omsagent-daemonset.yaml", + "omsagent-daemonset.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsContainerMonitoringEnabled(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(ContainerMonitoringAddonName), + }, + IPMASQAgentAddonName: { + "ip-masq-agent.yaml", + "ip-masq-agent.yaml", + true, + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(IPMASQAgentAddonName), + }, + DefaultAzureCNINetworkMonitorAddonName: { + "azure-cni-networkmonitor.yaml", + "azure-cni-networkmonitor.yaml", + profile.OrchestratorProfile.IsAzureCNI(), + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultAzureCNINetworkMonitorAddonName), + }, + DefaultDNSAutoscalerAddonName: { + "dns-autoscaler.yaml", + "dns-autoscaler.yaml", + // TODO enable this when it has been smoke tested + //common.IsKubernetesVersionGe(profile.OrchestratorProfile.OrchestratorVersion, "1.12.0"), + false, + profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultDNSAutoscalerAddonName), + }, + } +} + func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSetting { return []kubernetesFeatureSetting{ { @@ -37,32 +135,12 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSet common.IsKubernetesVersionGe(profile.OrchestratorProfile.OrchestratorVersion, "1.12.0"), profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultCoreDNSAddonName), }, - { - "dns-autoscaler.yaml", - "dns-autoscaler.yaml", - // TODO enable this when it has been smoke tested - //common.IsKubernetesVersionGe(profile.OrchestratorProfile.OrchestratorVersion, "1.12.0"), - false, - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultDNSAutoscalerAddonName), - }, { "kubernetesmasteraddons-kube-proxy-daemonset.yaml", "kube-proxy-daemonset.yaml", true, profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultKubeProxyAddonName), }, - { - "kubernetesmasteraddons-nvidia-device-plugin-daemonset.yaml", - "nvidia-device-plugin.yaml", - profile.IsNVIDIADevicePluginEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(NVIDIADevicePluginAddonName), - }, - { - "kubernetesmasteraddons-kubernetes-dashboard-deployment.yaml", - "kubernetes-dashboard-deployment.yaml", - profile.OrchestratorProfile.KubernetesConfig.IsDashboardEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultDashboardAddonName), - }, { "kubernetesmasteraddons-unmanaged-azure-storage-classes.yaml", "azure-storage-classes.yaml", @@ -75,36 +153,6 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSet profile.AgentPoolProfiles[0].StorageProfile == api.ManagedDisks, profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultAzureStorageClassesAddonName), }, - { - "kubernetesmasteraddons-tiller-deployment.yaml", - "kube-tiller-deployment.yaml", - profile.OrchestratorProfile.KubernetesConfig.IsTillerEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultTillerAddonName), - }, - { - "kubernetesmasteraddons-aad-pod-identity-deployment.yaml", - "aad-pod-identity-deployment.yaml", - profile.OrchestratorProfile.KubernetesConfig.IsAADPodIdentityEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultAADPodIdentityAddonName), - }, - { - "kubernetesmasteraddons-aci-connector-deployment.yaml", - "aci-connector-deployment.yaml", - profile.OrchestratorProfile.KubernetesConfig.IsACIConnectorEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultACIConnectorAddonName), - }, - { - "kubernetesmasteraddons-cluster-autoscaler-deployment.yaml", - "cluster-autoscaler-deployment.yaml", - profile.OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultClusterAutoscalerAddonName), - }, - { - "kubernetesmasteraddons-kube-rescheduler-deployment.yaml", - "kube-rescheduler-deployment.yaml", - profile.OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultReschedulerAddonName), - }, { "kubernetesmasteraddons-azure-npm-daemonset.yaml", "azure-npm-daemonset.yaml", @@ -142,60 +190,18 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSet true, profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultAzureCloudProviderDeploymentAddonName), }, - { - "kubernetesmasteraddons-metrics-server-deployment.yaml", - "kube-metrics-server-deployment.yaml", - profile.OrchestratorProfile.IsMetricsServerEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultMetricsServerAddonName), - }, - { - "kubernetesmasteraddons-omsagent-daemonset.yaml", - "omsagent-daemonset.yaml", - profile.OrchestratorProfile.KubernetesConfig.IsContainerMonitoringEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(ContainerMonitoringAddonName), - }, - { - "azure-cni-networkmonitor.yaml", - "azure-cni-networkmonitor.yaml", - profile.OrchestratorProfile.IsAzureCNI(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultAzureCNINetworkMonitorAddonName), - }, { "kubernetesmaster-audit-policy.yaml", "audit-policy.yaml", common.IsKubernetesVersionGe(profile.OrchestratorProfile.OrchestratorVersion, "1.8.0"), profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultAuditPolicyAddonName), }, - { - "kubernetesmasteraddons-blobfuse-flexvolume-installer.yaml", - "blobfuse-flexvolume-installer.yaml", - profile.OrchestratorProfile.KubernetesConfig.IsBlobfuseFlexVolumeEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultBlobfuseFlexVolumeAddonName), - }, - { - "kubernetesmasteraddons-smb-flexvolume-installer.yaml", - "smb-flexvolume-installer.yaml", - profile.OrchestratorProfile.KubernetesConfig.IsSMBFlexVolumeEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultSMBFlexVolumeAddonName), - }, - { - "kubernetesmasteraddons-keyvault-flexvolume-installer.yaml", - "keyvault-flexvolume-installer.yaml", - profile.OrchestratorProfile.KubernetesConfig.IsKeyVaultFlexVolumeEnabled(), - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultKeyVaultFlexVolumeAddonName), - }, { "kubernetesmasteraddons-elb-svc.yaml", "elb-svc.yaml", profile.OrchestratorProfile.KubernetesConfig.LoadBalancerSku == "Standard", profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultELBSVCAddonName), }, - { - "ip-masq-agent.yaml", - "ip-masq-agent.yaml", - true, - profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(IPMASQAgentAddonName), - }, } } @@ -268,6 +274,19 @@ func kubernetesArtifactSettingsInitAgent(profile *api.Properties) []kubernetesFe } } +func getAddonString(input, destinationPath, destinationFile string) string { + addonString := getBase64CustomScriptFromStr(input) + contents := []string{ + fmt.Sprintf("- path: %s/%s", destinationPath, destinationFile), + " permissions: \\\"0644\\\"", + " encoding: gzip", + " owner: \\\"root\\\"", + " content: !!binary |", + fmt.Sprintf(" %s\\n\\n", addonString), + } + return strings.Join(contents, "\\n") +} + func substituteConfigString(input string, kubernetesFeatureSettings []kubernetesFeatureSetting, sourcePath string, destinationPath string, placeholder string, orchestratorVersion string) string { var config string diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index c90a056a8..a84d4e166 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -50,8 +50,6 @@ const ( DefaultAuditPolicyAddonName = "audit-policy" // DefaultTillerAddonName is the name of the tiller addon deployment DefaultTillerAddonName = "tiller" - // DefaultTillerMaxHistory limits the maximum number of revisions saved per release. Use 0 for no limit. - DefaultTillerMaxHistory = 0 // DefaultAADPodIdentityAddonName is the name of the aad-pod-identity addon deployment DefaultAADPodIdentityAddonName = "aad-pod-identity" // DefaultACIConnectorAddonName is the name of the aci-connector addon deployment diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index dc1f3a0dd..836639375 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -11,6 +11,7 @@ import ( "net" "net/http" "regexp" + "sort" "strconv" "strings" "text/template" @@ -692,6 +693,78 @@ func getDCOSProvisionScript(script string) string { return strings.Replace(strings.Replace(provisionScript, "\r\n", "\n", -1), "\n", "\n\n ", -1) } +func getAddonFuncMap(addon api.KubernetesAddon) template.FuncMap { + return template.FuncMap{ + "ContainerImage": func(name string) string { + i := addon.GetAddonContainersIndexByName(name) + return addon.Containers[i].Image + }, + + "ContainerCPUReqs": func(name string) string { + i := addon.GetAddonContainersIndexByName(name) + return addon.Containers[i].CPURequests + }, + + "ContainerCPULimits": func(name string) string { + i := addon.GetAddonContainersIndexByName(name) + return addon.Containers[i].CPULimits + }, + + "ContainerMemReqs": func(name string) string { + i := addon.GetAddonContainersIndexByName(name) + return addon.Containers[i].MemoryRequests + }, + + "ContainerMemLimits": func(name string) string { + i := addon.GetAddonContainersIndexByName(name) + return addon.Containers[i].MemoryLimits + }, + "ContainerConfig": func(name string) string { + return addon.Config[name] + }, + } +} + +func getContainerAddonsString(properties *api.Properties, sourcePath string) string { + var result string + settingsMap := kubernetesContainerAddonSettingsInit(properties) + + var addonNames []string + + for addonName := range settingsMap { + addonNames = append(addonNames, addonName) + } + + sort.Strings(addonNames) + + for _, addonName := range addonNames { + setting := settingsMap[addonName] + if setting.isEnabled { + var input string + if setting.rawScript != "" { + input = setting.rawScript + } else { + addon := properties.OrchestratorProfile.KubernetesConfig.GetAddonByName(addonName) + templ := template.New("addon resolver template").Funcs(getAddonFuncMap(addon)) + addonFile := sourcePath + "/" + setting.sourceFile + addonFileBytes, err := Asset(addonFile) + if err != nil { + return "" + } + _, err = templ.Parse(string(addonFileBytes)) + if err != nil { + return "" + } + var buffer bytes.Buffer + templ.Execute(&buffer, addon) + input = buffer.String() + } + result += getAddonString(input, "/etc/kubernetes/addons", setting.destinationFile) + } + } + return result +} + func getDCOSAgentProvisionScript(profile *api.AgentPoolProfile, orchProfile *api.OrchestratorProfile, bootstrapIP string) string { // add the provision script scriptname := dcos2Provision diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go index 550253b04..5ecce35de 100644 --- a/pkg/acsengine/params_k8s.go +++ b/pkg/acsengine/params_k8s.go @@ -50,18 +50,6 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params } addValue(parametersMap, "kubernetesDNSSidecarSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["k8s-dns-sidecar"]) addValue(parametersMap, "kubernetesHeapsterSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["heapster"]) - if kubernetesConfig.IsTillerEnabled() { - tillerAddon := kubernetesConfig.GetAddonByName(DefaultTillerAddonName) - tillerIndex := tillerAddon.GetAddonContainersIndexByName(DefaultTillerAddonName) - if tillerIndex > -1 { - addValue(parametersMap, "kubernetesTillerCPURequests", tillerAddon.Containers[tillerIndex].CPURequests) - addValue(parametersMap, "kubernetesTillerCPULimit", tillerAddon.Containers[tillerIndex].CPULimits) - addValue(parametersMap, "kubernetesTillerMemoryRequests", tillerAddon.Containers[tillerIndex].MemoryRequests) - addValue(parametersMap, "kubernetesTillerMemoryLimit", tillerAddon.Containers[tillerIndex].MemoryLimits) - addValue(parametersMap, "kubernetesTillerMaxHistory", tillerAddon.Config["max-history"]) - addValue(parametersMap, "kubernetesTillerSpec", tillerAddon.Containers[tillerIndex].Image) - } - } if kubernetesConfig.IsAADPodIdentityEnabled() { aadPodIdentityAddon := kubernetesConfig.GetAddonByName(DefaultAADPodIdentityAddonName) aadIndex := aadPodIdentityAddon.GetAddonContainersIndexByName(DefaultAADPodIdentityAddonName) @@ -70,20 +58,7 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params } } if kubernetesConfig.IsACIConnectorEnabled() { - aciConnectorAddon := kubernetesConfig.GetAddonByName(DefaultACIConnectorAddonName) - aciConnectorIndex := aciConnectorAddon.GetAddonContainersIndexByName(DefaultACIConnectorAddonName) - if aciConnectorIndex > -1 { - addValue(parametersMap, "kubernetesACIConnectorEnabled", true) - addValue(parametersMap, "kubernetesACIConnectorNodeName", aciConnectorAddon.Config["nodeName"]) - addValue(parametersMap, "kubernetesACIConnectorOS", aciConnectorAddon.Config["os"]) - addValue(parametersMap, "kubernetesACIConnectorTaint", aciConnectorAddon.Config["taint"]) - addValue(parametersMap, "kubernetesACIConnectorRegion", aciConnectorAddon.Config["region"]) - addValue(parametersMap, "kubernetesACIConnectorCPURequests", aciConnectorAddon.Containers[aciConnectorIndex].CPURequests) - addValue(parametersMap, "kubernetesACIConnectorCPULimit", aciConnectorAddon.Containers[aciConnectorIndex].CPULimits) - addValue(parametersMap, "kubernetesACIConnectorMemoryRequests", aciConnectorAddon.Containers[aciConnectorIndex].MemoryRequests) - addValue(parametersMap, "kubernetesACIConnectorMemoryLimit", aciConnectorAddon.Containers[aciConnectorIndex].MemoryLimits) - addValue(parametersMap, "kubernetesACIConnectorSpec", aciConnectorAddon.Containers[aciConnectorIndex].Image) - } + addValue(parametersMap, "kubernetesACIConnectorEnabled", true) } else { addValue(parametersMap, "kubernetesACIConnectorEnabled", false) } @@ -92,130 +67,17 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params clusterAutoScalerIndex := clusterAutoscalerAddon.GetAddonContainersIndexByName(DefaultClusterAutoscalerAddonName) if clusterAutoScalerIndex > -1 { addValue(parametersMap, "kubernetesClusterAutoscalerAzureCloud", cloudSpecConfig.CloudName) - addValue(parametersMap, "kubernetesClusterAutoscalerCPURequests", clusterAutoscalerAddon.Containers[clusterAutoScalerIndex].CPURequests) - addValue(parametersMap, "kubernetesClusterAutoscalerCPULimit", clusterAutoscalerAddon.Containers[clusterAutoScalerIndex].CPULimits) - addValue(parametersMap, "kubernetesClusterAutoscalerMemoryRequests", clusterAutoscalerAddon.Containers[clusterAutoScalerIndex].MemoryRequests) - addValue(parametersMap, "kubernetesClusterAutoscalerMemoryLimit", clusterAutoscalerAddon.Containers[clusterAutoScalerIndex].MemoryLimits) - addValue(parametersMap, "kubernetesClusterAutoscalerMinNodes", clusterAutoscalerAddon.Config["minNodes"]) - addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"]) addValue(parametersMap, "kubernetesClusterAutoscalerEnabled", true) addValue(parametersMap, "kubernetesClusterAutoscalerUseManagedIdentity", strings.ToLower(strconv.FormatBool(kubernetesConfig.UseManagedIdentity))) - addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[clusterAutoScalerIndex].Image) - } } else { addValue(parametersMap, "kubernetesClusterAutoscalerEnabled", false) } - flexVolumeDriverConfig := map[string]string{} - bfFlexVolumeInstallerAddon := kubernetesConfig.GetAddonByName(DefaultBlobfuseFlexVolumeAddonName) - bfFlexVolumeIndex := bfFlexVolumeInstallerAddon.GetAddonContainersIndexByName(DefaultBlobfuseFlexVolumeAddonName) - if bfFlexVolumeIndex > -1 { - flexVolumeDriverConfig["kubernetesBlobfuseFlexVolumeInstallerCPURequests"] = bfFlexVolumeInstallerAddon.Containers[bfFlexVolumeIndex].CPURequests - flexVolumeDriverConfig["kubernetesBlobfuseFlexVolumeInstallerCPULimit"] = bfFlexVolumeInstallerAddon.Containers[bfFlexVolumeIndex].CPULimits - flexVolumeDriverConfig["kubernetesBlobfuseFlexVolumeInstallerMemoryRequests"] = bfFlexVolumeInstallerAddon.Containers[bfFlexVolumeIndex].MemoryRequests - flexVolumeDriverConfig["kubernetesBlobfuseFlexVolumeInstallerMemoryLimit"] = bfFlexVolumeInstallerAddon.Containers[bfFlexVolumeIndex].MemoryLimits - } - smbFlexVolumeInstallerAddon := kubernetesConfig.GetAddonByName(DefaultSMBFlexVolumeAddonName) - smbFlexVolumeIndex := smbFlexVolumeInstallerAddon.GetAddonContainersIndexByName(DefaultSMBFlexVolumeAddonName) - if smbFlexVolumeIndex > -1 { - flexVolumeDriverConfig["kubernetesSMBFlexVolumeInstallerCPURequests"] = smbFlexVolumeInstallerAddon.Containers[smbFlexVolumeIndex].CPURequests - flexVolumeDriverConfig["kubernetesSMBFlexVolumeInstallerCPULimit"] = smbFlexVolumeInstallerAddon.Containers[smbFlexVolumeIndex].CPULimits - flexVolumeDriverConfig["kubernetesSMBFlexVolumeInstallerMemoryRequests"] = smbFlexVolumeInstallerAddon.Containers[smbFlexVolumeIndex].MemoryRequests - flexVolumeDriverConfig["kubernetesSMBFlexVolumeInstallerMemoryLimit"] = smbFlexVolumeInstallerAddon.Containers[smbFlexVolumeIndex].MemoryLimits - } - addValue(parametersMap, "flexVolumeDriverConfig", flexVolumeDriverConfig) - if kubernetesConfig.IsKeyVaultFlexVolumeEnabled() { - kvFlexVolumeInstallerAddon := kubernetesConfig.GetAddonByName(DefaultKeyVaultFlexVolumeAddonName) - kvFlexVolumeIndex := kvFlexVolumeInstallerAddon.GetAddonContainersIndexByName(DefaultKeyVaultFlexVolumeAddonName) - if kvFlexVolumeIndex > -1 { - addValue(parametersMap, "kubernetesKeyVaultFlexVolumeInstallerCPURequests", kvFlexVolumeInstallerAddon.Containers[kvFlexVolumeIndex].CPURequests) - addValue(parametersMap, "kubernetesKeyVaultFlexVolumeInstallerCPULimit", kvFlexVolumeInstallerAddon.Containers[kvFlexVolumeIndex].CPULimits) - addValue(parametersMap, "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests", kvFlexVolumeInstallerAddon.Containers[kvFlexVolumeIndex].MemoryRequests) - addValue(parametersMap, "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit", kvFlexVolumeInstallerAddon.Containers[kvFlexVolumeIndex].MemoryLimits) - } - } - if kubernetesConfig.IsDashboardEnabled() { - dashboardAddon := kubernetesConfig.GetAddonByName(DefaultDashboardAddonName) - dashboardIndex := dashboardAddon.GetAddonContainersIndexByName(DefaultDashboardAddonName) - if dashboardIndex > -1 { - addValue(parametersMap, "kubernetesDashboardCPURequests", dashboardAddon.Containers[dashboardIndex].CPURequests) - addValue(parametersMap, "kubernetesDashboardCPULimit", dashboardAddon.Containers[dashboardIndex].CPULimits) - addValue(parametersMap, "kubernetesDashboardMemoryRequests", dashboardAddon.Containers[dashboardIndex].MemoryRequests) - addValue(parametersMap, "kubernetesDashboardMemoryLimit", dashboardAddon.Containers[dashboardIndex].MemoryLimits) - addValue(parametersMap, "kubernetesDashboardSpec", dashboardAddon.Containers[dashboardIndex].Image) - } - } - if kubernetesConfig.IsReschedulerEnabled() { - reschedulerAddon := kubernetesConfig.GetAddonByName(DefaultReschedulerAddonName) - reschedulerIndex := reschedulerAddon.GetAddonContainersIndexByName(DefaultReschedulerAddonName) - if reschedulerIndex > -1 { - addValue(parametersMap, "kubernetesReschedulerCPURequests", reschedulerAddon.Containers[reschedulerIndex].CPURequests) - addValue(parametersMap, "kubernetesReschedulerCPULimit", reschedulerAddon.Containers[reschedulerIndex].CPULimits) - addValue(parametersMap, "kubernetesReschedulerMemoryRequests", reschedulerAddon.Containers[reschedulerIndex].MemoryRequests) - addValue(parametersMap, "kubernetesReschedulerMemoryLimit", reschedulerAddon.Containers[reschedulerIndex].MemoryLimits) - addValue(parametersMap, "kubernetesReschedulerSpec", reschedulerAddon.Containers[reschedulerIndex].Image) - } - } - if properties.OrchestratorProfile.IsMetricsServerEnabled() { - metricsServerAddon := kubernetesConfig.GetAddonByName(DefaultMetricsServerAddonName) - metricsServerIndex := metricsServerAddon.GetAddonContainersIndexByName(DefaultMetricsServerAddonName) - if metricsServerIndex > -1 { - if metricsServerAddon.Containers[metricsServerIndex].Image != "" { - addValue(parametersMap, "kubernetesMetricsServerSpec", metricsServerAddon.Containers[metricsServerIndex].Image) - } else { - addValue(parametersMap, "kubernetesMetricsServerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents[DefaultMetricsServerAddonName]) - } - } - } - if properties.IsNVIDIADevicePluginEnabled() { - nvidiaDevicePluginAddon := kubernetesConfig.GetAddonByName(NVIDIADevicePluginAddonName) - nvidiaPluginIndex := nvidiaDevicePluginAddon.GetAddonContainersIndexByName(NVIDIADevicePluginAddonName) - if nvidiaPluginIndex > -1 { - addValue(parametersMap, "kubernetesNVIDIADevicePluginCPURequests", nvidiaDevicePluginAddon.Containers[nvidiaPluginIndex].CPURequests) - addValue(parametersMap, "kubernetesNVIDIADevicePluginCPULimit", nvidiaDevicePluginAddon.Containers[nvidiaPluginIndex].CPULimits) - addValue(parametersMap, "kubernetesNVIDIADevicePluginMemoryRequests", nvidiaDevicePluginAddon.Containers[nvidiaPluginIndex].MemoryRequests) - addValue(parametersMap, "kubernetesNVIDIADevicePluginMemoryLimit", nvidiaDevicePluginAddon.Containers[nvidiaPluginIndex].MemoryLimits) - addValue(parametersMap, "kubernetesNVIDIADevicePluginSpec", nvidiaDevicePluginAddon.Containers[nvidiaPluginIndex].Image) - } - } - if kubernetesConfig.IsContainerMonitoringEnabled() { - containerMonitoringAddon := kubernetesConfig.GetAddonByName(ContainerMonitoringAddonName) - omsagentIndex := containerMonitoringAddon.GetAddonContainersIndexByName("omsagent") - if omsagentIndex > -1 { - addValue(parametersMap, "omsAgentVersion", containerMonitoringAddon.Config["omsAgentVersion"]) - addValue(parametersMap, "omsAgentDockerProviderVersion", containerMonitoringAddon.Config["dockerProviderVersion"]) - addValue(parametersMap, "omsAgentWorkspaceGuid", containerMonitoringAddon.Config["workspaceGuid"]) - addValue(parametersMap, "omsAgentWorkspaceKey", containerMonitoringAddon.Config["workspaceKey"]) - addValue(parametersMap, "kubernetesOMSAgentCPURequests", containerMonitoringAddon.Containers[omsagentIndex].CPURequests) - addValue(parametersMap, "kubernetesOMSAgentCPULimit", containerMonitoringAddon.Containers[omsagentIndex].CPULimits) - addValue(parametersMap, "kubernetesOMSAgentMemoryRequests", containerMonitoringAddon.Containers[omsagentIndex].MemoryRequests) - addValue(parametersMap, "kubernetesOMSAgentMemoryLimit", containerMonitoringAddon.Containers[omsagentIndex].MemoryLimits) - addValue(parametersMap, "omsAgentImage", containerMonitoringAddon.Containers[omsagentIndex].Image) - } - } - if kubernetesConfig.IsIPMasqAgentEnabled() { - ipMasqAgentAddon := kubernetesConfig.GetAddonByName(IPMASQAgentAddonName) - ipMasqAgentIndex := ipMasqAgentAddon.GetAddonContainersIndexByName(IPMASQAgentAddonName) - if ipMasqAgentIndex > -1 { - addValue(parametersMap, "kubernetesIPMasqAgentCPURequests", ipMasqAgentAddon.Containers[ipMasqAgentIndex].CPURequests) - addValue(parametersMap, "kubernetesIPMasqAgentMemoryRequests", ipMasqAgentAddon.Containers[ipMasqAgentIndex].MemoryRequests) - addValue(parametersMap, "kubernetesIPMasqAgentCPULimit", ipMasqAgentAddon.Containers[ipMasqAgentIndex].CPULimits) - addValue(parametersMap, "kubernetesIPMasqAgentMemoryLimit", ipMasqAgentAddon.Containers[ipMasqAgentIndex].MemoryLimits) - } - } if kubernetesConfig.LoadBalancerSku == "Standard" { random := rand.New(rand.NewSource(time.Now().UnixNano())) elbsvcName := random.Int() addValue(parametersMap, "kuberneteselbsvcname", fmt.Sprintf("%d", elbsvcName)) } - - if properties.OrchestratorProfile.IsAzureCNI() { - azureCNINetworkmonitorAddon := kubernetesConfig.GetAddonByName(AzureCNINetworkMonitoringAddonName) - azureCNIIndex := azureCNINetworkmonitorAddon.GetAddonContainersIndexByName(AzureCNINetworkMonitoringAddonName) - if azureCNIIndex > -1 { - addValue(parametersMap, "AzureCNINetworkMonitorImageURL", azureCNINetworkmonitorAddon.Containers[azureCNIIndex].Image) - } - } if common.IsKubernetesVersionGe(k8sVersion, "1.12.0") { addValue(parametersMap, "kubernetesCoreDNSSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+k8sComponents["coredns"]) } else { diff --git a/pkg/acsengine/params_k8s_test.go b/pkg/acsengine/params_k8s_test.go index f76cca2c5..19e556583 100644 --- a/pkg/acsengine/params_k8s_test.go +++ b/pkg/acsengine/params_k8s_test.go @@ -4,8 +4,6 @@ import ( "path" "testing" - "github.com/Azure/acs-engine/pkg/helpers" - "github.com/Azure/acs-engine/pkg/api" "github.com/Azure/acs-engine/pkg/i18n" "github.com/leonelquinteros/gotext" @@ -48,127 +46,6 @@ func TestAssignKubernetesParameters(t *testing.T) { } } -func TestKubernetesParamsAddons(t *testing.T) { - tests := map[string]struct { - addon api.KubernetesAddon - expectedParams []string - }{ - "nvidia-device-plugin": { - api.KubernetesAddon{ - Name: api.NVIDIADevicePluginAddonName, - Enabled: helpers.PointerToBool(true), - Containers: []api.KubernetesContainerSpec{ - { - Name: api.NVIDIADevicePluginAddonName, - CPURequests: "50m", - MemoryRequests: "150Mi", - CPULimits: "50m", - MemoryLimits: "150Mi", - }, - }, - }, - []string{ - "kubernetesNVIDIADevicePluginCPURequests", - "kubernetesNVIDIADevicePluginMemoryRequests", - "kubernetesNVIDIADevicePluginCPULimit", - "kubernetesNVIDIADevicePluginMemoryLimit", - "kubernetesNVIDIADevicePluginSpec", - }, - }, - - "container-monitoring": { - api.KubernetesAddon{ - Name: api.ContainerMonitoringAddonName, - Enabled: helpers.PointerToBool(true), - Containers: []api.KubernetesContainerSpec{ - { - Name: "omsagent", - CPURequests: "50m", - MemoryRequests: "150Mi", - CPULimits: "50m", - MemoryLimits: "150Mi", - }, - }, - }, - []string{ - "omsAgentVersion", - "omsAgentDockerProviderVersion", - "omsAgentWorkspaceGuid", - "omsAgentWorkspaceKey", - "kubernetesOMSAgentCPURequests", - "kubernetesOMSAgentCPULimit", - "kubernetesOMSAgentMemoryLimit", - "kubernetesOMSAgentMemoryRequests", - "omsAgentImage", - }, - }, - - "kubernetes-dashboard": { - api.KubernetesAddon{ - Name: api.DefaultDashboardAddonName, - Enabled: helpers.PointerToBool(true), - Containers: []api.KubernetesContainerSpec{ - { - Name: api.DefaultDashboardAddonName, - CPURequests: "50m", - MemoryRequests: "150Mi", - CPULimits: "50m", - MemoryLimits: "150Mi", - }, - }, - }, - []string{ - "kubernetesDashboardCPURequests", - "kubernetesDashboardCPULimit", - "kubernetesDashboardMemoryRequests", - "kubernetesDashboardMemoryLimit", - "kubernetesDashboardSpec", - }, - }, - - "rescheduler": { - api.KubernetesAddon{ - Name: api.DefaultReschedulerAddonName, - Enabled: helpers.PointerToBool(true), - Containers: []api.KubernetesContainerSpec{ - { - Name: api.DefaultReschedulerAddonName, - CPURequests: "50m", - MemoryRequests: "150Mi", - CPULimits: "50m", - MemoryLimits: "150Mi", - }, - }, - }, - []string{ - "kubernetesReschedulerCPURequests", - "kubernetesReschedulerCPULimit", - "kubernetesReschedulerMemoryRequests", - "kubernetesReschedulerMemoryLimit", - "kubernetesReschedulerSpec", - }, - }, - } - - for name, test := range tests { - test := test - t.Run(name, func(t *testing.T) { - t.Parallel() - properties := api.GetK8sDefaultProperties(false) - properties.OrchestratorProfile.KubernetesConfig.Addons = []api.KubernetesAddon{test.addon} - parametersMap := paramsMap{} - assignKubernetesParameters(properties, parametersMap, api.AzureCloudSpec, DefaultGeneratorCode) - - for _, expectedParam := range test.expectedParams { - if !isKeyPresent(expectedParam, parametersMap) { - t.Errorf("expected key %s to be present in the map", expectedParam) - } - } - }) - } - -} - func isKeyPresent(key string, paramMap map[string]interface{}) bool { _, ok := paramMap[key] return ok diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index 5dff9ebed..ddfaff057 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -193,6 +193,10 @@ func (t *TemplateGenerator) getMasterCustomData(cs *api.ContainerService, textFi customFilesReader, "MASTER_CUSTOM_FILES_PLACEHOLDER") + addonStr := getContainerAddonsString(cs.Properties, "k8s/containeraddons") + + str = strings.Replace(str, "MASTER_CONTAINER_ADDONS_PLACEHOLDER", addonStr, -1) + // return the custom data return fmt.Sprintf("\"customData\": \"[base64(concat('%s'))]\",", str) } diff --git a/pkg/api/addons.go b/pkg/api/addons.go index 624b71a19..6b7a16266 100644 --- a/pkg/api/addons.go +++ b/pkg/api/addons.go @@ -80,6 +80,7 @@ func (cs *ContainerService) setAddonsConfig(isUpdate bool) { MemoryRequests: "10Mi", CPULimits: "50m", MemoryLimits: "10Mi", + Image: "andyzhangx/blobfuse-flexvol-installer", }, }, } @@ -94,6 +95,7 @@ func (cs *ContainerService) setAddonsConfig(isUpdate bool) { MemoryRequests: "10Mi", CPULimits: "50m", MemoryLimits: "10Mi", + Image: "andyzhangx/smb-flexvol-installer", }, }, } @@ -108,6 +110,7 @@ func (cs *ContainerService) setAddonsConfig(isUpdate bool) { MemoryRequests: "10Mi", CPULimits: "50m", MemoryLimits: "10Mi", + Image: "mcr.microsoft.com/k8s/flexvolume/keyvault-flexvolume:v0.0.5", }, }, } @@ -198,6 +201,7 @@ func (cs *ContainerService) setAddonsConfig(isUpdate bool) { MemoryRequests: "50Mi", CPULimits: "50m", MemoryLimits: "250Mi", + Image: specConfig.KubernetesImageBase + "ip-masq-agent-amd64:v2.0.0", }, }, } @@ -223,6 +227,19 @@ func (cs *ContainerService) setAddonsConfig(isUpdate bool) { }, } + defaultDNSAutoScalerAddonsConfig := KubernetesAddon{ + Name: DefaultDNSAutoscalerAddonName, + Enabled: helpers.PointerToBool(DefaultDNSAutoscalerAddonEnabled), + Containers: []KubernetesContainerSpec{ + { + Name: DefaultDNSAutoscalerAddonName, + Image: specConfig.KubernetesImageBase + "cluster-proportional-autoscaler-amd64:1.1.1", + CPURequests: "20m", + MemoryRequests: "10Mi", + }, + }, + } + defaultAddons := []KubernetesAddon{ defaultTillerAddonsConfig, defaultACIConnectorAddonsConfig, @@ -238,6 +255,7 @@ func (cs *ContainerService) setAddonsConfig(isUpdate bool) { defaultAzureCNINetworkMonitorAddonsConfig, defaultAzureNetworkPolicyAddonsConfig, defaultIPMasqAgentAddonsConfig, + defaultDNSAutoScalerAddonsConfig, } // Add default addons specification, if no user-provided spec exists if o.KubernetesConfig.Addons == nil { diff --git a/pkg/api/const.go b/pkg/api/const.go index 0b3a296b0..e47cc4c44 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -145,8 +145,8 @@ const ( DefaultNVIDIADevicePluginAddonEnabled = false // DefaultContainerMonitoringAddonEnabled determines the acs-engine provided default for enabling kubernetes container monitoring addon DefaultContainerMonitoringAddonEnabled = false - // DefaultAzureCNINetworkMonitoringAddonEnabled Azure CNI networkmonitor addon default - DefaultAzureCNINetworkMonitoringAddonEnabled = false + // DefaultDNSAutoscalerAddonEnabled determines the acs-engine provided default for dns-autoscaler addon + DefaultDNSAutoscalerAddonEnabled = false // IPMasqAgentAddonEnabled enables the ip-masq-agent addon IPMasqAgentAddonEnabled = true // DefaultTillerAddonName is the name of the tiller addon deployment @@ -194,6 +194,8 @@ const ( ARMVirtualNetworksResourceType = "virtualNetworks" // DefaultAcceleratedNetworkingWindowsEnabled determines the acs-engine provided default for enabling accelerated networking on Windows nodes DefaultAcceleratedNetworkingWindowsEnabled = false + // DefaultDNSAutoscalerAddonName is the name of the dns-autoscaler addon + DefaultDNSAutoscalerAddonName = "dns-autoscaler" ) const ( diff --git a/pkg/api/defaults_test.go b/pkg/api/defaults_test.go index 6ce0d6fc7..ad3dc3b0d 100644 --- a/pkg/api/defaults_test.go +++ b/pkg/api/defaults_test.go @@ -186,16 +186,17 @@ func TestAssignDefaultAddonImages(t *testing.T) { DefaultTillerAddonName: "gcr.io/kubernetes-helm/tiller:v2.8.1", DefaultACIConnectorAddonName: "microsoft/virtual-kubelet:latest", DefaultClusterAutoscalerAddonName: "k8s.gcr.io/cluster-autoscaler:v1.2.2", - DefaultBlobfuseFlexVolumeAddonName: "", - DefaultSMBFlexVolumeAddonName: "", - DefaultKeyVaultFlexVolumeAddonName: "", + DefaultBlobfuseFlexVolumeAddonName: "andyzhangx/blobfuse-flexvol-installer", + DefaultSMBFlexVolumeAddonName: "andyzhangx/smb-flexvol-installer", + DefaultKeyVaultFlexVolumeAddonName: "mcr.microsoft.com/k8s/flexvolume/keyvault-flexvolume:v0.0.5", DefaultDashboardAddonName: "k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.0", DefaultReschedulerAddonName: "k8s.gcr.io/rescheduler:v0.3.1", DefaultMetricsServerAddonName: "k8s.gcr.io/metrics-server-amd64:v0.2.1", NVIDIADevicePluginAddonName: "nvidia/k8s-device-plugin:1.10", ContainerMonitoringAddonName: "microsoft/oms:ciprod10162018-2", - IPMASQAgentAddonName: "", + IPMASQAgentAddonName: "k8s.gcr.io/ip-masq-agent-amd64:v2.0.0", AzureCNINetworkMonitoringAddonName: "containernetworking/networkmonitor:v0.0.4", + DefaultDNSAutoscalerAddonName: "k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.1", } var addons []KubernetesAddon diff --git a/pkg/api/types.go b/pkg/api/types.go index bc893141a..1108fd47c 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1236,7 +1236,7 @@ func (k *KubernetesConfig) IsAADPodIdentityEnabled() bool { // IsACIConnectorEnabled checks if the ACI Connector addon is enabled func (k *KubernetesConfig) IsACIConnectorEnabled() bool { - return k.isAddonEnabled(DefaultACIConnectorAddonName, DefaultAADPodIdentityAddonEnabled) + return k.isAddonEnabled(DefaultACIConnectorAddonName, DefaultACIConnectorAddonEnabled) } // IsClusterAutoscalerEnabled checks if the cluster autoscaler addon is enabled