feat: generic cluster-init component (#3023)

cmd/upgrade.go

@@ -12,12 +12,14 @@ import (
 	"time"
 
 	"github.com/Azure/aks-engine/pkg/api"
+	"github.com/Azure/aks-engine/pkg/api/common"
 	"github.com/Azure/aks-engine/pkg/armhelpers"
 	"github.com/Azure/aks-engine/pkg/armhelpers/utils"
 	"github.com/Azure/aks-engine/pkg/engine"
 	"github.com/Azure/aks-engine/pkg/helpers"
 	"github.com/Azure/aks-engine/pkg/i18n"
 	"github.com/Azure/aks-engine/pkg/operations/kubernetesupgrade"
+	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/leonelquinteros/gotext"
 	"github.com/pkg/errors"
 
@@ -35,16 +37,17 @@ type upgradeCmd struct {
 	authProvider
 
 	// user input
-	resourceGroupName           string
-	apiModelPath                string
-	deploymentDirectory         string
-	upgradeVersion              string
-	location                    string
-	kubeconfigPath              string
-	timeoutInMinutes            int
-	cordonDrainTimeoutInMinutes int
-	force                       bool
-	controlPlaneOnly            bool
+	resourceGroupName                        string
+	apiModelPath                             string
+	deploymentDirectory                      string
+	upgradeVersion                           string
+	location                                 string
+	kubeconfigPath                           string
+	timeoutInMinutes                         int
+	cordonDrainTimeoutInMinutes              int
+	force                                    bool
+	controlPlaneOnly                         bool
+	disableClusterInitComponentDuringUpgrade bool
 
 	// derived
 	containerService    *api.ContainerService
@@ -161,6 +164,14 @@ func (uc *upgradeCmd) loadCluster() error {
 		return errors.Wrap(err, "error parsing the api model")
 	}
 
+	// The cluster-init component is a cluster create-only feature, temporarily disable if enabled
+	if i := api.GetComponentsIndexByName(uc.containerService.Properties.OrchestratorProfile.KubernetesConfig.Components, common.ClusterInitComponentName); i > -1 {
+		if uc.containerService.Properties.OrchestratorProfile.KubernetesConfig.Components[i].IsEnabled() {
+			uc.disableClusterInitComponentDuringUpgrade = true
+			uc.containerService.Properties.OrchestratorProfile.KubernetesConfig.Components[i].Enabled = to.BoolPtr(false)
+		}
+	}
+
 	if uc.containerService.Properties.IsAzureStackCloud() {
 		writeCustomCloudProfile(uc.containerService)
 		if err = uc.containerService.Properties.SetAzureStackCloudSpec(api.AzureStackCloudSpecParams{
@@ -296,6 +307,12 @@ func (uc *upgradeCmd) run(cmd *cobra.Command, args []string) error {
 	}
 
 	// Save the new apimodel to reflect the cluster's state.
+	// Restore the original cluster-init component enabled value, if it was disabled during upgrade
+	if uc.disableClusterInitComponentDuringUpgrade {
+		if i := api.GetComponentsIndexByName(uc.containerService.Properties.OrchestratorProfile.KubernetesConfig.Components, common.ClusterInitComponentName); i > -1 {
+			uc.containerService.Properties.OrchestratorProfile.KubernetesConfig.Components[i].Enabled = to.BoolPtr(true)
+		}
+	}
 	apiloader := &api.Apiloader{
 		Translator: &i18n.Translator{
 			Locale: uc.locale,

docs/topics/clusterdefinitions.md

@@ -248,6 +248,7 @@ The reason for the unsightly base64-encoded input type is to optimize delivery p
 | kube-apiserver | true | 1 per master node | The Kubernetes API server validates and configures data for the api objects which include pods, services, replicationcontrollers, and others. The API Server services REST operations and provides the frontend to the cluster's shared state through which all other components interact. Official docs [here](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/). |
 | kube-scheduler | true | 1 per master node | The Kubernetes scheduler is a policy-rich, topology-aware, workload-specific function that significantly impacts availability, performance, and capacity. The scheduler needs to take into account individual and collective resource requirements, quality of service requirements, hardware/software/policy constraints, affinity and anti-affinity specifications, data locality, inter-workload interference, deadlines, and so on. Official docs [here](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/). |
 | kube-addon-manager | true | 1 per master node | Addon manager provides a standard way to deliver additional Kubernetes componentry. The addons supported by AKS Engine (documented [above](#addons)) are installed into the cluster using Addon manager. Official docs [here](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/addon-manager). |
+| cluster-init | false | n/a | This is an interface to deliver Kubernetes resource configuration that may be tightly coupled to an ARM deployment. For example, if you use AKS Engine in an automated pipeline and you can scale up a cluster quickly by loading a Kubernetes specification immediately after bootstrapping the control plane, give the base64-encoded YAML representation of that spec to `cluster-init`. That data is decoded at cluster creation time to the path `/opt/azure/containers/cluster-init.yaml` on the first master VM, and then loaded into the cluster via `kubectl apply -f`. See[`kubectl apply`](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#apply) for more documentation on how to specify the source YAML. Note: the `cluster-init` spec is ignored during `aks-engine upgrade` and `aks-engine scale`operations. |
 
 To give a bit more info on the `components` property: Currently, there are two configuration vectors available for use. Firstly, you have the option to pass in a custom image reference for the container that the AKS Engine-provided specs implement (all components are implemented as single-container Pod resources); in addition you may provide an alternate command string to execute inside the container. E.g.:
 
@@ -335,6 +336,20 @@ cGF0aDogL3Zhci9saWIvd2FhZ2VudC9NYW5hZ2VkSWRlbnRpdHktU2V0dGluZ3MK"
 }
 ```
 
+The above is the pattern we use to pass in a `cluster-init` spec for loading at cluster bootstrap time. E.g.:
+
+```json
+"kubernetesConfig": {
+    "components": [
+        {
+            "name": "cluster-init",
+            "enabled": true,
+            "data": "YXBpVmVyc2lvbjogdjEKa2luZDogUG9kCm1ldGFkYXRhOgogIG5hbWU6IGFrcy1lbmdpbmUtcG9kLWluaXQKc3BlYzoKICBjb250YWluZXJzOgogIC0gbmFtZTogYWtzLWVuZ2luZS1wb2QtaW5pdAogICAgaW1hZ2U6IGJ1c3lib3g6MS4zMS4xCiAgICBhcmdzOgogICAgLSAvYmluL3NoCiAgICAtIC1jCiAgICAtIHdoaWxlIHRydWU7IGRvIHNsZWVwIDYwMDsgZG9uZQogIG5vZGVTZWxlY3RvcjoKICAgIGJldGEua3ViZXJuZXRlcy5pby9vczogbGludXgKLS0tCmFwaVZlcnNpb246IGJhdGNoL3YxCmtpbmQ6IEpvYgptZXRhZGF0YToKICBuYW1lOiBha3MtZW5naW5lLWpvYi1pbml0CnNwZWM6CiAgdGVtcGxhdGU6CiAgICBzcGVjOgogICAgICBjb250YWluZXJzOgogICAgICAtIGltYWdlOiBidXN5Ym94OjEuMzEuMQogICAgICAgIG5hbWU6IGJ1c3lib3gtYWdlbnQKICAgICAgICBjb21tYW5kOiBbJ3NoJywgJy1jJywgJ1sgJChlY2hvICJIZWxsbywgV29ybGQhIiB8IHNoYTI1NnN1bSB8IGN1dCAtZCIgIiAtZjEpID0gImM5OGMyNGI2NzdlZmY0NDg2MGFmZWE2ZjQ5M2JiYWVjNWJiMWM0Y2JiMjA5YzZmYzJiYmI0N2Y2NmZmMmFkMzEiIF0nXQogICAgICByZXN0YXJ0UG9saWN5OiBOZXZlcgogICAgICBub2RlU2VsZWN0b3I6CiAgICAgICAgYmV0YS5rdWJlcm5ldGVzLmlvL29zOiBsaW51eAogIGJhY2tvZmZMaW1pdDogMAo="
+        }
+    ]
+}
+```
+
 <a name="feat-kubelet-config"></a>
 
 #### kubeletConfig

parts/k8s/cloud-init/artifacts/cse_config.sh

@@ -4,6 +4,7 @@ NODE_NAME=$(hostname)
 PRIVATE_IP=$(hostname -I | cut -d' ' -f1)
 ETCD_PEER_URL="https://${PRIVATE_IP}:2380"
 ETCD_CLIENT_URL="https://${PRIVATE_IP}:2379"
+KUBECTL="/usr/local/bin/kubectl --kubeconfig=/home/$ADMINUSER/.kube/config"
 
 systemctlEnableAndStart() {
   systemctl_restart 100 5 30 $1

parts/k8s/cloud-init/artifacts/cse_helpers.sh

@@ -7,7 +7,6 @@ DEBIAN_OS_NAME="DEBIAN"
 if ! echo "${UBUNTU_OS_NAME} ${RHEL_OS_NAME} ${DEBIAN_OS_NAME}" | grep -q "${OS}"; then
   OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID_LIKE=(.*))$/, a) { print toupper(a[2] a[3]); exit }')
 fi
-KUBECTL=/usr/local/bin/kubectl
 DOCKER=/usr/bin/docker
 export GPU_DV=418.40.04
 export GPU_DEST=/usr/local/nvidia

parts/k8s/cloud-init/artifacts/cse_main.sh

@@ -235,6 +235,12 @@ if [[ -n ${MASTER_NODE} ]]; then
   {{if IsAzurePolicyAddonEnabled}}
   time_metric "EnsureLabelExclusionForAzurePolicyAddon" ensureLabelExclusionForAzurePolicyAddon
   {{end}}
+  {{- if HasClusterInitComponent}}
+  if [[ $NODE_INDEX == 0 ]]; then
+    retrycmd 120 5 30 $KUBECTL apply -f /opt/azure/containers/cluster-init.yaml --server-dry-run=true || exit {{GetCSEErrorCode "ERR_CLUSTER_INIT_FAIL"}}
+    retrycmd 120 5 30 $KUBECTL apply -f /opt/azure/containers/cluster-init.yaml || exit {{GetCSEErrorCode "ERR_CLUSTER_INIT_FAIL"}}
+  fi
+  {{end}}
 fi
 
 {{- if not IsVHDDistroForAllNodes}}

pkg/api/common/const.go

@@ -288,6 +288,8 @@ const (
 	APIServerComponentName = "kube-apiserver"
 	// AddonManagerComponentName is the name of the kube-addon-manager component
 	AddonManagerComponentName = "kube-addon-manager"
+	// ClusterInitComponentName is the name of the cluster-init component
+	ClusterInitComponentName = "cluster-init"
 )
 
 const WindowsArtifactComponentName = "windowszip"

pkg/api/components.go

@@ -99,7 +99,7 @@ func (cs *ContainerService) setComponentsConfig(isUpgrade bool) {
 	// Honor custom{component}Image fields
 	useHyperkube := !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.17.0")
 	for _, component := range defaultComponents {
-		if i := getComponentsIndexByName(kubernetesConfig.Components, component.Name); i > -1 {
+		if i := GetComponentsIndexByName(kubernetesConfig.Components, component.Name); i > -1 {
 			var customComponentImage string
 			switch component.Name {
 			case common.APIServerComponentName:
@@ -139,7 +139,7 @@ func (cs *ContainerService) setComponentsConfig(isUpgrade bool) {
 
 func appendComponentIfNotPresent(components []KubernetesComponent, component KubernetesComponent) []KubernetesComponent {
 	if component.Name != "" {
-		i := getComponentsIndexByName(components, component.Name)
+		i := GetComponentsIndexByName(components, component.Name)
 		if i < 0 {
 			return append(components, component)
 		}
@@ -147,7 +147,7 @@ func appendComponentIfNotPresent(components []KubernetesComponent, component Kub
 	return components
 }
 
-func getComponentsIndexByName(components []KubernetesComponent, name string) int {
+func GetComponentsIndexByName(components []KubernetesComponent, name string) int {
 	for i := range components {
 		if components[i].Name == name {
 			return i
@@ -214,7 +214,7 @@ func assignDefaultComponentVals(component, defaultComponent KubernetesComponent,
 }
 
 func synthesizeComponentsConfig(components []KubernetesComponent, defaultComponent KubernetesComponent, isUpgrade bool) {
-	i := getComponentsIndexByName(components, defaultComponent.Name)
+	i := GetComponentsIndexByName(components, defaultComponent.Name)
 	if i >= 0 {
 		components[i] = assignDefaultComponentVals(components[i], defaultComponent, isUpgrade)
 	}

pkg/api/components_test.go

@@ -501,12 +501,12 @@ func TestSetComponentsConfig(t *testing.T) {
 				common.APIServerComponentName,
 				common.AddonManagerComponentName,
 			} {
-				component := test.cs.Properties.OrchestratorProfile.KubernetesConfig.Components[getComponentsIndexByName(test.cs.Properties.OrchestratorProfile.KubernetesConfig.Components, componentName)]
+				component := test.cs.Properties.OrchestratorProfile.KubernetesConfig.Components[GetComponentsIndexByName(test.cs.Properties.OrchestratorProfile.KubernetesConfig.Components, componentName)]
 				if component.IsEnabled() {
-					if i := getComponentsIndexByName(test.expectedComponents, componentName); i == -1 {
+					if i := GetComponentsIndexByName(test.expectedComponents, componentName); i == -1 {
 						t.Fatalf("got component %s that we weren't expecting", component.Name)
 					}
-					expectedComponent := test.expectedComponents[getComponentsIndexByName(test.expectedComponents, componentName)]
+					expectedComponent := test.expectedComponents[GetComponentsIndexByName(test.expectedComponents, componentName)]
 					if to.Bool(component.Enabled) != to.Bool(expectedComponent.Enabled) {
 						t.Fatalf("expected component %s to have Enabled value %t, instead got %t", expectedComponent.Name, to.Bool(expectedComponent.Enabled), to.Bool(component.Enabled))
 					}
@@ -550,7 +550,7 @@ func TestSetComponentsConfig(t *testing.T) {
 						}
 					}
 				} else {
-					if i := getComponentsIndexByName(test.expectedComponents, componentName); i > -1 {
+					if i := GetComponentsIndexByName(test.expectedComponents, componentName); i > -1 {
 						if to.Bool(test.expectedComponents[i].Enabled) {
 							t.Fatalf("expected component %s to be enabled, instead it was disabled", componentName)
 						}
@@ -681,9 +681,9 @@ func TestGetComponentsIndexByName(t *testing.T) {
 		c := c
 		t.Run(c.name, func(t *testing.T) {
 			t.Parallel()
-			result := getComponentsIndexByName(c.components, c.componentName)
+			result := GetComponentsIndexByName(c.components, c.componentName)
 			if result != c.expected {
-				t.Fatalf("expected getComponentsIndexByName() result %d to be equal to %d", result, c.expected)
+				t.Fatalf("expected GetComponentsIndexByName() result %d to be equal to %d", result, c.expected)
 			}
 		})
 	}
@@ -692,7 +692,7 @@ func TestGetComponentsIndexByName(t *testing.T) {
 func TestAssignDefaultComponentVals(t *testing.T) {
 	containerServiceMap := getContainerServicesMap()
 	defaultOneDotFifteenComponents := getDefaultComponents(getContainerServicesMap()["1.15"])
-	controllerManagerComponent := defaultOneDotFifteenComponents[getComponentsIndexByName(defaultOneDotFifteenComponents, common.ControllerManagerComponentName)]
+	controllerManagerComponent := defaultOneDotFifteenComponents[GetComponentsIndexByName(defaultOneDotFifteenComponents, common.ControllerManagerComponentName)]
 	cases := []struct {
 		name             string
 		component        KubernetesComponent
@@ -884,7 +884,7 @@ func TestAssignDefaultComponentVals(t *testing.T) {
 
 func TestSynthesizeComponentsConfig(t *testing.T) {
 	defaultOneDotFifteenComponents := getDefaultComponents(getContainerServicesMap()["1.15"])
-	i := getComponentsIndexByName(defaultOneDotFifteenComponents, common.ControllerManagerComponentName)
+	i := GetComponentsIndexByName(defaultOneDotFifteenComponents, common.ControllerManagerComponentName)
 	defaultControllerManagerComponent := defaultOneDotFifteenComponents[i]
 	customOneDotFifteenComponents := defaultOneDotFifteenComponents
 	customControllerManagerComponent := KubernetesComponent{
@@ -934,7 +934,7 @@ func TestSynthesizeComponentsConfig(t *testing.T) {
 		t.Run(c.name, func(t *testing.T) {
 			t.Parallel()
 			synthesizeComponentsConfig(c.components, c.defaultComponent, c.isUpgrade)
-			i := getComponentsIndexByName(c.components, common.ControllerManagerComponentName)
+			i := GetComponentsIndexByName(c.components, common.ControllerManagerComponentName)
 			if !reflect.DeepEqual(c.components[i], c.expected) {
 				t.Fatalf("expected synthesizeComponentsConfig() result %v to be equal to %v", c.components[i], c.expected)
 			}
@@ -1296,7 +1296,7 @@ func overwriteDefaultComponents(components []KubernetesComponent, cs *ContainerS
 	var ret []KubernetesComponent
 	defaults := getDefaultComponents(cs)
 	for _, defaultComponent := range defaults {
-		i := getComponentsIndexByName(components, defaultComponent.Name)
+		i := GetComponentsIndexByName(components, defaultComponent.Name)
 		if i > -1 {
 			ret = append(ret, components[i])
 		} else {

pkg/engine/artifacts.go

@@ -52,6 +52,10 @@ func kubernetesComponentSettingsInit(p *api.Properties) map[string]kubernetesCom
 			base64Data:      k.GetComponentData(common.AddonManagerComponentName),
 			destinationFile: addonManagerComponentDestinationFilename,
 		},
+		common.ClusterInitComponentName: {
+			base64Data:      k.GetComponentData(common.ClusterInitComponentName),
+			destinationFile: clusterInitComponentDestinationFilename,
+		},
 	}
 }
 

pkg/engine/artifacts_test.go

@@ -1191,6 +1191,7 @@ func TestKubernetesComponentSettingsInit(t *testing.T) {
 		expectedCloudControllerManager kubernetesComponentFileSpec
 		expectedAPIServer              kubernetesComponentFileSpec
 		expectedAddonManager           kubernetesComponentFileSpec
+		expectedClusterInit            kubernetesComponentFileSpec
 	}{
 		{
 			name: "components with data",
@@ -1220,6 +1221,10 @@ func TestKubernetesComponentSettingsInit(t *testing.T) {
 								Name: common.AddonManagerComponentName,
 								Data: base64Data,
 							},
+							{
+								Name: common.ClusterInitComponentName,
+								Data: base64Data,
+							},
 						},
 					},
 				},
@@ -1249,6 +1254,10 @@ func TestKubernetesComponentSettingsInit(t *testing.T) {
 				base64Data:      base64Data,
 				destinationFile: addonManagerComponentDestinationFilename,
 			},
+			expectedClusterInit: kubernetesComponentFileSpec{
+				base64Data:      base64Data,
+				destinationFile: clusterInitComponentDestinationFilename,
+			},
 		},
 		{
 			name: "components with no data",
@@ -1302,6 +1311,10 @@ func TestKubernetesComponentSettingsInit(t *testing.T) {
 				base64Data:      "",
 				destinationFile: addonManagerComponentDestinationFilename,
 			},
+			expectedClusterInit: kubernetesComponentFileSpec{
+				base64Data:      "",
+				destinationFile: clusterInitComponentDestinationFilename,
+			},
 		},
 		{
 			name: "no components in Properties object",
@@ -1331,6 +1344,10 @@ func TestKubernetesComponentSettingsInit(t *testing.T) {
 				base64Data:      "",
 				destinationFile: addonManagerComponentDestinationFilename,
 			},
+			expectedClusterInit: kubernetesComponentFileSpec{
+				base64Data:      "",
+				destinationFile: clusterInitComponentDestinationFilename,
+			},
 		},
 	}
 
@@ -1391,6 +1408,13 @@ func TestKubernetesComponentSettingsInit(t *testing.T) {
 					if c.expectedAddonManager.destinationFile != componentFileSpec[component].destinationFile {
 						t.Fatalf("Expected %s to be %s", componentFileSpec[component].destinationFile, c.expectedAddonManager.destinationFile)
 					}
+				case common.ClusterInitComponentName:
+					if c.expectedClusterInit.base64Data != componentFileSpec[component].base64Data {
+						t.Fatalf("Expected %s to be %s", componentFileSpec[component].base64Data, c.expectedClusterInit.base64Data)
+					}
+					if c.expectedClusterInit.destinationFile != componentFileSpec[component].destinationFile {
+						t.Fatalf("Expected %s to be %s", componentFileSpec[component].destinationFile, c.expectedClusterInit.destinationFile)
+					}
 				}
 			}
 		})

pkg/engine/const.go

@@ -277,4 +277,5 @@ const (
 	apiServerComponentDestinationFilename              string = "kube-apiserver.yaml"
 	addonManagerComponentSourceFilename                string = "kubernetesmaster-kube-addon-manager.yaml"
 	addonManagerComponentDestinationFilename           string = "kube-addon-manager.yaml"
+	clusterInitComponentDestinationFilename            string = "cluster-init.yaml"
 )

pkg/engine/cse.go

@@ -72,6 +72,7 @@ var cseErrorCodes = map[string]int{
 	"ERR_BCC_INSTALL_TIMEOUT":                    168,
 	"ERR_BPFTRACE_BIN_DOWNLOAD_FAIL":             169,
 	"ERR_BPFTRACE_TOOLS_DOWNLOAD_FAIL":           170,
+	"ERR_CLUSTER_INIT_FAIL":                      180,
 }
 
 func GetCSEErrorCode(errorType string) int {

pkg/engine/cse_test.go

@@ -85,6 +85,7 @@ func TestGetCSEErrorCode(t *testing.T) {
 				"ERR_BCC_INSTALL_TIMEOUT":                    168,
 				"ERR_BPFTRACE_BIN_DOWNLOAD_FAIL":             169,
 				"ERR_BPFTRACE_TOOLS_DOWNLOAD_FAIL":           170,
+				"ERR_CLUSTER_INIT_FAIL":                      180,
 			},
 		},
 		{

pkg/engine/engine.go

@@ -865,7 +865,7 @@ func getComponentsString(cs *api.ContainerService, sourcePath string) string {
 				if err != nil {
 					return ""
 				}
-			} else {
+			} else if setting.sourceFile != "" {
 				orchProfile := properties.OrchestratorProfile
 				versions := strings.Split(orchProfile.OrchestratorVersion, ".")
 				templ := template.New("component resolver template").Funcs(getComponentFuncMap(component, cs))
@@ -882,7 +882,11 @@ func getComponentsString(cs *api.ContainerService, sourcePath string) string {
 				templ.Execute(&buffer, component)
 				input = buffer.String()
 			}
-			result += getComponentString(input, "/etc/kubernetes/manifests", setting.destinationFile)
+			if componentName == common.ClusterInitComponentName {
+				result += getComponentString(input, "/opt/azure/containers", setting.destinationFile)
+			} else {
+				result += getComponentString(input, "/etc/kubernetes/manifests", setting.destinationFile)
+			}
 		}
 	}
 	return result

pkg/engine/template_generator.go

@@ -314,6 +314,10 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap {
 		"IsAzureCNI": func() bool {
 			return cs.Properties.OrchestratorProfile.IsAzureCNI()
 		},
+		"HasClusterInitComponent": func() bool {
+			_, enabled := cs.Properties.OrchestratorProfile.KubernetesConfig.IsComponentEnabled(common.ClusterInitComponentName)
+			return enabled
+		},
 		"HasCosmosEtcd": func() bool {
 			return cs.Properties.MasterProfile != nil && cs.Properties.MasterProfile.HasCosmosEtcd()
 		},

pkg/engine/template_generator_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/Azure/aks-engine/pkg/telemetry"
 
 	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/pkg/errors"
 )
 
@@ -182,6 +183,7 @@ func TestGetContainerServiceFuncMap(t *testing.T) {
 		expectedGetCSEErrorCodeVals           []int
 		expectedHasVHDDistroNodes             bool
 		expectedIsVHDDistroForAllNodes        bool
+		expectedHasClusterInitComponent       bool
 	}{
 		{
 			name: "1.15 release",
@@ -630,6 +632,45 @@ func TestGetContainerServiceFuncMap(t *testing.T) {
 			expectedGetPrivateAzureRegistryServer: "my-server",
 			expectedGetSysctlDConfigKeyVals:       "",
 		},
+		{
+			name: "cluster-init config",
+			cs: &api.ContainerService{
+				Properties: &api.Properties{
+					OrchestratorProfile: &api.OrchestratorProfile{
+						OrchestratorType:    api.Kubernetes,
+						OrchestratorVersion: "1.17.0-beta.1",
+						KubernetesConfig: &api.KubernetesConfig{
+							ContainerRuntime:        api.Docker,
+							KubernetesImageBaseType: common.KubernetesImageBaseTypeGCR,
+							Components: []api.KubernetesComponent{
+								{
+									Name:    common.ClusterInitComponentName,
+									Enabled: to.BoolPtr(true),
+								},
+							},
+						},
+					},
+					AgentPoolProfiles: []*api.AgentPoolProfile{
+						{
+							Name:                "pool1",
+							Count:               1,
+							AvailabilityProfile: api.VirtualMachineScaleSets,
+						},
+					},
+				},
+			},
+			expectedHasCustomSearchDomain:        false,
+			expectedGetSearchDomainName:          "",
+			expectedGetSearchDomainRealmUser:     "",
+			expectedGetSearchDomainRealmPassword: "",
+			expectedHasCustomNodesDNS:            false,
+			expectedGetHyperkubeImageReference:   "",
+			expectedGetTargetEnvironment:         "AzurePublicCloud",
+			expectedIsNSeriesSKU:                 false,
+			expectedIsDockerContainerRuntime:     true,
+			expectedGetSysctlDConfigKeyVals:      "",
+			expectedHasClusterInitComponent:      true,
+		},
 		{
 			name: "sysctl config",
 			cs: &api.ContainerService{
@@ -819,6 +860,11 @@ func TestGetContainerServiceFuncMap(t *testing.T) {
 					t.Errorf("expected funcMap invocation of GetCSEErrorCode to return %d, instead got %d", errorCodes[i], ret[0].Interface())
 				}
 			}
+			v = reflect.ValueOf(funcMap["HasClusterInitComponent"])
+			ret = v.Call(make([]reflect.Value, 0))
+			if ret[0].Interface() != c.expectedHasClusterInitComponent {
+				t.Errorf("expected funcMap invocation of HasClusterInitComponent to return %t, instead got %t", c.expectedHasClusterInitComponent, ret[0].Interface())
+			}
 		})
 	}
 }

pkg/engine/templates_generated.go

@@ -39746,6 +39746,7 @@ NODE_NAME=$(hostname)
 PRIVATE_IP=$(hostname -I | cut -d' ' -f1)
 ETCD_PEER_URL="https://${PRIVATE_IP}:2380"
 ETCD_CLIENT_URL="https://${PRIVATE_IP}:2379"
+KUBECTL="/usr/local/bin/kubectl --kubeconfig=/home/$ADMINUSER/.kube/config"
 
 systemctlEnableAndStart() {
   systemctl_restart 100 5 30 $1
@@ -40564,7 +40565,6 @@ DEBIAN_OS_NAME="DEBIAN"
 if ! echo "${UBUNTU_OS_NAME} ${RHEL_OS_NAME} ${DEBIAN_OS_NAME}" | grep -q "${OS}"; then
   OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID_LIKE=(.*))$/, a) { print toupper(a[2] a[3]); exit }')
 fi
-KUBECTL=/usr/local/bin/kubectl
 DOCKER=/usr/bin/docker
 export GPU_DV=418.40.04
 export GPU_DEST=/usr/local/nvidia
@@ -41268,6 +41268,12 @@ if [[ -n ${MASTER_NODE} ]]; then
   {{if IsAzurePolicyAddonEnabled}}
   time_metric "EnsureLabelExclusionForAzurePolicyAddon" ensureLabelExclusionForAzurePolicyAddon
   {{end}}
+  {{- if HasClusterInitComponent}}
+  if [[ $NODE_INDEX == 0 ]]; then
+    retrycmd 120 5 30 $KUBECTL apply -f /opt/azure/containers/cluster-init.yaml --server-dry-run=true || exit {{GetCSEErrorCode "ERR_CLUSTER_INIT_FAIL"}}
+    retrycmd 120 5 30 $KUBECTL apply -f /opt/azure/containers/cluster-init.yaml || exit {{GetCSEErrorCode "ERR_CLUSTER_INIT_FAIL"}}
+  fi
+  {{end}}
 fi
 
 {{- if not IsVHDDistroForAllNodes}}

test/e2e/config/config.go

@@ -36,6 +36,8 @@ type Config struct {
 	CleanUpIfFail       bool          `envconfig:"CLEANUP_IF_FAIL" default:"true"`
 	RetainSSH           bool          `envconfig:"RETAIN_SSH" default:"true"`
 	StabilityIterations int           `envconfig:"STABILITY_ITERATIONS"`
+	ClusterInitPodName  string        `envconfig:"CLUSTER_INIT_POD_NAME" default:""`
+	ClusterInitJobName  string        `envconfig:"CLUSTER_INIT_JOB_NAME" default:""`
 	Timeout             time.Duration `envconfig:"TIMEOUT" default:"20m"`
 	LBTimeout           time.Duration `envconfig:"LB_TIMEOUT" default:"20m"`
 	CurrentWorkingDir   string

test/e2e/kubernetes/kubernetes_test.go

@@ -174,6 +174,19 @@ var _ = AfterSuite(func() {
 
 var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", func() {
 	Describe("regardless of agent pool type", func() {
+		It("should check for cluster-init pod", func() {
+			if cfg.ClusterInitPodName != "" {
+				By(fmt.Sprintf("Ensuring that cluster-init Pod \"%s\" is Running", cfg.ClusterInitPodName))
+				running, err := pod.WaitOnSuccesses(cfg.ClusterInitPodName, "default", kubeSystemPodsReadinessChecks, sleepBetweenRetriesWhenWaitingForPodReady, cfg.Timeout)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(running).To(Equal(true))
+			}
+			if cfg.ClusterInitJobName != "" {
+				ready, err := job.WaitOnSucceeded(cfg.ClusterInitJobName, "default", 30*time.Second, cfg.Timeout)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(ready).To(Equal(true))
+			}
+		})
 		It("should validate host OS DNS", func() {
 			if cfg.BlockSSHPort {
 				Skip("SSH port is blocked")

test/e2e/test_cluster_configs/everything.json

@@ -4,7 +4,9 @@
 		"REGION_OPTIONS": "eastus,westeurope,westus2",
 		"GINKGO_SKIP": "should report all nodes in a Ready state",
 		"TEST_PVC": true,
-		"LB_TEST_TIMEOUT": "10m"
+		"LB_TEST_TIMEOUT": "10m",
+		"CLUSTER_INIT_POD_NAME": "aks-engine-pod-init",
+		"CLUSTER_INIT_JOB_NAME": "aks-engine-job-init"
 	},
 	"options": {
 		"allowedOrchestratorVersions": [
@@ -58,6 +60,13 @@
 							"name": "rescheduler",
 							"enabled": true
 						}
+					],
+					"components": [
+						{
+							"name": "cluster-init",
+							"enabled": true,
+							"data": "YXBpVmVyc2lvbjogdjEKa2luZDogUG9kCm1ldGFkYXRhOgogIG5hbWU6IGFrcy1lbmdpbmUtcG9kLWluaXQKc3BlYzoKICBjb250YWluZXJzOgogIC0gbmFtZTogYWtzLWVuZ2luZS1wb2QtaW5pdAogICAgaW1hZ2U6IGJ1c3lib3g6MS4zMS4xCiAgICBhcmdzOgogICAgLSAvYmluL3NoCiAgICAtIC1jCiAgICAtIHdoaWxlIHRydWU7IGRvIHNsZWVwIDYwMDsgZG9uZQogIG5vZGVTZWxlY3RvcjoKICAgIGJldGEua3ViZXJuZXRlcy5pby9vczogbGludXgKLS0tCmFwaVZlcnNpb246IGJhdGNoL3YxCmtpbmQ6IEpvYgptZXRhZGF0YToKICBuYW1lOiBha3MtZW5naW5lLWpvYi1pbml0CnNwZWM6CiAgdGVtcGxhdGU6CiAgICBzcGVjOgogICAgICBjb250YWluZXJzOgogICAgICAtIGltYWdlOiBidXN5Ym94OjEuMzEuMQogICAgICAgIG5hbWU6IGJ1c3lib3gtYWdlbnQKICAgICAgICBjb21tYW5kOiBbJ3NoJywgJy1jJywgJ1sgJChlY2hvICJIZWxsbywgV29ybGQhIiB8IHNoYTI1NnN1bSB8IGN1dCAtZCIgIiAtZjEpID0gImM5OGMyNGI2NzdlZmY0NDg2MGFmZWE2ZjQ5M2JiYWVjNWJiMWM0Y2JiMjA5YzZmYzJiYmI0N2Y2NmZmMmFkMzEiIF0nXQogICAgICByZXN0YXJ0UG9saWN5OiBOZXZlcgogICAgICBub2RlU2VsZWN0b3I6CiAgICAgICAgYmV0YS5rdWJlcm5ldGVzLmlvL29zOiBsaW51eAogIGJhY2tvZmZMaW1pdDogMAo="
+						}
 					]
 				}
 			},