downgrade to customscript extension 1.5 from 2.0, this required refactoring since node rate was unreliable with current approach

acstgen/acstgen.go

@@ -69,6 +69,7 @@ func prettyPrintArmTemplate(template string) (string, error) {
 		// there is a bug in ARM where it doesn't correctly translate back '\u003e' (>)
 		{">", "GREATERTHAN"},
 		{"<", "LESSTHAN"},
+		{"&", "AMPERSAND"},
 	}
 
 	template = translateJSON(template, translateParams, false)

acstgen/parts/kubernetesagentcustomdata.yml

@@ -142,4 +142,11 @@ write_files:
             --network-plugin=kubenet \
             --reconcile-cidr \
             --v=2
-    ExecStop=/usr/bin/docker stop -t 2 kubelet
+    ExecStop=/usr/bin/docker stop -t 2 kubelet
+
+- path: "/opt/azure/containers/provision.sh"
+  permissions: "0744"
+  encoding: gzip
+  owner: "root"
+  content: !!binary |
+    AGENT_PROVISION_B64_GZIP_STR

acstgen/parts/kubernetesagentcustomscript.sh

@@ -3,11 +3,24 @@
 ###########################################################
 # START SECRET DATA - ECHO DISABLED
 ###########################################################
+
+TID=$1
+SID=$2
+RGP=$3
+LOC=$4
+SUB=$5
+NSG=$6
+VNT=$7
+RTB=$8
+SVCPrincipalClientId=$9
+SVCPrincipalClientSecret=${10}
+CLIENTPRIVATEKEY=${11}
+
 CLIENTKEY=/etc/kubernetes/certs/client.key
 touch $CLIENTKEY
 chmod 0644 $CLIENTKEY
 chown root:root $CLIENTKEY
-echo {{{clientPrivateKey}}} | /usr/bin/base64 --decode > $CLIENTKEY 
+echo $CLIENTPRIVATEKEY | /usr/bin/base64 --decode > $CLIENTKEY 
 
 AZUREJSON=/etc/kubernetes/azure.json
 touch $AZUREJSON
@@ -17,8 +30,8 @@ AZURECONTENT=$(cat <<EOF
 {
     "tenantId": "$TID",
     "subscriptionId": "$SID",
-    "aadClientId": "{{{servicePrincipalClientId}}}",
+    "aadClientId": "$SVCPrincipalClientId",
-    "aadClientSecret": "{{{servicePrincipalClientSecret}}}",
+    "aadClientSecret": "$SVCPrincipalClientSecret",
     "resourceGroup": "$RGP",
     "location": "$LOC",
     "subnetName": "$SUB",
@@ -37,7 +50,7 @@ echo "$AZURECONTENT" > $AZUREJSON
 set -x
 
 # wait for docker to be available
-ensuredocker()
+ensuredockerbinary()
 {
     dockerfound=1
     for i in {1..600}; do
@@ -54,7 +67,7 @@ ensuredocker()
         exit 1
     fi
 }
-ensuredocker
+ensuredockerbinary
 
 # start all the services
 /bin/systemctl restart docker

acstgen/parts/kubernetesagentresources.t

@@ -154,13 +154,13 @@
       "type": "Microsoft.Compute/virtualMachines/extensions",
       "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(),'/cse', copyIndex())]",
       "properties": {
-        "publisher": "Microsoft.Azure.Extensions",
+        "publisher": "Microsoft.OSTCExtensions",
-        "type": "CustomScript",
+        "type": "CustomScriptForLinux",
-        "typeHandlerVersion": "2.0",
+        "typeHandlerVersion": "1.5",
         "autoUpgradeMinorVersion": true,
         "settings": {},
         "protectedSettings": {
-          "commandToExecute": "[concat('export TID=',variables('tenantID'),';export SID=',variables('subscriptionId'),';export RGP=',variables('resourceGroup'),';export LOC=',variables('location'),'export SUB=',variables('subnetName'),'export NSG=',variables('nsgName'),'export VNT=',variables('virtualNetworkName'),'export RTB=',variables('routeTableName'),';/bin/echo {{GetKubernetesAgentCustomScript}} | /usr/bin/base64 --decode | /bin/gunzip | /bin/bash')]"
+          "commandToExecute": "[concat('/usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh ',variables('tenantID'),' ',variables('subscriptionId'),' ',variables('resourceGroup'),' ',variables('location'),' ',variables('subnetName'),' ',variables('nsgName'),' ',variables('virtualNetworkName'),' ',variables('routeTableName'),' {{GetAgentSecrets}} >> /var/log/azure/cluster-provision.log 2>&1 &\" &')]"
         }
       }
     }

acstgen/parts/kubernetesmastercustomdata.yml

@@ -278,4 +278,11 @@ write_files:
             --cloud-provider=azure \
             --cloud-config=/etc/kubernetes/azure.json \
             --v=2
-    ExecStop=/usr/bin/docker stop -t 2 kubelet
+    ExecStop=/usr/bin/docker stop -t 2 kubelet
+
+- path: "/opt/azure/containers/provision.sh"
+  permissions: "0744"
+  encoding: gzip
+  owner: "root"
+  content: !!binary |
+    MASTER_PROVISION_B64_GZIP_STR

acstgen/parts/kubernetesmastercustomscript.sh

@@ -3,17 +3,30 @@
 ###########################################################
 # START SECRET DATA - ECHO DISABLED
 ###########################################################
+TID=$1
+SID=$2
+RGP=$3
+LOC=$4
+SUB=$5
+NSG=$6
+VNT=$7
+RTB=$8
+SVCPrincipalClientId=$9
+SVCPrincipalClientSecret=${10}
+CLIENTPRIVATEKEY=${11}
+SERVERPRIVATEKEY=${12}
+
 APISERVERKEY=/etc/kubernetes/certs/apiserver.key
 touch $APISERVERKEY
 chmod 0644 $APISERVERKEY
 chown root:root $APISERVERKEY
-echo {{{apiServerPrivateKey}}} | /usr/bin/base64 --decode > $APISERVERKEY
+echo $SERVERPRIVATEKEY | /usr/bin/base64 --decode > $APISERVERKEY
 
 CLIENTKEY=/etc/kubernetes/certs/client.key
 touch $CLIENTKEY
 chmod 0644 $CLIENTKEY
 chown root:root $CLIENTKEY
-echo {{{clientPrivateKey}}} | /usr/bin/base64 --decode > $CLIENTKEY 
+echo $CLIENTPRIVATEKEY | /usr/bin/base64 --decode > $CLIENTKEY 
 
 AZUREJSON=/etc/kubernetes/azure.json
 touch $AZUREJSON
@@ -23,8 +36,8 @@ AZURECONTENT=$(cat <<EOF
 {
     "tenantId": "$TID",
     "subscriptionId": "$SID",
-    "aadClientId": "{{{servicePrincipalClientId}}}",
+    "aadClientId": "$SVCPrincipalClientId",
-    "aadClientSecret": "{{{servicePrincipalClientSecret}}}",
+    "aadClientSecret": "$SVCPrincipalClientSecret",
     "resourceGroup": "$RGP",
     "location": "$LOC",
     "subnetName": "$SUB",

acstgen/parts/kubernetesmasterresources.t

@@ -311,13 +311,13 @@
       "type": "Microsoft.Compute/virtualMachines/extensions",
       "name": "[concat(variables('masterVMNamePrefix'), copyIndex(),'/cse', copyIndex())]",
       "properties": {
-        "publisher": "Microsoft.Azure.Extensions",
+        "publisher": "Microsoft.OSTCExtensions",
-        "type": "CustomScript",
+        "type": "CustomScriptForLinux",
-        "typeHandlerVersion": "2.0",
+        "typeHandlerVersion": "1.5",
         "autoUpgradeMinorVersion": true,
         "settings": {},
         "protectedSettings": {
-          "commandToExecute": "[concat('export TID=',variables('tenantID'),';export SID=',variables('subscriptionId'),';export RGP=',variables('resourceGroup'),';export LOC=',variables('location'),'export SUB=',variables('subnetName'),'export NSG=',variables('nsgName'),'export VNT=',variables('virtualNetworkName'),'export RTB=',variables('routeTableName'),';/bin/echo {{GetKubernetesMasterCustomScript}} | /usr/bin/base64 --decode | /bin/gunzip | /bin/bash')]"
+          "commandToExecute": "[concat('/usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh ',variables('tenantID'),' ',variables('subscriptionId'),' ',variables('resourceGroup'),' ',variables('location'),' ',variables('subnetName'),' ',variables('nsgName'),' ',variables('virtualNetworkName'),' ',variables('routeTableName'),' {{GetMasterSecrets}} >> /var/log/azure/cluster-provision.log 2>&1 &\" &')]"
         }
       }
     }

acstgen/parts/kubernetesmastervars.t

@@ -30,7 +30,7 @@
     "osImageSKU": "16.04.0-LTS", 
     "osImageVersion": "16.04.201606270",
     "resourceGroup": "[resourceGroup().name]", 
-    "routeTableName": "[concat(variables('masterVMNamePrefix'),'-routetable')]",
+    "routeTableName": "[concat(variables('masterVMNamePrefix'),'routetable')]",
     "routeTableID": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]",
     "sshNatPorts": [22,2201,2202,2203,2204],
     "sshKeyPath": "[concat('/home/',variables('username'),'/.ssh/authorized_keys')]", 

acstgen/tgen/template.go

@@ -280,24 +280,29 @@ func getTemplateFuncMap(acsCluster *vlabs.AcsCluster, partsDirectory string) map
 			return base64.StdEncoding.EncodeToString([]byte(s))
 		},
 		"GetKubernetesMasterCustomScript": func() string {
-			return getMasterBase64CustomScript(acsCluster, kubernetesMasterCustomScript, partsDirectory)
+			return getBase64CustomScript(acsCluster, kubernetesMasterCustomScript, partsDirectory)
 		},
 		"GetKubernetesMasterCustomData": func() string {
 			str, e := getSingleLineForTemplate(kubernetesMasterCustomDataYaml, partsDirectory)
 			if e != nil {
 				return ""
 			}
+			// add the master provisioning script
+			masterProvisionB64GzipStr := getBase64CustomScript(acsCluster, kubernetesMasterCustomScript, partsDirectory)
+			str = strings.Replace(str, "MASTER_PROVISION_B64_GZIP_STR", masterProvisionB64GzipStr, -1)
+
+			// return the custom data
 			return fmt.Sprintf("\"customData\": \"[base64(concat('%s'))]\",", str)
 		},
-		"GetKubernetesAgentCustomScript": func() string {
-			return getMasterBase64CustomScript(acsCluster, kubernetesAgentCustomScript, partsDirectory)
-		},
 		"GetKubernetesAgentCustomData": func(profile *vlabs.AgentPoolProfile) string {
 			str, e := getSingleLineForTemplate(kubernetesAgentCustomDataYaml, partsDirectory)
 			if e != nil {
 				return ""
 			}
-			str = strings.Replace(str, "{{.Name}}", profile.Name, -1)
+			// add the agent provisioning script
+			agentProvisionB64GzipStr := getBase64CustomScript(acsCluster, kubernetesAgentCustomScript, partsDirectory)
+			str = strings.Replace(str, "AGENT_PROVISION_B64_GZIP_STR", agentProvisionB64GzipStr, -1)
+
 			return fmt.Sprintf("\"customData\": \"[base64(concat('%s'))]\",", str)
 		},
 		"GetKubernetesKubeConfig": func() string {
@@ -307,6 +312,15 @@ func getTemplateFuncMap(acsCluster *vlabs.AcsCluster, partsDirectory string) map
 			}
 			return str
 		},
+		"GetMasterSecrets": func() string {
+			clientPrivateKey := base64.StdEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.ClientPrivateKey))
+			serverPrivateKey := base64.StdEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.APIServerPrivateKey))
+			return fmt.Sprintf("%s %s %s %s", acsCluster.ServicePrincipalProfile.ClientID, acsCluster.ServicePrincipalProfile.Secret, clientPrivateKey, serverPrivateKey)
+		},
+		"GetAgentSecrets": func() string {
+			clientPrivateKey := base64.StdEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.ClientPrivateKey))
+			return fmt.Sprintf("%s %s %s", acsCluster.ServicePrincipalProfile.ClientID, acsCluster.ServicePrincipalProfile.Secret, clientPrivateKey)
+		},
 		"AnyAgentHasDisks": func() bool {
 			for _, agentProfile := range acsCluster.AgentPoolProfiles {
 				if agentProfile.HasDisks() {
@@ -589,8 +603,8 @@ func getSingleLineForTemplate(yamlFilename string, partsDirectory string) (strin
 	return yamlStr, nil
 }
 
-// getMasterBase64CustomScript will return a base64 of the CSE
+// getBase64CustomScript will return a base64 of the CSE
-func getMasterBase64CustomScript(a *vlabs.AcsCluster, csFilename string, partsDirectory string) string {
+func getBase64CustomScript(a *vlabs.AcsCluster, csFilename string, partsDirectory string) string {
 	csFile := path.Join(partsDirectory, csFilename)
 	if _, err := os.Stat(csFile); os.IsNotExist(err) {
 		panic(err.Error())
@@ -601,10 +615,7 @@ func getMasterBase64CustomScript(a *vlabs.AcsCluster, csFilename string, partsDi
 	}
 	// translate the parameters
 	csStr := string(b)
-	csStr = strings.Replace(csStr, "{{{apiServerPrivateKey}}}", base64.StdEncoding.EncodeToString([]byte(a.CertificateProfile.APIServerPrivateKey)), -1)
+	csStr = strings.Replace(csStr, "\r\n", "\n", -1)
-	csStr = strings.Replace(csStr, "{{{clientPrivateKey}}}", base64.StdEncoding.EncodeToString([]byte(a.CertificateProfile.ClientPrivateKey)), -1)
-	csStr = strings.Replace(csStr, "{{{servicePrincipalClientId}}}", a.ServicePrincipalProfile.ClientID, -1)
-	csStr = strings.Replace(csStr, "{{{servicePrincipalClientSecret}}}", a.ServicePrincipalProfile.Secret, -1)
 
 	var gzipB bytes.Buffer
 	w := gzip.NewWriter(&gzipB)