diff --git a/acstgen/acstgen.go b/acstgen/acstgen.go index b7611feff..a3af4ffb8 100644 --- a/acstgen/acstgen.go +++ b/acstgen/acstgen.go @@ -69,6 +69,7 @@ func prettyPrintArmTemplate(template string) (string, error) { // there is a bug in ARM where it doesn't correctly translate back '\u003e' (>) {">", "GREATERTHAN"}, {"<", "LESSTHAN"}, + {"&", "AMPERSAND"}, } template = translateJSON(template, translateParams, false) diff --git a/acstgen/parts/kubernetesagentcustomdata.yml b/acstgen/parts/kubernetesagentcustomdata.yml index 0549779ff..31160c6e2 100644 --- a/acstgen/parts/kubernetesagentcustomdata.yml +++ b/acstgen/parts/kubernetesagentcustomdata.yml @@ -142,4 +142,11 @@ write_files: --network-plugin=kubenet \ --reconcile-cidr \ --v=2 - ExecStop=/usr/bin/docker stop -t 2 kubelet \ No newline at end of file + ExecStop=/usr/bin/docker stop -t 2 kubelet + +- path: "/opt/azure/containers/provision.sh" + permissions: "0744" + encoding: gzip + owner: "root" + content: !!binary | + AGENT_PROVISION_B64_GZIP_STR \ No newline at end of file diff --git a/acstgen/parts/kubernetesagentcustomscript.sh b/acstgen/parts/kubernetesagentcustomscript.sh index 793e6e23e..38646aa44 100644 --- a/acstgen/parts/kubernetesagentcustomscript.sh +++ b/acstgen/parts/kubernetesagentcustomscript.sh @@ -3,11 +3,24 @@ ########################################################### # START SECRET DATA - ECHO DISABLED ########################################################### + +TID=$1 +SID=$2 +RGP=$3 +LOC=$4 +SUB=$5 +NSG=$6 +VNT=$7 +RTB=$8 +SVCPrincipalClientId=$9 +SVCPrincipalClientSecret=${10} +CLIENTPRIVATEKEY=${11} + CLIENTKEY=/etc/kubernetes/certs/client.key touch $CLIENTKEY chmod 0644 $CLIENTKEY chown root:root $CLIENTKEY -echo {{{clientPrivateKey}}} | /usr/bin/base64 --decode > $CLIENTKEY +echo $CLIENTPRIVATEKEY | /usr/bin/base64 --decode > $CLIENTKEY AZUREJSON=/etc/kubernetes/azure.json touch $AZUREJSON @@ -17,8 +30,8 @@ AZURECONTENT=$(cat < $AZUREJSON set -x # wait for docker to be available -ensuredocker() +ensuredockerbinary() { dockerfound=1 for i in {1..600}; do @@ -54,7 +67,7 @@ ensuredocker() exit 1 fi } -ensuredocker +ensuredockerbinary # start all the services /bin/systemctl restart docker diff --git a/acstgen/parts/kubernetesagentresources.t b/acstgen/parts/kubernetesagentresources.t index add79d9be..fe35a8dc7 100644 --- a/acstgen/parts/kubernetesagentresources.t +++ b/acstgen/parts/kubernetesagentresources.t @@ -154,13 +154,13 @@ "type": "Microsoft.Compute/virtualMachines/extensions", "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(),'/cse', copyIndex())]", "properties": { - "publisher": "Microsoft.Azure.Extensions", - "type": "CustomScript", - "typeHandlerVersion": "2.0", + "publisher": "Microsoft.OSTCExtensions", + "type": "CustomScriptForLinux", + "typeHandlerVersion": "1.5", "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('export TID=',variables('tenantID'),';export SID=',variables('subscriptionId'),';export RGP=',variables('resourceGroup'),';export LOC=',variables('location'),'export SUB=',variables('subnetName'),'export NSG=',variables('nsgName'),'export VNT=',variables('virtualNetworkName'),'export RTB=',variables('routeTableName'),';/bin/echo {{GetKubernetesAgentCustomScript}} | /usr/bin/base64 --decode | /bin/gunzip | /bin/bash')]" + "commandToExecute": "[concat('/usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh ',variables('tenantID'),' ',variables('subscriptionId'),' ',variables('resourceGroup'),' ',variables('location'),' ',variables('subnetName'),' ',variables('nsgName'),' ',variables('virtualNetworkName'),' ',variables('routeTableName'),' {{GetAgentSecrets}} >> /var/log/azure/cluster-provision.log 2>&1 &\" &')]" } } } \ No newline at end of file diff --git a/acstgen/parts/kubernetesmastercustomdata.yml b/acstgen/parts/kubernetesmastercustomdata.yml index 83d063652..a62526d91 100644 --- a/acstgen/parts/kubernetesmastercustomdata.yml +++ b/acstgen/parts/kubernetesmastercustomdata.yml @@ -278,4 +278,11 @@ write_files: --cloud-provider=azure \ --cloud-config=/etc/kubernetes/azure.json \ --v=2 - ExecStop=/usr/bin/docker stop -t 2 kubelet \ No newline at end of file + ExecStop=/usr/bin/docker stop -t 2 kubelet + +- path: "/opt/azure/containers/provision.sh" + permissions: "0744" + encoding: gzip + owner: "root" + content: !!binary | + MASTER_PROVISION_B64_GZIP_STR \ No newline at end of file diff --git a/acstgen/parts/kubernetesmastercustomscript.sh b/acstgen/parts/kubernetesmastercustomscript.sh index 04c64d4e1..0e2f03a31 100644 --- a/acstgen/parts/kubernetesmastercustomscript.sh +++ b/acstgen/parts/kubernetesmastercustomscript.sh @@ -3,17 +3,30 @@ ########################################################### # START SECRET DATA - ECHO DISABLED ########################################################### +TID=$1 +SID=$2 +RGP=$3 +LOC=$4 +SUB=$5 +NSG=$6 +VNT=$7 +RTB=$8 +SVCPrincipalClientId=$9 +SVCPrincipalClientSecret=${10} +CLIENTPRIVATEKEY=${11} +SERVERPRIVATEKEY=${12} + APISERVERKEY=/etc/kubernetes/certs/apiserver.key touch $APISERVERKEY chmod 0644 $APISERVERKEY chown root:root $APISERVERKEY -echo {{{apiServerPrivateKey}}} | /usr/bin/base64 --decode > $APISERVERKEY +echo $SERVERPRIVATEKEY | /usr/bin/base64 --decode > $APISERVERKEY CLIENTKEY=/etc/kubernetes/certs/client.key touch $CLIENTKEY chmod 0644 $CLIENTKEY chown root:root $CLIENTKEY -echo {{{clientPrivateKey}}} | /usr/bin/base64 --decode > $CLIENTKEY +echo $CLIENTPRIVATEKEY | /usr/bin/base64 --decode > $CLIENTKEY AZUREJSON=/etc/kubernetes/azure.json touch $AZUREJSON @@ -23,8 +36,8 @@ AZURECONTENT=$(cat <> /var/log/azure/cluster-provision.log 2>&1 &\" &')]" } } } \ No newline at end of file diff --git a/acstgen/parts/kubernetesmastervars.t b/acstgen/parts/kubernetesmastervars.t index 330a1952f..3be924ba8 100644 --- a/acstgen/parts/kubernetesmastervars.t +++ b/acstgen/parts/kubernetesmastervars.t @@ -30,7 +30,7 @@ "osImageSKU": "16.04.0-LTS", "osImageVersion": "16.04.201606270", "resourceGroup": "[resourceGroup().name]", - "routeTableName": "[concat(variables('masterVMNamePrefix'),'-routetable')]", + "routeTableName": "[concat(variables('masterVMNamePrefix'),'routetable')]", "routeTableID": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]", "sshNatPorts": [22,2201,2202,2203,2204], "sshKeyPath": "[concat('/home/',variables('username'),'/.ssh/authorized_keys')]", diff --git a/acstgen/tgen/template.go b/acstgen/tgen/template.go index f91b63302..57c4a00cd 100644 --- a/acstgen/tgen/template.go +++ b/acstgen/tgen/template.go @@ -280,24 +280,29 @@ func getTemplateFuncMap(acsCluster *vlabs.AcsCluster, partsDirectory string) map return base64.StdEncoding.EncodeToString([]byte(s)) }, "GetKubernetesMasterCustomScript": func() string { - return getMasterBase64CustomScript(acsCluster, kubernetesMasterCustomScript, partsDirectory) + return getBase64CustomScript(acsCluster, kubernetesMasterCustomScript, partsDirectory) }, "GetKubernetesMasterCustomData": func() string { str, e := getSingleLineForTemplate(kubernetesMasterCustomDataYaml, partsDirectory) if e != nil { return "" } + // add the master provisioning script + masterProvisionB64GzipStr := getBase64CustomScript(acsCluster, kubernetesMasterCustomScript, partsDirectory) + str = strings.Replace(str, "MASTER_PROVISION_B64_GZIP_STR", masterProvisionB64GzipStr, -1) + + // return the custom data return fmt.Sprintf("\"customData\": \"[base64(concat('%s'))]\",", str) }, - "GetKubernetesAgentCustomScript": func() string { - return getMasterBase64CustomScript(acsCluster, kubernetesAgentCustomScript, partsDirectory) - }, "GetKubernetesAgentCustomData": func(profile *vlabs.AgentPoolProfile) string { str, e := getSingleLineForTemplate(kubernetesAgentCustomDataYaml, partsDirectory) if e != nil { return "" } - str = strings.Replace(str, "{{.Name}}", profile.Name, -1) + // add the agent provisioning script + agentProvisionB64GzipStr := getBase64CustomScript(acsCluster, kubernetesAgentCustomScript, partsDirectory) + str = strings.Replace(str, "AGENT_PROVISION_B64_GZIP_STR", agentProvisionB64GzipStr, -1) + return fmt.Sprintf("\"customData\": \"[base64(concat('%s'))]\",", str) }, "GetKubernetesKubeConfig": func() string { @@ -307,6 +312,15 @@ func getTemplateFuncMap(acsCluster *vlabs.AcsCluster, partsDirectory string) map } return str }, + "GetMasterSecrets": func() string { + clientPrivateKey := base64.StdEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.ClientPrivateKey)) + serverPrivateKey := base64.StdEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.APIServerPrivateKey)) + return fmt.Sprintf("%s %s %s %s", acsCluster.ServicePrincipalProfile.ClientID, acsCluster.ServicePrincipalProfile.Secret, clientPrivateKey, serverPrivateKey) + }, + "GetAgentSecrets": func() string { + clientPrivateKey := base64.StdEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.ClientPrivateKey)) + return fmt.Sprintf("%s %s %s", acsCluster.ServicePrincipalProfile.ClientID, acsCluster.ServicePrincipalProfile.Secret, clientPrivateKey) + }, "AnyAgentHasDisks": func() bool { for _, agentProfile := range acsCluster.AgentPoolProfiles { if agentProfile.HasDisks() { @@ -589,8 +603,8 @@ func getSingleLineForTemplate(yamlFilename string, partsDirectory string) (strin return yamlStr, nil } -// getMasterBase64CustomScript will return a base64 of the CSE -func getMasterBase64CustomScript(a *vlabs.AcsCluster, csFilename string, partsDirectory string) string { +// getBase64CustomScript will return a base64 of the CSE +func getBase64CustomScript(a *vlabs.AcsCluster, csFilename string, partsDirectory string) string { csFile := path.Join(partsDirectory, csFilename) if _, err := os.Stat(csFile); os.IsNotExist(err) { panic(err.Error()) @@ -601,10 +615,7 @@ func getMasterBase64CustomScript(a *vlabs.AcsCluster, csFilename string, partsDi } // translate the parameters csStr := string(b) - csStr = strings.Replace(csStr, "{{{apiServerPrivateKey}}}", base64.StdEncoding.EncodeToString([]byte(a.CertificateProfile.APIServerPrivateKey)), -1) - csStr = strings.Replace(csStr, "{{{clientPrivateKey}}}", base64.StdEncoding.EncodeToString([]byte(a.CertificateProfile.ClientPrivateKey)), -1) - csStr = strings.Replace(csStr, "{{{servicePrincipalClientId}}}", a.ServicePrincipalProfile.ClientID, -1) - csStr = strings.Replace(csStr, "{{{servicePrincipalClientSecret}}}", a.ServicePrincipalProfile.Secret, -1) + csStr = strings.Replace(csStr, "\r\n", "\n", -1) var gzipB bytes.Buffer w := gzip.NewWriter(&gzipB)