From d7d160ab1c62b30b67f2b1c68011ed27d76092d7 Mon Sep 17 00:00:00 2001 From: Anthony Howe Date: Sat, 8 Oct 2016 21:39:24 -0700 Subject: [PATCH] 1. fixed many CR comments 2. output to files instead of stdout 3. writing out kubeconfig files now 4. templates are now split into a parameters file and a template file --- .gitignore | 5 + acstgen/acstgen.go | 98 ++++++++----- acstgen/api/vlabs/doc.go | 2 +- acstgen/api/vlabs/types.go | 54 ++++--- acstgen/api/vlabs/validate.go | 22 ++- acstgen/clusterdefinitions/kubernetes.json | 8 +- acstgen/parts/agentparams.t | 5 - acstgen/parts/kubeconfig.json | 4 +- acstgen/parts/kubernetes.json | 36 ----- acstgen/parts/kubernetesbase.t | 2 + acstgen/parts/kubernetesmastercustomdata.yml | 4 +- acstgen/parts/kubernetesmastervars.t | 12 +- acstgen/parts/kubernetesparams.t | 30 ++-- acstgen/parts/masterparams.t | 7 - acstgen/parts/swarmmastervars.t | 2 +- acstgen/templategenerator/defaults.go | 111 --------------- acstgen/templategenerator/doc.go | 2 - .../Get-AzureConstants.ps1 | 2 +- .../{templategenerator => tgen}/azureconst.go | 2 +- acstgen/{templategenerator => tgen}/const.go | 9 +- acstgen/tgen/defaults.go | 114 +++++++++++++++ acstgen/tgen/doc.go | 2 + .../{templategenerator => tgen}/template.go | 133 ++++++++++++++++-- acstgen/util/doc.go | 2 +- acstgen/util/pki.go | 64 +++++++-- 25 files changed, 449 insertions(+), 283 deletions(-) create mode 100644 .gitignore delete mode 100644 acstgen/parts/kubernetes.json delete mode 100644 acstgen/templategenerator/defaults.go delete mode 100644 acstgen/templategenerator/doc.go rename acstgen/{templategenerator => tgen}/Get-AzureConstants.ps1 (99%) rename acstgen/{templategenerator => tgen}/azureconst.go (99%) rename acstgen/{templategenerator => tgen}/const.go (66%) create mode 100644 acstgen/tgen/defaults.go create mode 100644 acstgen/tgen/doc.go rename acstgen/{templategenerator => tgen}/template.go (72%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..1fbb03205 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.anthony.json +deployVM0.ps1 +noparams.json +output* +acstgen.exe \ No newline at end of file diff --git a/acstgen/acstgen.go b/acstgen/acstgen.go index 8d8c3ca8f..b7611feff 100644 --- a/acstgen/acstgen.go +++ b/acstgen/acstgen.go @@ -10,7 +10,7 @@ import ( "strings" "./api/vlabs" - tgen "./templategenerator" + "./tgen" ) // loadAcsCluster loads an ACS Cluster API Model from a JSON file @@ -62,10 +62,13 @@ func prettyPrintJSON(content string) (string, error) { func prettyPrintArmTemplate(template string) (string, error) { translateParams := [][]string{ - {"parameters", "dparameters"}, - {"variables", "evariables"}, - {"resources", "fresources"}, - {"outputs", "zoutputs"}, + {"\"parameters\"", "\"dparameters\""}, + {"\"variables\"", "\"evariables\""}, + {"\"resources\"", "\"fresources\""}, + {"\"outputs\"", "\"zoutputs\""}, + // there is a bug in ARM where it doesn't correctly translate back '\u003e' (>) + {">", "GREATERTHAN"}, + {"<", "LESSTHAN"}, } template = translateJSON(template, translateParams, false) @@ -78,12 +81,12 @@ func prettyPrintArmTemplate(template string) (string, error) { return template, nil } -func writeArtifacts(acsCluster *vlabs.AcsCluster, artifactsDir string) error { +func writeArtifacts(acsCluster *vlabs.AcsCluster, template string, parameters, artifactsDir string, templateDirectory string, certsGenerated bool) error { if len(artifactsDir) == 0 { - artifactsDir = fmt.Sprintf("k8s-%s", acsCluster.OrchestratorProfile.ClusterID) + artifactsDir = fmt.Sprintf("%s-%s", acsCluster.OrchestratorProfile.OrchestratorType, tgen.GenerateClusterID(acsCluster)) + artifactsDir = path.Join("output", artifactsDir) } - //b, err := json.Marshal(acsCluster) b, err := json.MarshalIndent(acsCluster, "", " ") if err != nil { return err @@ -92,23 +95,53 @@ func writeArtifacts(acsCluster *vlabs.AcsCluster, artifactsDir string) error { if e := saveFile(artifactsDir, "apimodel.json", b); e != nil { return e } - if e := saveFileString(artifactsDir, "ca.key", acsCluster.OrchestratorProfile.GetCAPrivateKey()); e != nil { + + if e := saveFileString(artifactsDir, "azuredeploy.json", template); e != nil { return e } - if e := saveFileString(artifactsDir, "ca.crt", acsCluster.OrchestratorProfile.CaCertificate); e != nil { + + if e := saveFileString(artifactsDir, "azuredeploy.parameters.json", parameters); e != nil { return e } - if e := saveFileString(artifactsDir, "apiserver.key", acsCluster.OrchestratorProfile.ApiServerPrivateKey); e != nil { - return e - } - if e := saveFileString(artifactsDir, "apiserver.crt", acsCluster.OrchestratorProfile.ApiServerCertificate); e != nil { - return e - } - if e := saveFileString(artifactsDir, "client.key", acsCluster.OrchestratorProfile.ClientPrivateKey); e != nil { - return e - } - if e := saveFileString(artifactsDir, "client.crt", acsCluster.OrchestratorProfile.ClientCertificate); e != nil { - return e + + if certsGenerated { + if acsCluster.OrchestratorProfile.OrchestratorType == vlabs.Kubernetes { + directory := path.Join(artifactsDir, "kubeconfig") + for _, location := range tgen.AzureLocations { + b, gkcerr := tgen.GenerateKubeConfig(acsCluster, templateDirectory, location) + if gkcerr != nil { + return gkcerr + } + if e := saveFileString(directory, fmt.Sprintf("kubeconfig.%s.json", location), b); e != nil { + return e + } + } + } + + if e := saveFileString(artifactsDir, "ca.key", acsCluster.CertificateProfile.GetCAPrivateKey()); e != nil { + return e + } + if e := saveFileString(artifactsDir, "ca.crt", acsCluster.CertificateProfile.CaCertificate); e != nil { + return e + } + if e := saveFileString(artifactsDir, "apiserver.key", acsCluster.CertificateProfile.APIServerPrivateKey); e != nil { + return e + } + if e := saveFileString(artifactsDir, "apiserver.crt", acsCluster.CertificateProfile.APIServerCertificate); e != nil { + return e + } + if e := saveFileString(artifactsDir, "client.key", acsCluster.CertificateProfile.ClientPrivateKey); e != nil { + return e + } + if e := saveFileString(artifactsDir, "client.crt", acsCluster.CertificateProfile.ClientCertificate); e != nil { + return e + } + if e := saveFileString(artifactsDir, "kubectlClient.key", acsCluster.CertificateProfile.KubeConfigPrivateKey); e != nil { + return e + } + if e := saveFileString(artifactsDir, "kubectlClient.crt", acsCluster.CertificateProfile.KubeConfigCertificate); e != nil { + return e + } } return nil @@ -139,7 +172,7 @@ func usage(errs ...error) { for _, err := range errs { fmt.Fprintf(os.Stderr, "error: %s\n\n", err.Error()) } - fmt.Fprintf(os.Stderr, "usage: %s ClusterDefinitionFile\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "usage: %s [OPTIONS] ClusterDefinitionFile\n", os.Args[0]) fmt.Fprintf(os.Stderr, " read the ClusterDefinitionFile and output an arm template") fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "options:\n") @@ -148,12 +181,12 @@ func usage(errs ...error) { var templateDirectory = flag.String("templateDirectory", "./parts", "directory containing base template files") var noPrettyPrint = flag.Bool("noPrettyPrint", false, "do not pretty print output") -var noArtifacts = flag.Bool("noArtifacts", false, "does not generate artifacts (api model, json, cert files)") var artifactsDir = flag.String("artifacts", "", "directory where artifacts will be written") func main() { var acsCluster *vlabs.AcsCluster var template string + var parameters string var err error flag.Parse() @@ -190,23 +223,24 @@ func main() { os.Exit(1) } - if template, err = tgen.GenerateTemplate(acsCluster, *templateDirectory); err != nil { + if template, parameters, err = tgen.GenerateTemplate(acsCluster, *templateDirectory); err != nil { fmt.Fprintf(os.Stderr, "error generating template %s: %s", jsonFile, err.Error()) os.Exit(1) } - if certsGenerated && !*noArtifacts { - if err = writeArtifacts(acsCluster, *artifactsDir); err != nil { - fmt.Fprintf(os.Stderr, "error writing artifacts %s", err.Error()) - os.Exit(1) - } - } - if !*noPrettyPrint { if template, err = prettyPrintArmTemplate(template); err != nil { fmt.Fprintf(os.Stderr, "error pretty printing template %s", err.Error()) os.Exit(1) } + if parameters, err = prettyPrintArmTemplate(parameters); err != nil { + fmt.Fprintf(os.Stderr, "error pretty printing template %s", err.Error()) + os.Exit(1) + } + } + + if err = writeArtifacts(acsCluster, template, parameters, *artifactsDir, *templateDirectory, certsGenerated); err != nil { + fmt.Fprintf(os.Stderr, "error writing artifacts %s", err.Error()) + os.Exit(1) } - fmt.Print(template) } diff --git a/acstgen/api/vlabs/doc.go b/acstgen/api/vlabs/doc.go index 5a4c4b850..28063c9a2 100644 --- a/acstgen/api/vlabs/doc.go +++ b/acstgen/api/vlabs/doc.go @@ -1,2 +1,2 @@ // Package vlabs stores an experimental api model for acs -package vlabs // import "./api/vlabs" +package vlabs diff --git a/acstgen/api/vlabs/types.go b/acstgen/api/vlabs/types.go index 7b5076a27..0892c1892 100644 --- a/acstgen/api/vlabs/types.go +++ b/acstgen/api/vlabs/types.go @@ -2,23 +2,41 @@ package vlabs // AcsCluster represents the ACS cluster definition type AcsCluster struct { - OrchestratorProfile OrchestratorProfile `json:"orchestratorProfile"` - MasterProfile MasterProfile `json:"masterProfile"` - AgentPoolProfiles []AgentPoolProfile `json:"agentPoolProfiles"` - LinuxProfile LinuxProfile `json:"linuxProfile"` + OrchestratorProfile OrchestratorProfile `json:"orchestratorProfile"` + MasterProfile MasterProfile `json:"masterProfile"` + AgentPoolProfiles []AgentPoolProfile `json:"agentPoolProfiles"` + LinuxProfile LinuxProfile `json:"linuxProfile"` + ServicePrincipalProfile ServicePrincipalProfile `json:"servicePrincipalProfile"` + CertificateProfile CertificateProfile `json:"certificateProfile"` } // OrchestratorProfile represents the type of orchestrator type OrchestratorProfile struct { - OrchestratorType string `json:"orchestratorType"` - ServicePrincipalClientID string `json:"servicePrincipalClientID,omitempty"` - ServicePrincipalClientSecret string `json:"servicePrincipalClientSecret,omitempty"` - ApiServerCertificate string `json:"apiServerCertificate,omitempty"` - ApiServerPrivateKey string `json:"apiServerPrivateKey,omitempty"` - CaCertificate string `json:"caCertificate,omitempty"` - ClientCertificate string `json:"clientCertificate,omitempty"` - ClientPrivateKey string `json:"clientPrivateKey,omitempty"` - ClusterID string `json:"clusterid,omitempty"` + OrchestratorType string `json:"orchestratorType"` +} + +// ServicePrincipalProfile contains the client and secret used by the cluster for Azure Resource CRUD +type ServicePrincipalProfile struct { + ClientID string `json:"servicePrincipalClientID,omitempty"` + Secret string `json:"servicePrincipalClientSecret,omitempty"` +} + +// CertificateProfile represents the definition of the master cluster +type CertificateProfile struct { + // CaCertificate is the certificate authority certificate. + CaCertificate string `json:"caCertificate,omitempty"` + // ApiServerCertificate is the rest api server certificate, and signed by the CA + APIServerCertificate string `json:"apiServerCertificate,omitempty"` + // ApiServerPrivateKey is the rest api server private key, and signed by the CA + APIServerPrivateKey string `json:"apiServerPrivateKey,omitempty"` + // ClientCertificate is the certificate used by the client kubelet services and signed by the CA + ClientCertificate string `json:"clientCertificate,omitempty"` + // ClientPrivateKey is the private key used by the client kubelet services and signed by the CA + ClientPrivateKey string `json:"clientPrivateKey,omitempty"` + // KubeConfigCertificate is the client certificate used for kubectl cli and signed by the CA + KubeConfigCertificate string `json:"kubeConfigCertificate,omitempty"` + // KubeConfigPrivateKey is the client private key used for kubectl cli and signed by the CA + KubeConfigPrivateKey string `json:"kubeConfigPrivateKey,omitempty"` // caPrivateKey is an internal field only set if generation required caPrivateKey string } @@ -64,13 +82,13 @@ type APIObject interface { } // GetCAPrivateKey returns the ca private key -func (o *OrchestratorProfile) GetCAPrivateKey() string { - return o.caPrivateKey +func (c *CertificateProfile) GetCAPrivateKey() string { + return c.caPrivateKey } -// SetCAPrivateKey returns the ca private key -func (o *OrchestratorProfile) SetCAPrivateKey(caPrivateKey string) { - o.caPrivateKey = caPrivateKey +// SetCAPrivateKey sets the ca private key +func (c *CertificateProfile) SetCAPrivateKey(caPrivateKey string) { + c.caPrivateKey = caPrivateKey } // IsCustomVNET returns true if the customer brought their own VNET diff --git a/acstgen/api/vlabs/validate.go b/acstgen/api/vlabs/validate.go index 9299ce38b..4c7e3724a 100644 --- a/acstgen/api/vlabs/validate.go +++ b/acstgen/api/vlabs/validate.go @@ -19,18 +19,6 @@ func (o *OrchestratorProfile) Validate() error { return fmt.Errorf("OrchestratorProfile has unknown orchestrator: %s", o.OrchestratorType) } - if o.OrchestratorType == Kubernetes && len(o.ServicePrincipalClientID) == 0 { - return fmt.Errorf("the service principal client ID must be specified with Orchestrator %s", o.OrchestratorType) - } - - if o.OrchestratorType == Kubernetes && len(o.ServicePrincipalClientSecret) == 0 { - return fmt.Errorf("the service principal client secrect must be specified with Orchestrator %s", o.OrchestratorType) - } - - if o.OrchestratorType != Kubernetes && (len(o.ServicePrincipalClientID) > 0 || len(o.ServicePrincipalClientSecret) > 0) { - return fmt.Errorf("Service principal and secret is not required for orchestrator %s", o.OrchestratorType) - } - return nil } @@ -118,6 +106,14 @@ func (a *AcsCluster) Validate() error { if e := validateUniqueProfileNames(a.AgentPoolProfiles); e != nil { return e } + if a.OrchestratorProfile.OrchestratorType == Kubernetes && len(a.ServicePrincipalProfile.ClientID) == 0 { + return fmt.Errorf("the service principal client ID must be specified with Orchestrator %s", a.OrchestratorProfile.OrchestratorType) + } + + if a.OrchestratorProfile.OrchestratorType == Kubernetes && len(a.ServicePrincipalProfile.Secret) == 0 { + return fmt.Errorf("the service principal client secrect must be specified with Orchestrator %s", a.OrchestratorProfile.OrchestratorType) + } + for _, agentPoolProfile := range a.AgentPoolProfiles { if e := agentPoolProfile.Validate(); e != nil { return e @@ -226,7 +222,7 @@ func validateVNET(a *AcsCluster) error { } masterFirstIP := net.ParseIP(a.MasterProfile.FirstConsecutiveStaticIP) - if masterFirstIP != nil { + if masterFirstIP == nil { return fmt.Errorf("MasterProfile.FirstConsecutiveStaticIP (with VNET Subnet specification) '%s' is an invalid IP address", a.MasterProfile.FirstConsecutiveStaticIP) } } diff --git a/acstgen/clusterdefinitions/kubernetes.json b/acstgen/clusterdefinitions/kubernetes.json index e8bf4f973..34fce22de 100644 --- a/acstgen/clusterdefinitions/kubernetes.json +++ b/acstgen/clusterdefinitions/kubernetes.json @@ -1,8 +1,6 @@ { "orchestratorProfile": { - "orchestratorType": "Kubernetes", - "servicePrincipalClientID": "REPLACE-WITH-ServicePrincipalClientID", - "servicePrincipalClientSecret": "REPLACE-WITH-myServicePrincipalClientSecret" + "orchestratorType": "Kubernetes" }, "masterProfile": { "count": 1, @@ -32,5 +30,9 @@ } ] } + }, + "servicePrincipalProfile": { + "servicePrincipalClientID": "REPLACE-WITH-ServicePrincipalClientID", + "servicePrincipalClientSecret": "REPLACE-WITH-myServicePrincipalClientSecret" } } \ No newline at end of file diff --git a/acstgen/parts/agentparams.t b/acstgen/parts/agentparams.t index ac3d3c872..30d139d8f 100644 --- a/acstgen/parts/agentparams.t +++ b/acstgen/parts/agentparams.t @@ -101,7 +101,6 @@ 99, 100 ], - "defaultValue": {{.Count}}, "metadata": { "description": "The number of Mesos agents for the cluster. This value can be from 1 to 100" }, @@ -109,7 +108,6 @@ }, "{{.Name}}VMSize": { {{GetAgentAllowedSizes}} - "defaultValue": "{{.VMSize}}", "metadata": { "description": "The size of the Virtual Machine." }, @@ -117,7 +115,6 @@ }, {{if .IsCustomVNET}} "{{.Name}}VnetSubnetID": { - "defaultValue": "{{.VnetSubnetID}}", "metadata": { "description": "Sets the vnet subnet of agent pool '{{.Name}}'." }, @@ -125,7 +122,6 @@ } {{else}} "{{.Name}}Subnet": { - "defaultValue": "{{.GetSubnet}}", "metadata": { "description": "Sets the subnet of agent pool '{{.Name}}'." }, @@ -134,7 +130,6 @@ {{end}} {{if IsPublic .Ports}} ,"{{.Name}}EndpointDNSNamePrefix": { - "defaultValue": "{{.DNSPrefix}}", "metadata": { "description": "Sets the Domain name label for the agent pool IP Address. The concatenation of the domain name label and the regional DNS zone make up the fully qualified domain name associated with the public IP address." }, diff --git a/acstgen/parts/kubeconfig.json b/acstgen/parts/kubeconfig.json index 8336ac22b..bfdeba2e2 100644 --- a/acstgen/parts/kubeconfig.json +++ b/acstgen/parts/kubeconfig.json @@ -24,8 +24,8 @@ { "name": "{{{resourceGroup}}}-admin", "user": { - "client-certificate-data": "<<>>", - "client-key-data": "<<>>" + "client-certificate-data": "<<>>", + "client-key-data": "<<>>" } } ] diff --git a/acstgen/parts/kubernetes.json b/acstgen/parts/kubernetes.json deleted file mode 100644 index e8bf4f973..000000000 --- a/acstgen/parts/kubernetes.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "orchestratorProfile": { - "orchestratorType": "Kubernetes", - "servicePrincipalClientID": "REPLACE-WITH-ServicePrincipalClientID", - "servicePrincipalClientSecret": "REPLACE-WITH-myServicePrincipalClientSecret" - }, - "masterProfile": { - "count": 1, - "dnsPrefix": "mgmtanhowe1004f", - "vmSize": "Standard_D2_v2" - }, - "agentPoolProfiles": [ - { - "name": "agentpool1", - "count": 3, - "vmSize": "Standard_D2_v2", - "isStateful": true - }, - { - "name": "agentpool2", - "count": 3, - "vmSize": "Standard_D2_v2", - "isStateful": true - } - ], - "linuxProfile": { - "adminUsername": "azureuser", - "ssh": { - "publicKeys": [ - { - "keyData": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8fhkh3jpHUQsrUIezFB5k4Rq9giJM8G1Cr0u2IRMiqG++nat5hbOr3gODpTA0h11q9bzb6nJtK7NtDzIHx+w3YNIVpcTGLiUEsfUbY53IHg7Nl/p3/gkST3g0R6BSL7Hg45SfyvpH7kwY30MoVHG/6P3go4SKlYoHXlgaaNr3fMwUTIeE9ofvyS3fcr6xxlsoB6luKuEs50h0NGsE4QEnbfSY4Yd/C1ucc3mEw+QFXBIsENHfHfZYrLNHm2L8MXYVmAH8k//5sFs4Migln9GiUgEQUT6uOjowsZyXBbXwfT11og+syPkAq4eqjiC76r0w6faVihdBYVoc/UcyupgH azureuser@linuxvm" - } - ] - } - } -} \ No newline at end of file diff --git a/acstgen/parts/kubernetesbase.t b/acstgen/parts/kubernetesbase.t index 412680249..f47319c38 100644 --- a/acstgen/parts/kubernetesbase.t +++ b/acstgen/parts/kubernetesbase.t @@ -9,7 +9,9 @@ "variables": { {{range $index, $agent := .AgentPoolProfiles}} {{template "kubernetesagentvars.t" .}} + {{if .HasDisks}} "{{.Name}}DataAccountName": "[concat(variables('storageAccountBaseName'), 'data{{$index}}')]", + {{end}} "{{.Name}}Index": {{$index}}, "{{.Name}}AccountName": "[concat(variables('storageAccountBaseName'), 'agnt{{$index}}')]", {{end}} diff --git a/acstgen/parts/kubernetesmastercustomdata.yml b/acstgen/parts/kubernetesmastercustomdata.yml index 188b91fb4..2f2b6fdca 100644 --- a/acstgen/parts/kubernetesmastercustomdata.yml +++ b/acstgen/parts/kubernetesmastercustomdata.yml @@ -8,7 +8,7 @@ packages: - traceroute runcmd: -- /bin/bash -c "/bin/echo DAEMON_ARGS=--advertise-client-urls {{{singleQuote}}}http://127.0.0.1:2379,http://{{{masterPrivateIp}}}:2379{{{singleQuote}}} --listen-client-urls {{{singleQuote}}}http://0.0.0.0:2379,http://0.0.0.0:4001{{{singleQuote}}} | tee -a /etc/default/etcd" +- /bin/bash -c "/bin/echo DAEMON_ARGS=--advertise-client-urls ""http://127.0.0.1:2379,http://{{{masterPrivateIp}}}:2379"" --listen-client-urls ""http://0.0.0.0:2379,http://0.0.0.0:4001"" | tee -a /etc/default/etcd" - /usr/bin/curl -sSL --retry 12 --retry-delay 10 https://get.docker.com/ > /tmp/install-docker - /bin/bash -c "/bin/bash /tmp/install-docker" - /usr/bin/curl -sSL --retry 12 --retry-delay 10 https://storage.googleapis.com/kubernetes-release/release/v1.3.8/bin/linux/amd64/kubectl > /usr/local/bin/kubectl @@ -128,7 +128,7 @@ write_files: command: - "/hyperkube" - "apiserver" - - "--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,PersistentVolumeLabel,ResourceQuota" + - "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota" - "--address=0.0.0.0" - "--allow-privileged" - "--insecure-port=8080" diff --git a/acstgen/parts/kubernetesmastervars.t b/acstgen/parts/kubernetesmastervars.t index 35b5226d6..4a80dc32e 100644 --- a/acstgen/parts/kubernetesmastervars.t +++ b/acstgen/parts/kubernetesmastervars.t @@ -6,11 +6,15 @@ "caCertificate": "[parameters('caCertificate')]", "clientCertificate": "[parameters('clientCertificate')]", "clientPrivateKey": "[parameters('clientPrivateKey')]", + "kubeConfigCertificate": "[parameters('kubeConfigCertificate')]", + "kubeConfigPrivateKey": "[parameters('kubeConfigPrivateKey')]", "kubernetesHyperkubeSpec": "[parameters('kubernetesHyperkubeSpec')]", "servicePrincipalClientId": "[parameters('servicePrincipalClientId')]", "servicePrincipalClientSecret": "[parameters('servicePrincipalClientSecret')]", "username": "[parameters('linuxAdminUsername')]", +{{if AnyAgentHasDisks}} "dataStorageAccountPrefixSeed": 97, +{{end}} "masterFqdnPrefix": "[parameters('masterEndpointDNSNamePrefix')]", "masterPrivateIp": "[parameters('firstConsecutiveStaticIP')]", "masterVMSize": "[parameters('masterVMSize')]", @@ -18,8 +22,8 @@ "masterCount": {{.MasterProfile.Count}}, "apiVersionDefault": "2016-03-30", "apiVersionStorage": "2015-06-15", - "kubeDnsServiceIp": "10.3.0.10", - "kubeServiceCidr": "10.3.0.0/16", + "kubeDnsServiceIp": "10.0.0.10", + "kubeServiceCidr": "10.0.0.0/16", "location": "[resourceGroup().location]", "masterAvailabilitySet": "master-availabilityset", "nsgName": "[concat(variables('masterVMNamePrefix'), 'nsg')]", @@ -35,10 +39,9 @@ "osImageVersion": "16.04.201606270", "resourceGroup": "[resourceGroup().name]", "routeTableName": "[concat(variables('masterVMNamePrefix'),'-routetable')]", - "singleQuote": "'", "sshNatPorts": [22,2201,2202,2203,2204], "sshKeyPath": "[concat('/home/',variables('username'),'/.ssh/authorized_keys')]", - "storageAccountBaseName": "[concat(uniqueString(concat(variables('masterFqdnPrefix'),resourceGroup().location)))]", + "storageAccountBaseName": "[uniqueString(concat(variables('masterFqdnPrefix'),resourceGroup().location))]", "storageAccountPrefixes": [ "0", "6", @@ -94,6 +97,7 @@ "masterLbIPConfigName": "[concat(variables('orchestratorName'), '-master-lbFrontEnd-', variables('nameSuffix'))]", "masterLbName": "[concat(variables('orchestratorName'), '-master-lb-', variables('nameSuffix'))]", "masterLbBackendPoolName": "[concat(variables('orchestratorName'), '-master-pool-', variables('nameSuffix'))]", + "masterFirstAddrComment": "these MasterFirstAddrComment are used to place multiple masters consecutively in the address space", "masterFirstAddrOctets": "[split(parameters('firstConsecutiveStaticIP'),'.')]", "masterFirstAddrOctet4": "[variables('masterFirstAddrOctets')[3]]", "masterFirstAddrPrefix": "[concat(variables('masterFirstAddrOctets')[0],'.',variables('masterFirstAddrOctets')[1],'.',variables('masterFirstAddrOctets')[2],'.')]", diff --git a/acstgen/parts/kubernetesparams.t b/acstgen/parts/kubernetesparams.t index e15c06fbd..bc11d0f50 100644 --- a/acstgen/parts/kubernetesparams.t +++ b/acstgen/parts/kubernetesparams.t @@ -1,54 +1,58 @@ "apiServerCertificate": { - "defaultValue": "{{Base64 .OrchestratorProfile.ApiServerCertificate}}", "metadata": { - "description": "The AD Tenant Id" + "description": "The base 64 server certificate used on the master" }, "type": "string" }, "apiServerPrivateKey": { - "defaultValue": "{{Base64 .OrchestratorProfile.ApiServerPrivateKey}}", "metadata": { - "description": "User name for the Linux Virtual Machines (SSH or Password)." + "description": "The base 64 server private key used on the master." }, "type": "securestring" }, "caCertificate": { - "defaultValue": "{{Base64 .OrchestratorProfile.CaCertificate}}", "metadata": { - "description": "The certificate authority certificate" + "description": "The base 64 certificate authority certificate" }, "type": "string" }, "clientCertificate": { - "defaultValue": "{{Base64 .OrchestratorProfile.ClientCertificate}}", "metadata": { - "description": "The client certificate used to communicate with the master" + "description": "The base 64 client certificate used to communicate with the master" }, "type": "string" }, "clientPrivateKey": { - "defaultValue": "{{Base64 .OrchestratorProfile.ClientPrivateKey}}", "metadata": { - "description": "The client private key used to communicate with the master" + "description": "The base 64 client private key used to communicate with the master" + }, + "type": "securestring" + }, + "kubeConfigCertificate": { + "metadata": { + "description": "The base 64 certificate used by cli to communicate with the master" + }, + "type": "string" + }, + "kubeConfigPrivateKey": { + "metadata": { + "description": "The base 64 private key used by cli to communicate with the master" }, "type": "securestring" }, "kubernetesHyperkubeSpec": { - "defaultValue": "gcr.io/google_containers/hyperkube-amd64:v1.4.0-beta.10", "metadata": { "description": "The container spec for hyperkube." }, "type": "string" }, "servicePrincipalClientId": { - "defaultValue": "{{.OrchestratorProfile.ServicePrincipalClientID}}", "metadata": { "description": "Client ID (used by cloudprovider)" }, "type": "string" }, "servicePrincipalClientSecret": { - "defaultValue": "{{.OrchestratorProfile.ServicePrincipalClientSecret}}", "metadata": { "description": "The Service Principal Client Secret." }, diff --git a/acstgen/parts/masterparams.t b/acstgen/parts/masterparams.t index 3611be66c..dcb92e13c 100644 --- a/acstgen/parts/masterparams.t +++ b/acstgen/parts/masterparams.t @@ -1,12 +1,10 @@ "linuxAdminUsername": { - "defaultValue": "{{.LinuxProfile.AdminUsername}}", "metadata": { "description": "User name for the Linux Virtual Machines (SSH or Password)." }, "type": "string" }, "masterEndpointDNSNamePrefix": { - "defaultValue": "{{.MasterProfile.DNSPrefix}}", "metadata": { "description": "Sets the Domain name label for the master IP Address. The concatenation of the domain name label and the regional DNS zone make up the fully qualified domain name associated with the public IP address." }, @@ -14,7 +12,6 @@ }, {{if .MasterProfile.IsCustomVNET}} "masterVnetSubnetID": { - "defaultValue": "{{.MasterProfile.VnetSubnetID}}", "metadata": { "description": "Sets the vnet subnet of the master." }, @@ -22,7 +19,6 @@ }, {{else}} "masterSubnet": { - "defaultValue": "{{.MasterProfile.GetSubnet}}", "metadata": { "description": "Sets the subnet of the master node(s)." }, @@ -30,7 +26,6 @@ }, {{end}} "firstConsecutiveStaticIP": { - "defaultValue": "{{.MasterProfile.FirstConsecutiveStaticIP}}", "metadata": { "description": "Sets the static IP of the first master" }, @@ -38,14 +33,12 @@ }, "masterVMSize": { {{GetMasterAllowedSizes}} - "defaultValue": "{{.MasterProfile.VMSize}}", "metadata": { "description": "The size of the Virtual Machine." }, "type": "string" }, "sshRSAPublicKey": { - "defaultValue": "{{GetLinuxProfileFirstSSHPublicKey}}", "metadata": { "description": "SSH public key used for auth to all Linux machines. Not Required. If not set, you must provide a password key." }, diff --git a/acstgen/parts/swarmmastervars.t b/acstgen/parts/swarmmastervars.t index c2be62f86..188cedc50 100644 --- a/acstgen/parts/swarmmastervars.t +++ b/acstgen/parts/swarmmastervars.t @@ -35,7 +35,7 @@ "postInstallScriptURI": "disabled", "sshKeyPath": "[concat('/home/', variables('adminUsername'), '/.ssh/authorized_keys')]", "sshRSAPublicKey": "[parameters('sshRSAPublicKey')]", - "storageAccountBaseName": "[concat(uniqueString(concat(variables('masterEndpointDNSNamePrefix'),resourceGroup().location)))]", + "storageAccountBaseName": "[uniqueString(concat(variables('masterEndpointDNSNamePrefix'),resourceGroup().location))]", "storageAccountPrefixes": [ "0", "6", diff --git a/acstgen/templategenerator/defaults.go b/acstgen/templategenerator/defaults.go deleted file mode 100644 index cf61f01cb..000000000 --- a/acstgen/templategenerator/defaults.go +++ /dev/null @@ -1,111 +0,0 @@ -package templategenerator - -import ( - "fmt" - "hash/fnv" - "math/rand" - "net" - - "./../api/vlabs" - "./../util" -) - -// SetAcsClusterDefaults for an AcsCluster, returns true if certs are generated -func SetAcsClusterDefaults(a *vlabs.AcsCluster) (bool, error) { - - if len(a.OrchestratorProfile.ClusterID) == 0 { - a.OrchestratorProfile.ClusterID = generateClusterID(a) - } - - setMasterNetworkDefaults(a) - - setAgentNetworkDefaults(a) - - certsGenerated, e := setDefaultCerts(&a.OrchestratorProfile, &a.MasterProfile) - if e != nil { - return false, e - } - return certsGenerated, nil -} - -// generateClusterID creates a unique 8 string cluster ID -func generateClusterID(acsCluster *vlabs.AcsCluster) string { - uniqueNameSuffixSize := 8 - // the name suffix uniquely identifies the cluster and is generated off a hash - // from the master dns name - h := fnv.New64a() - h.Write([]byte(acsCluster.MasterProfile.DNSPrefix)) - rand.Seed(int64(h.Sum64())) - return fmt.Sprintf("%08d", rand.Uint32())[:uniqueNameSuffixSize] -} - -// SetMasterNetworkDefaults for masters -func setMasterNetworkDefaults(a *vlabs.AcsCluster) { - if !a.MasterProfile.IsCustomVNET() { - if a.OrchestratorProfile.OrchestratorType == vlabs.Kubernetes { - a.MasterProfile.SetSubnet(DefaultKubernetesMasterSubnet) - a.MasterProfile.FirstConsecutiveStaticIP = DefaultFirstConsecutiveKubernetesStaticIP - } else { - a.MasterProfile.SetSubnet(DefaultMasterSubnet) - a.MasterProfile.FirstConsecutiveStaticIP = DefaultFirstConsecutiveStaticIP - } - } -} - -// SetAgentNetworkDefaults for agents -func setAgentNetworkDefaults(a *vlabs.AcsCluster) { - // configure the subnets if not in custom VNET - if !a.MasterProfile.IsCustomVNET() { - subnetCounter := 0 - for i := range a.AgentPoolProfiles { - profile := &a.AgentPoolProfiles[i] - - if a.OrchestratorProfile.OrchestratorType == vlabs.Kubernetes { - profile.SetSubnet(a.MasterProfile.GetSubnet()) - } else { - profile.SetSubnet(fmt.Sprintf(DefaultAgentSubnetTemplate, subnetCounter)) - } - - subnetCounter++ - } - } -} - -func setDefaultCerts(o *vlabs.OrchestratorProfile, m *vlabs.MasterProfile) (bool, error) { - certsGenerated := false - // auto generate certs if none of them have been set by customer - if len(o.ApiServerCertificate) > 0 || len(o.ApiServerPrivateKey) > 0 || - len(o.CaCertificate) > 0 || - len(o.ClientCertificate) > 0 || len(o.ClientPrivateKey) > 0 { - return certsGenerated, nil - } - masterWildCardFQDN := FormatAzureProdFQDN(m.DNSPrefix, "*") - masterExtraFQDNs := FormatAzureProdFQDNs(m.DNSPrefix) - firstMasterIP := net.ParseIP(m.FirstConsecutiveStaticIP) - - if firstMasterIP == nil { - return false, fmt.Errorf("MasterProfile.FirstConsecutiveStaticIP '%s' is an invalid IP address", m.FirstConsecutiveStaticIP) - } - - ips := []net.IP{firstMasterIP} - - for i := 1; i < m.Count; i++ { - ips = append(ips, net.IP{firstMasterIP[12], firstMasterIP[13], firstMasterIP[14], firstMasterIP[15] + byte(i)}) - } - - caPair, apiServerPair, clientPair, err := util.CreatePki(masterWildCardFQDN, masterExtraFQDNs, ips, DefaultKubernetesClusterDomain) - if err != nil { - return false, err - } - - certsGenerated = true - - o.ApiServerCertificate = apiServerPair.CertificatePem - o.ApiServerPrivateKey = apiServerPair.PrivateKeyPem - o.CaCertificate = caPair.CertificatePem - o.SetCAPrivateKey(caPair.PrivateKeyPem) - o.ClientCertificate = clientPair.CertificatePem - o.ClientPrivateKey = clientPair.PrivateKeyPem - - return certsGenerated, nil -} diff --git a/acstgen/templategenerator/doc.go b/acstgen/templategenerator/doc.go deleted file mode 100644 index 4a3e9088d..000000000 --- a/acstgen/templategenerator/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package templategenerator takes an ACS cluster model and generates the corresponding template -package templategenerator // import "./templategenerator" diff --git a/acstgen/templategenerator/Get-AzureConstants.ps1 b/acstgen/tgen/Get-AzureConstants.ps1 similarity index 99% rename from acstgen/templategenerator/Get-AzureConstants.ps1 rename to acstgen/tgen/Get-AzureConstants.ps1 index 04b3c20e0..cb813077a 100644 --- a/acstgen/templategenerator/Get-AzureConstants.ps1 +++ b/acstgen/tgen/Get-AzureConstants.ps1 @@ -99,7 +99,7 @@ Get-FileContents() { $Locations ) - $text = "package templategenerator" + $text = "package tgen" $text += @" diff --git a/acstgen/templategenerator/azureconst.go b/acstgen/tgen/azureconst.go similarity index 99% rename from acstgen/templategenerator/azureconst.go rename to acstgen/tgen/azureconst.go index 8e9ba9871..fe07e337c 100644 --- a/acstgen/templategenerator/azureconst.go +++ b/acstgen/tgen/azureconst.go @@ -1,4 +1,4 @@ -package templategenerator +package tgen import "fmt" diff --git a/acstgen/templategenerator/const.go b/acstgen/tgen/const.go similarity index 66% rename from acstgen/templategenerator/const.go rename to acstgen/tgen/const.go index 55d541e5f..eaea0479c 100644 --- a/acstgen/templategenerator/const.go +++ b/acstgen/tgen/const.go @@ -1,11 +1,11 @@ -package templategenerator +package tgen const ( // BaseLBPriority specifies the base lb priority. BaseLBPriority = 200 - // DefaultMasterSubnet specifies the default master subnet + // DefaultMasterSubnet specifies the default master subnet for DCOS or Swarm DefaultMasterSubnet = "172.16.0.0/24" - // DefaultFirstConsecutiveStaticIP specifies the static IP address on master 0 + // DefaultFirstConsecutiveStaticIP specifies the static IP address on master 0 for DCOS or Swarm DefaultFirstConsecutiveStaticIP = "172.16.0.5" // DefaultKubernetesMasterSubnet specifies the default kubernetes master subnet DefaultKubernetesMasterSubnet = "10.240.0.0/16" @@ -15,4 +15,7 @@ const ( DefaultAgentSubnetTemplate = "10.%d.0.0/24" // DefaultKubernetesClusterDomain is the dns suffix used in the cluster (used as a SAN in the PKI generation) DefaultKubernetesClusterDomain = "cluster.local" + // KubernetesHyperkubeSpec is the hyperkube version used for Kubernetes setup + // The latest stable version can be found here: https://storage.googleapis.com/kubernetes-release/release/stable.txt + KubernetesHyperkubeSpec = "gcr.io/google_containers/hyperkube-amd64:v1.4.0" ) diff --git a/acstgen/tgen/defaults.go b/acstgen/tgen/defaults.go new file mode 100644 index 000000000..36f9e879e --- /dev/null +++ b/acstgen/tgen/defaults.go @@ -0,0 +1,114 @@ +package tgen + +import ( + "fmt" + "net" + + "./../api/vlabs" + "./../util" +) + +// SetAcsClusterDefaults for an AcsCluster, returns true if certs are generated +func SetAcsClusterDefaults(a *vlabs.AcsCluster) (bool, error) { + + setMasterNetworkDefaults(a) + + setAgentNetworkDefaults(a) + + certsGenerated, e := setDefaultCerts(a) + if e != nil { + return false, e + } + return certsGenerated, nil +} + +// SetMasterNetworkDefaults for masters +func setMasterNetworkDefaults(a *vlabs.AcsCluster) { + if !a.MasterProfile.IsCustomVNET() { + if a.OrchestratorProfile.OrchestratorType == vlabs.Kubernetes { + a.MasterProfile.SetSubnet(DefaultKubernetesMasterSubnet) + a.MasterProfile.FirstConsecutiveStaticIP = DefaultFirstConsecutiveKubernetesStaticIP + } else { + a.MasterProfile.SetSubnet(DefaultMasterSubnet) + a.MasterProfile.FirstConsecutiveStaticIP = DefaultFirstConsecutiveStaticIP + } + } +} + +// SetAgentNetworkDefaults for agents +func setAgentNetworkDefaults(a *vlabs.AcsCluster) { + // configure the subnets if not in custom VNET + if !a.MasterProfile.IsCustomVNET() { + subnetCounter := 0 + for i := range a.AgentPoolProfiles { + profile := &a.AgentPoolProfiles[i] + + if a.OrchestratorProfile.OrchestratorType == vlabs.Kubernetes { + profile.SetSubnet(a.MasterProfile.GetSubnet()) + } else { + profile.SetSubnet(fmt.Sprintf(DefaultAgentSubnetTemplate, subnetCounter)) + } + + subnetCounter++ + } + } +} + +func setDefaultCerts(a *vlabs.AcsCluster) (bool, error) { + if !certGenerationRequired(a) { + return false, nil + } + + masterWildCardFQDN := FormatAzureProdFQDN(a.MasterProfile.DNSPrefix, "*") + masterExtraFQDNs := FormatAzureProdFQDNs(a.MasterProfile.DNSPrefix) + firstMasterIP := net.ParseIP(a.MasterProfile.FirstConsecutiveStaticIP) + + if firstMasterIP == nil { + return false, fmt.Errorf("MasterProfile.FirstConsecutiveStaticIP '%s' is an invalid IP address", a.MasterProfile.FirstConsecutiveStaticIP) + } + + ips := []net.IP{firstMasterIP} + + for i := 1; i < a.MasterProfile.Count; i++ { + ips = append(ips, net.IP{firstMasterIP[12], firstMasterIP[13], firstMasterIP[14], firstMasterIP[15] + byte(i)}) + } + + caPair, apiServerPair, clientPair, kubeConfigPair, err := util.CreatePki(masterWildCardFQDN, masterExtraFQDNs, ips, DefaultKubernetesClusterDomain) + if err != nil { + return false, err + } + + a.CertificateProfile.APIServerCertificate = apiServerPair.CertificatePem + a.CertificateProfile.APIServerPrivateKey = apiServerPair.PrivateKeyPem + a.CertificateProfile.CaCertificate = caPair.CertificatePem + a.CertificateProfile.SetCAPrivateKey(caPair.PrivateKeyPem) + a.CertificateProfile.ClientCertificate = clientPair.CertificatePem + a.CertificateProfile.ClientPrivateKey = clientPair.PrivateKeyPem + a.CertificateProfile.KubeConfigCertificate = kubeConfigPair.CertificatePem + a.CertificateProfile.KubeConfigPrivateKey = kubeConfigPair.PrivateKeyPem + + return true, nil +} + +func certGenerationRequired(a *vlabs.AcsCluster) bool { + if len(a.CertificateProfile.APIServerCertificate) > 0 || len(a.CertificateProfile.APIServerPrivateKey) > 0 || + len(a.CertificateProfile.CaCertificate) > 0 || + len(a.CertificateProfile.ClientCertificate) > 0 || len(a.CertificateProfile.ClientPrivateKey) > 0 { + return false + } + + switch a.OrchestratorProfile.OrchestratorType { + case vlabs.DCOS: + return false + case vlabs.DCOS184: + return false + case vlabs.DCOS173: + return false + case vlabs.Swarm: + return false + case vlabs.Kubernetes: + return true + default: + return false + } +} diff --git a/acstgen/tgen/doc.go b/acstgen/tgen/doc.go new file mode 100644 index 000000000..4dc42108a --- /dev/null +++ b/acstgen/tgen/doc.go @@ -0,0 +1,2 @@ +// Package tgen takes an ACS cluster model and generates the corresponding template +package tgen diff --git a/acstgen/templategenerator/template.go b/acstgen/tgen/template.go similarity index 72% rename from acstgen/templategenerator/template.go rename to acstgen/tgen/template.go index ae8757d46..04a51b539 100644 --- a/acstgen/templategenerator/template.go +++ b/acstgen/tgen/template.go @@ -1,11 +1,14 @@ -package templategenerator +package tgen import ( "bytes" "encoding/base64" + "encoding/json" "errors" "fmt" + "hash/fnv" "io/ioutil" + "math/rand" "os" "path" "regexp" @@ -69,7 +72,7 @@ func VerifyFiles(partsDirectory string) error { } // GenerateTemplate generates the template from the API Model -func GenerateTemplate(acsCluster *vlabs.AcsCluster, partsDirectory string) (string, error) { +func GenerateTemplate(acsCluster *vlabs.AcsCluster, partsDirectory string) (string, string, error) { var err error var templ *template.Template @@ -89,25 +92,117 @@ func GenerateTemplate(acsCluster *vlabs.AcsCluster, partsDirectory string) (stri files = append(commonTemplateFiles, kubernetesTemplateFiles...) baseFile = kubernetesBaseFile } else { - return "", fmt.Errorf("orchestrator '%s' is unsupported", acsCluster.OrchestratorProfile.OrchestratorType) + return "", "", fmt.Errorf("orchestrator '%s' is unsupported", acsCluster.OrchestratorProfile.OrchestratorType) } for _, file := range files { templateFile := path.Join(partsDirectory, file) bytes, e := ioutil.ReadFile(templateFile) if e != nil { - return "", fmt.Errorf("Error reading file %s: %s", templateFile, e.Error()) + return "", "", fmt.Errorf("Error reading file %s: %s", templateFile, e.Error()) } if _, err = templ.New(file).Parse(string(bytes)); err != nil { - return "", err + return "", "", err } } var b bytes.Buffer if err = templ.ExecuteTemplate(&b, baseFile, acsCluster); err != nil { - return "", err + return "", "", err } - return b.String(), nil + var parametersMap *map[string]interface{} + if parametersMap, err = getParameters(acsCluster); err != nil { + return "", "", err + } + var parameterBytes []byte + if parameterBytes, err = json.Marshal(parametersMap); err != nil { + return "", "", err + } + + return b.String(), string(parameterBytes), nil +} + +// GenerateClusterID creates a unique 8 string cluster ID +func GenerateClusterID(acsCluster *vlabs.AcsCluster) string { + uniqueNameSuffixSize := 8 + // the name suffix uniquely identifies the cluster and is generated off a hash + // from the master dns name + h := fnv.New64a() + h.Write([]byte(acsCluster.MasterProfile.DNSPrefix)) + rand.Seed(int64(h.Sum64())) + return fmt.Sprintf("%08d", rand.Uint32())[:uniqueNameSuffixSize] +} + +// GenerateKubeConfig returns a JSON string representing the KubeConfig +func GenerateKubeConfig(acsCluster *vlabs.AcsCluster, templateDirectory string, location string) (string, error) { + kubeTemplateFile := path.Join(templateDirectory, kubeConfigJSON) + if _, err := os.Stat(kubeTemplateFile); os.IsNotExist(err) { + return "", fmt.Errorf("file %s does not exist, did you specify the correct template directory?", kubeTemplateFile) + } + b, err := ioutil.ReadFile(kubeTemplateFile) + if err != nil { + return "", fmt.Errorf("error reading kube config template file %s: %s", kubeTemplateFile, err.Error()) + } + kubeconfig := string(b) + // variable replacement + kubeconfig = strings.Replace(kubeconfig, "<<>>", base64.URLEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.CaCertificate)), -1) + kubeconfig = strings.Replace(kubeconfig, "<<>>", FormatAzureProdFQDN(acsCluster.MasterProfile.DNSPrefix, location), -1) + kubeconfig = strings.Replace(kubeconfig, "{{{resourceGroup}}}", acsCluster.MasterProfile.DNSPrefix, -1) + kubeconfig = strings.Replace(kubeconfig, "<<>>", base64.URLEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.KubeConfigCertificate)), -1) + kubeconfig = strings.Replace(kubeconfig, "<<>>", base64.URLEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.KubeConfigPrivateKey)), -1) + + return kubeconfig, nil +} + +func getParameters(acsCluster *vlabs.AcsCluster) (*map[string]interface{}, error) { + parametersMap := &map[string]interface{}{} + + // Master Parameters + addValue(parametersMap, "linuxAdminUsername", acsCluster.LinuxProfile.AdminUsername) + addValue(parametersMap, "masterEndpointDNSNamePrefix", acsCluster.MasterProfile.DNSPrefix) + if acsCluster.MasterProfile.IsCustomVNET() { + addValue(parametersMap, "masterVnetSubnetID", acsCluster.MasterProfile.VnetSubnetID) + } else { + addValue(parametersMap, "masterSubnet", acsCluster.MasterProfile.GetSubnet()) + } + addValue(parametersMap, "firstConsecutiveStaticIP", acsCluster.MasterProfile.FirstConsecutiveStaticIP) + addValue(parametersMap, "masterVMSize", acsCluster.MasterProfile.VMSize) + addValue(parametersMap, "sshRSAPublicKey", acsCluster.LinuxProfile.SSH.PublicKeys[0].KeyData) + + // Kubernetes Parameters + if acsCluster.OrchestratorProfile.OrchestratorType == vlabs.Kubernetes { + addValue(parametersMap, "apiServerCertificate", base64.URLEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.APIServerCertificate))) + addValue(parametersMap, "apiServerPrivateKey", base64.URLEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.APIServerPrivateKey))) + addValue(parametersMap, "caCertificate", base64.URLEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.CaCertificate))) + addValue(parametersMap, "clientCertificate", base64.URLEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.ClientCertificate))) + addValue(parametersMap, "clientPrivateKey", base64.URLEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.ClientPrivateKey))) + addValue(parametersMap, "kubeConfigCertificate", base64.URLEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.KubeConfigCertificate))) + addValue(parametersMap, "kubeConfigPrivateKey", base64.URLEncoding.EncodeToString([]byte(acsCluster.CertificateProfile.KubeConfigPrivateKey))) + addValue(parametersMap, "kubernetesHyperkubeSpec", KubernetesHyperkubeSpec) + addValue(parametersMap, "servicePrincipalClientId", acsCluster.ServicePrincipalProfile.ClientID) + addValue(parametersMap, "servicePrincipalClientSecret", acsCluster.ServicePrincipalProfile.Secret) + } + + // Agent parameters + for _, agentProfile := range acsCluster.AgentPoolProfiles { + addValue(parametersMap, fmt.Sprintf("%sCount", agentProfile.Name), agentProfile.Count) + addValue(parametersMap, fmt.Sprintf("%sVMSize", agentProfile.Name), agentProfile.VMSize) + if agentProfile.IsCustomVNET() { + addValue(parametersMap, fmt.Sprintf("%sVnetSubnetID", agentProfile.Name), agentProfile.VnetSubnetID) + } else { + addValue(parametersMap, fmt.Sprintf("%sSubnet", agentProfile.Name), agentProfile.GetSubnet()) + } + if len(agentProfile.Ports) > 0 { + addValue(parametersMap, fmt.Sprintf("%sEndpointDNSNamePrefix", agentProfile.Name), agentProfile.DNSPrefix) + } + } + + return parametersMap, nil +} + +func addValue(m *map[string]interface{}, k string, v interface{}) { + (*m)[k] = *(&map[string]interface{}{}) + (*m)[k].(map[string]interface{})["Value"] = v } // getTemplateFuncMap returns all functions used in template generation @@ -147,11 +242,8 @@ func getTemplateFuncMap(acsCluster *vlabs.AcsCluster, partsDirectory string) map "GetDCOSGUID": func() string { return getPackageGUID(acsCluster.OrchestratorProfile.OrchestratorType, acsCluster.MasterProfile.Count) }, - "GetLinuxProfileFirstSSHPublicKey": func() string { - return acsCluster.LinuxProfile.SSH.PublicKeys[0].KeyData - }, "GetUniqueNameSuffix": func() string { - return acsCluster.OrchestratorProfile.ClusterID + return GenerateClusterID(acsCluster) }, "GetVNETAddressPrefixes": func() string { return getVNETAddressPrefixes(acsCluster) @@ -195,6 +287,14 @@ func getTemplateFuncMap(acsCluster *vlabs.AcsCluster, partsDirectory string) map } return str }, + "AnyAgentHasDisks": func() bool { + for _, agentProfile := range acsCluster.AgentPoolProfiles { + if agentProfile.HasDisks() { + return true + } + } + return false + }, // inspired by http://stackoverflow.com/questions/18276173/calling-a-template-with-several-pipeline-parameters/18276968#18276968 "dict": func(values ...interface{}) (map[string]interface{}, error) { if len(values)%2 != 0 { @@ -447,6 +547,7 @@ func getSingleLineForTemplate(yamlFilename string, partsDirectory string) (strin if err != nil { return "", fmt.Errorf("error reading yaml file %s: %s", yamlFile, err.Error()) } + // template.JSEscapeString leaves undesirable chars that don't work with pretty print yamlStr := string(b) yamlStr = strings.Replace(yamlStr, "\\", "\\\\", -1) yamlStr = strings.Replace(yamlStr, "\r\n", "\\n", -1) @@ -454,10 +555,16 @@ func getSingleLineForTemplate(yamlFilename string, partsDirectory string) (strin yamlStr = strings.Replace(yamlStr, "\"", "\\\"", -1) // variable replacement - rVariable := regexp.MustCompile("{{{([^}]*)}}}") + rVariable, e1 := regexp.Compile("{{{([^}]*)}}}") + if e1 != nil { + return "", e1 + } yamlStr = rVariable.ReplaceAllString(yamlStr, "',variables('$1'),'") // verbatim replacement - rVerbatim := regexp.MustCompile("<<<([^>]*)>>>") + rVerbatim, e2 := regexp.Compile("<<<([^>]*)>>>") + if e2 != nil { + return "", e2 + } yamlStr = rVerbatim.ReplaceAllString(yamlStr, "',$1,'") return yamlStr, nil } diff --git a/acstgen/util/doc.go b/acstgen/util/doc.go index c53b7005c..97ed24d51 100644 --- a/acstgen/util/doc.go +++ b/acstgen/util/doc.go @@ -1,2 +1,2 @@ // Package util provides various utility functions for the generator -package util // import "./util" \ No newline at end of file +package util diff --git a/acstgen/util/pki.go b/acstgen/util/pki.go index 2bcbbfdea..b0e57422b 100644 --- a/acstgen/util/pki.go +++ b/acstgen/util/pki.go @@ -1,4 +1,3 @@ -// util package comes from https://github.com/colemickens/azkube/blob/master/util/pki.go package util import ( @@ -11,6 +10,7 @@ import ( "fmt" "math/big" "net" + "os" "time" ) @@ -24,7 +24,7 @@ type PkiKeyCertPair struct { PrivateKeyPem string } -func CreatePki(masterFQDN string, extraFQDNs []string, extraIPs []net.IP, clusterDomain string) (*PkiKeyCertPair, *PkiKeyCertPair, *PkiKeyCertPair, error) { +func CreatePki(masterFQDN string, extraFQDNs []string, extraIPs []net.IP, clusterDomain string) (*PkiKeyCertPair, *PkiKeyCertPair, *PkiKeyCertPair, *PkiKeyCertPair, error) { extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes")) extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.default")) extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.default.svc")) @@ -32,23 +32,59 @@ func CreatePki(masterFQDN string, extraFQDNs []string, extraIPs []net.IP, cluste extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.kube-system")) extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.kube-system.svc")) extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.kube-system.svc.%s", clusterDomain)) - + start := time.Now() caCertificate, caPrivateKey, err := createCertificate("ca", nil, nil, false, "", nil, nil) if err != nil { - return nil, nil, nil, err - } - apiServerCertificate, apiServerPrivateKey, err := createCertificate("apiserver", caCertificate, caPrivateKey, true, masterFQDN, extraFQDNs, extraIPs) - if err != nil { - return nil, nil, nil, err - } - clientCertificate, clientPrivateKey, err := createCertificate("client", caCertificate, caPrivateKey, false, "", nil, nil) - if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } + var ( + apiServerCertificate *x509.Certificate + apiServerPrivateKey *rsa.PrivateKey + clientCertificate *x509.Certificate + clientPrivateKey *rsa.PrivateKey + kubeConfigCertificate *x509.Certificate + kubeConfigPrivateKey *rsa.PrivateKey + ) + errors := make(chan error) + + go func() { + var err error + apiServerCertificate, apiServerPrivateKey, err = createCertificate("apiserver", caCertificate, caPrivateKey, true, masterFQDN, extraFQDNs, extraIPs) + errors <- err + }() + + go func() { + var err error + clientCertificate, clientPrivateKey, err = createCertificate("client", caCertificate, caPrivateKey, false, "", nil, nil) + errors <- err + }() + + go func() { + var err error + kubeConfigCertificate, kubeConfigPrivateKey, err = createCertificate("client", caCertificate, caPrivateKey, false, "", nil, nil) + errors <- err + }() + + e1 := <-errors + e2 := <-errors + e3 := <-errors + if e1 != nil { + return nil, nil, nil, nil, e1 + } + if e2 != nil { + return nil, nil, nil, nil, e2 + } + if e3 != nil { + return nil, nil, nil, nil, e2 + } + + fmt.Fprintf(os.Stderr, "cert creation took %s\n", time.Since(start)) return &PkiKeyCertPair{CertificatePem: string(certificateToPem(caCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(caPrivateKey))}, &PkiKeyCertPair{CertificatePem: string(certificateToPem(apiServerCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(apiServerPrivateKey))}, - &PkiKeyCertPair{CertificatePem: string(certificateToPem(clientCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(clientPrivateKey))}, nil + &PkiKeyCertPair{CertificatePem: string(certificateToPem(clientCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(clientPrivateKey))}, + &PkiKeyCertPair{CertificatePem: string(certificateToPem(kubeConfigCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(kubeConfigPrivateKey))}, + nil } func createCertificate(commonName string, caCertificate *x509.Certificate, caPrivateKey *rsa.PrivateKey, isServer bool, FQDN string, extraFQDNs []string, extraIPs []net.IP) (*x509.Certificate, *rsa.PrivateKey, error) { @@ -72,7 +108,7 @@ func createCertificate(commonName string, caCertificate *x509.Certificate, caPri template.IsCA = isCA } else if isServer { extraFQDNs = append(extraFQDNs, FQDN) - extraIPs = append(extraIPs, net.ParseIP("10.3.0.1")) + extraIPs = append(extraIPs, net.ParseIP("10.0.0.1")) template.DNSNames = extraFQDNs template.IPAddresses = extraIPs