Refactor registry credentials to DcosConfig (#1137)

examples/dcos-private-registry/README.md

@@ -0,0 +1,67 @@
+# Private Registry Support
+
+ACS can deploy credentials to private registries to agent nodes DC/OS clusters.
+
+The credentials are specified in the orchestrator profile in the apimodel:
+```
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "DCOS",
+      "dcosConfig" : { 
+        "Registry" : "",
+        "RegistryUser" : "",
+        "RegistryPassword" : ""
+      }
+    },
+```
+
+The agent provisioning process will then create a tar archive containing a docker config as documented at: [Using a Private Docker Registry](https://docs.mesosphere.com/1.9/deploying-services/private-docker-registry/)
+
+## Example
+Let's provision a DC/OS cluster with credentials to an [Azure Container Registry](https://azure.microsoft.com/en-us/services/container-registry/) deployed to every agent node.
+
+- First, [provision an Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-managed-get-started-portal).  
+
+- Enable Admin Access and note the registry credentials
+<img src="../../docs/images/acrblade.png" alt="ACR Blade with Admin Access enabled" style="width: 50%; height: 50%;"/>
+
+- Clone [acs-engine](http://github.com/azure/acs-engine) and [start the container with the dev environment](https://github.com/Azure/acs-engine/blob/master/docs/acsengine.md).
+
+- Edit the API model to include the credentials
+```
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "DCOS",
+      "registry" : "xtophregistry.azurecr.io",
+      "registryUser" : "xtophregistry",
+      "registryPassword" : "aN//=+l==Z+/A=3hXhA+mSX=rXwB/UgW"
+    },
+```
+
+- Run acs-engine to create ARM templates
+```
+./acs-engine generate examples/dcos-private-registry/dcos.json
+```
+
+- Deploy the cluster
+```
+az group create -l eastus -n cluster-rg
+az group deployment create -g cluster-rg --template-file _output/dcoscluster/azuredeploy.json --parameters @_output/dcoscluster/azuredeploy.parameters.json
+```
+
+- Create a Service to deploy a container from the ACR
+<img src="../../docs/images/dcos-create-service-from-reg.png" alt="Service Creation from Registry" style="width: 50%; height: 50%;"/>
+
+- Add the credential path on the agent using the JSON editor
+<img src="../../docs/images/dcos-create-service-json.png" alt="JSON editor with credential path" style="width: 50%; height: 50%;"/>
+
+- See the Service running
+<img src="../../docs/images/dcos-running-service-from-reg.png" alt="Running Service" style="width: 50%; height: 50%;"/>
+
+- Check the credential deployment
+<img src="../../docs/images/dcos-running-service-from-reg-files.png" alt="Running Service" style="width: 50%; height: 50%;"/>
+
+## Limitations
+- The API model currenlty only supports credentials to a single registry.
+- Not tested with Kubernetes clusters
+- Credentials have to be updated on each node 

examples/dcos-private-registry/dcos.json

@@ -0,0 +1,46 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "DCOS",
+      "dcosConfig" : {
+        "registry" : "",
+        "registryUser" : "",
+        "registryPassword" : ""
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentprivate",
+        "count": 1,
+        "vmSize": "Standard_D2_v2"
+      },
+      {
+        "name": "agentpublic",
+        "count": 1,
+        "vmSize": "Standard_D2_v2",
+        "dnsPrefix": "",
+        "ports": [
+          80,
+          443,
+          8080
+        ]
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    }
+  }
+}

parts/agentparams.t

@@ -76,3 +76,17 @@
       "type": "string"
     }
 {{end}} 
+{{if HasPrivateRegistry}}
+  ,"registry": {
+      "metadata": {
+        "description": "Private Container Registry"
+      }, 
+      "type": "string"
+    },
+  "registryKey": {
+      "metadata": {
+        "description": "base64 encoded key to the Private Container Registry"
+      }, 
+      "type": "string"
+    }
+  {{end}}

parts/dcos/dcosagentvars.t

@@ -55,3 +55,10 @@
 
     {{end}}
 {{end}}
+{{if HasPrivateRegistry}}
+    "registry" : "[tolower(parameters('registry'))]",
+    "registryKey" : "[parameters('registryKey')]",
+{{else}}
+    "registry" : "",
+    "registryKey" : "",
+{{end}}

parts/dcos/dcoscustomdata184.t

@@ -68,6 +68,15 @@ runcmd: PREPROVISION_EXTENSION
   - mask
   - --now
   - lxc-net.service
+- - tar
+  - czf 
+  - /etc/docker.tar.gz
+  - -C
+  - /tmp/xtoph
+  - .docker
+- - rm 
+  - -rf 
+  - /tmp/xtoph
 - /opt/azure/containers/provision.sh
 - - cp
   - -p
@@ -349,13 +358,14 @@ write_files:
     '
   path: /etc/systemd/system/dcos-setup.service
   permissions: '0644'
+- path: /var/lib/dcos/mesos-slave-common
+  content: 'ATTRIBUTES_STR'
 - content: ''
   path: /etc/mesosphere/roles/azure
 - content: 'PROVISION_STR'
   path: "/opt/azure/containers/provision.sh"
   permissions: "0744"
   owner: "root"
-- path: /var/lib/dcos/mesos-slave-common
-  content: 'ATTRIBUTES_STR'
-  permissions: "0644"
-  owner: "root"
+- content: '{ "auths": { "{{{registry}}}": { "auth" : "{{{registryKey}}}" } } }'
+  path: "/tmp/xtoph/.docker/config.json"
+  owner: "root"

parts/dcos/dcoscustomdata187.t

@@ -68,6 +68,15 @@ runcmd: PREPROVISION_EXTENSION
   - mask
   - --now
   - lxc-net.service
+- - tar
+  - czf 
+  - /etc/docker.tar.gz
+  - -C
+  - /tmp/xtoph
+  - .docker
+- - rm 
+  - -rf 
+  - /tmp/xtoph
 - /opt/azure/containers/provision.sh
 - - cp
   - -p
@@ -350,3 +359,6 @@ write_files:
   content: 'ATTRIBUTES_STR'
   permissions: "0644"
   owner: "root"
+- content: '{ "auths": { "{{{registry}}}": { "auth" : "{{{registryKey}}}" } } }'
+  path: "/tmp/xtoph/.docker/config.json"
+  owner: "root"

parts/dcos/dcoscustomdata188.t

@@ -68,6 +68,15 @@ runcmd: PREPROVISION_EXTENSION
   - mask
   - --now
   - lxc-net.service
+- - tar
+  - czf 
+  - /etc/docker.tar.gz
+  - -C
+  - /tmp/xtoph
+  - .docker
+- - rm 
+  - -rf 
+  - /tmp/xtoph
 - /opt/azure/containers/provision.sh
 - - cp
   - -p
@@ -350,3 +359,6 @@ write_files:
   content: 'ATTRIBUTES_STR'
   permissions: "0644"
   owner: "root"
+- content: '{ "auths": { "{{{registry}}}": { "auth" : "{{{registryKey}}}" } } }'
+  path: "/tmp/xtoph/.docker/config.json"
+  owner: "root"

parts/dcos/dcoscustomdata190.t

@@ -85,6 +85,15 @@ runcmd: PREPROVISION_EXTENSION
 - sed -i "s/^Port 22$/Port 22\nPort 2222/1" /etc/ssh/sshd_config
 - service ssh restart 
 - /opt/azure/containers/setup_ephemeral_disk.sh
+- - tar
+  - czf 
+  - /etc/docker.tar.gz
+  - -C
+  - /tmp/xtoph
+  - .docker
+- - rm 
+  - -rf 
+  - /tmp/xtoph
 - /opt/azure/containers/provision.sh
 - - cp
   - -p
@@ -119,7 +128,6 @@ runcmd: PREPROVISION_EXTENSION
   - --no-block
   - start
   - dcos-setup.service
-- /opt/azure/containers/add_admin_to_docker_group.sh
 write_files:
 - content: 'https://dcosio.azureedge.net/dcos/stable
 
@@ -305,11 +313,8 @@ write_files:
   content: 'ATTRIBUTES_STR'
   permissions: "0644"
   owner: "root"
-- content: |
-    #!/bin/bash
-    adduser {{{adminUsername}}} docker
-  path: "/opt/azure/containers/add_admin_to_docker_group.sh"
-  permissions: "0744"
+- content: '{ "auths": { "{{{registry}}}": { "auth" : "{{{registryKey}}}" } } }'
+  path: "/tmp/xtoph/.docker/config.json"
   owner: "root"
 - content: |
     #!/bin/bash

pkg/acsengine/engine.go

@@ -727,6 +727,11 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s
 			if properties.OrchestratorProfile.DcosConfig.DcosBootstrapURL != "" {
 				dcosBootstrapURL = properties.OrchestratorProfile.DcosConfig.DcosBootstrapURL
 			}
+
+			if len(properties.OrchestratorProfile.DcosConfig.Registry) > 0 {
+				addValue(parametersMap, "registry", properties.OrchestratorProfile.DcosConfig.Registry)
+				addValue(parametersMap, "registryKey", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", properties.OrchestratorProfile.DcosConfig.RegistryUser, properties.OrchestratorProfile.DcosConfig.RegistryPass))))
+			}
 		}
 
 		addValue(parametersMap, "dcosBootstrapURL", dcosBootstrapURL)
@@ -926,6 +931,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 			}
 			return strings.TrimSuffix(buf.String(), ", ")
 		},
+		"HasPrivateRegistry": func() bool {
+			if cs.Properties.OrchestratorProfile.DcosConfig != nil {
+				return len(cs.Properties.OrchestratorProfile.DcosConfig.Registry) > 0
+			}
+			return false
+		},
 		"RequiresFakeAgentOutput": func() bool {
 			return cs.Properties.OrchestratorProfile.OrchestratorType == api.Kubernetes
 		},
@@ -994,7 +1005,7 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 			return getDataDisks(profile)
 		},
 		"GetDCOSMasterCustomData": func() string {
-			masterProvisionScript := getDCOSMasterProvisionScript()
+			masterProvisionScript := getDCOSMasterProvisionScript(*cs.Properties.OrchestratorProfile)
 			masterAttributeContents := getDCOSMasterCustomNodeLabels()
 			masterPreprovisionExtension := ""
 			if cs.Properties.MasterProfile.PreprovisionExtension != nil {
@@ -1011,7 +1022,7 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 			return fmt.Sprintf("\"customData\": \"[base64(concat('#cloud-config\\n\\n', '%s'))]\",", str)
 		},
 		"GetDCOSAgentCustomData": func(profile *api.AgentPoolProfile) string {
-			agentProvisionScript := getDCOSAgentProvisionScript(profile)
+			agentProvisionScript := getDCOSAgentProvisionScript(profile, *cs.Properties.OrchestratorProfile)
 			attributeContents := getDCOSAgentCustomNodeLabels(profile)
 			agentPreprovisionExtension := ""
 			if profile.PreprovisionExtension != nil {
@@ -2169,7 +2180,7 @@ func getBase64CustomScriptFromStr(str string) string {
 	return base64.StdEncoding.EncodeToString(gzipB.Bytes())
 }
 
-func getDCOSAgentProvisionScript(profile *api.AgentPoolProfile) string {
+func getDCOSAgentProvisionScript(profile *api.AgentPoolProfile, orchProfile api.OrchestratorProfile) string {
 	// add the provision script
 
 	var scriptname string
@@ -2197,13 +2208,20 @@ func getDCOSAgentProvisionScript(profile *api.AgentPoolProfile) string {
 	} else {
 		roleFileContents = "touch /etc/mesosphere/roles/slave"
 	}
-
 	provisionScript = strings.Replace(provisionScript, "ROLESFILECONTENTS", roleFileContents, -1)
 
-	return provisionScript
+	var b bytes.Buffer
+	b.WriteString(provisionScript)
+	b.WriteString("\n")
+
+	if len(orchProfile.DcosConfig.Registry) == 0 {
+		b.WriteString("rm /etc/docker.tar.gz\n")
+	}
+
+	return b.String()
 }
 
-func getDCOSMasterProvisionScript() string {
+func getDCOSMasterProvisionScript(orchProfile api.OrchestratorProfile) string {
 	// add the provision script
 	bp, err1 := Asset(dcosProvision)
 	if err1 != nil {
@@ -2219,8 +2237,12 @@ func getDCOSMasterProvisionScript() string {
 	roleFileContents := `touch /etc/mesosphere/roles/master
 touch /etc/mesosphere/roles/azure_master`
 	provisionScript = strings.Replace(provisionScript, "ROLESFILECONTENTS", roleFileContents, -1)
+	var b bytes.Buffer
+	b.WriteString(provisionScript)
+	b.WriteString("\n")
+
+	return b.String()
 
-	return provisionScript
 }
 
 // getSingleLineForTemplate returns the file as a single line for embedding in an arm template
@@ -2244,7 +2266,7 @@ func getSingleLineDCOSCustomData(orchestratorType, orchestratorVersion string,
 
 	b, err := Asset(yamlFilename)
 	if err != nil {
-		panic(fmt.Sprintf("BUG: %s", err.Error()))
+		panic(fmt.Sprintf("BUG getting yaml custom data file: %s", err.Error()))
 	}
 
 	// transform the provision script content

pkg/api/converterfromapi.go

@@ -621,6 +621,7 @@ func convertOrchestratorProfileToV20170701(api *OrchestratorProfile, o *v2017070
 
 func convertOrchestratorProfileToVLabs(api *OrchestratorProfile, o *vlabs.OrchestratorProfile) {
 	o.OrchestratorType = api.OrchestratorType
+
 	if api.OrchestratorVersion != "" {
 		o.OrchestratorVersion = api.OrchestratorVersion
 		sv, _ := semver.NewVersion(o.OrchestratorVersion)
@@ -641,6 +642,17 @@ func convertOrchestratorProfileToVLabs(api *OrchestratorProfile, o *vlabs.Orches
 func convertDcosConfigToVLabs(api *DcosConfig, vlabs *vlabs.DcosConfig) {
 	vlabs.DcosBootstrapURL = api.DcosBootstrapURL
 	vlabs.DcosWindowsBootstrapURL = api.DcosWindowsBootstrapURL
+	if api.Registry != "" {
+		vlabs.Registry = api.Registry
+	}
+
+	if api.RegistryUser != "" {
+		vlabs.RegistryUser = api.RegistryUser
+	}
+
+	if api.RegistryPass != "" {
+		vlabs.RegistryPass = api.RegistryPass
+	}
 }
 
 func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.KubernetesConfig) {

pkg/api/convertertoapi.go

@@ -587,6 +587,18 @@ func convertVLabsOrchestratorProfile(vp *vlabs.Properties, api *OrchestratorProf
 func convertVLabsDcosConfig(vlabs *vlabs.DcosConfig, api *DcosConfig) {
 	api.DcosBootstrapURL = vlabs.DcosBootstrapURL
 	api.DcosWindowsBootstrapURL = vlabs.DcosWindowsBootstrapURL
+
+	if len(vlabs.Registry) > 0 {
+		api.Registry = vlabs.Registry
+	}
+
+	if len(vlabs.RegistryUser) > 0 {
+		api.RegistryUser = vlabs.RegistryUser
+	}
+
+	if len(vlabs.RegistryPass) > 0 {
+		api.RegistryPass = vlabs.RegistryPass
+	}
 }
 
 func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *KubernetesConfig) {
@@ -877,6 +889,7 @@ func convertVLabsAgentPoolProfile(vlabs *vlabs.AgentPoolProfile, api *AgentPoolP
 	api.Subnet = vlabs.GetSubnet()
 	api.IPAddressCount = vlabs.IPAddressCount
 	api.FQDN = vlabs.FQDN
+
 	api.CustomNodeLabels = map[string]string{}
 	for k, v := range vlabs.CustomNodeLabels {
 		api.CustomNodeLabels[k] = v

pkg/api/types.go

@@ -296,6 +296,9 @@ type KubernetesConfig struct {
 type DcosConfig struct {
 	DcosBootstrapURL        string `json:"dcosBootstrapURL,omitempty"`
 	DcosWindowsBootstrapURL string `json:"dcosWindowsBootstrapURL,omitempty"`
+	Registry                string `json:"registry,omitempty"`
+	RegistryUser            string `json:"registryUser,omitempty"`
+	RegistryPass            string `json:"registryPassword,omitempty"`
 }
 
 // MasterProfile represents the definition of the master cluster

pkg/api/vlabs/types.go

@@ -288,6 +288,9 @@ type KubernetesConfig struct {
 type DcosConfig struct {
 	DcosBootstrapURL        string `json:"dcosBootstrapURL,omitempty"`
 	DcosWindowsBootstrapURL string `json:"dcosWindowsBootstrapURL,omitempty"`
+	Registry                string `json:"registry,omitempty"`
+	RegistryUser            string `json:"registryUser,omitempty"`
+	RegistryPass            string `json:"registryPassword,omitempty"`
 }
 
 // MasterProfile represents the definition of the master cluster