fix kubernetes attached volumes

acstgen/api/vlabs/const.go

@@ -21,4 +21,10 @@ const (
 	MaxPort = 65535
 	// MaxDisks specifies the maximum attached disks to add to the cluster
 	MaxDisks = 4
+	// StorageExternal equates to VMSS where attached disks are unsupported (Default)
+	StorageExternal = "External"
+	// StorageVolumes equates to AS where attached disks are supported
+	StorageVolumes = "Volumes"
+	// StorageHAVolumes are managed disks that provide fault domain coverage for volumes.
+	StorageHAVolumes = "HAVolumes"
 )

acstgen/api/vlabs/types.go

@@ -59,7 +59,7 @@ type AgentPoolProfile struct {
 	VMSize       string `json:"vmSize"`
 	DNSPrefix    string `json:"dnsPrefix,omitempty"`
 	Ports        []int  `json:"ports,omitempty"`
-	IsStateful   bool   `json:"isStateful,omitempty"`
+	StorageType  string `json:"storageType,omitempty"`
 	DiskSizesGB  []int  `json:"diskSizesGB,omitempty"`
 	VnetSubnetID string `json:"vnetSubnetID,omitempty"`
 	// subnet is internal
@@ -111,6 +111,11 @@ func (a *AgentPoolProfile) IsCustomVNET() bool {
 	return len(a.VnetSubnetID) > 0
 }
 
+// IsVolumeBasedStorage returns true if the customer specified disks
+func (a *AgentPoolProfile) IsVolumeBasedStorage() bool {
+	return a.StorageType == StorageVolumes
+}
+
 // HasDisks returns true if the customer specified disks
 func (a *AgentPoolProfile) HasDisks() bool {
 	return len(a.DiskSizesGB) > 0

acstgen/api/vlabs/validate.go

@@ -69,8 +69,8 @@ func (a *AgentPoolProfile) Validate() error {
 			return e
 		}
 	}
-	if len(a.DiskSizesGB) > 0 && !a.IsStateful {
-		return fmt.Errorf("Disks were specified on a non stateful cluster named '%s'.  Ensure you add '\"isStateful\": true' to the model", a.Name)
+	if len(a.DiskSizesGB) > 0 && (a.StorageType != StorageVolumes && a.StorageType != StorageHAVolumes) {
+		return fmt.Errorf("Storage Type %s does not support attached disks for cluster named '%s'.  Specify storage as either %s or %s", a.StorageType, a.Name, StorageVolumes, StorageHAVolumes)
 	}
 	if len(a.DiskSizesGB) > MaxDisks {
 		return fmt.Errorf("A maximum of %d disks may be specified.  %d disks were specified for cluster named '%s'", MaxDisks, len(a.DiskSizesGB), a.Name)
@@ -118,11 +118,21 @@ func (a *AcsCluster) Validate() error {
 		if e := agentPoolProfile.Validate(); e != nil {
 			return e
 		}
-		if a.OrchestratorProfile.OrchestratorType == Swarm && agentPoolProfile.IsStateful {
-			return errors.New("stateful deployments are not supported with Swarm, please let us know if you want this feature")
+		switch agentPoolProfile.StorageType {
+		case StorageVolumes:
+		case StorageHAVolumes:
+		case StorageExternal:
+		case "":
+		default:
+			{
+				return fmt.Errorf("unknown storage type '%s' for agent pool '%s'.  Specify one of %s, %s, or %s", agentPoolProfile.StorageType, agentPoolProfile.Name, StorageExternal, StorageVolumes, StorageHAVolumes)
+			}
 		}
-		if a.OrchestratorProfile.OrchestratorType == Kubernetes && !agentPoolProfile.IsStateful {
-			return errors.New("stateless (VMSS) deployments are not supported with Kubernetes, Kubernetes requires the ability to attach/detach disks.  To fix specify 'isStateful=true'")
+		if agentPoolProfile.StorageType == StorageHAVolumes {
+			return errors.New("HA volumes are currently unsupported")
+		}
+		if a.OrchestratorProfile.OrchestratorType == Kubernetes && (agentPoolProfile.StorageType == StorageExternal || len(agentPoolProfile.StorageType) == 0) {
+			return fmt.Errorf("External storage deployments (VMSS) are not supported with Kubernetes since Kubernetes requires the ability to attach/detach disks.  To fix specify \"StorageType\":\"%s\"", StorageVolumes)
 		}
 		if a.OrchestratorProfile.OrchestratorType == Kubernetes && len(agentPoolProfile.DNSPrefix) > 0 {
 			return errors.New("DNSPrefix not support for agent pools in Kubernetes - Kubernetes marks its own clusters public")

acstgen/clusterdefinitions/kubernetes.json

@@ -12,13 +12,13 @@
       "name": "agentpool1",
       "count": 3,
       "vmSize": "Standard_D2_v2",
-      "isStateful": true
+      "StorageType": "Volumes"
     },
     {
       "name": "agentpool2",
       "count": 3,
       "vmSize": "Standard_D2_v2",
-      "isStateful": true
+      "StorageType": "Volumes"
     }
   ],
   "linuxProfile": {
@@ -32,7 +32,7 @@
     }
   },
   "servicePrincipalProfile": {
-    "servicePrincipalClientID": "REPLACE-WITH-ServicePrincipalClientID",
-    "servicePrincipalClientSecret": "REPLACE-WITH-myServicePrincipalClientSecret"
+    "servicePrincipalClientID": "[REPLACE-WITH-ServicePrincipalClientID",
+    "servicePrincipalClientSecret": "[REPLACE-WITH-myServicePrincipalClientSecret"
   }
 }

acstgen/clusterdefinitions/vnet/dcosvnet.json

@@ -14,7 +14,7 @@
       "name": "agentprivate",
       "count": 3,
       "vmSize": "Standard_D2_v2",
-      "isStateful": true,
+      "StorageType": "Volumes",
       "vnetSubnetId": "/subscriptions/b52fce95-de5f-4b37-afca-db203a5d0b6a/resourceGroups/anhoweExampleRG/providers/Microsoft.Network/virtualNetworks/ExampleCustomVNET/subnets/ExampleAgentSubnet"
     },
     {

acstgen/clusterdefinitions/vnet/kubernetesvnet.json

@@ -15,14 +15,14 @@
       "count": 2,
       "vmSize": "Standard_D2_v2",
       "vnetSubnetId": "/subscriptions/b52fce95-de5f-4b37-afca-db203a5d0b6a/resourceGroups/anhoweKubeVnet/providers/Microsoft.Network/virtualNetworks/KubernetesCustomVNET/subnets/KubernetesSubnet",
-      "isStateful": true
+      "StorageType": "Volumes"
     },
     {
       "name": "agentpri2",
       "count": 2,
       "vmSize": "Standard_D2_v2",
       "vnetSubnetId": "/subscriptions/b52fce95-de5f-4b37-afca-db203a5d0b6a/resourceGroups/anhoweKubeVnet/providers/Microsoft.Network/virtualNetworks/KubernetesCustomVNET/subnets/KubernetesSubnet",
-      "isStateful": true
+      "StorageType": "Volumes"
     }
   ],
   "linuxProfile": {
@@ -36,7 +36,7 @@
     }
   },
   "servicePrincipalProfile": {
-    "servicePrincipalClientID": "REPLACE-WITH-ServicePrincipalClientID",
-    "servicePrincipalClientSecret": "REPLACE-WITH-myServicePrincipalClientSecret"
+    "servicePrincipalClientID": "[REPLACE-WITH-ServicePrincipalClientID",
+    "servicePrincipalClientSecret": "[REPLACE-WITH-myServicePrincipalClientSecret"
   }
 }

acstgen/clusterdefinitions/volumes/dcosvolumes-preAttachedDisks.json

@@ -8,53 +8,25 @@
     "vmSize": "Standard_D2_v2"
   },
   "agentPoolProfiles": [
-    {
-      "name": "agtnodisks1",
-      "count": 1,
-      "vmSize": "Standard_D2_v2",
-      "isStateful": true
-    },
     {
       "name": "agent128",
       "count": 3,
       "vmSize": "Standard_D2_v2",
-      "isStateful": true,
-      "diskSizesGB": [10, 10, 10, 10]
+      "StorageType": "Volumes",
+      "diskSizesGB": [128, 128, 128, 128]
     },
     {
       "name": "agent1public",
       "count": 3,
       "vmSize": "Standard_D2_v2",
       "dnsPrefix": "appanhowe0928g",
-      "isStateful": true,
+      "StorageType": "Volumes",
       "diskSizesGB": [1],
       "ports": [
         80,
         443,
         8080
       ]
-    },
-    {
-      "name": "agtnodisks2",
-      "count": 1,
-      "vmSize": "Standard_D2_v2",
-      "isStateful": true
-    },
-    {
-      "name": "agtnodisks3",
-      "count": 1,
-      "vmSize": "Standard_D2_v2",
-      "isStateful": true
-    },
-    {
-      "name": "astateless1",
-      "count": 2,
-      "vmSize": "Standard_D2_v2"
-    },
-    {
-      "name": "astateless2",
-      "count": 2,
-      "vmSize": "Standard_D2_v2"
     }
   ],
   "linuxProfile": {

acstgen/clusterdefinitions/volumes/dcosvolumes.json

@@ -12,13 +12,13 @@
       "name": "agtnodisks1",
       "count": 1,
       "vmSize": "Standard_D2_v2",
-      "isStateful": true
+      "StorageType": "Volumes"
     },
     {
       "name": "agtnodisks2",
       "count": 1,
       "vmSize": "Standard_D2_v2",
-      "isStateful": true
+      "StorageType": "Volumes"
     }
   ],
   "linuxProfile": {

acstgen/clusterdefinitions/volumes/kubernetes-preAttachedDisks.json

@@ -0,0 +1,40 @@
+{
+  "orchestratorProfile": {
+    "orchestratorType": "Kubernetes"
+  },
+  "masterProfile": {
+    "count": 1,
+    "dnsPrefix": "mgmtanhowe1004f",
+		"vmSize": "Standard_D2_v2"
+  },
+  "agentPoolProfiles": [
+    {
+      "name": "agentpool1",
+      "count": 3,
+      "vmSize": "Standard_D2_v2",
+      "StorageType": "Volumes",
+      "diskSizesGB": [128, 128, 128, 128]
+    },
+    {
+      "name": "agentpool2",
+      "count": 3,
+      "vmSize": "Standard_D2_v2",
+      "StorageType": "Volumes",
+      "diskSizesGB": [10, 10, 10, 10]
+    }
+  ],
+  "linuxProfile": {
+    "adminUsername": "azureuser",
+    "ssh": {
+      "publicKeys": [
+        {
+          "keyData": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8fhkh3jpHUQsrUIezFB5k4Rq9giJM8G1Cr0u2IRMiqG++nat5hbOr3gODpTA0h11q9bzb6nJtK7NtDzIHx+w3YNIVpcTGLiUEsfUbY53IHg7Nl/p3/gkST3g0R6BSL7Hg45SfyvpH7kwY30MoVHG/6P3go4SKlYoHXlgaaNr3fMwUTIeE9ofvyS3fcr6xxlsoB6luKuEs50h0NGsE4QEnbfSY4Yd/C1ucc3mEw+QFXBIsENHfHfZYrLNHm2L8MXYVmAH8k//5sFs4Migln9GiUgEQUT6uOjowsZyXBbXwfT11og+syPkAq4eqjiC76r0w6faVihdBYVoc/UcyupgH azureuser@linuxvm"
+        }
+      ]
+    }
+  },
+  "servicePrincipalProfile": {
+    "servicePrincipalClientID": "[REPLACE-WITH-ServicePrincipalClientID",
+    "servicePrincipalClientSecret": "[REPLACE-WITH-myServicePrincipalClientSecret"
+  }
+}

acstgen/parts/dcosagentvars.t

@@ -1,6 +1,6 @@
     "{{.Name}}StorageAccountOffset": "[mul(variables('maxStorageAccountsPerAgent'),variables('{{.Name}}Index'))]",
     "{{.Name}}Count": "[parameters('{{.Name}}Count')]",
-{{if .IsStateful}}
+{{if .IsVolumeBasedStorage}}
     "{{.Name}}AvailabilitySet": "[concat('{{.Name}}-availabilitySet-', variables('nameSuffix'))]",
     "{{.Name}}StorageAccountsCount": "[add(div(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')), mod(add(mod(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')),2), add(mod(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')),1)))]",
 {{else}}

acstgen/parts/dcosbase.t

@@ -8,7 +8,7 @@
   "variables": {
     {{range $index, $agent := .AgentPoolProfiles}}
         {{template "dcosagentvars.t" .}}
-        {{if .IsStateful}}
+        {{if .IsVolumeBasedStorage}}
           "{{.Name}}DataAccountName": "[concat(variables('storageAccountBaseName'), 'data{{$index}}')]",
         {{end}}
         "{{.Name}}Index": {{$index}},
@@ -21,7 +21,7 @@
   },
   "resources": [
     {{range .AgentPoolProfiles}}
-      {{if .IsStateful}}
+      {{if .IsVolumeBasedStorage}}
         {{template "dcosagentresourcesdisks.t" .}},
       {{else}}
         {{template "dcosagentresources.t" .}},

acstgen/parts/kubernetesagentvars.t

@@ -1,11 +1,7 @@
     "{{.Name}}StorageAccountOffset": "[mul(variables('maxStorageAccountsPerAgent'),variables('{{.Name}}Index'))]",
     "{{.Name}}Count": "[parameters('{{.Name}}Count')]",
-{{if .IsStateful}}
     "{{.Name}}AvailabilitySet": "[concat('{{.Name}}-availabilitySet-', variables('nameSuffix'))]",
     "{{.Name}}StorageAccountsCount": "[add(div(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')), mod(add(mod(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')),2), add(mod(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')),1)))]",
-{{else}}
-    "{{.Name}}StorageAccountsCount": "[variables('maxStorageAccountsPerAgent')]",
-{{end}}  
     "{{.Name}}VMNamePrefix": "[concat(variables('orchestratorName'), '-{{.Name}}-', variables('nameSuffix'), '-')]", 
     "{{.Name}}VMSize": "[parameters('{{.Name}}VMSize')]",
 {{if .IsCustomVNET}}

acstgen/tgen/defaults.go

@@ -15,6 +15,8 @@ func SetAcsClusterDefaults(a *vlabs.AcsCluster) (bool, error) {
 
 	setAgentNetworkDefaults(a)
 
+	setStorageDefaults(a)
+
 	certsGenerated, e := setDefaultCerts(a)
 	if e != nil {
 		return false, e
@@ -54,6 +56,16 @@ func setAgentNetworkDefaults(a *vlabs.AcsCluster) {
 	}
 }
 
+// setStorageDefaults for agents
+func setStorageDefaults(a *vlabs.AcsCluster) {
+	for i := range a.AgentPoolProfiles {
+		profile := &a.AgentPoolProfiles[i]
+		if len(profile.StorageType) == 0 {
+			profile.StorageType = vlabs.StorageExternal
+		}
+	}
+}
+
 func setDefaultCerts(a *vlabs.AcsCluster) (bool, error) {
 	if !certGenerationRequired(a) {
 		return false, nil