Enable boot diagnostics on RP nodes

The RP ARM template is updated to enable boot diagnostics on the VMSS. A new
storage account is also provisioned to store the bootstrap logs.

Each VMSS in a region has its own storage account.

Within the storage account, a new container is automatically created for each
node in the vmss. Only the latest bootstrap logs are stored.

This change affects the rp-production.json ARM template only.

Signed-off-by: Ivan Sim <isim@redhat.com>

deploy/config.yaml.example

@@ -1,33 +0,0 @@
-rps:
-- location: eastus
-  subscriptionId: 00000000-0000-0000-0000-000000000000
-  resourceGroupName: aro-eastus
-  configuration:
-    databaseAccountName: ''
-    keyvaultPrefix: ''
-configuration:
-  acrResourceId: ''
-  rpVersionStorageAccountName: ''
-  acrReplicaDisabled: true
-  adminApiCaBundle: ''
-  adminApiClientCertCommonName: ''
-  clusterParentDomainName: ''
-  extraClusterKeyvaultAccessPolicies: []
-  extraCosmosDBIPs: ''
-  extraServiceKeyvaultAccessPolicies: []
-  fpServerCertCommonName: ''
-  fpServicePrincipalId: ''
-  globalMonitoringKeyVaultUri: ''
-  globalResourceGroupName: ''
-  globalResourceGroupLocation: ''
-  globalSubscriptionId: ''
-  mdmFrontendUrl: ''
-  mdsdConfigVersion: ''
-  mdsdEnvironment: ''
-  rpImagePrefix: ''
-  rpMode: ''
-  rpParentDomainName: ''
-  subscriptionResourceGroupName: ''
-  subscriptionResourceGroupLocation: ''
-  sshPublicKey: ''
-  vmSize: ''

deploy/generate.go

@@ -1,5 +1,5 @@
 //go:generate go run ../hack/gendeploy
-//go:generate go run ../vendor/github.com/go-bindata/go-bindata/go-bindata -nometadata -pkg $GOPACKAGE -ignore=generate.go -ignore=config.yaml -ignore=config.yaml.example -o ../pkg/deploy/bindata.go .
+//go:generate go run ../vendor/github.com/go-bindata/go-bindata/go-bindata -nometadata -pkg $GOPACKAGE -ignore=generate.go -ignore=config.yaml -o ../pkg/deploy/bindata.go .
 //go:generate gofmt -s -l -w ../pkg/deploy/bindata.go
 
 package deploy

deploy/rp-production-parameters.json

@@ -47,6 +47,9 @@
         "sshPublicKey": {
             "value": ""
         },
+        "storageAccountName": {
+            "value": ""
+        },
         "subscriptionResourceGroupName": {
             "value": ""
         },

deploy/rp-production.json

@@ -50,6 +50,9 @@
         "sshPublicKey": {
             "type": "string"
         },
+        "storageAccountName": {
+            "type": "string"
+        },
         "subscriptionResourceGroupName": {
             "type": "string"
         },
@@ -209,6 +212,12 @@
                             }
                         ]
                     },
+                    "diagnosticsProfile": {
+                        "bootDiagnostics": {
+                            "enabled": true,
+                            "storageUri": "[concat('https://', parameters('storageAccountName'), '.blob.core.windows.net/')]"
+                        }
+                    },
                     "extensionProfile": {
                         "extensions": [
                             {
@@ -242,9 +251,20 @@
             "dependsOn": [
                 "[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceGroup().id, parameters('rpServicePrincipalId'), 'RP / Reader'))]",
                 "[resourceId('Microsoft.Network/virtualNetworks', 'rp-vnet')]",
-                "[resourceId('Microsoft.Network/loadBalancers', 'rp-lb')]"
+                "[resourceId('Microsoft.Network/loadBalancers', 'rp-lb')]",
+                "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]"
             ]
         },
+        {
+            "sku": {
+                "name": "Standard_LRS"
+            },
+            "location": "[resourceGroup().location]",
+            "name": "[parameters('storageAccountName')]",
+            "type": "Microsoft.Storage/storageAccounts",
+            "condition": "[parameters('fullDeploy')]",
+            "apiVersion": "2019-04-01"
+        },
         {
             "properties": {
                 "severity": 2,

pkg/deploy/config.go

@@ -54,6 +54,7 @@ type Configuration struct {
 	SubscriptionResourceGroupName      *string       `json:"subscriptionResourceGroupName,omitempty" value:"required"`
 	SubscriptionResourceGroupLocation  *string       `json:"subscriptionResourceGroupLocation,omitempty" value:"required"`
 	SSHPublicKey                       *string       `json:"sshPublicKey,omitempty" value:"required"`
+	StorageAccountName                 *string       `json:"storageAccountName,omitempty" value:"required"`
 	VMSize                             *string       `json:"vmSize,omitempty" value:"required"`
 }
 

pkg/deploy/config_test.go

@@ -137,6 +137,7 @@ func TestConfigRequiredValues(t *testing.T) {
 	SSHPublicKey := "SSHPublicKey"
 	SubscriptionResourceGroupName := "SubscriptionResourceGroupName"
 	SubscriptionResourceGroupLocation := "SubscriptionResourceGroupLocation"
+	StorageAccountName := "StorageAccountName"
 	VMSize := "VMSize"
 
 	for _, tt := range []struct {
@@ -175,6 +176,7 @@ func TestConfigRequiredValues(t *testing.T) {
 					SSHPublicKey:                       &SSHPublicKey,
 					SubscriptionResourceGroupName:      &SubscriptionResourceGroupName,
 					SubscriptionResourceGroupLocation:  &SubscriptionResourceGroupLocation,
+					StorageAccountName:                 &StorageAccountName,
 					VMSize:                             &VMSize,
 				},
 			},
@@ -189,7 +191,7 @@ func TestConfigRequiredValues(t *testing.T) {
 					ExtraCosmosDBIPs: []string{ExtraCosmosDBIPs},
 				},
 			},
-			expect: fmt.Errorf("Configuration has missing fields: %s", "[RPVersionStorageAccountName AdminAPIClientCertCommonName ClusterParentDomainName DatabaseAccountName ExtraClusterKeyvaultAccessPolicies ExtraServiceKeyvaultAccessPolicies FPServicePrincipalID GlobalMonitoringKeyVaultURI GlobalResourceGroupName GlobalResourceGroupLocation GlobalSubscriptionID KeyvaultPrefix MDMFrontendURL MDSDConfigVersion MDSDEnvironment RPImagePrefix RPNSGSourceAddressPrefixes RPParentDomainName SubscriptionResourceGroupName SubscriptionResourceGroupLocation SSHPublicKey VMSize]"),
+			expect: fmt.Errorf("Configuration has missing fields: %s", "[RPVersionStorageAccountName AdminAPIClientCertCommonName ClusterParentDomainName DatabaseAccountName ExtraClusterKeyvaultAccessPolicies ExtraServiceKeyvaultAccessPolicies FPServicePrincipalID GlobalMonitoringKeyVaultURI GlobalResourceGroupName GlobalResourceGroupLocation GlobalSubscriptionID KeyvaultPrefix MDMFrontendURL MDSDConfigVersion MDSDEnvironment RPImagePrefix RPNSGSourceAddressPrefixes RPParentDomainName SubscriptionResourceGroupName SubscriptionResourceGroupLocation SSHPublicKey StorageAccountName VMSize]"),
 		},
 	} {
 		valid := tt.config.validate()

pkg/deploy/generator/resources.go

@@ -1054,6 +1054,12 @@ done
 							},
 						},
 					},
+					DiagnosticsProfile: &mgmtcompute.DiagnosticsProfile{
+						BootDiagnostics: &mgmtcompute.BootDiagnostics{
+							Enabled:    to.BoolPtr(true),
+							StorageURI: to.StringPtr("[concat('https://', parameters('storageAccountName'), '.blob.core.windows.net/')]"),
+						},
+					},
 				},
 				Overprovision: to.BoolPtr(false),
 			},
@@ -1072,6 +1078,7 @@ done
 			"[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceGroup().id, parameters('rpServicePrincipalId'), 'RP / Reader'))]",
 			"[resourceId('Microsoft.Network/virtualNetworks', 'rp-vnet')]",
 			"[resourceId('Microsoft.Network/loadBalancers', 'rp-lb')]",
+			"[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]",
 		},
 	}
 }
@@ -1582,6 +1589,21 @@ func (g *generator) rpVersionStorageAccount() []*arm.Resource {
 	}
 }
 
+func (g *generator) storageAccount() *arm.Resource {
+	return &arm.Resource{
+		Resource: &mgmtstorage.Account{
+			Name:     to.StringPtr("[parameters('storageAccountName')]"),
+			Type:     to.StringPtr("Microsoft.Storage/storageAccounts"),
+			Location: to.StringPtr("[resourceGroup().location]"),
+			Sku: &mgmtstorage.Sku{
+				Name: "Standard_LRS",
+			},
+		},
+		Condition:  g.conditionStanza("fullDeploy"),
+		APIVersion: azureclient.APIVersions["Microsoft.Storage"],
+	}
+}
+
 func (g *generator) devCIPool() *arm.Resource {
 	parts := []string{
 		fmt.Sprintf("base64ToString('%s')", base64.StdEncoding.EncodeToString([]byte("set -e\n\n"))),

pkg/deploy/generator/templates.go

@@ -93,6 +93,7 @@ func (g *generator) rpTemplate() *arm.Template {
 			"rpImage",
 			"rpMode",
 			"sshPublicKey",
+			"storageAccountName",
 			"subscriptionResourceGroupName",
 			"vmssName",
 			"vmSize",
@@ -115,6 +116,7 @@ func (g *generator) rpTemplate() *arm.Template {
 
 	if g.production {
 		t.Resources = append(t.Resources, g.pip(), g.lb(), g.vmss(),
+			g.storageAccount(),
 			g.lbAlert(30.0, 2, "rp-availability-alert", "PT5M", "PT5M", "DipAvailability"), // triggers on all 3 RPs being down for 3.5min, can't be >=0.3 due to deploys going down to 32% at times.
 			g.lbAlert(67.0, 3, "rp-degraded-alert", "PT15M", "PT6H", "DipAvailability"),    // 1/3 backend down for 1h or 2/3 down for 3h in the last 6h
 			g.lbAlert(33.0, 2, "rp-vnet-alert", "PT5M", "PT5M", "VipAvailability"))         // this will trigger only if the Azure network infrastructure between the loadBalancers and VMs is down for 3.5min