Merge pull request #28 from jim-minter/deployment_retry

Grab bag

.gitignore

@@ -10,6 +10,7 @@ __pycache__
 /env*
 !/env.example
 /id_rsa
+/proxy
 /pyenv*
 /python/az/aro/build
 /python/az/aro/dist

Dockerfile.proxy

@@ -0,0 +1,5 @@
+FROM registry.access.redhat.com/ubi8/ubi-minimal
+COPY proxy /usr/local/bin
+ENTRYPOINT ["proxy"]
+EXPOSE 8443/tcp
+USER 1000

Makefile

@@ -48,9 +48,16 @@ client: generate
 generate:
 	go generate ./...
 
-image: aro
+image-aro: aro
 	docker pull registry.access.redhat.com/ubi8/ubi-minimal
-	docker build -t arosvc.azurecr.io/aro:$(COMMIT) .
+	docker build -f Dockerfile.aro -t arosvc.azurecr.io/aro:$(COMMIT) .
+
+image-proxy: proxy
+	docker pull registry.access.redhat.com/ubi8/ubi-minimal
+	docker build -f Dockerfile.proxy -t arosvc.azurecr.io/proxy:latest .
+
+proxy:
+	go build -ldflags "-X main.gitCommit=$(COMMIT)" ./hack/proxy
 
 pyenv${PYTHON_VERSION}:
 	virtualenv --python=/usr/bin/python${PYTHON_VERSION} pyenv${PYTHON_VERSION}
@@ -87,4 +94,4 @@ test-python: generate pyenv${PYTHON_VERSION}
 		azdev linter && \
 		azdev style
 
-.PHONY: aro az clean client generate image secrets secrets-update test-go test-python
+.PHONY: aro az clean client generate image-aro proxy secrets secrets-update test-go test-python

README.md

@@ -14,7 +14,8 @@ service, please see the following links:
 ## Quickstarts
 
 * If you have a whitelisted subscription and want to use `az aro` to create a
-  cluster, follow [using `az aro`](docs/using-az-aro.md).
+  cluster using the production RP, follow [using `az
+  aro`](docs/using-az-aro.md).
 
 * If you want to deploy a development RP, follow [deploy development
   RP](docs/deploy-development-rp.md).

cmd/aro/rp.go

@@ -27,7 +27,7 @@ func rp(ctx context.Context, log *logrus.Entry) error {
 		return err
 	}
 
-	db, err := database.NewDatabase(ctx, env, uuid, "ARO")
+	db, err := database.NewDatabase(env, uuid)
 	if err != nil {
 		return err
 	}

deploy/databases-development.json

@@ -0,0 +1,92 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "databaseAccountName": {
+            "type": "string"
+        },
+        "databaseName": {
+            "type": "string"
+        }
+    },
+    "resources": [
+        {
+            "properties": {
+                "resource": {
+                    "id": "[parameters('databaseName')]"
+                },
+                "options": {
+                    "x-ms-offer-throughput": "400"
+                }
+            },
+            "name": "[concat(parameters('databaseAccountName'), '/', parameters('databaseName'))]",
+            "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-08-01"
+        },
+        {
+            "properties": {
+                "resource": {
+                    "id": "AsyncOperations",
+                    "partitionKey": {
+                        "paths": [
+                            "/id"
+                        ],
+                        "kind": "Hash"
+                    },
+                    "defaultTtl": 604800
+                },
+                "options": {}
+            },
+            "name": "[concat(parameters('databaseAccountName'), '/', parameters('databaseName'), '/AsyncOperations')]",
+            "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-08-01"
+        },
+        {
+            "properties": {
+                "resource": {
+                    "id": "OpenShiftClusters",
+                    "partitionKey": {
+                        "paths": [
+                            "/partitionKey"
+                        ],
+                        "kind": "Hash"
+                    },
+                    "uniqueKeyPolicy": {
+                        "uniqueKeys": [
+                            {
+                                "paths": [
+                                    "/key"
+                                ]
+                            }
+                        ]
+                    }
+                },
+                "options": {}
+            },
+            "name": "[concat(parameters('databaseAccountName'), '/', parameters('databaseName'), '/OpenShiftClusters')]",
+            "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-08-01"
+        },
+        {
+            "properties": {
+                "resource": {
+                    "id": "Subscriptions",
+                    "partitionKey": {
+                        "paths": [
+                            "/id"
+                        ],
+                        "kind": "Hash"
+                    }
+                },
+                "options": {}
+            },
+            "name": "[concat(parameters('databaseAccountName'), '/', parameters('databaseName'), '/Subscriptions')]",
+            "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-08-01"
+        }
+    ]
+}

deploy/env-development.json

@@ -0,0 +1,207 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "proxyCert": {
+            "type": "string"
+        },
+        "proxyClientCert": {
+            "type": "string"
+        },
+        "proxyDomainNameLabel": {
+            "type": "string"
+        },
+        "proxyImage": {
+            "type": "string"
+        },
+        "proxyImageAuth": {
+            "type": "securestring"
+        },
+        "proxyKey": {
+            "type": "securestring"
+        },
+        "sshPublicKey": {
+            "type": "string"
+        },
+        "vpnCACertificate": {
+            "type": "string"
+        }
+    },
+    "resources": [
+        {
+            "sku": {
+                "name": "Standard"
+            },
+            "properties": {
+                "publicIPAllocationMethod": "Static"
+            },
+            "name": "dev-vpn-pip",
+            "type": "Microsoft.Network/publicIPAddresses",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-07-01"
+        },
+        {
+            "properties": {
+                "addressSpace": {
+                    "addressPrefixes": [
+                        "10.0.0.0/9"
+                    ]
+                },
+                "subnets": [
+                    {
+                        "properties": {
+                            "addressPrefix": "10.0.0.0/24"
+                        },
+                        "name": "GatewaySubnet"
+                    }
+                ]
+            },
+            "name": "dev-vnet",
+            "type": "Microsoft.Network/virtualNetworks",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-07-01"
+        },
+        {
+            "properties": {
+                "ipConfigurations": [
+                    {
+                        "properties": {
+                            "subnet": {
+                                "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'dev-vnet', 'GatewaySubnet')]"
+                            },
+                            "publicIPAddress": {
+                                "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'dev-vpn-pip')]"
+                            }
+                        },
+                        "name": "default"
+                    }
+                ],
+                "vpnType": "RouteBased",
+                "sku": {
+                    "name": "VpnGw1",
+                    "tier": "VpnGw1"
+                },
+                "vpnClientConfiguration": {
+                    "vpnClientAddressPool": {
+                        "addressPrefixes": [
+                            "192.168.255.0/24"
+                        ]
+                    },
+                    "vpnClientRootCertificates": [
+                        {
+                            "properties": {
+                                "publicCertData": "[parameters('vpnCACertificate')]"
+                            },
+                            "name": "dev-vpn-ca"
+                        }
+                    ],
+                    "vpnClientProtocols": [
+                        "OpenVPN"
+                    ]
+                }
+            },
+            "name": "dev-vpn",
+            "type": "Microsoft.Network/virtualNetworkGateways",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-07-01",
+            "dependsOn": [
+                "[resourceId('Microsoft.Network/publicIPAddresses', 'dev-vpn-pip')]",
+                "[resourceId('Microsoft.Network/virtualNetworks', 'dev-vnet')]"
+            ]
+        },
+        {
+            "sku": {
+                "name": "Standard_D2s_v3",
+                "tier": "Standard",
+                "capacity": 1
+            },
+            "properties": {
+                "upgradePolicy": {
+                    "mode": "Manual"
+                },
+                "virtualMachineProfile": {
+                    "osProfile": {
+                        "computerNamePrefix": "dev-proxy-",
+                        "adminUsername": "cloud-user",
+                        "linuxConfiguration": {
+                            "disablePasswordAuthentication": true,
+                            "ssh": {
+                                "publicKeys": [
+                                    {
+                                        "path": "/home/cloud-user/.ssh/authorized_keys",
+                                        "keyData": "[parameters('sshPublicKey')]"
+                                    }
+                                ]
+                            }
+                        }
+                    },
+                    "storageProfile": {
+                        "imageReference": {
+                            "publisher": "RedHat",
+                            "offer": "RHEL",
+                            "sku": "7-RAW",
+                            "version": "latest"
+                        },
+                        "osDisk": {
+                            "createOption": "FromImage",
+                            "managedDisk": {
+                                "storageAccountType": "Premium_LRS"
+                            }
+                        }
+                    },
+                    "networkProfile": {
+                        "networkInterfaceConfigurations": [
+                            {
+                                "name": "dev-proxy-vmss-nic",
+                                "properties": {
+                                    "primary": true,
+                                    "ipConfigurations": [
+                                        {
+                                            "name": "dev-proxy-vmss-ipconfig",
+                                            "properties": {
+                                                "subnet": {
+                                                    "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"
+                                                },
+                                                "primary": true,
+                                                "publicIPAddressConfiguration": {
+                                                    "name": "dev-proxy-vmss-pip",
+                                                    "properties": {
+                                                        "dnsSettings": {
+                                                            "domainNameLabel": "[parameters('proxyDomainNameLabel')]"
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    ]
+                                }
+                            }
+                        ]
+                    },
+                    "extensionProfile": {
+                        "extensions": [
+                            {
+                                "name": "dev-proxy-vmss-cse",
+                                "properties": {
+                                    "publisher": "Microsoft.Azure.Extensions",
+                                    "type": "CustomScript",
+                                    "typeHandlerVersion": "2.0",
+                                    "autoUpgradeMinorVersion": true,
+                                    "settings": {},
+                                    "protectedSettings": {
+                                        "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'PROXYIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImage')),''')\n','PROXYIMAGEAUTH=$(base64 -d \u003c\u003c\u003c''',base64(parameters('proxyImageAuth')),''')\n','PROXYCERT=''',parameters('proxyCert'),'''\n','PROXYCLIENTCERT=''',parameters('proxyClientCert'),'''\n','PROXYKEY=''',parameters('proxyKey'),'''\n','\n',base64ToString('eXVtIC15IHVwZGF0ZSAteCBXQUxpbnV4QWdlbnQKCnl1bSAteSBpbnN0YWxsIGRvY2tlcgoKZmlyZXdhbGwtY21kIC0tYWRkLXBvcnQ9NDQzL3RjcCAtLXBlcm1hbmVudAoKbWtkaXIgL3Jvb3QvLmRvY2tlcgpjYXQgPi9yb290Ly5kb2NrZXIvY29uZmlnLmpzb24gPDxFT0YKewoJImF1dGhzIjogewoJCSIke1BST1hZSU1BR0UlJS8qfSI6IHsKCQkJImF1dGgiOiAiJFBST1hZSU1BR0VBVVRIIgoJCX0KCX0KfQpFT0YKCm1rZGlyIC9ldGMvcHJveHkKYmFzZTY0IC1kIDw8PCIkUFJPWFlDRVJUIiA+L2V0Yy9wcm94eS9wcm94eS5jcnQKYmFzZTY0IC1kIDw8PCIkUFJPWFlLRVkiID4vZXRjL3Byb3h5L3Byb3h5LmtleQpiYXNlNjQgLWQgPDw8IiRQUk9YWUNMSUVOVENFUlQiID4vZXRjL3Byb3h5L3Byb3h5LWNsaWVudC5jcnQKY2hvd24gLVIgMTAwMDoxMDAwIC9ldGMvcHJveHkKY2htb2QgMDYwMCAvZXRjL3Byb3h5L3Byb3h5LmtleQoKY2F0ID4vZXRjL3N5c2NvbmZpZy9wcm94eSA8PEVPRgpQUk9YWV9JTUFHRT0nJFBST1hZSU1BR0UnCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL3Byb3h5LnNlcnZpY2UgPDxFT0YKW1VuaXRdCkFmdGVyPWRvY2tlci5zZXJ2aWNlClJlcXVpcmVzPWRvY2tlci5zZXJ2aWNlCgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL3Byb3h5CkV4ZWNTdGFydFByZT0tL3Vzci9iaW4vZG9ja2VyIHJtIC1mICVuCkV4ZWNTdGFydFByZT0vdXNyL2Jpbi9kb2NrZXIgcHVsbCBcJFBST1hZX0lNQUdFCkV4ZWNTdGFydD0vdXNyL2Jpbi9kb2NrZXIgcnVuIC0tcm0gLS1uYW1lICVuIC1wIDQ0Mzo4NDQzIC12IC9ldGMvcHJveHk6L3NlY3JldHMgXCRQUk9YWV9JTUFHRQpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlbgpSZXN0YXJ0PWFsd2F5cwoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKc3lzdGVtY3RsIGVuYWJsZSBwcm94eS5zZXJ2aWNlCgooc2xlZXAgMzA7IHJlYm9vdCkgJgo=')))]"
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                },
+                "overprovision": false
+            },
+            "name": "dev-proxy-vmss",
+            "type": "Microsoft.Compute/virtualMachineScaleSets",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-03-01"
+        }
+    ]
+}

deploy/rbac-development.json

@@ -9,7 +9,6 @@
             "type": "string"
         }
     },
-    "variables": {},
     "resources": [
         {
             "name": "79ed474a-7267-4ff8-b226-96140be062a2",
@@ -61,6 +60,8 @@
                             "Microsoft.Network/privateDnsZones/SRV/write",
                             "Microsoft.Network/privateDnsZones/virtualNetworkLinks/write",
                             "Microsoft.Network/privateDnsZones/write",
+                            "Microsoft.Network/privateLinkServices/read",
+                            "Microsoft.Network/privateLinkServices/write",
                             "Microsoft.Network/publicIPAddresses/delete",
                             "Microsoft.Network/publicIPAddresses/join/action",
                             "Microsoft.Network/publicIPAddresses/read",
@@ -68,6 +69,7 @@
                             "Microsoft.Network/routeTables/read",
                             "Microsoft.Network/routeTables/write",
                             "Microsoft.Resources/deployments/operationStatuses/read",
+                            "Microsoft.Resources/deployments/read",
                             "Microsoft.Resources/deployments/write",
                             "Microsoft.Resources/subscriptions/resourceGroups/delete",
                             "Microsoft.Resources/subscriptions/resourceGroups/read",
@@ -80,27 +82,6 @@
                 "roleName": "ARO v4 Development First Party Managed Resource Group"
             }
         },
-        {
-            "name": "f3fe7bc1-0ef9-4681-a68c-c1fa285d6128",
-            "apiVersion": "2017-09-01",
-            "type": "Microsoft.Authorization/roleDefinitions",
-            "properties": {
-                "assignableScopes": [
-                    "[concat('/subscriptions/', subscription().subscriptionId)]"
-                ],
-                "permissions": [
-                    {
-                        "actions": [
-                            "Microsoft.Network/virtualNetworks/join/action",
-                            "Microsoft.Network/virtualNetworks/subnets/join/action",
-                            "Microsoft.Network/virtualNetworks/subnets/read",
-                            "Microsoft.Network/virtualNetworks/subnets/write"
-                        ]
-                    }
-                ],
-                "roleName": "ARO v4 Development Subnet Contributor"
-            }
-        },
         {
             "name": "[guid(subscription().id, 'FP / ARO v4 FP Subscription')]",
             "apiVersion": "2017-09-01",

deploy/rp-development-nsg.json

@@ -0,0 +1,49 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+    "contentVersion": "1.0.0.0",
+    "resources": [
+        {
+            "properties": {
+                "securityRules": [
+                    {
+                        "properties": {
+                            "protocol": "Tcp",
+                            "sourcePortRange": "*",
+                            "destinationPortRange": "443",
+                            "sourceAddressPrefix": "*",
+                            "destinationAddressPrefix": "*",
+                            "access": "Allow",
+                            "priority": 120,
+                            "direction": "Inbound"
+                        },
+                        "name": "rp_in"
+                    },
+                    {
+                        "properties": {
+                            "protocol": "Tcp",
+                            "sourcePortRange": "*",
+                            "destinationPortRange": "22",
+                            "sourceAddressPrefix": "*",
+                            "destinationAddressPrefix": "*",
+                            "access": "Allow",
+                            "priority": 100,
+                            "direction": "Inbound"
+                        },
+                        "name": "ssh_in"
+                    }
+                ]
+            },
+            "name": "rp-nsg",
+            "type": "Microsoft.Network/networkSecurityGroups",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-07-01"
+        },
+        {
+            "properties": {},
+            "name": "rp-pe-nsg",
+            "type": "Microsoft.Network/networkSecurityGroups",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-07-01"
+        }
+    ]
+}

deploy/rp-development.json

@@ -11,6 +11,9 @@
         "domainName": {
             "type": "string"
         },
+        "fpServicePrincipalId": {
+            "type": "string"
+        },
         "keyvaultName": {
             "type": "string"
         },
@@ -72,6 +75,42 @@
             "location": "[resourceGroup().location]",
             "apiVersion": "2016-10-01"
         },
+        {
+            "properties": {
+                "addressSpace": {
+                    "addressPrefixes": [
+                        "10.0.0.0/8"
+                    ]
+                },
+                "subnets": [
+                    {
+                        "properties": {
+                            "addressPrefix": "10.0.0.0/24",
+                            "networkSecurityGroup": {
+                                "id": "[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-nsg')]",
+                                "tags": null
+                            }
+                        },
+                        "name": "rp-subnet"
+                    },
+                    {
+                        "properties": {
+                            "addressPrefix": "10.1.0.0/16",
+                            "networkSecurityGroup": {
+                                "id": "[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-pe-nsg')]",
+                                "tags": null
+                            },
+                            "privateEndpointNetworkPolicies": "Disabled"
+                        },
+                        "name": "rp-pe-subnet"
+                    }
+                ]
+            },
+            "name": "rp-vnet",
+            "type": "Microsoft.Network/virtualNetworks",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-07-01"
+        },
         {
             "kind": "GlobalDocumentDB",
             "properties": {
@@ -83,8 +122,7 @@
                         "locationName": "[resourceGroup().location]"
                     }
                 ],
-                "databaseAccountOfferType": "Standard",
-                "disableKeyBasedMetadataWriteAccess": true
+                "databaseAccountOfferType": "Standard"
             },
             "name": "[parameters('databaseAccountName')]",
             "type": "Microsoft.DocumentDB/databaseAccounts",
@@ -94,90 +132,6 @@
             },
             "apiVersion": "2019-08-01"
         },
-        {
-            "properties": {
-                "resource": {
-                    "id": "ARO"
-                },
-                "options": {}
-            },
-            "name": "[concat(parameters('databaseAccountName'), '/ARO')]",
-            "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases",
-            "apiVersion": "2019-08-01",
-            "dependsOn": [
-                "[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]"
-            ]
-        },
-        {
-            "properties": {
-                "resource": {
-                    "id": "AsyncOperations",
-                    "partitionKey": {
-                        "paths": [
-                            "/id"
-                        ],
-                        "kind": "Hash"
-                    },
-                    "defaultTtl": 604800
-                },
-                "options": {}
-            },
-            "name": "[concat(parameters('databaseAccountName'), '/ARO/AsyncOperations')]",
-            "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers",
-            "apiVersion": "2019-08-01",
-            "dependsOn": [
-                "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), 'ARO')]"
-            ]
-        },
-        {
-            "properties": {
-                "resource": {
-                    "id": "OpenShiftClusters",
-                    "partitionKey": {
-                        "paths": [
-                            "/partitionKey"
-                        ],
-                        "kind": "Hash"
-                    },
-                    "uniqueKeyPolicy": {
-                        "uniqueKeys": [
-                            {
-                                "paths": [
-                                    "/key"
-                                ]
-                            }
-                        ]
-                    }
-                },
-                "options": {}
-            },
-            "name": "[concat(parameters('databaseAccountName'), '/ARO/OpenShiftClusters')]",
-            "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers",
-            "apiVersion": "2019-08-01",
-            "dependsOn": [
-                "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), 'ARO')]"
-            ]
-        },
-        {
-            "properties": {
-                "resource": {
-                    "id": "Subscriptions",
-                    "partitionKey": {
-                        "paths": [
-                            "/id"
-                        ],
-                        "kind": "Hash"
-                    }
-                },
-                "options": {}
-            },
-            "name": "[concat(parameters('databaseAccountName'), '/ARO/Subscriptions')]",
-            "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers",
-            "apiVersion": "2019-08-01",
-            "dependsOn": [
-                "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), 'ARO')]"
-            ]
-        },
         {
             "name": "[guid(resourceGroup().id, 'RP / Reader')]",
             "type": "Microsoft.Authorization/roleAssignments",
@@ -189,6 +143,17 @@
             },
             "apiVersion": "2018-09-01-preview"
         },
+        {
+            "name": "[guid(resourceGroup().id, 'FP / Network Contributor')]",
+            "type": "Microsoft.Authorization/roleAssignments",
+            "properties": {
+                "scope": "[resourceGroup().id]",
+                "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7')]",
+                "principalId": "[parameters('fpServicePrincipalId')]",
+                "principalType": "ServicePrincipal"
+            },
+            "apiVersion": "2018-09-01-preview"
+        },
         {
             "name": "[concat(parameters('databaseAccountName'), '/Microsoft.Authorization/', guid(resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName')), 'RP / DocumentDB Account Contributor'))]",
             "type": "Microsoft.DocumentDB/databaseAccounts/providers/roleAssignments",
@@ -204,12 +169,12 @@
             ]
         },
         {
-            "name": "[concat(parameters('domainName'), '/Microsoft.Authorization/', guid(resourceId('Microsoft.Network/dnsZones', parameters('domainName')), 'RP / DNS Zone Contributor'))]",
+            "name": "[concat(parameters('domainName'), '/Microsoft.Authorization/', guid(resourceId('Microsoft.Network/dnsZones', parameters('domainName')), 'FP / DNS Zone Contributor'))]",
             "type": "Microsoft.Network/dnsZones/providers/roleAssignments",
             "properties": {
                 "scope": "[resourceId('Microsoft.Network/dnsZones', parameters('domainName'))]",
                 "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314')]",
-                "principalId": "[parameters('rpServicePrincipalId')]",
+                "principalId": "[parameters('fpServicePrincipalId')]",
                 "principalType": "ServicePrincipal"
             },
             "apiVersion": "2018-09-01-preview",

deploy/rp-production-nsg.json

@@ -2,12 +2,6 @@
     "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
     "contentVersion": "1.0.0.0",
     "resources": [
-        {
-            "name": "rp-identity",
-            "location": "[resourceGroup().location]",
-            "type": "Microsoft.ManagedIdentity/userAssignedIdentities",
-            "apiVersion": "2018-11-30"
-        },
         {
             "properties": {
                 "securityRules": [
@@ -30,6 +24,19 @@
             "type": "Microsoft.Network/networkSecurityGroups",
             "location": "[resourceGroup().location]",
             "apiVersion": "2019-07-01"
+        },
+        {
+            "properties": {},
+            "name": "rp-pe-nsg",
+            "type": "Microsoft.Network/networkSecurityGroups",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-07-01"
+        },
+        {
+            "name": "rp-identity",
+            "location": "[resourceGroup().location]",
+            "type": "Microsoft.ManagedIdentity/userAssignedIdentities",
+            "apiVersion": "2018-11-30"
         }
     ],
     "outputs": {

deploy/rp-production-parameters.json

@@ -14,6 +14,9 @@
         "extraKeyvaultAccessPolicies": {
             "value": []
         },
+        "fpServicePrincipalId": {
+            "value": ""
+        },
         "keyvaultName": {
             "value": ""
         },

deploy/rp-production.json

@@ -29,6 +29,9 @@
             "type": "array",
             "defaultValue": []
         },
+        "fpServicePrincipalId": {
+            "type": "string"
+        },
         "keyvaultName": {
             "type": "string"
         },
@@ -49,45 +52,6 @@
         }
     },
     "resources": [
-        {
-            "properties": {
-                "addressSpace": {
-                    "addressPrefixes": [
-                        "10.0.0.0/8"
-                    ]
-                },
-                "subnets": [
-                    {
-                        "properties": {
-                            "addressPrefix": "10.0.0.0/24",
-                            "networkSecurityGroup": {
-                                "id": "[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-nsg')]",
-                                "tags": null
-                            },
-                            "serviceEndpoints": [
-                                {
-                                    "service": "Microsoft.KeyVault",
-                                    "locations": [
-                                        "*"
-                                    ]
-                                },
-                                {
-                                    "service": "Microsoft.AzureCosmosDB",
-                                    "locations": [
-                                        "*"
-                                    ]
-                                }
-                            ]
-                        },
-                        "name": "rp-subnet"
-                    }
-                ]
-            },
-            "name": "rp-vnet",
-            "type": "Microsoft.Network/virtualNetworks",
-            "location": "[resourceGroup().location]",
-            "apiVersion": "2019-07-01"
-        },
         {
             "sku": {
                 "name": "Standard"
@@ -289,6 +253,56 @@
             "location": "[resourceGroup().location]",
             "apiVersion": "2016-10-01"
         },
+        {
+            "properties": {
+                "addressSpace": {
+                    "addressPrefixes": [
+                        "10.0.0.0/8"
+                    ]
+                },
+                "subnets": [
+                    {
+                        "properties": {
+                            "addressPrefix": "10.0.0.0/24",
+                            "networkSecurityGroup": {
+                                "id": "[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-nsg')]",
+                                "tags": null
+                            },
+                            "serviceEndpoints": [
+                                {
+                                    "service": "Microsoft.KeyVault",
+                                    "locations": [
+                                        "*"
+                                    ]
+                                },
+                                {
+                                    "service": "Microsoft.AzureCosmosDB",
+                                    "locations": [
+                                        "*"
+                                    ]
+                                }
+                            ]
+                        },
+                        "name": "rp-subnet"
+                    },
+                    {
+                        "properties": {
+                            "addressPrefix": "10.1.0.0/16",
+                            "networkSecurityGroup": {
+                                "id": "[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-pe-nsg')]",
+                                "tags": null
+                            },
+                            "privateEndpointNetworkPolicies": "Disabled"
+                        },
+                        "name": "rp-pe-subnet"
+                    }
+                ]
+            },
+            "name": "rp-vnet",
+            "type": "Microsoft.Network/virtualNetworks",
+            "location": "[resourceGroup().location]",
+            "apiVersion": "2019-07-01"
+        },
         {
             "kind": "GlobalDocumentDB",
             "properties": {
@@ -324,12 +338,15 @@
         {
             "properties": {
                 "resource": {
-                    "id": "ARO"
+                    "id": "['ARO']"
                 },
-                "options": {}
+                "options": {
+                    "x-ms-offer-throughput": "400"
+                }
             },
-            "name": "[concat(parameters('databaseAccountName'), '/ARO')]",
+            "name": "[concat(parameters('databaseAccountName'), '/', 'ARO')]",
             "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases",
+            "location": "[resourceGroup().location]",
             "apiVersion": "2019-08-01",
             "dependsOn": [
                 "[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]"
@@ -349,11 +366,12 @@
                 },
                 "options": {}
             },
-            "name": "[concat(parameters('databaseAccountName'), '/ARO/AsyncOperations')]",
+            "name": "[concat(parameters('databaseAccountName'), '/', 'ARO', '/AsyncOperations')]",
             "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers",
+            "location": "[resourceGroup().location]",
             "apiVersion": "2019-08-01",
             "dependsOn": [
-                "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), 'ARO')]"
+                "[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]"
             ]
         },
         {
@@ -378,11 +396,12 @@
                 },
                 "options": {}
             },
-            "name": "[concat(parameters('databaseAccountName'), '/ARO/OpenShiftClusters')]",
+            "name": "[concat(parameters('databaseAccountName'), '/', 'ARO', '/OpenShiftClusters')]",
             "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers",
+            "location": "[resourceGroup().location]",
             "apiVersion": "2019-08-01",
             "dependsOn": [
-                "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), 'ARO')]"
+                "[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]"
             ]
         },
         {
@@ -398,11 +417,12 @@
                 },
                 "options": {}
             },
-            "name": "[concat(parameters('databaseAccountName'), '/ARO/Subscriptions')]",
+            "name": "[concat(parameters('databaseAccountName'), '/', 'ARO', '/Subscriptions')]",
             "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers",
+            "location": "[resourceGroup().location]",
             "apiVersion": "2019-08-01",
             "dependsOn": [
-                "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), 'ARO')]"
+                "[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]"
             ]
         },
         {
@@ -416,6 +436,17 @@
             },
             "apiVersion": "2018-09-01-preview"
         },
+        {
+            "name": "[guid(resourceGroup().id, 'FP / Network Contributor')]",
+            "type": "Microsoft.Authorization/roleAssignments",
+            "properties": {
+                "scope": "[resourceGroup().id]",
+                "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7')]",
+                "principalId": "[parameters('fpServicePrincipalId')]",
+                "principalType": "ServicePrincipal"
+            },
+            "apiVersion": "2018-09-01-preview"
+        },
         {
             "name": "[concat(parameters('databaseAccountName'), '/Microsoft.Authorization/', guid(resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName')), 'RP / DocumentDB Account Contributor'))]",
             "type": "Microsoft.DocumentDB/databaseAccounts/providers/roleAssignments",
@@ -431,12 +462,12 @@
             ]
         },
         {
-            "name": "[concat(parameters('domainName'), '/Microsoft.Authorization/', guid(resourceId('Microsoft.Network/dnsZones', parameters('domainName')), 'RP / DNS Zone Contributor'))]",
+            "name": "[concat(parameters('domainName'), '/Microsoft.Authorization/', guid(resourceId('Microsoft.Network/dnsZones', parameters('domainName')), 'FP / DNS Zone Contributor'))]",
             "type": "Microsoft.Network/dnsZones/providers/roleAssignments",
             "properties": {
                 "scope": "[resourceId('Microsoft.Network/dnsZones', parameters('domainName'))]",
                 "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314')]",
-                "principalId": "[parameters('rpServicePrincipalId')]",
+                "principalId": "[parameters('fpServicePrincipalId')]",
                 "principalType": "ServicePrincipal"
             },
             "apiVersion": "2018-09-01-preview",

docs/deploy-development-rp.md

@@ -1,6 +1,6 @@
 # Deploy development RP
 
-## Prerequisites - all
+## Prerequisites
 
 1. Install [Go 1.13](https://golang.org/dl) or later, if you haven't already.
 
@@ -8,9 +8,12 @@
    you don't have one installed already.  The `az` client supports Python 2.7
    and Python 3.5+.  A recent Python 3.x version is recommended.
 
-1. Install the
-   [`az`](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) client,
-   if you haven't already.
+1. Install the [Azure
+   CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli), if you
+   haven't already.
+
+1. Install [OpenVPN](https://openvpn.net/community-downloads), if you haven't
+   already.
 
 1. Log in to Azure:
 
@@ -21,23 +24,19 @@
 1. Git clone this repository to your local machine:
 
    ```
-   go get github.com/Azure/ARO-RP/...
+   go get -u github.com/Azure/ARO-RP/...
+   cd ${GOPATH:-$HOME/go}/src/github.com/Azure/ARO-RP
    ```
 
-1. You will need the `Contributor` and `User Access Administrator` roles on your
-   subscription.
+1. Non-Red Hat ARO engineering: if you don't have access to a shared development
+   environment and secrets, follow [prepare a shared RP development
+   environment](docs/prepare-a-shared-rp-development-environment.md).
 
+1. Place your shared development environment secrets in `secrets` (Red Hat ARO
+   engineering: run `make secrets`).
 
-## Configuration - Red Hat ARO engineering
-
-1. Fetch the development environment secrets:
-
-   ```
-   make secrets
-   ```
-
-1. Edit and source your environment file.  The required environment variable
-   configuration is documented immediately below:
+1. Copy, edit (if necessary) and source your environment file.  The required
+   environment variable configuration is documented immediately below:
 
    ```
    cp env.example env
@@ -45,150 +44,30 @@
    . ./env
    ```
 
-   * LOCATION: Location of the resource group where the development RP will run
-     (default: `eastus`).
+   * LOCATION: Location of the shared RP development environment (default:
+     `eastus`).
 
-   * RESOURCEGROUP: Name of the new resource group into which you will deploy
-     your RP assets.
-
-   * RP_MODE: Set to `development` to enable the RP to read its development
-     configuration, and the `az aro` client to connect to the development RP.
-
-
-## Configuration - non-Red Hat ARO engineering
-
-1. Edit and source your environment file.  The required environment variable
-   configuration is documented immediately below:
+1. Create your own RP database:
 
    ```
-   cp env.example env
-   vi env
-   . ./env
-   ```
-
-   * LOCATION: Location of the resource group where the development RP will run
-     (default: `eastus`).
-
-   * RESOURCEGROUP: Name of the new resource group into which you will deploy
-     your RP assets.
-
-   * RP_MODE: Set to `development` to enable the RP to read its development
-     configuration, and the `az aro` client to connect to the development RP.
-
-   * AZURE_TENANT_ID: Azure tenant UUID.
-
-   * AZURE_SUBSCRIPTION_ID: Azure subscription UUID.
-
-   * AZURE_ARM_CLIENT_{ID,SECRET}: Credentials of an AAD application which fakes
-     up the ARM layer.
-
-     Later it will be granted:
-
-     * `User Access Administrator` on your subscription.
-
-   * AZURE_FP_CLIENT_ID: Client ID of an AAD application which fakes up the
-     first party application.
-
-     Later it will be granted:
-
-     * `ARO v4 FP Subscription` on your subscription.
-
-     This application requires client certificate authentication to be enabled.
-     A suitable key/certificate file can be generated using the following helper
-     utility; then configure it in AAD.
-
-     ```
-     go run ./hack/genkey -extKeyUsage client firstparty-development
-     ```
-
-   * AZURE_CLIENT_{ID,SECRET}: Credentials of an AAD application which fakes up
-     the RP identity.
-
-     Later it will be granted:
-
-     * `Reader` on RESOURCEGROUP.
-     * `Secrets / Get` on the key vault in RESOURCEGROUP.
-     * `DocumentDB Account Contributor` on the CosmosDB resource in RESOURCEGROUP.
-     * `DNS Zone Contributor` on the DNS zone in RESOURCEGROUP.
-
-   * PULL_SECRET: A cluster pull secret retrieved from [Red Hat OpenShift
-     Cluster
-     Manager](https://cloud.redhat.com/openshift/install/azure/installer-provisioned)
-
-   * ADMIN_OBJECT_ID: AAD object ID (e.g. an AAD group, or your AAD user) for
-     key vault admin(s)
-
-   * DOMAIN_RESOURCEGROUP, DOMAIN_NAME: Resource group and name of a publicly
-     resolvable parent DNS zone resource in your Azure subscription.
-
-1. Set up the RP role definitions and assignments in your Azure subscription.
-   This mimics the RBAC that ARM sets up.  With at least `User Access
-   Administrator` permissions on your subscription, do:
-
-   ```
-   az deployment create \
-     -l $LOCATION \
-     --template-file deploy/rbac-development.json \
-     --parameters \
-       "armServicePrincipalId=$ARM_SERVICEPRINCIPAL_ID" \
-       "fpServicePrincipalId=$FP_SERVICEPRINCIPAL_ID"
-   ```
-
-1. Create an RP serving key/certificate.  A suitable key/certificate file
-   can be generated using the following helper utility:
-
-   ```
-   go run ./hack/genkey localhost
-   ```
-
-
-## Deploy development RP - all
-
-1. Create the resource group and deploy the RP resources:
-
-   ```
-   az group create -g "$RESOURCEGROUP" -l "$LOCATION"
-
    az group deployment create \
      -g "$RESOURCEGROUP" \
-     --template-file deploy/rp-development.json \
+     -n "databases-development-$USER" \
+     --template-file deploy/databases-development.json \
      --parameters \
-       "adminObjectId=$ADMIN_OBJECT_ID" \
        "databaseAccountName=$COSMOSDB_ACCOUNT" \
-       "domainName=$DOMAIN" \
-       "keyvaultName=$KEYVAULT_NAME" \
-       "rpServicePrincipalId=$SERVICEPRINCIPAL_ID"
-   ```
-
-1. Load the keys/certificates into the key vault:
-
-   ```
-   az keyvault certificate import \
-     --vault-name "$KEYVAULT_NAME" \
-     --name rp-firstparty \
-     --file "$FP_KEYFILE"
-   az keyvault certificate import \
-     --vault-name "$KEYVAULT_NAME" \
-     --name rp-server \
-     --file "$KEYFILE"
-   ```
-
-1. Create nameserver records in the parent DNS zone:
-
-   ```
-   az network dns record-set ns create --resource-group "$DOMAIN_RESOURCEGROUP" --zone "$(cut -d. -f2- <<<"$DOMAIN")" --name "$(cut -d. -f1 <<<"$DOMAIN")"
-
-   for ns in "$(az network dns zone show --resource-group "$RESOURCEGROUP" --name "$DOMAIN" --query nameServers -o tsv)"; do
-     az network dns record-set ns add-record \
-       --resource-group "$DOMAIN_RESOURCEGROUP" \
-       --zone "$(cut -d. -f2- <<<"$DOMAIN")" \
-       --record-set-name "$(cut -d. -f1 <<<"$DOMAIN")" \
-       --nsdname "$ns"
-   done
+       "databaseName=$DATABASE_NAME" \
+     >/dev/null
    ```
 
 
-## Running the RP and creating a cluster
+## Run the RP and create a cluster
+
+1. Source your environment file.
+
+   ```
+   . ./env
+   ```
 
 1. Run the RP
 
@@ -201,7 +80,7 @@
 
    ```
    curl -k -X PUT \
-     -H 'Content-Type: application/json'
+     -H 'Content-Type: application/json' \
      -d '{"state": "Registered", "properties": {"tenantId": "'"$AZURE_TENANT_ID"'"}}' \
      "https://localhost:8443/subscriptions/$AZURE_SUBSCRIPTION_ID?api-version=2.0"
    ```
@@ -244,11 +123,12 @@
   ```
   hack/get-admin-kubeconfig.sh "/subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$RESOURCEGROUP/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER"
   export KUBECONFIG=admin.kubeconfig
-  oc version
   ```
 
 * "SSH" to a cluster node:
 
+  * First, get the admin kubeconfig and `export KUBECONFIG` as detailed above.
+
   ```
-  hack/ssh.sh [aro-master-0]
+  hack/ssh.sh [aro-master-{0,1,2}]
   ```

docs/prepare-a-shared-rp-development-environment.md

@@ -0,0 +1,346 @@
+# Prepare a shared RP development environment
+
+Follow these steps to build a shared RP development environment and secrets
+file.  A single RP development environment can be shared across multiple
+developers and/or CI flows.  It may include multiple resource groups in multiple
+locations.
+
+## Prerequisites
+
+1. You will need `Contributor` and `User Access Administrator` roles on your
+   Azure subscription, as well as the ability to create and configure AAD
+   applications.
+
+1. You will need a publicly resolvable DNS Zone resource in your Azure
+   subscription.  Set PARENT_DOMAIN_NAME and PARENT_DOMAIN_RESOURCEGROUP to the name and
+   resource group of the DNS Zone resource:
+
+   ```
+   PARENT_DOMAIN_NAME=osadev.cloud
+   PARENT_DOMAIN_RESOURCEGROUP=dns
+   ```
+
+1. You will need an AAD object (this could be your AAD user, or an AAD group of
+   which you are a member) which will be able to administer certificates in the
+   development environment key vault(s).  Set ADMIN_OBJECT_ID to the object ID.
+
+   ```
+   ADMIN_OBJECT_ID="$(az ad group show -g Engineering --query objectId -o tsv)"
+   ```
+
+1. You will need the ARO RP-specific pull secret (ask one of the
+   @azure-red-hat-openshift GitHub team for this):
+
+   ```
+   PULL_SECRET=...
+   ```
+
+1. Install [Go 1.13](https://golang.org/dl) or later, if you haven't already.
+
+1. Install the [Azure
+   CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli), if you
+   haven't already.
+
+1. Log in to Azure:
+
+   ```
+   az login
+
+   AZURE_TENANT_ID=$(az account show --query tenantId -o tsv)
+   AZURE_SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+   ```
+
+1. Git clone this repository to your local machine:
+
+   ```
+   go get -u github.com/Azure/ARO-RP/...
+   cd ${GOPATH:-$HOME/go}/src/github.com/Azure/ARO-RP
+   ```
+
+1. Prepare the secrets directory:
+
+   ```
+   mkdir -p secrets
+   ```
+
+
+## AAD applications
+
+1. Create an AAD application which will fake up the ARM layer:
+
+   ```
+   AZURE_ARM_CLIENT_SECRET="$(uuidgen)"
+   AZURE_ARM_CLIENT_ID="$(az ad app create \
+     --display-name aro-v4-arm-shared \
+     --end-date '2299-12-31T11:59:59+00:00' \
+     --identifier-uris "https://$(uuidgen)/" \
+     --key-type password \
+     --password "$AZURE_ARM_CLIENT_SECRET" \
+     --query appId \
+     -o tsv)"
+   az ad sp create --id "$AZURE_ARM_CLIENT_ID" >/dev/null
+   ```
+
+   Later this application will be granted:
+
+   * `User Access Administrator` on your subscription.
+
+1. Create an AAD application which will fake up the first party application.
+
+   This application requires client certificate authentication to be enabled.  A
+   suitable key/certificate file can be generated using the following helper
+   utility:
+
+   ```
+   go run ./hack/genkey -client firstparty-development
+   mv firstparty-development.* secrets
+   ```
+
+   Now create the application:
+
+   ```
+   AZURE_FP_CLIENT_ID="$(az ad app create \
+     --display-name aro-v4-fp-shared \
+     --identifier-uris "https://$(uuidgen)/" \
+     --query appId \
+     -o tsv)"
+   az ad app credential reset \
+     --id "$AZURE_FP_CLIENT_ID" \
+     --cert "$(base64 -w0 <secrets/firstparty-development.crt)" >/dev/null
+   az ad sp create --id "$AZURE_FP_CLIENT_ID" >/dev/null
+   ```
+
+   Later this application will be granted:
+
+   * `ARO v4 FP Subscription` on your subscription.
+   * `DNS Zone Contributor` on the DNS zone in RESOURCEGROUP.
+   * `Network Contributor` on RESOURCEGROUP.
+
+1. Create an AAD application which will fake up the RP identity.
+
+   ```
+   AZURE_CLIENT_SECRET="$(uuidgen)"
+   AZURE_CLIENT_ID="$(az ad app create \
+     --display-name aro-v4-rp-shared \
+     --end-date '2299-12-31T11:59:59+00:00' \
+     --identifier-uris "https://$(uuidgen)/" \
+     --key-type password \
+     --password "$AZURE_CLIENT_SECRET" \
+     --query appId \
+     -o tsv)"
+   az ad sp create --id "$AZURE_CLIENT_ID" >/dev/null
+   ```
+
+   Later this application will be granted:
+
+   * `Reader` on RESOURCEGROUP.
+   * `Secrets / Get` on the key vault in RESOURCEGROUP.
+   * `DocumentDB Account Contributor` on the CosmosDB resource in RESOURCEGROUP.
+
+1. Set up the RP role definitions and subscription role assignments in your
+   Azure subscription. This mimics the RBAC that ARM sets up.  With at least
+   `User Access Administrator` permissions on your subscription, do:
+
+   ```
+   az deployment create \
+     -l eastus \
+     --template-file deploy/rbac-development.json \
+     --parameters \
+       "armServicePrincipalId=$(az ad sp list --all --query "[?appId=='$AZURE_ARM_CLIENT_ID'].objectId" -o tsv)" \
+       "fpServicePrincipalId=$(az ad sp list --all --query "[?appId=='$AZURE_FP_CLIENT_ID'].objectId" -o tsv)" \
+     >/dev/null
+   ```
+
+
+## Certificates
+
+1. Create the VPN CA key/certificate.  A suitable key/certificate file can be
+   generated using the following helper utility:
+
+   ```
+   go run ./hack/genkey -ca vpn-ca
+   mv vpn-ca.* secrets
+   ```
+
+1. Create the VPN client key/certificate.  A suitable key/certificate file can be
+   generated using the following helper utility:
+
+   ```
+   go run ./hack/genkey -client -keyFile secrets/vpn-ca.key -certFile secrets/vpn-ca.crt vpn-client
+   mv vpn-client.* secrets
+   ```
+
+1. Create the proxy serving key/certificate.  A suitable key/certificate file
+   can be generated using the following helper utility:
+
+   ```
+   go run ./hack/genkey proxy
+   mv proxy.* secrets
+   ```
+
+1. Create the proxy client key/certificate.  A suitable key/certificate file can
+   be generated using the following helper utility:
+
+   ```
+   go run ./hack/genkey -client proxy-client
+   mv proxy-client.* secrets
+   ```
+
+1. Create the proxy ssh key/certificate.  A suitable key/certificate file can
+   be generated using the following helper utility:
+
+   ```
+   ssh-keygen -f secrets/proxy_id_rsa -N ''
+   ```
+
+1. Create an RP serving key/certificate.  A suitable key/certificate file
+   can be generated using the following helper utility:
+
+   ```
+   go run ./hack/genkey localhost
+   mv localhost.* secrets
+   ```
+
+
+# Environment file
+
+1. Choose the resource group prefix.  The resource group location will be
+   appended to the prefix to make the resource group name.
+
+   ```
+   RESOURCEGROUP_PREFIX=v4
+   ```
+
+1. Choose the proxy domain name label.  This final proxy hostname will be of the
+   form `vm0.$PROXY_DOMAIN_NAME_LABEL.$LOCATION.cloudapp.azure.com`.
+
+   ```
+   PROXY_DOMAIN_NAME_LABEL=aroproxy
+   ```
+
+1. Create the secrets/env file:
+
+   ```
+   cat >secrets/env <<EOF
+   export AZURE_TENANT_ID='$AZURE_TENANT_ID'
+   export AZURE_SUBSCRIPTION_ID='$AZURE_SUBSCRIPTION_ID'
+   export AZURE_ARM_CLIENT_ID='$AZURE_ARM_CLIENT_ID'
+   export AZURE_ARM_CLIENT_SECRET='$AZURE_ARM_CLIENT_SECRET'
+   export AZURE_FP_CLIENT_ID='$AZURE_FP_CLIENT_ID'
+   export AZURE_CLIENT_ID='$AZURE_CLIENT_ID'
+   export AZURE_CLIENT_SECRET='$AZURE_CLIENT_SECRET'
+   export RESOURCEGROUP="$RESOURCEGROUP_PREFIX-\$LOCATION"
+   export PROXY_HOSTNAME="vm0.$PROXY_DOMAIN_NAME_LABEL.\$LOCATION.cloudapp.azure.com"
+   export DATABASE_NAME="\$USER"
+   export RP_MODE=development
+   export PULL_SECRET='$PULL_SECRET'
+   COSMOSDB_ACCOUNT="\$RESOURCEGROUP"
+   EOF
+   ```
+
+
+## Deploy shared RP development environment (once per location)
+
+1. Copy, edit (if necessary) and source your environment file.  The required
+   environment variable configuration is documented immediately below:
+
+   ```
+   cp env.example env
+   vi env
+   . ./env
+   ```
+
+   * LOCATION: Location of the shared RP development environment (default:
+     `eastus`).
+
+1. Create the resource group and deploy the RP resources:
+
+   ```
+   DOMAIN_NAME="$RESOURCEGROUP"
+   KEYVAULT_NAME="$RESOURCEGROUP"
+
+   az group create -g "$RESOURCEGROUP" -l "$LOCATION"
+
+   az group deployment create \
+     -g "$RESOURCEGROUP" \
+     --template-file deploy/rp-development-nsg.json \
+     >/dev/null
+
+   az group deployment create \
+     -g "$RESOURCEGROUP" \
+     --template-file deploy/rp-development.json \
+     --parameters \
+       "adminObjectId=$ADMIN_OBJECT_ID" \
+       "databaseAccountName=$COSMOSDB_ACCOUNT" \
+       "domainName=$DOMAIN_NAME.$PARENT_DOMAIN_NAME" \
+       "fpServicePrincipalId=$(az ad sp list --all --query "[?appId=='$AZURE_FP_CLIENT_ID'].objectId" -o tsv)" \
+       "keyvaultName=$KEYVAULT_NAME" \
+       "rpServicePrincipalId=$(az ad sp list --all --query "[?appId=='$AZURE_CLIENT_ID'].objectId" -o tsv)" \
+     >/dev/null
+
+   az group deployment create \
+     -g "$RESOURCEGROUP" \
+     --template-file deploy/env-development.json \
+     --parameters \
+       "proxyCert=$(base64 -w0 <secrets/proxy.crt)" \
+       "proxyClientCert=$(base64 -w0 <secrets/proxy-client.crt)" \
+       "proxyDomainNameLabel=$(cut -d. -f2 <<<$PROXY_HOSTNAME)" \
+       "proxyImage=arosvc.azurecr.io/proxy:latest" \
+       "proxyImageAuth=$(jq -r '.auths["arosvc.azurecr.io"].auth' <<<$PULL_SECRET)" \
+       "proxyKey=$(base64 -w0 <secrets/proxy.key)" \
+       "sshPublicKey=$(<secrets/proxy_id_rsa.pub)" \
+       "vpnCACertificate=$(base64 -w0 <secrets/vpn-ca.crt)" \
+     >/dev/null
+   ```
+
+1. Load the keys/certificates into the key vault:
+
+   ```
+   az keyvault certificate import \
+     --vault-name "$KEYVAULT_NAME" \
+     --name rp-firstparty \
+     --file secrets/firstparty-development.pem \
+     >/dev/null
+   az keyvault certificate import \
+     --vault-name "$KEYVAULT_NAME" \
+     --name rp-server \
+     --file secrets/localhost.pem \
+     >/dev/null
+   ```
+
+1. Create nameserver records in the parent DNS zone:
+
+   ```
+   az network dns record-set ns create \
+     --resource-group "$PARENT_DOMAIN_RESOURCEGROUP" \
+     --zone "$PARENT_DOMAIN_NAME" \
+     --name "$DOMAIN_NAME" \
+     >/dev/null
+
+   for ns in $(az network dns zone show \
+     --resource-group "$RESOURCEGROUP" \
+     --name "$DOMAIN_NAME.$PARENT_DOMAIN_NAME" \
+     --query nameServers -o tsv); do
+     az network dns record-set ns add-record \
+       --resource-group "$PARENT_DOMAIN_RESOURCEGROUP" \
+       --zone "$PARENT_DOMAIN_NAME" \
+       --record-set-name "$DOMAIN_NAME" \
+       --nsdname "$ns" \
+       >/dev/null
+   done
+   ```
+
+1. Store the VPN client configuration:
+
+   ```
+   curl -so vpnclientconfiguration.zip "$(az network vnet-gateway vpn-client generate \
+     -g "$RESOURCEGROUP" \
+     -n dev-vpn \
+     -o tsv)"
+   export CLIENTCERTIFICATE="$(openssl x509 -inform der -in secrets/vpn-client.crt)"
+   export PRIVATEKEY="$(openssl rsa -inform der -in secrets/vpn-client.key)"
+   unzip -qc vpnclientconfiguration.zip 'OpenVPN\\vpnconfig.ovpn' \
+     | envsubst \
+     | grep -v '^log ' >"secrets/vpn-$LOCATION.ovpn"
+   rm vpnclientconfiguration.zip
+   ```

docs/using-az-aro.md

@@ -79,14 +79,29 @@ cluster:
    attached.  Your cluster will be deployed into these subnets.
 
    ```
-   RESOURCEGROUP=cluster-rg
+   LOCATION=eastus
+   RESOURCEGROUP="v4-$LOCATION"
    CLUSTER=cluster
 
-   az group create -g "$RESOURCEGROUP" -l eastus
-   az network vnet create -g "$RESOURCEGROUP" -n vnet --address-prefixes 10.0.0.0/9
+   az group create -g "$RESOURCEGROUP" -l $LOCATION
+   az network vnet create \
+     -g "$RESOURCEGROUP" \
+     -n dev-vnet \
+     --address-prefixes 10.0.0.0/9
    for subnet in "$CLUSTER-master" "$CLUSTER-worker"; do
-     az network vnet subnet create -g "$RESOURCEGROUP" --vnet-name vnet -n "$subnet" --address-prefixes 10.$((RANDOM & 127)).$((RANDOM & 255)).0/24
+     az network vnet subnet create \
+       -g "$RESOURCEGROUP" \
+       --vnet-name dev-vnet \
+       -n "$subnet" \
+       --address-prefixes 10.$((RANDOM & 127)).$((RANDOM & 255)).0/24 \
+       >/dev/null
    done
+   az network vnet subnet update \
+     -g "$RESOURCEGROUP" \
+     --vnet-name dev-vnet \
+     -n "$CLUSTER-master" \
+     --disable-private-link-service-network-policies true \
+     >/dev/null
    ```
 
 1. A cluster AAD application (client ID and secret) and service principal, or
@@ -104,10 +119,15 @@ cluster:
 1. Create a cluster:
 
    ```
-   az aro create -g "$RESOURCEGROUP" -n "$CLUSTER" --vnet vnet --master-subnet "$CLUSTER-master" --worker-subnet "$CLUSTER-worker"
+   az aro create \
+     -g "$RESOURCEGROUP" \
+     -n "$CLUSTER" \
+     --vnet dev-vnet \
+     --master-subnet "$CLUSTER-master" \
+     --worker-subnet "$CLUSTER-worker"
    ```
 
-   Note: cluster creation takes about 45 minutes.
+   Note: cluster creation takes about 35 minutes.
 
 1. Access the cluster console:
 
@@ -144,6 +164,6 @@ cluster:
 
    # (optionally)
    for subnet in "$CLUSTER-master" "$CLUSTER-worker"; do
-     az network vnet subnet delete -g "$RESOURCEGROUP" --vnet-name vnet -n "$subnet"
+     az network vnet subnet delete -g "$RESOURCEGROUP" --vnet-name dev-vnet -n "$subnet"
    done
    ```

env.example

@@ -1,33 +1,3 @@
 export LOCATION=eastus
-export RESOURCEGROUP=<resourcegroup>
-export RP_MODE=development
 
-# RH ARO engineering: uncomment the following line only and run `make secrets`:
-#. secrets/env
-
-# non-RH ARO engineering: uncomment from here:
-#export AZURE_TENANT_ID=<xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx>
-#export AZURE_SUBSCRIPTION_ID=<xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx>
-#export AZURE_ARM_CLIENT_ID=<xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx>
-#export AZURE_ARM_CLIENT_SECRET=<secret>
-#export AZURE_FP_CLIENT_ID=<xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx>
-#export AZURE_CLIENT_ID=<xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx>
-#export AZURE_CLIENT_SECRET=<secret>
-#export PULL_SECRET='<secret-json-object>'
-#
-#ADMIN_OBJECT_ID=<xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx>
-#DOMAIN_RESOURCEGROUP=<resourcegroup>
-#DOMAIN_NAME=<domain>
-#
-#ARM_SERVICEPRINCIPAL_ID="$(az ad sp list --all --query "[?appId=='$AZURE_ARM_CLIENT_ID'].objectId" -o tsv)"
-#FP_SERVICEPRINCIPAL_ID="$(az ad sp list --all --query "[?appId=='$AZURE_FP_CLIENT_ID'].objectId" -o tsv)"
-#SERVICEPRINCIPAL_ID="$(az ad sp list --all --query "[?appId=='$AZURE_CLIENT_ID'].objectId" -o tsv)"
-#
-#FP_KEYFILE=firstparty-development.pem
-#KEYFILE=localhost.pem
-
-# the following should hopefully need not be changed:
-
-COSMOSDB_ACCOUNT=$RESOURCEGROUP
-DOMAIN=$RESOURCEGROUP.$DOMAIN_NAME
-KEYVAULT_NAME=$RESOURCEGROUP
+. secrets/env

hack/db/db.go

@@ -28,7 +28,7 @@ func run(ctx context.Context, log *logrus.Entry) error {
 		return err
 	}
 
-	db, err := database.NewDatabase(ctx, env, "", "ARO")
+	db, err := database.NewDatabase(env, "")
 	if err != nil {
 		return err
 	}

hack/gendeploy/gendeploy.go

@@ -13,12 +13,17 @@ func run() error {
 		return err
 	}
 
-	err = deploy.GenerateNSGTemplate()
+	err = deploy.GenerateNSGTemplates()
 	if err != nil {
 		return err
 	}
 
-	return deploy.GenerateRPParameterTemplate()
+	err = deploy.GenerateRPParameterTemplate()
+	if err != nil {
+		return err
+	}
+
+	return deploy.GenerateDevelopmentTemplate()
 }
 
 func main() {

hack/genkey/genkey.go

@@ -5,21 +5,53 @@ package main
 
 import (
 	"bytes"
+	"crypto/rsa"
 	"crypto/x509"
 	"encoding/pem"
 	"flag"
+	"fmt"
 	"io/ioutil"
-	"strings"
+	"os"
 
 	"github.com/Azure/ARO-RP/pkg/util/tls"
 )
 
 var (
-	extKeyUsage = flag.String("extKeyUsage", "server", "server or client")
+	client   = flag.Bool("client", false, "generate client certificate")
+	ca       = flag.Bool("ca", false, "generate ca certificate")
+	keyFile  = flag.String("keyFile", "", `file containing signing key in der format (default "" - self-signed)`)
+	certFile = flag.String("certFile", "", `file containing signing certificate in der format (default "" - self-signed)`)
 )
 
 func run(name string) error {
-	key, cert, err := tls.GenerateKeyAndCertificate(name, strings.EqualFold(*extKeyUsage, "client"))
+	var signingKey *rsa.PrivateKey
+	var signingCert *x509.Certificate
+
+	if *keyFile != "" {
+		b, err := ioutil.ReadFile(*keyFile)
+		if err != nil {
+			return err
+		}
+
+		signingKey, err = x509.ParsePKCS1PrivateKey(b)
+		if err != nil {
+			return err
+		}
+	}
+
+	if *certFile != "" {
+		b, err := ioutil.ReadFile(*certFile)
+		if err != nil {
+			return err
+		}
+
+		signingCert, err = x509.ParseCertificate(b)
+		if err != nil {
+			return err
+		}
+	}
+
+	key, cert, err := tls.GenerateKeyAndCertificate(name, signingKey, signingCert, *ca, *client)
 	if err != nil {
 		return err
 	}
@@ -56,9 +88,20 @@ func run(name string) error {
 	return ioutil.WriteFile(name+".pem", buf.Bytes(), 0600)
 }
 
+func usage() {
+	fmt.Fprintf(flag.CommandLine.Output(), "usage: %s commonName\n", os.Args[0])
+	flag.PrintDefaults()
+}
+
 func main() {
+	flag.Usage = usage
 	flag.Parse()
 
+	if len(flag.Args()) != 1 {
+		flag.Usage()
+		os.Exit(2)
+	}
+
 	if err := run(flag.Arg(0)); err != nil {
 		panic(err)
 	}

hack/proxy/proxy.go

@@ -0,0 +1,153 @@
+package main
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"flag"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+
+	utillog "github.com/Azure/ARO-RP/pkg/util/log"
+)
+
+var (
+	certFile       = flag.String("certFile", "secrets/proxy.crt", "file containing server certificate")
+	keyFile        = flag.String("keyFile", "secrets/proxy.key", "file containing server key")
+	clientCertFile = flag.String("clientCertFile", "secrets/proxy-client.crt", "file containing client certificate")
+	subnet         = flag.String("subnet", "10.0.0.0/8", "allowed subnet")
+
+	gitCommit = "unknown"
+)
+
+func run() error {
+	_, subnet, err := net.ParseCIDR(*subnet)
+	if err != nil {
+		return err
+	}
+
+	b, err := ioutil.ReadFile(*clientCertFile)
+	if err != nil {
+		return err
+	}
+
+	clientCert, err := x509.ParseCertificate(b)
+	if err != nil {
+		return err
+	}
+
+	pool := x509.NewCertPool()
+	pool.AddCert(clientCert)
+
+	cert, err := ioutil.ReadFile(*certFile)
+	if err != nil {
+		return err
+	}
+
+	b, err = ioutil.ReadFile(*keyFile)
+	if err != nil {
+		return err
+	}
+
+	key, err := x509.ParsePKCS1PrivateKey(b)
+	if err != nil {
+		return err
+	}
+
+	l, err := tls.Listen("tcp", ":8443", &tls.Config{
+		Certificates: []tls.Certificate{
+			{
+				Certificate: [][]byte{
+					cert,
+				},
+				PrivateKey: key,
+			},
+		},
+		ClientCAs:  pool,
+		ClientAuth: tls.RequireAndVerifyClientCert,
+		CipherSuites: []uint16{
+			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+		},
+		PreferServerCipherSuites: true,
+		SessionTicketsDisabled:   true,
+		MinVersion:               tls.VersionTLS12,
+		CurvePreferences: []tls.CurveID{
+			tls.CurveP256,
+			tls.X25519,
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	return http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodConnect {
+			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+			return
+		}
+
+		ip, _, err := net.SplitHostPort(r.Host)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		if !subnet.Contains(net.ParseIP(ip)) {
+			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+			return
+		}
+
+		proxy(w, r)
+	}))
+}
+
+func proxy(w http.ResponseWriter, r *http.Request) {
+	c2, err := net.Dial("tcp", r.Host)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	hijacker, ok := w.(http.Hijacker)
+	if !ok {
+		http.Error(w, "hijacking not supported", http.StatusInternalServerError)
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+
+	c1, buf, err := hijacker.Hijack()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	go func() {
+		io.Copy(c2, buf)
+		c2.(*net.TCPConn).CloseWrite()
+	}()
+
+	io.Copy(c1, c2)
+	c1.(*tls.Conn).CloseWrite()
+}
+
+func main() {
+	log := utillog.GetLogger()
+
+	log.Printf("starting, git commit %s", gitCommit)
+
+	flag.Parse()
+
+	if err := run(); err != nil {
+		log.Fatal(err)
+	}
+}

hack/rhcos-sas/rhcos-sas.go

@@ -6,16 +6,21 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage"
+	mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage"
 	azstorage "github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
+	"github.com/Azure/go-autorest/autorest/date"
 	"github.com/openshift/installer/pkg/rhcos"
 
 	_ "github.com/Azure/ARO-RP/pkg/install"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage"
+	utillog "github.com/Azure/ARO-RP/pkg/util/log"
 )
 
 func run(ctx context.Context) error {
@@ -28,20 +33,28 @@ func run(ctx context.Context) error {
 		return err
 	}
 
-	accounts := storage.NewAccountsClient(subscriptionID)
-	accounts.Authorizer = authorizer
+	accounts := storage.NewAccountsClient(subscriptionID, authorizer)
 
-	keys, err := accounts.ListKeys(ctx, resourceGroup, accountName, "")
+	t := time.Now().UTC().Truncate(time.Second)
+
+	res, err := accounts.ListAccountSAS(ctx, resourceGroup, accountName, mgmtstorage.AccountSasParameters{
+		Services:               "b",
+		ResourceTypes:          "co",
+		Permissions:            "cr",
+		Protocols:              mgmtstorage.HTTPS,
+		SharedAccessStartTime:  &date.Time{Time: t},
+		SharedAccessExpiryTime: &date.Time{Time: t.Add(24 * time.Hour)},
+	})
 	if err != nil {
 		return err
 	}
 
-	storageClient, err := azstorage.NewBasicClient(accountName, *(*keys.Keys)[0].Value)
+	v, err := url.ParseQuery(*res.AccountSasToken)
 	if err != nil {
 		return err
 	}
 
-	blobService := storageClient.GetBlobService()
+	blobService := azstorage.NewAccountSASClient(accountName, v, azure.PublicCloud).GetBlobService()
 
 	c := blobService.GetContainerReference("rhcos")
 
@@ -92,7 +105,9 @@ func run(ctx context.Context) error {
 }
 
 func main() {
+	log := utillog.GetLogger()
+
 	if err := run(context.Background()); err != nil {
-		panic(err)
+		log.Fatal(err)
 	}
 }

hack/ssh.sh

@@ -1,12 +1,5 @@
 #!/bin/bash -e
 
-if [[ ! -e admin.kubeconfig ]]; then
-    echo "run hack/get-admin-kubeconfig.sh first" >&2
-    exit 1
-fi
-
-export KUBECONFIG=admin.kubeconfig
-
 while [[ $1 == -* ]]; do
   if [[ $1 == -- ]]; then
     shift

pkg/api/v20191231preview/openshiftcluster_validatedynamic.go

@@ -19,7 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/Azure/ARO-RP/pkg/api"
-	"github.com/Azure/ARO-RP/pkg/util/azureclient/authorization"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization"
 	utilpermissions "github.com/Azure/ARO-RP/pkg/util/permissions"
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )
@@ -143,7 +143,12 @@ func (dv *dynamicValidator) validateVnetPermissions(ctx context.Context, client
 		return err
 	}
 
-	permissions, err := client.ListForResource(ctx, vnetID)
+	r, err := azure.ParseResourceID(vnetID)
+	if err != nil {
+		return err
+	}
+
+	permissions, err := client.ListForResource(ctx, r.ResourceGroup, r.Provider, r.ResourceType, "", r.ResourceName)
 	if err != nil {
 		if err, ok := err.(autorest.DetailedError); ok {
 			if err.StatusCode == http.StatusNotFound {
@@ -211,6 +216,12 @@ func (dv *dynamicValidator) validateSubnet(ctx context.Context, path, typ, subne
 		return nil, err
 	}
 
+	if strings.EqualFold(dv.oc.Properties.MasterProfile.SubnetID, subnetID) {
+		if !strings.EqualFold(*s.PrivateLinkServiceNetworkPolicies, "Disabled") {
+			return nil, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidLinkedVNet, path, "The provided "+typ+" VM subnet '%s' is invalid: must have privateLinkServiceNetworkPolicies disabled.", subnetID)
+		}
+	}
+
 	if dv.oc.Properties.ProvisioningState == api.ProvisioningStateCreating {
 		if s.SubnetPropertiesFormat != nil &&
 			s.SubnetPropertiesFormat.NetworkSecurityGroup != nil {

pkg/backend/openshiftcluster/create.go

@@ -108,7 +108,7 @@ func (m *Manager) Create(ctx context.Context) error {
 				Name: m.doc.OpenShiftCluster.Properties.DomainName,
 			},
 			SSHKey:     sshkey.Type() + " " + base64.StdEncoding.EncodeToString(sshkey.Marshal()),
-			BaseDomain: m.env.DNS().Domain(),
+			BaseDomain: m.dns.Domain(),
 			Networking: &types.Networking{
 				MachineCIDR: ipnet.MustParseCIDR("127.0.0.0/8"), // dummy
 				NetworkType: "OpenShiftSDN",
@@ -192,7 +192,12 @@ func (m *Manager) Create(ctx context.Context) error {
 		return err
 	}
 
-	return install.NewInstaller(m.log, m.env, m.db, m.fpAuthorizer, r.SubscriptionID).Install(ctx, m.doc, installConfig, platformCreds, image)
+	i, err := install.NewInstaller(m.log, m.env, m.db, m.doc)
+	if err != nil {
+		return err
+	}
+
+	return i.Install(ctx, installConfig, platformCreds, image)
 }
 
 var rxRHCOS = regexp.MustCompile(`rhcos-((\d+)\.\d+\.\d{8})\d{4}\.\d+-azure\.x86_64\.vhd`)

pkg/backend/openshiftcluster/delete.go

@@ -15,8 +15,7 @@ import (
 
 func (m *Manager) Delete(ctx context.Context) error {
 	m.log.Printf("deleting dns")
-
-	err := m.env.DNS().Delete(ctx, m.doc.OpenShiftCluster)
+	err := m.dns.Delete(ctx, m.doc.OpenShiftCluster)
 	if err != nil {
 		return err
 	}
@@ -65,6 +64,12 @@ func (m *Manager) Delete(ctx context.Context) error {
 		return err
 	}
 
+	m.log.Print("deleting private endpoint")
+	err = m.privateendpoint.Delete(ctx, m.doc)
+	if err != nil {
+		return err
+	}
+
 	m.log.Printf("deleting resource group %s", m.doc.OpenShiftCluster.Properties.ResourceGroup)
 	return m.groups.DeleteAndWait(ctx, m.doc.OpenShiftCluster.Properties.ResourceGroup)
 }

pkg/backend/openshiftcluster/openshiftcluster.go

@@ -11,7 +11,9 @@ import (
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/database"
 	"github.com/Azure/ARO-RP/pkg/env"
-	"github.com/Azure/ARO-RP/pkg/util/azureclient/resources"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/resources"
+	"github.com/Azure/ARO-RP/pkg/util/dns"
+	"github.com/Azure/ARO-RP/pkg/util/privateendpoint"
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )
 
@@ -21,6 +23,9 @@ type Manager struct {
 	db           database.OpenShiftClusters
 	fpAuthorizer autorest.Authorizer
 
+	privateendpoint privateendpoint.Manager
+	dns             dns.Manager
+
 	groups resources.GroupsClient
 
 	subnets subnet.Manager
@@ -34,6 +39,11 @@ func NewManager(log *logrus.Entry, env env.Interface, db database.OpenShiftClust
 		return nil, err
 	}
 
+	localFPAuthorizer, err := env.FPAuthorizer(env.TenantID(), azure.PublicCloud.ResourceManagerEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
 	fpAuthorizer, err := env.FPAuthorizer(doc.OpenShiftCluster.Properties.ServicePrincipalProfile.TenantID, azure.PublicCloud.ResourceManagerEndpoint)
 	if err != nil {
 		return nil, err
@@ -45,6 +55,9 @@ func NewManager(log *logrus.Entry, env env.Interface, db database.OpenShiftClust
 		db:           db,
 		fpAuthorizer: fpAuthorizer,
 
+		privateendpoint: privateendpoint.NewManager(env, localFPAuthorizer),
+		dns:             dns.NewManager(env, localFPAuthorizer),
+
 		subnets: subnet.NewManager(r.SubscriptionID, fpAuthorizer),
 		groups:  resources.NewGroupsClient(r.SubscriptionID, fpAuthorizer),
 

pkg/backend/openshiftcluster/update.go

@@ -30,7 +30,12 @@ func find(xs interface{}, f func(int, int) bool) interface{} {
 }
 
 func (m *Manager) Update(ctx context.Context) error {
-	restConfig, err := restconfig.RestConfig(m.doc.OpenShiftCluster.Properties.AdminKubeconfig)
+	ip, err := m.privateendpoint.GetIP(ctx, m.doc)
+	if err != nil {
+		return err
+	}
+
+	restConfig, err := restconfig.RestConfig(ctx, m.env, m.doc, ip)
 	if err != nil {
 		return err
 	}

pkg/database/asyncoperations.go

@@ -4,7 +4,6 @@ package database
 // Licensed under the Apache License 2.0.
 
 import (
-	"context"
 	"fmt"
 	"net/http"
 	"strings"
@@ -26,7 +25,7 @@ type AsyncOperations interface {
 }
 
 // NewAsyncOperations returns a new AsyncOperations
-func NewAsyncOperations(ctx context.Context, uuid string, dbc cosmosdb.DatabaseClient, dbid, collid string) (AsyncOperations, error) {
+func NewAsyncOperations(uuid string, dbc cosmosdb.DatabaseClient, dbid, collid string) (AsyncOperations, error) {
 	collc := cosmosdb.NewCollectionClient(dbc, dbid)
 
 	return &asyncOperations{

pkg/database/database.go

@@ -4,7 +4,6 @@ package database
 // Licensed under the Apache License 2.0.
 
 import (
-	"context"
 	"crypto/tls"
 	"net/http"
 	"time"
@@ -24,8 +23,8 @@ type Database struct {
 }
 
 // NewDatabase returns a new Database
-func NewDatabase(ctx context.Context, env env.Interface, uuid, dbid string) (db *Database, err error) {
-	databaseAccount, masterKey := env.CosmosDB(ctx)
+func NewDatabase(env env.Interface, uuid string) (db *Database, err error) {
+	databaseAccount, masterKey := env.CosmosDB()
 
 	h := &codec.JsonHandle{
 		BasicHandle: codec.BasicHandle{
@@ -55,17 +54,17 @@ func NewDatabase(ctx context.Context, env env.Interface, uuid, dbid string) (db
 
 	db = &Database{}
 
-	db.AsyncOperations, err = NewAsyncOperations(ctx, uuid, dbc, dbid, "AsyncOperations")
+	db.AsyncOperations, err = NewAsyncOperations(uuid, dbc, env.DatabaseName(), "AsyncOperations")
 	if err != nil {
 		return nil, err
 	}
 
-	db.OpenShiftClusters, err = NewOpenShiftClusters(ctx, uuid, dbc, dbid, "OpenShiftClusters")
+	db.OpenShiftClusters, err = NewOpenShiftClusters(uuid, dbc, env.DatabaseName(), "OpenShiftClusters")
 	if err != nil {
 		return nil, err
 	}
 
-	db.Subscriptions, err = NewSubscriptions(ctx, uuid, dbc, dbid, "Subscriptions")
+	db.Subscriptions, err = NewSubscriptions(uuid, dbc, env.DatabaseName(), "Subscriptions")
 	if err != nil {
 		return nil, err
 	}

pkg/database/generate.go

@@ -3,5 +3,5 @@ package database
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../vendor/github.com/golang/mock/mockgen -destination=../util/mocks/mock_$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/$GOPACKAGE AsyncOperations,OpenShiftClusters,Subscriptions
-//go:generate go run ../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../util/mocks/mock_$GOPACKAGE/$GOPACKAGE.go
+//go:generate go run ../../vendor/github.com/golang/mock/mockgen -destination=../util/mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/$GOPACKAGE AsyncOperations,OpenShiftClusters,Subscriptions
+//go:generate go run ../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../util/mocks/$GOPACKAGE/$GOPACKAGE.go

pkg/database/openshiftclusters.go

@@ -4,7 +4,6 @@ package database
 // Licensed under the Apache License 2.0.
 
 import (
-	"context"
 	"fmt"
 	"net/http"
 	"strings"
@@ -34,7 +33,7 @@ type OpenShiftClusters interface {
 }
 
 // NewOpenShiftClusters returns a new OpenShiftClusters
-func NewOpenShiftClusters(ctx context.Context, uuid string, dbc cosmosdb.DatabaseClient, dbid, collid string) (OpenShiftClusters, error) {
+func NewOpenShiftClusters(uuid string, dbc cosmosdb.DatabaseClient, dbid, collid string) (OpenShiftClusters, error) {
 	collc := cosmosdb.NewCollectionClient(dbc, dbid)
 
 	triggers := []*cosmosdb.Trigger{

pkg/database/subscriptions.go

@@ -4,7 +4,6 @@ package database
 // Licensed under the Apache License 2.0.
 
 import (
-	"context"
 	"fmt"
 	"net/http"
 	"strings"
@@ -29,7 +28,7 @@ type Subscriptions interface {
 }
 
 // NewSubscriptions returns a new Subscriptions
-func NewSubscriptions(ctx context.Context, uuid string, dbc cosmosdb.DatabaseClient, dbid, collid string) (Subscriptions, error) {
+func NewSubscriptions(uuid string, dbc cosmosdb.DatabaseClient, dbid, collid string) (Subscriptions, error) {
 	collc := cosmosdb.NewCollectionClient(dbc, dbid)
 
 	triggers := []*cosmosdb.Trigger{

pkg/deploy/dev.go

@@ -0,0 +1,137 @@
+package deploy
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"encoding/json"
+	"io/ioutil"
+
+	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+	"github.com/Azure/go-autorest/autorest/to"
+
+	"github.com/Azure/ARO-RP/pkg/util/arm"
+)
+
+func GenerateDevelopmentTemplate() error {
+	t := &arm.Template{
+		Schema:         "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+		ContentVersion: "1.0.0.0",
+		Parameters:     map[string]*arm.TemplateParameter{},
+		Resources: []*arm.Resource{
+			{
+				Resource: &mgmtnetwork.PublicIPAddress{
+					Sku: &mgmtnetwork.PublicIPAddressSku{
+						Name: mgmtnetwork.PublicIPAddressSkuNameStandard,
+					},
+					PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{
+						PublicIPAllocationMethod: mgmtnetwork.Static,
+					},
+					Name:     to.StringPtr("dev-vpn-pip"),
+					Type:     to.StringPtr("Microsoft.Network/publicIPAddresses"),
+					Location: to.StringPtr("[resourceGroup().location]"),
+				},
+				APIVersion: apiVersions["network"],
+			},
+			{
+				Resource: &mgmtnetwork.VirtualNetwork{
+					VirtualNetworkPropertiesFormat: &mgmtnetwork.VirtualNetworkPropertiesFormat{
+						AddressSpace: &mgmtnetwork.AddressSpace{
+							AddressPrefixes: &[]string{
+								"10.0.0.0/9",
+							},
+						},
+						Subnets: &[]mgmtnetwork.Subnet{
+							{
+								SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{
+									AddressPrefix: to.StringPtr("10.0.0.0/24"),
+								},
+								Name: to.StringPtr("GatewaySubnet"),
+							},
+						},
+					},
+					Name:     to.StringPtr("dev-vnet"),
+					Type:     to.StringPtr("Microsoft.Network/virtualNetworks"),
+					Location: to.StringPtr("[resourceGroup().location]"),
+				},
+				APIVersion: apiVersions["network"],
+			},
+			{
+				Resource: &mgmtnetwork.VirtualNetworkGateway{
+					VirtualNetworkGatewayPropertiesFormat: &mgmtnetwork.VirtualNetworkGatewayPropertiesFormat{
+						IPConfigurations: &[]mgmtnetwork.VirtualNetworkGatewayIPConfiguration{
+							{
+								VirtualNetworkGatewayIPConfigurationPropertiesFormat: &mgmtnetwork.VirtualNetworkGatewayIPConfigurationPropertiesFormat{
+									Subnet: &mgmtnetwork.SubResource{
+										ID: to.StringPtr("[resourceId('Microsoft.Network/virtualNetworks/subnets', 'dev-vnet', 'GatewaySubnet')]"),
+									},
+									PublicIPAddress: &mgmtnetwork.SubResource{
+										ID: to.StringPtr("[resourceId('Microsoft.Network/publicIPAddresses', 'dev-vpn-pip')]"),
+									},
+								},
+								Name: to.StringPtr("default"),
+							},
+						},
+						VpnType: mgmtnetwork.RouteBased,
+						Sku: &mgmtnetwork.VirtualNetworkGatewaySku{
+							Name: mgmtnetwork.VirtualNetworkGatewaySkuNameVpnGw1,
+							Tier: mgmtnetwork.VirtualNetworkGatewaySkuTierVpnGw1,
+						},
+						VpnClientConfiguration: &mgmtnetwork.VpnClientConfiguration{
+							VpnClientAddressPool: &mgmtnetwork.AddressSpace{
+								AddressPrefixes: &[]string{"192.168.255.0/24"},
+							},
+							VpnClientRootCertificates: &[]mgmtnetwork.VpnClientRootCertificate{
+								{
+									VpnClientRootCertificatePropertiesFormat: &mgmtnetwork.VpnClientRootCertificatePropertiesFormat{
+										PublicCertData: to.StringPtr("[parameters('vpnCACertificate')]"),
+									},
+									Name: to.StringPtr("dev-vpn-ca"),
+								},
+							},
+							VpnClientProtocols: &[]mgmtnetwork.VpnClientProtocol{
+								mgmtnetwork.OpenVPN,
+							},
+						},
+					},
+					Name:     to.StringPtr("dev-vpn"),
+					Type:     to.StringPtr("Microsoft.Network/virtualNetworkGateways"),
+					Location: to.StringPtr("[resourceGroup().location]"),
+				},
+				APIVersion: apiVersions["network"],
+				DependsOn: []string{
+					"[resourceId('Microsoft.Network/publicIPAddresses', 'dev-vpn-pip')]",
+					"[resourceId('Microsoft.Network/virtualNetworks', 'dev-vnet')]",
+				},
+			},
+			proxyVmss(),
+		},
+	}
+
+	for _, param := range []string{
+		"proxyCert",
+		"proxyClientCert",
+		"proxyDomainNameLabel",
+		"proxyImage",
+		"proxyImageAuth",
+		"proxyKey",
+		"sshPublicKey",
+		"vpnCACertificate",
+	} {
+		typ := "string"
+		switch param {
+		case "proxyImageAuth", "proxyKey":
+			typ = "securestring"
+		}
+		t.Parameters[param] = &arm.TemplateParameter{Type: typ}
+	}
+
+	b, err := json.MarshalIndent(t, "", "    ")
+	if err != nil {
+		return err
+	}
+
+	b = append(b, byte('\n'))
+
+	return ioutil.WriteFile("env-development.json", b, 0666)
+}

pkg/deploy/nsg.go

@@ -7,66 +7,117 @@ import (
 	"encoding/json"
 	"io/ioutil"
 
-	"github.com/Azure/azure-sdk-for-go/services/msi/mgmt/2018-11-30/msi"
-	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+	mgmtmsi "github.com/Azure/azure-sdk-for-go/services/msi/mgmt/2018-11-30/msi"
+	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
 	"github.com/Azure/go-autorest/autorest/to"
 
 	"github.com/Azure/ARO-RP/pkg/util/arm"
 )
 
-func GenerateNSGTemplate() error {
-	t := &arm.Template{
-		Schema:         "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
-		ContentVersion: "1.0.0.0",
-		Resources: []*arm.Resource{
-			{
-				Resource: &msi.Identity{
-					Name:     to.StringPtr("rp-identity"),
-					Location: to.StringPtr("[resourceGroup().location]"),
-					Type:     "Microsoft.ManagedIdentity/userAssignedIdentities",
-				},
-				APIVersion: apiVersions["msi"],
-			},
-			{
-				Resource: &network.SecurityGroup{
-					SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{
-						SecurityRules: &[]network.SecurityRule{
-							{
-								SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
-									Protocol:                 network.SecurityRuleProtocolTCP,
-									SourcePortRange:          to.StringPtr("*"),
-									DestinationPortRange:     to.StringPtr("443"),
-									SourceAddressPrefix:      to.StringPtr("*"),
-									DestinationAddressPrefix: to.StringPtr("*"),
-									Access:                   network.SecurityRuleAccessAllow,
-									Priority:                 to.Int32Ptr(120),
-									Direction:                network.SecurityRuleDirectionInbound,
-								},
-								Name: to.StringPtr("rp_in"),
-							},
+func GenerateNSGTemplates() error {
+	for _, i := range []struct {
+		templateFile string
+		production   bool
+	}{
+		{
+			templateFile: "rp-development-nsg.json",
+		},
+		{
+			templateFile: "rp-production-nsg.json",
+			production:   true,
+		},
+	} {
+		nsg := &mgmtnetwork.SecurityGroup{
+			SecurityGroupPropertiesFormat: &mgmtnetwork.SecurityGroupPropertiesFormat{
+				SecurityRules: &[]mgmtnetwork.SecurityRule{
+					{
+						SecurityRulePropertiesFormat: &mgmtnetwork.SecurityRulePropertiesFormat{
+							Protocol:                 mgmtnetwork.SecurityRuleProtocolTCP,
+							SourcePortRange:          to.StringPtr("*"),
+							DestinationPortRange:     to.StringPtr("443"),
+							SourceAddressPrefix:      to.StringPtr("*"),
+							DestinationAddressPrefix: to.StringPtr("*"),
+							Access:                   mgmtnetwork.SecurityRuleAccessAllow,
+							Priority:                 to.Int32Ptr(120),
+							Direction:                mgmtnetwork.SecurityRuleDirectionInbound,
 						},
+						Name: to.StringPtr("rp_in"),
 					},
-					Name:     to.StringPtr("rp-nsg"),
-					Type:     to.StringPtr("Microsoft.Network/networkSecurityGroups"),
-					Location: to.StringPtr("[resourceGroup().location]"),
 				},
-				APIVersion: apiVersions["network"],
 			},
-		},
-		Outputs: map[string]*arm.Output{
-			"rpServicePrincipalId": {
-				Type:  "string",
-				Value: "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'rp-identity'), '2018-11-30').principalId]",
+			Name:     to.StringPtr("rp-nsg"),
+			Type:     to.StringPtr("Microsoft.Network/networkSecurityGroups"),
+			Location: to.StringPtr("[resourceGroup().location]"),
+		}
+
+		if !i.production {
+			*nsg.SecurityRules = append(*nsg.SecurityRules, mgmtnetwork.SecurityRule{
+				SecurityRulePropertiesFormat: &mgmtnetwork.SecurityRulePropertiesFormat{
+					Protocol:                 mgmtnetwork.SecurityRuleProtocolTCP,
+					SourcePortRange:          to.StringPtr("*"),
+					DestinationPortRange:     to.StringPtr("22"),
+					SourceAddressPrefix:      to.StringPtr("*"),
+					DestinationAddressPrefix: to.StringPtr("*"),
+					Access:                   mgmtnetwork.SecurityRuleAccessAllow,
+					Priority:                 to.Int32Ptr(100),
+					Direction:                mgmtnetwork.SecurityRuleDirectionInbound,
+				},
+				Name: to.StringPtr("ssh_in"),
+			})
+		}
+
+		t := &arm.Template{
+			Schema:         "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+			ContentVersion: "1.0.0.0",
+			Resources: []*arm.Resource{
+				{
+					Resource:   nsg,
+					APIVersion: apiVersions["network"],
+				},
+				{
+					Resource: &mgmtnetwork.SecurityGroup{
+						SecurityGroupPropertiesFormat: &mgmtnetwork.SecurityGroupPropertiesFormat{},
+						Name:                          to.StringPtr("rp-pe-nsg"),
+						Type:                          to.StringPtr("Microsoft.Network/networkSecurityGroups"),
+						Location:                      to.StringPtr("[resourceGroup().location]"),
+					},
+					APIVersion: apiVersions["network"],
+				},
 			},
-		},
+		}
+
+		if i.production {
+			t.Resources = append(t.Resources,
+				&arm.Resource{
+					Resource: &mgmtmsi.Identity{
+						Name:     to.StringPtr("rp-identity"),
+						Location: to.StringPtr("[resourceGroup().location]"),
+						Type:     "Microsoft.ManagedIdentity/userAssignedIdentities",
+					},
+					APIVersion: apiVersions["msi"],
+				},
+			)
+
+			t.Outputs = map[string]*arm.Output{
+				"rpServicePrincipalId": {
+					Type:  "string",
+					Value: "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'rp-identity'), '2018-11-30').principalId]",
+				},
+			}
+		}
+
+		b, err := json.MarshalIndent(t, "", "    ")
+		if err != nil {
+			return err
+		}
+
+		b = append(b, byte('\n'))
+
+		err = ioutil.WriteFile(i.templateFile, b, 0666)
+		if err != nil {
+			return err
+		}
 	}
 
-	b, err := json.MarshalIndent(t, "", "    ")
-	if err != nil {
-		return err
-	}
-
-	b = append(b, byte('\n'))
-
-	return ioutil.WriteFile("rp-production-nsg.json", b, 0666)
+	return nil
 }

pkg/deploy/proxy.go

@@ -0,0 +1,188 @@
+package deploy
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
+	"github.com/Azure/go-autorest/autorest/to"
+
+	"github.com/Azure/ARO-RP/pkg/util/arm"
+)
+
+func proxyVmss() *arm.Resource {
+	parts := []string{
+		fmt.Sprintf("base64ToString('%s')", base64.StdEncoding.EncodeToString([]byte("set -ex\n\n"))),
+	}
+
+	for _, variable := range []string{"proxyImage", "proxyImageAuth"} {
+		parts = append(parts,
+			fmt.Sprintf("'%s=$(base64 -d <<<'''", strings.ToUpper(variable)),
+			fmt.Sprintf("base64(parameters('%s'))", variable),
+			"''')\n'",
+		)
+	}
+
+	for _, variable := range []string{"proxyCert", "proxyClientCert", "proxyKey"} {
+		parts = append(parts,
+			fmt.Sprintf("'%s='''", strings.ToUpper(variable)),
+			fmt.Sprintf("parameters('%s')", variable),
+			"'''\n'",
+		)
+	}
+
+	trailer := base64.StdEncoding.EncodeToString([]byte(`yum -y update -x WALinuxAgent
+
+yum -y install docker
+
+firewall-cmd --add-port=443/tcp --permanent
+
+mkdir /root/.docker
+cat >/root/.docker/config.json <<EOF
+{
+	"auths": {
+		"${PROXYIMAGE%%/*}": {
+			"auth": "$PROXYIMAGEAUTH"
+		}
+	}
+}
+EOF
+
+mkdir /etc/proxy
+base64 -d <<<"$PROXYCERT" >/etc/proxy/proxy.crt
+base64 -d <<<"$PROXYKEY" >/etc/proxy/proxy.key
+base64 -d <<<"$PROXYCLIENTCERT" >/etc/proxy/proxy-client.crt
+chown -R 1000:1000 /etc/proxy
+chmod 0600 /etc/proxy/proxy.key
+
+cat >/etc/sysconfig/proxy <<EOF
+PROXY_IMAGE='$PROXYIMAGE'
+EOF
+
+cat >/etc/systemd/system/proxy.service <<EOF
+[Unit]
+After=docker.service
+Requires=docker.service
+
+[Service]
+EnvironmentFile=/etc/sysconfig/proxy
+ExecStartPre=-/usr/bin/docker rm -f %n
+ExecStartPre=/usr/bin/docker pull \$PROXY_IMAGE
+ExecStart=/usr/bin/docker run --rm --name %n -p 443:8443 -v /etc/proxy:/secrets \$PROXY_IMAGE
+ExecStop=/usr/bin/docker stop %n
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl enable proxy.service
+
+(sleep 30; reboot) &
+`))
+
+	parts = append(parts, "'\n'", fmt.Sprintf("base64ToString('%s')", trailer))
+
+	script := fmt.Sprintf("[base64(concat(%s))]", strings.Join(parts, ","))
+
+	return &arm.Resource{
+		Resource: &mgmtcompute.VirtualMachineScaleSet{
+			Sku: &mgmtcompute.Sku{
+				Name:     to.StringPtr(string(mgmtcompute.VirtualMachineSizeTypesStandardD2sV3)),
+				Tier:     to.StringPtr("Standard"),
+				Capacity: to.Int64Ptr(1),
+			},
+			VirtualMachineScaleSetProperties: &mgmtcompute.VirtualMachineScaleSetProperties{
+				UpgradePolicy: &mgmtcompute.UpgradePolicy{
+					Mode: mgmtcompute.Manual,
+				},
+				VirtualMachineProfile: &mgmtcompute.VirtualMachineScaleSetVMProfile{
+					OsProfile: &mgmtcompute.VirtualMachineScaleSetOSProfile{
+						ComputerNamePrefix: to.StringPtr("dev-proxy-"),
+						AdminUsername:      to.StringPtr("cloud-user"),
+						LinuxConfiguration: &mgmtcompute.LinuxConfiguration{
+							DisablePasswordAuthentication: to.BoolPtr(true),
+							SSH: &mgmtcompute.SSHConfiguration{
+								PublicKeys: &[]mgmtcompute.SSHPublicKey{
+									{
+										Path:    to.StringPtr("/home/cloud-user/.ssh/authorized_keys"),
+										KeyData: to.StringPtr("[parameters('sshPublicKey')]"),
+									},
+								},
+							},
+						},
+					},
+					StorageProfile: &mgmtcompute.VirtualMachineScaleSetStorageProfile{
+						ImageReference: &mgmtcompute.ImageReference{
+							Publisher: to.StringPtr("RedHat"),
+							Offer:     to.StringPtr("RHEL"),
+							Sku:       to.StringPtr("7-RAW"),
+							Version:   to.StringPtr("latest"),
+						},
+						OsDisk: &mgmtcompute.VirtualMachineScaleSetOSDisk{
+							CreateOption: mgmtcompute.DiskCreateOptionTypesFromImage,
+							ManagedDisk: &mgmtcompute.VirtualMachineScaleSetManagedDiskParameters{
+								StorageAccountType: mgmtcompute.StorageAccountTypesPremiumLRS,
+							},
+						},
+					},
+					NetworkProfile: &mgmtcompute.VirtualMachineScaleSetNetworkProfile{
+						NetworkInterfaceConfigurations: &[]mgmtcompute.VirtualMachineScaleSetNetworkConfiguration{
+							{
+								Name: to.StringPtr("dev-proxy-vmss-nic"),
+								VirtualMachineScaleSetNetworkConfigurationProperties: &mgmtcompute.VirtualMachineScaleSetNetworkConfigurationProperties{
+									Primary: to.BoolPtr(true),
+									IPConfigurations: &[]mgmtcompute.VirtualMachineScaleSetIPConfiguration{
+										{
+											Name: to.StringPtr("dev-proxy-vmss-ipconfig"),
+											VirtualMachineScaleSetIPConfigurationProperties: &mgmtcompute.VirtualMachineScaleSetIPConfigurationProperties{
+												Subnet: &mgmtcompute.APIEntityReference{
+													ID: to.StringPtr("[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"),
+												},
+												Primary: to.BoolPtr(true),
+												PublicIPAddressConfiguration: &mgmtcompute.VirtualMachineScaleSetPublicIPAddressConfiguration{
+													Name: to.StringPtr("dev-proxy-vmss-pip"),
+													VirtualMachineScaleSetPublicIPAddressConfigurationProperties: &mgmtcompute.VirtualMachineScaleSetPublicIPAddressConfigurationProperties{
+														DNSSettings: &mgmtcompute.VirtualMachineScaleSetPublicIPAddressConfigurationDNSSettings{
+															DomainNameLabel: to.StringPtr("[parameters('proxyDomainNameLabel')]"),
+														},
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+					ExtensionProfile: &mgmtcompute.VirtualMachineScaleSetExtensionProfile{
+						Extensions: &[]mgmtcompute.VirtualMachineScaleSetExtension{
+							{
+								Name: to.StringPtr("dev-proxy-vmss-cse"),
+								VirtualMachineScaleSetExtensionProperties: &mgmtcompute.VirtualMachineScaleSetExtensionProperties{
+									Publisher:               to.StringPtr("Microsoft.Azure.Extensions"),
+									Type:                    to.StringPtr("CustomScript"),
+									TypeHandlerVersion:      to.StringPtr("2.0"),
+									AutoUpgradeMinorVersion: to.BoolPtr(true),
+									Settings:                map[string]interface{}{},
+									ProtectedSettings: map[string]interface{}{
+										"script": script,
+									},
+								},
+							},
+						},
+					},
+				},
+				Overprovision: to.BoolPtr(false),
+			},
+			Name:     to.StringPtr("dev-proxy-vmss"),
+			Type:     to.StringPtr("Microsoft.Compute/virtualMachineScaleSets"),
+			Location: to.StringPtr("[resourceGroup().location]"),
+		},
+		APIVersion: apiVersions["compute"],
+	}
+}

pkg/deploy/rp.go

@@ -11,12 +11,12 @@ import (
 	"io/ioutil"
 	"strings"
 
-	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
-	"github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2019-08-01/documentdb"
-	"github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2018-05-01/dns"
-	"github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2016-10-01/keyvault"
-	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
-	"github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization"
+	mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
+	mgmtdocumentdb "github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2019-08-01/documentdb"
+	mgmtdns "github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2018-05-01/dns"
+	mgmtkeyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2016-10-01/keyvault"
+	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+	mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization"
 	"github.com/Azure/go-autorest/autorest/to"
 	uuid "github.com/satori/go.uuid"
 
@@ -49,33 +49,48 @@ func newGenerator(production bool) *generator {
 }
 
 func (g *generator) vnet() *arm.Resource {
+	subnet := mgmtnetwork.Subnet{
+		SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{
+			AddressPrefix: to.StringPtr("10.0.0.0/24"),
+			NetworkSecurityGroup: &mgmtnetwork.SecurityGroup{
+				ID: to.StringPtr("[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-nsg')]"),
+			},
+		},
+		Name: to.StringPtr("rp-subnet"),
+	}
+
+	if g.production {
+		subnet.ServiceEndpoints = &[]mgmtnetwork.ServiceEndpointPropertiesFormat{
+			{
+				Service:   to.StringPtr("Microsoft.KeyVault"),
+				Locations: &[]string{"*"},
+			},
+			{
+				Service:   to.StringPtr("Microsoft.AzureCosmosDB"),
+				Locations: &[]string{"*"},
+			},
+		}
+	}
+
 	return &arm.Resource{
-		Resource: &network.VirtualNetwork{
-			VirtualNetworkPropertiesFormat: &network.VirtualNetworkPropertiesFormat{
-				AddressSpace: &network.AddressSpace{
+		Resource: &mgmtnetwork.VirtualNetwork{
+			VirtualNetworkPropertiesFormat: &mgmtnetwork.VirtualNetworkPropertiesFormat{
+				AddressSpace: &mgmtnetwork.AddressSpace{
 					AddressPrefixes: &[]string{
 						"10.0.0.0/8",
 					},
 				},
-				Subnets: &[]network.Subnet{
+				Subnets: &[]mgmtnetwork.Subnet{
+					subnet,
 					{
-						SubnetPropertiesFormat: &network.SubnetPropertiesFormat{
-							AddressPrefix: to.StringPtr("10.0.0.0/24"),
-							NetworkSecurityGroup: &network.SecurityGroup{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-nsg')]"),
-							},
-							ServiceEndpoints: &[]network.ServiceEndpointPropertiesFormat{
-								{
-									Service:   to.StringPtr("Microsoft.KeyVault"),
-									Locations: &[]string{"*"},
-								},
-								{
-									Service:   to.StringPtr("Microsoft.AzureCosmosDB"),
-									Locations: &[]string{"*"},
-								},
+						SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{
+							AddressPrefix: to.StringPtr("10.1.0.0/16"),
+							NetworkSecurityGroup: &mgmtnetwork.SecurityGroup{
+								ID: to.StringPtr("[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-pe-nsg')]"),
 							},
+							PrivateEndpointNetworkPolicies: to.StringPtr("Disabled"),
 						},
-						Name: to.StringPtr("rp-subnet"),
+						Name: to.StringPtr("rp-pe-subnet"),
 					},
 				},
 			},
@@ -88,71 +103,68 @@ func (g *generator) vnet() *arm.Resource {
 }
 
 func (g *generator) pip() *arm.Resource {
-	pip := &network.PublicIPAddress{
-		Sku: &network.PublicIPAddressSku{
-			Name: network.PublicIPAddressSkuNameStandard,
-		},
-		PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
-			PublicIPAllocationMethod: network.Static,
-		},
-		Name:     to.StringPtr("rp-pip"),
-		Type:     to.StringPtr("Microsoft.Network/publicIPAddresses"),
-		Location: to.StringPtr("[resourceGroup().location]"),
-	}
-
 	return &arm.Resource{
-		Resource:   pip,
+		Resource: &mgmtnetwork.PublicIPAddress{
+			Sku: &mgmtnetwork.PublicIPAddressSku{
+				Name: mgmtnetwork.PublicIPAddressSkuNameStandard,
+			},
+			PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{
+				PublicIPAllocationMethod: mgmtnetwork.Static,
+			},
+			Name:     to.StringPtr("rp-pip"),
+			Type:     to.StringPtr("Microsoft.Network/publicIPAddresses"),
+			Location: to.StringPtr("[resourceGroup().location]"),
+		},
 		APIVersion: apiVersions["network"],
 	}
-
 }
 
 func (g *generator) lb() *arm.Resource {
 	return &arm.Resource{
-		Resource: &network.LoadBalancer{
-			Sku: &network.LoadBalancerSku{
-				Name: network.LoadBalancerSkuNameStandard,
+		Resource: &mgmtnetwork.LoadBalancer{
+			Sku: &mgmtnetwork.LoadBalancerSku{
+				Name: mgmtnetwork.LoadBalancerSkuNameStandard,
 			},
-			LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{
-				FrontendIPConfigurations: &[]network.FrontendIPConfiguration{
+			LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{
+				FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
 					{
-						FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
-							PublicIPAddress: &network.PublicIPAddress{
+						FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{
+							PublicIPAddress: &mgmtnetwork.PublicIPAddress{
 								ID: to.StringPtr("[resourceId('Microsoft.Network/publicIPAddresses', 'rp-pip')]"),
 							},
 						},
 						Name: to.StringPtr("rp-frontend"),
 					},
 				},
-				BackendAddressPools: &[]network.BackendAddressPool{
+				BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
 					{
 						Name: to.StringPtr("rp-backend"),
 					},
 				},
-				LoadBalancingRules: &[]network.LoadBalancingRule{
+				LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{
 					{
-						LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-							FrontendIPConfiguration: &network.SubResource{
+						LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
+							FrontendIPConfiguration: &mgmtnetwork.SubResource{
 								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'rp-lb', 'rp-frontend')]"),
 							},
-							BackendAddressPool: &network.SubResource{
+							BackendAddressPool: &mgmtnetwork.SubResource{
 								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'rp-lb', 'rp-backend')]"),
 							},
-							Probe: &network.SubResource{
+							Probe: &mgmtnetwork.SubResource{
 								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'rp-lb', 'rp-probe')]"),
 							},
-							Protocol:         network.TransportProtocolTCP,
-							LoadDistribution: network.LoadDistributionDefault,
+							Protocol:         mgmtnetwork.TransportProtocolTCP,
+							LoadDistribution: mgmtnetwork.LoadDistributionDefault,
 							FrontendPort:     to.Int32Ptr(443),
 							BackendPort:      to.Int32Ptr(443),
 						},
 						Name: to.StringPtr("rp-lbrule"),
 					},
 				},
-				Probes: &[]network.Probe{
+				Probes: &[]mgmtnetwork.Probe{
 					{
-						ProbePropertiesFormat: &network.ProbePropertiesFormat{
-							Protocol:       network.ProbeProtocolHTTPS,
+						ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
+							Protocol:       mgmtnetwork.ProbeProtocolHTTPS,
 							Port:           to.Int32Ptr(443),
 							NumberOfProbes: to.Int32Ptr(2),
 							RequestPath:    to.StringPtr("/healthz/ready"),
@@ -253,69 +265,71 @@ systemctl enable arorp.service
 
 	script := fmt.Sprintf("[base64(concat(%s))]", strings.Join(parts, ","))
 
-	vmss := &compute.VirtualMachineScaleSet{
-		Sku: &compute.Sku{
-			Name:     to.StringPtr(string(compute.VirtualMachineSizeTypesStandardD2sV3)),
-			Tier:     to.StringPtr("Standard"),
-			Capacity: to.Int64Ptr(3),
-		},
-		VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
-			UpgradePolicy: &compute.UpgradePolicy{
-				Mode: compute.Manual,
+	return &arm.Resource{
+		Resource: &mgmtcompute.VirtualMachineScaleSet{
+			Sku: &mgmtcompute.Sku{
+				Name:     to.StringPtr(string(mgmtcompute.VirtualMachineSizeTypesStandardD2sV3)),
+				Tier:     to.StringPtr("Standard"),
+				Capacity: to.Int64Ptr(3),
 			},
-			VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{
-				OsProfile: &compute.VirtualMachineScaleSetOSProfile{
-					ComputerNamePrefix: to.StringPtr("rp-"),
-					AdminUsername:      to.StringPtr("cloud-user"),
-					LinuxConfiguration: &compute.LinuxConfiguration{
-						DisablePasswordAuthentication: to.BoolPtr(true),
-						SSH: &compute.SSHConfiguration{
-							PublicKeys: &[]compute.SSHPublicKey{
-								{
-									Path:    to.StringPtr("/home/cloud-user/.ssh/authorized_keys"),
-									KeyData: to.StringPtr("[parameters('sshPublicKey')]"),
+			VirtualMachineScaleSetProperties: &mgmtcompute.VirtualMachineScaleSetProperties{
+				UpgradePolicy: &mgmtcompute.UpgradePolicy{
+					Mode: mgmtcompute.Manual,
+				},
+				VirtualMachineProfile: &mgmtcompute.VirtualMachineScaleSetVMProfile{
+					OsProfile: &mgmtcompute.VirtualMachineScaleSetOSProfile{
+						ComputerNamePrefix: to.StringPtr("rp-"),
+						AdminUsername:      to.StringPtr("cloud-user"),
+						LinuxConfiguration: &mgmtcompute.LinuxConfiguration{
+							DisablePasswordAuthentication: to.BoolPtr(true),
+							SSH: &mgmtcompute.SSHConfiguration{
+								PublicKeys: &[]mgmtcompute.SSHPublicKey{
+									{
+										Path:    to.StringPtr("/home/cloud-user/.ssh/authorized_keys"),
+										KeyData: to.StringPtr("[parameters('sshPublicKey')]"),
+									},
 								},
 							},
 						},
 					},
-				},
-				StorageProfile: &compute.VirtualMachineScaleSetStorageProfile{
-					ImageReference: &compute.ImageReference{
-						Publisher: to.StringPtr("RedHat"),
-						Offer:     to.StringPtr("RHEL"),
-						Sku:       to.StringPtr("7-RAW"),
-						Version:   to.StringPtr("latest"),
-					},
-					OsDisk: &compute.VirtualMachineScaleSetOSDisk{
-						CreateOption: compute.DiskCreateOptionTypesFromImage,
-						ManagedDisk: &compute.VirtualMachineScaleSetManagedDiskParameters{
-							StorageAccountType: compute.StorageAccountTypesPremiumLRS,
+					StorageProfile: &mgmtcompute.VirtualMachineScaleSetStorageProfile{
+						ImageReference: &mgmtcompute.ImageReference{
+							Publisher: to.StringPtr("RedHat"),
+							Offer:     to.StringPtr("RHEL"),
+							Sku:       to.StringPtr("7-RAW"),
+							Version:   to.StringPtr("latest"),
+						},
+						OsDisk: &mgmtcompute.VirtualMachineScaleSetOSDisk{
+							CreateOption: mgmtcompute.DiskCreateOptionTypesFromImage,
+							ManagedDisk: &mgmtcompute.VirtualMachineScaleSetManagedDiskParameters{
+								StorageAccountType: mgmtcompute.StorageAccountTypesPremiumLRS,
+							},
 						},
 					},
-				},
-				NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{
-					HealthProbe: &compute.APIEntityReference{
-						ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'rp-lb', 'rp-probe')]"),
-					},
-					NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{
-						{
-							Name: to.StringPtr("rp-vmss-nic"),
-							VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{
-								Primary: to.BoolPtr(true),
-								IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{
-									{
-										Name: to.StringPtr("rp-vmss-ipconfig"),
-										VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{
-											Subnet: &compute.APIEntityReference{
-												ID: to.StringPtr("[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"),
-											},
-											Primary: to.BoolPtr(true),
-											PublicIPAddressConfiguration: &compute.VirtualMachineScaleSetPublicIPAddressConfiguration{
-												Name: to.StringPtr("rp-vmss-pip"),
-											},
-											LoadBalancerBackendAddressPools: &[]compute.SubResource{
-												{
-													ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'rp-lb', 'rp-backend')]"),
+					NetworkProfile: &mgmtcompute.VirtualMachineScaleSetNetworkProfile{
+						HealthProbe: &mgmtcompute.APIEntityReference{
+							ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'rp-lb', 'rp-probe')]"),
+						},
+						NetworkInterfaceConfigurations: &[]mgmtcompute.VirtualMachineScaleSetNetworkConfiguration{
+							{
+								Name: to.StringPtr("rp-vmss-nic"),
+								VirtualMachineScaleSetNetworkConfigurationProperties: &mgmtcompute.VirtualMachineScaleSetNetworkConfigurationProperties{
+									Primary: to.BoolPtr(true),
+									IPConfigurations: &[]mgmtcompute.VirtualMachineScaleSetIPConfiguration{
+										{
+											Name: to.StringPtr("rp-vmss-ipconfig"),
+											VirtualMachineScaleSetIPConfigurationProperties: &mgmtcompute.VirtualMachineScaleSetIPConfigurationProperties{
+												Subnet: &mgmtcompute.APIEntityReference{
+													ID: to.StringPtr("[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"),
+												},
+												Primary: to.BoolPtr(true),
+												PublicIPAddressConfiguration: &mgmtcompute.VirtualMachineScaleSetPublicIPAddressConfiguration{
+													Name: to.StringPtr("rp-vmss-pip"),
+												},
+												LoadBalancerBackendAddressPools: &[]mgmtcompute.SubResource{
+													{
+														ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'rp-lb', 'rp-backend')]"),
+													},
 												},
 											},
 										},
@@ -324,40 +338,36 @@ systemctl enable arorp.service
 							},
 						},
 					},
-				},
-				ExtensionProfile: &compute.VirtualMachineScaleSetExtensionProfile{
-					Extensions: &[]compute.VirtualMachineScaleSetExtension{
-						{
-							Name: to.StringPtr("rp-vmss-cse"),
-							VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
-								Publisher:               to.StringPtr("Microsoft.Azure.Extensions"),
-								Type:                    to.StringPtr("CustomScript"),
-								TypeHandlerVersion:      to.StringPtr("2.0"),
-								AutoUpgradeMinorVersion: to.BoolPtr(true),
-								Settings:                map[string]interface{}{},
-								ProtectedSettings: map[string]interface{}{
-									"script": script,
+					ExtensionProfile: &mgmtcompute.VirtualMachineScaleSetExtensionProfile{
+						Extensions: &[]mgmtcompute.VirtualMachineScaleSetExtension{
+							{
+								Name: to.StringPtr("rp-vmss-cse"),
+								VirtualMachineScaleSetExtensionProperties: &mgmtcompute.VirtualMachineScaleSetExtensionProperties{
+									Publisher:               to.StringPtr("Microsoft.Azure.Extensions"),
+									Type:                    to.StringPtr("CustomScript"),
+									TypeHandlerVersion:      to.StringPtr("2.0"),
+									AutoUpgradeMinorVersion: to.BoolPtr(true),
+									Settings:                map[string]interface{}{},
+									ProtectedSettings: map[string]interface{}{
+										"script": script,
+									},
 								},
 							},
 						},
 					},
 				},
+				Overprovision: to.BoolPtr(false),
 			},
-			Overprovision: to.BoolPtr(false),
-		},
-		Identity: &compute.VirtualMachineScaleSetIdentity{
-			Type: compute.ResourceIdentityTypeUserAssigned,
-			UserAssignedIdentities: map[string]*compute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue{
-				"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'rp-identity')]": {},
+			Identity: &mgmtcompute.VirtualMachineScaleSetIdentity{
+				Type: mgmtcompute.ResourceIdentityTypeUserAssigned,
+				UserAssignedIdentities: map[string]*mgmtcompute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue{
+					"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'rp-identity')]": {},
+				},
 			},
+			Name:     to.StringPtr("rp-vmss"),
+			Type:     to.StringPtr("Microsoft.Compute/virtualMachineScaleSets"),
+			Location: to.StringPtr("[resourceGroup().location]"),
 		},
-		Name:     to.StringPtr("rp-vmss"),
-		Type:     to.StringPtr("Microsoft.Compute/virtualMachineScaleSets"),
-		Location: to.StringPtr("[resourceGroup().location]"),
-	}
-
-	return &arm.Resource{
-		Resource:   vmss,
 		APIVersion: apiVersions["compute"],
 		DependsOn: []string{
 			"[resourceId('Microsoft.Network/virtualNetworks', 'rp-vnet')]",
@@ -368,8 +378,8 @@ systemctl enable arorp.service
 
 func (g *generator) zone() *arm.Resource {
 	return &arm.Resource{
-		Resource: &dns.Zone{
-			ZoneProperties: &dns.ZoneProperties{},
+		Resource: &mgmtdns.Zone{
+			ZoneProperties: &mgmtdns.ZoneProperties{},
 			Name:           to.StringPtr("[parameters('domainName')]"),
 			Type:           to.StringPtr("Microsoft.Network/dnsZones"),
 			Location:       to.StringPtr("global"),
@@ -378,27 +388,27 @@ func (g *generator) zone() *arm.Resource {
 	}
 }
 
-func (g *generator) accessPolicyEntry() keyvault.AccessPolicyEntry {
-	return keyvault.AccessPolicyEntry{
+func (g *generator) accessPolicyEntry() mgmtkeyvault.AccessPolicyEntry {
+	return mgmtkeyvault.AccessPolicyEntry{
 		TenantID: &tenantUUIDHack,
 		ObjectID: to.StringPtr("[parameters('rpServicePrincipalId')]"),
-		Permissions: &keyvault.Permissions{
-			Secrets: &[]keyvault.SecretPermissions{
-				keyvault.SecretPermissionsGet,
+		Permissions: &mgmtkeyvault.Permissions{
+			Secrets: &[]mgmtkeyvault.SecretPermissions{
+				mgmtkeyvault.SecretPermissionsGet,
 			},
 		},
 	}
 }
 
 func (g *generator) vault() *arm.Resource {
-	vault := &keyvault.Vault{
-		Properties: &keyvault.VaultProperties{
+	vault := &mgmtkeyvault.Vault{
+		Properties: &mgmtkeyvault.VaultProperties{
 			TenantID: &tenantUUIDHack,
-			Sku: &keyvault.Sku{
-				Name:   keyvault.Standard,
+			Sku: &mgmtkeyvault.Sku{
+				Name:   mgmtkeyvault.Standard,
 				Family: to.StringPtr("A"),
 			},
-			AccessPolicies: &[]keyvault.AccessPolicyEntry{},
+			AccessPolicies: &[]mgmtkeyvault.AccessPolicyEntry{},
 		},
 		Name:     to.StringPtr("[parameters('keyvaultName')]"),
 		Type:     to.StringPtr("Microsoft.KeyVault/vaults"),
@@ -406,27 +416,27 @@ func (g *generator) vault() *arm.Resource {
 	}
 
 	if !g.production {
-		vault.Properties.AccessPolicies = &[]keyvault.AccessPolicyEntry{
+		vault.Properties.AccessPolicies = &[]mgmtkeyvault.AccessPolicyEntry{
 			g.accessPolicyEntry(),
 			{
 				TenantID: &tenantUUIDHack,
 				ObjectID: to.StringPtr("[parameters('adminObjectId')]"),
-				Permissions: &keyvault.Permissions{
-					Certificates: &[]keyvault.CertificatePermissions{
-						keyvault.Create,
-						keyvault.Delete,
-						keyvault.Deleteissuers,
-						keyvault.Get,
-						keyvault.Getissuers,
-						keyvault.Import,
-						keyvault.List,
-						keyvault.Listissuers,
-						keyvault.Managecontacts,
-						keyvault.Manageissuers,
-						keyvault.Purge,
-						keyvault.Recover,
-						keyvault.Setissuers,
-						keyvault.Update,
+				Permissions: &mgmtkeyvault.Permissions{
+					Certificates: &[]mgmtkeyvault.CertificatePermissions{
+						mgmtkeyvault.Create,
+						mgmtkeyvault.Delete,
+						mgmtkeyvault.Deleteissuers,
+						mgmtkeyvault.Get,
+						mgmtkeyvault.Getissuers,
+						mgmtkeyvault.Import,
+						mgmtkeyvault.List,
+						mgmtkeyvault.Listissuers,
+						mgmtkeyvault.Managecontacts,
+						mgmtkeyvault.Manageissuers,
+						mgmtkeyvault.Purge,
+						mgmtkeyvault.Recover,
+						mgmtkeyvault.Setissuers,
+						mgmtkeyvault.Update,
 					},
 				},
 			},
@@ -439,20 +449,19 @@ func (g *generator) vault() *arm.Resource {
 	}
 }
 
-func (g *generator) cosmosdb() []*arm.Resource {
-	cosmosdb := &documentdb.DatabaseAccountCreateUpdateParameters{
-		Kind: documentdb.GlobalDocumentDB,
-		DatabaseAccountCreateUpdateProperties: &documentdb.DatabaseAccountCreateUpdateProperties{
-			ConsistencyPolicy: &documentdb.ConsistencyPolicy{
-				DefaultConsistencyLevel: documentdb.Strong,
+func (g *generator) cosmosdb(databaseName string) []*arm.Resource {
+	cosmosdb := &mgmtdocumentdb.DatabaseAccountCreateUpdateParameters{
+		Kind: mgmtdocumentdb.GlobalDocumentDB,
+		DatabaseAccountCreateUpdateProperties: &mgmtdocumentdb.DatabaseAccountCreateUpdateProperties{
+			ConsistencyPolicy: &mgmtdocumentdb.ConsistencyPolicy{
+				DefaultConsistencyLevel: mgmtdocumentdb.Strong,
 			},
-			Locations: &[]documentdb.Location{
+			Locations: &[]mgmtdocumentdb.Location{
 				{
 					LocationName: to.StringPtr("[resourceGroup().location]"),
 				},
 			},
-			DatabaseAccountOfferType:           to.StringPtr(string(documentdb.Standard)),
-			DisableKeyBasedMetadataWriteAccess: to.BoolPtr(true),
+			DatabaseAccountOfferType: to.StringPtr(string(mgmtdocumentdb.Standard)),
 		},
 		Name:     to.StringPtr("[parameters('databaseAccountName')]"),
 		Type:     to.StringPtr("Microsoft.DocumentDB/databaseAccounts"),
@@ -470,69 +479,89 @@ func (g *generator) cosmosdb() []*arm.Resource {
 	if g.production {
 		cosmosdb.IPRangeFilter = to.StringPtr("[concat('104.42.195.92,40.76.54.131,52.176.6.30,52.169.50.45,52.187.184.26', if(equals(parameters('extraCosmosDBIPs'), ''), '', ','), parameters('extraCosmosDBIPs'))]")
 		cosmosdb.IsVirtualNetworkFilterEnabled = to.BoolPtr(true)
-		cosmosdb.VirtualNetworkRules = &[]documentdb.VirtualNetworkRule{
+		cosmosdb.VirtualNetworkRules = &[]mgmtdocumentdb.VirtualNetworkRule{
 			{
 				ID: to.StringPtr("[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"),
 			},
 		}
+		cosmosdb.DisableKeyBasedMetadataWriteAccess = to.BoolPtr(true)
 
 		r.DependsOn = append(r.DependsOn, "[resourceId('Microsoft.Network/virtualNetworks', 'rp-vnet')]")
 	}
 
-	return []*arm.Resource{
+	rs := []*arm.Resource{
 		r,
+	}
+
+	if g.production {
+		rs = append(rs, g.database(databaseName, true)...)
+	}
+
+	return rs
+}
+
+func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Resource {
+	var dependsOn []string
+
+	if addDependsOn {
+		dependsOn = []string{
+			"[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]",
+		}
+	}
+
+	return []*arm.Resource{
 		{
-			Resource: &documentdb.SQLDatabaseCreateUpdateParameters{
-				SQLDatabaseCreateUpdateProperties: &documentdb.SQLDatabaseCreateUpdateProperties{
-					Resource: &documentdb.SQLDatabaseResource{
-						ID: to.StringPtr("ARO"),
+			Resource: &mgmtdocumentdb.SQLDatabaseCreateUpdateParameters{
+				SQLDatabaseCreateUpdateProperties: &mgmtdocumentdb.SQLDatabaseCreateUpdateProperties{
+					Resource: &mgmtdocumentdb.SQLDatabaseResource{
+						ID: to.StringPtr("[" + databaseName + "]"),
+					},
+					Options: map[string]*string{
+						"x-ms-offer-throughput": to.StringPtr("400"),
 					},
-					Options: map[string]*string{},
 				},
-				Name: to.StringPtr("[concat(parameters('databaseAccountName'), '/ARO')]"),
-				Type: to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases"),
+				Name:     to.StringPtr("[concat(parameters('databaseAccountName'), '/', " + databaseName + ")]"),
+				Type:     to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases"),
+				Location: to.StringPtr("[resourceGroup().location]"),
 			},
 			APIVersion: apiVersions["documentdb"],
-			DependsOn: []string{
-				"[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]",
-			},
+			DependsOn:  dependsOn,
 		},
 		{
-			Resource: &documentdb.SQLContainerCreateUpdateParameters{
-				SQLContainerCreateUpdateProperties: &documentdb.SQLContainerCreateUpdateProperties{
-					Resource: &documentdb.SQLContainerResource{
+			Resource: &mgmtdocumentdb.SQLContainerCreateUpdateParameters{
+				SQLContainerCreateUpdateProperties: &mgmtdocumentdb.SQLContainerCreateUpdateProperties{
+					Resource: &mgmtdocumentdb.SQLContainerResource{
 						ID: to.StringPtr("AsyncOperations"),
-						PartitionKey: &documentdb.ContainerPartitionKey{
+						PartitionKey: &mgmtdocumentdb.ContainerPartitionKey{
 							Paths: &[]string{
 								"/id",
 							},
-							Kind: documentdb.PartitionKindHash,
+							Kind: mgmtdocumentdb.PartitionKindHash,
 						},
 						DefaultTTL: to.Int32Ptr(7 * 86400), // 7 days
 					},
 					Options: map[string]*string{},
 				},
-				Name: to.StringPtr("[concat(parameters('databaseAccountName'), '/ARO/AsyncOperations')]"),
-				Type: to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"),
+				Name:     to.StringPtr("[concat(parameters('databaseAccountName'), '/', " + databaseName + ", '/AsyncOperations')]"),
+				Type:     to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"),
+				Location: to.StringPtr("[resourceGroup().location]"),
 			},
 			APIVersion: apiVersions["documentdb"],
-			DependsOn: []string{
-				"[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), 'ARO')]",
-			},
+			DependsOn:  dependsOn,
 		},
 		{
-			Resource: &documentdb.SQLContainerCreateUpdateParameters{
-				SQLContainerCreateUpdateProperties: &documentdb.SQLContainerCreateUpdateProperties{
-					Resource: &documentdb.SQLContainerResource{
+			Resource: &mgmtdocumentdb.SQLContainerCreateUpdateParameters{
+				SQLContainerCreateUpdateProperties: &mgmtdocumentdb.SQLContainerCreateUpdateProperties{
+					Resource: &mgmtdocumentdb.SQLContainerResource{
 						ID: to.StringPtr("OpenShiftClusters"),
-						PartitionKey: &documentdb.ContainerPartitionKey{
+						PartitionKey: &mgmtdocumentdb.ContainerPartitionKey{
 							Paths: &[]string{
 								"/partitionKey",
 							},
-							Kind: documentdb.PartitionKindHash,
+							Kind: mgmtdocumentdb.PartitionKindHash,
 						},
-						UniqueKeyPolicy: &documentdb.UniqueKeyPolicy{
-							UniqueKeys: &[]documentdb.UniqueKey{
+						UniqueKeyPolicy: &mgmtdocumentdb.UniqueKeyPolicy{
+							UniqueKeys: &[]mgmtdocumentdb.UniqueKey{
 								{
 									Paths: &[]string{
 										"/key",
@@ -543,35 +572,33 @@ func (g *generator) cosmosdb() []*arm.Resource {
 					},
 					Options: map[string]*string{},
 				},
-				Name: to.StringPtr("[concat(parameters('databaseAccountName'), '/ARO/OpenShiftClusters')]"),
-				Type: to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"),
+				Name:     to.StringPtr("[concat(parameters('databaseAccountName'), '/', " + databaseName + ", '/OpenShiftClusters')]"),
+				Type:     to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"),
+				Location: to.StringPtr("[resourceGroup().location]"),
 			},
 			APIVersion: apiVersions["documentdb"],
-			DependsOn: []string{
-				"[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), 'ARO')]",
-			},
+			DependsOn:  dependsOn,
 		},
 		{
-			Resource: &documentdb.SQLContainerCreateUpdateParameters{
-				SQLContainerCreateUpdateProperties: &documentdb.SQLContainerCreateUpdateProperties{
-					Resource: &documentdb.SQLContainerResource{
+			Resource: &mgmtdocumentdb.SQLContainerCreateUpdateParameters{
+				SQLContainerCreateUpdateProperties: &mgmtdocumentdb.SQLContainerCreateUpdateProperties{
+					Resource: &mgmtdocumentdb.SQLContainerResource{
 						ID: to.StringPtr("Subscriptions"),
-						PartitionKey: &documentdb.ContainerPartitionKey{
+						PartitionKey: &mgmtdocumentdb.ContainerPartitionKey{
 							Paths: &[]string{
 								"/id",
 							},
-							Kind: documentdb.PartitionKindHash,
+							Kind: mgmtdocumentdb.PartitionKindHash,
 						},
 					},
 					Options: map[string]*string{},
 				},
-				Name: to.StringPtr("[concat(parameters('databaseAccountName'), '/ARO/Subscriptions')]"),
-				Type: to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"),
+				Name:     to.StringPtr("[concat(parameters('databaseAccountName'), '/', " + databaseName + ", '/Subscriptions')]"),
+				Type:     to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers"),
+				Location: to.StringPtr("[resourceGroup().location]"),
 			},
 			APIVersion: apiVersions["documentdb"],
-			DependsOn: []string{
-				"[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlDatabases', parameters('databaseAccountName'), 'ARO')]",
-			},
+			DependsOn:  dependsOn,
 		},
 	}
 }
@@ -579,27 +606,40 @@ func (g *generator) cosmosdb() []*arm.Resource {
 func (g *generator) rbac() []*arm.Resource {
 	return []*arm.Resource{
 		{
-			Resource: &authorization.RoleAssignment{
+			Resource: &mgmtauthorization.RoleAssignment{
 				Name: to.StringPtr("[guid(resourceGroup().id, 'RP / Reader')]"),
 				Type: to.StringPtr("Microsoft.Authorization/roleAssignments"),
-				RoleAssignmentPropertiesWithScope: &authorization.RoleAssignmentPropertiesWithScope{
+				RoleAssignmentPropertiesWithScope: &mgmtauthorization.RoleAssignmentPropertiesWithScope{
 					Scope:            to.StringPtr("[resourceGroup().id]"),
 					RoleDefinitionID: to.StringPtr("[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]"),
 					PrincipalID:      to.StringPtr("[parameters('rpServicePrincipalId')]"),
-					PrincipalType:    authorization.ServicePrincipal,
+					PrincipalType:    mgmtauthorization.ServicePrincipal,
 				},
 			},
 			APIVersion: apiVersions["authorization"],
 		},
 		{
-			Resource: &authorization.RoleAssignment{
+			Resource: &mgmtauthorization.RoleAssignment{
+				Name: to.StringPtr("[guid(resourceGroup().id, 'FP / Network Contributor')]"),
+				Type: to.StringPtr("Microsoft.Authorization/roleAssignments"),
+				RoleAssignmentPropertiesWithScope: &mgmtauthorization.RoleAssignmentPropertiesWithScope{
+					Scope:            to.StringPtr("[resourceGroup().id]"),
+					RoleDefinitionID: to.StringPtr("[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7')]"),
+					PrincipalID:      to.StringPtr("[parameters('fpServicePrincipalId')]"),
+					PrincipalType:    mgmtauthorization.ServicePrincipal,
+				},
+			},
+			APIVersion: apiVersions["authorization"],
+		},
+		{
+			Resource: &mgmtauthorization.RoleAssignment{
 				Name: to.StringPtr("[concat(parameters('databaseAccountName'), '/Microsoft.Authorization/', guid(resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName')), 'RP / DocumentDB Account Contributor'))]"),
 				Type: to.StringPtr("Microsoft.DocumentDB/databaseAccounts/providers/roleAssignments"),
-				RoleAssignmentPropertiesWithScope: &authorization.RoleAssignmentPropertiesWithScope{
+				RoleAssignmentPropertiesWithScope: &mgmtauthorization.RoleAssignmentPropertiesWithScope{
 					Scope:            to.StringPtr("[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]"),
 					RoleDefinitionID: to.StringPtr("[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '5bd9cd88-fe45-4216-938b-f97437e15450')]"),
 					PrincipalID:      to.StringPtr("[parameters('rpServicePrincipalId')]"),
-					PrincipalType:    authorization.ServicePrincipal,
+					PrincipalType:    mgmtauthorization.ServicePrincipal,
 				},
 			},
 			APIVersion: apiVersions["authorization"],
@@ -608,14 +648,14 @@ func (g *generator) rbac() []*arm.Resource {
 			},
 		},
 		{
-			Resource: &authorization.RoleAssignment{
-				Name: to.StringPtr("[concat(parameters('domainName'), '/Microsoft.Authorization/', guid(resourceId('Microsoft.Network/dnsZones', parameters('domainName')), 'RP / DNS Zone Contributor'))]"),
+			Resource: &mgmtauthorization.RoleAssignment{
+				Name: to.StringPtr("[concat(parameters('domainName'), '/Microsoft.Authorization/', guid(resourceId('Microsoft.Network/dnsZones', parameters('domainName')), 'FP / DNS Zone Contributor'))]"),
 				Type: to.StringPtr("Microsoft.Network/dnsZones/providers/roleAssignments"),
-				RoleAssignmentPropertiesWithScope: &authorization.RoleAssignmentPropertiesWithScope{
+				RoleAssignmentPropertiesWithScope: &mgmtauthorization.RoleAssignmentPropertiesWithScope{
 					Scope:            to.StringPtr("[resourceId('Microsoft.Network/dnsZones', parameters('domainName'))]"),
 					RoleDefinitionID: to.StringPtr("[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'befefa01-2a29-4197-83a8-272ff33ce314')]"),
-					PrincipalID:      to.StringPtr("[parameters('rpServicePrincipalId')]"),
-					PrincipalType:    authorization.ServicePrincipal,
+					PrincipalID:      to.StringPtr("[parameters('fpServicePrincipalId')]"),
+					PrincipalType:    mgmtauthorization.ServicePrincipal,
 				},
 			},
 			APIVersion: apiVersions["authorization"],
@@ -635,7 +675,7 @@ func (g *generator) template() *arm.Template {
 
 	if g.production {
 		t.Variables = map[string]interface{}{
-			"keyvaultAccessPolicies": []keyvault.AccessPolicyEntry{
+			"keyvaultAccessPolicies": []mgmtkeyvault.AccessPolicyEntry{
 				g.accessPolicyEntry(),
 			},
 		}
@@ -644,6 +684,7 @@ func (g *generator) template() *arm.Template {
 	params := []string{
 		"databaseAccountName",
 		"domainName",
+		"fpServicePrincipalId",
 		"keyvaultName",
 		"rpServicePrincipalId",
 	}
@@ -669,15 +710,19 @@ func (g *generator) template() *arm.Template {
 		}
 		t.Parameters["extraKeyvaultAccessPolicies"] = &arm.TemplateParameter{
 			Type:         "array",
-			DefaultValue: []keyvault.AccessPolicyEntry{},
+			DefaultValue: []mgmtkeyvault.AccessPolicyEntry{},
 		}
 	}
 
 	if g.production {
-		t.Resources = append(t.Resources, g.vnet(), g.pip(), g.lb(), g.vmss())
+		t.Resources = append(t.Resources, g.pip(), g.lb(), g.vmss())
+	}
+	t.Resources = append(t.Resources, g.zone(), g.vault(), g.vnet())
+	if g.production {
+		t.Resources = append(t.Resources, g.cosmosdb("'ARO'")...)
+	} else {
+		t.Resources = append(t.Resources, g.cosmosdb("parameters('databaseName')")...)
 	}
-	t.Resources = append(t.Resources, g.zone(), g.vault())
-	t.Resources = append(t.Resources, g.cosmosdb()...)
 	t.Resources = append(t.Resources, g.rbac()...)
 
 	return t
@@ -714,7 +759,31 @@ func GenerateRPTemplates() error {
 		}
 	}
 
-	return nil
+	t := &arm.Template{
+		Schema:         "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+		ContentVersion: "1.0.0.0",
+		Parameters: map[string]*arm.TemplateParameter{
+			"databaseAccountName": {
+				Type: "string",
+			},
+			"databaseName": {
+				Type: "string",
+			},
+		},
+	}
+
+	g := newGenerator(false)
+
+	t.Resources = append(t.Resources, g.database("parameters('databaseName')", false)...)
+
+	b, err := json.MarshalIndent(t, "", "    ")
+	if err != nil {
+		return err
+	}
+
+	b = append(b, byte('\n'))
+
+	return ioutil.WriteFile("databases-development.json", b, 0666)
 }
 
 func GenerateRPParameterTemplate() error {

pkg/env/dev.go

@@ -4,13 +4,19 @@ package env
 // Licensed under the Apache License 2.0.
 
 import (
+	"bufio"
 	"context"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"io/ioutil"
 	"net"
+	"net/http"
 	"os"
+	"time"
 
 	mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/authorization/mgmt/2015-07-01/authorization"
-	"github.com/Azure/azure-sdk-for-go/services/graphrbac/1.6/graphrbac"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/Azure/go-autorest/autorest/azure"
@@ -20,12 +26,21 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/Azure/ARO-RP/pkg/api"
-	"github.com/Azure/ARO-RP/pkg/util/azureclient/authorization"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/graphrbac"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization"
 	"github.com/Azure/ARO-RP/pkg/util/clientauthorizer"
 	"github.com/Azure/ARO-RP/pkg/util/instancemetadata"
-	utilpermissions "github.com/Azure/ARO-RP/pkg/util/permissions"
 )
 
+type conn struct {
+	net.Conn
+	r *bufio.Reader
+}
+
+func (c *conn) Read(b []byte) (int, error) {
+	return c.r.Read(b)
+}
+
 type refreshableAuthorizer struct {
 	autorest.Authorizer
 	sp *adal.ServicePrincipalToken
@@ -47,6 +62,10 @@ type dev struct {
 	permissions     authorization.PermissionsClient
 	roleassignments authorization.RoleAssignmentsClient
 	applications    graphrbac.ApplicationsClient
+
+	proxyPool       *x509.CertPool
+	proxyClientCert []byte
+	proxyClientKey  *rsa.PrivateKey
 }
 
 func newDev(ctx context.Context, log *logrus.Entry, instancemetadata instancemetadata.InstanceMetadata, clientauthorizer clientauthorizer.ClientAuthorizer) (*dev, error) {
@@ -56,7 +75,9 @@ func newDev(ctx context.Context, log *logrus.Entry, instancemetadata instancemet
 		"AZURE_FP_CLIENT_ID",
 		"AZURE_SUBSCRIPTION_ID",
 		"AZURE_TENANT_ID",
+		"DATABASE_NAME",
 		"LOCATION",
+		"PROXY_HOSTNAME",
 		"PULL_SECRET",
 		"RESOURCEGROUP",
 	} {
@@ -65,9 +86,7 @@ func newDev(ctx context.Context, log *logrus.Entry, instancemetadata instancemet
 		}
 	}
 
-	tenantID := os.Getenv("AZURE_TENANT_ID")
-
-	armAuthorizer, err := auth.NewClientCredentialsConfig(os.Getenv("AZURE_ARM_CLIENT_ID"), os.Getenv("AZURE_ARM_CLIENT_SECRET"), tenantID).Authorizer()
+	armAuthorizer, err := auth.NewClientCredentialsConfig(os.Getenv("AZURE_ARM_CLIENT_ID"), os.Getenv("AZURE_ARM_CLIENT_SECRET"), instancemetadata.TenantID()).Authorizer()
 	if err != nil {
 		return nil, err
 	}
@@ -75,7 +94,6 @@ func newDev(ctx context.Context, log *logrus.Entry, instancemetadata instancemet
 	d := &dev{
 		log:             log,
 		roleassignments: authorization.NewRoleAssignmentsClient(instancemetadata.SubscriptionID(), armAuthorizer),
-		applications:    graphrbac.NewApplicationsClient(tenantID),
 	}
 
 	d.prod, err = newProd(ctx, log, instancemetadata, clientauthorizer)
@@ -83,21 +101,115 @@ func newDev(ctx context.Context, log *logrus.Entry, instancemetadata instancemet
 		return nil, err
 	}
 
-	d.applications.Authorizer, err = d.FPAuthorizer(tenantID, azure.PublicCloud.GraphEndpoint)
+	fpGraphAuthorizer, err := d.FPAuthorizer(instancemetadata.TenantID(), azure.PublicCloud.GraphEndpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	fpAuthorizer, err := d.FPAuthorizer(tenantID, azure.PublicCloud.ResourceManagerEndpoint)
+	d.applications = graphrbac.NewApplicationsClient(instancemetadata.TenantID(), fpGraphAuthorizer)
+
+	fpAuthorizer, err := d.FPAuthorizer(instancemetadata.TenantID(), azure.PublicCloud.ResourceManagerEndpoint)
 	if err != nil {
 		return nil, err
 	}
 
 	d.permissions = authorization.NewPermissionsClient(instancemetadata.SubscriptionID(), fpAuthorizer)
 
+	b, err := ioutil.ReadFile("secrets/proxy.crt")
+	if err != nil {
+		return nil, err
+	}
+
+	cert, err := x509.ParseCertificate(b)
+	if err != nil {
+		return nil, err
+	}
+
+	d.proxyPool = x509.NewCertPool()
+	d.proxyPool.AddCert(cert)
+
+	d.proxyClientCert, err = ioutil.ReadFile("secrets/proxy-client.crt")
+	if err != nil {
+		return nil, err
+	}
+
+	b, err = ioutil.ReadFile("secrets/proxy-client.key")
+	if err != nil {
+		return nil, err
+	}
+
+	d.proxyClientKey, err = x509.ParsePKCS1PrivateKey(b)
+	if err != nil {
+		return nil, err
+	}
+
 	return d, nil
 }
 
+func (d *dev) DatabaseName() string {
+	return os.Getenv("DATABASE_NAME")
+}
+
+func (d *dev) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	if network != "tcp" {
+		return nil, fmt.Errorf("unimplemented network %q", network)
+	}
+
+	c, err := (&net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+	}).DialContext(ctx, network, os.Getenv("PROXY_HOSTNAME")+":443")
+	if err != nil {
+		return nil, err
+	}
+
+	c = tls.Client(c, &tls.Config{
+		RootCAs: d.proxyPool,
+		Certificates: []tls.Certificate{
+			{
+				Certificate: [][]byte{
+					d.proxyClientCert,
+				},
+				PrivateKey: d.proxyClientKey,
+			},
+		},
+		ServerName: "proxy",
+	})
+
+	err = c.(*tls.Conn).Handshake()
+	if err != nil {
+		c.Close()
+		return nil, err
+	}
+
+	r := bufio.NewReader(c)
+
+	req, err := http.NewRequest(http.MethodConnect, "", nil)
+	if err != nil {
+		c.Close()
+		return nil, err
+	}
+	req.Host = address
+
+	err = req.Write(c)
+	if err != nil {
+		c.Close()
+		return nil, err
+	}
+
+	resp, err := http.ReadResponse(r, req)
+	if err != nil {
+		c.Close()
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		c.Close()
+		return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode)
+	}
+
+	return &conn{Conn: c, r: r}, nil
+}
+
 func (d *dev) Listen() (net.Listener, error) {
 	// in dev mode there is no authentication, so for safety we only listen on
 	// localhost
@@ -145,25 +257,5 @@ func (d *dev) CreateARMResourceGroupRoleAssignment(ctx context.Context, fpAuthor
 	}
 
 	d.log.Print("development mode: refreshing authorizer")
-	err = fpAuthorizer.(*refreshableAuthorizer).Refresh()
-	if err != nil {
-		return err
-	}
-
-	// try removing the code below after a bit if we don't hit the error
-	permissions, err := d.permissions.ListForResourceGroup(ctx, oc.Properties.ResourceGroup)
-	if err != nil {
-		return err
-	}
-
-	ok, err := utilpermissions.CanDoAction(permissions, "Microsoft.Storage/storageAccounts/write")
-	if err != nil {
-		return err
-	}
-
-	if !ok {
-		return fmt.Errorf("Microsoft.Storage/storageAccounts/write permission not found")
-	}
-
-	return nil
+	return fpAuthorizer.(*refreshableAuthorizer).Refresh()
 }

pkg/env/env.go

@@ -15,7 +15,6 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/Azure/ARO-RP/pkg/util/clientauthorizer"
-	"github.com/Azure/ARO-RP/pkg/util/dns"
 	"github.com/Azure/ARO-RP/pkg/util/instancemetadata"
 )
 
@@ -23,8 +22,10 @@ type Interface interface {
 	clientauthorizer.ClientAuthorizer
 	instancemetadata.InstanceMetadata
 
-	CosmosDB(context.Context) (string, string)
-	DNS() dns.Manager
+	CosmosDB() (string, string)
+	DatabaseName() string
+	DialContext(context.Context, string, string) (net.Conn, error)
+	Domain() string
 	FPAuthorizer(string, string) (autorest.Authorizer, error)
 	GetSecret(context.Context, string) (*rsa.PrivateKey, []*x509.Certificate, error)
 	Listen() (net.Listener, error)

pkg/env/prod.go

@@ -12,18 +12,19 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb"
-	"github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
-	keyvaultmgmt "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2016-10-01/keyvault"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
 	"github.com/sirupsen/logrus"
 
+	basekeyvault "github.com/Azure/ARO-RP/pkg/util/azureclient/keyvault"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/dns"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/documentdb"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/keyvault"
 	"github.com/Azure/ARO-RP/pkg/util/clientauthorizer"
-	"github.com/Azure/ARO-RP/pkg/util/dns"
 	"github.com/Azure/ARO-RP/pkg/util/instancemetadata"
 )
 
@@ -31,13 +32,12 @@ type prod struct {
 	instancemetadata.InstanceMetadata
 	clientauthorizer.ClientAuthorizer
 
-	keyvault keyvault.BaseClient
+	keyvault basekeyvault.BaseClient
 
-	dns dns.Manager
-
-	vaultURI                 string
 	cosmosDBAccountName      string
 	cosmosDBPrimaryMasterKey string
+	domain                   string
+	vaultURI                 string
 
 	fpCertificate *x509.Certificate
 	fpPrivateKey  *rsa.PrivateKey
@@ -52,11 +52,16 @@ func newProd(ctx context.Context, log *logrus.Entry, instancemetadata instanceme
 		}
 	}
 
+	kvAuthorizer, err := auth.NewAuthorizerFromEnvironmentWithResource(azure.PublicCloud.ResourceIdentifiers.KeyVault)
+	if err != nil {
+		return nil, err
+	}
+
 	p := &prod{
 		InstanceMetadata: instancemetadata,
 		ClientAuthorizer: clientauthorizer,
 
-		keyvault: keyvault.New(),
+		keyvault: basekeyvault.New(kvAuthorizer),
 	}
 
 	rpAuthorizer, err := auth.NewAuthorizerFromEnvironment()
@@ -64,22 +69,17 @@ func newProd(ctx context.Context, log *logrus.Entry, instancemetadata instanceme
 		return nil, err
 	}
 
-	p.keyvault.Authorizer, err = auth.NewAuthorizerFromEnvironmentWithResource(azure.PublicCloud.ResourceIdentifiers.KeyVault)
-	if err != nil {
-		return nil, err
-	}
-
-	err = p.populateVaultURI(ctx, rpAuthorizer)
-	if err != nil {
-		return nil, err
-	}
-
 	err = p.populateCosmosDB(ctx, rpAuthorizer)
 	if err != nil {
 		return nil, err
 	}
 
-	p.dns, err = dns.NewManager(ctx, instancemetadata, rpAuthorizer)
+	err = p.populateDomain(ctx, rpAuthorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	err = p.populateVaultURI(ctx, rpAuthorizer)
 	if err != nil {
 		return nil, err
 	}
@@ -95,28 +95,8 @@ func newProd(ctx context.Context, log *logrus.Entry, instancemetadata instanceme
 	return p, nil
 }
 
-func (p *prod) populateVaultURI(ctx context.Context, rpAuthorizer autorest.Authorizer) error {
-	vaults := keyvaultmgmt.NewVaultsClient(p.SubscriptionID())
-	vaults.Authorizer = rpAuthorizer
-
-	page, err := vaults.ListByResourceGroup(ctx, p.ResourceGroup(), nil)
-	if err != nil {
-		return err
-	}
-
-	vs := page.Values()
-	if len(vs) != 1 {
-		return fmt.Errorf("found at least %d vaults, expected 1", len(vs))
-	}
-
-	p.vaultURI = *vs[0].Properties.VaultURI
-
-	return nil
-}
-
 func (p *prod) populateCosmosDB(ctx context.Context, rpAuthorizer autorest.Authorizer) error {
-	databaseaccounts := documentdb.NewDatabaseAccountsClient(p.SubscriptionID())
-	databaseaccounts.Authorizer = rpAuthorizer
+	databaseaccounts := documentdb.NewDatabaseAccountsClient(p.SubscriptionID(), rpAuthorizer)
 
 	accts, err := databaseaccounts.ListByResourceGroup(ctx, p.ResourceGroup())
 	if err != nil {
@@ -138,12 +118,57 @@ func (p *prod) populateCosmosDB(ctx context.Context, rpAuthorizer autorest.Autho
 	return nil
 }
 
-func (p *prod) CosmosDB(context.Context) (string, string) {
+func (p *prod) populateDomain(ctx context.Context, rpAuthorizer autorest.Authorizer) error {
+	zones := dns.NewZonesClient(p.SubscriptionID(), rpAuthorizer)
+
+	zs, err := zones.ListByResourceGroup(ctx, p.ResourceGroup(), nil)
+	if err != nil {
+		return err
+	}
+
+	if len(zs) != 1 {
+		return fmt.Errorf("found %d zones, expected 1", len(zs))
+	}
+
+	p.domain = *zs[0].Name
+
+	return nil
+}
+
+func (p *prod) populateVaultURI(ctx context.Context, rpAuthorizer autorest.Authorizer) error {
+	vaults := keyvault.NewVaultsClient(p.SubscriptionID(), rpAuthorizer)
+
+	vs, err := vaults.ListByResourceGroup(ctx, p.ResourceGroup(), nil)
+	if err != nil {
+		return err
+	}
+
+	if len(vs) != 1 {
+		return fmt.Errorf("found %d vaults, expected 1", len(vs))
+	}
+
+	p.vaultURI = *vs[0].Properties.VaultURI
+
+	return nil
+}
+
+func (p *prod) CosmosDB() (string, string) {
 	return p.cosmosDBAccountName, p.cosmosDBPrimaryMasterKey
 }
 
-func (p *prod) DNS() dns.Manager {
-	return p.dns
+func (p *prod) DatabaseName() string {
+	return "ARO"
+}
+
+func (p *prod) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	return (&net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+	}).DialContext(ctx, network, address)
+}
+
+func (p *prod) Domain() string {
+	return p.domain
 }
 
 func (p *prod) FPAuthorizer(tenantID, resource string) (autorest.Authorizer, error) {

pkg/frontend/security_test.go

@@ -21,7 +21,7 @@ import (
 func TestSecurity(t *testing.T) {
 	ctx := context.Background()
 
-	validclientkey, validclientcerts, err := utiltls.GenerateKeyAndCertificate("validclient", true)
+	validclientkey, validclientcerts, err := utiltls.GenerateKeyAndCertificate("validclient", nil, nil, false, true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -31,12 +31,12 @@ func TestSecurity(t *testing.T) {
 
 	env := env.NewTest(l, validclientcerts[0].Raw)
 
-	env.TLSKey, env.TLSCerts, err = utiltls.GenerateKeyAndCertificate("server", false)
+	env.TLSKey, env.TLSCerts, err = utiltls.GenerateKeyAndCertificate("server", nil, nil, false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	invalidclientkey, invalidclientcerts, err := utiltls.GenerateKeyAndCertificate("invalidclient", true)
+	invalidclientkey, invalidclientcerts, err := utiltls.GenerateKeyAndCertificate("invalidclient", nil, nil, false, true)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/install/0-installstorage.go

@@ -12,9 +12,11 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
-	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
-	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage"
+	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+	mgmtresources "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
+	mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
 	"github.com/openshift/installer/pkg/asset/installconfig"
@@ -37,7 +39,7 @@ var apiVersions = map[string]string{
 	"storage":       "2019-04-01",
 }
 
-func (i *Installer) installStorage(ctx context.Context, doc *api.OpenShiftClusterDocument, installConfig *installconfig.InstallConfig, platformCreds *installconfig.PlatformCreds, image *releaseimage.Image) error {
+func (i *Installer) installStorage(ctx context.Context, installConfig *installconfig.InstallConfig, platformCreds *installconfig.PlatformCreds, image *releaseimage.Image) error {
 	clusterID := &installconfig.ClusterID{
 		UUID:    uuid.NewV4().String(),
 		InfraID: "aro",
@@ -62,20 +64,20 @@ func (i *Installer) installStorage(ctx context.Context, doc *api.OpenShiftCluste
 	bootstrap := g[reflect.TypeOf(&bootstrap.Bootstrap{})].(*bootstrap.Bootstrap)
 
 	i.log.Print("creating resource group")
-	group := resources.Group{
+	group := mgmtresources.Group{
 		Location:  &installConfig.Config.Azure.Region,
-		ManagedBy: to.StringPtr(doc.OpenShiftCluster.ID),
+		ManagedBy: to.StringPtr(i.doc.OpenShiftCluster.ID),
 	}
 	if _, ok := i.env.(env.Dev); ok {
 		group.ManagedBy = nil
 	}
-	_, err := i.groups.CreateOrUpdate(ctx, doc.OpenShiftCluster.Properties.ResourceGroup, group)
+	_, err := i.groups.CreateOrUpdate(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup, group)
 	if err != nil {
 		return err
 	}
 
 	if development, ok := i.env.(env.Dev); ok {
-		err = development.CreateARMResourceGroupRoleAssignment(ctx, i.fpAuthorizer, doc.OpenShiftCluster)
+		err = development.CreateARMResourceGroupRoleAssignment(ctx, i.fpAuthorizer, i.doc.OpenShiftCluster)
 		if err != nil {
 			return err
 		}
@@ -87,63 +89,63 @@ func (i *Installer) installStorage(ctx context.Context, doc *api.OpenShiftCluste
 			ContentVersion: "1.0.0.0",
 			Resources: []*arm.Resource{
 				{
-					Resource: &storage.Account{
-						Sku: &storage.Sku{
+					Resource: &mgmtstorage.Account{
+						Sku: &mgmtstorage.Sku{
 							Name: "Standard_LRS",
 						},
-						Name:     to.StringPtr("cluster" + doc.OpenShiftCluster.Properties.StorageSuffix),
+						Name:     to.StringPtr("cluster" + i.doc.OpenShiftCluster.Properties.StorageSuffix),
 						Location: &installConfig.Config.Azure.Region,
 						Type:     to.StringPtr("Microsoft.Storage/storageAccounts"),
 					},
 					APIVersion: apiVersions["storage"],
 				},
 				{
-					Resource: &storage.BlobContainer{
-						Name: to.StringPtr("cluster" + doc.OpenShiftCluster.Properties.StorageSuffix + "/default/ignition"),
+					Resource: &mgmtstorage.BlobContainer{
+						Name: to.StringPtr("cluster" + i.doc.OpenShiftCluster.Properties.StorageSuffix + "/default/ignition"),
 						Type: to.StringPtr("Microsoft.Storage/storageAccounts/blobServices/containers"),
 					},
 					APIVersion: apiVersions["storage"],
 					DependsOn: []string{
-						"Microsoft.Storage/storageAccounts/cluster" + doc.OpenShiftCluster.Properties.StorageSuffix,
+						"Microsoft.Storage/storageAccounts/cluster" + i.doc.OpenShiftCluster.Properties.StorageSuffix,
 					},
 				},
 				{
-					Resource: &storage.BlobContainer{
-						Name: to.StringPtr("cluster" + doc.OpenShiftCluster.Properties.StorageSuffix + "/default/aro"),
+					Resource: &mgmtstorage.BlobContainer{
+						Name: to.StringPtr("cluster" + i.doc.OpenShiftCluster.Properties.StorageSuffix + "/default/aro"),
 						Type: to.StringPtr("Microsoft.Storage/storageAccounts/blobServices/containers"),
 					},
 					APIVersion: apiVersions["storage"],
 					DependsOn: []string{
-						"Microsoft.Storage/storageAccounts/cluster" + doc.OpenShiftCluster.Properties.StorageSuffix,
+						"Microsoft.Storage/storageAccounts/cluster" + i.doc.OpenShiftCluster.Properties.StorageSuffix,
 					},
 				},
 				{
-					Resource: &network.SecurityGroup{
-						SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{
-							SecurityRules: &[]network.SecurityRule{
+					Resource: &mgmtnetwork.SecurityGroup{
+						SecurityGroupPropertiesFormat: &mgmtnetwork.SecurityGroupPropertiesFormat{
+							SecurityRules: &[]mgmtnetwork.SecurityRule{
 								{
-									SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
-										Protocol:                 network.SecurityRuleProtocolTCP,
+									SecurityRulePropertiesFormat: &mgmtnetwork.SecurityRulePropertiesFormat{
+										Protocol:                 mgmtnetwork.SecurityRuleProtocolTCP,
 										SourcePortRange:          to.StringPtr("*"),
 										DestinationPortRange:     to.StringPtr("6443"),
 										SourceAddressPrefix:      to.StringPtr("*"),
 										DestinationAddressPrefix: to.StringPtr("*"),
-										Access:                   network.SecurityRuleAccessAllow,
+										Access:                   mgmtnetwork.SecurityRuleAccessAllow,
 										Priority:                 to.Int32Ptr(101),
-										Direction:                network.SecurityRuleDirectionInbound,
+										Direction:                mgmtnetwork.SecurityRuleDirectionInbound,
 									},
 									Name: to.StringPtr("apiserver_in"),
 								},
 								{
-									SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
-										Protocol:                 network.SecurityRuleProtocolTCP,
+									SecurityRulePropertiesFormat: &mgmtnetwork.SecurityRulePropertiesFormat{
+										Protocol:                 mgmtnetwork.SecurityRuleProtocolTCP,
 										SourcePortRange:          to.StringPtr("*"),
 										DestinationPortRange:     to.StringPtr("22"),
 										SourceAddressPrefix:      to.StringPtr("*"),
 										DestinationAddressPrefix: to.StringPtr("*"),
-										Access:                   network.SecurityRuleAccessAllow,
+										Access:                   mgmtnetwork.SecurityRuleAccessAllow,
 										Priority:                 to.Int32Ptr(103),
-										Direction:                network.SecurityRuleDirectionInbound,
+										Direction:                mgmtnetwork.SecurityRuleDirectionInbound,
 									},
 									Name: to.StringPtr("bootstrap_ssh_in"),
 								},
@@ -156,7 +158,7 @@ func (i *Installer) installStorage(ctx context.Context, doc *api.OpenShiftCluste
 					APIVersion: apiVersions["network"],
 				},
 				{
-					Resource: &network.SecurityGroup{
+					Resource: &mgmtnetwork.SecurityGroup{
 						Name:     to.StringPtr("aro-node-nsg"),
 						Type:     to.StringPtr("Microsoft.Network/networkSecurityGroups"),
 						Location: &installConfig.Config.Azure.Region,
@@ -167,19 +169,29 @@ func (i *Installer) installStorage(ctx context.Context, doc *api.OpenShiftCluste
 		}
 
 		i.log.Print("deploying storage template")
-		err = i.deployments.CreateOrUpdateAndWait(ctx, doc.OpenShiftCluster.Properties.ResourceGroup, "azuredeploy", resources.Deployment{
-			Properties: &resources.DeploymentProperties{
+		err = i.deployments.CreateOrUpdateAndWait(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup, "azuredeploy", mgmtresources.Deployment{
+			Properties: &mgmtresources.DeploymentProperties{
 				Template: t,
-				Mode:     resources.Incremental,
+				Mode:     mgmtresources.Incremental,
 			},
 		})
 		if err != nil {
-			return err
+			if detailedError, ok := err.(autorest.DetailedError); ok {
+				if requestError, ok := detailedError.Original.(azure.RequestError); ok &&
+					requestError.ServiceError != nil &&
+					requestError.ServiceError.Code == "DeploymentActive" {
+					i.log.Print("waiting for storage template")
+					err = i.deployments.Wait(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup, "azuredeploy")
+				}
+			}
+			if err != nil {
+				return err
+			}
 		}
 	}
 
 	{
-		blobService, err := i.getBlobService(ctx, doc.OpenShiftCluster)
+		blobService, err := i.getBlobService(ctx)
 		if err != nil {
 			return err
 		}
@@ -205,8 +217,8 @@ func (i *Installer) installStorage(ctx context.Context, doc *api.OpenShiftCluste
 	}
 
 	for _, subnetID := range []string{
-		doc.OpenShiftCluster.Properties.MasterProfile.SubnetID,
-		doc.OpenShiftCluster.Properties.WorkerProfiles[0].SubnetID,
+		i.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID,
+		i.doc.OpenShiftCluster.Properties.WorkerProfiles[0].SubnetID,
 	} {
 		i.log.Printf("attaching network security group to subnet %s", subnetID)
 
@@ -217,10 +229,10 @@ func (i *Installer) installStorage(ctx context.Context, doc *api.OpenShiftCluste
 		}
 
 		if s.SubnetPropertiesFormat == nil {
-			s.SubnetPropertiesFormat = &network.SubnetPropertiesFormat{}
+			s.SubnetPropertiesFormat = &mgmtnetwork.SubnetPropertiesFormat{}
 		}
 
-		nsgID, err := subnet.NetworkSecurityGroupID(doc.OpenShiftCluster, subnetID)
+		nsgID, err := subnet.NetworkSecurityGroupID(i.doc.OpenShiftCluster, subnetID)
 		if err != nil {
 			return err
 		}
@@ -233,7 +245,7 @@ func (i *Installer) installStorage(ctx context.Context, doc *api.OpenShiftCluste
 			return fmt.Errorf("tried to overwrite non-nil network security group")
 		}
 
-		s.SubnetPropertiesFormat.NetworkSecurityGroup = &network.SecurityGroup{
+		s.SubnetPropertiesFormat.NetworkSecurityGroup = &mgmtnetwork.SecurityGroup{
 			ID: to.StringPtr(nsgID),
 		}
 
@@ -243,7 +255,7 @@ func (i *Installer) installStorage(ctx context.Context, doc *api.OpenShiftCluste
 		}
 	}
 
-	_, err = i.db.Patch(doc.Key, func(doc *api.OpenShiftClusterDocument) error {
+	i.doc, err = i.db.Patch(i.doc.Key, func(doc *api.OpenShiftClusterDocument) error {
 		// used for the SAS token with which the bootstrap node retrieves its
 		// ignition payload
 		doc.OpenShiftCluster.Properties.Install.Now = time.Now().UTC()

pkg/install/1-installresources.go

@@ -11,12 +11,12 @@ import (
 	"reflect"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/services/authorization/mgmt/2015-07-01/authorization"
-	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
-	"github.com/Azure/azure-sdk-for-go/services/graphrbac/1.6/graphrbac"
-	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
-	"github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns"
-	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
+	mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/authorization/mgmt/2015-07-01/authorization"
+	mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
+	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+	mgmtprivatedns "github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns"
+	mgmtresources "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
+	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
 	"github.com/Azure/go-autorest/autorest/to"
@@ -27,14 +27,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 
-	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/util/arm"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/graphrbac"
 	"github.com/Azure/ARO-RP/pkg/util/restconfig"
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )
 
-func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClusterDocument) error {
-	g, err := i.getGraph(ctx, doc.OpenShiftCluster)
+func (i *Installer) installResources(ctx context.Context) error {
+	g, err := i.getGraph(ctx)
 	if err != nil {
 		return err
 	}
@@ -43,12 +43,12 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 	machinesMaster := g[reflect.TypeOf(&machines.Master{})].(*machines.Master)
 	machineMaster := g[reflect.TypeOf(&machine.Master{})].(*machine.Master)
 
-	vnetID, _, err := subnet.Split(doc.OpenShiftCluster.Properties.MasterProfile.SubnetID)
+	vnetID, _, err := subnet.Split(i.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID)
 	if err != nil {
 		return err
 	}
 
-	masterSubnet, err := i.subnets.Get(ctx, doc.OpenShiftCluster.Properties.MasterProfile.SubnetID)
+	masterSubnet, err := i.subnets.Get(ctx, i.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID)
 	if err != nil {
 		return err
 	}
@@ -65,9 +65,9 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 		lbIP = cidr.Dec(cidr.Dec(last))
 	}
 
-	srvRecords := make([]privatedns.SrvRecord, len(machinesMaster.MachineFiles))
+	srvRecords := make([]mgmtprivatedns.SrvRecord, len(machinesMaster.MachineFiles))
 	for i := 0; i < len(machinesMaster.MachineFiles); i++ {
-		srvRecords[i] = privatedns.SrvRecord{
+		srvRecords[i] = mgmtprivatedns.SrvRecord{
 			Priority: to.Int32Ptr(10),
 			Weight:   to.Int32Ptr(10),
 			Port:     to.Int32Ptr(2380),
@@ -77,18 +77,17 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 
 	var objectID string
 	{
-		spp := &doc.OpenShiftCluster.Properties.ServicePrincipalProfile
+		spp := &i.doc.OpenShiftCluster.Properties.ServicePrincipalProfile
 
 		conf := auth.NewClientCredentialsConfig(spp.ClientID, spp.ClientSecret, spp.TenantID)
 		conf.Resource = azure.PublicCloud.GraphEndpoint
 
-		spAuthorizer, err := conf.Authorizer()
+		spGraphAuthorizer, err := conf.Authorizer()
 		if err != nil {
 			return err
 		}
 
-		applications := graphrbac.NewApplicationsClient(spp.TenantID)
-		applications.Authorizer = spAuthorizer
+		applications := graphrbac.NewApplicationsClient(spp.TenantID, spGraphAuthorizer)
 
 		res, err := applications.GetServicePrincipalsIDByAppID(ctx, spp.ClientID)
 		if err != nil {
@@ -109,10 +108,10 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 			},
 			Resources: []*arm.Resource{
 				{
-					Resource: &authorization.RoleAssignment{
+					Resource: &mgmtauthorization.RoleAssignment{
 						Name: to.StringPtr("[guid(resourceGroup().id, 'SP / Contributor')]"),
 						Type: to.StringPtr("Microsoft.Authorization/roleAssignments"),
-						Properties: &authorization.RoleAssignmentPropertiesWithScope{
+						Properties: &mgmtauthorization.RoleAssignmentPropertiesWithScope{
 							Scope:            to.StringPtr("[resourceGroup().id]"),
 							RoleDefinitionID: to.StringPtr("[resourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]"),
 							PrincipalID:      to.StringPtr(objectID),
@@ -121,7 +120,7 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					APIVersion: apiVersions["authorization"],
 				},
 				{
-					Resource: &privatedns.PrivateZone{
+					Resource: &mgmtprivatedns.PrivateZone{
 						Name:     to.StringPtr(installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain),
 						Type:     to.StringPtr("Microsoft.Network/privateDnsZones"),
 						Location: to.StringPtr("global"),
@@ -129,12 +128,12 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					APIVersion: apiVersions["privatedns"],
 				},
 				{
-					Resource: &privatedns.RecordSet{
+					Resource: &mgmtprivatedns.RecordSet{
 						Name: to.StringPtr(installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain + "/api-int"),
 						Type: to.StringPtr("Microsoft.Network/privateDnsZones/A"),
-						RecordSetProperties: &privatedns.RecordSetProperties{
+						RecordSetProperties: &mgmtprivatedns.RecordSetProperties{
 							TTL: to.Int64Ptr(300),
-							ARecords: &[]privatedns.ARecord{
+							ARecords: &[]mgmtprivatedns.ARecord{
 								{
 									Ipv4Address: to.StringPtr(lbIP.String()),
 								},
@@ -147,12 +146,12 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					},
 				},
 				{
-					Resource: &privatedns.RecordSet{
+					Resource: &mgmtprivatedns.RecordSet{
 						Name: to.StringPtr(installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain + "/api"),
 						Type: to.StringPtr("Microsoft.Network/privateDnsZones/A"),
-						RecordSetProperties: &privatedns.RecordSetProperties{
+						RecordSetProperties: &mgmtprivatedns.RecordSetProperties{
 							TTL: to.Int64Ptr(300),
-							ARecords: &[]privatedns.ARecord{
+							ARecords: &[]mgmtprivatedns.ARecord{
 								{
 									Ipv4Address: to.StringPtr(lbIP.String()),
 								},
@@ -165,10 +164,10 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					},
 				},
 				{
-					Resource: &privatedns.RecordSet{
+					Resource: &mgmtprivatedns.RecordSet{
 						Name: to.StringPtr(installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain + "/_etcd-server-ssl._tcp"),
 						Type: to.StringPtr("Microsoft.Network/privateDnsZones/SRV"),
-						RecordSetProperties: &privatedns.RecordSetProperties{
+						RecordSetProperties: &mgmtprivatedns.RecordSetProperties{
 							TTL:        to.Int64Ptr(60),
 							SrvRecords: &srvRecords,
 						},
@@ -179,12 +178,12 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					},
 				},
 				{
-					Resource: &privatedns.RecordSet{
+					Resource: &mgmtprivatedns.RecordSet{
 						Name: to.StringPtr("[concat('" + installConfig.Config.ObjectMeta.Name + "." + installConfig.Config.BaseDomain + "/etcd-', copyIndex())]"),
 						Type: to.StringPtr("Microsoft.Network/privateDnsZones/A"),
-						RecordSetProperties: &privatedns.RecordSetProperties{
+						RecordSetProperties: &mgmtprivatedns.RecordSetProperties{
 							TTL: to.Int64Ptr(60),
-							ARecords: &[]privatedns.ARecord{
+							ARecords: &[]mgmtprivatedns.ARecord{
 								{
 									Ipv4Address: to.StringPtr("[reference(resourceId('Microsoft.Network/networkInterfaces', concat('aro-master', copyIndex(), '-nic')), '2019-07-01').ipConfigurations[0].properties.privateIPAddress]"),
 								},
@@ -202,9 +201,9 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					},
 				},
 				{
-					Resource: &privatedns.VirtualNetworkLink{
-						VirtualNetworkLinkProperties: &privatedns.VirtualNetworkLinkProperties{
-							VirtualNetwork: &privatedns.SubResource{
+					Resource: &mgmtprivatedns.VirtualNetworkLink{
+						VirtualNetworkLinkProperties: &mgmtprivatedns.VirtualNetworkLinkProperties{
+							VirtualNetwork: &mgmtprivatedns.SubResource{
 								ID: to.StringPtr(vnetID),
 							},
 							RegistrationEnabled: to.BoolPtr(false),
@@ -219,9 +218,47 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 						"privatednscopy",
 					},
 				},
+				{
+					Resource: &mgmtnetwork.PrivateLinkService{
+						PrivateLinkServiceProperties: &mgmtnetwork.PrivateLinkServiceProperties{
+							LoadBalancerFrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
+								{
+									ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'aro-internal-lb', 'internal-lb-ip')]"),
+								},
+							},
+							IPConfigurations: &[]mgmtnetwork.PrivateLinkServiceIPConfiguration{
+								{
+									PrivateLinkServiceIPConfigurationProperties: &mgmtnetwork.PrivateLinkServiceIPConfigurationProperties{
+										Subnet: &mgmtnetwork.Subnet{
+											ID: to.StringPtr(i.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID),
+										},
+									},
+									Name: to.StringPtr("aro-pls-nic"),
+								},
+							},
+							Visibility: &mgmtnetwork.PrivateLinkServicePropertiesVisibility{
+								Subscriptions: &[]string{
+									i.env.SubscriptionID(),
+								},
+							},
+							AutoApproval: &mgmtnetwork.PrivateLinkServicePropertiesAutoApproval{
+								Subscriptions: &[]string{
+									i.env.SubscriptionID(),
+								},
+							},
+						},
+						Name:     to.StringPtr("aro-pls"),
+						Type:     to.StringPtr("Microsoft.Network/privateLinkServices"),
+						Location: &installConfig.Config.Azure.Region,
+					},
+					APIVersion: apiVersions["network"],
+					DependsOn: []string{
+						"Microsoft.Network/loadBalancers/aro-internal-lb",
+					},
+				},
 				{
 					// TODO: upstream doesn't appear to wire this in to any vnet - investigate.
-					Resource: &network.RouteTable{
+					Resource: &mgmtnetwork.RouteTable{
 						Name:     to.StringPtr("aro-node-routetable"),
 						Type:     to.StringPtr("Microsoft.Network/routeTables"),
 						Location: &installConfig.Config.Azure.Region,
@@ -230,12 +267,12 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 				},
 				{
 					// TODO: we will want to remove this
-					Resource: &network.PublicIPAddress{
-						Sku: &network.PublicIPAddressSku{
-							Name: network.PublicIPAddressSkuNameStandard,
+					Resource: &mgmtnetwork.PublicIPAddress{
+						Sku: &mgmtnetwork.PublicIPAddressSku{
+							Name: mgmtnetwork.PublicIPAddressSkuNameStandard,
 						},
-						PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
-							PublicIPAllocationMethod: network.Static,
+						PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{
+							PublicIPAllocationMethod: mgmtnetwork.Static,
 						},
 						Name:     to.StringPtr("aro-bootstrap-pip"),
 						Type:     to.StringPtr("Microsoft.Network/publicIPAddresses"),
@@ -244,14 +281,14 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					APIVersion: apiVersions["network"],
 				},
 				{
-					Resource: &network.PublicIPAddress{
-						Sku: &network.PublicIPAddressSku{
-							Name: network.PublicIPAddressSkuNameStandard,
+					Resource: &mgmtnetwork.PublicIPAddress{
+						Sku: &mgmtnetwork.PublicIPAddressSku{
+							Name: mgmtnetwork.PublicIPAddressSkuNameStandard,
 						},
-						PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
-							PublicIPAllocationMethod: network.Static,
-							DNSSettings: &network.PublicIPAddressDNSSettings{
-								DomainNameLabel: &doc.OpenShiftCluster.Properties.DomainName,
+						PublicIPAddressPropertiesFormat: &mgmtnetwork.PublicIPAddressPropertiesFormat{
+							PublicIPAllocationMethod: mgmtnetwork.Static,
+							DNSSettings: &mgmtnetwork.PublicIPAddressDNSSettings{
+								DomainNameLabel: &i.doc.OpenShiftCluster.Properties.DomainName,
 							},
 						},
 						Name:     to.StringPtr("aro-pip"),
@@ -261,40 +298,40 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					APIVersion: apiVersions["network"],
 				},
 				{
-					Resource: &network.LoadBalancer{
-						Sku: &network.LoadBalancerSku{
-							Name: network.LoadBalancerSkuNameStandard,
+					Resource: &mgmtnetwork.LoadBalancer{
+						Sku: &mgmtnetwork.LoadBalancerSku{
+							Name: mgmtnetwork.LoadBalancerSkuNameStandard,
 						},
-						LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{
-							FrontendIPConfigurations: &[]network.FrontendIPConfiguration{
+						LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{
+							FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
 								{
-									FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
-										PublicIPAddress: &network.PublicIPAddress{
+									FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{
+										PublicIPAddress: &mgmtnetwork.PublicIPAddress{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/publicIPAddresses', 'aro-pip')]"),
 										},
 									},
 									Name: to.StringPtr("public-lb-ip"),
 								},
 							},
-							BackendAddressPools: &[]network.BackendAddressPool{
+							BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
 								{
 									Name: to.StringPtr("aro-public-lb-control-plane"),
 								},
 							},
-							LoadBalancingRules: &[]network.LoadBalancingRule{
+							LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{
 								{
-									LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-										FrontendIPConfiguration: &network.SubResource{
+									LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
+										FrontendIPConfiguration: &mgmtnetwork.SubResource{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'aro-public-lb', 'public-lb-ip')]"),
 										},
-										BackendAddressPool: &network.SubResource{
+										BackendAddressPool: &mgmtnetwork.SubResource{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'aro-public-lb', 'aro-public-lb-control-plane')]"),
 										},
-										Probe: &network.SubResource{
+										Probe: &mgmtnetwork.SubResource{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'aro-public-lb', 'api-internal-probe')]"),
 										},
-										Protocol:             network.TransportProtocolTCP,
-										LoadDistribution:     network.LoadDistributionDefault,
+										Protocol:             mgmtnetwork.TransportProtocolTCP,
+										LoadDistribution:     mgmtnetwork.LoadDistributionDefault,
 										FrontendPort:         to.Int32Ptr(6443),
 										BackendPort:          to.Int32Ptr(6443),
 										IdleTimeoutInMinutes: to.Int32Ptr(30),
@@ -302,10 +339,10 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 									Name: to.StringPtr("api-internal"),
 								},
 							},
-							Probes: &[]network.Probe{
+							Probes: &[]mgmtnetwork.Probe{
 								{
-									ProbePropertiesFormat: &network.ProbePropertiesFormat{
-										Protocol:          network.ProbeProtocolTCP,
+									ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
+										Protocol:          mgmtnetwork.ProbeProtocolTCP,
 										Port:              to.Int32Ptr(6443),
 										IntervalInSeconds: to.Int32Ptr(10),
 										NumberOfProbes:    to.Int32Ptr(3),
@@ -325,42 +362,42 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					},
 				},
 				{
-					Resource: &network.LoadBalancer{
-						Sku: &network.LoadBalancerSku{
-							Name: network.LoadBalancerSkuNameStandard,
+					Resource: &mgmtnetwork.LoadBalancer{
+						Sku: &mgmtnetwork.LoadBalancerSku{
+							Name: mgmtnetwork.LoadBalancerSkuNameStandard,
 						},
-						LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{
-							FrontendIPConfigurations: &[]network.FrontendIPConfiguration{
+						LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{
+							FrontendIPConfigurations: &[]mgmtnetwork.FrontendIPConfiguration{
 								{
-									FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
+									FrontendIPConfigurationPropertiesFormat: &mgmtnetwork.FrontendIPConfigurationPropertiesFormat{
 										PrivateIPAddress:          to.StringPtr(lbIP.String()),
-										PrivateIPAllocationMethod: network.Static,
-										Subnet: &network.Subnet{
-											ID: to.StringPtr(doc.OpenShiftCluster.Properties.MasterProfile.SubnetID),
+										PrivateIPAllocationMethod: mgmtnetwork.Static,
+										Subnet: &mgmtnetwork.Subnet{
+											ID: to.StringPtr(i.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID),
 										},
 									},
 									Name: to.StringPtr("internal-lb-ip"),
 								},
 							},
-							BackendAddressPools: &[]network.BackendAddressPool{
+							BackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
 								{
 									Name: to.StringPtr("aro-internal-controlplane"),
 								},
 							},
-							LoadBalancingRules: &[]network.LoadBalancingRule{
+							LoadBalancingRules: &[]mgmtnetwork.LoadBalancingRule{
 								{
-									LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-										FrontendIPConfiguration: &network.SubResource{
+									LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
+										FrontendIPConfiguration: &mgmtnetwork.SubResource{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'aro-internal-lb', 'internal-lb-ip')]"),
 										},
-										BackendAddressPool: &network.SubResource{
+										BackendAddressPool: &mgmtnetwork.SubResource{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'aro-internal-lb', 'aro-internal-controlplane')]"),
 										},
-										Probe: &network.SubResource{
+										Probe: &mgmtnetwork.SubResource{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'aro-internal-lb', 'api-internal-probe')]"),
 										},
-										Protocol:             network.TransportProtocolTCP,
-										LoadDistribution:     network.LoadDistributionDefault,
+										Protocol:             mgmtnetwork.TransportProtocolTCP,
+										LoadDistribution:     mgmtnetwork.LoadDistributionDefault,
 										FrontendPort:         to.Int32Ptr(6443),
 										BackendPort:          to.Int32Ptr(6443),
 										IdleTimeoutInMinutes: to.Int32Ptr(30),
@@ -368,18 +405,18 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 									Name: to.StringPtr("api-internal"),
 								},
 								{
-									LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-										FrontendIPConfiguration: &network.SubResource{
+									LoadBalancingRulePropertiesFormat: &mgmtnetwork.LoadBalancingRulePropertiesFormat{
+										FrontendIPConfiguration: &mgmtnetwork.SubResource{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', 'aro-internal-lb', 'internal-lb-ip')]"),
 										},
-										BackendAddressPool: &network.SubResource{
+										BackendAddressPool: &mgmtnetwork.SubResource{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'aro-internal-lb', 'aro-internal-controlplane')]"),
 										},
-										Probe: &network.SubResource{
+										Probe: &mgmtnetwork.SubResource{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'aro-internal-lb', 'sint-probe')]"),
 										},
-										Protocol:             network.TransportProtocolTCP,
-										LoadDistribution:     network.LoadDistributionDefault,
+										Protocol:             mgmtnetwork.TransportProtocolTCP,
+										LoadDistribution:     mgmtnetwork.LoadDistributionDefault,
 										FrontendPort:         to.Int32Ptr(22623),
 										BackendPort:          to.Int32Ptr(22623),
 										IdleTimeoutInMinutes: to.Int32Ptr(30),
@@ -387,10 +424,10 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 									Name: to.StringPtr("sint"),
 								},
 							},
-							Probes: &[]network.Probe{
+							Probes: &[]mgmtnetwork.Probe{
 								{
-									ProbePropertiesFormat: &network.ProbePropertiesFormat{
-										Protocol:          network.ProbeProtocolTCP,
+									ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
+										Protocol:          mgmtnetwork.ProbeProtocolTCP,
 										Port:              to.Int32Ptr(6443),
 										IntervalInSeconds: to.Int32Ptr(10),
 										NumberOfProbes:    to.Int32Ptr(3),
@@ -398,8 +435,8 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 									Name: to.StringPtr("api-internal-probe"),
 								},
 								{
-									ProbePropertiesFormat: &network.ProbePropertiesFormat{
-										Protocol:          network.ProbeProtocolTCP,
+									ProbePropertiesFormat: &mgmtnetwork.ProbePropertiesFormat{
+										Protocol:          mgmtnetwork.ProbeProtocolTCP,
 										Port:              to.Int32Ptr(22623),
 										IntervalInSeconds: to.Int32Ptr(10),
 										NumberOfProbes:    to.Int32Ptr(3),
@@ -415,12 +452,12 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					APIVersion: apiVersions["network"],
 				},
 				{
-					Resource: &network.Interface{
-						InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
-							IPConfigurations: &[]network.InterfaceIPConfiguration{
+					Resource: &mgmtnetwork.Interface{
+						InterfacePropertiesFormat: &mgmtnetwork.InterfacePropertiesFormat{
+							IPConfigurations: &[]mgmtnetwork.InterfaceIPConfiguration{
 								{
-									InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
-										LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
+									InterfaceIPConfigurationPropertiesFormat: &mgmtnetwork.InterfaceIPConfigurationPropertiesFormat{
+										LoadBalancerBackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
 											{
 												ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'aro-public-lb', 'aro-public-lb-control-plane')]"),
 											},
@@ -428,10 +465,10 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 												ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'aro-internal-lb', 'aro-internal-controlplane')]"),
 											},
 										},
-										Subnet: &network.Subnet{
-											ID: to.StringPtr(doc.OpenShiftCluster.Properties.MasterProfile.SubnetID),
+										Subnet: &mgmtnetwork.Subnet{
+											ID: to.StringPtr(i.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID),
 										},
-										PublicIPAddress: &network.PublicIPAddress{
+										PublicIPAddress: &mgmtnetwork.PublicIPAddress{
 											ID: to.StringPtr("[resourceId('Microsoft.Network/publicIPAddresses', 'aro-bootstrap-pip')]"),
 										},
 									},
@@ -451,12 +488,12 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					},
 				},
 				{
-					Resource: &network.Interface{
-						InterfacePropertiesFormat: &network.InterfacePropertiesFormat{
-							IPConfigurations: &[]network.InterfaceIPConfiguration{
+					Resource: &mgmtnetwork.Interface{
+						InterfacePropertiesFormat: &mgmtnetwork.InterfacePropertiesFormat{
+							IPConfigurations: &[]mgmtnetwork.InterfaceIPConfiguration{
 								{
-									InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{
-										LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
+									InterfaceIPConfigurationPropertiesFormat: &mgmtnetwork.InterfaceIPConfigurationPropertiesFormat{
+										LoadBalancerBackendAddressPools: &[]mgmtnetwork.BackendAddressPool{
 											{
 												ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'aro-public-lb', 'aro-public-lb-control-plane')]"),
 											},
@@ -464,8 +501,8 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 												ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'aro-internal-lb', 'aro-internal-controlplane')]"),
 											},
 										},
-										Subnet: &network.Subnet{
-											ID: to.StringPtr(doc.OpenShiftCluster.Properties.MasterProfile.SubnetID),
+										Subnet: &mgmtnetwork.Subnet{
+											ID: to.StringPtr(i.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID),
 										},
 									},
 									Name: to.StringPtr("pipConfig"),
@@ -487,48 +524,48 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					},
 				},
 				{
-					Resource: &compute.VirtualMachine{
-						VirtualMachineProperties: &compute.VirtualMachineProperties{
-							HardwareProfile: &compute.HardwareProfile{
-								VMSize: compute.VirtualMachineSizeTypesStandardD4sV3,
+					Resource: &mgmtcompute.VirtualMachine{
+						VirtualMachineProperties: &mgmtcompute.VirtualMachineProperties{
+							HardwareProfile: &mgmtcompute.HardwareProfile{
+								VMSize: mgmtcompute.VirtualMachineSizeTypesStandardD4sV3,
 							},
-							StorageProfile: &compute.StorageProfile{
-								ImageReference: &compute.ImageReference{
+							StorageProfile: &mgmtcompute.StorageProfile{
+								ImageReference: &mgmtcompute.ImageReference{
 									Publisher: &installConfig.Config.Azure.Image.Publisher,
 									Offer:     &installConfig.Config.Azure.Image.Offer,
 									Sku:       &installConfig.Config.Azure.Image.SKU,
 									Version:   &installConfig.Config.Azure.Image.Version,
 								},
-								OsDisk: &compute.OSDisk{
+								OsDisk: &mgmtcompute.OSDisk{
 									Name:         to.StringPtr("aro-bootstrap_OSDisk"),
-									Caching:      compute.CachingTypesReadWrite,
-									CreateOption: compute.DiskCreateOptionTypesFromImage,
+									Caching:      mgmtcompute.CachingTypesReadWrite,
+									CreateOption: mgmtcompute.DiskCreateOptionTypesFromImage,
 									DiskSizeGB:   to.Int32Ptr(100),
-									ManagedDisk: &compute.ManagedDiskParameters{
-										StorageAccountType: compute.StorageAccountTypesPremiumLRS,
+									ManagedDisk: &mgmtcompute.ManagedDiskParameters{
+										StorageAccountType: mgmtcompute.StorageAccountTypesPremiumLRS,
 									},
 								},
 							},
-							OsProfile: &compute.OSProfile{
+							OsProfile: &mgmtcompute.OSProfile{
 								ComputerName:  to.StringPtr("aro-bootstrap-vm"),
 								AdminUsername: to.StringPtr("core"),
 								AdminPassword: to.StringPtr("NotActuallyApplied!"),
-								CustomData:    to.StringPtr(`[base64(concat('{"ignition":{"version":"2.2.0","config":{"replace":{"source":"https://cluster` + doc.OpenShiftCluster.Properties.StorageSuffix + `.blob.core.windows.net/ignition/bootstrap.ign?', listAccountSas(resourceId('Microsoft.Storage/storageAccounts', 'cluster` + doc.OpenShiftCluster.Properties.StorageSuffix + `'), '2019-04-01', parameters('sas')).accountSasToken, '"}}}}'))]`),
-								LinuxConfiguration: &compute.LinuxConfiguration{
+								CustomData:    to.StringPtr(`[base64(concat('{"ignition":{"version":"2.2.0","config":{"replace":{"source":"https://cluster` + i.doc.OpenShiftCluster.Properties.StorageSuffix + `.blob.core.windows.net/ignition/bootstrap.ign?', listAccountSas(resourceId('Microsoft.Storage/storageAccounts', 'cluster` + i.doc.OpenShiftCluster.Properties.StorageSuffix + `'), '2019-04-01', parameters('sas')).accountSasToken, '"}}}}'))]`),
+								LinuxConfiguration: &mgmtcompute.LinuxConfiguration{
 									DisablePasswordAuthentication: to.BoolPtr(false),
 								},
 							},
-							NetworkProfile: &compute.NetworkProfile{
-								NetworkInterfaces: &[]compute.NetworkInterfaceReference{
+							NetworkProfile: &mgmtcompute.NetworkProfile{
+								NetworkInterfaces: &[]mgmtcompute.NetworkInterfaceReference{
 									{
 										ID: to.StringPtr("[resourceId('Microsoft.Network/networkInterfaces', 'aro-bootstrap-nic')]"),
 									},
 								},
 							},
-							DiagnosticsProfile: &compute.DiagnosticsProfile{
-								BootDiagnostics: &compute.BootDiagnostics{
+							DiagnosticsProfile: &mgmtcompute.DiagnosticsProfile{
+								BootDiagnostics: &mgmtcompute.BootDiagnostics{
 									Enabled:    to.BoolPtr(true),
-									StorageURI: to.StringPtr("https://cluster" + doc.OpenShiftCluster.Properties.StorageSuffix + ".blob.core.windows.net/"),
+									StorageURI: to.StringPtr("https://cluster" + i.doc.OpenShiftCluster.Properties.StorageSuffix + ".blob.core.windows.net/"),
 								},
 							},
 						},
@@ -544,48 +581,48 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 					},
 				},
 				{
-					Resource: &compute.VirtualMachine{
-						VirtualMachineProperties: &compute.VirtualMachineProperties{
-							HardwareProfile: &compute.HardwareProfile{
-								VMSize: compute.VirtualMachineSizeTypes(installConfig.Config.ControlPlane.Platform.Azure.InstanceType),
+					Resource: &mgmtcompute.VirtualMachine{
+						VirtualMachineProperties: &mgmtcompute.VirtualMachineProperties{
+							HardwareProfile: &mgmtcompute.HardwareProfile{
+								VMSize: mgmtcompute.VirtualMachineSizeTypes(installConfig.Config.ControlPlane.Platform.Azure.InstanceType),
 							},
-							StorageProfile: &compute.StorageProfile{
-								ImageReference: &compute.ImageReference{
+							StorageProfile: &mgmtcompute.StorageProfile{
+								ImageReference: &mgmtcompute.ImageReference{
 									Publisher: &installConfig.Config.Azure.Image.Publisher,
 									Offer:     &installConfig.Config.Azure.Image.Offer,
 									Sku:       &installConfig.Config.Azure.Image.SKU,
 									Version:   &installConfig.Config.Azure.Image.Version,
 								},
-								OsDisk: &compute.OSDisk{
+								OsDisk: &mgmtcompute.OSDisk{
 									Name:         to.StringPtr("[concat('aro-master-', copyIndex(), '_OSDisk')]"),
-									Caching:      compute.CachingTypesReadOnly,
-									CreateOption: compute.DiskCreateOptionTypesFromImage,
+									Caching:      mgmtcompute.CachingTypesReadOnly,
+									CreateOption: mgmtcompute.DiskCreateOptionTypesFromImage,
 									DiskSizeGB:   &installConfig.Config.ControlPlane.Platform.Azure.OSDisk.DiskSizeGB,
-									ManagedDisk: &compute.ManagedDiskParameters{
-										StorageAccountType: compute.StorageAccountTypesPremiumLRS,
+									ManagedDisk: &mgmtcompute.ManagedDiskParameters{
+										StorageAccountType: mgmtcompute.StorageAccountTypesPremiumLRS,
 									},
 								},
 							},
-							OsProfile: &compute.OSProfile{
+							OsProfile: &mgmtcompute.OSProfile{
 								ComputerName:  to.StringPtr("[concat('aro-master-', copyIndex())]"),
 								AdminUsername: to.StringPtr("core"),
 								AdminPassword: to.StringPtr("NotActuallyApplied!"),
 								CustomData:    to.StringPtr(base64.StdEncoding.EncodeToString(machineMaster.File.Data)),
-								LinuxConfiguration: &compute.LinuxConfiguration{
+								LinuxConfiguration: &mgmtcompute.LinuxConfiguration{
 									DisablePasswordAuthentication: to.BoolPtr(false),
 								},
 							},
-							NetworkProfile: &compute.NetworkProfile{
-								NetworkInterfaces: &[]compute.NetworkInterfaceReference{
+							NetworkProfile: &mgmtcompute.NetworkProfile{
+								NetworkInterfaces: &[]mgmtcompute.NetworkInterfaceReference{
 									{
 										ID: to.StringPtr("[resourceId('Microsoft.Network/networkInterfaces', concat('aro-master', copyIndex(), '-nic'))]"),
 									},
 								},
 							},
-							DiagnosticsProfile: &compute.DiagnosticsProfile{
-								BootDiagnostics: &compute.BootDiagnostics{
+							DiagnosticsProfile: &mgmtcompute.DiagnosticsProfile{
+								BootDiagnostics: &mgmtcompute.BootDiagnostics{
 									Enabled:    to.BoolPtr(true),
-									StorageURI: to.StringPtr("https://cluster" + doc.OpenShiftCluster.Properties.StorageSuffix + ".blob.core.windows.net/"),
+									StorageURI: to.StringPtr("https://cluster" + i.doc.OpenShiftCluster.Properties.StorageSuffix + ".blob.core.windows.net/"),
 								},
 							},
 						},
@@ -611,14 +648,14 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 		}
 
 		i.log.Print("deploying resources template")
-		err = i.deployments.CreateOrUpdateAndWait(ctx, doc.OpenShiftCluster.Properties.ResourceGroup, "azuredeploy", resources.Deployment{
-			Properties: &resources.DeploymentProperties{
+		err = i.deployments.CreateOrUpdateAndWait(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup, "azuredeploy", mgmtresources.Deployment{
+			Properties: &mgmtresources.DeploymentProperties{
 				Template: t,
 				Parameters: map[string]interface{}{
 					"sas": map[string]interface{}{
 						"value": map[string]interface{}{
-							"signedStart":         doc.OpenShiftCluster.Properties.Install.Now.Format(time.RFC3339),
-							"signedExpiry":        doc.OpenShiftCluster.Properties.Install.Now.Add(24 * time.Hour).Format(time.RFC3339),
+							"signedStart":         i.doc.OpenShiftCluster.Properties.Install.Now.Format(time.RFC3339),
+							"signedExpiry":        i.doc.OpenShiftCluster.Properties.Install.Now.Add(24 * time.Hour).Format(time.RFC3339),
 							"signedPermission":    "rl",
 							"signedResourceTypes": "o",
 							"signedServices":      "b",
@@ -626,23 +663,46 @@ func (i *Installer) installResources(ctx context.Context, doc *api.OpenShiftClus
 						},
 					},
 				},
-				Mode: resources.Incremental,
+				Mode: mgmtresources.Incremental,
 			},
 		})
 		if err != nil {
-			return err
+			if detailedError, ok := err.(autorest.DetailedError); ok {
+				if requestError, ok := detailedError.Original.(azure.RequestError); ok &&
+					requestError.ServiceError != nil &&
+					requestError.ServiceError.Code == "DeploymentActive" {
+					i.log.Print("waiting for resources template")
+					err = i.deployments.Wait(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup, "azuredeploy")
+				}
+			}
+			if err != nil {
+				return err
+			}
 		}
 	}
 
 	{
-		err = i.env.DNS().CreateOrUpdate(ctx, doc.OpenShiftCluster)
+		i.log.Print("creating private endpoint")
+		err = i.privateendpoint.Create(ctx, i.doc)
 		if err != nil {
 			return err
 		}
 	}
 
 	{
-		restConfig, err := restconfig.RestConfig(doc.OpenShiftCluster.Properties.AdminKubeconfig)
+		err = i.dns.CreateOrUpdate(ctx, i.doc.OpenShiftCluster)
+		if err != nil {
+			return err
+		}
+	}
+
+	{
+		ip, err := i.privateendpoint.GetIP(ctx, i.doc)
+		if err != nil {
+			return err
+		}
+
+		restConfig, err := restconfig.RestConfig(ctx, i.env, i.doc, ip)
 		if err != nil {
 			return err
 		}

pkg/install/2-removebootstrap.go

@@ -19,8 +19,8 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/restconfig"
 )
 
-func (i *Installer) removeBootstrap(ctx context.Context, doc *api.OpenShiftClusterDocument) error {
-	g, err := i.getGraph(ctx, doc.OpenShiftCluster)
+func (i *Installer) removeBootstrap(ctx context.Context) error {
+	g, err := i.getGraph(ctx)
 	if err != nil {
 		return err
 	}
@@ -29,7 +29,7 @@ func (i *Installer) removeBootstrap(ctx context.Context, doc *api.OpenShiftClust
 
 	{
 		i.log.Print("removing bootstrap vm")
-		err := i.virtualmachines.DeleteAndWait(ctx, doc.OpenShiftCluster.Properties.ResourceGroup, "aro-bootstrap")
+		err := i.virtualmachines.DeleteAndWait(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup, "aro-bootstrap")
 		if err != nil {
 			return err
 		}
@@ -37,7 +37,7 @@ func (i *Installer) removeBootstrap(ctx context.Context, doc *api.OpenShiftClust
 
 	{
 		i.log.Print("removing bootstrap disk")
-		err := i.disks.DeleteAndWait(ctx, doc.OpenShiftCluster.Properties.ResourceGroup, "aro-bootstrap_OSDisk")
+		err := i.disks.DeleteAndWait(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup, "aro-bootstrap_OSDisk")
 		if err != nil {
 			return err
 		}
@@ -45,7 +45,7 @@ func (i *Installer) removeBootstrap(ctx context.Context, doc *api.OpenShiftClust
 
 	{
 		i.log.Print("removing bootstrap nic")
-		err = i.interfaces.DeleteAndWait(ctx, doc.OpenShiftCluster.Properties.ResourceGroup, "aro-bootstrap-nic")
+		err = i.interfaces.DeleteAndWait(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup, "aro-bootstrap-nic")
 		if err != nil {
 			return err
 		}
@@ -53,14 +53,19 @@ func (i *Installer) removeBootstrap(ctx context.Context, doc *api.OpenShiftClust
 
 	{
 		i.log.Print("removing bootstrap ip")
-		err = i.publicipaddresses.DeleteAndWait(ctx, doc.OpenShiftCluster.Properties.ResourceGroup, "aro-bootstrap-pip")
+		err = i.publicipaddresses.DeleteAndWait(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup, "aro-bootstrap-pip")
 		if err != nil {
 			return err
 		}
 	}
 
 	{
-		restConfig, err := restconfig.RestConfig(doc.OpenShiftCluster.Properties.AdminKubeconfig)
+		ip, err := i.privateendpoint.GetIP(ctx, i.doc)
+		if err != nil {
+			return err
+		}
+
+		restConfig, err := restconfig.RestConfig(ctx, i.env, i.doc, ip)
 		if err != nil {
 			return err
 		}
@@ -109,7 +114,7 @@ func (i *Installer) removeBootstrap(ctx context.Context, doc *api.OpenShiftClust
 		}
 	}
 
-	ips, err := i.publicipaddresses.List(ctx, doc.OpenShiftCluster.Properties.ResourceGroup)
+	ips, err := i.publicipaddresses.List(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup)
 	if err != nil {
 		return err
 	}
@@ -126,15 +131,15 @@ func (i *Installer) removeBootstrap(ctx context.Context, doc *api.OpenShiftClust
 			return fmt.Errorf("routerIP not found")
 		}
 
-		err = i.env.DNS().CreateOrUpdateRouter(ctx, doc.OpenShiftCluster, routerIP)
+		err = i.dns.CreateOrUpdateRouter(ctx, i.doc.OpenShiftCluster, routerIP)
 		if err != nil {
 			return err
 		}
 	}
 
-	_, err = i.db.Patch(doc.Key, func(doc *api.OpenShiftClusterDocument) error {
-		doc.OpenShiftCluster.Properties.APIServerURL = "https://api." + doc.OpenShiftCluster.Properties.DomainName + "." + i.env.DNS().Domain() + ":6443/"
-		doc.OpenShiftCluster.Properties.ConsoleURL = "https://console-openshift-console.apps." + doc.OpenShiftCluster.Properties.DomainName + "." + i.env.DNS().Domain() + "/"
+	i.doc, err = i.db.Patch(i.doc.Key, func(doc *api.OpenShiftClusterDocument) error {
+		doc.OpenShiftCluster.Properties.APIServerURL = "https://api." + doc.OpenShiftCluster.Properties.DomainName + "." + i.dns.Domain() + ":6443/"
+		doc.OpenShiftCluster.Properties.ConsoleURL = "https://console-openshift-console.apps." + doc.OpenShiftCluster.Properties.DomainName + "." + i.dns.Domain() + "/"
 		doc.OpenShiftCluster.Properties.KubeadminPassword = kubeadminPassword.Password
 		return nil
 	})

pkg/install/install.go

@@ -22,10 +22,12 @@ import (
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/database"
 	"github.com/Azure/ARO-RP/pkg/env"
-	"github.com/Azure/ARO-RP/pkg/util/azureclient/compute"
-	"github.com/Azure/ARO-RP/pkg/util/azureclient/network"
-	"github.com/Azure/ARO-RP/pkg/util/azureclient/resources"
-	"github.com/Azure/ARO-RP/pkg/util/azureclient/storage"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/resources"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage"
+	"github.com/Azure/ARO-RP/pkg/util/dns"
+	"github.com/Azure/ARO-RP/pkg/util/privateendpoint"
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )
 
@@ -33,8 +35,12 @@ type Installer struct {
 	log          *logrus.Entry
 	env          env.Interface
 	db           database.OpenShiftClusters
+	doc          *api.OpenShiftClusterDocument
 	fpAuthorizer autorest.Authorizer
 
+	privateendpoint privateendpoint.Manager
+	dns             dns.Manager
+
 	disks             compute.DisksClient
 	virtualmachines   compute.VirtualMachinesClient
 	interfaces        network.InterfacesClient
@@ -46,27 +52,48 @@ type Installer struct {
 	subnets subnet.Manager
 }
 
-func NewInstaller(log *logrus.Entry, env env.Interface, db database.OpenShiftClusters, fpAuthorizer autorest.Authorizer, subscriptionID string) *Installer {
+func NewInstaller(log *logrus.Entry, env env.Interface, db database.OpenShiftClusters, doc *api.OpenShiftClusterDocument) (*Installer, error) {
+	r, err := azure.ParseResourceID(doc.OpenShiftCluster.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	localFPAuthorizer, err := env.FPAuthorizer(env.TenantID(), azure.PublicCloud.ResourceManagerEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	fpAuthorizer, err := env.FPAuthorizer(doc.OpenShiftCluster.Properties.ServicePrincipalProfile.TenantID, azure.PublicCloud.ResourceManagerEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Installer{
 		log:          log,
 		env:          env,
 		db:           db,
+		doc:          doc,
 		fpAuthorizer: fpAuthorizer,
 
-		disks:             compute.NewDisksClient(subscriptionID, fpAuthorizer),
-		virtualmachines:   compute.NewVirtualMachinesClient(subscriptionID, fpAuthorizer),
-		interfaces:        network.NewInterfacesClient(subscriptionID, fpAuthorizer),
-		publicipaddresses: network.NewPublicIPAddressesClient(subscriptionID, fpAuthorizer),
-		deployments:       resources.NewDeploymentsClient(subscriptionID, fpAuthorizer),
-		groups:            resources.NewGroupsClient(subscriptionID, fpAuthorizer),
-		accounts:          storage.NewAccountsClient(subscriptionID, fpAuthorizer),
+		privateendpoint: privateendpoint.NewManager(env, localFPAuthorizer),
+		dns:             dns.NewManager(env, localFPAuthorizer),
 
-		subnets: subnet.NewManager(subscriptionID, fpAuthorizer),
-	}
+		disks:             compute.NewDisksClient(r.SubscriptionID, fpAuthorizer),
+		virtualmachines:   compute.NewVirtualMachinesClient(r.SubscriptionID, fpAuthorizer),
+		interfaces:        network.NewInterfacesClient(r.SubscriptionID, fpAuthorizer),
+		publicipaddresses: network.NewPublicIPAddressesClient(r.SubscriptionID, fpAuthorizer),
+		deployments:       resources.NewDeploymentsClient(r.SubscriptionID, fpAuthorizer),
+		groups:            resources.NewGroupsClient(r.SubscriptionID, fpAuthorizer),
+		accounts:          storage.NewAccountsClient(r.SubscriptionID, fpAuthorizer),
+
+		subnets: subnet.NewManager(r.SubscriptionID, fpAuthorizer),
+	}, nil
 }
 
-func (i *Installer) Install(ctx context.Context, doc *api.OpenShiftClusterDocument, installConfig *installconfig.InstallConfig, platformCreds *installconfig.PlatformCreds, image *releaseimage.Image) error {
-	doc, err := i.db.Patch(doc.Key, func(doc *api.OpenShiftClusterDocument) error {
+func (i *Installer) Install(ctx context.Context, installConfig *installconfig.InstallConfig, platformCreds *installconfig.PlatformCreds, image *releaseimage.Image) error {
+	var err error
+
+	i.doc, err = i.db.Patch(i.doc.Key, func(doc *api.OpenShiftClusterDocument) error {
 		if doc.OpenShiftCluster.Properties.Install == nil {
 			doc.OpenShiftCluster.Properties.Install = &api.Install{}
 		}
@@ -77,38 +104,37 @@ func (i *Installer) Install(ctx context.Context, doc *api.OpenShiftClusterDocume
 	}
 
 	for {
-		i.log.Printf("starting phase %s", doc.OpenShiftCluster.Properties.Install.Phase)
-		switch doc.OpenShiftCluster.Properties.Install.Phase {
+		i.log.Printf("starting phase %s", i.doc.OpenShiftCluster.Properties.Install.Phase)
+		switch i.doc.OpenShiftCluster.Properties.Install.Phase {
 		case api.InstallPhaseDeployStorage:
-			err := i.installStorage(ctx, doc, installConfig, platformCreds, image)
+			err := i.installStorage(ctx, installConfig, platformCreds, image)
 			if err != nil {
 				return err
 			}
 
 		case api.InstallPhaseDeployResources:
-			err := i.installResources(ctx, doc)
+			err := i.installResources(ctx)
 			if err != nil {
 				return err
 			}
 
 		case api.InstallPhaseRemoveBootstrap:
-			err := i.removeBootstrap(ctx, doc)
+			err := i.removeBootstrap(ctx)
 			if err != nil {
 				return err
 			}
 
-			_, err = i.db.Patch(doc.Key, func(doc *api.OpenShiftClusterDocument) error {
+			i.doc, err = i.db.Patch(i.doc.Key, func(doc *api.OpenShiftClusterDocument) error {
 				doc.OpenShiftCluster.Properties.Install = nil
 				return nil
 			})
 			return err
 
 		default:
-			return fmt.Errorf("unrecognised phase %s", doc.OpenShiftCluster.Properties.Install.Phase)
+			return fmt.Errorf("unrecognised phase %s", i.doc.OpenShiftCluster.Properties.Install.Phase)
 		}
 
-		var err error
-		doc, err = i.db.Patch(doc.Key, func(doc *api.OpenShiftClusterDocument) error {
+		i.doc, err = i.db.Patch(i.doc.Key, func(doc *api.OpenShiftClusterDocument) error {
 			doc.OpenShiftCluster.Properties.Install.Phase++
 			return nil
 		})
@@ -118,10 +144,10 @@ func (i *Installer) Install(ctx context.Context, doc *api.OpenShiftClusterDocume
 	}
 }
 
-func (i *Installer) getBlobService(ctx context.Context, oc *api.OpenShiftCluster) (*azstorage.BlobStorageClient, error) {
+func (i *Installer) getBlobService(ctx context.Context) (*azstorage.BlobStorageClient, error) {
 	t := time.Now().UTC().Truncate(time.Second)
 
-	res, err := i.accounts.ListAccountSAS(ctx, oc.Properties.ResourceGroup, "cluster"+oc.Properties.StorageSuffix, mgmtstorage.AccountSasParameters{
+	res, err := i.accounts.ListAccountSAS(ctx, i.doc.OpenShiftCluster.Properties.ResourceGroup, "cluster"+i.doc.OpenShiftCluster.Properties.StorageSuffix, mgmtstorage.AccountSasParameters{
 		Services:               "b",
 		ResourceTypes:          "o",
 		Permissions:            "crw",
@@ -138,15 +164,15 @@ func (i *Installer) getBlobService(ctx context.Context, oc *api.OpenShiftCluster
 		return nil, err
 	}
 
-	c := azstorage.NewAccountSASClient("cluster"+oc.Properties.StorageSuffix, v, azure.PublicCloud).GetBlobService()
+	c := azstorage.NewAccountSASClient("cluster"+i.doc.OpenShiftCluster.Properties.StorageSuffix, v, azure.PublicCloud).GetBlobService()
 
 	return &c, nil
 }
 
-func (i *Installer) getGraph(ctx context.Context, oc *api.OpenShiftCluster) (graph, error) {
+func (i *Installer) getGraph(ctx context.Context) (graph, error) {
 	i.log.Print("retrieving graph")
 
-	blobService, err := i.getBlobService(ctx, oc)
+	blobService, err := i.getBlobService(ctx)
 	if err != nil {
 		return nil, err
 	}

pkg/util/azureclient/graphrbac/applications.go

@@ -0,0 +1,35 @@
+package graphrbac
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+//go:generate go run ../../../../vendor/github.com/golang/mock/mockgen -destination=../../../util/mocks/azureclient/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/$GOPACKAGE ApplicationsClient
+//go:generate go run ../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../util/mocks/azureclient/$GOPACKAGE/$GOPACKAGE.go
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/graphrbac/1.6/graphrbac"
+	"github.com/Azure/go-autorest/autorest"
+)
+
+// ApplicationsClient is a minimal interface for azure ApplicationsClient
+type ApplicationsClient interface {
+	GetServicePrincipalsIDByAppID(ctx context.Context, applicationID string) (result graphrbac.ServicePrincipalObjectResult, err error)
+}
+
+type applicationsClient struct {
+	graphrbac.ApplicationsClient
+}
+
+var _ ApplicationsClient = &applicationsClient{}
+
+// NewApplicationsClient creates a new ApplicationsClient
+func NewApplicationsClient(tenantID string, authorizer autorest.Authorizer) ApplicationsClient {
+	client := graphrbac.NewApplicationsClient(tenantID)
+	client.Authorizer = authorizer
+
+	return &applicationsClient{
+		ApplicationsClient: client,
+	}
+}

pkg/util/azureclient/keyvault/base.go

@@ -0,0 +1,35 @@
+package keyvault
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+//go:generate go run ../../../../vendor/github.com/golang/mock/mockgen -destination=../../../util/mocks/azureclient/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/$GOPACKAGE BaseClient
+//go:generate go run ../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../util/mocks/azureclient/$GOPACKAGE/$GOPACKAGE.go
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
+	"github.com/Azure/go-autorest/autorest"
+)
+
+// BaseClient is a minimal interface for azure BaseClient
+type BaseClient interface {
+	GetSecret(ctx context.Context, vaultBaseURL string, secretName string, secretVersion string) (result keyvault.SecretBundle, err error)
+}
+
+type baseClient struct {
+	keyvault.BaseClient
+}
+
+var _ BaseClient = &baseClient{}
+
+// New creates a new BaseClient
+func New(authorizer autorest.Authorizer) BaseClient {
+	client := keyvault.New()
+	client.Authorizer = authorizer
+
+	return &baseClient{
+		BaseClient: client,
+	}
+}

pkg/util/azureclient/mgmt/authorization/permissions.go

@@ -3,8 +3,8 @@ package authorization
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../../../vendor/github.com/golang/mock/mockgen -destination=../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/$GOPACKAGE PermissionsClient,RoleAssignmentsClient
-//go:generate go run ../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE PermissionsClient,RoleAssignmentsClient
+//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go
 
 import (
 	"github.com/Azure/azure-sdk-for-go/services/authorization/mgmt/2015-07-01/authorization"

pkg/util/azureclient/mgmt/authorization/permissions_addons.go

@@ -7,22 +7,16 @@ import (
 	"context"
 
 	"github.com/Azure/azure-sdk-for-go/services/authorization/mgmt/2015-07-01/authorization"
-	"github.com/Azure/go-autorest/autorest/azure"
 )
 
 // PermissionsClientAddons contains addons for PermissionsClient
 type PermissionsClientAddons interface {
-	ListForResource(ctx context.Context, resourceID string) (permissions []authorization.Permission, err error)
+	ListForResource(ctx context.Context, resourceGroupName string, resourceProviderNamespace string, parentResourcePath string, resourceType string, resourceName string) (permissions []authorization.Permission, err error)
 	ListForResourceGroup(ctx context.Context, resourceGroupName string) (permissions []authorization.Permission, err error)
 }
 
-func (c *permissionsClient) ListForResource(ctx context.Context, resourceID string) (permissions []authorization.Permission, error error) {
-	r, err := azure.ParseResourceID(resourceID)
-	if err != nil {
-		return nil, err
-	}
-
-	page, err := c.PermissionsClient.ListForResource(ctx, r.ResourceGroup, r.Provider, r.ResourceType, "", r.ResourceName)
+func (c *permissionsClient) ListForResource(ctx context.Context, resourceGroupName string, resourceProviderNamespace string, parentResourcePath string, resourceType string, resourceName string) (permissions []authorization.Permission, err error) {
+	page, err := c.PermissionsClient.ListForResource(ctx, resourceGroupName, resourceProviderNamespace, parentResourcePath, resourceType, resourceName)
 	if err != nil {
 		return nil, err
 	}

pkg/util/azureclient/mgmt/authorization/roleassignments.go

@@ -3,8 +3,8 @@ package authorization
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../../../vendor/github.com/golang/mock/mockgen -destination=../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/$GOPACKAGE RoleAssignmentsClient
-//go:generate go run ../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE RoleAssignmentsClient
+//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go
 
 import (
 	"context"

pkg/util/azureclient/mgmt/compute/disks.go

@@ -3,8 +3,8 @@ package compute
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../../../vendor/github.com/golang/mock/mockgen -destination=../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/$GOPACKAGE DisksClient,VirtualMachinesClient
-//go:generate go run ../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE DisksClient,VirtualMachinesClient
+//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go
 
 import (
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"

pkg/util/azureclient/mgmt/compute/disks_addons.go

@@ -18,5 +18,5 @@ func (c *disksClient) DeleteAndWait(ctx context.Context, resourceGroupName strin
 		return err
 	}
 
-	return future.WaitForCompletionRef(ctx, c.DisksClient.Client)
+	return future.WaitForCompletionRef(ctx, c.Client)
 }

pkg/util/azureclient/mgmt/compute/virtualmachines_addons.go

@@ -21,7 +21,7 @@ func (c *virtualMachinesClient) CreateOrUpdateAndWait(ctx context.Context, resou
 		return err
 	}
 
-	return future.WaitForCompletionRef(ctx, c.VirtualMachinesClient.Client)
+	return future.WaitForCompletionRef(ctx, c.Client)
 }
 
 func (c *virtualMachinesClient) DeleteAndWait(ctx context.Context, resourceGroupName string, VMName string) error {
@@ -30,5 +30,5 @@ func (c *virtualMachinesClient) DeleteAndWait(ctx context.Context, resourceGroup
 		return err
 	}
 
-	return future.WaitForCompletionRef(ctx, c.VirtualMachinesClient.Client)
+	return future.WaitForCompletionRef(ctx, c.Client)
 }

pkg/util/azureclient/mgmt/dns/recordsets.go

@@ -3,8 +3,8 @@ package dns
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../../../vendor/github.com/golang/mock/mockgen -destination=../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/$GOPACKAGE RecordSetsClient
-//go:generate go run ../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE RecordSetsClient,ZonesClient
+//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go
 
 import (
 	"context"

pkg/util/azureclient/mgmt/dns/zones.go

@@ -0,0 +1,30 @@
+package dns
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2018-05-01/dns"
+	"github.com/Azure/go-autorest/autorest"
+)
+
+// ZonesClient is a minimal interface for azure ZonesClient
+type ZonesClient interface {
+	ZonesClientAddons
+}
+
+type zonesClient struct {
+	dns.ZonesClient
+}
+
+var _ ZonesClient = &zonesClient{}
+
+// NewZonesClient creates a new ZonesClient
+func NewZonesClient(subscriptionID string, authorizer autorest.Authorizer) ZonesClient {
+	client := dns.NewZonesClient(subscriptionID)
+	client.Authorizer = authorizer
+
+	return &zonesClient{
+		ZonesClient: client,
+	}
+}

pkg/util/azureclient/mgmt/dns/zones_addons.go

@@ -0,0 +1,33 @@
+package dns
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2018-05-01/dns"
+)
+
+// ZonesClientAddons contains addons for ZonesClient
+type ZonesClientAddons interface {
+	ListByResourceGroup(ctx context.Context, resourceGroupName string, top *int32) (zones []dns.Zone, err error)
+}
+
+func (c *zonesClient) ListByResourceGroup(ctx context.Context, resourceGroupName string, top *int32) (zones []dns.Zone, err error) {
+	page, err := c.ZonesClient.ListByResourceGroup(ctx, resourceGroupName, top)
+	if err != nil {
+		return nil, err
+	}
+
+	for page.NotDone() {
+		zones = append(zones, page.Values()...)
+
+		err = page.Next()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return zones, nil
+}

pkg/util/azureclient/mgmt/documentdb/databaseaccounts.go

@@ -0,0 +1,36 @@
+package documentdb
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE DatabaseAccountsClient
+//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb"
+	"github.com/Azure/go-autorest/autorest"
+)
+
+// DatabaseAccountsClient is a minimal interface for azure DatabaseAccountsClient
+type DatabaseAccountsClient interface {
+	ListByResourceGroup(ctx context.Context, resourceGroupName string) (result documentdb.DatabaseAccountsListResult, err error)
+	ListKeys(ctx context.Context, resourceGroupName string, accountName string) (result documentdb.DatabaseAccountListKeysResult, err error)
+}
+
+type databaseAccountsClient struct {
+	documentdb.DatabaseAccountsClient
+}
+
+var _ DatabaseAccountsClient = &databaseAccountsClient{}
+
+// NewDatabaseAccountsClient creates a new DatabaseAccountsClient
+func NewDatabaseAccountsClient(subscriptionID string, authorizer autorest.Authorizer) DatabaseAccountsClient {
+	client := documentdb.NewDatabaseAccountsClient(subscriptionID)
+	client.Authorizer = authorizer
+
+	return &databaseAccountsClient{
+		DatabaseAccountsClient: client,
+	}
+}

pkg/util/azureclient/mgmt/keyvault/vaults.go

@@ -0,0 +1,33 @@
+package keyvault
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE VaultsClient
+//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go
+
+import (
+	"github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2016-10-01/keyvault"
+	"github.com/Azure/go-autorest/autorest"
+)
+
+// VaultsClient is a minimal interface for azure VaultsClient
+type VaultsClient interface {
+	VaultsClientAddons
+}
+
+type vaultsClient struct {
+	keyvault.VaultsClient
+}
+
+var _ VaultsClient = &vaultsClient{}
+
+// NewVaultsClient creates a new VaultsClient
+func NewVaultsClient(subscriptionID string, authorizer autorest.Authorizer) VaultsClient {
+	client := keyvault.NewVaultsClient(subscriptionID)
+	client.Authorizer = authorizer
+
+	return &vaultsClient{
+		VaultsClient: client,
+	}
+}

pkg/util/azureclient/mgmt/keyvault/vaults_addons.go

@@ -0,0 +1,33 @@
+package keyvault
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2016-10-01/keyvault"
+)
+
+// VaultsClientAddons contains addons for VaultsClient
+type VaultsClientAddons interface {
+	ListByResourceGroup(ctx context.Context, resourceGroupName string, top *int32) (vaults []keyvault.Vault, err error)
+}
+
+func (c *vaultsClient) ListByResourceGroup(ctx context.Context, resourceGroupName string, top *int32) (vaults []keyvault.Vault, err error) {
+	page, err := c.VaultsClient.ListByResourceGroup(ctx, resourceGroupName, top)
+	if err != nil {
+		return nil, err
+	}
+
+	for page.NotDone() {
+		vaults = append(vaults, page.Values()...)
+
+		err = page.Next()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return vaults, nil
+}

pkg/util/azureclient/mgmt/network/interfaces.go

@@ -3,8 +3,8 @@ package network
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../../../vendor/github.com/golang/mock/mockgen -destination=../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/$GOPACKAGE InterfacesClient,PublicIPAddressesClient
-//go:generate go run ../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE InterfacesClient,PrivateEndpointsClient,PublicIPAddressesClient,SubnetsClient
+//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go
 
 import (
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"

pkg/util/azureclient/mgmt/network/interfaces_addons.go

@@ -18,5 +18,5 @@ func (c *interfacesClient) DeleteAndWait(ctx context.Context, resourceGroupName
 		return err
 	}
 
-	return future.WaitForCompletionRef(ctx, c.InterfacesClient.Client)
+	return future.WaitForCompletionRef(ctx, c.Client)
 }

pkg/util/azureclient/mgmt/network/privateendpoints.go

@@ -0,0 +1,33 @@
+package network
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+	"github.com/Azure/go-autorest/autorest"
+)
+
+// PrivateEndpointsClient is a minimal interface for azure PrivateEndpointsClient
+type PrivateEndpointsClient interface {
+	Get(ctx context.Context, resourceGroupName string, privateEndpointName string, expand string) (result network.PrivateEndpoint, err error)
+	PrivateEndpointsClientAddons
+}
+
+type privateEndpointsClient struct {
+	network.PrivateEndpointsClient
+}
+
+var _ PrivateEndpointsClient = &privateEndpointsClient{}
+
+// NewPrivateEndpointsClient creates a new PrivateEndpointsClient
+func NewPrivateEndpointsClient(subscriptionID string, authorizer autorest.Authorizer) PrivateEndpointsClient {
+	client := network.NewPrivateEndpointsClient(subscriptionID)
+	client.Authorizer = authorizer
+
+	return &privateEndpointsClient{
+		PrivateEndpointsClient: client,
+	}
+}

pkg/util/azureclient/mgmt/network/privateendpoints_addons.go

@@ -0,0 +1,34 @@
+package network
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+)
+
+// PrivateEndpointsClientAddons contains addons for PrivateEndpointsClient
+type PrivateEndpointsClientAddons interface {
+	CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, privateEndpointName string, parameters network.PrivateEndpoint) (err error)
+	DeleteAndWait(ctx context.Context, resourceGroupName string, publicIPAddressName string) (err error)
+}
+
+func (c *privateEndpointsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, privateEndpointName string, parameters network.PrivateEndpoint) error {
+	future, err := c.CreateOrUpdate(ctx, resourceGroupName, privateEndpointName, parameters)
+	if err != nil {
+		return err
+	}
+
+	return future.WaitForCompletionRef(ctx, c.Client)
+}
+
+func (c *privateEndpointsClient) DeleteAndWait(ctx context.Context, resourceGroupName string, publicIPAddressName string) error {
+	future, err := c.Delete(ctx, resourceGroupName, publicIPAddressName)
+	if err != nil {
+		return err
+	}
+
+	return future.WaitForCompletionRef(ctx, c.Client)
+}

pkg/util/azureclient/mgmt/network/publicipaddresses_addons.go

@@ -20,7 +20,7 @@ func (c *publicIPAddressesClient) DeleteAndWait(ctx context.Context, resourceGro
 		return err
 	}
 
-	return future.WaitForCompletionRef(ctx, c.PublicIPAddressesClient.Client)
+	return future.WaitForCompletionRef(ctx, c.Client)
 }
 
 func (c *publicIPAddressesClient) List(ctx context.Context, resourceGroupName string) (ips []network.PublicIPAddress, err error) {

pkg/util/azureclient/mgmt/network/subnets.go

@@ -0,0 +1,33 @@
+package network
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+	"github.com/Azure/go-autorest/autorest"
+)
+
+// SubnetsClient is a minimal interface for azure SubnetsClient
+type SubnetsClient interface {
+	Get(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string, expand string) (result network.Subnet, err error)
+	SubnetsClientAddons
+}
+
+type subnetsClient struct {
+	network.SubnetsClient
+}
+
+var _ SubnetsClient = &subnetsClient{}
+
+// NewSubnetsClient creates a new SubnetsClient
+func NewSubnetsClient(subscriptionID string, authorizer autorest.Authorizer) SubnetsClient {
+	client := network.NewSubnetsClient(subscriptionID)
+	client.Authorizer = authorizer
+
+	return &subnetsClient{
+		SubnetsClient: client,
+	}
+}

pkg/util/azureclient/mgmt/network/subnets_addons.go

@@ -0,0 +1,24 @@
+package network
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+)
+
+// SubnetsClientAddons contains addons for SubnetsClient
+type SubnetsClientAddons interface {
+	CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string, subnetParameters network.Subnet) (err error)
+}
+
+func (c *subnetsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, virtualNetworkName string, subnetName string, subnetParameters network.Subnet) error {
+	future, err := c.CreateOrUpdate(ctx, resourceGroupName, virtualNetworkName, subnetName, subnetParameters)
+	if err != nil {
+		return err
+	}
+
+	return future.WaitForCompletionRef(ctx, c.Client)
+}

pkg/util/azureclient/mgmt/resources/deployments.go

@@ -3,8 +3,8 @@ package resources
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../../../vendor/github.com/golang/mock/mockgen -destination=../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/$GOPACKAGE DeploymentsClient,GroupsClient
-//go:generate go run ../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE DeploymentsClient,GroupsClient
+//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go
 
 import (
 	"time"

pkg/util/azureclient/mgmt/resources/deployments_addons.go

@@ -0,0 +1,43 @@
+package resources
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+// DeploymentsClientAddons contains addons for DeploymentsClient
+type DeploymentsClientAddons interface {
+	CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, deploymentName string, parameters resources.Deployment) error
+	Wait(ctx context.Context, resourceGroupName string, deploymentName string) error
+}
+
+func (c *deploymentsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, deploymentName string, parameters resources.Deployment) error {
+	future, err := c.CreateOrUpdate(ctx, resourceGroupName, deploymentName, parameters)
+	if err != nil {
+		return err
+	}
+
+	return future.WaitForCompletionRef(ctx, c.Client)
+}
+
+func (c *deploymentsClient) Wait(ctx context.Context, resourceGroupName string, deploymentName string) error {
+	return wait.Poll(c.Client.PollingDelay, c.Client.PollingDuration, func() (bool, error) {
+		deployment, err := c.Get(ctx, resourceGroupName, deploymentName)
+		if err != nil {
+			return false, err
+		}
+
+		switch *deployment.Properties.ProvisioningState {
+		case "Canceled", "Failed":
+			return false, fmt.Errorf("got provisioningState %q", *deployment.Properties.ProvisioningState)
+		}
+
+		return *deployment.Properties.ProvisioningState == "Succeeded", nil
+	})
+}

pkg/util/azureclient/mgmt/resources/groups_addons.go

@@ -18,5 +18,5 @@ func (c *groupsClient) DeleteAndWait(ctx context.Context, resourceGroupName stri
 		return err
 	}
 
-	return future.WaitForCompletionRef(ctx, c.GroupsClient.Client)
+	return future.WaitForCompletionRef(ctx, c.Client)
 }

pkg/util/azureclient/mgmt/storage/accounts.go

@@ -3,8 +3,8 @@ package storage
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../../../vendor/github.com/golang/mock/mockgen -destination=../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/$GOPACKAGE AccountsClient
-//go:generate go run ../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../util/mocks/mock_azureclient/mock_$GOPACKAGE/$GOPACKAGE.go
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE AccountsClient
+//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go
 
 import (
 	"context"

pkg/util/azureclient/mgmt/storage/accounts_addons.go

@@ -20,5 +20,5 @@ func (c *accountsClient) CreateAndWait(ctx context.Context, resourceGroupName st
 		return err
 	}
 
-	return future.WaitForCompletionRef(ctx, c.AccountsClient.Client)
+	return future.WaitForCompletionRef(ctx, c.Client)
 }

pkg/util/azureclient/resources/deployments_addons.go

@@ -1,24 +0,0 @@
-package resources
-
-// Copyright (c) Microsoft Corporation.
-// Licensed under the Apache License 2.0.
-
-import (
-	"context"
-
-	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
-)
-
-// DeploymentsClientAddons contains addons for DeploymentsClient
-type DeploymentsClientAddons interface {
-	CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, deploymentName string, parameters resources.Deployment) error
-}
-
-func (c *deploymentsClient) CreateOrUpdateAndWait(ctx context.Context, resourceGroupName string, deploymentName string, parameters resources.Deployment) error {
-	future, err := c.CreateOrUpdate(ctx, resourceGroupName, deploymentName, parameters)
-	if err != nil {
-		return err
-	}
-
-	return future.WaitForCompletionRef(ctx, c.DeploymentsClient.Client)
-}

pkg/util/dns/dns.go

@@ -5,14 +5,14 @@ package dns
 
 import (
 	"context"
-	"fmt"
 
-	"github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2018-05-01/dns"
+	mgmtdns "github.com/Azure/azure-sdk-for-go/services/dns/mgmt/2018-05-01/dns"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/to"
 
 	"github.com/Azure/ARO-RP/pkg/api"
-	"github.com/Azure/ARO-RP/pkg/util/instancemetadata"
+	"github.com/Azure/ARO-RP/pkg/env"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/dns"
 )
 
 type Manager interface {
@@ -23,49 +23,27 @@ type Manager interface {
 }
 
 type manager struct {
-	instancemetadata instancemetadata.InstanceMetadata
-
+	env        env.Interface
 	recordsets dns.RecordSetsClient
-
-	domain string
 }
 
-func NewManager(ctx context.Context, instancemetadata instancemetadata.InstanceMetadata, rpAuthorizer autorest.Authorizer) (Manager, error) {
-	m := &manager{
-		instancemetadata: instancemetadata,
+func NewManager(env env.Interface, localFPAuthorizer autorest.Authorizer) Manager {
+	return &manager{
+		env: env,
 
-		recordsets: dns.NewRecordSetsClient(instancemetadata.SubscriptionID()),
+		recordsets: dns.NewRecordSetsClient(env.SubscriptionID(), localFPAuthorizer),
 	}
-
-	m.recordsets.Authorizer = rpAuthorizer
-
-	zones := dns.NewZonesClient(instancemetadata.SubscriptionID())
-	zones.Authorizer = rpAuthorizer
-
-	page, err := zones.ListByResourceGroup(ctx, m.instancemetadata.ResourceGroup(), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	zs := page.Values()
-	if len(zs) != 1 {
-		return nil, fmt.Errorf("found at least %d zones, expected 1", len(zs))
-	}
-
-	m.domain = *zs[0].Name
-
-	return m, nil
 }
 
 func (m *manager) Domain() string {
-	return m.domain
+	return m.env.Domain()
 }
 
 func (m *manager) CreateOrUpdate(ctx context.Context, oc *api.OpenShiftCluster) error {
-	_, err := m.recordsets.CreateOrUpdate(ctx, m.instancemetadata.ResourceGroup(), m.domain, "api."+oc.Properties.DomainName, dns.CNAME, dns.RecordSet{
-		RecordSetProperties: &dns.RecordSetProperties{
+	_, err := m.recordsets.CreateOrUpdate(ctx, m.env.ResourceGroup(), m.Domain(), "api."+oc.Properties.DomainName, mgmtdns.CNAME, mgmtdns.RecordSet{
+		RecordSetProperties: &mgmtdns.RecordSetProperties{
 			TTL: to.Int64Ptr(300),
-			CnameRecord: &dns.CnameRecord{
+			CnameRecord: &mgmtdns.CnameRecord{
 				Cname: to.StringPtr(oc.Properties.DomainName + "." + oc.Location + ".cloudapp.azure.com"),
 			},
 		},
@@ -75,10 +53,10 @@ func (m *manager) CreateOrUpdate(ctx context.Context, oc *api.OpenShiftCluster)
 }
 
 func (m *manager) CreateOrUpdateRouter(ctx context.Context, oc *api.OpenShiftCluster, routerIP string) error {
-	_, err := m.recordsets.CreateOrUpdate(ctx, m.instancemetadata.ResourceGroup(), m.domain, "*.apps."+oc.Properties.DomainName, dns.A, dns.RecordSet{
-		RecordSetProperties: &dns.RecordSetProperties{
+	_, err := m.recordsets.CreateOrUpdate(ctx, m.env.ResourceGroup(), m.Domain(), "*.apps."+oc.Properties.DomainName, mgmtdns.A, mgmtdns.RecordSet{
+		RecordSetProperties: &mgmtdns.RecordSetProperties{
 			TTL: to.Int64Ptr(300),
-			ARecords: &[]dns.ARecord{
+			ARecords: &[]mgmtdns.ARecord{
 				{
 					Ipv4Address: to.StringPtr(routerIP),
 				},
@@ -90,12 +68,12 @@ func (m *manager) CreateOrUpdateRouter(ctx context.Context, oc *api.OpenShiftClu
 }
 
 func (m *manager) Delete(ctx context.Context, oc *api.OpenShiftCluster) error {
-	_, err := m.recordsets.Delete(ctx, m.instancemetadata.ResourceGroup(), m.domain, "api."+oc.Properties.DomainName, dns.CNAME, "")
+	_, err := m.recordsets.Delete(ctx, m.env.ResourceGroup(), m.Domain(), "api."+oc.Properties.DomainName, mgmtdns.CNAME, "")
 	if err != nil {
 		return err
 	}
 
-	_, err = m.recordsets.Delete(ctx, m.instancemetadata.ResourceGroup(), m.domain, "*.apps."+oc.Properties.DomainName, dns.A, "")
+	_, err = m.recordsets.Delete(ctx, m.env.ResourceGroup(), m.Domain(), "*.apps."+oc.Properties.DomainName, mgmtdns.A, "")
 
 	return err
 }

pkg/util/instancemetadata/dev.go

@@ -9,6 +9,7 @@ import (
 
 func NewDev() InstanceMetadata {
 	return &instanceMetadata{
+		tenantID:       os.Getenv("AZURE_TENANT_ID"),
 		subscriptionID: os.Getenv("AZURE_SUBSCRIPTION_ID"),
 		location:       os.Getenv("LOCATION"),
 		resourceGroup:  os.Getenv("RESOURCEGROUP"),

pkg/util/instancemetadata/instancemetadata.go

@@ -4,17 +4,23 @@ package instancemetadata
 // Licensed under the Apache License 2.0.
 
 type InstanceMetadata interface {
+	TenantID() string
 	SubscriptionID() string
 	Location() string
 	ResourceGroup() string
 }
 
 type instanceMetadata struct {
+	tenantID       string
 	subscriptionID string
 	location       string
 	resourceGroup  string
 }
 
+func (im *instanceMetadata) TenantID() string {
+	return im.tenantID
+}
+
 func (im *instanceMetadata) SubscriptionID() string {
 	return im.subscriptionID
 }

pkg/util/instancemetadata/prod.go

@@ -8,12 +8,29 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+
+	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/dgrijalva/jwt-go"
 )
 
+type azureClaim struct {
+	TenantID string `json:"tid,omitempty"`
+}
+
+func (*azureClaim) Valid() error {
+	return fmt.Errorf("unimplemented")
+}
+
 func NewProd() (InstanceMetadata, error) {
 	im := &instanceMetadata{}
 
-	err := im.populateInstanceMetadata()
+	err := im.populateTenantIDFromMSI()
+	if err != nil {
+		return nil, err
+	}
+
+	err = im.populateInstanceMetadata()
 	if err != nil {
 		return nil, err
 	}
@@ -21,6 +38,34 @@ func NewProd() (InstanceMetadata, error) {
 	return im, nil
 }
 
+func (im *instanceMetadata) populateTenantIDFromMSI() error {
+	msiEndpoint, err := adal.GetMSIVMEndpoint()
+	if err != nil {
+		return err
+	}
+
+	token, err := adal.NewServicePrincipalTokenFromMSI(msiEndpoint, azure.PublicCloud.ResourceManagerEndpoint)
+	if err != nil {
+		return err
+	}
+
+	err = token.EnsureFresh()
+	if err != nil {
+		return err
+	}
+
+	p := &jwt.Parser{}
+	c := &azureClaim{}
+	_, _, err = p.ParseUnverified(token.OAuthToken(), c)
+	if err != nil {
+		return err
+	}
+
+	im.tenantID = c.TenantID
+
+	return nil
+}
+
 func (im *instanceMetadata) populateInstanceMetadata() error {
 	req, err := http.NewRequest(http.MethodGet, "http://169.254.169.254/metadata/instance/compute?api-version=2019-03-11", nil)
 	if err != nil {

pkg/util/mocks/azureclient/graphrbac/graphrbac.go

@@ -0,0 +1,51 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/graphrbac (interfaces: ApplicationsClient)
+
+// Package mock_graphrbac is a generated GoMock package.
+package mock_graphrbac
+
+import (
+	context "context"
+	reflect "reflect"
+
+	graphrbac "github.com/Azure/azure-sdk-for-go/services/graphrbac/1.6/graphrbac"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockApplicationsClient is a mock of ApplicationsClient interface
+type MockApplicationsClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockApplicationsClientMockRecorder
+}
+
+// MockApplicationsClientMockRecorder is the mock recorder for MockApplicationsClient
+type MockApplicationsClientMockRecorder struct {
+	mock *MockApplicationsClient
+}
+
+// NewMockApplicationsClient creates a new mock instance
+func NewMockApplicationsClient(ctrl *gomock.Controller) *MockApplicationsClient {
+	mock := &MockApplicationsClient{ctrl: ctrl}
+	mock.recorder = &MockApplicationsClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockApplicationsClient) EXPECT() *MockApplicationsClientMockRecorder {
+	return m.recorder
+}
+
+// GetServicePrincipalsIDByAppID mocks base method
+func (m *MockApplicationsClient) GetServicePrincipalsIDByAppID(arg0 context.Context, arg1 string) (graphrbac.ServicePrincipalObjectResult, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetServicePrincipalsIDByAppID", arg0, arg1)
+	ret0, _ := ret[0].(graphrbac.ServicePrincipalObjectResult)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetServicePrincipalsIDByAppID indicates an expected call of GetServicePrincipalsIDByAppID
+func (mr *MockApplicationsClientMockRecorder) GetServicePrincipalsIDByAppID(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServicePrincipalsIDByAppID", reflect.TypeOf((*MockApplicationsClient)(nil).GetServicePrincipalsIDByAppID), arg0, arg1)
+}

pkg/util/mocks/azureclient/keyvault/keyvault.go

@@ -0,0 +1,51 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/keyvault (interfaces: BaseClient)
+
+// Package mock_keyvault is a generated GoMock package.
+package mock_keyvault
+
+import (
+	context "context"
+	reflect "reflect"
+
+	keyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockBaseClient is a mock of BaseClient interface
+type MockBaseClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockBaseClientMockRecorder
+}
+
+// MockBaseClientMockRecorder is the mock recorder for MockBaseClient
+type MockBaseClientMockRecorder struct {
+	mock *MockBaseClient
+}
+
+// NewMockBaseClient creates a new mock instance
+func NewMockBaseClient(ctrl *gomock.Controller) *MockBaseClient {
+	mock := &MockBaseClient{ctrl: ctrl}
+	mock.recorder = &MockBaseClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockBaseClient) EXPECT() *MockBaseClientMockRecorder {
+	return m.recorder
+}
+
+// GetSecret mocks base method
+func (m *MockBaseClient) GetSecret(arg0 context.Context, arg1, arg2, arg3 string) (keyvault.SecretBundle, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetSecret", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(keyvault.SecretBundle)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetSecret indicates an expected call of GetSecret
+func (mr *MockBaseClientMockRecorder) GetSecret(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSecret", reflect.TypeOf((*MockBaseClient)(nil).GetSecret), arg0, arg1, arg2, arg3)
+}

pkg/util/mocks/azureclient/mgmt/authorization/authorization.go

@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/authorization (interfaces: RoleAssignmentsClient)
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization (interfaces: RoleAssignmentsClient)
 
 // Package mock_authorization is a generated GoMock package.
 package mock_authorization

pkg/util/mocks/azureclient/mgmt/compute/compute.go

@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/compute (interfaces: DisksClient,VirtualMachinesClient)
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute (interfaces: DisksClient,VirtualMachinesClient)
 
 // Package mock_compute is a generated GoMock package.
 package mock_compute

pkg/util/mocks/azureclient/mgmt/dns/dns.go

@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/dns (interfaces: RecordSetsClient)
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/dns (interfaces: RecordSetsClient,ZonesClient)
 
 // Package mock_dns is a generated GoMock package.
 package mock_dns
@@ -65,3 +65,41 @@ func (mr *MockRecordSetsClientMockRecorder) Delete(arg0, arg1, arg2, arg3, arg4,
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockRecordSetsClient)(nil).Delete), arg0, arg1, arg2, arg3, arg4, arg5)
 }
+
+// MockZonesClient is a mock of ZonesClient interface
+type MockZonesClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockZonesClientMockRecorder
+}
+
+// MockZonesClientMockRecorder is the mock recorder for MockZonesClient
+type MockZonesClientMockRecorder struct {
+	mock *MockZonesClient
+}
+
+// NewMockZonesClient creates a new mock instance
+func NewMockZonesClient(ctrl *gomock.Controller) *MockZonesClient {
+	mock := &MockZonesClient{ctrl: ctrl}
+	mock.recorder = &MockZonesClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockZonesClient) EXPECT() *MockZonesClientMockRecorder {
+	return m.recorder
+}
+
+// ListByResourceGroup mocks base method
+func (m *MockZonesClient) ListByResourceGroup(arg0 context.Context, arg1 string, arg2 *int32) ([]dns.Zone, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListByResourceGroup", arg0, arg1, arg2)
+	ret0, _ := ret[0].([]dns.Zone)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListByResourceGroup indicates an expected call of ListByResourceGroup
+func (mr *MockZonesClientMockRecorder) ListByResourceGroup(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListByResourceGroup", reflect.TypeOf((*MockZonesClient)(nil).ListByResourceGroup), arg0, arg1, arg2)
+}

pkg/util/mocks/azureclient/mgmt/documentdb/documentdb.go

@@ -0,0 +1,66 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/documentdb (interfaces: DatabaseAccountsClient)
+
+// Package mock_documentdb is a generated GoMock package.
+package mock_documentdb
+
+import (
+	context "context"
+	reflect "reflect"
+
+	documentdb "github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockDatabaseAccountsClient is a mock of DatabaseAccountsClient interface
+type MockDatabaseAccountsClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockDatabaseAccountsClientMockRecorder
+}
+
+// MockDatabaseAccountsClientMockRecorder is the mock recorder for MockDatabaseAccountsClient
+type MockDatabaseAccountsClientMockRecorder struct {
+	mock *MockDatabaseAccountsClient
+}
+
+// NewMockDatabaseAccountsClient creates a new mock instance
+func NewMockDatabaseAccountsClient(ctrl *gomock.Controller) *MockDatabaseAccountsClient {
+	mock := &MockDatabaseAccountsClient{ctrl: ctrl}
+	mock.recorder = &MockDatabaseAccountsClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockDatabaseAccountsClient) EXPECT() *MockDatabaseAccountsClientMockRecorder {
+	return m.recorder
+}
+
+// ListByResourceGroup mocks base method
+func (m *MockDatabaseAccountsClient) ListByResourceGroup(arg0 context.Context, arg1 string) (documentdb.DatabaseAccountsListResult, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListByResourceGroup", arg0, arg1)
+	ret0, _ := ret[0].(documentdb.DatabaseAccountsListResult)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListByResourceGroup indicates an expected call of ListByResourceGroup
+func (mr *MockDatabaseAccountsClientMockRecorder) ListByResourceGroup(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListByResourceGroup", reflect.TypeOf((*MockDatabaseAccountsClient)(nil).ListByResourceGroup), arg0, arg1)
+}
+
+// ListKeys mocks base method
+func (m *MockDatabaseAccountsClient) ListKeys(arg0 context.Context, arg1, arg2 string) (documentdb.DatabaseAccountListKeysResult, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListKeys", arg0, arg1, arg2)
+	ret0, _ := ret[0].(documentdb.DatabaseAccountListKeysResult)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListKeys indicates an expected call of ListKeys
+func (mr *MockDatabaseAccountsClientMockRecorder) ListKeys(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListKeys", reflect.TypeOf((*MockDatabaseAccountsClient)(nil).ListKeys), arg0, arg1, arg2)
+}

pkg/util/mocks/azureclient/mgmt/keyvault/keyvault.go

@@ -0,0 +1,51 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/keyvault (interfaces: VaultsClient)
+
+// Package mock_keyvault is a generated GoMock package.
+package mock_keyvault
+
+import (
+	context "context"
+	reflect "reflect"
+
+	keyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2016-10-01/keyvault"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockVaultsClient is a mock of VaultsClient interface
+type MockVaultsClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockVaultsClientMockRecorder
+}
+
+// MockVaultsClientMockRecorder is the mock recorder for MockVaultsClient
+type MockVaultsClientMockRecorder struct {
+	mock *MockVaultsClient
+}
+
+// NewMockVaultsClient creates a new mock instance
+func NewMockVaultsClient(ctrl *gomock.Controller) *MockVaultsClient {
+	mock := &MockVaultsClient{ctrl: ctrl}
+	mock.recorder = &MockVaultsClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockVaultsClient) EXPECT() *MockVaultsClientMockRecorder {
+	return m.recorder
+}
+
+// ListByResourceGroup mocks base method
+func (m *MockVaultsClient) ListByResourceGroup(arg0 context.Context, arg1 string, arg2 *int32) ([]keyvault.Vault, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListByResourceGroup", arg0, arg1, arg2)
+	ret0, _ := ret[0].([]keyvault.Vault)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListByResourceGroup indicates an expected call of ListByResourceGroup
+func (mr *MockVaultsClientMockRecorder) ListByResourceGroup(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListByResourceGroup", reflect.TypeOf((*MockVaultsClient)(nil).ListByResourceGroup), arg0, arg1, arg2)
+}

pkg/util/mocks/azureclient/mgmt/network/network.go

@@ -0,0 +1,220 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network (interfaces: InterfacesClient,PrivateEndpointsClient,PublicIPAddressesClient,SubnetsClient)
+
+// Package mock_network is a generated GoMock package.
+package mock_network
+
+import (
+	context "context"
+	reflect "reflect"
+
+	network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockInterfacesClient is a mock of InterfacesClient interface
+type MockInterfacesClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockInterfacesClientMockRecorder
+}
+
+// MockInterfacesClientMockRecorder is the mock recorder for MockInterfacesClient
+type MockInterfacesClientMockRecorder struct {
+	mock *MockInterfacesClient
+}
+
+// NewMockInterfacesClient creates a new mock instance
+func NewMockInterfacesClient(ctrl *gomock.Controller) *MockInterfacesClient {
+	mock := &MockInterfacesClient{ctrl: ctrl}
+	mock.recorder = &MockInterfacesClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockInterfacesClient) EXPECT() *MockInterfacesClientMockRecorder {
+	return m.recorder
+}
+
+// DeleteAndWait mocks base method
+func (m *MockInterfacesClient) DeleteAndWait(arg0 context.Context, arg1, arg2 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteAndWait", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteAndWait indicates an expected call of DeleteAndWait
+func (mr *MockInterfacesClientMockRecorder) DeleteAndWait(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockInterfacesClient)(nil).DeleteAndWait), arg0, arg1, arg2)
+}
+
+// MockPrivateEndpointsClient is a mock of PrivateEndpointsClient interface
+type MockPrivateEndpointsClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockPrivateEndpointsClientMockRecorder
+}
+
+// MockPrivateEndpointsClientMockRecorder is the mock recorder for MockPrivateEndpointsClient
+type MockPrivateEndpointsClientMockRecorder struct {
+	mock *MockPrivateEndpointsClient
+}
+
+// NewMockPrivateEndpointsClient creates a new mock instance
+func NewMockPrivateEndpointsClient(ctrl *gomock.Controller) *MockPrivateEndpointsClient {
+	mock := &MockPrivateEndpointsClient{ctrl: ctrl}
+	mock.recorder = &MockPrivateEndpointsClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockPrivateEndpointsClient) EXPECT() *MockPrivateEndpointsClientMockRecorder {
+	return m.recorder
+}
+
+// CreateOrUpdateAndWait mocks base method
+func (m *MockPrivateEndpointsClient) CreateOrUpdateAndWait(arg0 context.Context, arg1, arg2 string, arg3 network.PrivateEndpoint) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateOrUpdateAndWait", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CreateOrUpdateAndWait indicates an expected call of CreateOrUpdateAndWait
+func (mr *MockPrivateEndpointsClientMockRecorder) CreateOrUpdateAndWait(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateAndWait", reflect.TypeOf((*MockPrivateEndpointsClient)(nil).CreateOrUpdateAndWait), arg0, arg1, arg2, arg3)
+}
+
+// DeleteAndWait mocks base method
+func (m *MockPrivateEndpointsClient) DeleteAndWait(arg0 context.Context, arg1, arg2 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteAndWait", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteAndWait indicates an expected call of DeleteAndWait
+func (mr *MockPrivateEndpointsClientMockRecorder) DeleteAndWait(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockPrivateEndpointsClient)(nil).DeleteAndWait), arg0, arg1, arg2)
+}
+
+// Get mocks base method
+func (m *MockPrivateEndpointsClient) Get(arg0 context.Context, arg1, arg2, arg3 string) (network.PrivateEndpoint, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3)
+	ret0, _ := ret[0].(network.PrivateEndpoint)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Get indicates an expected call of Get
+func (mr *MockPrivateEndpointsClientMockRecorder) Get(arg0, arg1, arg2, arg3 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockPrivateEndpointsClient)(nil).Get), arg0, arg1, arg2, arg3)
+}
+
+// MockPublicIPAddressesClient is a mock of PublicIPAddressesClient interface
+type MockPublicIPAddressesClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockPublicIPAddressesClientMockRecorder
+}
+
+// MockPublicIPAddressesClientMockRecorder is the mock recorder for MockPublicIPAddressesClient
+type MockPublicIPAddressesClientMockRecorder struct {
+	mock *MockPublicIPAddressesClient
+}
+
+// NewMockPublicIPAddressesClient creates a new mock instance
+func NewMockPublicIPAddressesClient(ctrl *gomock.Controller) *MockPublicIPAddressesClient {
+	mock := &MockPublicIPAddressesClient{ctrl: ctrl}
+	mock.recorder = &MockPublicIPAddressesClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockPublicIPAddressesClient) EXPECT() *MockPublicIPAddressesClientMockRecorder {
+	return m.recorder
+}
+
+// DeleteAndWait mocks base method
+func (m *MockPublicIPAddressesClient) DeleteAndWait(arg0 context.Context, arg1, arg2 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteAndWait", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// DeleteAndWait indicates an expected call of DeleteAndWait
+func (mr *MockPublicIPAddressesClientMockRecorder) DeleteAndWait(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockPublicIPAddressesClient)(nil).DeleteAndWait), arg0, arg1, arg2)
+}
+
+// List mocks base method
+func (m *MockPublicIPAddressesClient) List(arg0 context.Context, arg1 string) ([]network.PublicIPAddress, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "List", arg0, arg1)
+	ret0, _ := ret[0].([]network.PublicIPAddress)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// List indicates an expected call of List
+func (mr *MockPublicIPAddressesClientMockRecorder) List(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockPublicIPAddressesClient)(nil).List), arg0, arg1)
+}
+
+// MockSubnetsClient is a mock of SubnetsClient interface
+type MockSubnetsClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockSubnetsClientMockRecorder
+}
+
+// MockSubnetsClientMockRecorder is the mock recorder for MockSubnetsClient
+type MockSubnetsClientMockRecorder struct {
+	mock *MockSubnetsClient
+}
+
+// NewMockSubnetsClient creates a new mock instance
+func NewMockSubnetsClient(ctrl *gomock.Controller) *MockSubnetsClient {
+	mock := &MockSubnetsClient{ctrl: ctrl}
+	mock.recorder = &MockSubnetsClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockSubnetsClient) EXPECT() *MockSubnetsClientMockRecorder {
+	return m.recorder
+}
+
+// CreateOrUpdateAndWait mocks base method
+func (m *MockSubnetsClient) CreateOrUpdateAndWait(arg0 context.Context, arg1, arg2, arg3 string, arg4 network.Subnet) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateOrUpdateAndWait", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CreateOrUpdateAndWait indicates an expected call of CreateOrUpdateAndWait
+func (mr *MockSubnetsClientMockRecorder) CreateOrUpdateAndWait(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateAndWait", reflect.TypeOf((*MockSubnetsClient)(nil).CreateOrUpdateAndWait), arg0, arg1, arg2, arg3, arg4)
+}
+
+// Get mocks base method
+func (m *MockSubnetsClient) Get(arg0 context.Context, arg1, arg2, arg3, arg4 string) (network.Subnet, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3, arg4)
+	ret0, _ := ret[0].(network.Subnet)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Get indicates an expected call of Get
+func (mr *MockSubnetsClientMockRecorder) Get(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockSubnetsClient)(nil).Get), arg0, arg1, arg2, arg3, arg4)
+}

pkg/util/mocks/azureclient/mgmt/resources/resources.go

@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/resources (interfaces: DeploymentsClient,GroupsClient)
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/resources (interfaces: DeploymentsClient,GroupsClient)
 
 // Package mock_resources is a generated GoMock package.
 package mock_resources
@@ -50,6 +50,20 @@ func (mr *MockDeploymentsClientMockRecorder) CreateOrUpdateAndWait(arg0, arg1, a
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateAndWait", reflect.TypeOf((*MockDeploymentsClient)(nil).CreateOrUpdateAndWait), arg0, arg1, arg2, arg3)
 }
 
+// Wait mocks base method
+func (m *MockDeploymentsClient) Wait(arg0 context.Context, arg1, arg2 string) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Wait", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Wait indicates an expected call of Wait
+func (mr *MockDeploymentsClientMockRecorder) Wait(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Wait", reflect.TypeOf((*MockDeploymentsClient)(nil).Wait), arg0, arg1, arg2)
+}
+
 // MockGroupsClient is a mock of GroupsClient interface
 type MockGroupsClient struct {
 	ctrl     *gomock.Controller

pkg/util/mocks/azureclient/mgmt/storage/storage.go

@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/storage (interfaces: AccountsClient)
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage (interfaces: AccountsClient)
 
 // Package mock_storage is a generated GoMock package.
 package mock_storage

pkg/util/mocks/mock_azureclient/azureclient.go

@@ -1,49 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/Azure/ARO-RP/pkg/util/azureclient (interfaces: Client)
-
-// Package mock_azureclient is a generated GoMock package.
-package mock_azureclient
-
-import (
-	reflect "reflect"
-
-	autorest "github.com/Azure/go-autorest/autorest"
-	gomock "github.com/golang/mock/gomock"
-)
-
-// MockClient is a mock of Client interface
-type MockClient struct {
-	ctrl     *gomock.Controller
-	recorder *MockClientMockRecorder
-}
-
-// MockClientMockRecorder is the mock recorder for MockClient
-type MockClientMockRecorder struct {
-	mock *MockClient
-}
-
-// NewMockClient creates a new mock instance
-func NewMockClient(ctrl *gomock.Controller) *MockClient {
-	mock := &MockClient{ctrl: ctrl}
-	mock.recorder = &MockClientMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockClient) EXPECT() *MockClientMockRecorder {
-	return m.recorder
-}
-
-// Client mocks base method
-func (m *MockClient) Client() autorest.Client {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Client")
-	ret0, _ := ret[0].(autorest.Client)
-	return ret0
-}
-
-// Client indicates an expected call of Client
-func (mr *MockClientMockRecorder) Client() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Client", reflect.TypeOf((*MockClient)(nil).Client))
-}

pkg/util/mocks/mock_azureclient/mock_network/network.go

@@ -1,102 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/network (interfaces: InterfacesClient,PublicIPAddressesClient)
-
-// Package mock_network is a generated GoMock package.
-package mock_network
-
-import (
-	context "context"
-	reflect "reflect"
-
-	network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
-	gomock "github.com/golang/mock/gomock"
-)
-
-// MockInterfacesClient is a mock of InterfacesClient interface
-type MockInterfacesClient struct {
-	ctrl     *gomock.Controller
-	recorder *MockInterfacesClientMockRecorder
-}
-
-// MockInterfacesClientMockRecorder is the mock recorder for MockInterfacesClient
-type MockInterfacesClientMockRecorder struct {
-	mock *MockInterfacesClient
-}
-
-// NewMockInterfacesClient creates a new mock instance
-func NewMockInterfacesClient(ctrl *gomock.Controller) *MockInterfacesClient {
-	mock := &MockInterfacesClient{ctrl: ctrl}
-	mock.recorder = &MockInterfacesClientMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockInterfacesClient) EXPECT() *MockInterfacesClientMockRecorder {
-	return m.recorder
-}
-
-// DeleteAndWait mocks base method
-func (m *MockInterfacesClient) DeleteAndWait(arg0 context.Context, arg1, arg2 string) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "DeleteAndWait", arg0, arg1, arg2)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// DeleteAndWait indicates an expected call of DeleteAndWait
-func (mr *MockInterfacesClientMockRecorder) DeleteAndWait(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockInterfacesClient)(nil).DeleteAndWait), arg0, arg1, arg2)
-}
-
-// MockPublicIPAddressesClient is a mock of PublicIPAddressesClient interface
-type MockPublicIPAddressesClient struct {
-	ctrl     *gomock.Controller
-	recorder *MockPublicIPAddressesClientMockRecorder
-}
-
-// MockPublicIPAddressesClientMockRecorder is the mock recorder for MockPublicIPAddressesClient
-type MockPublicIPAddressesClientMockRecorder struct {
-	mock *MockPublicIPAddressesClient
-}
-
-// NewMockPublicIPAddressesClient creates a new mock instance
-func NewMockPublicIPAddressesClient(ctrl *gomock.Controller) *MockPublicIPAddressesClient {
-	mock := &MockPublicIPAddressesClient{ctrl: ctrl}
-	mock.recorder = &MockPublicIPAddressesClientMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockPublicIPAddressesClient) EXPECT() *MockPublicIPAddressesClientMockRecorder {
-	return m.recorder
-}
-
-// DeleteAndWait mocks base method
-func (m *MockPublicIPAddressesClient) DeleteAndWait(arg0 context.Context, arg1, arg2 string) error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "DeleteAndWait", arg0, arg1, arg2)
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// DeleteAndWait indicates an expected call of DeleteAndWait
-func (mr *MockPublicIPAddressesClientMockRecorder) DeleteAndWait(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockPublicIPAddressesClient)(nil).DeleteAndWait), arg0, arg1, arg2)
-}
-
-// List mocks base method
-func (m *MockPublicIPAddressesClient) List(arg0 context.Context, arg1 string) ([]network.PublicIPAddress, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "List", arg0, arg1)
-	ret0, _ := ret[0].([]network.PublicIPAddress)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// List indicates an expected call of List
-func (mr *MockPublicIPAddressesClientMockRecorder) List(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockPublicIPAddressesClient)(nil).List), arg0, arg1)
-}

pkg/util/permissions/permissions.go

@@ -7,11 +7,11 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/Azure/azure-sdk-for-go/services/authorization/mgmt/2015-07-01/authorization"
+	mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/authorization/mgmt/2015-07-01/authorization"
 )
 
 // CanDoAction returns true if a given action is granted by a set of permissions
-func CanDoAction(ps []authorization.Permission, a string) (bool, error) {
+func CanDoAction(ps []mgmtauthorization.Permission, a string) (bool, error) {
 	for _, p := range ps {
 		var matched bool
 

pkg/util/permissions/permissions_test.go

@@ -6,13 +6,13 @@ package permissions
 import (
 	"testing"
 
-	"github.com/Azure/azure-sdk-for-go/services/authorization/mgmt/2015-07-01/authorization"
+	mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/authorization/mgmt/2015-07-01/authorization"
 )
 
 func TestCanDoAction(t *testing.T) {
 	tests := []struct {
 		name        string
-		permissions []authorization.Permission
+		permissions []mgmtauthorization.Permission
 		action      string
 		want        bool
 	}{
@@ -22,7 +22,7 @@ func TestCanDoAction(t *testing.T) {
 		},
 		{
 			name: "has permission - exact",
-			permissions: []authorization.Permission{
+			permissions: []mgmtauthorization.Permission{
 				{
 					Actions:    &[]string{"Microsoft.Compute/virtualMachines/*"},
 					NotActions: &[]string{},
@@ -37,7 +37,7 @@ func TestCanDoAction(t *testing.T) {
 		},
 		{
 			name: "has permission - wildcard",
-			permissions: []authorization.Permission{{
+			permissions: []mgmtauthorization.Permission{{
 				Actions:    &[]string{"Microsoft.Network/virtualNetworks/subnets/*/action"},
 				NotActions: &[]string{},
 			}},
@@ -46,7 +46,7 @@ func TestCanDoAction(t *testing.T) {
 		},
 		{
 			name: "has permission - exact, conflict",
-			permissions: []authorization.Permission{
+			permissions: []mgmtauthorization.Permission{
 				{
 					Actions:    &[]string{"Microsoft.Network/virtualNetworks/subnets/join/action"},
 					NotActions: &[]string{},
@@ -61,7 +61,7 @@ func TestCanDoAction(t *testing.T) {
 		},
 		{
 			name: "has permission excluded - exact",
-			permissions: []authorization.Permission{{
+			permissions: []mgmtauthorization.Permission{{
 				Actions:    &[]string{"Microsoft.Network/*"},
 				NotActions: &[]string{"Microsoft.Network/virtualNetworks/subnets/join/action"},
 			}},
@@ -69,7 +69,7 @@ func TestCanDoAction(t *testing.T) {
 		},
 		{
 			name: "has permission excluded - wildcard",
-			permissions: []authorization.Permission{{
+			permissions: []mgmtauthorization.Permission{{
 				Actions:    &[]string{"Microsoft.Network/*"},
 				NotActions: &[]string{"Microsoft.Network/virtualNetworks/subnets/*/action"},
 			}},

pkg/util/privateendpoint/privateendpoint.go

@@ -0,0 +1,76 @@
+package privateendpoint
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/Azure/go-autorest/autorest/to"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/env"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network"
+)
+
+const prefix = "rp-pe-"
+
+type Manager interface {
+	Create(context.Context, *api.OpenShiftClusterDocument) error
+	Delete(context.Context, *api.OpenShiftClusterDocument) error
+	GetIP(context.Context, *api.OpenShiftClusterDocument) (string, error)
+}
+
+type manager struct {
+	env env.Interface
+
+	privateendpoints network.PrivateEndpointsClient
+}
+
+func NewManager(env env.Interface, localFPAuthorizer autorest.Authorizer) Manager {
+	return &manager{
+		env: env,
+
+		privateendpoints: network.NewPrivateEndpointsClient(env.SubscriptionID(), localFPAuthorizer),
+	}
+}
+
+func (m *manager) Create(ctx context.Context, doc *api.OpenShiftClusterDocument) error {
+	r, err := azure.ParseResourceID(doc.OpenShiftCluster.ID)
+	if err != nil {
+		return err
+	}
+
+	return m.privateendpoints.CreateOrUpdateAndWait(ctx, m.env.ResourceGroup(), prefix+doc.ID, mgmtnetwork.PrivateEndpoint{
+		PrivateEndpointProperties: &mgmtnetwork.PrivateEndpointProperties{
+			Subnet: &mgmtnetwork.Subnet{
+				ID: to.StringPtr("/subscriptions/" + m.env.SubscriptionID() + "/resourceGroups/" + m.env.ResourceGroup() + "/providers/Microsoft.Network/virtualNetworks/rp-vnet/subnets/rp-pe-subnet"),
+			},
+			ManualPrivateLinkServiceConnections: &[]mgmtnetwork.PrivateLinkServiceConnection{
+				{
+					Name: to.StringPtr("rp-plsconnection"),
+					PrivateLinkServiceConnectionProperties: &mgmtnetwork.PrivateLinkServiceConnectionProperties{
+						PrivateLinkServiceID: to.StringPtr("/subscriptions/" + r.SubscriptionID + "/resourceGroups/" + doc.OpenShiftCluster.Properties.ResourceGroup + "/providers/Microsoft.Network/privateLinkServices/aro-pls"),
+					},
+				},
+			},
+		},
+		Location: &doc.OpenShiftCluster.Location,
+	})
+}
+
+func (m *manager) Delete(ctx context.Context, doc *api.OpenShiftClusterDocument) error {
+	return m.privateendpoints.DeleteAndWait(ctx, m.env.ResourceGroup(), prefix+doc.ID)
+}
+
+func (m *manager) GetIP(ctx context.Context, doc *api.OpenShiftClusterDocument) (string, error) {
+	pe, err := m.privateendpoints.Get(ctx, m.env.ResourceGroup(), prefix+doc.ID, "networkInterfaces")
+	if err != nil {
+		return "", nil
+	}
+
+	return *(*(*pe.PrivateEndpointProperties.NetworkInterfaces)[0].IPConfigurations)[0].PrivateIPAddress, nil
+}

pkg/util/restconfig/restconfig.go

@@ -4,16 +4,41 @@ package restconfig
 // Licensed under the Apache License 2.0.
 
 import (
+	"context"
+	"fmt"
+	"net"
+
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/env"
 )
 
 // RestConfig returns the Kubernetes *rest.Config for a kubeconfig
-func RestConfig(b []byte) (*rest.Config, error) {
-	config, err := clientcmd.Load(b)
+func RestConfig(ctx context.Context, env env.Interface, doc *api.OpenShiftClusterDocument, ip string) (*rest.Config, error) {
+	config, err := clientcmd.Load(doc.OpenShiftCluster.Properties.AdminKubeconfig)
 	if err != nil {
 		return nil, err
 	}
 
-	return clientcmd.NewDefaultClientConfig(*config, &clientcmd.ConfigOverrides{}).ClientConfig()
+	restconfig, err := clientcmd.NewDefaultClientConfig(*config, &clientcmd.ConfigOverrides{}).ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	restconfig.Dial = func(ctx context.Context, network, address string) (net.Conn, error) {
+		if network != "tcp" {
+			return nil, fmt.Errorf("unimplemented network %q", network)
+		}
+
+		_, port, err := net.SplitHostPort(address)
+		if err != nil {
+			return nil, err
+		}
+
+		return env.DialContext(ctx, network, ip+":"+port)
+	}
+
+	return restconfig, nil
 }

pkg/util/subnet/generate.go

@@ -3,5 +3,5 @@ package subnet
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../../vendor/github.com/golang/mock/mockgen -destination=../mocks/mock_$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/$GOPACKAGE Manager
-//go:generate go run ../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../mocks/mock_$GOPACKAGE/$GOPACKAGE.go
+//go:generate go run ../../../vendor/github.com/golang/mock/mockgen -destination=../mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/$GOPACKAGE Manager
+//go:generate go run ../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../mocks/$GOPACKAGE/$GOPACKAGE.go

pkg/util/subnet/subnet.go

@@ -8,16 +8,17 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
+	mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/azure"
 
 	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network"
 )
 
 type Manager interface {
-	Get(ctx context.Context, subnetID string) (*network.Subnet, error)
-	CreateOrUpdate(ctx context.Context, subnetID string, subnet *network.Subnet) error
+	Get(ctx context.Context, subnetID string) (*mgmtnetwork.Subnet, error)
+	CreateOrUpdate(ctx context.Context, subnetID string, subnet *mgmtnetwork.Subnet) error
 }
 
 type manager struct {
@@ -25,17 +26,13 @@ type manager struct {
 }
 
 func NewManager(subscriptionID string, spAuthorizer autorest.Authorizer) Manager {
-	m := &manager{
-		subnets: network.NewSubnetsClient(subscriptionID),
+	return &manager{
+		subnets: network.NewSubnetsClient(subscriptionID, spAuthorizer),
 	}
-
-	m.subnets.Authorizer = spAuthorizer
-
-	return m
 }
 
 // Get retrieves the linked subnet
-func (m *manager) Get(ctx context.Context, subnetID string) (*network.Subnet, error) {
+func (m *manager) Get(ctx context.Context, subnetID string) (*mgmtnetwork.Subnet, error) {
 	vnetID, subnetName, err := Split(subnetID)
 	if err != nil {
 		return nil, err
@@ -55,7 +52,7 @@ func (m *manager) Get(ctx context.Context, subnetID string) (*network.Subnet, er
 }
 
 // CreateOrUpdate updates the linked subnet
-func (m *manager) CreateOrUpdate(ctx context.Context, subnetID string, subnet *network.Subnet) error {
+func (m *manager) CreateOrUpdate(ctx context.Context, subnetID string, subnet *mgmtnetwork.Subnet) error {
 	vnetID, subnetName, err := Split(subnetID)
 	if err != nil {
 		return err
@@ -66,12 +63,7 @@ func (m *manager) CreateOrUpdate(ctx context.Context, subnetID string, subnet *n
 		return err
 	}
 
-	future, err := m.subnets.CreateOrUpdate(ctx, r.ResourceGroup, r.ResourceName, subnetName, *subnet)
-	if err != nil {
-		return err
-	}
-
-	return future.WaitForCompletionRef(ctx, m.subnets.Client)
+	return m.subnets.CreateOrUpdateAndWait(ctx, r.ResourceGroup, r.ResourceName, subnetName, *subnet)
 }
 
 // Split splits the given subnetID into a vnetID and subnetName