Azure
/
ARO-RP
зеркало из https://github.com/Azure/ARO-RP.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #233 from mjudeikis/vault.move 

Move vault to pre-deploy and split managedIdentity
This commit is contained in:
Jim Minter 2020-03-12 16:17:43 -06:00 коммит произвёл GitHub
Родитель fe68521a1f f7f080ef3c
Коммит ca627bae63
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
21 изменённых файлов: 847 добавлений и 653 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
.pipelines/templates/template-deploy-shared-env.yml
Просмотреть файл

@ -8,8 +8,8 @@ steps:
. ./hack/devtools/deploy-shared-env.sh
create_infra_rg
deploy_rp_dev_nsg
validate_arm_template_state "rp-development-nsg"
deploy_rp_dev_predeploy
validate_arm_template_state "rp-development-predeploy"
deploy_rp_dev
validate_arm_template_state "rp-development"
deploy_env_dev

49
deploy/rp-development-nsg.json
Просмотреть файл

@ -1,49 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"resources": [
{
"properties": {
"securityRules": [
{
"properties": {
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "443",
"sourceAddressPrefix": "*",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 120,
"direction": "Inbound"
},
"name": "rp_in"
},
{
"properties": {
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "22",
"sourceAddressPrefix": "*",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 100,
"direction": "Inbound"
},
"name": "ssh_in"
}
]
},
"name": "rp-nsg",
"type": "Microsoft.Network/networkSecurityGroups",
"location": "[resourceGroup().location]",
"apiVersion": "2019-07-01"
},
{
"properties": {},
"name": "rp-pe-nsg",
"type": "Microsoft.Network/networkSecurityGroups",
"location": "[resourceGroup().location]",
"apiVersion": "2019-07-01"
}
]
}

148
deploy/rp-development-predeploy.json Normal file
Просмотреть файл

@ -0,0 +1,148 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"adminObjectId": {
"type": "string"
},
"fpServicePrincipalId": {
"type": "string"
},
"keyvaultPrefix": {
"type": "string",
"maxLength": 20
},
"rpServicePrincipalId": {
"type": "string"
}
},
"resources": [
{
"properties": {
"securityRules": [
{
"properties": {
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "443",
"sourceAddressPrefix": "*",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 120,
"direction": "Inbound"
},
"name": "rp_in"
},
{
"properties": {
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "22",
"sourceAddressPrefix": "*",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 100,
"direction": "Inbound"
},
"name": "ssh_in"
}
]
},
"name": "rp-nsg",
"type": "Microsoft.Network/networkSecurityGroups",
"location": "[resourceGroup().location]",
"apiVersion": "2019-07-01"
},
{
"properties": {},
"name": "rp-pe-nsg",
"type": "Microsoft.Network/networkSecurityGroups",
"location": "[resourceGroup().location]",
"apiVersion": "2019-07-01"
},
{
"properties": {
"tenantId": "[subscription().tenantId]",
"sku": {
"family": "A",
"name": "standard"
},
"accessPolicies": [
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('fpServicePrincipalId')]",
"permissions": {
"secrets": [
"get"
],
"certificates": [
"create",
"delete"
]
}
},
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('adminObjectId')]",
"permissions": {
"certificates": [
"get",
"list"
]
}
}
]
},
"name": "[concat(parameters('keyvaultPrefix'), '-cls')]",
"type": "Microsoft.KeyVault/vaults",
"location": "[resourceGroup().location]",
"tags": {
"vault": "clusters"
},
"apiVersion": "2016-10-01"
},
{
"properties": {
"tenantId": "[subscription().tenantId]",
"sku": {
"family": "A",
"name": "standard"
},
"accessPolicies": [
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('rpServicePrincipalId')]",
"permissions": {
"secrets": [
"get"
]
}
},
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('adminObjectId')]",
"permissions": {
"secrets": [
"set",
"list"
],
"certificates": [
"delete",
"get",
"import",
"list"
]
}
}
]
},
"name": "[concat(parameters('keyvaultPrefix'), '-svc')]",
"type": "Microsoft.KeyVault/vaults",
"location": "[resourceGroup().location]",
"tags": {
"vault": "service"
},
"apiVersion": "2016-10-01"
}
]
}

91
deploy/rp-development.json
Просмотреть файл

@ -2,9 +2,6 @@
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"adminObjectId": {
"type": "string"
},
"databaseAccountName": {
"type": "string"
},
@ -14,10 +11,6 @@
"fpServicePrincipalId": {
"type": "string"
},
"keyvaultPrefix": {
"type": "string",
"maxLength": 20
},
"rpServicePrincipalId": {
"type": "string"
}
@ -30,90 +23,6 @@
"location": "global",
"apiVersion": "2018-05-01"
},
{
"properties": {
"tenantId": "[subscription().tenantId]",
"sku": {
"family": "A",
"name": "standard"
},
"accessPolicies": [
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('fpServicePrincipalId')]",
"permissions": {
"secrets": [
"get"
],
"certificates": [
"create",
"delete"
]
}
},
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('adminObjectId')]",
"permissions": {
"certificates": [
"get",
"list"
]
}
}
]
},
"name": "[concat(parameters('keyvaultPrefix'), '-cls')]",
"type": "Microsoft.KeyVault/vaults",
"location": "[resourceGroup().location]",
"tags": {
"vault": "clusters"
},
"apiVersion": "2016-10-01"
},
{
"properties": {
"tenantId": "[subscription().tenantId]",
"sku": {
"family": "A",
"name": "standard"
},
"accessPolicies": [
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('rpServicePrincipalId')]",
"permissions": {
"secrets": [
"get"
]
}
},
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('adminObjectId')]",
"permissions": {
"secrets": [
"set",
"list"
],
"certificates": [
"delete",
"get",
"import",
"list"
]
}
}
]
},
"name": "[concat(parameters('keyvaultPrefix'), '-svc')]",
"type": "Microsoft.KeyVault/vaults",
"location": "[resourceGroup().location]",
"tags": {
"vault": "service"
},
"apiVersion": "2016-10-01"
},
{
"properties": {
"addressSpace": {

18
deploy/rp-production-managed-identity.json Normal file
Просмотреть файл

@ -0,0 +1,18 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"resources": [
{
"type": "Microsoft.ManagedIdentity/userAssignedIdentities",
"name": "rp-identity",
"location": "[resourceGroup().location]",
"apiVersion": "2018-11-30"
}
],
"outputs": {
"rpServicePrincipalId": {
"type": "string",
"value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'rp-identity'), '2018-11-30').principalId]"
}
}
}

48
deploy/rp-production-nsg.json
Просмотреть файл

@ -1,48 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"resources": [
{
"properties": {
"securityRules": [
{
"properties": {
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "443",
"sourceAddressPrefix": "*",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 120,
"direction": "Inbound"
},
"name": "rp_in"
}
]
},
"name": "rp-nsg",
"type": "Microsoft.Network/networkSecurityGroups",
"location": "[resourceGroup().location]",
"apiVersion": "2019-07-01"
},
{
"properties": {},
"name": "rp-pe-nsg",
"type": "Microsoft.Network/networkSecurityGroups",
"location": "[resourceGroup().location]",
"apiVersion": "2019-07-01"
},
{
"type": "Microsoft.ManagedIdentity/userAssignedIdentities",
"name": "rp-identity",
"location": "[resourceGroup().location]",
"apiVersion": "2018-11-30"
}
],
"outputs": {
"rpServicePrincipalId": {
"type": "string",
"value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'rp-identity'), '2018-11-30').principalId]"
}
}
}

6
deploy/rp-production-parameters.json
Просмотреть файл

@ -17,15 +17,9 @@
"extraCosmosDBIPs": {
"value": ""
},
"extraKeyvaultAccessPolicies": {
"value": []
},
"fpServicePrincipalId": {
"value": ""
},
"keyvaultPrefix": {
"value": ""
},
"mdmFrontendUrl": {
"value": ""
},

18
deploy/rp-production-predeploy-parameters.json Normal file
Просмотреть файл

@ -0,0 +1,18 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"extraKeyvaultAccessPolicies": {
"value": []
},
"fpServicePrincipalId": {
"value": ""
},
"keyvaultPrefix": {
"value": ""
},
"rpServicePrincipalId": {
"value": ""
}
}
}

114
deploy/rp-production-predeploy.json Normal file
Просмотреть файл

@ -0,0 +1,114 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"variables": {
"clustersKeyvaultAccessPolicies": [
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('fpServicePrincipalId')]",
"permissions": {
"secrets": [
"get"
],
"certificates": [
"create",
"delete"
]
}
}
],
"serviceKeyvaultAccessPolicies": [
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('rpServicePrincipalId')]",
"permissions": {
"secrets": [
"get"
]
}
}
]
},
"parameters": {
"extraKeyvaultAccessPolicies": {
"type": "array",
"defaultValue": []
},
"fpServicePrincipalId": {
"type": "string"
},
"keyvaultPrefix": {
"type": "string",
"maxLength": 20
},
"rpServicePrincipalId": {
"type": "string"
}
},
"resources": [
{
"properties": {
"securityRules": [
{
"properties": {
"protocol": "Tcp",
"sourcePortRange": "*",
"destinationPortRange": "443",
"sourceAddressPrefix": "*",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 120,
"direction": "Inbound"
},
"name": "rp_in"
}
]
},
"name": "rp-nsg",
"type": "Microsoft.Network/networkSecurityGroups",
"location": "[resourceGroup().location]",
"apiVersion": "2019-07-01"
},
{
"properties": {},
"name": "rp-pe-nsg",
"type": "Microsoft.Network/networkSecurityGroups",
"location": "[resourceGroup().location]",
"apiVersion": "2019-07-01"
},
{
"properties": {
"tenantId": "[subscription().tenantId]",
"sku": {
"family": "A",
"name": "standard"
},
"accessPolicies": "[concat(variables('clustersKeyvaultAccessPolicies'), parameters('extraKeyvaultAccessPolicies'))]"
},
"name": "[concat(parameters('keyvaultPrefix'), '-cls')]",
"type": "Microsoft.KeyVault/vaults",
"location": "[resourceGroup().location]",
"tags": {
"vault": "clusters"
},
"apiVersion": "2016-10-01"
},
{
"properties": {
"tenantId": "[subscription().tenantId]",
"sku": {
"family": "A",
"name": "standard"
},
"accessPolicies": "[concat(variables('serviceKeyvaultAccessPolicies'), parameters('extraKeyvaultAccessPolicies'))]"
},
"name": "[concat(parameters('keyvaultPrefix'), '-svc')]",
"type": "Microsoft.KeyVault/vaults",
"location": "[resourceGroup().location]",
"tags": {
"vault": "service"
},
"apiVersion": "2016-10-01"
}
]
}

70
deploy/rp-production.json
Просмотреть файл

@ -1,34 +1,6 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"variables": {
"clustersKeyvaultAccessPolicies": [
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('fpServicePrincipalId')]",
"permissions": {
"secrets": [
"get"
],
"certificates": [
"create",
"delete"
]
}
}
],
"serviceKeyvaultAccessPolicies": [
{
"tenantId": "[subscription().tenantId]",
"objectId": "[parameters('rpServicePrincipalId')]",
"permissions": {
"secrets": [
"get"
]
}
}
]
},
"parameters": {
"adminApiCaBundle": {
"type": "string"
@ -46,17 +18,9 @@
"type": "string",
"defaultValue": ""
},
"extraKeyvaultAccessPolicies": {
"type": "array",
"defaultValue": []
},
"fpServicePrincipalId": {
"type": "string"
},
"keyvaultPrefix": {
"type": "string",
"maxLength": 20
},
"mdmFrontendUrl": {
"type": "string"
},
@ -282,40 +246,6 @@
"location": "global",
"apiVersion": "2018-05-01"
},
{
"properties": {
"tenantId": "[subscription().tenantId]",
"sku": {
"family": "A",
"name": "standard"
},
"accessPolicies": "[concat(variables('clustersKeyvaultAccessPolicies'), parameters('extraKeyvaultAccessPolicies'))]"
},
"name": "[concat(parameters('keyvaultPrefix'), '-cls')]",
"type": "Microsoft.KeyVault/vaults",
"location": "[resourceGroup().location]",
"tags": {
"vault": "clusters"
},
"apiVersion": "2016-10-01"
},
{
"properties": {
"tenantId": "[subscription().tenantId]",
"sku": {
"family": "A",
"name": "standard"
},
"accessPolicies": "[concat(variables('serviceKeyvaultAccessPolicies'), parameters('extraKeyvaultAccessPolicies'))]"
},
"name": "[concat(parameters('keyvaultPrefix'), '-svc')]",
"type": "Microsoft.KeyVault/vaults",
"location": "[resourceGroup().location]",
"tags": {
"vault": "service"
},
"apiVersion": "2016-10-01"
},
{
"properties": {
"addressSpace": {

40
docs/deploy-upgrade-rp.md
Просмотреть файл

@ -15,8 +15,8 @@ the following environment variables:
* RESOURCEGROUP: RP resource group name
* RP_PARAMETERS_FILE: location of environment parameters file (if
RP_PREDEPLOY_ONLY is not set)
* RP_PARAMETERS_FILE: location of environment parameters file (same variable
used for predeploy and deploy)
* Optional:
@ -24,7 +24,6 @@ the following environment variables:
* RP_PREDEPLOY_ONLY: exit after pre-deploy step
Notes:
* If the deployment tool is run on an existing resource group, it will update
@ -32,8 +31,37 @@ Notes:
* The new RP VMSS will be created with postfix `-short_gitcommit`.
* Parameters file exampleL `deploy/rp-production-parameters.json`.
* Parameters file example `deploy/rp-production-parameters.json`.
* The utility will not re-deploy rp-production-nsg.json if the deployment
already exists. If you want to re-deploy rp-production-nsg.json, delete
* The utility will not re-deploy rp-production-predeploy.json if the deployment
already exists. If you want to re-deploy rp-production-predeploy.json, delete
existing deployment object.
## Deployment logical order:
* Deploy managed identity `rp-production-managed-identity.json`. This will
produce `rpServicePrincipalId` required by next deployments.
* Deploy pre-deploy resources `rp-production-predeploy.json` with
`rp-production-predeploy-parameters.json`.
* Deploy main deployment resources `rp-production.json` with
`rp-production-parameters.json`.
## Utility example
```bash
# run pre-deploy phase only
export RP_PARAMETERS_FILE=rp-production-predeploy-parameters.json
RP_PREDEPLOY_ONLY=true go run ./cmd/aro deploy
# deploy RP under name test
export RP_VERSION="test"
export RP_PARAMETERS_FILE=rp-production-parameters.json
go run ./cmd/aro deploy
# deploy second VMSS instance with name test2 and retire test
export RP_VERSION="test2"
export RP_PARAMETERS_FILE=rp-production-parameters.json
go run ./cmd/aro deploy
```

4
docs/prepare-a-shared-rp-development-environment.md
Просмотреть файл

@ -278,8 +278,8 @@ each of the bash functions below.
. ./hack/devtools/deploy-shared-env.sh
# Create the RG
create_infra_rg
# Deploy NSG
deploy_rp_dev_nsg
# Deploy the predeployment ARM template
deploy_rp_dev_predeploy
# Deploy the infrastructure resources such as Cosmos, KV, Vnet...
deploy_rp_dev
# Deploy the proxy and VPN

15
hack/devtools/deploy-shared-env.sh
Просмотреть файл

@ -6,12 +6,17 @@ create_infra_rg() {
az group create -g "$RESOURCEGROUP" -l "$LOCATION" >/dev/null
}
deploy_rp_dev_nsg() {
echo "########## Deploying rp-development-nsg in RG $RESOURCEGROUP ##########"
deploy_rp_dev_predeploy() {
echo "########## Deploying rp-development-predeploy in RG $RESOURCEGROUP ##########"
az group deployment create \
-g "$RESOURCEGROUP" \
-n rp-development-nsg \
--template-file deploy/rp-development-nsg.json
-n rp-development-predeploy \
--template-file deploy/rp-development-predeploy.json \
--parameters \
"adminObjectId=$ADMIN_OBJECT_ID" \
"fpServicePrincipalId=$(az ad sp list --filter "appId eq '$AZURE_FP_CLIENT_ID'" --query '[].objectId' -o tsv)" \
"keyvaultPrefix=$KEYVAULT_PREFIX" \
"rpServicePrincipalId=$(az ad sp list --filter "appId eq '$AZURE_CLIENT_ID'" --query '[].objectId' -o tsv)" >/dev/null
}
deploy_rp_dev() {
@ -21,11 +26,9 @@ deploy_rp_dev() {
-n rp-development \
--template-file deploy/rp-development.json \
--parameters \
"adminObjectId=$ADMIN_OBJECT_ID" \
"databaseAccountName=$COSMOSDB_ACCOUNT" \
"domainName=$DOMAIN_NAME.$PARENT_DOMAIN_NAME" \
"fpServicePrincipalId=$(az ad sp list --filter "appId eq '$AZURE_FP_CLIENT_ID'" --query '[].objectId' -o tsv)" \
"keyvaultPrefix=$KEYVAULT_PREFIX" \
"rpServicePrincipalId=$(az ad sp list --filter "appId eq '$AZURE_CLIENT_ID'" --query '[].objectId' -o tsv)" >/dev/null
}

16
hack/gendeploy/gendeploy.go
Просмотреть файл

@ -8,22 +8,14 @@ import (
)
func run() error {
err := generator.GenerateRPTemplates()
// dev artifacts
err := generator.New(false).Artifacts()
if err != nil {
return err
}
err = generator.GenerateNSGTemplates()
if err != nil {
return err
}
err = generator.GenerateRPParameterTemplate()
if err != nil {
return err
}
return generator.GenerateDevelopmentTemplate()
// prod artifacts
return generator.New(true).Artifacts()
}
func main() {

116
pkg/deploy/bindata.go
Просмотреть файл

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

121
pkg/deploy/deploy.go
Просмотреть файл

@ -76,44 +76,37 @@ func New(ctx context.Context, log *logrus.Entry, authorizer autorest.Authorizer,
}
}
// PreDeploy deploys NSG and ManagedIdentity, needed for man deployment
func (d *deployer) PreDeploy(ctx context.Context) (rpServicePrincipalID string, err error) {
group := azresources.Group{
// PreDeploy deploys managed identity, NSGs and keyvaults, needed for main
// deployment
func (d *deployer) PreDeploy(ctx context.Context) (string, error) {
_, err := d.groups.CreateOrUpdate(ctx, d.resourceGroup, azresources.Group{
Location: &d.location,
}
_, err = d.groups.CreateOrUpdate(ctx, d.resourceGroup, group)
})
if err != nil {
return "", err
}
deploymentName := "rp-production-nsg"
// deploy managed identity if needed and get rpServicePrincipalID
rpServicePrincipalID, err := d.deployManageIdentity(ctx)
if err != nil {
return "", err
}
// deploy NSGs, keyvaults
err = d.deployPreDeploy(ctx, rpServicePrincipalID)
if err != nil {
return "", err
}
return rpServicePrincipalID, nil
}
func (d *deployer) deployManageIdentity(ctx context.Context) (string, error) {
deploymentName := "rp-production-managed-identity"
deployment, err := d.deployments.Get(ctx, d.resourceGroup, deploymentName)
if isDeploymentNotFoundError(err) {
var b []byte // must not shadow err
b, err = Asset(generator.FileRPProductionNSG)
if err != nil {
return "", err
}
var template map[string]interface{}
err = json.Unmarshal(b, &template)
if err != nil {
return "", err
}
d.log.Printf("predeploying to %s", d.resourceGroup)
err = d.deployments.CreateOrUpdateAndWait(ctx, d.resourceGroup, deploymentName, azresources.Deployment{
Properties: &azresources.DeploymentProperties{
Template: template,
Mode: azresources.Incremental,
},
})
if err != nil {
return "", err
}
deployment, err = d.deployments.Get(ctx, d.resourceGroup, deploymentName)
deployment, err = d._deployManageIdentity(ctx, deploymentName)
}
if err != nil {
return "", err
@ -122,6 +115,70 @@ func (d *deployer) PreDeploy(ctx context.Context) (rpServicePrincipalID string,
return deployment.Properties.Outputs.(map[string]interface{})["rpServicePrincipalId"].(map[string]interface{})["value"].(string), nil
}
func (d *deployer) _deployManageIdentity(ctx context.Context, deploymentName string) (azresources.DeploymentExtended, error) {
b, err := Asset(generator.FileRPProductionManagedIdentity)
if err != nil {
return azresources.DeploymentExtended{}, nil
}
var template map[string]interface{}
err = json.Unmarshal(b, &template)
if err != nil {
return azresources.DeploymentExtended{}, nil
}
d.log.Infof("deploying managed identity to %s", d.resourceGroup)
err = d.deployments.CreateOrUpdateAndWait(ctx, d.resourceGroup, deploymentName, azresources.Deployment{
Properties: &azresources.DeploymentProperties{
Template: template,
Mode: azresources.Incremental,
},
})
if err != nil {
return azresources.DeploymentExtended{}, nil
}
return d.deployments.Get(ctx, d.resourceGroup, deploymentName)
}
func (d *deployer) deployPreDeploy(ctx context.Context, rpServicePrincipalID string) error {
deploymentName := "rp-production-predeploy"
_, err := d.deployments.Get(ctx, d.resourceGroup, deploymentName)
if err == nil || !isDeploymentNotFoundError(err) {
return err
}
b, err := Asset(generator.FileRPProductionPredeploy)
if err != nil {
return err
}
var template map[string]interface{}
err = json.Unmarshal(b, &template)
if err != nil {
return err
}
parameters, err := getParameters()
if err != nil {
return err
}
parameters.Parameters["rpServicePrincipalId"] = &arm.ParametersParameter{
Value: rpServicePrincipalID,
}
d.log.Infof("predeploying to %s", d.resourceGroup)
return d.deployments.CreateOrUpdateAndWait(ctx, d.resourceGroup, deploymentName, azresources.Deployment{
Properties: &azresources.DeploymentProperties{
Template: template,
Mode: azresources.Incremental,
Parameters: parameters.Parameters,
},
})
}
func (d *deployer) Deploy(ctx context.Context, rpServicePrincipalID string) error {
b, err := Asset(generator.FileRPProduction)
if err != nil {
@ -220,7 +277,7 @@ func (d *deployer) removeOldScalesets(ctx context.Context) error {
}
d.log.Printf("waiting for %s instances to terminate", *vmss.Name)
err = wait.PollImmediateUntil(10*time.Second, func() (ready bool, err error) {
err = wait.PollImmediateUntil(10*time.Second, func() (bool, error) {
for _, vm := range scalesetVMs {
u := fmt.Sprintf("https://vm%s.%s.%s.cloudapp.azure.com/healthz/ready", *vm.InstanceID, *vmss.Name, d.location)

13
pkg/deploy/generator/const.go
Просмотреть файл

@ -14,11 +14,14 @@ const (
// Template file constants
const (
FileRPProductionNSG = "rp-production-nsg.json"
FileRPProduction = "rp-production.json"
FileRPProductionManagedIdentity = "rp-production-managed-identity.json"
FileRPProductionPredeploy = "rp-production-predeploy.json"
fileRPProductionPredeployParameters = "rp-production-predeploy-parameters.json"
FileRPProduction = "rp-production.json"
fileRPProductionParameters = "rp-production-parameters.json"
fileEnvDevelopment = "env-development.json"
fileRPDevelopmentNSG = "rp-development-nsg.json"
fileRPDevelopment = "rp-development.json"
fileDatabaseDevelopment = "databases-development.json"
fileRPProductionParameters = "rp-production-parameters.json"
fileRPDevelopmentPredeploy = "rp-development-predeploy.json"
fileRPDevelopment = "rp-development.json"
)

319
pkg/deploy/generator/generators.go
Просмотреть файл

@ -4,320 +4,87 @@ package generator
// Licensed under the Apache License 2.0.
import (
"bytes"
"encoding/json"
"io/ioutil"
mgmtkeyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2016-10-01/keyvault"
uuid "github.com/satori/go.uuid"
"github.com/Azure/ARO-RP/pkg/util/arm"
)
var apiVersions = map[string]string{
"authorization": "2018-09-01-preview",
"compute": "2019-03-01",
"dns": "2018-05-01",
"documentdb": "2019-08-01",
"keyvault": "2016-10-01",
"msi": "2018-11-30",
"network": "2019-07-01",
// Generator defines generator main interface
type Generator interface {
Artifacts() error
}
const (
tenantIDHack = "13805ec3-a223-47ad-ad65-8b2baf92c0fb"
)
var (
tenantUUIDHack = uuid.Must(uuid.FromString(tenantIDHack))
)
type generator struct {
production bool
}
func newGenerator(production bool) *generator {
func New(production bool) Generator {
return &generator{
production: production,
}
}
// GenerateRPTemplates generates RP templates for production and development
// outputs: database-development.json, rp-development.json, rp-production.json
func GenerateRPTemplates() error {
for _, i := range []struct {
templateFile string
g *generator
}{
{
templateFile: fileRPDevelopment,
g: newGenerator(false),
},
{
templateFile: FileRPProduction,
g: newGenerator(true),
},
} {
b, err := json.MarshalIndent(i.g.rpTemplate(), "", " ")
func (g *generator) Artifacts() error {
if g.production {
err := g.writeTemplate(g.managedIdentityTemplate(), FileRPProductionManagedIdentity)
if err != nil {
return err
}
err = g.writeTemplate(g.preDeployTemplate(), FileRPProductionPredeploy)
if err != nil {
return err
}
err = g.writeTemplate(g.rpTemplate(), FileRPProduction)
if err != nil {
return err
}
err = g.writeParameters(g.rpPreDeployParameters(), fileRPProductionPredeployParameters)
if err != nil {
return err
}
err = g.writeParameters(g.rpParameters(), fileRPProductionParameters)
if err != nil {
return err
}
// :-(
b = bytes.ReplaceAll(b, []byte(tenantIDHack), []byte("[subscription().tenantId]"))
if i.g.production {
b = bytes.Replace(b, []byte(`"accessPolicies": []`), []byte(`"accessPolicies": "[concat(variables('clustersKeyvaultAccessPolicies'), parameters('extraKeyvaultAccessPolicies'))]"`), 1)
b = bytes.Replace(b, []byte(`"accessPolicies": []`), []byte(`"accessPolicies": "[concat(variables('serviceKeyvaultAccessPolicies'), parameters('extraKeyvaultAccessPolicies'))]"`), 1)
} else {
err := g.writeTemplate(g.sharedDevelopmentEnvTemplate(), fileEnvDevelopment)
if err != nil {
return err
}
b = append(b, byte('\n'))
err = ioutil.WriteFile(i.templateFile, b, 0666)
err = g.writeTemplate(g.databaseTemplate(), fileDatabaseDevelopment)
if err != nil {
return err
}
err = g.writeTemplate(g.preDeployTemplate(), fileRPDevelopmentPredeploy)
if err != nil {
return err
}
err = g.writeTemplate(g.rpTemplate(), fileRPDevelopment)
if err != nil {
return err
}
}
t := &arm.Template{
Schema: "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
ContentVersion: "1.0.0.0",
Parameters: map[string]*arm.TemplateParameter{
"databaseAccountName": {
Type: "string",
},
"databaseName": {
Type: "string",
},
},
}
return nil
}
g := newGenerator(false)
t.Resources = append(t.Resources, g.database("parameters('databaseName')", false)...)
b, err := json.MarshalIndent(t, "", " ")
func (g *generator) writeTemplate(t *arm.Template, output string) error {
b, err := g.templateFixup(t)
if err != nil {
return err
}
b = append(b, byte('\n'))
return ioutil.WriteFile(fileDatabaseDevelopment, b, 0666)
return ioutil.WriteFile(output, b, 0666)
}
// GenerateRPParameterTemplate generates RP parameters file
// output: rp-production-parameters.json
func GenerateRPParameterTemplate() error {
t := newGenerator(true).rpTemplate()
p := &arm.Parameters{
Schema: "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
ContentVersion: "1.0.0.0",
Parameters: map[string]*arm.ParametersParameter{},
}
for name, tp := range t.Parameters {
param := &arm.ParametersParameter{Value: tp.DefaultValue}
if param.Value == nil {
param.Value = ""
}
p.Parameters[name] = param
}
func (g *generator) writeParameters(p *arm.Parameters, output string) error {
b, err := json.MarshalIndent(p, "", " ")
if err != nil {
return err
}
b = append(b, byte('\n'))
err = ioutil.WriteFile(fileRPProductionParameters, b, 0666)
if err != nil {
return err
}
return nil
}
// GenerateNSGTemplates generates Network security group file for development and production
// outputs: rp-development-ngs.json, rp-production-ngs.json
func GenerateNSGTemplates() error {
for _, i := range []struct {
templateFile string
g *generator
}{
{
templateFile: fileRPDevelopmentNSG,
g: newGenerator(false),
},
{
templateFile: FileRPProductionNSG,
g: newGenerator(true),
},
} {
t := &arm.Template{
Schema: "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
ContentVersion: "1.0.0.0",
Parameters: map[string]*arm.TemplateParameter{},
}
t.Resources = append(t.Resources, i.g.securityGroupRP(), i.g.securityGroupPE())
if i.g.production {
t.Resources = append(t.Resources, i.g.managedIdentity())
t.Outputs = map[string]*arm.Output{
"rpServicePrincipalId": {
Type: "string",
Value: "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'rp-identity'), '2018-11-30').principalId]",
},
}
}
b, err := json.MarshalIndent(t, "", " ")
if err != nil {
return err
}
b = append(b, byte('\n'))
err = ioutil.WriteFile(i.templateFile, b, 0666)
if err != nil {
return err
}
}
return nil
}
// GenerateDevelopmentTemplate shared RP template for development
// outputs: env-development.json
func GenerateDevelopmentTemplate() error {
g := newGenerator(false)
t := &arm.Template{
Schema: "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
ContentVersion: "1.0.0.0",
Parameters: map[string]*arm.TemplateParameter{},
}
t.Resources = append(t.Resources,
g.devVpnPip(),
g.devVnet(),
g.devVPN(),
g.proxyVmss())
for _, param := range []string{
"proxyCert",
"proxyClientCert",
"proxyDomainNameLabel",
"proxyImage",
"proxyImageAuth",
"proxyKey",
"publicIPAddressAllocationMethod",
"publicIPAddressSkuName",
"sshPublicKey",
"vpnCACertificate",
} {
typ := "string"
var defaultValue interface{}
switch param {
case "proxyImageAuth", "proxyKey":
typ = "securestring"
case "publicIPAddressAllocationMethod":
defaultValue = "Static"
case "publicIPAddressSkuName":
defaultValue = "Standard"
}
t.Parameters[param] = &arm.TemplateParameter{
Type: typ,
DefaultValue: defaultValue,
}
}
b, err := json.MarshalIndent(t, "", " ")
if err != nil {
return err
}
b = append(b, byte('\n'))
return ioutil.WriteFile(fileEnvDevelopment, b, 0666)
}
func (g *generator) rpTemplate() *arm.Template {
t := &arm.Template{
Schema: "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
ContentVersion: "1.0.0.0",
Parameters: map[string]*arm.TemplateParameter{},
}
if g.production {
t.Variables = map[string]interface{}{
"clustersKeyvaultAccessPolicies": g.clustersKeyvaultAccessPolicies(),
"serviceKeyvaultAccessPolicies": g.serviceKeyvaultAccessPolicies(),
}
}
params := []string{
"databaseAccountName",
"domainName",
"fpServicePrincipalId",
"keyvaultPrefix",
"rpServicePrincipalId",
}
if g.production {
params = append(params,
"extraCosmosDBIPs",
"extraKeyvaultAccessPolicies",
"mdmFrontendUrl",
"mdsdConfigVersion",
"mdsdEnvironment",
"pullSecret",
"rpImage",
"rpImageAuth",
"rpMode",
"sshPublicKey",
"vmssName",
"adminApiCaBundle",
"adminApiClientCertCommonName",
)
} else {
params = append(params,
"adminObjectId",
)
}
for _, param := range params {
p := &arm.TemplateParameter{Type: "string"}
switch param {
case "extraCosmosDBIPs", "rpMode":
p.DefaultValue = ""
case "extraKeyvaultAccessPolicies":
p.Type = "array"
p.DefaultValue = []mgmtkeyvault.AccessPolicyEntry{}
case "pullSecret", "rpImageAuth":
p.Type = "securestring"
case "keyvaultPrefix":
p.MaxLength = 24 - max(len(kvClusterSuffix), len(kvServiceSuffix))
}
t.Parameters[param] = p
}
if g.production {
t.Resources = append(t.Resources, g.pip(), g.lb(), g.vmss())
}
// clustersKeyvault must preceed serviceKeyvault due to terrible bytes.Replace below
t.Resources = append(t.Resources, g.zone(),
g.clustersKeyvault(), g.serviceKeyvault(),
g.rpvnet(), g.pevnet(),
g.halfPeering("rp-vnet", "rp-pe-vnet-001"),
g.halfPeering("rp-pe-vnet-001", "rp-vnet"))
if g.production {
t.Resources = append(t.Resources, g.cosmosdb("'ARO'")...)
} else {
t.Resources = append(t.Resources, g.cosmosdb("pparameters('databaseName')")...)
}
t.Resources = append(t.Resources, g.rbac()...)
return t
return ioutil.WriteFile(output, b, 0666)
}

23
pkg/deploy/generator/resources.go
Просмотреть файл

@ -16,10 +16,29 @@ import (
mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-07-01/network"
mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization"
"github.com/Azure/go-autorest/autorest/to"
uuid "github.com/satori/go.uuid"
"github.com/Azure/ARO-RP/pkg/util/arm"
)
var apiVersions = map[string]string{
"authorization": "2018-09-01-preview",
"compute": "2019-03-01",
"dns": "2018-05-01",
"documentdb": "2019-08-01",
"keyvault": "2016-10-01",
"msi": "2018-11-30",
"network": "2019-07-01",
}
const (
tenantIDHack = "13805ec3-a223-47ad-ad65-8b2baf92c0fb"
)
var (
tenantUUIDHack = uuid.Must(uuid.FromString(tenantIDHack))
)
func (g *generator) managedIdentity() *arm.Resource {
return &arm.Resource{
Resource: &mgmtmsi.Identity{
@ -1068,7 +1087,7 @@ func (g *generator) serviceKeyvault() *arm.Resource {
}
}
func (g *generator) cosmosdb(databaseName string) []*arm.Resource {
func (g *generator) cosmosdb() []*arm.Resource {
cosmosdb := &mgmtdocumentdb.DatabaseAccountCreateUpdateParameters{
Kind: mgmtdocumentdb.GlobalDocumentDB,
DatabaseAccountCreateUpdateProperties: &mgmtdocumentdb.DatabaseAccountCreateUpdateProperties{
@ -1113,7 +1132,7 @@ func (g *generator) cosmosdb(databaseName string) []*arm.Resource {
}
if g.production {
rs = append(rs, g.database(databaseName, true)...)
rs = append(rs, g.database("'ARO'", true)...)
}
return rs

256
pkg/deploy/generator/templates.go Normal file
Просмотреть файл

@ -0,0 +1,256 @@
package generator
// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.
import (
"bytes"
"encoding/json"
"github.com/Azure/ARO-RP/pkg/util/arm"
)
func (g *generator) managedIdentityTemplate() *arm.Template {
t := templateStanza()
t.Resources = append(t.Resources,
g.managedIdentity(),
)
t.Outputs = map[string]*arm.Output{
"rpServicePrincipalId": {
Type: "string",
Value: "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', 'rp-identity'), '2018-11-30').principalId]",
},
}
return t
}
func (g *generator) rpTemplate() *arm.Template {
t := templateStanza()
params := []string{
"databaseAccountName",
"domainName",
"fpServicePrincipalId",
"rpServicePrincipalId",
}
if g.production {
params = append(params,
"adminApiCaBundle",
"adminApiClientCertCommonName",
"extraCosmosDBIPs",
"mdmFrontendUrl",
"mdsdConfigVersion",
"mdsdEnvironment",
"pullSecret",
"rpImage",
"rpImageAuth",
"rpMode",
"sshPublicKey",
"vmssName",
)
}
for _, param := range params {
p := &arm.TemplateParameter{Type: "string"}
switch param {
case "extraCosmosDBIPs", "rpMode":
p.DefaultValue = ""
case "pullSecret", "rpImageAuth":
p.Type = "securestring"
}
t.Parameters[param] = p
}
if g.production {
t.Resources = append(t.Resources, g.pip(), g.lb(), g.vmss())
}
t.Resources = append(t.Resources, g.zone(),
g.rpvnet(), g.pevnet(),
g.halfPeering("rp-vnet", "rp-pe-vnet-001"),
g.halfPeering("rp-pe-vnet-001", "rp-vnet"))
t.Resources = append(t.Resources, g.cosmosdb()...)
t.Resources = append(t.Resources, g.rbac()...)
return t
}
func (g *generator) databaseTemplate() *arm.Template {
t := templateStanza()
t.Resources = append(t.Resources,
g.database("parameters('databaseName')", false)...)
t.Parameters = map[string]*arm.TemplateParameter{
"databaseAccountName": {
Type: "string",
},
"databaseName": {
Type: "string",
},
}
return t
}
func (g *generator) rpParameters() *arm.Parameters {
t := g.rpTemplate()
p := parametersStanza()
for name, tp := range t.Parameters {
param := &arm.ParametersParameter{Value: tp.DefaultValue}
if param.Value == nil {
param.Value = ""
}
p.Parameters[name] = param
}
return p
}
func (g *generator) preDeployTemplate() *arm.Template {
t := templateStanza()
if g.production {
t.Variables = map[string]interface{}{
"clustersKeyvaultAccessPolicies": g.clustersKeyvaultAccessPolicies(),
"serviceKeyvaultAccessPolicies": g.serviceKeyvaultAccessPolicies(),
}
}
params := []string{
"keyvaultPrefix",
"rpServicePrincipalId",
"fpServicePrincipalId",
}
if g.production {
params = append(params,
"extraKeyvaultAccessPolicies",
)
} else {
params = append(params,
"adminObjectId",
)
}
for _, param := range params {
p := &arm.TemplateParameter{Type: "string"}
switch param {
case "extraKeyvaultAccessPolicies":
p.Type = "array"
p.DefaultValue = []interface{}{}
case "keyvaultPrefix":
p.MaxLength = 24 - max(len(kvClusterSuffix), len(kvServiceSuffix))
}
t.Parameters[param] = p
}
t.Resources = append(t.Resources,
g.securityGroupRP(),
g.securityGroupPE(),
// clustersKeyvault must preceed serviceKeyvault due to terrible
// bytes.Replace in templateFixup
g.clustersKeyvault(),
g.serviceKeyvault(),
)
return t
}
func (g *generator) rpPreDeployParameters() *arm.Parameters {
t := g.preDeployTemplate()
p := parametersStanza()
for name, tp := range t.Parameters {
param := &arm.ParametersParameter{Value: tp.DefaultValue}
if param.Value == nil {
param.Value = ""
}
p.Parameters[name] = param
}
return p
}
func (g *generator) sharedDevelopmentEnvTemplate() *arm.Template {
t := templateStanza()
t.Resources = append(t.Resources,
g.devVpnPip(),
g.devVnet(),
g.devVPN(),
g.proxyVmss())
for _, param := range []string{
"proxyCert",
"proxyClientCert",
"proxyDomainNameLabel",
"proxyImage",
"proxyImageAuth",
"proxyKey",
"publicIPAddressAllocationMethod",
"publicIPAddressSkuName",
"sshPublicKey",
"vpnCACertificate",
} {
typ := "string"
var defaultValue interface{}
switch param {
case "proxyImageAuth", "proxyKey":
typ = "securestring"
case "publicIPAddressAllocationMethod":
defaultValue = "Static"
case "publicIPAddressSkuName":
defaultValue = "Standard"
}
t.Parameters[param] = &arm.TemplateParameter{
Type: typ,
DefaultValue: defaultValue,
}
}
return t
}
func max(a, b int) int {
if a > b {
return a
}
return b
}
func (g *generator) templateFixup(t *arm.Template) ([]byte, error) {
b, err := json.MarshalIndent(t, "", " ")
if err != nil {
return nil, err
}
// :-(
b = bytes.ReplaceAll(b, []byte(tenantIDHack), []byte("[subscription().tenantId]"))
if g.production {
b = bytes.Replace(b, []byte(`"accessPolicies": []`), []byte(`"accessPolicies": "[concat(variables('clustersKeyvaultAccessPolicies'), parameters('extraKeyvaultAccessPolicies'))]"`), 1)
b = bytes.Replace(b, []byte(`"accessPolicies": []`), []byte(`"accessPolicies": "[concat(variables('serviceKeyvaultAccessPolicies'), parameters('extraKeyvaultAccessPolicies'))]"`), 1)
}
return append(b, byte('\n')), nil
}
func templateStanza() *arm.Template {
return &arm.Template{
Schema: "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
ContentVersion: "1.0.0.0",
Parameters: map[string]*arm.TemplateParameter{},
}
}
func parametersStanza() *arm.Parameters {
return &arm.Parameters{
Schema: "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
ContentVersion: "1.0.0.0",
Parameters: map[string]*arm.ParametersParameter{},
}
}

11
pkg/deploy/generator/util.go
Просмотреть файл

@ -1,11 +0,0 @@
package generator
// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.
func max(a, b int) int {
if a > b {
return a
}
return b
}