package deploy // Copyright (c) Microsoft Corporation. // Licensed under the Apache License 2.0. import ( "context" "encoding/json" "reflect" "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore" mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features" "github.com/Azure/go-autorest/autorest/azure" "github.com/jongio/azidext/go/azidext" "github.com/sirupsen/logrus" "github.com/Azure/ARO-RP/pkg/deploy/vmsscleaner" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/util/arm" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/dns" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/msi" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage" "github.com/Azure/ARO-RP/pkg/util/keyvault" ) var _ Deployer = (*deployer)(nil) type Deployer interface { PreDeploy(context.Context, int) error DeployRP(context.Context) error DeployGateway(context.Context) error UpgradeRP(context.Context) error UpgradeGateway(context.Context) error SaveVersion(context.Context) error } type deployer struct { log *logrus.Entry env env.Core globaldeployments features.DeploymentsClient globalgroups features.ResourceGroupsClient globalrecordsets dns.RecordSetsClient globalaccounts storage.AccountsClient globaluserassignedidentities msi.UserAssignedIdentitiesClient deployments features.DeploymentsClient groups features.ResourceGroupsClient userassignedidentities msi.UserAssignedIdentitiesClient providers features.ProvidersClient publicipaddresses network.PublicIPAddressesClient resourceskus compute.ResourceSkusClient roleassignments authorization.RoleAssignmentsClient vmss compute.VirtualMachineScaleSetsClient vmssvms compute.VirtualMachineScaleSetVMsClient zones dns.ZonesClient clusterKeyvault keyvault.Manager portalKeyvault keyvault.Manager serviceKeyvault keyvault.Manager config *RPConfig version string vmssCleaner vmsscleaner.Interface } // KnownDeploymentErrorType represents a type of error we encounter during an // RP/gateway deployment that we know how to handle via automation. type KnownDeploymentErrorType string const ( KnownDeploymentErrorTypeRPLBNotFound KnownDeploymentErrorType = "RPLBNotFound" ) // New initiates new deploy utility object func New(ctx context.Context, log *logrus.Entry, _env env.Core, config *RPConfig, version string, tokenCredential azcore.TokenCredential) (Deployer, error) { err := config.validate() if err != nil { return nil, err } scopes := []string{_env.Environment().ResourceManagerScope} authorizer := azidext.NewTokenCredentialAdapter(tokenCredential, scopes) scopes = []string{_env.Environment().KeyVaultScope} kvAuthorizer := azidext.NewTokenCredentialAdapter(tokenCredential, scopes) vmssClient := compute.NewVirtualMachineScaleSetsClient(_env.Environment(), config.SubscriptionID, authorizer) return &deployer{ log: log, env: _env, globaldeployments: features.NewDeploymentsClient(_env.Environment(), *config.Configuration.GlobalSubscriptionID, authorizer), globalgroups: features.NewResourceGroupsClient(_env.Environment(), *config.Configuration.GlobalSubscriptionID, authorizer), globalrecordsets: dns.NewRecordSetsClient(_env.Environment(), *config.Configuration.GlobalSubscriptionID, authorizer), globalaccounts: storage.NewAccountsClient(_env.Environment(), *config.Configuration.GlobalSubscriptionID, authorizer), globaluserassignedidentities: msi.NewUserAssignedIdentitiesClient(_env.Environment(), *config.Configuration.GlobalSubscriptionID, authorizer), deployments: features.NewDeploymentsClient(_env.Environment(), config.SubscriptionID, authorizer), groups: features.NewResourceGroupsClient(_env.Environment(), config.SubscriptionID, authorizer), userassignedidentities: msi.NewUserAssignedIdentitiesClient(_env.Environment(), config.SubscriptionID, authorizer), providers: features.NewProvidersClient(_env.Environment(), config.SubscriptionID, authorizer), roleassignments: authorization.NewRoleAssignmentsClient(_env.Environment(), config.SubscriptionID, authorizer), resourceskus: compute.NewResourceSkusClient(_env.Environment(), config.SubscriptionID, authorizer), publicipaddresses: network.NewPublicIPAddressesClient(_env.Environment(), config.SubscriptionID, authorizer), vmss: vmssClient, vmssvms: compute.NewVirtualMachineScaleSetVMsClient(_env.Environment(), config.SubscriptionID, authorizer), zones: dns.NewZonesClient(_env.Environment(), config.SubscriptionID, authorizer), clusterKeyvault: keyvault.NewManager(kvAuthorizer, "https://"+*config.Configuration.KeyvaultPrefix+env.ClusterKeyvaultSuffix+"."+_env.Environment().KeyVaultDNSSuffix+"/"), portalKeyvault: keyvault.NewManager(kvAuthorizer, "https://"+*config.Configuration.KeyvaultPrefix+env.PortalKeyvaultSuffix+"."+_env.Environment().KeyVaultDNSSuffix+"/"), serviceKeyvault: keyvault.NewManager(kvAuthorizer, "https://"+*config.Configuration.KeyvaultPrefix+env.ServiceKeyvaultSuffix+"."+_env.Environment().KeyVaultDNSSuffix+"/"), config: config, version: version, vmssCleaner: vmsscleaner.New(log, vmssClient), }, nil } // getParameters returns an *arm.Parameters populated with parameter names and // values. The names are taken from the ps argument and the values are taken // from d.config.Configuration. func (d *deployer) getParameters(ps map[string]interface{}) *arm.Parameters { m := map[string]interface{}{} v := reflect.ValueOf(*d.config.Configuration) for i := 0; i < v.NumField(); i++ { if v.Field(i).IsNil() { continue } m[strings.SplitN(v.Type().Field(i).Tag.Get("json"), ",", 2)[0]] = v.Field(i).Interface() } parameters := &arm.Parameters{ Parameters: map[string]*arm.ParametersParameter{}, } for p := range ps { // do not convert empty fields // makes default values templates work v, ok := m[p] if !ok { continue } switch p { case "gatewayDomains", "gatewayFeatures", "portalAccessGroupIds", "portalElevatedGroupIds", "rpFeatures": v = strings.Join(v.([]string), ",") } parameters.Parameters[p] = &arm.ParametersParameter{ Value: v, } } return parameters } func (d *deployer) deploy(ctx context.Context, rgName, deploymentName, vmssName string, deployment mgmtfeatures.Deployment) (err error) { numAttempts := 3 for i := 0; i < numAttempts; i++ { d.log.Printf("deploying %s", deploymentName) err = d.deployments.CreateOrUpdateAndWait(ctx, rgName, deploymentName, deployment) serviceErr, isServiceError := err.(*azure.ServiceError) // As long as this is not the final deployment attempt, // unconditionally log the error before inspecting it. if err != nil && i < numAttempts-1 { d.log.Print(err) } // Check for a known error that we know how to handle. if isServiceError { errorType, checkTypeErr := d.checkForKnownError(serviceErr, i) if checkTypeErr != nil { d.log.Printf("Encountered an error in checkForKnownError: %s", checkTypeErr) } // On new RP deployments, we get a spurious DeploymentFailed error // from the Microsoft.Insights/metricAlerts resources indicating // that rp-lb can't be found, even though it exists and the // resources correctly have a dependsOn stanza referring to it. // Retry once, and only if this error is encountered on the first // deployment attempt. if errorType == KnownDeploymentErrorTypeRPLBNotFound { d.log.Print("Deployment encountered known ResourceNotFound error for RP LB; retrying.") continue } } // For errors we don't know how to handle, delete the failed VMSS and retry the deployment. if err != nil && *d.config.Configuration.VMSSCleanupEnabled { if retry := d.vmssCleaner.RemoveFailedNewScaleset(ctx, rgName, vmssName); retry { continue } } break } return err } // checkForKnownError is a helper function that checks the errors nested within an Azure ServiceError // for a known error and returns the corresponding KnownDeploymentErrorType if applicable. func (d *deployer) checkForKnownError(serviceErr *azure.ServiceError, deployAttempt int) (KnownDeploymentErrorType, error) { if serviceErr.Code != "DeploymentFailed" || len(serviceErr.Details) == 0 { return "", nil } outerErr := azure.ServiceError{} jsonEncoded, err := json.Marshal(serviceErr.Details[0]) if err != nil { return "", err } err = json.Unmarshal(jsonEncoded, &outerErr) if err != nil { return "", err } innerErr := azure.ServiceError{} err = json.Unmarshal([]byte(outerErr.Message), &innerErr) if err != nil { return "", err } isFirstAttempt := deployAttempt < 1 isRPLBNotFound := innerErr.Code == "ResourceNotFound" && strings.Contains(innerErr.Message, "Microsoft.Network/loadBalancers/rp-lb") if isFirstAttempt && isRPLBNotFound { return KnownDeploymentErrorTypeRPLBNotFound, nil } return "", nil }