quota checking with static mapping between VMSize and number of cores

pkg/api/error.go

@@ -66,6 +66,7 @@ var (
 	CloudErrorCodeInvalidOperationID                 = "InvalidOperationID"
 	CloudErrorCodeDuplicateClientID                  = "DuplicateClientID"
 	CloudErrorCodeDuplicateDomain                    = "DuplicateDomain"
+	CloudErrorCodeResourceQuotaExceeded              = "ResourceQuotaExceeded"
 )
 
 // NewCloudError returns a new CloudError

pkg/api/openshiftcluster.go

@@ -155,6 +155,7 @@ type MasterProfile struct {
 type VMSize string
 
 // VMSize constants
+// add required resources in pkg/api/validate/quota.go when adding a new VMSize
 const (
 	VMSizeStandardD2sV3 VMSize = "Standard_D2s_v3"
 	VMSizeStandardD4sV3 VMSize = "Standard_D4s_v3"

pkg/api/validate/openshiftcluster_validatedynamic.go

@@ -22,6 +22,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/env"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute"
 	utilpermissions "github.com/Azure/ARO-RP/pkg/util/permissions"
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )
@@ -73,6 +74,12 @@ func (dv *openShiftClusterDynamicValidator) Dynamic(ctx context.Context, oc *api
 		return err
 	}
 
+	spUsage := compute.NewUsageClient(r.SubscriptionID, spAuthorizer)
+	err = dv.validateQuotas(ctx, oc, spUsage)
+	if err != nil {
+		return err
+	}
+
 	fpAuthorizer, err := dv.env.FPAuthorizer(oc.Properties.ServicePrincipalProfile.TenantID, azure.PublicCloud.ResourceManagerEndpoint)
 	if err != nil {
 		return err

pkg/api/validate/quota.go

@@ -0,0 +1,61 @@
+package validate
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute"
+)
+
+func addRequiredResources(requiredResources map[string]int, vmSize api.VMSize, count int) error {
+	requiredResources["virtualMachines"] += count
+	requiredResources["PremiumDiskCount"] += count
+	switch vmSize {
+	case api.VMSizeStandardD2sV3:
+		requiredResources["standardDSv3Family"] += (count * 2)
+		requiredResources["cores"] += (count * 2)
+	case api.VMSizeStandardD4sV3:
+		requiredResources["standardDSv3Family"] += (count * 4)
+		requiredResources["cores"] += (count * 4)
+	case api.VMSizeStandardD8sV3:
+		requiredResources["standardDSv3Family"] += (count * 8)
+		requiredResources["cores"] += (count * 8)
+	default:
+		//will only happen if pkg/api verification allows new VMSizes
+		return fmt.Errorf("unexpected node VMSize %s", vmSize)
+	}
+	return nil
+}
+
+// validateQuotas checks usage quotas vs. resources required by cluster before cluster creation
+func (dv *openShiftClusterDynamicValidator) validateQuotas(ctx context.Context, oc *api.OpenShiftCluster, uc compute.UsageClient) error {
+	requiredResources := map[string]int{}
+	addRequiredResources(requiredResources, oc.Properties.MasterProfile.VMSize, 3)
+	//worker node resource calculation
+	for _, w := range oc.Properties.WorkerProfiles {
+		addRequiredResources(requiredResources, w.VMSize, w.Count)
+	}
+
+	usages, err := uc.List(ctx, oc.Location)
+	if err != nil {
+		return err
+	}
+	//check requirements vs. usage
+
+	// we're only checking the limits returned by the Usage API and ignoring usage limits missing from the results
+	// rationale:
+	// 1. if the Usage API doesn't send a limit because a resource is no longer limited, RP will continue cluster creation without impact
+	// 2. if the Usage API doesn't send a limit that is still enforced, cluster creation will fail on the backend and we will get an error in the RP logs
+	for _, usage := range usages {
+		required, present := requiredResources[*usage.Name.Value]
+		if present && int64(required) > (*usage.Limit-int64(*usage.CurrentValue)) {
+			return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeResourceQuotaExceeded, "", "Resource quota of %s exceeded. Maximum allowed: %d, Current in use: %d, Additional requested: %d.", *usage.Name.Value, *usage.Limit, *usage.CurrentValue, required)
+		}
+	}
+	return nil
+}

pkg/api/validate/quota_test.go

@@ -0,0 +1,165 @@
+package validate
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"testing"
+
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
+	"github.com/Azure/go-autorest/autorest/to"
+	"github.com/golang/mock/gomock"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+	mock_compute "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/compute"
+)
+
+func TestQuotaCheck(t *testing.T) {
+	ctx := context.Background()
+
+	oc := &api.OpenShiftCluster{
+		Location: "ocLocation",
+		Properties: api.Properties{
+			MasterProfile: api.MasterProfile{
+				VMSize: "Standard_D8s_v3",
+			},
+			WorkerProfiles: []api.WorkerProfile{
+				{
+					VMSize: "Standard_D8s_v3",
+					Count:  10,
+				},
+			},
+		},
+	}
+
+	dv := openShiftClusterDynamicValidator{}
+	type test struct {
+		name    string
+		mocks   func(*test, *mock_compute.MockUsageClient)
+		wantErr string
+	}
+	for _, tt := range []*test{
+		{
+			name: "allow when there's enough resources - limits set to exact requirements, offset by 100 of current value",
+			mocks: func(tt *test, uc *mock_compute.MockUsageClient) {
+				uc.EXPECT().
+					List(ctx, "ocLocation").
+					Return([]compute.Usage{
+						{
+							Name: &compute.UsageName{
+								Value: to.StringPtr("cores"),
+							},
+							CurrentValue: to.Int32Ptr(100),
+							Limit:        to.Int64Ptr(204),
+						},
+						{
+							Name: &compute.UsageName{
+								Value: to.StringPtr("virtualMachines"),
+							},
+							CurrentValue: to.Int32Ptr(100),
+							Limit:        to.Int64Ptr(113),
+						},
+						{
+							Name: &compute.UsageName{
+								Value: to.StringPtr("standardDSv3Family"),
+							},
+							CurrentValue: to.Int32Ptr(100),
+							Limit:        to.Int64Ptr(204),
+						},
+						{
+							Name: &compute.UsageName{
+								Value: to.StringPtr("PremiumDiskCount"),
+							},
+							CurrentValue: to.Int32Ptr(100),
+							Limit:        to.Int64Ptr(113),
+						},
+					}, nil)
+			},
+		},
+		{
+			name:    "not enough cores",
+			wantErr: "400: ResourceQuotaExceeded: : Resource quota of cores exceeded. Maximum allowed: 204, Current in use: 101, Additional requested: 104.",
+			mocks: func(tt *test, uc *mock_compute.MockUsageClient) {
+				uc.EXPECT().
+					List(ctx, "ocLocation").
+					Return([]compute.Usage{
+						{
+							Name: &compute.UsageName{
+								Value: to.StringPtr("cores"),
+							},
+							CurrentValue: to.Int32Ptr(101),
+							Limit:        to.Int64Ptr(204),
+						},
+					}, nil)
+			},
+		},
+		{
+			name:    "not enough virtualMachines",
+			wantErr: "400: ResourceQuotaExceeded: : Resource quota of virtualMachines exceeded. Maximum allowed: 113, Current in use: 101, Additional requested: 13.",
+			mocks: func(tt *test, uc *mock_compute.MockUsageClient) {
+				uc.EXPECT().
+					List(ctx, "ocLocation").
+					Return([]compute.Usage{
+						{
+							Name: &compute.UsageName{
+								Value: to.StringPtr("virtualMachines"),
+							},
+							CurrentValue: to.Int32Ptr(101),
+							Limit:        to.Int64Ptr(113),
+						},
+					}, nil)
+			},
+		},
+		{
+			name:    "not enough standardDSv3Family",
+			wantErr: "400: ResourceQuotaExceeded: : Resource quota of standardDSv3Family exceeded. Maximum allowed: 204, Current in use: 101, Additional requested: 104.",
+			mocks: func(tt *test, uc *mock_compute.MockUsageClient) {
+				uc.EXPECT().
+					List(ctx, "ocLocation").
+					Return([]compute.Usage{
+						{
+							Name: &compute.UsageName{
+								Value: to.StringPtr("standardDSv3Family"),
+							},
+							CurrentValue: to.Int32Ptr(101),
+							Limit:        to.Int64Ptr(204),
+						},
+					}, nil)
+			},
+		},
+		{
+			name:    "not enough premium disks",
+			wantErr: "400: ResourceQuotaExceeded: : Resource quota of PremiumDiskCount exceeded. Maximum allowed: 113, Current in use: 101, Additional requested: 13.",
+			mocks: func(tt *test, uc *mock_compute.MockUsageClient) {
+				uc.EXPECT().
+					List(ctx, "ocLocation").
+					Return([]compute.Usage{
+						{
+							Name: &compute.UsageName{
+								Value: to.StringPtr("PremiumDiskCount"),
+							},
+							CurrentValue: to.Int32Ptr(101),
+							Limit:        to.Int64Ptr(113),
+						},
+					}, nil)
+			},
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			usageClient := mock_compute.NewMockUsageClient(controller)
+			if tt.mocks != nil {
+				tt.mocks(tt, usageClient)
+			}
+
+			err := dv.validateQuotas(ctx, oc, usageClient)
+			if err != nil && err.Error() != tt.wantErr ||
+				err == nil && tt.wantErr != "" {
+				t.Error(err)
+			}
+		})
+	}
+}

pkg/util/azureclient/mgmt/compute/disks.go

@@ -3,7 +3,7 @@ package compute
 // Copyright (c) Microsoft Corporation.
 // Licensed under the Apache License 2.0.
 
-//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE DisksClient,ResourceSkusClient,VirtualMachinesClient
+//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE DisksClient,ResourceSkusClient,VirtualMachinesClient,UsageClient
 //go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go
 
 import (

pkg/util/azureclient/mgmt/compute/usage.go

@@ -0,0 +1,30 @@
+package compute
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
+	"github.com/Azure/go-autorest/autorest"
+)
+
+// UsageClient is a minimal interface for azure UsageClient
+type UsageClient interface {
+	UsageClientAddons
+}
+
+type usageClient struct {
+	compute.UsageClient
+}
+
+var _ UsageClient = &usageClient{}
+
+// NewUsageClient creates a new UsageClient
+func NewUsageClient(tenantID string, authorizer autorest.Authorizer) UsageClient {
+	client := compute.NewUsageClient(tenantID)
+	client.Authorizer = authorizer
+
+	return &usageClient{
+		UsageClient: client,
+	}
+}

pkg/util/azureclient/mgmt/compute/usage_addons.go

@@ -0,0 +1,33 @@
+package compute
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
+)
+
+// UsageClientAddons contains addons to UsageClient
+type UsageClientAddons interface {
+	List(ctx context.Context, location string) (result []compute.Usage, err error)
+}
+
+func (u *usageClient) List(ctx context.Context, location string) (result []compute.Usage, err error) {
+	page, err := u.UsageClient.List(ctx, location)
+	if err != nil {
+		return nil, err
+	}
+
+	for page.NotDone() {
+		result = append(result, page.Values()...)
+
+		err = page.Next()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return result, nil
+}

pkg/util/mocks/azureclient/mgmt/compute/compute.go

@@ -1,5 +1,5 @@
 // Code generated by MockGen. DO NOT EDIT.
-// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute (interfaces: DisksClient,ResourceSkusClient,VirtualMachinesClient)
+// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute (interfaces: DisksClient,ResourceSkusClient,VirtualMachinesClient,UsageClient)
 
 // Package mock_compute is a generated GoMock package.
 package mock_compute
@@ -137,3 +137,41 @@ func (mr *MockVirtualMachinesClientMockRecorder) DeleteAndWait(arg0, arg1, arg2
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockVirtualMachinesClient)(nil).DeleteAndWait), arg0, arg1, arg2)
 }
+
+// MockUsageClient is a mock of UsageClient interface
+type MockUsageClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockUsageClientMockRecorder
+}
+
+// MockUsageClientMockRecorder is the mock recorder for MockUsageClient
+type MockUsageClientMockRecorder struct {
+	mock *MockUsageClient
+}
+
+// NewMockUsageClient creates a new mock instance
+func NewMockUsageClient(ctrl *gomock.Controller) *MockUsageClient {
+	mock := &MockUsageClient{ctrl: ctrl}
+	mock.recorder = &MockUsageClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockUsageClient) EXPECT() *MockUsageClientMockRecorder {
+	return m.recorder
+}
+
+// List mocks base method
+func (m *MockUsageClient) List(arg0 context.Context, arg1 string) ([]compute.Usage, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "List", arg0, arg1)
+	ret0, _ := ret[0].([]compute.Usage)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// List indicates an expected call of List
+func (mr *MockUsageClientMockRecorder) List(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockUsageClient)(nil).List), arg0, arg1)
+}