Azure
/
aks-engine
зеркало из https://github.com/Azure/aks-engine.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

feat: use all VMAS fault domains in a region (#1090)

This commit is contained in:
Matt Boersma 2019-06-05 17:40:56 -06:00 коммит произвёл Jack Francis
Родитель 2318d1a557
Коммит 10a312534d
21 изменённых файлов: 590 добавлений и 9 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

13
cmd/scale.go
Просмотреть файл

@ -223,6 +223,8 @@ func (sc *scaleCmd) run(cmd *cobra.Command, args []string) error {
indexes := make([]int, 0)
indexToVM := make(map[int]string)
if sc.agentPool.IsAvailabilitySets() {
availabilitySetIDs := []string{}
for vmsListPage, err := sc.client.ListVirtualMachines(ctx, sc.resourceGroupName); vmsListPage.NotDone(); err = vmsListPage.Next() {
if err != nil {
return errors.Wrap(err, "failed to get vms in the resource group")
@ -235,6 +237,10 @@ func (sc *scaleCmd) run(cmd *cobra.Command, args []string) error {
continue
}
if vm.AvailabilitySet != nil {
availabilitySetIDs = append(availabilitySetIDs, *vm.AvailabilitySet.ID)
}
osPublisher := vm.StorageProfile.ImageReference.Publisher
if osPublisher != nil && strings.EqualFold(*osPublisher, "MicrosoftWindowsServer") {
_, _, winPoolIndex, index, err = utils.WindowsVMNameParts(vmName)
@ -260,6 +266,13 @@ func (sc *scaleCmd) run(cmd *cobra.Command, args []string) error {
}
highestUsedIndex = indexes[len(indexes)-1]
// set the VMAS platformFaultDomainCount to match the existing value
fdCount, err := sc.client.GetAvailabilitySetFaultDomainCount(ctx, sc.resourceGroupName, availabilitySetIDs)
if err != nil {
return err
}
sc.containerService.SetPlatformFaultDomainCount(fdCount)
// Scale down Scenario
if currentNodeCount > sc.newDesiredAgentCount {
if sc.masterFQDN == "" {

11
pkg/api/types.go
Просмотреть файл

@ -469,6 +469,7 @@ type MasterProfile struct {
ImageRef *ImageReference `json:"imageReference,omitempty"`
CustomFiles *[]CustomFile `json:"customFiles,omitempty"`
AvailabilityProfile string `json:"availabilityProfile"`
PlatformFaultDomainCount *int `json:"platformFaultDomainCount"`
AgentSubnet string `json:"agentSubnet,omitempty"`
AvailabilityZones []string `json:"availabilityZones,omitempty"`
SinglePlacementGroup *bool `json:"singlePlacementGroup,omitempty"`
@ -518,6 +519,7 @@ type AgentPoolProfile struct {
Ports []int `json:"ports,omitempty"`
ProvisioningState ProvisioningState `json:"provisioningState,omitempty"`
AvailabilityProfile string `json:"availabilityProfile"`
PlatformFaultDomainCount *int `json:"platformFaultDomainCount"`
ScaleSetPriority string `json:"scaleSetPriority,omitempty"`
ScaleSetEvictionPolicy string `json:"scaleSetEvictionPolicy,omitempty"`
StorageProfile string `json:"storageProfile,omitempty"`
@ -1842,6 +1844,15 @@ func (cs *ContainerService) GetAzureProdFQDN() string {
return FormatProdFQDNByLocation(cs.Properties.MasterProfile.DNSPrefix, cs.Location, cs.Properties.GetCustomCloudName())
}
// SetPlatformFaultDomainCount sets the fault domain count value for all VMASes in a cluster.
func (cs *ContainerService) SetPlatformFaultDomainCount(count int) {
// Assume that all VMASes in the cluster share a value for platformFaultDomainCount
cs.Properties.MasterProfile.PlatformFaultDomainCount = &count
for _, pool := range cs.Properties.AgentPoolProfiles {
pool.PlatformFaultDomainCount = &count
}
}
// FormatAzureProdFQDNByLocation constructs an Azure prod fqdn
func FormatAzureProdFQDNByLocation(fqdnPrefix string, location string) string {
targetEnv := helpers.GetCloudTargetEnv(location)

26
pkg/api/types_test.go
Просмотреть файл

@ -4956,3 +4956,29 @@ func TestKubernetesConfigIsAddonEnabled(t *testing.T) {
}
}
}
func TestSetPlatformFaultDomainCount(t *testing.T) {
// check that the default value is nil
cs := CreateMockContainerService("testcluster", defaultTestClusterVer, 1, 3, false)
if cs.Properties.MasterProfile.PlatformFaultDomainCount != nil {
t.Errorf("expected master platformFaultDomainCount to be nil, not %v", cs.Properties.MasterProfile.PlatformFaultDomainCount)
}
for _, pool := range cs.Properties.AgentPoolProfiles {
if pool.PlatformFaultDomainCount != nil {
t.Errorf("expected agent platformFaultDomainCount to be nil, not %v", pool.PlatformFaultDomainCount)
}
}
// check that pfdc can be set to legal values
for i := 1; i <= 3; i++ {
cs.SetPlatformFaultDomainCount(i)
if *cs.Properties.MasterProfile.PlatformFaultDomainCount != i {
t.Errorf("expected master platformFaultDomainCount to be %d, not %v", i, cs.Properties.MasterProfile.PlatformFaultDomainCount)
}
for _, pool := range cs.Properties.AgentPoolProfiles {
if *pool.PlatformFaultDomainCount != i {
t.Errorf("expected agent platformFaultDomainCount to be %d, not %v", i, pool.PlatformFaultDomainCount)
}
}
}
}

4
pkg/armhelpers/azureclient.go
Просмотреть файл

@ -67,6 +67,7 @@ type AzureClient struct {
virtualMachineScaleSetVMsClient compute.VirtualMachineScaleSetVMsClient
virtualMachineExtensionsClient compute.VirtualMachineExtensionsClient
disksClient compute.DisksClient
availabilitySetsClient compute.AvailabilitySetsClient
applicationsClient graphrbac.ApplicationsClient
servicePrincipalsClient graphrbac.ServicePrincipalsClient
@ -348,6 +349,7 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armAuthor
virtualMachineScaleSetVMsClient: compute.NewVirtualMachineScaleSetVMsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
virtualMachineExtensionsClient: compute.NewVirtualMachineExtensionsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
disksClient: compute.NewDisksClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
availabilitySetsClient: compute.NewAvailabilitySetsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
applicationsClient: graphrbac.NewApplicationsClientWithBaseURI(env.GraphEndpoint, tenantID),
servicePrincipalsClient: graphrbac.NewServicePrincipalsClientWithBaseURI(env.GraphEndpoint, tenantID),
@ -366,6 +368,7 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armAuthor
c.virtualMachineScaleSetsClient.Authorizer = armAuthorizer
c.virtualMachineScaleSetVMsClient.Authorizer = armAuthorizer
c.disksClient.Authorizer = armAuthorizer
c.availabilitySetsClient.Authorizer = armAuthorizer
c.deploymentsClient.PollingDelay = time.Second * 5
c.resourcesClient.PollingDelay = time.Second * 5
@ -384,6 +387,7 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armAuthor
c.virtualMachineScaleSetsClient.PollingDuration = DefaultARMOperationTimeout
c.virtualMachineScaleSetVMsClient.PollingDuration = DefaultARMOperationTimeout
c.virtualMachinesClient.PollingDuration = DefaultARMOperationTimeout
c.availabilitySetsClient.PollingDuration = DefaultARMOperationTimeout
c.applicationsClient.Authorizer = graphAuthorizer
c.servicePrincipalsClient.Authorizer = graphAuthorizer

4
pkg/armhelpers/azurestack/azureclient.go
Просмотреть файл

@ -63,6 +63,7 @@ type AzureClient struct {
virtualMachineScaleSetVMsClient compute.VirtualMachineScaleSetVMsClient
virtualMachineExtensionsClient compute.VirtualMachineExtensionsClient
disksClient compute.DisksClient
availabilitySetsClient compute.AvailabilitySetsClient
applicationsClient graphrbac.ApplicationsClient
servicePrincipalsClient graphrbac.ServicePrincipalsClient
@ -213,6 +214,7 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armAuthor
virtualMachineScaleSetVMsClient: compute.NewVirtualMachineScaleSetVMsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
virtualMachineExtensionsClient: compute.NewVirtualMachineExtensionsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
disksClient: compute.NewDisksClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
availabilitySetsClient: compute.NewAvailabilitySetsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID),
applicationsClient: graphrbac.NewApplicationsClientWithBaseURI(env.GraphEndpoint, tenantID),
servicePrincipalsClient: graphrbac.NewServicePrincipalsClientWithBaseURI(env.GraphEndpoint, tenantID),
@ -231,6 +233,7 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armAuthor
c.virtualMachineScaleSetsClient.Authorizer = armAuthorizer
c.virtualMachineScaleSetVMsClient.Authorizer = armAuthorizer
c.disksClient.Authorizer = armAuthorizer
c.availabilitySetsClient.Authorizer = armAuthorizer
c.deploymentsClient.PollingDelay = time.Second * 5
c.resourcesClient.PollingDelay = time.Second * 5
@ -249,6 +252,7 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armAuthor
c.virtualMachineScaleSetsClient.PollingDuration = DefaultARMOperationTimeout
c.virtualMachineScaleSetVMsClient.PollingDuration = DefaultARMOperationTimeout
c.virtualMachinesClient.PollingDuration = DefaultARMOperationTimeout
c.availabilitySetsClient.PollingDuration = DefaultARMOperationTimeout
c.applicationsClient.Authorizer = graphAuthorizer
c.servicePrincipalsClient.Authorizer = graphAuthorizer

35
pkg/armhelpers/azurestack/compute.go
Просмотреть файл

@ -6,10 +6,12 @@ package azurestack
import (
"context"
"fmt"
"strings"
"github.com/Azure/aks-engine/pkg/armhelpers"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-03-30/compute"
azcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
log "github.com/sirupsen/logrus"
)
// ListVirtualMachines returns (the first page of) the machines in the specified resource group.
@ -160,3 +162,36 @@ func (az *AzureClient) SetVirtualMachineScaleSetCapacity(ctx context.Context, re
_, err = future.Result(az.virtualMachineScaleSetsClient)
return err
}
// GetAvailabilitySet retrieves the specified VM availability set.
func (az *AzureClient) GetAvailabilitySet(ctx context.Context, resourceGroup, availabilitySetName string) (azcompute.AvailabilitySet, error) {
azVMAS := azcompute.AvailabilitySet{}
vmas, err := az.availabilitySetsClient.Get(ctx, resourceGroup, availabilitySetName)
if err != nil {
log.Printf("fail to get availability set, %v", err)
return azVMAS, err
}
if err = DeepCopy(&azVMAS, vmas); err != nil {
log.Printf("fail to convert availability set, %v", err)
return azVMAS, err
}
return azVMAS, nil
}
// GetAvailabilitySetFaultDomainCount returns the first existing fault domain count it finds from the IDs provided.
func (az *AzureClient) GetAvailabilitySetFaultDomainCount(ctx context.Context, resourceGroup string, vmasIDs []string) (int, error) {
var count int
for _, id := range vmasIDs {
// extract the last element of the id for VMAS name
ss := strings.Split(id, "/")
name := ss[len(ss)-1]
vmas, err := az.GetAvailabilitySet(ctx, resourceGroup, name)
if err != nil {
return 0, err
}
// Assume that all VMASes in the cluster share a value for platformFaultDomainCount
count = int(*vmas.AvailabilitySetProperties.PlatformFaultDomainCount)
break
}
return count, nil
}

63
pkg/armhelpers/azurestack/compute_test.go
Просмотреть файл

@ -185,3 +185,66 @@ func TestDeleteVirtualMachine(t *testing.T) {
t.Error(err)
}
}
func TestGetAvailabilitySet(t *testing.T) {
mc, err := NewHTTPMockClient()
if err != nil {
t.Fatalf("failed to create HttpMockClient - %s", err)
}
mc.Activate()
defer mc.DeactivateAndReset()
mc.RegisterLogin()
env := mc.GetEnvironment()
azureClient, err := NewAzureClientWithClientSecret(env, subscriptionID, "clientID", "secret")
if err != nil {
t.Fatalf("can not get client %s", err)
}
mc.RegisterGetAvailabilitySet()
vmas, err := azureClient.GetAvailabilitySet(context.Background(), resourceGroup, virtualMachineAvailabilitySetName)
if err != nil {
t.Fatalf("can't get availability set: %s", err)
}
var expected int32 = 3
if *vmas.PlatformFaultDomainCount != expected {
t.Fatalf("expected PlatformFaultDomainCount of %d but got %v", expected, *vmas.PlatformFaultDomainCount)
}
if *vmas.PlatformUpdateDomainCount != expected {
t.Fatalf("expected PlatformUpdateDomainCount of %d but got %v", expected, *vmas.PlatformUpdateDomainCount)
}
l := "eastus"
if *vmas.Location != l {
t.Fatalf("expected Location of %s but got %v", l, *vmas.Location)
}
}
func TestGetAvailabilitySetFaultDomainCount(t *testing.T) {
mc, err := NewHTTPMockClient()
if err != nil {
t.Fatalf("failed to create HttpMockClient - %s", err)
}
mc.Activate()
defer mc.DeactivateAndReset()
mc.RegisterLogin()
env := mc.GetEnvironment()
azureClient, err := NewAzureClientWithClientSecret(env, subscriptionID, "clientID", "secret")
if err != nil {
t.Fatalf("can not get client %s", err)
}
mc.RegisterGetAvailabilitySetFaultDomainCount()
count, err := azureClient.GetAvailabilitySetFaultDomainCount(context.Background(), resourceGroup, []string{"id1", "id2"})
if err != nil {
t.Fatalf("can't get availability set platform fault domain count: %s", err)
}
expected := 3
if count != expected {
t.Fatalf("platform fault domain count: expected %d but got %d", expected, count)
}
}

27
pkg/armhelpers/azurestack/httpMockClient.go
Просмотреть файл

@ -23,6 +23,7 @@ const (
deploymentName = "testDeplomentName"
deploymentStatus = "08586474508192185203"
virtualMachineScaleSetName = "vmscalesetName"
virtualMachineAvailabilitySetName = "vmavailabilitysetName"
virtualMachineName = "testVirtualMachineName"
virtualNicName = "testVirtualNicName"
virutalDiskName = "testVirtualdickName"
@ -35,6 +36,7 @@ const (
filePathGetVirtualMachine = "httpMockClientData/getVirtualMachine.json"
fileDeployVirtualMachine = "httpMockClientData/deployVMResponse.json"
fileDeployVirtualMachineError = "httpMockClientData/deploymentVMError.json"
filePathGetAvailabilitySet = "httpMockClientData/getAvailabilitySet.json"
)
//HTTPMockClient is an wrapper of httpmock
@ -60,6 +62,7 @@ type HTTPMockClient struct {
ResponseGetVirtualMachine string
ResponseDeployVirtualMachine string
ResponseDeployVirtualMachineError string
ResponseGetAvailabilitySet string
}
//VirtualMachineScaleSetListValues is an wrapper of virtual machine scale set list response values
@ -125,6 +128,10 @@ func NewHTTPMockClient() (HTTPMockClient, error) {
if err != nil {
return client, err
}
client.ResponseGetAvailabilitySet, err = readFromFile(filePathGetAvailabilitySet)
if err != nil {
return client, err
}
return client, nil
}
@ -194,6 +201,26 @@ func (mc HTTPMockClient) RegisterListVirtualMachines() {
)
}
// RegisterGetAvailabilitySet registers the mock response for GetAvailabilitySet.
func (mc HTTPMockClient) RegisterGetAvailabilitySet() {
httpmock.RegisterResponder("GET", fmt.Sprintf("https://management.azure.com/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/availabilitySets/vmavailabilitysetName?api-version=%s", mc.SubscriptionID, mc.ResourceGroup, mc.ComputeAPIVersion),
func(req *http.Request) (*http.Response, error) {
resp := httpmock.NewStringResponse(200, mc.ResponseGetAvailabilitySet)
return resp, nil
},
)
}
// RegisterGetAvailabilitySetFaultDomainCount registers a mock response for GetAvailabilitySet.
func (mc HTTPMockClient) RegisterGetAvailabilitySetFaultDomainCount() {
httpmock.RegisterResponder("GET", fmt.Sprintf("https://management.azure.com/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/availabilitySets/id1?api-version=%s", mc.SubscriptionID, mc.ResourceGroup, mc.ComputeAPIVersion),
func(req *http.Request) (*http.Response, error) {
resp := httpmock.NewStringResponse(200, mc.ResponseGetAvailabilitySet)
return resp, nil
},
)
}
// RegisterGetVirtualMachine registers the mock response for GetVirtualMachine
func (mc HTTPMockClient) RegisterGetVirtualMachine() {
httpmock.RegisterResponder("GET", fmt.Sprintf("https://management.azure.com/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s?api-version=%s", mc.SubscriptionID, mc.ResourceGroup, mc.VirtualMachineName, mc.ComputeAPIVersion),

18
pkg/armhelpers/azurestack/httpMockClientData/getAvailabilitySet.json Normal file
Просмотреть файл

@ -0,0 +1,18 @@
{
"properties": {
"platformUpdateDomainCount": 3,
"platformFaultDomainCount": 3,
"virtualMachines": [
{
"id": "/subscriptions/12345678-1234-4fe0-86da-28abc43fc4c7/resourceGroups/MYGROUP/providers/Microsoft.Compute/virtualMachines/K8S-MASTER-26399701-0"
}
]
},
"type": "Microsoft.Compute/availabilitySets",
"location": "eastus",
"id": "/subscriptions/12345678-1234-4fe0-86da-28abc43fc4c7/resourceGroups/mygroup/providers/Microsoft.Compute/availabilitySets/master-availabilityset-26399701",
"name": "master-availabilityset-26399701",
"sku": {
"name": "Aligned"
}
}

24
pkg/armhelpers/compute.go
Просмотреть файл

@ -5,6 +5,7 @@ package armhelpers
import (
"context"
"strings"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
)
@ -126,3 +127,26 @@ func (az *AzureClient) SetVirtualMachineScaleSetCapacity(ctx context.Context, re
_, err = future.Result(az.virtualMachineScaleSetsClient)
return err
}
// GetAvailabilitySet retrieves the specified VM availability set.
func (az *AzureClient) GetAvailabilitySet(ctx context.Context, resourceGroup, availabilitySetName string) (compute.AvailabilitySet, error) {
return az.availabilitySetsClient.Get(ctx, resourceGroup, availabilitySetName)
}
// GetAvailabilitySetFaultDomainCount returns the first existing fault domain count it finds from the IDs provided.
func (az *AzureClient) GetAvailabilitySetFaultDomainCount(ctx context.Context, resourceGroup string, vmasIDs []string) (int, error) {
var count int
for _, id := range vmasIDs {
// extract the last element of the id for VMAS name
ss := strings.Split(id, "/")
name := ss[len(ss)-1]
vmas, err := az.GetAvailabilitySet(ctx, resourceGroup, name)
if err != nil {
return 0, err
}
// Assume that all VMASes in the cluster share a value for platformFaultDomainCount
count = int(*vmas.AvailabilitySetProperties.PlatformFaultDomainCount)
break
}
return count, nil
}

7
pkg/armhelpers/interfaces.go
Просмотреть файл

@ -128,6 +128,13 @@ type AKSEngineClient interface {
// SetVirtualMachineScaleSetCapacity sets the VMSS capacity
SetVirtualMachineScaleSetCapacity(ctx context.Context, resourceGroup, virtualMachineScaleSet string, sku compute.Sku, location string) error
// GetAvailabilitySet retrieves the specified VM availability set.
GetAvailabilitySet(ctx context.Context, resourceGroup, availabilitySet string) (compute.AvailabilitySet, error)
// GetAvailabilitySetFaultDomainCount returns the first platform fault domain count it finds from the
// VM availability set IDs provided.
GetAvailabilitySetFaultDomainCount(ctx context.Context, resourceGroup string, vmasIDs []string) (int, error)
//
// STORAGE

16
pkg/armhelpers/mockclients.go
Просмотреть файл

@ -558,7 +558,11 @@ func (mc *MockAKSEngineClient) ListVirtualMachines(ctx context.Context, resource
if mc.FakeListVirtualMachineResult == nil {
mc.FakeListVirtualMachineResult = func() []compute.VirtualMachine {
return []compute.VirtualMachine{mc.MakeFakeVirtualMachine(DefaultFakeVMName, defaultK8sVersionForFakeVMs)}
machine := mc.MakeFakeVirtualMachine(DefaultFakeVMName, defaultK8sVersionForFakeVMs)
machine.AvailabilitySet = &compute.SubResource{
ID: to.StringPtr("MockAvailabilitySet"),
}
return []compute.VirtualMachine{machine}
}
}
vms := mc.FakeListVirtualMachineResult()
@ -783,6 +787,16 @@ func (mc *MockAKSEngineClient) ListVirtualMachineScaleSetVMs(ctx context.Context
}, nil
}
// GetAvailabilitySet mock
func (mc *MockAKSEngineClient) GetAvailabilitySet(ctx context.Context, resourceGroup, availabilitySetName string) (compute.AvailabilitySet, error) {
return compute.AvailabilitySet{}, nil
}
// GetAvailabilitySetFaultDomainCount mock
func (mc *MockAKSEngineClient) GetAvailabilitySetFaultDomainCount(ctx context.Context, resourceGroup string, vmasIDs []string) (int, error) {
return 3, nil
}
//GetStorageClient mock
func (mc *MockAKSEngineClient) GetStorageClient(ctx context.Context, resourceGroup, accountName string) (AKSStorageClient, error) {
if mc.FailGetStorageClient {

2
pkg/engine/armresources_test.go
Просмотреть файл

@ -149,7 +149,6 @@ func TestGenerateARMResourcesWithVMSSAgentPool(t *testing.T) {
Name: to.StringPtr("Aligned"),
},
AvailabilitySetProperties: &compute.AvailabilitySetProperties{
PlatformFaultDomainCount: to.Int32Ptr(2),
PlatformUpdateDomainCount: to.Int32Ptr(3),
},
},
@ -991,7 +990,6 @@ func TestGenerateARMResourceWithVMASAgents(t *testing.T) {
AvailabilitySet: compute.AvailabilitySet{
AvailabilitySetProperties: &compute.AvailabilitySetProperties{
PlatformUpdateDomainCount: to.Int32Ptr(3),
PlatformFaultDomainCount: to.Int32Ptr(2),
},
Sku: &compute.Sku{
Name: to.StringPtr("Aligned"),

42
pkg/engine/armtype.go
Просмотреть файл

@ -5,6 +5,8 @@ package engine
import (
"encoding/json"
"regexp"
"strings"
"github.com/Azure/azure-sdk-for-go/services/cosmos-db/mgmt/2015-04-08/documentdb"
"github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2018-02-14/keyvault"
@ -65,6 +67,46 @@ type AvailabilitySetARM struct {
compute.AvailabilitySet
}
// MarshalJSON is the custom marshaler for an AvailabilitySetARM.
// It acts as a decorator by replacing the JSON field "platformFaultDomainCount"
// with an ARM expression if the value was not set.
func (a AvailabilitySetARM) MarshalJSON() ([]byte, error) {
// alias the type to avoid infinite recursion in marshaling
type Alias AvailabilitySetARM
bytes, err := json.Marshal((Alias)(a))
if err != nil {
return nil, err
}
if a.AvailabilitySet.PlatformFaultDomainCount != nil {
return bytes, nil
}
// armExpr is evaluated by Azure Resource Manager at deployment time:
// if location is in the three-fault-domain list, return 3
// else if location is "canary" (testing), return 1
// else return 2
// NOTE: use fault_domains_expr.py to update this ARM expression.
armExpr := `"[
if( contains(
split('canadacentral,centralus,eastus,eastus2,northcentralus,northeurope,southcentralus,westeurope,westus', ','),
variables('location') ),
3,
if( equals('centraluseuap', variables('location') ),
1,
2
))]"`
// strip all whitespace
armExpr = strings.Join(strings.Fields(armExpr), "")
// insert ARM expression for platformFaultDomainCount as the first JSON property
// NOTE: this relies on this field being omitted in JSON when its value is nil.
re := regexp.MustCompile(`"properties" *: *{ *"`)
s := re.ReplaceAllLiteralString(string(bytes), `"properties":{"platformFaultDomainCount":`+armExpr+`,"`)
return []byte(s), nil
}
// StorageAccountARM embeds the ARMResource type in storage.Account.
type StorageAccountARM struct {
ARMResource

116
pkg/engine/armtype_test.go
Просмотреть файл

@ -9,6 +9,9 @@ import (
"testing"
"github.com/Azure/aks-engine/pkg/api"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
"github.com/Azure/go-autorest/autorest/to"
. "github.com/onsi/gomega"
)
func TestMarshalJSON(t *testing.T) {
@ -38,6 +41,117 @@ func TestMarshalJSON(t *testing.T) {
}
armObject := CreateCustomScriptExtension(cs)
jsonObj, _ := json.MarshalIndent(armObject, "", " ")
jsonObj, err := json.MarshalIndent(armObject, "", " ")
if err != nil {
t.Error(err)
}
// TODO: why print this? Let's validate it.
fmt.Println(string(jsonObj))
}
func TestMarshalJSONAvailabilitySetARM(t *testing.T) {
g := NewGomegaWithT(t)
type VMASTestDatum struct {
avSet AvailabilitySetARM
json string
}
vmasTestData := []VMASTestDatum{
{
avSet: AvailabilitySetARM{
ARMResource: ARMResource{
APIVersion: "[variables('apiVersionCompute')]",
},
AvailabilitySet: compute.AvailabilitySet{
Name: to.StringPtr("[variables('masterAvailabilitySet')]"),
Location: to.StringPtr("[variables('location')]"),
Type: to.StringPtr("Microsoft.Compute/availabilitySets"),
Sku: &compute.Sku{
Name: to.StringPtr("Aligned"),
},
AvailabilitySetProperties: &compute.AvailabilitySetProperties{
PlatformFaultDomainCount: to.Int32Ptr(3),
PlatformUpdateDomainCount: to.Int32Ptr(3),
},
},
},
json: `{
"apiVersion": "[variables('apiVersionCompute')]",
"properties": {
"platformFaultDomainCount": 3,
"platformUpdateDomainCount": 3
},
"sku": {
"name": "Aligned"
},
"name": "[variables('masterAvailabilitySet')]",
"type": "Microsoft.Compute/availabilitySets",
"location": "[variables('location')]",
"tags": null
}`},
{
avSet: AvailabilitySetARM{
ARMResource: ARMResource{
APIVersion: "[variables('apiVersionCompute')]",
},
AvailabilitySet: compute.AvailabilitySet{
Name: to.StringPtr("[variables('masterAvailabilitySet')]"),
Location: to.StringPtr("[variables('location')]"),
Type: to.StringPtr("Microsoft.Compute/availabilitySets"),
Sku: &compute.Sku{
Name: to.StringPtr("Aligned"),
},
AvailabilitySetProperties: &compute.AvailabilitySetProperties{
PlatformUpdateDomainCount: to.Int32Ptr(3),
},
},
},
json: `{
"apiVersion": "[variables('apiVersionCompute')]",
"properties": {
"platformFaultDomainCount": "[if(contains(split('canadacentral,centralus,eastus,eastus2,northcentralus,northeurope,southcentralus,westeurope,westus',','),variables('location')),3,if(equals('centraluseuap',variables('location')),1,2))]",
"platformUpdateDomainCount": 3
},
"sku": {
"name": "Aligned"
},
"name": "[variables('masterAvailabilitySet')]",
"type": "Microsoft.Compute/availabilitySets",
"location": "[variables('location')]",
"tags": null
}`},
{
avSet: AvailabilitySetARM{
ARMResource: ARMResource{
APIVersion: "[variables('apiVersionCompute')]",
},
AvailabilitySet: compute.AvailabilitySet{
Name: to.StringPtr("[variables('masterAvailabilitySet')]"),
Location: to.StringPtr("[variables('location')]"),
Type: to.StringPtr("Microsoft.Compute/availabilitySets"),
Sku: &compute.Sku{
Name: to.StringPtr("Aligned"),
},
AvailabilitySetProperties: &compute.AvailabilitySetProperties{},
},
},
json: `{
"apiVersion": "[variables('apiVersionCompute')]",
"properties": {},
"sku": {
"name": "Aligned"
},
"name": "[variables('masterAvailabilitySet')]",
"type": "Microsoft.Compute/availabilitySets",
"location": "[variables('location')]",
"tags": null
}`},
}
for _, vmasTestDatum := range vmasTestData {
output, err := json.Marshal(vmasTestDatum.avSet)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(string(output)).To(MatchJSON(vmasTestDatum.json))
}
}

10
pkg/engine/availabilitysets.go
Просмотреть файл

@ -26,9 +26,12 @@ func CreateAvailabilitySet(cs *api.ContainerService, isManagedDisks bool) Availa
if !cs.Properties.MasterProfile.HasAvailabilityZones() {
if isManagedDisks {
avSet.AvailabilitySetProperties = &compute.AvailabilitySetProperties{
PlatformFaultDomainCount: to.Int32Ptr(2),
PlatformUpdateDomainCount: to.Int32Ptr(3),
}
if cs.Properties.MasterProfile.PlatformFaultDomainCount != nil {
p := int32(*cs.Properties.MasterProfile.PlatformFaultDomainCount)
avSet.PlatformFaultDomainCount = to.Int32Ptr(p)
}
avSet.Sku = &compute.Sku{
Name: to.StringPtr("Aligned"),
}
@ -57,7 +60,10 @@ func createAgentAvailabilitySets(profile *api.AgentPoolProfile) AvailabilitySetA
}
if profile.IsManagedDisks() {
avSet.PlatformFaultDomainCount = to.Int32Ptr(2)
if profile.PlatformFaultDomainCount != nil {
p := int32(*profile.PlatformFaultDomainCount)
avSet.PlatformFaultDomainCount = to.Int32Ptr(p)
}
avSet.PlatformUpdateDomainCount = to.Int32Ptr(3)
avSet.Sku = &compute.Sku{
Name: to.StringPtr("Aligned"),

70
pkg/engine/availabilitysets_test.go
Просмотреть файл

@ -65,7 +65,6 @@ func TestCreateAvailabilitySet(t *testing.T) {
Name: to.StringPtr("Aligned"),
},
AvailabilitySetProperties: &compute.AvailabilitySetProperties{
PlatformFaultDomainCount: to.Int32Ptr(2),
PlatformUpdateDomainCount: to.Int32Ptr(3),
},
},
@ -106,6 +105,41 @@ func TestCreateAvailabilitySet(t *testing.T) {
t.Errorf("unexpected error while comparing availability sets: %s", diff)
}
// Test availability set with platform fault domain count set
count := 3
cs = &api.ContainerService{
Properties: &api.Properties{
MasterProfile: &api.MasterProfile{
PlatformFaultDomainCount: &count,
},
},
}
avSet = CreateAvailabilitySet(cs, true)
expectedAvSet = AvailabilitySetARM{
ARMResource: ARMResource{
APIVersion: "[variables('apiVersionCompute')]",
},
AvailabilitySet: compute.AvailabilitySet{
Name: to.StringPtr("[variables('masterAvailabilitySet')]"),
Location: to.StringPtr("[variables('location')]"),
Type: to.StringPtr("Microsoft.Compute/availabilitySets"),
Sku: &compute.Sku{
Name: to.StringPtr("Aligned"),
},
AvailabilitySetProperties: &compute.AvailabilitySetProperties{
PlatformFaultDomainCount: to.Int32Ptr(int32(count)),
PlatformUpdateDomainCount: to.Int32Ptr(3),
},
},
}
diff = cmp.Diff(avSet, expectedAvSet)
if diff != "" {
t.Errorf("unexpected error while comparing availability sets: %s", diff)
}
}
func TestCreateAgentAvailabilitySets(t *testing.T) {
@ -152,7 +186,6 @@ func TestCreateAgentAvailabilitySets(t *testing.T) {
Location: to.StringPtr("[variables('location')]"),
Type: to.StringPtr("Microsoft.Compute/availabilitySets"),
AvailabilitySetProperties: &compute.AvailabilitySetProperties{
PlatformFaultDomainCount: to.Int32Ptr(2),
PlatformUpdateDomainCount: to.Int32Ptr(3),
},
Sku: &compute.Sku{
@ -167,4 +200,37 @@ func TestCreateAgentAvailabilitySets(t *testing.T) {
t.Errorf("unexpected error while comparing availability sets: %s", diff)
}
// Test availability set with platform fault domain count set
count := 3
profile = &api.AgentPoolProfile{
Name: "foobar",
StorageProfile: api.ManagedDisks,
PlatformFaultDomainCount: &count,
}
avSet = createAgentAvailabilitySets(profile)
expectedAvSet = AvailabilitySetARM{
ARMResource: ARMResource{
APIVersion: "[variables('apiVersionCompute')]",
},
AvailabilitySet: compute.AvailabilitySet{
Name: to.StringPtr("[variables('foobarAvailabilitySet')]"),
Location: to.StringPtr("[variables('location')]"),
Type: to.StringPtr("Microsoft.Compute/availabilitySets"),
AvailabilitySetProperties: &compute.AvailabilitySetProperties{
PlatformFaultDomainCount: to.Int32Ptr(int32(count)),
PlatformUpdateDomainCount: to.Int32Ptr(3),
},
Sku: &compute.Sku{
Name: to.StringPtr("Aligned"),
},
},
}
diff = cmp.Diff(avSet, expectedAvSet)
if diff != "" {
t.Errorf("unexpected error while comparing availability sets: %s", diff)
}
}

1
pkg/engine/masterarmresources_test.go
Просмотреть файл

@ -361,7 +361,6 @@ func TestCreateKubernetesMasterResourcesPVC(t *testing.T) {
AvailabilitySet: compute.AvailabilitySet{
AvailabilitySetProperties: &compute.AvailabilitySetProperties{
PlatformUpdateDomainCount: to.Int32Ptr(3),
PlatformFaultDomainCount: to.Int32Ptr(2),
},
Sku: &compute.Sku{
Name: to.StringPtr("Aligned"),

13
pkg/operations/kubernetesupgrade/upgradecluster.go
Просмотреть файл

@ -244,6 +244,8 @@ func (uc *UpgradeCluster) getClusterNodeStatus(kubeClient armhelpers.KubernetesC
}
}
availabilitySetIDs := []string{}
for vmListPage, err := uc.Client.ListVirtualMachines(ctx, resourceGroup); vmListPage.NotDone(); err = vmListPage.Next() {
if err != nil {
return err
@ -258,6 +260,10 @@ func (uc *UpgradeCluster) getClusterNodeStatus(kubeClient armhelpers.KubernetesC
}
currentVersion := uc.getNodeVersion(kubeClient, strings.ToLower(*vm.Name), vm.Tags, true)
if vm.AvailabilitySet != nil {
availabilitySetIDs = append(availabilitySetIDs, *vm.AvailabilitySet.ID)
}
if uc.Force {
if currentVersion == "" {
currentVersion = "Unknown"
@ -283,6 +289,13 @@ func (uc *UpgradeCluster) getClusterNodeStatus(kubeClient armhelpers.KubernetesC
}
}
// set the VMAS platformFaultDomainCount to match the existing value
fdCount, err := uc.Client.GetAvailabilitySetFaultDomainCount(ctx, resourceGroup, availabilitySetIDs)
if err != nil {
return err
}
uc.DataModel.SetPlatformFaultDomainCount(fdCount)
return nil
}

26
pkg/operations/kubernetesupgrade/upgradecluster_test.go
Просмотреть файл

@ -614,6 +614,32 @@ var _ = Describe("Upgrade Kubernetes cluster tests", func() {
Expect(*uc.MasterVMs).To(HaveLen(1))
Expect(*uc.UpgradedMasterVMs).To(HaveLen(0))
})
It("Should set platform fault domain count based on availability sets", func() {
cs := api.CreateMockContainerService("testcluster", "1.9.11", 3, 2, false)
cs.Properties.OrchestratorProfile.KubernetesConfig = &api.KubernetesConfig{}
cs.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = true
uc := UpgradeCluster{
Translator: &i18n.Translator{},
Logger: log.NewEntry(log.New()),
}
mockClient := armhelpers.MockAKSEngineClient{}
uc.Client = &mockClient
uc.ClusterTopology = ClusterTopology{}
uc.SubscriptionID = "DEC923E3-1EF1-4745-9516-37906D56DEC4"
uc.ResourceGroup = "TestRg"
uc.DataModel = cs
uc.NameSuffix = "12345678"
uc.AgentPoolsToUpgrade = map[string]bool{"agentpool1": true}
err := uc.UpgradeCluster(&mockClient, "kubeConfig", TestAKSEngineVersion)
Expect(err).To(BeNil())
Expect(*cs.Properties.MasterProfile.PlatformFaultDomainCount).To(Equal(3))
for _, pool := range cs.Properties.AgentPoolProfiles {
Expect(*pool.PlatformFaultDomainCount).To(Equal(3))
}
})
})
It("Should not fail if no managed identity is returned by azure during upgrade operation", func() {

71
scripts/fault_domains_expr.py Executable file
Просмотреть файл

@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""
Generates an Azure Resource Manager (ARM) expression that will evaluate at
deployment time to the number of fault domains available in a given location.
Also generates Go code referencing the expression.
Since there is no API to query the fault domain count for an Azure location,
this script parses the public documentation to find out.
"""
import os
import re
import textwrap
import urllib.request
ARM_EXPR = """\
"[
if( contains(
split('{}', ','),
variables('location') ),
3,
if( equals('centraluseuap', variables('location') ),
1,
2
))]"\
"""
GO_CODE = """\
// armExpr is evaluated by Azure Resource Manager at deployment time:
// if location is in the three-fault-domain list, return 3
// else if location is "canary" (testing), return 1
// else return 2
// NOTE: use {} to update this ARM expression.
armExpr := `{}`
// strip all whitespace
armExpr = strings.Join(strings.Fields(armExpr), "")
""".format(os.path.basename(__file__), ARM_EXPR)
SOURCE_DOC = 'https://raw.githubusercontent.com/MicrosoftDocs/azure-docs/master/includes/managed-disks-common-fault-domain-region-list.md' # pylint: disable=line-too-long
def main():
"""
Print an ARM expression that returns the fault domain count for an Azure
location, as well as the associated Go code, ready for copy-and-paste.
"""
regex = re.compile(r"""
\|\s* # vertical bar followed by whitespace
([A-Za-z0-9 ]*?) # >= 0 location name characters (lazy, capture)
\s+\|\s* # >= 1 whitespace chars, vertical bar, more whitespace
(\d) # a single digit (capture)
\s*\| # whitespace followed by a vertical bar
""", re.VERBOSE)
markdown = urllib.request.urlopen(SOURCE_DOC).read().decode("utf8")
# Since the canary region is hard-coded, only the regions with three
# fault domains need to be included in the expression.
threes = (m[0].replace(' ', '').lower() for m in regex.findall(markdown)
if int(m[1]) == 3)
threes = ','.join(sorted(threes))
print("\n\033[1;36mARM expression minified:\033[0;0m \n")
print("".join(ARM_EXPR.format(threes).split()))
print("\n\033[1;34mGo code snippet:\033[0;0m \n")
print(textwrap.indent(GO_CODE.format(threes), "\t"))
if __name__ == '__main__':
main()