зеркало из https://github.com/Azure/AgentBaker.git
feat: create private acr in e2e tests (#5096)
This commit is contained in:
Alison
GitHub
Родитель
5a6b5f03ad
Коммит
78c5d85c95
123
e2e/aks_model.go
123
e2e/aks_model.go
|
|
@ -2,13 +2,16 @@ package e2e
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/agentbakere2e/config"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"
|
||||
|
|
@ -63,21 +66,21 @@ func getBaseClusterModel(clusterName string) *armcontainerservice.ManagedCluster
|
|||
}
|
||||
}
|
||||
|
||||
func addAirgapNetworkSettings(ctx context.Context, t *testing.T, cluster *Cluster) error {
|
||||
t.Logf("Adding network settings for airgap cluster %s in rg %s\n", *cluster.Model.Name, *cluster.Model.Properties.NodeResourceGroup)
|
||||
func addAirgapNetworkSettings(ctx context.Context, t *testing.T, clusterModel *armcontainerservice.ManagedCluster) error {
|
||||
t.Logf("Adding network settings for airgap cluster %s in rg %s\n", *clusterModel.Name, *clusterModel.Properties.NodeResourceGroup)
|
||||
|
||||
vnet, err := getClusterVNet(ctx, *cluster.Model.Properties.NodeResourceGroup)
|
||||
vnet, err := getClusterVNet(ctx, *clusterModel.Properties.NodeResourceGroup)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subnetId := vnet.subnetId
|
||||
|
||||
nsgParams, err := airGapSecurityGroup(config.Config.Location, *cluster.Model.Properties.Fqdn)
|
||||
nsgParams, err := airGapSecurityGroup(config.Config.Location, *clusterModel.Properties.Fqdn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
nsg, err := createAirgapSecurityGroup(ctx, cluster.Model, nsgParams, nil)
|
||||
nsg, err := createAirgapSecurityGroup(ctx, clusterModel, nsgParams, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -91,16 +94,16 @@ func addAirgapNetworkSettings(ctx context.Context, t *testing.T, cluster *Cluste
|
|||
},
|
||||
},
|
||||
}
|
||||
if err = updateSubnet(ctx, cluster.Model, subnetParameters, vnet.name); err != nil {
|
||||
if err = updateSubnet(ctx, clusterModel, subnetParameters, vnet.name); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = addPrivateEndpointForACR(ctx, t, *cluster.Model.Properties.NodeResourceGroup, vnet)
|
||||
err = addPrivateEndpointForACR(ctx, t, *clusterModel.Properties.NodeResourceGroup, vnet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t.Logf("updated cluster %s subnet with airgap settings", *cluster.Model.Name)
|
||||
t.Logf("updated cluster %s subnet with airgap settings", *clusterModel.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -148,11 +151,12 @@ func airGapSecurityGroup(location, clusterFQDN string) (armnetwork.SecurityGroup
|
|||
}
|
||||
|
||||
func addPrivateEndpointForACR(ctx context.Context, t *testing.T, nodeResourceGroup string, vnet VNet) error {
|
||||
t.Logf("Adding private endpoint for ACR in rg %s\n", nodeResourceGroup)
|
||||
t.Logf("Checking if private endpoint for private container registry is in rg %s\n", nodeResourceGroup)
|
||||
|
||||
var err error
|
||||
var exists bool
|
||||
privateEndpointName := "PE-for-ABE2ETests"
|
||||
exists, err := privateEndpointExists(ctx, t, nodeResourceGroup, privateEndpointName)
|
||||
if err != nil {
|
||||
if exists, err = privateEndpointExists(ctx, t, nodeResourceGroup, privateEndpointName); err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
|
|
@ -160,32 +164,28 @@ func addPrivateEndpointForACR(ctx context.Context, t *testing.T, nodeResourceGro
|
|||
return nil
|
||||
}
|
||||
|
||||
peResp, err := createPrivateEndpoint(ctx, t, nodeResourceGroup, privateEndpointName, vnet)
|
||||
if err != nil {
|
||||
var peResp armnetwork.PrivateEndpointsClientCreateOrUpdateResponse
|
||||
if peResp, err = createPrivateEndpoint(ctx, t, nodeResourceGroup, privateEndpointName, config.PrivateACRName, vnet); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
privateZoneName := "privatelink.azurecr.io"
|
||||
pzResp, err := createPrivateZone(ctx, t, nodeResourceGroup, privateZoneName)
|
||||
if err != nil {
|
||||
var pzResp armprivatedns.PrivateZonesClientCreateOrUpdateResponse
|
||||
if pzResp, err = createPrivateZone(ctx, t, nodeResourceGroup, privateZoneName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = createPrivateDNSLink(ctx, t, vnet, nodeResourceGroup, privateZoneName)
|
||||
if err != nil {
|
||||
if err = createPrivateDNSLink(ctx, t, vnet, nodeResourceGroup, privateZoneName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = addRecordSetToPrivateDNSZone(ctx, t, peResp, nodeResourceGroup, privateZoneName)
|
||||
if err != nil {
|
||||
if err = addRecordSetToPrivateDNSZone(ctx, t, peResp, nodeResourceGroup, privateZoneName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = addDNSZoneGroup(ctx, t, pzResp, nodeResourceGroup, privateZoneName, *peResp.Name)
|
||||
if err != nil {
|
||||
if err = addDNSZoneGroup(ctx, t, pzResp, nodeResourceGroup, privateZoneName, *peResp.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -201,7 +201,82 @@ func privateEndpointExists(ctx context.Context, t *testing.T, nodeResourceGroup,
|
|||
return false, nil
|
||||
}
|
||||
|
||||
func createPrivateEndpoint(ctx context.Context, t *testing.T, nodeResourceGroup, privateEndpointName string, vnet VNet) (armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, error) {
|
||||
func createPrivateAzureContainerRegistry(ctx context.Context, t *testing.T, resourceGroup, privateACRName string) error {
|
||||
t.Logf("Creating private Azure Container Registry in rg %s\n", resourceGroup)
|
||||
|
||||
acr, err := config.Azure.RegistriesClient.Get(ctx, resourceGroup, privateACRName, nil)
|
||||
if err == nil {
|
||||
t.Logf("Private ACR already exists at id %s, skipping creation", *acr.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// check if error is anything but not found
|
||||
var azErr *azcore.ResponseError
|
||||
if errors.As(err, &azErr) && azErr.StatusCode != 404 {
|
||||
return fmt.Errorf("failed to get private ACR: %w", err)
|
||||
}
|
||||
|
||||
t.Logf("ACR does not exist, creating...")
|
||||
createParams := armcontainerregistry.Registry{
|
||||
Location: to.Ptr(config.Config.Location),
|
||||
SKU: &armcontainerregistry.SKU{
|
||||
Name: to.Ptr(armcontainerregistry.SKUNamePremium),
|
||||
},
|
||||
Properties: &armcontainerregistry.RegistryProperties{
|
||||
AdminUserEnabled: to.Ptr(false),
|
||||
AnonymousPullEnabled: to.Ptr(true), // required to pull images from the private ACR without authentication
|
||||
},
|
||||
}
|
||||
pollerResp, err := config.Azure.RegistriesClient.BeginCreate(
|
||||
ctx,
|
||||
resourceGroup,
|
||||
privateACRName,
|
||||
createParams,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create private ACR in BeginCreate: %w", err)
|
||||
}
|
||||
_, err = pollerResp.PollUntilDone(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create private ACR during polling: %w", err)
|
||||
}
|
||||
|
||||
t.Logf("Private Azure Container Registry created\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
func addCacheRuelsToPrivateAzureContainerRegistry(ctx context.Context, t *testing.T, resourceGroup, privateACRName string) error {
|
||||
cacheParams := armcontainerregistry.CacheRule{
|
||||
Properties: &armcontainerregistry.CacheRuleProperties{
|
||||
SourceRepository: to.Ptr("mcr.microsoft.com/*"),
|
||||
TargetRepository: to.Ptr("aks/*"),
|
||||
},
|
||||
}
|
||||
cacheCreateResp, err := config.Azure.CacheRulesClient.BeginCreate(
|
||||
ctx,
|
||||
resourceGroup,
|
||||
privateACRName,
|
||||
"aks-managed-rule",
|
||||
cacheParams,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create cache rule in BeginCreate: %w", err)
|
||||
}
|
||||
_, err = cacheCreateResp.PollUntilDone(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create cache rule in polling: %w", err)
|
||||
}
|
||||
|
||||
t.Logf("Cache rule created\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
func createPrivateEndpoint(ctx context.Context, t *testing.T, nodeResourceGroup, privateEndpointName, acrName string, vnet VNet) (armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, error) {
|
||||
t.Logf("Creating Private Endpoint in rg %s\n", nodeResourceGroup)
|
||||
acrID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerRegistry/registries/%s", config.Config.SubscriptionID, config.ResourceGroupName, acrName)
|
||||
|
||||
peParams := armnetwork.PrivateEndpoint{
|
||||
Location: to.Ptr(config.Config.Location),
|
||||
Properties: &armnetwork.PrivateEndpointProperties{
|
||||
|
|
@ -212,7 +287,7 @@ func createPrivateEndpoint(ctx context.Context, t *testing.T, nodeResourceGroup,
|
|||
{
|
||||
Name: to.Ptr(privateEndpointName),
|
||||
Properties: &armnetwork.PrivateLinkServiceConnectionProperties{
|
||||
PrivateLinkServiceID: to.Ptr("/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.ContainerRegistry/registries/aksvhdtestcr"),
|
||||
PrivateLinkServiceID: to.Ptr(acrID),
|
||||
GroupIDs: []*string{to.Ptr("registry")},
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ func ClusterKubenetAirgap(ctx context.Context, t *testing.T) (*Cluster, error) {
|
|||
clusterKubenetAirgapOnce.Do(func() {
|
||||
cluster, err := prepareCluster(ctx, t, getKubenetClusterModel("abe2e-kubenet-airgap"), true)
|
||||
if err == nil {
|
||||
err = addAirgapNetworkSettings(ctx, t, cluster)
|
||||
err = addAirgapNetworkSettings(ctx, t, cluster.Model)
|
||||
}
|
||||
clusterKubenetAirgap, clusterKubenetAirgapError = cluster, err
|
||||
})
|
||||
|
|
@ -102,6 +102,16 @@ func nodeBootstrappingConfig(ctx context.Context, t *testing.T, kube *Kubeclient
|
|||
func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster, isAirgap bool) (*Cluster, error) {
|
||||
cluster.Name = to.Ptr(fmt.Sprintf("%s-%s", *cluster.Name, hash(cluster)))
|
||||
|
||||
// private acr must be created before we add the debug daemonsets
|
||||
if isAirgap {
|
||||
if err := createPrivateAzureContainerRegistry(ctx, t, config.ResourceGroupName, config.PrivateACRName); err != nil {
|
||||
return nil, fmt.Errorf("failed to create private acr: %w", err)
|
||||
}
|
||||
if err := addCacheRuelsToPrivateAzureContainerRegistry(ctx, t, config.ResourceGroupName, config.PrivateACRName); err != nil {
|
||||
return nil, fmt.Errorf("failed to add cache rules to private acr: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
cluster, err := getOrCreateCluster(ctx, t, cluster)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -123,16 +133,19 @@ func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerserv
|
|||
return nil, fmt.Errorf("get kube client using cluster %q: %w", *cluster.Name, err)
|
||||
}
|
||||
|
||||
if err := ensureDebugDaemonsets(ctx, kube, isAirgap); err != nil {
|
||||
return nil, fmt.Errorf("ensure debug daemonsets for %q: %w", *cluster.Name, err)
|
||||
}
|
||||
|
||||
t.Logf("node resource group: %s", *cluster.Properties.NodeResourceGroup)
|
||||
subnetID, err := getClusterSubnetID(ctx, *cluster.Properties.NodeResourceGroup)
|
||||
subnetID, err := getClusterSubnetID(ctx, *cluster.Properties.NodeResourceGroup, t)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get cluster subnet: %w", err)
|
||||
}
|
||||
|
||||
t.Logf("ensuring debug daemonsets")
|
||||
if err := ensureDebugDaemonsets(ctx, t, kube, isAirgap); err != nil {
|
||||
return nil, fmt.Errorf("ensure debug daemonsets for %q: %w", *cluster.Name, err)
|
||||
}
|
||||
|
||||
// nodeBootstrappingConfig requires the debug deamonset to already be created
|
||||
t.Logf("getting the node bootstrapping configuration for cluster")
|
||||
nbc, err := nodeBootstrappingConfig(ctx, t, kube)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get node bootstrapping configuration: %w", err)
|
||||
|
|
@ -159,7 +172,7 @@ func getOrCreateCluster(ctx context.Context, t *testing.T, cluster *armcontainer
|
|||
existingCluster, err := config.Azure.AKS.Get(ctx, config.ResourceGroupName, *cluster.Name, nil)
|
||||
var azErr *azcore.ResponseError
|
||||
if errors.As(err, &azErr) && azErr.StatusCode == 404 {
|
||||
return createNewAKSClusterWithRetry(ctx, t, cluster)
|
||||
return createNewAKSClusterWithRetry(ctx, t, cluster, config.ResourceGroupName)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get cluster %q: %w", *cluster.Name, err)
|
||||
|
|
@ -172,7 +185,7 @@ func getOrCreateCluster(ctx context.Context, t *testing.T, cluster *armcontainer
|
|||
return waitUntilClusterReady(ctx, config.ResourceGroupName, *cluster.Name)
|
||||
default:
|
||||
// this operation will try to update the cluster if it's in a failed state
|
||||
return createNewAKSClusterWithRetry(ctx, t, cluster)
|
||||
return createNewAKSClusterWithRetry(ctx, t, cluster, config.ResourceGroupName)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -202,12 +215,12 @@ func createNewAKSCluster(ctx context.Context, t *testing.T, cluster *armcontaine
|
|||
// that retries creating a cluster if it fails with a 409 Conflict error
|
||||
// clusters are reused, and sometimes a cluster can be in UPDATING or DELETING state
|
||||
// simple retry should be sufficient to avoid such conflicts
|
||||
func createNewAKSClusterWithRetry(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster) (*armcontainerservice.ManagedCluster, error) {
|
||||
func createNewAKSClusterWithRetry(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster, resourceGroup string) (*armcontainerservice.ManagedCluster, error) {
|
||||
maxRetries := 10
|
||||
retryInterval := 30 * time.Second
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||
t.Logf("Attempt %d: creating or updating cluster %s in rg %s\n", attempt+1, *cluster.Name, *cluster.Location)
|
||||
t.Logf("Attempt %d: creating or updating cluster %s in region %s and rg %s\n", attempt+1, *cluster.Name, *cluster.Location, resourceGroup)
|
||||
|
||||
createdCluster, err := createNewAKSCluster(ctx, t, cluster)
|
||||
if err == nil {
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import (
|
|||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"
|
||||
|
|
@ -45,6 +46,8 @@ type AzureClient struct {
|
|||
VMSSVM *armcompute.VirtualMachineScaleSetVMsClient
|
||||
VNet *armnetwork.VirtualNetworksClient
|
||||
VirutalNetworkLinksClient *armprivatedns.VirtualNetworkLinksClient
|
||||
RegistriesClient *armcontainerregistry.RegistriesClient
|
||||
CacheRulesClient *armcontainerregistry.CacheRulesClient
|
||||
}
|
||||
|
||||
func mustNewAzureClient(subscription string) *AzureClient {
|
||||
|
|
@ -118,6 +121,16 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
|
|||
return nil, fmt.Errorf("create core client: %w", err)
|
||||
}
|
||||
|
||||
cloud.RegistriesClient, err = armcontainerregistry.NewRegistriesClient(subscription, credential, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create registry client: %w", err)
|
||||
}
|
||||
|
||||
cloud.CacheRulesClient, err = armcontainerregistry.NewCacheRulesClient(subscription, credential, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create cache rules client: %w", err)
|
||||
}
|
||||
|
||||
cloud.PrivateEndpointClient, err = armnetwork.NewPrivateEndpointsClient(subscription, credential, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create private endpoint client: %w", err)
|
||||
|
|
|
|||
|
|
@ -9,9 +9,11 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
Config = mustLoadConfig()
|
||||
Azure = mustNewAzureClient(Config.SubscriptionID)
|
||||
ResourceGroupName = "abe2e-" + Config.Location
|
||||
Config = mustLoadConfig()
|
||||
Azure = mustNewAzureClient(Config.SubscriptionID)
|
||||
ResourceGroupName = "abe2e-" + Config.Location
|
||||
PrivateACRName = "privateacre2e"
|
||||
|
||||
DefaultPollUntilDoneOptions = &runtime.PollUntilDoneOptions{
|
||||
Frequency: time.Second,
|
||||
}
|
||||
|
|
@ -27,7 +29,7 @@ type Configuration struct {
|
|||
SIGVersionTagValue string `env:"SIG_VERSION_TAG_VALUE" envDefault:"refs/heads/master"`
|
||||
TagsToRun string `env:"TAGS_TO_RUN"`
|
||||
TagsToSkip string `env:"TAGS_TO_SKIP"`
|
||||
TestTimeout time.Duration `env:"TEST_TIMEOUT" envDefault:"20m"`
|
||||
TestTimeout time.Duration `env:"TEST_TIMEOUT" envDefault:"35m"`
|
||||
E2ELoggingDir string `env:"LOGGING_DIR" envDefault:"scenario-logs"`
|
||||
IgnoreScenariosWithMissingVHD bool `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD"`
|
||||
SkipTestsWithSKUCapacityIssue bool `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE"`
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ func extractLogsFromVM(ctx context.Context, t *testing.T, vmssName, privateIP, s
|
|||
"node-bootstrapper": "cat /var/log/azure/node-bootstrapper.log",
|
||||
}
|
||||
|
||||
podName, err := getHostNetworkDebugPodName(ctx, cluster.Kube)
|
||||
podName, err := getHostNetworkDebugPodName(ctx, cluster.Kube, t)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get debug pod name: %w", err)
|
||||
}
|
||||
|
|
@ -100,7 +100,7 @@ func extractClusterParameters(ctx context.Context, t *testing.T, kube *Kubeclien
|
|||
"/var/lib/kubelet/bootstrap-kubeconfig": "cat /var/lib/kubelet/bootstrap-kubeconfig",
|
||||
}
|
||||
|
||||
podName, err := getHostNetworkDebugPodName(ctx, kube)
|
||||
podName, err := getHostNetworkDebugPodName(ctx, kube, t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ require (
|
|||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xP
|
|||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 h1:zDeQI/PaWztI2tcrGO/9RIMey9NvqYbnyttf/0P3QWM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0/go.mod h1:zflC9v4VfViJrSvcvplqws/yGXVbUEMZi/iHpZdSPWA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.1 h1:VNLfijkPSLB25P1l52CXGyeaD8Aj0gcsigmbOpJKwhk=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.1/go.mod h1:E7ltexgRDmeJ0fJWv0D/HLwY2xbDdN+uv+X2uZtOx3w=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0 h1:5n7dPVqsWfVKw+ZiEKSd3Kzu7gwBkbEBkeXb8rgaE9Q=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v5 v5.0.0/go.mod h1:HcZY0PHPo/7d75p99lB6lK0qYOP4vLRJUBpiehYXtLQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.0.0 h1:EK0ZY1qKWzaWyRNFDsrwRfgVBMGbs+m71yie+y11+Tc=
|
||||
|
|
|
|||
17
e2e/kube.go
17
e2e/kube.go
|
|
@ -3,9 +3,10 @@ package e2e
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/agentbakere2e/config"
|
||||
"k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/apps/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
|
@ -76,23 +77,25 @@ func getClusterKubeconfigBytes(ctx context.Context, resourceGroupName, clusterNa
|
|||
}
|
||||
|
||||
// this is a bit ugly, but we don't want to execute this piece concurrently with other tests
|
||||
func ensureDebugDaemonsets(ctx context.Context, kube *Kubeclient, isAirgap bool) error {
|
||||
hostDS := getDebugDaemonsetTemplate(hostNetworkDebugAppLabel, "nodepool1", true, isAirgap)
|
||||
func ensureDebugDaemonsets(ctx context.Context, t *testing.T, kube *Kubeclient, isAirgap bool) error {
|
||||
// airgap set to false since acr does not exist during cluster creation
|
||||
hostDS := getDebugDaemonsetTemplate(t, hostNetworkDebugAppLabel, "nodepool1", true, isAirgap)
|
||||
if err := createDebugDaemonset(ctx, kube, hostDS); err != nil {
|
||||
return err
|
||||
}
|
||||
nonHostDS := getDebugDaemonsetTemplate(podNetworkDebugAppLabel, "nodepool2", false, isAirgap)
|
||||
nonHostDS := getDebugDaemonsetTemplate(t, podNetworkDebugAppLabel, "nodepool2", false, isAirgap)
|
||||
if err := createDebugDaemonset(ctx, kube, nonHostDS); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getDebugDaemonsetTemplate(deploymentName, targetNodeLabel string, isHostNetwork, isAirgap bool) string {
|
||||
func getDebugDaemonsetTemplate(t *testing.T, deploymentName, targetNodeLabel string, isHostNetwork, isAirgap bool) string {
|
||||
image := "mcr.microsoft.com/cbl-mariner/base/core:2.0"
|
||||
if isAirgap {
|
||||
image = "aksvhdtestcr.azurecr.io/aks/cbl-mariner/base/core:2.0"
|
||||
image = fmt.Sprintf("%s.azurecr.io/aks/cbl-mariner/base/core:2.0", config.PrivateACRName)
|
||||
}
|
||||
t.Logf("using image %s for debug daemonset", image)
|
||||
|
||||
return fmt.Sprintf(`apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
|
|
@ -148,7 +151,7 @@ func createDebugDaemonset(ctx context.Context, kube *Kubeclient, manifest string
|
|||
return nil
|
||||
}
|
||||
|
||||
func getClusterSubnetID(ctx context.Context, mcResourceGroupName string) (string, error) {
|
||||
func getClusterSubnetID(ctx context.Context, mcResourceGroupName string, t *testing.T) (string, error) {
|
||||
pager := config.Azure.VNet.NewListPager(mcResourceGroupName, nil)
|
||||
for pager.More() {
|
||||
nextResult, err := pager.NextPage(ctx)
|
||||
|
|
|
|||
10
e2e/pod.go
10
e2e/pod.go
|
|
@ -20,7 +20,7 @@ const (
|
|||
)
|
||||
|
||||
// Returns the name of a pod that's a member of the 'debug' daemonset, running on an aks-nodepool node.
|
||||
func getHostNetworkDebugPodName(ctx context.Context, kube *Kubeclient) (string, error) {
|
||||
func getHostNetworkDebugPodName(ctx context.Context, kube *Kubeclient, t *testing.T) (string, error) {
|
||||
podList := corev1.PodList{}
|
||||
if err := kube.Dynamic.List(ctx, &podList, client.MatchingLabels{"app": hostNetworkDebugAppLabel}); err != nil {
|
||||
return "", fmt.Errorf("failed to list debug pod: %w", err)
|
||||
|
|
@ -29,7 +29,7 @@ func getHostNetworkDebugPodName(ctx context.Context, kube *Kubeclient) (string,
|
|||
return "", fmt.Errorf("failed to find host debug pod")
|
||||
}
|
||||
pod := podList.Items[0]
|
||||
err := waitUntilPodReady(ctx, kube, pod.Name)
|
||||
err := waitUntilPodReady(ctx, kube, pod.Name, t)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to wait for pod to be in running state: %w", err)
|
||||
}
|
||||
|
|
@ -38,7 +38,7 @@ func getHostNetworkDebugPodName(ctx context.Context, kube *Kubeclient) (string,
|
|||
|
||||
// Returns the name of a pod that's a member of the 'debugnonhost' daemonset running in the cluster - this will return
|
||||
// the name of the pod that is running on the node created for specifically for the test case which is running validation checks.
|
||||
func getPodNetworkDebugPodNameForVMSS(ctx context.Context, kube *Kubeclient, vmssName string) (string, error) {
|
||||
func getPodNetworkDebugPodNameForVMSS(ctx context.Context, kube *Kubeclient, vmssName string, t *testing.T) (string, error) {
|
||||
podList := corev1.PodList{}
|
||||
if err := kube.Dynamic.List(ctx, &podList, client.MatchingLabels{"app": podNetworkDebugAppLabel}); err != nil {
|
||||
return "", fmt.Errorf("failed to list debug pod: %w", err)
|
||||
|
|
@ -46,7 +46,7 @@ func getPodNetworkDebugPodNameForVMSS(ctx context.Context, kube *Kubeclient, vms
|
|||
|
||||
for _, pod := range podList.Items {
|
||||
if strings.Contains(pod.Spec.NodeName, vmssName) {
|
||||
err := waitUntilPodReady(ctx, kube, pod.Name)
|
||||
err := waitUntilPodReady(ctx, kube, pod.Name, t)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to wait for pod to be in running state: %w", err)
|
||||
}
|
||||
|
|
@ -89,7 +89,7 @@ func ensurePod(ctx context.Context, t *testing.T, namespace string, kube *Kubecl
|
|||
t.Logf("couldn't not delete pod %s: %v", podName, err)
|
||||
}
|
||||
})
|
||||
if err := waitUntilPodReady(ctx, kube, podName); err != nil {
|
||||
if err := waitUntilPodReady(ctx, kube, podName, t); err != nil {
|
||||
return fmt.Errorf("failed to wait for pod to be in running state: %w", err)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"sigs.k8s.io/yaml"
|
||||
|
|
@ -58,10 +59,35 @@ func waitUntilNodeReady(ctx context.Context, t *testing.T, kube *Kubeclient, vms
|
|||
return nodeName
|
||||
}
|
||||
|
||||
func waitUntilPodReady(ctx context.Context, kube *Kubeclient, podName string) error {
|
||||
func waitUntilPodReady(ctx context.Context, kube *Kubeclient, podName string, t *testing.T) error {
|
||||
lastLogTime := time.Now()
|
||||
logInterval := 5 * time.Minute // log every 5 minutes
|
||||
|
||||
return wait.PollUntilContextCancel(ctx, defaultPollInterval, true, func(ctx context.Context) (bool, error) {
|
||||
currentLogTime := time.Now()
|
||||
|
||||
pod, err := kube.Typed.CoreV1().Pods(defaultNamespace).Get(ctx, podName, metav1.GetOptions{})
|
||||
|
||||
printLog := false
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
remaining := time.Until(deadline)
|
||||
if currentLogTime.Sub(lastLogTime) > logInterval {
|
||||
// this logs every 5 minutes to reduce spam, iterations of poller are continuning as normal.
|
||||
t.Logf("pod %s status: %s time before timeout: %v", podName, pod.Status.Phase, remaining)
|
||||
lastLogTime = currentLogTime
|
||||
printLog = true
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// pod might not be created yet, let the poller continue
|
||||
if errors.IsNotFound(err) {
|
||||
if printLog {
|
||||
// this logs every 5 minutes to reduce spam, iterations of poller are continuning as normal.
|
||||
t.Logf("pod %s not found yet. Err %v", podName, err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ func Test_azurelinuxv2AirGap(t *testing.T) {
|
|||
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
|
||||
PrivateEgress: &datamodel.PrivateEgress{
|
||||
Enabled: true,
|
||||
ContainerRegistryServer: "aksvhdtestcr.azurecr.io/aks",
|
||||
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks", config.PrivateACRName),
|
||||
},
|
||||
}
|
||||
},
|
||||
|
|
@ -97,7 +97,7 @@ func Test_azurelinuxv2ARM64AirGap(t *testing.T) {
|
|||
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
|
||||
PrivateEgress: &datamodel.PrivateEgress{
|
||||
Enabled: true,
|
||||
ContainerRegistryServer: "aksvhdtestcr.azurecr.io/aks",
|
||||
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks", config.PrivateACRName),
|
||||
},
|
||||
}
|
||||
},
|
||||
|
|
@ -295,7 +295,7 @@ func Test_marinerv2AirGap(t *testing.T) {
|
|||
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
|
||||
PrivateEgress: &datamodel.PrivateEgress{
|
||||
Enabled: true,
|
||||
ContainerRegistryServer: "aksvhdtestcr.azurecr.io/aks",
|
||||
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks", config.PrivateACRName),
|
||||
},
|
||||
}
|
||||
},
|
||||
|
|
@ -345,7 +345,7 @@ func Test_marinerv2ARM64AirGap(t *testing.T) {
|
|||
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
|
||||
PrivateEgress: &datamodel.PrivateEgress{
|
||||
Enabled: true,
|
||||
ContainerRegistryServer: "aksvhdtestcr.azurecr.io/aks",
|
||||
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks", config.PrivateACRName),
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -666,7 +666,7 @@ func Test_ubuntu2204AirGap(t *testing.T) {
|
|||
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
|
||||
PrivateEgress: &datamodel.PrivateEgress{
|
||||
Enabled: true,
|
||||
ContainerRegistryServer: "aksvhdtestcr.azurecr.io/aks",
|
||||
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks", config.PrivateACRName),
|
||||
},
|
||||
}
|
||||
},
|
||||
|
|
@ -691,7 +691,7 @@ func Test_Ubuntu2204Gen2ContainerdAirgapped_K8sNotCached(t *testing.T) {
|
|||
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
|
||||
PrivateEgress: &datamodel.PrivateEgress{
|
||||
Enabled: true,
|
||||
ContainerRegistryServer: "aksvhdtestcr.azurecr.io/aks",
|
||||
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks", config.PrivateACRName),
|
||||
},
|
||||
}
|
||||
},
|
||||
|
|
@ -1153,7 +1153,7 @@ func Test_ubuntu2204WasmAirGap(t *testing.T) {
|
|||
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
|
||||
PrivateEgress: &datamodel.PrivateEgress{
|
||||
Enabled: true,
|
||||
ContainerRegistryServer: "aksvhdtestcr.azurecr.io/aks",
|
||||
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks", config.PrivateACRName),
|
||||
},
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"fmt"
|
||||
|
||||
"github.com/Azure/agentbaker/pkg/agent/datamodel"
|
||||
"github.com/Azure/agentbakere2e/config"
|
||||
)
|
||||
|
||||
// this is huge, but accurate, so leave it here.
|
||||
|
|
@ -451,7 +452,7 @@ func baseTemplate(location string) *datamodel.NodeBootstrappingConfiguration {
|
|||
func getHTTPServerTemplate(podName, nodeName string, isAirgap bool) string {
|
||||
image := "mcr.microsoft.com/cbl-mariner/busybox:2.0"
|
||||
if isAirgap {
|
||||
image = "aksvhdtestcr.azurecr.io/aks/cbl-mariner/busybox:2.0"
|
||||
image = fmt.Sprintf("%s.azurecr.io/aks/cbl-mariner/busybox:2.0", config.PrivateACRName)
|
||||
}
|
||||
|
||||
return fmt.Sprintf(`apiVersion: v1
|
||||
|
|
|
|||
|
|
@ -34,12 +34,12 @@ func validateWasm(ctx context.Context, t *testing.T, kube *Kubeclient, nodeName
|
|||
}
|
||||
|
||||
func runLiveVMValidators(ctx context.Context, t *testing.T, vmssName, privateIP, sshPrivateKey string, opts *scenarioRunOpts) error {
|
||||
hostPodName, err := getHostNetworkDebugPodName(ctx, opts.clusterConfig.Kube)
|
||||
hostPodName, err := getHostNetworkDebugPodName(ctx, opts.clusterConfig.Kube, t)
|
||||
if err != nil {
|
||||
return fmt.Errorf("while running live validator for node %s, unable to get debug pod name: %w", vmssName, err)
|
||||
}
|
||||
|
||||
nonHostPodName, err := getPodNetworkDebugPodNameForVMSS(ctx, opts.clusterConfig.Kube, vmssName)
|
||||
nonHostPodName, err := getPodNetworkDebugPodNameForVMSS(ctx, opts.clusterConfig.Kube, vmssName, t)
|
||||
if err != nil {
|
||||
return fmt.Errorf("while running live validator for node %s, unable to get non host debug pod name: %w", vmssName, err)
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче