From b5a7f14ecd4b9cc6e53ccccd7f997c11c447ae8b Mon Sep 17 00:00:00 2001 From: Santhosh Prabhu <6684582+santhoshmprabhu@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:26:39 -0800 Subject: [PATCH] test: pipeline changes for cilium nodesubnet (#3083) * chore: separate pipeline changes for cilium nodesubnet * fix:make linter happy * chore: remove node subnet cilium version tag * fix: make private fork version tags compatible --- .pipelines/pipeline.yaml | 16 +++++++++ .../cilium-nodesubnet-e2e-job-template.yaml | 2 +- .../cilium-nodesubnet-e2e-step-template.yaml | 2 +- hack/aks/Makefile | 35 ++++++++++--------- hack/aks/README.md | 1 + .../cilium-nodesubnet/ipconfigupdate.go | 34 +++++++++--------- .../ip-masq-agent/config-custom.yaml | 1 + .../ip-masq-agent/config-reconcile.yaml | 1 + 8 files changed, 55 insertions(+), 37 deletions(-) diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 6631f80f0..6fa30d2c8 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -261,6 +261,17 @@ stages: k8sVersion: "" dependsOn: "containerize" + # Cilium Nodesubnet E2E tests + - template: singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-job-template.yaml + parameters: + name: "cilium_nodesubnet_e2e" + displayName: Cilium NodeSubnet + clusterType: nodesubnet-byocni-nokubeproxy-up + clusterName: "cilndsubnete2e" + vmSize: Standard_B2s + k8sVersion: "" + dependsOn: "containerize" + # Cilium Overlay E2E tests - template: singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml parameters: @@ -405,6 +416,7 @@ stages: - azure_overlay_stateless_e2e - aks_swift_e2e - cilium_e2e + - cilium_nodesubnet_e2e - cilium_overlay_e2e - cilium_h_overlay_e2e - aks_ubuntu_22_linux_e2e @@ -425,6 +437,10 @@ stages: cilium_e2e: name: cilium_e2e clusterName: "ciliume2e" + region: $(REGION_AKS_CLUSTER_TEST) + cilium_nodesubnet_e2e: + name: cilium_nodesubnet_e2e + clusterName: "cilndsubnete2e" region: $(REGION_AKS_CLUSTER_TEST) cilium_overlay_e2e: name: cilium_overlay_e2e diff --git a/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-job-template.yaml index 416be76c0..694d0b4e9 100644 --- a/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-job-template.yaml +++ b/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-job-template.yaml @@ -71,9 +71,9 @@ stages: os: ${{ parameters.os }} datapath: true dns: true + cni: cilium portforward: true service: true - hostport: true dependsOn: ${{ parameters.name }} - job: failedE2ELogs diff --git a/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml index 835a8dd1f..c6fc01f77 100644 --- a/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml @@ -58,7 +58,7 @@ steps: kubectl cluster-info kubectl get po -owide -A echo "install Cilium ${CILIUM_VERSION_TAG}" - export DIR=${CILIUM_VERSION_TAG%.*} + export DIR=$(echo ${CILIUM_VERSION_TAG#v} | cut -d. -f1,2) echo "installing files from ${DIR}" echo "deploy Cilium ConfigMap" kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-config/cilium-config.yaml diff --git a/hack/aks/Makefile b/hack/aks/Makefile index e88c7e001..2df03770b 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -8,17 +8,18 @@ AZIMG = mcr.microsoft.com/azure-cli AZCLI ?= docker run --rm -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh -v $(PWD):/root/tmpsrc $(AZIMG) az # overrideable defaults -AUTOUPGRADE ?= patch -K8S_VER ?= 1.30 -NODE_COUNT ?= 2 -NODE_COUNT_WIN ?= $(NODE_COUNT) -NODEUPGRADE ?= NodeImage -OS ?= linux # Used to signify if you want to bring up a windows nodePool on byocni clusters -OS_SKU ?= Ubuntu -OS_SKU_WIN ?= Windows2022 -REGION ?= westus2 -VM_SIZE ?= Standard_B2s -VM_SIZE_WIN ?= Standard_B2s +AUTOUPGRADE ?= patch +K8S_VER ?= 1.30 +NODE_COUNT ?= 2 +NODE_COUNT_WIN ?= $(NODE_COUNT) +NODEUPGRADE ?= NodeImage +OS ?= linux # Used to signify if you want to bring up a windows nodePool on byocni clusters +OS_SKU ?= Ubuntu +OS_SKU_WIN ?= Windows2022 +REGION ?= westus2 +VM_SIZE ?= Standard_B2s +VM_SIZE_WIN ?= Standard_B2s +KUBE_PROXY_JSON_PATH ?= ./kube-proxy.json # overrideable variables SUB ?= $(AZURE_SUBSCRIPTION) @@ -103,13 +104,13 @@ nodesubnet-byocni-nokubeproxy-up: rg-up overlay-net-up ## Brings up an NodeSubne --kubernetes-version $(K8S_VER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ + --load-balancer-sku standard \ --max-pods 250 \ --network-plugin none \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --os-sku $(OS_SKU) \ --no-ssh-key \ - --kube-proxy-config ./kube-proxy.json \ + --kube-proxy-config $(KUBE_PROXY_JSON_PATH) \ --yes @$(MAKE) set-kubeconf @@ -146,7 +147,7 @@ overlay-byocni-nokubeproxy-up: rg-up overlay-net-up ## Brings up an Overlay BYO --pod-cidr 192.168.0.0/16 \ --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ --no-ssh-key \ - --kube-proxy-config ./kube-proxy.json \ + --kube-proxy-config $(KUBE_PROXY_JSON_PATH) \ --yes @$(MAKE) set-kubeconf @@ -215,7 +216,7 @@ swift-byocni-nokubeproxy-up: rg-up swift-net-up ## Bring up a SWIFT BYO CNI clus --pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \ --no-ssh-key \ --os-sku $(OS_SKU) \ - --kube-proxy-config ./kube-proxy.json \ + --kube-proxy-config $(KUBE_PROXY_JSON_PATH) \ --yes @$(MAKE) set-kubeconf @@ -305,7 +306,7 @@ vnetscale-swift-byocni-nokubeproxy-up: rg-up vnetscale-swift-net-up ## Bring up --pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \ --no-ssh-key \ --os-sku $(OS_SKU) \ - --kube-proxy-config ./kube-proxy.json \ + --kube-proxy-config $(KUBE_PROXY_JSON_PATH) \ --yes @$(MAKE) set-kubeconf @@ -434,7 +435,7 @@ dualstack-byocni-nokubeproxy-up: rg-up overlay-net-up ## Brings up a Dualstack o --ip-families ipv4,ipv6 \ --aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureOverlayDualStackPreview \ --no-ssh-key \ - --kube-proxy-config ./kube-proxy.json \ + --kube-proxy-config $(KUBE_PROXY_JSON_PATH) \ --yes @$(MAKE) set-kubeconf diff --git a/hack/aks/README.md b/hack/aks/README.md index 8e1febd0e..0ff3ed481 100644 --- a/hack/aks/README.md +++ b/hack/aks/README.md @@ -24,6 +24,7 @@ AKS Clusters byocni-up Alias to swift-byocni-up cilium-up Alias to swift-cilium-up up Alias to swift-up + nodesubnet-byocni-nokubeproxy-up Bring up a Nodesubnet BYO CNI cluster. Does not include secondary IP configs. overlay-byocni-up Bring up a Overlay BYO CNI cluster overlay-byocni-nokubeproxy-up Bring up a Overlay BYO CNI cluster without kube-proxy overlay-cilium-up Bring up a Overlay Cilium cluster diff --git a/test/integration/cilium-nodesubnet/ipconfigupdate.go b/test/integration/cilium-nodesubnet/ipconfigupdate.go index 2a8fa8a84..214f4655f 100644 --- a/test/integration/cilium-nodesubnet/ipconfigupdate.go +++ b/test/integration/cilium-nodesubnet/ipconfigupdate.go @@ -12,14 +12,15 @@ import ( "github.com/pkg/errors" ) -func runCommand(command string) (string, error) { - cmd := exec.Command("bash", "-c", command) +func runAzCommand(params ...string) (string, error) { var out bytes.Buffer var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr var err error + fmt.Println("Running Azure CLI command ", strings.Join(params, " ")) for i := 0; i < 3; i++ { + cmd := exec.Command("az", params...) + cmd.Stdout = &out + cmd.Stderr = &stderr err = cmd.Run() if err == nil { break @@ -27,7 +28,7 @@ func runCommand(command string) (string, error) { } if err != nil { - return "", errors.Wrap(err, "command failed") + return "", errors.Wrap(err, "command failed "+stderr.String()) } return out.String(), nil @@ -60,16 +61,14 @@ func main() { os.Exit(1) } - command := fmt.Sprintf("az vmss list -g %s --query '[0].name' -o tsv", resourceGroup) - result, err := runCommand(command) + result, err := runAzCommand("vmss", "list", "-g", resourceGroup, "--query", "[0].name", "-o", "tsv") if err != nil { fmt.Printf("Command failed with error: %s\n", err) os.Exit(1) } vmssName := strings.TrimSpace(result) - command = fmt.Sprintf("az vmss show -g %s -n %s", resourceGroup, vmssName) - result, err = runCommand(command) + result, err = runAzCommand("vmss", "show", "-g", resourceGroup, "-n", vmssName) if err != nil { fmt.Printf("Command failed with error: %s\n", err) os.Exit(1) @@ -104,8 +103,14 @@ func main() { for i := 2; i <= secondaryConfigCount+1; i++ { ipConfig := make(map[string]interface{}) for k, v := range primaryIPConfig { + // only the primary config needs loadBalancerBackendAddressPools. Azure doesn't allow + // secondary IP configs to be associated load balancer backend pools. + if k == "loadBalancerBackendAddressPools" { + continue + } ipConfig[k] = v } + ipConfigName := fmt.Sprintf("ipconfig%d", i) if !contains(usedIPConfigNames, ipConfigName) { ipConfig["name"] = ipConfigName @@ -125,20 +130,13 @@ func main() { os.Exit(1) } - escapedNetworkProfileJSON := strings.ReplaceAll(string(networkProfileJSON), `\`, `\\`) - escapedNetworkProfileJSON = strings.ReplaceAll(escapedNetworkProfileJSON, `'`, `\'`) - - command = fmt.Sprintf("az vmss update -g %s -n %s --set virtualMachineProfile.networkProfile='%s'", resourceGroup, vmssName, escapedNetworkProfileJSON) - fmt.Println("Command to update VMSS: ", command) - _, err = runCommand(command) + _, err = runAzCommand("vmss", "update", "-g", resourceGroup, "-n", vmssName, "--set", fmt.Sprintf("virtualMachineProfile.networkProfile=%s", networkProfileJSON)) if err != nil { fmt.Printf("Command failed with error: %s\n", err) os.Exit(1) } - command = fmt.Sprintf("az vmss update-instances -g %s -n %s --instance-ids '*'", resourceGroup, vmssName) - fmt.Println("Command to update VMSS instances: ", command) - _, err = runCommand(command) + _, err = runAzCommand("vmss", "update-instances", "-g", resourceGroup, "-n", vmssName, "--instance-ids", "*") if err != nil { fmt.Printf("Command failed with error: %s\n", err) os.Exit(1) diff --git a/test/integration/manifests/ip-masq-agent/config-custom.yaml b/test/integration/manifests/ip-masq-agent/config-custom.yaml index 4bdc6cc5e..3ace54131 100644 --- a/test/integration/manifests/ip-masq-agent/config-custom.yaml +++ b/test/integration/manifests/ip-masq-agent/config-custom.yaml @@ -13,5 +13,6 @@ data: - 192.168.0.0/16 - 100.64.0.0/10 - 10.244.0.0/16 + - 10.10.0.0/16 masqLinkLocal: false masqLinkLocalIPv6: true diff --git a/test/integration/manifests/ip-masq-agent/config-reconcile.yaml b/test/integration/manifests/ip-masq-agent/config-reconcile.yaml index 0f715267d..67944cd91 100644 --- a/test/integration/manifests/ip-masq-agent/config-reconcile.yaml +++ b/test/integration/manifests/ip-masq-agent/config-reconcile.yaml @@ -11,4 +11,5 @@ data: - 192.168.0.0/16 - 100.64.0.0/10 - 10.244.0.0/16 + - 10.10.0.0/16 masqLinkLocal: true