From 1b5d3a107688e6374e300fe9b5d6b52e4cd33f3e Mon Sep 17 00:00:00 2001 From: John Payne <89417863+jpayne3506@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:34:46 -0800 Subject: [PATCH] ci: Add exit code capture for PR pipeline (#3125) * ci: Add exit code capture for PR pipeline * ci: ignore errors from cilium status --- .../singletenancy/aks-swift/aks-swift-e2e.steps.yaml | 1 + .../singletenancy/aks-swift/e2e-step-template.yaml | 1 + .pipelines/singletenancy/aks/aks-e2e.steps.yaml | 1 + .pipelines/singletenancy/aks/e2e-step-template.yaml | 1 + .../azure-cni-overlay-e2e-step-template.yaml | 2 ++ .../azure-cni-overlay-e2e.steps.yaml | 2 ++ .../cilium-dualstackoverlay-e2e-step-template.yaml | 2 ++ .../cilium-dualstackoverlay-e2e.steps.yaml | 2 ++ .../cilium-nodesubnet-e2e-step-template.yaml | 7 +++++-- .../cilium-overlay-e2e-step-template.yaml | 8 ++++---- .../cilium-overlay-e2e.steps.yaml | 8 ++++---- .../cilium-overlay-e2e-step-template.yaml | 9 +++++---- .../cilium-overlay/cilium-overlay-e2e.steps.yaml | 11 ++++++----- .../cilium/cilium-e2e-step-template.yaml | 6 +++++- .pipelines/singletenancy/cilium/cilium-e2e.steps.yaml | 2 ++ .../dualstackoverlay-e2e-step-template.yaml | 2 ++ .../dualstack-overlay/dualstackoverlay-e2e.steps.yaml | 2 ++ .pipelines/templates/cilium-cli.steps.yaml | 1 + .pipelines/templates/cilium-cli.yaml | 4 ++++ .pipelines/templates/cilium-tests.yaml | 10 +++++----- 20 files changed, 57 insertions(+), 25 deletions(-) diff --git a/.pipelines/singletenancy/aks-swift/aks-swift-e2e.steps.yaml b/.pipelines/singletenancy/aks-swift/aks-swift-e2e.steps.yaml index 8cb3edb19..f8177d101 100644 --- a/.pipelines/singletenancy/aks-swift/aks-swift-e2e.steps.yaml +++ b/.pipelines/singletenancy/aks-swift/aks-swift-e2e.steps.yaml @@ -68,6 +68,7 @@ steps: workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index 1e5a923d4..00cc69aed 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -71,6 +71,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS diff --git a/.pipelines/singletenancy/aks/aks-e2e.steps.yaml b/.pipelines/singletenancy/aks/aks-e2e.steps.yaml index b5627ceb9..85dd35dfb 100644 --- a/.pipelines/singletenancy/aks/aks-e2e.steps.yaml +++ b/.pipelines/singletenancy/aks/aks-e2e.steps.yaml @@ -70,6 +70,7 @@ steps: displayName: "Restart Nodes" - script: | + set -e kubectl get pods -A -o wide echo "Deploying test pods" diff --git a/.pipelines/singletenancy/aks/e2e-step-template.yaml b/.pipelines/singletenancy/aks/e2e-step-template.yaml index 397bac27d..ccb7df849 100644 --- a/.pipelines/singletenancy/aks/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks/e2e-step-template.yaml @@ -64,6 +64,7 @@ steps: done displayName: "Restart Nodes" - script: | + set -e kubectl get pods -A -o wide echo "Deploying test pods" cd test/integration/load diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml index 633c427e6..8d8de54f5 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml @@ -64,6 +64,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -129,6 +130,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.steps.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.steps.yaml index a6bf382b4..cf0a0310f 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.steps.yaml @@ -68,6 +68,7 @@ steps: workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -138,6 +139,7 @@ steps: workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml index 879850a58..701f79e63 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml @@ -92,6 +92,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -106,6 +107,7 @@ steps: retryCountOnTaskFailure: 3 - script: | + set -e echo "Run Cilium Connectivity Tests" cilium status cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption,!no-unexpected-packet-drops' --force-deploy diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml index dea896aeb..8d0b262de 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml @@ -96,6 +96,7 @@ steps: workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -110,6 +111,7 @@ steps: retryCountOnTaskFailure: 3 - script: | + set -e echo "Run Cilium Connectivity Tests" cilium status cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption,!no-unexpected-packet-drops' --force-deploy diff --git a/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml index c6fc01f77..242ec88bd 100644 --- a/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml @@ -53,7 +53,7 @@ steps: inlineScript: | set -e make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} - ls -lah + ls -lah pwd kubectl cluster-info kubectl get po -owide -A @@ -81,5 +81,8 @@ steps: retryCountOnTaskFailure: 3 name: "nodeSubnetE2ETests" displayName: "Run NodeSubnet E2E" - + - template: ../../templates/cilium-tests.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + scaleup: ${{ parameters.scaleup }} diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml index ec8530183..ff52d7a66 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml @@ -96,6 +96,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -110,6 +111,7 @@ steps: retryCountOnTaskFailure: 3 - script: | + set -e echo "Run Cilium Connectivity Tests" cilium status cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' --force-deploy @@ -135,19 +137,17 @@ steps: displayName: "Run Hubble Connectivity Tests" - script: | + set -e echo "validate pod IP assignment and check systemd-networkd restart" kubectl get pod -owide -A - # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. - # Saves 17 minutes - kubectl delete deploy -n $(ciliumNamespace) echo-external-node if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then echo "Check cilium identities in $(ciliumNamespace) namepsace during nightly run" echo "expect the identities to be deleted when the namespace is deleted" kubectl get ciliumidentity | grep cilium-test fi make test-validate-state - echo "delete cilium connectivity test resources and re-validate state" + echo "delete cilium connectivity test resources and re-validate state" # TODO Delete this and the next 4 lines if connectivity no longer has bug kubectl delete ns $(ciliumNamespace) kubectl get pod -owide -A make test-validate-state diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml index 8634bff50..f52800e74 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml @@ -93,6 +93,7 @@ steps: workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -107,6 +108,7 @@ steps: retryCountOnTaskFailure: 3 - script: | + set -e echo "Run Cilium Connectivity Tests" cilium status cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' --force-deploy @@ -134,19 +136,17 @@ steps: displayName: "Run Hubble Connectivity Tests" - script: | + set -e echo "validate pod IP assignment and check systemd-networkd restart" kubectl get pod -owide -A - # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. - # Saves 17 minutes - kubectl delete deploy -n $(ciliumNamespace) echo-external-node if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then echo "Check cilium identities in $(ciliumNamespace) namepsace during nightly run" echo "expect the identities to be deleted when the namespace is deleted" kubectl get ciliumidentity | grep cilium-test fi make test-validate-state - echo "delete cilium connectivity test resources and re-validate state" + echo "delete cilium connectivity test resources and re-validate state" # TODO Delete this and the next 4 lines if connectivity no longer has bug kubectl delete ns $(ciliumNamespace) kubectl get pod -owide -A make test-validate-state diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index a045b67f7..ccd2fffce 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -118,6 +118,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -132,6 +133,7 @@ steps: retryCountOnTaskFailure: 3 - script: | + set -e echo "Run Cilium Connectivity Tests" cilium status if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ] @@ -162,11 +164,9 @@ steps: displayName: "Run Hubble Connectivity Tests" - script: | + set -e echo "validate pod IP assignment and check systemd-networkd restart" kubectl get pod -owide -A - # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. - # Saves 17 minutes - kubectl delete deploy -n $(ciliumNamespace) echo-external-node if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then echo "Check cilium identities in $(ciliumNamespace) namepsace during nightly run" echo "expect the identities to be deleted when the namespace is deleted" @@ -207,7 +207,8 @@ steps: name: "CiliumIdentities" displayName: "Verify Cilium Identities Deletion" - - script: | + - script: | # TODO REMOVE THIS STEP, make test-load covers this + set -e echo "validate pod IP assignment before CNS restart" kubectl get pod -owide -A make test-validate-state diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml index 5fad1a332..dea722df9 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml @@ -115,6 +115,7 @@ steps: workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -129,6 +130,7 @@ steps: retryCountOnTaskFailure: 3 - script: | + set -e echo "Run Cilium Connectivity Tests" cilium status if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ] @@ -160,18 +162,16 @@ steps: displayName: "Run Hubble Connectivity Tests" - script: | + set -e echo "validate pod IP assignment and check systemd-networkd restart" kubectl get pod -owide -A - # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. - # Saves 17 minutes - kubectl delete deploy -n $(ciliumNamespace) echo-external-node if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then echo "Check cilium identities in $(ciliumNamespace) namepsace during nightly run" echo "expect the identities to be deleted when the namespace is deleted" kubectl get ciliumidentity | grep cilium-test fi make test-validate-state - echo "delete cilium connectivity test resources and re-validate state" + echo "delete cilium connectivity test resources and re-validate state" # TODO Delete this and the next 4 lines if connectivity no longer has bug kubectl delete ns $(ciliumNamespace) kubectl get pod -owide -A make test-validate-state @@ -206,7 +206,8 @@ steps: name: "CiliumIdentities" displayName: "Verify Cilium Identities Deletion" - - script: | + - script: | # TODO REMOVE THIS STEP, make test-load covers this + set -e echo "validate pod IP assignment before CNS restart" kubectl get pod -owide -A make test-validate-state diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 53e7544a6..528428554 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -64,4 +64,8 @@ steps: name: "aziliumTest" displayName: "Run Azilium E2E" - - template: ../../templates/cilium-tests.yaml \ No newline at end of file + - template: ../../templates/cilium-tests.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + scaleup: ${{ parameters.scaleup }} + diff --git a/.pipelines/singletenancy/cilium/cilium-e2e.steps.yaml b/.pipelines/singletenancy/cilium/cilium-e2e.steps.yaml index 9f8650efd..12e6f4e2a 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e.steps.yaml @@ -93,6 +93,7 @@ steps: workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -107,6 +108,7 @@ steps: retryCountOnTaskFailure: 3 - script: | + set -e echo "Run Cilium Connectivity Tests" cilium status cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' --force-deploy diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 9dcbe186d..dc9dc3f01 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -77,6 +77,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -134,6 +135,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.steps.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.steps.yaml index 7630f0c8b..7764b08f0 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.steps.yaml @@ -82,6 +82,7 @@ steps: workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -143,6 +144,7 @@ steps: workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS diff --git a/.pipelines/templates/cilium-cli.steps.yaml b/.pipelines/templates/cilium-cli.steps.yaml index 0da16ede7..a6d6f7d15 100644 --- a/.pipelines/templates/cilium-cli.steps.yaml +++ b/.pipelines/templates/cilium-cli.steps.yaml @@ -1,5 +1,6 @@ steps: - script: | + set -e echo "install cilium CLI" if [[ ${CILIUM_VERSION_TAG#v} =~ ^1.1[1-3].[0-9]{1,2}|1.1[1-3].[0-9]{1,2}-[0-9]{1,6} ]]; then echo "Cilium Agent Version ${BASH_REMATCH[0]}" diff --git a/.pipelines/templates/cilium-cli.yaml b/.pipelines/templates/cilium-cli.yaml index a237e4a07..2c3034365 100644 --- a/.pipelines/templates/cilium-cli.yaml +++ b/.pipelines/templates/cilium-cli.yaml @@ -1,5 +1,6 @@ steps: - script: | + set -e echo "install cilium CLI" if [[ ${CILIUM_VERSION_TAG#v} =~ ^1.1[1-3].[0-9]{1,2}|1.1[1-3].[0-9]{1,2}-[0-9]{1,6} ]]; then echo "Cilium Agent Version ${BASH_REMATCH[0]}" @@ -17,6 +18,9 @@ steps: sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + + # We can ignore failures from cilium status as cilium agent will not be in ready status until CNS is installed. + set +e cilium status cilium version name: "installCiliumCLI" diff --git a/.pipelines/templates/cilium-tests.yaml b/.pipelines/templates/cilium-tests.yaml index e46015286..d43743b5c 100644 --- a/.pipelines/templates/cilium-tests.yaml +++ b/.pipelines/templates/cilium-tests.yaml @@ -31,6 +31,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS @@ -45,6 +46,7 @@ steps: retryCountOnTaskFailure: 3 - script: | + set -e echo "Run Cilium Connectivity Tests" cilium status cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' --force-deploy @@ -55,13 +57,11 @@ steps: displayName: "Run Cilium Connectivity Tests" - script: | + set -e echo "validate pod IP assignment and check systemd-networkd restart" kubectl get pod -owide -A - # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. - # Saves 17 minutes - kubectl delete deploy -n $(ciliumNamespace) echo-external-node make test-validate-state - echo "delete cilium connectivity test resources and re-validate state" + echo "delete cilium connectivity test resources and re-validate state" # TODO Delete this and the next 4 lines if connectivity no longer has bug kubectl delete ns $(ciliumNamespace) kubectl get pod -owide -A make test-validate-state @@ -83,4 +83,4 @@ steps: kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]' fi name: "testAsyncDelete" - displayName: "Verify Async Delete when CNS is down" \ No newline at end of file + displayName: "Verify Async Delete when CNS is down"