ci: Cleanup Dual OS ACN PR pipeline step templates (#2482)

* ci: cleanup dual OS templates

* ci: add dynamic windows vmsku to overlay

* fix: overlay dependson

* revert: test-load changes

* ci: enable multple OS log template in same stage

.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml

@@ -43,9 +43,8 @@ stages:
     pool:
       name: $(BUILD_POOL_NAME_DEFAULT)
     jobs:
-      - job: ${{ parameters.name }}
-        displayName: Azure CNI Overlay Test Suite - (${{ parameters.name }})
-        timeoutInMinutes: 120
+      - job: ${{ parameters.name }}_linux
+        displayName: Azure CNI Overlay Test Suite | Linux - (${{ parameters.name }})
         pool:
           name: $(BUILD_POOL_NAME_DEFAULT)
           demands:
@@ -56,13 +55,31 @@ stages:
             parameters:
               name: ${{ parameters.name }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: linux
+
+      - job: ${{ parameters.name }}_windows
+        displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }})
+        dependsOn: ${{ parameters.name }}_linux
+        pool:
+          name: $(BUILD_POOL_NAME_DEFAULT)
+          demands:
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        steps:
+          - template: azure-cni-overlay-e2e-step-template.yaml
+            parameters:
+              name: ${{ parameters.name }}
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: windows
+              vmSizeWin: ${{ parameters.vmSize }} # Matching linux vmSize
 
       - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
         parameters:
           sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
           clusterName: ${{ parameters.clusterName }}-$(commitID)
           os: ${{ parameters.os }}
-          dependsOn: ${{ parameters.name }}
+          dependsOn: ${{ parameters.name }}_windows
+          datapath: true
           dns: true
           portforward: true
           hostport: true
@@ -74,24 +91,36 @@ stages:
           clusterName: ${{ parameters.clusterName }}-$(commitID)
           os: windows
           dependsOn: cni_${{ parameters.os }}
+          datapath: true
           dns: true
           portforward: true
           hostport: true
           service: true
           hybridWin: true
-          datapath: true
 
-      - job: failedE2ELogs
-        displayName: "Failure Logs"
+      - job: failedE2ELogs_linux
+        displayName: "Linux Failure Logs"
         dependsOn:
-          - ${{ parameters.name }}
-          - cni_${{ parameters.os }}
-          - cni_windows
-        condition: failed()
+          - ${{ parameters.name }}_linux
+          - cni_linux
+        condition: in(dependencies.${{ parameters.name }}_linux.result, 'Failed')
         steps:
           - template: ../../templates/log-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
-              os: ${{ parameters.os }}
+              os: linux
+              cni: cniv2
+
+      - job: failedE2ELogs_windows
+        displayName: "Windows Failure Logs"
+        dependsOn:
+          - ${{ parameters.name }}_windows
+          - cni_windows
+        condition: in(dependencies.${{ parameters.name }}_windows.result, 'Failed')
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: windows
               cni: cniv2
 

.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml

@@ -1,15 +1,9 @@
 parameters:
   name: ""
   clusterName: ""
+  os: ""
 
 steps:
-  - bash: |
-      echo $UID
-      sudo rm -rf $(System.DefaultWorkingDirectory)/*
-    displayName: "Set up OS environment"
-
-  - checkout: self
-
   - bash: |
       go version
       go env
@@ -36,130 +30,102 @@ steps:
         make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
     name: "kubeconfig"
     displayName: "Set Kubeconfig"
-  
-  - script: |
-      echo "Start Integration Tests on Overlay Cluster"
-      sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true
-    retryCountOnTaskFailure: 3
-    name: "integrationTest"
-    displayName: "Run CNS Integration Tests on AKS Overlay"
 
-  - script: |
-      echo "Status of the nodes and pods after the test"
-      kubectl get nodes -o wide
-      kubectl get pods -A -o wide
-      echo "Logs will be available as a build artifact"
-      ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
-      echo $ARTIFACT_DIR
-      sudo rm -rf $ARTIFACT_DIR
-      sudo mkdir $ARTIFACT_DIR
-      sudo cp test/integration/logs/* $ARTIFACT_DIR
-    name: "GetLogs"
-    displayName: "Get logs"
-    condition: always()
+  - ${{ if eq(parameters.os, 'linux') }}:
+    - script: |
+        echo "Start Integration Tests on Overlay Cluster"
+        sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true
+      retryCountOnTaskFailure: 2
+      name: "integrationTest"
+      displayName: "Run CNS Integration Tests on AKS Overlay"
 
-  - task: AzureCLI@1
-    inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-      scriptLocation: "inlineScript"
-      scriptType: "bash"
-      addSpnToEnvironment: true
-      inlineScript: |
-        set -e
-        clusterName=${{ parameters.clusterName }}
-        echo "Restarting nodes"
-        for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
-          make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
-        done
-    displayName: "Restart Nodes"
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          set -e
+          clusterName=${{ parameters.clusterName }}
+          echo "Restarting nodes"
+          for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
+            make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
+          done
+      displayName: "Restart Nodes"
 
-  - task: AzureCLI@1
-    inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-      scriptLocation: "inlineScript"
-      scriptType: "bash"
-      addSpnToEnvironment: true
-      inlineScript: |
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          cd test/integration/load
+          clusterName=${{ parameters.clusterName }}
+          make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
+          make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
+          kubectl get pods -owide -A
+          echo "Validating Node Restart"
+          CNI_TYPE=cniv2 RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
+      displayName: "Validate Node Restart"
+      retryCountOnTaskFailure: 3
+
+    - script: |
+        echo "validate pod IP assignment before CNS restart"
+        kubectl get pod -owide -A
+        echo "validate pod state before CNS restarts"
         cd test/integration/load
-        clusterName=${{ parameters.clusterName }}
-        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
-        make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
-        kubectl get pods -owide -A
-        echo "Validating Node Restart"
-        CNI_TYPE=cniv2 RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
-    displayName: "Validate Node Restart"
-    retryCountOnTaskFailure: 3
+        CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
+        kubectl rollout restart ds azure-cns -n kube-system
+        kubectl rollout status ds azure-cns -n kube-system
+        kubectl get pod -owide -A
+        echo "validate pods after CNS restart"
+        CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
+      name: "restartCNS_ValidatePodState"
+      displayName: "Restart CNS and validate pod state"
+      retryCountOnTaskFailure: 3
 
-  - script: |
-      echo "validate pod IP assignment before CNS restart"
-      kubectl get pod -owide -A
-      echo "validate pod state before CNS restarts"
-      cd test/integration/load
-      CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
-      kubectl rollout restart ds azure-cns -n kube-system
-      kubectl rollout status ds azure-cns -n kube-system
-      kubectl get pod -owide -A
-      echo "validate pods after CNS restart"
-      CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
-    name: "restartCNS_ValidatePodState"
-    displayName: "Restart CNS and validate pod state"
-    retryCountOnTaskFailure: 3
+    - script: |
+        echo "Run wireserver and metadata connectivity Tests"
+        bash test/network/wireserver_metadata_test.sh
+      retryCountOnTaskFailure: 3
+      name: "WireserverMetadataConnectivityTests"
+      displayName: "Run Wireserver and Metadata Connectivity Tests"
 
-  - task: PublishBuildArtifacts@1
-    inputs:
-      artifactName: test-output
-      pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output"
-    condition: always()
+  - ${{ if eq(parameters.os, 'windows') }}:
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }}
+          echo "Windows node are successfully added to v4 Overlay Cluster"
+          kubectl cluster-info
+          kubectl get node -owide
+          kubectl get po -owide -A
+      name: "Add_Windows_Node"
+      displayName: "Add windows node on v4 overlay cluster"
 
-  - script: |
-      echo "Run wireserver and metadata connectivity Tests"
-      bash test/network/wireserver_metadata_test.sh
-    retryCountOnTaskFailure: 3
-    name: "WireserverMetadataConnectivityTests"
-    displayName: "Run Wireserver and Metadata Connectivity Tests"
+    - script: |
+        nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
+        for node in $nodeList; do
+            taint=`kubectl describe node $node | grep Taints | awk '{print $2}'`
+            if [ $taint == "node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule" ]; then
+                kubectl taint nodes $node node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule-
+            fi
+        done
+        sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=windows CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true VALIDATE_V4OVERLAY=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true
+      name: "WindowsOverlayControlPlaneScaleTests"
+      displayName: "Windows v4Overlay ControlPlane Scale Tests"
+      retryCountOnTaskFailure: 2
 
-  - script: |
-      ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
-      echo $ARTIFACT_DIR
-      sudo rm -rf $ARTIFACT_DIR
-      sudo rm -rf test/integration/logs
-    name: "Cleanupartifactdir"
-    displayName: "Cleanup artifact dir"
-    condition: always()
-
-  - task: AzureCLI@2
-    inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-      scriptLocation: "inlineScript"
-      scriptType: "bash"
-      addSpnToEnvironment: true
-      inlineScript: |
-        set -e
-        make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}
-        echo "Windows node are successfully added to v4 Overlay Cluster"
-        kubectl cluster-info
-        kubectl get node -owide
-        kubectl get po -owide -A
-    name: "Add_Windows_Node"
-    displayName: "Add windows node on v4 overlay cluster"
-
-  - script: |
-      nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
-      for node in $nodeList; do
-          taint=`kubectl describe node $node | grep Taints | awk '{print $2}'`
-          if [ $taint == "node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule" ]; then
-              kubectl taint nodes $node node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule-
-          fi
-      done
-      sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=windows CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true VALIDATE_V4OVERLAY=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true
-    name: "WindowsOverlayControlPlaneScaleTests"
-    displayName: "Windows v4Overlay ControlPlane Scale Tests"
-    retryCountOnTaskFailure: 3
-
-  - script: |
-      echo "IPv4 Overlay DataPath Test"
-      cd test/integration/datapath
-      sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$
-    name: "WindowsV4OverlayDatapathTests"
-    displayName: "Windows v4Overlay Datapath Tests"
-    retryCountOnTaskFailure: 3
+    - script: |
+        echo "IPv4 Overlay DataPath Test"
+        cd test/integration/datapath
+        sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$
+      name: "WindowsV4OverlayDatapathTests"
+      displayName: "Windows v4Overlay Datapath Tests"
+      retryCountOnTaskFailure: 3

.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml

@@ -44,8 +44,8 @@ stages:
     pool:
       name: $(BUILD_POOL_NAME_DEFAULT)
     jobs:
-      - job: ${{ parameters.name }}
-        displayName: DualStack Overlay Test Suite - (${{ parameters.name }})
+      - job: ${{ parameters.name }}_linux
+        displayName: DualStack Overlay Test Suite | Linux - (${{ parameters.name }})
         pool:
           name: $(BUILD_POOL_NAME_DEFAULT)
           demands:
@@ -56,13 +56,30 @@ stages:
             parameters:
               name: ${{ parameters.name }}
               clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: linux
+
+      - job: ${{ parameters.name }}_windows
+        displayName: DualStack Overlay Test Suite | Windows - (${{ parameters.name }})
+        dependsOn: ${{ parameters.name }}_linux
+        pool:
+          name: $(BUILD_POOL_NAME_DEFAULT)
+          demands:
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        steps:
+          - template: dualstackoverlay-e2e-step-template.yaml
+            parameters:
+              name: ${{ parameters.name }}
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: windows
+              vmSizeWin: ${{ parameters.vmSize }} # Matching linux vmSize
 
       - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
         parameters:
           sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
           clusterName: ${{ parameters.clusterName }}-$(commitID)
-          os: ${{ parameters.os }}
-          dependsOn: ${{ parameters.name }}
+          os: linux
+          dependsOn: ${{ parameters.name }}_windows
           dualstack: true
           dns: true
           portforward: true
@@ -74,7 +91,7 @@ stages:
           sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
           clusterName: ${{ parameters.clusterName }}-$(commitID)
           os: windows
-          dependsOn: cni_${{ parameters.os }}
+          dependsOn: cni_linux
           dualstack: true
           dns: true
           portforward: true
@@ -82,16 +99,28 @@ stages:
           hostport: true
           hybridWin: true
 
-      - job: failedE2ELogs
-        displayName: "Failure Logs"
+      - job: failedE2ELogs_linux
+        displayName: "Linux Failure Logs"
         dependsOn:
-          - ${{ parameters.name }}
-          - cni_${{ parameters.os }}
-          - cni_windows
-        condition: failed()
+          - ${{ parameters.name }}_linux
+          - cni_linux
+        condition: in(dependencies.${{ parameters.name }}_linux.result, 'Failed')
         steps:
           - template: ../../templates/log-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
-              os: ${{ parameters.os }}
+              os: linux
+              cni: cniv2
+
+      - job: failedE2ELogs_windows
+        displayName: "Windows Failure Logs"
+        dependsOn:
+          - ${{ parameters.name }}_windows
+          - cni_windows
+        condition: in(dependencies.${{ parameters.name }}_windows.result, 'Failed')
+        steps:
+          - template: ../../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: windows
               cni: cniv2

.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml

@@ -2,6 +2,7 @@ parameters:
   name: ""
   clusterName: ""
   cni: "dualstack"
+  os: ""
 
 steps:
   - bash: |
@@ -31,115 +32,107 @@ steps:
     name: "kubeconfig"
     displayName: "Set Kubeconfig"
 
-  - script: |
-      kubectl cluster-info
-      kubectl get node
-      sudo -E env "PATH=$PATH" make test-integration CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true
-      kubectl get po -owide -A
-    retryCountOnTaskFailure: 3
-    name: "integrationTest"
-    displayName: "Run CNS Integration Tests on AKS DualStack Overlay"
-
-  - script: |
-      set -e
-      cd test/integration/load
-      echo "DualStack Overlay Linux control plane Node properties test"
-      CNI_TYPE=dualstack go test -timeout 30m -tags load -run ^TestDualStackProperties$
-      echo "DualStack Overlay Linux control plane Load test"
-      go test -timeout 30m -tags load -run ^TestLoad$
-      echo "DualStack Overlay Linux control plane CNS validation test"
-      CNI_TYPE=dualstack go test -timeout 30m -tags load -run ^TestValidateState$
-      cd ../datapath
-      echo "Dualstack Overlay Linux datapath IPv6 test"
-      go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true
-      echo "Dualstack Overlay Linux datapath IPv4 test"
-      go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration
-      echo "cleaning up load-test namespace"
-      kubectl delete ns load-test
-    retryCountOnTaskFailure: 3
-    name: "DualStack_Overlay_Linux_Tests"
-    displayName: "DualStack Overlay Linux Tests"
-
-  - task: AzureCLI@1
-    inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-      scriptLocation: "inlineScript"
-      scriptType: "bash"
-      addSpnToEnvironment: true
-      inlineScript: |
-        set -e
-        clusterName=${{ parameters.clusterName }}
-        echo "Restarting nodes"
-        for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) --query "[].name" -o tsv); do
-          make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) VMSS_NAME=${val}
-        done
-    displayName: "Restart Nodes"
-
-  - task: AzureCLI@1
-    inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-      scriptLocation: "inlineScript"
-      scriptType: "bash"
-      addSpnToEnvironment: true
-      inlineScript: |
-        cd test/integration/load
-        clusterName=${{ parameters.clusterName }}
-        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
-        make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST)
-        kubectl get pods -A
-        echo "Validating Node Restart"
-        CNI_TYPE=dualstack RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
-    displayName: "Validate Node Restart"
-    retryCountOnTaskFailure: 3
-
-  - script: |
-      echo "validate pod IP assignment before CNS restart"
-      kubectl get pod -owide -A
-      echo "validate pod state before CNS restarts"
-      cd test/integration/load
-      CNI_TYPE=dualstack go test -timeout 30m -tags load -run ^TestValidateState$
-      kubectl rollout restart ds azure-cns -n kube-system
-      kubectl rollout status ds azure-cns -n kube-system
-      kubectl get pod -owide -A
-      echo "validate pods after CNS restarts"
-      CNI_TYPE=dualstack go test -timeout 30m -tags load -run ^TestValidateState$
-    name: "restartCNS_ValidatePodState"
-    displayName: "Restart CNS and Validate Pod State"
-    retryCountOnTaskFailure: 3
-
-  - task: AzureCLI@2
-    inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-      scriptLocation: "inlineScript"
-      scriptType: "bash"
-      addSpnToEnvironment: true
-      inlineScript: |
-        set -e
-        make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}
-        echo "Windows nodes have been successfully added to DualStack Overlay Cluster"
+  - ${{ if eq(parameters.os, 'linux') }}:
+    - script: |
         kubectl cluster-info
-        kubectl get node -owide
+        kubectl get node
+        sudo -E env "PATH=$PATH" make test-integration CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true
         kubectl get po -owide -A
-    name: "Add_Windows_Node"
-    displayName: "Add windows node"
+      retryCountOnTaskFailure: 3
+      name: "integrationTest"
+      displayName: "Run CNS Integration Tests on AKS DualStack Overlay"
 
-  - script: |
-      nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
-      for node in $nodeList; do
-          taint=`kubectl describe node $node | grep Taints | awk '{print $2}'`
-          if [ $taint == "node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule" ]; then
-              kubectl taint nodes $node node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule-
-          fi
-      done
-      sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=windows CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true
-    name: "WindowsDualStackOverlayControlPlaneScaleTests"
-    displayName: "Windows DualStack Overlay ControlPlane Scale Tests"
-    retryCountOnTaskFailure: 3
+    - script: |
+        set -e
+        cd test/integration/datapath
+        echo "Dualstack Overlay Linux datapath IPv6 test"
+        go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true
+        echo "Dualstack Overlay Linux datapath IPv4 test"
+        go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration
+      retryCountOnTaskFailure: 3
+      name: "DualStack_Overlay_Linux_Tests"
+      displayName: "DualStack Overlay Linux Tests"
 
-  - script: |
-      echo "DualStack Overlay DataPath Test"
-      cd test/integration/datapath
-      sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$
-    name: "WindowsDualStackOverlayDatapathTests"
-    displayName: "Windows DualStack Overlay Datapath Tests"
-    retryCountOnTaskFailure: 3
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          set -e
+          clusterName=${{ parameters.clusterName }}
+          echo "Restarting nodes"
+          for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) --query "[].name" -o tsv); do
+            make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) VMSS_NAME=${val}
+          done
+      displayName: "Restart Nodes"
+
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          cd test/integration/load
+          clusterName=${{ parameters.clusterName }}
+          make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
+          make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST)
+          kubectl get pods -A
+          echo "Validating Node Restart"
+          CNI_TYPE=dualstack RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
+      displayName: "Validate Node Restart"
+      retryCountOnTaskFailure: 3
+
+    - script: |
+        echo "validate pod IP assignment before CNS restart"
+        kubectl get pod -owide -A
+        echo "validate pod state before CNS restarts"
+        cd test/integration/load
+        CNI_TYPE=dualstack go test -timeout 30m -tags load -run ^TestValidateState$
+        kubectl rollout restart ds azure-cns -n kube-system
+        kubectl rollout status ds azure-cns -n kube-system
+        kubectl get pod -owide -A
+        echo "validate pods after CNS restarts"
+        CNI_TYPE=dualstack go test -timeout 30m -tags load -run ^TestValidateState$
+      name: "restartCNS_ValidatePodState"
+      displayName: "Restart CNS and Validate Pod State"
+      retryCountOnTaskFailure: 3
+
+  - ${{ if eq(parameters.os, 'windows') }}:
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }}
+          echo "Windows nodes have been successfully added to DualStack Overlay Cluster"
+          kubectl cluster-info
+          kubectl get node -owide
+          kubectl get po -owide -A
+      name: "Add_Windows_Node"
+      displayName: "Add windows node"
+
+    - script: |
+        nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
+        for node in $nodeList; do
+            taint=`kubectl describe node $node | grep Taints | awk '{print $2}'`
+            if [ $taint == "node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule" ]; then
+                kubectl taint nodes $node node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule-
+            fi
+        done
+        sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=windows CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true
+      name: "WindowsDualStackOverlayControlPlaneScaleTests"
+      displayName: "Windows DualStack Overlay ControlPlane Scale Tests"
+      retryCountOnTaskFailure: 3
+
+    - script: |
+        echo "DualStack Overlay DataPath Test"
+        cd test/integration/datapath
+        sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$
+      name: "WindowsDualStackOverlayDatapathTests"
+      displayName: "Windows DualStack Overlay Datapath Tests"
+      retryCountOnTaskFailure: 3

.pipelines/templates/log-template.yaml

@@ -20,7 +20,7 @@ parameters:
   logType: "failure"
   os: ""
   cni: ""
-  jobName: "failedE2ELogs"
+  jobName: "FailedE2ELogs"
 
 steps:
   - task: AzureCLI@1
@@ -255,14 +255,14 @@ steps:
         array=($podList)
 
         if [ -z ${array[0]} ]; then
-          echo There are no kube-system pods in a non-ready state.
+          echo "There are no kube-system pods in a non-ready state."
         else
-          mkdir -p $acnLogs/non-ready
-          echo Directory created: $acnLogs/non-ready
-          echo Capturing failed pods
+          mkdir -p $acnLogs/${{ parameters.os }}non-ready
+          echo "Directory created: $acnLogs/${{ parameters.os }}non-ready"
+          echo "Capturing failed pods"
           for pod in $podList; do
-            kubectl describe pod -n kube-system $pod > $acnLogs/non-ready/$pod.txt
-            echo $acnLogs/non-ready/$pod.txt
+            kubectl describe pod -n kube-system $pod > $acnLogs/${{ parameters.os }}non-ready/$pod.txt
+            echo "$acnLogs/${{ parameters.os }}non-ready/$pod.txt"
           done
         fi
       displayName: Failure Logs
@@ -270,6 +270,6 @@ steps:
 
   - publish: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt)
     condition: always()
-    artifact: ${{ parameters.clusterName }}_${{ parameters.jobName }}_Attempt_#$(System.StageAttempt)
+    artifact: ${{ parameters.clusterName }}_${{ parameters.os }}${{ parameters.jobName }}_Attempt_#$(System.StageAttempt)
     name: acnLogs_${{ parameters.logType }}
     displayName: Publish Cluster logs