pipelines: Adding a new stage for NPM continous integration (#1580)

* pipelines: Adding a new stage for NPM continous integration * pipelines: Adding a new stage for NPM continous integration * pipelines: Adding a new stage for NPM continous integration * removing download phase * removing download phase * remove v1 default
2022-09-07 14:19:19 -07:00 · 2022-09-07 14:19:19 -07:00 · 1c43a5cf37
--- a/.pipelines/npm/npm-conformance-tests-latest-release.yaml
+++ b/.pipelines/npm/npm-conformance-tests-latest-release.yaml
@ -0,0 +1,446 @@
+trigger:
+  - master
+
+variables:
+  - name: VNET_NAME
+    value: npm-vnet
+  - name: NUM_PARALLEL_JOBS_FOR_STRESS_TEST
+    value: "3"
+
+jobs:
+  - job: setup
+    displayName: "Configure Test Environment"
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+      demands:
+        - agent.os -equals Linux
+        - Role -equals Build
+    steps:
+      - checkout: self
+
+      - script: |
+          go version
+          go env
+          which go
+          echo $PATH
+          mkdir -p '$(GOBIN)'
+          mkdir -p '$(GOPATH)/pkg'
+          BUILD_NUMBER=$(Build.BuildNumber)
+          RG=e2e-$(echo "npm-`date "+%Y-%m-%d-%S"`")
+
+          git --version
+          TAG=$(git -c "versionsort.suffix=-" ls-remote --tags https://github.com/Azure/azure-container-networking  | grep -v "zapai" | grep -v "ipam" | grep -v "{}"| cut --delimiter="/" --fields=3 | sort -V | tail -1)
+          echo "Resource group: $RG"
+          echo "Image tag: $TAG"
+
+          echo "##vso[task.setvariable variable=RESOURCE_GROUP;isOutput=true;]$RG"
+          echo "##vso[task.setvariable variable=TAG;isOutput=true;]$TAG"
+
+        name: "EnvironmentalVariables"
+        displayName: "Set environmental variables"
+        condition: always()
+
+  - job: k8se2e
+    displayName: "Build Kubernetes Test Suite"
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+      demands:
+        - agent.os -equals Linux
+        - Role -equals Build
+    steps:
+      - checkout: self
+
+      - script: |
+          echo "cloning vakr"
+          git clone https://github.com/vakalapa/kubernetes.git
+
+        displayName: "Clone Kubernetes Repo"
+
+      - bash: |
+          cd kubernetes
+          git checkout vakr/sleepinnpmtests
+          export PATH=$PATH:/usr/local/go/bin/
+          make WHAT=test/e2e/e2e.test
+        displayName: "Build Kubernetes e2e.test"
+
+      - publish: $(System.DefaultWorkingDirectory)/kubernetes/_output/local/bin/linux/amd64
+        artifact: Test
+
+  - job: Create_Cluster_and_Run_Test
+    timeoutInMinutes: 360
+    displayName: "Run Kubernetes Network Policy Test Suite"
+    strategy:
+      matrix:
+        v2-default:
+          AZURE_CLUSTER: "conformance-v2-default"
+          PROFILE: "v2-default"
+          IS_STRESS_TEST: "false"
+        v2-default-ws22:
+          AZURE_CLUSTER: "conformance-v2-default-ws22"
+          PROFILE: "v2-default-ws22"
+          IS_STRESS_TEST: "false"
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+      demands:
+        - agent.os -equals Linux
+        - Role -equals Build
+    dependsOn: [setup, k8se2e]
+    variables:
+      RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
+      TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
+      FQDN: empty
+    steps:
+      - checkout: none
+      - download: current
+        artifact: Test
+
+      - task: AzureCLI@2
+        displayName: "Create resource group"
+        inputs:
+          azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          scriptType: "bash"
+          scriptLocation: "inlineScript"
+          inlineScript: |
+            az group create -n $(RESOURCE_GROUP) -l $(LOCATION) -o table
+            echo created RG $(RESOURCE_GROUP) in $(LOCATION)
+            az version
+
+      - task: AzureCLI@2
+        displayName: "Deploy NPM to Test Cluster"
+        inputs:
+          azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          scriptType: "bash"
+          scriptLocation: "inlineScript"
+          failOnStderr: true
+          inlineScript: |
+            # get kubectl 
+            curl -LO https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl
+            chmod +x kubectl
+            echo Cluster $(AZURE_CLUSTER)
+            echo Resource $(RESOURCE_GROUP)
+
+            if [[ $(AZURE_CLUSTER) == *ws22 ]] # * is used for pattern matching
+            then
+              az extension add --name aks-preview
+              az extension update --name aks-preview
+
+              # Enable Microsoft.ContainerService/AKSWindows2022Preview
+              az feature register --namespace Microsoft.ContainerService --name AKSWindows2022Preview
+              az provider register -n Microsoft.ContainerService
+
+              echo "creating WS22 Cluster";
+              az aks create \
+                  --resource-group $(RESOURCE_GROUP) \
+                  --name $(AZURE_CLUSTER) \
+                  --generate-ssh-keys \
+                  --windows-admin-username e2eadmin \
+                  --windows-admin-password alpha@numeric!password2 \
+                  --network-plugin azure \
+                  --vm-set-type VirtualMachineScaleSets \
+                  --kubernetes-version v1.23.5 \
+                  --node-vm-size Standard_D4s_v3 \
+                  --node-count 1
+
+              # don't schedule anything on the linux system pool
+              echo "Updating $(AZURE_CLUSTER) to not schedule anything on linux pool..."
+              az aks nodepool update \
+                --cluster-name $(AZURE_CLUSTER) \
+                -g $(RESOURCE_GROUP) \
+                -n nodepool1 \
+                --node-taints CriticalAddonsOnly=true:NoSchedule
+
+              echo "Adding Windows nodepool to $(AZURE_CLUSTER) to group $(RESOURCE_GROUP)"
+              az aks nodepool add \
+                  --resource-group $(RESOURCE_GROUP) \
+                  --cluster-name $(AZURE_CLUSTER) \
+                  --name awin22 \
+                  --os-type Windows \
+                  --os-sku Windows2022 \
+                  --node-vm-size Standard_D4s_v3 \
+                  --node-count 2
+
+              az aks get-credentials -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --file ./kubeconfig
+              ./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/examples/windows/azure-npm.yaml
+              ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-$(TAG)
+
+            else
+              echo "Creating Linux Cluster"; 
+              az aks create --no-ssh-key \
+              --resource-group $(RESOURCE_GROUP) \
+              --name $(AZURE_CLUSTER) \
+              --network-plugin azure
+
+              az aks get-credentials -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --file ./kubeconfig
+
+              # deploy azure-npm
+              ./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/azure-npm.yaml
+
+              # swap azure-npm image with one built during run
+              ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:linux-amd64-$(TAG)
+
+              # swap NPM profile with one specified as parameter
+              ./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/profiles/$(PROFILE).yaml
+              ./kubectl --kubeconfig=./kubeconfig rollout restart ds azure-npm -n kube-system
+            fi
+
+            ./kubectl --kubeconfig=./kubeconfig describe daemonset azure-npm -n kube-system
+
+            FQDN=`az aks show -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --query fqdn -o tsv`
+            echo "##vso[task.setvariable variable=FQDN]$FQDN"
+      - bash: |
+          echo "sleeping 3 minutes to allow NPM pods to restart"
+          sleep 180
+          set -o pipefail
+
+          ## create the output folder and include the kubeconfig there
+          npmLogsFolder=$(System.DefaultWorkingDirectory)/npmLogs_$(AZURE_CLUSTER)
+          mkdir -p $npmLogsFolder
+          cp ./kubeconfig $npmLogsFolder/kubeconfig
+
+          ## write to all NPM pod logs in the background (do this in the background instead of after to make sure the logs aren't truncated)
+          npmPodList=`kubectl --kubeconfig=./kubeconfig get pods -n kube-system | grep npm | awk '{print $1}'`
+          echo "Found NPM pods: $npmPodList"
+
+
+          ## Run all Conformance tests in the background
+          echo $FQDN
+          chmod +x $(Pipeline.Workspace)/Test/e2e.test
+          runConformance () {
+              KUBERNETES_SERVICE_HOST="$FQDN" KUBERNETES_SERVICE_PORT=443 $(Pipeline.Workspace)/Test/e2e.test --provider=local --ginkgo.focus="NetworkPolicy" --ginkgo.skip="SCTP" --kubeconfig=./kubeconfig
+              # there can't be a command after e2e.test because the exit code is important
+          }
+
+          runConformanceWindows () {
+            git clone https://github.com/huntergregory/kubernetes.git
+            cd kubernetes
+            git checkout sleep-before-probing
+            export PATH=$PATH:/usr/local/go/bin/
+            make WHAT=test/e2e/e2e.test
+            cd ..
+
+            nomatch1="should enforce policy based on PodSelector or NamespaceSelector"
+            nomatch2="should enforce policy based on NamespaceSelector with MatchExpressions using default ns label"
+            nomatch3="should enforce policy based on PodSelector and NamespaceSelector"
+            nomatch4="should enforce policy based on Multiple PodSelectors and NamespaceSelectors"
+            cidrExcept1="should ensure an IP overlapping both IPBlock.CIDR and IPBlock.Except is allowed"
+            cidrExcept2="should enforce except clause while egress access to server in CIDR block"
+            namedPorts="named port"
+            wrongK8sVersion="Netpol API"
+            toSkip="\[LinuxOnly\]|$nomatch1|$nomatch2|$nomatch3|$nomatch4|$cidrExcept1|$cidrExcept2|$namedPorts|$wrongK8sVersion|SCTP"
+            KUBERNETES_SERVICE_HOST="$FQDN" KUBERNETES_SERVICE_PORT=443 $(System.DefaultWorkingDirectory)/kubernetes/_output/local/bin/linux/amd64/e2e.test --provider=local --ginkgo.focus="NetworkPolicy" --ginkgo.skip="$toSkip"  --node-os-distro=windows --allowed-not-ready-nodes=1 --kubeconfig=./kubeconfig
+          }
+
+          exitCode=0
+          if [ $(IS_STRESS_TEST) == "true" ]; then
+              echo "Running $NUM_PARALLEL_JOBS_FOR_STRESS_TEST conformance tests at once and writing outputs to files"
+              declare -a conformancePIDs
+              for round in $(seq 1 $NUM_PARALLEL_JOBS_FOR_STRESS_TEST); do
+                  # for each iteration, run the conformance test and echos in the background, and write the output of the conformance test to a file
+                  
+                   # run the conformance test in the foreground and write the output to stdout and a file
+                  if [[ $(AZURE_CLUSTER) == *ws22 ]] # * is used for pattern matching
+                  then
+                    echo "starting conformance test #$round" && \
+                        runConformanceWindows | tee $npmLogsFolder/conformance-results-$round && \
+                        echo "finished conformance test #$round" &
+                    pidOfConformanceTest=$!
+                    conformancePIDs+=($pidOfConformanceTest)
+                  else
+                    echo "starting windows conformance test #$round" && \
+                        runConformance > $npmLogsFolder/conformance-results-$round && \
+                        echo "finished conformance test #$round" &
+                    pidOfConformanceTest=$!
+                    conformancePIDs+=($pidOfConformanceTest)
+                  fi
+              done
+
+              # wait until all conformance tests finish and take note of any failed tests
+              for round in $(seq 1 $NUM_PARALLEL_JOBS_FOR_STRESS_TEST); do
+                  i=$((round-1))
+                  wait ${conformancePIDs[$i]}
+                  exitCode=$?
+                  if [ $exitCode != 0 ]; then
+                      echo "conformance test #$round failed"
+                      break
+                  fi
+              done
+          else
+              # run the conformance test in the foreground and write the output to stdout and a file
+            if [[ $(AZURE_CLUSTER) == *ws22 ]] # * is used for pattern matching
+            then
+              runConformanceWindows | tee $npmLogsFolder/conformance-results
+              exitCode=$?
+            else
+              runConformance | tee $npmLogsFolder/conformance-results
+              exitCode=$?
+            fi
+          fi
+          # kill the background processes (the logs) that have this process' pid (i.e. $$) as a parent
+
+          for npmPod in $npmPodList; do
+              ./kubectl --kubeconfig=./kubeconfig logs -n kube-system $npmPod > $npmLogsFolder/$npmPod-logs.txt
+          done
+
+          exit $exitCode
+        displayName: "Run Test Suite and Get Logs"
+        failOnStderr: false
+
+      - publish: $(System.DefaultWorkingDirectory)/npmLogs_$(AZURE_CLUSTER)
+        condition: always()
+        artifact: NpmLogs_$(AZURE_CLUSTER)
+
+  - job: Create_Windows_Cluster_and_Run_Test
+    timeoutInMinutes: 360
+    displayName: "Run Windows Cyclonus"
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+      demands:
+        - agent.os -equals Linux
+        - Role -equals Build
+    dependsOn: [setup]
+    variables:
+      RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
+      TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
+      FQDN: empty
+    strategy:
+      matrix:
+        v2-windows:
+          PROFILE: "cyc-ws22"
+    steps:
+      - checkout: self
+      - download: none
+
+      - task: AzureCLI@2
+        displayName: "Create AKS Cluster"
+        inputs:
+          azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          scriptType: "bash"
+          scriptLocation: "inlineScript"
+          failOnStderr: true
+          inlineScript: |
+            az extension add --name aks-preview
+            az extension update --name aks-preview
+
+            # Enable Microsoft.ContainerService/AKSWindows2022Preview
+            az feature register --namespace Microsoft.ContainerService --name AKSWindows2022Preview
+            az provider register -n Microsoft.ContainerService
+
+            export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
+
+            echo "Creating resource group named $CLUSTER_NAME"
+            az group create --name $CLUSTER_NAME -l $(LOCATION) -o table
+
+            echo "Creating resource group named $CLUSTER_NAME"
+            az aks create \
+                --resource-group $CLUSTER_NAME \
+                --name $CLUSTER_NAME \
+                --generate-ssh-keys \
+                --windows-admin-username e2eadmin \
+                --windows-admin-password alpha@numeric!password2 \
+                --network-plugin azure \
+                --vm-set-type VirtualMachineScaleSets \
+                --node-vm-size Standard_D4s_v3 \
+                --kubernetes-version v1.23.5 \
+                --node-count 1
+
+            # don't schedule anything on the linux system pool
+            echo "Updating $CLUSTER_NAME to not schedule anything on linux pool..."
+            az aks nodepool update \
+              --cluster-name $CLUSTER_NAME \
+              -g $CLUSTER_NAME \
+              -n nodepool1 \
+              --node-taints CriticalAddonsOnly=true:NoSchedule
+
+            echo "Adding Windows nodepool to $CLUSTER_NAME"
+            az aks nodepool add \
+                --resource-group $CLUSTER_NAME \
+                --cluster-name $CLUSTER_NAME \
+                --name awin22 \
+                --os-type Windows \
+                --os-sku Windows2022 \
+                --node-vm-size Standard_D4s_v3 \
+                --node-count 3
+
+            echo "Getting credentials to $CLUSTER_NAME"
+            az aks get-credentials -g $CLUSTER_NAME -n $CLUSTER_NAME --overwrite-existing --file ./kubeconfig
+            mkdir -p ~/.kube/
+            cp ./kubeconfig ~/.kube/config
+
+      - task: AzureCLI@2
+        displayName: "Deploy NPM to Test Cluster"
+        inputs:
+          azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          scriptType: "bash"
+          scriptLocation: "inlineScript"
+          failOnStderr: true
+          inlineScript: |
+            export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
+
+            curl -LO https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl
+            chmod +x kubectl
+
+            # deploy azure-npm
+            ./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/examples/windows/azure-npm.yaml
+
+            # swap azure-npm image with one built during run
+            ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-$(TAG)
+
+            echo "sleeping 3 minutes to allow NPM pods to restart"
+            sleep 180
+
+            ./kubectl --kubeconfig=./kubeconfig get po -n kube-system -owide -A
+
+            echo "Showing cluster status for $CLUSTER_NAME"
+            FQDN=`az aks show -n $CLUSTER_NAME -g $CLUSTER_NAME --query fqdn -o tsv`
+            echo "##vso[task.setvariable variable=FQDN]$FQDN"
+
+      - script: |
+          cat ~/.kube/config
+          curl -fsSL github.com/mattfenwick/cyclonus/releases/latest/download/cyclonus_linux_amd64.tar.gz | tar -zxv
+        name: download_cyclonus
+        displayName: "Download Cyclonus"
+        failOnStderr: false
+        condition: always()
+
+      - script: |
+          ./test/cyclonus/test-cyclonus-windows.sh
+        name: cyclonus
+        displayName: "Run Cyclonus Test"
+        failOnStderr: false
+        condition: always()
+
+      - bash: |
+          export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
+          cp cyclonus-$CLUSTER_NAME $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/cyclonus-$CLUSTER_NAME
+          echo "Getting cluster state for $CLUSTER_NAME"
+          mkdir -p $(System.DefaultWorkingDirectory)/$CLUSTER_NAME
+          kubectl logs -n kube-system -l k8s-app=azure-npm --tail -1 --prefix > $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/npm-logs_$(PROFILE).txt
+          cp ./kubeconfig $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/.kubeconfig
+        condition: always()
+
+      - publish: $(System.DefaultWorkingDirectory)/$(RESOURCE_GROUP)-$(PROFILE)
+        condition: always()
+        artifact: NpmLogs-$(RESOURCE_GROUP)-$(PROFILE)
+
+  - job: clean_up
+    displayName: "Cleanup"
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+      demands:
+        - agent.os -equals Linux
+        - Role -equals Build
+    dependsOn:
+      [Create_Cluster_and_Run_Test, Create_Windows_Cluster_and_Run_Test, setup]
+    variables:
+      RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
+    steps:
+      - checkout: none
+      - task: AzureCLI@2
+        displayName: "Delete Test Cluster Resource Group"
+        inputs:
+          azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+          scriptType: "bash"
+          scriptLocation: "inlineScript"
+          inlineScript: |
+            echo Deleting $(RESOURCE_GROUP)
+            az group delete -n $(RESOURCE_GROUP) --yes