Azure
/
azure-container-networking
зеркало из https://github.com/Azure/azure-container-networking.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Add Cyclonus Windows job (#1363) 

* add cyclonus windows job

* control plane jobs

* add cyclonus to conformance

* run cyclonus from pipeline vm

* containerize

* cluster name

* rg name

* shorter rg

* k8s version

* update wait timeout

* logging

* get kubeconfigs

* date

* kubeconfig

* shorter name

* job timeout

* bigger vm

* increase vm size

* kubernetes version

* feature preview register

* increase node size

* exit code

* failonstderr

* move windows specifics closer

* omit failstderr

* wait for npm pods to restart

* update conformance scope

* retry

* seperate conformance for platforms

* revert branch

* hunter's branch
This commit is contained in:
Mathew Merrick 2022-08-08 17:43:44 +00:00 коммит произвёл GitHub
Родитель 3a854f5766
Коммит ef95e7d9f4
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
7 изменённых файлов: 568 добавлений и 252 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

686
.pipelines/npm/npm-conformance-tests.yaml
Просмотреть файл

@ -1,270 +1,486 @@
trigger:
- master
- master
variables:
- name: VNET_NAME
value: npm-vnet
- name: NUM_PARALLEL_JOBS_FOR_STRESS_TEST
value: "3"
- name: VNET_NAME
value: npm-vnet
- name: NUM_PARALLEL_JOBS_FOR_STRESS_TEST
value: "3"
jobs:
- job: setup
displayName: "Configure Test Environment"
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals Build
steps:
- job: setup
displayName: "Configure Test Environment"
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals Build
steps:
- checkout: self
- checkout: self
- script: |
go version
go env
which go
echo $PATH
mkdir -p '$(GOBIN)'
mkdir -p '$(GOPATH)/pkg'
BUILD_NUMBER=$(Build.BuildNumber)
RG=e2e-$(echo "npm-`date "+%Y-%m-%d-%S"`")
TAG=$(make version)-conformance-test
echo "Resource group: $RG"
echo "Image tag: $TAG"
- script: |
go version
go env
which go
echo $PATH
mkdir -p '$(GOBIN)'
mkdir -p '$(GOPATH)/pkg'
BUILD_NUMBER=$(Build.BuildNumber)
RG=kubernetes-$(echo "npm-conformance-`date "+%Y-%m-%d-%S"`")
TAG=$(make version)-conformance-test
echo "Resource group: $RG"
echo "Image tag: $TAG"
echo "##vso[task.setvariable variable=RESOURCE_GROUP;isOutput=true;]$RG"
echo "##vso[task.setvariable variable=TAG;isOutput=true;]$TAG"
echo "##vso[task.setvariable variable=RESOURCE_GROUP;isOutput=true;]$RG"
echo "##vso[task.setvariable variable=TAG;isOutput=true;]$TAG"
name: "EnvironmentalVariables"
displayName: "Set environmental variables"
condition: always()
name: "EnvironmentalVariables"
displayName: "Set environmental variables"
condition: always()
- job: containerize
dependsOn: [setup]
displayName: Build Images
variables:
TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
pool:
name: "$(BUILD_POOL_NAME_DEFAULT)"
strategy:
matrix:
npm_linux_amd64:
arch: amd64
os: linux
name: npm
steps:
- template: ../containers/container-template.yaml
parameters:
name: $(name)
os: $(os)
arch: $(arch)
tag: $(TAG)
- job: Build_test
displayName: "Build NPM and Kubernetes Test Suite"
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals Build
dependsOn: [setup]
variables:
GOBIN: "$(GOPATH)/bin" # Go binaries path
GOPATH: "$(System.DefaultWorkingDirectory)/gopath" # Go workspace path
RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
- job: containerize_windows
dependsOn: [setup]
displayName: Build Images
variables:
TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
pool:
name: "$(BUILD_POOL_NAME_DEFAULT_WINDOWS)"
strategy:
matrix:
npm_windows_amd64:
arch: amd64
name: npm
steps:
- template: ../containers/container-template-windows.yaml
parameters:
name: $(name)
arch: $(arch)
tag: $(TAG)
steps:
- script: |
rm -rf ./*
displayName: "Set up OS environment"
- job: k8se2e
displayName: "Build Kubernetes Test Suite"
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals Build
steps:
- checkout: self
- checkout: self
- script: |
echo "cloning vakr"
git clone https://github.com/vakalapa/kubernetes.git
- task: Docker@2
displayName: Docker Login
inputs:
containerRegistry: $(ACR_SERVICE_CONNECTION)
command: 'login'
addPipelineData: false
displayName: "Clone Kubernetes Repo"
- script: |
echo Tag: $(TAG)
echo ResourceGroup: $(RESOURCE_GROUP)
make npm-image TAG=$(TAG)
make npm-image-push TAG=$(TAG)
displayName: 'Build and Push NPM Image'
- bash: |
cd kubernetes
git checkout vakr/sleepinnpmtests
export PATH=$PATH:/usr/local/go/bin/
make WHAT=test/e2e/e2e.test
displayName: "Build Kubernetes e2e.test"
- task: Docker@2
displayName: Docker Logout
inputs:
containerRegistry: $(ACR_SERVICE_CONNECTION)
command: 'logout'
addPipelineData: false
- publish: $(System.DefaultWorkingDirectory)/kubernetes/_output/local/bin/linux/amd64
artifact: Test
- script: git clone https://github.com/vakalapa/kubernetes.git
displayName: 'Clone Kubernetes Repo'
- bash: |
cd kubernetes
git checkout vakr/sleepinnpmtests
export PATH=$PATH:/usr/local/go/bin/
make WHAT=test/e2e/e2e.test
displayName: 'Build Kubernetes e2e.test'
- publish: $(System.DefaultWorkingDirectory)/kubernetes/_output/local/bin/linux/amd64
artifact: Test
- job: Create_Cluster_and_Run_Test
timeoutInMinutes: 360
displayName: "Run Kubernetes Network Policy Test Suite"
strategy:
matrix:
v1-default:
AZURE_CLUSTER: "conformance-v1-default"
PROFILE: "v1-default"
IS_STRESS_TEST: "false"
v2-default:
AZURE_CLUSTER: "conformance-v2-default"
PROFILE: "v2-default"
IS_STRESS_TEST: "false"
v2-default-ws22:
AZURE_CLUSTER: "conformance-v2-default-ws22"
PROFILE: "v2-default-ws22"
IS_STRESS_TEST: "false"
v2-default-stress:
AZURE_CLUSTER: "conformance-v2-default-stress"
PROFILE: "v2-default"
IS_STRESS_TEST: "true"
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals Build
dependsOn: [setup, k8se2e, containerize]
variables:
RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
FQDN: empty
steps:
- checkout: none
- download: current
artifact: Test
- job: Create_RG
displayName: "Create AKS Resource Group"
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals Build
dependsOn: [setup]
variables:
RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
steps:
- script: |
sudo rm -rf ./*
displayName: "Set up OS environment"
- task: AzureCLI@2
displayName: "Create resource group"
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptType: "bash"
scriptLocation: "inlineScript"
inlineScript: |
az group create -n $(RESOURCE_GROUP) -l $(LOCATION) -o table
echo created RG $(RESOURCE_GROUP) in $(LOCATION)
az version
- checkout: self
- task: AzureCLI@2
displayName: "Deploy NPM to Test Cluster"
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptType: "bash"
scriptLocation: "inlineScript"
failOnStderr: true
inlineScript: |
# get kubectl
curl -LO https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl
chmod +x kubectl
echo Cluster $(AZURE_CLUSTER)
echo Resource $(RESOURCE_GROUP)
- task: AzureCLI@2
displayName: "Deploy"
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptType: 'bash'
scriptLocation: 'inlineScript'
inlineScript: |
az group create -n $(RESOURCE_GROUP) -l $(LOCATION) -o table
echo created RG $(RESOURCE_GROUP) in $(LOCATION)
az version
if [[ $(AZURE_CLUSTER) == *ws22 ]] # * is used for pattern matching
then
az extension add --name aks-preview
az extension update --name aks-preview
- job: Create_Cluster_and_Run_Test
timeoutInMinutes: 360
displayName: "Run Kubernetes Network Policy Test Suite"
strategy:
matrix:
v1-default:
AZURE_CLUSTER: 'v1-default-cluster'
PROFILE: 'v1-default'
IS_STRESS_TEST: 'false'
v2-default:
AZURE_CLUSTER: 'v2-default-cluster'
PROFILE: 'v2-default'
IS_STRESS_TEST: 'false'
v2-default-stress:
AZURE_CLUSTER: 'v2-default-stress-cluster'
PROFILE: 'v2-default'
IS_STRESS_TEST: 'true'
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals Build
dependsOn: [Create_RG, Build_test, setup]
variables:
RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
FQDN: empty
steps:
- checkout: none
- download: current
artifact: Test
- task: AzureCLI@2
displayName: "Deploy NPM to Test Cluster"
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptType: 'bash'
scriptLocation: 'inlineScript'
inlineScript: |
az aks create --no-ssh-key \
--resource-group $(RESOURCE_GROUP) \
--name $(AZURE_CLUSTER) \
--network-plugin azure
# Enable Microsoft.ContainerService/AKSWindows2022Preview
az feature register --namespace Microsoft.ContainerService --name AKSWindows2022Preview
az provider register -n Microsoft.ContainerService
echo Cluster $(AZURE_CLUSTER)
echo Resource $(RESOURCE_GROUP)
echo "creating WS22 Cluster";
az aks create \
--resource-group $(RESOURCE_GROUP) \
--name $(AZURE_CLUSTER) \
--generate-ssh-keys \
--windows-admin-username e2eadmin \
--windows-admin-password alpha@numeric!password2 \
--network-plugin azure \
--vm-set-type VirtualMachineScaleSets \
--kubernetes-version v1.23 \
--node-vm-size Standard_D4s_v3 \
--node-count 1
az aks get-credentials -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --file ./kubeconfig
# get kubectl
curl -LO https://dl.k8s.io/release/v1.20.0/bin/linux/amd64/kubectl
chmod +x kubectl
# don't schedule anything on the linux system pool
echo "Updating $(AZURE_CLUSTER) to not schedule anything on linux pool..."
az aks nodepool update \
--cluster-name $(AZURE_CLUSTER) \
-g $(RESOURCE_GROUP) \
-n nodepool1 \
--node-taints CriticalAddonsOnly=true:NoSchedule
# deploy azure-npm
./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/azure-npm.yaml
echo "Adding Windows nodepool to $(AZURE_CLUSTER) to group $(RESOURCE_GROUP)"
az aks nodepool add \
--resource-group $(RESOURCE_GROUP) \
--cluster-name $(AZURE_CLUSTER) \
--name awin22 \
--os-type Windows \
--os-sku Windows2022 \
--node-vm-size Standard_D4s_v3 \
--node-count 2
# swap azure-npm image with one built during run
./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:$(TAG)
az aks get-credentials -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --file ./kubeconfig
./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/examples/windows/azure-npm.yaml
./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-$(TAG)
# swap NPM profile with one specified as parameter
./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/profiles/$(PROFILE).yaml
./kubectl --kubeconfig=./kubeconfig rollout restart ds azure-npm -n kube-system
else
echo "Creating Linux Cluster";
az aks create --no-ssh-key \
--resource-group $(RESOURCE_GROUP) \
--name $(AZURE_CLUSTER) \
--network-plugin azure
./kubectl --kubeconfig=./kubeconfig describe daemonset azure-npm -n kube-system
az aks get-credentials -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --file ./kubeconfig
FQDN=`az aks show -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --query fqdn -o tsv`
echo "##vso[task.setvariable variable=FQDN]$FQDN"
# deploy azure-npm
./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/azure-npm.yaml
- bash: |
echo "sleeping 3 minutes to allow NPM pods to restart"
sleep 180
# swap azure-npm image with one built during run
./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:linux-amd64-$(TAG)
## create the output folder and include the kubeconfig there
npmLogsFolder=$(System.DefaultWorkingDirectory)/npmLogs_$(AZURE_CLUSTER)
mkdir -p $npmLogsFolder
cp ./kubeconfig $npmLogsFolder/kubeconfig
# swap NPM profile with one specified as parameter
./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/profiles/$(PROFILE).yaml
./kubectl --kubeconfig=./kubeconfig rollout restart ds azure-npm -n kube-system
fi
## write to all NPM pod logs in the background (do this in the background instead of after to make sure the logs aren't truncated)
npmPodList=`kubectl --kubeconfig=./kubeconfig get pods -n kube-system | grep npm | awk '{print $1}'`
echo "Found NPM pods: $npmPodList"
for npmPod in $npmPodList; do
./kubectl --kubeconfig=./kubeconfig logs -n kube-system $npmPod -f > $npmLogsFolder/$npmPod-logs.txt &
done
./kubectl --kubeconfig=./kubeconfig describe daemonset azure-npm -n kube-system
## Run all Conformance tests in the background
echo $FQDN
chmod +x $(Pipeline.Workspace)/Test/e2e.test
runConformance () {
KUBERNETES_SERVICE_HOST="$FQDN" KUBERNETES_SERVICE_PORT=443 $(Pipeline.Workspace)/Test/e2e.test --provider=local --ginkgo.focus="NetworkPolicy" --ginkgo.skip="SCTP" --kubeconfig=./kubeconfig
# there can't be a command after e2e.test because the exit code is important
}
FQDN=`az aks show -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --query fqdn -o tsv`
echo "##vso[task.setvariable variable=FQDN]$FQDN"
- bash: |
echo "sleeping 3 minutes to allow NPM pods to restart"
sleep 180
set -o pipefail
exitCode=0
if [ $(IS_STRESS_TEST) == "true" ]; then
echo "Running $NUM_PARALLEL_JOBS_FOR_STRESS_TEST conformance tests at once and writing outputs to files"
declare -a conformancePIDs
for round in $(seq 1 $NUM_PARALLEL_JOBS_FOR_STRESS_TEST); do
# for each iteration, run the conformance test and echos in the background, and write the output of the conformance test to a file
echo "starting conformance test #$round" && \
runConformance > $npmLogsFolder/conformance-results-$round && \
echo "finished conformance test #$round" &
pidOfConformanceTest=$!
conformancePIDs+=($pidOfConformanceTest)
done
## create the output folder and include the kubeconfig there
npmLogsFolder=$(System.DefaultWorkingDirectory)/npmLogs_$(AZURE_CLUSTER)
mkdir -p $npmLogsFolder
cp ./kubeconfig $npmLogsFolder/kubeconfig
# wait until all conformance tests finish and take note of any failed tests
for round in $(seq 1 $NUM_PARALLEL_JOBS_FOR_STRESS_TEST); do
i=$((round-1))
wait ${conformancePIDs[$i]}
## write to all NPM pod logs in the background (do this in the background instead of after to make sure the logs aren't truncated)
npmPodList=`kubectl --kubeconfig=./kubeconfig get pods -n kube-system | grep npm | awk '{print $1}'`
echo "Found NPM pods: $npmPodList"
## Run all Conformance tests in the background
echo $FQDN
chmod +x $(Pipeline.Workspace)/Test/e2e.test
runConformance () {
KUBERNETES_SERVICE_HOST="$FQDN" KUBERNETES_SERVICE_PORT=443 $(Pipeline.Workspace)/Test/e2e.test --provider=local --ginkgo.focus="NetworkPolicy" --ginkgo.skip="SCTP" --kubeconfig=./kubeconfig
# there can't be a command after e2e.test because the exit code is important
}
runConformanceWindows () {
git clone https://github.com/huntergregory/kubernetes.git
cd kubernetes
git checkout sleep-before-probing
export PATH=$PATH:/usr/local/go/bin/
make WHAT=test/e2e/e2e.test
cd ..
nomatch1="should enforce policy based on PodSelector or NamespaceSelector"
nomatch2="should enforce policy based on NamespaceSelector with MatchExpressions using default ns label"
nomatch3="should enforce policy based on PodSelector and NamespaceSelector"
nomatch4="should enforce policy based on Multiple PodSelectors and NamespaceSelectors"
cidrExcept1="should ensure an IP overlapping both IPBlock.CIDR and IPBlock.Except is allowed"
cidrExcept2="should enforce except clause while egress access to server in CIDR block"
namedPorts="named port"
wrongK8sVersion="Netpol API"
toSkip="\[LinuxOnly\]|$nomatch1|$nomatch2|$nomatch3|$nomatch4|$cidrExcept1|$cidrExcept2|$namedPorts|$wrongK8sVersion|SCTP"
KUBERNETES_SERVICE_HOST="$FQDN" KUBERNETES_SERVICE_PORT=443 $(System.DefaultWorkingDirectory)/kubernetes/_output/local/bin/linux/amd64/e2e.test --provider=local --ginkgo.focus="NetworkPolicy" --ginkgo.skip="$toSkip" --node-os-distro=windows --allowed-not-ready-nodes=1 --kubeconfig=./kubeconfig
}
exitCode=0
if [ $(IS_STRESS_TEST) == "true" ]; then
echo "Running $NUM_PARALLEL_JOBS_FOR_STRESS_TEST conformance tests at once and writing outputs to files"
declare -a conformancePIDs
for round in $(seq 1 $NUM_PARALLEL_JOBS_FOR_STRESS_TEST); do
# for each iteration, run the conformance test and echos in the background, and write the output of the conformance test to a file
# run the conformance test in the foreground and write the output to stdout and a file
if [[ $(AZURE_CLUSTER) == *ws22 ]] # * is used for pattern matching
then
echo "starting conformance test #$round" && \
runConformanceWindows | tee $npmLogsFolder/conformance-results-$round && \
echo "finished conformance test #$round" &
pidOfConformanceTest=$!
conformancePIDs+=($pidOfConformanceTest)
else
echo "starting windows conformance test #$round" && \
runConformance > $npmLogsFolder/conformance-results-$round && \
echo "finished conformance test #$round" &
pidOfConformanceTest=$!
conformancePIDs+=($pidOfConformanceTest)
fi
done
# wait until all conformance tests finish and take note of any failed tests
for round in $(seq 1 $NUM_PARALLEL_JOBS_FOR_STRESS_TEST); do
i=$((round-1))
wait ${conformancePIDs[$i]}
exitCode=$?
if [ $exitCode != 0 ]; then
echo "conformance test #$round failed"
break
fi
done
else
# run the conformance test in the foreground and write the output to stdout and a file
if [[ $(AZURE_CLUSTER) == *ws22 ]] # * is used for pattern matching
then
runConformanceWindows | tee $npmLogsFolder/conformance-results
exitCode=$?
if [ $exitCode != 0 ]; then
echo "conformance test #$round failed"
break
fi
done
else
# run the conformance test in the foreground and write the output to stdout and a file
runConformance > $npmLogsFolder/conformance-results
exitCode=$?
fi
# kill the background processes (the logs) that have this process' pid (i.e. $$) as a parent
pkill -P $$
exit $exitCode
displayName: "Run Test Suite and Get Logs"
- publish: $(System.DefaultWorkingDirectory)/npmLogs_$(AZURE_CLUSTER)
condition: always()
artifact: NpmLogs_$(AZURE_CLUSTER)
else
runConformance | tee $npmLogsFolder/conformance-results
exitCode=$?
fi
fi
# kill the background processes (the logs) that have this process' pid (i.e. $$) as a parent
- job: Clean_up
displayName: "Cleanup"
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals Build
dependsOn: [Create_Cluster_and_Run_Test, setup]
variables:
RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
steps:
- checkout: none
- task: AzureCLI@2
displayName: "Delete Test Cluster Resource Group"
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptType: 'bash'
scriptLocation: 'inlineScript'
inlineScript: |
echo Deleting $(RESOURCE_GROUP)
az group delete -n $(RESOURCE_GROUP) --yes
for npmPod in $npmPodList; do
./kubectl --kubeconfig=./kubeconfig logs -n kube-system $npmPod > $npmLogsFolder/$npmPod-logs.txt
done
exit $exitCode
displayName: "Run Test Suite and Get Logs"
failOnStderr: false
- publish: $(System.DefaultWorkingDirectory)/npmLogs_$(AZURE_CLUSTER)
condition: always()
artifact: NpmLogs_$(AZURE_CLUSTER)
- job: Create_Windows_Cluster_and_Run_Test
timeoutInMinutes: 360
displayName: "Run Windows Cyclonus"
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals Build
dependsOn: [containerize_windows, setup]
variables:
RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
FQDN: empty
strategy:
matrix:
v2-windows:
PROFILE: "cyc-ws22"
steps:
- checkout: self
- download: current
artifact: Test
- task: AzureCLI@2
displayName: "Create AKS Cluster"
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptType: "bash"
scriptLocation: "inlineScript"
failOnStderr: true
inlineScript: |
az extension add --name aks-preview
az extension update --name aks-preview
# Enable Microsoft.ContainerService/AKSWindows2022Preview
az feature register --namespace Microsoft.ContainerService --name AKSWindows2022Preview
az provider register -n Microsoft.ContainerService
export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
echo "Creating resource group named $CLUSTER_NAME"
az group create --name $CLUSTER_NAME -l $(LOCATION) -o table
echo "Creating resource group named $CLUSTER_NAME"
az aks create \
--resource-group $CLUSTER_NAME \
--name $CLUSTER_NAME \
--generate-ssh-keys \
--windows-admin-username e2eadmin \
--windows-admin-password alpha@numeric!password2 \
--network-plugin azure \
--vm-set-type VirtualMachineScaleSets \
--node-vm-size Standard_D4s_v3 \
--kubernetes-version v1.23 \
--node-count 1
# don't schedule anything on the linux system pool
echo "Updating $CLUSTER_NAME to not schedule anything on linux pool..."
az aks nodepool update \
--cluster-name $CLUSTER_NAME \
-g $CLUSTER_NAME \
-n nodepool1 \
--node-taints CriticalAddonsOnly=true:NoSchedule
echo "Adding Windows nodepool to $CLUSTER_NAME"
az aks nodepool add \
--resource-group $CLUSTER_NAME \
--cluster-name $CLUSTER_NAME \
--name awin22 \
--os-type Windows \
--os-sku Windows2022 \
--node-vm-size Standard_D4s_v3 \
--node-count 3
echo "Getting credentials to $CLUSTER_NAME"
az aks get-credentials -g $CLUSTER_NAME -n $CLUSTER_NAME --overwrite-existing --file ./kubeconfig
mkdir -p ~/.kube/
cp ./kubeconfig ~/.kube/config
- task: AzureCLI@2
displayName: "Deploy NPM to Test Cluster"
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptType: "bash"
scriptLocation: "inlineScript"
failOnStderr: true
inlineScript: |
export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
curl -LO https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl
chmod +x kubectl
# deploy azure-npm
./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/examples/windows/azure-npm.yaml
# swap azure-npm image with one built during run
./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-$(TAG)
echo "sleeping 3 minutes to allow NPM pods to restart"
sleep 180
./kubectl --kubeconfig=./kubeconfig get po -n kube-system -owide -A
echo "Showing cluster status for $CLUSTER_NAME"
FQDN=`az aks show -n $CLUSTER_NAME -g $CLUSTER_NAME --query fqdn -o tsv`
echo "##vso[task.setvariable variable=FQDN]$FQDN"
- bash: |
cat ~/.kube/config
curl -fsSL github.com/mattfenwick/cyclonus/releases/latest/download/cyclonus_linux_amd64.tar.gz | tar -zxv
./test/cyclonus/test-cyclonus-windows.sh
name: cyclonus
displayName: "Run Cyclonus Test"
failOnStderr: false
condition: always()
- bash: |
export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
cp cyclonus-$CLUSTER_NAME $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/cyclonus-$CLUSTER_NAME
echo "Getting cluster state for $CLUSTER_NAME"
mkdir -p $(System.DefaultWorkingDirectory)/$CLUSTER_NAME
kubectl logs -n kube-system -l k8s-app=azure-npm --tail -1 --prefix > $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/npm-logs_$(PROFILE).txt
cp ./kubeconfig $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/.kubeconfig
condition: always()
- publish: $(System.DefaultWorkingDirectory)/$(RESOURCE_GROUP)-$(PROFILE)
condition: always()
artifact: NpmLogs-$(RESOURCE_GROUP)-$(PROFILE)
- job: clean_up
displayName: "Cleanup"
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
demands:
- agent.os -equals Linux
- Role -equals Build
dependsOn:
[Create_Cluster_and_Run_Test, Create_Windows_Cluster_and_Run_Test, setup]
variables:
RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
steps:
- checkout: none
- task: AzureCLI@2
displayName: "Delete Test Cluster Resource Group"
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptType: "bash"
scriptLocation: "inlineScript"
inlineScript: |
echo Deleting $(RESOURCE_GROUP)
az group delete -n $(RESOURCE_GROUP) --yes

4
Makefile
Просмотреть файл

@ -545,6 +545,10 @@ test-cyclonus: ## run the cyclonus test for npm.
cd test/cyclonus && bash ./test-cyclonus.sh
cd ..
test-cyclonus-windows: ## run the cyclonus test for npm.
cd test/cyclonus && bash ./test-cyclonus.sh windows
cd ..
test-extended-cyclonus: ## run the cyclonus test for npm.
cd test/cyclonus && bash ./test-cyclonus.sh extended
cd ..

40
npm/examples/windows/azure-npm.yaml
Просмотреть файл

@ -15,7 +15,7 @@ metadata:
addonmanager.kubernetes.io/mode: EnsureExists
rules:
- apiGroups:
- ""
- ""
resources:
- pods
- nodes
@ -25,7 +25,7 @@ rules:
- list
- watch
- apiGroups:
- networking.k8s.io
- networking.k8s.io
resources:
- networkpolicies
verbs:
@ -65,17 +65,17 @@ spec:
metadata:
labels:
k8s-app: azure-npm
annotations:
azure.npm/scrapeable: ''
annotations:
azure.npm/scrapeable: ""
spec:
priorityClassName: system-node-critical
tolerations:
- operator: "Exists"
effect: NoExecute
- operator: "Exists"
effect: NoSchedule
- key: CriticalAddonsOnly
operator: Exists
- operator: "Exists"
effect: NoExecute
- operator: "Exists"
effect: NoSchedule
- key: CriticalAddonsOnly
operator: Exists
securityContext:
windowsOptions:
hostProcess: true
@ -85,7 +85,15 @@ spec:
- name: azure-npm
image: mcr.microsoft.com/containernetworking/azure-npm:v1.4.29
command: ["powershell.exe"]
args: ['.\setkubeconfigpath.ps1', ';', 'powershell.exe', '.\npm.exe', "start", '--kubeconfig=.\kubeconfig']
args:
[
'.\setkubeconfigpath.ps1',
";",
"powershell.exe",
'.\npm.exe',
"start",
'--kubeconfig=.\kubeconfig',
]
resources:
limits:
cpu: 250m
@ -101,14 +109,14 @@ spec:
- name: NPM_CONFIG
value: .\\etc\\azure-npm\\azure-npm.json
volumeMounts:
- name: azure-npm-config
mountPath: .\\etc\\azure-npm
- name: azure-npm-config
mountPath: .\\etc\\azure-npm
nodeSelector:
kubernetes.io/os: windows
volumes:
- name: azure-npm-config
configMap:
name: azure-npm-config
- name: azure-npm-config
configMap:
name: azure-npm-config
serviceAccountName: azure-npm
---
apiVersion: v1

32
test/cyclonus/install-cyclonus-controlplane.yaml Normal file
Просмотреть файл

@ -0,0 +1,32 @@
apiVersion: batch/v1
kind: Job
metadata:
name: cyclonus
namespace: kube-system
labels:
app: cyclonus
spec:
template:
spec:
restartPolicy: Never
tolerations:
- key: "node-role.kubernetes.io/master"
operator: "Exists"
effect: "NoSchedule"
nodeSelector:
node-role.kubernetes.io/master: ""
containers:
- command:
- ./cyclonus
- generate
- --noisy=true
- --ignore-loopback=true
- --cleanup-namespaces=true
- --perturbation-wait-seconds=5
- --pod-creation-timeout-seconds=30
- --job-timeout-seconds=2
- --server-protocol=TCP,UDP
name: cyclonus
imagePullPolicy: IfNotPresent
image: acnpublic.azurecr.io/cyclonus:v0.4.6.8
serviceAccount: cyclonus

29
test/cyclonus/install-cyclonus-windows.yaml Normal file
Просмотреть файл

@ -0,0 +1,29 @@
apiVersion: batch/v1
kind: Job
metadata:
name: cyclonus
namespace: kube-system
labels:
app: cyclonus
spec:
template:
spec:
restartPolicy: Never
containers:
- name: cyclonus
command: ["cyclonus.exe"]
args:
[
"generate",
"--noisy=true",
"--ignore-loopback=true",
"--cleanup-namespaces=true",
"--perturbation-wait-seconds=10",
"--pod-creation-timeout-seconds=30",
"--job-timeout-seconds=30",
"--server-protocol=TCP,UDP",
"--verbosity=debug",
]
imagePullPolicy: IfNotPresent
image: acnpublic.azurecr.io/cyclonus-windows:v0.4.6.8
serviceAccount: cyclonus

19
test/cyclonus/test-cyclonus-windows.sh Executable file
Просмотреть файл

@ -0,0 +1,19 @@
curl -fsSL github.com/mattfenwick/cyclonus/releases/latest/download/cyclonus_linux_amd64.tar.gz | tar -zxv
./cyclonus_linux_amd64/cyclonus generate \
--noisy=true \
--retries=7 \
--ignore-loopback=true \
--cleanup-namespaces=true \
--perturbation-wait-seconds=20 \
--pod-creation-timeout-seconds=480 \
--job-timeout-seconds=15 \
--server-protocol=TCP,UDP \
--exclude sctp,named-port,ip-block-with-except,multi-peer,upstream-e2e,example,end-port,namespaces-by-default-label | tee cyclonus-$CLUSTER_NAME
rc=0
cat cyclonus-$CLUSTER_NAME | grep "failed" > /dev/null 2>&1 || rc=$?
echo $rc
if [ $rc -eq 0 ]; then
echo "failures detected"
exit 1
fi

10
test/cyclonus/test-cyclonus.sh
Просмотреть файл

@ -3,7 +3,14 @@
set -eo pipefail
set -xv
cyclonusProfile="./install-cyclonus.yaml"
if [ -z "$1" == "windows"]
then
cyclonusProfile="./install-cyclonus-windows.yaml"
else
cyclonusProfile="./install-cyclonus.yaml"
fi
if [ -z "$1" ]
then
echo "Running with default profile: $cyclonusProfile"
@ -14,6 +21,7 @@ then
echo "Running with exclude SCTP profile with 214 testcases: $cyclonusProfile"
fi
echo "Running with cyclonus profile: $cyclonusProfile"
kubectl delete --ignore-not-found=true clusterrolebinding cyclonus
kubectl delete --ignore-not-found=true sa cyclonus -n kube-system
kubectl delete --ignore-not-found=true -f $cyclonusProfile