ci: test async delete in cilium stages (#2725)
* add async delete testing to cilium stages
* Revert "revert: async delete changes for azure-ipam (#2346)"
This reverts commit 5b9aaef6ad.
* tidy
* use exit code in script
* add async delete to cilium load test
* tidy
This commit is contained in:
Camryn Lee
GitHub
Родитель
5e1b2adf6c
Коммит
512d2d4672
|
|
@ -218,6 +218,15 @@ stages:
|
|||
retryCountOnTaskFailure: 6
|
||||
name: "CiliumConnectivityTests"
|
||||
displayName: "Run Cilium Connectivity Tests"
|
||||
- script: |
|
||||
cd hack/scripts
|
||||
chmod +x async-delete-test.sh
|
||||
./async-delete-test.sh
|
||||
if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then
|
||||
kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]'
|
||||
fi
|
||||
name: "testAsyncDelete"
|
||||
displayName: "Verify Async Delete when CNS is down"
|
||||
- template: ../k8s-e2e/k8s-e2e-job-template.yaml
|
||||
parameters:
|
||||
sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
|
||||
|
|
|
|||
|
|
@ -126,3 +126,13 @@ steps:
|
|||
retryCountOnTaskFailure: 3
|
||||
name: "WireserverMetadataConnectivityTests"
|
||||
displayName: "Run Wireserver and Metadata Connectivity Tests"
|
||||
|
||||
- script: |
|
||||
cd hack/scripts
|
||||
chmod +x async-delete-test.sh
|
||||
./async-delete-test.sh
|
||||
if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then
|
||||
kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]'
|
||||
fi
|
||||
name: "testAsyncDelete"
|
||||
displayName: "Verify Async Delete when CNS is down"
|
||||
|
|
|
|||
|
|
@ -165,6 +165,16 @@ steps:
|
|||
retryCountOnTaskFailure: 3
|
||||
name: "WireserverMetadataConnectivityTests"
|
||||
displayName: "Run Wireserver and Metadata Connectivity Tests"
|
||||
|
||||
- script: |
|
||||
cd hack/scripts
|
||||
chmod +x async-delete-test.sh
|
||||
./async-delete-test.sh
|
||||
if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then
|
||||
kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]'
|
||||
fi
|
||||
name: "testAsyncDelete"
|
||||
displayName: "Verify Async Delete when CNS is down"
|
||||
|
||||
- script: |
|
||||
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
|
||||
|
|
|
|||
|
|
@ -199,3 +199,13 @@ steps:
|
|||
retryCountOnTaskFailure: 3
|
||||
name: "WireserverMetadataConnectivityTests"
|
||||
displayName: "Run Wireserver and Metadata Connectivity Tests"
|
||||
|
||||
- script: |
|
||||
cd hack/scripts
|
||||
chmod +x async-delete-test.sh
|
||||
./async-delete-test.sh
|
||||
if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then
|
||||
kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]'
|
||||
fi
|
||||
name: "testAsyncDelete"
|
||||
displayName: "Verify Async Delete when CNS is down"
|
||||
|
|
|
|||
|
|
@ -131,3 +131,13 @@ steps:
|
|||
retryCountOnTaskFailure: 3
|
||||
name: "WireserverMetadataConnectivityTests"
|
||||
displayName: "Run Wireserver and Metadata Connectivity Tests"
|
||||
|
||||
- script: |
|
||||
cd hack/scripts
|
||||
chmod +x async-delete-test.sh
|
||||
./async-delete-test.sh
|
||||
if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then
|
||||
kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]'
|
||||
fi
|
||||
name: "testAsyncDelete"
|
||||
displayName: "Verify Async Delete when CNS is down"
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ require (
|
|||
github.com/coreos/go-iptables v0.7.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/go-logr/logr v1.4.1 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.20.0 // indirect
|
||||
github.com/go-openapi/jsonreference v0.20.2 // indirect
|
||||
|
|
@ -71,6 +72,7 @@ require (
|
|||
golang.org/x/mod v0.14.0 // indirect
|
||||
golang.org/x/net v0.23.0 // indirect
|
||||
golang.org/x/oauth2 v0.16.0 // indirect
|
||||
golang.org/x/sync v0.6.0 // indirect
|
||||
golang.org/x/sys v0.18.0 // indirect
|
||||
golang.org/x/term v0.18.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
|
||||
|
|
@ -10,6 +11,7 @@ import (
|
|||
"github.com/Azure/azure-container-networking/azure-ipam/ipconfig"
|
||||
"github.com/Azure/azure-container-networking/cns"
|
||||
cnscli "github.com/Azure/azure-container-networking/cns/client"
|
||||
"github.com/Azure/azure-container-networking/cns/fsnotify"
|
||||
cniSkel "github.com/containernetworking/cni/pkg/skel"
|
||||
cniTypes "github.com/containernetworking/cni/pkg/types"
|
||||
types100 "github.com/containernetworking/cni/pkg/types/100"
|
||||
|
|
@ -17,6 +19,10 @@ import (
|
|||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
watcherPath = "/var/run/azure-vnet/deleteIDs"
|
||||
)
|
||||
|
||||
// IPAMPlugin is the struct for the delegated azure-ipam plugin
|
||||
// https://www.cni.dev/docs/spec/#section-4-plugin-delegation
|
||||
type IPAMPlugin struct {
|
||||
|
|
@ -154,6 +160,7 @@ func (p *IPAMPlugin) CmdAdd(args *cniSkel.CmdArgs) error {
|
|||
|
||||
// CmdDel handles CNI delete commands.
|
||||
func (p *IPAMPlugin) CmdDel(args *cniSkel.CmdArgs) error {
|
||||
var connectionErr *cnscli.ConnectionFailureErr
|
||||
p.logger.Info("DEL called", zap.Any("args", args))
|
||||
|
||||
// Create ip config request from args
|
||||
|
|
@ -181,8 +188,28 @@ func (p *IPAMPlugin) CmdDel(args *cniSkel.CmdArgs) error {
|
|||
err = p.cnsClient.ReleaseIPAddress(context.TODO(), ipconfigReq)
|
||||
|
||||
if err != nil {
|
||||
p.logger.Error("Failed to release IP address to CNS using ReleaseIPAddress", zap.Error(err), zap.Any("request", ipconfigReq))
|
||||
return cniTypes.NewError(ErrRequestIPConfigFromCNS, err.Error(), "failed to release IP address from CNS using ReleaseIPAddress")
|
||||
if errors.As(err, &connectionErr) {
|
||||
p.logger.Info("Failed to release IP address from CNS due to connection failure, saving to watcher to delete")
|
||||
addErr := fsnotify.AddFile(args.ContainerID, args.ContainerID, watcherPath)
|
||||
if addErr != nil {
|
||||
p.logger.Error("Failed to add file to watcher", zap.String("containerID", args.ContainerID), zap.Error(addErr))
|
||||
return cniTypes.NewError(cniTypes.ErrTryAgainLater, addErr.Error(), fmt.Sprintf("failed to add file to watcher with containerID %s", args.ContainerID))
|
||||
} else {
|
||||
p.logger.Info("File successfully added to watcher directory")
|
||||
}
|
||||
} else {
|
||||
p.logger.Error("Failed to release IP address to CNS using ReleaseIPAddress", zap.Error(err), zap.Any("request", ipconfigReq))
|
||||
return cniTypes.NewError(ErrRequestIPConfigFromCNS, err.Error(), "failed to release IP address from CNS using ReleaseIPAddress")
|
||||
}
|
||||
}
|
||||
} else if errors.As(err, &connectionErr) {
|
||||
p.logger.Info("Failed to release IP addresses from CNS due to connection failure, saving to watcher to delete")
|
||||
addErr := fsnotify.AddFile(args.ContainerID, args.ContainerID, watcherPath)
|
||||
if addErr != nil {
|
||||
p.logger.Error("Failed to add file to watcher", zap.String("containerID", args.ContainerID), zap.Error(addErr))
|
||||
return cniTypes.NewError(cniTypes.ErrTryAgainLater, addErr.Error(), fmt.Sprintf("failed to add file to watcher with containerID %s", args.ContainerID))
|
||||
} else {
|
||||
p.logger.Info("File successfully added to watcher directory")
|
||||
}
|
||||
} else {
|
||||
p.logger.Error("Failed to release IP addresses from CNS", zap.Error(err), zap.Any("request", req))
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ do
|
|||
break
|
||||
else
|
||||
echo "##[error]async delete failure. file still exists in deleteIDs directory."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче