fix: Using retryOnConflict to fix v4overlay scale tests failing. (#2314)

* fix: ctry using retryOnConflict for v4overlay scale test

* fix: typo

.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml

@@ -154,5 +154,5 @@ steps:
       cd test/integration/datapath
       sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$
     name: "WindowsDualStackOverlayDatapathTests"
-    displayName: "Windows DaulStack Overlay Datapath Tests"
+    displayName: "Windows DualStack Overlay Datapath Tests"
     retryCountOnTaskFailure: 3

test/internal/kubernetes/utils.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/client-go/util/homedir"
+	k8sRetry "k8s.io/client-go/util/retry"
 )
 
 const (
@@ -301,14 +302,27 @@ func WaitForPodDaemonset(ctx context.Context, clientset *kubernetes.Clientset, n
 }
 
 func MustUpdateReplica(ctx context.Context, deploymentsClient typedappsv1.DeploymentInterface, deploymentName string, replicas int32) {
-	deployment, err := deploymentsClient.Get(ctx, deploymentName, metav1.GetOptions{})
-	if err != nil {
-		panic(errors.Wrapf(err, "could not get deployment %s", deploymentName))
-	}
+	retryErr := k8sRetry.RetryOnConflict(k8sRetry.DefaultRetry, func() error {
+		// Get the latest Deployment resource.
+		deployment, getErr := deploymentsClient.Get(ctx, deploymentName, metav1.GetOptions{})
+		if getErr != nil {
+			return fmt.Errorf("failed to get deployment: %w", getErr)
+		}
 
-	deployment.Spec.Replicas = Int32ToPtr(replicas)
-	if _, err := deploymentsClient.Update(ctx, deployment, metav1.UpdateOptions{}); err != nil {
-		panic(errors.Wrapf(err, "could not update deployment %s", deploymentName))
+		// Modify the number of replicas.
+		deployment.Spec.Replicas = Int32ToPtr(replicas)
+
+		// Attempt to update the Deployment.
+		_, updateErr := deploymentsClient.Update(ctx, deployment, metav1.UpdateOptions{})
+		if updateErr != nil {
+			return fmt.Errorf("failed to update deployment: %w", updateErr)
+		}
+
+		return nil // No error, operation succeeded.
+	})
+
+	if retryErr != nil {
+		panic(errors.Wrapf(retryErr, "could not update deployment %s", deploymentName))
 	}
 }