test: make e2e singleCommandTimeout configurable (#4372)

test/e2e/cluster.sh

@@ -39,6 +39,9 @@ fi
 if [ "$STABILITY_ITERATIONS" == "" ]; then
   STABILITY_ITERATIONS=3
 fi
+if [ "$SINGLE_COMMAND_TIMEOUT_MINUTES" == "" ]; then
+  SINGLE_COMMAND_TIMEOUT_MINUTES=1
+fi
 if [ "$STABILITY_TIMEOUT_SECONDS" == "" ]; then
   STABILITY_TIMEOUT_SECONDS=5
 fi
@@ -224,6 +227,7 @@ docker run --rm \
 -e SERVICE_MANAGEMENT_VM_DNS_SUFFIX="${SERVICE_MANAGEMENT_VM_DNS_SUFFIX}" \
 -e RESOURCE_MANAGER_VM_DNS_SUFFIX="${RESOURCE_MANAGER_VM_DNS_SUFFIX}" \
 -e STABILITY_ITERATIONS=${STABILITY_ITERATIONS} \
+-e SINGLE_COMMAND_TIMEOUT_MINUTES=${SINGLE_COMMAND_TIMEOUT_MINUTES} \
 -e STABILITY_TIMEOUT_SECONDS=${STABILITY_TIMEOUT_SECONDS} \
 -e ARC_CLIENT_ID=${ARC_CLIENT_ID:-$AZURE_CLIENT_ID} \
 -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
@@ -356,6 +360,7 @@ if [ "${ROTATE_CERTS}" = "true" ]; then
     -e SERVICE_MANAGEMENT_VM_DNS_SUFFIX="${SERVICE_MANAGEMENT_VM_DNS_SUFFIX}" \
     -e RESOURCE_MANAGER_VM_DNS_SUFFIX="${RESOURCE_MANAGER_VM_DNS_SUFFIX}" \
     -e STABILITY_ITERATIONS=${STABILITY_ITERATIONS} \
+    -e SINGLE_COMMAND_TIMEOUT_MINUTES=${SINGLE_COMMAND_TIMEOUT_MINUTES} \
     -e STABILITY_TIMEOUT_SECONDS=${STABILITY_TIMEOUT_SECONDS} \
     -e ARC_CLIENT_ID=${ARC_CLIENT_ID:-$AZURE_CLIENT_ID} \
     -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
@@ -437,6 +442,7 @@ if [ -n "$ADD_NODE_POOL_INPUT" ]; then
     -e SERVICE_MANAGEMENT_VM_DNS_SUFFIX="${SERVICE_MANAGEMENT_VM_DNS_SUFFIX}" \
     -e RESOURCE_MANAGER_VM_DNS_SUFFIX="${RESOURCE_MANAGER_VM_DNS_SUFFIX}" \
     -e STABILITY_ITERATIONS=${STABILITY_ITERATIONS} \
+    -e SINGLE_COMMAND_TIMEOUT_MINUTES=${SINGLE_COMMAND_TIMEOUT_MINUTES} \
     -e STABILITY_TIMEOUT_SECONDS=${STABILITY_TIMEOUT_SECONDS} \
     -e ARC_CLIENT_ID=${ARC_CLIENT_ID:-$AZURE_CLIENT_ID} \
     -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
@@ -552,6 +558,7 @@ if [ "${SCALE_CLUSTER}" = "true" ]; then
     -e SERVICE_MANAGEMENT_VM_DNS_SUFFIX="${SERVICE_MANAGEMENT_VM_DNS_SUFFIX}" \
     -e RESOURCE_MANAGER_VM_DNS_SUFFIX="${RESOURCE_MANAGER_VM_DNS_SUFFIX}" \
     -e STABILITY_ITERATIONS=${STABILITY_ITERATIONS} \
+    -e SINGLE_COMMAND_TIMEOUT_MINUTES=${SINGLE_COMMAND_TIMEOUT_MINUTES} \
     -e STABILITY_TIMEOUT_SECONDS=${STABILITY_TIMEOUT_SECONDS} \
     -e ARC_CLIENT_ID=${ARC_CLIENT_ID:-$AZURE_CLIENT_ID} \
     -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
@@ -642,6 +649,7 @@ if [ "${UPGRADE_CLUSTER}" = "true" ]; then
       -e SERVICE_MANAGEMENT_VM_DNS_SUFFIX="${SERVICE_MANAGEMENT_VM_DNS_SUFFIX}" \
       -e RESOURCE_MANAGER_VM_DNS_SUFFIX="${RESOURCE_MANAGER_VM_DNS_SUFFIX}" \
       -e STABILITY_ITERATIONS=${STABILITY_ITERATIONS} \
+      -e SINGLE_COMMAND_TIMEOUT_MINUTES=${SINGLE_COMMAND_TIMEOUT_MINUTES} \
       -e STABILITY_TIMEOUT_SECONDS=${STABILITY_TIMEOUT_SECONDS} \
       -e ARC_CLIENT_ID=${ARC_CLIENT_ID:-$AZURE_CLIENT_ID} \
       -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
@@ -721,6 +729,7 @@ if [ "${SCALE_CLUSTER}" = "true" ]; then
     -e SERVICE_MANAGEMENT_VM_DNS_SUFFIX="${SERVICE_MANAGEMENT_VM_DNS_SUFFIX}" \
     -e RESOURCE_MANAGER_VM_DNS_SUFFIX="${RESOURCE_MANAGER_VM_DNS_SUFFIX}" \
     -e STABILITY_ITERATIONS=${STABILITY_ITERATIONS} \
+    -e SINGLE_COMMAND_TIMEOUT_MINUTES=${SINGLE_COMMAND_TIMEOUT_MINUTES} \
     -e STABILITY_TIMEOUT_SECONDS=${STABILITY_TIMEOUT_SECONDS} \
     -e ARC_CLIENT_ID=${ARC_CLIENT_ID:-$AZURE_CLIENT_ID} \
     -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \

test/e2e/config/config.go

@@ -36,6 +36,7 @@ type Config struct {
 	CleanUpIfFail                      bool          `envconfig:"CLEANUP_IF_FAIL" default:"false"`
 	RetainSSH                          bool          `envconfig:"RETAIN_SSH" default:"true"`
 	StabilityIterations                int           `envconfig:"STABILITY_ITERATIONS" default:"3"`
+	SingleCommandTimeoutMinutes        int           `envconfig:"SINGLE_COMMAND_TIMEOUT_MINUTES" default:"1"`
 	StabilityTimeoutSeconds            int           `envconfig:"STABILITY_TIMEOUT_SECONDS" default:"5"`
 	ClusterInitPodName                 string        `envconfig:"CLUSTER_INIT_POD_NAME" default:""`
 	ClusterInitJobName                 string        `envconfig:"CLUSTER_INIT_JOB_NAME" default:""`

test/e2e/kubernetes/kubernetes_test.go

@@ -60,7 +60,6 @@ const (
 	sleepBetweenRetriesWhenWaitingForPodReady = 1 * time.Second
 	sleepBetweenRetriesRemoteSSHCommand       = 3 * time.Second
 	timeoutWhenWaitingForPodOutboundAccess    = 1 * time.Minute
-	singleCommandTimeout                      = 1 * time.Minute
 	validateNetworkPolicyTimeout              = 3 * time.Minute
 	podLookupRetries                          = 5
 	sigPublishingTimeout                      = 8 * time.Hour // :(
@@ -80,6 +79,7 @@ var (
 	clusterAutoscalerAddon          api.KubernetesAddon
 	deploymentReplicasCount         int
 	dnsAddonName                    string
+	singleCommandTimeout            time.Duration
 	stabilityCommandTimeout         time.Duration
 	env                             azure.Environment
 	azureClient                     *armhelpers.AzureClient
@@ -174,6 +174,7 @@ var _ = BeforeSuite(func() {
 	}
 	Expect(dnsAddonName).NotTo(Equal(""))
 
+	singleCommandTimeout = time.Duration(cfg.SingleCommandTimeoutMinutes) * time.Minute
 	stabilityCommandTimeout = time.Duration(cfg.StabilityTimeoutSeconds) * time.Second
 
 	if !cfg.IsCustomCloudProfile() {