E2E: rationalize node check + kube-system check, no kms (#4273)

.circleci/config.yml

@@ -72,7 +72,7 @@ jobs:
           echo 'export ORCHESTRATOR_RELEASE=1.10' >> $BASH_ENV
           echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
           echo 'export CREATE_VNET=true' >> $BASH_ENV
-          echo 'export ENABLE_KMS_ENCRYPTION=true' >> $BASH_ENV
+          echo 'export ENABLE_KMS_ENCRYPTION=false' >> $BASH_ENV
           echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV
           echo 'export CLEANUP_IF_FAIL=${CLEANUP_IF_FAIL_LINUX}' >> $BASH_ENV
           echo 'export RETAIN_SSH=false' >> $BASH_ENV
@@ -98,7 +98,7 @@ jobs:
           echo 'export ORCHESTRATOR_RELEASE=1.11' >> $BASH_ENV
           echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
           echo 'export CREATE_VNET=true' >> $BASH_ENV
-          echo 'export ENABLE_KMS_ENCRYPTION=true' >> $BASH_ENV
+          echo 'export ENABLE_KMS_ENCRYPTION=false' >> $BASH_ENV
           echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV
           echo 'export CLEANUP_IF_FAIL=${CLEANUP_IF_FAIL_LINUX}' >> $BASH_ENV
           echo 'export RETAIN_SSH=false' >> $BASH_ENV
@@ -124,7 +124,7 @@ jobs:
           echo 'export ORCHESTRATOR_RELEASE=1.12' >> $BASH_ENV
           echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
           echo 'export CREATE_VNET=true' >> $BASH_ENV
-          echo 'export ENABLE_KMS_ENCRYPTION=true' >> $BASH_ENV
+          echo 'export ENABLE_KMS_ENCRYPTION=false' >> $BASH_ENV
           echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV
           echo 'export CLEANUP_IF_FAIL=${CLEANUP_IF_FAIL_LINUX}' >> $BASH_ENV
           echo 'export RETAIN_SSH=false' >> $BASH_ENV
@@ -150,7 +150,7 @@ jobs:
           echo 'export ORCHESTRATOR_RELEASE=1.13' >> $BASH_ENV
           echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
           echo 'export CREATE_VNET=true' >> $BASH_ENV
-          echo 'export ENABLE_KMS_ENCRYPTION=true' >> $BASH_ENV
+          echo 'export ENABLE_KMS_ENCRYPTION=false' >> $BASH_ENV
           echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV
           echo 'export CLEANUP_IF_FAIL=${CLEANUP_IF_FAIL_LINUX}' >> $BASH_ENV
           echo 'export RETAIN_SSH=false' >> $BASH_ENV

test/e2e/engine/template.go

@@ -183,6 +183,9 @@ func Build(cfg *config.Config, masterSubnetID string, agentSubnetID string, isVM
 	}
 
 	if config.EnableKMSEncryption && config.ClientObjectID != "" {
+		if cs.ContainerService.Properties.OrchestratorProfile.KubernetesConfig == nil {
+			cs.ContainerService.Properties.OrchestratorProfile.KubernetesConfig = &vlabs.KubernetesConfig{}
+		}
 		cs.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.EnableEncryptionWithExternalKms = &config.EnableKMSEncryption
 		cs.ContainerService.Properties.ServicePrincipalProfile.ObjectID = config.ClientObjectID
 	}

test/e2e/kubernetes/kubernetes_test.go

@@ -139,10 +139,15 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
 			}
 		})
 
-		It("should have have the appropriate node count", func() {
-			nodeList, err := node.Get()
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(nodeList.Nodes)).To(Equal(eng.NodeCount()))
+		It("should report all nodes in a Ready state", func() {
+			ready := node.WaitOnReady(eng.NodeCount(), 10*time.Second, cfg.Timeout)
+			cmd := exec.Command("kubectl", "get", "nodes", "-o", "wide")
+			out, _ := cmd.CombinedOutput()
+			log.Printf("%s\n", out)
+			if !ready {
+				log.Printf("Error: Not all nodes in a healthy state\n")
+			}
+			Expect(ready).To(Equal(true))
 		})
 
 		It("should have DNS pod running", func() {

test/e2e/kubernetes/node/node.go

@@ -71,14 +71,22 @@ type List struct {
 // AreAllReady returns a bool depending on cluster state
 func AreAllReady(nodeCount int) bool {
 	list, _ := Get()
+	var ready int
 	if list != nil && len(list.Nodes) == nodeCount {
 		for _, node := range list.Nodes {
+			nodeReady := false
 			for _, condition := range node.Status.Conditions {
-				if condition.Type == "KubeletReady" && condition.Status == "false" {
-					return false
+				if condition.Type == "Ready" && condition.Status == "True" {
+					ready++
+					nodeReady = true
 				}
 			}
+			if !nodeReady {
+				return false
+			}
 		}
+	}
+	if ready == nodeCount {
 		return true
 	}
 	return false

test/e2e/runner/cli_provisioner.go

@@ -255,6 +255,9 @@ func (cli *CLIProvisioner) waitForNodes() error {
 		if !cli.IsPrivate() {
 			log.Println("Waiting on nodes to go into ready state...")
 			ready := node.WaitOnReady(cli.Engine.NodeCount(), 10*time.Second, cli.Config.Timeout)
+			cmd := exec.Command("kubectl", "get", "nodes", "-o", "wide")
+			out, _ := cmd.CombinedOutput()
+			log.Printf("%s\n", out)
 			if !ready {
 				return errors.New("Error: Not all nodes in a healthy state")
 			}