k8s component tests should happen before api tests (#3991)

This commit is contained in:
Jack Francis 2018-10-10 22:32:33 -07:00 коммит произвёл GitHub
Родитель ba8b8d6348
Коммит c447f9d8ee
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
1 изменённых файлов: 222 добавлений и 222 удалений

Просмотреть файл

@ -110,153 +110,6 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
Expect(len(nodeList.Nodes)).To(Equal(eng.NodeCount())) Expect(len(nodeList.Nodes)).To(Equal(eng.NodeCount()))
}) })
It("should have stable external container networking", func() {
name := fmt.Sprintf("alpine-%s", cfg.Name)
command := fmt.Sprintf("nc -vz 8.8.8.8 53 || nc -vz 8.8.4.4 53")
successes, err := pod.RunCommandMultipleTimes(pod.RunLinuxPod, "alpine", name, command, cfg.StabilityIterations)
Expect(err).NotTo(HaveOccurred())
Expect(successes).To(Equal(cfg.StabilityIterations))
})
It("should have stable internal container networking", func() {
name := fmt.Sprintf("alpine-%s", cfg.Name)
var command string
if common.IsKubernetesVersionGe(eng.ExpandedDefinition.Properties.OrchestratorProfile.OrchestratorVersion, "1.12.0") {
command = fmt.Sprintf("nc -vz kubernetes 443 && nc -vz kubernetes.default.svc 443 && nc -vz kubernetes.default.svc.cluster.local 443")
} else {
command = fmt.Sprintf("nc -vz kubernetes 443")
}
successes, err := pod.RunCommandMultipleTimes(pod.RunLinuxPod, "alpine", name, command, cfg.StabilityIterations)
Expect(err).NotTo(HaveOccurred())
Expect(successes).To(Equal(cfg.StabilityIterations))
})
It("should be able to launch a long-running container networking DNS liveness pod", func() {
if !eng.HasNetworkPolicy("calico") {
var err error
var p *pod.Pod
p, err = pod.CreatePodFromFile(filepath.Join(WorkloadDir, "dns-liveness.yaml"), "dns-liveness", "default")
if cfg.SoakClusterName == "" {
Expect(err).NotTo(HaveOccurred())
} else {
if err != nil {
p, err = pod.Get("dns-liveness", "default")
Expect(err).NotTo(HaveOccurred())
}
}
running, err := p.WaitOnReady(5*time.Second, 2*time.Minute)
Expect(err).NotTo(HaveOccurred())
Expect(running).To(Equal(true))
}
})
It("should have functional host OS DNS", func() {
kubeConfig, err := GetConfig()
Expect(err).NotTo(HaveOccurred())
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
ifconfigCmd := fmt.Sprintf("ifconfig -a -v")
var cmd *exec.Cmd
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd)
util.PrintCommand(cmd)
out, err := cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
resolvCmd := fmt.Sprintf("cat /etc/resolv.conf")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
By("Ensuring that we have a valid connection to our resolver")
digCmd := fmt.Sprintf("dig +short +search +answer `hostname`")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
nodeList, err := node.Get()
Expect(err).NotTo(HaveOccurred())
for _, node := range nodeList.Nodes {
By("Ensuring that we get a DNS lookup answer response for each node hostname")
digCmd := fmt.Sprintf("dig +short +search +answer %s | grep -v -e '^$'", node.Metadata.Name)
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
Expect(err).NotTo(HaveOccurred())
}
By("Ensuring that we get a DNS lookup answer response for external names")
digCmd = fmt.Sprintf("dig +short +search www.bing.com | grep -v -e '^$'")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
if err != nil {
log.Printf("Error while querying DNS: %s\n", out)
}
digCmd = fmt.Sprintf("dig +short +search google.com | grep -v -e '^$'")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
By("Ensuring that we get a DNS lookup answer response for external names using external resolver")
digCmd = fmt.Sprintf("dig +short +search www.bing.com @8.8.8.8 | grep -v -e '^$'")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
digCmd = fmt.Sprintf("dig +short +search google.com @8.8.8.8 | grep -v -e '^$'")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
})
It("should have functional container networking DNS", func() {
By("Ensuring that we have functional DNS resolution from a container")
j, err := job.CreateJobFromFile(filepath.Join(WorkloadDir, "validate-dns.yaml"), "validate-dns", "default")
Expect(err).NotTo(HaveOccurred())
ready, err := j.WaitOnReady(5*time.Second, cfg.Timeout)
delErr := j.Delete(deleteResourceRetries)
if delErr != nil {
fmt.Printf("could not delete job %s\n", j.Metadata.Name)
fmt.Println(delErr)
}
Expect(err).NotTo(HaveOccurred())
Expect(ready).To(Equal(true))
By("Ensuring that we have stable external DNS resolution from a container")
name := fmt.Sprintf("alpine-%s", cfg.Name)
command := fmt.Sprintf("nc -vz bbc.co.uk 80 || nc -vz google.com 443 || nc -vz microsoft.com 80")
successes, err := pod.RunCommandMultipleTimes(pod.RunLinuxPod, "alpine", name, command, cfg.StabilityIterations)
Expect(err).NotTo(HaveOccurred())
Expect(successes).To(Equal(cfg.StabilityIterations))
})
It("should have DNS pod running", func() { It("should have DNS pod running", func() {
var err error var err error
var running bool var running bool
@ -329,81 +182,6 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
} }
}) })
It("should be able to access the dashboard from each node", func() {
if hasDashboard, dashboardAddon := eng.HasAddon("kubernetes-dashboard"); hasDashboard {
By("Ensuring that the kubernetes-dashboard pod is Running")
running, err := pod.WaitOnReady("kubernetes-dashboard", "kube-system", 3, 30*time.Second, cfg.Timeout)
Expect(err).NotTo(HaveOccurred())
Expect(running).To(Equal(true))
By("Ensuring that the kubernetes-dashboard service is Running")
s, err := service.Get("kubernetes-dashboard", "kube-system")
Expect(err).NotTo(HaveOccurred())
if !eng.HasWindowsAgents() {
By("Gathering connection information to determine whether or not to connect via HTTP or HTTPS")
dashboardPort := 443
version, err := node.Version()
Expect(err).NotTo(HaveOccurred())
re := regexp.MustCompile("1.(5|6|7|8).")
if re.FindString(version) != "" {
dashboardPort = 80
}
port := s.GetNodePort(dashboardPort)
kubeConfig, err := GetConfig()
Expect(err).NotTo(HaveOccurred())
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
if dashboardPort == 80 {
By("Ensuring that we can connect via HTTP to the dashboard on any one node")
} else {
By("Ensuring that we can connect via HTTPS to the dashboard on any one node")
}
nodeList, err := node.Get()
Expect(err).NotTo(HaveOccurred())
for _, node := range nodeList.Nodes {
success := false
for i := 0; i < 60; i++ {
address := node.Status.GetAddressByType("InternalIP")
if address == nil {
log.Printf("One of our nodes does not have an InternalIP value!: %s\n", node.Metadata.Name)
}
Expect(address).NotTo(BeNil())
dashboardURL := fmt.Sprintf("http://%s:%v", address.Address, port)
curlCMD := fmt.Sprintf("curl --max-time 60 %s", dashboardURL)
var cmd *exec.Cmd
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD)
util.PrintCommand(cmd)
out, err := cmd.CombinedOutput()
if err == nil {
success = true
break
}
if i > 58 {
log.Printf("Error while connecting to Windows dashboard:%s\n", err)
log.Println(string(out))
}
time.Sleep(10 * time.Second)
}
Expect(success).To(BeTrue())
}
By("Ensuring that the correct resources have been applied")
// Assuming one dashboard pod
pods, err := pod.GetAllByPrefix("kubernetes-dashboard", "kube-system")
Expect(err).NotTo(HaveOccurred())
for i, c := range dashboardAddon.Containers {
err := pods[0].Spec.Containers[i].ValidateResources(c)
Expect(err).NotTo(HaveOccurred())
}
}
} else {
Skip("kubernetes-dashboard disabled for this cluster, will not test")
}
})
It("should have ip-masq-agent running", func() { It("should have ip-masq-agent running", func() {
if hasIPMasqAgent, IPMasqAgentAddon := eng.HasAddon("ip-masq-agent"); hasIPMasqAgent { if hasIPMasqAgent, IPMasqAgentAddon := eng.HasAddon("ip-masq-agent"); hasIPMasqAgent {
running, err := pod.WaitOnReady("azure-ip-masq-agent", "kube-system", 3, 30*time.Second, cfg.Timeout) running, err := pod.WaitOnReady("azure-ip-masq-agent", "kube-system", 3, 30*time.Second, cfg.Timeout)
@ -630,6 +408,228 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
} }
} }
}) })
It("should have stable external container networking", func() {
name := fmt.Sprintf("alpine-%s", cfg.Name)
command := fmt.Sprintf("nc -vz 8.8.8.8 53 || nc -vz 8.8.4.4 53")
successes, err := pod.RunCommandMultipleTimes(pod.RunLinuxPod, "alpine", name, command, cfg.StabilityIterations)
Expect(err).NotTo(HaveOccurred())
Expect(successes).To(Equal(cfg.StabilityIterations))
})
It("should have stable internal container networking", func() {
name := fmt.Sprintf("alpine-%s", cfg.Name)
var command string
if common.IsKubernetesVersionGe(eng.ExpandedDefinition.Properties.OrchestratorProfile.OrchestratorVersion, "1.12.0") {
command = fmt.Sprintf("nc -vz kubernetes 443 && nc -vz kubernetes.default.svc 443 && nc -vz kubernetes.default.svc.cluster.local 443")
} else {
command = fmt.Sprintf("nc -vz kubernetes 443")
}
successes, err := pod.RunCommandMultipleTimes(pod.RunLinuxPod, "alpine", name, command, cfg.StabilityIterations)
Expect(err).NotTo(HaveOccurred())
Expect(successes).To(Equal(cfg.StabilityIterations))
})
It("should be able to launch a long-running container networking DNS liveness pod", func() {
if !eng.HasNetworkPolicy("calico") {
var err error
var p *pod.Pod
p, err = pod.CreatePodFromFile(filepath.Join(WorkloadDir, "dns-liveness.yaml"), "dns-liveness", "default")
if cfg.SoakClusterName == "" {
Expect(err).NotTo(HaveOccurred())
} else {
if err != nil {
p, err = pod.Get("dns-liveness", "default")
Expect(err).NotTo(HaveOccurred())
}
}
running, err := p.WaitOnReady(5*time.Second, 2*time.Minute)
Expect(err).NotTo(HaveOccurred())
Expect(running).To(Equal(true))
}
})
It("should have functional host OS DNS", func() {
kubeConfig, err := GetConfig()
Expect(err).NotTo(HaveOccurred())
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
ifconfigCmd := fmt.Sprintf("ifconfig -a -v")
var cmd *exec.Cmd
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd)
util.PrintCommand(cmd)
out, err := cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
resolvCmd := fmt.Sprintf("cat /etc/resolv.conf")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
By("Ensuring that we have a valid connection to our resolver")
digCmd := fmt.Sprintf("dig +short +search +answer `hostname`")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
nodeList, err := node.Get()
Expect(err).NotTo(HaveOccurred())
for _, node := range nodeList.Nodes {
By("Ensuring that we get a DNS lookup answer response for each node hostname")
digCmd := fmt.Sprintf("dig +short +search +answer %s | grep -v -e '^$'", node.Metadata.Name)
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
Expect(err).NotTo(HaveOccurred())
}
By("Ensuring that we get a DNS lookup answer response for external names")
digCmd = fmt.Sprintf("dig +short +search www.bing.com | grep -v -e '^$'")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
if err != nil {
log.Printf("Error while querying DNS: %s\n", out)
}
digCmd = fmt.Sprintf("dig +short +search google.com | grep -v -e '^$'")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
By("Ensuring that we get a DNS lookup answer response for external names using external resolver")
digCmd = fmt.Sprintf("dig +short +search www.bing.com @8.8.8.8 | grep -v -e '^$'")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
digCmd = fmt.Sprintf("dig +short +search google.com @8.8.8.8 | grep -v -e '^$'")
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
if err != nil {
log.Printf("Error while querying DNS: %s\n", err)
}
})
It("should have functional container networking DNS", func() {
By("Ensuring that we have functional DNS resolution from a container")
j, err := job.CreateJobFromFile(filepath.Join(WorkloadDir, "validate-dns.yaml"), "validate-dns", "default")
Expect(err).NotTo(HaveOccurred())
ready, err := j.WaitOnReady(5*time.Second, cfg.Timeout)
delErr := j.Delete(deleteResourceRetries)
if delErr != nil {
fmt.Printf("could not delete job %s\n", j.Metadata.Name)
fmt.Println(delErr)
}
Expect(err).NotTo(HaveOccurred())
Expect(ready).To(Equal(true))
By("Ensuring that we have stable external DNS resolution from a container")
name := fmt.Sprintf("alpine-%s", cfg.Name)
command := fmt.Sprintf("nc -vz bbc.co.uk 80 || nc -vz google.com 443 || nc -vz microsoft.com 80")
successes, err := pod.RunCommandMultipleTimes(pod.RunLinuxPod, "alpine", name, command, cfg.StabilityIterations)
Expect(err).NotTo(HaveOccurred())
Expect(successes).To(Equal(cfg.StabilityIterations))
})
It("should be able to access the dashboard from each node", func() {
if hasDashboard, dashboardAddon := eng.HasAddon("kubernetes-dashboard"); hasDashboard {
By("Ensuring that the kubernetes-dashboard pod is Running")
running, err := pod.WaitOnReady("kubernetes-dashboard", "kube-system", 3, 30*time.Second, cfg.Timeout)
Expect(err).NotTo(HaveOccurred())
Expect(running).To(Equal(true))
By("Ensuring that the kubernetes-dashboard service is Running")
s, err := service.Get("kubernetes-dashboard", "kube-system")
Expect(err).NotTo(HaveOccurred())
if !eng.HasWindowsAgents() {
By("Gathering connection information to determine whether or not to connect via HTTP or HTTPS")
dashboardPort := 443
version, err := node.Version()
Expect(err).NotTo(HaveOccurred())
re := regexp.MustCompile("1.(5|6|7|8).")
if re.FindString(version) != "" {
dashboardPort = 80
}
port := s.GetNodePort(dashboardPort)
kubeConfig, err := GetConfig()
Expect(err).NotTo(HaveOccurred())
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
if dashboardPort == 80 {
By("Ensuring that we can connect via HTTP to the dashboard on any one node")
} else {
By("Ensuring that we can connect via HTTPS to the dashboard on any one node")
}
nodeList, err := node.Get()
Expect(err).NotTo(HaveOccurred())
for _, node := range nodeList.Nodes {
success := false
for i := 0; i < 60; i++ {
address := node.Status.GetAddressByType("InternalIP")
if address == nil {
log.Printf("One of our nodes does not have an InternalIP value!: %s\n", node.Metadata.Name)
}
Expect(address).NotTo(BeNil())
dashboardURL := fmt.Sprintf("http://%s:%v", address.Address, port)
curlCMD := fmt.Sprintf("curl --max-time 60 %s", dashboardURL)
var cmd *exec.Cmd
cmd = exec.Command("ssh", "-i", masterSSHPrivateKeyFilepath, "-p", masterSSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD)
util.PrintCommand(cmd)
out, err := cmd.CombinedOutput()
if err == nil {
success = true
break
}
if i > 58 {
log.Printf("Error while connecting to Windows dashboard:%s\n", err)
log.Println(string(out))
}
time.Sleep(10 * time.Second)
}
Expect(success).To(BeTrue())
}
By("Ensuring that the correct resources have been applied")
// Assuming one dashboard pod
pods, err := pod.GetAllByPrefix("kubernetes-dashboard", "kube-system")
Expect(err).NotTo(HaveOccurred())
for i, c := range dashboardAddon.Containers {
err := pods[0].Spec.Containers[i].ValidateResources(c)
Expect(err).NotTo(HaveOccurred())
}
}
} else {
Skip("kubernetes-dashboard disabled for this cluster, will not test")
}
})
}) })
Describe("with a linux agent pool", func() { Describe("with a linux agent pool", func() {