Merge pull request #1063 from hasbro17/haseeb/op-upgrade-heal-1-member

e2e/upgradetest: 1 member recovery test

test/e2e/e2eutil/util.go

@@ -33,9 +33,9 @@ import (
 	"k8s.io/client-go/pkg/api/v1"
 )
 
-func KillMembers(kubecli kubernetes.Interface, ns string, names ...string) error {
+func KillMembers(kubecli kubernetes.Interface, namespace string, names ...string) error {
 	for _, name := range names {
-		err := kubecli.CoreV1().Pods(ns).Delete(name, metav1.NewDeleteOptions(0))
+		err := kubecli.CoreV1().Pods(namespace).Delete(name, metav1.NewDeleteOptions(0))
 		if err != nil && !k8sutil.IsKubernetesResourceNotFoundError(err) {
 			return err
 		}

test/e2e/upgradetest/framework/framework.go

@@ -15,9 +15,13 @@
 package framework
 
 import (
+	"time"
+
 	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
+	"github.com/coreos/etcd-operator/pkg/util/retryutil"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/pkg/api/v1"
@@ -98,10 +102,26 @@ func (f *Framework) CreateOperator() error {
 
 func (f *Framework) DeleteOperator() error {
 	foreground := metav1.DeletePropagationForeground
-	return f.KubeCli.AppsV1beta1().Deployments(f.KubeNS).Delete("etcd-operator", &metav1.DeleteOptions{
+	err := f.KubeCli.AppsV1beta1().Deployments(f.KubeNS).Delete("etcd-operator", &metav1.DeleteOptions{
 		GracePeriodSeconds: func(t int64) *int64 { return &t }(0),
 		PropagationPolicy:  &foreground,
 	})
+	if err != nil {
+		return err
+	}
+
+	// Wait until the etcd-operator pod is actually gone and not just terminating
+	ls := labels.SelectorFromSet(map[string]string{"name": "etcd-operator"})
+	return retryutil.Retry(5*time.Second, 5, func() (bool, error) {
+		podList, err := f.KubeCli.CoreV1().Pods(f.Config.KubeNS).List(metav1.ListOptions{LabelSelector: ls.String()})
+		if err != nil {
+			return false, err
+		}
+		if len(podList.Items) == 0 {
+			return true, nil
+		}
+		return false, nil
+	})
 }
 
 func (f *Framework) UpgradeOperator() error {

test/e2e/upgradetest/upgrade_test.go

@@ -72,3 +72,45 @@ func TestResize(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestHealOneMemberForOldCluster(t *testing.T) {
+	err := testF.CreateOperator()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := testF.DeleteOperator()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	if err = k8sutil.WaitEtcdTPRReady(testF.KubeCli.CoreV1().RESTClient(), 3*time.Second, 30*time.Second, testF.KubeNS); err != nil {
+		t.Fatalf("failed to see cluster TPR get created in time: %v", err)
+	}
+
+	testEtcd, err := e2eutil.CreateCluster(t, testF.KubeCli, testF.KubeNS, e2eutil.NewCluster("upgrade-test-", 3))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if err := e2eutil.DeleteCluster(t, testF.KubeCli, testEtcd, &e2eutil.StorageCheckerOptions{}); err != nil {
+			t.Fatal(err)
+		}
+	}()
+	names, err := e2eutil.WaitUntilSizeReached(t, testF.KubeCli, 3, 60*time.Second, testEtcd)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = testF.UpgradeOperator()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := e2eutil.KillMembers(testF.KubeCli, testF.KubeNS, names[2]); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := e2eutil.WaitUntilSizeReached(t, testF.KubeCli, 3, 60*time.Second, testEtcd); err != nil {
+		t.Fatalf("failed to heal one member: %v", err)
+	}
+}