Merge pull request #289 from coreos/better_monkey

chaos: add probability

cmd/controller/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/coreos/kube-etcd-controller/pkg/controller"
 	"github.com/coreos/kube-etcd-controller/pkg/util/k8sutil"
 	"github.com/coreos/kube-etcd-controller/version"
+	"golang.org/x/time/rate"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/client/leaderelection"
 	"k8s.io/kubernetes/pkg/client/record"
@@ -125,10 +126,10 @@ func run(stop <-chan struct{}) {
 
 		switch chaosLevel {
 		case 1:
-			logrus.Infof("chaos level = 1: randomly kill one etcd pod every 10 seconds")
+			logrus.Infof("chaos level = 1: randomly kill one etcd pod every 30 seconds at 50%")
 			m := chaos.NewMonkeys(cfg.KubeCli)
 			ls := labels.SelectorFromSet(map[string]string{"app": "etcd"})
-			go m.CrushPods(ctx, cfg.Namespace, ls, 0.1)
+			go m.CrushPods(ctx, cfg.Namespace, ls, rate.Every(30*time.Second), 0.5)
 		default:
 		}
 

pkg/chaos/chaos.go

@@ -35,19 +35,24 @@ func NewMonkeys(k8s *unversioned.Client) *Monkeys {
 }
 
 // TODO: respect context in k8s operations.
-func (m *Monkeys) CrushPods(ctx context.Context, ns string, ls labels.Selector, killRate float64) {
+func (m *Monkeys) CrushPods(ctx context.Context, ns string, ls labels.Selector, killRate rate.Limit, killProbability float64) {
 	burst := int(killRate)
 	if burst <= 0 {
 		burst = 1
 	}
-	limiter := rate.NewLimiter(rate.Limit(killRate), burst)
+	limiter := rate.NewLimiter(killRate, burst)
 	for {
 		err := limiter.Wait(ctx)
 		if err != nil { // user cancellation
-			logrus.Infof("crushPods is cancelled for selector %v by the user: %v", ls.String(), err)
+			logrus.Infof("crushPods is canceled for selector %v by the user: %v", ls.String(), err)
 			return
 		}
 
+		if p := rand.Float64(); p > killProbability {
+			logrus.Infof("skip killing pod: probability: %v, got p: %v", killProbability, p)
+			continue
+		}
+
 		pods, err := m.k8s.Pods(ns).List(api.ListOptions{LabelSelector: ls})
 		if err != nil {
 			logrus.Errorf("failed to list pods for selector %v: %v", ls.String(), err)