Correct the logic for multi-container scenarios (after we dropped the container dimansion).

Add a test case to cover that.

The metric figure in Geneva now matches the RESTART column figure shown on oc get pods.

pkg/monitor/cluster/podconditions.go

@@ -130,23 +130,27 @@ func (mon *Monitor) _emitPodContainerRestartCounter(ps *corev1.PodList) {
 			continue
 		}
 
+		//Sum up the total number of restarts in the pod to match the number of restarts shown in the 'oc get pods' display
+		t := int32(0)
 		for _, cs := range p.Status.ContainerStatuses {
-			if cs.RestartCount < restartCounterThreshold {
-				continue
-			}
+			t += cs.RestartCount
+		}
 
-			mon.emitGauge("pod.restartcounter", int64(cs.RestartCount), map[string]string{
+		if t < restartCounterThreshold {
+			continue
+		}
+
+		mon.emitGauge("pod.restartcounter", int64(t), map[string]string{
+			"name":      p.Name,
+			"namespace": p.Namespace,
+		})
+
+		if mon.hourlyRun {
+			mon.log.WithFields(logrus.Fields{
+				"metric":    "pod.restartcounter",
 				"name":      p.Name,
 				"namespace": p.Namespace,
-			})
-
-			if mon.hourlyRun {
-				mon.log.WithFields(logrus.Fields{
-					"metric":    "pod.restartcounter",
-					"name":      p.Name,
-					"namespace": p.Namespace,
-				}).Print()
-			}
+			}).Print()
 		}
 	}
 }

pkg/monitor/cluster/podconditions_test.go

@@ -148,7 +148,7 @@ func TestEmitPodContainerStatuses(t *testing.T) {
 func TestEmitPodContainerRestartCounter(t *testing.T) {
 
 	cli := fake.NewSimpleClientset(
-		&corev1.Pod{ // metrics and log entry expected
+		&corev1.Pod{ // #1 metrics and log entry expected
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "podname1",
 				Namespace: "openshift",
@@ -165,7 +165,7 @@ func TestEmitPodContainerRestartCounter(t *testing.T) {
 				NodeName: "fake-node-name",
 			},
 		},
-		&corev1.Pod{ // no metrics expected
+		&corev1.Pod{ // #2 no metrics expected
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "podname2",
 				Namespace: "openshift",
@@ -182,7 +182,7 @@ func TestEmitPodContainerRestartCounter(t *testing.T) {
 				NodeName: "fake-node-name",
 			},
 		},
-		&corev1.Pod{ // metrics and log entry expected
+		&corev1.Pod{ // #3 metrics and log entry expected
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "podname3",
 				Namespace: "openshift",
@@ -199,7 +199,7 @@ func TestEmitPodContainerRestartCounter(t *testing.T) {
 				NodeName: "fake-node-name",
 			},
 		},
-		&corev1.Pod{ // no metrics expected
+		&corev1.Pod{ // #4 no metrics expected
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "podname4",
 				Namespace: "openshift",
@@ -216,7 +216,7 @@ func TestEmitPodContainerRestartCounter(t *testing.T) {
 				NodeName: "fake-node-name",
 			},
 		},
-		&corev1.Pod{ // no metrics expected
+		&corev1.Pod{ // #5 no metrics expected
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "not-system-namespace",
 				Namespace: "default",
@@ -233,6 +233,27 @@ func TestEmitPodContainerRestartCounter(t *testing.T) {
 				NodeName: "fake-node-name",
 			},
 		},
+		&corev1.Pod{ // #6 Multi-container pod
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "multi-container-pod",
+				Namespace: "openshift-test",
+			},
+			Status: corev1.PodStatus{
+				ContainerStatuses: []corev1.ContainerStatus{
+					{
+						Name:         "firstcontainer",
+						RestartCount: restartCounterThreshold,
+					},
+					{
+						Name:         "secondcontainer",
+						RestartCount: restartCounterThreshold,
+					},
+				},
+			},
+			Spec: corev1.PodSpec{
+				NodeName: "fake-node-name",
+			},
+		},
 	)
 
 	controller := gomock.NewController(t)
@@ -263,10 +284,16 @@ func TestEmitPodContainerRestartCounter(t *testing.T) {
 
 	// Expecting data for 'podname4' to be dropped
 
+	m.EXPECT().EmitGauge("pod.restartcounter", int64(restartCounterThreshold*2), map[string]string{
+		"name":      "multi-container-pod",
+		"namespace": "openshift-test",
+	})
+
 	ps, _ := cli.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
 	mon._emitPodContainerRestartCounter(ps)
 
-	assert.Equal(t, 2, len(hook.Entries))
+	// Matches the number of emitted messages
+	assert.Equal(t, 3, len(hook.Entries))
 
 	// the order of the log entries does not seem to be stable, so testing one entry only
 	// and no test for specific values, except for the metric