Merge pull request #218 from jim-minter/splitm

split service and cluster monitoring

cmd/aro/monitor.go

@@ -5,6 +5,8 @@ package main
 
 import (
 	"context"
+	"fmt"
+	"os"
 
 	uuid "github.com/satori/go.uuid"
 	"github.com/sirupsen/logrus"
@@ -20,27 +22,43 @@ func monitor(ctx context.Context, log *logrus.Entry) error {
 	uuid := uuid.NewV4().String()
 	log.Printf("uuid %s", uuid)
 
-	env, err := env.NewEnv(ctx, log)
+	_env, err := env.NewEnv(ctx, log)
 	if err != nil {
 		return err
 	}
 
-	m, err := statsd.New(ctx, log.WithField("component", "metrics"), env)
+	if _, ok := _env.(env.Dev); !ok {
+		for _, key := range []string{
+			"MDM_ACCOUNT",
+			"MDM_NAMESPACE",
+		} {
+			if _, found := os.LookupEnv(key); !found {
+				return fmt.Errorf("environment variable %q unset", key)
+			}
+		}
+	}
+
+	m, err := statsd.New(ctx, log.WithField("component", "metrics"), _env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"))
 	if err != nil {
 		return err
 	}
 
-	cipher, err := encryption.NewXChaCha20Poly1305(ctx, env)
+	clusterm, err := statsd.New(ctx, log.WithField("component", "metrics"), _env, os.Getenv("CLUSTER_MDM_ACCOUNT"), os.Getenv("CLUSTER_MDM_NAMESPACE"))
 	if err != nil {
 		return err
 	}
 
-	db, err := database.NewDatabase(ctx, log.WithField("component", "database"), env, m, cipher, uuid)
+	cipher, err := encryption.NewXChaCha20Poly1305(ctx, _env)
 	if err != nil {
 		return err
 	}
 
-	mon := pkgmonitor.NewMonitor(log.WithField("component", "monitor"), env, db, m)
+	db, err := database.NewDatabase(ctx, log.WithField("component", "database"), _env, m, cipher, uuid)
+	if err != nil {
+		return err
+	}
+
+	mon := pkgmonitor.NewMonitor(log.WithField("component", "monitor"), _env, db, m, clusterm)
 
 	return mon.Run(ctx)
 }

cmd/aro/rp.go

@@ -41,7 +41,7 @@ func rp(ctx context.Context, log *logrus.Entry) error {
 		return err
 	}
 
-	m, err := statsd.New(ctx, log.WithField("component", "metrics"), env)
+	m, err := statsd.New(ctx, log.WithField("component", "metrics"), env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"))
 	if err != nil {
 		return err
 	}

pkg/deploy/rp.go

@@ -519,8 +519,10 @@ WantedBy=multi-user.target
 EOF
 
 cat >/etc/sysconfig/aro-monitor <<EOF
-MDM_ACCOUNT='$CLUSTERMDMMONITORINGACCOUNT'
-MDM_NAMESPACE='$CLUSTERMDMMETRICNAMESPACE'
+MDM_ACCOUNT='$RPMDMMONITORINGACCOUNT'
+MDM_NAMESPACE='$RPMDMMETRICNAMESPACE'
+CLUSTER_MDM_ACCOUNT='$CLUSTERMDMMONITORINGACCOUNT'
+CLUSTER_MDM_NAMESPACE='$CLUSTERMDMMETRICNAMESPACE'
 RPIMAGE='$RPIMAGE'
 RP_MODE='$RPMODE'
 EOF

pkg/metrics/statsd/statsd.go

@@ -37,13 +37,13 @@ type statsd struct {
 }
 
 // New returns a new metrics.Interface
-func New(ctx context.Context, log *logrus.Entry, env env.Interface) (metrics.Interface, error) {
+func New(ctx context.Context, log *logrus.Entry, env env.Interface, account, namespace string) (metrics.Interface, error) {
 	s := &statsd{
 		log: log,
 		env: env,
 
-		account:   os.Getenv("MDM_ACCOUNT"),
-		namespace: os.Getenv("MDM_NAMESPACE"),
+		account:   account,
+		namespace: namespace,
 
 		ch: make(chan *metric, 1024),
 

pkg/monitor/monitor.go

@@ -21,12 +21,13 @@ import (
 )
 
 type monitor struct {
-	baseLog *logrus.Entry
-	env     env.Interface
-	db      *database.Database
-	m       metrics.Interface
-	mu      sync.Mutex
-	docs    sync.Map
+	baseLog  *logrus.Entry
+	env      env.Interface
+	db       *database.Database
+	m        metrics.Interface
+	clusterm metrics.Interface
+	mu       sync.Mutex
+	docs     sync.Map
 
 	isMaster    bool
 	bucketCount int
@@ -39,12 +40,13 @@ type Runnable interface {
 	Run(context.Context) error
 }
 
-func NewMonitor(log *logrus.Entry, env env.Interface, db *database.Database, m metrics.Interface) Runnable {
+func NewMonitor(log *logrus.Entry, env env.Interface, db *database.Database, m, clusterm metrics.Interface) Runnable {
 	return &monitor{
-		baseLog: log,
-		env:     env,
-		db:      db,
-		m:       m,
+		baseLog:  log,
+		env:      env,
+		db:       db,
+		m:        m,
+		clusterm: clusterm,
 
 		bucketCount: bucket.Buckets,
 		buckets:     map[int]struct{}{},

pkg/monitor/worker.go

@@ -161,7 +161,7 @@ func (mon *monitor) workOne(ctx context.Context, log *logrus.Entry, doc *api.Ope
 		return err
 	}
 
-	mon.m.EmitGauge("monitoring.apiserver.health", 1, map[string]string{
+	mon.clusterm.EmitGauge("monitoring.apiserver.health", 1, map[string]string{
 		"resource": doc.OpenShiftCluster.ID,
 		"code":     strconv.FormatInt(int64(statusCode), 10),
 	})