From 066e88eabc8a3ed180651159fe3f1ce5736a403e Mon Sep 17 00:00:00 2001 From: Michael Demmer Date: Mon, 30 Apr 2018 14:39:26 -0700 Subject: [PATCH] add prometheus support for stats.NewHistogram Signed-off-by: Michael Demmer --- go/stats/histogram.go | 19 +++++- go/stats/histogram_test.go | 5 +- go/stats/prometheusbackend/collectors.go | 58 ++++++++++++++++--- .../prometheusbackend/prometheusbackend.go | 16 ++++- .../prometheusbackend_test.go | 23 ++++++++ go/stats/timings.go | 4 +- .../tabletserver/tabletenv/tabletenv.go | 4 +- 7 files changed, 110 insertions(+), 19 deletions(-) diff --git a/go/stats/histogram.go b/go/stats/histogram.go index 993f321a74..b12f3235ff 100644 --- a/go/stats/histogram.go +++ b/go/stats/histogram.go @@ -27,6 +27,8 @@ import ( // splitting the counts under different buckets // using specified cutoffs. type Histogram struct { + help string + labelName string cutoffs []int64 labels []string countLabel string @@ -41,24 +43,25 @@ type Histogram struct { // based on the cutoffs. The buckets are categorized using the // following criterion: cutoff[i-1] < value <= cutoff[i]. Anything // higher than the highest cutoff is labeled as "inf". -func NewHistogram(name string, cutoffs []int64) *Histogram { +func NewHistogram(name, help string, cutoffs []int64) *Histogram { labels := make([]string, len(cutoffs)+1) for i, v := range cutoffs { labels[i] = fmt.Sprintf("%d", v) } labels[len(labels)-1] = "inf" - return NewGenericHistogram(name, cutoffs, labels, "Count", "Total") + return NewGenericHistogram(name, help, cutoffs, labels, "Count", "Total") } // NewGenericHistogram creates a histogram where all the labels are // supplied by the caller. The number of labels has to be one more than // the number of cutoffs because the last label captures everything that // exceeds the highest cutoff. -func NewGenericHistogram(name string, cutoffs []int64, labels []string, countLabel, totalLabel string) *Histogram { +func NewGenericHistogram(name, help string, cutoffs []int64, labels []string, countLabel, totalLabel string) *Histogram { if len(cutoffs) != len(labels)-1 { panic("mismatched cutoff and label lengths") } h := &Histogram{ + help: help, cutoffs: cutoffs, labels: labels, countLabel: countLabel, @@ -160,3 +163,13 @@ func (h *Histogram) Buckets() []int64 { } return buckets } + +// Help returns the help string. +func (h *Histogram) Help() string { + return h.help +} + +// LabelName returns the label name. +func (h *Histogram) LabelName() string { + return h.labelName +} diff --git a/go/stats/histogram_test.go b/go/stats/histogram_test.go index f0cb07105e..a0e577e79f 100644 --- a/go/stats/histogram_test.go +++ b/go/stats/histogram_test.go @@ -23,7 +23,7 @@ import ( func TestHistogram(t *testing.T) { clear() - h := NewHistogram("hist1", []int64{1, 5}) + h := NewHistogram("hist1", "desc1", []int64{1, 5}) for i := 0; i < 10; i++ { h.Add(int64(i)) } @@ -57,6 +57,7 @@ func TestGenericHistogram(t *testing.T) { clear() h := NewGenericHistogram( "histgen", + "generic histogram", []int64{1, 5}, []string{"one", "five", "max"}, "count", @@ -78,7 +79,7 @@ func TestHistogramHook(t *testing.T) { }) name := "hist2" - v := NewHistogram(name, []int64{1}) + v := NewHistogram(name, "", []int64{1}) if gotname != name { t.Errorf("got %v; want %v", gotname, name) } diff --git a/go/stats/prometheusbackend/collectors.go b/go/stats/prometheusbackend/collectors.go index 9cc2254bcf..b48d9f967c 100644 --- a/go/stats/prometheusbackend/collectors.go +++ b/go/stats/prometheusbackend/collectors.go @@ -129,8 +129,9 @@ func (c *metricsFuncWithMultiLabelsCollector) Collect(ch chan<- prometheus.Metri } type timingsCollector struct { - t *stats.Timings - desc *prometheus.Desc + t *stats.Timings + cutoffs []float64 + desc *prometheus.Desc } // Describe implements Collector. @@ -145,16 +146,15 @@ func (c *timingsCollector) Collect(ch chan<- prometheus.Metric) { c.desc, uint64(his.Count()), float64(his.Total()), - makePromBucket(his.Cutoffs(), his.Buckets()), + makeCumulativeBuckets(c.cutoffs, his.Buckets()), cat) } } -func makePromBucket(cutoffs []int64, buckets []int64) map[float64]uint64 { +func makeCumulativeBuckets(cutoffs []float64, buckets []int64) map[float64]uint64 { output := make(map[float64]uint64) last := uint64(0) - for i := range cutoffs { - key := float64(cutoffs[i]) / 1000000000 + for i, key := range cutoffs { //TODO(zmagg): int64 => uint64 conversion. error if it overflows? output[key] = uint64(buckets[i]) + last last = output[key] @@ -163,8 +163,9 @@ func makePromBucket(cutoffs []int64, buckets []int64) map[float64]uint64 { } type multiTimingsCollector struct { - mt *stats.MultiTimings - desc *prometheus.Desc + mt *stats.MultiTimings + cutoffs []float64 + desc *prometheus.Desc } // Describe implements Collector. @@ -180,7 +181,46 @@ func (c *multiTimingsCollector) Collect(ch chan<- prometheus.Metric) { c.desc, uint64(his.Count()), float64(his.Total()), - makePromBucket(his.Cutoffs(), his.Buckets()), + makeCumulativeBuckets(c.cutoffs, his.Buckets()), labelValues...) } } + +type histogramCollector struct { + h *stats.Histogram + cutoffs []float64 + desc *prometheus.Desc +} + +func newHistogramCollector(h *stats.Histogram, name string) { + collector := &histogramCollector{ + h: h, + cutoffs: make([]float64, len(h.Cutoffs())), + desc: prometheus.NewDesc( + name, + h.Help(), + []string{}, + nil), + } + + for i, val := range h.Cutoffs() { + collector.cutoffs[i] = float64(val) + } + + prometheus.MustRegister(collector) +} + +// Describe implements Collector. +func (c *histogramCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.desc +} + +// Collect implements Collector. +func (c *histogramCollector) Collect(ch chan<- prometheus.Metric) { + ch <- prometheus.MustNewConstHistogram( + c.desc, + uint64(c.h.Count()), + float64(c.h.Total()), + makeCumulativeBuckets(c.cutoffs, c.h.Buckets()), + ) +} diff --git a/go/stats/prometheusbackend/prometheusbackend.go b/go/stats/prometheusbackend/prometheusbackend.go index 1bfb822f05..bbe00dca82 100644 --- a/go/stats/prometheusbackend/prometheusbackend.go +++ b/go/stats/prometheusbackend/prometheusbackend.go @@ -65,6 +65,8 @@ func (be *PromBackend) publishPrometheusMetric(name string, v expvar.Var) { be.newTiming(st, name) case *stats.MultiTimings: be.newMultiTiming(st, name) + case *stats.Histogram: + newHistogramCollector(st, be.buildPromName(name)) default: logUnsupported.Infof("Not exporting to Prometheus an unsupported metric type of %T: %s", st, name) } @@ -138,7 +140,8 @@ func (be *PromBackend) newMetricsFuncWithMultiLabels(cfml *stats.CountersFuncWit func (be *PromBackend) newTiming(t *stats.Timings, name string) { collector := &timingsCollector{ - t: t, + t: t, + cutoffs: make([]float64, len(t.Cutoffs())), desc: prometheus.NewDesc( be.buildPromName(name), t.Help(), @@ -146,12 +149,17 @@ func (be *PromBackend) newTiming(t *stats.Timings, name string) { nil), } + for i, val := range t.Cutoffs() { + collector.cutoffs[i] = float64(val) / 1000000000 + } + prometheus.MustRegister(collector) } func (be *PromBackend) newMultiTiming(mt *stats.MultiTimings, name string) { collector := &multiTimingsCollector{ - mt: mt, + mt: mt, + cutoffs: make([]float64, len(mt.Cutoffs())), desc: prometheus.NewDesc( be.buildPromName(name), mt.Help(), @@ -159,6 +167,10 @@ func (be *PromBackend) newMultiTiming(mt *stats.MultiTimings, name string) { nil), } + for i, val := range mt.Cutoffs() { + collector.cutoffs[i] = float64(val) / 1000000000 + } + prometheus.MustRegister(collector) } diff --git a/go/stats/prometheusbackend/prometheusbackend_test.go b/go/stats/prometheusbackend/prometheusbackend_test.go index 61f5dbe12f..354202c9d9 100644 --- a/go/stats/prometheusbackend/prometheusbackend_test.go +++ b/go/stats/prometheusbackend/prometheusbackend_test.go @@ -290,6 +290,29 @@ func TestPrometheusMultiTimings_PanicWrongLength(t *testing.T) { c.Add([]string{"label1"}, time.Duration(100000000)) } +func TestPrometheusHistogram(t *testing.T) { + name := "blah_hist" + hist := stats.NewHistogram(name, "help", []int64{1, 5, 10}) + hist.Add(2) + hist.Add(3) + hist.Add(6) + + response := testMetricsHandler(t) + var s []string + + s = append(s, fmt.Sprintf("%s_%s_bucket{le=\"1\"} %d", namespace, name, 0)) + s = append(s, fmt.Sprintf("%s_%s_bucket{le=\"5\"} %d", namespace, name, 2)) + s = append(s, fmt.Sprintf("%s_%s_bucket{le=\"10\"} %d", namespace, name, 3)) + s = append(s, fmt.Sprintf("%s_%s_sum %d", namespace, name, 1)) + s = append(s, fmt.Sprintf("%s_%s_count %d", namespace, name, 3)) + + for _, line := range s { + if !strings.Contains(response.Body.String(), line) { + t.Fatalf("Expected result to contain %s, got %s", line, response.Body.String()) + } + } +} + func testMetricsHandler(t *testing.T) *httptest.ResponseRecorder { req, _ := http.NewRequest("GET", "/metrics", nil) response := httptest.NewRecorder() diff --git a/go/stats/timings.go b/go/stats/timings.go index 37a662eb91..1077b4b7db 100644 --- a/go/stats/timings.go +++ b/go/stats/timings.go @@ -52,7 +52,7 @@ func NewTimings(name, help, labelName string, categories ...string) *Timings { labelName: labelName, } for _, cat := range categories { - t.histograms[cat] = NewGenericHistogram("", bucketCutoffs, bucketLabels, "Count", "Time") + t.histograms[cat] = NewGenericHistogram("", "", bucketCutoffs, bucketLabels, "Count", "Time") } if name != "" { publish(name, t) @@ -74,7 +74,7 @@ func (t *Timings) Add(name string, elapsed time.Duration) { t.mu.Lock() hist, ok = t.histograms[name] if !ok { - hist = NewGenericHistogram("", bucketCutoffs, bucketLabels, "Count", "Time") + hist = NewGenericHistogram("", "", bucketCutoffs, bucketLabels, "Count", "Time") t.histograms[name] = hist } t.mu.Unlock() diff --git a/go/vt/vttablet/tabletserver/tabletenv/tabletenv.go b/go/vt/vttablet/tabletserver/tabletenv/tabletenv.go index cc65b57ce9..0fd1dfc774 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/tabletenv.go +++ b/go/vt/vttablet/tabletserver/tabletenv/tabletenv.go @@ -93,7 +93,9 @@ var ( "Total transaction latency for each CallerID", []string{"CallerID", "Conclusion"}) // ResultStats shows the histogram of number of rows returned. - ResultStats = stats.NewHistogram("Results", []int64{0, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000}) + ResultStats = stats.NewHistogram("Results", + "Distribution of rows returned", + []int64{0, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000}) // TableaclAllowed tracks the number allows. TableaclAllowed = stats.NewCountersWithMultiLabels( "TableACLAllowed",