Sloop keys histogram (#109)

Added Sloop keys Histogram for debug support

pkg/sloop/common/utilities.go

@@ -0,0 +1,25 @@
+package common
+
+import (
+	"fmt"
+	"strings"
+)
+
+func BoolToFloat(value bool) float64 {
+	if value {
+		return 1
+	}
+	return 0
+}
+
+func ParseKey(key string) (error, []string) {
+	parts := strings.Split(key, "/")
+	if len(parts) != 7 {
+		return fmt.Errorf("key should have 6 parts: %v", key), parts
+	}
+	if parts[0] != "" {
+		return fmt.Errorf("key should start with /: %v", key), parts
+	}
+
+	return nil, parts
+}

pkg/sloop/common/utilities_test.go

@@ -0,0 +1,37 @@
+package common
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func Test_boolToFloat(t *testing.T) {
+	assert.Equal(t, float64(1), BoolToFloat(true))
+	assert.Equal(t, float64(0), BoolToFloat(false))
+}
+
+func Test_ParseKey_2_Parts(t *testing.T) {
+	keyWith2Parts := "/part1/part2"
+	err, _ := ParseKey(keyWith2Parts)
+
+	assert.NotNil(t, err)
+	assert.Equal(t, fmt.Errorf("key should have 6 parts: %v", keyWith2Parts), err)
+}
+
+func Test_ParseKey_Start_Parts(t *testing.T) {
+	keyWith2Parts := "part1/part2/part3/part4/part5/part6/part7"
+	err, _ := ParseKey(keyWith2Parts)
+
+	assert.NotNil(t, err)
+	assert.Equal(t, fmt.Errorf("key should start with /: %v", keyWith2Parts), err)
+}
+
+
+func Test_ParseKey_Success(t *testing.T) {
+	keyWith2Parts := "/part1/part2/part3/part4/part5/part6"
+	err, parts := ParseKey(keyWith2Parts)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 7, len(parts))
+}

pkg/sloop/store/typed/eventcounttable.go

@@ -10,9 +10,9 @@ package typed
 import (
 	"fmt"
 	badger "github.com/dgraph-io/badger/v2"
+	"github.com/salesforce/sloop/pkg/sloop/common"
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped"
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap"
-	"strings"
 	"time"
 )
 
@@ -38,13 +38,11 @@ func (*EventCountKey) TableName() string {
 }
 
 func (k *EventCountKey) Parse(key string) error {
-	parts := strings.Split(key, "/")
-	if len(parts) != 7 {
-		return fmt.Errorf("Key should have 6 parts: %v", key)
-	}
-	if parts[0] != "" {
-		return fmt.Errorf("Key should start with /: %v", key)
+	err, parts := common.ParseKey(key)
+	if err != nil {
+		return err
 	}
+
 	if parts[1] != k.TableName() {
 		return fmt.Errorf("Second part of key (%v) should be %v", key, k.TableName())
 	}

pkg/sloop/store/typed/resourcesummarytable.go

@@ -9,8 +9,8 @@ package typed
 
 import (
 	"fmt"
+	"github.com/salesforce/sloop/pkg/sloop/common"
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped"
-	"strings"
 	"time"
 )
 
@@ -44,13 +44,11 @@ func (*ResourceSummaryKey) TableName() string {
 }
 
 func (k *ResourceSummaryKey) Parse(key string) error {
-	parts := strings.Split(key, "/")
-	if len(parts) != 7 {
-		return fmt.Errorf("Key should have 6 parts: %v", key)
-	}
-	if parts[0] != "" {
-		return fmt.Errorf("Key should start with /: %v", key)
+	err, parts := common.ParseKey(key)
+	if err != nil {
+		return err
 	}
+
 	if parts[1] != k.TableName() {
 		return fmt.Errorf("Second part of key (%v) should be %v", key, k.TableName())
 	}

pkg/sloop/store/typed/watchactivitytable.go

@@ -10,8 +10,8 @@ package typed
 import (
 	"fmt"
 	"github.com/dgraph-io/badger/v2"
+	"github.com/salesforce/sloop/pkg/sloop/common"
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap"
-	"strings"
 )
 
 // Key is /<partition>/<kind>/<namespace>/<name>
@@ -42,13 +42,11 @@ func (*WatchActivityKey) TableName() string {
 }
 
 func (k *WatchActivityKey) Parse(key string) error {
-	parts := strings.Split(key, "/")
-	if len(parts) != 7 {
-		return fmt.Errorf("Key should have 6 parts: %v", key)
-	}
-	if parts[0] != "" {
-		return fmt.Errorf("Key should start with /: %v", key)
+	err, parts := common.ParseKey(key)
+	if err != nil {
+		return err
 	}
+
 	if parts[1] != k.TableName() {
 		return fmt.Errorf("Second part of key (%v) should be %v", key, k.TableName())
 	}

pkg/sloop/store/typed/watchtable.go

@@ -10,8 +10,8 @@ package typed
 import (
 	"fmt"
 	"github.com/pkg/errors"
+	"github.com/salesforce/sloop/pkg/sloop/common"
 	"strconv"
-	"strings"
 	"time"
 )
 
@@ -44,13 +44,11 @@ func (*WatchTableKey) TableName() string {
 }
 
 func (k *WatchTableKey) Parse(key string) error {
-	parts := strings.Split(key, "/")
-	if len(parts) != 7 {
-		return fmt.Errorf("Key should have 6 parts: %v", key)
-	}
-	if parts[0] != "" {
-		return fmt.Errorf("Key should start with /: %v", key)
+	err, parts := common.ParseKey(key)
+	if err != nil {
+		return err
 	}
+
 	if parts[1] != k.TableName() {
 		return fmt.Errorf("Second part of key (%v) should be %v", key, k.TableName())
 	}

pkg/sloop/store/untyped/badgerwrap/api.go

@@ -69,9 +69,9 @@ type Item interface {
 	Value(fn func(val []byte) error) error
 	ValueCopy(dst []byte) ([]byte, error)
 	//	DiscardEarlierVersions() bool
-	//	EstimatedSize() int64
+	EstimatedSize() int64
 	//	ExpiresAt() uint64
-	//	IsDeletedOrExpired() bool
+	IsDeletedOrExpired() bool
 	KeyCopy(dst []byte) []byte
 	//	KeySize() int64
 	//	String() string

pkg/sloop/store/untyped/badgerwrap/badger.go

@@ -128,6 +128,14 @@ func (i *BadgerItem) KeyCopy(dst []byte) []byte {
 	return i.item.KeyCopy(dst)
 }
 
+func (i *BadgerItem) EstimatedSize() int64 {
+	return i.item.EstimatedSize()
+}
+
+func (i *BadgerItem) IsDeletedOrExpired() bool {
+	return i.item.IsDeletedOrExpired()
+}
+
 // Iterator
 
 func (i *BadgerIterator) Close() {

pkg/sloop/store/untyped/badgerwrap/mock.go

@@ -179,6 +179,14 @@ func (i *MockItem) ValueCopy(dst []byte) ([]byte, error) {
 	return newcopy, nil
 }
 
+func (i *MockItem) EstimatedSize() int64 {
+	return int64(len(i.key) + len(i.value))
+}
+
+func (i *MockItem) IsDeletedOrExpired() bool {
+	return false
+}
+
 func (i *MockItem) KeyCopy(dst []byte) []byte {
 	copy(dst, i.key)
 	newcopy := make([]byte, len(i.key))

pkg/sloop/storemanager/storemanager.go

@@ -12,6 +12,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/salesforce/sloop/pkg/sloop/common"
 	"github.com/salesforce/sloop/pkg/sloop/store/typed"
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped"
 	"github.com/spf13/afero"
@@ -97,7 +98,7 @@ func (sm *StoreManager) gcLoop() {
 		before := time.Now()
 		metricGcRunning.Set(1)
 		cleanUpPerformed, numOfDeletedKeys, numOfKeysToDelete, err := doCleanup(sm.tables, sm.config.TimeLimit, sm.config.SizeLimitBytes, sm.stats, sm.config.DeletionBatchSize)
-		metricGcCleanUpPerformed.Set(boolToFloat(cleanUpPerformed))
+		metricGcCleanUpPerformed.Set(common.BoolToFloat(cleanUpPerformed))
 		metricGcDeletedNumberOfKeys.Set(numOfDeletedKeys)
 		metricGcNumberOfKeysToDelete.Set(numOfKeysToDelete)
 		metricGcRunning.Set(0)
@@ -177,23 +178,22 @@ func doCleanup(tables typed.Tables, timeLimit time.Duration, sizeLimitBytes int,
 		return false, 0, 0, nil
 	}
 
+	minPartitionAge, err := untyped.GetAgeOfPartitionInHours(minPartition)
+	if err == nil {
+		metricAgeOfMinimumPartition.Set(minPartitionAge)
+	}
+
+	maxPartitionAge, err := untyped.GetAgeOfPartitionInHours(maxPartition)
+	if err == nil {
+		metricAgeOfMaximumPartition.Set(maxPartitionAge)
+	}
+
 	var totalNumOfDeletedKeys float64 = 0
 	var totalNumOfKeysToDelete float64 = 0
 	anyCleanupPerformed := false
 	if cleanUpTimeCondition(minPartition, maxPartition, timeLimit) || cleanUpFileSizeCondition(stats, sizeLimitBytes) {
 		partStart, partEnd, err := untyped.GetTimeRangeForPartition(minPartition)
 		glog.Infof("GC removing partition %q with data from %v to %v (err %v)", minPartition, partStart, partEnd, err)
-		minPartitionAge := 0.0
-		minPartitionAge, err = untyped.GetAgeOfPartitionInHours(minPartition)
-		if err != nil {
-			metricAgeOfMinimumPartition.Set(minPartitionAge)
-		}
-
-		maxPartitionAge, err := untyped.GetAgeOfPartitionInHours(maxPartition)
-		if err != nil {
-			metricAgeOfMaximumPartition.Set(maxPartitionAge)
-		}
-
 		var errMessages []string
 		for _, tableName := range tables.GetTableNames() {
 			prefix := fmt.Sprintf("/%s/%s", tableName, minPartition)

pkg/sloop/storemanager/utilities.go

@@ -1,8 +0,0 @@
-package storemanager
-
-func boolToFloat(value bool) float64 {
-	if value {
-		return 1
-	}
-	return 0
-}

pkg/sloop/storemanager/utilities_test.go

@@ -1,11 +0,0 @@
-package storemanager
-
-import (
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-func Test_boolToFloat(t *testing.T) {
-	assert.Equal(t, float64(1), boolToFloat(true))
-	assert.Equal(t, float64(0), boolToFloat(false))
-}

pkg/sloop/webfiles/debug.html

@@ -16,6 +16,7 @@ For full license text, see LICENSE.txt file in the repo root or https://opensour
 
 <ul>
     <li><a href="/debug/listkeys/">Query Data Store</a> - Allows you to query keys and values from the badger store</li>
+    <li><a href="/debug/histogram/">Sloop Keys Histogram</a> - View the keys histogram</li>
     <li><a href="/debug/config/">Config</a> - View the current active config for Sloop</li>
     <li><a href="/debug/tables/">Tables</a> - View Badger LSM Table Info</li>
     <li><a href="/debug/requests">Badger Requests</a></li>

pkg/sloop/webfiles/debughistogram.html

@@ -0,0 +1,49 @@
+<!--
+Copyright (c) 2020, salesforce.com, inc.
+All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+-->
+<html>
+<head>
+    <title>Sloop Keys Histogram</title>
+    <link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
+</head>
+<body>
+[ <a href="/">Home</a> ][ <a href="/debug/">Debug Menu</a> ]<br/>
+
+<h2>Sloop Keys Histogram</h2>
+
+<table bgcolor="#f5f5f5" width="500px"><tr><td style="padding: 20px">
+<form action="/debug/histogram/" method="get">
+
+    <label for="prefix">Prefix of keys to filter: </label><br><br>
+    <input type="text" name="prefix" id="prefix"><br><br>
+    <label> Enter * to get all keys histogram </label><br><br>
+    <label> Format of key is /tablename/partitionId/ </label><br><br><br>
+    <input type="submit">
+</form>
+</td></tr></table>
+<br/>
+
+<table border="1">
+    <tr><td>Total keys</td><td>{{.TotalKeys}}</td></tr>
+    <tr><td>Deleted Keys</td><td><pre>{{.DeletedKeys}}</pre></td></tr>
+</table>
+
+<br/>
+
+<b>Partitions List</b>:<br/>
+<br/>
+<table border="1">
+    <tr><td>Table</td><td>Partition ID</td><td>Number of Keys</td><td>Estimated Size</td><td>Minimum Size</td><td>Maximum Size</td><td>Average Size</td></tr>
+    {{range $key, $value := .HistogramMap}}
+        <tr><td>{{$key.TableName}}</td><td>{{$key.PartitionID}}</td><td>{{$value.TotalKeys}}</td><td>{{$value.TotalSize}}</td><td>{{$value.MinimumSize}}</td><td>{{$value.MaximumSize}}</td><td>{{$value.AverageSize}}</td></tr>
+    {{end}}
+</table>
+</body>
+
+<script src="/webfiles/filter.js"></script>
+<script>
+</script>
+</html>

pkg/sloop/webserver/debug.go

@@ -13,6 +13,8 @@ import (
 	"fmt"
 	"github.com/dgraph-io/badger/v2"
 	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"github.com/salesforce/sloop/pkg/sloop/common"
 	"github.com/salesforce/sloop/pkg/sloop/store/typed"
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap"
 	"html/template"
@@ -162,6 +164,116 @@ func listKeysHandler(tables typed.Tables) http.HandlerFunc {
 	}
 }
 
+type sloopKeyInfo struct {
+	MinimumSize int64
+	MaximumSize int64
+	TotalKeys   int64
+	TotalSize   int64
+	AverageSize int64
+}
+
+type sloopKey struct {
+	TableName   string
+	PartitionID string
+}
+
+type histogram struct {
+	HistogramMap map[sloopKey]*sloopKeyInfo
+	TotalKeys    int
+	DeletedKeys  int
+}
+
+// returns TableName, PartitionId, error.
+func parseSloopKey(item badgerwrap.Item) (string, string, error) {
+	key := item.Key()
+	err, parts := common.ParseKey(string(key))
+	if err != nil {
+		return "", "", err
+	}
+
+	var tableName = parts[1]
+	var partitionId = parts[2]
+	return tableName, partitionId, nil
+}
+
+func histogramHandler(tables typed.Tables) http.HandlerFunc {
+	return func(writer http.ResponseWriter, request *http.Request) {
+		var result histogram
+		prefix := request.URL.Query().Get("prefix")
+		if len(prefix) > 0 {
+
+			if prefix == "*" {
+				prefix = ""
+			}
+
+			err := tables.Db().View(func(txn badgerwrap.Txn) error {
+				iterOpt := badger.DefaultIteratorOptions
+				iterOpt.Prefix = []byte(prefix)
+				iterOpt.PrefetchValues = false
+				itr := txn.NewIterator(iterOpt)
+				defer itr.Close()
+
+				totalKeys := 0
+				totalDeletedExpiredKeys := 0
+				var sloopMap = make(map[sloopKey]*sloopKeyInfo)
+				for itr.Rewind(); itr.Valid(); itr.Next() {
+					item := itr.Item()
+					tableName, partitionId, err := parseSloopKey(item)
+					if err != nil {
+						return errors.Wrapf(err, "failed to parse information about key: %x",
+							item.Key())
+					}
+					totalKeys++
+
+					if item.IsDeletedOrExpired() {
+						totalDeletedExpiredKeys++
+					}
+
+					size := item.EstimatedSize()
+					sloopKey := sloopKey{tableName, partitionId}
+					if sloopMap[sloopKey] == nil {
+						sloopMap[sloopKey] = &sloopKeyInfo{size, size, 1, size, size}
+					} else {
+						sloopMap[sloopKey].TotalKeys++
+						sloopMap[sloopKey].TotalSize += size
+						sloopMap[sloopKey].AverageSize = sloopMap[sloopKey].TotalSize / sloopMap[sloopKey].TotalKeys
+						if size < sloopMap[sloopKey].MinimumSize {
+							sloopMap[sloopKey].MinimumSize = size
+						}
+
+						if size > sloopMap[sloopKey].MaximumSize {
+							sloopMap[sloopKey].MaximumSize = size
+						}
+					}
+				}
+
+				result.TotalKeys = totalKeys
+				result.DeletedKeys = totalDeletedExpiredKeys
+				result.HistogramMap = sloopMap
+				return nil
+			})
+
+			if err != nil {
+				logWebError(err, "Could not get histogram", request, writer)
+				return
+			}
+		}
+		writer.Header().Set("content-type", "text/html")
+
+		t, err := template.New(debugHistogramFile).ParseFiles(path.Join(webFiles, debugHistogramFile))
+		if err != nil {
+			logWebError(err, "failed to parse histogram template", request, writer)
+			return
+		}
+
+		err = t.ExecuteTemplate(writer, debugHistogramFile, result)
+		if err != nil {
+			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
+			return
+		}
+	}
+}
+
 func configHandler(config string) http.HandlerFunc {
 	return func(writer http.ResponseWriter, request *http.Request) {
 		t, err := template.New(debugConfigTemplateFile).ParseFiles(path.Join(webFiles, debugConfigTemplateFile))

pkg/sloop/webserver/webserver.go

@@ -40,6 +40,7 @@ import (
 const (
 	debugViewKeyTemplateFile      = "debugviewkey.html"
 	debugListKeysTemplateFile     = "debuglistkeys.html"
+	debugHistogramFile            = "debughistogram.html"
 	debugConfigTemplateFile       = "debugconfig.html"
 	debugTemplateFile             = "debug.html"
 	debugBadgerTablesTemplateFile = "debugtables.html"
@@ -176,6 +177,7 @@ func Run(config WebConfig, tables typed.Tables) error {
 	// Debug pages
 	server.mux.HandleFunc("/debug/", debugHandler())
 	server.mux.HandleFunc("/debug/listkeys/", listKeysHandler(tables))
+	server.mux.HandleFunc("/debug/histogram/", histogramHandler(tables))
 	server.mux.HandleFunc("/debug/tables/", debugBadgerTablesHandler(tables.Db()))
 	server.mux.HandleFunc("/debug/view/", viewKeyHandler(tables))
 	server.mux.HandleFunc("/debug/config/", configHandler(config.ConfigYaml))