diff --git a/pkg/sloop/common/utilities.go b/pkg/sloop/common/utilities.go new file mode 100644 index 0000000..ec4ef49 --- /dev/null +++ b/pkg/sloop/common/utilities.go @@ -0,0 +1,25 @@ +package common + +import ( + "fmt" + "strings" +) + +func BoolToFloat(value bool) float64 { + if value { + return 1 + } + return 0 +} + +func ParseKey(key string) (error, []string) { + parts := strings.Split(key, "/") + if len(parts) != 7 { + return fmt.Errorf("key should have 6 parts: %v", key), parts + } + if parts[0] != "" { + return fmt.Errorf("key should start with /: %v", key), parts + } + + return nil, parts +} diff --git a/pkg/sloop/common/utilities_test.go b/pkg/sloop/common/utilities_test.go new file mode 100644 index 0000000..c7c8275 --- /dev/null +++ b/pkg/sloop/common/utilities_test.go @@ -0,0 +1,37 @@ +package common + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_boolToFloat(t *testing.T) { + assert.Equal(t, float64(1), BoolToFloat(true)) + assert.Equal(t, float64(0), BoolToFloat(false)) +} + +func Test_ParseKey_2_Parts(t *testing.T) { + keyWith2Parts := "/part1/part2" + err, _ := ParseKey(keyWith2Parts) + + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("key should have 6 parts: %v", keyWith2Parts), err) +} + +func Test_ParseKey_Start_Parts(t *testing.T) { + keyWith2Parts := "part1/part2/part3/part4/part5/part6/part7" + err, _ := ParseKey(keyWith2Parts) + + assert.NotNil(t, err) + assert.Equal(t, fmt.Errorf("key should start with /: %v", keyWith2Parts), err) +} + + +func Test_ParseKey_Success(t *testing.T) { + keyWith2Parts := "/part1/part2/part3/part4/part5/part6" + err, parts := ParseKey(keyWith2Parts) + + assert.Nil(t, err) + assert.Equal(t, 7, len(parts)) +} \ No newline at end of file diff --git a/pkg/sloop/store/typed/eventcounttable.go b/pkg/sloop/store/typed/eventcounttable.go index 3602217..fa18214 100644 --- a/pkg/sloop/store/typed/eventcounttable.go +++ b/pkg/sloop/store/typed/eventcounttable.go @@ -10,9 +10,9 @@ package typed import ( "fmt" badger "github.com/dgraph-io/badger/v2" + "github.com/salesforce/sloop/pkg/sloop/common" "github.com/salesforce/sloop/pkg/sloop/store/untyped" "github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap" - "strings" "time" ) @@ -38,13 +38,11 @@ func (*EventCountKey) TableName() string { } func (k *EventCountKey) Parse(key string) error { - parts := strings.Split(key, "/") - if len(parts) != 7 { - return fmt.Errorf("Key should have 6 parts: %v", key) - } - if parts[0] != "" { - return fmt.Errorf("Key should start with /: %v", key) + err, parts := common.ParseKey(key) + if err != nil { + return err } + if parts[1] != k.TableName() { return fmt.Errorf("Second part of key (%v) should be %v", key, k.TableName()) } diff --git a/pkg/sloop/store/typed/resourcesummarytable.go b/pkg/sloop/store/typed/resourcesummarytable.go index 1df0f65..8c21f20 100644 --- a/pkg/sloop/store/typed/resourcesummarytable.go +++ b/pkg/sloop/store/typed/resourcesummarytable.go @@ -9,8 +9,8 @@ package typed import ( "fmt" + "github.com/salesforce/sloop/pkg/sloop/common" "github.com/salesforce/sloop/pkg/sloop/store/untyped" - "strings" "time" ) @@ -44,13 +44,11 @@ func (*ResourceSummaryKey) TableName() string { } func (k *ResourceSummaryKey) Parse(key string) error { - parts := strings.Split(key, "/") - if len(parts) != 7 { - return fmt.Errorf("Key should have 6 parts: %v", key) - } - if parts[0] != "" { - return fmt.Errorf("Key should start with /: %v", key) + err, parts := common.ParseKey(key) + if err != nil { + return err } + if parts[1] != k.TableName() { return fmt.Errorf("Second part of key (%v) should be %v", key, k.TableName()) } diff --git a/pkg/sloop/store/typed/watchactivitytable.go b/pkg/sloop/store/typed/watchactivitytable.go index eb38b94..a287396 100644 --- a/pkg/sloop/store/typed/watchactivitytable.go +++ b/pkg/sloop/store/typed/watchactivitytable.go @@ -10,8 +10,8 @@ package typed import ( "fmt" "github.com/dgraph-io/badger/v2" + "github.com/salesforce/sloop/pkg/sloop/common" "github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap" - "strings" ) // Key is //// @@ -42,13 +42,11 @@ func (*WatchActivityKey) TableName() string { } func (k *WatchActivityKey) Parse(key string) error { - parts := strings.Split(key, "/") - if len(parts) != 7 { - return fmt.Errorf("Key should have 6 parts: %v", key) - } - if parts[0] != "" { - return fmt.Errorf("Key should start with /: %v", key) + err, parts := common.ParseKey(key) + if err != nil { + return err } + if parts[1] != k.TableName() { return fmt.Errorf("Second part of key (%v) should be %v", key, k.TableName()) } diff --git a/pkg/sloop/store/typed/watchtable.go b/pkg/sloop/store/typed/watchtable.go index 98f8847..0c952d7 100644 --- a/pkg/sloop/store/typed/watchtable.go +++ b/pkg/sloop/store/typed/watchtable.go @@ -10,8 +10,8 @@ package typed import ( "fmt" "github.com/pkg/errors" + "github.com/salesforce/sloop/pkg/sloop/common" "strconv" - "strings" "time" ) @@ -44,13 +44,11 @@ func (*WatchTableKey) TableName() string { } func (k *WatchTableKey) Parse(key string) error { - parts := strings.Split(key, "/") - if len(parts) != 7 { - return fmt.Errorf("Key should have 6 parts: %v", key) - } - if parts[0] != "" { - return fmt.Errorf("Key should start with /: %v", key) + err, parts := common.ParseKey(key) + if err != nil { + return err } + if parts[1] != k.TableName() { return fmt.Errorf("Second part of key (%v) should be %v", key, k.TableName()) } diff --git a/pkg/sloop/store/untyped/badgerwrap/api.go b/pkg/sloop/store/untyped/badgerwrap/api.go index d1b8fdb..7deec11 100644 --- a/pkg/sloop/store/untyped/badgerwrap/api.go +++ b/pkg/sloop/store/untyped/badgerwrap/api.go @@ -69,9 +69,9 @@ type Item interface { Value(fn func(val []byte) error) error ValueCopy(dst []byte) ([]byte, error) // DiscardEarlierVersions() bool - // EstimatedSize() int64 + EstimatedSize() int64 // ExpiresAt() uint64 - // IsDeletedOrExpired() bool + IsDeletedOrExpired() bool KeyCopy(dst []byte) []byte // KeySize() int64 // String() string diff --git a/pkg/sloop/store/untyped/badgerwrap/badger.go b/pkg/sloop/store/untyped/badgerwrap/badger.go index 23591ac..542eb0a 100644 --- a/pkg/sloop/store/untyped/badgerwrap/badger.go +++ b/pkg/sloop/store/untyped/badgerwrap/badger.go @@ -128,6 +128,14 @@ func (i *BadgerItem) KeyCopy(dst []byte) []byte { return i.item.KeyCopy(dst) } +func (i *BadgerItem) EstimatedSize() int64 { + return i.item.EstimatedSize() +} + +func (i *BadgerItem) IsDeletedOrExpired() bool { + return i.item.IsDeletedOrExpired() +} + // Iterator func (i *BadgerIterator) Close() { diff --git a/pkg/sloop/store/untyped/badgerwrap/mock.go b/pkg/sloop/store/untyped/badgerwrap/mock.go index 405f61a..3ceac3a 100644 --- a/pkg/sloop/store/untyped/badgerwrap/mock.go +++ b/pkg/sloop/store/untyped/badgerwrap/mock.go @@ -179,6 +179,14 @@ func (i *MockItem) ValueCopy(dst []byte) ([]byte, error) { return newcopy, nil } +func (i *MockItem) EstimatedSize() int64 { + return int64(len(i.key) + len(i.value)) +} + +func (i *MockItem) IsDeletedOrExpired() bool { + return false +} + func (i *MockItem) KeyCopy(dst []byte) []byte { copy(dst, i.key) newcopy := make([]byte, len(i.key)) diff --git a/pkg/sloop/storemanager/storemanager.go b/pkg/sloop/storemanager/storemanager.go index 627d194..9be8680 100644 --- a/pkg/sloop/storemanager/storemanager.go +++ b/pkg/sloop/storemanager/storemanager.go @@ -12,6 +12,7 @@ import ( "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/salesforce/sloop/pkg/sloop/common" "github.com/salesforce/sloop/pkg/sloop/store/typed" "github.com/salesforce/sloop/pkg/sloop/store/untyped" "github.com/spf13/afero" @@ -97,7 +98,7 @@ func (sm *StoreManager) gcLoop() { before := time.Now() metricGcRunning.Set(1) cleanUpPerformed, numOfDeletedKeys, numOfKeysToDelete, err := doCleanup(sm.tables, sm.config.TimeLimit, sm.config.SizeLimitBytes, sm.stats, sm.config.DeletionBatchSize) - metricGcCleanUpPerformed.Set(boolToFloat(cleanUpPerformed)) + metricGcCleanUpPerformed.Set(common.BoolToFloat(cleanUpPerformed)) metricGcDeletedNumberOfKeys.Set(numOfDeletedKeys) metricGcNumberOfKeysToDelete.Set(numOfKeysToDelete) metricGcRunning.Set(0) @@ -177,23 +178,22 @@ func doCleanup(tables typed.Tables, timeLimit time.Duration, sizeLimitBytes int, return false, 0, 0, nil } + minPartitionAge, err := untyped.GetAgeOfPartitionInHours(minPartition) + if err == nil { + metricAgeOfMinimumPartition.Set(minPartitionAge) + } + + maxPartitionAge, err := untyped.GetAgeOfPartitionInHours(maxPartition) + if err == nil { + metricAgeOfMaximumPartition.Set(maxPartitionAge) + } + var totalNumOfDeletedKeys float64 = 0 var totalNumOfKeysToDelete float64 = 0 anyCleanupPerformed := false if cleanUpTimeCondition(minPartition, maxPartition, timeLimit) || cleanUpFileSizeCondition(stats, sizeLimitBytes) { partStart, partEnd, err := untyped.GetTimeRangeForPartition(minPartition) glog.Infof("GC removing partition %q with data from %v to %v (err %v)", minPartition, partStart, partEnd, err) - minPartitionAge := 0.0 - minPartitionAge, err = untyped.GetAgeOfPartitionInHours(minPartition) - if err != nil { - metricAgeOfMinimumPartition.Set(minPartitionAge) - } - - maxPartitionAge, err := untyped.GetAgeOfPartitionInHours(maxPartition) - if err != nil { - metricAgeOfMaximumPartition.Set(maxPartitionAge) - } - var errMessages []string for _, tableName := range tables.GetTableNames() { prefix := fmt.Sprintf("/%s/%s", tableName, minPartition) diff --git a/pkg/sloop/storemanager/utilities.go b/pkg/sloop/storemanager/utilities.go deleted file mode 100644 index 11f843f..0000000 --- a/pkg/sloop/storemanager/utilities.go +++ /dev/null @@ -1,8 +0,0 @@ -package storemanager - -func boolToFloat(value bool) float64 { - if value { - return 1 - } - return 0 -} diff --git a/pkg/sloop/storemanager/utilities_test.go b/pkg/sloop/storemanager/utilities_test.go deleted file mode 100644 index d9f43de..0000000 --- a/pkg/sloop/storemanager/utilities_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package storemanager - -import ( - "github.com/stretchr/testify/assert" - "testing" -) - -func Test_boolToFloat(t *testing.T) { - assert.Equal(t, float64(1), boolToFloat(true)) - assert.Equal(t, float64(0), boolToFloat(false)) -} diff --git a/pkg/sloop/webfiles/debug.html b/pkg/sloop/webfiles/debug.html index f587313..61245c3 100644 --- a/pkg/sloop/webfiles/debug.html +++ b/pkg/sloop/webfiles/debug.html @@ -16,6 +16,7 @@ For full license text, see LICENSE.txt file in the repo root or https://opensour
  • Query Data Store - Allows you to query keys and values from the badger store
    • +
  • Sloop Keys Histogram - View the keys histogram
  • Config - View the current active config for Sloop
  • Tables - View Badger LSM Table Info
  • Badger Requests
    • diff --git a/pkg/sloop/webfiles/debughistogram.html b/pkg/sloop/webfiles/debughistogram.html new file mode 100644 index 0000000..6362549 --- /dev/null +++ b/pkg/sloop/webfiles/debughistogram.html @@ -0,0 +1,49 @@ + + + + Sloop Keys Histogram + + + +[ Home ][ Debug Menu ]
    + +

    Sloop Keys Histogram

    + +
    +
    + +

    +

    +

    +


    + +
    +
    +
    + + + + +
    Total keys{{.TotalKeys}}
    Deleted Keys
    {{.DeletedKeys}}
    + +
    + +Partitions List:
    +
    + + + {{range $key, $value := .HistogramMap}} + + {{end}} +
    TablePartition IDNumber of KeysEstimated SizeMinimum SizeMaximum SizeAverage Size
    {{$key.TableName}}{{$key.PartitionID}}{{$value.TotalKeys}}{{$value.TotalSize}}{{$value.MinimumSize}}{{$value.MaximumSize}}{{$value.AverageSize}}
    + + + + + diff --git a/pkg/sloop/webserver/debug.go b/pkg/sloop/webserver/debug.go index 0735c7d..8899ec0 100644 --- a/pkg/sloop/webserver/debug.go +++ b/pkg/sloop/webserver/debug.go @@ -13,6 +13,8 @@ import ( "fmt" "github.com/dgraph-io/badger/v2" "github.com/golang/glog" + "github.com/pkg/errors" + "github.com/salesforce/sloop/pkg/sloop/common" "github.com/salesforce/sloop/pkg/sloop/store/typed" "github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap" "html/template" @@ -162,6 +164,116 @@ func listKeysHandler(tables typed.Tables) http.HandlerFunc { } } +type sloopKeyInfo struct { + MinimumSize int64 + MaximumSize int64 + TotalKeys int64 + TotalSize int64 + AverageSize int64 +} + +type sloopKey struct { + TableName string + PartitionID string +} + +type histogram struct { + HistogramMap map[sloopKey]*sloopKeyInfo + TotalKeys int + DeletedKeys int +} + +// returns TableName, PartitionId, error. +func parseSloopKey(item badgerwrap.Item) (string, string, error) { + key := item.Key() + err, parts := common.ParseKey(string(key)) + if err != nil { + return "", "", err + } + + var tableName = parts[1] + var partitionId = parts[2] + return tableName, partitionId, nil +} + +func histogramHandler(tables typed.Tables) http.HandlerFunc { + return func(writer http.ResponseWriter, request *http.Request) { + var result histogram + prefix := request.URL.Query().Get("prefix") + if len(prefix) > 0 { + + if prefix == "*" { + prefix = "" + } + + err := tables.Db().View(func(txn badgerwrap.Txn) error { + iterOpt := badger.DefaultIteratorOptions + iterOpt.Prefix = []byte(prefix) + iterOpt.PrefetchValues = false + itr := txn.NewIterator(iterOpt) + defer itr.Close() + + totalKeys := 0 + totalDeletedExpiredKeys := 0 + var sloopMap = make(map[sloopKey]*sloopKeyInfo) + for itr.Rewind(); itr.Valid(); itr.Next() { + item := itr.Item() + tableName, partitionId, err := parseSloopKey(item) + if err != nil { + return errors.Wrapf(err, "failed to parse information about key: %x", + item.Key()) + } + totalKeys++ + + if item.IsDeletedOrExpired() { + totalDeletedExpiredKeys++ + } + + size := item.EstimatedSize() + sloopKey := sloopKey{tableName, partitionId} + if sloopMap[sloopKey] == nil { + sloopMap[sloopKey] = &sloopKeyInfo{size, size, 1, size, size} + } else { + sloopMap[sloopKey].TotalKeys++ + sloopMap[sloopKey].TotalSize += size + sloopMap[sloopKey].AverageSize = sloopMap[sloopKey].TotalSize / sloopMap[sloopKey].TotalKeys + if size < sloopMap[sloopKey].MinimumSize { + sloopMap[sloopKey].MinimumSize = size + } + + if size > sloopMap[sloopKey].MaximumSize { + sloopMap[sloopKey].MaximumSize = size + } + } + } + + result.TotalKeys = totalKeys + result.DeletedKeys = totalDeletedExpiredKeys + result.HistogramMap = sloopMap + return nil + }) + + if err != nil { + logWebError(err, "Could not get histogram", request, writer) + return + } + } + writer.Header().Set("content-type", "text/html") + + t, err := template.New(debugHistogramFile).ParseFiles(path.Join(webFiles, debugHistogramFile)) + if err != nil { + logWebError(err, "failed to parse histogram template", request, writer) + return + } + + err = t.ExecuteTemplate(writer, debugHistogramFile, result) + if err != nil { + logWebError(err, "Template.ExecuteTemplate failed", request, writer) + return + } + } +} + func configHandler(config string) http.HandlerFunc { return func(writer http.ResponseWriter, request *http.Request) { t, err := template.New(debugConfigTemplateFile).ParseFiles(path.Join(webFiles, debugConfigTemplateFile)) diff --git a/pkg/sloop/webserver/webserver.go b/pkg/sloop/webserver/webserver.go index 36c41c3..9179a3b 100644 --- a/pkg/sloop/webserver/webserver.go +++ b/pkg/sloop/webserver/webserver.go @@ -40,6 +40,7 @@ import ( const ( debugViewKeyTemplateFile = "debugviewkey.html" debugListKeysTemplateFile = "debuglistkeys.html" + debugHistogramFile = "debughistogram.html" debugConfigTemplateFile = "debugconfig.html" debugTemplateFile = "debug.html" debugBadgerTablesTemplateFile = "debugtables.html" @@ -176,6 +177,7 @@ func Run(config WebConfig, tables typed.Tables) error { // Debug pages server.mux.HandleFunc("/debug/", debugHandler()) server.mux.HandleFunc("/debug/listkeys/", listKeysHandler(tables)) + server.mux.HandleFunc("/debug/histogram/", histogramHandler(tables)) server.mux.HandleFunc("/debug/tables/", debugBadgerTablesHandler(tables.Db())) server.mux.HandleFunc("/debug/view/", viewKeyHandler(tables)) server.mux.HandleFunc("/debug/config/", configHandler(config.ConfigYaml))