Add better badger debugging (#101)

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/spf13/afero v1.2.2
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect
-	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
+	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
 	golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect
 	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect

pkg/sloop/server/internal/config/config.go

@@ -30,34 +30,35 @@ type SloopConfig struct {
 	LeftBarLinks  []webserver.LinkTemplate         `json:"leftBarLinks"`
 	ResourceLinks []webserver.ResourceLinkTemplate `json:"resourceLinks"`
 	// Normal fields that can come from file or cmd line
-	DisableKubeWatcher      bool          `json:"disableKubeWatch"`
+	DisableKubeWatcher       bool          `json:"disableKubeWatch"`
-	KubeWatchResyncInterval time.Duration `json:"kubeWatchResyncInterval"`
+	KubeWatchResyncInterval  time.Duration `json:"kubeWatchResyncInterval"`
-	WebFilesPath            string        `json:"webfilesPath"`
+	WebFilesPath             string        `json:"webfilesPath"`
-	Port                    int           `json:"port"`
+	Port                     int           `json:"port"`
-	StoreRoot               string        `json:"storeRoot"`
+	StoreRoot                string        `json:"storeRoot"`
-	MaxLookback             time.Duration `json:"maxLookBack"`
+	MaxLookback              time.Duration `json:"maxLookBack"`
-	MaxDiskMb               int           `json:"maxDiskMb"`
+	MaxDiskMb                int           `json:"maxDiskMb"`
-	DebugPlaybackFile       string        `json:"debugPlaybackFile"`
+	DebugPlaybackFile        string        `json:"debugPlaybackFile"`
-	DebugRecordFile         string        `json:"debugRecordFile"`
+	DebugRecordFile          string        `json:"debugRecordFile"`
-	UseMockBadger           bool          `json:"mockBadger"`
+	UseMockBadger            bool          `json:"mockBadger"`
-	DisableStoreManager     bool          `json:"disableStoreManager"`
+	DisableStoreManager      bool          `json:"disableStoreManager"`
-	CleanupFrequency        time.Duration `json:"cleanupFrequency" validate:"min=1h,max=120h"`
+	CleanupFrequency         time.Duration `json:"cleanupFrequency" validate:"min=1h,max=120h"`
-	KeepMinorNodeUpdates    bool          `json:"keepMinorNodeUpdates"`
+	KeepMinorNodeUpdates     bool          `json:"keepMinorNodeUpdates"`
-	DefaultNamespace        string        `json:"defaultNamespace"`
+	DefaultNamespace         string        `json:"defaultNamespace"`
-	DefaultKind             string        `json:"defaultKind"`
+	DefaultKind              string        `json:"defaultKind"`
-	DefaultLookback         string        `json:"defaultLookback"`
+	DefaultLookback          string        `json:"defaultLookback"`
-	UseKubeContext          string        `json:"context"`
+	UseKubeContext           string        `json:"context"`
-	DisplayContext          string        `json:"displayContext"`
+	DisplayContext           string        `json:"displayContext"`
-	ApiServerHost           string        `json:"apiServerHost"`
+	ApiServerHost            string        `json:"apiServerHost"`
-	WatchCrds               bool          `json:"watchCrds"`
+	WatchCrds                bool          `json:"watchCrds"`
-	RestoreDatabaseFile     string        `json:"restoreDatabaseFile"`
+	RestoreDatabaseFile      string        `json:"restoreDatabaseFile"`
-	BadgerDiscardRatio      float64       `json:"badgerDiscardRatio"`
+	BadgerDiscardRatio       float64       `json:"badgerDiscardRatio"`
-	BadgerVLogGCFreq        time.Duration `json:"badgerVLogGCFreq"`
+	BadgerVLogGCFreq         time.Duration `json:"badgerVLogGCFreq"`
-	BadgerMaxTableSize      int64         `json:"badgerMaxTableSize"`
+	BadgerMaxTableSize       int64         `json:"badgerMaxTableSize"`
-	BadgerKeepL0InMemory    bool          `json:"badgerKeepL0InMemory"`
+	BadgerKeepL0InMemory     bool          `json:"badgerKeepL0InMemory"`
-	BadgerVLogFileSize      int64         `json:"badgerVLogFileSize"`
+	BadgerVLogFileSize       int64         `json:"badgerVLogFileSize"`
-	BadgerVLogMaxEntries    uint          `json:"badgerVLogMaxEntries"`
+	BadgerVLogMaxEntries     uint          `json:"badgerVLogMaxEntries"`
-	BadgerUseLSMOnlyOptions bool          `json:"badgerUseLSMOnlyOptions"`
+	BadgerUseLSMOnlyOptions  bool          `json:"badgerUseLSMOnlyOptions"`
+	BadgerEnableEventLogging bool          `json:"badgerEnableEventLogging"`
 }
 
 func registerFlags(fs *flag.FlagSet, config *SloopConfig) {
@@ -91,6 +92,7 @@ func registerFlags(fs *flag.FlagSet, config *SloopConfig) {
 	fs.Int64Var(&config.BadgerVLogFileSize, "badger-vlog-file-size", 0, "Max size in bytes per value log file. 0 = use badger default")
 	fs.UintVar(&config.BadgerVLogMaxEntries, "badger-vlog-max-entries", 0, "Max number of entries per value log files. 0 = use badger default")
 	fs.BoolVar(&config.BadgerUseLSMOnlyOptions, "badger-use-lsm-only-options", true, "Sets a higher valueThreshold so values would be collocated with LSM tree reducing vlog disk usage")
+	fs.BoolVar(&config.BadgerEnableEventLogging, "badger-enable-event-logging", false, "Turns on badger event logging")
 }
 
 // This will first check if a config file is specified on cmd line using a temporary flagSet

pkg/sloop/server/server.go

@@ -58,13 +58,14 @@ func RealMain() error {
 
 	storeRootWithKubeContext := path.Join(conf.StoreRoot, kubeContext)
 	storeConfig := &untyped.Config{
-		RootPath:                storeRootWithKubeContext,
+		RootPath:                 storeRootWithKubeContext,
-		ConfigPartitionDuration: time.Duration(1) * time.Hour,
+		ConfigPartitionDuration:  time.Duration(1) * time.Hour,
-		BadgerMaxTableSize:      conf.BadgerMaxTableSize,
+		BadgerMaxTableSize:       conf.BadgerMaxTableSize,
-		BadgerKeepL0InMemory:    conf.BadgerKeepL0InMemory,
+		BadgerKeepL0InMemory:     conf.BadgerKeepL0InMemory,
-		BadgerVLogFileSize:      conf.BadgerVLogFileSize,
+		BadgerVLogFileSize:       conf.BadgerVLogFileSize,
-		BadgerVLogMaxEntries:    conf.BadgerVLogMaxEntries,
+		BadgerVLogMaxEntries:     conf.BadgerVLogMaxEntries,
-		BadgerUseLSMOnlyOptions: conf.BadgerUseLSMOnlyOptions,
+		BadgerUseLSMOnlyOptions:  conf.BadgerUseLSMOnlyOptions,
+		BadgerEnableEventLogging: conf.BadgerEnableEventLogging,
 	}
 	db, err := untyped.OpenStore(factory, storeConfig)
 	if err != nil {

pkg/sloop/store/untyped/store.go

@@ -17,13 +17,14 @@ import (
 )
 
 type Config struct {
-	RootPath                string
+	RootPath                 string
-	ConfigPartitionDuration time.Duration
+	ConfigPartitionDuration  time.Duration
-	BadgerMaxTableSize      int64
+	BadgerMaxTableSize       int64
-	BadgerKeepL0InMemory    bool
+	BadgerKeepL0InMemory     bool
-	BadgerVLogFileSize      int64
+	BadgerVLogFileSize       int64
-	BadgerVLogMaxEntries    uint
+	BadgerVLogMaxEntries     uint
-	BadgerUseLSMOnlyOptions bool
+	BadgerUseLSMOnlyOptions  bool
+	BadgerEnableEventLogging bool
 }
 
 func OpenStore(factory badgerwrap.Factory, config *Config) (badgerwrap.DB, error) {
@@ -44,6 +45,10 @@ func OpenStore(factory badgerwrap.Factory, config *Config) (badgerwrap.DB, error
 		opts = badger.DefaultOptions(config.RootPath)
 	}
 
+	if config.BadgerEnableEventLogging {
+		opts = opts.WithEventLogging(true)
+	}
+
 	if config.BadgerMaxTableSize != 0 {
 		opts = opts.WithMaxTableSize(config.BadgerMaxTableSize)
 	}

pkg/sloop/webfiles/debug.html

@@ -0,0 +1,28 @@
+<!--
+Copyright (c) 2019, salesforce.com, inc.
+All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+-->
+<html>
+<head>
+    <title>Sloop Debug Menu</title>
+    <link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
+</head>
+<body>
+[ <a href="/">Home</a> ]<br/><br/>
+
+<h2>Sloop Debug Menu</h2>
+
+<ul>
+    <li><a href="/debug/listkeys/">Query Data Store</a> - Allows you to query keys and values from the badger store</li>
+    <li><a href="/debug/config/">Config</a> - View the current active config for Sloop</li>
+    <li><a href="/debug/tables/">Tables</a> - View Badger LSM Table Info</li>
+    <li><a href="/debug/requests">Badger Requests</a></li>
+    <li><a href="/debug/events">Badger Events</a></li>
+    <li><a href="/debug/vars">Badger Metrics</a></li>
+    <li><a href="/metrics">Sloop Metrics</a></li>
+</ul>
+
+</body>
+</html>

pkg/sloop/webfiles/debugconfig.html

@@ -6,12 +6,14 @@ For full license text, see LICENSE.txt file in the repo root or https://opensour
 -->
 <html>
 <head>
-    <title>Sloop Debug</title>
+    <title>Sloop Debug Config</title>
     <link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
 </head>
 <body>
-[ <a href="/">Home</a> ][ <a href="/debug/">List Keys</a> ][ <a href="/config/">Config</a> ]<br><br>
+[ <a href="/">Home</a> ][ <a href="/debug/">Debug Menu</a> ]<br/>
-<b>Current Config</b>:<br>
+
+<h2>Current Config</h2>
+
 <table border="1"><tr><td>
 <pre>{{.}}</pre>
 </td></tr></table>

pkg/sloop/webfiles/debuglistkeys.html

@@ -6,14 +6,16 @@ For full license text, see LICENSE.txt file in the repo root or https://opensour
 -->
 <html>
 <head>
-    <title>Sloop Debug</title>
+    <title>Sloop Debug Query Badger</title>
     <link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
 </head>
 <body>
-[ <a href="/">Home</a> ][ <a href="/debug/">List Keys</a> ][ <a href="/debug/config/">Config</a> ]<br/><br/>
+[ <a href="/">Home</a> ][ <a href="/debug/">Debug Menu</a> ]<br/>
+
+<h2>Query Sloop's Badger DB Directly</h2>
 
 <table bgcolor="silver" width="500px"><tr><td style="padding: 20px">
-<form action="/debug/" method="get">
+<form action="/debug/listkeys/" method="get">
     <label for="table">Time Range:</label><br/>
     <select name="table" id="table">
         <option value="watch">watch</option>

pkg/sloop/webfiles/debugtables.html

@@ -0,0 +1,36 @@
+<!--
+Copyright (c) 2019, salesforce.com, inc.
+All rights reserved.
+SPDX-License-Identifier: BSD-3-Clause
+For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+-->
+<html>
+<head>
+    <title>Sloop Debug Badger Tables</title>
+    <link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
+</head>
+<body>
+[ <a href="/">Home</a> ][ <a href="/debug/">Debug Menu</a> ]<br/>
+
+<h2>Badger Tables</h2>
+
+<table border="1">
+    <tr>
+        <th>Level</th>
+        <th>KeyCount</th>
+        <th>Left</th>
+        <th>Right</th>
+        <th>ID</th>
+    </tr>
+{{range .}}
+    <tr>
+        <td>{{.Level}}</td>
+        <td>{{.KeyCount}}</td>
+        <td>{{.LeftKey}}</td>
+        <td>{{.RightKey}}</td>
+        <td>{{.ID}}</td>
+    </tr>
+{{end}}
+</table>
+</body>
+</html>

pkg/sloop/webfiles/debugviewkey.html

@@ -6,11 +6,12 @@ For full license text, see LICENSE.txt file in the repo root or https://opensour
 -->
 <html>
   <head>
-      <title>Sloop Debug</title>
+      <title>Sloop Debug View Record</title>
       <link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
   </head>
 <body>
-[ <a href="/">Home</a> ][ <a href="/debug/">Back</a>]<br>
+[ <a href="/">Home</a> ][ <a href="/debug/">Debug Menu</a> ]<br/>
+
 <b>Record View</b><br>
 <table border="1">
     <tr><td>Key</td><td>{{.Key}}</td></tr>

pkg/sloop/webfiles/index.html

@@ -66,9 +66,8 @@ For full license text, see LICENSE.txt file in the repo root or https://opensour
         <br><br>
 
         <h2>Links</h2>
-        <a href="/debug/">List Keys</a><br/>
+        <a href="/debug/">Sloop Debug Menu</a><br/>
-        <a href="/debug/config">View Config</a><br/>
+        <a href="" id="datafilelink">Data File For This Query</a><br/>
-        <a href="" id="datafilelink">Query Data File</a><br/>
         <a href="https://github.com/salesforce/sloop" target="_blank">Source Code on GitHub</a><br/>
 {{range .LeftBarLinks}}
         <a href="{{.Url}}" target="_blank">{{.Text}}</a><br/>

pkg/sloop/webserver/debug.go

@@ -176,3 +176,53 @@ func configHandler(config string) http.HandlerFunc {
 		}
 	}
 }
+
+func debugHandler() http.HandlerFunc {
+	return func(writer http.ResponseWriter, request *http.Request) {
+		t, err := template.New(debugTemplateFile).ParseFiles(path.Join(webFiles, debugTemplateFile))
+		if err != nil {
+			logWebError(err, "failed to parse template", request, writer)
+			return
+		}
+		err = t.ExecuteTemplate(writer, debugTemplateFile, nil)
+		if err != nil {
+			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
+			return
+		}
+	}
+}
+
+// Make a copy with string keys instead of []byte keys
+type badgerTableInfo struct {
+	Level    int
+	LeftKey  string
+	RightKey string
+	KeyCount uint64
+	ID       uint64
+}
+
+func debugBadgerTablesHandler(db badgerwrap.DB) http.HandlerFunc {
+	return func(writer http.ResponseWriter, request *http.Request) {
+		t, err := template.New(debugBadgerTablesTemplateFile).ParseFiles(path.Join(webFiles, debugBadgerTablesTemplateFile))
+		if err != nil {
+			logWebError(err, "failed to parse template", request, writer)
+			return
+		}
+		data := []badgerTableInfo{}
+		for _, table := range db.Tables(true) {
+			thisTable := badgerTableInfo{
+				Level:    table.Level,
+				LeftKey:  string(table.Left),
+				RightKey: string(table.Right),
+				KeyCount: table.KeyCount,
+				ID:       table.ID,
+			}
+			data = append(data, thisTable)
+		}
+		err = t.ExecuteTemplate(writer, debugBadgerTablesTemplateFile, data)
+		if err != nil {
+			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
+			return
+		}
+	}
+}

pkg/sloop/webserver/webserver.go

@@ -9,6 +9,7 @@ package webserver
 
 import (
 	"context"
+	"expvar"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -32,14 +33,18 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"golang.org/x/net/trace"
 )
 
 const (
-	debugViewKeyTemplateFile  = "debugviewkey.html"
+	debugViewKeyTemplateFile      = "debugviewkey.html"
-	debugListKeysTemplateFile = "debuglistkeys.html"
+	debugListKeysTemplateFile     = "debuglistkeys.html"
-	debugConfigTemplateFile   = "debugconfig.html"
+	debugConfigTemplateFile       = "debugconfig.html"
-	indexTemplateFile         = "index.html"
+	debugTemplateFile             = "debug.html"
-	resourceTemplateFile      = "resource.html"
+	debugBadgerTablesTemplateFile = "debugtables.html"
+	indexTemplateFile             = "index.html"
+	resourceTemplateFile          = "resource.html"
 )
 
 type WebConfig struct {
@@ -167,9 +172,18 @@ func Run(config WebConfig, tables typed.Tables) error {
 	server.mux.HandleFunc("/data/backup", backupHandler(tables.Db(), config.CurrentContext))
 	server.mux.HandleFunc("/data", queryHandler(tables, config.MaxLookback))
 	server.mux.HandleFunc("/resource", resourceHandler(config.ResourceLinks))
-	server.mux.HandleFunc("/debug/", listKeysHandler(tables))
+	// Debug pages
+	server.mux.HandleFunc("/debug/", debugHandler())
+	server.mux.HandleFunc("/debug/listkeys/", listKeysHandler(tables))
+	server.mux.HandleFunc("/debug/tables/", debugBadgerTablesHandler(tables.Db()))
 	server.mux.HandleFunc("/debug/view/", viewKeyHandler(tables))
 	server.mux.HandleFunc("/debug/config/", configHandler(config.ConfigYaml))
+	// Badger uses the trace package, which registers /debug/requests and /debug/events
+	server.mux.HandleFunc("/debug/requests", trace.Traces)
+	server.mux.HandleFunc("/debug/events", trace.Events)
+	// Badger also uses expvar which exposes prometheus compatible metrics on /debug/vars
+	server.mux.HandleFunc("/debug/vars", expvar.Handler().ServeHTTP)
+
 	server.mux.HandleFunc("/healthz", healthHandler())
 	server.mux.Handle("/metrics", promhttp.Handler())
 	server.mux.HandleFunc("/", indexHandler(config))