Unity-Technologies
/
sloop
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Add better badger debugging (#101)

This commit is contained in:
Thomas Hargrove 2020-01-31 00:08:34 +05:30 коммит произвёл GitHub
Родитель a9469f43b5
Коммит ba2c651886
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
12 изменённых файлов: 200 добавлений и 60 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
go.mod
Просмотреть файл

@ -27,7 +27,7 @@ require (
github.com/spf13/afero v1.2.2 github.com/spf13/afero v1.2.2
github.com/stretchr/testify v1.4.0 github.com/stretchr/testify v1.4.0
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect
golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect

2
pkg/sloop/server/internal/config/config.go
Просмотреть файл

@ -58,6 +58,7 @@ type SloopConfig struct {
BadgerVLogFileSize int64 `json:"badgerVLogFileSize"` BadgerVLogFileSize int64 `json:"badgerVLogFileSize"`
BadgerVLogMaxEntries uint `json:"badgerVLogMaxEntries"` BadgerVLogMaxEntries uint `json:"badgerVLogMaxEntries"`
BadgerUseLSMOnlyOptions bool `json:"badgerUseLSMOnlyOptions"` BadgerUseLSMOnlyOptions bool `json:"badgerUseLSMOnlyOptions"`
BadgerEnableEventLogging bool `json:"badgerEnableEventLogging"`
} }
func registerFlags(fs *flag.FlagSet, config *SloopConfig) { func registerFlags(fs *flag.FlagSet, config *SloopConfig) {
@ -91,6 +92,7 @@ func registerFlags(fs *flag.FlagSet, config *SloopConfig) {
fs.Int64Var(&config.BadgerVLogFileSize, "badger-vlog-file-size", 0, "Max size in bytes per value log file. 0 = use badger default") fs.Int64Var(&config.BadgerVLogFileSize, "badger-vlog-file-size", 0, "Max size in bytes per value log file. 0 = use badger default")
fs.UintVar(&config.BadgerVLogMaxEntries, "badger-vlog-max-entries", 0, "Max number of entries per value log files. 0 = use badger default") fs.UintVar(&config.BadgerVLogMaxEntries, "badger-vlog-max-entries", 0, "Max number of entries per value log files. 0 = use badger default")
fs.BoolVar(&config.BadgerUseLSMOnlyOptions, "badger-use-lsm-only-options", true, "Sets a higher valueThreshold so values would be collocated with LSM tree reducing vlog disk usage") fs.BoolVar(&config.BadgerUseLSMOnlyOptions, "badger-use-lsm-only-options", true, "Sets a higher valueThreshold so values would be collocated with LSM tree reducing vlog disk usage")
fs.BoolVar(&config.BadgerEnableEventLogging, "badger-enable-event-logging", false, "Turns on badger event logging")
} }
// This will first check if a config file is specified on cmd line using a temporary flagSet // This will first check if a config file is specified on cmd line using a temporary flagSet

1
pkg/sloop/server/server.go
Просмотреть файл

@ -65,6 +65,7 @@ func RealMain() error {
BadgerVLogFileSize: conf.BadgerVLogFileSize, BadgerVLogFileSize: conf.BadgerVLogFileSize,
BadgerVLogMaxEntries: conf.BadgerVLogMaxEntries, BadgerVLogMaxEntries: conf.BadgerVLogMaxEntries,
BadgerUseLSMOnlyOptions: conf.BadgerUseLSMOnlyOptions, BadgerUseLSMOnlyOptions: conf.BadgerUseLSMOnlyOptions,
BadgerEnableEventLogging: conf.BadgerEnableEventLogging,
} }
db, err := untyped.OpenStore(factory, storeConfig) db, err := untyped.OpenStore(factory, storeConfig)
if err != nil { if err != nil {

5
pkg/sloop/store/untyped/store.go
Просмотреть файл

@ -24,6 +24,7 @@ type Config struct {
BadgerVLogFileSize int64 BadgerVLogFileSize int64
BadgerVLogMaxEntries uint BadgerVLogMaxEntries uint
BadgerUseLSMOnlyOptions bool BadgerUseLSMOnlyOptions bool
BadgerEnableEventLogging bool
} }
func OpenStore(factory badgerwrap.Factory, config *Config) (badgerwrap.DB, error) { func OpenStore(factory badgerwrap.Factory, config *Config) (badgerwrap.DB, error) {
@ -44,6 +45,10 @@ func OpenStore(factory badgerwrap.Factory, config *Config) (badgerwrap.DB, error
opts = badger.DefaultOptions(config.RootPath) opts = badger.DefaultOptions(config.RootPath)
} }
if config.BadgerEnableEventLogging {
opts = opts.WithEventLogging(true)
}
if config.BadgerMaxTableSize != 0 { if config.BadgerMaxTableSize != 0 {
opts = opts.WithMaxTableSize(config.BadgerMaxTableSize) opts = opts.WithMaxTableSize(config.BadgerMaxTableSize)
} }

28
pkg/sloop/webfiles/debug.html Normal file
Просмотреть файл

@ -0,0 +1,28 @@
<!--
Copyright (c) 2019, salesforce.com, inc.
All rights reserved.
SPDX-License-Identifier: BSD-3-Clause
For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
-->
<html>
<head>
<title>Sloop Debug Menu</title>
<link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
</head>
<body>
[ <a href="/">Home</a> ]<br/><br/>
<h2>Sloop Debug Menu</h2>
<ul>
<li><a href="/debug/listkeys/">Query Data Store</a> - Allows you to query keys and values from the badger store</li>
<li><a href="/debug/config/">Config</a> - View the current active config for Sloop</li>
<li><a href="/debug/tables/">Tables</a> - View Badger LSM Table Info</li>
<li><a href="/debug/requests">Badger Requests</a></li>
<li><a href="/debug/events">Badger Events</a></li>
<li><a href="/debug/vars">Badger Metrics</a></li>
<li><a href="/metrics">Sloop Metrics</a></li>
</ul>
</body>
</html>

8
pkg/sloop/webfiles/debugconfig.html
Просмотреть файл

@ -6,12 +6,14 @@ For full license text, see LICENSE.txt file in the repo root or https://opensour
--> -->
<html> <html>
<head> <head>
<title>Sloop Debug</title> <title>Sloop Debug Config</title>
<link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' /> <link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
</head> </head>
<body> <body>
[ <a href="/">Home</a> ][ <a href="/debug/">List Keys</a> ][ <a href="/config/">Config</a> ]<br><br> [ <a href="/">Home</a> ][ <a href="/debug/">Debug Menu</a> ]<br/>
<b>Current Config</b>:<br>
<h2>Current Config</h2>
<table border="1"><tr><td> <table border="1"><tr><td>
<pre>{{.}}</pre> <pre>{{.}}</pre>
</td></tr></table> </td></tr></table>

8
pkg/sloop/webfiles/debuglistkeys.html
Просмотреть файл

@ -6,14 +6,16 @@ For full license text, see LICENSE.txt file in the repo root or https://opensour
--> -->
<html> <html>
<head> <head>
<title>Sloop Debug</title> <title>Sloop Debug Query Badger</title>
<link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' /> <link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
</head> </head>
<body> <body>
[ <a href="/">Home</a> ][ <a href="/debug/">List Keys</a> ][ <a href="/debug/config/">Config</a> ]<br/><br/> [ <a href="/">Home</a> ][ <a href="/debug/">Debug Menu</a> ]<br/>
<h2>Query Sloop's Badger DB Directly</h2>
<table bgcolor="silver" width="500px"><tr><td style="padding: 20px"> <table bgcolor="silver" width="500px"><tr><td style="padding: 20px">
<form action="/debug/" method="get"> <form action="/debug/listkeys/" method="get">
<label for="table">Time Range:</label><br/> <label for="table">Time Range:</label><br/>
<select name="table" id="table"> <select name="table" id="table">
<option value="watch">watch</option> <option value="watch">watch</option>

36
pkg/sloop/webfiles/debugtables.html Normal file
Просмотреть файл

@ -0,0 +1,36 @@
<!--
Copyright (c) 2019, salesforce.com, inc.
All rights reserved.
SPDX-License-Identifier: BSD-3-Clause
For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
-->
<html>
<head>
<title>Sloop Debug Badger Tables</title>
<link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
</head>
<body>
[ <a href="/">Home</a> ][ <a href="/debug/">Debug Menu</a> ]<br/>
<h2>Badger Tables</h2>
<table border="1">
<tr>
<th>Level</th>
<th>KeyCount</th>
<th>Left</th>
<th>Right</th>
<th>ID</th>
</tr>
{{range .}}
<tr>
<td>{{.Level}}</td>
<td>{{.KeyCount}}</td>
<td>{{.LeftKey}}</td>
<td>{{.RightKey}}</td>
<td>{{.ID}}</td>
</tr>
{{end}}
</table>
</body>
</html>

5
pkg/sloop/webfiles/debugviewkey.html
Просмотреть файл

@ -6,11 +6,12 @@ For full license text, see LICENSE.txt file in the repo root or https://opensour
--> -->
<html> <html>
<head> <head>
<title>Sloop Debug</title> <title>Sloop Debug View Record</title>
<link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' /> <link rel='shortcut icon' type='image/x-icon' href='/webfiles/favicon.ico' />
</head> </head>
<body> <body>
[ <a href="/">Home</a> ][ <a href="/debug/">Back</a>]<br> [ <a href="/">Home</a> ][ <a href="/debug/">Debug Menu</a> ]<br/>
<b>Record View</b><br> <b>Record View</b><br>
<table border="1"> <table border="1">
<tr><td>Key</td><td>{{.Key}}</td></tr> <tr><td>Key</td><td>{{.Key}}</td></tr>

5
pkg/sloop/webfiles/index.html
Просмотреть файл

@ -66,9 +66,8 @@ For full license text, see LICENSE.txt file in the repo root or https://opensour
<br><br> <br><br>
<h2>Links</h2> <h2>Links</h2>
<a href="/debug/">List Keys</a><br/> <a href="/debug/">Sloop Debug Menu</a><br/>
<a href="/debug/config">View Config</a><br/> <a href="" id="datafilelink">Data File For This Query</a><br/>
<a href="" id="datafilelink">Query Data File</a><br/>
<a href="https://github.com/salesforce/sloop" target="_blank">Source Code on GitHub</a><br/> <a href="https://github.com/salesforce/sloop" target="_blank">Source Code on GitHub</a><br/>
{{range .LeftBarLinks}} {{range .LeftBarLinks}}
<a href="{{.Url}}" target="_blank">{{.Text}}</a><br/> <a href="{{.Url}}" target="_blank">{{.Text}}</a><br/>

50
pkg/sloop/webserver/debug.go
Просмотреть файл

@ -176,3 +176,53 @@ func configHandler(config string) http.HandlerFunc {
} }
} }
} }
func debugHandler() http.HandlerFunc {
return func(writer http.ResponseWriter, request *http.Request) {
t, err := template.New(debugTemplateFile).ParseFiles(path.Join(webFiles, debugTemplateFile))
if err != nil {
logWebError(err, "failed to parse template", request, writer)
return
}
err = t.ExecuteTemplate(writer, debugTemplateFile, nil)
if err != nil {
logWebError(err, "Template.ExecuteTemplate failed", request, writer)
return
}
}
}
// Make a copy with string keys instead of []byte keys
type badgerTableInfo struct {
Level int
LeftKey string
RightKey string
KeyCount uint64
ID uint64
}
func debugBadgerTablesHandler(db badgerwrap.DB) http.HandlerFunc {
return func(writer http.ResponseWriter, request *http.Request) {
t, err := template.New(debugBadgerTablesTemplateFile).ParseFiles(path.Join(webFiles, debugBadgerTablesTemplateFile))
if err != nil {
logWebError(err, "failed to parse template", request, writer)
return
}
data := []badgerTableInfo{}
for _, table := range db.Tables(true) {
thisTable := badgerTableInfo{
Level: table.Level,
LeftKey: string(table.Left),
RightKey: string(table.Right),
KeyCount: table.KeyCount,
ID: table.ID,
}
data = append(data, thisTable)
}
err = t.ExecuteTemplate(writer, debugBadgerTablesTemplateFile, data)
if err != nil {
logWebError(err, "Template.ExecuteTemplate failed", request, writer)
return
}
}
}

16
pkg/sloop/webserver/webserver.go
Просмотреть файл

@ -9,6 +9,7 @@ package webserver
import ( import (
"context" "context"
"expvar"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log" "log"
@ -32,12 +33,16 @@ import (
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promauto"
"golang.org/x/net/trace"
) )
const ( const (
debugViewKeyTemplateFile = "debugviewkey.html" debugViewKeyTemplateFile = "debugviewkey.html"
debugListKeysTemplateFile = "debuglistkeys.html" debugListKeysTemplateFile = "debuglistkeys.html"
debugConfigTemplateFile = "debugconfig.html" debugConfigTemplateFile = "debugconfig.html"
debugTemplateFile = "debug.html"
debugBadgerTablesTemplateFile = "debugtables.html"
indexTemplateFile = "index.html" indexTemplateFile = "index.html"
resourceTemplateFile = "resource.html" resourceTemplateFile = "resource.html"
) )
@ -167,9 +172,18 @@ func Run(config WebConfig, tables typed.Tables) error {
server.mux.HandleFunc("/data/backup", backupHandler(tables.Db(), config.CurrentContext)) server.mux.HandleFunc("/data/backup", backupHandler(tables.Db(), config.CurrentContext))
server.mux.HandleFunc("/data", queryHandler(tables, config.MaxLookback)) server.mux.HandleFunc("/data", queryHandler(tables, config.MaxLookback))
server.mux.HandleFunc("/resource", resourceHandler(config.ResourceLinks)) server.mux.HandleFunc("/resource", resourceHandler(config.ResourceLinks))
server.mux.HandleFunc("/debug/", listKeysHandler(tables)) // Debug pages
server.mux.HandleFunc("/debug/", debugHandler())
server.mux.HandleFunc("/debug/listkeys/", listKeysHandler(tables))
server.mux.HandleFunc("/debug/tables/", debugBadgerTablesHandler(tables.Db()))
server.mux.HandleFunc("/debug/view/", viewKeyHandler(tables)) server.mux.HandleFunc("/debug/view/", viewKeyHandler(tables))
server.mux.HandleFunc("/debug/config/", configHandler(config.ConfigYaml)) server.mux.HandleFunc("/debug/config/", configHandler(config.ConfigYaml))
// Badger uses the trace package, which registers /debug/requests and /debug/events
server.mux.HandleFunc("/debug/requests", trace.Traces)
server.mux.HandleFunc("/debug/events", trace.Events)
// Badger also uses expvar which exposes prometheus compatible metrics on /debug/vars
server.mux.HandleFunc("/debug/vars", expvar.Handler().ServeHTTP)
server.mux.HandleFunc("/healthz", healthHandler()) server.mux.HandleFunc("/healthz", healthHandler())
server.mux.Handle("/metrics", promhttp.Handler()) server.mux.Handle("/metrics", promhttp.Handler())
server.mux.HandleFunc("/", indexHandler(config)) server.mux.HandleFunc("/", indexHandler(config))