Data Backup Endpoint (#56)

* Add /data/backup endpoint to dump database

* Use tables.Db() instead of raw db

* Add backupHandler doc

* Provide better error message for since parsing issues

pkg/sloop/server/server.go

@@ -9,23 +9,26 @@ package server
 
 import (
 	"flag"
+	"os"
+	"path"
+	"strings"
+	"time"
+
 	"github.com/pkg/errors"
+
 	"github.com/salesforce/sloop/pkg/sloop/ingress"
 	"github.com/salesforce/sloop/pkg/sloop/server/internal/config"
 	"github.com/salesforce/sloop/pkg/sloop/store/typed"
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped"
-	"os"
-	"path"
-	"strings"
 
 	"github.com/golang/glog"
 
+	"github.com/spf13/afero"
+
 	"github.com/salesforce/sloop/pkg/sloop/processing"
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap"
 	"github.com/salesforce/sloop/pkg/sloop/storemanager"
 	"github.com/salesforce/sloop/pkg/sloop/webserver"
-	"github.com/spf13/afero"
-	"time"
 )
 
 const alsologtostderr = "alsologtostderr"

pkg/sloop/store/untyped/badgerwrap/api.go

@@ -8,6 +8,8 @@
 package badgerwrap
 
 import (
+	"io"
+
 	"github.com/dgraph-io/badger"
 )
 
@@ -25,7 +27,7 @@ type DB interface {
 	DropPrefix(prefix []byte) error
 	Size() (lsm, vlog int64)
 	Tables(withKeysCount bool) []badger.TableInfo
-	//	Backup(w io.Writer, since uint64) (uint64, error)
+	Backup(w io.Writer, since uint64) (uint64, error)
 	//	DropAll() error
 	//	Flatten(workers int) error
 	//	GetMergeOperator(key []byte, f MergeFunc, dur time.Duration) *MergeOperator

pkg/sloop/store/untyped/badgerwrap/badger.go

@@ -8,6 +8,8 @@
 package badgerwrap
 
 import (
+	"io"
+
 	"github.com/dgraph-io/badger"
 	"github.com/pkg/errors"
 )
@@ -74,6 +76,10 @@ func (b *BadgerDb) Tables(withKeysCount bool) []badger.TableInfo {
 	return b.db.Tables(withKeysCount)
 }
 
+func (b *BadgerDb) Backup(w io.Writer, since uint64) (uint64, error) {
+	return b.db.Backup(w, since)
+}
+
 // Transaction
 
 func (t *BadgerTxn) Get(key []byte) (Item, error) {

pkg/sloop/store/untyped/badgerwrap/mock.go

@@ -9,10 +9,12 @@ package badgerwrap
 
 import (
 	"fmt"
-	"github.com/dgraph-io/badger"
+	"io"
 	"sort"
 	"strings"
 	"sync"
+
+	"github.com/dgraph-io/badger"
 )
 
 // This mock simulates badger using an in-memory store
@@ -108,6 +110,10 @@ func (b *MockDb) Tables(withKeysCount bool) []badger.TableInfo {
 	}
 }
 
+func (b *MockDb) Backup(w io.Writer, since uint64) (uint64, error) {
+	return 0, nil
+}
+
 // Transaction
 
 func (t *MockTxn) Get(key []byte) (Item, error) {

pkg/sloop/webserver/webserver.go

@@ -8,27 +8,30 @@
 package webserver
 
 import (
-	"github.com/salesforce/sloop/pkg/sloop/queries"
-	"github.com/salesforce/sloop/pkg/sloop/store/typed"
-	"log"
-
-	"github.com/golang/glog"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-
 	"context"
 	"fmt"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"io/ioutil"
+	"log"
 	"mime"
 	"net/http"
 	"os"
 	"os/signal"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
+
+	"github.com/salesforce/sloop/pkg/sloop/queries"
+	"github.com/salesforce/sloop/pkg/sloop/store/typed"
+	"github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap"
+
+	"github.com/golang/glog"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 const (
@@ -100,6 +103,37 @@ func webFileHandler(w http.ResponseWriter, r *http.Request) {
 	glog.V(2).Infof("webFileHandler successfully returned file %v for %v", fixedUrl, r.URL)
 }
 
+// backupHandler streams a download of a backup of the database.
+// It is a simple HTTP translation of the Badger DB's built-in online backup function.
+// If the optional `since` query parameter is provided, the backup will only include versions since the version provided.
+func backupHandler(db badgerwrap.DB, currentContext string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		sinceStr := r.URL.Query().Get("since")
+		if sinceStr == "" {
+			sinceStr = "0"
+		}
+		since, err := strconv.ParseUint(sinceStr, 10, 64)
+		if err != nil {
+			logWebError(err, "Error parsing 'since' parameter. Must be expressed as a positive integer.", r, w)
+			return
+		}
+
+		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=sloop-%s-%d.bak", currentContext, since))
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.Header().Set("Transfer-Encoding", "chunked")
+
+		_, err = db.Backup(w, since)
+		if err != nil {
+			logWebError(err, "Error writing backup", r, w)
+			return
+		}
+
+		if flusher, ok := w.(http.Flusher); ok {
+			flusher.Flush()
+		}
+	}
+}
+
 // Returns json to feed into dhtmlgantt
 // Info on data format: https://docs.dhtmlx.com/gantt/desktop__loading.html
 
@@ -130,6 +164,7 @@ func Run(config WebConfig, tables typed.Tables) error {
 	server := &Server{}
 	server.mux = http.NewServeMux()
 	server.mux.HandleFunc("/webfiles/", webFileHandler)
+	server.mux.HandleFunc("/data/backup", backupHandler(tables.Db(), config.CurrentContext))
 	server.mux.HandleFunc("/data", queryHandler(tables, config.MaxLookback))
 	server.mux.HandleFunc("/resource", resourceHandler(config.ResourceLinks))
 	server.mux.HandleFunc("/debug/", listKeysHandler(tables))