@W-8594741 - Import webfiles dynamically (#134)

* code changes to reflect webfiles as binary data

* missed a few error logging and added comments

* added test file and checking whether asset exists

* ran go mod tidy

* added const back

* checking locally first and then bin data

* updated names of func to make local and added run time checks of bin data

* readwebfiles unit tests

* fixed issue w filepath const

* ran go fmt

* minor changes

* switched expected and actual output assert

* msg to string const

* removed middle parameter in read webfile

* formatted

* changes in test file

* changes in test file

* made the error msg into a variable

* changes to dockerfile, go releaser, and comments

.goreleaser.yml

@@ -16,7 +16,7 @@ archives:
     builds:
       - sloop
     files:
-      - ./pkg/sloop/webfiles/**/*
+      - ./pkg/sloop/webserver/webfiles/**/*
 dockers:
   - goos: linux
     goarch: amd64
@@ -25,7 +25,7 @@ dockers:
     binaries:
       - sloop
     extra_files:
-      - pkg/sloop/webfiles
+      - pkg/sloop/webserver/webfiles
     image_templates:
       - "salesforce/sloop:latest"
       - "salesforce/sloop:{{ .Tag }}"

Dockerfile

@@ -6,7 +6,9 @@ RUN curl -o /aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com
 
 FROM gcr.io/distroless/base
 COPY sloop /sloop
-COPY pkg/sloop/webfiles /pkg/sloop/webfiles
+# The copy statement below can be uncommented to reflect changes to any webfiles as compared
+# to the binary version of the files in use.
+# COPY pkg/sloop/webserver/webfiles /webfiles
 COPY --from=build /aws-iam-authenticator /aws-iam-authenticator
 ENV PATH="/:${PATH}"
 CMD ["/sloop"]

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/googleapis/gnostic v0.3.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
 	github.com/imdario/mergo v0.3.8 // indirect
+	github.com/jteeuwen/go-bindata v3.0.7+incompatible
 	github.com/nsf/jsondiff v0.0.0-20190712045011-8443391ee9b6
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.2.1

go.sum

@@ -237,6 +237,8 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jteeuwen/go-bindata v3.0.7+incompatible h1:91Uy4d9SYVr1kyTJ15wJsog+esAZZl7JmEfTkwmhJts=
+github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=

pkg/sloop/queries/params.go

@@ -8,7 +8,7 @@
 package queries
 
 // Parameters are shared between webserver and here
-// Keep this in sync with pkg/sloop/webfiles/filter.js
+// Keep this in sync with pkg/sloop/webserver/webfiles/filter.js
 const (
 	LookbackParam  = "lookback"
 	NamespaceParam = "namespace"

pkg/sloop/server/internal/config/config.go

@@ -79,7 +79,7 @@ func registerFlags(fs *flag.FlagSet, config *SloopConfig) {
 	fs.BoolVar(&config.DisableKubeWatcher, "disable-kube-watch", false, "Turn off kubernetes watch")
 	fs.DurationVar(&config.KubeWatchResyncInterval, "kube-watch-resync-interval", 30*time.Minute,
 		"OPTIONAL: Kubernetes watch resync interval")
-	fs.StringVar(&config.WebFilesPath, "web-files-path", "./pkg/sloop/webfiles", "Path to web files")
+	fs.StringVar(&config.WebFilesPath, "web-files-path", "./pkg/sloop/webserver/webfiles", "Path to web files")
 	fs.StringVar(&config.BindAddress, "bind-address", "", "Web server bind ip address.")
 	fs.IntVar(&config.Port, "port", 8080, "Web server port")
 	fs.StringVar(&config.StoreRoot, "store-root", "./data", "Path to store history data")

pkg/sloop/server/internal/testFiles/testConfig.txt

@@ -4,7 +4,7 @@
 	"resourceLinks": null,
 	"disableKubeWatch": false,
 	"kubeWatchResyncInterval": 1800000000000,
-	"webfilesPath": "./pkg/sloop/webfiles",
+	"webfilesPath": "./pkg/sloop/webserver/webfiles",
 	"bindAddress": "",
 	"port": 8080,
 	"storeRoot": "./data",

pkg/sloop/server/internal/testFiles/testconfig.json

@@ -4,7 +4,7 @@
 	"resourceLinks": null,
 	"disableKubeWatch": true,
 	"kubeWatchResyncInterval": 1800000000000,
-	"webfilesPath": "./pkg/sloop/webfiles",
+	"webfilesPath": "./pkg/sloop/webserver/webfiles",
 	"bindAddress": "",
 	"port": 8080,
 	"storeRoot": "./data",

pkg/sloop/server/internal/testFiles/testconfig.yaml

@@ -4,7 +4,7 @@ leftBarLinks:
 resourceLinks: 
 disableKubeWatch: false
 kubeWatchResyncInterval: 1800000000000
-webfilesPath: "./pkg/sloop/webfiles"
+webfilesPath: "./pkg/sloop/webserver/webfiles"
 bindAddress: ''
 port: 8080
 storeRoot: "./data"

pkg/sloop/webserver/debug.go

@@ -19,7 +19,6 @@ import (
 	"github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap"
 	"html/template"
 	"net/http"
-	"path"
 	"regexp"
 	"strings"
 )
@@ -97,13 +96,12 @@ func viewKeyHandler(tables typed.Tables) http.HandlerFunc {
 		}
 		data.Payload = template.HTML(string(prettyJson))
 
-		t, err := template.New(debugViewKeyTemplateFile).ParseFiles(path.Join(webFiles, debugViewKeyTemplateFile))
+		debugViewKeyTemplate, err := getTemplate(debugViewKeyTemplateFile, _webfilesDebugviewkeyHtml)
 		if err != nil {
 			logWebError(err, "failed to parse template", request, writer)
 			return
 		}
-
-		err = t.ExecuteTemplate(writer, debugViewKeyTemplateFile, data)
+		err = debugViewKeyTemplate.Execute(writer, data)
 		if err != nil {
 			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
 			return
@@ -163,7 +161,7 @@ func listKeysHandler(tables typed.Tables) http.HandlerFunc {
 
 		writer.Header().Set("content-type", "text/html")
 
-		t, err := template.New(debugListKeysTemplateFile).ParseFiles(path.Join(webFiles, debugListKeysTemplateFile))
+		debugListKeysTemplate, err := getTemplate(debugListKeysTemplateFile, _webfilesDebuglistkeysHtml)
 		if err != nil {
 			logWebError(err, "failed to parse template", request, writer)
 			return
@@ -173,7 +171,7 @@ func listKeysHandler(tables typed.Tables) http.HandlerFunc {
 		result.TotalKeys = totalCount
 		result.TotalSize = totalSize
 		result.KeysMatched = count
-		err = t.ExecuteTemplate(writer, debugListKeysTemplateFile, result)
+		err = debugListKeysTemplate.Execute(writer, result)
 		if err != nil {
 			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
 			return
@@ -302,13 +300,12 @@ func histogramHandler(tables typed.Tables) http.HandlerFunc {
 		}
 		writer.Header().Set("content-type", "text/html")
 
-		t, err := template.New(debugHistogramFile).ParseFiles(path.Join(webFiles, debugHistogramFile))
+		debugHistogramTemplate, err := getTemplate(debugHistogramFile, _webfilesDebughistogramHtml)
 		if err != nil {
 			logWebError(err, "failed to parse histogram template", request, writer)
 			return
 		}
-
-		err = t.ExecuteTemplate(writer, debugHistogramFile, result)
+		err = debugHistogramTemplate.Execute(writer, result)
 		if err != nil {
 			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
 			return
@@ -318,12 +315,12 @@ func histogramHandler(tables typed.Tables) http.HandlerFunc {
 
 func configHandler(config string) http.HandlerFunc {
 	return func(writer http.ResponseWriter, request *http.Request) {
-		t, err := template.New(debugConfigTemplateFile).ParseFiles(path.Join(webFiles, debugConfigTemplateFile))
+		debugConfigTemplate, err := getTemplate(debugConfigTemplateFile, _webfilesDebugconfigHtml)
 		if err != nil {
 			logWebError(err, "failed to parse template", request, writer)
 			return
 		}
-		err = t.ExecuteTemplate(writer, debugConfigTemplateFile, config)
+		err = debugConfigTemplate.Execute(writer, config)
 		if err != nil {
 			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
 			return
@@ -333,12 +330,12 @@ func configHandler(config string) http.HandlerFunc {
 
 func debugHandler() http.HandlerFunc {
 	return func(writer http.ResponseWriter, request *http.Request) {
-		t, err := template.New(debugTemplateFile).ParseFiles(path.Join(webFiles, debugTemplateFile))
+		debugTemplate, err := getTemplate(debugTemplateFile, _webfilesDebugHtml)
 		if err != nil {
 			logWebError(err, "failed to parse template", request, writer)
 			return
 		}
-		err = t.ExecuteTemplate(writer, debugTemplateFile, nil)
+		err = debugTemplate.Execute(writer, nil)
 		if err != nil {
 			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
 			return
@@ -358,7 +355,7 @@ type badgerTableInfo struct {
 
 func debugBadgerTablesHandler(db badgerwrap.DB) http.HandlerFunc {
 	return func(writer http.ResponseWriter, request *http.Request) {
-		t, err := template.New(debugBadgerTablesTemplateFile).ParseFiles(path.Join(webFiles, debugBadgerTablesTemplateFile))
+		debugBadgerTablesTemplate, err := getTemplate(debugBadgerTablesTemplateFile, _webfilesDebugtablesHtml)
 		if err != nil {
 			logWebError(err, "failed to parse template", request, writer)
 			return
@@ -375,7 +372,7 @@ func debugBadgerTablesHandler(db badgerwrap.DB) http.HandlerFunc {
 			}
 			data = append(data, thisTable)
 		}
-		err = t.ExecuteTemplate(writer, debugBadgerTablesTemplateFile, data)
+		err = debugBadgerTablesTemplate.Execute(writer, data)
 		if err != nil {
 			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
 			return

pkg/sloop/webserver/indexhandler.go

@@ -8,9 +8,7 @@
 package webserver
 
 import (
-	"html/template"
 	"net/http"
-	"path"
 )
 
 type indexData struct {
@@ -23,12 +21,11 @@ type indexData struct {
 
 func indexHandler(config WebConfig) http.HandlerFunc {
 	return func(writer http.ResponseWriter, request *http.Request) {
-		t, err := template.New(indexTemplateFile).ParseFiles(path.Join(webFiles, indexTemplateFile))
+		indexTemplate, err := getTemplate(indexTemplateFile, _webfilesIndexHtml)
 		if err != nil {
 			logWebError(err, "Template.New failed", request, writer)
 			return
 		}
-
 		data := indexData{}
 		data.DefaultLookback = config.DefaultLookback
 		data.DefaultNamespace = config.DefaultNamespace
@@ -40,7 +37,7 @@ func indexHandler(config WebConfig) http.HandlerFunc {
 			return
 		}
 
-		err = t.ExecuteTemplate(writer, indexTemplateFile, data)
+		err = indexTemplate.Execute(writer, data)
 		if err != nil {
 			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
 			return

pkg/sloop/webserver/resourcehandler.go

@@ -13,7 +13,6 @@ import (
 	"github.com/salesforce/sloop/pkg/sloop/queries"
 	"html/template"
 	"net/http"
-	"path"
 	"strings"
 	"time"
 )
@@ -55,12 +54,11 @@ func runTextTemplate(templateStr string, data interface{}) (string, error) {
 
 func resourceHandler(resLinks []ResourceLinkTemplate) http.HandlerFunc {
 	return func(writer http.ResponseWriter, request *http.Request) {
-		t, err := template.New(resourceTemplateFile).ParseFiles(path.Join(webFiles, resourceTemplateFile))
+		resourceTemplate, err := getTemplate(resourceTemplateFile, _webfilesResourceHtml)
 		if err != nil {
 			logWebError(err, "Template.New failed", request, writer)
 			return
 		}
-
 		d := resourceData{}
 		d.Namespace = cleanStringFromParam(request, queries.NamespaceParam, "")
 		d.Name = cleanStringFromParam(request, queries.NameParam, "")
@@ -90,7 +88,7 @@ func resourceHandler(resLinks []ResourceLinkTemplate) http.HandlerFunc {
 		dataParams = fmt.Sprintf("?query=%v&namespace=%v&start_time=%v&end_time=%v&kind=%v&name=%v", "GetResPayload", d.Namespace, queryStart, queryEnd, d.Kind, d.Name)
 		d.PayloadUrl = "/data" + dataParams
 
-		err = t.ExecuteTemplate(writer, resourceTemplateFile, d)
+		err = resourceTemplate.Execute(writer, d)
 		if err != nil {
 			logWebError(err, "Template.ExecuteTemplate failed", request, writer)
 			return

pkg/sloop/webserver/webfiles.go

@@ -0,0 +1,54 @@
+package webserver
+
+import (
+	"fmt"
+	_ "github.com/jteeuwen/go-bindata"
+	"github.com/spf13/afero"
+	"html/template"
+	"path"
+	"sort"
+	"strings"
+)
+
+const (
+	prefix      = "webfiles/"
+	errorString = "Webfile %v is invalid.  Must start with %v"
+)
+
+// go-bindata -o bindata.go webfiles
+// ReadWebfile is a function which finds the webfiles that have been predefined and converted to binary format.
+// sample input : filepath= "webfiles/index.html"
+func readWebfile(filepath string, fs *afero.Afero) ([]byte, error) {
+	if !strings.HasPrefix(filepath, prefix) {
+		return nil, fmt.Errorf(errorString, filepath, prefix)
+	}
+	data, err := fs.ReadFile(filepath)
+	if err == nil {
+		return data, err
+	}
+	files := AssetNames()
+	//if file exists in binary form
+	if sort.SearchStrings(files, filepath) != 0 {
+		return Asset(filepath)
+	} else {
+		return nil, err
+	}
+}
+
+// Example input:
+// templateName = index.html
+// Get Template function creates a new template of the webfile passed as a string after first reading the file by
+// calling ReadWebfile ().
+func getTemplate(templateName string, _ []byte) (*template.Template, error) {
+	fs := afero.Afero{Fs: afero.NewOsFs()}
+	data, err := readWebfile((path.Join(prefix, templateName)), &fs)
+	if err != nil {
+		return nil, err
+	}
+	newTemplate := template.New(templateName)
+	newTemplate, err = newTemplate.Parse(string(data))
+	if err != nil {
+		return nil, err
+	}
+	return newTemplate, nil
+}

pkg/sloop/webserver/webfiles_test.go

@@ -0,0 +1,48 @@
+package webserver
+
+import (
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"path"
+	"testing"
+)
+
+const (
+	defaultFileMode = os.FileMode(0755)
+	someContents1   = "contents abcd"
+	filePath        = "webfiles/index.html"
+)
+
+func Test_BindataReadWebfile_True(t *testing.T) {
+	expectedOutput, err := Asset(filePath)
+	assert.Nil(t, err)
+
+	actualOutput, _ := readWebfile(filePath, &afero.Afero{afero.NewMemMapFs()})
+	assert.Equal(t, expectedOutput, actualOutput)
+}
+
+func Test_LocalReadWebfile_True(t *testing.T) {
+	notExpectedOutput, _ := Asset(filePath)
+
+	fs := &afero.Afero{afero.NewMemMapFs()}
+	writeFile(t, fs, filePath, someContents1)
+
+	actualOutput, _ := readWebfile(filePath, fs)
+
+	assert.NotEqual(t, notExpectedOutput, actualOutput)
+	assert.Equal(t, []uint8(someContents1), actualOutput)
+}
+
+func Test_FilenotinReqdFormat_False(t *testing.T) {
+	filePath := "index.html"
+	_, err := readWebfile(filePath, &afero.Afero{afero.NewMemMapFs()})
+	assert.Errorf(t, err, errorString, filePath, prefix)
+}
+
+func writeFile(t *testing.T, fs *afero.Afero, filePath string, content string) {
+	err := fs.MkdirAll(path.Dir(filePath), defaultFileMode)
+	assert.Nil(t, err)
+	err = fs.WriteFile(filePath, []byte(content), defaultFileMode)
+	assert.Nil(t, err)
+}

pkg/sloop/webserver/webserver.go

@@ -11,7 +11,7 @@ import (
 	"context"
 	"expvar"
 	"fmt"
-	"io/ioutil"
+	"github.com/spf13/afero"
 	"log"
 	"mime"
 	"net/http"
@@ -96,8 +96,8 @@ func webFileHandler(w http.ResponseWriter, r *http.Request) {
 		logWebError(nil, "Not allowed", r, w)
 		return
 	}
-	fullPath := path.Join(webFiles, fixedUrl)
-	data, err := ioutil.ReadFile(fullPath)
+	fullPath := path.Join(prefix, fixedUrl)
+	data, err := readWebfile(fullPath, &afero.Afero{afero.NewOsFs()})
 	if err != nil {
 		logWebError(err, "Error reading web file: "+fixedUrl, r, w)
 		return