Add redis mirror finder (#4)

* Intermediate progress.

* Rudimentary Redis Scenario working

* Formatting

* A few things I thought of last night.

* Adding test for RedisFinder_FindMirrors

* Adding license information

Gopkg.lock

@@ -7,6 +7,12 @@
   revision = "629574ca2a5df945712d3079857300b5e4da0236"
   version = "v1.4.2"
 
+[[projects]]
+  name = "github.com/go-redis/redis"
+  packages = [".","internal","internal/consistenthash","internal/hashtag","internal/pool","internal/proto"]
+  revision = "9c885d5ba8a22f260badcffde1649915fbeebfc0"
+  version = "v6.7.5"
+
 [[projects]]
   branch = "master"
   name = "github.com/hashicorp/hcl"
@@ -100,6 +106,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "e02c290d61ce4875ebc4567cffa3cc526ec74d268a768333cc3827e3576a40e4"
+  inputs-digest = "af5b5496f0dbacdaf738399cca08959f8768f1e50d072b0dcbd6aca7b8f1130a"
   solver-name = "gps-cdcl"
   solver-version = 1

mirrorcat/cmd/root.go

@@ -51,6 +51,7 @@ func init() {
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
+	viper.SetEnvPrefix("mirrorcat")
 	if cfgFile != "" {
 		// Use config file from the flag.
 		viper.SetConfigFile(cfgFile)

mirrorcat/cmd/start.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/Azure/mirrorcat"
+	"github.com/go-redis/redis"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -46,6 +47,25 @@ var startCmd = &cobra.Command{
 		port := viper.GetInt("port")
 		log.Printf("Listening on port %d\n", port)
 
+		if options, err := redis.ParseURL(viper.GetString("redis-connection")); err != nil {
+			log.Println("Unable to connect to Redis Because: ", err)
+		} else {
+			client := redis.NewClient(options)
+
+			go func() {
+				log.Print("Connecting to Redis at ", options.Addr)
+
+				allMirrors = append(allMirrors, mirrorcat.RedisFinder(*client))
+
+				_, err := client.Keys("*").Result()
+				if err != nil {
+					log.Print("Unable to connect to Redis because: ", err)
+				} else {
+					log.Print("Successfully connected to Redis.")
+				}
+			}()
+		}
+
 		if http.ListenAndServe(fmt.Sprintf(":%d", port), nil) != nil {
 			return
 		}
@@ -88,6 +108,9 @@ func init() {
 	startCmd.Flags().UintP("clone-depth", "c", 0, "The number of commits to checkout while cloning the original repository. (The default behavior is to clone all of the commits in the original repository.)")
 	viper.BindPFlag("clone-depth", startCmd.Flags().Lookup("clone-depth"))
 
+	startCmd.Flags().StringP("redis-connection", "r", "", "The host to contact Redis with, if it's relevant.")
+	viper.BindPFlag("redis-connection", startCmd.Flags().Lookup("redis-connection"))
+
 	viper.SetDefault("port", DefaultPort)
 	viper.SetDefault("clone-depth", DefaultCloneDepth)
 }
@@ -133,7 +156,7 @@ func handleGitHubPushEvent(resp http.ResponseWriter, req *http.Request) {
 	}
 	mirrors := make(chan mirrorcat.RemoteRef)
 
-	go staticMirrors.FindMirrors(ctx, original, mirrors)
+	go allMirrors.FindMirrors(ctx, original, mirrors)
 
 	bodyWriter := json.NewEncoder(resp)
 loop:
@@ -165,6 +188,7 @@ loop:
 	log.Println("Request Completed.")
 }
 
+var allMirrors = mirrorcat.MergeFinder{staticMirrors}
 var staticMirrors = mirrorcat.NewDefaultMirrorFinder()
 
 var populateStaticMirrors = func() func() error {

mirrorcat/cmd/version.go

@@ -24,6 +24,8 @@ import (
 	"fmt"
 	"runtime"
 
+	"github.com/spf13/viper"
+
 	"github.com/spf13/cobra"
 )
 
@@ -36,6 +38,16 @@ var versionCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println("Built from commit:", commit)
 		fmt.Println(runtime.Version(), runtime.GOOS)
+		if viper.GetBool("license") {
+			fmt.Println()
+			fmt.Println(`This Software was written by Microsoft Corp. and Contributors and is Licensed under the MIT license.
+The text of which can be found here:
+https://github.com/Azure/mirrorcat/blob/master/LICENSE
+
+A portion of this Software was written by the github.com/go-redis/redis Authors and is licensed under the BSD-2 Clause.
+The License for this software can be found here:
+https://github.com/go-redis/redis/blob/master/LICENSE`)
+		}
 	},
 }
 
@@ -55,4 +67,7 @@ func init() {
 	// Cobra supports local flags which will only run when this command
 	// is called directly, e.g.:
 	// versionCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+
+	versionCmd.Flags().BoolP("license", "l", false, "Provide this flag to see license information about this project.")
+	viper.BindPFlag("license", versionCmd.Flags().Lookup("license"))
 }

redis_finder.go

@@ -0,0 +1,70 @@
+package mirrorcat
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/go-redis/redis"
+)
+
+// RedisFinder implementes the MirrorFinder interface against a Redis Cache.
+type RedisFinder redis.Client
+
+// RedisRemoteRef allows easy conversion to `string` from a `mirrorcat.RemoteRef`.
+type RedisRemoteRef RemoteRef
+
+// ParseRedisRemoteRef reads a string formatted as a RedisRemoteRef and reads it into
+// a `mirrorcat.RemoteRef`.
+func ParseRedisRemoteRef(input string) (RedisRemoteRef, error) {
+	splitPoint := strings.IndexRune(input, ':')
+	if splitPoint < 0 {
+		return RedisRemoteRef{}, fmt.Errorf("%q does not resemble a RedisRemoteRef", input)
+	}
+
+	return RedisRemoteRef{
+		Ref:        input[:splitPoint],
+		Repository: input[splitPoint+1:],
+	}, nil
+}
+
+func (rrr RedisRemoteRef) String() string {
+	return fmt.Sprintf("%s:%s", rrr.Ref, rrr.Repository)
+}
+
+// FindMirrors scrapes a Redis Cache, looking for any mirror entries.
+//
+// It is expected that the Redis Cache will contain a key which is the result of
+// `mirrorcat.RedisRemoteRef(original).String()`. At that key, MirrorCat expects to
+// find a Set of strings matching the format of the key, but targeting other repositories
+// and refs.
+func (rf RedisFinder) FindMirrors(ctx context.Context, original RemoteRef, results chan<- RemoteRef) error {
+	defer close(results)
+
+	base := redis.Client(rf)
+
+	memberCmd := base.SMembers(RedisRemoteRef(original).String())
+
+	mirrors, err := memberCmd.Result()
+	if err != nil {
+		return err
+	}
+
+	log.Printf("Found %d Redis entries for mirror %q", len(mirrors), RedisRemoteRef(original).String())
+
+	for _, item := range mirrors {
+		parsed, err := ParseRedisRemoteRef(item)
+		if err != nil {
+			return err
+		}
+		select {
+		case results <- RemoteRef(parsed):
+			// Intentionally Left Blank
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+
+	return nil
+}

redis_finder_test.go

@@ -0,0 +1,177 @@
+package mirrorcat_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/Azure/mirrorcat"
+	"github.com/go-redis/redis"
+	"github.com/spf13/viper"
+)
+
+func ExampleRedisRemoteRef_String() {
+	subject := mirrorcat.RemoteRef{
+		Repository: "https://github.com/Azure/mirrorcat",
+		Ref:        "master",
+	}
+
+	marshaled := mirrorcat.RedisRemoteRef(subject).String()
+	fmt.Println(marshaled)
+
+	// Output: master:https://github.com/Azure/mirrorcat
+}
+
+func ExampleParseRedisRemoteRef() {
+	unmarshaled, err := mirrorcat.ParseRedisRemoteRef("master:https://github.com/Azure/mirrorcat")
+	if err != nil {
+		return
+	}
+
+	fmt.Println("Repository:", unmarshaled.Repository)
+	fmt.Println("Ref:", unmarshaled.Ref)
+
+	// Output:
+	// Repository: https://github.com/Azure/mirrorcat
+	// Ref: master
+}
+
+func TestParseRedisRemoteRef(t *testing.T) {
+	testCases := []struct {
+		string
+		want mirrorcat.RedisRemoteRef
+	}{
+		{":", mirrorcat.RedisRemoteRef{}},
+		{"left:right", mirrorcat.RedisRemoteRef{Ref: "left", Repository: "right"}},
+		{"branch:https://hostname:1234/folk?person=Pete%20Seeger", mirrorcat.RedisRemoteRef{Ref: "branch", Repository: "https://hostname:1234/folk?person=Pete%20Seeger"}},
+	}
+
+	for _, tc := range testCases {
+		t.Run("", func(t *testing.T) {
+			unmarshaled, err := mirrorcat.ParseRedisRemoteRef(tc.string)
+
+			if err != nil {
+				t.Log("unexpected err: ", err)
+				t.Fail()
+			}
+
+			if unmarshaled.Repository != tc.want.Repository {
+				t.Logf("got: %q want: %q", unmarshaled.Repository, tc.want.Repository)
+				t.Fail()
+			}
+
+			if unmarshaled.Ref != tc.want.Ref {
+				t.Logf("got: %q want: %q", unmarshaled.Ref, tc.want.Ref)
+				t.Fail()
+			}
+		})
+	}
+}
+
+func TestParseRedisRemoteRef_Invalid(t *testing.T) {
+	testCases := []string{
+		"",
+		"github.com/Azure/mirrorcat",
+	}
+
+	for _, tc := range testCases {
+		t.Run("", func(t *testing.T) {
+			unmarshaled, err := mirrorcat.ParseRedisRemoteRef(tc)
+			if err == nil {
+				t.Log("expected a non-nil error for:", tc)
+				t.Log("got:", err)
+				t.Fail()
+			}
+			if unmarshaled.Repository != "" || unmarshaled.Ref != "" {
+				t.Log("expected the empty RedisRemoteRef for:", tc)
+				t.Fail()
+			}
+		})
+	}
+}
+
+func TestRedisFinder_FindMirrors(t *testing.T) {
+	viper.BindEnv("redis-connection", "MIRRORCAT_REDIS_CONNECTION")
+	viper.SetDefault("redis-connection", "redis://localhost:6379")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	t.Log("using redis connection:", viper.GetString("redis-connection"))
+	connectionOptions, err := redis.ParseURL(viper.GetString("redis-connection"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	const testKey = "master:testRepo"
+	expected := map[mirrorcat.RemoteRef]struct{}{
+		mirrorcat.RemoteRef{Repository: "testRepo", Ref: "dev"}:  struct{}{},
+		mirrorcat.RemoteRef{Repository: "otherRepo", Ref: "dev"}: struct{}{},
+	}
+
+	expectedArr := make([]interface{}, 0, len(expected))
+	for item := range expected {
+		expectedArr = append(expectedArr, mirrorcat.RedisRemoteRef(item).String())
+	}
+
+	client := redis.NewClient(connectionOptions)
+	_, err = client.SAdd(testKey, expectedArr...).Result()
+	defer func() {
+		_, err = client.Del(testKey).Result()
+		if err != nil {
+			t.Log("Unable to cleanup Redis Instance: ", err)
+		} else {
+			t.Log("Redis Instance cleaned up.")
+		}
+	}()
+
+	if err != nil {
+		t.Log("Unable to connect to Redis instance: ", err)
+		t.SkipNow()
+	}
+
+	subject := mirrorcat.RedisFinder(*client)
+
+	testRepo, err := mirrorcat.ParseRedisRemoteRef(testKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	results, errs := make(chan mirrorcat.RemoteRef), make(chan error, 1)
+
+	go func() {
+		select {
+		case errs <- subject.FindMirrors(ctx, mirrorcat.RemoteRef(testRepo), results):
+		case <-ctx.Done():
+			errs <- ctx.Err()
+		}
+	}()
+
+loop:
+	for {
+		select {
+		case err = <-errs:
+			if err != nil {
+				t.Fatal(err)
+			}
+		case seen, ok := <-results:
+			if !ok {
+				break loop
+			}
+
+			_, ok = expected[seen]
+			if ok {
+				delete(expected, seen)
+			} else {
+				t.Log("unexpected result: ", seen)
+				t.Fail()
+			}
+		}
+	}
+
+	for unseen := range expected {
+		t.Log("didn't see expected value: ", unseen)
+		t.Fail()
+	}
+}