internal/report, cmd/vulnreport: refactor reports.GetAllExisting() to take a git repository.

This change makes a few things possible: we add testing for GetAllExisting, and it allows us to add xref-like functionality to the worker in the future (see https://b.corp.google.com/issues/259438778)

Change-Id: Ia183d29f0d77c01dc7cb4b228be0b9f85f8ada62
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/454015
Reviewed-by: Tatiana Bradley <tatiana@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Maceo Thompson <maceothompson@google.com>

cmd/vulnreport/main.go

@@ -142,7 +142,11 @@ func main() {
 		}
 		cmdFunc = func(name string) error { return setDates(name, commitDates) }
 	case "xref":
-		_, existingByFile, err := report.GetAllExisting()
+		repo, err := gitrepo.Open(ctx, ".")
+		if err != nil {
+			log.Fatal(err)
+		}
+		_, existingByFile, err := report.GetAllExisting(repo)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -241,7 +245,11 @@ func setupCreate(ctx context.Context, args []string) ([]int, *createCfg, error)
 	if *githubToken == "" {
 		return nil, nil, fmt.Errorf("githubToken must be provided")
 	}
-	existingByIssue, existingByFile, err := report.GetAllExisting()
+	localRepo, err := gitrepo.Open(ctx, ".")
+	if err != nil {
+		log.Fatal(err)
+	}
+	existingByIssue, existingByFile, err := report.GetAllExisting(localRepo)
 	if err != nil {
 		return nil, nil, err
 	}

internal/report/reports.go

@@ -5,10 +5,14 @@
 package report
 
 import (
-	"os"
+	"fmt"
 	"path/filepath"
 
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/object"
 	"golang.org/x/vulndb/internal/derrors"
+	"golang.org/x/vulndb/internal/gitrepo"
+	"gopkg.in/yaml.v3"
 )
 
 var (
@@ -21,35 +25,46 @@ var (
 	ExcludedDir = "data/excluded"
 )
 
-func GetAllExisting() (byIssue map[int]*Report, byFile map[string]*Report, err error) {
+func GetAllExisting(repo *git.Repository) (byIssue map[int]*Report, byFile map[string]*Report, err error) {
 	defer derrors.Wrap(&err, "GetAllExisting")
+	root, err := gitrepo.Root(repo)
+	if err != nil {
+		return nil, nil, err
+	}
 
 	byIssue = make(map[int]*Report)
 	byFile = make(map[string]*Report)
-	for _, dir := range []string{YAMLDir, ExcludedDir} {
-		f, err := os.Open(dir)
-		if err != nil {
-			return nil, nil, err
+
+	if err = root.Files().ForEach(func(f *object.File) error {
+		name := f.Name
+		if !(filepath.Dir(name) == YAMLDir || filepath.Dir(name) == ExcludedDir) ||
+			filepath.Ext(name) != ".yaml" {
+			return nil
 		}
-		defer f.Close()
-		names, err := f.Readdirnames(0)
+
+		reader, err := f.Reader()
 		if err != nil {
-			return nil, nil, err
+			return err
 		}
-		for _, name := range names {
-			name := filepath.Join(dir, name)
+		d := yaml.NewDecoder(reader)
+		d.KnownFields(true)
+		var r Report
+		if err := d.Decode(&r); err != nil {
+			return fmt.Errorf("yaml.Decode: %v", err)
+		}
+
 		_, _, iss, err := ParseFilepath(name)
 		if err != nil {
+			return err
+		}
+
+		byFile[name] = &r
+		byIssue[iss] = &r
+
+		return nil
+	}); err != nil {
 		return nil, nil, err
 	}
-			r, err := Read(name)
-			if err != nil {
-				return nil, nil, err
-			}
-			byIssue[iss] = r
-			byFile[name] = r
-		}
-	}
 
 	return byIssue, byFile, nil
 }

internal/report/reports_test.go

@@ -0,0 +1,75 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package report
+
+import (
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/vulndb/internal/gitrepo"
+)
+
+var (
+	r1 = Report{
+		Modules: []*Module{
+			{Module: "std"},
+		},
+		CVEMetadata: &CVEMeta{
+			ID: "CVE-9999-0001",
+		},
+	}
+	r2 = Report{
+		Modules: []*Module{
+			{Module: "example.com/fake/module"},
+		},
+		CVEMetadata: &CVEMeta{
+			ID: "CVE-9999-0002",
+		},
+	}
+	r4 = Report{
+		Modules: []*Module{
+			{Module: "example.com/another/module"},
+		},
+
+		GHSAs: []string{
+			"GHSA-9999-abcd-efgh",
+		},
+	}
+	r5 = Report{
+		Modules: []*Module{
+			{Module: "example.com/adiff/module"},
+		},
+		CVEs: []string{"CVE-9999-0002"},
+	}
+)
+
+func TestGetAllExisting(t *testing.T) {
+
+	wantByIssue := map[int]*Report{1: &r1, 2: &r2, 4: &r4, 5: &r5}
+	wantByFile := map[string]*Report{
+		"data/reports/GO-9999-0001.yaml":  &r1,
+		"data/excluded/GO-9999-0002.yaml": &r2,
+		"data/reports/GO-9999-0004.yaml":  &r4,
+		"data/reports/GO-9999-0005.yaml":  &r5,
+	}
+
+	repo, err := gitrepo.ReadTxtarRepo("testdata/repo.txtar", time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gotByIssue, gotByFile, err := GetAllExisting(repo)
+	if err != nil {
+		t.Fatalf("GetAllExisting() error = %v, ", err)
+	}
+	if diff := cmp.Diff(gotByIssue, wantByIssue); diff != "" {
+		t.Errorf("GetAllExisting(): byIssue mismatch (-got, +want): %s", diff)
+	}
+
+	if diff := cmp.Diff(gotByFile, wantByFile); diff != "" {
+		t.Errorf("GetAllExisting() byFile mismatch (-got, +want): %s", diff)
+	}
+}

internal/report/testdata/repo.txtar

@@ -0,0 +1,29 @@
+# Copyright 2022 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+Repo for testing reports.GetAllExisting, semi in the shape of
+github.com/golang/vulndb
+
+-- data/reports/GO-9999-0001.yaml --
+modules:
+  - module: std
+cve_metadata:
+  id: CVE-9999-0001
+
+-- data/excluded/GO-9999-0002.yaml --
+modules:
+  - module: example.com/fake/module
+cve_metadata:
+    id: CVE-9999-0002
+
+-- data/reports/GO-9999-0004.yaml --
+modules:
+  - module: example.com/another/module
+ghsas:
+  - GHSA-9999-abcd-efgh
+
+-- data/reports/GO-9999-0005.yaml --
+modules:
+  - module: example.com/adiff/module
+cves:
+  - CVE-9999-0002