From 8440bccb87d96463be37eee410e33e5457788969 Mon Sep 17 00:00:00 2001 From: Maceo Thompson Date: Wed, 23 Nov 2022 16:05:04 -0500 Subject: [PATCH] internal/report, cmd/vulnreport: refactor reports.GetAllExisting() to take a git repository. This change makes a few things possible: we add testing for GetAllExisting, and it allows us to add xref-like functionality to the worker in the future (see https://b.corp.google.com/issues/259438778) Change-Id: Ia183d29f0d77c01dc7cb4b228be0b9f85f8ada62 Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/454015 Reviewed-by: Tatiana Bradley TryBot-Result: Gopher Robot Run-TryBot: Maceo Thompson --- cmd/vulnreport/main.go | 12 ++++- internal/report/reports.go | 57 ++++++++++++++-------- internal/report/reports_test.go | 75 +++++++++++++++++++++++++++++ internal/report/testdata/repo.txtar | 29 +++++++++++ 4 files changed, 150 insertions(+), 23 deletions(-) create mode 100644 internal/report/reports_test.go create mode 100644 internal/report/testdata/repo.txtar diff --git a/cmd/vulnreport/main.go b/cmd/vulnreport/main.go index e8769955..a9819080 100644 --- a/cmd/vulnreport/main.go +++ b/cmd/vulnreport/main.go @@ -142,7 +142,11 @@ func main() { } cmdFunc = func(name string) error { return setDates(name, commitDates) } case "xref": - _, existingByFile, err := report.GetAllExisting() + repo, err := gitrepo.Open(ctx, ".") + if err != nil { + log.Fatal(err) + } + _, existingByFile, err := report.GetAllExisting(repo) if err != nil { log.Fatal(err) } @@ -241,7 +245,11 @@ func setupCreate(ctx context.Context, args []string) ([]int, *createCfg, error) if *githubToken == "" { return nil, nil, fmt.Errorf("githubToken must be provided") } - existingByIssue, existingByFile, err := report.GetAllExisting() + localRepo, err := gitrepo.Open(ctx, ".") + if err != nil { + log.Fatal(err) + } + existingByIssue, existingByFile, err := report.GetAllExisting(localRepo) if err != nil { return nil, nil, err } diff --git a/internal/report/reports.go b/internal/report/reports.go index 9044223d..cd1f2fc4 100644 --- a/internal/report/reports.go +++ b/internal/report/reports.go @@ -5,10 +5,14 @@ package report import ( - "os" + "fmt" "path/filepath" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" "golang.org/x/vulndb/internal/derrors" + "golang.org/x/vulndb/internal/gitrepo" + "gopkg.in/yaml.v3" ) var ( @@ -21,34 +25,45 @@ var ( ExcludedDir = "data/excluded" ) -func GetAllExisting() (byIssue map[int]*Report, byFile map[string]*Report, err error) { +func GetAllExisting(repo *git.Repository) (byIssue map[int]*Report, byFile map[string]*Report, err error) { defer derrors.Wrap(&err, "GetAllExisting") + root, err := gitrepo.Root(repo) + if err != nil { + return nil, nil, err + } byIssue = make(map[int]*Report) byFile = make(map[string]*Report) - for _, dir := range []string{YAMLDir, ExcludedDir} { - f, err := os.Open(dir) + + if err = root.Files().ForEach(func(f *object.File) error { + name := f.Name + if !(filepath.Dir(name) == YAMLDir || filepath.Dir(name) == ExcludedDir) || + filepath.Ext(name) != ".yaml" { + return nil + } + + reader, err := f.Reader() if err != nil { - return nil, nil, err + return err } - defer f.Close() - names, err := f.Readdirnames(0) + d := yaml.NewDecoder(reader) + d.KnownFields(true) + var r Report + if err := d.Decode(&r); err != nil { + return fmt.Errorf("yaml.Decode: %v", err) + } + + _, _, iss, err := ParseFilepath(name) if err != nil { - return nil, nil, err - } - for _, name := range names { - name := filepath.Join(dir, name) - _, _, iss, err := ParseFilepath(name) - if err != nil { - return nil, nil, err - } - r, err := Read(name) - if err != nil { - return nil, nil, err - } - byIssue[iss] = r - byFile[name] = r + return err } + + byFile[name] = &r + byIssue[iss] = &r + + return nil + }); err != nil { + return nil, nil, err } return byIssue, byFile, nil diff --git a/internal/report/reports_test.go b/internal/report/reports_test.go new file mode 100644 index 00000000..c179894c --- /dev/null +++ b/internal/report/reports_test.go @@ -0,0 +1,75 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package report + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "golang.org/x/vulndb/internal/gitrepo" +) + +var ( + r1 = Report{ + Modules: []*Module{ + {Module: "std"}, + }, + CVEMetadata: &CVEMeta{ + ID: "CVE-9999-0001", + }, + } + r2 = Report{ + Modules: []*Module{ + {Module: "example.com/fake/module"}, + }, + CVEMetadata: &CVEMeta{ + ID: "CVE-9999-0002", + }, + } + r4 = Report{ + Modules: []*Module{ + {Module: "example.com/another/module"}, + }, + + GHSAs: []string{ + "GHSA-9999-abcd-efgh", + }, + } + r5 = Report{ + Modules: []*Module{ + {Module: "example.com/adiff/module"}, + }, + CVEs: []string{"CVE-9999-0002"}, + } +) + +func TestGetAllExisting(t *testing.T) { + + wantByIssue := map[int]*Report{1: &r1, 2: &r2, 4: &r4, 5: &r5} + wantByFile := map[string]*Report{ + "data/reports/GO-9999-0001.yaml": &r1, + "data/excluded/GO-9999-0002.yaml": &r2, + "data/reports/GO-9999-0004.yaml": &r4, + "data/reports/GO-9999-0005.yaml": &r5, + } + + repo, err := gitrepo.ReadTxtarRepo("testdata/repo.txtar", time.Now()) + if err != nil { + t.Fatal(err) + } + + gotByIssue, gotByFile, err := GetAllExisting(repo) + if err != nil { + t.Fatalf("GetAllExisting() error = %v, ", err) + } + if diff := cmp.Diff(gotByIssue, wantByIssue); diff != "" { + t.Errorf("GetAllExisting(): byIssue mismatch (-got, +want): %s", diff) + } + + if diff := cmp.Diff(gotByFile, wantByFile); diff != "" { + t.Errorf("GetAllExisting() byFile mismatch (-got, +want): %s", diff) + } +} diff --git a/internal/report/testdata/repo.txtar b/internal/report/testdata/repo.txtar new file mode 100644 index 00000000..9157bec1 --- /dev/null +++ b/internal/report/testdata/repo.txtar @@ -0,0 +1,29 @@ +# Copyright 2022 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. +Repo for testing reports.GetAllExisting, semi in the shape of +github.com/golang/vulndb + +-- data/reports/GO-9999-0001.yaml -- +modules: + - module: std +cve_metadata: + id: CVE-9999-0001 + +-- data/excluded/GO-9999-0002.yaml -- +modules: + - module: example.com/fake/module +cve_metadata: + id: CVE-9999-0002 + +-- data/reports/GO-9999-0004.yaml -- +modules: + - module: example.com/another/module +ghsas: + - GHSA-9999-abcd-efgh + +-- data/reports/GO-9999-0005.yaml -- +modules: + - module: example.com/adiff/module +cves: + - CVE-9999-0002 \ No newline at end of file