go
/
vulndb
зеркало из https://github.com/golang/vulndb.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

internal/cvelistrepo: package for the cvelist repo 

Create a package for working with github.com/CVEProject/cvelist.

Change-Id: Iadac53c0f13ba658bc20a9bd4f74d58ade5a4f44
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/375716
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Julie Qiu <julie@golang.org>
This commit is contained in:
Jonathan Amsterdam 2022-01-05 16:18:25 -05:00
Родитель 416c73c9eb
Коммит 12508860a4
12 изменённых файлов: 353 добавлений и 308 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7
cmd/vulnreport/main.go
Просмотреть файл

@ -19,12 +19,11 @@ import (
"os"
"golang.org/x/vulndb/internal"
"golang.org/x/vulndb/internal/cvelistrepo"
"golang.org/x/vulndb/internal/derrors"
"golang.org/x/vulndb/internal/gitrepo"
"golang.org/x/vulndb/internal/issues"
"golang.org/x/vulndb/internal/report"
"golang.org/x/vulndb/internal/stdlib"
"golang.org/x/vulndb/internal/worker"
"gopkg.in/yaml.v2"
)
@ -61,7 +60,7 @@ func main() {
if err != nil {
log.Fatal(err)
}
repoPath := gitrepo.CVEListRepoURL
repoPath := cvelistrepo.URL
if *localRepoPath != "" {
repoPath = *localRepoPath
}
@ -111,7 +110,7 @@ func create(ctx context.Context, issueNumber int, ghToken, issueRepo, repoPath s
if !strings.HasPrefix(cveID, "CVE") {
return fmt.Errorf("expected last element of title to be the CVE ID; got %q", iss.Title)
}
cve, err := worker.FetchCVE(ctx, repoPath, cveID)
cve, err := cvelistrepo.FetchCVE(ctx, repoPath, cveID)
if err != nil {
return err
}

4
cmd/worker/main.go
Просмотреть файл

@ -22,7 +22,7 @@ import (
"time"
"golang.org/x/vulndb/internal"
"golang.org/x/vulndb/internal/gitrepo"
"golang.org/x/vulndb/internal/cvelistrepo"
"golang.org/x/vulndb/internal/issues"
"golang.org/x/vulndb/internal/worker"
"golang.org/x/vulndb/internal/worker/log"
@ -184,7 +184,7 @@ func listCVEsCommand(ctx context.Context, triageState string) error {
}
func updateCommand(ctx context.Context, commitHash string) error {
repoPath := gitrepo.CVEListRepoURL
repoPath := cvelistrepo.URL
if *localRepoPath != "" {
repoPath = *localRepoPath
}

159
internal/cvelistrepo/cvelistrepo.go Normal file
Просмотреть файл

@ -0,0 +1,159 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package cvelistrepo supports working with the repo
// containing the list of CVEs.
package cvelistrepo
import (
"context"
"encoding/json"
"fmt"
"io"
"path"
"sort"
"strconv"
"strings"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
"golang.org/x/vulndb/internal/cveschema"
"golang.org/x/vulndb/internal/derrors"
"golang.org/x/vulndb/internal/gitrepo"
)
// URL is the URL of the cvelist repo.
const URL = "https://github.com/CVEProject/cvelist"
// A File is a file in the cvelist repo that contains a CVE.
type File struct {
DirPath string
Filename string
TreeHash plumbing.Hash
BlobHash plumbing.Hash
Year int
Number int
}
// Files returns all the CVE files in the given repo commit, sorted by
// name.
func Files(repo *git.Repository, commit *object.Commit) (_ []File, err error) {
defer derrors.Wrap(&err, "CVEFiles(%s)", commit.Hash)
root, err := repo.TreeObject(commit.TreeHash)
if err != nil {
return nil, fmt.Errorf("TreeObject: %v", err)
}
files, err := walkFiles(repo, root, "", nil)
if err != nil {
return nil, err
}
sort.Slice(files, func(i, j int) bool {
// Compare the year and the number, as ints. Using the ID directly
// would put CVE-2014-100009 before CVE-2014-10001.
if files[i].Year != files[j].Year {
return files[i].Year < files[j].Year
}
return files[i].Number < files[j].Number
})
return files, nil
}
// walkFiles collects CVE files from a repo tree.
func walkFiles(repo *git.Repository, tree *object.Tree, dirpath string, files []File) ([]File, error) {
for _, e := range tree.Entries {
if e.Mode == filemode.Dir {
dir, err := repo.TreeObject(e.Hash)
if err != nil {
return nil, err
}
files, err = walkFiles(repo, dir, path.Join(dirpath, e.Name), files)
if err != nil {
return nil, err
}
} else if isCVEFilename(e.Name) {
// e.Name is CVE-YEAR-NUMBER.json
year, err := strconv.Atoi(e.Name[4:8])
if err != nil {
return nil, err
}
number, err := strconv.Atoi(e.Name[9 : len(e.Name)-5])
if err != nil {
return nil, err
}
files = append(files, File{
DirPath: dirpath,
Filename: e.Name,
TreeHash: tree.Hash,
BlobHash: e.Hash,
Year: year,
Number: number,
})
}
}
return files, nil
}
// isCVEFilename reports whether name is the basename of a CVE file.
func isCVEFilename(name string) bool {
return strings.HasPrefix(name, "CVE-") && path.Ext(name) == ".json"
}
// blobReader returns a reader to the blob with the given hash.
func blobReader(repo *git.Repository, hash plumbing.Hash) (io.Reader, error) {
blob, err := repo.BlobObject(hash)
if err != nil {
return nil, err
}
return blob.Reader()
}
// FetchCVE fetches the CVE file for cveID from the CVElist repo and returns
// the parsed info.
func FetchCVE(ctx context.Context, repoPath, cveID string) (_ *cveschema.CVE, err error) {
defer derrors.Wrap(&err, "FetchCVE(repo, commit, %s)", cveID)
repo, err := gitrepo.CloneOrOpen(ctx, repoPath)
if err != nil {
return nil, err
}
ref, err := repo.Reference(plumbing.HEAD, true)
if err != nil {
return nil, err
}
ch := ref.Hash()
commit, err := repo.CommitObject(ch)
if err != nil {
return nil, err
}
files, err := Files(repo, commit)
if err != nil {
return nil, err
}
for _, f := range files {
if strings.Contains(f.Filename, cveID) {
cve, err := ParseCVE(repo, f)
if err != nil {
return nil, err
}
return cve, nil
}
}
return nil, fmt.Errorf("not found")
}
// ParseCVE parses the CVE file f into a CVE.
func ParseCVE(repo *git.Repository, f File) (*cveschema.CVE, error) {
// Read CVE from repo.
r, err := blobReader(repo, f.BlobHash)
if err != nil {
return nil, err
}
cve := &cveschema.CVE{}
if err := json.NewDecoder(r).Decode(cve); err != nil {
return nil, err
}
return cve, nil
}

57
internal/cvelistrepo/cvelistrepo_test.go Normal file
Просмотреть файл

@ -0,0 +1,57 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package cvelistrepo
import (
"testing"
"time"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/vulndb/internal/gitrepo"
)
func TestRepoCVEFiles(t *testing.T) {
repo, err := gitrepo.ReadTxtarRepo("testdata/basic.txtar", time.Now())
if err != nil {
t.Fatal(err)
}
commit := headCommit(t, repo)
if err != nil {
t.Fatal(err)
}
got, err := Files(repo, commit)
if err != nil {
t.Fatal(err)
}
want := []File{
{DirPath: "2020/9xxx", Filename: "CVE-2020-9283.json", Year: 2020, Number: 9283},
{DirPath: "2021/0xxx", Filename: "CVE-2021-0001.json", Year: 2021, Number: 1},
{DirPath: "2021/0xxx", Filename: "CVE-2021-0010.json", Year: 2021, Number: 10},
{DirPath: "2021/1xxx", Filename: "CVE-2021-1384.json", Year: 2021, Number: 1384},
}
opt := cmpopts.IgnoreFields(File{}, "TreeHash", "BlobHash")
if diff := cmp.Diff(want, got, cmp.AllowUnexported(File{}), opt); diff != "" {
t.Errorf("mismatch (-want, +got):\n%s", diff)
}
}
// headCommit returns the commit at the repo HEAD.
func headCommit(t *testing.T, repo *git.Repository) *object.Commit {
h, err := gitrepo.HeadHash(repo)
if err != nil {
t.Fatal(err)
}
commit, err := repo.CommitObject(h)
if err != nil {
t.Fatal(err)
}
return commit
}

0
internal/worker/testdata/basic.txtar → internal/cvelistrepo/testdata/basic.txtar поставляемый
Просмотреть файл

61
internal/gitrepo/gitrepo.go
Просмотреть файл

@ -8,17 +8,18 @@ package gitrepo
import (
"context"
"strings"
"time"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/storage/memory"
"golang.org/x/tools/txtar"
"golang.org/x/vulndb/internal/derrors"
"golang.org/x/vulndb/internal/worker/log"
)
const CVEListRepoURL = "https://github.com/CVEProject/cvelist"
// Clone returns a repo by cloning the repo at repoURL.
func Clone(ctx context.Context, repoURL string) (repo *git.Repository, err error) {
defer derrors.Wrap(&err, "gitrepo.Clone(%q)", repoURL)
@ -65,3 +66,59 @@ func Root(repo *git.Repository) (root *object.Tree, err error) {
}
return repo.TreeObject(commit.TreeHash)
}
// ReadTxtarRepo converts a txtar file to a single-commit
// repo. It is intended for testing.
func ReadTxtarRepo(filename string, now time.Time) (_ *git.Repository, err error) {
defer derrors.Wrap(&err, "readTxtarRepo(%q)", filename)
mfs := memfs.New()
ar, err := txtar.ParseFile(filename)
if err != nil {
return nil, err
}
for _, f := range ar.Files {
file, err := mfs.Create(f.Name)
if err != nil {
return nil, err
}
if _, err := file.Write(f.Data); err != nil {
return nil, err
}
if err := file.Close(); err != nil {
return nil, err
}
}
repo, err := git.Init(memory.NewStorage(), mfs)
if err != nil {
return nil, err
}
wt, err := repo.Worktree()
if err != nil {
return nil, err
}
for _, f := range ar.Files {
if _, err := wt.Add(f.Name); err != nil {
return nil, err
}
}
_, err = wt.Commit("", &git.CommitOptions{All: true, Author: &object.Signature{
Name: "Joe Random",
Email: "joe@example.com",
When: now,
}})
if err != nil {
return nil, err
}
return repo, nil
}
// HeadHash returns the hash of the repo's HEAD.
func HeadHash(repo *git.Repository) (plumbing.Hash, error) {
ref, err := repo.Reference(plumbing.HEAD, true)
if err != nil {
return plumbing.ZeroHash, err
}
return ref.Hash(), nil
}

82
internal/worker/repo_test.go
Просмотреть файл

@ -6,85 +6,3 @@
// +build go1.17
package worker
import (
"testing"
"time"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/storage/memory"
"golang.org/x/tools/txtar"
"golang.org/x/vulndb/internal/derrors"
)
// readTxtarRepo converts a txtar file to a single-commit
// repo.
func readTxtarRepo(filename string, now time.Time) (_ *git.Repository, err error) {
defer derrors.Wrap(&err, "readTxtarRepo(%q)", filename)
mfs := memfs.New()
ar, err := txtar.ParseFile(filename)
if err != nil {
return nil, err
}
for _, f := range ar.Files {
file, err := mfs.Create(f.Name)
if err != nil {
return nil, err
}
if _, err := file.Write(f.Data); err != nil {
return nil, err
}
if err := file.Close(); err != nil {
return nil, err
}
}
repo, err := git.Init(memory.NewStorage(), mfs)
if err != nil {
return nil, err
}
wt, err := repo.Worktree()
if err != nil {
return nil, err
}
for _, f := range ar.Files {
if _, err := wt.Add(f.Name); err != nil {
return nil, err
}
}
_, err = wt.Commit("", &git.CommitOptions{All: true, Author: &object.Signature{
Name: "Joe Random",
Email: "joe@example.com",
When: now,
}})
if err != nil {
return nil, err
}
return repo, nil
}
// headCommit returns the commit at the repo HEAD.
func headCommit(t *testing.T, repo *git.Repository) *object.Commit {
h, err := headHash(repo)
if err != nil {
t.Fatal(err)
}
commit, err := repo.CommitObject(h)
if err != nil {
t.Fatal(err)
}
return commit
}
// headHash returns the hash of the repo's HEAD.
func headHash(repo *git.Repository) (plumbing.Hash, error) {
ref, err := repo.Reference(plumbing.HEAD, true)
if err != nil {
return plumbing.ZeroHash, err
}
return ref.Hash(), nil
}

6
internal/worker/server.go
Просмотреть файл

@ -21,8 +21,8 @@ import (
"github.com/google/safehtml/template"
"golang.org/x/sync/errgroup"
"golang.org/x/vulndb/internal"
"golang.org/x/vulndb/internal/cvelistrepo"
"golang.org/x/vulndb/internal/derrors"
"golang.org/x/vulndb/internal/gitrepo"
"golang.org/x/vulndb/internal/issues"
"golang.org/x/vulndb/internal/worker/log"
"golang.org/x/vulndb/internal/worker/store"
@ -226,7 +226,7 @@ func (s *Server) indexPage(w http.ResponseWriter, r *http.Request) error {
}
page := indexPage{
CVEListRepoURL: gitrepo.CVEListRepoURL,
CVEListRepoURL: cvelistrepo.URL,
Namespace: s.cfg.Namespace,
Updates: updates,
CVEsNeedingIssue: needingIssue,
@ -254,7 +254,7 @@ func (s *Server) doUpdate(r *http.Request) error {
if f := r.FormValue("force"); f == "true" {
force = true
}
err := UpdateCommit(r.Context(), gitrepo.CVEListRepoURL, "HEAD", s.cfg.Store, pkgsiteURL, force)
err := UpdateCommit(r.Context(), cvelistrepo.URL, "HEAD", s.cfg.Store, pkgsiteURL, force)
if cerr := new(CheckUpdateError); errors.As(err, &cerr) {
return &serverError{
status: http.StatusPreconditionFailed,

182
internal/worker/update.go
Просмотреть файл

@ -6,22 +6,16 @@ package worker
import (
"context"
"encoding/json"
"fmt"
"io"
"path"
"sort"
"strconv"
"strings"
"time"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
"golang.org/x/vulndb/internal/cvelistrepo"
"golang.org/x/vulndb/internal/cveschema"
"golang.org/x/vulndb/internal/derrors"
"golang.org/x/vulndb/internal/gitrepo"
"golang.org/x/vulndb/internal/worker/log"
"golang.org/x/vulndb/internal/worker/store"
)
@ -97,7 +91,7 @@ func (u *updater) update(ctx context.Context) (ur *store.CommitUpdateRecord, err
// It is cheaper to read all the files from the repo and compare
// them to the DB in bulk, than to walk the repo and process
// each file individually.
files, err := repoCVEFiles(u.repo, commit)
files, err := cvelistrepo.Files(u.repo, commit)
if err != nil {
return nil, err
}
@ -143,9 +137,9 @@ func (u *updater) update(ctx context.Context) (ur *store.CommitUpdateRecord, err
// See https://cloud.google.com/firestore/quotas.
const maxTransactionWrites = 500
func (u *updater) updateDirectory(ctx context.Context, dirFiles []repoFile) (_ updateStats, err error) {
dirPath := dirFiles[0].dirPath
dirHash := dirFiles[0].treeHash.String()
func (u *updater) updateDirectory(ctx context.Context, dirFiles []cvelistrepo.File) (_ updateStats, err error) {
dirPath := dirFiles[0].DirPath
dirHash := dirFiles[0].TreeHash.String()
// A non-empty directory hash means that we have fully processed the directory
// with that hash. If the stored hash matches the current one, we can skip
@ -191,9 +185,9 @@ func (u *updater) updateDirectory(ctx context.Context, dirFiles []repoFile) (_ u
return stats, nil
}
func (u *updater) updateBatch(ctx context.Context, batch []repoFile) (numAdds, numMods int, err error) {
startID := idFromFilename(batch[0].filename)
endID := idFromFilename(batch[len(batch)-1].filename)
func (u *updater) updateBatch(ctx context.Context, batch []cvelistrepo.File) (numAdds, numMods int, err error) {
startID := idFromFilename(batch[0].Filename)
endID := idFromFilename(batch[len(batch)-1].Filename)
defer derrors.Wrap(&err, "updateBatch(%s-%s)", startID, endID)
err = u.st.RunTransaction(ctx, func(ctx context.Context, tx store.Transaction) error {
@ -213,9 +207,9 @@ func (u *updater) updateBatch(ctx context.Context, batch []repoFile) (numAdds, n
}
// Determine what needs to be added and modified.
for _, f := range batch {
id := idFromFilename(f.filename)
id := idFromFilename(f.Filename)
old := idToRecord[id]
if old != nil && old.BlobHash == f.blobHash.String() {
if old != nil && old.BlobHash == f.BlobHash.String() {
// No change; do nothing.
continue
}
@ -241,9 +235,9 @@ func (u *updater) updateBatch(ctx context.Context, batch []repoFile) (numAdds, n
// handleCVE determines how to change the store for a single CVE.
// The CVE will definitely be either added, if it's new, or modified, if it's
// already in the DB.
func (u *updater) handleCVE(f repoFile, old *store.CVERecord, tx store.Transaction) (added bool, err error) {
defer derrors.Wrap(&err, "handleCVE(%s)", f.filename)
cve, err := parseCVE(u.repo, f)
func (u *updater) handleCVE(f cvelistrepo.File, old *store.CVERecord, tx store.Transaction) (added bool, err error) {
defer derrors.Wrap(&err, "handleCVE(%s)", f.Filename)
cve, err := cvelistrepo.ParseCVE(u.repo, f)
if err != nil {
return false, err
}
@ -263,10 +257,10 @@ func (u *updater) handleCVE(f repoFile, old *store.CVERecord, tx store.Transacti
}
}
pathname := path.Join(f.dirPath, f.filename)
pathname := path.Join(f.DirPath, f.Filename)
// If the CVE is not in the database, add it.
if old == nil {
cr := store.NewCVERecord(cve, pathname, f.blobHash.String())
cr := store.NewCVERecord(cve, pathname, f.BlobHash.String())
cr.CommitHash = u.commitHash.String()
switch {
case result != nil:
@ -286,7 +280,7 @@ func (u *updater) handleCVE(f repoFile, old *store.CVERecord, tx store.Transacti
// Change to an existing record.
mod := *old // copy the old one
mod.Path = pathname
mod.BlobHash = f.blobHash.String()
mod.BlobHash = f.BlobHash.String()
mod.CVEState = cve.State
mod.CommitHash = u.commitHash.String()
switch old.TriageState {
@ -334,52 +328,6 @@ func (u *updater) handleCVE(f repoFile, old *store.CVERecord, tx store.Transacti
return false, nil
}
// FetchCVE fetches the CVE file for cveID from the CVElist repo and returns
// the parsed info.
func FetchCVE(ctx context.Context, repoPath, cveID string) (_ *cveschema.CVE, err error) {
defer derrors.Wrap(&err, "FetchCVE(repo, commit, %s)", cveID)
repo, err := gitrepo.CloneOrOpen(ctx, repoPath)
if err != nil {
return nil, err
}
ref, err := repo.Reference(plumbing.HEAD, true)
if err != nil {
return nil, err
}
ch := ref.Hash()
commit, err := repo.CommitObject(ch)
if err != nil {
return nil, err
}
files, err := repoCVEFiles(repo, commit)
if err != nil {
return nil, err
}
for _, f := range files {
if strings.Contains(f.filename, cveID) {
cve, err := parseCVE(repo, f)
if err != nil {
return nil, err
}
return cve, nil
}
}
return nil, fmt.Errorf("not found")
}
func parseCVE(repo *git.Repository, f repoFile) (*cveschema.CVE, error) {
// Read CVE from repo.
r, err := blobReader(repo, f.blobHash)
if err != nil {
return nil, err
}
cve := &cveschema.CVE{}
if err := json.NewDecoder(r).Decode(cve); err != nil {
return nil, err
}
return cve, nil
}
// copyRemoving returns a copy of cve with any reference that has a given URL removed.
func copyRemoving(cve *cveschema.CVE, refURLs []string) *cveschema.CVE {
remove := map[string]bool{}
@ -397,86 +345,18 @@ func copyRemoving(cve *cveschema.CVE, refURLs []string) *cveschema.CVE {
return &c
}
type repoFile struct {
dirPath string
filename string
treeHash plumbing.Hash
blobHash plumbing.Hash
year int
number int
}
// repoCVEFiles returns all the CVE files in the given repo commit, sorted by
// name.
func repoCVEFiles(repo *git.Repository, commit *object.Commit) (_ []repoFile, err error) {
defer derrors.Wrap(&err, "repoCVEFiles(%s)", commit.Hash)
root, err := repo.TreeObject(commit.TreeHash)
if err != nil {
return nil, fmt.Errorf("TreeObject: %v", err)
}
files, err := walkFiles(repo, root, "", nil)
if err != nil {
return nil, err
}
sort.Slice(files, func(i, j int) bool {
// Compare the year and the number, as ints. Using the ID directly
// would put CVE-2014-100009 before CVE-2014-10001.
if files[i].year != files[j].year {
return files[i].year < files[j].year
}
return files[i].number < files[j].number
})
return files, nil
}
// walkFiles collects CVE files from a repo tree.
func walkFiles(repo *git.Repository, tree *object.Tree, dirpath string, files []repoFile) ([]repoFile, error) {
for _, e := range tree.Entries {
if e.Mode == filemode.Dir {
dir, err := repo.TreeObject(e.Hash)
if err != nil {
return nil, err
}
files, err = walkFiles(repo, dir, path.Join(dirpath, e.Name), files)
if err != nil {
return nil, err
}
} else if isCVEFilename(e.Name) {
// e.Name is CVE-YEAR-NUMBER.json
year, err := strconv.Atoi(e.Name[4:8])
if err != nil {
return nil, err
}
number, err := strconv.Atoi(e.Name[9 : len(e.Name)-5])
if err != nil {
return nil, err
}
files = append(files, repoFile{
dirPath: dirpath,
filename: e.Name,
treeHash: tree.Hash,
blobHash: e.Hash,
year: year,
number: number,
})
}
}
return files, nil
}
// Collect files by directory, verifying that directories are contiguous in
// the list of files. Our directory hash optimization depends on that.
func groupFilesByDirectory(files []repoFile) ([][]repoFile, error) {
func groupFilesByDirectory(files []cvelistrepo.File) ([][]cvelistrepo.File, error) {
if len(files) == 0 {
return nil, nil
}
var (
result [][]repoFile
curDir []repoFile
result [][]cvelistrepo.File
curDir []cvelistrepo.File
)
for _, f := range files {
if len(curDir) > 0 && f.dirPath != curDir[0].dirPath {
if len(curDir) > 0 && f.DirPath != curDir[0].DirPath {
result = append(result, curDir)
curDir = nil
}
@ -487,29 +367,15 @@ func groupFilesByDirectory(files []repoFile) ([][]repoFile, error) {
}
seen := map[string]bool{}
for _, dir := range result {
if seen[dir[0].dirPath] {
return nil, fmt.Errorf("directory %s is not contiguous in the sorted list of files", dir[0].dirPath)
if seen[dir[0].DirPath] {
return nil, fmt.Errorf("directory %s is not contiguous in the sorted list of files", dir[0].DirPath)
}
seen[dir[0].dirPath] = true
seen[dir[0].DirPath] = true
}
return result, nil
}
// blobReader returns a reader to the blob with the given hash.
func blobReader(repo *git.Repository, hash plumbing.Hash) (io.Reader, error) {
blob, err := repo.BlobObject(hash)
if err != nil {
return nil, err
}
return blob.Reader()
}
// idFromFilename extracts the CVE ID from its filename.
// idFromFilename extracts the CVE ID from a filename.
func idFromFilename(name string) string {
return strings.TrimSuffix(path.Base(name), path.Ext(name))
}
// isCVEFilename reports whether name is the basename of a CVE file.
func isCVEFilename(name string) bool {
return strings.HasPrefix(name, "CVE-") && path.Ext(name) == ".json"
}

97
internal/worker/update_test.go
Просмотреть файл

@ -14,40 +14,14 @@ import (
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/vulndb/internal/cvelistrepo"
"golang.org/x/vulndb/internal/cveschema"
"golang.org/x/vulndb/internal/gitrepo"
"golang.org/x/vulndb/internal/worker/store"
)
func TestRepoCVEFiles(t *testing.T) {
repo, err := readTxtarRepo("testdata/basic.txtar", time.Now())
if err != nil {
t.Fatal(err)
}
commit := headCommit(t, repo)
if err != nil {
t.Fatal(err)
}
got, err := repoCVEFiles(repo, commit)
if err != nil {
t.Fatal(err)
}
want := []repoFile{
{dirPath: "2020/9xxx", filename: "CVE-2020-9283.json", year: 2020, number: 9283},
{dirPath: "2021/0xxx", filename: "CVE-2021-0001.json", year: 2021, number: 1},
{dirPath: "2021/0xxx", filename: "CVE-2021-0010.json", year: 2021, number: 10},
{dirPath: "2021/1xxx", filename: "CVE-2021-1384.json", year: 2021, number: 1384},
}
opt := cmpopts.IgnoreFields(repoFile{}, "treeHash", "blobHash")
if diff := cmp.Diff(want, got, cmp.AllowUnexported(repoFile{}), opt); diff != "" {
t.Errorf("mismatch (-want, +got):\n%s", diff)
}
}
const clearString = "**CLEAR**"
var clearCVE = &cveschema.CVE{}
@ -94,11 +68,11 @@ func modify(r, m *store.CVERecord) *store.CVERecord {
func TestDoUpdate(t *testing.T) {
ctx := context.Background()
repo, err := readTxtarRepo("testdata/basic.txtar", time.Now())
repo, err := gitrepo.ReadTxtarRepo("../cvelistrepo/testdata/basic.txtar", time.Now())
if err != nil {
t.Fatal(err)
}
h, err := headHash(repo)
h, err := gitrepo.HeadHash(repo)
if err != nil {
t.Fatal(err)
}
@ -309,43 +283,43 @@ func TestDoUpdate(t *testing.T) {
func TestGroupFilesByDirectory(t *testing.T) {
for _, test := range []struct {
in []repoFile
want [][]repoFile
in []cvelistrepo.File
want [][]cvelistrepo.File
}{
{in: nil, want: nil},
{
in: []repoFile{{dirPath: "a"}},
want: [][]repoFile{{{dirPath: "a"}}},
in: []cvelistrepo.File{{DirPath: "a"}},
want: [][]cvelistrepo.File{{{DirPath: "a"}}},
},
{
in: []repoFile{
{dirPath: "a", filename: "f1"},
{dirPath: "a", filename: "f2"},
in: []cvelistrepo.File{
{DirPath: "a", Filename: "f1"},
{DirPath: "a", Filename: "f2"},
},
want: [][]repoFile{{
{dirPath: "a", filename: "f1"},
{dirPath: "a", filename: "f2"},
want: [][]cvelistrepo.File{{
{DirPath: "a", Filename: "f1"},
{DirPath: "a", Filename: "f2"},
}},
},
{
in: []repoFile{
{dirPath: "a", filename: "f1"},
{dirPath: "a", filename: "f2"},
{dirPath: "b", filename: "f1"},
{dirPath: "c", filename: "f1"},
{dirPath: "c", filename: "f2"},
in: []cvelistrepo.File{
{DirPath: "a", Filename: "f1"},
{DirPath: "a", Filename: "f2"},
{DirPath: "b", Filename: "f1"},
{DirPath: "c", Filename: "f1"},
{DirPath: "c", Filename: "f2"},
},
want: [][]repoFile{
want: [][]cvelistrepo.File{
{
{dirPath: "a", filename: "f1"},
{dirPath: "a", filename: "f2"},
{DirPath: "a", Filename: "f1"},
{DirPath: "a", Filename: "f2"},
},
{
{dirPath: "b", filename: "f1"},
{DirPath: "b", Filename: "f1"},
},
{
{dirPath: "c", filename: "f1"},
{dirPath: "c", filename: "f2"},
{DirPath: "c", Filename: "f1"},
{DirPath: "c", Filename: "f2"},
},
},
},
@ -354,12 +328,12 @@ func TestGroupFilesByDirectory(t *testing.T) {
if err != nil {
t.Fatalf("%v: %v", test.in, err)
}
if diff := cmp.Diff(got, test.want, cmp.AllowUnexported(repoFile{})); diff != "" {
if diff := cmp.Diff(got, test.want, cmp.AllowUnexported(cvelistrepo.File{})); diff != "" {
t.Errorf("%v: (-want, +got)\n%s", test.in, diff)
}
}
_, err := groupFilesByDirectory([]repoFile{{dirPath: "a"}, {dirPath: "b"}, {dirPath: "a"}})
_, err := groupFilesByDirectory([]cvelistrepo.File{{DirPath: "a"}, {DirPath: "b"}, {DirPath: "a"}})
if err == nil {
t.Error("got nil, want error")
}
@ -387,3 +361,16 @@ func createCVERecords(t *testing.T, s store.Store, crs []*store.CVERecord) {
t.Fatal(err)
}
}
// headCommit returns the commit at the repo HEAD.
func headCommit(t *testing.T, repo *git.Repository) *object.Commit {
h, err := gitrepo.HeadHash(repo)
if err != nil {
t.Fatal(err)
}
commit, err := repo.CommitObject(h)
if err != nil {
t.Fatal(err)
}
return commit
}

3
internal/worker/worker.go
Просмотреть файл

@ -21,6 +21,7 @@ import (
"golang.org/x/sync/errgroup"
"golang.org/x/time/rate"
vulnc "golang.org/x/vuln/client"
"golang.org/x/vulndb/internal/cvelistrepo"
"golang.org/x/vulndb/internal/cveschema"
"golang.org/x/vulndb/internal/derrors"
"golang.org/x/vulndb/internal/gitrepo"
@ -273,7 +274,7 @@ func newBody(cr *store.CVERecord) (string, error) {
if err := issueTemplate.Execute(&b, issueTemplateData{
Heading: fmt.Sprintf(
"In [%s](%s/tree/%s/%s), the reference URL [%s](%s) (and possibly others) refers to something in Go.",
cr.ID, gitrepo.CVEListRepoURL, cr.CommitHash, cr.Path, cr.Module, cr.Module),
cr.ID, cvelistrepo.URL, cr.CommitHash, cr.Path, cr.Module, cr.Module),
Report: string(out),
Pre: "```",
}); err != nil {

3
internal/worker/worker_test.go
Просмотреть файл

@ -17,6 +17,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/vulndb/internal/cveschema"
"golang.org/x/vulndb/internal/gitrepo"
"golang.org/x/vulndb/internal/issues"
"golang.org/x/vulndb/internal/worker/log"
"golang.org/x/vulndb/internal/worker/store"
@ -25,7 +26,7 @@ import (
func TestCheckUpdate(t *testing.T) {
ctx := context.Background()
tm := time.Date(2021, 1, 26, 0, 0, 0, 0, time.Local)
repo, err := readTxtarRepo("testdata/basic.txtar", tm)
repo, err := gitrepo.ReadTxtarRepo("../cvelistrepo/testdata/basic.txtar", tm)
if err != nil {
t.Fatal(err)
}