From 3455efa7786fef5cf1009639819c6bbb11d48043 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Thu, 25 Mar 2021 09:17:03 -0700 Subject: [PATCH] all: use consistent index type across packages All reports still need to have their published fields set. Change-Id: I64feda32742bb5f85e310211f8da270e4346ad6b Reviewed-on: https://team-review.git.corp.google.com/c/golang/vulndb/+/1036000 Reviewed-by: Roland Shoemaker --- client/cache.go | 12 ++++++------ client/cache_test.go | 2 +- client/client.go | 12 ++++++------ cmd/gendb/main.go | 40 +++++++--------------------------------- osv/json.go | 13 ++++++++++++- report/report.go | 22 +++++++++++++--------- 6 files changed, 45 insertions(+), 56 deletions(-) diff --git a/client/cache.go b/client/cache.go index 3ef8d8a..05d031c 100644 --- a/client/cache.go +++ b/client/cache.go @@ -25,7 +25,7 @@ import ( // $GOPATH/pkg/mod/cache/download/vulndb/{db hostname}/indexes/index.json // { // Retrieved time.Time -// Index map[string]time.Time +// Index osv.DBIndex // } // // Each package also has a JSON file which contains the array of vulnerability @@ -35,8 +35,8 @@ import ( // []*osv.Entry type Cache interface { - ReadIndex(string) (map[string]time.Time, time.Time, error) - WriteIndex(string, map[string]time.Time, time.Time) error + ReadIndex(string) (osv.DBIndex, time.Time, error) + WriteIndex(string, osv.DBIndex, time.Time) error ReadEntries(string, string) ([]*osv.Entry, error) WriteEntries(string, string, []*osv.Entry) error } @@ -54,10 +54,10 @@ var cacheRoot = filepath.Join(build.Default.GOPATH, "/pkg/mod/cache/download/vul type cachedIndex struct { Retrieved time.Time - Index map[string]time.Time + Index osv.DBIndex } -func (c *fsCache) ReadIndex(dbName string) (map[string]time.Time, time.Time, error) { +func (c *fsCache) ReadIndex(dbName string) (osv.DBIndex, time.Time, error) { b, err := os.ReadFile(filepath.Join(cacheRoot, dbName, "index.json")) if err != nil { if os.IsNotExist(err) { @@ -72,7 +72,7 @@ func (c *fsCache) ReadIndex(dbName string) (map[string]time.Time, time.Time, err return index.Index, index.Retrieved, nil } -func (c *fsCache) WriteIndex(dbName string, index map[string]time.Time, retrieved time.Time) error { +func (c *fsCache) WriteIndex(dbName string, index osv.DBIndex, retrieved time.Time) error { path := filepath.Join(cacheRoot, dbName) if err := os.MkdirAll(path, 0777); err != nil { return err diff --git a/client/cache_test.go b/client/cache_test.go index bd5e7f2..9c474fe 100644 --- a/client/cache_test.go +++ b/client/cache_test.go @@ -38,7 +38,7 @@ func TestCache(t *testing.T) { } now := time.Now() - expectedIdx := map[string]time.Time{ + expectedIdx := osv.DBIndex{ "a.vuln.example.com": time.Time{}.Add(time.Hour), "b.vuln.example.com": time.Time{}.Add(time.Hour * 2), "c.vuln.example.com": time.Time{}.Add(time.Hour * 3), diff --git a/client/client.go b/client/client.go index 63694eb..c42ac2f 100644 --- a/client/client.go +++ b/client/client.go @@ -19,7 +19,7 @@ type dbIndex struct{} type source interface { Get([]string) ([]*osv.Entry, error) - Index() (map[string]time.Time, error) + Index() (osv.DBIndex, error) } type localSource struct { @@ -44,8 +44,8 @@ func (ls *localSource) Get(packages []string) ([]*osv.Entry, error) { return entries, nil } -func (ls *localSource) Index() (map[string]time.Time, error) { - var index map[string]time.Time +func (ls *localSource) Index() (osv.DBIndex, error) { + var index osv.DBIndex b, err := os.ReadFile(filepath.Join(ls.dir, "index.json")) if err != nil { return nil, err @@ -63,8 +63,8 @@ type httpSource struct { dbName string } -func (hs *httpSource) Index() (map[string]time.Time, error) { - var cachedIndex map[string]time.Time +func (hs *httpSource) Index() (osv.DBIndex, error) { + var cachedIndex osv.DBIndex var cachedIndexRetrieved *time.Time if hs.cache != nil { @@ -104,7 +104,7 @@ func (hs *httpSource) Index() (map[string]time.Time, error) { if err != nil { return nil, err } - var index map[string]time.Time + var index osv.DBIndex if err = json.Unmarshal(b, &index); err != nil { return nil, err } diff --git a/cmd/gendb/main.go b/cmd/gendb/main.go index f2526b0..b024817 100644 --- a/cmd/gendb/main.go +++ b/cmd/gendb/main.go @@ -9,18 +9,12 @@ import ( "path/filepath" "reflect" "strings" - "time" "github.com/BurntSushi/toml" "golang.org/x/vulndb/osv" "golang.org/x/vulndb/report" ) -type IndexEntry struct { - LastModified time.Time - LastNewFinding time.Time -} - func fail(why string) { fmt.Fprintln(os.Stderr, why) os.Exit(1) @@ -77,46 +71,26 @@ func main() { } } - index := map[string]*IndexEntry{} - if content, err := ioutil.ReadFile(filepath.Join(*jsonDir, "index.json")); err == nil { - err = json.Unmarshal(content, &index) - if err != nil { - fail(fmt.Sprintf("failed to parse index: %s", err)) - } - } else if err != nil && !os.IsNotExist(err) { - fail(fmt.Sprintf("failed to read index %q: %s", filepath.Join(*jsonDir, "index.json"), err)) - } - - // TODO(bracewell): I'm pretty sure the freshness stuff is basically - // completely broken at the moment. - now := time.Now() - for path, v := range jsonVulns { + index := make(osv.DBIndex, len(jsonVulns)) + for path, vulns := range jsonVulns { outPath := filepath.Join(*jsonDir, path) - content, err := json.Marshal(v) + content, err := json.Marshal(vulns) if err != nil { fail(fmt.Sprintf("failed to marshal json: %s", err)) } - // fmt.Println("making", filepath.Dir(outPath)) err = os.MkdirAll(filepath.Dir(outPath), 0700) if err != nil { fail(fmt.Sprintf("failed to create directory %q: %s", filepath.Dir(outPath), err)) } - // if there is already an index entry, only update the file - // if the set of vulns differ from what is already on disk - if _, ok := index[path]; ok && matchesCurrent(outPath, v) { - // fmt.Println("skipping", outPath) - continue - } - // fmt.Println("writing", outPath, string(content)) err = ioutil.WriteFile(outPath+".json", content, 0644) if err != nil { fail(fmt.Sprintf("failed to write %q: %s", outPath+".json", err)) } - if index[path] == nil { - index[path] = &IndexEntry{} + for _, v := range vulns { + if v.LastModified.After(index[path]) { + index[path] = v.LastModified + } } - index[path].LastModified = now - // also need to set the LastNewFinding, somewhat more complicated... } indexJSON, err := json.Marshal(index) diff --git a/osv/json.go b/osv/json.go index acf5eb5..19c249c 100644 --- a/osv/json.go +++ b/osv/json.go @@ -7,6 +7,13 @@ import ( "golang.org/x/vulndb/report" ) +// DBIndex contains a mapping of vulnerable packages to the +// last time a new vulnerability was added to the database. +// TODO: this is probably not the correct place to put this +// type, since it's not really an OSV/CVF thing, but rather +// vulndb implementatiion detail. +type DBIndex map[string]time.Time + type Severity int const ( @@ -166,7 +173,11 @@ func Generate(id string, url string, r report.Report) []Entry { // It would be better if this was just a recursive thing probably for _, additional := range r.AdditionalPackages { entryCopy := entry - entryCopy.Package.Name = additional.Package + additionalImportPath := additional.Module + if additional.Package != "" { + additionalImportPath = additional.Package + } + entryCopy.Package.Name = additionalImportPath entryCopy.EcosystemSpecific.Symbols = additional.Symbols entryCopy.Affects = generateAffects(additional.Versions) diff --git a/report/report.go b/report/report.go index aa39392..2b94a85 100644 --- a/report/report.go +++ b/report/report.go @@ -1,5 +1,7 @@ package report +import "time" + type VersionRange struct { Introduced string Fixed string @@ -26,15 +28,17 @@ type Report struct { Symbols []string Versions []VersionRange } `toml:"additional_packages"` - Versions []VersionRange - Description string - Severity string - CVE string - Credit string - Symbols []string - OS []string - Arch []string - Links struct { + Versions []VersionRange + Description string + Published time.Time + LastModified time.Time `toml:"last_modified"` + Severity string + CVE string + Credit string + Symbols []string + OS []string + Arch []string + Links struct { PR string Commit string Context []string