internal/database: add New function

Add a function, New, which generates a new Database struct from a git
repo. The git repo must contain a folder "data/osv" with OSV files.

Adds integration tests to ensure that the current Generate logic is the same
as running New then Write. (Generate will eventually be replaced by these
functions.)

Test data is updated to allow for testing with respect to a git repo,
and to test timestamp logic.

For golang/go#56417

Change-Id: Iae88c5bb8d788bcf025af6d9fb700d87b1834455
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/450976
Reviewed-by: Damien Neil <dneil@google.com>
Run-TryBot: Tatiana Bradley <tatiana@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Tatiana Bradley <tatiana@golang.org>

internal/database/database.go

@@ -7,8 +7,17 @@
 package database
 
 import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/object"
 	"golang.org/x/vuln/client"
 	"golang.org/x/vuln/osv"
+	"golang.org/x/vulndb/internal/derrors"
+	"golang.org/x/vulndb/internal/gitrepo"
 )
 
 // Database is an in-memory representation of a Go vulnerability database,
@@ -19,17 +28,23 @@ type Database struct {
 	Index client.DBIndex
 	// Map from each Go ID to its OSV entry.
 	// Represents $dbPath/ID/index.json and the contents of $dbPath/ID/
-	EntriesByID map[string]*osv.Entry
+	EntriesByID EntriesByID
 	// Map from each module path to a list of corresponding OSV entries.
 	// Each map entry represents the contents of a $dbPath/$modulePath.json
 	// file.
-	EntriesByModule map[string][]*osv.Entry
+	EntriesByModule EntriesByModule
 	// Map from each alias (CVE and GHSA) ID to a list of Go IDs for that
 	// alias.
 	// Represents $dbPath/aliases.json
-	IDsByAlias map[string][]string
+	IDsByAlias IDsByAlias
 }
 
+type (
+	EntriesByID     map[string]*osv.Entry
+	EntriesByModule map[string][]*osv.Entry
+	IDsByAlias      map[string][]string
+)
+
 const (
 	// indexFile is the name of the file that contains the database
 	// index.
@@ -63,3 +78,82 @@ const (
 	// that will contain info on toolchain (cmd/...) vulnerabilities.
 	toolchainFileName = "toolchain"
 )
+
+func New(ctx context.Context, repo *git.Repository) (_ *Database, err error) {
+	defer derrors.Wrap(&err, "New()")
+
+	d := &Database{
+		Index:           make(client.DBIndex),
+		EntriesByID:     make(EntriesByID),
+		EntriesByModule: make(EntriesByModule),
+		IDsByAlias:      make(IDsByAlias),
+	}
+
+	root, err := gitrepo.Root(repo)
+	if err != nil {
+		return nil, err
+	}
+
+	commitDates, err := gitrepo.AllCommitDates(repo, gitrepo.HeadReference, osvDir)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = root.Files().ForEach(func(f *object.File) error {
+		if filepath.Dir(f.Name) != osvDir ||
+			filepath.Ext(f.Name) != ".json" {
+			return nil
+		}
+
+		// Read the entry.
+		contents, err := f.Contents()
+		if err != nil {
+			return fmt.Errorf("could not read contents of file %s: %v", f.Name, err)
+		}
+		var entry osv.Entry
+		err = json.Unmarshal([]byte(contents), &entry)
+		if err != nil {
+			return err
+		}
+
+		// Set the modified and published times.
+		dates, ok := commitDates[f.Name]
+		if !ok {
+			return fmt.Errorf("can't find git repo commit dates for %q", f.Name)
+		}
+		addTimestamps(&entry, dates)
+
+		d.addEntry(&entry)
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return d, nil
+}
+
+func (d *Database) addEntry(entry *osv.Entry) {
+	for _, module := range ModulesForEntry(*entry) {
+		d.EntriesByModule[module] = append(d.EntriesByModule[module], entry)
+		if entry.Modified.After(d.Index[module]) {
+			d.Index[module] = entry.Modified
+		}
+	}
+	d.EntriesByID[entry.ID] = entry
+	for _, alias := range entry.Aliases {
+		d.IDsByAlias[alias] = append(d.IDsByAlias[alias], entry.ID)
+	}
+}
+
+func addTimestamps(entry *osv.Entry, dates gitrepo.Dates) {
+	// If a report contains a published field, consider it
+	// the authoritative source of truth.
+	// Otherwise, use the time of the earliest commit in the git history.
+	if entry.Published.IsZero() {
+		entry.Published = dates.Oldest
+	}
+
+	// The modified time is the time of the latest commit for the file.
+	entry.Modified = dates.Newest
+}

internal/database/database_test.go

@@ -0,0 +1,95 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package database
+
+import (
+	"context"
+	"flag"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/vulndb/internal/gitrepo"
+)
+
+var (
+	integration = flag.Bool("integration", false, "test with respect to current contents of vulndb")
+
+	testRepoDir = "testdata/repo.txtar"
+)
+
+func TestNew(t *testing.T) {
+	ctx := context.Background()
+	testRepo, err := gitrepo.ReadTxtarRepo(testRepoDir, jan2002)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := New(ctx, testRepo)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := valid
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("unexpected diff (want-, got+):\n%s", diff)
+	}
+}
+
+func TestNewWriteLoad(t *testing.T) {
+	ctx := context.Background()
+	testRepo, err := gitrepo.ReadTxtarRepo(testRepoDir, jan2002)
+	if err != nil {
+		t.Fatal(err)
+	}
+	new, err := New(ctx, testRepo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	writeDir := t.TempDir()
+	if err = new.Write(writeDir, true); err != nil {
+		t.Fatal(err)
+	}
+	if err = cmpDirHashes(validDir, writeDir); err != nil {
+		t.Error(err)
+	}
+
+	loaded, err := Load(writeDir)
+	if err != nil {
+		t.Error(err)
+	}
+	if diff := cmp.Diff(loaded, new); diff != "" {
+		t.Errorf("unexpected diff (loaded-, new+):\n%s", diff)
+	}
+}
+
+func TestNewWriteLoadIntegration(t *testing.T) {
+	if !*integration {
+		t.Skip("Skipping integration tests, use flag -integration to run")
+	}
+
+	ctx := context.Background()
+
+	repo, err := gitrepo.Open(ctx, "../..")
+	if err != nil {
+		t.Fatal(err)
+	}
+	new, err := New(ctx, repo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	writeDir := t.TempDir()
+	if err = new.Write(writeDir, true); err != nil {
+		t.Fatal(err)
+	}
+
+	loaded, err := Load(writeDir)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if diff := cmp.Diff(loaded, new); diff != "" {
+		t.Errorf("unexpected diff (loaded-, new+):\n%s", diff)
+	}
+}

internal/database/generate_test.go

@@ -5,9 +5,79 @@
 package database
 
 import (
+	"context"
+	"os"
 	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/vulndb/internal/gitrepo"
 )
 
 func TestGenerate(t *testing.T) {
-	// TODO(https://github.com/golang/go#56417): Write tests for Generate.
+	// TODO(https://github.com/golang/go#56417): Write unit tests for Generate.
+}
+
+func TestGenerateIntegration(t *testing.T) {
+	// Generate (in its current state) can only be tested with respect to the
+	// real contents of vulndb.
+	if !*integration {
+		t.Skip("Skipping integration tests, use flag -integration to run")
+	}
+
+	moveToVulnDBRoot(t)
+
+	ctx := context.Background()
+
+	genDir := t.TempDir()
+	err := Generate(ctx, ".", genDir, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	repo, err := gitrepo.Open(ctx, ".")
+	if err != nil {
+		t.Fatal(err)
+	}
+	new, err := New(ctx, repo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Generate equivalent to New then Write", func(t *testing.T) {
+		writeDir := t.TempDir()
+		if err = new.Write(writeDir, false); err != nil {
+			t.Fatal(err)
+		}
+		if err = cmpDirHashes(genDir, writeDir); err != nil {
+			t.Error(err)
+		}
+	})
+
+	t.Run("New equivalent to Generate then Load", func(t *testing.T) {
+		loaded, err := Load(genDir)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if diff := cmp.Diff(loaded, new); diff != "" {
+			t.Errorf("unexpected diff (loaded-, new+):\n%s", diff)
+		}
+	})
+}
+
+func moveToVulnDBRoot(t *testing.T) {
+	// Store current working directory and move into vulndb/ folder.
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chdir("../.."); err != nil {
+		t.Fatal(err)
+	}
+
+	// Restore state from before test.
+	t.Cleanup(func() {
+		if err = os.Chdir(wd); err != nil {
+			t.Log(err)
+		}
+	})
 }

internal/database/load.go

@@ -20,19 +20,19 @@ func Load(dbPath string) (_ *Database, err error) {
 
 	d := &Database{
 		Index:      make(client.DBIndex),
-		IDsByAlias: make(map[string][]string),
+		IDsByAlias: make(IDsByAlias),
 	}
 
 	if err := unmarshalFromFile(filepath.Join(dbPath, indexFile), &d.Index); err != nil {
 		return nil, err
 	}
 
-	d.EntriesByModule, err = getEntriesByModule(dbPath, d.Index)
+	d.EntriesByModule, err = loadEntriesByModule(dbPath, d.Index)
 	if err != nil {
 		return nil, err
 	}
 
-	d.EntriesByID, err = getEntriesByID(dbPath)
+	d.EntriesByID, err = loadEntriesByID(dbPath)
 	if err != nil {
 		return nil, err
 	}
@@ -44,13 +44,13 @@ func Load(dbPath string) (_ *Database, err error) {
 	return d, nil
 }
 
-func getEntriesByID(dbPath string) (map[string]*osv.Entry, error) {
+func loadEntriesByID(dbPath string) (EntriesByID, error) {
 	var ids []string
 	if err := unmarshalFromFile(filepath.Join(dbPath, idDirectory, indexFile), &ids); err != nil {
 		return nil, err
 	}
 
-	entriesByID := make(map[string]*osv.Entry, len(ids))
+	entriesByID := make(EntriesByID, len(ids))
 	for _, id := range ids {
 		var entry osv.Entry
 		err := unmarshalFromFile(filepath.Join(dbPath, idDirectory, id+".json"), &entry)
@@ -62,8 +62,8 @@ func getEntriesByID(dbPath string) (map[string]*osv.Entry, error) {
 	return entriesByID, nil
 }
 
-func getEntriesByModule(dbPath string, index client.DBIndex) (map[string][]*osv.Entry, error) {
-	entriesByModule := make(map[string][]*osv.Entry, len(index))
+func loadEntriesByModule(dbPath string, index client.DBIndex) (EntriesByModule, error) {
+	entriesByModule := make(EntriesByModule, len(index))
 	for _, module := range maps.Keys(index) {
 		emodule, err := client.EscapeModulePath(module)
 		if err != nil {

internal/database/load_test.go

@@ -14,14 +14,16 @@ import (
 )
 
 var (
-	validDir          = "testdata/db/valid"
-	testModifiedTime1 = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
-	testModifiedTime2 = time.Date(2002, 1, 1, 0, 0, 0, 0, time.UTC)
-	testOSV1          = &osv.Entry{
+	validDir = "testdata/db/valid"
+	jan1999  = time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC)
+	jan2000  = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+	jan2002  = time.Date(2002, 1, 1, 0, 0, 0, 0, time.UTC)
+	testOSV1 = &osv.Entry{
 		ID:        "GO-1999-0001",
-		Published: time.Date(1999, time.January, 1, 0, 0, 0, 0, time.UTC), Modified: testModifiedTime1,
-		Aliases: []string{"CVE-1999-1111"},
-		Details: "Some details",
+		Published: jan1999,
+		Modified:  jan2002,
+		Aliases:   []string{"CVE-1999-1111"},
+		Details:   "Some details",
 		Affected: []osv.Affected{
 			{
 				Package: osv.Package{
@@ -46,9 +48,10 @@ var (
 		}}
 	testOSV2 = &osv.Entry{
 		ID:        "GO-2000-0002",
-		Published: time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC), Modified: testModifiedTime2,
-		Aliases: []string{"CVE-1999-2222"},
-		Details: "Some details",
+		Published: jan2000,
+		Modified:  jan2002,
+		Aliases:   []string{"CVE-1999-2222"},
+		Details:   "Some details",
 		Affected: []osv.Affected{
 			{
 				Package: osv.Package{
@@ -69,9 +72,10 @@ var (
 		}}
 	testOSV3 = &osv.Entry{
 		ID:        "GO-2000-0003",
-		Published: time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC), Modified: testModifiedTime2,
-		Aliases: []string{"CVE-1999-3333", "GHSA-xxxx-yyyy-zzzz"},
-		Details: "Some details",
+		Published: jan2002,
+		Modified:  jan2002,
+		Aliases:   []string{"CVE-1999-3333", "GHSA-xxxx-yyyy-zzzz"},
+		Details:   "Some details",
 		Affected: []osv.Affected{
 			{
 				Package: osv.Package{
@@ -99,15 +103,15 @@ var (
 
 var valid = &Database{
 	Index: client.DBIndex{
-		"example.com/module":  testModifiedTime1,
-		"example.com/module2": testModifiedTime2,
+		"example.com/module":  jan2002,
+		"example.com/module2": jan2002,
 	},
-	EntriesByID: map[string]*osv.Entry{"GO-1999-0001": testOSV1, "GO-2000-0002": testOSV2, "GO-2000-0003": testOSV3},
-	EntriesByModule: map[string][]*osv.Entry{
+	EntriesByID: EntriesByID{"GO-1999-0001": testOSV1, "GO-2000-0002": testOSV2, "GO-2000-0003": testOSV3},
+	EntriesByModule: EntriesByModule{
 		"example.com/module":  {testOSV1},
 		"example.com/module2": {testOSV2, testOSV3},
 	},
-	IDsByAlias: map[string][]string{
+	IDsByAlias: IDsByAlias{
 		"CVE-1999-1111":       {"GO-1999-0001"},
 		"CVE-1999-2222":       {"GO-2000-0002"},
 		"CVE-1999-3333":       {"GO-2000-0003"},

internal/database/testdata/db/valid/ID/GO-1999-0001.json

@@ -1,7 +1,7 @@
 {
   "id": "GO-1999-0001",
   "published": "1999-01-01T00:00:00Z",
-  "modified": "2000-01-01T00:00:00Z",
+  "modified": "2002-01-01T00:00:00Z",
   "aliases": [
     "CVE-1999-1111"
   ],

internal/database/testdata/db/valid/ID/GO-2000-0003.json

@@ -1,6 +1,6 @@
 {
   "id": "GO-2000-0003",
-  "published": "2000-01-01T00:00:00Z",
+  "published": "2002-01-01T00:00:00Z",
   "modified": "2002-01-01T00:00:00Z",
   "aliases": [
     "CVE-1999-3333",

internal/database/testdata/db/valid/example.com/module.json

@@ -2,7 +2,7 @@
   {
     "id": "GO-1999-0001",
     "published": "1999-01-01T00:00:00Z",
-    "modified": "2000-01-01T00:00:00Z",
+    "modified": "2002-01-01T00:00:00Z",
     "aliases": [
       "CVE-1999-1111"
     ],

internal/database/testdata/db/valid/example.com/module2.json

@@ -50,7 +50,7 @@
   },
   {
     "id": "GO-2000-0003",
-    "published": "2000-01-01T00:00:00Z",
+    "published": "2002-01-01T00:00:00Z",
     "modified": "2002-01-01T00:00:00Z",
     "aliases": [
       "CVE-1999-3333",

internal/database/testdata/db/valid/index.json

@@ -1,4 +1,4 @@
 {
-  "example.com/module": "2000-01-01T00:00:00Z",
+  "example.com/module": "2002-01-01T00:00:00Z",
   "example.com/module2": "2002-01-01T00:00:00Z"
 }

internal/database/testdata/osv/GO-1999-0001.json

@@ -1,55 +0,0 @@
-{
-  "id": "GO-1999-0001",
-  "published": "1999-01-01T00:00:00Z",
-  "modified": "2000-01-01T00:00:00Z",
-  "aliases": [
-    "CVE-1999-1111"
-  ],
-  "details": "Some details",
-  "affected": [
-    {
-      "package": {
-        "name": "example.com/module",
-        "ecosystem": "Go"
-      },
-      "ranges": [
-        {
-          "type": "SEMVER",
-          "events": [
-            {
-              "introduced": "0"
-            },
-            {
-              "fixed": "1.1.0"
-            },
-            {
-              "introduced": "1.2.0"
-            },
-            {
-              "fixed": "1.2.2"
-            }
-          ]
-        }
-      ],
-      "database_specific": {
-        "url": "https://pkg.go.dev/vuln/GO-1999-0001"
-      },
-      "ecosystem_specific": {
-        "imports": [
-          {
-            "path": "package",
-            "symbols": [
-              "Symbol"
-            ]
-          }
-        ]
-      }
-    }
-  ],
-  "references": [
-    {
-      "type": "FIX",
-      "url": "https://example.com/cl/123"
-    }
-  ]
-}

internal/database/testdata/osv/GO-2000-0002.json

@@ -1,49 +0,0 @@
-{
-  "id": "GO-2000-0002",
-  "published": "2000-01-01T00:00:00Z",
-  "modified": "2001-01-01T00:00:00Z",
-  "aliases": [
-    "CVE-1999-2222"
-  ],
-  "details": "Some details",
-  "affected": [
-    {
-      "package": {
-        "name": "example.com/module2",
-        "ecosystem": "Go"
-      },
-      "ranges": [
-        {
-          "type": "SEMVER",
-          "events": [
-            {
-              "introduced": "0"
-            },
-            {
-              "fixed": "1.2.0"
-            }
-          ]
-        }
-      ],
-      "database_specific": {
-        "url": "https://pkg.go.dev/vuln/GO-2000-0002"
-      },
-      "ecosystem_specific": {
-        "imports": [
-          {
-            "path": "package",
-            "symbols": [
-              "Symbol"
-            ]
-          }
-        ]
-      }
-    }
-  ],
-  "references": [
-    {
-      "type": "FIX",
-      "url": "https://example.com/cl/543"
-    }
-  ]
-}

internal/database/testdata/osv/GO-2000-0003.json

@@ -1,50 +0,0 @@
-{
-  "id": "GO-2000-0003",
-  "published": "2000-01-01T00:00:00Z",
-  "modified": "2002-01-01T00:00:00Z",
-  "aliases": [
-    "CVE-1999-3333",
-    "GHSA-xxxx-yyyy-zzzz"
-  ],
-  "details": "Some details",
-  "affected": [
-    {
-      "package": {
-        "name": "example.com/module2",
-        "ecosystem": "Go"
-      },
-      "ranges": [
-        {
-          "type": "SEMVER",
-          "events": [
-            {
-              "introduced": "0"
-            },
-            {
-              "fixed": "1.1.0"
-            }
-          ]
-        }
-      ],
-      "database_specific": {
-        "url": "https://pkg.go.dev/vuln/GO-2000-0003"
-      },
-      "ecosystem_specific": {
-        "imports": [
-          {
-            "path": "package",
-            "symbols": [
-              "Symbol"
-            ]
-          }
-        ]
-      }
-    }
-  ],
-  "references": [
-    {
-      "type": "FIX",
-      "url": "https://example.com/cl/000"
-    }
-  ]
-}

internal/database/testdata/repo.txtar

@@ -0,0 +1,161 @@
+-- data/osv/GO-1999-0001.json --
+{
+  "id": "GO-1999-0001",
+  "published": "1999-01-01T00:00:00Z",
+  "modified": "0001-01-01T00:00:00Z",
+  "aliases": [
+    "CVE-1999-1111"
+  ],
+  "details": "Some details",
+  "affected": [
+    {
+      "package": {
+        "name": "example.com/module",
+        "ecosystem": "Go"
+      },
+      "ranges": [
+        {
+          "type": "SEMVER",
+          "events": [
+            {
+              "introduced": "0"
+            },
+            {
+              "fixed": "1.1.0"
+            },
+            {
+              "introduced": "1.2.0"
+            },
+            {
+              "fixed": "1.2.2"
+            }
+          ]
+        }
+      ],
+      "database_specific": {
+        "url": "https://pkg.go.dev/vuln/GO-1999-0001"
+      },
+      "ecosystem_specific": {
+        "imports": [
+          {
+            "path": "package",
+            "symbols": [
+              "Symbol"
+            ]
+          }
+        ]
+      }
+    }
+  ],
+  "references": [
+    {
+      "type": "FIX",
+      "url": "https://example.com/cl/123"
+    }
+  ]
+}
+
+-- data/osv/GO-2000-0002.json --
+{
+  "id": "GO-2000-0002",
+  "published": "2000-01-01T00:00:00Z",
+  "modified": "0001-01-01T00:00:00Z",
+  "aliases": [
+    "CVE-1999-2222"
+  ],
+  "details": "Some details",
+  "affected": [
+    {
+      "package": {
+        "name": "example.com/module2",
+        "ecosystem": "Go"
+      },
+      "ranges": [
+        {
+          "type": "SEMVER",
+          "events": [
+            {
+              "introduced": "0"
+            },
+            {
+              "fixed": "1.2.0"
+            }
+          ]
+        }
+      ],
+      "database_specific": {
+        "url": "https://pkg.go.dev/vuln/GO-2000-0002"
+      },
+      "ecosystem_specific": {
+        "imports": [
+          {
+            "path": "package",
+            "symbols": [
+              "Symbol"
+            ]
+          }
+        ]
+      }
+    }
+  ],
+  "references": [
+    {
+      "type": "FIX",
+      "url": "https://example.com/cl/543"
+    }
+  ]
+}
+
+-- data/osv/GO-2000-0003.json --
+{
+  "id": "GO-2000-0003",
+  "published": "0001-01-01T00:00:00Z",
+  "modified": "0001-01-01T00:00:00Z",
+  "aliases": [
+    "CVE-1999-3333",
+    "GHSA-xxxx-yyyy-zzzz"
+  ],
+  "details": "Some details",
+  "affected": [
+    {
+      "package": {
+        "name": "example.com/module2",
+        "ecosystem": "Go"
+      },
+      "ranges": [
+        {
+          "type": "SEMVER",
+          "events": [
+            {
+              "introduced": "0"
+            },
+            {
+              "fixed": "1.1.0"
+            }
+          ]
+        }
+      ],
+      "database_specific": {
+        "url": "https://pkg.go.dev/vuln/GO-2000-0003"
+      },
+      "ecosystem_specific": {
+        "imports": [
+          {
+            "path": "package",
+            "symbols": [
+              "Symbol"
+            ]
+          }
+        ]
+      }
+    }
+  ],
+  "references": [
+    {
+      "type": "FIX",
+      "url": "https://example.com/cl/000"
+    }
+  ]
+}
+
+