internal/postgres: add ReInsertLatestVersion

New latest-version info can retract the current latest good version,
resulting in an older version becoming the new latest good version.

Add a function that updates the search_documents table when
that happens.

A later CL will also update imports_unique, and will call this
function from the worker.

For golang/go#44710

Change-Id: I09569e27d384ed7defb06a4e81facdb96c22caeb
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/303312
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Jamal Carvalho <jamal@golang.org>
Reviewed-by: Julie Qiu <julie@golang.org>

internal/postgres/insert_module.go

@@ -632,6 +632,95 @@ func insertReadmes(ctx context.Context, db *database.DB,
 	return db.BulkUpsert(ctx, "readmes", readmeCols, readmeValues, []string{"unit_id"})
 }
 
+// ReInsertLatestVersion checks that the latest good version matches the version
+// in search_documents. If it doesn't, it inserts the latest good version into
+// search_documents and imports_unique.
+func (db *DB) ReInsertLatestVersion(ctx context.Context, modulePath string) (err error) {
+	defer derrors.WrapStack(&err, "ReInsertLatestVersion(%q)", modulePath)
+
+	return db.db.Transact(ctx, sql.LevelRepeatableRead, func(tx *database.DB) error {
+		// Hold the lock on the module path throughout.
+		if err := lock(ctx, tx, modulePath); err != nil {
+			return err
+		}
+
+		lmv, _, err := getLatestModuleVersions(ctx, tx, modulePath)
+		if err != nil {
+			return err
+		}
+		if lmv.GoodVersion == "" {
+			// TODO(golang/go#44710): once we are confident that
+			// latest_module_versions is accurate and up to date, we can assume
+			// that a missing GoodVersion should mean that there are no good
+			// versions remaining, and we should remove the current module from
+			// search_documents.
+			log.Debugf(ctx, "ReInsertLatestVersion(%q): no good version", modulePath)
+			return nil
+		}
+		// Is the latest good version in search_documents?
+		var x int
+		switch err := tx.QueryRow(ctx, `
+			SELECT 1
+			FROM search_documents
+			WHERE module_path = $1
+			AND version = $2
+		`, modulePath, lmv.GoodVersion).Scan(&x); err {
+		case sql.ErrNoRows:
+			break
+		case nil:
+			log.Debugf(ctx, "ReInsertLatestVersion(%q): good version %s found in search_documents; doing nothing",
+				modulePath, lmv.GoodVersion)
+			return nil
+		default:
+			return err
+		}
+
+		// The latest good version is not in search_documents. Is this an
+		// alternative module path?
+		alt, err := isAlternativeModulePath(ctx, tx, modulePath)
+		if err != nil {
+			return err
+		}
+		if alt {
+			log.Debugf(ctx, "ReInsertLatestVersion(%q): alternative module path; doing nothing", modulePath)
+			return nil
+		}
+
+		// Not an alternative module path. Read the module information at the
+		// latest good version.
+		pkgMetas, err := getPackagesInUnit(ctx, tx, modulePath, modulePath, lmv.GoodVersion, db.bypassLicenseCheck)
+		if err != nil {
+			return err
+		}
+		// We only need the readme for the module.
+		readme, err := getModuleReadme(ctx, tx, modulePath, lmv.GoodVersion)
+		if err != nil {
+			return err
+		}
+		// Insert into search_documents.
+		for _, pkg := range pkgMetas {
+			if isInternalPackage(pkg.Path) {
+				continue
+			}
+			args := UpsertSearchDocumentArgs{
+				PackagePath: pkg.Path,
+				ModulePath:  modulePath,
+				Version:     lmv.GoodVersion,
+				Synopsis:    pkg.Synopsis,
+			}
+			if pkg.Path == modulePath && readme != nil {
+				args.ReadmeFilePath = readme.Filepath
+				args.ReadmeContents = readme.Contents
+			}
+			if err := UpsertSearchDocument(ctx, tx, args); err != nil {
+				return err
+			}
+		}
+		log.Debugf(ctx, "ReInsertLatestVersion(%q): re-inserted at latest good version %s", modulePath, lmv.GoodVersion)
+		return nil
+	})
+}
+
 // lock obtains an exclusive, transaction-scoped advisory lock on modulePath.
 func lock(ctx context.Context, tx *database.DB, modulePath string) (err error) {
 	defer derrors.WrapStack(&err, "lock(%s)", modulePath)

internal/postgres/insert_module_test.go

@@ -651,3 +651,61 @@ func TestIsAlternativeModulePath(t *testing.T) {
 		}
 	}
 }
+
+func TestReInsertLatestVersion(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	testDB, release := acquire(t)
+	defer release()
+
+	const modulePath = "m.com/a"
+
+	insert := func(version string, status int, modfile string) {
+		m := sample.Module(modulePath, version, "pkg")
+		lmv := addLatest(ctx, t, testDB, modulePath, version, "module "+modulePath+"\n"+modfile)
+		m.Packages()[0].Documentation[0].Synopsis = version
+		if status == 200 {
+			MustInsertModuleLMV(ctx, t, testDB, m, lmv)
+		}
+		if err := testDB.UpsertModuleVersionState(ctx, &ModuleVersionStateForUpsert{
+			ModulePath: modulePath,
+			Version:    version,
+			AppVersion: "app",
+			Timestamp:  time.Now(),
+			Status:     status,
+		}); err != nil {
+			t.Fatal(err)
+		}
+		if err := testDB.ReInsertLatestVersion(ctx, modulePath); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	check := func(wantVersion string) {
+		var gotVersion, gotSynopsis string
+		if err := testDB.db.QueryRow(ctx, `
+			SELECT version, synopsis
+			FROM search_documents
+			WHERE module_path = 'm.com/a'
+			AND package_path = 'm.com/a/pkg'
+		`).Scan(&gotVersion, &gotSynopsis); err != nil {
+			t.Fatal(err)
+		}
+		if gotVersion != wantVersion && gotSynopsis != wantVersion {
+			t.Fatalf("got (%s, %s), want %s for both", gotVersion, gotSynopsis, wantVersion)
+		}
+	}
+
+	// Insert a good module. It should be in search_documents.
+	insert("v1.1.0", 200, "")
+	check("v1.1.0")
+
+	// Insert a higher, good version. It should replace the first.
+	insert("v1.2.0", 200, "")
+	check("v1.2.0")
+
+	// Now an even higher, bad version comes along that retracts v1.2.0.
+	// The search_documents table should go back to v1.1.0.
+	insert("v1.3.0", 400, "retract v1.2.0")
+	check("v1.1.0")
+}

internal/postgres/search.go

@@ -467,7 +467,7 @@ var upsertSearchStatement = fmt.Sprintf(`
 			END)
 	;`, hllRegisterCount)
 
-// upsertSearchDocuments adds search information for mod ot the search_documents table.
+// upsertSearchDocuments adds search information for mod to the search_documents table.
 // It assumes that all non-redistributable data has been removed from mod.
 func upsertSearchDocuments(ctx context.Context, ddb *database.DB, mod *internal.Module) (err error) {
 	defer derrors.WrapStack(&err, "upsertSearchDocuments(ctx, %q, %q)", mod.ModulePath, mod.Version)
@@ -513,6 +513,7 @@ func UpsertSearchDocument(ctx context.Context, ddb *database.DB, args UpsertSear
 	defer derrors.WrapStack(&err, "DB.UpsertSearchDocument(ctx, ddb, %q, %q)", args.PackagePath, args.ModulePath)
 
 	// Only summarize the README if the package and module have the same path.
+	// If this changes, fix DB.ReInsertLatestVersion.
 	if args.PackagePath != args.ModulePath {
 		args.ReadmeFilePath = ""
 		args.ReadmeContents = ""

internal/postgres/unit.go

@@ -448,7 +448,11 @@ func (db *DB) getImports(ctx context.Context, unitID int) (_ []string, err error
 // getPackagesInUnit returns all of the packages in a unit from a
 // module version, including the package that lives at fullPath, if present.
 func (db *DB) getPackagesInUnit(ctx context.Context, fullPath, modulePath, resolvedVersion string) (_ []*internal.PackageMeta, err error) {
-	defer derrors.WrapStack(&err, "DB.getPackagesInUnit(ctx, %q, %q, %q)", fullPath, modulePath, resolvedVersion)
+	return getPackagesInUnit(ctx, db.db, fullPath, modulePath, resolvedVersion, db.bypassLicenseCheck)
+}
+
+func getPackagesInUnit(ctx context.Context, db *database.DB, fullPath, modulePath, resolvedVersion string, bypassLicenseCheck bool) (_ []*internal.PackageMeta, err error) {
+	defer derrors.WrapStack(&err, "getPackagesInUnit(ctx, %q, %q, %q)", fullPath, modulePath, resolvedVersion)
 	defer middleware.ElapsedStat(ctx, "getPackagesInUnit")()
 
 	query := `
@@ -512,7 +516,7 @@ func (db *DB) getPackagesInUnit(ctx context.Context, fullPath, modulePath, resol
 		}
 		return nil
 	}
-	if err := db.db.RunQuery(ctx, query, collect, modulePath, resolvedVersion); err != nil {
+	if err := db.RunQuery(ctx, query, collect, modulePath, resolvedVersion); err != nil {
 		return nil, err
 	}
 
@@ -523,7 +527,7 @@ func (db *DB) getPackagesInUnit(ctx context.Context, fullPath, modulePath, resol
 	}
 	sort.Slice(packages, func(i, j int) bool { return packages[i].Path < packages[j].Path })
 	for _, p := range packages {
-		if db.bypassLicenseCheck {
+		if bypassLicenseCheck {
 			p.IsRedistributable = true
 		} else {
 			p.RemoveNonRedistributableData()
@@ -678,9 +682,13 @@ func (db *DB) getPathsInModule(ctx context.Context, modulePath, resolvedVersion
 
 // GetModuleReadme returns the README corresponding to the modulePath and version.
 func (db *DB) GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (_ *internal.Readme, err error) {
-	defer derrors.WrapStack(&err, "GetModuleReadme(ctx, %q, %q)", modulePath, resolvedVersion)
+	return getModuleReadme(ctx, db.db, modulePath, resolvedVersion)
+}
+
+func getModuleReadme(ctx context.Context, db *database.DB, modulePath, resolvedVersion string) (_ *internal.Readme, err error) {
+	defer derrors.WrapStack(&err, "getModuleReadme(ctx, %q, %q)", modulePath, resolvedVersion)
 	var readme internal.Readme
-	err = db.db.QueryRow(ctx, `
+	err = db.QueryRow(ctx, `
 		SELECT file_path, contents
 		FROM modules m
 		INNER JOIN units u