cmd/go: preserve sums for indirect deps fetched by 'go mod download'

Previously, commands that wrote go.sum (except 'go mod tidy') would
retain sums for zip files of directly required modules. Sums of
indirect dependencies wouldn't be retained unless they were used to
load packages.

With this change, sums for indirect dependencies will be retained if
they're available. This allows users to add missing sums with
'go mod download example.com/mod', which previously only worked for
directly required modules.

Note that 'go mod download' without arguments now adds sums for every
module in the build list. That matches 1.15 behavior.

For #41103

Change-Id: I4cce2bf1c73578dae836bdb5adb32da071554f1a
Reviewed-on: https://go-review.googlesource.com/c/go/+/282692
Trust: Jay Conrod <jayconrod@google.com>
Run-TryBot: Jay Conrod <jayconrod@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

src/cmd/go/internal/modload/init.go

@@ -976,9 +976,12 @@ func WriteGoMod() {
 // It also contains entries for go.mod files needed for MVS (the version
 // of these entries ends with "/go.mod").
 //
-// If addDirect is true, the set also includes sums for modules directly
+// If keepBuildListZips is true, the set also includes sums for zip files for
-// required by go.mod, as represented by the index, with replacements applied.
+// all modules in the build list with replacements applied. 'go get' and
-func keepSums(addDirect bool) map[module.Version]bool {
+// 'go mod download' may add sums to this set when adding a requirement on a
+// module without a root package or when downloading a direct or indirect
+// dependency.
+func keepSums(keepBuildListZips bool) map[module.Version]bool {
 	// Re-derive the build list using the current list of direct requirements.
 	// Keep the sum for the go.mod of each visited module version (or its
 	// replacement).
@@ -1007,11 +1010,6 @@ func keepSums(addDirect bool) map[module.Version]bool {
 		panic(fmt.Sprintf("unexpected error reloading build list: %v", err))
 	}
 
-	// Add entries for modules in the build list with paths that are prefixes of
-	// paths of loaded packages. We need to retain sums for modules needed to
-	// report ambiguous import errors. We use our re-derived build list,
-	// since the global build list may have been tidied.
-	if loaded != nil {
 	actualMods := make(map[string]module.Version)
 	for _, m := range buildList[1:] {
 		if r := Replacement(m); r.Path != "" {
@@ -1020,6 +1018,12 @@ func keepSums(addDirect bool) map[module.Version]bool {
 			actualMods[m.Path] = m
 		}
 	}
+
+	// Add entries for modules in the build list with paths that are prefixes of
+	// paths of loaded packages. We need to retain sums for modules needed to
+	// report ambiguous import errors. We use our re-derived build list,
+	// since the global build list may have been tidied.
+	if loaded != nil {
 		for _, pkg := range loaded.pkgs {
 			if pkg.testOf != nil || pkg.inStd || module.CheckImportPath(pkg.path) != nil {
 				continue
@@ -1032,17 +1036,13 @@ func keepSums(addDirect bool) map[module.Version]bool {
 		}
 	}
 
-	// Add entries for modules directly required by go.mod.
+	// Add entries for the zip of each module in the build list.
-	if addDirect {
+	// We might not need all of these (tidy does not add them), but they may be
-		for m := range index.require {
+	// added by a specific 'go get' or 'go mod download' command to resolve
-			var kept module.Version
+	// missing import sum errors.
-			if r := Replacement(m); r.Path != "" {
+	if keepBuildListZips {
-				kept = r
+		for _, m := range actualMods {
-			} else {
+			keep[m] = true
-				kept = m
-			}
-			keep[kept] = true
-			keep[module.Version{Path: kept.Path, Version: kept.Version + "/go.mod"}] = true
 		}
 	}
 
@@ -1062,9 +1062,8 @@ func (r *keepSumReqs) Required(m module.Version) ([]module.Version, error) {
 }
 
 func TrimGoSum() {
-	// Don't retain sums for direct requirements in go.mod. When TrimGoSum is
+	// Don't retain sums for the zip file of every module in the build list.
-	// called, go.mod has not been updated, and it may contain requirements on
+	// We may not need them all to build the main module's packages.
-	// modules deleted from the build list.
+	keepBuildListZips := false
-	addDirect := false
+	modfetch.TrimGoSum(keepSums(keepBuildListZips))
-	modfetch.TrimGoSum(keepSums(addDirect))
 }

src/cmd/go/testdata/script/mod_sum_ambiguous.txt

@@ -10,6 +10,14 @@ go mod tidy
 grep '^example.com/ambiguous/a v1.0.0 h1:' go.sum
 grep '^example.com/ambiguous/a/b v0.0.0-empty h1:' go.sum
 
+# 'go mod download' should also add sums.
+cp go.sum.buildlist-only go.sum
+go mod download example.com/ambiguous/a
+grep '^example.com/ambiguous/a v1.0.0 h1:' go.sum
+! grep '^example.com/ambiguous/a/b v0.0.0-empty h1:' go.sum
+go mod download example.com/ambiguous/a/b
+grep '^example.com/ambiguous/a/b v0.0.0-empty h1:' go.sum
+
 # If two modules could provide a package, and we're missing a sum for one,
 # we should see a missing sum error, even if we have a sum for a module that
 # provides the package.