internal/task: dynamically find nested modules to tidy during tagging

Fetch the git tree where go.mod is being updated, walk over its content
to find nested modules to tidy. This way even if new nested modules are
added, they'll be handled without needing additional attention here.

The existing test cases continue to pass (i.e., the x/tools/gopls module
is still being tidied). It required adding a minimal HTTP(S) git server
to FakeGerrit, to act like the real Gerrit server, or rather just enough
for gitfs to be able to successfully clone from it.

For golang/go#68873.

Change-Id: Ic571f2edc6345d25558591e531d58ecce70f1851
Reviewed-on: https://go-review.googlesource.com/c/build/+/617778
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Dmitri Shuralyov <dmitshur@golang.org>

internal/task/fakes.go

@@ -349,24 +349,105 @@ func (g *FakeGerrit) GerritURL() string {
 	return g.serverURL
 }
 
-func (g *FakeGerrit) serveHTTP(w http.ResponseWriter, r *http.Request) {
-	parts := strings.Split(r.URL.Path, "/")
-	if len(parts) != 4 {
+func (g *FakeGerrit) serveHTTP(w http.ResponseWriter, req *http.Request) {
+	switch url := req.URL.String(); {
+	// Serve a revision tarball (.tar.gz) like Gerrit does.
+	case strings.HasSuffix(url, ".tar.gz"):
+		parts := strings.Split(req.URL.Path, "/")
+		if len(parts) != 4 {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		repo, err := g.repo(parts[1])
+		if err != nil {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		rev := strings.TrimSuffix(parts[3], ".tar.gz")
+		archive, err := repo.dir.RunCommand(req.Context(), "archive", "--format=tgz", rev)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		http.ServeContent(w, req, parts[3], time.Now(), bytes.NewReader(archive))
+		return
+
+	// Serve a git repository over HTTP like Gerrit does.
+	case strings.HasSuffix(url, "/info/refs?service=git-upload-pack"):
+		parts := strings.Split(url[:len(url)-len("/info/refs?service=git-upload-pack")], "/")
+		if len(parts) != 2 {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		repo, err := g.repo(parts[1])
+		if err != nil {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		g.serveGitInfoRefsUploadPack(w, req, repo)
+		return
+	case strings.HasSuffix(url, "/git-upload-pack"):
+		parts := strings.Split(url[:len(url)-len("/git-upload-pack")], "/")
+		if len(parts) != 2 {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		repo, err := g.repo(parts[1])
+		if err != nil {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		g.serveGitUploadPack(w, req, repo)
+		return
+
+	default:
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
-	repo, err := g.repo(parts[1])
-	if err != nil {
-		w.WriteHeader(http.StatusNotFound)
+}
+func (*FakeGerrit) serveGitInfoRefsUploadPack(w http.ResponseWriter, req *http.Request, repo *FakeRepo) {
+	if req.Method != http.MethodGet {
+		w.Header().Set("Allow", http.MethodGet)
+		http.Error(w, "method should be GET", http.StatusMethodNotAllowed)
 		return
 	}
-	rev := strings.TrimSuffix(parts[3], ".tar.gz")
-	archive, err := repo.dir.RunCommand(r.Context(), "archive", "--format=tgz", rev)
+	cmd := exec.CommandContext(req.Context(), "git", "upload-pack", "--strict", "--advertise-refs", ".")
+	cmd.Dir = filepath.Join(repo.dir.dir, ".git")
+	cmd.Env = append(os.Environ(), "GIT_PROTOCOL="+req.Header.Get("Git-Protocol"))
+	var buf bytes.Buffer
+	cmd.Stdout = &buf
+	err := cmd.Run()
 	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	http.ServeContent(w, r, parts[3], time.Now(), bytes.NewReader(archive))
+	w.Header().Set("Content-Type", "application/x-git-upload-pack-advertisement")
+	io.WriteString(w, "001e# service=git-upload-pack\n0000")
+	io.Copy(w, &buf)
+}
+func (*FakeGerrit) serveGitUploadPack(w http.ResponseWriter, req *http.Request, repo *FakeRepo) {
+	if req.Method != http.MethodPost {
+		w.Header().Set("Allow", http.MethodPost)
+		http.Error(w, "method should be POST", http.StatusMethodNotAllowed)
+		return
+	}
+	if req.Header.Get("Content-Type") != "application/x-git-upload-pack-request" {
+		http.Error(w, "unexpected Content-Type", http.StatusBadRequest)
+		return
+	}
+	cmd := exec.CommandContext(req.Context(), "git", "upload-pack", "--strict", "--stateless-rpc", ".")
+	cmd.Dir = filepath.Join(repo.dir.dir, ".git")
+	cmd.Env = append(os.Environ(), "GIT_PROTOCOL="+req.Header.Get("Git-Protocol"))
+	cmd.Stdin = req.Body
+	var buf bytes.Buffer
+	cmd.Stdout = &buf
+	err := cmd.Run()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/x-git-upload-pack-result")
+	io.Copy(w, &buf)
 }
 
 func (*FakeGerrit) QueryChanges(_ context.Context, query string) ([]*gerrit.ChangeInfo, error) {

internal/task/gerrit.go

@@ -58,7 +58,7 @@ type GerritClient interface {
 }
 
 type RealGerritClient struct {
-	Gitiles string
+	Gitiles string // Gitiles server URL, without trailing slash. For example, "https://go.googlesource.com".
 	Client  *gerrit.Client
 }
 

internal/task/tagx.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io/fs"
 	"net/url"
+	pathpkg "path"
 	"reflect"
 	"regexp"
 	"strconv"
@@ -19,6 +20,7 @@ import (
 
 	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
 	"golang.org/x/build/gerrit"
+	"golang.org/x/build/internal/gitfs"
 	"golang.org/x/build/internal/releasetargets"
 	"golang.org/x/build/internal/relui/groups"
 	wf "golang.org/x/build/internal/workflow"
@@ -389,32 +391,47 @@ func (x *TagXReposTasks) UpdateGoMod(ctx *wf.TaskContext, repo TagRepo, deps []T
 	}
 	script.WriteString("\n")
 
-	// Tidy the root module.
-	// Also tidy nested modules with a replace directive.
-	// TODO(go.dev/issue/68873): Dynamically detect and handle nested modules, instead of the fixed list below.
-	dirs := []string{"."}
-	switch repo.Name {
-	case "exp":
-		dirs = append(dirs, "slog/benchmarks/zap_benchmarks")     // A local replace directive as of 2023-09-05.
-		dirs = append(dirs, "slog/benchmarks/zerolog_benchmarks") // A local replace directive as of 2023-09-05.
-	case "oscar":
-		dirs = append(dirs, "internal/devtools", "internal/gaby", "internal/gcp", "internal/syncdb") // Using a checked-in go.work as of 2024-10-03.
-	case "telemetry":
-		dirs = append(dirs, "godev") // A local replace directive as of 2023-09-05.
-	case "tools":
-		dirs = append(dirs, "gopls") // A local replace directive as of 2023-09-05.
+	// Tidy the root module and nested modules.
+	// Look for the nested modules dynamically. See go.dev/issue/68873.
+	gitRepo, err := gitfs.NewRepo(x.Gerrit.GitilesURL() + "/" + repo.Name)
+	if err != nil {
+		return nil, err
+	}
+	head, err := gitRepo.Resolve("refs/heads/" + branch)
+	if err != nil {
+		return nil, err
+	}
+	ctx.Printf("Using commit %q as the branch %q head.", head, branch)
+	rootFS, err := gitRepo.CloneHash(head)
+	if err != nil {
+		return nil, err
 	}
 	var outputs []string
-	for _, dir := range dirs {
-		dropToolchain := ""
-		if !repo.HasToolchain {
-			// Don't introduce a toolchain directive if it wasn't already there.
-			// TODO(go.dev/issue/68873): Read the nested module's go.mod. For now, re-use decision from the top-level module.
-			dropToolchain = " && go get toolchain@none"
+	if err := fs.WalkDir(rootFS, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
 		}
-		script.WriteString(fmt.Sprintf("(cd %v && touch go.sum && go mod tidy%s)\n", dir, dropToolchain))
-		outputs = append(outputs, dir+"/go.mod", dir+"/go.sum")
+		if path != "." && d.IsDir() && (strings.HasPrefix(d.Name(), ".") || strings.HasPrefix(d.Name(), "_") || d.Name() == "testdata") {
+			// Skip directories that begin with ".", "_", or are named "testdata".
+			return fs.SkipDir
+		}
+		if d.Name() == "go.mod" && !d.IsDir() { // A go.mod file.
+			dir := pathpkg.Dir(path)
+			dropToolchain := ""
+			if !repo.HasToolchain {
+				// Don't introduce a toolchain directive if it wasn't already there.
+				// TODO(go.dev/issue/68873): Read the nested module's go.mod. For now, re-use decision from the top-level module.
+				dropToolchain = " && go get toolchain@none"
+			}
+			script.WriteString(fmt.Sprintf("(cd %v && touch go.sum && go mod tidy%s)\n", dir, dropToolchain))
+			outputs = append(outputs, dir+"/go.mod", dir+"/go.sum")
+		}
+		return nil
+	}); err != nil {
+		return nil, err
 	}
+
+	// Execute the script to generate updated go.mod/go.sum files.
 	build, err := x.CloudBuild.RunScript(ctx, script.String(), repo.Name, outputs)
 	if err != nil {
 		return nil, err