go/packages: use native overlay support for 1.16

This change modifies go/packages to use the go command's -overlay flag
if used with Go 1.16. It does so by adding a new Overlay field to the
gocommand.Invocation struct. go/packages writes out the overlay files
as expected by go list before invoking a `go list` command.

Fixes golang/go#41598

Change-Id: Iec5edf19ce2936d5a633d076905622c2cf779bcc
Reviewed-on: https://go-review.googlesource.com/c/tools/+/263984
Trust: Rebecca Stambler <rstambler@golang.org>
Run-TryBot: Rebecca Stambler <rstambler@golang.org>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Heschi Kreinick <heschi@google.com>

go/packages/golist.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"go/types"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -208,6 +209,9 @@ extractQueries:
 		}
 	}
 
+	// Only use go/packages' overlay processing if we're using a Go version
+	// below 1.16. Otherwise, go list handles it.
+	if goVersion, err := state.getGoVersion(); err == nil && goVersion < 16 {
 		modifiedPkgs, needPkgs, err := state.processGolistOverlay(response)
 		if err != nil {
 			return nil, err
@@ -228,12 +232,10 @@ extractQueries:
 				if !ok {
 					response.addPackage(&Package{
 						ID: id,
-					Errors: []Error{
-						{
+						Errors: []Error{{
 							Kind: ListError,
 							Msg:  fmt.Sprintf("package %s expected but not seen", id),
-						},
-					},
+						}},
 					})
 					continue
 				}
@@ -261,6 +263,7 @@ extractQueries:
 				}
 			}
 		}
+	}
 
 	sizeswg.Wait()
 	if sizeserr != nil {
@@ -835,6 +838,26 @@ func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer,
 	cfg := state.cfg
 
 	inv := state.cfgInvocation()
+
+	// For Go versions 1.16 and above, `go list` accepts overlays directly via
+	// the -overlay flag. Set it, if it's available.
+	//
+	// The check for "list" is not necessarily required, but we should avoid
+	// getting the go version if possible.
+	if verb == "list" {
+		goVersion, err := state.getGoVersion()
+		if err != nil {
+			return nil, err
+		}
+		if goVersion >= 16 {
+			filename, cleanup, err := state.writeOverlays()
+			if err != nil {
+				return nil, err
+			}
+			defer cleanup()
+			inv.Overlay = filename
+		}
+	}
 	inv.Verb = verb
 	inv.Args = args
 	gocmdRunner := cfg.gocmdRunner
@@ -976,6 +999,67 @@ func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer,
 	return stdout, nil
 }
 
+// OverlayJSON is the format overlay files are expected to be in.
+// The Replace map maps from overlaid paths to replacement paths:
+// the Go command will forward all reads trying to open
+// each overlaid path to its replacement path, or consider the overlaid
+// path not to exist if the replacement path is empty.
+//
+// From golang/go#39958.
+type OverlayJSON struct {
+	Replace map[string]string `json:"replace,omitempty"`
+}
+
+// writeOverlays writes out files for go list's -overlay flag, as described
+// above.
+func (state *golistState) writeOverlays() (filename string, cleanup func(), err error) {
+	// Do nothing if there are no overlays in the config.
+	if len(state.cfg.Overlay) == 0 {
+		return "", func() {}, nil
+	}
+	dir, err := ioutil.TempDir("", "gopackages-*")
+	if err != nil {
+		return "", nil, err
+	}
+	// The caller must clean up this directory, unless this function returns an
+	// error.
+	cleanup = func() {
+		os.RemoveAll(dir)
+	}
+	defer func() {
+		if err != nil {
+			cleanup()
+		}
+	}()
+	overlays := map[string]string{}
+	for k, v := range state.cfg.Overlay {
+		// Create a unique filename for the overlaid files, to avoid
+		// creating nested directories.
+		noSeparator := strings.Join(strings.Split(filepath.ToSlash(k), "/"), "")
+		f, err := ioutil.TempFile(dir, fmt.Sprintf("*-%s", noSeparator))
+		if err != nil {
+			return "", func() {}, err
+		}
+		if _, err := f.Write(v); err != nil {
+			return "", func() {}, err
+		}
+		if err := f.Close(); err != nil {
+			return "", func() {}, err
+		}
+		overlays[k] = f.Name()
+	}
+	b, err := json.Marshal(OverlayJSON{Replace: overlays})
+	if err != nil {
+		return "", func() {}, err
+	}
+	// Write out the overlay file that contains the filepath mappings.
+	filename = filepath.Join(dir, "overlay.json")
+	if err := ioutil.WriteFile(filename, b, 0665); err != nil {
+		return "", func() {}, err
+	}
+	return filename, cleanup, nil
+}
+
 func containsGoFile(s []string) bool {
 	for _, f := range s {
 		if strings.HasSuffix(f, ".go") {

go/packages/overlay_test.go

@@ -87,6 +87,9 @@ func testOverlayChangesBothPackageNames(t *testing.T, exporter packagestest.Expo
 		{"fake [fake.test]", "foox", 2},
 		{"fake.test", "main", 1},
 	}
+	if len(initial) != 3 {
+		t.Fatalf("expected 3 packages, got %v", len(initial))
+	}
 	for i := 0; i < 3; i++ {
 		if ok := checkPkg(t, initial[i], want[i].id, want[i].name, want[i].count); !ok {
 			t.Errorf("%d: got {%s %s %d}, expected %v", i, initial[i].ID,
@@ -102,7 +105,8 @@ func TestOverlayChangesTestPackageName(t *testing.T) {
 	packagestest.TestAll(t, testOverlayChangesTestPackageName)
 }
 func testOverlayChangesTestPackageName(t *testing.T, exporter packagestest.Exporter) {
-	log.SetFlags(log.Lshortfile)
+	testenv.NeedsGo1Point(t, 16)
+
 	exported := packagestest.Export(t, exporter, []packagestest.Module{{
 		Name: "fake",
 		Files: map[string]interface{}{
@@ -127,10 +131,13 @@ func testOverlayChangesTestPackageName(t *testing.T, exporter packagestest.Expor
 		id, name string
 		count    int
 	}{
-		{"fake", "foo", 0},
+		{"fake", "foox", 0},
 		{"fake [fake.test]", "foox", 1},
 		{"fake.test", "main", 1},
 	}
+	if len(initial) != 3 {
+		t.Fatalf("expected 3 packages, got %v", len(initial))
+	}
 	for i := 0; i < 3; i++ {
 		if ok := checkPkg(t, initial[i], want[i].id, want[i].name, want[i].count); !ok {
 			t.Errorf("got {%s %s %d}, expected %v", initial[i].ID,
@@ -329,6 +336,9 @@ func testOverlayDeps(t *testing.T, exporter packagestest.Exporter) {
 
 	// Find package golang.org/fake/c
 	sort.Slice(pkgs, func(i, j int) bool { return pkgs[i].ID < pkgs[j].ID })
+	if len(pkgs) != 2 {
+		t.Fatalf("expected 2 packages, got %v", len(pkgs))
+	}
 	pkgc := pkgs[0]
 	if pkgc.ID != "golang.org/fake/c" {
 		t.Errorf("expected first package in sorted list to be \"golang.org/fake/c\", got %v", pkgc.ID)
@@ -804,6 +814,9 @@ func testInvalidFilesBeforeOverlayContains(t *testing.T, exporter packagestest.E
 				if err != nil {
 					t.Fatal(err)
 				}
+				if len(initial) != 1 {
+					t.Fatalf("expected 1 packages, got %v", len(initial))
+				}
 				pkg := initial[0]
 				if pkg.ID != tt.wantID {
 					t.Fatalf("expected package ID %q, got %q", tt.wantID, pkg.ID)
@@ -986,7 +999,7 @@ func Hi() {
 				}
 			}
 			if match == nil {
-				t.Fatalf(`expected package path "golang.org/fake/a", got %q`, match.PkgPath)
+				t.Fatalf(`expected package path "golang.org/fake/a", got none`)
 			}
 			if match.PkgPath != "golang.org/fake/a" {
 				t.Fatalf(`expected package path "golang.org/fake/a", got %q`, match.PkgPath)
@@ -1072,6 +1085,9 @@ replace (
 	if err != nil {
 		t.Error(err)
 	}
+	if len(initial) != 1 {
+		t.Fatalf(`expected 1 package, got %v`, len(initial))
+	}
 	pkg := initial[0]
 	if pkg.PkgPath != "b.com/inner" {
 		t.Fatalf(`expected package path "b.com/inner", got %q`, pkg.PkgPath)

internal/gocommand/invoke.go

@@ -132,6 +132,7 @@ type Invocation struct {
 	BuildFlags []string
 	ModFlag    string
 	ModFile    string
+	Overlay    string
 	Env        []string
 	WorkingDir string
 	Logf       func(format string, args ...interface{})
@@ -171,6 +172,11 @@ func (i *Invocation) run(ctx context.Context, stdout, stderr io.Writer) error {
 			goArgs = append(goArgs, "-mod="+i.ModFlag)
 		}
 	}
+	appendOverlayFlag := func() {
+		if i.Overlay != "" {
+			goArgs = append(goArgs, "-overlay="+i.Overlay)
+		}
+	}
 
 	switch i.Verb {
 	case "env", "version":
@@ -189,6 +195,7 @@ func (i *Invocation) run(ctx context.Context, stdout, stderr io.Writer) error {
 		goArgs = append(goArgs, i.BuildFlags...)
 		appendModFile()
 		appendModFlag()
+		appendOverlayFlag()
 		goArgs = append(goArgs, i.Args...)
 	}
 	cmd := exec.Command("go", goArgs...)