internal/gcimporter: load cached export data for packages individually

In short tests, also avoid creating export data for all of std.

This change applies the same improvements made to the equivalent std
packages in CL 454497 and CL 454498. (It may even fix the reverse
builders that are currently timing out in x/tools, although I suspect
there is still a bit of work to do for those.)

For golang/go#56967.
Updates golang/go#47257.

Change-Id: I82e72557da5f917203637513122932c7942a98e9
Reviewed-on: https://go-review.googlesource.com/c/tools/+/454499
Auto-Submit: Bryan Mills <bcmills@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Michael Matloob <matloob@golang.org>
Run-TryBot: Bryan Mills <bcmills@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>

internal/gcimporter/gcimporter.go

@@ -13,6 +13,7 @@ package gcimporter // import "golang.org/x/tools/internal/gcimporter"
 
 import (
 	"bufio"
+	"bytes"
 	"errors"
 	"fmt"
 	"go/build"
@@ -22,14 +23,13 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"path"
+	"os/exec"
 	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"text/scanner"
-
-	"golang.org/x/tools/internal/goroot"
 )
 
 const (
@@ -41,23 +41,45 @@ const (
 	trace = false
 )
 
-func lookupGorootExport(pkgpath, srcRoot, srcDir string) (string, bool) {
-	pkgpath = filepath.ToSlash(pkgpath)
-	m, err := goroot.PkgfileMap()
-	if err != nil {
-		return "", false
+var exportMap sync.Map // package dir → func() (string, bool)
+
+// lookupGorootExport returns the location of the export data
+// (normally found in the build cache, but located in GOROOT/pkg
+// in prior Go releases) for the package located in pkgDir.
+//
+// (We use the package's directory instead of its import path
+// mainly to simplify handling of the packages in src/vendor
+// and cmd/vendor.)
+func lookupGorootExport(pkgDir string) (string, bool) {
+	f, ok := exportMap.Load(pkgDir)
+	if !ok {
+		var (
+			listOnce   sync.Once
+			exportPath string
+		)
+		f, _ = exportMap.LoadOrStore(pkgDir, func() (string, bool) {
+			listOnce.Do(func() {
+				cmd := exec.Command("go", "list", "-export", "-f", "{{.Export}}", pkgDir)
+				cmd.Dir = build.Default.GOROOT
+				var output []byte
+				output, err := cmd.Output()
+				if err != nil {
+					return
+				}
+
+				exports := strings.Split(string(bytes.TrimSpace(output)), "\n")
+				if len(exports) != 1 {
+					return
+				}
+
+				exportPath = exports[0]
+			})
+
+			return exportPath, exportPath != ""
+		})
 	}
-	if export, ok := m[pkgpath]; ok {
-		return export, true
-	}
-	vendorPrefix := "vendor"
-	if strings.HasPrefix(srcDir, filepath.Join(srcRoot, "cmd")) {
-		vendorPrefix = path.Join("cmd", vendorPrefix)
-	}
-	pkgpath = path.Join(vendorPrefix, pkgpath)
-	fmt.Fprintln(os.Stderr, "looking up ", pkgpath)
-	export, ok := m[pkgpath]
-	return export, ok
+
+	return f.(func() (string, bool))()
 }
 
 var pkgExts = [...]string{".a", ".o"}
@@ -83,8 +105,8 @@ func FindPkg(path, srcDir string) (filename, id string) {
 		bp, _ := build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
 		if bp.PkgObj == "" {
 			var ok bool
-			if bp.Goroot {
-				filename, ok = lookupGorootExport(path, bp.SrcRoot, srcDir)
+			if bp.Goroot && bp.Dir != "" {
+				filename, ok = lookupGorootExport(bp.Dir)
 			}
 			if !ok {
 				id = path // make sure we have an id to print in error message

internal/gcimporter/gcimporter_test.go

@@ -27,6 +27,7 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/tools/internal/goroot"
 	"golang.org/x/tools/internal/testenv"
 )
 
@@ -65,8 +66,20 @@ func compilePkg(t *testing.T, dirname, filename, outdirname string, packagefiles
 	}
 	objname := basename + ".o"
 	outname := filepath.Join(outdirname, objname)
-	importcfgfile := filepath.Join(outdirname, basename) + ".importcfg"
-	testenv.WriteImportcfg(t, importcfgfile, packagefiles)
+
+	importcfgfile := os.DevNull
+	if len(packagefiles) > 0 {
+		importcfgfile = filepath.Join(outdirname, basename) + ".importcfg"
+		importcfg := new(bytes.Buffer)
+		fmt.Fprintf(importcfg, "# import config")
+		for k, v := range packagefiles {
+			fmt.Fprintf(importcfg, "\npackagefile %s=%s\n", k, v)
+		}
+		if err := os.WriteFile(importcfgfile, importcfg.Bytes(), 0655); err != nil {
+			t.Fatal(err)
+		}
+	}
+
 	importreldir := strings.ReplaceAll(outdirname, string(os.PathSeparator), "/")
 	cmd := exec.Command("go", "tool", "compile", "-p", pkg, "-D", importreldir, "-importcfg", importcfgfile, "-o", outname, filename)
 	cmd.Dir = dirname
@@ -109,7 +122,16 @@ func TestImportTestdata(t *testing.T) {
 	tmpdir := mktmpdir(t)
 	defer os.RemoveAll(tmpdir)
 
-	compile(t, "testdata", testfile, filepath.Join(tmpdir, "testdata"), nil)
+	packageFiles := map[string]string{}
+	for _, pkg := range []string{"go/ast", "go/token"} {
+		export, _ := FindPkg(pkg, "testdata")
+		if export == "" {
+			t.Fatalf("no export data found for %s", pkg)
+		}
+		packageFiles[pkg] = export
+	}
+
+	compile(t, "testdata", testfile, filepath.Join(tmpdir, "testdata"), packageFiles)
 
 	// filename should end with ".go"
 	filename := testfile[:len(testfile)-3]
@@ -137,6 +159,10 @@ func TestImportTestdata(t *testing.T) {
 }
 
 func TestImportTypeparamTests(t *testing.T) {
+	if testing.Short() {
+		t.Skipf("in short mode, skipping test that requires export data for all of std")
+	}
+
 	testenv.NeedsGo1Point(t, 18) // requires generics
 
 	// This package only handles gc export data.
@@ -191,7 +217,11 @@ func TestImportTypeparamTests(t *testing.T) {
 
 			// Compile and import, and compare the resulting package with the package
 			// that was type-checked directly.
-			compile(t, rootDir, entry.Name(), filepath.Join(tmpdir, "testdata"), nil)
+			pkgFiles, err := goroot.PkgfileMap()
+			if err != nil {
+				t.Fatal(err)
+			}
+			compile(t, rootDir, entry.Name(), filepath.Join(tmpdir, "testdata"), pkgFiles)
 			pkgName := strings.TrimSuffix(entry.Name(), ".go")
 			imported := importPkg(t, "./testdata/"+pkgName, tmpdir)
 			checked := checkFile(t, filename, src)
@@ -589,7 +619,13 @@ func TestIssue13566(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	compilePkg(t, "testdata", "a.go", testoutdir, nil, apkg(testoutdir))
+
+	jsonExport, _ := FindPkg("encoding/json", "testdata")
+	if jsonExport == "" {
+		t.Fatalf("no export data found for encoding/json")
+	}
+
+	compilePkg(t, "testdata", "a.go", testoutdir, map[string]string{"encoding/json": jsonExport}, apkg(testoutdir))
 	compile(t, testoutdir, bpath, testoutdir, map[string]string{apkg(testoutdir): filepath.Join(testoutdir, "a.o")})
 
 	// import must succeed (test for issue at hand)

internal/goroot/importcfg.go

@@ -29,9 +29,7 @@ func Importcfg() (string, error) {
 	}
 	fmt.Fprintf(&icfg, "# import config")
 	for importPath, export := range m {
-		if importPath != "unsafe" && export != "" { // unsafe
-			fmt.Fprintf(&icfg, "\npackagefile %s=%s", importPath, export)
-		}
+		fmt.Fprintf(&icfg, "\npackagefile %s=%s", importPath, export)
 	}
 	s := icfg.String()
 	return s, nil
@@ -63,7 +61,9 @@ func PkgfileMap() (map[string]string, error) {
 				return
 			}
 			importPath, export := sp[0], sp[1]
-			m[importPath] = export
+			if export != "" {
+				m[importPath] = export
+			}
 		}
 		stdlibPkgfileMap = m
 	})