internal/packagestest: fork go/packages/packagestest

Also, update all imports outside of that package.

Updates golang/go#70229

Change-Id: I8b08f892ec86d560c0406319c2954eb9912d78ad
Reviewed-on: https://go-review.googlesource.com/c/tools/+/625920
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Robert Findley <rfindley@google.com>

cmd/bundle/main_test.go

@@ -11,7 +11,7 @@ import (
 	"runtime"
 	"testing"
 
-	"golang.org/x/tools/go/packages/packagestest"
+	"golang.org/x/tools/internal/packagestest"
 )
 
 func TestBundle(t *testing.T) { packagestest.TestAll(t, testBundle) }

cmd/godoc/godoc_test.go

@@ -21,7 +21,7 @@ import (
 	"testing"
 	"time"
 
-	"golang.org/x/tools/go/packages/packagestest"
+	"golang.org/x/tools/internal/packagestest"
 	"golang.org/x/tools/internal/testenv"
 )
 

go/analysis/unitchecker/unitchecker_test.go

@@ -17,7 +17,7 @@ import (
 	"golang.org/x/tools/go/analysis/passes/findcall"
 	"golang.org/x/tools/go/analysis/passes/printf"
 	"golang.org/x/tools/go/analysis/unitchecker"
-	"golang.org/x/tools/go/packages/packagestest"
+	"golang.org/x/tools/internal/packagestest"
 )
 
 func TestMain(m *testing.M) {

go/buildutil/allpackages_test.go

@@ -17,7 +17,7 @@ import (
 	"testing"
 
 	"golang.org/x/tools/go/buildutil"
-	"golang.org/x/tools/go/packages/packagestest"
+	"golang.org/x/tools/internal/packagestest"
 )
 
 func TestAllPackages(t *testing.T) {

go/buildutil/util_test.go

@@ -13,7 +13,7 @@ import (
 	"testing"
 
 	"golang.org/x/tools/go/buildutil"
-	"golang.org/x/tools/go/packages/packagestest"
+	"golang.org/x/tools/internal/packagestest"
 )
 
 func TestContainingPackage(t *testing.T) {

go/packages/overlay_test.go

@@ -14,7 +14,7 @@ import (
 	"testing"
 
 	"golang.org/x/tools/go/packages"
-	"golang.org/x/tools/go/packages/packagestest"
+	"golang.org/x/tools/internal/packagestest"
 	"golang.org/x/tools/internal/testenv"
 )
 

go/packages/packages_test.go

@@ -27,8 +27,8 @@ import (
 	"time"
 
 	"golang.org/x/tools/go/packages"
-	"golang.org/x/tools/go/packages/packagestest"
 	"golang.org/x/tools/internal/packagesinternal"
+	"golang.org/x/tools/internal/packagestest"
 	"golang.org/x/tools/internal/testenv"
 	"golang.org/x/tools/internal/testfiles"
 )

go/ssa/ssautil/load_test.go

@@ -17,9 +17,9 @@ import (
 	"testing"
 
 	"golang.org/x/tools/go/packages"
-	"golang.org/x/tools/go/packages/packagestest"
 	"golang.org/x/tools/go/ssa"
 	"golang.org/x/tools/go/ssa/ssautil"
+	"golang.org/x/tools/internal/packagestest"
 	"golang.org/x/tools/internal/testenv"
 )
 

internal/imports/fix_test.go

@@ -20,8 +20,8 @@ import (
 	"sync/atomic"
 	"testing"
 
-	"golang.org/x/tools/go/packages/packagestest"
 	"golang.org/x/tools/internal/gocommand"
+	"golang.org/x/tools/internal/packagestest"
 	"golang.org/x/tools/internal/stdlib"
 )
 

internal/packagestest/expect.go

@@ -0,0 +1,468 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packagestest
+
+import (
+	"fmt"
+	"go/token"
+	"os"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"golang.org/x/tools/go/packages"
+	"golang.org/x/tools/internal/expect"
+)
+
+const (
+	markMethod    = "mark"
+	eofIdentifier = "EOF"
+)
+
+// Expect invokes the supplied methods for all expectation notes found in
+// the exported source files.
+//
+// All exported go source files are parsed to collect the expectation
+// notes.
+// See the documentation for expect.Parse for how the notes are collected
+// and parsed.
+//
+// The methods are supplied as a map of name to function, and those functions
+// will be matched against the expectations by name.
+// Notes with no matching function will be skipped, and functions with no
+// matching notes will not be invoked.
+// If there are no registered markers yet, a special pass will be run first
+// which adds any markers declared with @mark(Name, pattern) or @name. These
+// call the Mark method to add the marker to the global set.
+// You can register the "mark" method to override these in your own call to
+// Expect. The bound Mark function is usable directly in your method map, so
+//
+//	exported.Expect(map[string]interface{}{"mark": exported.Mark})
+//
+// replicates the built in behavior.
+//
+// # Method invocation
+//
+// When invoking a method the expressions in the parameter list need to be
+// converted to values to be passed to the method.
+// There are a very limited set of types the arguments are allowed to be.
+//
+//	expect.Note : passed the Note instance being evaluated.
+//	string : can be supplied either a string literal or an identifier.
+//	int : can only be supplied an integer literal.
+//	*regexp.Regexp : can only be supplied a regular expression literal
+//	token.Pos : has a file position calculated as described below.
+//	token.Position : has a file position calculated as described below.
+//	expect.Range: has a start and end position as described below.
+//	interface{} : will be passed any value
+//
+// # Position calculation
+//
+// There is some extra handling when a parameter is being coerced into a
+// token.Pos, token.Position or Range type argument.
+//
+// If the parameter is an identifier, it will be treated as the name of an
+// marker to look up (as if markers were global variables).
+//
+// If it is a string or regular expression, then it will be passed to
+// expect.MatchBefore to look up a match in the line at which it was declared.
+//
+// It is safe to call this repeatedly with different method sets, but it is
+// not safe to call it concurrently.
+func (e *Exported) Expect(methods map[string]interface{}) error {
+	if err := e.getNotes(); err != nil {
+		return err
+	}
+	if err := e.getMarkers(); err != nil {
+		return err
+	}
+	var err error
+	ms := make(map[string]method, len(methods))
+	for name, f := range methods {
+		mi := method{f: reflect.ValueOf(f)}
+		mi.converters = make([]converter, mi.f.Type().NumIn())
+		for i := 0; i < len(mi.converters); i++ {
+			mi.converters[i], err = e.buildConverter(mi.f.Type().In(i))
+			if err != nil {
+				return fmt.Errorf("invalid method %v: %v", name, err)
+			}
+		}
+		ms[name] = mi
+	}
+	for _, n := range e.notes {
+		if n.Args == nil {
+			// simple identifier form, convert to a call to mark
+			n = &expect.Note{
+				Pos:  n.Pos,
+				Name: markMethod,
+				Args: []interface{}{n.Name, n.Name},
+			}
+		}
+		mi, ok := ms[n.Name]
+		if !ok {
+			continue
+		}
+		params := make([]reflect.Value, len(mi.converters))
+		args := n.Args
+		for i, convert := range mi.converters {
+			params[i], args, err = convert(n, args)
+			if err != nil {
+				return fmt.Errorf("%v: %v", e.ExpectFileSet.Position(n.Pos), err)
+			}
+		}
+		if len(args) > 0 {
+			return fmt.Errorf("%v: unwanted args got %+v extra", e.ExpectFileSet.Position(n.Pos), args)
+		}
+		//TODO: catch the error returned from the method
+		mi.f.Call(params)
+	}
+	return nil
+}
+
+// A Range represents an interval within a source file in go/token notation.
+type Range struct {
+	TokFile    *token.File // non-nil
+	Start, End token.Pos   // both valid and within range of TokFile
+}
+
+// Mark adds a new marker to the known set.
+func (e *Exported) Mark(name string, r Range) {
+	if e.markers == nil {
+		e.markers = make(map[string]Range)
+	}
+	e.markers[name] = r
+}
+
+func (e *Exported) getNotes() error {
+	if e.notes != nil {
+		return nil
+	}
+	notes := []*expect.Note{}
+	var dirs []string
+	for _, module := range e.written {
+		for _, filename := range module {
+			dirs = append(dirs, filepath.Dir(filename))
+		}
+	}
+	for filename := range e.Config.Overlay {
+		dirs = append(dirs, filepath.Dir(filename))
+	}
+	pkgs, err := packages.Load(e.Config, dirs...)
+	if err != nil {
+		return fmt.Errorf("unable to load packages for directories %s: %v", dirs, err)
+	}
+	seen := make(map[token.Position]struct{})
+	for _, pkg := range pkgs {
+		for _, filename := range pkg.GoFiles {
+			content, err := e.FileContents(filename)
+			if err != nil {
+				return err
+			}
+			l, err := expect.Parse(e.ExpectFileSet, filename, content)
+			if err != nil {
+				return fmt.Errorf("failed to extract expectations: %v", err)
+			}
+			for _, note := range l {
+				pos := e.ExpectFileSet.Position(note.Pos)
+				if _, ok := seen[pos]; ok {
+					continue
+				}
+				notes = append(notes, note)
+				seen[pos] = struct{}{}
+			}
+		}
+	}
+	if _, ok := e.written[e.primary]; !ok {
+		e.notes = notes
+		return nil
+	}
+	// Check go.mod markers regardless of mode, we need to do this so that our marker count
+	// matches the counts in the summary.txt.golden file for the test directory.
+	if gomod, found := e.written[e.primary]["go.mod"]; found {
+		// If we are in Modules mode, then we need to check the contents of the go.mod.temp.
+		if e.Exporter == Modules {
+			gomod += ".temp"
+		}
+		l, err := goModMarkers(e, gomod)
+		if err != nil {
+			return fmt.Errorf("failed to extract expectations for go.mod: %v", err)
+		}
+		notes = append(notes, l...)
+	}
+	e.notes = notes
+	return nil
+}
+
+func goModMarkers(e *Exported, gomod string) ([]*expect.Note, error) {
+	if _, err := os.Stat(gomod); os.IsNotExist(err) {
+		// If there is no go.mod file, we want to be able to continue.
+		return nil, nil
+	}
+	content, err := e.FileContents(gomod)
+	if err != nil {
+		return nil, err
+	}
+	if e.Exporter == GOPATH {
+		return expect.Parse(e.ExpectFileSet, gomod, content)
+	}
+	gomod = strings.TrimSuffix(gomod, ".temp")
+	// If we are in Modules mode, copy the original contents file back into go.mod
+	if err := os.WriteFile(gomod, content, 0644); err != nil {
+		return nil, nil
+	}
+	return expect.Parse(e.ExpectFileSet, gomod, content)
+}
+
+func (e *Exported) getMarkers() error {
+	if e.markers != nil {
+		return nil
+	}
+	// set markers early so that we don't call getMarkers again from Expect
+	e.markers = make(map[string]Range)
+	return e.Expect(map[string]interface{}{
+		markMethod: e.Mark,
+	})
+}
+
+var (
+	noteType       = reflect.TypeOf((*expect.Note)(nil))
+	identifierType = reflect.TypeOf(expect.Identifier(""))
+	posType        = reflect.TypeOf(token.Pos(0))
+	positionType   = reflect.TypeOf(token.Position{})
+	rangeType      = reflect.TypeOf(Range{})
+	fsetType       = reflect.TypeOf((*token.FileSet)(nil))
+	regexType      = reflect.TypeOf((*regexp.Regexp)(nil))
+	exportedType   = reflect.TypeOf((*Exported)(nil))
+)
+
+// converter converts from a marker's argument parsed from the comment to
+// reflect values passed to the method during Invoke.
+// It takes the args remaining, and returns the args it did not consume.
+// This allows a converter to consume 0 args for well known types, or multiple
+// args for compound types.
+type converter func(*expect.Note, []interface{}) (reflect.Value, []interface{}, error)
+
+// method is used to track information about Invoke methods that is expensive to
+// calculate so that we can work it out once rather than per marker.
+type method struct {
+	f          reflect.Value // the reflect value of the passed in method
+	converters []converter   // the parameter converters for the method
+}
+
+// buildConverter works out what function should be used to go from an ast expressions to a reflect
+// value of the type expected by a method.
+// It is called when only the target type is know, it returns converters that are flexible across
+// all supported expression types for that target type.
+func (e *Exported) buildConverter(pt reflect.Type) (converter, error) {
+	switch {
+	case pt == noteType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			return reflect.ValueOf(n), args, nil
+		}, nil
+	case pt == fsetType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			return reflect.ValueOf(e.ExpectFileSet), args, nil
+		}, nil
+	case pt == exportedType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			return reflect.ValueOf(e), args, nil
+		}, nil
+	case pt == posType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			r, remains, err := e.rangeConverter(n, args)
+			if err != nil {
+				return reflect.Value{}, nil, err
+			}
+			return reflect.ValueOf(r.Start), remains, nil
+		}, nil
+	case pt == positionType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			r, remains, err := e.rangeConverter(n, args)
+			if err != nil {
+				return reflect.Value{}, nil, err
+			}
+			return reflect.ValueOf(e.ExpectFileSet.Position(r.Start)), remains, nil
+		}, nil
+	case pt == rangeType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			r, remains, err := e.rangeConverter(n, args)
+			if err != nil {
+				return reflect.Value{}, nil, err
+			}
+			return reflect.ValueOf(r), remains, nil
+		}, nil
+	case pt == identifierType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			if len(args) < 1 {
+				return reflect.Value{}, nil, fmt.Errorf("missing argument")
+			}
+			arg := args[0]
+			args = args[1:]
+			switch arg := arg.(type) {
+			case expect.Identifier:
+				return reflect.ValueOf(arg), args, nil
+			default:
+				return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to string", arg)
+			}
+		}, nil
+
+	case pt == regexType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			if len(args) < 1 {
+				return reflect.Value{}, nil, fmt.Errorf("missing argument")
+			}
+			arg := args[0]
+			args = args[1:]
+			if _, ok := arg.(*regexp.Regexp); !ok {
+				return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to *regexp.Regexp", arg)
+			}
+			return reflect.ValueOf(arg), args, nil
+		}, nil
+
+	case pt.Kind() == reflect.String:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			if len(args) < 1 {
+				return reflect.Value{}, nil, fmt.Errorf("missing argument")
+			}
+			arg := args[0]
+			args = args[1:]
+			switch arg := arg.(type) {
+			case expect.Identifier:
+				return reflect.ValueOf(string(arg)), args, nil
+			case string:
+				return reflect.ValueOf(arg), args, nil
+			default:
+				return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to string", arg)
+			}
+		}, nil
+	case pt.Kind() == reflect.Int64:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			if len(args) < 1 {
+				return reflect.Value{}, nil, fmt.Errorf("missing argument")
+			}
+			arg := args[0]
+			args = args[1:]
+			switch arg := arg.(type) {
+			case int64:
+				return reflect.ValueOf(arg), args, nil
+			default:
+				return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to int", arg)
+			}
+		}, nil
+	case pt.Kind() == reflect.Bool:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			if len(args) < 1 {
+				return reflect.Value{}, nil, fmt.Errorf("missing argument")
+			}
+			arg := args[0]
+			args = args[1:]
+			b, ok := arg.(bool)
+			if !ok {
+				return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to bool", arg)
+			}
+			return reflect.ValueOf(b), args, nil
+		}, nil
+	case pt.Kind() == reflect.Slice:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			converter, err := e.buildConverter(pt.Elem())
+			if err != nil {
+				return reflect.Value{}, nil, err
+			}
+			result := reflect.MakeSlice(reflect.SliceOf(pt.Elem()), 0, len(args))
+			for range args {
+				value, remains, err := converter(n, args)
+				if err != nil {
+					return reflect.Value{}, nil, err
+				}
+				result = reflect.Append(result, value)
+				args = remains
+			}
+			return result, args, nil
+		}, nil
+	default:
+		if pt.Kind() == reflect.Interface && pt.NumMethod() == 0 {
+			return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+				if len(args) < 1 {
+					return reflect.Value{}, nil, fmt.Errorf("missing argument")
+				}
+				return reflect.ValueOf(args[0]), args[1:], nil
+			}, nil
+		}
+		return nil, fmt.Errorf("param has unexpected type %v (kind %v)", pt, pt.Kind())
+	}
+}
+
+func (e *Exported) rangeConverter(n *expect.Note, args []interface{}) (Range, []interface{}, error) {
+	tokFile := e.ExpectFileSet.File(n.Pos)
+	if len(args) < 1 {
+		return Range{}, nil, fmt.Errorf("missing argument")
+	}
+	arg := args[0]
+	args = args[1:]
+	switch arg := arg.(type) {
+	case expect.Identifier:
+		// handle the special identifiers
+		switch arg {
+		case eofIdentifier:
+			// end of file identifier
+			eof := tokFile.Pos(tokFile.Size())
+			return newRange(tokFile, eof, eof), args, nil
+		default:
+			// look up an marker by name
+			mark, ok := e.markers[string(arg)]
+			if !ok {
+				return Range{}, nil, fmt.Errorf("cannot find marker %v", arg)
+			}
+			return mark, args, nil
+		}
+	case string:
+		start, end, err := expect.MatchBefore(e.ExpectFileSet, e.FileContents, n.Pos, arg)
+		if err != nil {
+			return Range{}, nil, err
+		}
+		if !start.IsValid() {
+			return Range{}, nil, fmt.Errorf("%v: pattern %s did not match", e.ExpectFileSet.Position(n.Pos), arg)
+		}
+		return newRange(tokFile, start, end), args, nil
+	case *regexp.Regexp:
+		start, end, err := expect.MatchBefore(e.ExpectFileSet, e.FileContents, n.Pos, arg)
+		if err != nil {
+			return Range{}, nil, err
+		}
+		if !start.IsValid() {
+			return Range{}, nil, fmt.Errorf("%v: pattern %s did not match", e.ExpectFileSet.Position(n.Pos), arg)
+		}
+		return newRange(tokFile, start, end), args, nil
+	default:
+		return Range{}, nil, fmt.Errorf("cannot convert %v to pos", arg)
+	}
+}
+
+// newRange creates a new Range from a token.File and two valid positions within it.
+func newRange(file *token.File, start, end token.Pos) Range {
+	fileBase := file.Base()
+	fileEnd := fileBase + file.Size()
+	if !start.IsValid() {
+		panic("invalid start token.Pos")
+	}
+	if !end.IsValid() {
+		panic("invalid end token.Pos")
+	}
+	if int(start) < fileBase || int(start) > fileEnd {
+		panic(fmt.Sprintf("invalid start: %d not in [%d, %d]", start, fileBase, fileEnd))
+	}
+	if int(end) < fileBase || int(end) > fileEnd {
+		panic(fmt.Sprintf("invalid end: %d not in [%d, %d]", end, fileBase, fileEnd))
+	}
+	if start > end {
+		panic("invalid start: greater than end")
+	}
+	return Range{
+		TokFile: file,
+		Start:   start,
+		End:     end,
+	}
+}

internal/packagestest/expect_test.go

@@ -0,0 +1,71 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packagestest_test
+
+import (
+	"go/token"
+	"testing"
+
+	"golang.org/x/tools/internal/expect"
+	"golang.org/x/tools/internal/packagestest"
+)
+
+func TestExpect(t *testing.T) {
+	exported := packagestest.Export(t, packagestest.GOPATH, []packagestest.Module{{
+		Name:  "golang.org/fake",
+		Files: packagestest.MustCopyFileTree("testdata"),
+	}})
+	defer exported.Cleanup()
+	checkCount := 0
+	if err := exported.Expect(map[string]interface{}{
+		"check": func(src, target token.Position) {
+			checkCount++
+		},
+		"boolArg": func(n *expect.Note, yes, no bool) {
+			if !yes {
+				t.Errorf("Expected boolArg first param to be true")
+			}
+			if no {
+				t.Errorf("Expected boolArg second param to be false")
+			}
+		},
+		"intArg": func(n *expect.Note, i int64) {
+			if i != 42 {
+				t.Errorf("Expected intarg to be 42")
+			}
+		},
+		"stringArg": func(n *expect.Note, name expect.Identifier, value string) {
+			if string(name) != value {
+				t.Errorf("Got string arg %v expected %v", value, name)
+			}
+		},
+		"directNote": func(n *expect.Note) {},
+		"range": func(r packagestest.Range) {
+			if r.Start == token.NoPos || r.Start == 0 {
+				t.Errorf("Range had no valid starting position")
+			}
+			if r.End == token.NoPos || r.End == 0 {
+				t.Errorf("Range had no valid ending position")
+			} else if r.End <= r.Start {
+				t.Errorf("Range ending was not greater than start")
+			}
+		},
+		"checkEOF": func(n *expect.Note, p token.Pos) {
+			if p <= n.Pos {
+				t.Errorf("EOF was before the checkEOF note")
+			}
+		},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	// We expect to have walked the @check annotations in all .go files,
+	// including _test.go files (XTest or otherwise). But to have walked the
+	// non-_test.go files only once. Hence wantCheck = 3 (testdata/test.go) + 1
+	// (testdata/test_test.go) + 1 (testdata/x_test.go)
+	wantCheck := 7
+	if wantCheck != checkCount {
+		t.Fatalf("Expected @check count of %v; got %v", wantCheck, checkCount)
+	}
+}

internal/packagestest/export.go

@@ -0,0 +1,666 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package packagestest creates temporary projects on disk for testing go tools on.
+
+By changing the exporter used, you can create projects for multiple build
+systems from the same description, and run the same tests on them in many
+cases.
+
+# Example
+
+As an example of packagestest use, consider the following test that runs
+the 'go list' command on the specified modules:
+
+	// TestGoList exercises the 'go list' command in module mode and in GOPATH mode.
+	func TestGoList(t *testing.T) { packagestest.TestAll(t, testGoList) }
+	func testGoList(t *testing.T, x packagestest.Exporter) {
+		e := packagestest.Export(t, x, []packagestest.Module{
+			{
+				Name: "gopher.example/repoa",
+				Files: map[string]interface{}{
+					"a/a.go": "package a",
+				},
+			},
+			{
+				Name: "gopher.example/repob",
+				Files: map[string]interface{}{
+					"b/b.go": "package b",
+				},
+			},
+		})
+		defer e.Cleanup()
+
+		cmd := exec.Command("go", "list", "gopher.example/...")
+		cmd.Dir = e.Config.Dir
+		cmd.Env = e.Config.Env
+		out, err := cmd.Output()
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Logf("'go list gopher.example/...' with %s mode layout:\n%s", x.Name(), out)
+	}
+
+TestGoList uses TestAll to exercise the 'go list' command with all
+exporters known to packagestest. Currently, packagestest includes
+exporters that produce module mode layouts and GOPATH mode layouts.
+Running the test with verbose output will print:
+
+	=== RUN   TestGoList
+	=== RUN   TestGoList/GOPATH
+	=== RUN   TestGoList/Modules
+	--- PASS: TestGoList (0.21s)
+	    --- PASS: TestGoList/GOPATH (0.03s)
+	        main_test.go:36: 'go list gopher.example/...' with GOPATH mode layout:
+	            gopher.example/repoa/a
+	            gopher.example/repob/b
+	    --- PASS: TestGoList/Modules (0.18s)
+	        main_test.go:36: 'go list gopher.example/...' with Modules mode layout:
+	            gopher.example/repoa/a
+	            gopher.example/repob/b
+*/
+package packagestest
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"go/token"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"golang.org/x/tools/go/packages"
+	"golang.org/x/tools/internal/expect"
+	"golang.org/x/tools/internal/testenv"
+)
+
+var (
+	skipCleanup = flag.Bool("skip-cleanup", false, "Do not delete the temporary export folders") // for debugging
+)
+
+// ErrUnsupported indicates an error due to an operation not supported on the
+// current platform.
+var ErrUnsupported = errors.New("operation is not supported")
+
+// Module is a representation of a go module.
+type Module struct {
+	// Name is the base name of the module as it would be in the go.mod file.
+	Name string
+	// Files is the set of source files for all packages that make up the module.
+	// The keys are the file fragment that follows the module name, the value can
+	// be a string or byte slice, in which case it is the contents of the
+	// file, otherwise it must be a Writer function.
+	Files map[string]interface{}
+
+	// Overlay is the set of source file overlays for the module.
+	// The keys are the file fragment as in the Files configuration.
+	// The values are the in memory overlay content for the file.
+	Overlay map[string][]byte
+}
+
+// A Writer is a function that writes out a test file.
+// It is provided the name of the file to write, and may return an error if it
+// cannot write the file.
+// These are used as the content of the Files map in a Module.
+type Writer func(filename string) error
+
+// Exported is returned by the Export function to report the structure that was produced on disk.
+type Exported struct {
+	// Config is a correctly configured packages.Config ready to be passed to packages.Load.
+	// Exactly what it will contain varies depending on the Exporter being used.
+	Config *packages.Config
+
+	// Modules is the module description that was used to produce this exported data set.
+	Modules []Module
+
+	ExpectFileSet *token.FileSet // The file set used when parsing expectations
+
+	Exporter Exporter                     // the exporter used
+	temp     string                       // the temporary directory that was exported to
+	primary  string                       // the first non GOROOT module that was exported
+	written  map[string]map[string]string // the full set of exported files
+	notes    []*expect.Note               // The list of expectations extracted from go source files
+	markers  map[string]Range             // The set of markers extracted from go source files
+}
+
+// Exporter implementations are responsible for converting from the generic description of some
+// test data to a driver specific file layout.
+type Exporter interface {
+	// Name reports the name of the exporter, used in logging and sub-test generation.
+	Name() string
+	// Filename reports the system filename for test data source file.
+	// It is given the base directory, the module the file is part of and the filename fragment to
+	// work from.
+	Filename(exported *Exported, module, fragment string) string
+	// Finalize is called once all files have been written to write any extra data needed and modify
+	// the Config to match. It is handed the full list of modules that were encountered while writing
+	// files.
+	Finalize(exported *Exported) error
+}
+
+// All is the list of known exporters.
+// This is used by TestAll to run tests with all the exporters.
+var All = []Exporter{GOPATH, Modules}
+
+// TestAll invokes the testing function once for each exporter registered in
+// the All global.
+// Each exporter will be run as a sub-test named after the exporter being used.
+func TestAll(t *testing.T, f func(*testing.T, Exporter)) {
+	t.Helper()
+	for _, e := range All {
+		e := e // in case f calls t.Parallel
+		t.Run(e.Name(), func(t *testing.T) {
+			t.Helper()
+			f(t, e)
+		})
+	}
+}
+
+// BenchmarkAll invokes the testing function once for each exporter registered in
+// the All global.
+// Each exporter will be run as a sub-test named after the exporter being used.
+func BenchmarkAll(b *testing.B, f func(*testing.B, Exporter)) {
+	b.Helper()
+	for _, e := range All {
+		e := e // in case f calls t.Parallel
+		b.Run(e.Name(), func(b *testing.B) {
+			b.Helper()
+			f(b, e)
+		})
+	}
+}
+
+// Export is called to write out a test directory from within a test function.
+// It takes the exporter and the build system agnostic module descriptions, and
+// uses them to build a temporary directory.
+// It returns an Exported with the results of the export.
+// The Exported.Config is prepared for loading from the exported data.
+// You must invoke Exported.Cleanup on the returned value to clean up.
+// The file deletion in the cleanup can be skipped by setting the skip-cleanup
+// flag when invoking the test, allowing the temporary directory to be left for
+// debugging tests.
+//
+// If the Writer for any file within any module returns an error equivalent to
+// ErrUnspported, Export skips the test.
+func Export(t testing.TB, exporter Exporter, modules []Module) *Exported {
+	t.Helper()
+	if exporter == Modules {
+		testenv.NeedsTool(t, "go")
+	}
+
+	dirname := strings.Replace(t.Name(), "/", "_", -1)
+	dirname = strings.Replace(dirname, "#", "_", -1) // duplicate subtests get a #NNN suffix.
+	temp, err := os.MkdirTemp("", dirname)
+	if err != nil {
+		t.Fatal(err)
+	}
+	exported := &Exported{
+		Config: &packages.Config{
+			Dir:     temp,
+			Env:     append(os.Environ(), "GOPACKAGESDRIVER=off", "GOROOT="), // Clear GOROOT to work around #32849.
+			Overlay: make(map[string][]byte),
+			Tests:   true,
+			Mode:    packages.LoadImports,
+		},
+		Modules:       modules,
+		Exporter:      exporter,
+		temp:          temp,
+		primary:       modules[0].Name,
+		written:       map[string]map[string]string{},
+		ExpectFileSet: token.NewFileSet(),
+	}
+	if testing.Verbose() {
+		exported.Config.Logf = t.Logf
+	}
+	defer func() {
+		if t.Failed() || t.Skipped() {
+			exported.Cleanup()
+		}
+	}()
+	for _, module := range modules {
+		// Create all parent directories before individual files. If any file is a
+		// symlink to a directory, that directory must exist before the symlink is
+		// created or else it may be created with the wrong type on Windows.
+		// (See https://golang.org/issue/39183.)
+		for fragment := range module.Files {
+			fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment))
+			if err := os.MkdirAll(filepath.Dir(fullpath), 0755); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		for fragment, value := range module.Files {
+			fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment))
+			written, ok := exported.written[module.Name]
+			if !ok {
+				written = map[string]string{}
+				exported.written[module.Name] = written
+			}
+			written[fragment] = fullpath
+			switch value := value.(type) {
+			case Writer:
+				if err := value(fullpath); err != nil {
+					if errors.Is(err, ErrUnsupported) {
+						t.Skip(err)
+					}
+					t.Fatal(err)
+				}
+			case string:
+				if err := os.WriteFile(fullpath, []byte(value), 0644); err != nil {
+					t.Fatal(err)
+				}
+			default:
+				t.Fatalf("Invalid type %T in files, must be string or Writer", value)
+			}
+		}
+		for fragment, value := range module.Overlay {
+			fullpath := exporter.Filename(exported, module.Name, filepath.FromSlash(fragment))
+			exported.Config.Overlay[fullpath] = value
+		}
+	}
+	if err := exporter.Finalize(exported); err != nil {
+		t.Fatal(err)
+	}
+	testenv.NeedsGoPackagesEnv(t, exported.Config.Env)
+	return exported
+}
+
+// Script returns a Writer that writes out contents to the file and sets the
+// executable bit on the created file.
+// It is intended for source files that are shell scripts.
+func Script(contents string) Writer {
+	return func(filename string) error {
+		return os.WriteFile(filename, []byte(contents), 0755)
+	}
+}
+
+// Link returns a Writer that creates a hard link from the specified source to
+// the required file.
+// This is used to link testdata files into the generated testing tree.
+//
+// If hard links to source are not supported on the destination filesystem, the
+// returned Writer returns an error for which errors.Is(_, ErrUnsupported)
+// returns true.
+func Link(source string) Writer {
+	return func(filename string) error {
+		linkErr := os.Link(source, filename)
+
+		if linkErr != nil && !builderMustSupportLinks() {
+			// Probe to figure out whether Link failed because the Link operation
+			// isn't supported.
+			if stat, err := openAndStat(source); err == nil {
+				if err := createEmpty(filename, stat.Mode()); err == nil {
+					// Successfully opened the source and created the destination,
+					// but the result is empty and not a hard-link.
+					return &os.PathError{Op: "Link", Path: filename, Err: ErrUnsupported}
+				}
+			}
+		}
+
+		return linkErr
+	}
+}
+
+// Symlink returns a Writer that creates a symlink from the specified source to the
+// required file.
+// This is used to link testdata files into the generated testing tree.
+//
+// If symlinks to source are not supported on the destination filesystem, the
+// returned Writer returns an error for which errors.Is(_, ErrUnsupported)
+// returns true.
+func Symlink(source string) Writer {
+	if !strings.HasPrefix(source, ".") {
+		if absSource, err := filepath.Abs(source); err == nil {
+			if _, err := os.Stat(source); !os.IsNotExist(err) {
+				source = absSource
+			}
+		}
+	}
+	return func(filename string) error {
+		symlinkErr := os.Symlink(source, filename)
+
+		if symlinkErr != nil && !builderMustSupportLinks() {
+			// Probe to figure out whether Symlink failed because the Symlink
+			// operation isn't supported.
+			fullSource := source
+			if !filepath.IsAbs(source) {
+				// Compute the target path relative to the parent of filename, not the
+				// current working directory.
+				fullSource = filepath.Join(filename, "..", source)
+			}
+			stat, err := openAndStat(fullSource)
+			mode := os.ModePerm
+			if err == nil {
+				mode = stat.Mode()
+			} else if !errors.Is(err, os.ErrNotExist) {
+				// We couldn't open the source, but it might exist. We don't expect to be
+				// able to portably create a symlink to a file we can't see.
+				return symlinkErr
+			}
+
+			if err := createEmpty(filename, mode|0644); err == nil {
+				// Successfully opened the source (or verified that it does not exist) and
+				// created the destination, but we couldn't create it as a symlink.
+				// Probably the OS just doesn't support symlinks in this context.
+				return &os.PathError{Op: "Symlink", Path: filename, Err: ErrUnsupported}
+			}
+		}
+
+		return symlinkErr
+	}
+}
+
+// builderMustSupportLinks reports whether we are running on a Go builder
+// that is known to support hard and symbolic links.
+func builderMustSupportLinks() bool {
+	if os.Getenv("GO_BUILDER_NAME") == "" {
+		// Any OS can be configured to mount an exotic filesystem.
+		// Don't make assumptions about what users are running.
+		return false
+	}
+
+	switch runtime.GOOS {
+	case "windows", "plan9":
+		// Some versions of Windows and all versions of plan9 do not support
+		// symlinks by default.
+		return false
+
+	default:
+		// All other platforms should support symlinks by default, and our builders
+		// should not do anything unusual that would violate that.
+		return true
+	}
+}
+
+// openAndStat attempts to open source for reading.
+func openAndStat(source string) (os.FileInfo, error) {
+	src, err := os.Open(source)
+	if err != nil {
+		return nil, err
+	}
+	stat, err := src.Stat()
+	src.Close()
+	if err != nil {
+		return nil, err
+	}
+	return stat, nil
+}
+
+// createEmpty creates an empty file or directory (depending on mode)
+// at dst, with the same permissions as mode.
+func createEmpty(dst string, mode os.FileMode) error {
+	if mode.IsDir() {
+		return os.Mkdir(dst, mode.Perm())
+	}
+
+	f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode.Perm())
+	if err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(dst) // best-effort
+		return err
+	}
+
+	return nil
+}
+
+// Copy returns a Writer that copies a file from the specified source to the
+// required file.
+// This is used to copy testdata files into the generated testing tree.
+func Copy(source string) Writer {
+	return func(filename string) error {
+		stat, err := os.Stat(source)
+		if err != nil {
+			return err
+		}
+		if !stat.Mode().IsRegular() {
+			// cannot copy non-regular files (e.g., directories,
+			// symlinks, devices, etc.)
+			return fmt.Errorf("cannot copy non regular file %s", source)
+		}
+		return copyFile(filename, source, stat.Mode().Perm())
+	}
+}
+
+func copyFile(dest, source string, perm os.FileMode) error {
+	src, err := os.Open(source)
+	if err != nil {
+		return err
+	}
+	defer src.Close()
+
+	dst, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(dst, src)
+	if closeErr := dst.Close(); err == nil {
+		err = closeErr
+	}
+	return err
+}
+
+// GroupFilesByModules attempts to map directories to the modules within each directory.
+// This function assumes that the folder is structured in the following way:
+//
+//	dir/
+//		primarymod/
+//			*.go files
+//			packages
+//			go.mod (optional)
+//		modules/
+//			repoa/
+//				mod1/
+//					*.go files
+//					packages
+//					go.mod (optional)
+//
+// It scans the directory tree anchored at root and adds a Copy writer to the
+// map for every file found.
+// This is to enable the common case in tests where you have a full copy of the
+// package in your testdata.
+func GroupFilesByModules(root string) ([]Module, error) {
+	root = filepath.FromSlash(root)
+	primarymodPath := filepath.Join(root, "primarymod")
+
+	_, err := os.Stat(primarymodPath)
+	if os.IsNotExist(err) {
+		return nil, fmt.Errorf("could not find primarymod folder within %s", root)
+	}
+
+	primarymod := &Module{
+		Name:    root,
+		Files:   make(map[string]interface{}),
+		Overlay: make(map[string][]byte),
+	}
+	mods := map[string]*Module{
+		root: primarymod,
+	}
+	modules := []Module{*primarymod}
+
+	if err := filepath.Walk(primarymodPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		fragment, err := filepath.Rel(primarymodPath, path)
+		if err != nil {
+			return err
+		}
+		primarymod.Files[filepath.ToSlash(fragment)] = Copy(path)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	modulesPath := filepath.Join(root, "modules")
+	if _, err := os.Stat(modulesPath); os.IsNotExist(err) {
+		return modules, nil
+	}
+
+	var currentRepo, currentModule string
+	updateCurrentModule := func(dir string) {
+		if dir == currentModule {
+			return
+		}
+		// Handle the case where we step into a nested directory that is a module
+		// and then step out into the parent which is also a module.
+		// Example:
+		// - repoa
+		//   - moda
+		//     - go.mod
+		//     - v2
+		//       - go.mod
+		//     - what.go
+		//   - modb
+		for dir != root {
+			if mods[dir] != nil {
+				currentModule = dir
+				return
+			}
+			dir = filepath.Dir(dir)
+		}
+	}
+
+	if err := filepath.Walk(modulesPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		enclosingDir := filepath.Dir(path)
+		// If the path is not a directory, then we want to add the path to
+		// the files map of the currentModule.
+		if !info.IsDir() {
+			updateCurrentModule(enclosingDir)
+			fragment, err := filepath.Rel(currentModule, path)
+			if err != nil {
+				return err
+			}
+			mods[currentModule].Files[filepath.ToSlash(fragment)] = Copy(path)
+			return nil
+		}
+		// If the path is a directory and it's enclosing folder is equal to
+		// the modules folder, then the path is a new repo.
+		if enclosingDir == modulesPath {
+			currentRepo = path
+			return nil
+		}
+		// If the path is a directory and it's enclosing folder is not the same
+		// as the current repo and it is not of the form `v1`,`v2`,...
+		// then the path is a folder/package of the current module.
+		if enclosingDir != currentRepo && !versionSuffixRE.MatchString(filepath.Base(path)) {
+			return nil
+		}
+		// If the path is a directory and it's enclosing folder is the current repo
+		// then the path is a new module.
+		module, err := filepath.Rel(modulesPath, path)
+		if err != nil {
+			return err
+		}
+		mods[path] = &Module{
+			Name:    filepath.ToSlash(module),
+			Files:   make(map[string]interface{}),
+			Overlay: make(map[string][]byte),
+		}
+		currentModule = path
+		modules = append(modules, *mods[path])
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return modules, nil
+}
+
+// MustCopyFileTree returns a file set for a module based on a real directory tree.
+// It scans the directory tree anchored at root and adds a Copy writer to the
+// map for every file found. It skips copying files in nested modules.
+// This is to enable the common case in tests where you have a full copy of the
+// package in your testdata.
+// This will panic if there is any kind of error trying to walk the file tree.
+func MustCopyFileTree(root string) map[string]interface{} {
+	result := map[string]interface{}{}
+	if err := filepath.Walk(filepath.FromSlash(root), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			// skip nested modules.
+			if path != root {
+				if fi, err := os.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() {
+					return filepath.SkipDir
+				}
+			}
+			return nil
+		}
+		fragment, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		result[filepath.ToSlash(fragment)] = Copy(path)
+		return nil
+	}); err != nil {
+		log.Panic(fmt.Sprintf("MustCopyFileTree failed: %v", err))
+	}
+	return result
+}
+
+// Cleanup removes the temporary directory (unless the --skip-cleanup flag was set)
+// It is safe to call cleanup multiple times.
+func (e *Exported) Cleanup() {
+	if e.temp == "" {
+		return
+	}
+	if *skipCleanup {
+		log.Printf("Skipping cleanup of temp dir: %s", e.temp)
+		return
+	}
+	// Make everything read-write so that the Module exporter's module cache can be deleted.
+	filepath.Walk(e.temp, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if info.IsDir() {
+			os.Chmod(path, 0777)
+		}
+		return nil
+	})
+	os.RemoveAll(e.temp) // ignore errors
+	e.temp = ""
+}
+
+// Temp returns the temporary directory that was generated.
+func (e *Exported) Temp() string {
+	return e.temp
+}
+
+// File returns the full path for the given module and file fragment.
+func (e *Exported) File(module, fragment string) string {
+	if m := e.written[module]; m != nil {
+		return m[fragment]
+	}
+	return ""
+}
+
+// FileContents returns the contents of the specified file.
+// It will use the overlay if the file is present, otherwise it will read it
+// from disk.
+func (e *Exported) FileContents(filename string) ([]byte, error) {
+	if content, found := e.Config.Overlay[filename]; found {
+		return content, nil
+	}
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return content, nil
+}

internal/packagestest/export_test.go

@@ -0,0 +1,234 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packagestest_test
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"testing"
+
+	"golang.org/x/tools/internal/packagestest"
+)
+
+var testdata = []packagestest.Module{{
+	Name: "golang.org/fake1",
+	Files: map[string]interface{}{
+		"a.go": packagestest.Symlink("testdata/a.go"), // broken symlink
+		"b.go": "invalid file contents",
+	},
+	Overlay: map[string][]byte{
+		"b.go": []byte("package fake1"),
+		"c.go": []byte("package fake1"),
+	},
+}, {
+	Name: "golang.org/fake2",
+	Files: map[string]interface{}{
+		"other/a.go": "package fake2",
+	},
+}, {
+	Name: "golang.org/fake2/v2",
+	Files: map[string]interface{}{
+		"other/a.go": "package fake2",
+	},
+}, {
+	Name: "golang.org/fake3@v1.0.0",
+	Files: map[string]interface{}{
+		"other/a.go": "package fake3",
+	},
+}, {
+	Name: "golang.org/fake3@v1.1.0",
+	Files: map[string]interface{}{
+		"other/a.go": "package fake3",
+	},
+}}
+
+type fileTest struct {
+	module, fragment, expect string
+	check                    func(t *testing.T, exported *packagestest.Exported, filename string)
+}
+
+func checkFiles(t *testing.T, exported *packagestest.Exported, tests []fileTest) {
+	for _, test := range tests {
+		expect := filepath.Join(exported.Temp(), filepath.FromSlash(test.expect))
+		got := exported.File(test.module, test.fragment)
+		if got == "" {
+			t.Errorf("File %v missing from the output", expect)
+		} else if got != expect {
+			t.Errorf("Got file %v, expected %v", got, expect)
+		}
+		if test.check != nil {
+			test.check(t, exported, got)
+		}
+	}
+}
+
+func checkLink(expect string) func(t *testing.T, exported *packagestest.Exported, filename string) {
+	expect = filepath.FromSlash(expect)
+	return func(t *testing.T, exported *packagestest.Exported, filename string) {
+		if target, err := os.Readlink(filename); err != nil {
+			t.Errorf("Error checking link %v: %v", filename, err)
+		} else if target != expect {
+			t.Errorf("Link %v does not match, got %v expected %v", filename, target, expect)
+		}
+	}
+}
+
+func checkContent(expect string) func(t *testing.T, exported *packagestest.Exported, filename string) {
+	return func(t *testing.T, exported *packagestest.Exported, filename string) {
+		if content, err := exported.FileContents(filename); err != nil {
+			t.Errorf("Error reading %v: %v", filename, err)
+		} else if string(content) != expect {
+			t.Errorf("Content of %v does not match, got %v expected %v", filename, string(content), expect)
+		}
+	}
+}
+
+func TestGroupFilesByModules(t *testing.T) {
+	for _, tt := range []struct {
+		testdir string
+		want    []packagestest.Module
+	}{
+		{
+			testdir: "testdata/groups/one",
+			want: []packagestest.Module{
+				{
+					Name: "testdata/groups/one",
+					Files: map[string]interface{}{
+						"main.go": true,
+					},
+				},
+				{
+					Name: "example.com/extra",
+					Files: map[string]interface{}{
+						"help.go": true,
+					},
+				},
+			},
+		},
+		{
+			testdir: "testdata/groups/two",
+			want: []packagestest.Module{
+				{
+					Name: "testdata/groups/two",
+					Files: map[string]interface{}{
+						"main.go":           true,
+						"expect/yo.go":      true,
+						"expect/yo_test.go": true,
+					},
+				},
+				{
+					Name: "example.com/extra",
+					Files: map[string]interface{}{
+						"yo.go":        true,
+						"geez/help.go": true,
+					},
+				},
+				{
+					Name: "example.com/extra/v2",
+					Files: map[string]interface{}{
+						"me.go":        true,
+						"geez/help.go": true,
+					},
+				},
+				{
+					Name: "example.com/tempmod",
+					Files: map[string]interface{}{
+						"main.go": true,
+					},
+				},
+				{
+					Name: "example.com/what@v1.0.0",
+					Files: map[string]interface{}{
+						"main.go": true,
+					},
+				},
+				{
+					Name: "example.com/what@v1.1.0",
+					Files: map[string]interface{}{
+						"main.go": true,
+					},
+				},
+			},
+		},
+	} {
+		t.Run(tt.testdir, func(t *testing.T) {
+			got, err := packagestest.GroupFilesByModules(tt.testdir)
+			if err != nil {
+				t.Fatalf("could not group files %v", err)
+			}
+			if len(got) != len(tt.want) {
+				t.Fatalf("%s: wanted %d modules but got %d", tt.testdir, len(tt.want), len(got))
+			}
+			for i, w := range tt.want {
+				g := got[i]
+				if filepath.FromSlash(g.Name) != filepath.FromSlash(w.Name) {
+					t.Fatalf("%s: wanted module[%d].Name to be %s but got %s", tt.testdir, i, filepath.FromSlash(w.Name), filepath.FromSlash(g.Name))
+				}
+				for fh := range w.Files {
+					if _, ok := g.Files[fh]; !ok {
+						t.Fatalf("%s, module[%d]: wanted %s but could not find", tt.testdir, i, fh)
+					}
+				}
+				for fh := range g.Files {
+					if _, ok := w.Files[fh]; !ok {
+						t.Fatalf("%s, module[%d]: found unexpected file %s", tt.testdir, i, fh)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestMustCopyFiles(t *testing.T) {
+	// Create the following test directory structure in a temporary directory.
+	src := map[string]string{
+		// copies all files under the specified directory.
+		"go.mod": "module example.com",
+		"m.go":   "package m",
+		"a/a.go": "package a",
+		// contents from a nested module shouldn't be copied.
+		"nested/go.mod": "module example.com/nested",
+		"nested/m.go":   "package nested",
+		"nested/b/b.go": "package b",
+	}
+
+	tmpDir, err := os.MkdirTemp("", t.Name())
+	if err != nil {
+		t.Fatalf("failed to create a temporary directory: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	for fragment, contents := range src {
+		fullpath := filepath.Join(tmpDir, filepath.FromSlash(fragment))
+		if err := os.MkdirAll(filepath.Dir(fullpath), 0755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(fullpath, []byte(contents), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	copied := packagestest.MustCopyFileTree(tmpDir)
+	var got []string
+	for fragment := range copied {
+		got = append(got, filepath.ToSlash(fragment))
+	}
+	want := []string{"go.mod", "m.go", "a/a.go"}
+
+	sort.Strings(got)
+	sort.Strings(want)
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("packagestest.MustCopyFileTree = %v, want %v", got, want)
+	}
+
+	// packagestest.Export is happy.
+	exported := packagestest.Export(t, packagestest.Modules, []packagestest.Module{{
+		Name:  "example.com",
+		Files: packagestest.MustCopyFileTree(tmpDir),
+	}})
+	defer exported.Cleanup()
+}

internal/packagestest/gopath.go

@@ -0,0 +1,77 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packagestest
+
+import (
+	"path"
+	"path/filepath"
+)
+
+// GOPATH is the exporter that produces GOPATH layouts.
+// Each "module" is put in it's own GOPATH entry to help test complex cases.
+// Given the two files
+//
+//	golang.org/repoa#a/a.go
+//	golang.org/repob#b/b.go
+//
+// You would get the directory layout
+//
+//	/sometemporarydirectory
+//	├── repoa
+//	│   └── src
+//	│       └── golang.org
+//	│           └── repoa
+//	│               └── a
+//	│                   └── a.go
+//	└── repob
+//	    └── src
+//	        └── golang.org
+//	            └── repob
+//	                └── b
+//	                    └── b.go
+//
+// GOPATH would be set to
+//
+//	/sometemporarydirectory/repoa;/sometemporarydirectory/repob
+//
+// and the working directory would be
+//
+//	/sometemporarydirectory/repoa/src
+var GOPATH = gopath{}
+
+type gopath struct{}
+
+func (gopath) Name() string {
+	return "GOPATH"
+}
+
+func (gopath) Filename(exported *Exported, module, fragment string) string {
+	return filepath.Join(gopathDir(exported, module), "src", module, fragment)
+}
+
+func (gopath) Finalize(exported *Exported) error {
+	exported.Config.Env = append(exported.Config.Env, "GO111MODULE=off")
+	gopath := ""
+	for module := range exported.written {
+		if gopath != "" {
+			gopath += string(filepath.ListSeparator)
+		}
+		dir := gopathDir(exported, module)
+		gopath += dir
+		if module == exported.primary {
+			exported.Config.Dir = filepath.Join(dir, "src")
+		}
+	}
+	exported.Config.Env = append(exported.Config.Env, "GOPATH="+gopath)
+	return nil
+}
+
+func gopathDir(exported *Exported, module string) string {
+	dir := path.Base(module)
+	if versionSuffixRE.MatchString(dir) {
+		dir = path.Base(path.Dir(module)) + "_" + dir
+	}
+	return filepath.Join(exported.temp, dir)
+}

internal/packagestest/gopath_test.go

@@ -0,0 +1,28 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packagestest_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"golang.org/x/tools/internal/packagestest"
+)
+
+func TestGOPATHExport(t *testing.T) {
+	exported := packagestest.Export(t, packagestest.GOPATH, testdata)
+	defer exported.Cleanup()
+	// Check that the cfg contains all the right bits
+	var expectDir = filepath.Join(exported.Temp(), "fake1", "src")
+	if exported.Config.Dir != expectDir {
+		t.Errorf("Got working directory %v expected %v", exported.Config.Dir, expectDir)
+	}
+	checkFiles(t, exported, []fileTest{
+		{"golang.org/fake1", "a.go", "fake1/src/golang.org/fake1/a.go", checkLink("testdata/a.go")},
+		{"golang.org/fake1", "b.go", "fake1/src/golang.org/fake1/b.go", checkContent("package fake1")},
+		{"golang.org/fake2", "other/a.go", "fake2/src/golang.org/fake2/other/a.go", checkContent("package fake2")},
+		{"golang.org/fake2/v2", "other/a.go", "fake2_v2/src/golang.org/fake2/v2/other/a.go", checkContent("package fake2")},
+	})
+}

internal/packagestest/modules.go

@@ -0,0 +1,223 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packagestest
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"golang.org/x/tools/internal/gocommand"
+	"golang.org/x/tools/internal/proxydir"
+)
+
+// Modules is the exporter that produces module layouts.
+// Each "repository" is put in its own module, and the module file generated
+// will have replace directives for all other modules.
+// Given the two files
+//
+//	golang.org/repoa#a/a.go
+//	golang.org/repob#b/b.go
+//
+// You would get the directory layout
+//
+//	/sometemporarydirectory
+//	├── repoa
+//	│   ├── a
+//	│   │   └── a.go
+//	│   └── go.mod
+//	└── repob
+//	    ├── b
+//	    │   └── b.go
+//	    └── go.mod
+//
+// and the working directory would be
+//
+//	/sometemporarydirectory/repoa
+var Modules = modules{}
+
+type modules struct{}
+
+type moduleAtVersion struct {
+	module  string
+	version string
+}
+
+func (modules) Name() string {
+	return "Modules"
+}
+
+func (modules) Filename(exported *Exported, module, fragment string) string {
+	if module == exported.primary {
+		return filepath.Join(primaryDir(exported), fragment)
+	}
+	return filepath.Join(moduleDir(exported, module), fragment)
+}
+
+func (modules) Finalize(exported *Exported) error {
+	// Write out the primary module. This module can use symlinks and
+	// other weird stuff, and will be the working dir for the go command.
+	// It depends on all the other modules.
+	primaryDir := primaryDir(exported)
+	if err := os.MkdirAll(primaryDir, 0755); err != nil {
+		return err
+	}
+	exported.Config.Dir = primaryDir
+	if exported.written[exported.primary] == nil {
+		exported.written[exported.primary] = make(map[string]string)
+	}
+
+	// Create a map of modulepath -> {module, version} for modulepaths
+	// that are of the form `repoa/mod1@v1.1.0`.
+	versions := make(map[string]moduleAtVersion)
+	for module := range exported.written {
+		if splt := strings.Split(module, "@"); len(splt) > 1 {
+			versions[module] = moduleAtVersion{
+				module:  splt[0],
+				version: splt[1],
+			}
+		}
+	}
+
+	// If the primary module already has a go.mod, write the contents to a temp
+	// go.mod for now and then we will reset it when we are getting all the markers.
+	if gomod := exported.written[exported.primary]["go.mod"]; gomod != "" {
+		contents, err := os.ReadFile(gomod)
+		if err != nil {
+			return err
+		}
+		if err := os.WriteFile(gomod+".temp", contents, 0644); err != nil {
+			return err
+		}
+	}
+
+	exported.written[exported.primary]["go.mod"] = filepath.Join(primaryDir, "go.mod")
+	var primaryGomod bytes.Buffer
+	fmt.Fprintf(&primaryGomod, "module %s\nrequire (\n", exported.primary)
+	for other := range exported.written {
+		if other == exported.primary {
+			continue
+		}
+		version := moduleVersion(other)
+		// If other is of the form `repo1/mod1@v1.1.0`,
+		// then we need to extract the module and the version.
+		if v, ok := versions[other]; ok {
+			other = v.module
+			version = v.version
+		}
+		fmt.Fprintf(&primaryGomod, "\t%v %v\n", other, version)
+	}
+	fmt.Fprintf(&primaryGomod, ")\n")
+	if err := os.WriteFile(filepath.Join(primaryDir, "go.mod"), primaryGomod.Bytes(), 0644); err != nil {
+		return err
+	}
+
+	// Create the mod cache so we can rename it later, even if we don't need it.
+	if err := os.MkdirAll(modCache(exported), 0755); err != nil {
+		return err
+	}
+
+	// Write out the go.mod files for the other modules.
+	for module, files := range exported.written {
+		if module == exported.primary {
+			continue
+		}
+		dir := moduleDir(exported, module)
+		modfile := filepath.Join(dir, "go.mod")
+		// If other is of the form `repo1/mod1@v1.1.0`,
+		// then we need to extract the module name without the version.
+		if v, ok := versions[module]; ok {
+			module = v.module
+		}
+		if err := os.WriteFile(modfile, []byte("module "+module+"\n"), 0644); err != nil {
+			return err
+		}
+		files["go.mod"] = modfile
+	}
+
+	// Zip up all the secondary modules into the proxy dir.
+	modProxyDir := filepath.Join(exported.temp, "modproxy")
+	for module, files := range exported.written {
+		if module == exported.primary {
+			continue
+		}
+		version := moduleVersion(module)
+		// If other is of the form `repo1/mod1@v1.1.0`,
+		// then we need to extract the module and the version.
+		if v, ok := versions[module]; ok {
+			module = v.module
+			version = v.version
+		}
+		if err := writeModuleFiles(modProxyDir, module, version, files); err != nil {
+			return fmt.Errorf("creating module proxy dir for %v: %v", module, err)
+		}
+	}
+
+	// Discard the original mod cache dir, which contained the files written
+	// for us by Export.
+	if err := os.Rename(modCache(exported), modCache(exported)+".orig"); err != nil {
+		return err
+	}
+	exported.Config.Env = append(exported.Config.Env,
+		"GO111MODULE=on",
+		"GOPATH="+filepath.Join(exported.temp, "modcache"),
+		"GOMODCACHE=",
+		"GOPROXY="+proxydir.ToURL(modProxyDir),
+		"GOSUMDB=off",
+	)
+
+	// Run go mod download to recreate the mod cache dir with all the extra
+	// stuff in cache. All the files created by Export should be recreated.
+	inv := gocommand.Invocation{
+		Verb:       "mod",
+		Args:       []string{"download", "all"},
+		Env:        exported.Config.Env,
+		BuildFlags: exported.Config.BuildFlags,
+		WorkingDir: exported.Config.Dir,
+	}
+	_, err := new(gocommand.Runner).Run(context.Background(), inv)
+	return err
+}
+
+func writeModuleFiles(rootDir, module, ver string, filePaths map[string]string) error {
+	fileData := make(map[string][]byte)
+	for name, path := range filePaths {
+		contents, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		fileData[name] = contents
+	}
+	return proxydir.WriteModuleVersion(rootDir, module, ver, fileData)
+}
+
+func modCache(exported *Exported) string {
+	return filepath.Join(exported.temp, "modcache/pkg/mod")
+}
+
+func primaryDir(exported *Exported) string {
+	return filepath.Join(exported.temp, path.Base(exported.primary))
+}
+
+func moduleDir(exported *Exported, module string) string {
+	if strings.Contains(module, "@") {
+		return filepath.Join(modCache(exported), module)
+	}
+	return filepath.Join(modCache(exported), path.Dir(module), path.Base(module)+"@"+moduleVersion(module))
+}
+
+var versionSuffixRE = regexp.MustCompile(`v\d+`)
+
+func moduleVersion(module string) string {
+	if versionSuffixRE.MatchString(path.Base(module)) {
+		return path.Base(module) + ".0.0"
+	}
+	return "v1.0.0"
+}

internal/packagestest/modules_test.go

@@ -0,0 +1,32 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packagestest_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"golang.org/x/tools/internal/packagestest"
+)
+
+func TestModulesExport(t *testing.T) {
+	exported := packagestest.Export(t, packagestest.Modules, testdata)
+	defer exported.Cleanup()
+	// Check that the cfg contains all the right bits
+	var expectDir = filepath.Join(exported.Temp(), "fake1")
+	if exported.Config.Dir != expectDir {
+		t.Errorf("Got working directory %v expected %v", exported.Config.Dir, expectDir)
+	}
+	checkFiles(t, exported, []fileTest{
+		{"golang.org/fake1", "go.mod", "fake1/go.mod", nil},
+		{"golang.org/fake1", "a.go", "fake1/a.go", checkLink("testdata/a.go")},
+		{"golang.org/fake1", "b.go", "fake1/b.go", checkContent("package fake1")},
+		{"golang.org/fake2", "go.mod", "modcache/pkg/mod/golang.org/fake2@v1.0.0/go.mod", nil},
+		{"golang.org/fake2", "other/a.go", "modcache/pkg/mod/golang.org/fake2@v1.0.0/other/a.go", checkContent("package fake2")},
+		{"golang.org/fake2/v2", "other/a.go", "modcache/pkg/mod/golang.org/fake2/v2@v2.0.0/other/a.go", checkContent("package fake2")},
+		{"golang.org/fake3@v1.1.0", "other/a.go", "modcache/pkg/mod/golang.org/fake3@v1.1.0/other/a.go", checkContent("package fake3")},
+		{"golang.org/fake3@v1.0.0", "other/a.go", "modcache/pkg/mod/golang.org/fake3@v1.0.0/other/a.go", nil},
+	})
+}

internal/packagestest/testdata/groups/one/modules/example.com/extra/help.go

@@ -0,0 +1 @@
+package extra

internal/packagestest/testdata/groups/one/primarymod/main.go

@@ -0,0 +1 @@
+package one

internal/packagestest/testdata/groups/two/modules/example.com/extra/geez/help.go

@@ -0,0 +1 @@
+package example.com/extra/geez

internal/packagestest/testdata/groups/two/modules/example.com/extra/v2/geez/help.go

@@ -0,0 +1 @@
+package example.com/extra/geez

internal/packagestest/testdata/groups/two/modules/example.com/extra/v2/me.go

@@ -0,0 +1 @@
+package example.com/extra

internal/packagestest/testdata/groups/two/modules/example.com/extra/yo.go

@@ -0,0 +1 @@
+package example.com/extra

internal/packagestest/testdata/groups/two/modules/example.com/tempmod/main.go

@@ -0,0 +1 @@
+package example.com/tempmod

internal/packagestest/testdata/groups/two/modules/example.com/what@v1.0.0/main.go

@@ -0,0 +1 @@
+package example.com/what

internal/packagestest/testdata/groups/two/modules/example.com/what@v1.1.0/main.go

@@ -0,0 +1 @@
+package example.com/what

internal/packagestest/testdata/groups/two/primarymod/expect/yo.go

@@ -0,0 +1,3 @@
+package expect
+
+var X int //@check("X", "X")

internal/packagestest/testdata/groups/two/primarymod/expect/yo_test.go

@@ -0,0 +1,10 @@
+package expect_test
+
+import (
+	"testdata/groups/two/expect"
+	"testing"
+)
+
+func TestX(t *testing.T) {
+	_ = expect.X //@check("X", "X")
+}

internal/packagestest/testdata/groups/two/primarymod/main.go

@@ -0,0 +1 @@
+package two

internal/packagestest/testdata/test.go

@@ -0,0 +1,24 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fake1
+
+// This is a test file for the behaviors in Exported.Expect.
+
+type AThing string //@AThing,mark(StringThing, "AThing"),mark(REThing,re`.T.*g`)
+
+type Match string //@check("Match",re`[[:upper:]]`)
+
+//@check(AThing, StringThing)
+//@check(AThing, REThing)
+
+//@boolArg(true, false)
+//@intArg(42)
+//@stringArg(PlainString, "PlainString")
+//@stringArg(IdentAsString,IdentAsString)
+//@directNote()
+//@range(AThing)
+
+// The following test should remain at the bottom of the file
+//@checkEOF(EOF)

internal/packagestest/testdata/test_test.go

@@ -0,0 +1,3 @@
+package fake1
+
+type ATestType string //@check("ATestType","ATestType")

internal/packagestest/testdata/x_test.go

@@ -0,0 +1,3 @@
+package fake1_test
+
+type AnXTestType string //@check("AnXTestType","AnXTestType")

refactor/importgraph/graph_test.go

@@ -17,7 +17,7 @@ import (
 	"strings"
 	"testing"
 
-	"golang.org/x/tools/go/packages/packagestest"
+	"golang.org/x/tools/internal/packagestest"
 	"golang.org/x/tools/refactor/importgraph"
 
 	_ "crypto/hmac" // just for test, below