go/packages: support pre go1.10.4 using go/loader

If go list fails because it doesn't support the new flags added
for Go 1.11 (which will be also released in Go 1.10.4) try again
using the Loader to approximate the packages requested.

This implementation is incomplete. It will never support test packages
because of the two phase test loading process of the Loader. It
also doesn't reliably have access to export data so it will
always do an upgraded whole-program query. But we'll try to get
the best level of support we can for the go/packages interface
given the limitations of the loader.

Once Go 1.12 is released, we'll delete this support. By then, most
Go users should have at least switched to Go 1.10.4.

Change-Id: I5248e20980032695a86b052caa9ff368ecf7b142
Reviewed-on: https://go-review.googlesource.com/125616
Reviewed-by: Ian Cottrell <iancottrell@google.com>

go/packages/loader_fallback.go

@@ -0,0 +1,123 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// TODO(matloob): Delete this file once Go 1.12 is released.
+
+// This file provides backwards compatibility support for
+// loading for versions of Go earlier than 1.10.4. This support is meant to
+// assist with migration to the Package API until there's
+// widespread adoption of these newer Go versions.
+// This support will be removed once Go 1.12 is released
+// in Q1 2019.
+
+// The support is incomplete. These are some of the missing
+// features:
+// - the Tests option has no behavior, and test packages are
+//   never returned.
+// - Package.OtherFiles are always missing even if the package
+//   contains non-go sources.
+
+package packages
+
+import (
+	"fmt"
+	"go/build"
+	legacy "golang.org/x/tools/go/loader"
+	"golang.org/x/tools/imports"
+	"strings"
+)
+
+func loaderFallback(dir string, env []string, patterns []string) ([]*Package, error) {
+	cfg := legacy.Config{}
+	cfg.Cwd = dir
+	cfg.AllowErrors = true
+	cfg.FromArgs(patterns, false) // test packages are not supported
+
+	// Set build ctx
+	buildCtx := build.Default
+	for _, ev := range env {
+		sp := strings.Split(ev, "=")
+		if len(sp) != 2 {
+			continue
+		}
+		evar, val := sp[0], sp[1]
+		switch evar {
+		case "GOPATH":
+			buildCtx.GOPATH = val
+		case "GOROOT":
+			buildCtx.GOROOT = val
+		case "GOARCH":
+			buildCtx.GOARCH = val
+		case "GOOS":
+			buildCtx.GOOS = val
+		}
+	}
+	cfg.Build = &buildCtx
+
+	lprog, err := cfg.Load()
+	if err != nil {
+		if err.Error() == "no initial packages were loaded" {
+			return nil, fmt.Errorf("packages not found") // Return same error as golist-based code
+		}
+		return nil, fmt.Errorf("failed to load packages with legacy loader: %v", err)
+	}
+
+	allpkgs := make(map[string]*loaderPackage)
+
+	initial := make(map[*legacy.PackageInfo]bool)
+	for _, lpkg := range lprog.InitialPackages() {
+		initial[lpkg] = true
+	}
+	for _, lpkg := range lprog.AllPackages {
+		id := lpkg.Pkg.Path()
+
+		var goFiles []string
+		for _, f := range lpkg.Files {
+			goFiles = append(goFiles, lprog.Fset.File(f.Pos()).Name())
+		}
+
+		pkgimports := make(map[string]string)
+		for _, imppkg := range lpkg.Pkg.Imports() {
+			// TODO(matloob): Is the import path of a package always VendorlessPath(path)?
+			pkgimports[imports.VendorlessPath(imppkg.Path())] = imppkg.Path()
+		}
+
+		allpkgs[id] = &loaderPackage{
+			Package: &Package{
+				ID:         id,
+				Name:       lpkg.Pkg.Name(),
+				GoFiles:    goFiles,
+				Fset:       lprog.Fset,
+				Syntax:     lpkg.Files,
+				Errors:     lpkg.Errors,
+				Types:      lpkg.Pkg,
+				TypesInfo:  &lpkg.Info,
+				IllTyped:   !lpkg.TransitivelyErrorFree,
+				OtherFiles: nil, // Never set for the fallback, because we can't extract from loader.
+			},
+			imports: pkgimports,
+		}
+	}
+
+	// Do a second pass to populate imports.
+	for _, pkg := range allpkgs {
+		pkg.Imports = make(map[string]*Package)
+		for imppath, impid := range pkg.imports {
+			target, ok := allpkgs[impid]
+			if !ok {
+				// return nil, fmt.Errorf("could not load package: %v", impid)
+				continue
+			}
+			pkg.Imports[imppath] = target.Package
+		}
+	}
+
+	// Grab the initial set of packages.
+	var packages []*Package
+	for _, lpkg := range lprog.InitialPackages() {
+		packages = append(packages, allpkgs[lpkg.Pkg.Path()].Package)
+	}
+
+	return packages, nil
+}

go/packages/packages.go

@@ -269,6 +269,9 @@ func (ld *loader) load(patterns ...string) ([]*Package, error) {
 	export := ld.Mode > LoadImports && ld.Mode < LoadAllSyntax
 	deps := ld.Mode >= LoadImports
 	list, err := golistPackages(ld.Context, ld.Dir, ld.Env, export, ld.Tests, deps, patterns)
+	if _, ok := err.(GoTooOldError); ok {
+		return loaderFallback(ld.Dir, ld.Env, patterns)
+	}
 	if err != nil {
 		return nil, err
 	}

go/packages/packages110_test.go

@@ -6,16 +6,6 @@
 
 package packages_test
 
-import (
-	"testing"
-
-	"golang.org/x/tools/go/packages"
-)
-
-func TestGoIsTooOld(t *testing.T) {
-	_, err := packages.Load(nil, "errors")
-
-	if _, ok := err.(packages.GoTooOldError); !ok {
-		t.Fatalf("using go/packages with pre-Go 1.11 go: err=%v, want ErrGoTooOld", err)
-	}
+func init() {
+	usesLegacyLoader = true
 }

go/packages/packages_test.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build go1.11
-
 package packages_test
 
 import (
@@ -26,6 +24,10 @@ import (
 	"golang.org/x/tools/go/packages"
 )
 
+// TODO(matloob): remove this once Go 1.12 is released as we will end support
+// for the loader-backed implementation then.
+var usesLegacyLoader = false
+
 // TODO(adonovan): more test cases to write:
 //
 // - When the tests fail, make them print a 'cd & load' command
@@ -131,7 +133,7 @@ func TestMetadataImportGraph(t *testing.T) {
   subdir/d_test [subdir/d.test] -> subdir/d [subdir/d.test]
 `[1:]
 
-	if graph != wantGraph {
+	if graph != wantGraph && !usesLegacyLoader {
 		t.Errorf("wrong import graph: got <<%s>>, want <<%s>>", graph, wantGraph)
 	}
 
@@ -151,6 +153,10 @@ func TestMetadataImportGraph(t *testing.T) {
 		{"subdir/d.test", "main", "command", "0.go"},
 		{"unsafe", "unsafe", "package", ""},
 	} {
+		if usesLegacyLoader && test.id == "subdir/d.test" {
+			// Legacy Loader does not support tests.
+			continue
+		}
 		p, ok := all[test.id]
 		if !ok {
 			t.Errorf("no package %s", test.id)
@@ -181,11 +187,16 @@ func TestMetadataImportGraph(t *testing.T) {
 		t.Errorf("failed to obtain metadata for ad-hoc package: %s", err)
 	} else {
 		got := fmt.Sprintf("%s %s", initial[0].ID, srcs(initial[0]))
-		if want := "command-line-arguments [c.go]"; got != want {
+		if want := "command-line-arguments [c.go]"; got != want && !usesLegacyLoader {
 			t.Errorf("oops: got %s, want %s", got, want)
 		}
 	}
 
+	if usesLegacyLoader {
+		// TODO(matloob): Wildcards are not yet supported.
+		return
+	}
+
 	// Wildcards
 	// See StdlibTest for effective test of "std" wildcard.
 	// TODO(adonovan): test "all" returns everything in the current module.
@@ -202,7 +213,7 @@ func TestMetadataImportGraph(t *testing.T) {
 	}
 }
 
-func TestOptionsDir(t *testing.T) {
+func TestOptionsDir_Go110(t *testing.T) {
 	tmp, cleanup := makeTree(t, map[string]string{
 		"src/a/a.go":   `package a; const Name = "a" `,
 		"src/a/b/b.go": `package b; const Name = "a/b"`,
@@ -260,7 +271,7 @@ func (ec *errCollector) add(err error) {
 	ec.mu.Unlock()
 }
 
-func TestTypeCheckOK(t *testing.T) {
+func TestTypeCheckOK_Go110(t *testing.T) {
 	tmp, cleanup := makeTree(t, map[string]string{
 		"src/a/a.go": `package a; import "b"; const A = "a" + b.B`,
 		"src/b/b.go": `package b; import "c"; const B = "b" + c.C`,
@@ -296,6 +307,10 @@ func TestTypeCheckOK(t *testing.T) {
 		t.Errorf("wrong import graph: got <<%s>>, want <<%s>>", graph, wantGraph)
 	}
 
+	// TODO(matloob): The go/loader based support loads everything from source
+	// because it doesn't do a build and the .a files don't exist.
+	// Can we simulate its existance?
+
 	for _, test := range []struct {
 		id         string
 		wantType   bool
@@ -307,6 +322,10 @@ func TestTypeCheckOK(t *testing.T) {
 		{"d", true, false},  // export data package
 		{"e", false, false}, // no package
 	} {
+		if usesLegacyLoader && test.id == "d" || test.id == "e" {
+			// legacyLoader always does a whole-program load.
+			continue
+		}
 		p := all[test.id]
 		if p == nil {
 			t.Errorf("missing package: %s", test.id)
@@ -392,6 +411,11 @@ func TestTypeCheckError(t *testing.T) {
 		{"d", false, false, true, nil},  // missing export data
 		{"e", false, false, false, nil}, // type info not requested (despite type error)
 	} {
+		if usesLegacyLoader && test.id == "c" || test.id == "d" || test.id == "e" {
+			// Behavior is different for legacy loader because it always loads wholeProgram.
+			// TODO(matloob): can we run more of this test? Can we put export data into the test GOPATH?
+			continue
+		}
 		p := all[test.id]
 		if p == nil {
 			t.Errorf("missing package: %s", test.id)
@@ -429,6 +453,10 @@ func TestTypeCheckError(t *testing.T) {
 // This function tests use of the ParseFile hook to supply
 // alternative file contents to the parser and type-checker.
 func TestWholeProgramOverlay(t *testing.T) {
+	if usesLegacyLoader {
+		t.Skip("not yet supported in go/loader based implementation")
+	}
+
 	type M = map[string]string
 
 	tmp, cleanup := makeTree(t, M{
@@ -489,6 +517,10 @@ func TestWholeProgramOverlay(t *testing.T) {
 }
 
 func TestWholeProgramImportErrors(t *testing.T) {
+	if usesLegacyLoader {
+		t.Skip("not yet supported in go/loader based implementation")
+	}
+
 	tmp, cleanup := makeTree(t, map[string]string{
 		"src/unicycle/unicycle.go": `package unicycle; import _ "unicycle"`,
 		"src/bicycle1/bicycle1.go": `package bicycle1; import _ "bicycle2"`,