internal/etl: support several build contexts

When reading package files, try multiple GOOS/GOARCH combinations to see if the some set of build constraints results in a non-empty package. The build contexts we try are the same as those used by godoc.org. See af0f2af807/doc/builder.go (L464-L470). Fixes b/132621615. Change-Id: I44ec87469394392561086ef93d3dced5075c5a9d Reviewed-on: https://team-review.git.corp.google.com/c/golang/discovery/+/553144 Reviewed-by: Robert Findley <rfindley@google.com>
2019-09-17 18:09:58 -04:00 · 2019-09-17 18:09:58 -04:00 · 0ae71ff38d
--- a/internal/etl/fetch.go
+++ b/internal/etl/fetch.go
@ -421,11 +421,37 @@ type BadPackageError struct {

 func (bpe *BadPackageError) Error() string { return bpe.Err.Error() }

-// loadPackage loads a Go package made of .go files in zipGoFiles
-// using the default build context. modulePath is stdlib.ModulePath for the
-// Go standard library and the module path for all other modules.
-// innerPath is the path of the Go package directory relative to
-// the module root.
+// Go environments used to construct build contexts in loadPackage.
+var goEnvs = []struct{ GOOS, GOARCH string }{
+	{"linux", "amd64"},
+	{"windows", "amd64"},
+	{"darwin", "amd64"},
+	{"js", "wasm"},
+	{"linux", "js"},
+}
+
+// loadPackage loads a Go package by calling loadPackageWithBuildContext, trying
+// several build contexts in turn. The first build context in the list to produce
+// a non-empty package is used. If none of them result in a package, then
+// loadPackage returns nil, nil.
+func loadPackage(zipGoFiles []*zip.File, innerPath, modulePath string) (*internal.Package, error) {
+	for _, env := range goEnvs {
+		pkg, err := loadPackageWithBuildContext(env.GOOS, env.GOARCH, zipGoFiles, innerPath, modulePath)
+		if err != nil {
+			return nil, err
+		}
+		if pkg != nil {
+			return pkg, nil
+		}
+	}
+	return nil, nil
+}
+
+// loadPackageWithBuildContext loads a Go package made of .go files in zipGoFiles
+// using a build context constructed from the given GOOS and GOARCH values.
+// modulePath is stdlib.ModulePath for the Go standard library and the module
+// path for all other modules. innerPath is the path of the Go package directory
+// relative to the module root.
 //
 // zipGoFiles must contain only .go files that have been verified
 // to be of reasonable size.
@ -436,7 +462,7 @@ func (bpe *BadPackageError) Error() string { return bpe.Err.Error() }
 // or all .go files have been excluded by constraints.
 // A *BadPackageError error is returned if the directory
 // contains .go files but do not make up a valid package.
-func loadPackage(zipGoFiles []*zip.File, innerPath, modulePath string) (*internal.Package, error) {
+func loadPackageWithBuildContext(goos, goarch string, zipGoFiles []*zip.File, innerPath, modulePath string) (*internal.Package, error) {
 	var (
 		// files is a map of file names to their contents.
 		//
@ -475,8 +501,8 @@ func loadPackage(zipGoFiles []*zip.File, innerPath, modulePath string) (*interna
 	// bctx is the build context. It's used to make decisions about which
 	// of the .go files are included or excluded by build constraints.
 	bctx := &build.Context{
-		GOOS:        "linux",
-		GOARCH:      "amd64",
+		GOOS:        goos,
+		GOARCH:      goarch,
 		CgoEnabled:  true,
 		Compiler:    build.Default.Compiler,
 		ReleaseTags: build.Default.ReleaseTags,
--- a/internal/etl/fetch_test.go
+++ b/internal/etl/fetch_test.go
@ -184,50 +184,90 @@ func TestFetchVersion(t *testing.T) {

 	modulePath := "github.com/my/module"
 	version := "v1.0.0"
-	contents := map[string]string{
-		"README.md":  "THIS IS A README",
-		"foo/foo.go": "// package foo exports a helpful constant.\npackage foo\nimport \"net/http\"\nconst OK = http.StatusOK",
-		"LICENSE.md": testhelper.MITLicense,
+	wantVersionInfo := internal.VersionInfo{
+		ModulePath:     "github.com/my/module",
+		Version:        "v1.0.0",
+		CommitTime:     testProxyCommitTime,
+		ReadmeFilePath: "README.md",
+		ReadmeContents: []byte("THIS IS A README"),
+		VersionType:    internal.VersionTypeRelease,
+		RepositoryURL:  "https://github.com/my/module",
 	}
-	want := &internal.Version{
-		VersionInfo: internal.VersionInfo{
-			ModulePath:     "github.com/my/module",
-			Version:        "v1.0.0",
-			CommitTime:     testProxyCommitTime,
-			ReadmeFilePath: "README.md",
-			ReadmeContents: []byte("THIS IS A README"),
-			VersionType:    internal.VersionTypeRelease,
-			RepositoryURL:  "https://github.com/my/module",
+	wantLicenses := []*license.License{
+		{
+			Metadata: &license.Metadata{Types: []string{"MIT"}, FilePath: "LICENSE.md"},
+			Contents: []byte(testhelper.MITLicense),
 		},
-		Packages: []*internal.Package{
-			{
-				Path:     "github.com/my/module/foo",
-				V1Path:   "github.com/my/module/foo",
-				Name:     "foo",
-				Synopsis: "package foo exports a helpful constant.",
-				Licenses: []*license.Metadata{{Types: []string{"MIT"}, FilePath: "LICENSE.md"}},
-				Imports:  []string{"net/http"},
+	}
+	for _, test := range []struct {
+		name     string
+		contents map[string]string
+		want     *internal.Version
+	}{
+		{
+			name: "basic",
+			contents: map[string]string{
+				"README.md":  "THIS IS A README",
+				"foo/foo.go": "// package foo exports a helpful constant.\npackage foo\nimport \"net/http\"\nconst OK = http.StatusOK",
+				"LICENSE.md": testhelper.MITLicense,
+			},
+			want: &internal.Version{
+				VersionInfo: wantVersionInfo,
+				Packages: []*internal.Package{
+					{
+						Path:     "github.com/my/module/foo",
+						V1Path:   "github.com/my/module/foo",
+						Name:     "foo",
+						Synopsis: "package foo exports a helpful constant.",
+						Licenses: []*license.Metadata{{Types: []string{"MIT"}, FilePath: "LICENSE.md"}},
+						Imports:  []string{"net/http"},
+					},
+				},
+				Licenses: wantLicenses,
 			},
 		},
-		Licenses: []*license.License{
-			{
-				Metadata: &license.Metadata{Types: []string{"MIT"}, FilePath: "LICENSE.md"},
-				Contents: []byte(testhelper.MITLicense),
+		{
+			name: "wasm",
+			contents: map[string]string{
+				"README.md":  "THIS IS A README",
+				"LICENSE.md": testhelper.MITLicense,
+				"js/js.go": `
+					// +build js,wasm
+
+					// Package js only works with wasm.
+					package js
+					type Value int`,
+			},
+			want: &internal.Version{
+				VersionInfo: wantVersionInfo,
+				Packages: []*internal.Package{
+					{
+						Path:     "github.com/my/module/js",
+						V1Path:   "github.com/my/module/js",
+						Name:     "js",
+						Synopsis: "Package js only works with wasm.",
+						Licenses: []*license.Metadata{{Types: []string{"MIT"}, FilePath: "LICENSE.md"}},
+						Imports:  []string{},
+					},
+				},
+				Licenses: wantLicenses,
 			},
 		},
-	}
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			client, teardownProxy := proxy.SetupTestProxy(t, []*proxy.TestVersion{
+				proxy.NewTestVersion(t, modulePath, version, test.contents),
+			})
+			defer teardownProxy()

-	client, teardownProxy := proxy.SetupTestProxy(t, []*proxy.TestVersion{
-		proxy.NewTestVersion(t, modulePath, version, contents),
-	})
-	defer teardownProxy()
-
-	got, err := FetchVersion(ctx, modulePath, version, client)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(internal.Package{}, "DocumentationHTML")); diff != "" {
-		t.Errorf("fetchVersion(%q, %q) diff:\n%s", modulePath, version, diff)
+			got, err := FetchVersion(ctx, modulePath, version, client)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(test.want, got, cmpopts.IgnoreFields(internal.Package{}, "DocumentationHTML")); diff != "" {
+				t.Errorf("mismatch (-want +got):\n%s", diff)
+			}
+		})
 	}
 }