From eac14482e2f038cb8bcb9c68ee7a5784d1b0a188 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Wed, 21 Apr 2021 06:47:39 -0400 Subject: [PATCH] internal/postgres: read only one doc source from DB If GetUnit is passed a build context, it will read documentation source only for that build context. It still reads all the build context values, just only one serialized blob of file ASTs. This sped up getUnitWithAllFields by about 2x on net/http, using detail-stats against the staging DB from a local frontend. Change-Id: Ib8e988940b7f8f8ef2c270129c4338bb3d56b9eb Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/312269 Trust: Jonathan Amsterdam Run-TryBot: Jonathan Amsterdam Reviewed-by: Jamal Carvalho --- internal/build_context.go | 11 +- internal/datasource.go | 3 +- internal/frontend/imports.go | 2 +- internal/frontend/license.go | 2 +- internal/frontend/main.go | 13 +- internal/localdatasource/datasource.go | 11 +- internal/localdatasource/datasource_test.go | 2 +- internal/postgres/insert_module_test.go | 7 +- internal/postgres/licenses_test.go | 4 +- internal/postgres/unit.go | 125 +++++++++++--------- internal/postgres/unit_test.go | 64 +++++++++- internal/proxydatasource/datasource.go | 7 +- internal/proxydatasource/datasource_test.go | 2 +- internal/proxydatasource/details.go | 8 +- internal/unit.go | 3 +- internal/worker/fetch_test.go | 4 +- internal/worker/fetcherror_test.go | 2 +- internal/worker/refetch_test.go | 4 +- 18 files changed, 183 insertions(+), 91 deletions(-) diff --git a/internal/build_context.go b/internal/build_context.go index 823c5476..63b1b77e 100644 --- a/internal/build_context.go +++ b/internal/build_context.go @@ -18,6 +18,15 @@ func (b BuildContext) String() string { return fmt.Sprintf("%s/%s", b.GOOS, b.GOARCH) } +// Match reports whether its receiver, which acts like a pattern, matches its +// target, an ordinary BuildContext. In addition to the usual values, a pattern +// can have an empty GOOS or GOARCH, which means "match anything." +func (pattern BuildContext) Match(target BuildContext) bool { + match := func(pat, targ string) bool { return pat == "" || targ == All || pat == targ } + + return match(pattern.GOOS, target.GOOS) && match(pattern.GOARCH, target.GOARCH) +} + // All represents all values for a build context element (GOOS or GOARCH). const All = "all" @@ -81,7 +90,7 @@ func (d *Documentation) BuildContext() BuildContext { // match the first element of docs, if there is one. func DocumentationForBuildContext(docs []*Documentation, bc BuildContext) *Documentation { for _, d := range docs { - if (bc.GOOS == "" || d.GOOS == All || bc.GOOS == d.GOOS) && (bc.GOARCH == "" || d.GOARCH == All || bc.GOARCH == d.GOARCH) { + if bc.Match(BuildContext{d.GOOS, d.GOARCH}) { return d } } diff --git a/internal/datasource.go b/internal/datasource.go index 0e702220..657e1eab 100644 --- a/internal/datasource.go +++ b/internal/datasource.go @@ -16,7 +16,8 @@ type DataSource interface { GetNestedModules(ctx context.Context, modulePath string) ([]*ModuleInfo, error) // GetUnit returns information about a directory, which may also be a // module and/or package. The module and version must both be known. - GetUnit(ctx context.Context, pathInfo *UnitMeta, fields FieldSet) (_ *Unit, err error) + // The BuildContext selects the documentation to read. + GetUnit(ctx context.Context, pathInfo *UnitMeta, fields FieldSet, bc BuildContext) (_ *Unit, err error) // GetUnitMeta returns information about a path. GetUnitMeta(ctx context.Context, path, requestedModulePath, requestedVersion string) (_ *UnitMeta, err error) // GetModuleReadme gets the readme for the module. diff --git a/internal/frontend/imports.go b/internal/frontend/imports.go index bcf3994d..624af4c3 100644 --- a/internal/frontend/imports.go +++ b/internal/frontend/imports.go @@ -42,7 +42,7 @@ func fetchImportsDetails(ctx context.Context, ds internal.DataSource, pkgPath, m ModulePath: modulePath, Version: resolvedVersion, }, - }, internal.WithImports) + }, internal.WithImports, internal.BuildContext{}) if err != nil { return nil, err } diff --git a/internal/frontend/license.go b/internal/frontend/license.go index 7dfeff72..41c03a81 100644 --- a/internal/frontend/license.go +++ b/internal/frontend/license.go @@ -37,7 +37,7 @@ type LicenseMetadata struct { // fetchLicensesDetails fetches license data for the package version specified by // path and version from the database and returns a LicensesDetails. func fetchLicensesDetails(ctx context.Context, ds internal.DataSource, um *internal.UnitMeta) (*LicensesDetails, error) { - u, err := ds.GetUnit(ctx, um, internal.WithLicenses) + u, err := ds.GetUnit(ctx, um, internal.WithLicenses, internal.BuildContext{}) if err != nil { return nil, err } diff --git a/internal/frontend/main.go b/internal/frontend/main.go index adabd94a..59b19096 100644 --- a/internal/frontend/main.go +++ b/internal/frontend/main.go @@ -109,7 +109,7 @@ type File struct { func fetchMainDetails(ctx context.Context, ds internal.DataSource, um *internal.UnitMeta, expandReadme bool, bc internal.BuildContext) (_ *MainDetails, err error) { defer middleware.ElapsedStat(ctx, "fetchMainDetails")() - unit, err := ds.GetUnit(ctx, um, internal.WithMain) + unit, err := ds.GetUnit(ctx, um, internal.WithMain, bc) if err != nil { return nil, err } @@ -135,15 +135,18 @@ func fetchMainDetails(ctx context.Context, ds internal.DataSource, um *internal. ) unit.Documentation = cleanDocumentation(unit.Documentation) - doc := internal.DocumentationForBuildContext(unit.Documentation, bc) + // There should be at most one Documentation. + var doc *internal.Documentation + if len(unit.Documentation) > 0 { + doc = unit.Documentation[0] + } + versionToNameToUnitSymbol := map[string]map[string]*internal.UnitSymbol{} if doc != nil { synopsis = doc.Synopsis goos = doc.GOOS goarch = doc.GOARCH - for _, v := range unit.Documentation { - buildContexts = append(buildContexts, v.BuildContext()) - } + buildContexts = unit.BuildContexts end := middleware.ElapsedStat(ctx, "DecodePackage") docPkg, err := godoc.DecodePackage(doc.Source) end() diff --git a/internal/localdatasource/datasource.go b/internal/localdatasource/datasource.go index e8c3609b..69a0c6cf 100644 --- a/internal/localdatasource/datasource.go +++ b/internal/localdatasource/datasource.go @@ -76,6 +76,15 @@ func (ds *DataSource) fetch(ctx context.Context, modulePath, localPath string) e unit.IsRedistributable = true } + for _, unit := range fr.Module.Units { + for _, d := range unit.Documentation { + unit.BuildContexts = append(unit.BuildContexts, internal.BuildContext{ + GOOS: d.GOOS, + GOARCH: d.GOARCH, + }) + } + } + ds.mu.Lock() defer ds.mu.Unlock() ds.loadedModules[fr.ModulePath] = fr.Module @@ -99,7 +108,7 @@ func getFullPath(modulePath string) string { // GetUnit returns information about a unit. Both the module path and package // path must be known. -func (ds *DataSource) GetUnit(ctx context.Context, pathInfo *internal.UnitMeta, fields internal.FieldSet) (_ *internal.Unit, err error) { +func (ds *DataSource) GetUnit(ctx context.Context, pathInfo *internal.UnitMeta, fields internal.FieldSet, bc internal.BuildContext) (_ *internal.Unit, err error) { defer derrors.Wrap(&err, "GetUnit(%q, %q)", pathInfo.Path, pathInfo.ModulePath) modulepath := pathInfo.ModulePath diff --git a/internal/localdatasource/datasource_test.go b/internal/localdatasource/datasource_test.go index 8bee7907..4ce3c5bb 100644 --- a/internal/localdatasource/datasource_test.go +++ b/internal/localdatasource/datasource_test.go @@ -240,7 +240,7 @@ func TestGetUnit(t *testing.T) { Path: test.path, ModuleInfo: internal.ModuleInfo{ModulePath: test.modulePath}, } - got, err := ds.GetUnit(ctx, um, 0) + got, err := ds.GetUnit(ctx, um, 0, internal.BuildContext{}) if !test.wantLoaded { if err == nil { t.Fatalf("returned not loaded module %q", test.path) diff --git a/internal/postgres/insert_module_test.go b/internal/postgres/insert_module_test.go index 6478f160..67d13adc 100644 --- a/internal/postgres/insert_module_test.go +++ b/internal/postgres/insert_module_test.go @@ -100,7 +100,7 @@ func checkModule(ctx context.Context, t *testing.T, db *DB, want *internal.Modul } for _, wantu := range want.Units { - got, err := db.GetUnit(ctx, &wantu.UnitMeta, internal.AllFields) + got, err := db.GetUnit(ctx, &wantu.UnitMeta, internal.AllFields, internal.BuildContext{}) if err != nil { t.Fatal(err) } @@ -111,6 +111,7 @@ func checkModule(ctx context.Context, t *testing.T, db *DB, want *internal.Modul } } wantu.Subdirectories = subdirectories + wantu.BuildContexts = got.BuildContexts opts := cmp.Options{ cmpopts.IgnoreFields(licenses.Metadata{}, "Coverage", "OldCoverage"), cmp.AllowUnexported(source.Info{}, safehtml.HTML{}), @@ -163,7 +164,7 @@ func TestInsertModuleLicenseCheck(t *testing.T) { MustInsertModule(ctx, t, db, mod) // New model - u, err := db.GetUnit(ctx, newUnitMeta(mod.ModulePath, mod.ModulePath, mod.Version), internal.AllFields) + u, err := db.GetUnit(ctx, newUnitMeta(mod.ModulePath, mod.ModulePath, mod.Version), internal.AllFields, internal.BuildContext{}) if err != nil { t.Fatal(err) } @@ -306,7 +307,7 @@ func TestInsertModuleNewCoverage(t *testing.T) { }, } MustInsertModule(ctx, t, testDB, m) - u, err := testDB.GetUnit(ctx, newUnitMeta(m.ModulePath, m.ModulePath, m.Version), internal.AllFields) + u, err := testDB.GetUnit(ctx, newUnitMeta(m.ModulePath, m.ModulePath, m.Version), internal.AllFields, internal.BuildContext{}) if err != nil { t.Fatal(err) } diff --git a/internal/postgres/licenses_test.go b/internal/postgres/licenses_test.go index 9ebcd861..5b43469d 100644 --- a/internal/postgres/licenses_test.go +++ b/internal/postgres/licenses_test.go @@ -95,7 +95,7 @@ func TestGetLicenses(t *testing.T) { }, } { t.Run(test.name, func(t *testing.T) { - u, err := testDB.GetUnit(ctx, newUnitMeta(test.fullPath, test.modulePath, test.version), internal.WithLicenses) + u, err := testDB.GetUnit(ctx, newUnitMeta(test.fullPath, test.modulePath, test.version), internal.WithLicenses, internal.BuildContext{}) if !errors.Is(err, test.err) { t.Fatal(err) } @@ -185,7 +185,7 @@ func TestGetLicensesBypass(t *testing.T) { if bypass { db = bypassDB } - u, err := db.GetUnit(ctx, newUnitMeta(sample.ModulePath, sample.ModulePath, m.Version), internal.WithLicenses) + u, err := db.GetUnit(ctx, newUnitMeta(sample.ModulePath, sample.ModulePath, m.Version), internal.WithLicenses, internal.BuildContext{}) if err != nil { t.Fatal(err) } diff --git a/internal/postgres/unit.go b/internal/postgres/unit.go index 4c5b44c2..687fcc77 100644 --- a/internal/postgres/unit.go +++ b/internal/postgres/unit.go @@ -225,12 +225,13 @@ func (db *DB) getLatestUnitVersion(ctx context.Context, fullPath, requestedModul // GetUnit returns a unit from the database, along with all of the data // associated with that unit. -func (db *DB) GetUnit(ctx context.Context, um *internal.UnitMeta, fields internal.FieldSet) (_ *internal.Unit, err error) { - defer derrors.WrapStack(&err, "GetUnit(ctx, %q, %q, %q)", um.Path, um.ModulePath, um.Version) +// If bc is not nil, get only the Documentation that matches it (or nil if none do). +func (db *DB) GetUnit(ctx context.Context, um *internal.UnitMeta, fields internal.FieldSet, bc internal.BuildContext) (_ *internal.Unit, err error) { + defer derrors.WrapStack(&err, "GetUnit(ctx, %q, %q, %q, %v)", um.Path, um.ModulePath, um.Version, bc) u := &internal.Unit{UnitMeta: *um} if fields&internal.WithMain != 0 { - u, err = db.getUnitWithAllFields(ctx, um) + u, err = db.getUnitWithAllFields(ctx, um, bc) if err != nil { return nil, err } @@ -396,16 +397,53 @@ func getPackagesInUnit(ctx context.Context, db *database.DB, fullPath, modulePat return packages, nil } -func (db *DB) getUnitWithAllFields(ctx context.Context, um *internal.UnitMeta) (_ *internal.Unit, err error) { +func (db *DB) getUnitWithAllFields(ctx context.Context, um *internal.UnitMeta, bc internal.BuildContext) (_ *internal.Unit, err error) { defer derrors.WrapStack(&err, "getUnitWithAllFields(ctx, %q, %q, %q)", um.Path, um.ModulePath, um.Version) defer middleware.ElapsedStat(ctx, "getUnitWithAllFields")() - // Get README and import counts. + // Get build contexts and unit ID. + var unitID int + var bcs []internal.BuildContext + err = db.db.RunQuery(ctx, ` + SELECT d.goos, d.goarch, u.id + FROM units u + INNER JOIN paths p ON p.id = u.path_id + INNER JOIN modules m ON m.id = u.module_id + LEFT JOIN documentation d ON d.unit_id = u.id + WHERE + p.path = $1 + AND m.module_path = $2 + AND m.version = $3 + `, func(rows *sql.Rows) error { + var bc internal.BuildContext + // GOOS and GOARCH will be NULL if there are no documentation rows for + // the unit, but we still want the unit ID. + if err := rows.Scan(database.NullIsEmpty(&bc.GOOS), database.NullIsEmpty(&bc.GOARCH), &unitID); err != nil { + return err + } + if bc.GOOS != "" && bc.GOARCH != "" { + bcs = append(bcs, bc) + } + return nil + }, um.Path, um.ModulePath, um.Version) + if err != nil { + return nil, err + } + sort.Slice(bcs, func(i, j int) bool { return internal.CompareBuildContexts(bcs[i], bcs[j]) < 0 }) + var bcMatched internal.BuildContext + for _, c := range bcs { + if bc.Match(c) { + bcMatched = c + break + } + } + // Get README, documentation and import counts. query := ` SELECT - u.id, r.file_path, r.contents, + d.synopsis, + d.source, COALESCE(( SELECT COUNT(unit_id) FROM package_imports @@ -420,49 +458,51 @@ func (db *DB) getUnitWithAllFields(ctx context.Context, um *internal.UnitMeta) ( WHERE package_path = $1 ), 0) AS num_imported_by FROM units u - INNER JOIN paths p - ON p.id = u.path_id - INNER JOIN modules m - ON u.module_id = m.id LEFT JOIN readmes r ON r.unit_id = u.id - WHERE - p.path = $1 - AND m.module_path = $2 - AND m.version = $3;` + LEFT JOIN ( + SELECT synopsis, source, goos, goarch, unit_id + FROM documentation d + WHERE d.GOOS = $3 AND d.GOARCH = $4 + ) d + ON d.unit_id = u.id + WHERE u.id = $2 + ` var ( - unitID int - r internal.Readme - u internal.Unit + r internal.Readme + u internal.Unit ) + u.BuildContexts = bcs + var goos, goarch interface{} + if bcMatched.GOOS != "" { + goos = bcMatched.GOOS + goarch = bcMatched.GOARCH + } + doc := &internal.Documentation{GOOS: bcMatched.GOOS, GOARCH: bcMatched.GOARCH} end := middleware.ElapsedStat(ctx, "getUnitWithAllFields-readme-and-imports") - err = db.db.QueryRow(ctx, query, um.Path, um.ModulePath, um.Version).Scan( - &unitID, + err = db.db.QueryRow(ctx, query, um.Path, unitID, goos, goarch).Scan( database.NullIsEmpty(&r.Filepath), database.NullIsEmpty(&r.Contents), + database.NullIsEmpty(&doc.Synopsis), + &doc.Source, &u.NumImports, &u.NumImportedBy, ) switch err { case sql.ErrNoRows: - return nil, derrors.NotFound + // Neither a README nor documentation; that's OK, continue. case nil: if r.Filepath != "" && um.ModulePath != stdlib.ModulePath { u.Readme = &r } + if doc.GOOS != "" { + u.Documentation = []*internal.Documentation{doc} + } default: return nil, err } end() - - // Get documentation. - docs, err := db.getDocumentation(ctx, unitID) - if err != nil { - return nil, err - } - u.Documentation = docs - // Get other info. pkgs, err := db.getPackagesInUnit(ctx, um.Path, um.ModulePath, um.Version) if err != nil { @@ -473,35 +513,6 @@ func (db *DB) getUnitWithAllFields(ctx context.Context, um *internal.UnitMeta) ( return &u, nil } -func (db *DB) getDocumentation(ctx context.Context, unitID int) (_ []*internal.Documentation, err error) { - defer derrors.WrapStack(&err, "getDocumentation(ctx, %d)", unitID) - defer middleware.ElapsedStat(ctx, "getDocumentation")() - - var docs []*internal.Documentation - // Get documentation. There can be multiple rows. - query := ` - SELECT goos, goarch, synopsis, source - FROM documentation - WHERE unit_id = $1` - if err := db.db.RunQuery(ctx, query, func(rows *sql.Rows) error { - var d internal.Documentation - if err := rows.Scan(&d.GOOS, &d.GOARCH, &d.Synopsis, &d.Source); err != nil { - return err - } - docs = append(docs, &d) - return nil - }, unitID); err != nil { - return nil, err - } - // Sort documentation by GOOS/GOARCH. - sort.Slice(docs, func(i, j int) bool { - ci := docs[i].BuildContext() - cj := docs[j].BuildContext() - return internal.CompareBuildContexts(ci, cj) < 0 - }) - return docs, nil -} - type dbPath struct { id int64 path string diff --git a/internal/postgres/unit_test.go b/internal/postgres/unit_test.go index 47de46d6..40fc7be7 100644 --- a/internal/postgres/unit_test.go +++ b/internal/postgres/unit_test.go @@ -724,7 +724,8 @@ func TestGetUnit(t *testing.T) { u := unit("a.com/twodoc/p", "a.com/twodoc", "v1.2.3", "p", nil, []string{"p"}) - u.Documentation = docs2 + u.Documentation = docs2[:1] + u.BuildContexts = []internal.BuildContext{internal.BuildContextLinux, internal.BuildContextWindows} u.Subdirectories[0].Synopsis = docs2[0].Synopsis return u }(), @@ -747,7 +748,7 @@ func TestGetUnit(t *testing.T) { func checkUnit(ctx context.Context, t *testing.T, db *DB, um *internal.UnitMeta, want *internal.Unit, experiments ...string) { t.Helper() ctx = experiment.NewContext(ctx, experiments...) - got, err := db.GetUnit(ctx, um, internal.AllFields) + got, err := db.GetUnit(ctx, um, internal.AllFields, internal.BuildContext{}) if err != nil { t.Fatal(err) } @@ -799,6 +800,7 @@ func TestGetUnitFieldSet(t *testing.T) { // Add/remove fields based on the FieldSet specified. if fields&internal.WithMain != 0 { u.Documentation = []*internal.Documentation{sample.Doc} + u.BuildContexts = []internal.BuildContext{internal.BuildContextAll} u.Readme = readme u.NumImports = len(sample.Imports()) u.Subdirectories = []*internal.PackageMeta{ @@ -850,7 +852,7 @@ func TestGetUnitFieldSet(t *testing.T) { test.want.Name, test.want.IsRedistributable, ) - got, err := testDB.GetUnit(ctx, um, test.fields) + got, err := testDB.GetUnit(ctx, um, test.fields, internal.BuildContext{}) if err != nil { t.Fatal(err) } @@ -869,6 +871,59 @@ func TestGetUnitFieldSet(t *testing.T) { } } +func TestGetUnitBuildContext(t *testing.T) { + t.Parallel() + testDB, release := acquire(t) + defer release() + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + // Add a module that has documentation for two Go build contexts. + m := sample.Module("a.com/twodoc", "v1.2.3", "p") + pkg := m.Packages()[0] + linuxDoc := sample.Documentation("linux", "amd64", `package p; var L int`) + windowsDoc := sample.Documentation("windows", "amd64", `package p; var W int`) + pkg.Documentation = []*internal.Documentation{linuxDoc, windowsDoc} + MustInsertModule(ctx, t, testDB, m) + + um := sample.UnitMeta( + "a.com/twodoc/p", + "a.com/twodoc", + "v1.2.3", + "p", + true) + for _, test := range []struct { + goos, goarch string + want *internal.Documentation + }{ + {"", "", linuxDoc}, + {"linux", "amd64", linuxDoc}, + {"windows", "amd64", windowsDoc}, + {"linux", "", linuxDoc}, + {"wasm", "js", nil}, + } { + t.Run(fmt.Sprintf("%s-%s", test.goos, test.goarch), func(t *testing.T) { + bc := internal.BuildContext{GOOS: test.goos, GOARCH: test.goarch} + u, err := testDB.GetUnit(ctx, um, internal.WithMain, bc) + if err != nil { + t.Fatal(err) + } + got := u.Documentation + var want []*internal.Documentation + if test.want != nil { + want = []*internal.Documentation{test.want} + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("mismatch (-want, +got):\n%s", diff) + } + wantb := []internal.BuildContext{internal.BuildContextLinux, internal.BuildContextWindows} + if got := u.BuildContexts; !cmp.Equal(got, wantb) { + t.Errorf("got %v, want %v", got, wantb) + } + }) + } +} + func unit(fullPath, modulePath, version, name string, readme *internal.Readme, suffixes []string) *internal.Unit { u := &internal.Unit{ UnitMeta: internal.UnitMeta{ @@ -892,6 +947,7 @@ func unit(fullPath, modulePath, version, name string, readme *internal.Readme, s u.Imports = imps u.NumImports = len(imps) u.Documentation = []*internal.Documentation{sample.Doc} + u.BuildContexts = []internal.BuildContext{internal.BuildContextAll} } return u } @@ -927,7 +983,7 @@ func TestGetUnitBypass(t *testing.T) { {bypassDB, false}, } { pathInfo := newUnitMeta(m.ModulePath, m.ModulePath, m.Version) - d, err := test.db.GetUnit(ctx, pathInfo, internal.AllFields) + d, err := test.db.GetUnit(ctx, pathInfo, internal.AllFields, internal.BuildContext{}) if err != nil { t.Fatal(err) } diff --git a/internal/proxydatasource/datasource.go b/internal/proxydatasource/datasource.go index 56a18fac..d986d769 100644 --- a/internal/proxydatasource/datasource.go +++ b/internal/proxydatasource/datasource.go @@ -85,7 +85,7 @@ type versionEntry struct { // getModule retrieves a version from the cache, or failing that queries and // processes the version from the proxy. -func (ds *DataSource) getModule(ctx context.Context, modulePath, version string) (_ *internal.Module, err error) { +func (ds *DataSource) getModule(ctx context.Context, modulePath, version string, _ internal.BuildContext) (_ *internal.Module, err error) { defer derrors.Wrap(&err, "getModule(%q, %q)", modulePath, version) key := versionKey{modulePath, version} @@ -106,6 +106,7 @@ func (ds *DataSource) getModule(ctx context.Context, modulePath, version string) } else { m.RemoveNonRedistributableData() } + // // Use the go.mod file at the raw latest version to fill in deprecation // and retraction information. lmv, err := fetch.LatestModuleVersions(ctx, modulePath, ds.proxyClient, nil) @@ -178,9 +179,9 @@ func (ds *DataSource) findModule(ctx context.Context, pkgPath string, version st } // getUnit returns information about a unit. -func (ds *DataSource) getUnit(ctx context.Context, fullPath, modulePath, version string) (_ *internal.Unit, err error) { +func (ds *DataSource) getUnit(ctx context.Context, fullPath, modulePath, version string, bc internal.BuildContext) (_ *internal.Unit, err error) { var m *internal.Module - m, err = ds.getModule(ctx, modulePath, version) + m, err = ds.getModule(ctx, modulePath, version, bc) if err != nil { return nil, err } diff --git a/internal/proxydatasource/datasource_test.go b/internal/proxydatasource/datasource_test.go index e7f3ad27..a3d05be5 100644 --- a/internal/proxydatasource/datasource_test.go +++ b/internal/proxydatasource/datasource_test.go @@ -218,7 +218,7 @@ func TestBypass(t *testing.T) { if err != nil { t.Fatal(err) } - got, err := ds.GetUnit(ctx, um, 0) + got, err := ds.GetUnit(ctx, um, 0, internal.BuildContext{}) if err != nil { t.Fatal(err) } diff --git a/internal/proxydatasource/details.go b/internal/proxydatasource/details.go index cc8667d8..71481e27 100644 --- a/internal/proxydatasource/details.go +++ b/internal/proxydatasource/details.go @@ -15,16 +15,16 @@ import ( ) // GetUnit returns information about a directory at a path. -func (ds *DataSource) GetUnit(ctx context.Context, um *internal.UnitMeta, field internal.FieldSet) (_ *internal.Unit, err error) { +func (ds *DataSource) GetUnit(ctx context.Context, um *internal.UnitMeta, field internal.FieldSet, bc internal.BuildContext) (_ *internal.Unit, err error) { defer derrors.Wrap(&err, "GetUnit(%q, %q, %q)", um.Path, um.ModulePath, um.Version) - return ds.getUnit(ctx, um.Path, um.ModulePath, um.Version) + return ds.getUnit(ctx, um.Path, um.ModulePath, um.Version, bc) } // GetModuleInfo returns the ModuleInfo as fetched from the proxy for module // version specified by modulePath and version. func (ds *DataSource) GetModuleInfo(ctx context.Context, modulePath, version string) (_ *internal.ModuleInfo, err error) { defer derrors.Wrap(&err, "GetModuleInfo(%q, %q)", modulePath, version) - m, err := ds.getModule(ctx, modulePath, version) + m, err := ds.getModule(ctx, modulePath, version, internal.BuildContext{}) if err != nil { return nil, err } @@ -43,7 +43,7 @@ func (ds *DataSource) GetUnitMeta(ctx context.Context, path, inModulePath, inVer } inVersion = info.Version } - m, err := ds.getModule(ctx, inModulePath, inVersion) + m, err := ds.getModule(ctx, inModulePath, inVersion, internal.BuildContext{}) if err != nil { return nil, err } diff --git a/internal/unit.go b/internal/unit.go index caa458f0..325fd55b 100644 --- a/internal/unit.go +++ b/internal/unit.go @@ -45,7 +45,8 @@ func (um *UnitMeta) IsModule() bool { type Unit struct { UnitMeta Readme *Readme - Documentation []*Documentation + BuildContexts []BuildContext // set only on read + Documentation []*Documentation // at most one on read Subdirectories []*PackageMeta Imports []string LicenseContents []*licenses.License diff --git a/internal/worker/fetch_test.go b/internal/worker/fetch_test.go index 7bed065a..d577c582 100644 --- a/internal/worker/fetch_test.go +++ b/internal/worker/fetch_test.go @@ -317,14 +317,14 @@ func TestFetchAndUpdateState(t *testing.T) { t.Fatalf("testDB.GetUnitMeta(ctx, %q, %q) mismatch (-want +got):\n%s", test.modulePath, test.version, diff) } - gotPkg, err := testDB.GetUnit(ctx, got, internal.WithMain) + gotPkg, err := testDB.GetUnit(ctx, got, internal.WithMain, internal.BuildContext{}) if err != nil { t.Fatal(err) } if diff := cmp.Diff(test.want, gotPkg, cmp.AllowUnexported(source.Info{}), - cmpopts.IgnoreFields(internal.Unit{}, "Documentation"), + cmpopts.IgnoreFields(internal.Unit{}, "Documentation", "BuildContexts"), cmpopts.IgnoreFields(internal.Unit{}, "Subdirectories")); diff != "" { t.Errorf("mismatch on readme (-want +got):\n%s", diff) } diff --git a/internal/worker/fetcherror_test.go b/internal/worker/fetcherror_test.go index d875b6ff..291b4c14 100644 --- a/internal/worker/fetcherror_test.go +++ b/internal/worker/fetcherror_test.go @@ -418,7 +418,7 @@ func checkPackage(ctx context.Context, t *testing.T, pkgPath string) { t.Fatalf("testDB.GetUnitMeta(%q, %q, %q): isPackage = false; want = true", pkgPath, internal.UnknownModulePath, sample.VersionString) } - dir, err := testDB.GetUnit(ctx, um, internal.WithMain) + dir, err := testDB.GetUnit(ctx, um, internal.WithMain, internal.BuildContext{}) if err != nil { t.Fatal(err) } diff --git a/internal/worker/refetch_test.go b/internal/worker/refetch_test.go index 0b746dcd..5f9e2944 100644 --- a/internal/worker/refetch_test.go +++ b/internal/worker/refetch_test.go @@ -129,13 +129,13 @@ func TestReFetch(t *testing.T) { t.Fatalf("testDB.GetUnitMeta(ctx, %q, %q) mismatch (-want +got):\n%s", want.ModulePath, want.Version, diff) } - gotPkg, err := testDB.GetUnit(ctx, got, internal.WithMain) + gotPkg, err := testDB.GetUnit(ctx, got, internal.WithMain, internal.BuildContext{}) if err != nil { t.Fatal(err) } if diff := cmp.Diff(want, gotPkg, cmp.AllowUnexported(source.Info{}), - cmpopts.IgnoreFields(internal.Unit{}, "Documentation"), + cmpopts.IgnoreFields(internal.Unit{}, "Documentation", "BuildContexts"), cmpopts.IgnoreFields(licenses.Metadata{}, "Coverage", "OldCoverage"), cmpopts.IgnoreFields(internal.UnitMeta{}, "HasGoMod")); diff != "" { t.Errorf("mismatch on readme (-want +got):\n%s", diff)