diff --git a/all.bash b/all.bash index 39c37183..63ff1feb 100755 --- a/all.bash +++ b/all.bash @@ -47,13 +47,13 @@ modified_files() { fi } + # Helper for modified_files. It asks git for all modified, added or deleted # files, and keeps only the latter two. diff_files() { - git diff --name-status $* | awk '$1 != "D" { print $2 }' + git diff --name-status $* | awk '$1 ~ /^R/ { print $3; next } $1 != "D" { print $2 }' } - # codedirs lists directories that contain discovery code. If they include # directories containing external code, those directories must be excluded in # findcode below. diff --git a/internal/fetch/raw_latest.go b/internal/fetch/raw_latest.go deleted file mode 100644 index e6735dd6..00000000 --- a/internal/fetch/raw_latest.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package fetch - -import ( - "context" - "errors" - - "golang.org/x/pkgsite/internal" - "golang.org/x/pkgsite/internal/derrors" - "golang.org/x/pkgsite/internal/log" - "golang.org/x/pkgsite/internal/proxy" - "golang.org/x/pkgsite/internal/stdlib" - "golang.org/x/pkgsite/internal/version" -) - -// RawLatestInfo uses the proxy to get information about the raw latest version -// of modulePath. If it cannot obtain it, it returns (nil, nil). -// -// The hasGoMod function that is passed in should check if version v of the -// module has a go.mod file, using a source other than the proxy (e.g. a -// database). If it doesn't have enough information to decide, it should return -// an error that wraps derrors.NotFound. -func RawLatestInfo(ctx context.Context, modulePath string, prox *proxy.Client, hasGoMod func(v string) (bool, error)) (_ *internal.RawLatestInfo, err error) { - defer derrors.WrapStack(&err, "RawLatestInfo(%q)", modulePath) - - // No raw latest info for std; no deprecations or retractions. - if modulePath == stdlib.ModulePath { - return nil, nil - } - - v, err := fetchRawLatestVersion(ctx, modulePath, prox, hasGoMod) - if err != nil { - return nil, err - } - modBytes, err := prox.Mod(ctx, modulePath, v) - if err != nil { - return nil, err - } - return internal.NewRawLatestInfo(modulePath, v, modBytes) -} - -// fetchRawLatestVersion uses the proxy to determine the latest -// version of a module independent of retractions or other modifications. -// -// This meaning of "latest" is defined at https://golang.org/ref/mod#version-queries. -// That definition does not deal with a subtlety involving -// incompatible versions. The actual definition is embodied in the go command's -// queryMatcher.filterVersions method. This code is a rewrite of that method at Go -// version 1.16 -// (https://go.googlesource.com/go/+/refs/tags/go1.16/src/cmd/go/internal/modload/query.go#441). -func fetchRawLatestVersion(ctx context.Context, modulePath string, prox *proxy.Client, hasGoMod func(v string) (bool, error)) (v string, err error) { - defer derrors.WrapStack(&err, "fetchRawLatestVersion(%q)", modulePath) - - defer func() { - log.Debugf(ctx, "fetchRawLatestVersion(%q) => (%q, %v)", modulePath, v, err) - }() - - // Get tagged versions from the proxy's list endpoint. - taggedVersions, err := prox.Versions(ctx, modulePath) - if err != nil { - return "", err - } - // If there are no tagged versions, use the proxy's @latest endpoint. - if len(taggedVersions) == 0 { - latestInfo, err := prox.Info(ctx, modulePath, internal.LatestVersion) - if err != nil { - return "", err - } - return latestInfo.Version, nil - } - - // Find the latest of all tagged versions. - hasGoModFunc := func(v string) (bool, error) { - var ( - has bool - err error - ) - if hasGoMod == nil { - err = derrors.NotFound - } else { - has, err = hasGoMod(v) - } - if err == nil { - return has, nil - } else if !errors.Is(err, derrors.NotFound) { - return false, err - } else { - // hasGoMod doesn't know; download the zip. - zr, err := prox.Zip(ctx, modulePath, v) - if err != nil { - return false, err - } - return hasGoModFile(zr, modulePath, v), nil - } - } - return version.Latest(taggedVersions, hasGoModFunc) - -} diff --git a/internal/fetch/raw_latest_test.go b/internal/fetch/raw_latest_test.go deleted file mode 100644 index 1261ef96..00000000 --- a/internal/fetch/raw_latest_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package fetch - -import ( - "context" - "testing" - - "golang.org/x/pkgsite/internal/proxy" -) - -func TestFetchRawLatestVersion(t *testing.T) { - prox, teardown := proxy.SetupTestClient(t, testModules) - defer teardown() - - for _, test := range []struct { - module string - want string - }{ - {"example.com/basic", "v1.1.0"}, - {"example.com/single", "v1.0.0"}, - } { - got, err := fetchRawLatestVersion(context.Background(), test.module, prox, nil) - if err != nil { - t.Fatal(err) - } - if got != test.want { - t.Errorf("%s: got %s, want %s", test.module, got, test.want) - } - } -} - -func TestRawLatestInfo(t *testing.T) { - // fetchRawLatestVersion is tested above. - // Contents of the go.mod file are tested in proxydatasource. - // Here, just test that there is a parsed go.mod file. - prox, teardown := proxy.SetupTestClient(t, testModules) - defer teardown() - - const module = "example.com/basic" - got, err := RawLatestInfo(context.Background(), module, prox, nil) - if err != nil { - t.Fatal(err) - } - if got.ModulePath != module || got.Version != "v1.1.0" || got.GoModFile == nil { - t.Errorf("got (%q, %q, %p), want (%q, 'v1.1.0', )", got.ModulePath, got.Version, got.GoModFile, module) - } - -} diff --git a/internal/latest.go b/internal/latest.go index cd8b2b90..d42b805f 100644 --- a/internal/latest.go +++ b/internal/latest.go @@ -6,6 +6,7 @@ package internal import ( "fmt" + "strings" "golang.org/x/mod/modfile" "golang.org/x/mod/semver" @@ -42,6 +43,25 @@ func NewLatestModuleVersions(modulePath, raw, cooked, good string, modBytes []by }, nil } +// isDeprecated reports whether the go.mod deprecates this module. +// It looks for "Deprecated" comments in the line comments before and next to +// the module declaration. If it finds one, it returns true along with the text +// after "Deprecated:". Otherwise it returns false, "". +func isDeprecated(mf *modfile.File) (bool, string) { + const prefix = "Deprecated:" + + if mf.Module == nil { + return false, "" + } + for _, comment := range append(mf.Module.Syntax.Before, mf.Module.Syntax.Suffix...) { + text := strings.TrimSpace(strings.TrimPrefix(comment.Token, "//")) + if strings.HasPrefix(text, prefix) { + return true, strings.TrimSpace(text[len(prefix):]) + } + } + return false, "" +} + // PopulateModuleInfo uses the LatestModuleVersions to populate fields of the given module. func (li *LatestModuleVersions) PopulateModuleInfo(mi *ModuleInfo) { mi.Deprecated = li.deprecated diff --git a/internal/raw_latest_test.go b/internal/latest_test.go similarity index 97% rename from internal/raw_latest_test.go rename to internal/latest_test.go index a5c2b42d..5f21e57a 100644 --- a/internal/raw_latest_test.go +++ b/internal/latest_test.go @@ -85,7 +85,7 @@ func TestIsRetracted(t *testing.T) { if err != nil { t.Fatal(err) } - gotIs, gotRationale := isRetracted(mf, "v1.2.3") + gotIs, gotRationale := IsRetracted(mf, "v1.2.3") if gotIs != test.wantIs || gotRationale != test.wantRationale { t.Errorf("%s: got (%t, %q), want(%t, %q)", test.name, gotIs, gotRationale, test.wantIs, test.wantRationale) } diff --git a/internal/postgres/version.go b/internal/postgres/version.go index e3f46dbc..49ddfb03 100644 --- a/internal/postgres/version.go +++ b/internal/postgres/version.go @@ -268,63 +268,6 @@ func (db *DB) getLatestMinorModuleVersionInfo(ctx context.Context, unitPath, mod } } -// GetRawLatestInfo returns the row of the raw_latest_versions table for modulePath. -// If the module path is not found, it returns nil, nil. -func (db *DB) GetRawLatestInfo(ctx context.Context, modulePath string) (_ *internal.RawLatestInfo, err error) { - defer derrors.WrapStack(&err, "GetRawLatestInfo(%q)", modulePath) - - var ( - version string - goModBytes []byte - ) - err = db.db.QueryRow(ctx, ` - SELECT r.version, r.go_mod_bytes - FROM raw_latest_versions r - INNER JOIN paths p ON p.id = r.module_path_id - WHERE p.path = $1`, - modulePath).Scan(&version, &goModBytes) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, err - } - return internal.NewRawLatestInfo(modulePath, version, goModBytes) -} - -// UpdateRawLatestInfo upserts its argument into the raw_latest_versions table -// if the row doesn't exist, or the new version is later. -func (db *DB) UpdateRawLatestInfo(ctx context.Context, info *internal.RawLatestInfo) (err error) { - defer derrors.WrapStack(&err, "UpdateRawLatestInfo(%q)", info.ModulePath) - - // We need RepeatableRead here because the INSERT...ON CONFLICT does a read. - return db.db.Transact(ctx, sql.LevelRepeatableRead, func(tx *database.DB) error { - var ( - id int - curVersion string - ) - err = tx.QueryRow(ctx, ` - SELECT p.id, r.version - FROM raw_latest_versions r - INNER JOIN paths p ON p.id = r.module_path_id - WHERE p.path = $1`, - info.ModulePath).Scan(&id, &curVersion) - switch { - case err == sql.ErrNoRows: - // Fall through to upsert. - case err != nil: - return err - default: - if !shouldUpdateRawLatest(info.Version, curVersion) { - return nil - } - } - - log.Debugf(ctx, "updating raw latest from %q to %q", curVersion, info.Version) - return upsertRawLatestInfo(ctx, tx, id, info) - }) -} - func shouldUpdateRawLatest(newVersion, curVersion string) bool { // Only update if the new one is later according to version.Later // (semver except that release > prerelease). that avoids a race @@ -341,37 +284,6 @@ func shouldUpdateRawLatest(newVersion, curVersion string) bool { (version.IsIncompatible(curVersion) && !version.IsIncompatible(newVersion)) } -func upsertRawLatestInfo(ctx context.Context, tx *database.DB, id int, info *internal.RawLatestInfo) (err error) { - defer derrors.WrapStack(&err, "upsertRawLatestInfo(%d, %q, %q)", id, info.ModulePath, info.Version) - - // If the row doesn't exist, get a path ID for the module path. - if id == 0 { - id, err = upsertPath(ctx, tx, info.ModulePath) - if err != nil { - return err - } - } - - // Convert the go.mod file into bytes. - goModBytes, err := info.GoModFile.Format() - if err != nil { - return err - } - _, err = tx.Exec(ctx, ` - INSERT INTO raw_latest_versions ( - module_path_id, - version, - go_mod_bytes - ) VALUES ($1, $2, $3) - ON CONFLICT (module_path_id) - DO UPDATE SET - version=excluded.version, - go_mod_bytes=excluded.go_mod_bytes - `, - id, info.Version, goModBytes) - return err -} - // GetLatestModuleVersions returns the row of the latest_module_versions table for modulePath. // If the module path is not found, it returns nil, nil. func (db *DB) GetLatestModuleVersions(ctx context.Context, modulePath string) (_ *internal.LatestModuleVersions, err error) { diff --git a/internal/postgres/version_test.go b/internal/postgres/version_test.go index 7dbfe603..f6aa2995 100644 --- a/internal/postgres/version_test.go +++ b/internal/postgres/version_test.go @@ -64,7 +64,7 @@ func TestGetVersions(t *testing.T) { for _, m := range testModules { MustInsertModule(ctx, t, testDB, m) } - // Add raw latest version info for rootModule. + // Add latest version info for rootModule. addLatest(ctx, t, testDB, rootModule, "v1.1.0", ` module golang.org/foo/bar // Deprecated: use other retract v1.0.3 // security flaw @@ -369,33 +369,6 @@ func TestGetLatestInfo(t *testing.T) { } } -func TestRawLatestInfo(t *testing.T) { - t.Parallel() - testDB, release := acquire(t) - defer release() - ctx := context.Background() - - check := func(v string) { - info, err := internal.NewRawLatestInfo("m", v, []byte(`module m`)) - if err != nil { - t.Fatal(err) - } - if err := testDB.UpdateRawLatestInfo(ctx, info); err != nil { - t.Fatal(err) - } - got, err := testDB.GetRawLatestInfo(ctx, "m") - if err != nil { - t.Fatal(err) - } - if diff := cmp.Diff(info, got, cmp.AllowUnexported(internal.RawLatestInfo{})); diff != "" { - t.Fatalf("mismatch (-want, +got): %s", diff) - } - } - - check("v1.0.0") - check("v1.2.3") -} - func TestShouldUpdateRawLatest(t *testing.T) { for _, test := range []struct { new, cur string diff --git a/internal/raw_latest.go b/internal/raw_latest.go deleted file mode 100644 index eb9c20c0..00000000 --- a/internal/raw_latest.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package internal - -import ( - "fmt" - "strings" - - "golang.org/x/mod/modfile" - "golang.org/x/mod/semver" -) - -// RawLatestInfo describes the "raw" latest version of a module: -// the latest version without considering retractions or the like. -// The go.mod file of the raw latest version establishes whether -// the module is deprecated, and what versions are retracted. -type RawLatestInfo struct { - ModulePath string - Version string - GoModFile *modfile.File - deprecated bool - deprecationComment string -} - -func NewRawLatestInfo(modulePath, version string, modBytes []byte) (*RawLatestInfo, error) { - f, err := modfile.ParseLax(fmt.Sprintf("%s@%s/go.mod", modulePath, version), modBytes, nil) - if err != nil { - return nil, err - } - dep, comment := isDeprecated(f) - return &RawLatestInfo{ - ModulePath: modulePath, - Version: version, - GoModFile: f, - deprecated: dep, - deprecationComment: comment, - }, nil -} - -// PopulateModuleInfo uses the RawLatestInfo to populate fields of the given module. -func (r *RawLatestInfo) PopulateModuleInfo(mi *ModuleInfo) { - if r == nil { - return - } - mi.Deprecated = r.deprecated - mi.DeprecationComment = r.deprecationComment - mi.Retracted, mi.RetractionRationale = isRetracted(r.GoModFile, mi.Version) -} - -// isDeprecated reports whether the go.mod deprecates this module. -// It looks for "Deprecated" comments in the line comments before and next to -// the module declaration. If it finds one, it returns true along with the text -// after "Deprecated:". Otherwise it returns false, "". -func isDeprecated(mf *modfile.File) (bool, string) { - const prefix = "Deprecated:" - - if mf.Module == nil { - return false, "" - } - for _, comment := range append(mf.Module.Syntax.Before, mf.Module.Syntax.Suffix...) { - text := strings.TrimSpace(strings.TrimPrefix(comment.Token, "//")) - if strings.HasPrefix(text, prefix) { - return true, strings.TrimSpace(text[len(prefix):]) - } - } - return false, "" -} - -// isRetracted reports whether the go.mod file retracts the version. -// If so, it returns true along with the rationale for the retraction. -func isRetracted(mf *modfile.File, resolvedVersion string) (bool, string) { - for _, r := range mf.Retract { - if semver.Compare(resolvedVersion, r.Low) >= 0 && semver.Compare(resolvedVersion, r.High) <= 0 { - return true, r.Rationale - } - } - return false, "" -} diff --git a/internal/worker/fetch.go b/internal/worker/fetch.go index 1b6029ad..788c65f0 100644 --- a/internal/worker/fetch.go +++ b/internal/worker/fetch.go @@ -394,7 +394,7 @@ func logTaskResult(ctx context.Context, ft *fetchTask, prefix string) { prefix, ft.ModulePath, ft.ResolvedVersion, ft.Status, len(ft.PackageVersionStates), ft.Error, msg) } -// fetchAndUpdateLatest fetches information about the raw latest version from the proxy, +// fetchAndUpdateLatest fetches information about the latest versions from the proxy, // and updates the database if the version has changed. func (f *Fetcher) fetchAndUpdateLatest(ctx context.Context, modulePath string) (err error) { defer derrors.Wrap(&err, "fetchAndUpdateLatest(%q)", modulePath)