зеркало из https://github.com/golang/pkgsite.git
many: remove raw-latest code
The raw-latest system is replaced by the latest-version system. For golang/go#44710 Change-Id: Id59f03aa3078c05e2a91b58d92633cc73dea4c3e Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/298234 Trust: Jonathan Amsterdam <jba@google.com> Run-TryBot: Jonathan Amsterdam <jba@google.com> TryBot-Result: kokoro <noreply+kokoro@google.com> Reviewed-by: Julie Qiu <julie@golang.org>
This commit is contained in:
Jonathan Amsterdam
Родитель
1fabe838f6
Коммит
a310f439ee
4
all.bash
4
all.bash
|
|
@ -47,13 +47,13 @@ modified_files() {
|
|||
fi
|
||||
}
|
||||
|
||||
|
||||
# Helper for modified_files. It asks git for all modified, added or deleted
|
||||
# files, and keeps only the latter two.
|
||||
diff_files() {
|
||||
git diff --name-status $* | awk '$1 != "D" { print $2 }'
|
||||
git diff --name-status $* | awk '$1 ~ /^R/ { print $3; next } $1 != "D" { print $2 }'
|
||||
}
|
||||
|
||||
|
||||
# codedirs lists directories that contain discovery code. If they include
|
||||
# directories containing external code, those directories must be excluded in
|
||||
# findcode below.
|
||||
|
|
|
|||
|
|
@ -1,101 +0,0 @@
|
|||
// Copyright 2021 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"golang.org/x/pkgsite/internal"
|
||||
"golang.org/x/pkgsite/internal/derrors"
|
||||
"golang.org/x/pkgsite/internal/log"
|
||||
"golang.org/x/pkgsite/internal/proxy"
|
||||
"golang.org/x/pkgsite/internal/stdlib"
|
||||
"golang.org/x/pkgsite/internal/version"
|
||||
)
|
||||
|
||||
// RawLatestInfo uses the proxy to get information about the raw latest version
|
||||
// of modulePath. If it cannot obtain it, it returns (nil, nil).
|
||||
//
|
||||
// The hasGoMod function that is passed in should check if version v of the
|
||||
// module has a go.mod file, using a source other than the proxy (e.g. a
|
||||
// database). If it doesn't have enough information to decide, it should return
|
||||
// an error that wraps derrors.NotFound.
|
||||
func RawLatestInfo(ctx context.Context, modulePath string, prox *proxy.Client, hasGoMod func(v string) (bool, error)) (_ *internal.RawLatestInfo, err error) {
|
||||
defer derrors.WrapStack(&err, "RawLatestInfo(%q)", modulePath)
|
||||
|
||||
// No raw latest info for std; no deprecations or retractions.
|
||||
if modulePath == stdlib.ModulePath {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
v, err := fetchRawLatestVersion(ctx, modulePath, prox, hasGoMod)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
modBytes, err := prox.Mod(ctx, modulePath, v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return internal.NewRawLatestInfo(modulePath, v, modBytes)
|
||||
}
|
||||
|
||||
// fetchRawLatestVersion uses the proxy to determine the latest
|
||||
// version of a module independent of retractions or other modifications.
|
||||
//
|
||||
// This meaning of "latest" is defined at https://golang.org/ref/mod#version-queries.
|
||||
// That definition does not deal with a subtlety involving
|
||||
// incompatible versions. The actual definition is embodied in the go command's
|
||||
// queryMatcher.filterVersions method. This code is a rewrite of that method at Go
|
||||
// version 1.16
|
||||
// (https://go.googlesource.com/go/+/refs/tags/go1.16/src/cmd/go/internal/modload/query.go#441).
|
||||
func fetchRawLatestVersion(ctx context.Context, modulePath string, prox *proxy.Client, hasGoMod func(v string) (bool, error)) (v string, err error) {
|
||||
defer derrors.WrapStack(&err, "fetchRawLatestVersion(%q)", modulePath)
|
||||
|
||||
defer func() {
|
||||
log.Debugf(ctx, "fetchRawLatestVersion(%q) => (%q, %v)", modulePath, v, err)
|
||||
}()
|
||||
|
||||
// Get tagged versions from the proxy's list endpoint.
|
||||
taggedVersions, err := prox.Versions(ctx, modulePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// If there are no tagged versions, use the proxy's @latest endpoint.
|
||||
if len(taggedVersions) == 0 {
|
||||
latestInfo, err := prox.Info(ctx, modulePath, internal.LatestVersion)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return latestInfo.Version, nil
|
||||
}
|
||||
|
||||
// Find the latest of all tagged versions.
|
||||
hasGoModFunc := func(v string) (bool, error) {
|
||||
var (
|
||||
has bool
|
||||
err error
|
||||
)
|
||||
if hasGoMod == nil {
|
||||
err = derrors.NotFound
|
||||
} else {
|
||||
has, err = hasGoMod(v)
|
||||
}
|
||||
if err == nil {
|
||||
return has, nil
|
||||
} else if !errors.Is(err, derrors.NotFound) {
|
||||
return false, err
|
||||
} else {
|
||||
// hasGoMod doesn't know; download the zip.
|
||||
zr, err := prox.Zip(ctx, modulePath, v)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return hasGoModFile(zr, modulePath, v), nil
|
||||
}
|
||||
}
|
||||
return version.Latest(taggedVersions, hasGoModFunc)
|
||||
|
||||
}
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
// Copyright 2021 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/pkgsite/internal/proxy"
|
||||
)
|
||||
|
||||
func TestFetchRawLatestVersion(t *testing.T) {
|
||||
prox, teardown := proxy.SetupTestClient(t, testModules)
|
||||
defer teardown()
|
||||
|
||||
for _, test := range []struct {
|
||||
module string
|
||||
want string
|
||||
}{
|
||||
{"example.com/basic", "v1.1.0"},
|
||||
{"example.com/single", "v1.0.0"},
|
||||
} {
|
||||
got, err := fetchRawLatestVersion(context.Background(), test.module, prox, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != test.want {
|
||||
t.Errorf("%s: got %s, want %s", test.module, got, test.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawLatestInfo(t *testing.T) {
|
||||
// fetchRawLatestVersion is tested above.
|
||||
// Contents of the go.mod file are tested in proxydatasource.
|
||||
// Here, just test that there is a parsed go.mod file.
|
||||
prox, teardown := proxy.SetupTestClient(t, testModules)
|
||||
defer teardown()
|
||||
|
||||
const module = "example.com/basic"
|
||||
got, err := RawLatestInfo(context.Background(), module, prox, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.ModulePath != module || got.Version != "v1.1.0" || got.GoModFile == nil {
|
||||
t.Errorf("got (%q, %q, %p), want (%q, 'v1.1.0', <non-nil>)", got.ModulePath, got.Version, got.GoModFile, module)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ package internal
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/mod/modfile"
|
||||
"golang.org/x/mod/semver"
|
||||
|
|
@ -42,6 +43,25 @@ func NewLatestModuleVersions(modulePath, raw, cooked, good string, modBytes []by
|
|||
}, nil
|
||||
}
|
||||
|
||||
// isDeprecated reports whether the go.mod deprecates this module.
|
||||
// It looks for "Deprecated" comments in the line comments before and next to
|
||||
// the module declaration. If it finds one, it returns true along with the text
|
||||
// after "Deprecated:". Otherwise it returns false, "".
|
||||
func isDeprecated(mf *modfile.File) (bool, string) {
|
||||
const prefix = "Deprecated:"
|
||||
|
||||
if mf.Module == nil {
|
||||
return false, ""
|
||||
}
|
||||
for _, comment := range append(mf.Module.Syntax.Before, mf.Module.Syntax.Suffix...) {
|
||||
text := strings.TrimSpace(strings.TrimPrefix(comment.Token, "//"))
|
||||
if strings.HasPrefix(text, prefix) {
|
||||
return true, strings.TrimSpace(text[len(prefix):])
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// PopulateModuleInfo uses the LatestModuleVersions to populate fields of the given module.
|
||||
func (li *LatestModuleVersions) PopulateModuleInfo(mi *ModuleInfo) {
|
||||
mi.Deprecated = li.deprecated
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ func TestIsRetracted(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotIs, gotRationale := isRetracted(mf, "v1.2.3")
|
||||
gotIs, gotRationale := IsRetracted(mf, "v1.2.3")
|
||||
if gotIs != test.wantIs || gotRationale != test.wantRationale {
|
||||
t.Errorf("%s: got (%t, %q), want(%t, %q)", test.name, gotIs, gotRationale, test.wantIs, test.wantRationale)
|
||||
}
|
||||
|
|
@ -268,63 +268,6 @@ func (db *DB) getLatestMinorModuleVersionInfo(ctx context.Context, unitPath, mod
|
|||
}
|
||||
}
|
||||
|
||||
// GetRawLatestInfo returns the row of the raw_latest_versions table for modulePath.
|
||||
// If the module path is not found, it returns nil, nil.
|
||||
func (db *DB) GetRawLatestInfo(ctx context.Context, modulePath string) (_ *internal.RawLatestInfo, err error) {
|
||||
defer derrors.WrapStack(&err, "GetRawLatestInfo(%q)", modulePath)
|
||||
|
||||
var (
|
||||
version string
|
||||
goModBytes []byte
|
||||
)
|
||||
err = db.db.QueryRow(ctx, `
|
||||
SELECT r.version, r.go_mod_bytes
|
||||
FROM raw_latest_versions r
|
||||
INNER JOIN paths p ON p.id = r.module_path_id
|
||||
WHERE p.path = $1`,
|
||||
modulePath).Scan(&version, &goModBytes)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return internal.NewRawLatestInfo(modulePath, version, goModBytes)
|
||||
}
|
||||
|
||||
// UpdateRawLatestInfo upserts its argument into the raw_latest_versions table
|
||||
// if the row doesn't exist, or the new version is later.
|
||||
func (db *DB) UpdateRawLatestInfo(ctx context.Context, info *internal.RawLatestInfo) (err error) {
|
||||
defer derrors.WrapStack(&err, "UpdateRawLatestInfo(%q)", info.ModulePath)
|
||||
|
||||
// We need RepeatableRead here because the INSERT...ON CONFLICT does a read.
|
||||
return db.db.Transact(ctx, sql.LevelRepeatableRead, func(tx *database.DB) error {
|
||||
var (
|
||||
id int
|
||||
curVersion string
|
||||
)
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT p.id, r.version
|
||||
FROM raw_latest_versions r
|
||||
INNER JOIN paths p ON p.id = r.module_path_id
|
||||
WHERE p.path = $1`,
|
||||
info.ModulePath).Scan(&id, &curVersion)
|
||||
switch {
|
||||
case err == sql.ErrNoRows:
|
||||
// Fall through to upsert.
|
||||
case err != nil:
|
||||
return err
|
||||
default:
|
||||
if !shouldUpdateRawLatest(info.Version, curVersion) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
log.Debugf(ctx, "updating raw latest from %q to %q", curVersion, info.Version)
|
||||
return upsertRawLatestInfo(ctx, tx, id, info)
|
||||
})
|
||||
}
|
||||
|
||||
func shouldUpdateRawLatest(newVersion, curVersion string) bool {
|
||||
// Only update if the new one is later according to version.Later
|
||||
// (semver except that release > prerelease). that avoids a race
|
||||
|
|
@ -341,37 +284,6 @@ func shouldUpdateRawLatest(newVersion, curVersion string) bool {
|
|||
(version.IsIncompatible(curVersion) && !version.IsIncompatible(newVersion))
|
||||
}
|
||||
|
||||
func upsertRawLatestInfo(ctx context.Context, tx *database.DB, id int, info *internal.RawLatestInfo) (err error) {
|
||||
defer derrors.WrapStack(&err, "upsertRawLatestInfo(%d, %q, %q)", id, info.ModulePath, info.Version)
|
||||
|
||||
// If the row doesn't exist, get a path ID for the module path.
|
||||
if id == 0 {
|
||||
id, err = upsertPath(ctx, tx, info.ModulePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Convert the go.mod file into bytes.
|
||||
goModBytes, err := info.GoModFile.Format()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO raw_latest_versions (
|
||||
module_path_id,
|
||||
version,
|
||||
go_mod_bytes
|
||||
) VALUES ($1, $2, $3)
|
||||
ON CONFLICT (module_path_id)
|
||||
DO UPDATE SET
|
||||
version=excluded.version,
|
||||
go_mod_bytes=excluded.go_mod_bytes
|
||||
`,
|
||||
id, info.Version, goModBytes)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetLatestModuleVersions returns the row of the latest_module_versions table for modulePath.
|
||||
// If the module path is not found, it returns nil, nil.
|
||||
func (db *DB) GetLatestModuleVersions(ctx context.Context, modulePath string) (_ *internal.LatestModuleVersions, err error) {
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ func TestGetVersions(t *testing.T) {
|
|||
for _, m := range testModules {
|
||||
MustInsertModule(ctx, t, testDB, m)
|
||||
}
|
||||
// Add raw latest version info for rootModule.
|
||||
// Add latest version info for rootModule.
|
||||
addLatest(ctx, t, testDB, rootModule, "v1.1.0", `
|
||||
module golang.org/foo/bar // Deprecated: use other
|
||||
retract v1.0.3 // security flaw
|
||||
|
|
@ -369,33 +369,6 @@ func TestGetLatestInfo(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestRawLatestInfo(t *testing.T) {
|
||||
t.Parallel()
|
||||
testDB, release := acquire(t)
|
||||
defer release()
|
||||
ctx := context.Background()
|
||||
|
||||
check := func(v string) {
|
||||
info, err := internal.NewRawLatestInfo("m", v, []byte(`module m`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := testDB.UpdateRawLatestInfo(ctx, info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := testDB.GetRawLatestInfo(ctx, "m")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if diff := cmp.Diff(info, got, cmp.AllowUnexported(internal.RawLatestInfo{})); diff != "" {
|
||||
t.Fatalf("mismatch (-want, +got): %s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
check("v1.0.0")
|
||||
check("v1.2.3")
|
||||
}
|
||||
|
||||
func TestShouldUpdateRawLatest(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
new, cur string
|
||||
|
|
|
|||
|
|
@ -1,80 +0,0 @@
|
|||
// Copyright 2021 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package internal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/mod/modfile"
|
||||
"golang.org/x/mod/semver"
|
||||
)
|
||||
|
||||
// RawLatestInfo describes the "raw" latest version of a module:
|
||||
// the latest version without considering retractions or the like.
|
||||
// The go.mod file of the raw latest version establishes whether
|
||||
// the module is deprecated, and what versions are retracted.
|
||||
type RawLatestInfo struct {
|
||||
ModulePath string
|
||||
Version string
|
||||
GoModFile *modfile.File
|
||||
deprecated bool
|
||||
deprecationComment string
|
||||
}
|
||||
|
||||
func NewRawLatestInfo(modulePath, version string, modBytes []byte) (*RawLatestInfo, error) {
|
||||
f, err := modfile.ParseLax(fmt.Sprintf("%s@%s/go.mod", modulePath, version), modBytes, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dep, comment := isDeprecated(f)
|
||||
return &RawLatestInfo{
|
||||
ModulePath: modulePath,
|
||||
Version: version,
|
||||
GoModFile: f,
|
||||
deprecated: dep,
|
||||
deprecationComment: comment,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PopulateModuleInfo uses the RawLatestInfo to populate fields of the given module.
|
||||
func (r *RawLatestInfo) PopulateModuleInfo(mi *ModuleInfo) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
mi.Deprecated = r.deprecated
|
||||
mi.DeprecationComment = r.deprecationComment
|
||||
mi.Retracted, mi.RetractionRationale = isRetracted(r.GoModFile, mi.Version)
|
||||
}
|
||||
|
||||
// isDeprecated reports whether the go.mod deprecates this module.
|
||||
// It looks for "Deprecated" comments in the line comments before and next to
|
||||
// the module declaration. If it finds one, it returns true along with the text
|
||||
// after "Deprecated:". Otherwise it returns false, "".
|
||||
func isDeprecated(mf *modfile.File) (bool, string) {
|
||||
const prefix = "Deprecated:"
|
||||
|
||||
if mf.Module == nil {
|
||||
return false, ""
|
||||
}
|
||||
for _, comment := range append(mf.Module.Syntax.Before, mf.Module.Syntax.Suffix...) {
|
||||
text := strings.TrimSpace(strings.TrimPrefix(comment.Token, "//"))
|
||||
if strings.HasPrefix(text, prefix) {
|
||||
return true, strings.TrimSpace(text[len(prefix):])
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// isRetracted reports whether the go.mod file retracts the version.
|
||||
// If so, it returns true along with the rationale for the retraction.
|
||||
func isRetracted(mf *modfile.File, resolvedVersion string) (bool, string) {
|
||||
for _, r := range mf.Retract {
|
||||
if semver.Compare(resolvedVersion, r.Low) >= 0 && semver.Compare(resolvedVersion, r.High) <= 0 {
|
||||
return true, r.Rationale
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
|
@ -394,7 +394,7 @@ func logTaskResult(ctx context.Context, ft *fetchTask, prefix string) {
|
|||
prefix, ft.ModulePath, ft.ResolvedVersion, ft.Status, len(ft.PackageVersionStates), ft.Error, msg)
|
||||
}
|
||||
|
||||
// fetchAndUpdateLatest fetches information about the raw latest version from the proxy,
|
||||
// fetchAndUpdateLatest fetches information about the latest versions from the proxy,
|
||||
// and updates the database if the version has changed.
|
||||
func (f *Fetcher) fetchAndUpdateLatest(ctx context.Context, modulePath string) (err error) {
|
||||
defer derrors.Wrap(&err, "fetchAndUpdateLatest(%q)", modulePath)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче