go
/
pkgsite
зеркало из https://github.com/golang/pkgsite.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

many: remove raw-latest code 

The raw-latest system is replaced by the latest-version system.

For golang/go#44710

Change-Id: Id59f03aa3078c05e2a91b58d92633cc73dea4c3e
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/298234
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
This commit is contained in:
Jonathan Amsterdam 2021-03-03 10:07:46 -05:00
Родитель 1fabe838f6
Коммит a310f439ee
9 изменённых файлов: 25 добавлений и 352 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
all.bash
Просмотреть файл

@ -47,13 +47,13 @@ modified_files() {
fi fi
} }
# Helper for modified_files. It asks git for all modified, added or deleted # Helper for modified_files. It asks git for all modified, added or deleted
# files, and keeps only the latter two. # files, and keeps only the latter two.
diff_files() { diff_files() {
git diff --name-status $* | awk '$1 != "D" { print $2 }' git diff --name-status $* | awk '$1 ~ /^R/ { print $3; next } $1 != "D" { print $2 }'
} }
# codedirs lists directories that contain discovery code. If they include # codedirs lists directories that contain discovery code. If they include
# directories containing external code, those directories must be excluded in # directories containing external code, those directories must be excluded in
# findcode below. # findcode below.

101
internal/fetch/raw_latest.go
Просмотреть файл

@ -1,101 +0,0 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fetch
import (
"context"
"errors"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/proxy"
"golang.org/x/pkgsite/internal/stdlib"
"golang.org/x/pkgsite/internal/version"
)
// RawLatestInfo uses the proxy to get information about the raw latest version
// of modulePath. If it cannot obtain it, it returns (nil, nil).
//
// The hasGoMod function that is passed in should check if version v of the
// module has a go.mod file, using a source other than the proxy (e.g. a
// database). If it doesn't have enough information to decide, it should return
// an error that wraps derrors.NotFound.
func RawLatestInfo(ctx context.Context, modulePath string, prox *proxy.Client, hasGoMod func(v string) (bool, error)) (_ *internal.RawLatestInfo, err error) {
defer derrors.WrapStack(&err, "RawLatestInfo(%q)", modulePath)
// No raw latest info for std; no deprecations or retractions.
if modulePath == stdlib.ModulePath {
return nil, nil
}
v, err := fetchRawLatestVersion(ctx, modulePath, prox, hasGoMod)
if err != nil {
return nil, err
}
modBytes, err := prox.Mod(ctx, modulePath, v)
if err != nil {
return nil, err
}
return internal.NewRawLatestInfo(modulePath, v, modBytes)
}
// fetchRawLatestVersion uses the proxy to determine the latest
// version of a module independent of retractions or other modifications.
//
// This meaning of "latest" is defined at https://golang.org/ref/mod#version-queries.
// That definition does not deal with a subtlety involving
// incompatible versions. The actual definition is embodied in the go command's
// queryMatcher.filterVersions method. This code is a rewrite of that method at Go
// version 1.16
// (https://go.googlesource.com/go/+/refs/tags/go1.16/src/cmd/go/internal/modload/query.go#441).
func fetchRawLatestVersion(ctx context.Context, modulePath string, prox *proxy.Client, hasGoMod func(v string) (bool, error)) (v string, err error) {
defer derrors.WrapStack(&err, "fetchRawLatestVersion(%q)", modulePath)
defer func() {
log.Debugf(ctx, "fetchRawLatestVersion(%q) => (%q, %v)", modulePath, v, err)
}()
// Get tagged versions from the proxy's list endpoint.
taggedVersions, err := prox.Versions(ctx, modulePath)
if err != nil {
return "", err
}
// If there are no tagged versions, use the proxy's @latest endpoint.
if len(taggedVersions) == 0 {
latestInfo, err := prox.Info(ctx, modulePath, internal.LatestVersion)
if err != nil {
return "", err
}
return latestInfo.Version, nil
}
// Find the latest of all tagged versions.
hasGoModFunc := func(v string) (bool, error) {
var (
has bool
err error
)
if hasGoMod == nil {
err = derrors.NotFound
} else {
has, err = hasGoMod(v)
}
if err == nil {
return has, nil
} else if !errors.Is(err, derrors.NotFound) {
return false, err
} else {
// hasGoMod doesn't know; download the zip.
zr, err := prox.Zip(ctx, modulePath, v)
if err != nil {
return false, err
}
return hasGoModFile(zr, modulePath, v), nil
}
}
return version.Latest(taggedVersions, hasGoModFunc)
}

51
internal/fetch/raw_latest_test.go
Просмотреть файл

@ -1,51 +0,0 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fetch
import (
"context"
"testing"
"golang.org/x/pkgsite/internal/proxy"
)
func TestFetchRawLatestVersion(t *testing.T) {
prox, teardown := proxy.SetupTestClient(t, testModules)
defer teardown()
for _, test := range []struct {
module string
want string
}{
{"example.com/basic", "v1.1.0"},
{"example.com/single", "v1.0.0"},
} {
got, err := fetchRawLatestVersion(context.Background(), test.module, prox, nil)
if err != nil {
t.Fatal(err)
}
if got != test.want {
t.Errorf("%s: got %s, want %s", test.module, got, test.want)
}
}
}
func TestRawLatestInfo(t *testing.T) {
// fetchRawLatestVersion is tested above.
// Contents of the go.mod file are tested in proxydatasource.
// Here, just test that there is a parsed go.mod file.
prox, teardown := proxy.SetupTestClient(t, testModules)
defer teardown()
const module = "example.com/basic"
got, err := RawLatestInfo(context.Background(), module, prox, nil)
if err != nil {
t.Fatal(err)
}
if got.ModulePath != module || got.Version != "v1.1.0" || got.GoModFile == nil {
t.Errorf("got (%q, %q, %p), want (%q, 'v1.1.0', <non-nil>)", got.ModulePath, got.Version, got.GoModFile, module)
}
}

20
internal/latest.go
Просмотреть файл

@ -6,6 +6,7 @@ package internal
import ( import (
"fmt" "fmt"
"strings"
"golang.org/x/mod/modfile" "golang.org/x/mod/modfile"
"golang.org/x/mod/semver" "golang.org/x/mod/semver"
@ -42,6 +43,25 @@ func NewLatestModuleVersions(modulePath, raw, cooked, good string, modBytes []by
}, nil }, nil
} }
// isDeprecated reports whether the go.mod deprecates this module.
// It looks for "Deprecated" comments in the line comments before and next to
// the module declaration. If it finds one, it returns true along with the text
// after "Deprecated:". Otherwise it returns false, "".
func isDeprecated(mf *modfile.File) (bool, string) {
const prefix = "Deprecated:"
if mf.Module == nil {
return false, ""
}
for _, comment := range append(mf.Module.Syntax.Before, mf.Module.Syntax.Suffix...) {
text := strings.TrimSpace(strings.TrimPrefix(comment.Token, "//"))
if strings.HasPrefix(text, prefix) {
return true, strings.TrimSpace(text[len(prefix):])
}
}
return false, ""
}
// PopulateModuleInfo uses the LatestModuleVersions to populate fields of the given module. // PopulateModuleInfo uses the LatestModuleVersions to populate fields of the given module.
func (li *LatestModuleVersions) PopulateModuleInfo(mi *ModuleInfo) { func (li *LatestModuleVersions) PopulateModuleInfo(mi *ModuleInfo) {
mi.Deprecated = li.deprecated mi.Deprecated = li.deprecated

2
internal/raw_latest_test.go → internal/latest_test.go
Просмотреть файл

@ -85,7 +85,7 @@ func TestIsRetracted(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
gotIs, gotRationale := isRetracted(mf, "v1.2.3") gotIs, gotRationale := IsRetracted(mf, "v1.2.3")
if gotIs != test.wantIs || gotRationale != test.wantRationale { if gotIs != test.wantIs || gotRationale != test.wantRationale {
t.Errorf("%s: got (%t, %q), want(%t, %q)", test.name, gotIs, gotRationale, test.wantIs, test.wantRationale) t.Errorf("%s: got (%t, %q), want(%t, %q)", test.name, gotIs, gotRationale, test.wantIs, test.wantRationale)
} }

88
internal/postgres/version.go
Просмотреть файл

@ -268,63 +268,6 @@ func (db *DB) getLatestMinorModuleVersionInfo(ctx context.Context, unitPath, mod
} }
} }
// GetRawLatestInfo returns the row of the raw_latest_versions table for modulePath.
// If the module path is not found, it returns nil, nil.
func (db *DB) GetRawLatestInfo(ctx context.Context, modulePath string) (_ *internal.RawLatestInfo, err error) {
defer derrors.WrapStack(&err, "GetRawLatestInfo(%q)", modulePath)
var (
version string
goModBytes []byte
)
err = db.db.QueryRow(ctx, `
SELECT r.version, r.go_mod_bytes
FROM raw_latest_versions r
INNER JOIN paths p ON p.id = r.module_path_id
WHERE p.path = $1`,
modulePath).Scan(&version, &goModBytes)
if err != nil {
if err == sql.ErrNoRows {
return nil, nil
}
return nil, err
}
return internal.NewRawLatestInfo(modulePath, version, goModBytes)
}
// UpdateRawLatestInfo upserts its argument into the raw_latest_versions table
// if the row doesn't exist, or the new version is later.
func (db *DB) UpdateRawLatestInfo(ctx context.Context, info *internal.RawLatestInfo) (err error) {
defer derrors.WrapStack(&err, "UpdateRawLatestInfo(%q)", info.ModulePath)
// We need RepeatableRead here because the INSERT...ON CONFLICT does a read.
return db.db.Transact(ctx, sql.LevelRepeatableRead, func(tx *database.DB) error {
var (
id int
curVersion string
)
err = tx.QueryRow(ctx, `
SELECT p.id, r.version
FROM raw_latest_versions r
INNER JOIN paths p ON p.id = r.module_path_id
WHERE p.path = $1`,
info.ModulePath).Scan(&id, &curVersion)
switch {
case err == sql.ErrNoRows:
// Fall through to upsert.
case err != nil:
return err
default:
if !shouldUpdateRawLatest(info.Version, curVersion) {
return nil
}
}
log.Debugf(ctx, "updating raw latest from %q to %q", curVersion, info.Version)
return upsertRawLatestInfo(ctx, tx, id, info)
})
}
func shouldUpdateRawLatest(newVersion, curVersion string) bool { func shouldUpdateRawLatest(newVersion, curVersion string) bool {
// Only update if the new one is later according to version.Later // Only update if the new one is later according to version.Later
// (semver except that release > prerelease). that avoids a race // (semver except that release > prerelease). that avoids a race
@ -341,37 +284,6 @@ func shouldUpdateRawLatest(newVersion, curVersion string) bool {
(version.IsIncompatible(curVersion) && !version.IsIncompatible(newVersion)) (version.IsIncompatible(curVersion) && !version.IsIncompatible(newVersion))
} }
func upsertRawLatestInfo(ctx context.Context, tx *database.DB, id int, info *internal.RawLatestInfo) (err error) {
defer derrors.WrapStack(&err, "upsertRawLatestInfo(%d, %q, %q)", id, info.ModulePath, info.Version)
// If the row doesn't exist, get a path ID for the module path.
if id == 0 {
id, err = upsertPath(ctx, tx, info.ModulePath)
if err != nil {
return err
}
}
// Convert the go.mod file into bytes.
goModBytes, err := info.GoModFile.Format()
if err != nil {
return err
}
_, err = tx.Exec(ctx, `
INSERT INTO raw_latest_versions (
module_path_id,
version,
go_mod_bytes
) VALUES ($1, $2, $3)
ON CONFLICT (module_path_id)
DO UPDATE SET
version=excluded.version,
go_mod_bytes=excluded.go_mod_bytes
`,
id, info.Version, goModBytes)
return err
}
// GetLatestModuleVersions returns the row of the latest_module_versions table for modulePath. // GetLatestModuleVersions returns the row of the latest_module_versions table for modulePath.
// If the module path is not found, it returns nil, nil. // If the module path is not found, it returns nil, nil.
func (db *DB) GetLatestModuleVersions(ctx context.Context, modulePath string) (_ *internal.LatestModuleVersions, err error) { func (db *DB) GetLatestModuleVersions(ctx context.Context, modulePath string) (_ *internal.LatestModuleVersions, err error) {

29
internal/postgres/version_test.go
Просмотреть файл

@ -64,7 +64,7 @@ func TestGetVersions(t *testing.T) {
for _, m := range testModules { for _, m := range testModules {
MustInsertModule(ctx, t, testDB, m) MustInsertModule(ctx, t, testDB, m)
} }
// Add raw latest version info for rootModule. // Add latest version info for rootModule.
addLatest(ctx, t, testDB, rootModule, "v1.1.0", ` addLatest(ctx, t, testDB, rootModule, "v1.1.0", `
module golang.org/foo/bar // Deprecated: use other module golang.org/foo/bar // Deprecated: use other
retract v1.0.3 // security flaw retract v1.0.3 // security flaw
@ -369,33 +369,6 @@ func TestGetLatestInfo(t *testing.T) {
} }
} }
func TestRawLatestInfo(t *testing.T) {
t.Parallel()
testDB, release := acquire(t)
defer release()
ctx := context.Background()
check := func(v string) {
info, err := internal.NewRawLatestInfo("m", v, []byte(`module m`))
if err != nil {
t.Fatal(err)
}
if err := testDB.UpdateRawLatestInfo(ctx, info); err != nil {
t.Fatal(err)
}
got, err := testDB.GetRawLatestInfo(ctx, "m")
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(info, got, cmp.AllowUnexported(internal.RawLatestInfo{})); diff != "" {
t.Fatalf("mismatch (-want, +got): %s", diff)
}
}
check("v1.0.0")
check("v1.2.3")
}
func TestShouldUpdateRawLatest(t *testing.T) { func TestShouldUpdateRawLatest(t *testing.T) {
for _, test := range []struct { for _, test := range []struct {
new, cur string new, cur string

80
internal/raw_latest.go
Просмотреть файл

@ -1,80 +0,0 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package internal
import (
"fmt"
"strings"
"golang.org/x/mod/modfile"
"golang.org/x/mod/semver"
)
// RawLatestInfo describes the "raw" latest version of a module:
// the latest version without considering retractions or the like.
// The go.mod file of the raw latest version establishes whether
// the module is deprecated, and what versions are retracted.
type RawLatestInfo struct {
ModulePath string
Version string
GoModFile *modfile.File
deprecated bool
deprecationComment string
}
func NewRawLatestInfo(modulePath, version string, modBytes []byte) (*RawLatestInfo, error) {
f, err := modfile.ParseLax(fmt.Sprintf("%s@%s/go.mod", modulePath, version), modBytes, nil)
if err != nil {
return nil, err
}
dep, comment := isDeprecated(f)
return &RawLatestInfo{
ModulePath: modulePath,
Version: version,
GoModFile: f,
deprecated: dep,
deprecationComment: comment,
}, nil
}
// PopulateModuleInfo uses the RawLatestInfo to populate fields of the given module.
func (r *RawLatestInfo) PopulateModuleInfo(mi *ModuleInfo) {
if r == nil {
return
}
mi.Deprecated = r.deprecated
mi.DeprecationComment = r.deprecationComment
mi.Retracted, mi.RetractionRationale = isRetracted(r.GoModFile, mi.Version)
}
// isDeprecated reports whether the go.mod deprecates this module.
// It looks for "Deprecated" comments in the line comments before and next to
// the module declaration. If it finds one, it returns true along with the text
// after "Deprecated:". Otherwise it returns false, "".
func isDeprecated(mf *modfile.File) (bool, string) {
const prefix = "Deprecated:"
if mf.Module == nil {
return false, ""
}
for _, comment := range append(mf.Module.Syntax.Before, mf.Module.Syntax.Suffix...) {
text := strings.TrimSpace(strings.TrimPrefix(comment.Token, "//"))
if strings.HasPrefix(text, prefix) {
return true, strings.TrimSpace(text[len(prefix):])
}
}
return false, ""
}
// isRetracted reports whether the go.mod file retracts the version.
// If so, it returns true along with the rationale for the retraction.
func isRetracted(mf *modfile.File, resolvedVersion string) (bool, string) {
for _, r := range mf.Retract {
if semver.Compare(resolvedVersion, r.Low) >= 0 && semver.Compare(resolvedVersion, r.High) <= 0 {
return true, r.Rationale
}
}
return false, ""
}

2
internal/worker/fetch.go
Просмотреть файл

@ -394,7 +394,7 @@ func logTaskResult(ctx context.Context, ft *fetchTask, prefix string) {
prefix, ft.ModulePath, ft.ResolvedVersion, ft.Status, len(ft.PackageVersionStates), ft.Error, msg) prefix, ft.ModulePath, ft.ResolvedVersion, ft.Status, len(ft.PackageVersionStates), ft.Error, msg)
} }
// fetchAndUpdateLatest fetches information about the raw latest version from the proxy, // fetchAndUpdateLatest fetches information about the latest versions from the proxy,
// and updates the database if the version has changed. // and updates the database if the version has changed.
func (f *Fetcher) fetchAndUpdateLatest(ctx context.Context, modulePath string) (err error) { func (f *Fetcher) fetchAndUpdateLatest(ctx context.Context, modulePath string) (err error) {
defer derrors.Wrap(&err, "fetchAndUpdateLatest(%q)", modulePath) defer derrors.Wrap(&err, "fetchAndUpdateLatest(%q)", modulePath)