зеркало из https://github.com/golang/pkgsite.git
internal, internal/fetch: add VersionType and getVersionType
The database needs an enum column called version_type to indicate the type of version a package has. So to be able to insert packages with the correct version type VersionType is being added along with function getVersionType to parse a package's version to determine what kind of VersionType it should have. getVersionType is also used to make sure that the version is valid in FetchAndInsertVersion. Change-Id: Ia15c23c3783dbb83934ce8b0b580a25bafa219c2 Reviewed-on: https://team-review.git.corp.google.com/c/golang/discovery/+/437333 Reviewed-by: Andrew Bonventre <andybons@google.com>
This commit is contained in:
Channing Kimble-Brown
Julie Qiu
Родитель
7010dcd7f2
Коммит
a6b5740369
8
go.sum
8
go.sum
|
|
@ -165,9 +165,12 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
|
|||
github.com/google/subcommands v0.0.0-20181012225330-46f0354f6315/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
|
||||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/wire v0.2.1 h1:TYj4Z2qjqxa2ufb34UJqVeO9aznL+i0fLO6TqThKZ7Y=
|
||||
github.com/google/wire v0.2.1/go.mod h1:ptBl5bWD3nzmJHVNwYHV3v4wdtKzBMlU2YbtKQCG9GI=
|
||||
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
|
||||
github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gax-go/v2 v2.0.3 h1:siORttZ36U2R/WjiJuDz8znElWBiAlO9rVt+mqJt0Cc=
|
||||
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
|
||||
github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
|
|
@ -431,6 +434,7 @@ github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZ
|
|||
go.etcd.io/etcd v3.3.11+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
|
||||
go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
|
||||
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b h1:6ayHMBPtdP3jNuk+Sfhso+PTB7ZJQ5E1FBo403m2H8w=
|
||||
go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
|
|
@ -461,6 +465,7 @@ golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73r
|
|||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/oauth2 v0.0.0-20180603041954-1e0a3fa8ba9a/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
|
|
@ -488,6 +493,7 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h
|
|||
golang.org/x/sys v0.0.0-20190130150945-aca44879d564 h1:o6ENHFwwr1TZ9CUPQcfo1HGvLP1OPsPOTB7xCIOPNmU=
|
||||
golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -500,6 +506,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
|
|||
golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c h1:vamGzbGri8IKo20MQncCuljcQ5uAO6kaCeawQPVblAI=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/xerrors v0.0.0-20190129162528-20feca13ea86 h1:kMgZCSynBSIN3PHpvuFeMExQwPWtUZ/xfnt2Yr2cp20=
|
||||
golang.org/x/xerrors v0.0.0-20190129162528-20feca13ea86/go.mod h1:/lyp46tcDBI65C0XC8F4d0/XVb7MT7RScVRech7dX/4=
|
||||
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
|
||||
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
|
||||
|
|
@ -507,6 +514,7 @@ google.golang.org/api v0.0.0-20180603000442-8e296ef26005/go.mod h1:4mhQ8q/RsB7i+
|
|||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI=
|
||||
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
|
||||
google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
|
|
|
|||
|
|
@ -50,6 +50,29 @@ type Package struct {
|
|||
Version *Version
|
||||
}
|
||||
|
||||
// VersionType defines the version types a module can have.
|
||||
type VersionType string
|
||||
|
||||
const (
|
||||
// VersionTypeRelease is a normal release.
|
||||
VersionTypeRelease = VersionType("release")
|
||||
|
||||
// VersionTypePrerelease is a version with a prerelease.
|
||||
VersionTypePrerelease = VersionType("prerelease")
|
||||
|
||||
// VersionTypePseudo appears to have a prerelease of the
|
||||
// form <commit date>-<commit hash>.
|
||||
VersionTypePseudo = VersionType("pseudo")
|
||||
|
||||
// VersionTypeInvalid indicates that a given version is
|
||||
// invalid.
|
||||
VersionTypeInvalid = VersionType("invalid")
|
||||
)
|
||||
|
||||
func (vt VersionType) String() string {
|
||||
return string(vt)
|
||||
}
|
||||
|
||||
// A VersionSource is the source of a record in the version logs.
|
||||
type VersionSource string
|
||||
|
||||
|
|
|
|||
|
|
@ -15,14 +15,17 @@ import (
|
|||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sos.googlesource.com/sos/license"
|
||||
|
||||
"golang.org/x/discovery/internal"
|
||||
"golang.org/x/discovery/internal/postgres"
|
||||
"golang.org/x/discovery/internal/proxy"
|
||||
"golang.org/x/discovery/internal/thirdparty/semver"
|
||||
"golang.org/x/tools/go/packages"
|
||||
)
|
||||
|
||||
|
|
@ -49,6 +52,38 @@ func ParseModulePathAndVersion(u *url.URL) (string, string, error) {
|
|||
return parts[0], parts[1], nil
|
||||
}
|
||||
|
||||
// parseVersion returns the VersionType of a given a version.
|
||||
func parseVersion(version string) internal.VersionType {
|
||||
if !semver.IsValid(version) {
|
||||
return internal.VersionTypeInvalid
|
||||
}
|
||||
|
||||
prerelease := semver.Prerelease(version)
|
||||
if prerelease == "" {
|
||||
return internal.VersionTypeRelease
|
||||
}
|
||||
prerelease = prerelease[1:] // remove starting dash
|
||||
|
||||
// if prerelease looks like a commit then return VersionTypePseudo
|
||||
matched, err := regexp.MatchString(`[0-9]{14}-[0-9a-z]{12}`, prerelease)
|
||||
if err != nil {
|
||||
log.Printf("regexp.MatchString(`[0-9]{14}-[0-9a-z]{12}`, %v): %v", prerelease, err)
|
||||
return internal.VersionType("regexp.MatchString error")
|
||||
}
|
||||
|
||||
if matched {
|
||||
rawTime := strings.Split(prerelease, "-")[0]
|
||||
layout := "20060102150405"
|
||||
t, err := time.Parse(layout, rawTime)
|
||||
|
||||
if err == nil && t.Before(time.Now()) {
|
||||
return internal.VersionTypePseudo
|
||||
}
|
||||
}
|
||||
|
||||
return internal.VersionTypePrerelease
|
||||
}
|
||||
|
||||
// FetchAndInsertVersion downloads the given module version from the module proxy, processes
|
||||
// the contents, and writes the data to the database. The fetch service will:
|
||||
// (1) Get the version commit time from the proxy
|
||||
|
|
@ -56,6 +91,11 @@ func ParseModulePathAndVersion(u *url.URL) (string, string, error) {
|
|||
// (3) Process the contents (series name, readme, license, and packages)
|
||||
// (4) Write the data to the discovery database
|
||||
func FetchAndInsertVersion(name, version string, proxyClient *proxy.Client, db *postgres.DB) error {
|
||||
versionType := parseVersion(version)
|
||||
if versionType == internal.VersionTypeInvalid {
|
||||
return fmt.Errorf("parseVersion(%q) = %v", version, versionType)
|
||||
}
|
||||
|
||||
info, err := proxyClient.GetInfo(name, version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("proxyClient.GetInfo(%q, %q): %v", name, version, err)
|
||||
|
|
|
|||
|
|
@ -445,3 +445,44 @@ func TestDetectLicense(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetch_parseVersion(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name, version string
|
||||
wantVersionType internal.VersionType
|
||||
}{
|
||||
{
|
||||
name: "valid_pseudo-version",
|
||||
version: "v1.0.0-20190311183353-d8887717615a",
|
||||
wantVersionType: internal.VersionTypePseudo,
|
||||
},
|
||||
{
|
||||
name: "invalid_pseudo-version_future_date",
|
||||
version: "v1.0.0-40000311183353-d8887717615a",
|
||||
wantVersionType: internal.VersionTypePrerelease,
|
||||
},
|
||||
{
|
||||
name: "valid_release",
|
||||
version: "v1.0.0",
|
||||
wantVersionType: internal.VersionTypeRelease,
|
||||
},
|
||||
{
|
||||
name: "valid_release",
|
||||
version: "v1.0.0-alpha.1",
|
||||
wantVersionType: internal.VersionTypePrerelease,
|
||||
},
|
||||
{
|
||||
name: "invalid_version",
|
||||
version: "not_a_version",
|
||||
wantVersionType: internal.VersionTypeInvalid,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if gotVt := parseVersion(tc.version); tc.wantVersionType != gotVt {
|
||||
t.Errorf("parseVersion(%v) = %v, want %v", tc.version, gotVt, tc.wantVersionType)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче