internal, internal/fetch: add VersionType and getVersionType

The database needs an enum column called version_type to indicate
the type of version a package has. So to be able to insert
packages with the correct version type VersionType is being added
along with function getVersionType to parse a package's
version to determine what kind of VersionType it should have.

getVersionType is also used to make sure that the version is valid
in FetchAndInsertVersion.

Change-Id: Ia15c23c3783dbb83934ce8b0b580a25bafa219c2
Reviewed-on: https://team-review.git.corp.google.com/c/golang/discovery/+/437333
Reviewed-by: Andrew Bonventre <andybons@google.com>

go.sum

@@ -165,9 +165,12 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
 github.com/google/subcommands v0.0.0-20181012225330-46f0354f6315/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/wire v0.2.1 h1:TYj4Z2qjqxa2ufb34UJqVeO9aznL+i0fLO6TqThKZ7Y=
 github.com/google/wire v0.2.1/go.mod h1:ptBl5bWD3nzmJHVNwYHV3v4wdtKzBMlU2YbtKQCG9GI=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
 github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/googleapis/gax-go/v2 v2.0.3 h1:siORttZ36U2R/WjiJuDz8znElWBiAlO9rVt+mqJt0Cc=
 github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
 github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -431,6 +434,7 @@ github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZ
 go.etcd.io/etcd v3.3.11+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
+go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b h1:6ayHMBPtdP3jNuk+Sfhso+PTB7ZJQ5E1FBo403m2H8w=
 go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -461,6 +465,7 @@ golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/oauth2 v0.0.0-20180603041954-1e0a3fa8ba9a/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -488,6 +493,7 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564 h1:o6ENHFwwr1TZ9CUPQcfo1HGvLP1OPsPOTB7xCIOPNmU=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -500,6 +506,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c h1:vamGzbGri8IKo20MQncCuljcQ5uAO6kaCeawQPVblAI=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/xerrors v0.0.0-20190129162528-20feca13ea86 h1:kMgZCSynBSIN3PHpvuFeMExQwPWtUZ/xfnt2Yr2cp20=
 golang.org/x/xerrors v0.0.0-20190129162528-20feca13ea86/go.mod h1:/lyp46tcDBI65C0XC8F4d0/XVb7MT7RScVRech7dX/4=
 gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
@@ -507,6 +514,7 @@ google.golang.org/api v0.0.0-20180603000442-8e296ef26005/go.mod h1:4mhQ8q/RsB7i+
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI=
 google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
 google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

internal/discovery.go

@@ -50,6 +50,29 @@ type Package struct {
 	Version  *Version
 }
 
+// VersionType defines the version types a module can have.
+type VersionType string
+
+const (
+	// VersionTypeRelease is a normal release.
+	VersionTypeRelease = VersionType("release")
+
+	// VersionTypePrerelease is a version with a prerelease.
+	VersionTypePrerelease = VersionType("prerelease")
+
+	// VersionTypePseudo appears to have a prerelease of the
+	// form <commit date>-<commit hash>.
+	VersionTypePseudo = VersionType("pseudo")
+
+	// VersionTypeInvalid indicates that a given version is
+	// invalid.
+	VersionTypeInvalid = VersionType("invalid")
+)
+
+func (vt VersionType) String() string {
+	return string(vt)
+}
+
 // A VersionSource is the source of a record in the version logs.
 type VersionSource string
 

internal/fetch/fetch.go

@@ -15,14 +15,17 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"sos.googlesource.com/sos/license"
 
 	"golang.org/x/discovery/internal"
 	"golang.org/x/discovery/internal/postgres"
 	"golang.org/x/discovery/internal/proxy"
+	"golang.org/x/discovery/internal/thirdparty/semver"
 	"golang.org/x/tools/go/packages"
 )
 
@@ -49,6 +52,38 @@ func ParseModulePathAndVersion(u *url.URL) (string, string, error) {
 	return parts[0], parts[1], nil
 }
 
+// parseVersion returns the VersionType of a given a version.
+func parseVersion(version string) internal.VersionType {
+	if !semver.IsValid(version) {
+		return internal.VersionTypeInvalid
+	}
+
+	prerelease := semver.Prerelease(version)
+	if prerelease == "" {
+		return internal.VersionTypeRelease
+	}
+	prerelease = prerelease[1:] // remove starting dash
+
+	// if prerelease looks like a commit then return VersionTypePseudo
+	matched, err := regexp.MatchString(`[0-9]{14}-[0-9a-z]{12}`, prerelease)
+	if err != nil {
+		log.Printf("regexp.MatchString(`[0-9]{14}-[0-9a-z]{12}`, %v): %v", prerelease, err)
+		return internal.VersionType("regexp.MatchString error")
+	}
+
+	if matched {
+		rawTime := strings.Split(prerelease, "-")[0]
+		layout := "20060102150405"
+		t, err := time.Parse(layout, rawTime)
+
+		if err == nil && t.Before(time.Now()) {
+			return internal.VersionTypePseudo
+		}
+	}
+
+	return internal.VersionTypePrerelease
+}
+
 // FetchAndInsertVersion downloads the given module version from the module proxy, processes
 // the contents, and writes the data to the database. The fetch service will:
 // (1) Get the version commit time from the proxy
@@ -56,6 +91,11 @@ func ParseModulePathAndVersion(u *url.URL) (string, string, error) {
 // (3) Process the contents (series name, readme, license, and packages)
 // (4) Write the data to the discovery database
 func FetchAndInsertVersion(name, version string, proxyClient *proxy.Client, db *postgres.DB) error {
+	versionType := parseVersion(version)
+	if versionType == internal.VersionTypeInvalid {
+		return fmt.Errorf("parseVersion(%q) = %v", version, versionType)
+	}
+
 	info, err := proxyClient.GetInfo(name, version)
 	if err != nil {
 		return fmt.Errorf("proxyClient.GetInfo(%q, %q): %v", name, version, err)

internal/fetch/fetch_test.go

@@ -445,3 +445,44 @@ func TestDetectLicense(t *testing.T) {
 		})
 	}
 }
+
+func TestFetch_parseVersion(t *testing.T) {
+	testCases := []struct {
+		name, version   string
+		wantVersionType internal.VersionType
+	}{
+		{
+			name:            "valid_pseudo-version",
+			version:         "v1.0.0-20190311183353-d8887717615a",
+			wantVersionType: internal.VersionTypePseudo,
+		},
+		{
+			name:            "invalid_pseudo-version_future_date",
+			version:         "v1.0.0-40000311183353-d8887717615a",
+			wantVersionType: internal.VersionTypePrerelease,
+		},
+		{
+			name:            "valid_release",
+			version:         "v1.0.0",
+			wantVersionType: internal.VersionTypeRelease,
+		},
+		{
+			name:            "valid_release",
+			version:         "v1.0.0-alpha.1",
+			wantVersionType: internal.VersionTypePrerelease,
+		},
+		{
+			name:            "invalid_version",
+			version:         "not_a_version",
+			wantVersionType: internal.VersionTypeInvalid,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if gotVt := parseVersion(tc.version); tc.wantVersionType != gotVt {
+				t.Errorf("parseVersion(%v) = %v, want %v", tc.version, gotVt, tc.wantVersionType)
+			}
+		})
+	}
+}