зеркало из https://github.com/golang/pkgsite.git
internal/cron: create proxy index cron
The proxy index cron is created, with a job to get new versions from the module index. The cron will: 1. query the module index for new versions since a given timestamp 2. write each version to the version_logs table 3. make a request to the fetch service for each version to be downloaded The fetch client is also fixed to make a GET request to https://<fetch-url>/<module>/@v/<version>. Change-Id: I838029d94f9b2782e0c1066ec7932931b47fe01e Reviewed-on: https://team-review.git.corp.google.com/c/golang/discovery/+/426749 Reviewed-by: Andrew Bonventre <andybons@google.com>
This commit is contained in:
Julie Qiu
Родитель
8f815ef30a
Коммит
cfb8fdf20b
|
|
@ -0,0 +1,76 @@
|
|||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// The fetch command runs a server that fetches modules from a proxy and writes
|
||||
// them to the discovery database.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
|
||||
"golang.org/x/discovery/internal/cron"
|
||||
"golang.org/x/discovery/internal/fetch"
|
||||
"golang.org/x/discovery/internal/postgres"
|
||||
)
|
||||
|
||||
var (
|
||||
indexURL = getEnv("GO_MODULE_INDEX_URL", "")
|
||||
fetchURL = getEnv("GO_DISCOVERY_FETCH_URL", "http://localhost:9000")
|
||||
user = getEnv("GO_DISCOVERY_DATABASE_USER", "postgres")
|
||||
password = getEnv("GO_DISCOVERY_DATABASE_PASSWORD", "")
|
||||
host = getEnv("GO_DISCOVERY_DATABASE_HOST", "localhost")
|
||||
dbname = getEnv("GO_DISCOVERY_DATABASE_NAME", "discovery-database")
|
||||
dbinfo = fmt.Sprintf("user=%s password=%s host=%s dbname=%s sslmode=disable", user, password, host, dbname)
|
||||
port = getEnv("PORT", "8000")
|
||||
)
|
||||
|
||||
func getEnv(key, fallback string) string {
|
||||
if value, ok := os.LookupEnv(key); ok {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func makeNewVersionsHandler(db *postgres.DB) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
logs, err := cron.NewVersionsFromProxyIndex(indexURL, db)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("Status %d (%s): %v", http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError), err),
|
||||
http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
client := fetch.New(fetchURL)
|
||||
for _, l := range logs {
|
||||
fmt.Fprintf(w, "Fetch requested: %q %q\n", l.Name, l.Version)
|
||||
go func(name, version string) {
|
||||
if err := client.FetchVersion(name, version); err != nil {
|
||||
log.Printf("client.FetchVersion(%q, %q): %v", name, version, err)
|
||||
}
|
||||
}(l.Name, l.Version)
|
||||
}
|
||||
fmt.Fprintf(w, "Done!")
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
db, err := postgres.Open(dbinfo)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres.Open(%q): %v", dbinfo, err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
http.HandleFunc("/new/", makeNewVersionsHandler(db))
|
||||
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, "Hello, Go Discovery Cron!")
|
||||
})
|
||||
|
||||
log.Printf("Listening on port %s", port)
|
||||
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))
|
||||
}
|
||||
|
|
@ -43,6 +43,7 @@ func makeFetchHandler(proxyClient *proxy.Client, db *postgres.DB) http.HandlerFu
|
|||
return
|
||||
}
|
||||
|
||||
log.Printf("Request received: %q", r.URL.Path)
|
||||
name, version, err := fetch.ParseNameAndVersion(r.URL)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("Status %d (%s): %v", http.StatusBadRequest, http.StatusText(http.StatusBadRequest), err),
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -5,8 +5,8 @@ go 1.12
|
|||
require (
|
||||
github.com/lib/pq v1.0.0
|
||||
gocloud.dev v0.11.0
|
||||
golang.org/x/tools v0.0.0-20190221204921-83362c3779f5
|
||||
google.golang.org/grpc v1.18.0
|
||||
golang.org/x/tools v0.0.0-20190306162903-69e0dcfa1121
|
||||
google.golang.org/grpc v1.19.0
|
||||
sos.googlesource.com/sos v1.0.0
|
||||
)
|
||||
|
||||
|
|
|
|||
20
go.sum
20
go.sum
|
|
@ -164,12 +164,9 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
|
|||
github.com/google/subcommands v0.0.0-20181012225330-46f0354f6315/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
|
||||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/wire v0.2.1 h1:TYj4Z2qjqxa2ufb34UJqVeO9aznL+i0fLO6TqThKZ7Y=
|
||||
github.com/google/wire v0.2.1/go.mod h1:ptBl5bWD3nzmJHVNwYHV3v4wdtKzBMlU2YbtKQCG9GI=
|
||||
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
|
||||
github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gax-go/v2 v2.0.3 h1:siORttZ36U2R/WjiJuDz8znElWBiAlO9rVt+mqJt0Cc=
|
||||
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
|
||||
github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
|
|
@ -274,7 +271,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
|
|||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
|
||||
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
|
|
@ -429,7 +425,6 @@ github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZ
|
|||
go.etcd.io/etcd v3.3.11+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
|
||||
go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
|
||||
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b h1:6ayHMBPtdP3jNuk+Sfhso+PTB7ZJQ5E1FBo403m2H8w=
|
||||
go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
|
|
@ -460,7 +455,6 @@ golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73r
|
|||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190119204137-ed066c81e75e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/oauth2 v0.0.0-20180603041954-1e0a3fa8ba9a/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
|
|
@ -486,7 +480,6 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h
|
|||
golang.org/x/sys v0.0.0-20190130150945-aca44879d564 h1:o6ENHFwwr1TZ9CUPQcfo1HGvLP1OPsPOTB7xCIOPNmU=
|
||||
golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -495,9 +488,10 @@ golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGm
|
|||
golang.org/x/tools v0.0.0-20181017214349-06f26fdaaa28/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190221204921-83362c3779f5 h1:ev5exjGDsOo0NPTB0qdCcE53BfWl1IICJlhgXgfT9fM=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/xerrors v0.0.0-20190129162528-20feca13ea86 h1:kMgZCSynBSIN3PHpvuFeMExQwPWtUZ/xfnt2Yr2cp20=
|
||||
golang.org/x/tools v0.0.0-20190306162903-69e0dcfa1121 h1:no7FcgCKNUGnAyqPewDIx0La6Y6an4a/T+VWe9mILpQ=
|
||||
golang.org/x/tools v0.0.0-20190306162903-69e0dcfa1121/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/xerrors v0.0.0-20190129162528-20feca13ea86/go.mod h1:/lyp46tcDBI65C0XC8F4d0/XVb7MT7RScVRech7dX/4=
|
||||
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
|
||||
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
|
||||
|
|
@ -505,7 +499,6 @@ google.golang.org/api v0.0.0-20180603000442-8e296ef26005/go.mod h1:4mhQ8q/RsB7i+
|
|||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI=
|
||||
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
|
||||
google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
|
|
@ -521,20 +514,19 @@ google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk
|
|||
google.golang.org/genproto v0.0.0-20190110221437-6909d8a4a91b/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/genproto v0.0.0-20190122204518-eef12c790cc0/go.mod h1:L3J43x8/uS+qIUoksaLKe6OS3nUKxOKuIFz1sl2/jx4=
|
||||
google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922/go.mod h1:L3J43x8/uS+qIUoksaLKe6OS3nUKxOKuIFz1sl2/jx4=
|
||||
google.golang.org/genproto v0.0.0-20190219182410-082222b4a5c5 h1:SdCO7As+ChE1iV3IjBleIIWlj8VjZWuYEUF5pjELOOQ=
|
||||
google.golang.org/genproto v0.0.0-20190219182410-082222b4a5c5/go.mod h1:L3J43x8/uS+qIUoksaLKe6OS3nUKxOKuIFz1sl2/jx4=
|
||||
google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.18.0 h1:IZl7mfBGfbhYx2p2rKRtYgDFw6SBz+kclmxYrCksPPA=
|
||||
google.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
|
||||
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fatih/pool.v2 v2.0.0/go.mod h1:8xVGeu1/2jr2wm5V9SPuMht2H5AEmf5aFMGSQixtjTY=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
|
|
@ -555,12 +547,12 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
|
|||
gopkg.in/vmihailenco/msgpack.v2 v2.9.1/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8=
|
||||
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20181108184350-ae8f1f9103cc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
k8s.io/api v0.0.0-20181128191700-6db15a15d2d3/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
|
||||
k8s.io/api v0.0.0-20181221193117-173ce66c1e39/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cron
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/discovery/internal"
|
||||
"golang.org/x/discovery/internal/postgres"
|
||||
)
|
||||
|
||||
// FetchAndStoreVersions queries indexURL for new versions and writes them to
|
||||
// the version_logs table.
|
||||
func FetchAndStoreVersions(indexURL string, db *postgres.DB) ([]*internal.VersionLog, error) {
|
||||
t, err := db.LatestProxyIndexUpdate()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("db.LatestProxyIndexUpdate(): %v", err)
|
||||
}
|
||||
|
||||
logs, err := getVersionsFromIndex(indexURL, t)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getVersionsFromIndex(%q, %v): %v", indexURL, t, err)
|
||||
}
|
||||
|
||||
if err = db.InsertVersionLogs(logs); err != nil {
|
||||
return nil, fmt.Errorf("db.InsertVersionLogs(%v): %v", logs, err)
|
||||
}
|
||||
return logs, nil
|
||||
}
|
||||
|
||||
// getVersionsFromIndex makes a request to indexURL/<since> and returns the
|
||||
// the response as a []*internal.VersionLog.
|
||||
func getVersionsFromIndex(indexURL string, since time.Time) ([]*internal.VersionLog, error) {
|
||||
latestUpdate := time.Now()
|
||||
|
||||
u := fmt.Sprintf("%s?since=%s", strings.TrimRight(indexURL, "/"), since.Format(time.RFC3339))
|
||||
r, err := http.Get(u)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http.Get(%q): %v", u, err)
|
||||
}
|
||||
defer r.Body.Close()
|
||||
|
||||
var logs []*internal.VersionLog
|
||||
dec := json.NewDecoder(r.Body)
|
||||
|
||||
// The module index returns a stream of JSON objects formatted with newline
|
||||
// as the delimiter. For each version log, we want to set source to
|
||||
// "proxy-index" and created_at to the time right before the proxy index is
|
||||
// queried.
|
||||
for dec.More() {
|
||||
var l internal.VersionLog
|
||||
if err := dec.Decode(&l); err != nil {
|
||||
log.Printf("dec.Decode: %v", err)
|
||||
continue
|
||||
}
|
||||
logs = append(logs, &l)
|
||||
l.Source = internal.VersionSourceProxyIndex
|
||||
l.CreatedAt = latestUpdate
|
||||
}
|
||||
return logs, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
// Copyright 2019 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cron
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/discovery/internal"
|
||||
"golang.org/x/discovery/internal/postgres"
|
||||
)
|
||||
|
||||
type testCase struct {
|
||||
index *httptest.Server
|
||||
fetch *httptest.Server
|
||||
}
|
||||
|
||||
func setupIndex(t *testing.T, versions []map[string]string) (func(t *testing.T), *httptest.Server) {
|
||||
index := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
for _, v := range versions {
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
}))
|
||||
|
||||
fn := func(t *testing.T) {
|
||||
index.Close()
|
||||
}
|
||||
return fn, index
|
||||
}
|
||||
|
||||
// versionLogArrayToString outputs a string for an array for version logs. It
|
||||
// is used for testing in printing errors.
|
||||
func versionLogArrayToString(logs []*internal.VersionLog) string {
|
||||
var b strings.Builder
|
||||
for _, l := range logs {
|
||||
fmt.Fprintf(&b, "%+v\n", l)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func TestGetVersionsFromIndex(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
indexInfo []map[string]string
|
||||
wantLogs []*internal.VersionLog
|
||||
}{
|
||||
{
|
||||
name: "valid_get_versions",
|
||||
indexInfo: []map[string]string{
|
||||
map[string]string{
|
||||
"name": "my/module",
|
||||
"version": "v1.0.0",
|
||||
},
|
||||
map[string]string{
|
||||
"name": "my/module",
|
||||
"version": "v1.1.0",
|
||||
},
|
||||
map[string]string{
|
||||
"name": "my/module/v2",
|
||||
"version": "v2.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty_get_versions",
|
||||
indexInfo: []map[string]string{},
|
||||
},
|
||||
} {
|
||||
wantLogs := []*internal.VersionLog{}
|
||||
for _, v := range tc.indexInfo {
|
||||
wantLogs = append(wantLogs, &internal.VersionLog{
|
||||
Name: v["name"],
|
||||
Version: v["version"],
|
||||
Source: internal.VersionSourceProxyIndex,
|
||||
})
|
||||
}
|
||||
if len(wantLogs) > 0 {
|
||||
tc.wantLogs = wantLogs
|
||||
}
|
||||
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
teardownTestCase, index := setupIndex(t, tc.indexInfo)
|
||||
defer teardownTestCase(t)
|
||||
logs, err := getVersionsFromIndex(index.URL, time.Time{})
|
||||
if err != nil {
|
||||
t.Fatalf("getVersionFromIndex(%q, %q) error: %v",
|
||||
index.URL, time.Time{}, err)
|
||||
}
|
||||
|
||||
for _, l := range logs {
|
||||
l.CreatedAt = time.Time{}
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(logs, tc.wantLogs) {
|
||||
t.Errorf("getVersionFromIndex(%q, %q) = %v; want = %v", index.URL, time.Time{}.String(), versionLogArrayToString(logs), versionLogArrayToString(tc.wantLogs))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewVersionFromProxyIndex(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
indexInfo []map[string]string
|
||||
oldVersionLogs []*internal.VersionLog
|
||||
wantVersionLogs []*internal.VersionLog
|
||||
}{
|
||||
{
|
||||
name: "version-logs-no-existing-entries",
|
||||
indexInfo: []map[string]string{
|
||||
map[string]string{
|
||||
"name": "my/module",
|
||||
"version": "v1.0.0",
|
||||
},
|
||||
},
|
||||
oldVersionLogs: nil,
|
||||
wantVersionLogs: []*internal.VersionLog{
|
||||
&internal.VersionLog{
|
||||
Name: "my/module",
|
||||
Version: "v1.0.0",
|
||||
Source: internal.VersionSourceProxyIndex,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "version-logs-existing-duplicate-entry",
|
||||
indexInfo: []map[string]string{
|
||||
map[string]string{
|
||||
"name": "my/module",
|
||||
"version": "v1.0.0",
|
||||
},
|
||||
map[string]string{
|
||||
"name": "my/module",
|
||||
"version": "v2.0.0",
|
||||
},
|
||||
},
|
||||
oldVersionLogs: []*internal.VersionLog{
|
||||
&internal.VersionLog{
|
||||
Name: "my/module",
|
||||
Version: "v1.0.0",
|
||||
Source: internal.VersionSourceProxyIndex,
|
||||
},
|
||||
},
|
||||
wantVersionLogs: []*internal.VersionLog{
|
||||
&internal.VersionLog{
|
||||
Name: "my/module",
|
||||
Version: "v1.0.0",
|
||||
Source: internal.VersionSourceProxyIndex,
|
||||
},
|
||||
&internal.VersionLog{
|
||||
Name: "my/module",
|
||||
Version: "v2.0.0",
|
||||
Source: internal.VersionSourceProxyIndex,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "version-logs-no-new-entries",
|
||||
indexInfo: []map[string]string{},
|
||||
oldVersionLogs: nil,
|
||||
wantVersionLogs: nil,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
teardownTestCase, index := setupIndex(t, tc.indexInfo)
|
||||
defer teardownTestCase(t)
|
||||
|
||||
cleanupDB, db := postgres.SetupCleanDB(t)
|
||||
defer cleanupDB(t)
|
||||
|
||||
if err := db.InsertVersionLogs(tc.oldVersionLogs); err != nil {
|
||||
t.Fatalf("db.InsertVersionLogs(%v): %v", tc.oldVersionLogs, err)
|
||||
}
|
||||
|
||||
got, err := FetchAndStoreVersions(index.URL, db)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchAndStoreVersions(%q, %v): %v", index.URL, db, err)
|
||||
}
|
||||
|
||||
// do not compare the timestamps, since they are set inside
|
||||
// NewVersionFromProxyIndex.
|
||||
for _, l := range got {
|
||||
l.CreatedAt = time.Time{}
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(got, tc.wantVersionLogs) {
|
||||
t.Fatalf("NewVersionFromProxyIndex(%q, %v) = %v; want %v",
|
||||
index.URL, db, versionLogArrayToString(got), versionLogArrayToString(tc.wantVersionLogs))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -4,7 +4,9 @@
|
|||
|
||||
package internal
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// A Series is a group of modules that share the same base path and are assumed
|
||||
// to be major-version variants.
|
||||
|
|
@ -53,10 +55,10 @@ type VersionSource string
|
|||
|
||||
const (
|
||||
// Version fetched by the Proxy Index Cron
|
||||
VersionLogProxyIndex = VersionSource("proxy-index")
|
||||
VersionSourceProxyIndex = VersionSource("proxy-index")
|
||||
|
||||
// Version requested by user from Frontend
|
||||
VersionLogFrontend = VersionSource("frontend")
|
||||
VersionSourceFrontend = VersionSource("frontend")
|
||||
)
|
||||
|
||||
func (vs VersionSource) String() string {
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@
|
|||
package fetch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
|
@ -24,27 +22,16 @@ func New(url string) *Client {
|
|||
}
|
||||
}
|
||||
|
||||
// FetchVersion makes a request to url for the given module version.
|
||||
// FetchVersion makes a request for the module with name and version.
|
||||
func (c *Client) FetchVersion(name, version string) error {
|
||||
data := map[string]string{
|
||||
"name": name,
|
||||
"version": version,
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := json.NewEncoder(&buf).Encode(data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
contentType := "application/json"
|
||||
r, err := http.Post(c.url, contentType, &buf)
|
||||
url := fmt.Sprintf("%s/%s/@v/%s", c.url, name, version)
|
||||
r, err := http.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("http.Post(%q, %q, %q) error: %v", c.url, contentType, data, err)
|
||||
return fmt.Errorf("http.Get(%q): %v", url, err)
|
||||
}
|
||||
|
||||
if r.StatusCode < 200 || r.StatusCode >= 300 {
|
||||
return fmt.Errorf("http.Post(%q, %q, %q) returned response: %d (%q)",
|
||||
c.url, contentType, data, r.StatusCode, r.Status)
|
||||
return fmt.Errorf("http.Get(%q) returned response: %d (%q)", url, r.StatusCode, r.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,11 +38,9 @@ func TestFetchVersionInvalidFetchURL(t *testing.T) {
|
|||
m := "module"
|
||||
v := "v1.5.2"
|
||||
|
||||
expectedData := map[string]string{"name": m, "version": v}
|
||||
expectedErr := fmt.Errorf("http.Post(%q, %q, %q) returned response: 400 (%q)",
|
||||
server.URL, "application/json", expectedData, "400 Bad Request")
|
||||
|
||||
if err := c.FetchVersion(m, v); err.Error() != expectedErr.Error() {
|
||||
t.Errorf("fetchVersion(%q, %q, %q) = %v; want %v", server.URL, m, v, err, expectedErr)
|
||||
url := fmt.Sprintf("%s/%s/@v/%s", server.URL, m, v)
|
||||
wantErr := fmt.Errorf(`http.Get(%q) returned response: 400 ("400 Bad Request")`, url)
|
||||
if err := c.FetchVersion(m, v); err.Error() != wantErr.Error() {
|
||||
t.Errorf("fetchVersion(%q) = %v; want %v", url, err, wantErr)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ func (db *DB) LatestProxyIndexUpdate() (time.Time, error) {
|
|||
LIMIT 1`
|
||||
|
||||
var createdAt time.Time
|
||||
row := db.QueryRow(query, internal.VersionLogProxyIndex)
|
||||
row := db.QueryRow(query, internal.VersionSourceProxyIndex)
|
||||
switch err := row.Scan(&createdAt); err {
|
||||
case sql.ErrNoRows:
|
||||
return time.Time{}, nil
|
||||
|
|
@ -81,7 +81,7 @@ func (db *DB) InsertVersionLogs(logs []*internal.VersionLog) error {
|
|||
for _, l := range logs {
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO version_logs(name, version, created_at, source, error)
|
||||
VALUES ($1, $2, $3, $4, $5);`,
|
||||
VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING;`,
|
||||
l.Name, l.Version, l.CreatedAt, l.Source, l.Error,
|
||||
); err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -157,19 +157,19 @@ func TestPostgress_InsertVersionLogs(t *testing.T) {
|
|||
Name: "testModule",
|
||||
Version: "v.1.0.0",
|
||||
CreatedAt: now.Add(-10 * time.Minute),
|
||||
Source: internal.VersionLogProxyIndex,
|
||||
Source: internal.VersionSourceProxyIndex,
|
||||
},
|
||||
&internal.VersionLog{
|
||||
Name: "testModule",
|
||||
Version: "v.1.1.0",
|
||||
CreatedAt: now,
|
||||
Source: internal.VersionLogProxyIndex,
|
||||
Source: internal.VersionSourceProxyIndex,
|
||||
},
|
||||
&internal.VersionLog{
|
||||
Name: "testModule/v2",
|
||||
Version: "v.2.0.0",
|
||||
CreatedAt: now,
|
||||
Source: internal.VersionLogProxyIndex,
|
||||
Source: internal.VersionSourceProxyIndex,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче