cmd/gerritbot,https: add scaffolding for gerritbot and https package

Update golang/go#18517

Change-Id: I1dabca341839c3c77b99991d345419425861c5d4
Reviewed-on: https://go-review.googlesource.com/70311
Reviewed-by: Sarah Adams <shadams@google.com>
This commit is contained in:
Andrew Bonventre 2017-10-12 14:19:44 -04:00
Родитель fa4b20e11b
Коммит 0e1a9cad1b
10 изменённых файлов: 507 добавлений и 0 удалений

2
cmd/gerritbot/.gitignore поставляемый Normal file
Просмотреть файл

@ -0,0 +1,2 @@
gerritbot
ca-certificates.crt

8
cmd/gerritbot/Dockerfile Normal file
Просмотреть файл

@ -0,0 +1,8 @@
# Copyright 2017 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
FROM scratch
LABEL maintainer "golang-dev@googlegroups.com"
COPY ca-certificates.crt /etc/ssl/certs/
COPY gerritbot /
ENTRYPOINT ["/gerritbot"]

158
cmd/gerritbot/Dockerfile.0 Normal file
Просмотреть файл

@ -0,0 +1,158 @@
# Copyright 2017 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
# Note that OpenSSH 6.5+ is required for the Github SSH private key, which requires
# at least Debian Jessie (not Wheezy). This uses Jessie:
FROM golang:1.8
LABEL maintainer "golang-dev@googlegroups.com"
# BEGIN deps (run `make update-deps` to update)
# Repo cloud.google.com/go at da30b6d (2017-09-23)
ENV REV=da30b6d6aeb462b223228cc0b4e7952cba499d90
RUN go get -d cloud.google.com/go/compute/metadata `#and 6 other pkgs` &&\
(cd /go/src/cloud.google.com/go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo github.com/golang/protobuf at 130e6b0 (2017-09-20)
ENV REV=130e6b02ab059e7b717a096f397c5b60111cae74
RUN go get -d github.com/golang/protobuf/proto `#and 6 other pkgs` &&\
(cd /go/src/github.com/golang/protobuf && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo github.com/google/go-github at c004bf0 (2017-10-08)
ENV REV=c004bf0c02fcab9d87a7f2e2f002fd008bf0784c
RUN go get -d github.com/google/go-github/github &&\
(cd /go/src/github.com/google/go-github && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo github.com/google/go-querystring at 53e6ce1 (2017-01-11)
ENV REV=53e6ce116135b80d037921a7fdd5138cf32d7a8a
RUN go get -d github.com/google/go-querystring/query &&\
(cd /go/src/github.com/google/go-querystring && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo github.com/googleapis/gax-go at 317e000 (2017-09-15)
ENV REV=317e0006254c44a0ac427cc52a0e083ff0b9622f
RUN go get -d github.com/googleapis/gax-go &&\
(cd /go/src/github.com/googleapis/gax-go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo github.com/gregjones/httpcache at 316c5e0 (2017-09-20)
ENV REV=316c5e0ff04e0ce21cfae9fbbb2ab39c7fc74e71
RUN go get -d github.com/gregjones/httpcache &&\
(cd /go/src/github.com/gregjones/httpcache && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo go4.org at 034d17a (2017-05-25)
ENV REV=034d17a462f7b2dcd1a4a73553ec5357ff6e6c6e
RUN go get -d go4.org/types &&\
(cd /go/src/go4.org && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/crypto at 847319b (2017-09-25)
ENV REV=847319b7fc94cab682988f93da778204da164588
RUN go get -d golang.org/x/crypto/acme `#and 2 other pkgs` &&\
(cd /go/src/golang.org/x/crypto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/net at 0744d00 (2017-09-22)
ENV REV=0744d001aa8470aaa53df28d32e5ceeb8af9bd70
RUN go get -d golang.org/x/net/context `#and 8 other pkgs` &&\
(cd /go/src/golang.org/x/net && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/oauth2 at 13449ad (2017-09-12)
ENV REV=13449ad91cb26cb47661c1b080790392170385fd
RUN go get -d golang.org/x/oauth2 `#and 5 other pkgs` &&\
(cd /go/src/golang.org/x/oauth2 && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/sync at f52d181 (2017-05-17)
ENV REV=f52d1811a62927559de87708c8913c1650ce4f26
RUN go get -d golang.org/x/sync/errgroup &&\
(cd /go/src/golang.org/x/sync && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo golang.org/x/text at 1cbadb4 (2017-09-15)
ENV REV=1cbadb444a806fd9430d14ad08967ed91da4fa0a
RUN go get -d golang.org/x/text/secure/bidirule `#and 4 other pkgs` &&\
(cd /go/src/golang.org/x/text && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo google.golang.org/api at cec5cc0 (2017-09-24)
ENV REV=cec5cc05d57604ba2d2eaaf5381d3248fac4814d
RUN go get -d google.golang.org/api/gensupport `#and 9 other pkgs` &&\
(cd /go/src/google.golang.org/api && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo google.golang.org/genproto at 1e559d0 (2017-09-18)
ENV REV=1e559d0a00eef8a9a43151db4665280bd8dd5886
RUN go get -d google.golang.org/genproto/googleapis/api/annotations `#and 3 other pkgs` &&\
(cd /go/src/google.golang.org/genproto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Repo google.golang.org/grpc at d4b75eb (2017-09-21)
ENV REV=d4b75ebd4f9f8c4a2b1cdadbdbe0d7920431ccca
RUN go get -d google.golang.org/grpc `#and 17 other pkgs` &&\
(cd /go/src/google.golang.org/grpc && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Optimization to speed up iterative development, not necessary for correctness:
RUN go install cloud.google.com/go/compute/metadata \
cloud.google.com/go/iam \
cloud.google.com/go/internal \
cloud.google.com/go/internal/optional \
cloud.google.com/go/internal/version \
cloud.google.com/go/storage \
github.com/golang/protobuf/proto \
github.com/golang/protobuf/protoc-gen-go/descriptor \
github.com/golang/protobuf/ptypes \
github.com/golang/protobuf/ptypes/any \
github.com/golang/protobuf/ptypes/duration \
github.com/golang/protobuf/ptypes/timestamp \
github.com/google/go-github/github \
github.com/google/go-querystring/query \
github.com/googleapis/gax-go \
github.com/gregjones/httpcache \
go4.org/types \
golang.org/x/crypto/acme \
golang.org/x/crypto/acme/autocert \
golang.org/x/net/context \
golang.org/x/net/context/ctxhttp \
golang.org/x/net/http2 \
golang.org/x/net/http2/hpack \
golang.org/x/net/idna \
golang.org/x/net/internal/timeseries \
golang.org/x/net/lex/httplex \
golang.org/x/net/trace \
golang.org/x/oauth2 \
golang.org/x/oauth2/google \
golang.org/x/oauth2/internal \
golang.org/x/oauth2/jws \
golang.org/x/oauth2/jwt \
golang.org/x/sync/errgroup \
golang.org/x/text/secure/bidirule \
golang.org/x/text/transform \
golang.org/x/text/unicode/bidi \
golang.org/x/text/unicode/norm \
google.golang.org/api/gensupport \
google.golang.org/api/googleapi \
google.golang.org/api/googleapi/internal/uritemplates \
google.golang.org/api/googleapi/transport \
google.golang.org/api/internal \
google.golang.org/api/iterator \
google.golang.org/api/option \
google.golang.org/api/storage/v1 \
google.golang.org/api/transport/http \
google.golang.org/genproto/googleapis/api/annotations \
google.golang.org/genproto/googleapis/iam/v1 \
google.golang.org/genproto/googleapis/rpc/status \
google.golang.org/grpc \
google.golang.org/grpc/balancer \
google.golang.org/grpc/codes \
google.golang.org/grpc/connectivity \
google.golang.org/grpc/credentials \
google.golang.org/grpc/grpclb/grpc_lb_v1/messages \
google.golang.org/grpc/grpclog \
google.golang.org/grpc/internal \
google.golang.org/grpc/keepalive \
google.golang.org/grpc/metadata \
google.golang.org/grpc/naming \
google.golang.org/grpc/peer \
google.golang.org/grpc/resolver \
google.golang.org/grpc/stats \
google.golang.org/grpc/status \
google.golang.org/grpc/tap \
google.golang.org/grpc/transport
# END deps
COPY . /go/src/golang.org/x/build/
RUN go install golang.org/x/build/cmd/gerritbot

51
cmd/gerritbot/Makefile Normal file
Просмотреть файл

@ -0,0 +1,51 @@
# Copyright 2017 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
MUTABLE_VERSION ?= latest
VERSION ?= $(shell git rev-parse --short HEAD)
IMAGE_STAGING := gcr.io/go-dashboard-dev/gerritbot
IMAGE_PROD := gcr.io/symbolic-datum-552/gerritbot
DOCKER_IMAGE_build0=build0/gerritbot:latest
DOCKER_CTR_build0=gerritbot-build0
build0: *.go Dockerfile.0
docker build --force-rm -f Dockerfile.0 --tag=$(DOCKER_IMAGE_build0) ../
gerritbot: build0
docker create --name $(DOCKER_CTR_build0) $(DOCKER_IMAGE_build0)
docker cp $(DOCKER_CTR_build0):/go/bin/$@ $@
docker rm $(DOCKER_CTR_build0)
ca-certificates.crt:
docker create --name $(DOCKER_CTR_build0) $(DOCKER_IMAGE_build0)
docker cp $(DOCKER_CTR_build0):/etc/ssl/certs/$@ $@
docker rm $(DOCKER_CTR_build0)
update-deps:
go install golang.org/x/build/cmd/gitlock
gitlock --update=Dockerfile.0 golang.org/x/build/cmd/gerritbot
docker-prod: Dockerfile gerritbot ca-certificates.crt
docker build --force-rm --tag=$(IMAGE_PROD):$(VERSION) .
docker tag $(IMAGE_PROD):$(VERSION) $(IMAGE_PROD):$(MUTABLE_VERSION)
docker-staging: Dockerfile gerritbot ca-certificates.crt
docker build --force-rm --tag=$(IMAGE_STAGING):$(VERSION) .
docker tag $(IMAGE_STAGING):$(VERSION) $(IMAGE_STAGING):$(MUTABLE_VERSION)
push-prod: docker-prod
gcloud docker -- push $(IMAGE_PROD):$(VERSION)
push-staging: docker-staging
gcloud docker -- push $(IMAGE_STAGING):$(VERSION)
deploy-prod: push-prod
kubectl set image deployment/gerritbot-deployment gerritbot=$(IMAGE_PROD):$(VERSION)
deploy-staging: push-staging
kubectl set image deployment/gerritbot-deployment gerritbot=$(IMAGE_STAGING):$(VERSION)
.PHONY: clean
clean:
$(RM) gerritbot
$(RM) ca-certificates.crt

7
cmd/gerritbot/README.md Normal file
Просмотреть файл

@ -0,0 +1,7 @@
<!-- Auto-generated by x/build/update-readmes.go -->
[![GoDoc](https://godoc.org/golang.org/x/build/cmd/gerritbot?status.svg)](https://godoc.org/golang.org/x/build/cmd/gerritbot)
# golang.org/x/build/cmd/gerritbot
The gerritbot binary converts GitHub Pull Requests to Gerrit Changes, updating the PR and Gerrit Change as appropriate.

Просмотреть файл

@ -0,0 +1,28 @@
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: gerritbot-deployment
spec:
replicas: 1
template:
metadata:
labels:
app: gerritbot
annotations:
container.seccomp.security.alpha.kubernetes.io/gerritbot: docker/default
container.apparmor.security.beta.kubernetes.io/gerritbot: runtime/default
spec:
containers:
- name: gerritbot
image: gcr.io/symbolic-datum-552/gerritbot:latest
imagePullPolicy: Always
command: ["/gerritbot", "-listen=:80", "-autocert-bucket=golang-gerritbot-autocert"]
ports:
- containerPort: 80
- containerPort: 443
resources:
requests:
cpu: "1"
memory: "1Gi"
limits:
memory: "2Gi"

104
cmd/gerritbot/gerritbot.go Normal file
Просмотреть файл

@ -0,0 +1,104 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// The gerritbot binary converts GitHub Pull Requests to Gerrit Changes,
// updating the PR and Gerrit Change as appropriate.
package main
import (
"context"
"flag"
"fmt"
"log"
"net/http"
"sync"
"time"
"golang.org/x/build/internal/https"
"golang.org/x/build/maintner"
"golang.org/x/build/maintner/godata"
)
func main() {
var (
listen = flag.String("listen", "localhost:6343", "listen address")
autocertBucket = flag.String("autocert-bucket", "", "if non-empty, listen on port 443 and serve a LetsEncrypt TLS cert using this Google Cloud Storage bucket as a cache")
)
flag.Parse()
ctx := context.Background()
initCorpus(ctx)
go corpusUpdateLoop(ctx)
https.ListenAndServe(http.HandlerFunc(handleIndex), &https.Options{
Addr: *listen,
AutocertCacheBucket: *autocertBucket,
})
}
func handleIndex(w http.ResponseWriter, r *http.Request) {
r.Header.Set("Content-Type", "text/html; charset=utf-8")
fmt.Fprintln(w, "Hello, GerritBot! 🤖")
}
var (
mu sync.RWMutex
corpus *maintner.Corpus
)
// initCorpus fetches a full maintner corpus, overwriting any existing data.
func initCorpus(ctx context.Context) error {
mu.Lock()
defer mu.Unlock()
var err error
corpus, err = godata.Get(ctx)
if err != nil {
return fmt.Errorf("godata.Get: %v", err)
}
return nil
}
// corpusUpdateLoop continuously updates the servers corpus until ctxs Done
// channel is closed.
func corpusUpdateLoop(ctx context.Context) {
log.Println("Starting corpus update loop ...")
for {
checkPullRequests()
err := corpus.UpdateWithLocker(ctx, &mu)
if err != nil {
if err == maintner.ErrSplit {
log.Println("Corpus out of sync. Re-fetching corpus.")
initCorpus(ctx)
} else {
log.Printf("corpus.Update: %v; sleeping 15s", err)
time.Sleep(15 * time.Second)
continue
}
}
select {
case <-ctx.Done():
return
default:
continue
}
}
}
func checkPullRequests() {
mu.RLock()
defer mu.RUnlock()
repo := corpus.GitHub().Repo("golang", "scratch")
repo.ForeachIssue(func(issue *maintner.GitHubIssue) error {
if issue.Closed || !issue.PullRequest {
return nil
}
return processPullRequest(issue)
})
}
// processPullRequest requires mu to be held.
func processPullRequest(pr *maintner.GitHubIssue) error {
return nil
}

Просмотреть файл

@ -0,0 +1,16 @@
apiVersion: v1
kind: Service
metadata:
name: gerritbot
spec:
ports:
- port: 80
targetPort: 80
name: http
- port: 443
targetPort: 443
name: https
selector:
app: gerritbot
type: LoadBalancer
loadBalancerIP: 35.188.188.192

7
internal/https/README.md Normal file
Просмотреть файл

@ -0,0 +1,7 @@
<!-- Auto-generated by x/build/update-readmes.go -->
[![GoDoc](https://godoc.org/golang.org/x/build/internal/https?status.svg)](https://godoc.org/golang.org/x/build/internal/https)
# golang.org/x/build/internal/https
Package https contains helpers for starting an HTTPS server.

126
internal/https/https.go Normal file
Просмотреть файл

@ -0,0 +1,126 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package https contains helpers for starting an HTTPS server.
package https // import "golang.org/x/build/internal/https"
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"strings"
"time"
"cloud.google.com/go/storage"
"golang.org/x/build/autocertcache"
"golang.org/x/crypto/acme/autocert"
"golang.org/x/net/http2"
)
// Options are the configuration parameters for the HTTP(S) server.
type Options struct {
// Addr specifies the host and port the server should listen on.
Addr string
// AutocertCacheBucket specifies the name of the GCS bucket for
// Lets Encrypt to use. If this is not specified, then HTTP traffic is
// served on Addr.
AutocertCacheBucket string
}
var defaultOptions = &Options{
Addr: "localhost:6343",
}
// ListenAndServe serves the given handler by HTTPS (and HTTP, redirecting to
// HTTPS) using the provided options.
func ListenAndServe(handler http.Handler, opt *Options) error {
if opt == nil {
opt = defaultOptions
}
ln, err := net.Listen("tcp", opt.Addr)
if err != nil {
return fmt.Errorf(`net.Listen("tcp", %q): %v`, opt.Addr, err)
}
errc := make(chan error)
if ln != nil {
go func() {
if opt.AutocertCacheBucket != "" {
handler = http.HandlerFunc(redirectToHTTPS)
}
errc <- fmt.Errorf("http.Serve = %v", http.Serve(ln, handler))
}()
}
if opt.AutocertCacheBucket != "" {
go func() { errc <- serveAutocertTLS(handler, opt.AutocertCacheBucket) }()
}
return <-errc
}
// redirectToHTTPS will redirect to the https version of the URL requested. If
// r.TLS is set or r.Host is empty, a 404 not found response is sent.
func redirectToHTTPS(w http.ResponseWriter, r *http.Request) {
if r.TLS != nil || r.Host == "" {
http.NotFound(w, r)
return
}
http.Redirect(w, r, "https://"+r.Host+r.RequestURI, http.StatusFound)
}
// serveAutocertTLS serves the handler h on port 443 using the given GCS bucket
// for its autocert cache. It will only serve on domains of the form *.golang.org.
func serveAutocertTLS(h http.Handler, bucket string) error {
ln, err := net.Listen("tcp", ":443")
if err != nil {
return err
}
defer ln.Close()
sc, err := storage.NewClient(context.Background())
if err != nil {
return fmt.Errorf("storage.NewClient: %v", err)
}
const hostSuffix = ".golang.org"
m := autocert.Manager{
Prompt: autocert.AcceptTOS,
HostPolicy: func(ctx context.Context, host string) error {
if !strings.HasSuffix(host, hostSuffix) {
return fmt.Errorf("refusing to serve autocert on provided domain (%q), must have the suffix %q",
host, hostSuffix)
}
return nil
},
Cache: autocertcache.NewGoogleCloudStorageCache(sc, bucket),
}
config := &tls.Config{
GetCertificate: m.GetCertificate,
NextProtos: []string{"h2", "http/1.1"},
}
tlsLn := tls.NewListener(tcpKeepAliveListener{ln.(*net.TCPListener)}, config)
server := &http.Server{
Addr: ln.Addr().String(),
Handler: h,
}
if err := http2.ConfigureServer(server, nil); err != nil {
return fmt.Errorf("http2.ConfigureServer: %v", err)
}
return server.Serve(tlsLn)
}
type tcpKeepAliveListener struct {
*net.TCPListener
}
func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
tc, err := ln.AcceptTCP()
if err != nil {
return
}
tc.SetKeepAlive(true)
tc.SetKeepAlivePeriod(3 * time.Minute)
return tc, nil
}