build/cmd/gerritbot/gerritbot.go

// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The gerritbot binary converts GitHub Pull Requests to Gerrit Changes,
// updating the PR and Gerrit Change as appropriate.
package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"cloud.google.com/go/compute/metadata"
	"github.com/google/go-github/github"
	"golang.org/x/build/gerrit"
	"golang.org/x/build/internal/https"
	"golang.org/x/build/maintner"
	"golang.org/x/build/maintner/godata"
	"golang.org/x/oauth2"
)

var (
	listen          = flag.String("listen", "localhost:6343", "listen address")
	autocertBucket  = flag.String("autocert-bucket", "", "if non-empty, listen on port 443 and serve a LetsEncrypt TLS cert using this Google Cloud Storage bucket as a cache")
	workdir         = flag.String("workdir", defaultWorkdir(), "where git repos and temporary worktrees are created")
	githubTokenFile = flag.String("github-token-file", filepath.Join(defaultWorkdir(), "github-token"), "file to load GitHub token from; should only contain the token text")
	gerritTokenFile = flag.String("gerrit-token-file", filepath.Join(defaultWorkdir(), "gerrit-token"), "file to load Gerrit token from; should be of form <git-email>:<token>")
)

func main() {
	flag.Parse()

	ghc, err := githubClient()
	if err != nil {
		log.Fatalf("githubClient(): %v", err)
	}
	gc, err := gerritClient()
	if err != nil {
		log.Fatalf("gerritClient(): %v", err)
	}
	b := newBot(ghc, gc)

	ctx := context.Background()
	b.initCorpus(ctx)
	go b.corpusUpdateLoop(ctx)

	https.ListenAndServe(http.HandlerFunc(handleIndex), &https.Options{
		Addr:                *listen,
		AutocertCacheBucket: *autocertBucket,
	})
}

func defaultWorkdir() string {
	// TODO(andybons): Use os.UserCacheDir (issue 22536) when it's available.
	return filepath.Join(home(), ".gerritbot")
}

func home() string {
	h := os.Getenv("HOME")
	if h != "" {
		return h
	}
	u, err := user.Current()
	if err != nil {
		log.Fatalf("user.Current(): %v", err)
	}
	return u.HomeDir
}

func githubClient() (*github.Client, error) {
	token, err := githubToken()
	if err != nil {
		return nil, err
	}
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
	tc := oauth2.NewClient(context.Background(), ts)
	return github.NewClient(tc), nil
}

func githubToken() (string, error) {
	if metadata.OnGCE() {
		token, err := metadata.ProjectAttributeValue("maintner-github-token")
		if err == nil {
			return token, nil
		}
	}
	slurp, err := ioutil.ReadFile(*githubTokenFile)
	if err != nil {
		return "", err
	}
	tok := strings.TrimSpace(string(slurp))
	if len(tok) == 0 {
		return "", fmt.Errorf("token from file %q cannot be empty", *githubTokenFile)
	}
	return tok, nil
}

func gerritClient() (*gerrit.Client, error) {
	username, token, err := gerritAuth()
	if err != nil {
		return nil, err
	}
	c := gerrit.NewClient("https://go-review.googlesource.com", gerrit.BasicAuth(username, token))
	return c, nil
}

func gerritAuth() (string, string, error) {
	var slurp string
	if metadata.OnGCE() {
		var err error
		slurp, err = metadata.ProjectAttributeValue("gobot-password")
		if err != nil {
			log.Printf(`Error retrieving Project Metadata "gobot-password": %v`, err)
		}
	}
	if len(slurp) == 0 {
		slurpBytes, err := ioutil.ReadFile(*gerritTokenFile)
		if err != nil {
			return "", "", err
		}
		slurp = string(slurpBytes)
	}
	f := strings.SplitN(strings.TrimSpace(slurp), ":", 2)
	if len(f) == 1 {
		// Assume the whole thing is the token.
		return "git-gobot.golang.org", f[0], nil
	}
	if len(f) != 2 || f[0] == "" || f[1] == "" {
		return "", "", fmt.Errorf("expected Gerrit token to be of form <git-email>:<token>")
	}
	return f[0], f[1], nil
}

func handleIndex(w http.ResponseWriter, r *http.Request) {
	r.Header.Set("Content-Type", "text/html; charset=utf-8")
	fmt.Fprintln(w, "Hello, GerritBot! 🤖")
}

const (
	// Footer that contains the last revision from GitHub that was successfully
	// imported to Gerrit.
	prefixGitFooterLastRev = "GitHub-Last-Rev:"

	// Footer containing the GitHub PR associated with the Gerrit Change.
	prefixGitFooterPR = "GitHub-Pull-Request:"

	// Footer containing the Gerrit Change ID.
	prefixGitFooterChangeID = "Change-Id:"
)

var (
	// GitHub repos we accept PRs for, mirroring them into Gerrit CLs.
	githubRepoWhitelist = map[string]bool{
		"golang/scratch": true,
	}
	// Gerrit projects we accept PRs for.
	gerritProjectWhitelist = map[string]bool{
		"scratch": true,
	}
)

type bot struct {
	githubClient *github.Client
	gerritClient *gerrit.Client

	sync.RWMutex // Protects all fields below
	corpus       *maintner.Corpus
	importedPRs  map[string]*maintner.GerritCL // GitHub owner/repo#n -> Gerrit CL

	// CLs that have been created/updated on Gerrit for GitHub PRs but are not yet
	// reflected in the maintner corpus yet.
	pendingCLs map[string]string // GitHub owner/repo#n -> latest GitHub SHA
}

func newBot(githubClient *github.Client, gerritClient *gerrit.Client) *bot {
	return &bot{
		githubClient: githubClient,
		gerritClient: gerritClient,
		importedPRs:  map[string]*maintner.GerritCL{},
		pendingCLs:   map[string]string{},
	}
}

// initCorpus fetches a full maintner corpus, overwriting any existing data.
func (b *bot) initCorpus(ctx context.Context) error {
	b.Lock()
	defer b.Unlock()
	var err error
	b.corpus, err = godata.Get(ctx)
	if err != nil {
		return fmt.Errorf("godata.Get: %v", err)
	}
	return nil
}

// corpusUpdateLoop continuously updates the server’s corpus until ctx’s Done
// channel is closed.
func (b *bot) corpusUpdateLoop(ctx context.Context) {
	log.Println("Starting corpus update loop ...")
	for {
		b.checkPullRequests()
		err := b.corpus.UpdateWithLocker(ctx, &b.RWMutex)
		if err != nil {
			if err == maintner.ErrSplit {
				log.Println("Corpus out of sync. Re-fetching corpus.")
				b.initCorpus(ctx)
			} else {
				log.Printf("corpus.Update: %v; sleeping 15s", err)
				time.Sleep(15 * time.Second)
				continue
			}
		}

		select {
		case <-ctx.Done():
			return
		default:
			continue
		}
	}
}

func (b *bot) checkPullRequests() {
	b.Lock()
	defer b.Unlock()
	b.corpus.Gerrit().ForeachProjectUnsorted(func(p *maintner.GerritProject) error {
		pname := p.Project()
		if !gerritProjectWhitelist[pname] {
			return nil
		}
		return p.ForeachOpenCL(func(cl *maintner.GerritCL) error {
			prv := cl.Footer(prefixGitFooterPR)
			if prv == "" {
				return nil
			}
			b.importedPRs[prv] = cl
			return nil
		})
	})

	if err := b.corpus.GitHub().ForeachRepo(func(ghr *maintner.GitHubRepo) error {
		id := ghr.ID()
		if !githubRepoWhitelist[id.Owner+"/"+id.Repo] {
			return nil
		}
		return ghr.ForeachIssue(func(issue *maintner.GitHubIssue) error {
			if issue.Closed || !issue.PullRequest || !issue.HasLabel("cla: yes") {
				return nil
			}
			ctx := context.Background()
			pr, err := b.getFullPR(ctx, id.Owner, id.Repo, int(issue.Number))
			if err != nil {
				return fmt.Errorf("getFullPR(ctx, %q, %q, %d): %v", id.Owner, id.Repo, issue.Number, err)
			}
			return b.processPullRequest(ctx, pr)
		})
	}); err != nil {
		log.Printf("corpus.GitHub().ForeachRepo(...): %v", err)
	}
}

// prShortLink returns text referencing a Pull Request that will be automatically
// converted into a link by GitHub.
func prShortLink(pr *github.PullRequest) string {
	repo := pr.GetBase().GetRepo()
	return fmt.Sprintf("%s/%s#%d", repo.GetOwner().GetLogin(), repo.GetName(), pr.GetNumber())
}

// processPullRequest is the entry point to the state machine of mirroring a PR
// with Gerrit. PRs that are up to date with their respective Gerrit changes are
// skipped, and any with a squashed commit SHA unequal to its Gerrit equivalent
// are imported. Those that have no associated Gerrit changes will result in one
// being created.
// TODO(andybons): Document comment mirroring once that is implemented.
// b's RWMutex read-write lock must be held.
func (b *bot) processPullRequest(ctx context.Context, pr *github.PullRequest) error {
	log.Printf("Processing PR %s ...", pr.GetHTMLURL())
	shortLink := prShortLink(pr)
	cl := b.importedPRs[shortLink]
	var lastRev string
	if cl != nil {
		lastRev = cl.Footer(prefixGitFooterLastRev)
	}
	if cl != nil && b.pendingCLs[shortLink] == lastRev {
		delete(b.pendingCLs, shortLink)
	}
	if b.pendingCLs[shortLink] != "" {
		return nil
	}

	if pr.GetCommits() == 0 {
		// Um. Wat?
		return fmt.Errorf("pr has 0 commits")
	}
	prHeadSHA := pr.Head.GetSHA()
	if pr.GetCommits() > 1 {
		repo := pr.GetBase().GetRepo()
		return b.postGitHubMessageNoDup(ctx, repo.GetOwner().GetLogin(), repo.GetName(), pr.GetNumber(),
			fmt.Sprintf("Head %v has %d commits. Please squash your commits into one.", prHeadSHA, pr.GetCommits()))
	}

	if cl == nil {
		gcl, err := b.gerritChangeForPR(pr)
		if err != nil {
			return fmt.Errorf("gerritChangeForPR(%+v): %v", pr, err)
		}
		if gcl != nil {
			b.pendingCLs[shortLink] = prHeadSHA
			return nil
		}
		if err := importGerritChangeFromPR(pr, nil); err != nil {
			return fmt.Errorf("importGerritChangeFromPR(%v, nil): %v", shortLink, err)
		}
		b.pendingCLs[shortLink] = prHeadSHA
		return nil
	}

	if lastRev == "" {
		log.Printf("Imported CL https://go-review.googlesource.com/q/%s does not have %s footer; skipping",
			cl.ChangeID(), prefixGitFooterLastRev)
		return nil
	}
	if pr.Head.GetSHA() == lastRev {
		log.Printf("Change https://go-review.googlesource.com/q/%s is up to date; nothing to do.",
			cl.ChangeID())
		return nil
	}
	// Import PR to existing Gerrit Change.
	if err := importGerritChangeFromPR(pr, cl); err != nil {
		return fmt.Errorf("importGerritChangeFromPR(%v, %v): %v", shortLink, cl, err)
	}
	b.pendingCLs[shortLink] = prHeadSHA
	return nil
}

// gerritChangeForPR returns the Gerrit Change info associated with the given PR.
// If no change exists for pr, it returns nil (with a nil error).
func (b *bot) gerritChangeForPR(pr *github.PullRequest) (*gerrit.ChangeInfo, error) {
	q := fmt.Sprintf(`is:open "%s %s"`, prefixGitFooterPR, prShortLink(pr))
	cs, err := b.gerritClient.QueryChanges(context.Background(), q)
	if err != nil {
		return nil, fmt.Errorf("c.QueryChanges(ctx, %q): %v", q, err)
	}
	if len(cs) == 0 {
		return nil, nil
	}
	// Return the first result no matter what.
	return cs[0], nil
}

// downloadRef calls the Gerrit API to retrieve the ref (such as refs/changes/16/81116/1)
// of the most recent patch set of the change with changeID.
func (b *bot) downloadRef(ctx context.Context, changeID string) (string, error) {
	opt := gerrit.QueryChangesOpt{Fields: []string{"CURRENT_REVISION"}}
	ch, err := b.gerritClient.GetChange(ctx, changeID, opt)
	if err != nil {
		return "", fmt.Errorf("c.GetChange(ctx, %q, %+v): %v", changeID, opt, err)
	}
	rev, ok := ch.Revisions[ch.CurrentRevision]
	if !ok {
		return "", fmt.Errorf("revisions[current_revision] is not present in %+v", ch)
	}
	return rev.Ref, nil
}

const gerritHostBase = "https://go.googlesource.com/"

// importGerritChangeFromPR mirrors the latest state of pr to cl. If cl is nil,
// then a new Gerrit Change is created.
func importGerritChangeFromPR(pr *github.PullRequest, cl *maintner.GerritCL) error {
	githubRepo := pr.GetBase().GetRepo()
	gerritRepo := gerritHostBase + githubRepo.GetName() // GitHub repo name should match Gerrit repo name.
	repoDir := filepath.Join(reposRoot(), url.PathEscape(gerritRepo))

	if _, err := os.Stat(repoDir); os.IsNotExist(err) {
		commitHook := filepath.Join(repoDir, "/hooks/commit-msg")

		cmds := []*exec.Cmd{
			exec.Command("git", "clone", "--bare", gerritRepo, repoDir),
			exec.Command("curl", "-Lo", commitHook, "https://gerrit-review.googlesource.com/tools/hooks/commit-msg"),
			exec.Command("chmod", "+x", commitHook),
			exec.Command("git", "-C", repoDir, "remote", "add", "github", githubRepo.GetGitURL()),
		}
		for _, c := range cmds {
			log.Printf("Executing %v", c.Args)
			if b, err := c.CombinedOutput(); err != nil {
				return fmt.Errorf("running %v: %s", c.Args, b)
			}
		}
	}

	worktree := fmt.Sprintf("worktree_%s_%s_%d", githubRepo.GetOwner().GetLogin(), githubRepo.GetName(), pr.GetNumber())
	worktreeDir := filepath.Join(*workdir, "tmp", worktree)
	// workTreeDir is created by the `git worktree add` command.
	defer func() {
		log.Println("Cleaning up...")
		for _, c := range []*exec.Cmd{
			exec.Command("git", "-C", worktreeDir, "checkout", "master"),
			exec.Command("git", "-C", worktreeDir, "branch", "-D", prShortLink(pr)),
			exec.Command("rm", "-rf", worktreeDir),
			exec.Command("git", "-C", repoDir, "worktree", "prune"),
			exec.Command("git", "-C", repoDir, "branch", "-D", worktree),
		} {
			log.Printf("Executing %v", c.Args)
			if b, err := c.CombinedOutput(); err != nil {
				log.Printf("running %v: %s", c.Args, b)
			}
		}
	}()
	for _, c := range []*exec.Cmd{
		exec.Command("rm", "-rf", worktreeDir),
		exec.Command("git", "-C", repoDir, "worktree", "prune"),
		exec.Command("git", "-C", repoDir, "worktree", "add", worktreeDir),
		exec.Command("git", "-C", worktreeDir, "pull", "origin", "master"),
	} {
		log.Printf("Executing %v", c.Args)
		if b, err := c.CombinedOutput(); err != nil {
			return fmt.Errorf("running %v: %s", c.Args, b)
		}
	}

	commitMsg := fmt.Sprintf("%s\n\n%s\n\n%s %s\n%s %s\n",
		pr.GetTitle(),
		pr.GetBody(),
		prefixGitFooterLastRev, pr.Head.GetSHA(),
		prefixGitFooterPR, prShortLink(pr))
	if cl != nil {
		commitMsg += fmt.Sprintf("%s %s\n", prefixGitFooterChangeID, cl.ChangeID())
	}
	for _, c := range []*exec.Cmd{
		exec.Command("git", "-C", worktreeDir, "fetch", "github", fmt.Sprintf("pull/%d/head", pr.GetNumber())),
		exec.Command("git", "-C", worktreeDir, "checkout", pr.Base.GetSHA(), "-b", prShortLink(pr)),
		exec.Command("git", "-C", worktreeDir, "merge", "FETCH_HEAD"),
		exec.Command("git", "-C", worktreeDir, "commit", "--amend", "-m", commitMsg),
		exec.Command("git", "-C", worktreeDir, "push", "origin", "HEAD:refs/for/"+pr.GetBase().GetRef()),
	} {
		log.Printf("Executing %v", c.Args)
		if b, err := c.CombinedOutput(); err != nil {
			return fmt.Errorf("running %v: %s", c.Args, b)
		}
	}
	return nil
}

func reposRoot() string {
	return filepath.Join(*workdir, "repos")
}

func (b *bot) getFullPR(ctx context.Context, owner, repo string, number int) (*github.PullRequest, error) {
	pr, resp, err := b.githubClient.PullRequests.Get(ctx, owner, repo, number)
	defer closeGitHubResp(resp)
	if err != nil {
		return nil, fmt.Errorf("ghc.PullRequests().Get(ctc, %q, %q, %d): %v", owner, repo, number, err)
	}
	return pr, nil
}

func closeGitHubResp(resp *github.Response) {
	if resp != nil && resp.Body != nil {
		resp.Body.Close()
	}
}

// postGitHubMessage posts the given message to the given issue. This is likely not the method
// you are looking to use. To ensure that duplicate messages are not posted, use postGitHubMessageNoDup.
func (b *bot) postGitHubMessage(ctx context.Context, owner, repo string, issueNum int, msg string) error {
	cmt := &github.IssueComment{Body: github.String(msg)}
	_, resp, err := b.githubClient.Issues.CreateComment(ctx, owner, repo, issueNum, cmt)
	defer closeGitHubResp(resp)
	if err != nil {
		return fmt.Errorf("b.githubClient.Issues.CreateComment(ctx, %q, %q, %d, %+v): %v", owner, repo, issueNum, cmt, err)
	}
	return nil
}

// postGitHubMessageNoDup ensures that the message being posted on an issue does not already have the
// same exact content.
func (b *bot) postGitHubMessageNoDup(ctx context.Context, owner, repo string, issueNum int, msg string) error {
	opts := &github.IssueListCommentsOptions{
		ListOptions: github.ListOptions{
			PerPage: 1000,
		},
	}
	comments, res, err := b.githubClient.Issues.ListComments(ctx, owner, repo, issueNum, opts)
	defer closeGitHubResp(res)
	if err != nil {
		return fmt.Errorf("b.githubClient.Issues.ListComments(%q, %q, %d, %+v): %v", owner, repo, issueNum, opts, err)
	}
	for _, ic := range comments {
		if ic.GetBody() == msg {
			return nil
		}
	}
	return b.postGitHubMessage(ctx, owner, repo, issueNum, msg)
}