cmd/coordinator, internal: move benchmark building to separate package

This moves getSourceTgz* to a new package called sourcecache. The
original functions couldn't be left in coordinator.go because that
would cause a single process to have multiple caches.

The code for building and running benchmarks is moved into the buildgo
package, and the public API becomes GoBuilder.EnumerateBenchmarks and
Run on the resulting BenchmarkItem objects.

Updates golang/go#19871

Change-Id: I28b660e1cdaa6d1c6b0378c08de30f5e58316cc6
Reviewed-on: https://go-review.googlesource.com/44211
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

cmd/coordinator/coordinator.go

@@ -55,8 +55,7 @@ import (
 	"golang.org/x/build/dashboard"
 	"golang.org/x/build/gerrit"
 	"golang.org/x/build/internal/buildgo"
-	"golang.org/x/build/internal/lru"
+	"golang.org/x/build/internal/sourcecache"
-	"golang.org/x/build/internal/singleflight"
 	"golang.org/x/build/livelog"
 	"golang.org/x/build/maintner/maintnerd/apipb"
 	"golang.org/x/build/types"
@@ -365,7 +364,12 @@ func init() {
 		log.Fatal(err)
 	}
 	watcherProxy = httputil.NewSingleHostReverseProxy(u)
-	watcherProxy.Transport = gitMirrorClient.Transport
+	watcherProxy.Transport = &http.Transport{
+		IdleConnTimeout: 30 * time.Second,
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return goKubeClient.DialServicePort(ctx, "gitmirror", "")
+		},
+	}
 }
 
 func handleDebugWatcher(w http.ResponseWriter, r *http.Request) {
@@ -1348,6 +1352,25 @@ func (st *buildStatus) buildletPool() BuildletPool {
 	return poolForConf(st.conf)
 }
 
+// parentRev returns the parent of this build's commit (but only if this build comes from a trySet).
+func (st *buildStatus) parentRev() (pbr buildgo.BuilderRev, err error) {
+	// TODO(quentin): Support non-try commits by asking maintnerd
+	// (at which point this no longer needs to be a method on buildStatus).
+	pbr = st.BuilderRev // copy
+	rev := st.trySet.ci.Revisions[st.trySet.ci.CurrentRevision]
+	if rev.Commit == nil {
+		err = fmt.Errorf("commit information missing for revision %q", st.trySet.ci.CurrentRevision)
+		return
+	}
+	if len(rev.Commit.Parents) == 0 {
+		// TODO(quentin): Log?
+		err = errors.New("commit has no parent")
+		return
+	}
+	pbr.Rev = rev.Commit.Parents[0].CommitID
+	return
+}
+
 func (st *buildStatus) expectedMakeBashDuration() time.Duration {
 	// TODO: base this on historical measurements, instead of statically configured.
 	// TODO: move this to dashboard/builders.go? But once we based on on historical
@@ -1688,6 +1711,16 @@ func (st *buildStatus) shouldBench() bool {
 	return st.isTry() && !st.IsSubrepo() && st.conf.RunBench
 }
 
+// goBuilder returns a GoBuilder for this buildStatus.
+func (st *buildStatus) goBuilder() buildgo.GoBuilder {
+	return buildgo.GoBuilder{
+		Logger:     st,
+		BuilderRev: st.BuilderRev,
+		Conf:       st.conf,
+		Goroot:     "go",
+	}
+}
+
 // runAllSharded runs make.bash and then shards the test execution.
 // remoteErr and err are as described at the top of this file.
 //
@@ -1698,13 +1731,7 @@ func (st *buildStatus) runAllSharded() (remoteErr, err error) {
 	st.getHelpersReadySoon()
 
 	if !st.useSnapshot() {
-		builder := buildgo.GoBuilder{
+		remoteErr, err = st.goBuilder().RunMake(st.bc, st)
-			Logger:     st,
-			BuilderRev: st.BuilderRev,
-			Conf:       st.conf,
-			Goroot:     "go",
-		}
-		remoteErr, err = builder.RunMake(st.bc, st)
 		if err != nil {
 			return nil, err
 		}
@@ -1876,7 +1903,7 @@ func (st *buildStatus) writeGoSourceTo(bc *buildlet.Client) error {
 		return sp.Done(fmt.Errorf("writing VERSION tgz: %v", err))
 	}
 
-	srcTar, err := getSourceTgz(st, "go", st.Rev)
+	srcTar, err := sourcecache.GetSourceTgz(st, "go", st.Rev)
 	if err != nil {
 		return err
 	}
@@ -1995,7 +2022,7 @@ func (st *buildStatus) shouldSkipTest(testName string) bool {
 	return false
 }
 
-func (st *buildStatus) newTestSet(names []string, benchmarks []*benchmarkItem) *testSet {
+func (st *buildStatus) newTestSet(names []string, benchmarks []*buildgo.BenchmarkItem) *testSet {
 	set := &testSet{
 		st: st,
 	}
@@ -2009,7 +2036,7 @@ func (st *buildStatus) newTestSet(names []string, benchmarks []*benchmarkItem) *
 		})
 	}
 	for _, bench := range benchmarks {
-		name := "bench:" + bench.name()
+		name := "bench:" + bench.Name()
 		set.items = append(set.items, &testItem{
 			set:      set,
 			name:     name,
@@ -2393,19 +2420,11 @@ func testDuration(builderName, testName string) time.Duration {
 	}
 	if strings.HasPrefix(testName, "bench:") {
 		// Assume benchmarks are roughly 20 seconds per run.
-		return 2 * benchRuns * 20 * time.Second
+		return 2 * 5 * 20 * time.Second
 	}
 	return minGoTestSpeed * 2
 }
 
-func fetchSubrepo(sl spanlog.Logger, bc *buildlet.Client, repo, rev string) error {
-	tgz, err := getSourceTgz(sl, repo, rev)
-	if err != nil {
-		return err
-	}
-	return bc.PutTar(tgz, "gopath/src/"+subrepoPrefix+repo)
-}
-
 func (st *buildStatus) runSubrepoTests() (remoteErr, err error) {
 	st.LogEventTime("fetching_subrepo", st.SubName)
 
@@ -2423,7 +2442,7 @@ func (st *buildStatus) runSubrepoTests() (remoteErr, err error) {
 	// fetch checks out the provided sub-repo to the buildlet's workspace.
 	fetch := func(repo, rev string) error {
 		fetched[repo] = true
-		return fetchSubrepo(st, st.bc, repo, rev)
+		return buildgo.FetchSubrepo(st, st.bc, repo, rev)
 	}
 
 	// findDeps uses 'go list' on the checked out repo to find its
@@ -2503,6 +2522,26 @@ func (st *buildStatus) runSubrepoTests() (remoteErr, err error) {
 	})
 }
 
+// affectedPkgs returns the name of every package affected by this commit.
+// The returned list may contain duplicates and is unsorted.
+// It is safe to call this on a nil trySet.
+func (ts *trySet) affectedPkgs() (pkgs []string) {
+	// TODO(quentin): Support non-try commits by asking maintnerd for the affected files.
+	if ts == nil || ts.ci == nil {
+		return
+	}
+	rev := ts.ci.Revisions[ts.ci.CurrentRevision]
+	for p := range rev.Files {
+		if strings.HasPrefix(p, "src/") {
+			pkg := path.Dir(p[len("src/"):])
+			if pkg != "" {
+				pkgs = append(pkgs, pkg)
+			}
+		}
+	}
+	return
+}
+
 // runTests is only called for builders which support a split make/run
 // (should be everything, at least soon). Currently (2015-05-27) iOS
 // and Android and Nacl do not.
@@ -2517,10 +2556,14 @@ func (st *buildStatus) runTests(helpers <-chan *buildlet.Client) (remoteErr, err
 	if err != nil {
 		return nil, fmt.Errorf("distTestList exec: %v", err)
 	}
-	var benches []*benchmarkItem
+	var benches []*buildgo.BenchmarkItem
 	if st.shouldBench() {
 		sp := st.CreateSpan("enumerate_benchmarks")
-		b, err := enumerateBenchmarks(st, st.conf, st.bc, "go", st.trySet)
+		rev := getRepoHead("benchmarks")
+		if rev == "" {
+			rev = "master" // should happen rarely; ok if it does.
+		}
+		b, err := st.goBuilder().EnumerateBenchmarks(st.bc, rev, st.trySet.affectedPkgs())
 		sp.Done(err)
 		if err == nil {
 			benches = b
@@ -2626,7 +2669,7 @@ func (st *buildStatus) runTests(helpers <-chan *buildlet.Client) (remoteErr, err
 		}
 
 		if ti.bench != nil {
-			for i, s := range ti.bench.output {
+			for i, s := range ti.bench.Output {
 				if i < len(benchFiles) {
 					benchFiles[i].out.WriteString(s)
 				}
@@ -2710,7 +2753,7 @@ func (st *buildStatus) benchFiles() []*benchFile {
 	if !st.shouldBench() {
 		return nil
 	}
-	// We know rev and rev.Commit.Parents[0] exist because benchmarkItem.buildParent has checked.
+	// We know rev and rev.Commit.Parents[0] exist because BenchmarkItem.buildParent has checked.
 	rev := st.trySet.ci.Revisions[st.trySet.ci.CurrentRevision]
 	ps := rev.PatchSetNumber
 	benchFiles := []*benchFile{
@@ -2794,7 +2837,11 @@ func (st *buildStatus) runTestsOnBuildlet(bc *buildlet.Client, tis []*testItem,
 	timeout := execTimeout(names)
 	var remoteErr, err error
 	if ti := tis[0]; ti.bench != nil {
-		remoteErr, err = ti.bench.run(st, bc, &buf)
+		pbr, perr := st.parentRev()
+		// TODO(quentin): Error if parent commit could not be determined?
+		if perr == nil {
+			remoteErr, err = ti.bench.Run(buildEnv, st, st.conf, bc, &buf, []buildgo.BuilderRev{st.BuilderRev, pbr})
+		}
 	} else {
 		remoteErr, err = bc.Exec(path.Join("go", "bin", "go"), buildlet.ExecOpts{
 			// We set Dir to "." instead of the default ("go/bin") so when the dist tests
@@ -2941,7 +2988,7 @@ type testItem struct {
 	name     string        // "go_test:sort"
 	duration time.Duration // optional approximate size
 
-	bench *benchmarkItem // If populated, this is a benchmark instead of a regular test.
+	bench *buildgo.BenchmarkItem // If populated, this is a benchmark instead of a regular test.
 
 	take chan token // buffered size 1: sending takes ownership of rest of fields:
 
@@ -3277,110 +3324,10 @@ func versionTgz(rev string) io.Reader {
 	return bytes.NewReader(buf.Bytes())
 }
 
-var sourceGroup singleflight.Group
-
-var sourceCache = lru.New(40) // git rev -> []byte
-
 func useGitMirror() bool {
 	return *mode != "dev"
 }
 
-// repo is go.googlesource.com repo ("go", "net", etc)
-// rev is git revision.
-func getSourceTgz(sl spanlog.Logger, repo, rev string) (tgz io.Reader, err error) {
-	sp := sl.CreateSpan("get_source")
-	defer func() { sp.Done(err) }()
-
-	key := fmt.Sprintf("%v-%v", repo, rev)
-	vi, err, _ := sourceGroup.Do(key, func() (interface{}, error) {
-		if tgzBytes, ok := sourceCache.Get(key); ok {
-			return tgzBytes, nil
-		}
-
-		if useGitMirror() {
-			sp := sl.CreateSpan("get_source_from_gitmirror")
-			tgzBytes, err := getSourceTgzFromGitMirror(repo, rev)
-			if err == nil {
-				sourceCache.Add(key, tgzBytes)
-				sp.Done(nil)
-				return tgzBytes, nil
-			}
-			log.Printf("Error fetching source %s/%s from watcher (after %v uptime): %v",
-				repo, rev, time.Since(processStartTime), err)
-			sp.Done(errors.New("timeout"))
-		}
-
-		sp := sl.CreateSpan("get_source_from_gerrit", fmt.Sprintf("%v from gerrit", key))
-		tgzBytes, err := getSourceTgzFromGerrit(repo, rev)
-		sp.Done(err)
-		if err == nil {
-			sourceCache.Add(key, tgzBytes)
-		}
-		return tgzBytes, err
-	})
-	if err != nil {
-		return nil, err
-	}
-	return bytes.NewReader(vi.([]byte)), nil
-}
-
-var gitMirrorClient = &http.Client{
-	Timeout: 30 * time.Second,
-	Transport: &http.Transport{
-		IdleConnTimeout: 30 * time.Second,
-		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
-			return goKubeClient.DialServicePort(ctx, "gitmirror", "")
-		},
-	},
-}
-
-var gerritHTTPClient = &http.Client{
-	Timeout: 30 * time.Second,
-}
-
-func getSourceTgzFromGerrit(repo, rev string) (tgz []byte, err error) {
-	return getSourceTgzFromURL(gerritHTTPClient, "gerrit", repo, rev, "https://go.googlesource.com/"+repo+"/+archive/"+rev+".tar.gz")
-}
-
-func getSourceTgzFromGitMirror(repo, rev string) (tgz []byte, err error) {
-	for i := 0; i < 2; i++ { // two tries; different pods maybe?
-		if i > 0 {
-			time.Sleep(1 * time.Second)
-		}
-		// The "gitmirror" hostname is unused:
-		tgz, err = getSourceTgzFromURL(gitMirrorClient, "gitmirror", repo, rev, "http://gitmirror/"+repo+".tar.gz?rev="+rev)
-		if err == nil {
-			return tgz, nil
-		}
-		if tr, ok := http.DefaultTransport.(*http.Transport); ok {
-			tr.CloseIdleConnections()
-		}
-	}
-	return nil, err
-}
-
-func getSourceTgzFromURL(hc *http.Client, source, repo, rev, urlStr string) (tgz []byte, err error) {
-	res, err := hc.Get(urlStr)
-	if err != nil {
-		return nil, fmt.Errorf("fetching %s/%s from %s: %v", repo, rev, source, err)
-	}
-	defer res.Body.Close()
-	if res.StatusCode/100 != 2 {
-		slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
-		return nil, fmt.Errorf("fetching %s/%s from %s: %v; body: %s", repo, rev, source, res.Status, slurp)
-	}
-	// TODO(bradfitz): finish golang.org/issue/11224
-	const maxSize = 50 << 20 // talks repo is over 25MB; go source is 7.8MB on 2015-06-15
-	slurp, err := ioutil.ReadAll(io.LimitReader(res.Body, maxSize+1))
-	if len(slurp) > maxSize && err == nil {
-		err = fmt.Errorf("body over %d bytes", maxSize)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("reading %s/%s from %s: %v", repo, rev, source, err)
-	}
-	return slurp, nil
-}
-
 var nl = []byte("\n")
 
 // repoHead contains the hashes of the latest master HEAD

cmd/coordinator/kube.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"sort"
 	"strconv"
 	"strings"
@@ -18,6 +19,7 @@ import (
 
 	"golang.org/x/build/buildlet"
 	"golang.org/x/build/dashboard"
+	"golang.org/x/build/internal/sourcecache"
 	"golang.org/x/build/kubernetes"
 	"golang.org/x/build/kubernetes/api"
 	"golang.org/x/build/kubernetes/gke"
@@ -70,6 +72,10 @@ func initKube() error {
 		return err
 	}
 
+	sourcecache.RegisterGitMirrorDial(func(ctx context.Context) (net.Conn, error) {
+		return goKubeClient.DialServicePort(ctx, "gitmirror", "")
+	})
+
 	go kubePool.pollCapacityLoop()
 	return nil
 }

internal/buildgo/benchmarks.go

@@ -2,37 +2,40 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package buildgo
 
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"path"
 	"strings"
 	"time"
 
+	"golang.org/x/build/buildenv"
 	"golang.org/x/build/buildlet"
 	"golang.org/x/build/cmd/coordinator/spanlog"
 	"golang.org/x/build/dashboard"
-	"golang.org/x/build/internal/buildgo"
+	"golang.org/x/build/internal/sourcecache"
 )
 
 // benchRuns is the number of times to run each benchmark binary
 const benchRuns = 5
 
-type benchmarkItem struct {
+// BenchmarkItem represents a single package's benchmarks, to be run on one or more commit.
+// After Run is called, the output of each commit can be retrieved from the Output field.
+type BenchmarkItem struct {
 	binary   string   // name of binary relative to goroot
 	args     []string // args to run binary with
 	preamble string   // string to print before benchmark results (e.g. "pkg: test/bench/go1\n")
-	output   []string // benchmark output for each commit
+	Output   []string // benchmark output for each commit
 
 	build func(bc *buildlet.Client, goroot string, w io.Writer) (remoteErr, err error) // how to build benchmark binary
 }
 
-func (b *benchmarkItem) name() string {
+// Name returns a string that uniquely identifies this benchmark.
+func (b *BenchmarkItem) Name() string {
 	return b.binary + " " + strings.Join(b.args, " ")
 }
 
@@ -82,7 +85,7 @@ func buildXBenchmark(sl spanlog.Logger, conf dashboard.BuildConfig, bc *buildlet
 		return nil, err
 	}
 	if err := bc.ListDir("gopath/src/golang.org/x/benchmarks", buildlet.ListDirOpts{}, func(buildlet.DirEntry) {}); err != nil {
-		if err := fetchSubrepo(sl, bc, "benchmarks", rev); err != nil {
+		if err := FetchSubrepo(sl, bc, "benchmarks", rev); err != nil {
 			return nil, err
 		}
 	}
@@ -96,43 +99,45 @@ func buildXBenchmark(sl spanlog.Logger, conf dashboard.BuildConfig, bc *buildlet
 	})
 }
 
-func enumerateBenchmarks(sl spanlog.Logger, conf dashboard.BuildConfig, bc *buildlet.Client, goroot string, trySet *trySet) ([]*benchmarkItem, error) {
+// EnumerateBenchmarks returns a slice of the benchmarks to be run for the built Go distribution found in gb.Goroot.
+// If benchmarksRev is non-empty, it is the revision of x/benchmarks to check out for additional benchmarks.
+// pkgs contains a list of possibly duplicate packages that will be searched for benchmarks.
+func (gb GoBuilder) EnumerateBenchmarks(bc *buildlet.Client, benchmarksRev string, pkgs []string) ([]*BenchmarkItem, error) {
 	workDir, err := bc.WorkDir()
 	if err != nil {
 		err = fmt.Errorf("buildBench, WorkDir: %v", err)
 		return nil, err
 	}
-	// Fetch x/benchmarks
-	rev := getRepoHead("benchmarks")
-	if rev == "" {
-		rev = "master" // should happen rarely; ok if it does.
-	}
 
-	if err := fetchSubrepo(sl, bc, "benchmarks", rev); err != nil {
+	// Fetch x/benchmarks
+	if benchmarksRev != "" {
+		if err := FetchSubrepo(gb.Logger, bc, "benchmarks", benchmarksRev); err != nil {
 			return nil, err
 		}
+	}
 
-	var out []*benchmarkItem
+	var out []*BenchmarkItem
 
 	// These regexes shard the go1 tests so each shard takes about 20s, ensuring no test runs for
 	for _, re := range []string{`^Benchmark[BF]`, `^Benchmark[HR]`, `^Benchmark[^BFHR]`} {
-		out = append(out, &benchmarkItem{
+		out = append(out, &BenchmarkItem{
 			binary:   "test/bench/go1/go1.test",
 			args:     []string{"-test.bench", re, "-test.benchmem"},
 			preamble: "pkg: test/bench/go1\n",
 			build: func(bc *buildlet.Client, goroot string, w io.Writer) (error, error) {
-				return buildGo1(conf, bc, goroot, w)
+				return buildGo1(gb.Conf, bc, goroot, w)
 			},
 		})
 	}
 
 	// Enumerate x/benchmarks
+	if benchmarksRev != "" {
 		var buf bytes.Buffer
-	remoteErr, err := bc.Exec(path.Join(goroot, "bin/go"), buildlet.ExecOpts{
+		remoteErr, err := bc.Exec(path.Join(gb.Goroot, "bin/go"), buildlet.ExecOpts{
 			Output: &buf,
 			ExtraEnv: []string{
-			"GOROOT=" + conf.FilePathJoin(workDir, goroot),
+				"GOROOT=" + gb.Conf.FilePathJoin(workDir, gb.Goroot),
-			"GOPATH=" + conf.FilePathJoin(workDir, "gopath"),
+				"GOPATH=" + gb.Conf.FilePathJoin(workDir, "gopath"),
 			},
 			Args: []string{"list", "-f", `{{if eq .Name "main"}}{{.ImportPath}}{{end}}`, "golang.org/x/benchmarks/..."},
 		})
@@ -145,31 +150,22 @@ func enumerateBenchmarks(sl spanlog.Logger, conf dashboard.BuildConfig, bc *buil
 		for _, pkg := range strings.Fields(buf.String()) {
 			pkg := pkg
 			name := "bench-" + path.Base(pkg) + ".exe"
-		out = append(out, &benchmarkItem{
+			out = append(out, &BenchmarkItem{
 				binary: name, args: nil, build: func(bc *buildlet.Client, goroot string, w io.Writer) (error, error) {
-				return buildXBenchmark(sl, conf, bc, goroot, w, rev, pkg, name)
+					return buildXBenchmark(gb.Logger, gb.Conf, bc, goroot, w, benchmarksRev, pkg, name)
 				}})
 		}
+	}
 	// Enumerate package benchmarks that were affected by the CL
-	if trySet != nil && trySet.ci != nil {
+	if len(pkgs) > 0 {
-		rev := trySet.ci.Revisions[trySet.ci.CurrentRevision]
-		var args []string
-		for p := range rev.Files {
-			if strings.HasPrefix(p, "src/") {
-				pkg := path.Dir(p[len("src/"):])
-				if pkg != "" {
-					args = append(args, pkg)
-				}
-			}
-		}
 		// Find packages that actually have benchmarks or tests.
 		var buf bytes.Buffer
-		remoteErr, err := bc.Exec(path.Join(goroot, "bin/go"), buildlet.ExecOpts{
+		remoteErr, err := bc.Exec(path.Join(gb.Goroot, "bin/go"), buildlet.ExecOpts{
 			Output: &buf,
 			ExtraEnv: []string{
-				"GOROOT=" + conf.FilePathJoin(workDir, goroot),
+				"GOROOT=" + gb.Conf.FilePathJoin(workDir, gb.Goroot),
 			},
-			Args: append([]string{"list", "-e", "-f", "{{if or (len .TestGoFiles) (len .XTestGoFiles)}}{{.ImportPath}}{{end}}"}, args...),
+			Args: append([]string{"list", "-e", "-f", "{{if or (len .TestGoFiles) (len .XTestGoFiles)}}{{.ImportPath}}{{end}}"}, pkgs...),
 		})
 		if remoteErr != nil {
 			return nil, remoteErr
@@ -186,11 +182,11 @@ func enumerateBenchmarks(sl spanlog.Logger, conf dashboard.BuildConfig, bc *buil
 				continue
 			}
 			name := "bench-" + strings.Replace(pkg, "/", "-", -1) + ".exe"
-			out = append(out, &benchmarkItem{
+			out = append(out, &BenchmarkItem{
 				binary: name,
 				args:   []string{"-test.bench", ".", "-test.benchmem", "-test.run", "^$", "-test.benchtime", "100ms"},
 				build: func(bc *buildlet.Client, goroot string, w io.Writer) (error, error) {
-					return buildPkg(conf, bc, goroot, w, pkg, name)
+					return buildPkg(gb.Conf, bc, goroot, w, pkg, name)
 				}})
 		}
 	}
@@ -225,38 +221,21 @@ func runOneBenchBinary(conf dashboard.BuildConfig, bc *buildlet.Client, w io.Wri
 	})
 }
 
-// parentRev returns the parent of this build's commit (but only if this build comes from a trySet).
+func buildRev(buildEnv *buildenv.Environment, sl spanlog.Logger, conf dashboard.BuildConfig, bc *buildlet.Client, w io.Writer, goroot string, br BuilderRev) error {
-func (st *buildStatus) parentRev() (pbr buildgo.BuilderRev, err error) {
-	pbr = st.BuilderRev // copy
-	rev := st.trySet.ci.Revisions[st.trySet.ci.CurrentRevision]
-	if rev.Commit == nil {
-		err = fmt.Errorf("commit information missing for revision %q", st.trySet.ci.CurrentRevision)
-		return
-	}
-	if len(rev.Commit.Parents) == 0 {
-		// TODO(quentin): Log?
-		err = errors.New("commit has no parent")
-		return
-	}
-	pbr.Rev = rev.Commit.Parents[0].CommitID
-	return
-}
-
-func buildRev(sl spanlog.Logger, conf dashboard.BuildConfig, bc *buildlet.Client, w io.Writer, goroot string, br buildgo.BuilderRev) error {
 	if br.SnapshotExists(context.TODO(), buildEnv) {
 		return bc.PutTarFromURL(br.SnapshotURL(buildEnv), goroot)
 	}
-	if err := bc.PutTar(versionTgz(br.Rev), goroot); err != nil {
+	if err := bc.PutTar(VersionTgz(br.Rev), goroot); err != nil {
 		return err
 	}
-	srcTar, err := getSourceTgz(sl, "go", br.Rev)
+	srcTar, err := sourcecache.GetSourceTgz(sl, "go", br.Rev)
 	if err != nil {
 		return err
 	}
 	if err := bc.PutTar(srcTar, goroot); err != nil {
 		return err
 	}
-	builder := buildgo.GoBuilder{
+	builder := GoBuilder{
 		Logger:     sl,
 		BuilderRev: br,
 		Conf:       conf,
@@ -269,18 +248,16 @@ func buildRev(sl spanlog.Logger, conf dashboard.BuildConfig, bc *buildlet.Client
 	return remoteErr
 }
 
-// run runs all the iterations of this benchmark on bc.
+// Run runs all the iterations of this benchmark on bc.
 // Build output is sent to w. Benchmark output is stored in b.output.
-// TODO(quentin): Take a list of commits so this can be used for non-try runs.
+// revs must contain exactly two revs. The first rev is assumed to be present in "go", and the second will be placed into "go-parent".
-func (b *benchmarkItem) run(st *buildStatus, bc *buildlet.Client, w io.Writer) (remoteErr, err error) {
+// TODO(quentin): Support len(revs) != 2.
+func (b *BenchmarkItem) Run(buildEnv *buildenv.Environment, sl spanlog.Logger, conf dashboard.BuildConfig, bc *buildlet.Client, w io.Writer, revs []BuilderRev) (remoteErr, err error) {
 	// Ensure we have a built parent repo.
 	if err := bc.ListDir("go-parent", buildlet.ListDirOpts{}, func(buildlet.DirEntry) {}); err != nil {
-		pbr, err := st.parentRev()
+		pbr := revs[1]
-		if err != nil {
+		sp := sl.CreateSpan("bench_build_parent", bc.Name())
-			return nil, err
+		err = buildRev(buildEnv, sl, conf, bc, w, "go-parent", pbr)
-		}
-		sp := st.CreateSpan("bench_build_parent", bc.Name())
-		err = buildRev(st, st.conf, bc, w, "go-parent", pbr)
 		sp.Done(err)
 		if err != nil {
 			return nil, err
@@ -288,7 +265,7 @@ func (b *benchmarkItem) run(st *buildStatus, bc *buildlet.Client, w io.Writer) (
 	}
 	// Build benchmark.
 	for _, goroot := range []string{"go", "go-parent"} {
-		sp := st.CreateSpan("bench_build", fmt.Sprintf("%s/%s: %s", goroot, b.binary, bc.Name()))
+		sp := sl.CreateSpan("bench_build", fmt.Sprintf("%s/%s: %s", goroot, b.binary, bc.Name()))
 		remoteErr, err = b.build(bc, goroot, w)
 		sp.Done(err)
 		if remoteErr != nil || err != nil {
@@ -314,8 +291,8 @@ func (b *benchmarkItem) run(st *buildStatus, bc *buildlet.Client, w io.Writer) (
 		for _, c := range commits {
 			fmt.Fprintf(&c.out, "iteration: %d\nstart-time: %s\n", i, time.Now().UTC().Format(time.RFC3339))
 			p := path.Join(c.path, b.binary)
-			sp := st.CreateSpan("run_one_bench", p)
+			sp := sl.CreateSpan("run_one_bench", p)
-			remoteErr, err = runOneBenchBinary(st.conf, bc, &c.out, c.path, p, b.args)
+			remoteErr, err = runOneBenchBinary(conf, bc, &c.out, c.path, p, b.args)
 			sp.Done(err)
 			if err != nil || remoteErr != nil {
 				c.out.WriteTo(w)
@@ -323,7 +300,7 @@ func (b *benchmarkItem) run(st *buildStatus, bc *buildlet.Client, w io.Writer) (
 			}
 		}
 	}
-	b.output = []string{
+	b.Output = []string{
 		commits[0].out.String(),
 		commits[1].out.String(),
 	}

internal/buildgo/buildgo.go

@@ -7,6 +7,9 @@
 package buildgo
 
 import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
@@ -19,8 +22,11 @@ import (
 	"golang.org/x/build/buildlet"
 	"golang.org/x/build/cmd/coordinator/spanlog"
 	"golang.org/x/build/dashboard"
+	"golang.org/x/build/internal/sourcecache"
 )
 
+const subrepoPrefix = "golang.org/x/"
+
 // BuilderRev is a build configuration type and a revision.
 type BuilderRev struct {
 	Name string // e.g. "linux-amd64-race"
@@ -163,3 +169,39 @@ func (gb GoBuilder) runConcurrentGoBuildStdCmd(bc *buildlet.Client, w io.Writer)
 	span.Done(nil)
 	return nil, nil
 }
+
+func FetchSubrepo(sl spanlog.Logger, bc *buildlet.Client, repo, rev string) error {
+	tgz, err := sourcecache.GetSourceTgz(sl, repo, rev)
+	if err != nil {
+		return err
+	}
+	return bc.PutTar(tgz, "gopath/src/"+subrepoPrefix+repo)
+}
+
+// VersionTgz returns an io.Reader of a *.tar.gz file containing only
+// a VERSION file containing the contents of the provided rev string.
+func VersionTgz(rev string) io.Reader {
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zw)
+
+	// Writing to a bytes.Buffer should never fail, so check
+	// errors with an explosion:
+	check := func(err error) {
+		if err != nil {
+			panic("previously assumed to never fail: " + err.Error())
+		}
+	}
+
+	contents := fmt.Sprintf("devel " + rev)
+	check(tw.WriteHeader(&tar.Header{
+		Name: "VERSION",
+		Mode: 0644,
+		Size: int64(len(contents)),
+	}))
+	_, err := io.WriteString(tw, contents)
+	check(err)
+	check(tw.Close())
+	check(zw.Close())
+	return bytes.NewReader(buf.Bytes())
+}

internal/sourcecache/source.go

@@ -0,0 +1,134 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package sourcecache provides a cache of code found in Git repositories.
+// It takes directly to the Gerrit instance at go.googlesource.com.
+// If RegisterGitMirrorDial is called, it will first try to get code from gitmirror before falling back on Gerrit.
+package sourcecache
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"time"
+
+	"golang.org/x/build/cmd/coordinator/spanlog"
+	"golang.org/x/build/internal/lru"
+	"golang.org/x/build/internal/singleflight"
+)
+
+var processStartTime = time.Now()
+
+var sourceGroup singleflight.Group
+
+var sourceCache = lru.New(40) // git rev -> []byte
+
+// GetSourceTgz returns a Reader that provides a tgz of the requested source revision.
+// repo is go.googlesource.com repo ("go", "net", etc)
+// rev is git revision.
+func GetSourceTgz(sl spanlog.Logger, repo, rev string) (tgz io.Reader, err error) {
+	sp := sl.CreateSpan("get_source")
+	defer func() { sp.Done(err) }()
+
+	key := fmt.Sprintf("%v-%v", repo, rev)
+	vi, err, _ := sourceGroup.Do(key, func() (interface{}, error) {
+		if tgzBytes, ok := sourceCache.Get(key); ok {
+			return tgzBytes, nil
+		}
+
+		if gitMirrorClient != nil {
+			sp := sl.CreateSpan("get_source_from_gitmirror")
+			tgzBytes, err := getSourceTgzFromGitMirror(repo, rev)
+			if err == nil {
+				sourceCache.Add(key, tgzBytes)
+				sp.Done(nil)
+				return tgzBytes, nil
+			}
+			log.Printf("Error fetching source %s/%s from watcher (after %v uptime): %v",
+				repo, rev, time.Since(processStartTime), err)
+			sp.Done(errors.New("timeout"))
+		}
+
+		sp := sl.CreateSpan("get_source_from_gerrit", fmt.Sprintf("%v from gerrit", key))
+		tgzBytes, err := getSourceTgzFromGerrit(repo, rev)
+		sp.Done(err)
+		if err == nil {
+			sourceCache.Add(key, tgzBytes)
+		}
+		return tgzBytes, err
+	})
+	if err != nil {
+		return nil, err
+	}
+	return bytes.NewReader(vi.([]byte)), nil
+}
+
+var gitMirrorClient *http.Client
+
+// RegisterGitMirrorDial registers a dial function which will be used to reach gitmirror.
+// If used, this function must be called before GetSourceTgz.
+func RegisterGitMirrorDial(dial func(context.Context) (net.Conn, error)) {
+	gitMirrorClient = &http.Client{
+		Timeout: 30 * time.Second,
+		Transport: &http.Transport{
+			IdleConnTimeout: 30 * time.Second,
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				return dial(ctx)
+			},
+		},
+	}
+}
+
+var gerritHTTPClient = &http.Client{
+	Timeout: 30 * time.Second,
+}
+
+func getSourceTgzFromGerrit(repo, rev string) (tgz []byte, err error) {
+	return getSourceTgzFromURL(gerritHTTPClient, "gerrit", repo, rev, "https://go.googlesource.com/"+repo+"/+archive/"+rev+".tar.gz")
+}
+
+func getSourceTgzFromGitMirror(repo, rev string) (tgz []byte, err error) {
+	for i := 0; i < 2; i++ { // two tries; different pods maybe?
+		if i > 0 {
+			time.Sleep(1 * time.Second)
+		}
+		// The "gitmirror" hostname is unused:
+		tgz, err = getSourceTgzFromURL(gitMirrorClient, "gitmirror", repo, rev, "http://gitmirror/"+repo+".tar.gz?rev="+rev)
+		if err == nil {
+			return tgz, nil
+		}
+		if tr, ok := http.DefaultTransport.(*http.Transport); ok {
+			tr.CloseIdleConnections()
+		}
+	}
+	return nil, err
+}
+
+func getSourceTgzFromURL(hc *http.Client, source, repo, rev, urlStr string) (tgz []byte, err error) {
+	res, err := hc.Get(urlStr)
+	if err != nil {
+		return nil, fmt.Errorf("fetching %s/%s from %s: %v", repo, rev, source, err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode/100 != 2 {
+		slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
+		return nil, fmt.Errorf("fetching %s/%s from %s: %v; body: %s", repo, rev, source, res.Status, slurp)
+	}
+	// TODO(bradfitz): finish golang.org/issue/11224
+	const maxSize = 50 << 20 // talks repo is over 25MB; go source is 7.8MB on 2015-06-15
+	slurp, err := ioutil.ReadAll(io.LimitReader(res.Body, maxSize+1))
+	if len(slurp) > maxSize && err == nil {
+		err = fmt.Errorf("body over %d bytes", maxSize)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("reading %s/%s from %s: %v", repo, rev, source, err)
+	}
+	return slurp, nil
+}