cmd/coordinator: test sharding, better status, logs

Also gomote updates which came about during the process of developing
and debugging this.

Change-Id: Ia53d674118a6b99bcdda7062d3b7161279b6ad52
Reviewed-on: https://go-review.googlesource.com/10463
Reviewed-by: Andrew Gerrand <adg@golang.org>

buildlet/buildletclient.go

@@ -105,6 +105,8 @@ func (c *Client) URL() string {
 	return "http://" + strings.TrimSuffix(c.ipPort, ":80")
 }
 
+func (c *Client) IPPort() string { return c.ipPort }
+
 func (c *Client) do(req *http.Request) (*http.Response, error) {
 	if c.password != "" {
 		req.SetBasicAuth("gomote", c.password)

buildlet/gce.go

@@ -232,7 +232,7 @@ OpLoop:
 	}
 	condRun(opts.OnGotInstanceInfo)
 
-	const timeout = 90 * time.Second
+	const timeout = 3 * time.Minute
 	var alive bool
 	impatientClient := &http.Client{
 		Timeout: 5 * time.Second,

cmd/buildlet/Makefile

@@ -41,3 +41,7 @@ dev-buildlet.windows-amd64: buildlet.go reverse.go buildlet_windows.go
 	GOOS=windows GOARCH=amd64 go build -o $@
 	cat $@ | (cd ../upload && go run upload.go --public dev-go-builder-data/buildlet.windows-amd64)
 
+dev-buildlet.plan9-386: buildlet.go reverse.go
+	GOOS=plan9 GOARCH=386 go build -o $@
+	cat $@ | (cd ../upload && go run upload.go --public dev-go-builder-data/buildlet.plan9-386)
+

cmd/buildlet/buildlet.go

@@ -74,6 +74,11 @@ func main() {
 	if runtime.GOOS == "plan9" {
 		log.SetOutput(&plan9LogWriter{w: os.Stderr})
 	}
+	if runtime.GOOS == "linux" {
+		if w, err := os.OpenFile("/dev/console", os.O_WRONLY, 0); err == nil {
+			log.SetOutput(w)
+		}
+	}
 	flag.Parse()
 
 	onGCE := metadata.OnGCE()
@@ -93,9 +98,12 @@ func main() {
 				log.Fatalf("error creating workdir: %v", err)
 			}
 		default:
-			dir, err := ioutil.TempDir("", "buildlet-scatch")
-			if err != nil {
-				log.Fatalf("error creating workdir with ioutil.TempDir: %v", err)
+			dir := filepath.Join(os.TempDir(), "workdir")
+			if err := os.RemoveAll(dir); err != nil { // should be no-op
+				log.Fatal(err)
+			}
+			if err := os.Mkdir(dir, 0755); err != nil {
+				log.Fatal(err)
 			}
 			*workDir = dir
 		}
@@ -632,11 +640,15 @@ func handleExec(w http.ResponseWriter, r *http.Request) {
 	cmd.Stderr = cmdOutput
 	cmd.Env = env
 
+	log.Printf("[%p] Running %s with args %q and env %q in dir %s",
+		cmd, cmd.Path, cmd.Args, cmd.Env, cmd.Dir)
+
 	if debug {
 		fmt.Fprintf(cmdOutput, ":: Running %s with args %q and env %q in dir %s\n\n",
 			cmd.Path, cmd.Args, cmd.Env, cmd.Dir)
 	}
 
+	t0 := time.Now()
 	err := cmd.Start()
 	if err == nil {
 		go func() {
@@ -661,7 +673,7 @@ func handleExec(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	w.Header().Set(hdrProcessState, state)
-	log.Printf("Run = %s", state)
+	log.Printf("[%p] Run = %s, after %v", cmd, state, time.Since(t0))
 }
 
 // setPathEnv returns a copy of the provided environment with any existing
@@ -679,10 +691,11 @@ func setPathEnv(env, path []string, workDir string) []string {
 		pathIdx  = -1
 		pathOrig = ""
 	)
+
 	for i, s := range env {
-		if strings.HasPrefix(s, "PATH=") {
+		if isPathEnvPair(s) {
 			pathIdx = i
-			pathOrig = strings.TrimPrefix(s, "PATH=")
+			pathOrig = s[len("PaTh="):] // in whatever case
 			break
 		}
 	}
@@ -706,8 +719,7 @@ func setPathEnv(env, path []string, workDir string) []string {
 
 	// Put the new PATH in env.
 	env = append([]string{}, env...)
-	// TODO(adg): check that this works on plan 9
-	pathEnv := "PATH=" + strings.Join(path, string(filepath.ListSeparator))
+	pathEnv := pathEnvVar() + "=" + strings.Join(path, pathSeparator())
 	if pathIdx >= 0 {
 		env[pathIdx] = pathEnv
 	} else {
@@ -717,6 +729,39 @@ func setPathEnv(env, path []string, workDir string) []string {
 	return env
 }
 
+// isPathEnvPair reports whether the key=value pair s represents
+// the operating system's path variable.
+func isPathEnvPair(s string) bool {
+	// On Unix it's PATH.
+	// On Plan 9 it's path.
+	// On Windows it's pAtH case-insensitive.
+	if runtime.GOOS == "windows" {
+		return len(s) >= 5 && strings.EqualFold(s[:5], "PATH=")
+	}
+	if runtime.GOOS == "plan9" {
+		return strings.HasPrefix(s, "path=")
+	}
+	return strings.HasPrefix(s, "PATH=")
+}
+
+// On Unix it's PATH.
+// On Plan 9 it's path.
+// On Windows it's pAtH case-insensitive.
+func pathEnvVar() string {
+	if runtime.GOOS == "plan9" {
+		return "path"
+	}
+	return "PATH"
+}
+
+func pathSeparator() string {
+	if runtime.GOOS == "plan9" {
+		return "\x00"
+	} else {
+		return string(filepath.ListSeparator)
+	}
+}
+
 func baseEnv() []string {
 	if runtime.GOOS == "windows" {
 		return windowsBaseEnv()
@@ -835,6 +880,7 @@ func handleRemoveAll(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	for _, p := range paths {
+		log.Printf("Removing %s", p)
 		p = filepath.Join(*workDir, filepath.FromSlash(p))
 		if err := os.RemoveAll(p); err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)

cmd/coordinator/Makefile

@@ -1,5 +1,5 @@
 coordinator: builders.go coordinator.go dash.go debug.go gce.go reverse.go status.go watcher.go
-	GOOS=linux go build -o coordinator .
+	GOOS=linux go build --ldflags="-X main.Version $$USER-$$(TZ=UTC date +%FT%T)Z" -o coordinator .
 
 # After "make upload", either reboot the machine, or ssh to it and:
 #   sudo systemctl restart gobuild.service

cmd/coordinator/coordinator_test.go

@@ -0,0 +1,21 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestPartitionGoTests(t *testing.T) {
+	var in []string
+	for name := range fixedTestDuration {
+		in = append(in, name)
+	}
+	sets := partitionGoTests(in)
+	for i, set := range sets {
+		t.Logf("set %d = \"-run=^(%s)$\"", i, strings.Join(set, "|"))
+	}
+}

cmd/coordinator/gce.go

@@ -14,6 +14,7 @@ import (
 	"io"
 	"log"
 	"path"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -108,7 +109,7 @@ func initGCE() error {
 	devCluster = projectID == "go-dashboard-dev"
 	if devCluster {
 		log.Printf("Running in dev cluster")
-		gcePool.vmCap = make(chan bool, 1)
+		gcePool.vmCap = make(chan bool, 5)
 	}
 
 	return nil
@@ -151,7 +152,7 @@ type gceBuildletPool struct {
 	vmCap chan bool
 
 	mu       sync.Mutex
-	instUsed map[string]bool // GCE VM instance name -> true
+	instUsed map[string]time.Time // GCE VM instance name -> creationTime
 }
 
 func (p *gceBuildletPool) SetEnabled(enabled bool) {
@@ -162,9 +163,11 @@ func (p *gceBuildletPool) SetEnabled(enabled bool) {
 	}
 }
 
-func (p *gceBuildletPool) GetBuildlet(typ, rev string, el eventTimeLogger) (*buildlet.Client, error) {
+func (p *gceBuildletPool) GetBuildlet(cancel Cancel, typ, rev string, el eventTimeLogger) (*buildlet.Client, error) {
 	el.logEventTime("awaiting_gce_quota")
-	p.awaitVMCountQuota()
+	if err := p.awaitVMCountQuota(cancel); err != nil {
+		return nil, err
+	}
 
 	// name is the project-wide unique name of the GCE instance. It can't be longer
 	// than 61 bytes.
@@ -173,8 +176,7 @@ func (p *gceBuildletPool) GetBuildlet(typ, rev string, el eventTimeLogger) (*bui
 
 	var needDelete bool
 
-	el.logEventTime("instance_name=" + instName)
-	el.logEventTime("creating_instance")
+	el.logEventTime("creating_gce_instance", instName)
 	log.Printf("Creating GCE VM %q for %s at %s", instName, typ, rev)
 	bc, err := buildlet.StartNewVM(tokenSource, instName, typ, buildlet.VMOpts{
 		ProjectID:   projectID,
@@ -191,7 +193,7 @@ func (p *gceBuildletPool) GetBuildlet(typ, rev string, el eventTimeLogger) (*bui
 			needDelete = true // redundant with OnInstanceRequested one, but fine.
 		},
 		OnGotInstanceInfo: func() {
-			el.logEventTime("waiting_for_buildlet")
+			el.logEventTime("got_instance_info", "waiting_for_buildlet...")
 		},
 	})
 	if err != nil {
@@ -215,23 +217,44 @@ func (p *gceBuildletPool) GetBuildlet(typ, rev string, el eventTimeLogger) (*bui
 
 func (p *gceBuildletPool) WriteHTMLStatus(w io.Writer) {
 	fmt.Fprintf(w, "<b>GCE pool</b> capacity: %d/%d", len(p.vmCap), cap(p.vmCap))
+	const show = 6 // must be even
+	active := p.instancesActive()
+	if len(active) > 0 {
+		fmt.Fprintf(w, "<ul>")
+		for i, inst := range active {
+			if i < show/2 || i >= len(active)-(show/2) {
+				fmt.Fprintf(w, "<li>%v, %v</li>\n", inst.name, time.Since(inst.creation))
+			} else if i == show/2 {
+				fmt.Fprintf(w, "<li>... %d of %d total omitted ...</li>\n", len(active)-show, len(active))
+			}
+		}
+		fmt.Fprintf(w, "</ul>")
+	}
 }
 
 func (p *gceBuildletPool) String() string {
 	return fmt.Sprintf("GCE pool capacity: %d/%d", len(p.vmCap), cap(p.vmCap))
 }
 
-func (p *gceBuildletPool) awaitVMCountQuota() { p.vmCap <- true }
-func (p *gceBuildletPool) putVMCountQuota()   { <-p.vmCap }
+// awaitVMCountQuota waits for quota.
+func (p *gceBuildletPool) awaitVMCountQuota(cancel Cancel) error {
+	select {
+	case p.vmCap <- true:
+		return nil
+	case <-cancel:
+		return ErrCanceled
+	}
+}
+func (p *gceBuildletPool) putVMCountQuota() { <-p.vmCap }
 
 func (p *gceBuildletPool) setInstanceUsed(instName string, used bool) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	if p.instUsed == nil {
-		p.instUsed = make(map[string]bool)
+		p.instUsed = make(map[string]time.Time)
 	}
 	if used {
-		p.instUsed[instName] = true
+		p.instUsed[instName] = time.Now()
 	} else {
 		delete(p.instUsed, instName)
 	}
@@ -240,9 +263,35 @@ func (p *gceBuildletPool) setInstanceUsed(instName string, used bool) {
 func (p *gceBuildletPool) instanceUsed(instName string) bool {
 	p.mu.Lock()
 	defer p.mu.Unlock()
-	return p.instUsed[instName]
+	_, ok := p.instUsed[instName]
+	return ok
 }
 
+func (p *gceBuildletPool) instancesActive() (ret []instanceTime) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	for name, create := range p.instUsed {
+		ret = append(ret, instanceTime{
+			name:     name,
+			creation: create,
+		})
+	}
+	sort.Sort(byCreationTime(ret))
+	return ret
+}
+
+// instanceTime is a GCE instance name and its creation time.
+type instanceTime struct {
+	name     string
+	creation time.Time
+}
+
+type byCreationTime []instanceTime
+
+func (s byCreationTime) Len() int           { return len(s) }
+func (s byCreationTime) Less(i, j int) bool { return s[i].creation.Before(s[j].creation) }
+func (s byCreationTime) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
 // cleanUpOldVMs loops forever and periodically enumerates virtual
 // machines and deletes those which have expired.
 //

cmd/coordinator/reverse.go

@@ -159,7 +159,7 @@ func (p *reverseBuildletPool) reverseHealthCheck() {
 	p.mu.Unlock()
 }
 
-func (p *reverseBuildletPool) GetBuildlet(machineType, rev string, el eventTimeLogger) (*buildlet.Client, error) {
+func (p *reverseBuildletPool) GetBuildlet(cancel Cancel, machineType, rev string, el eventTimeLogger) (*buildlet.Client, error) {
 	seenErrInUse := false
 	for {
 		b, err := p.tryToGrab(machineType)
@@ -175,6 +175,8 @@ func (p *reverseBuildletPool) GetBuildlet(machineType, rev string, el eventTimeL
 			// a best effort signal. So periodically try to grab
 			// a buildlet again.
 			case <-time.After(30 * time.Second):
+			case <-cancel:
+				return nil, ErrCanceled
 			}
 		} else if err != nil {
 			return nil, err

cmd/coordinator/reverse_test.go

@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build BROKEN
+
+// I get:
+// $ go test -v
+// ./reverse_test.go:15: can't find import: "golang.org/x/build/cmd/buildlet"
+
 package main
 
 import (

cmd/coordinator/status.go

@@ -27,6 +27,7 @@ func handleStatus(w http.ResponseWriter, r *http.Request) {
 		Uptime:   round(time.Now().Sub(processStartTime)),
 		Recent:   append([]*buildStatus{}, statusDone...),
 		DiskFree: df,
+		Version:  Version,
 	}
 	for _, st := range status {
 		data.Active = append(data.Active, st)
@@ -84,6 +85,7 @@ type statusData struct {
 	GCEPoolStatus     template.HTML // TODO: embed template
 	ReversePoolStatus template.HTML // TODO: embed template
 	DiskFree          string
+	Version           string
 }
 
 var statusTmpl = template.Must(template.New("status").Parse(`
@@ -101,7 +103,7 @@ var statusTmpl = template.Must(template.New("status").Parse(`
 </header>
 
 <h2>Running</h2>
-<p>{{printf "%d" .Total}} total builds active. Uptime {{printf "%s" .Uptime}}.
+<p>{{printf "%d" .Total}} total builds active. Uptime {{printf "%s" .Uptime}}. Version {{.Version}}.
 
 <ul>
 {{range .Active}}

cmd/gomote/gomote.go

@@ -24,11 +24,17 @@ import (
 	"fmt"
 	"os"
 	"sort"
+
+	"golang.org/x/build/dashboard"
 )
 
 var (
 	proj = flag.String("project", "symbolic-datum-552", "GCE project owning builders")
 	zone = flag.String("zone", "us-central1-a", "GCE zone")
+
+	// Mostly dev options:
+	dev            = flag.Bool("dev", false, "if true, the default project becomes the default dev project")
+	buildletBucket = flag.String("buildletbucket", "", "Optional alternate GCS bucket for the buildlet binaries.")
 )
 
 type command struct {
@@ -91,6 +97,13 @@ func main() {
 	registerCommands()
 	flag.Usage = usage
 	flag.Parse()
+	if *dev {
+		*proj = "go-dashboard-dev"
+		dashboard.BuildletBucket = "dev-go-builder-data"
+	}
+	if v := *buildletBucket; v != "" {
+		dashboard.BuildletBucket = v
+	}
 	args := flag.Args()
 	if len(args) == 0 {
 		usage()

cmd/gomote/run.go

@@ -27,6 +27,10 @@ func run(args []string) error {
 	fs.BoolVar(&debug, "debug", false, "write debug info about the command's execution before it begins")
 	var env stringSlice
 	fs.Var(&env, "e", "Environment variable KEY=value. The -e flag may be repeated multiple times to add multiple things to the environment.")
+	var path string
+	fs.StringVar(&path, "path", "", "Comma-separated list of ExecOpts.Path elements. The special string 'EMPTY' means to run without any $PATH. The empty string (default) does not modify the $PATH.")
+	var dir string
+	fs.StringVar(&dir, "dir", "", "Directory to run from. Defaults to the directory of the command, or the work directory if -system is true.")
 
 	fs.Parse(args)
 	if fs.NArg() < 2 {
@@ -44,12 +48,21 @@ func run(args []string) error {
 		return err
 	}
 
+	var pathOpt []string
+	if path == "EMPTY" {
+		pathOpt = []string{} // non-nil
+	} else if path != "" {
+		pathOpt = strings.Split(path, ",")
+	}
+
 	remoteErr, execErr := bc.Exec(cmd, buildlet.ExecOpts{
+		Dir:         dir,
 		SystemLevel: sys || strings.HasPrefix(cmd, "/"),
 		Output:      os.Stdout,
 		Args:        fs.Args()[2:],
 		ExtraEnv:    envutil.Dedup(conf.GOOS() == "windows", append(conf.Env(), []string(env)...)),
 		Debug:       debug,
+		Path:        pathOpt,
 	})
 	if execErr != nil {
 		return fmt.Errorf("Error trying to execute %s: %v", cmd, execErr)

dashboard/builders.go

@@ -30,6 +30,10 @@ type BuildConfig struct {
 	RegularDisk bool // if true, use spinning disk instead of SSD
 	TryOnly     bool // only used for trybots, and not regular builds
 
+	// NumTestHelpers is the number of _additional_ buildlets
+	// past the first help out with sharded tests.
+	NumTestHelpers int
+
 	// BuildletType optionally specifies the type of buildlet to
 	// request from the buildlet pool. If empty, it defaults to
 	// the value of Name.
@@ -59,12 +63,37 @@ func (c *BuildConfig) GOARCH() string {
 	return arch[:i]
 }
 
+// FilePathJoin is mostly like filepath.Join (without the cleaning) except
+// it uses the path separator of c.GOOS instead of the host system's.
+func (c *BuildConfig) FilePathJoin(x ...string) string {
+	if c.GOOS() == "windows" {
+		return strings.Join(x, "\\")
+	}
+	return strings.Join(x, "/")
+}
+
+// BuildletBucket is the GCS storage bucket which holds the buildlet binaries.
+// Tools working in the dev project may change this.
+var BuildletBucket = "go-builder-data"
+
+func fixBuildletBucket(u string) string {
+	if BuildletBucket == "go-builder-data" {
+		// Prod. Default case.
+		return u
+	}
+	// Dev project remapping:
+	return strings.Replace(u,
+		"//storage.googleapis.com/go-builder-data/",
+		"//storage.googleapis.com/"+BuildletBucket+"/",
+		1)
+}
+
 // BuildletBinaryURL returns the public URL of this builder's buildlet.
 func (c *BuildConfig) BuildletBinaryURL() string {
 	if c.buildletURL != "" {
-		return c.buildletURL
+		return fixBuildletBucket(c.buildletURL)
 	}
-	return "http://storage.googleapis.com/go-builder-data/buildlet." + c.GOOS() + "-" + c.GOARCH()
+	return fixBuildletBucket("http://storage.googleapis.com/go-builder-data/buildlet." + c.GOOS() + "-" + c.GOARCH())
 }
 
 // SetBuildletBinaryURL sets the public URL of this builder's buildlet.
@@ -190,18 +219,20 @@ func (c BuildConfig) ShortOwner() string {
 
 func init() {
 	addBuilder(BuildConfig{
-		Name:        "freebsd-amd64-gce93",
-		VMImage:     "freebsd-amd64-gce93",
-		machineType: "n1-highcpu-2",
-		Go14URL:     "https://storage.googleapis.com/go-builder-data/go1.4-freebsd-amd64.tar.gz",
+		Name:           "freebsd-amd64-gce93",
+		VMImage:        "freebsd-amd64-gce93",
+		machineType:    "n1-highcpu-2",
+		Go14URL:        "https://storage.googleapis.com/go-builder-data/go1.4-freebsd-amd64.tar.gz",
+		NumTestHelpers: 3,
 	})
 	addBuilder(BuildConfig{
-		Name:        "freebsd-amd64-gce101",
-		Notes:       "FreeBSD 10.1; GCE VM is built from script in build/env/freebsd-amd64",
-		VMImage:     "freebsd-amd64-gce101",
-		machineType: "n1-highcpu-2",
-		Go14URL:     "https://storage.googleapis.com/go-builder-data/go1.4-freebsd-amd64.tar.gz",
-		env:         []string{"CC=clang"},
+		Name:           "freebsd-amd64-gce101",
+		Notes:          "FreeBSD 10.1; GCE VM is built from script in build/env/freebsd-amd64",
+		VMImage:        "freebsd-amd64-gce101",
+		machineType:    "n1-highcpu-2",
+		Go14URL:        "https://storage.googleapis.com/go-builder-data/go1.4-freebsd-amd64.tar.gz",
+		env:            []string{"CC=clang"},
+		NumTestHelpers: 3,
 	})
 	addBuilder(BuildConfig{
 		Name:        "freebsd-amd64-race",
@@ -225,10 +256,11 @@ func init() {
 		env: []string{"GOARCH=386", "CC=clang"},
 	})
 	addBuilder(BuildConfig{
-		Name:        "linux-386",
-		VMImage:     "linux-buildlet-std",
-		buildletURL: "http://storage.googleapis.com/go-builder-data/buildlet.linux-amd64",
-		env:         []string{"GOROOT_BOOTSTRAP=/go1.4", "GOARCH=386", "GOHOSTARCH=386"},
+		Name:           "linux-386",
+		VMImage:        "linux-buildlet-std",
+		buildletURL:    "http://storage.googleapis.com/go-builder-data/buildlet.linux-amd64",
+		env:            []string{"GOROOT_BOOTSTRAP=/go1.4", "GOARCH=386", "GOHOSTARCH=386"},
+		NumTestHelpers: 3,
 	})
 	addBuilder(BuildConfig{
 		Name:        "linux-386-387",
@@ -238,9 +270,10 @@ func init() {
 		env:         []string{"GOROOT_BOOTSTRAP=/go1.4", "GOARCH=386", "GOHOSTARCH=386", "GO386=387"},
 	})
 	addBuilder(BuildConfig{
-		Name:    "linux-amd64",
-		VMImage: "linux-buildlet-std",
-		env:     []string{"GOROOT_BOOTSTRAP=/go1.4"},
+		Name:           "linux-amd64",
+		VMImage:        "linux-buildlet-std",
+		env:            []string{"GOROOT_BOOTSTRAP=/go1.4"},
+		NumTestHelpers: 3,
 	})
 	addBuilder(BuildConfig{
 		Name:        "all-compile",
@@ -275,6 +308,7 @@ func init() {
 		VMImage:     "linux-buildlet-std",
 		machineType: "n1-highcpu-4",
 		env:         []string{"GOROOT_BOOTSTRAP=/go1.4"},
+		// TODO(bradfitz): make race.bash shardable, then: NumTestHelpers: 3
 	})
 	addBuilder(BuildConfig{
 		Name:        "linux-386-clang",
@@ -394,28 +428,26 @@ func init() {
 		env:         []string{"GOROOT_BOOTSTRAP=/go1.4", "GOOS=nacl", "GOARCH=amd64p32", "GOHOSTOS=linux", "GOHOSTARCH=amd64"},
 	})
 	addBuilder(BuildConfig{
-		Name:        "openbsd-amd64-gce56",
-		Notes:       "OpenBSD 5.6; GCE VM is built from script in build/env/openbsd-amd64",
-		VMImage:     "openbsd-amd64-56",
-		machineType: "n1-highcpu-2",
-		Go14URL:     "https://storage.googleapis.com/go-builder-data/go1.4-openbsd-amd64.tar.gz",
+		Name:           "openbsd-amd64-gce56",
+		Notes:          "OpenBSD 5.6; GCE VM is built from script in build/env/openbsd-amd64",
+		VMImage:        "openbsd-amd64-56",
+		machineType:    "n1-highcpu-2",
+		Go14URL:        "https://storage.googleapis.com/go-builder-data/go1.4-openbsd-amd64.tar.gz",
+		NumTestHelpers: 3,
 	})
 	addBuilder(BuildConfig{
-		Name:        "openbsd-386-gce56",
-		Notes:       "OpenBSD 5.6; GCE VM is built from script in build/env/openbsd-386",
-		VMImage:     "openbsd-386-56",
-		machineType: "n1-highcpu-2",
-		Go14URL:     "https://storage.googleapis.com/go-builder-data/go1.4-openbsd-386.tar.gz",
+		Name:           "openbsd-386-gce56",
+		Notes:          "OpenBSD 5.6; GCE VM is built from script in build/env/openbsd-386",
+		VMImage:        "openbsd-386-56",
+		machineType:    "n1-highcpu-2",
+		Go14URL:        "https://storage.googleapis.com/go-builder-data/go1.4-openbsd-386.tar.gz",
+		NumTestHelpers: 3,
 	})
 	addBuilder(BuildConfig{
-		Name:    "plan9-386-gcepartial",
-		Notes:   "Plan 9 from 0intro; GCE VM is built from script in build/env/plan9-386; runs with GOTESTONLY=std (only stdlib tests)",
+		Name:    "plan9-386",
+		Notes:   "Plan 9 from 0intro; GCE VM is built from script in build/env/plan9-386",
 		VMImage: "plan9-386-v2",
 		Go14URL: "https://storage.googleapis.com/go-builder-data/go1.4-plan9-386.tar.gz",
-		// It's named "partial" because the buildlet only runs
-		// the standard library tests ("go test std cmd", basically).
-		// TODO: run a full Plan 9 builder, or a sharded one.
-		env: []string{"GOTESTONLY=^go_test:"},
 
 		// We *were* using n1-standard-1 because Plan 9 can only
 		// reliably use a single CPU. Using 2 or 4 and we see
@@ -439,19 +471,18 @@ func init() {
 		// single-core Plan 9. It will see 2 virtual cores and
 		// only use 1, but we hope that 1 will be more powerful
 		// and we'll stop timing out on tests.
-		//
-		// But then with the toolchain conversion to Go and
-		// using ramfs, it turns out we need more memory
-		// anyway, so use n1-highcpu-4.
-		machineType: "n1-highcpu-4",
+		machineType: "n1-highcpu-2",
+
+		NumTestHelpers: 5, // slow
 	})
 	addBuilder(BuildConfig{
-		Name:        "windows-amd64-gce",
-		VMImage:     "windows-buildlet-v2",
-		machineType: "n1-highcpu-2",
-		Go14URL:     "https://storage.googleapis.com/go-builder-data/go1.4-windows-amd64.tar.gz",
-		RegularDisk: true,
-		env:         []string{"GOARCH=amd64", "GOHOSTARCH=amd64"},
+		Name:           "windows-amd64-gce",
+		VMImage:        "windows-buildlet-v2",
+		machineType:    "n1-highcpu-2",
+		Go14URL:        "https://storage.googleapis.com/go-builder-data/go1.4-windows-amd64.tar.gz",
+		RegularDisk:    true,
+		env:            []string{"GOARCH=amd64", "GOHOSTARCH=amd64"},
+		NumTestHelpers: 3,
 	})
 	addBuilder(BuildConfig{
 		Name:        "windows-amd64-race",
@@ -463,19 +494,21 @@ func init() {
 		env:         []string{"GOARCH=amd64", "GOHOSTARCH=amd64"},
 	})
 	addBuilder(BuildConfig{
-		Name:        "windows-386-gce",
-		VMImage:     "windows-buildlet-v2",
-		machineType: "n1-highcpu-2",
-		buildletURL: "http://storage.googleapis.com/go-builder-data/buildlet.windows-amd64",
-		Go14URL:     "https://storage.googleapis.com/go-builder-data/go1.4-windows-386.tar.gz",
-		RegularDisk: true,
-		env:         []string{"GOARCH=386", "GOHOSTARCH=386"},
+		Name:           "windows-386-gce",
+		VMImage:        "windows-buildlet-v2",
+		machineType:    "n1-highcpu-2",
+		buildletURL:    "http://storage.googleapis.com/go-builder-data/buildlet.windows-amd64",
+		Go14URL:        "https://storage.googleapis.com/go-builder-data/go1.4-windows-386.tar.gz",
+		RegularDisk:    true,
+		env:            []string{"GOARCH=386", "GOHOSTARCH=386"},
+		NumTestHelpers: 3,
 	})
 	addBuilder(BuildConfig{
-		Name:      "darwin-amd64-10_10",
-		Notes:     "Mac Mini running OS X 10.10 (Yosemite)",
-		Go14URL:   "https://storage.googleapis.com/go-builder-data/go1.4-darwin-amd64.tar.gz",
-		IsReverse: true,
+		Name:           "darwin-amd64-10_10",
+		Notes:          "Mac Mini running OS X 10.10 (Yosemite)",
+		Go14URL:        "https://storage.googleapis.com/go-builder-data/go1.4-darwin-amd64.tar.gz",
+		IsReverse:      true,
+		NumTestHelpers: 1, // limited resources
 	})
 	addBuilder(BuildConfig{
 		Name:      "darwin-arm-a5ios",