dashboard/env: add go-commit-watcher image

LGTM=bradfitz
R=bradfitz, adg
CC=adg, golang-codereviews
https://golang.org/cl/179370043

README

@@ -8,7 +8,7 @@ app/:     an AppEngine server. The code that runs http://build.golang.org/
 builder/: gobuilder, a Go continuous build client
 coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages
           builds (using the builder in single-shot mode) in Docker containers.
-env/:     configuration files describing the environment of builders.
+env/:     configuration files describing the environment of builders and related binaries.
           Many builders are still configured ad-hoc.
 watcher/: a daemon that watches for new commits to the Go repository and
           its sub-repositories, and notifies the dashboard of those commits.

coordinator/main.go

@@ -37,6 +37,7 @@ var (
 var (
 	startTime = time.Now()
 	builders  = map[string]buildConfig{} // populated once at startup
+	watchers  = map[string]watchConfig{} // populated once at startup
 	donec     = make(chan builderRev)    // reports of finished builders
 
 	statusMu sync.Mutex
@@ -51,6 +52,7 @@ type imageInfo struct {
 }
 
 var images = map[string]*imageInfo{
+	"go-commit-watcher":          {url: "https://storage.googleapis.com/go-builder-data/docker-commit-watcher.tar.gz"},
 	"gobuilders/linux-x86-base":  {url: "https://storage.googleapis.com/go-builder-data/docker-linux.base.tar.gz"},
 	"gobuilders/linux-x86-clang": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.clang.tar.gz"},
 	"gobuilders/linux-x86-gccgo": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.gccgo.tar.gz"},
@@ -67,6 +69,12 @@ type buildConfig struct {
 	tool    string   // the tool this configuration is for
 }
 
+type watchConfig struct {
+	repo     string        // "https://code.google.com/p/go"
+	dash     string        // "https://build.golang.org/" (must end in /)
+	interval time.Duration // Polling interval
+}
+
 func main() {
 	flag.Parse()
 	addBuilder(buildConfig{name: "linux-386"})
@@ -96,6 +104,9 @@ func main() {
 	addBuilder(buildConfig{name: "linux-386-clang", image: "gobuilders/linux-x86-clang"})
 	addBuilder(buildConfig{name: "linux-amd64-clang", image: "gobuilders/linux-x86-clang"})
 
+	addWatcher(watchConfig{repo: "https://code.google.com/p/go", dash: "https://build.golang.org/"})
+	addWatcher(watchConfig{repo: "https://code.google.com/p/gofrontend", dash: "https://build.golang.org/gccgo/"})
+
 	if (*just != "") != (*rev != "") {
 		log.Fatalf("--just and --rev must be used together")
 	}
@@ -117,6 +128,12 @@ func main() {
 	http.HandleFunc("/logs", handleLogs)
 	go http.ListenAndServe(":80", nil)
 
+	for _, watcher := range watchers {
+		if err := startWatching(watchers[watcher.repo]); err != nil {
+			log.Printf("Error starting watcher for %s: %v", watcher.repo, err)
+		}
+	}
+
 	workc := make(chan builderRev)
 	for name, builder := range builders {
 		go findWorkLoop(name, builder.dashURL, workc)
@@ -129,10 +146,7 @@ func main() {
 			log.Printf("workc received %+v; len(status) = %v, maxBuilds = %v; cur = %p", work, len(status), *maxBuilds, status[work])
 			mayBuild := mayBuildRev(work)
 			if mayBuild {
-				out, _ := exec.Command("docker", "ps").Output()
-				numBuilds := bytes.Count(out, []byte("\n")) - 1
-				log.Printf("num current docker builds: %d", numBuilds)
-				if numBuilds > *maxBuilds {
+				if numBuilds() > *maxBuilds {
 					mayBuild = false
 				}
 			}
@@ -333,6 +347,40 @@ func addBuilder(c buildConfig) {
 	builders[c.name] = c
 }
 
+// returns the part after "docker run"
+func (conf watchConfig) dockerRunArgs() (args []string) {
+	if key := builderKey("watcher"); key != "" {
+		tmpKey := "/tmp/watcher.buildkey"
+		if _, err := os.Stat(tmpKey); err != nil {
+			if err := ioutil.WriteFile(tmpKey, []byte(key), 0600); err != nil {
+				log.Fatal(err)
+			}
+		}
+		args = append(args, "-v", tmpKey+":/.gobuildkey")
+	}
+	args = append(args,
+		"go-commit-watcher",
+		"/usr/local/bin/watcher",
+		"-repo="+conf.repo,
+		"-dash="+conf.dash,
+		"-poll="+conf.interval.String(),
+	)
+	return
+}
+
+func addWatcher(c watchConfig) {
+	if c.repo == "" {
+		c.repo = "https://code.google.com/p/go"
+	}
+	if c.dash == "" {
+		c.dash = "https://build.golang.org/"
+	}
+	if c.interval == 0 {
+		c.interval = 10 * time.Second
+	}
+	watchers[c.repo] = c
+}
+
 func condUpdateImage(img string) error {
 	ii := images[img]
 	if ii == nil {
@@ -373,6 +421,20 @@ func condUpdateImage(img string) error {
 	return nil
 }
 
+// numBuilds finds the number of go builder instances currently running.
+func numBuilds() int {
+	out, _ := exec.Command("docker", "ps").Output()
+	numBuilds := 0
+	ps := bytes.Split(out, []byte("\n"))
+	for _, p := range ps {
+		if bytes.HasPrefix(p, []byte("gobuilders/")) {
+			numBuilds++
+		}
+	}
+	log.Printf("num current docker builds: %d", numBuilds)
+	return numBuilds
+}
+
 func startBuilding(conf buildConfig, rev string) (*buildStatus, error) {
 	if err := condUpdateImage(conf.image); err != nil {
 		log.Printf("Failed to setup container for %v %v: %v", conf.name, rev, err)
@@ -411,6 +473,18 @@ type buildStatus struct {
 	// ...
 }
 
+func startWatching(conf watchConfig) error {
+	if err := condUpdateImage("go-commit-watcher"); err != nil {
+		log.Printf("Failed to setup container for commit watcher: %v", err)
+		return err
+	}
+
+	cmd := exec.Command("docker", append([]string{"run", "-d"}, conf.dockerRunArgs()...)...)
+	all, err := cmd.CombinedOutput()
+	log.Printf("Docker run for commit watcher = err:%v, output: %s", err, all)
+	return err
+}
+
 func builderKey(builder string) string {
 	master := masterKey()
 	if len(master) == 0 {

env/commit-watcher/Dockerfile

@@ -0,0 +1,17 @@
+# Copyright 2014 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# Commit watcher for Go repos.
+
+# Wheezy has Mercurial 2.2.2 and we need Mercurial >= 2.8, sid has 3.1.
+FROM debian:sid
+MAINTAINER golang-dev <golang-dev@googlegroups.com>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ADD /scripts/install-apt-deps.sh /scripts/
+RUN /scripts/install-apt-deps.sh
+
+ADD /scripts/build-commit-watcher.sh /scripts/
+RUN GO_REV=8c27884843c3 WATCHER_REV=ae08a5291439 /scripts/build-commit-watcher.sh && test -f /usr/local/bin/watcher

env/commit-watcher/Makefile

@@ -0,0 +1,9 @@
+# Copyright 2014 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+docker: Dockerfile
+	docker build -t go-commit-watcher .
+
+docker-commit-watcher.tar.gz: docker
+	docker save go-commit-watcher | gzip | (cd ../../coordinator/buildongce && go run create.go --write_object=go-builder-data/docker-commit-watcher.tar.gz)

env/commit-watcher/scripts/build-commit-watcher.sh

@@ -0,0 +1,20 @@
+set -ex
+
+export GOPATH=/gopath
+export GOROOT=/goroot
+PREFIX=/usr/local
+: ${GO_REV:?"need to be set to the golang repo revision used to build the commit watcher."}
+: ${WATCHER_REV:?"need to be set to the go.tools repo revision for the commit watcher."}
+
+mkdir -p $GOROOT
+curl -s https://storage.googleapis.com/gobuilder/go-snap.tar.gz | tar x --no-same-owner -zv -C $GOROOT
+(cd $GOROOT/src && hg pull -r $GO_REV -u && find && ./make.bash)
+
+GO_TOOLS=$GOPATH/src/golang.org/x/tools
+mkdir -p $GO_TOOLS
+curl -s https://storage.googleapis.com/gobuilder/go.tools-snap.tar.gz | tar x --no-same-owner -zv -C $GO_TOOLS
+
+mkdir -p $PREFIX/bin
+(cd $GO_TOOLS && hg pull -r $WATCHER_REV -u && GOBIN=$PREFIX/bin /goroot/bin/go install golang.org/x/tools/dashboard/watcher)
+
+rm -fR $GOROOT/bin $GOROOT/pkg $GOPATH

env/commit-watcher/scripts/install-apt-deps.sh

@@ -0,0 +1,12 @@
+set -ex
+
+apt-get update
+# For running curl to get the hg starter tarballs (faster than hg clone).
+apt-get install -y --no-install-recommends curl ca-certificates
+# For building Go's bootstrap 'dist' prog
+apt-get install -y --no-install-recommends gcc libc6-dev
+# For interacting with the Go source & subrepos:
+apt-get install -y --no-install-recommends mercurial git-core
+
+apt-get clean
+rm -fr /var/lib/apt/lists