dashboard: rearrange tree, in prep for packagification

Package main binaries now go under cmd, leaving the top level for
similarly-named packages.

I'll be moving a lot of data structures and functions out of the
commands and into common packages.

A future CL will also unify all the "+build whatever" restrictions
into one common build tag, since there will be so much package sharing
coming up. Probably they will become "+build extdep", to indicate that
they have dependencies outside of x/tools.

Change-Id: Idc198e0dfa933b5f5de8f2b581533e8d299d2293
Reviewed-on: https://go-review.googlesource.com/2738
Reviewed-by: Andrew Gerrand <adg@golang.org>

README

@@ -4,28 +4,37 @@
 
 The files in these directories constitute the continuous builder:
 
-app/:      a.k.a the "dashboard"; the App Engine code that runs http://build.golang.org/
-buildlet/: HTTP server that runs on a VM and is told what to write to disk
+app/: a.k.a the "dashboard"; the App Engine code that runs http://build.golang.org/
+
+cmd/:
+
+  buildlet/: HTTP server that runs on a VM and is told what to write to disk
            and what command to run. This is cross-compiled to different architectures
            and is the first program run when a builder VM comes up. It then
            is contacted by the coordinator to do a build. Not all builders use
            the buildlet (at least not yet).
-builder/: gobuilder, a Go continuous build client. The original Go builder program.
-coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages
+
+  builder/: gobuilder, a Go continuous build client. The original Go builder program.
+
+  coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages
           builds using Docker containers and/or VMs as needed.
+
+  retrybuilds/: a Go client program to delete build results from the dashboard (app)
+
+  upload/:  a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere.
+
+  watcher/: a daemon that watches for new commits to the Go repository and
+          its sub-repositories, and notifies the dashboard of those commits.
+
 env/:     configuration files describing the environment of builders and related
           binaries. Many builders are still configured ad-hoc, without a hermetic
           environment.
-retrybuilds/: a Go client program to delete build results from the dashboard (app)
+
 types/:   a Go package contain common types used by other pieces.
-upload/:  a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere.
-watcher/: a daemon that watches for new commits to the Go repository and
-          its sub-repositories, and notifies the dashboard of those commits.
+
 
 If you wish to run a Go builder, please email golang-dev@googlegroups.com first.
 There is documentation at https://golang.org/wiki/DashboardBuilders but
 depending on the type of builder, we may want to run it ourselves, after you
 prepare an environment description (resulting in a VM image) of it. See the env
 directory.
-
-

cmd/builder/doc.go

@@ -55,4 +55,4 @@ If the Google Code credentials are not provided the archival step
 will be skipped.
 
 */
-package main // import "golang.org/x/tools/dashboard/builder"
+package main // import "golang.org/x/tools/dashboard/cmd/builder"

cmd/coordinator/.gitignore

@@ -0,0 +1,3 @@
+buildongce/client-*.dat
+buildongce/token.dat
+coordinator

env/commit-watcher/Makefile

@@ -6,4 +6,4 @@ docker: Dockerfile
 	docker build -t go-commit-watcher .
 
 docker-commit-watcher.tar.gz: docker
-	docker save go-commit-watcher | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-commit-watcher.tar.gz)
+	docker save go-commit-watcher | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-commit-watcher.tar.gz)

env/linux-x86-base/Makefile

@@ -6,7 +6,7 @@ docker: Dockerfile
 	docker build -t gobuilders/linux-x86-base .
 
 docker-linux.base.tar.gz: docker
-	docker save gobuilders/linux-x86-base | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.base.tar.gz)
+	docker save gobuilders/linux-x86-base | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.base.tar.gz)
 
 check: docker
 	docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-base /usr/local/bin/builder -rev=20a10e7ddd1 -buildroot=/ -v -report=false linux-amd64-temp

env/linux-x86-clang/Makefile

@@ -6,7 +6,7 @@ docker: Dockerfile
 	docker build -t gobuilders/linux-x86-clang .
 
 docker-linux.clang.tar.gz: docker
-	docker save gobuilders/linux-x86-clang | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz)
+	docker save gobuilders/linux-x86-clang | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz)
 
 check: docker
 	docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-temp

env/linux-x86-gccgo/Makefile

@@ -6,7 +6,7 @@ docker: Dockerfile
 	docker build -t gobuilders/linux-x86-gccgo .
 
 docker-linux.gccgo.tar.gz: docker
-	docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.gccgo.tar.gz)
+	docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.gccgo.tar.gz)
 
 check: docker
 	docker run gobuilders/linux-x86-gccgo /usr/local/bin/builder -tool="gccgo" -rev=b9151e911a54 -v -cmd='make RUNTESTFLAGS="--target_board=unix/-m64" check-go' -report=false linux-amd64-gccgo-temp

env/linux-x86-nacl/Makefile

@@ -6,7 +6,7 @@ docker: Dockerfile
 	docker build -t gobuilders/linux-x86-nacl .
 
 upload: docker
-	docker save gobuilders/linux-x86-nacl | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.nacl.tar.gz)
+	docker save gobuilders/linux-x86-nacl | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.nacl.tar.gz)
 
 check: docker
 	docker run gobuilders/linux-x86-nacl /usr/local/bin/builder -rev=77e96c9208d0 -buildroot=/ -v -cmd=/usr/local/bin/build-command.pl -report=false nacl-amd64p32

env/linux-x86-sid/Makefile

@@ -6,7 +6,7 @@ docker: Dockerfile
 	docker build -t gobuilders/linux-x86-sid .
 
 docker-linux.sid.tar.gz: docker
-	docker save gobuilders/linux-x86-sid | gzip | (cd ../../upload && go run upload.go --public go-builder-data/docker-linux.sid.tar.gz)
+	docker save gobuilders/linux-x86-sid | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.sid.tar.gz)
 
 check: docker
 	docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-sid /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-sid

updater/updater.go

@@ -1,128 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main // import "golang.org/x/tools/dashboard/updater"
-
-import (
-	"bytes"
-	"encoding/json"
-	"encoding/xml"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"os"
-	"os/exec"
-	"strings"
-)
-
-var (
-	builder   = flag.String("builder", "", "builder name")
-	key       = flag.String("key", "", "builder key")
-	gopath    = flag.String("gopath", "", "path to go repo")
-	dashboard = flag.String("dashboard", "build.golang.org", "Go Dashboard Host")
-	batch     = flag.Int("batch", 100, "upload batch size")
-)
-
-// Do not benchmark beyond this commit.
-// There is little sense in benchmarking till first commit,
-// and the benchmark won't build anyway.
-const Go1Commit = "0051c7442fed" // test/bench/shootout: update timing.log to Go 1.
-
-// HgLog represents a single Mercurial revision.
-type HgLog struct {
-	Hash   string
-	Branch string
-	Files  string
-}
-
-func main() {
-	flag.Parse()
-	logs := hgLog()
-	var hashes []string
-	ngo1 := 0
-	for i := range logs {
-		if strings.HasPrefix(logs[i].Hash, Go1Commit) {
-			break
-		}
-		if needsBenchmarking(&logs[i]) {
-			hashes = append(hashes, logs[i].Hash)
-		}
-		ngo1++
-	}
-	fmt.Printf("found %v commits, %v after Go1, %v need benchmarking\n", len(logs), ngo1, len(hashes))
-	for i := 0; i < len(hashes); i += *batch {
-		j := i + *batch
-		if j > len(hashes) {
-			j = len(hashes)
-		}
-		fmt.Printf("sending %v-%v... ", i, j)
-		res := postCommits(hashes[i:j])
-		fmt.Printf("%s\n", res)
-	}
-}
-
-func hgLog() []HgLog {
-	var out bytes.Buffer
-	cmd := exec.Command("hg", "log", "--encoding=utf-8", "--template", xmlLogTemplate)
-	cmd.Dir = *gopath
-	cmd.Stdout = &out
-	cmd.Stderr = os.Stderr
-	err := cmd.Run()
-	if err != nil {
-		fmt.Printf("failed to execute 'hg log': %v\n", err)
-		os.Exit(1)
-	}
-	var top struct{ Log []HgLog }
-	err = xml.Unmarshal([]byte("<Top>"+out.String()+"</Top>"), &top)
-	if err != nil {
-		fmt.Printf("failed to parse log: %v\n", err)
-		os.Exit(1)
-	}
-	return top.Log
-}
-
-func needsBenchmarking(log *HgLog) bool {
-	if log.Branch != "" {
-		return false
-	}
-	for _, f := range strings.Split(log.Files, " ") {
-		if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
-			!strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
-			return true
-		}
-	}
-	return false
-}
-
-func postCommits(hashes []string) string {
-	args := url.Values{"builder": {*builder}, "key": {*key}}
-	cmd := fmt.Sprintf("http://%v/updatebenchmark?%v", *dashboard, args.Encode())
-	b, err := json.Marshal(hashes)
-	if err != nil {
-		return fmt.Sprintf("failed to encode request: %v\n", err)
-	}
-	r, err := http.Post(cmd, "text/json", bytes.NewReader(b))
-	if err != nil {
-		return fmt.Sprintf("failed to send http request: %v\n", err)
-	}
-	defer r.Body.Close()
-	if r.StatusCode != http.StatusOK {
-		return fmt.Sprintf("http request failed: %v\n", r.Status)
-	}
-	resp, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		return fmt.Sprintf("failed to read http response: %v\n", err)
-	}
-	return string(resp)
-}
-
-const xmlLogTemplate = `
-        <Log>
-        <Hash>{node|escape}</Hash>
-        <Branch>{branches}</Branch>
-        <Files>{files}</Files>
-        </Log>
-`