go.tools: add dashboard

Moving misc/dashboard to its new home. It will be deleted
(except for misc/dashboard/codereview) from its current home
if this is approved.

R=golang-dev, bradfitz, cmang, adg
CC=golang-dev
https://golang.org/cl/12180043

dashboard/README

@@ -0,0 +1,26 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+The files in this directory constitute the continuous builder:
+
+app/:     an AppEngine server
+builder/: gobuilder, a Go continuous build client
+
+If you wish to run a Go builder, please email golang-dev@googlegroups.com
+
+To run a builder:
+
+* Write the key ~gobuild/.gobuildkey 
+  You need to get it from someone who knows the key.
+  You may also use a filename of the form .gobuildkey-$BUILDER if you
+  wish to run builders for multiple targets.
+
+* Append your username and password googlecode.com credentials from
+    https://code.google.com/hosting/settings
+  to the buildkey file in the format "Username\nPassword\n".
+  (This is for uploading tarballs to the project downloads section,
+   and is an optional step.)
+
+* Build and run gobuilder (see its documentation for command-line options).
+

dashboard/app/app.yaml

@@ -0,0 +1,20 @@
+# Update with
+#	google_appengine/appcfg.py [-V test-build] update .
+#
+# Using -V test-build will run as test-build.golang.org.
+
+application: golang-org
+version: build
+runtime: go
+api_version: go1
+
+handlers:
+- url: /static
+  static_dir: static
+- url: /log/.+
+  script: _go_app
+- url: /(|commit|packages|result|tag|todo)
+  script: _go_app
+- url: /(init|buildtest|key|_ah/queue/go/delay)
+  script: _go_app
+  login: admin

dashboard/app/build/build.go

@@ -0,0 +1,332 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine
+
+package build
+
+import (
+	"bytes"
+	"compress/gzip"
+	"crypto/sha1"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"time"
+
+	"appengine"
+	"appengine/datastore"
+)
+
+const maxDatastoreStringLen = 500
+
+// A Package describes a package that is listed on the dashboard.
+type Package struct {
+	Kind    string // "subrepo", "external", or empty for the main Go tree
+	Name    string
+	Path    string // (empty for the main Go tree)
+	NextNum int    // Num of the next head Commit
+}
+
+func (p *Package) String() string {
+	return fmt.Sprintf("%s: %q", p.Path, p.Name)
+}
+
+func (p *Package) Key(c appengine.Context) *datastore.Key {
+	key := p.Path
+	if key == "" {
+		key = "go"
+	}
+	return datastore.NewKey(c, "Package", key, 0, nil)
+}
+
+// LastCommit returns the most recent Commit for this Package.
+func (p *Package) LastCommit(c appengine.Context) (*Commit, error) {
+	var commits []*Commit
+	_, err := datastore.NewQuery("Commit").
+		Ancestor(p.Key(c)).
+		Order("-Time").
+		Limit(1).
+		GetAll(c, &commits)
+	if _, ok := err.(*datastore.ErrFieldMismatch); ok {
+		// Some fields have been removed, so it's okay to ignore this error.
+		err = nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(commits) != 1 {
+		return nil, datastore.ErrNoSuchEntity
+	}
+	return commits[0], nil
+}
+
+// GetPackage fetches a Package by path from the datastore.
+func GetPackage(c appengine.Context, path string) (*Package, error) {
+	p := &Package{Path: path}
+	err := datastore.Get(c, p.Key(c), p)
+	if err == datastore.ErrNoSuchEntity {
+		return nil, fmt.Errorf("package %q not found", path)
+	}
+	if _, ok := err.(*datastore.ErrFieldMismatch); ok {
+		// Some fields have been removed, so it's okay to ignore this error.
+		err = nil
+	}
+	return p, err
+}
+
+// A Commit describes an individual commit in a package.
+//
+// Each Commit entity is a descendant of its associated Package entity.
+// In other words, all Commits with the same PackagePath belong to the same
+// datastore entity group.
+type Commit struct {
+	PackagePath string // (empty for Go commits)
+	Hash        string
+	ParentHash  string
+	Num         int // Internal monotonic counter unique to this package.
+
+	User string
+	Desc string `datastore:",noindex"`
+	Time time.Time
+
+	// ResultData is the Data string of each build Result for this Commit.
+	// For non-Go commits, only the Results for the current Go tip, weekly,
+	// and release Tags are stored here. This is purely de-normalized data.
+	// The complete data set is stored in Result entities.
+	ResultData []string `datastore:",noindex"`
+
+	FailNotificationSent bool
+}
+
+func (com *Commit) Key(c appengine.Context) *datastore.Key {
+	if com.Hash == "" {
+		panic("tried Key on Commit with empty Hash")
+	}
+	p := Package{Path: com.PackagePath}
+	key := com.PackagePath + "|" + com.Hash
+	return datastore.NewKey(c, "Commit", key, 0, p.Key(c))
+}
+
+func (c *Commit) Valid() error {
+	if !validHash(c.Hash) {
+		return errors.New("invalid Hash")
+	}
+	if c.ParentHash != "" && !validHash(c.ParentHash) { // empty is OK
+		return errors.New("invalid ParentHash")
+	}
+	return nil
+}
+
+// each result line is approx 105 bytes. This constant is a tradeoff between
+// build history and the AppEngine datastore limit of 1mb.
+const maxResults = 1000
+
+// AddResult adds the denormalized Result data to the Commit's Result field.
+// It must be called from inside a datastore transaction.
+func (com *Commit) AddResult(c appengine.Context, r *Result) error {
+	if err := datastore.Get(c, com.Key(c), com); err != nil {
+		return fmt.Errorf("getting Commit: %v", err)
+	}
+	com.ResultData = trim(append(com.ResultData, r.Data()), maxResults)
+	if _, err := datastore.Put(c, com.Key(c), com); err != nil {
+		return fmt.Errorf("putting Commit: %v", err)
+	}
+	return nil
+}
+
+func trim(s []string, n int) []string {
+	l := min(len(s), n)
+	return s[len(s)-l:]
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+// Result returns the build Result for this Commit for the given builder/goHash.
+func (c *Commit) Result(builder, goHash string) *Result {
+	for _, r := range c.ResultData {
+		p := strings.SplitN(r, "|", 4)
+		if len(p) != 4 || p[0] != builder || p[3] != goHash {
+			continue
+		}
+		return partsToHash(c, p)
+	}
+	return nil
+}
+
+// Results returns the build Results for this Commit for the given goHash.
+func (c *Commit) Results(goHash string) (results []*Result) {
+	for _, r := range c.ResultData {
+		p := strings.SplitN(r, "|", 4)
+		if len(p) != 4 || p[3] != goHash {
+			continue
+		}
+		results = append(results, partsToHash(c, p))
+	}
+	return
+}
+
+// partsToHash converts a Commit and ResultData substrings to a Result.
+func partsToHash(c *Commit, p []string) *Result {
+	return &Result{
+		Builder:     p[0],
+		Hash:        c.Hash,
+		PackagePath: c.PackagePath,
+		GoHash:      p[3],
+		OK:          p[1] == "true",
+		LogHash:     p[2],
+	}
+}
+
+// A Result describes a build result for a Commit on an OS/architecture.
+//
+// Each Result entity is a descendant of its associated Commit entity.
+type Result struct {
+	Builder     string // "os-arch[-note]"
+	Hash        string
+	PackagePath string // (empty for Go commits)
+
+	// The Go Commit this was built against (empty for Go commits).
+	GoHash string
+
+	OK      bool
+	Log     string `datastore:"-"`        // for JSON unmarshaling only
+	LogHash string `datastore:",noindex"` // Key to the Log record.
+
+	RunTime int64 // time to build+test in nanoseconds
+}
+
+func (r *Result) Key(c appengine.Context) *datastore.Key {
+	p := Package{Path: r.PackagePath}
+	key := r.Builder + "|" + r.PackagePath + "|" + r.Hash + "|" + r.GoHash
+	return datastore.NewKey(c, "Result", key, 0, p.Key(c))
+}
+
+func (r *Result) Valid() error {
+	if !validHash(r.Hash) {
+		return errors.New("invalid Hash")
+	}
+	if r.PackagePath != "" && !validHash(r.GoHash) {
+		return errors.New("invalid GoHash")
+	}
+	return nil
+}
+
+// Data returns the Result in string format
+// to be stored in Commit's ResultData field.
+func (r *Result) Data() string {
+	return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash)
+}
+
+// A Log is a gzip-compressed log file stored under the SHA1 hash of the
+// uncompressed log text.
+type Log struct {
+	CompressedLog []byte
+}
+
+func (l *Log) Text() ([]byte, error) {
+	d, err := gzip.NewReader(bytes.NewBuffer(l.CompressedLog))
+	if err != nil {
+		return nil, fmt.Errorf("reading log data: %v", err)
+	}
+	b, err := ioutil.ReadAll(d)
+	if err != nil {
+		return nil, fmt.Errorf("reading log data: %v", err)
+	}
+	return b, nil
+}
+
+func PutLog(c appengine.Context, text string) (hash string, err error) {
+	h := sha1.New()
+	io.WriteString(h, text)
+	b := new(bytes.Buffer)
+	z, _ := gzip.NewWriterLevel(b, gzip.BestCompression)
+	io.WriteString(z, text)
+	z.Close()
+	hash = fmt.Sprintf("%x", h.Sum(nil))
+	key := datastore.NewKey(c, "Log", hash, 0, nil)
+	_, err = datastore.Put(c, key, &Log{b.Bytes()})
+	return
+}
+
+// A Tag is used to keep track of the most recent Go weekly and release tags.
+// Typically there will be one Tag entity for each kind of hg tag.
+type Tag struct {
+	Kind string // "weekly", "release", or "tip"
+	Name string // the tag itself (for example: "release.r60")
+	Hash string
+}
+
+func (t *Tag) Key(c appengine.Context) *datastore.Key {
+	p := &Package{}
+	return datastore.NewKey(c, "Tag", t.Kind, 0, p.Key(c))
+}
+
+func (t *Tag) Valid() error {
+	if t.Kind != "weekly" && t.Kind != "release" && t.Kind != "tip" {
+		return errors.New("invalid Kind")
+	}
+	if !validHash(t.Hash) {
+		return errors.New("invalid Hash")
+	}
+	return nil
+}
+
+// Commit returns the Commit that corresponds with this Tag.
+func (t *Tag) Commit(c appengine.Context) (*Commit, error) {
+	com := &Commit{Hash: t.Hash}
+	err := datastore.Get(c, com.Key(c), com)
+	return com, err
+}
+
+// GetTag fetches a Tag by name from the datastore.
+func GetTag(c appengine.Context, tag string) (*Tag, error) {
+	t := &Tag{Kind: tag}
+	if err := datastore.Get(c, t.Key(c), t); err != nil {
+		if err == datastore.ErrNoSuchEntity {
+			return nil, errors.New("tag not found: " + tag)
+		}
+		return nil, err
+	}
+	if err := t.Valid(); err != nil {
+		return nil, err
+	}
+	return t, nil
+}
+
+// Packages returns packages of the specified kind.
+// Kind must be one of "external" or "subrepo".
+func Packages(c appengine.Context, kind string) ([]*Package, error) {
+	switch kind {
+	case "external", "subrepo":
+	default:
+		return nil, errors.New(`kind must be one of "external" or "subrepo"`)
+	}
+	var pkgs []*Package
+	q := datastore.NewQuery("Package").Filter("Kind=", kind)
+	for t := q.Run(c); ; {
+		pkg := new(Package)
+		_, err := t.Next(pkg)
+		if _, ok := err.(*datastore.ErrFieldMismatch); ok {
+			// Some fields have been removed, so it's okay to ignore this error.
+			err = nil
+		}
+		if err == datastore.Done {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		if pkg.Path != "" {
+			pkgs = append(pkgs, pkg)
+		}
+	}
+	return pkgs, nil
+}

dashboard/app/build/handler.go

@@ -0,0 +1,449 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine
+
+package build
+
+import (
+	"crypto/hmac"
+	"crypto/md5"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"appengine"
+	"appengine/datastore"
+	"cache"
+)
+
+const commitsPerPage = 30
+
+// commitHandler retrieves commit data or records a new commit.
+//
+// For GET requests it returns a Commit value for the specified
+// packagePath and hash.
+//
+// For POST requests it reads a JSON-encoded Commit value from the request
+// body and creates a new Commit entity. It also updates the "tip" Tag for
+// each new commit at tip.
+//
+// This handler is used by a gobuilder process in -commit mode.
+func commitHandler(r *http.Request) (interface{}, error) {
+	c := appengine.NewContext(r)
+	com := new(Commit)
+
+	if r.Method == "GET" {
+		com.PackagePath = r.FormValue("packagePath")
+		com.Hash = r.FormValue("hash")
+		if err := datastore.Get(c, com.Key(c), com); err != nil {
+			return nil, fmt.Errorf("getting Commit: %v", err)
+		}
+		return com, nil
+	}
+	if r.Method != "POST" {
+		return nil, errBadMethod(r.Method)
+	}
+
+	// POST request
+	defer r.Body.Close()
+	if err := json.NewDecoder(r.Body).Decode(com); err != nil {
+		return nil, fmt.Errorf("decoding Body: %v", err)
+	}
+	if len(com.Desc) > maxDatastoreStringLen {
+		com.Desc = com.Desc[:maxDatastoreStringLen]
+	}
+	if err := com.Valid(); err != nil {
+		return nil, fmt.Errorf("validating Commit: %v", err)
+	}
+	defer cache.Tick(c)
+	tx := func(c appengine.Context) error {
+		return addCommit(c, com)
+	}
+	return nil, datastore.RunInTransaction(c, tx, nil)
+}
+
+// addCommit adds the Commit entity to the datastore and updates the tip Tag.
+// It must be run inside a datastore transaction.
+func addCommit(c appengine.Context, com *Commit) error {
+	var tc Commit // temp value so we don't clobber com
+	err := datastore.Get(c, com.Key(c), &tc)
+	if err != datastore.ErrNoSuchEntity {
+		// if this commit is already in the datastore, do nothing
+		if err == nil {
+			return nil
+		}
+		return fmt.Errorf("getting Commit: %v", err)
+	}
+	// get the next commit number
+	p, err := GetPackage(c, com.PackagePath)
+	if err != nil {
+		return fmt.Errorf("GetPackage: %v", err)
+	}
+	com.Num = p.NextNum
+	p.NextNum++
+	if _, err := datastore.Put(c, p.Key(c), p); err != nil {
+		return fmt.Errorf("putting Package: %v", err)
+	}
+	// if this isn't the first Commit test the parent commit exists
+	if com.Num > 0 {
+		n, err := datastore.NewQuery("Commit").
+			Filter("Hash =", com.ParentHash).
+			Ancestor(p.Key(c)).
+			Count(c)
+		if err != nil {
+			return fmt.Errorf("testing for parent Commit: %v", err)
+		}
+		if n == 0 {
+			return errors.New("parent commit not found")
+		}
+	}
+	// update the tip Tag if this is the Go repo and this isn't on a release branch
+	if p.Path == "" && !strings.HasPrefix(com.Desc, "[release-branch") {
+		t := &Tag{Kind: "tip", Hash: com.Hash}
+		if _, err = datastore.Put(c, t.Key(c), t); err != nil {
+			return fmt.Errorf("putting Tag: %v", err)
+		}
+	}
+	// put the Commit
+	if _, err = datastore.Put(c, com.Key(c), com); err != nil {
+		return fmt.Errorf("putting Commit: %v", err)
+	}
+	return nil
+}
+
+// tagHandler records a new tag. It reads a JSON-encoded Tag value from the
+// request body and updates the Tag entity for the Kind of tag provided.
+//
+// This handler is used by a gobuilder process in -commit mode.
+func tagHandler(r *http.Request) (interface{}, error) {
+	if r.Method != "POST" {
+		return nil, errBadMethod(r.Method)
+	}
+
+	t := new(Tag)
+	defer r.Body.Close()
+	if err := json.NewDecoder(r.Body).Decode(t); err != nil {
+		return nil, err
+	}
+	if err := t.Valid(); err != nil {
+		return nil, err
+	}
+	c := appengine.NewContext(r)
+	defer cache.Tick(c)
+	_, err := datastore.Put(c, t.Key(c), t)
+	return nil, err
+}
+
+// Todo is a todoHandler response.
+type Todo struct {
+	Kind string // "build-go-commit" or "build-package"
+	Data interface{}
+}
+
+// todoHandler returns the next action to be performed by a builder.
+// It expects "builder" and "kind" query parameters and returns a *Todo value.
+// Multiple "kind" parameters may be specified.
+func todoHandler(r *http.Request) (interface{}, error) {
+	c := appengine.NewContext(r)
+	now := cache.Now(c)
+	key := "build-todo-" + r.Form.Encode()
+	var todo *Todo
+	if cache.Get(r, now, key, &todo) {
+		return todo, nil
+	}
+	var err error
+	builder := r.FormValue("builder")
+	for _, kind := range r.Form["kind"] {
+		var data interface{}
+		switch kind {
+		case "build-go-commit":
+			data, err = buildTodo(c, builder, "", "")
+		case "build-package":
+			packagePath := r.FormValue("packagePath")
+			goHash := r.FormValue("goHash")
+			data, err = buildTodo(c, builder, packagePath, goHash)
+		}
+		if data != nil || err != nil {
+			todo = &Todo{Kind: kind, Data: data}
+			break
+		}
+	}
+	if err == nil {
+		cache.Set(r, now, key, todo)
+	}
+	return todo, err
+}
+
+// buildTodo returns the next Commit to be built (or nil if none available).
+//
+// If packagePath and goHash are empty, it scans the first 20 Go Commits in
+// Num-descending order and returns the first one it finds that doesn't have a
+// Result for this builder.
+//
+// If provided with non-empty packagePath and goHash args, it scans the first
+// 20 Commits in Num-descending order for the specified packagePath and
+// returns the first that doesn't have a Result for this builder and goHash.
+func buildTodo(c appengine.Context, builder, packagePath, goHash string) (interface{}, error) {
+	p, err := GetPackage(c, packagePath)
+	if err != nil {
+		return nil, err
+	}
+
+	t := datastore.NewQuery("Commit").
+		Ancestor(p.Key(c)).
+		Limit(commitsPerPage).
+		Order("-Num").
+		Run(c)
+	for {
+		com := new(Commit)
+		if _, err := t.Next(com); err == datastore.Done {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		if com.Result(builder, goHash) == nil {
+			return com, nil
+		}
+	}
+
+	// Nothing left to do if this is a package (not the Go tree).
+	if packagePath != "" {
+		return nil, nil
+	}
+
+	// If there are no Go tree commits left to build,
+	// see if there are any subrepo commits that need to be built at tip.
+	// If so, ask the builder to build a go tree at the tip commit.
+	// TODO(adg): do the same for "weekly" and "release" tags.
+
+	tag, err := GetTag(c, "tip")
+	if err != nil {
+		return nil, err
+	}
+
+	// Check that this Go commit builds OK for this builder.
+	// If not, don't re-build as the subrepos will never get built anyway.
+	com, err := tag.Commit(c)
+	if err != nil {
+		return nil, err
+	}
+	if r := com.Result(builder, ""); r != nil && !r.OK {
+		return nil, nil
+	}
+
+	pkgs, err := Packages(c, "subrepo")
+	if err != nil {
+		return nil, err
+	}
+	for _, pkg := range pkgs {
+		com, err := pkg.LastCommit(c)
+		if err != nil {
+			c.Warningf("%v: no Commit found: %v", pkg, err)
+			continue
+		}
+		if com.Result(builder, tag.Hash) == nil {
+			return tag.Commit(c)
+		}
+	}
+
+	return nil, nil
+}
+
+// packagesHandler returns a list of the non-Go Packages monitored
+// by the dashboard.
+func packagesHandler(r *http.Request) (interface{}, error) {
+	kind := r.FormValue("kind")
+	c := appengine.NewContext(r)
+	now := cache.Now(c)
+	key := "build-packages-" + kind
+	var p []*Package
+	if cache.Get(r, now, key, &p) {
+		return p, nil
+	}
+	p, err := Packages(c, kind)
+	if err != nil {
+		return nil, err
+	}
+	cache.Set(r, now, key, p)
+	return p, nil
+}
+
+// resultHandler records a build result.
+// It reads a JSON-encoded Result value from the request body,
+// creates a new Result entity, and updates the relevant Commit entity.
+// If the Log field is not empty, resultHandler creates a new Log entity
+// and updates the LogHash field before putting the Commit entity.
+func resultHandler(r *http.Request) (interface{}, error) {
+	if r.Method != "POST" {
+		return nil, errBadMethod(r.Method)
+	}
+
+	c := appengine.NewContext(r)
+	res := new(Result)
+	defer r.Body.Close()
+	if err := json.NewDecoder(r.Body).Decode(res); err != nil {
+		return nil, fmt.Errorf("decoding Body: %v", err)
+	}
+	if err := res.Valid(); err != nil {
+		return nil, fmt.Errorf("validating Result: %v", err)
+	}
+	defer cache.Tick(c)
+	// store the Log text if supplied
+	if len(res.Log) > 0 {
+		hash, err := PutLog(c, res.Log)
+		if err != nil {
+			return nil, fmt.Errorf("putting Log: %v", err)
+		}
+		res.LogHash = hash
+	}
+	tx := func(c appengine.Context) error {
+		// check Package exists
+		if _, err := GetPackage(c, res.PackagePath); err != nil {
+			return fmt.Errorf("GetPackage: %v", err)
+		}
+		// put Result
+		if _, err := datastore.Put(c, res.Key(c), res); err != nil {
+			return fmt.Errorf("putting Result: %v", err)
+		}
+		// add Result to Commit
+		com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
+		if err := com.AddResult(c, res); err != nil {
+			return fmt.Errorf("AddResult: %v", err)
+		}
+		// Send build failure notifications, if necessary.
+		// Note this must run after the call AddResult, which
+		// populates the Commit's ResultData field.
+		return notifyOnFailure(c, com, res.Builder)
+	}
+	return nil, datastore.RunInTransaction(c, tx, nil)
+}
+
+// logHandler displays log text for a given hash.
+// It handles paths like "/log/hash".
+func logHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-type", "text/plain; charset=utf-8")
+	c := appengine.NewContext(r)
+	hash := r.URL.Path[len("/log/"):]
+	key := datastore.NewKey(c, "Log", hash, 0, nil)
+	l := new(Log)
+	if err := datastore.Get(c, key, l); err != nil {
+		logErr(w, r, err)
+		return
+	}
+	b, err := l.Text()
+	if err != nil {
+		logErr(w, r, err)
+		return
+	}
+	w.Write(b)
+}
+
+type dashHandler func(*http.Request) (interface{}, error)
+
+type dashResponse struct {
+	Response interface{}
+	Error    string
+}
+
+// errBadMethod is returned by a dashHandler when
+// the request has an unsuitable method.
+type errBadMethod string
+
+func (e errBadMethod) Error() string {
+	return "bad method: " + string(e)
+}
+
+// AuthHandler wraps a http.HandlerFunc with a handler that validates the
+// supplied key and builder query parameters.
+func AuthHandler(h dashHandler) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		c := appengine.NewContext(r)
+
+		// Put the URL Query values into r.Form to avoid parsing the
+		// request body when calling r.FormValue.
+		r.Form = r.URL.Query()
+
+		var err error
+		var resp interface{}
+
+		// Validate key query parameter for POST requests only.
+		key := r.FormValue("key")
+		builder := r.FormValue("builder")
+		if r.Method == "POST" && !validKey(c, key, builder) {
+			err = errors.New("invalid key: " + key)
+		}
+
+		// Call the original HandlerFunc and return the response.
+		if err == nil {
+			resp, err = h(r)
+		}
+
+		// Write JSON response.
+		dashResp := &dashResponse{Response: resp}
+		if err != nil {
+			c.Errorf("%v", err)
+			dashResp.Error = err.Error()
+		}
+		w.Header().Set("Content-Type", "application/json")
+		if err = json.NewEncoder(w).Encode(dashResp); err != nil {
+			c.Criticalf("encoding response: %v", err)
+		}
+	}
+}
+
+func keyHandler(w http.ResponseWriter, r *http.Request) {
+	builder := r.FormValue("builder")
+	if builder == "" {
+		logErr(w, r, errors.New("must supply builder in query string"))
+		return
+	}
+	c := appengine.NewContext(r)
+	fmt.Fprint(w, builderKey(c, builder))
+}
+
+func init() {
+	// admin handlers
+	http.HandleFunc("/init", initHandler)
+	http.HandleFunc("/key", keyHandler)
+
+	// authenticated handlers
+	http.HandleFunc("/commit", AuthHandler(commitHandler))
+	http.HandleFunc("/packages", AuthHandler(packagesHandler))
+	http.HandleFunc("/result", AuthHandler(resultHandler))
+	http.HandleFunc("/tag", AuthHandler(tagHandler))
+	http.HandleFunc("/todo", AuthHandler(todoHandler))
+
+	// public handlers
+	http.HandleFunc("/log/", logHandler)
+}
+
+func validHash(hash string) bool {
+	// TODO(adg): correctly validate a hash
+	return hash != ""
+}
+
+func validKey(c appengine.Context, key, builder string) bool {
+	if appengine.IsDevAppServer() {
+		return true
+	}
+	if key == secretKey(c) {
+		return true
+	}
+	return key == builderKey(c, builder)
+}
+
+func builderKey(c appengine.Context, builder string) string {
+	h := hmac.New(md5.New, []byte(secretKey(c)))
+	h.Write([]byte(builder))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func logErr(w http.ResponseWriter, r *http.Request, err error) {
+	appengine.NewContext(r).Errorf("Error: %v", err)
+	w.WriteHeader(http.StatusInternalServerError)
+	fmt.Fprint(w, "Error: ", err)
+}

dashboard/app/build/init.go

@@ -0,0 +1,68 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine
+
+package build
+
+import (
+	"fmt"
+	"net/http"
+
+	"appengine"
+	"appengine/datastore"
+	"cache"
+)
+
+// defaultPackages specifies the Package records to be created by initHandler.
+var defaultPackages = []*Package{
+	{Name: "Go", Kind: "go"},
+}
+
+// subRepos specifies the Go project sub-repositories.
+var subRepos = []string{
+	"blog",
+	"codereview",
+	"crypto",
+	"exp",
+	"image",
+	"net",
+	"talks",
+	"tools",
+}
+
+// Put subRepos into defaultPackages.
+func init() {
+	for _, name := range subRepos {
+		p := &Package{
+			Kind: "subrepo",
+			Name: "go." + name,
+			Path: "code.google.com/p/go." + name,
+		}
+		defaultPackages = append(defaultPackages, p)
+	}
+}
+
+func initHandler(w http.ResponseWriter, r *http.Request) {
+	c := appengine.NewContext(r)
+	defer cache.Tick(c)
+	for _, p := range defaultPackages {
+		err := datastore.Get(c, p.Key(c), new(Package))
+		if _, ok := err.(*datastore.ErrFieldMismatch); ok {
+			// Some fields have been removed, so it's okay to ignore this error.
+			err = nil
+		}
+		if err == nil {
+			continue
+		} else if err != datastore.ErrNoSuchEntity {
+			logErr(w, r, err)
+			return
+		}
+		if _, err := datastore.Put(c, p.Key(c), p); err != nil {
+			logErr(w, r, err)
+			return
+		}
+	}
+	fmt.Fprint(w, "OK")
+}

dashboard/app/build/key.go

@@ -0,0 +1,64 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine
+
+package build
+
+import (
+	"sync"
+
+	"appengine"
+	"appengine/datastore"
+)
+
+var theKey struct {
+	sync.RWMutex
+	BuilderKey
+}
+
+type BuilderKey struct {
+	Secret string
+}
+
+func (k *BuilderKey) Key(c appengine.Context) *datastore.Key {
+	return datastore.NewKey(c, "BuilderKey", "root", 0, nil)
+}
+
+func secretKey(c appengine.Context) string {
+	// check with rlock
+	theKey.RLock()
+	k := theKey.Secret
+	theKey.RUnlock()
+	if k != "" {
+		return k
+	}
+
+	// prepare to fill; check with lock and keep lock
+	theKey.Lock()
+	defer theKey.Unlock()
+	if theKey.Secret != "" {
+		return theKey.Secret
+	}
+
+	// fill
+	if err := datastore.Get(c, theKey.Key(c), &theKey.BuilderKey); err != nil {
+		if err == datastore.ErrNoSuchEntity {
+			// If the key is not stored in datastore, write it.
+			// This only happens at the beginning of a new deployment.
+			// The code is left here for SDK use and in case a fresh
+			// deployment is ever needed.  "gophers rule" is not the
+			// real key.
+			if !appengine.IsDevAppServer() {
+				panic("lost key from datastore")
+			}
+			theKey.Secret = "gophers rule"
+			datastore.Put(c, theKey.Key(c), &theKey.BuilderKey)
+			return theKey.Secret
+		}
+		panic("cannot load builder key: " + err.Error())
+	}
+
+	return theKey.Secret
+}

dashboard/app/build/notify.go

@@ -0,0 +1,168 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine
+
+package build
+
+import (
+	"appengine"
+	"appengine/datastore"
+	"appengine/delay"
+	"appengine/mail"
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"text/template"
+)
+
+const (
+	mailFrom   = "builder@golang.org" // use this for sending any mail
+	failMailTo = "golang-dev@googlegroups.com"
+	domain     = "build.golang.org"
+)
+
+// failIgnore is a set of builders that we don't email about because
+// they're too flaky.
+var failIgnore = map[string]bool{
+	"netbsd-386-bsiegert":   true,
+	"netbsd-amd64-bsiegert": true,
+}
+
+// notifyOnFailure checks whether the supplied Commit or the subsequent
+// Commit (if present) breaks the build for this builder.
+// If either of those commits break the build an email notification is sent
+// from a delayed task. (We use a task because this way the mail won't be
+// sent if the enclosing datastore transaction fails.)
+//
+// This must be run in a datastore transaction, and the provided *Commit must
+// have been retrieved from the datastore within that transaction.
+func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
+	if failIgnore[builder] {
+		return nil
+	}
+
+	// TODO(adg): implement notifications for packages
+	if com.PackagePath != "" {
+		return nil
+	}
+
+	p := &Package{Path: com.PackagePath}
+	var broken *Commit
+	cr := com.Result(builder, "")
+	if cr == nil {
+		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
+	}
+	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
+	if cr.OK {
+		// This commit is OK. Notify if next Commit is broken.
+		next := new(Commit)
+		q = q.Filter("ParentHash=", com.Hash)
+		if err := firstMatch(c, q, next); err != nil {
+			if err == datastore.ErrNoSuchEntity {
+				// OK at tip, no notification necessary.
+				return nil
+			}
+			return err
+		}
+		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
+			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
+			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
+			broken = next
+		}
+	} else {
+		// This commit is broken. Notify if the previous Commit is OK.
+		prev := new(Commit)
+		q = q.Filter("Hash=", com.ParentHash)
+		if err := firstMatch(c, q, prev); err != nil {
+			if err == datastore.ErrNoSuchEntity {
+				// No previous result, let the backfill of
+				// this result trigger the notification.
+				return nil
+			}
+			return err
+		}
+		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
+			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
+			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
+			broken = com
+		}
+	}
+	var err error
+	if broken != nil && !broken.FailNotificationSent {
+		c.Infof("%s is broken commit; notifying", broken.Hash)
+		sendFailMailLater.Call(c, broken, builder) // add task to queue
+		broken.FailNotificationSent = true
+		_, err = datastore.Put(c, broken.Key(c), broken)
+	}
+	return err
+}
+
+// firstMatch executes the query q and loads the first entity into v.
+func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
+	t := q.Limit(1).Run(c)
+	_, err := t.Next(v)
+	if err == datastore.Done {
+		err = datastore.ErrNoSuchEntity
+	}
+	return err
+}
+
+var (
+	sendFailMailLater = delay.Func("sendFailMail", sendFailMail)
+	sendFailMailTmpl  = template.Must(
+		template.New("notify.txt").
+			Funcs(template.FuncMap(tmplFuncs)).
+			ParseFiles("build/notify.txt"),
+	)
+)
+
+func init() {
+	gob.Register(&Commit{}) // for delay
+}
+
+// sendFailMail sends a mail notification that the build failed on the
+// provided commit and builder.
+func sendFailMail(c appengine.Context, com *Commit, builder string) {
+	// TODO(adg): handle packages
+
+	// get Result
+	r := com.Result(builder, "")
+	if r == nil {
+		c.Errorf("finding result for %q: %+v", builder, com)
+		return
+	}
+
+	// get Log
+	k := datastore.NewKey(c, "Log", r.LogHash, 0, nil)
+	l := new(Log)
+	if err := datastore.Get(c, k, l); err != nil {
+		c.Errorf("finding Log record %v: %v", r.LogHash, err)
+		return
+	}
+
+	// prepare mail message
+	var body bytes.Buffer
+	err := sendFailMailTmpl.Execute(&body, map[string]interface{}{
+		"Builder": builder, "Commit": com, "Result": r, "Log": l,
+		"Hostname": domain,
+	})
+	if err != nil {
+		c.Errorf("rendering mail template: %v", err)
+		return
+	}
+	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
+	msg := &mail.Message{
+		Sender:  mailFrom,
+		To:      []string{failMailTo},
+		ReplyTo: failMailTo,
+		Subject: subject,
+		Body:    body.String(),
+	}
+
+	// send mail
+	if err := mail.Send(c, msg); err != nil {
+		c.Errorf("sending mail: %v", err)
+	}
+}

dashboard/app/build/notify.txt

@@ -0,0 +1,9 @@
+Change {{shortHash .Commit.Hash}} broke the {{.Builder}} build:
+http://{{.Hostname}}/log/{{.Result.LogHash}}
+
+{{.Commit.Desc}}
+
+http://code.google.com/p/go/source/detail?r={{shortHash .Commit.Hash}}
+
+$ tail -200 < log
+{{printf "%s" .Log.Text | tail 200}}

dashboard/app/build/test.go

@@ -0,0 +1,258 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine
+
+package build
+
+// TODO(adg): test authentication
+
+import (
+	"appengine"
+	"appengine/datastore"
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"time"
+)
+
+func init() {
+	http.HandleFunc("/buildtest", testHandler)
+}
+
+var testEntityKinds = []string{
+	"Package",
+	"Commit",
+	"Result",
+	"Log",
+}
+
+const testPkg = "code.google.com/p/go.test"
+
+var testPackage = &Package{Name: "Test", Kind: "subrepo", Path: testPkg}
+
+var testPackages = []*Package{
+	{Name: "Go", Path: ""},
+	testPackage,
+}
+
+var tCommitTime = time.Now().Add(-time.Hour * 24 * 7)
+
+func tCommit(hash, parentHash, path string) *Commit {
+	tCommitTime.Add(time.Hour) // each commit should have a different time
+	return &Commit{
+		PackagePath: path,
+		Hash:        hash,
+		ParentHash:  parentHash,
+		Time:        tCommitTime,
+		User:        "adg",
+		Desc:        "change description " + hash,
+	}
+}
+
+var testRequests = []struct {
+	path string
+	vals url.Values
+	req  interface{}
+	res  interface{}
+}{
+	// Packages
+	{"/packages?kind=subrepo", nil, nil, []*Package{testPackage}},
+
+	// Go repo
+	{"/commit", nil, tCommit("0001", "0000", ""), nil},
+	{"/commit", nil, tCommit("0002", "0001", ""), nil},
+	{"/commit", nil, tCommit("0003", "0002", ""), nil},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
+	{"/result", nil, &Result{Builder: "linux-386", Hash: "0001", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
+	{"/result", nil, &Result{Builder: "linux-386", Hash: "0002", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
+
+	// multiple builders
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
+	{"/result", nil, &Result{Builder: "linux-amd64", Hash: "0003", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}},
+
+	// branches
+	{"/commit", nil, tCommit("0004", "0003", ""), nil},
+	{"/commit", nil, tCommit("0005", "0002", ""), nil},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
+	{"/result", nil, &Result{Builder: "linux-386", Hash: "0005", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}},
+	{"/result", nil, &Result{Builder: "linux-386", Hash: "0004", OK: false}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
+
+	// logs
+	{"/result", nil, &Result{Builder: "linux-386", Hash: "0003", OK: false, Log: "test"}, nil},
+	{"/log/a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", nil, nil, "test"},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, nil},
+
+	// repeat failure (shouldn't re-send mail)
+	{"/result", nil, &Result{Builder: "linux-386", Hash: "0003", OK: false, Log: "test"}, nil},
+
+	// non-Go repos
+	{"/commit", nil, tCommit("1001", "1000", testPkg), nil},
+	{"/commit", nil, tCommit("1002", "1001", testPkg), nil},
+	{"/commit", nil, tCommit("1003", "1002", testPkg), nil},
+	{"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1003"}}},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0001", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1002"}}},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1002", GoHash: "0001", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1001"}}},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0001", OK: true}, nil},
+	{"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, nil},
+	{"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0002"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1003"}}},
+
+	// re-build Go revision for stale subrepos
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
+	{"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0005", OK: false, Log: "boo"}, nil},
+	{"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, nil},
+}
+
+func testHandler(w http.ResponseWriter, r *http.Request) {
+	if !appengine.IsDevAppServer() {
+		fmt.Fprint(w, "These tests must be run under the dev_appserver.")
+		return
+	}
+	c := appengine.NewContext(r)
+	if err := nukeEntities(c, testEntityKinds); err != nil {
+		logErr(w, r, err)
+		return
+	}
+	if r.FormValue("nukeonly") != "" {
+		fmt.Fprint(w, "OK")
+		return
+	}
+
+	for _, p := range testPackages {
+		if _, err := datastore.Put(c, p.Key(c), p); err != nil {
+			logErr(w, r, err)
+			return
+		}
+	}
+
+	for i, t := range testRequests {
+		c.Infof("running test %d %s", i, t.path)
+		errorf := func(format string, args ...interface{}) {
+			fmt.Fprintf(w, "%d %s: ", i, t.path)
+			fmt.Fprintf(w, format, args...)
+			fmt.Fprintln(w)
+		}
+		var body io.ReadWriter
+		if t.req != nil {
+			body = new(bytes.Buffer)
+			json.NewEncoder(body).Encode(t.req)
+		}
+		url := "http://" + domain + t.path
+		if t.vals != nil {
+			url += "?" + t.vals.Encode()
+		}
+		req, err := http.NewRequest("POST", url, body)
+		if err != nil {
+			logErr(w, r, err)
+			return
+		}
+		if t.req != nil {
+			req.Method = "POST"
+		}
+		req.Header = r.Header
+		rec := httptest.NewRecorder()
+
+		// Make the request
+		http.DefaultServeMux.ServeHTTP(rec, req)
+
+		if rec.Code != 0 && rec.Code != 200 {
+			errorf(rec.Body.String())
+			return
+		}
+		resp := new(dashResponse)
+
+		// If we're expecting a *Todo value,
+		// prime the Response field with a Todo and a Commit inside it.
+		if _, ok := t.res.(*Todo); ok {
+			resp.Response = &Todo{Data: &Commit{}}
+		}
+
+		if strings.HasPrefix(t.path, "/log/") {
+			resp.Response = rec.Body.String()
+		} else {
+			err := json.NewDecoder(rec.Body).Decode(resp)
+			if err != nil {
+				errorf("decoding response: %v", err)
+				return
+			}
+		}
+		if e, ok := t.res.(string); ok {
+			g, ok := resp.Response.(string)
+			if !ok {
+				errorf("Response not string: %T", resp.Response)
+				return
+			}
+			if g != e {
+				errorf("response mismatch: got %q want %q", g, e)
+				return
+			}
+		}
+		if e, ok := t.res.(*Todo); ok {
+			g, ok := resp.Response.(*Todo)
+			if !ok {
+				errorf("Response not *Todo: %T", resp.Response)
+				return
+			}
+			if e.Data == nil && g.Data != nil {
+				errorf("Response.Data should be nil, got: %v", g.Data)
+				return
+			}
+			if g.Data == nil {
+				errorf("Response.Data is nil, want: %v", e.Data)
+				return
+			}
+			gd, ok := g.Data.(*Commit)
+			if !ok {
+				errorf("Response.Data not *Commit: %T", g.Data)
+				return
+			}
+			if eh := e.Data.(*Commit).Hash; eh != gd.Hash {
+				errorf("hashes don't match: got %q, want %q", gd.Hash, eh)
+				return
+			}
+		}
+		if t.res == nil && resp.Response != nil {
+			errorf("response mismatch: got %q expected <nil>",
+				resp.Response)
+			return
+		}
+	}
+	fmt.Fprint(w, "PASS\nYou should see only one mail notification (for 0003/linux-386) in the dev_appserver logs.")
+}
+
+func nukeEntities(c appengine.Context, kinds []string) error {
+	if !appengine.IsDevAppServer() {
+		return errors.New("can't nuke production data")
+	}
+	var keys []*datastore.Key
+	for _, kind := range kinds {
+		q := datastore.NewQuery(kind).KeysOnly()
+		for t := q.Run(c); ; {
+			k, err := t.Next(nil)
+			if err == datastore.Done {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			keys = append(keys, k)
+		}
+	}
+	return datastore.DeleteMulti(c, keys)
+}

dashboard/app/build/ui.go

@@ -0,0 +1,321 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// TODO(adg): packages at weekly/release
+// TODO(adg): some means to register new packages
+
+// +build appengine
+
+package build
+
+import (
+	"bytes"
+	"errors"
+	"html/template"
+	"net/http"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"appengine"
+	"appengine/datastore"
+	"cache"
+)
+
+func init() {
+	http.HandleFunc("/", uiHandler)
+}
+
+// uiHandler draws the build status page.
+func uiHandler(w http.ResponseWriter, r *http.Request) {
+	c := appengine.NewContext(r)
+	now := cache.Now(c)
+	const key = "build-ui"
+
+	page, _ := strconv.Atoi(r.FormValue("page"))
+	if page < 0 {
+		page = 0
+	}
+
+	// Used cached version of front page, if available.
+	if page == 0 {
+		var b []byte
+		if cache.Get(r, now, key, &b) {
+			w.Write(b)
+			return
+		}
+	}
+
+	commits, err := goCommits(c, page)
+	if err != nil {
+		logErr(w, r, err)
+		return
+	}
+	builders := commitBuilders(commits, "")
+
+	var tipState *TagState
+	if page == 0 {
+		// only show sub-repo state on first page
+		tipState, err = TagStateByName(c, "tip")
+		if err != nil {
+			logErr(w, r, err)
+			return
+		}
+	}
+
+	p := &Pagination{}
+	if len(commits) == commitsPerPage {
+		p.Next = page + 1
+	}
+	if page > 0 {
+		p.Prev = page - 1
+		p.HasPrev = true
+	}
+	data := &uiTemplateData{commits, builders, tipState, p}
+
+	var buf bytes.Buffer
+	if err := uiTemplate.Execute(&buf, data); err != nil {
+		logErr(w, r, err)
+		return
+	}
+
+	// Cache the front page.
+	if page == 0 {
+		cache.Set(r, now, key, buf.Bytes())
+	}
+
+	buf.WriteTo(w)
+}
+
+type Pagination struct {
+	Next, Prev int
+	HasPrev    bool
+}
+
+// goCommits gets a slice of the latest Commits to the Go repository.
+// If page > 0 it paginates by commitsPerPage.
+func goCommits(c appengine.Context, page int) ([]*Commit, error) {
+	q := datastore.NewQuery("Commit").
+		Ancestor((&Package{}).Key(c)).
+		Order("-Num").
+		Limit(commitsPerPage).
+		Offset(page * commitsPerPage)
+	var commits []*Commit
+	_, err := q.GetAll(c, &commits)
+	return commits, err
+}
+
+// commitBuilders returns the names of the builders that provided
+// Results for the provided commits.
+func commitBuilders(commits []*Commit, goHash string) []string {
+	builders := make(map[string]bool)
+	for _, commit := range commits {
+		for _, r := range commit.Results(goHash) {
+			builders[r.Builder] = true
+		}
+	}
+	return keys(builders)
+}
+
+func keys(m map[string]bool) (s []string) {
+	for k := range m {
+		s = append(s, k)
+	}
+	sort.Strings(s)
+	return
+}
+
+// TagState represents the state of all Packages at a Tag.
+type TagState struct {
+	Tag      *Commit
+	Packages []*PackageState
+}
+
+// PackageState represents the state of a Package at a Tag.
+type PackageState struct {
+	Package *Package
+	Commit  *Commit
+}
+
+// TagStateByName fetches the results for all Go subrepos at the specified Tag.
+func TagStateByName(c appengine.Context, name string) (*TagState, error) {
+	tag, err := GetTag(c, name)
+	if err != nil {
+		return nil, err
+	}
+	pkgs, err := Packages(c, "subrepo")
+	if err != nil {
+		return nil, err
+	}
+	var st TagState
+	for _, pkg := range pkgs {
+		com, err := pkg.LastCommit(c)
+		if err != nil {
+			c.Warningf("%v: no Commit found: %v", pkg, err)
+			continue
+		}
+		st.Packages = append(st.Packages, &PackageState{pkg, com})
+	}
+	st.Tag, err = tag.Commit(c)
+	if err != nil {
+		return nil, err
+	}
+	return &st, nil
+}
+
+type uiTemplateData struct {
+	Commits    []*Commit
+	Builders   []string
+	TipState   *TagState
+	Pagination *Pagination
+}
+
+var uiTemplate = template.Must(
+	template.New("ui.html").Funcs(tmplFuncs).ParseFiles("build/ui.html"),
+)
+
+var tmplFuncs = template.FuncMap{
+	"builderOS":        builderOS,
+	"builderArch":      builderArch,
+	"builderArchShort": builderArchShort,
+	"builderArchChar":  builderArchChar,
+	"builderTitle":     builderTitle,
+	"builderSpans":     builderSpans,
+	"repoURL":          repoURL,
+	"shortDesc":        shortDesc,
+	"shortHash":        shortHash,
+	"shortUser":        shortUser,
+	"tail":             tail,
+}
+
+func splitDash(s string) (string, string) {
+	i := strings.Index(s, "-")
+	if i >= 0 {
+		return s[:i], s[i+1:]
+	}
+	return s, ""
+}
+
+// builderOS returns the os tag for a builder string
+func builderOS(s string) string {
+	os, _ := splitDash(s)
+	return os
+}
+
+// builderArch returns the arch tag for a builder string
+func builderArch(s string) string {
+	_, arch := splitDash(s)
+	arch, _ = splitDash(arch) // chop third part
+	return arch
+}
+
+// builderArchShort returns a short arch tag for a builder string
+func builderArchShort(s string) string {
+	if strings.Contains(s+"-", "-race-") {
+		return "race"
+	}
+	arch := builderArch(s)
+	switch arch {
+	case "amd64":
+		return "x64"
+	}
+	return arch
+}
+
+// builderArchChar returns the architecture letter for a builder string
+func builderArchChar(s string) string {
+	arch := builderArch(s)
+	switch arch {
+	case "386":
+		return "8"
+	case "amd64":
+		return "6"
+	case "arm":
+		return "5"
+	}
+	return arch
+}
+
+type builderSpan struct {
+	N  int
+	OS string
+}
+
+// builderSpans creates a list of tags showing
+// the builder's operating system names, spanning
+// the appropriate number of columns.
+func builderSpans(s []string) []builderSpan {
+	var sp []builderSpan
+	for len(s) > 0 {
+		i := 1
+		os := builderOS(s[0])
+		for i < len(s) && builderOS(s[i]) == os {
+			i++
+		}
+		sp = append(sp, builderSpan{i, os})
+		s = s[i:]
+	}
+	return sp
+}
+
+// builderTitle formats "linux-amd64-foo" as "linux amd64 foo".
+func builderTitle(s string) string {
+	return strings.Replace(s, "-", " ", -1)
+}
+
+// shortDesc returns the first line of a description.
+func shortDesc(desc string) string {
+	if i := strings.Index(desc, "\n"); i != -1 {
+		desc = desc[:i]
+	}
+	return desc
+}
+
+// shortHash returns a short version of a hash.
+func shortHash(hash string) string {
+	if len(hash) > 12 {
+		hash = hash[:12]
+	}
+	return hash
+}
+
+// shortUser returns a shortened version of a user string.
+func shortUser(user string) string {
+	if i, j := strings.Index(user, "<"), strings.Index(user, ">"); 0 <= i && i < j {
+		user = user[i+1 : j]
+	}
+	if i := strings.Index(user, "@"); i >= 0 {
+		return user[:i]
+	}
+	return user
+}
+
+// repoRe matches Google Code repositories and subrepositories (without paths).
+var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+)(\.[a-z0-9\-]+)?$`)
+
+// repoURL returns the URL of a change at a Google Code repository or subrepo.
+func repoURL(hash, packagePath string) (string, error) {
+	if packagePath == "" {
+		return "https://code.google.com/p/go/source/detail?r=" + hash, nil
+	}
+	m := repoRe.FindStringSubmatch(packagePath)
+	if m == nil {
+		return "", errors.New("unrecognized package: " + packagePath)
+	}
+	url := "https://code.google.com/p/" + m[1] + "/source/detail?r=" + hash
+	if len(m) > 2 {
+		url += "&repo=" + m[2][1:]
+	}
+	return url, nil
+}
+
+// tail returns the trailing n lines of s.
+func tail(n int, s string) string {
+	lines := strings.Split(s, "\n")
+	if len(lines) < n {
+		return s
+	}
+	return strings.Join(lines[len(lines)-n:], "\n")
+}

dashboard/app/build/ui.html

@@ -0,0 +1,209 @@
+<!DOCTYPE HTML>
+<html>
+  <head>
+    <title>Go Build Dashboard</title>
+    <style>
+      body {
+        font-family: sans-serif;
+        padding: 0; margin: 0;
+      }
+      h1, h2 {
+        margin: 0;
+        padding: 5px;
+      }
+      h1 {
+        background: #eee;
+      }
+      h2 {
+        margin-top: 20px;
+      }
+      .build, .packages {
+        margin: 5px;
+        border-collapse: collapse;
+      }
+      .build td, .build th, .packages td, .packages th {
+        vertical-align: top;
+        padding: 2px 4px;
+        font-size: 10pt;
+      }
+      .build tr.commit:nth-child(2n) {
+        background-color: #f0f0f0;
+      }
+      .build .hash {
+        font-family: monospace;
+        font-size: 9pt;
+      }
+      .build .result {
+        text-align: center;
+        width: 2em;
+      }
+      .col-hash, .col-result {
+        border-right: solid 1px #ccc;
+      }
+      .build .arch {
+        font-size: 66%;
+        font-weight: normal;
+      }
+      .build .time {
+        color: #666;
+      }
+      .build .ok {
+        font-size: 83%;
+      }
+      .build .desc, .build .time, .build .user {
+        white-space: nowrap;
+      }
+      .paginate {
+        padding: 0.5em;
+      }
+      .paginate a {
+        padding: 0.5em;
+        background: #eee;
+        color: blue;
+      }
+      .paginate a.inactive {
+        color: #999;
+      }
+      .fail {
+        color: #C00;
+      }
+    </style>
+  </head>
+  <body>
+
+    <h1>Go Build Status</h1>
+
+  {{if $.Commits}}
+
+    <table class="build">
+      <colgroup class="col-hash"></colgroup>
+    {{range $.Builders | builderSpans}}
+      <colgroup class="col-result" span="{{.N}}"></colgroup>
+    {{end}}
+      <colgroup class="col-user"></colgroup>
+      <colgroup class="col-time"></colgroup>
+      <colgroup class="col-desc"></colgroup>
+      <tr>
+        <!-- extra row to make alternating colors use dark for first result -->
+      </tr>
+      <tr>
+        <th>&nbsp;</th>
+    {{range $.Builders | builderSpans}}
+        <th colspan="{{.N}}">{{.OS}}</th>
+    {{end}}
+        <th></th>
+        <th></th>
+        <th></th>
+      </tr>
+      <tr>
+        <th>&nbsp;</th>
+    {{range $.Builders}}
+        <th class="result arch" title="{{.}}">{{builderArchShort .}}</th>
+    {{end}}
+      </tr>
+    {{range $c := $.Commits}}
+      <tr class="commit">
+        <td class="hash"><a href="{{repoURL .Hash ""}}">{{shortHash .Hash}}</a></td>
+        {{range $.Builders}}
+          <td class="result">
+            {{with $c.Result . ""}}
+              {{if .OK}}
+                <span class="ok">ok</span>
+              {{else}}
+                <a href="/log/{{.LogHash}}" class="fail">fail</a>
+              {{end}}
+            {{else}}
+              &nbsp;
+            {{end}}
+          </td>
+        {{end}}
+        <td class="user" title="{{.User}}">{{shortUser .User}}</td>
+        <td class="time">{{.Time.Format "Mon 02 Jan 15:04"}}</td>
+        <td class="desc" title="{{.Desc}}">{{shortDesc .Desc}}</td>
+      </tr>
+    {{end}}
+    </table>
+
+    {{with $.Pagination}}
+    <div class="paginate">
+      <a {{if .HasPrev}}href="?page={{.Prev}}"{{else}}class="inactive"{{end}}>newer</a>
+      <a {{if .Next}}href="?page={{.Next}}"{{else}}class="inactive"{{end}}>older</a>
+      <a {{if .HasPrev}}href="."{{else}}class="inactive"{{end}}>latest</a>
+    </div>
+    {{end}}
+
+  {{else}}
+    <p>No commits to display. Hm.</p>
+  {{end}}
+
+  {{with $.TipState}}
+    {{$goHash := .Tag.Hash}}
+    <h2>
+      Sub-repositories at tip
+      <small>(<a href="{{repoURL .Tag.Hash ""}}">{{shortHash .Tag.Hash}}</a>)</small>
+    </h2>
+
+    <table class="build">
+      <colgroup class="col-package"></colgroup>
+      <colgroup class="col-hash"></colgroup>
+      {{range $.Builders | builderSpans}}
+        <colgroup class="col-result" span="{{.N}}"></colgroup>
+      {{end}}
+      <colgroup class="col-user"></colgroup>
+      <colgroup class="col-time"></colgroup>
+      <colgroup class="col-desc"></colgroup>
+      <tr>
+        <!-- extra row to make alternating colors use dark for first result -->
+      </tr>
+      <tr>
+        <th></th>
+        <th></th>
+        {{range $.Builders | builderSpans}}
+          <th colspan="{{.N}}">{{.OS}}</th>
+        {{end}}
+        <th></th>
+        <th></th>
+        <th></th>
+      </tr>
+      <tr>
+        <th></th>
+        <th></th>
+        {{range $.Builders}}
+          <th class="result arch" title="{{.}}">{{builderArchShort .}}</th>
+        {{end}}
+        <th></th>
+        <th></th>
+        <th></th>
+      </tr>
+    {{range $pkg := .Packages}}
+      <tr class="commit">
+        <td><a title="{{.Package.Path}}">{{.Package.Name}}</a></td>
+        <td class="hash">
+          {{$h := $pkg.Commit.Hash}}
+          <a href="{{repoURL $h $pkg.Commit.PackagePath}}">{{shortHash $h}}</a>
+        </td>
+        {{range $.Builders}}
+          <td class="result">
+            {{with $pkg.Commit.Result . $goHash}}
+              {{if .OK}}
+                <span class="ok">ok</span>
+              {{else}}
+                <a href="/log/{{.LogHash}}" class="fail">fail</a>
+              {{end}}
+            {{else}}
+              &nbsp;
+            {{end}}
+          </td>
+        {{end}}
+        {{with $pkg.Commit}}
+          <td class="user" title="{{.User}}">{{shortUser .User}}</td>
+          <td class="time">{{.Time.Format "Mon 02 Jan 15:04"}}</td>
+          <td class="desc" title="{{.Desc}}">{{shortDesc .Desc}}</td>
+        {{end}}
+      </tr>
+    {{end}}
+    </table>
+  {{end}}
+
+  </body>
+</html>

dashboard/app/cache/cache.go

@@ -0,0 +1,84 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appengine
+
+package cache
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"appengine"
+	"appengine/memcache"
+)
+
+const (
+	nocache = "nocache"
+	timeKey = "cachetime"
+	expiry  = 600 // 10 minutes
+)
+
+func newTime() uint64 { return uint64(time.Now().Unix()) << 32 }
+
+// Now returns the current logical datastore time to use for cache lookups.
+func Now(c appengine.Context) uint64 {
+	t, err := memcache.Increment(c, timeKey, 0, newTime())
+	if err != nil {
+		c.Errorf("cache.Now: %v", err)
+		return 0
+	}
+	return t
+}
+
+// Tick sets the current logical datastore time to a never-before-used time
+// and returns that time. It should be called to invalidate the cache.
+func Tick(c appengine.Context) uint64 {
+	t, err := memcache.Increment(c, timeKey, 1, newTime())
+	if err != nil {
+		c.Errorf("cache.Tick: %v", err)
+		return 0
+	}
+	return t
+}
+
+// Get fetches data for name at time now from memcache and unmarshals it into
+// value. It reports whether it found the cache record and logs any errors to
+// the admin console.
+func Get(r *http.Request, now uint64, name string, value interface{}) bool {
+	if now == 0 || r.FormValue(nocache) != "" {
+		return false
+	}
+	c := appengine.NewContext(r)
+	key := fmt.Sprintf("%s.%d", name, now)
+	_, err := memcache.JSON.Get(c, key, value)
+	if err == nil {
+		c.Debugf("cache hit %q", key)
+		return true
+	}
+	c.Debugf("cache miss %q", key)
+	if err != memcache.ErrCacheMiss {
+		c.Errorf("get cache %q: %v", key, err)
+	}
+	return false
+}
+
+// Set puts value into memcache under name at time now.
+// It logs any errors to the admin console.
+func Set(r *http.Request, now uint64, name string, value interface{}) {
+	if now == 0 || r.FormValue(nocache) != "" {
+		return
+	}
+	c := appengine.NewContext(r)
+	key := fmt.Sprintf("%s.%d", name, now)
+	err := memcache.JSON.Set(c, &memcache.Item{
+		Key:        key,
+		Object:     value,
+		Expiration: expiry,
+	})
+	if err != nil {
+		c.Errorf("set cache %q: %v", key, err)
+	}
+}

dashboard/builder/Makefile

@@ -0,0 +1,9 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+builder: $(shell ls *.go)
+	go build -o $@ $^
+
+clean:
+	rm -f builder

dashboard/builder/doc.go

@@ -0,0 +1,58 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+
+Go Builder is a continuous build client for the Go project.
+It integrates with the Go Dashboard AppEngine application.
+
+Go Builder is intended to run continuously as a background process.
+
+It periodically pulls updates from the Go Mercurial repository.
+
+When a newer revision is found, Go Builder creates a clone of the repository,
+runs all.bash, and reports build success or failure to the Go Dashboard.
+
+For a release revision (a change description that matches "release.YYYY-MM-DD"),
+Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
+Go Google Code project's downloads section.
+
+Usage:
+
+  gobuilder goos-goarch...
+
+  Several goos-goarch combinations can be provided, and the builder will
+  build them in serial.
+
+Optional flags:
+
+  -dashboard="godashboard.appspot.com": Go Dashboard Host
+    The location of the Go Dashboard application to which Go Builder will
+    report its results.
+
+  -release: Build and deliver binary release archive
+
+  -rev=N: Build revision N and exit
+
+  -cmd="./all.bash": Build command (specify absolute or relative to go/src)
+
+  -v: Verbose logging
+
+  -external: External package builder mode (will not report Go build
+     state to dashboard or issue releases)
+
+The key file should be located at $HOME/.gobuildkey or, for a builder-specific
+key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64).
+
+The build key file is a text file of the format:
+
+  godashboard-key
+  googlecode-username
+  googlecode-password
+
+If the Google Code credentials are not provided the archival step
+will be skipped.
+
+*/
+package main

dashboard/builder/exec.go

@@ -0,0 +1,79 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"time"
+)
+
+// run is a simple wrapper for exec.Run/Close
+func run(timeout time.Duration, envv []string, dir string, argv ...string) error {
+	if *verbose {
+		log.Println("run", argv)
+	}
+	cmd := exec.Command(argv[0], argv[1:]...)
+	cmd.Dir = dir
+	cmd.Env = envv
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	return waitWithTimeout(timeout, cmd)
+}
+
+// runLog runs a process and returns the combined stdout/stderr. It returns
+// process combined stdout and stderr output, exit status and error. The
+// error returned is nil, if process is started successfully, even if exit
+// status is not successful.
+func runLog(timeout time.Duration, envv []string, dir string, argv ...string) (string, bool, error) {
+	var b bytes.Buffer
+	ok, err := runOutput(timeout, envv, &b, dir, argv...)
+	return b.String(), ok, err
+}
+
+// runOutput runs a process and directs any output to the supplied writer.
+// It returns exit status and error. The error returned is nil, if process
+// is started successfully, even if exit status is not successful.
+func runOutput(timeout time.Duration, envv []string, out io.Writer, dir string, argv ...string) (bool, error) {
+	if *verbose {
+		log.Println("runOutput", argv)
+	}
+
+	cmd := exec.Command(argv[0], argv[1:]...)
+	cmd.Dir = dir
+	cmd.Env = envv
+	cmd.Stdout = out
+	cmd.Stderr = out
+
+	startErr := cmd.Start()
+	if startErr != nil {
+		return false, startErr
+	}
+	if err := waitWithTimeout(timeout, cmd); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func waitWithTimeout(timeout time.Duration, cmd *exec.Cmd) error {
+	errc := make(chan error, 1)
+	go func() {
+		errc <- cmd.Wait()
+	}()
+	var err error
+	select {
+	case <-time.After(timeout):
+		cmd.Process.Kill()
+		err = fmt.Errorf("timed out after %v", timeout)
+	case err = <-errc:
+	}
+	return err
+}

dashboard/builder/http.go

@@ -0,0 +1,167 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+type obj map[string]interface{}
+
+// dash runs the given method and command on the dashboard.
+// If args is non-nil it is encoded as the URL query string.
+// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST.
+// If resp is non-nil the server's response is decoded into the value pointed
+// to by resp (resp must be a pointer).
+func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
+	var r *http.Response
+	var err error
+	if *verbose {
+		log.Println("dash", meth, cmd, args, req)
+	}
+	cmd = "http://" + *dashboard + "/" + cmd
+	if len(args) > 0 {
+		cmd += "?" + args.Encode()
+	}
+	switch meth {
+	case "GET":
+		if req != nil {
+			log.Panicf("%s to %s with req", meth, cmd)
+		}
+		r, err = http.Get(cmd)
+	case "POST":
+		var body io.Reader
+		if req != nil {
+			b, err := json.Marshal(req)
+			if err != nil {
+				return err
+			}
+			body = bytes.NewBuffer(b)
+		}
+		r, err = http.Post(cmd, "text/json", body)
+	default:
+		log.Panicf("%s: invalid method %q", cmd, meth)
+		panic("invalid method: " + meth)
+	}
+	if err != nil {
+		return err
+	}
+	defer r.Body.Close()
+	if r.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad http response: %v", r.Status)
+	}
+	body := new(bytes.Buffer)
+	if _, err := body.ReadFrom(r.Body); err != nil {
+		return err
+	}
+
+	// Read JSON-encoded Response into provided resp
+	// and return an error if present.
+	var result = struct {
+		Response interface{}
+		Error    string
+	}{
+		// Put the provided resp in here as it can be a pointer to
+		// some value we should unmarshal into.
+		Response: resp,
+	}
+	if err = json.Unmarshal(body.Bytes(), &result); err != nil {
+		log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
+		return err
+	}
+	if result.Error != "" {
+		return errors.New(result.Error)
+	}
+
+	return nil
+}
+
+// todo returns the next hash to build.
+func (b *Builder) todo(kind, pkg, goHash string) (rev string, err error) {
+	args := url.Values{
+		"kind":        {kind},
+		"builder":     {b.name},
+		"packagePath": {pkg},
+		"goHash":      {goHash},
+	}
+	var resp *struct {
+		Kind string
+		Data struct {
+			Hash string
+		}
+	}
+	if err = dash("GET", "todo", args, nil, &resp); err != nil {
+		return "", err
+	}
+	if resp == nil {
+		return "", nil
+	}
+	if kind != resp.Kind {
+		return "", fmt.Errorf("expecting Kind %q, got %q", kind, resp.Kind)
+	}
+	return resp.Data.Hash, nil
+}
+
+// recordResult sends build results to the dashboard
+func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runTime time.Duration) error {
+	req := obj{
+		"Builder":     b.name,
+		"PackagePath": pkg,
+		"Hash":        hash,
+		"GoHash":      goHash,
+		"OK":          ok,
+		"Log":         buildLog,
+		"RunTime":     runTime,
+	}
+	args := url.Values{"key": {b.key}, "builder": {b.name}}
+	return dash("POST", "result", args, req, nil)
+}
+
+func postCommit(key, pkg string, l *HgLog) error {
+	t, err := time.Parse(time.RFC3339, l.Date)
+	if err != nil {
+		return fmt.Errorf("parsing %q: %v", l.Date, t)
+	}
+	return dash("POST", "commit", url.Values{"key": {key}}, obj{
+		"PackagePath": pkg,
+		"Hash":        l.Hash,
+		"ParentHash":  l.Parent,
+		"Time":        t.Format(time.RFC3339),
+		"User":        l.Author,
+		"Desc":        l.Desc,
+	}, nil)
+}
+
+func dashboardCommit(pkg, hash string) bool {
+	err := dash("GET", "commit", url.Values{
+		"packagePath": {pkg},
+		"hash":        {hash},
+	}, nil, nil)
+	return err == nil
+}
+
+func dashboardPackages(kind string) []string {
+	args := url.Values{"kind": []string{kind}}
+	var resp []struct {
+		Path string
+	}
+	if err := dash("GET", "packages", args, nil, &resp); err != nil {
+		log.Println("dashboardPackages:", err)
+		return nil
+	}
+	var pkgs []string
+	for _, r := range resp {
+		pkgs = append(pkgs, r.Path)
+	}
+	return pkgs
+}

dashboard/builder/main.go

@@ -0,0 +1,653 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"time"
+)
+
+const (
+	codeProject      = "go"
+	codePyScript     = "misc/dashboard/googlecode_upload.py"
+	hgUrl            = "https://code.google.com/p/go/"
+	mkdirPerm        = 0750
+	waitInterval     = 30 * time.Second // time to wait before checking for new revs
+	pkgBuildInterval = 24 * time.Hour   // rebuild packages every 24 hours
+)
+
+// These variables are copied from the gobuilder's environment
+// to the envv of its subprocesses.
+var extraEnv = []string{
+	"GOARM",
+
+	// For Unix derivatives.
+	"CC",
+	"PATH",
+	"TMPDIR",
+	"USER",
+
+	// For Plan 9.
+	"objtype",
+	"cputype",
+	"path",
+}
+
+type Builder struct {
+	goroot       *Repo
+	name         string
+	goos, goarch string
+	key          string
+}
+
+var (
+	buildroot      = flag.String("buildroot", defaultBuildRoot(), "Directory under which to build")
+	dashboard      = flag.String("dashboard", "build.golang.org", "Go Dashboard Host")
+	buildRelease   = flag.Bool("release", false, "Build and upload binary release archives")
+	buildRevision  = flag.String("rev", "", "Build specified revision and exit")
+	buildCmd       = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)")
+	failAll        = flag.Bool("fail", false, "fail all builds")
+	parallel       = flag.Bool("parallel", false, "Build multiple targets in parallel")
+	buildTimeout   = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests")
+	cmdTimeout     = flag.Duration("cmdTimeout", 10*time.Minute, "Maximum time to wait for an external command")
+	commitInterval = flag.Duration("commitInterval", 1*time.Minute, "Time to wait between polling for new commits (0 disables commit poller)")
+	verbose        = flag.Bool("v", false, "verbose")
+)
+
+var (
+	binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`)
+	releaseRe   = regexp.MustCompile(`^release\.r[0-9\-.]+`)
+	allCmd      = "all" + suffix
+	raceCmd     = "race" + suffix
+	cleanCmd    = "clean" + suffix
+	suffix      = defaultSuffix()
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0])
+		flag.PrintDefaults()
+		os.Exit(2)
+	}
+	flag.Parse()
+	if len(flag.Args()) == 0 {
+		flag.Usage()
+	}
+	goroot := &Repo{
+		Path: filepath.Join(*buildroot, "goroot"),
+	}
+
+	// set up work environment, use existing enviroment if possible
+	if goroot.Exists() || *failAll {
+		log.Print("Found old workspace, will use it")
+	} else {
+		if err := os.RemoveAll(*buildroot); err != nil {
+			log.Fatalf("Error removing build root (%s): %s", *buildroot, err)
+		}
+		if err := os.Mkdir(*buildroot, mkdirPerm); err != nil {
+			log.Fatalf("Error making build root (%s): %s", *buildroot, err)
+		}
+		var err error
+		goroot, err = RemoteRepo(hgUrl).Clone(goroot.Path, "tip")
+		if err != nil {
+			log.Fatal("Error cloning repository:", err)
+		}
+	}
+
+	// set up builders
+	builders := make([]*Builder, len(flag.Args()))
+	for i, name := range flag.Args() {
+		b, err := NewBuilder(goroot, name)
+		if err != nil {
+			log.Fatal(err)
+		}
+		builders[i] = b
+	}
+
+	if *failAll {
+		failMode(builders)
+		return
+	}
+
+	// if specified, build revision and return
+	if *buildRevision != "" {
+		hash, err := goroot.FullHash(*buildRevision)
+		if err != nil {
+			log.Fatal("Error finding revision: ", err)
+		}
+		for _, b := range builders {
+			if err := b.buildHash(hash); err != nil {
+				log.Println(err)
+			}
+		}
+		return
+	}
+
+	// Start commit watcher
+	go commitWatcher(goroot)
+
+	// go continuous build mode
+	// check for new commits and build them
+	for {
+		built := false
+		t := time.Now()
+		if *parallel {
+			done := make(chan bool)
+			for _, b := range builders {
+				go func(b *Builder) {
+					done <- b.build()
+				}(b)
+			}
+			for _ = range builders {
+				built = <-done || built
+			}
+		} else {
+			for _, b := range builders {
+				built = b.build() || built
+			}
+		}
+		// sleep if there was nothing to build
+		if !built {
+			time.Sleep(waitInterval)
+		}
+		// sleep if we're looping too fast.
+		dt := time.Now().Sub(t)
+		if dt < waitInterval {
+			time.Sleep(waitInterval - dt)
+		}
+	}
+}
+
+// go continuous fail mode
+// check for new commits and FAIL them
+func failMode(builders []*Builder) {
+	for {
+		built := false
+		for _, b := range builders {
+			built = b.failBuild() || built
+		}
+		// stop if there was nothing to fail
+		if !built {
+			break
+		}
+	}
+}
+
+func NewBuilder(goroot *Repo, name string) (*Builder, error) {
+	b := &Builder{
+		goroot: goroot,
+		name:   name,
+	}
+
+	// get goos/goarch from builder string
+	s := strings.SplitN(b.name, "-", 3)
+	if len(s) >= 2 {
+		b.goos, b.goarch = s[0], s[1]
+	} else {
+		return nil, fmt.Errorf("unsupported builder form: %s", name)
+	}
+
+	// read keys from keyfile
+	fn := ""
+	if runtime.GOOS == "windows" {
+		fn = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+	} else {
+		fn = os.Getenv("HOME")
+	}
+	fn = filepath.Join(fn, ".gobuildkey")
+	if s := fn + "-" + b.name; isFile(s) { // builder-specific file
+		fn = s
+	}
+	c, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return nil, fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err)
+	}
+	b.key = string(bytes.TrimSpace(bytes.SplitN(c, []byte("\n"), 2)[0]))
+	return b, nil
+}
+
+// buildCmd returns the build command to invoke.
+// Builders which contain the string '-race' in their
+// name will override *buildCmd and return raceCmd.
+func (b *Builder) buildCmd() string {
+	if strings.Contains(b.name, "-race") {
+		return raceCmd
+	}
+	return *buildCmd
+}
+
+// build checks for a new commit for this builder
+// and builds it if one is found.
+// It returns true if a build was attempted.
+func (b *Builder) build() bool {
+	hash, err := b.todo("build-go-commit", "", "")
+	if err != nil {
+		log.Println(err)
+		return false
+	}
+	if hash == "" {
+		return false
+	}
+
+	if err := b.buildHash(hash); err != nil {
+		log.Println(err)
+	}
+	return true
+}
+
+func (b *Builder) buildHash(hash string) error {
+	log.Println(b.name, "building", hash)
+
+	// create place in which to do work
+	workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
+	if err := os.Mkdir(workpath, mkdirPerm); err != nil {
+		return err
+	}
+	defer os.RemoveAll(workpath)
+
+	// pull before cloning to ensure we have the revision
+	if err := b.goroot.Pull(); err != nil {
+		return err
+	}
+
+	// clone repo at specified revision
+	if _, err := b.goroot.Clone(filepath.Join(workpath, "go"), hash); err != nil {
+		return err
+	}
+
+	srcDir := filepath.Join(workpath, "go", "src")
+
+	// build
+	var buildlog bytes.Buffer
+	logfile := filepath.Join(workpath, "build.log")
+	f, err := os.Create(logfile)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	w := io.MultiWriter(f, &buildlog)
+
+	cmd := b.buildCmd()
+	if !filepath.IsAbs(cmd) {
+		cmd = filepath.Join(srcDir, cmd)
+	}
+	startTime := time.Now()
+	ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, cmd)
+	runTime := time.Now().Sub(startTime)
+	errf := func() string {
+		if err != nil {
+			return fmt.Sprintf("error: %v", err)
+		}
+		if !ok {
+			return "failed"
+		}
+		return "success"
+	}
+	fmt.Fprintf(w, "Build complete, duration %v. Result: %v\n", runTime, errf())
+
+	if err != nil || !ok {
+		// record failure
+		return b.recordResult(false, "", hash, "", buildlog.String(), runTime)
+	}
+
+	// record success
+	if err = b.recordResult(true, "", hash, "", "", runTime); err != nil {
+		return fmt.Errorf("recordResult: %s", err)
+	}
+
+	// build Go sub-repositories
+	goRoot := filepath.Join(workpath, "go")
+	goPath := workpath
+	b.buildSubrepos(goRoot, goPath, hash)
+
+	return nil
+}
+
+// failBuild checks for a new commit for this builder
+// and fails it if one is found.
+// It returns true if a build was "attempted".
+func (b *Builder) failBuild() bool {
+	hash, err := b.todo("build-go-commit", "", "")
+	if err != nil {
+		log.Println(err)
+		return false
+	}
+	if hash == "" {
+		return false
+	}
+
+	log.Printf("fail %s %s\n", b.name, hash)
+
+	if err := b.recordResult(false, "", hash, "", "auto-fail mode run by "+os.Getenv("USER"), 0); err != nil {
+		log.Print(err)
+	}
+	return true
+}
+
+func (b *Builder) buildSubrepos(goRoot, goPath, goHash string) {
+	for _, pkg := range dashboardPackages("subrepo") {
+		// get the latest todo for this package
+		hash, err := b.todo("build-package", pkg, goHash)
+		if err != nil {
+			log.Printf("buildSubrepos %s: %v", pkg, err)
+			continue
+		}
+		if hash == "" {
+			continue
+		}
+
+		// build the package
+		if *verbose {
+			log.Printf("buildSubrepos %s: building %q", pkg, hash)
+		}
+		buildLog, err := b.buildSubrepo(goRoot, goPath, pkg, hash)
+		if err != nil {
+			if buildLog == "" {
+				buildLog = err.Error()
+			}
+			log.Printf("buildSubrepos %s: %v", pkg, err)
+		}
+
+		// record the result
+		err = b.recordResult(err == nil, pkg, hash, goHash, buildLog, 0)
+		if err != nil {
+			log.Printf("buildSubrepos %s: %v", pkg, err)
+		}
+	}
+}
+
+// buildSubrepo fetches the given package, updates it to the specified hash,
+// and runs 'go test -short pkg/...'. It returns the build log and any error.
+func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error) {
+	goTool := filepath.Join(goRoot, "bin", "go")
+	env := append(b.envv(), "GOROOT="+goRoot, "GOPATH="+goPath)
+
+	// add $GOROOT/bin and $GOPATH/bin to PATH
+	for i, e := range env {
+		const p = "PATH="
+		if !strings.HasPrefix(e, p) {
+			continue
+		}
+		sep := string(os.PathListSeparator)
+		env[i] = p + filepath.Join(goRoot, "bin") + sep + filepath.Join(goPath, "bin") + sep + e[len(p):]
+	}
+
+	// fetch package and dependencies
+	log, ok, err := runLog(*cmdTimeout, env, goPath, goTool, "get", "-d", pkg+"/...")
+	if err == nil && !ok {
+		err = fmt.Errorf("go exited with status 1")
+	}
+	if err != nil {
+		return log, err
+	}
+
+	// hg update to the specified hash
+	repo := Repo{Path: filepath.Join(goPath, "src", pkg)}
+	if err := repo.UpdateTo(hash); err != nil {
+		return "", err
+	}
+
+	// test the package
+	log, ok, err = runLog(*buildTimeout, env, goPath, goTool, "test", "-short", pkg+"/...")
+	if err == nil && !ok {
+		err = fmt.Errorf("go exited with status 1")
+	}
+	return log, err
+}
+
+// envv returns an environment for build/bench execution
+func (b *Builder) envv() []string {
+	if runtime.GOOS == "windows" {
+		return b.envvWindows()
+	}
+	e := []string{
+		"GOOS=" + b.goos,
+		"GOHOSTOS=" + b.goos,
+		"GOARCH=" + b.goarch,
+		"GOHOSTARCH=" + b.goarch,
+		"GOROOT_FINAL=/usr/local/go",
+	}
+	for _, k := range extraEnv {
+		if s, ok := getenvOk(k); ok {
+			e = append(e, k+"="+s)
+		}
+	}
+	return e
+}
+
+// windows version of envv
+func (b *Builder) envvWindows() []string {
+	start := map[string]string{
+		"GOOS":         b.goos,
+		"GOHOSTOS":     b.goos,
+		"GOARCH":       b.goarch,
+		"GOHOSTARCH":   b.goarch,
+		"GOROOT_FINAL": `c:\go`,
+		"GOBUILDEXIT":  "1", // exit all.bat with completion status.
+	}
+	for _, name := range extraEnv {
+		if s, ok := getenvOk(name); ok {
+			start[name] = s
+		}
+	}
+	skip := map[string]bool{
+		"GOBIN":   true,
+		"GOROOT":  true,
+		"INCLUDE": true,
+		"LIB":     true,
+	}
+	var e []string
+	for name, v := range start {
+		e = append(e, name+"="+v)
+		skip[name] = true
+	}
+	for _, kv := range os.Environ() {
+		s := strings.SplitN(kv, "=", 2)
+		name := strings.ToUpper(s[0])
+		switch {
+		case name == "":
+			// variables, like "=C:=C:\", just copy them
+			e = append(e, kv)
+		case !skip[name]:
+			e = append(e, kv)
+			skip[name] = true
+		}
+	}
+	return e
+}
+
+func isDirectory(name string) bool {
+	s, err := os.Stat(name)
+	return err == nil && s.IsDir()
+}
+
+func isFile(name string) bool {
+	s, err := os.Stat(name)
+	return err == nil && !s.IsDir()
+}
+
+// commitWatcher polls hg for new commits and tells the dashboard about them.
+func commitWatcher(goroot *Repo) {
+	if *commitInterval == 0 {
+		log.Printf("commitInterval is %s, disabling commitWatcher", *commitInterval)
+		return
+	}
+	// Create builder just to get master key.
+	b, err := NewBuilder(goroot, "mercurial-commit")
+	if err != nil {
+		log.Fatal(err)
+	}
+	key := b.key
+
+	for {
+		if *verbose {
+			log.Printf("poll...")
+		}
+		// Main Go repository.
+		commitPoll(goroot, "", key)
+		// Go sub-repositories.
+		for _, pkg := range dashboardPackages("subrepo") {
+			pkgroot := &Repo{
+				Path: filepath.Join(*buildroot, pkg),
+			}
+			commitPoll(pkgroot, pkg, key)
+		}
+		if *verbose {
+			log.Printf("sleep...")
+		}
+		time.Sleep(*commitInterval)
+	}
+}
+
+// logByHash is a cache of all Mercurial revisions we know about,
+// indexed by full hash.
+var logByHash = map[string]*HgLog{}
+
+// commitPoll pulls any new revisions from the hg server
+// and tells the server about them.
+func commitPoll(repo *Repo, pkg, key string) {
+	if !repo.Exists() {
+		var err error
+		repo, err = RemoteRepo(repoURL(pkg)).Clone(repo.Path, "tip")
+		if err != nil {
+			log.Printf("%s: hg clone failed: %v", pkg, err)
+			if err := os.RemoveAll(repo.Path); err != nil {
+				log.Printf("%s: %v", pkg, err)
+			}
+		}
+		return
+	}
+
+	logs, err := repo.Log() // repo.Log calls repo.Pull internally
+	if err != nil {
+		log.Printf("hg log: %v", err)
+		return
+	}
+
+	// Pass 1.  Fill in parents and add new log entries to logsByHash.
+	// Empty parent means take parent from next log entry.
+	// Non-empty parent has form 1234:hashhashhash; we want full hash.
+	for i := range logs {
+		l := &logs[i]
+		if l.Parent == "" && i+1 < len(logs) {
+			l.Parent = logs[i+1].Hash
+		} else if l.Parent != "" {
+			l.Parent, _ = repo.FullHash(l.Parent)
+		}
+		if *verbose {
+			log.Printf("hg log %s: %s < %s\n", pkg, l.Hash, l.Parent)
+		}
+		if logByHash[l.Hash] == nil {
+			// Make copy to avoid pinning entire slice when only one entry is new.
+			t := *l
+			logByHash[t.Hash] = &t
+		}
+	}
+
+	for _, l := range logs {
+		addCommit(pkg, l.Hash, key)
+	}
+}
+
+// addCommit adds the commit with the named hash to the dashboard.
+// key is the secret key for authentication to the dashboard.
+// It avoids duplicate effort.
+func addCommit(pkg, hash, key string) bool {
+	l := logByHash[hash]
+	if l == nil {
+		return false
+	}
+	if l.added {
+		return true
+	}
+
+	// Check for already added, perhaps in an earlier run.
+	if dashboardCommit(pkg, hash) {
+		log.Printf("%s already on dashboard\n", hash)
+		// Record that this hash is on the dashboard,
+		// as must be all its parents.
+		for l != nil {
+			l.added = true
+			l = logByHash[l.Parent]
+		}
+		return true
+	}
+
+	// Create parent first, to maintain some semblance of order.
+	if l.Parent != "" {
+		if !addCommit(pkg, l.Parent, key) {
+			return false
+		}
+	}
+
+	// Create commit.
+	if err := postCommit(key, pkg, l); err != nil {
+		log.Printf("failed to add %s to dashboard: %v", key, err)
+		return false
+	}
+	return true
+}
+
+var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`)
+
+// repoURL returns the repository URL for the supplied import path.
+func repoURL(importPath string) string {
+	m := repoRe.FindStringSubmatch(importPath)
+	if len(m) < 2 {
+		log.Printf("repoURL: couldn't decipher %q", importPath)
+		return ""
+	}
+	return "https://code.google.com/p/" + m[1]
+}
+
+// defaultSuffix returns file extension used for command files in
+// current os environment.
+func defaultSuffix() string {
+	switch runtime.GOOS {
+	case "windows":
+		return ".bat"
+	case "plan9":
+		return ".rc"
+	default:
+		return ".bash"
+	}
+}
+
+// defaultBuildRoot returns default buildroot directory.
+func defaultBuildRoot() string {
+	var d string
+	if runtime.GOOS == "windows" {
+		// will use c:\, otherwise absolute paths become too long
+		// during builder run, see http://golang.org/issue/3358.
+		d = `c:\`
+	} else {
+		d = os.TempDir()
+	}
+	return filepath.Join(d, "gobuilder")
+}
+
+func getenvOk(k string) (v string, ok bool) {
+	v = os.Getenv(k)
+	if v != "" {
+		return v, true
+	}
+	keq := k + "="
+	for _, kv := range os.Environ() {
+		if kv == keq {
+			return "", true
+		}
+	}
+	return "", false
+}

dashboard/builder/vcs.go

@@ -0,0 +1,148 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/xml"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+// Repo represents a mercurial repository.
+type Repo struct {
+	Path string
+	sync.Mutex
+}
+
+// RemoteRepo constructs a *Repo representing a remote repository.
+func RemoteRepo(url string) *Repo {
+	return &Repo{
+		Path: url,
+	}
+}
+
+// Clone clones the current Repo to a new destination
+// returning a new *Repo if successful.
+func (r *Repo) Clone(path, rev string) (*Repo, error) {
+	r.Lock()
+	defer r.Unlock()
+	if err := run(*cmdTimeout, nil, *buildroot, r.hgCmd("clone", "-r", rev, r.Path, path)...); err != nil {
+		return nil, err
+	}
+	return &Repo{
+		Path: path,
+	}, nil
+}
+
+// UpdateTo updates the working copy of this Repo to the
+// supplied revision.
+func (r *Repo) UpdateTo(hash string) error {
+	r.Lock()
+	defer r.Unlock()
+	return run(*cmdTimeout, nil, r.Path, r.hgCmd("update", hash)...)
+}
+
+// Exists reports whether this Repo represents a valid Mecurial repository.
+func (r *Repo) Exists() bool {
+	fi, err := os.Stat(filepath.Join(r.Path, ".hg"))
+	if err != nil {
+		return false
+	}
+	return fi.IsDir()
+}
+
+// Pull pulls changes from the default path, that is, the path
+// this Repo was cloned from.
+func (r *Repo) Pull() error {
+	r.Lock()
+	defer r.Unlock()
+	return run(*cmdTimeout, nil, r.Path, r.hgCmd("pull")...)
+}
+
+// Log returns the changelog for this repository.
+func (r *Repo) Log() ([]HgLog, error) {
+	if err := r.Pull(); err != nil {
+		return nil, err
+	}
+	const N = 50 // how many revisions to grab
+
+	r.Lock()
+	defer r.Unlock()
+	data, _, err := runLog(*cmdTimeout, nil, r.Path, r.hgCmd("log",
+		"--encoding=utf-8",
+		"--limit="+strconv.Itoa(N),
+		"--template="+xmlLogTemplate)...,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	var logStruct struct {
+		Log []HgLog
+	}
+	err = xml.Unmarshal([]byte("<Top>"+data+"</Top>"), &logStruct)
+	if err != nil {
+		log.Printf("unmarshal hg log: %v", err)
+		return nil, err
+	}
+	return logStruct.Log, nil
+}
+
+// FullHash returns the full hash for the given Mercurial revision.
+func (r *Repo) FullHash(rev string) (string, error) {
+	r.Lock()
+	defer r.Unlock()
+	s, _, err := runLog(*cmdTimeout, nil, r.Path,
+		r.hgCmd("log",
+			"--encoding=utf-8",
+			"--rev="+rev,
+			"--limit=1",
+			"--template={node}")...,
+	)
+	if err != nil {
+		return "", nil
+	}
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return "", fmt.Errorf("cannot find revision")
+	}
+	if len(s) != 40 {
+		return "", fmt.Errorf("hg returned invalid hash " + s)
+	}
+	return s, nil
+}
+
+func (r *Repo) hgCmd(args ...string) []string {
+	return append([]string{"hg", "--config", "extensions.codereview=!"}, args...)
+}
+
+// HgLog represents a single Mercurial revision.
+type HgLog struct {
+	Hash   string
+	Author string
+	Date   string
+	Desc   string
+	Parent string
+
+	// Internal metadata
+	added bool
+}
+
+// xmlLogTemplate is a template to pass to Mercurial to make
+// hg log print the log in valid XML for parsing with xml.Unmarshal.
+const xmlLogTemplate = `
+        <Log>
+        <Hash>{node|escape}</Hash>
+        <Parent>{parent|escape}</Parent>
+        <Author>{author|escape}</Author>
+        <Date>{date|rfc3339date}</Date>
+        <Desc>{desc|escape}</Desc>
+        </Log>
+`