app: modernize the App Engine code

+ Renames build/ to appengine/ as it contains the main entry point
  for the app.
+ Moves app.yaml files to the appengine/ folder as gcloud app deploy
  will complain if it can’t find any go files to build in the same
  directory as the app.yaml file (previously I think it recursively
  looked for them in the directory tree).
+ Update deploy steps in app.yaml.
+ Remove application and version fields in app.yaml as gcloud deploy
  doesn’t allow them anymore.
+ Update to use go1.8, including using the context package as
  appropriate. In some cases the x/net/context package must be
  used to match the function types specified by GAE.

Change-Id: Ib38563ba7245f37de1a2f8d7f034f02c0cd77863
Reviewed-on: https://go-review.googlesource.com/70410
Reviewed-by: Chris Broadfoot <cbro@golang.org>
This commit is contained in:
Andrew Bonventre 2017-10-12 16:08:53 -04:00
Родитель a328a7908a
Коммит bd4a55cdea
31 изменённых файлов: 180 добавлений и 161 удалений

11
app/appengine/README.md Normal file
Просмотреть файл

@ -0,0 +1,11 @@
# build.golang.org App Engine App
Update with
```sh
gcloud config set project golang-org
gcloud app deploy --no-promote -v {build|build-test} app.yaml
```
Using -v build will run as build.golang.org.
Using -v build-test will run as build-test.golang.org.

Просмотреть файл

@ -1,12 +1,5 @@
# Update with
# google_appengine/appcfg.py [-V build-test] update .
#
# Using -V build-test will run as build-test.golang.org.
application: golang-org
version: build
runtime: go
api_version: go1
api_version: go1.8
handlers:
- url: /static

Просмотреть файл

@ -9,6 +9,7 @@ package build
import (
"bytes"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
@ -19,10 +20,9 @@ import (
"strings"
"time"
"appengine"
"appengine/datastore"
"google.golang.org/appengine/datastore"
"cache"
"golang.org/x/build/app/cache"
"golang.org/x/build/internal/loghash"
)
@ -44,7 +44,7 @@ func (p *Package) String() string {
return fmt.Sprintf("%s: %q", p.Path, p.Name)
}
func (p *Package) Key(c appengine.Context) *datastore.Key {
func (p *Package) Key(c context.Context) *datastore.Key {
key := p.Path
if key == "" {
key = "go"
@ -53,7 +53,7 @@ func (p *Package) Key(c appengine.Context) *datastore.Key {
}
// LastCommit returns the most recent Commit for this Package.
func (p *Package) LastCommit(c appengine.Context) (*Commit, error) {
func (p *Package) LastCommit(c context.Context) (*Commit, error) {
var commits []*Commit
_, err := datastore.NewQuery("Commit").
Ancestor(p.Key(c)).
@ -70,7 +70,7 @@ func (p *Package) LastCommit(c appengine.Context) (*Commit, error) {
}
// GetPackage fetches a Package by path from the datastore.
func GetPackage(c appengine.Context, path string) (*Package, error) {
func GetPackage(c context.Context, path string) (*Package, error) {
p := &Package{Path: path}
err := datastore.Get(c, p.Key(c), p)
if err == datastore.ErrNoSuchEntity {
@ -116,7 +116,7 @@ type Commit struct {
buildingURLs map[builderAndGoHash]string
}
func (com *Commit) Key(c appengine.Context) *datastore.Key {
func (com *Commit) Key(c context.Context) *datastore.Key {
if com.Hash == "" {
panic("tried Key on Commit with empty Hash")
}
@ -135,7 +135,7 @@ func (c *Commit) Valid() error {
return nil
}
func putCommit(c appengine.Context, com *Commit) error {
func putCommit(c context.Context, com *Commit) error {
if err := com.Valid(); err != nil {
return fmt.Errorf("putting Commit: %v", err)
}
@ -154,7 +154,7 @@ const maxResults = 1000
// AddResult adds the denormalized Result data to the Commit's ResultData field.
// It must be called from inside a datastore transaction.
func (com *Commit) AddResult(c appengine.Context, r *Result) error {
func (com *Commit) AddResult(c context.Context, r *Result) error {
if err := datastore.Get(c, com.Key(c), com); err != nil {
return fmt.Errorf("getting Commit: %v", err)
}
@ -191,7 +191,7 @@ func (com *Commit) RemoveResult(r *Result) {
// AddPerfResult remembers that the builder has run the benchmark on the commit.
// It must be called from inside a datastore transaction.
func (com *Commit) AddPerfResult(c appengine.Context, builder, benchmark string) error {
func (com *Commit) AddPerfResult(c context.Context, builder, benchmark string) error {
if err := datastore.Get(c, com.Key(c), com); err != nil {
return fmt.Errorf("getting Commit: %v", err)
}
@ -307,7 +307,7 @@ type CommitRun struct {
NeedsBenchmarking []bool `datastore:",noindex"`
}
func (cr *CommitRun) Key(c appengine.Context) *datastore.Key {
func (cr *CommitRun) Key(c context.Context) *datastore.Key {
p := Package{Path: cr.PackagePath}
key := strconv.Itoa(cr.StartCommitNum)
return datastore.NewKey(c, "CommitRun", key, 0, p.Key(c))
@ -315,7 +315,7 @@ func (cr *CommitRun) Key(c appengine.Context) *datastore.Key {
// GetCommitRun loads and returns CommitRun that contains information
// for commit commitNum.
func GetCommitRun(c appengine.Context, commitNum int) (*CommitRun, error) {
func GetCommitRun(c context.Context, commitNum int) (*CommitRun, error) {
cr := &CommitRun{StartCommitNum: commitNum / PerfRunLength * PerfRunLength}
err := datastore.Get(c, cr.Key(c), cr)
if err != nil && err != datastore.ErrNoSuchEntity {
@ -331,7 +331,7 @@ func GetCommitRun(c appengine.Context, commitNum int) (*CommitRun, error) {
return cr, nil
}
func (cr *CommitRun) AddCommit(c appengine.Context, com *Commit) error {
func (cr *CommitRun) AddCommit(c context.Context, com *Commit) error {
if com.Num < cr.StartCommitNum || com.Num >= cr.StartCommitNum+PerfRunLength {
return fmt.Errorf("AddCommit: commit num %v out of range [%v, %v)",
com.Num, cr.StartCommitNum, cr.StartCommitNum+PerfRunLength)
@ -353,7 +353,7 @@ func (cr *CommitRun) AddCommit(c appengine.Context, com *Commit) error {
// GetCommits returns [startCommitNum, startCommitNum+n) commits.
// Commits information is partial (obtained from CommitRun),
// do not store them back into datastore.
func GetCommits(c appengine.Context, startCommitNum, n int) ([]*Commit, error) {
func GetCommits(c context.Context, startCommitNum, n int) ([]*Commit, error) {
if startCommitNum < 0 || n <= 0 {
return nil, fmt.Errorf("GetCommits: invalid args (%v, %v)", startCommitNum, n)
}
@ -436,7 +436,7 @@ type Result struct {
RunTime int64 // time to build+test in nanoseconds
}
func (r *Result) Key(c appengine.Context) *datastore.Key {
func (r *Result) Key(c context.Context) *datastore.Key {
p := Package{Path: r.PackagePath}
key := r.Builder + "|" + r.PackagePath + "|" + r.Hash + "|" + r.GoHash
return datastore.NewKey(c, "Result", key, 0, p.Key(c))
@ -477,7 +477,7 @@ type ParsedPerfResult struct {
Artifacts map[string]string
}
func (r *PerfResult) Key(c appengine.Context) *datastore.Key {
func (r *PerfResult) Key(c context.Context) *datastore.Key {
p := Package{Path: r.PackagePath}
key := r.CommitHash
return datastore.NewKey(c, "PerfResult", key, 0, p.Key(c))
@ -561,7 +561,7 @@ type PerfMetricRun struct {
Vals []int64 `datastore:",noindex"`
}
func (m *PerfMetricRun) Key(c appengine.Context) *datastore.Key {
func (m *PerfMetricRun) Key(c context.Context) *datastore.Key {
p := Package{Path: m.PackagePath}
key := m.Builder + "|" + m.Benchmark + "|" + m.Metric + "|" + strconv.Itoa(m.StartCommitNum)
return datastore.NewKey(c, "PerfMetricRun", key, 0, p.Key(c))
@ -569,7 +569,7 @@ func (m *PerfMetricRun) Key(c appengine.Context) *datastore.Key {
// GetPerfMetricRun loads and returns PerfMetricRun that contains information
// for commit commitNum.
func GetPerfMetricRun(c appengine.Context, builder, benchmark, metric string, commitNum int) (*PerfMetricRun, error) {
func GetPerfMetricRun(c context.Context, builder, benchmark, metric string, commitNum int) (*PerfMetricRun, error) {
startCommitNum := commitNum / PerfRunLength * PerfRunLength
m := &PerfMetricRun{Builder: builder, Benchmark: benchmark, Metric: metric, StartCommitNum: startCommitNum}
err := datastore.Get(c, m.Key(c), m)
@ -582,7 +582,7 @@ func GetPerfMetricRun(c appengine.Context, builder, benchmark, metric string, co
return m, nil
}
func (m *PerfMetricRun) AddMetric(c appengine.Context, commitNum int, v uint64) error {
func (m *PerfMetricRun) AddMetric(c context.Context, commitNum int, v uint64) error {
if commitNum < m.StartCommitNum || commitNum >= m.StartCommitNum+PerfRunLength {
return fmt.Errorf("AddMetric: CommitNum %v out of range [%v, %v)",
commitNum, m.StartCommitNum, m.StartCommitNum+PerfRunLength)
@ -596,7 +596,7 @@ func (m *PerfMetricRun) AddMetric(c appengine.Context, commitNum int, v uint64)
// GetPerfMetricsForCommits returns perf metrics for builder/benchmark/metric
// and commits [startCommitNum, startCommitNum+n).
func GetPerfMetricsForCommits(c appengine.Context, builder, benchmark, metric string, startCommitNum, n int) ([]uint64, error) {
func GetPerfMetricsForCommits(c context.Context, builder, benchmark, metric string, startCommitNum, n int) ([]uint64, error) {
if startCommitNum < 0 || n <= 0 {
return nil, fmt.Errorf("GetPerfMetricsForCommits: invalid args (%v, %v)", startCommitNum, n)
}
@ -657,14 +657,14 @@ type PerfConfig struct {
noise map[string]float64
}
func PerfConfigKey(c appengine.Context) *datastore.Key {
func PerfConfigKey(c context.Context) *datastore.Key {
p := Package{}
return datastore.NewKey(c, "PerfConfig", "PerfConfig", 0, p.Key(c))
}
const perfConfigCacheKey = "perf-config"
func GetPerfConfig(c appengine.Context, r *http.Request) (*PerfConfig, error) {
func GetPerfConfig(c context.Context, r *http.Request) (*PerfConfig, error) {
pc := new(PerfConfig)
now := cache.Now(c)
if cache.Get(c, r, now, perfConfigCacheKey, pc) {
@ -703,7 +703,7 @@ func (pc *PerfConfig) NoiseLevel(builder, benchmark, metric string) float64 {
// UpdatePerfConfig updates the PerfConfig entity with results of benchmarking.
// Returns whether it's a benchmark that we have not yet seem on the builder.
func UpdatePerfConfig(c appengine.Context, r *http.Request, req *PerfRequest) (newBenchmark bool, err error) {
func UpdatePerfConfig(c context.Context, r *http.Request, req *PerfRequest) (newBenchmark bool, err error) {
pc, err := GetPerfConfig(c, r)
if err != nil {
return false, err
@ -820,14 +820,14 @@ type PerfTodo struct {
CommitNums []int `datastore:",noindex"` // LIFO queue of commits to benchmark.
}
func (todo *PerfTodo) Key(c appengine.Context) *datastore.Key {
func (todo *PerfTodo) Key(c context.Context) *datastore.Key {
p := Package{Path: todo.PackagePath}
key := todo.Builder
return datastore.NewKey(c, "PerfTodo", key, 0, p.Key(c))
}
// AddCommitToPerfTodo adds the commit to all existing PerfTodo entities.
func AddCommitToPerfTodo(c appengine.Context, com *Commit) error {
func AddCommitToPerfTodo(c context.Context, com *Commit) error {
var todos []*PerfTodo
_, err := datastore.NewQuery("PerfTodo").
Ancestor((&Package{}).Key(c)).
@ -863,7 +863,7 @@ func (l *Log) Text() ([]byte, error) {
return b, nil
}
func PutLog(c appengine.Context, text string) (hash string, err error) {
func PutLog(c context.Context, text string) (hash string, err error) {
b := new(bytes.Buffer)
z, _ := gzip.NewWriterLevel(b, gzip.BestCompression)
io.WriteString(z, text)
@ -889,7 +889,7 @@ func (t *Tag) String() string {
return t.Name
}
func (t *Tag) Key(c appengine.Context) *datastore.Key {
func (t *Tag) Key(c context.Context) *datastore.Key {
p := &Package{}
s := t.Kind
if t.Kind == "release" {
@ -912,14 +912,14 @@ func (t *Tag) Valid() error {
}
// Commit returns the Commit that corresponds with this Tag.
func (t *Tag) Commit(c appengine.Context) (*Commit, error) {
func (t *Tag) Commit(c context.Context) (*Commit, error) {
com := &Commit{Hash: t.Hash}
err := datastore.Get(c, com.Key(c), com)
return com, err
}
// GetTag fetches a Tag by name from the datastore.
func GetTag(c appengine.Context, kind, name string) (*Tag, error) {
func GetTag(c context.Context, kind, name string) (*Tag, error) {
t := &Tag{Kind: kind, Name: name}
if err := datastore.Get(c, t.Key(c), t); err != nil {
return nil, err
@ -932,7 +932,7 @@ func GetTag(c appengine.Context, kind, name string) (*Tag, error) {
// Packages returns packages of the specified kind.
// Kind must be one of "external" or "subrepo".
func Packages(c appengine.Context, kind string) ([]*Package, error) {
func Packages(c context.Context, kind string) ([]*Package, error) {
switch kind {
case "external", "subrepo":
default:

Просмотреть файл

@ -9,8 +9,9 @@ package build
import (
"sync"
"appengine"
"appengine/datastore"
"golang.org/x/net/context"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/log"
)
// A global map of rarely-changing configuration values.
@ -35,7 +36,7 @@ type configEntity struct {
// Config returns the value for the given key
// or the empty string if no such key exists.
func Config(c appengine.Context, key string) string {
func Config(c context.Context, key string) string {
config.RLock()
v, ok := config.m[key]
config.RUnlock()
@ -55,7 +56,7 @@ func Config(c appengine.Context, key string) string {
k := datastore.NewKey(c, "Config", key, 0, nil)
ent := configEntity{}
if err := datastore.Get(c, k, &ent); err != nil {
c.Errorf("Get Config: %v", err)
log.Errorf(c, "Get Config: %v", err)
return ""
}
// Don't return or cache the dummy value.
@ -69,24 +70,24 @@ func Config(c appengine.Context, key string) string {
// initConfig is invoked by the initHandler to create an entity for each key in
// configKeys. This makes it easy to edit the configuration values using the
// Datastore Viewer in the App Engine dashboard.
func initConfig(c appengine.Context) {
func initConfig(c context.Context) {
for _, key := range configKeys {
err := datastore.RunInTransaction(c, func(c appengine.Context) error {
err := datastore.RunInTransaction(c, func(c context.Context) error {
k := datastore.NewKey(c, "Config", key, 0, nil)
ent := configEntity{}
if err := datastore.Get(c, k, &ent); err == nil {
c.Infof("huh? %v", key)
log.Infof(c, "huh? %v", key)
return nil
} else if err != datastore.ErrNoSuchEntity {
return err
}
ent.Value = configDummy
_, err := datastore.Put(c, k, &ent)
c.Infof("BLAH BLAH %v", key)
log.Infof(c, "BLAH BLAH %v", key)
return err
}, nil)
if err != nil {
c.Errorf("initConfig %v: %v", key, err)
log.Errorf(c, "initConfig %v: %v", key, err)
}
}
}

Просмотреть файл

Просмотреть файл

Просмотреть файл

@ -7,10 +7,11 @@
package build
import (
"context"
"net/http"
"strings"
"appengine"
"google.golang.org/appengine"
)
func handleFunc(path string, h http.HandlerFunc) {
@ -53,7 +54,7 @@ func dashboardForRequest(r *http.Request) *Dashboard {
// Context returns a namespaced context for this dashboard, or panics if it
// fails to create a new context.
func (d *Dashboard) Context(c appengine.Context) appengine.Context {
func (d *Dashboard) Context(c context.Context) context.Context {
if d.Namespace == "" {
return c
}

Просмотреть файл

@ -21,12 +21,13 @@ import (
"time"
"unicode/utf8"
"appengine"
"appengine/datastore"
"appengine/memcache"
"cache"
"key"
"golang.org/x/build/app/cache"
"golang.org/x/build/app/key"
"golang.org/x/net/context"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/log"
"google.golang.org/appengine/memcache"
)
const (
@ -108,7 +109,7 @@ func commitHandler(r *http.Request) (interface{}, error) {
return nil, fmt.Errorf("reading Body: %v", err)
}
if !bytes.Contains(body, needsBenchmarkingBytes) {
c.Warningf("old builder detected at %v", r.RemoteAddr)
log.Warningf(c, "old builder detected at %v", r.RemoteAddr)
return nil, fmt.Errorf("rejecting old builder request, body does not contain %s: %q", needsBenchmarkingBytes, body)
}
if err := json.Unmarshal(body, com); err != nil {
@ -119,7 +120,7 @@ func commitHandler(r *http.Request) (interface{}, error) {
return nil, fmt.Errorf("validating Commit: %v", err)
}
defer cache.Tick(c)
tx := func(c appengine.Context) error {
tx := func(c context.Context) error {
return addCommit(c, com)
}
return nil, datastore.RunInTransaction(c, tx, nil)
@ -129,7 +130,7 @@ var needsBenchmarkingBytes = []byte(`"NeedsBenchmarking"`)
// addCommit adds the Commit entity to the datastore and updates the tip Tag.
// It must be run inside a datastore transaction.
func addCommit(c appengine.Context, com *Commit) error {
func addCommit(c context.Context, com *Commit) error {
var ec Commit // existing commit
isUpdate := false
err := datastore.Get(c, com.Key(c), &ec)
@ -335,7 +336,7 @@ func todoHandler(r *http.Request) (interface{}, error) {
// If provided with non-empty packagePath and goHash args, it scans the first
// 20 Commits in Num-descending order for the specified packagePath and
// returns the first that doesn't have a Result for this builder and goHash.
func buildTodo(c appengine.Context, builder, packagePath, goHash string) (*Commit, error) {
func buildTodo(c context.Context, builder, packagePath, goHash string) (*Commit, error) {
p, err := GetPackage(c, packagePath)
if err != nil {
return nil, err
@ -390,7 +391,7 @@ func buildTodo(c appengine.Context, builder, packagePath, goHash string) (*Commi
for _, pkg := range pkgs {
com, err := pkg.LastCommit(c)
if err != nil {
c.Warningf("%v: no Commit found: %v", pkg, err)
log.Warningf(c, "%v: no Commit found: %v", pkg, err)
continue
}
if com.Result(builder, tag.Hash) == nil {
@ -402,7 +403,7 @@ func buildTodo(c appengine.Context, builder, packagePath, goHash string) (*Commi
}
// perfTodo returns the next Commit to be benchmarked (or nil if none available).
func perfTodo(c appengine.Context, builder string) (*Commit, error) {
func perfTodo(c context.Context, builder string) (*Commit, error) {
p := &Package{}
todo := &PerfTodo{Builder: builder}
err := datastore.Get(c, todo.Key(c), todo)
@ -448,9 +449,9 @@ func perfTodo(c appengine.Context, builder string) (*Commit, error) {
}
// buildPerfTodo creates PerfTodo for the builder with all commits. In a transaction.
func buildPerfTodo(c appengine.Context, builder string) (*PerfTodo, error) {
func buildPerfTodo(c context.Context, builder string) (*PerfTodo, error) {
todo := &PerfTodo{Builder: builder}
tx := func(c appengine.Context) error {
tx := func(c context.Context) error {
err := datastore.Get(c, todo.Key(c), todo)
if err != nil && err != datastore.ErrNoSuchEntity {
return fmt.Errorf("fetching PerfTodo: %v", err)
@ -499,7 +500,7 @@ func buildPerfTodo(c appengine.Context, builder string) (*PerfTodo, error) {
return todo, datastore.RunInTransaction(c, tx, nil)
}
func removeCommitFromPerfTodo(c appengine.Context, builder string, num int) error {
func removeCommitFromPerfTodo(c context.Context, builder string, num int) error {
todo := &PerfTodo{Builder: builder}
err := datastore.Get(c, todo.Key(c), todo)
if err != nil && err != datastore.ErrNoSuchEntity {
@ -604,7 +605,7 @@ func resultHandler(r *http.Request) (interface{}, error) {
}
res.LogHash = hash
}
tx := func(c appengine.Context) error {
tx := func(c context.Context) error {
// check Package exists
if _, err := GetPackage(c, res.PackagePath); err != nil {
return fmt.Errorf("GetPackage: %v", err)
@ -669,7 +670,7 @@ func perfResultHandler(r *http.Request) (interface{}, error) {
}
req.Artifacts[i].Body = hash
}
tx := func(c appengine.Context) error {
tx := func(c context.Context) error {
return addPerfResult(c, r, req)
}
return nil, datastore.RunInTransaction(c, tx, nil)
@ -678,7 +679,7 @@ func perfResultHandler(r *http.Request) (interface{}, error) {
// addPerfResult creates PerfResult and updates Commit, PerfTodo,
// PerfMetricRun and PerfConfig.
// MUST be called from inside a transaction.
func addPerfResult(c appengine.Context, r *http.Request, req *PerfRequest) error {
func addPerfResult(c context.Context, r *http.Request, req *PerfRequest) error {
// check Package exists
p, err := GetPackage(c, "")
if err != nil {
@ -745,7 +746,7 @@ func addPerfResult(c appengine.Context, r *http.Request, req *PerfRequest) error
}
// MUST be called from inside a transaction.
func checkPerfChanges(c appengine.Context, r *http.Request, com *Commit, builder string, res *PerfResult) error {
func checkPerfChanges(c context.Context, r *http.Request, com *Commit, builder string, res *PerfResult) error {
pc, err := GetPerfConfig(c, r)
if err != nil {
return err
@ -811,7 +812,7 @@ func checkPerfChanges(c appengine.Context, r *http.Request, com *Commit, builder
return nil
}
func comparePerfResults(c appengine.Context, pc *PerfConfig, builder string, prevRes, res *PerfResult) error {
func comparePerfResults(c context.Context, pc *PerfConfig, builder string, prevRes, res *PerfResult) error {
changes := significantPerfChanges(pc, builder, prevRes, res)
if len(changes) == 0 {
return nil
@ -869,7 +870,7 @@ func clearResultsHandler(r *http.Request) (interface{}, error) {
c := contextForRequest(r)
defer cache.Tick(c)
pkg := (&Package{}).Key(c) // TODO(adg): support clearing sub-repos
err := datastore.RunInTransaction(c, func(c appengine.Context) error {
err := datastore.RunInTransaction(c, func(c context.Context) error {
var coms []*Commit
keys, err := datastore.NewQuery("Commit").
Ancestor(pkg).
@ -958,12 +959,12 @@ func AuthHandler(h dashHandler) http.HandlerFunc {
// Write JSON response.
dashResp := &dashResponse{Response: resp}
if err != nil {
c.Errorf("%v", err)
log.Errorf(c, "%v", err)
dashResp.Error = err.Error()
}
w.Header().Set("Content-Type", "application/json")
if err = json.NewEncoder(w).Encode(dashResp); err != nil {
c.Criticalf("encoding response: %v", err)
log.Criticalf(c, "encoding response: %v", err)
}
}
}
@ -1002,7 +1003,7 @@ func validHash(hash string) bool {
return hash != ""
}
func validKey(c appengine.Context, key, builder string) bool {
func validKey(c context.Context, key, builder string) bool {
if isMasterKey(c, key) {
return true
}
@ -1012,23 +1013,24 @@ func validKey(c appengine.Context, key, builder string) bool {
return key == builderKey(c, builder)
}
func isMasterKey(c appengine.Context, k string) bool {
func isMasterKey(c context.Context, k string) bool {
return appengine.IsDevAppServer() || k == key.Secret(c)
}
func builderKey(c appengine.Context, builder string) string {
func builderKey(c context.Context, builder string) string {
h := hmac.New(md5.New, []byte(key.Secret(c)))
h.Write([]byte(builder))
return fmt.Sprintf("%x", h.Sum(nil))
}
func logErr(w http.ResponseWriter, r *http.Request, err error) {
contextForRequest(r).Errorf("Error: %v", err)
c := contextForRequest(r)
log.Errorf(c, "Error: %v", err)
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprint(w, "Error: ", html.EscapeString(err.Error()))
}
func contextForRequest(r *http.Request) appengine.Context {
func contextForRequest(r *http.Request) context.Context {
return dashboardForRequest(r).Context(appengine.NewContext(r))
}

Просмотреть файл

Просмотреть файл

@ -10,11 +10,11 @@ import (
"fmt"
"net/http"
"appengine"
"appengine/datastore"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
"cache"
"key"
"golang.org/x/build/app/cache"
"golang.org/x/build/app/key"
)
func initHandler(w http.ResponseWriter, r *http.Request) {

Просмотреть файл

@ -8,6 +8,7 @@ package build
import (
"bytes"
"context"
"encoding/gob"
"encoding/json"
"fmt"
@ -18,10 +19,11 @@ import (
"strings"
"text/template"
"appengine"
"appengine/datastore"
"appengine/delay"
"appengine/urlfetch"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/delay"
"google.golang.org/appengine/log"
"google.golang.org/appengine/urlfetch"
)
const (
@ -50,7 +52,7 @@ var ignoreFailure = map[string]bool{
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
func notifyOnFailure(c context.Context, com *Commit, builder string) error {
if ignoreFailure[builder] {
return nil
}
@ -79,8 +81,8 @@ func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
return err
}
if nr := next.Result(builder, ""); nr != nil && !nr.OK {
c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
log.Debugf(c, "commit ok: %#v\nresult: %#v", com, cr)
log.Debugf(c, "next commit broken: %#v\nnext result:%#v", next, nr)
broken = next
}
} else {
@ -96,8 +98,8 @@ func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
return err
}
if pr := prev.Result(builder, ""); pr != nil && pr.OK {
c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
log.Debugf(c, "commit broken: %#v\nresult: %#v", com, cr)
log.Debugf(c, "previous commit ok: %#v\nprevious result:%#v", prev, pr)
broken = com
}
}
@ -112,7 +114,7 @@ func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
}
// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
func firstMatch(c context.Context, q *datastore.Query, v interface{}) error {
t := q.Limit(1).Run(c)
_, err := t.Next(v)
if err == datastore.Done {
@ -124,12 +126,12 @@ func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
var (
notifyLater = delay.Func("notify", notify)
notifyTmpl = template.Must(template.New("notify.txt").
Funcs(template.FuncMap(tmplFuncs)).ParseFiles("build/notify.txt"))
Funcs(template.FuncMap(tmplFuncs)).ParseFiles("notify.txt"))
)
// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder, logHash string) {
func notify(c context.Context, com *Commit, builder, logHash string) {
var msg bytes.Buffer
err := notifyTmpl.Execute(&msg, struct {
Builder string
@ -137,19 +139,19 @@ func notify(c appengine.Context, com *Commit, builder, logHash string) {
Hostname string
}{builder, logHash, domain})
if err != nil {
c.Criticalf("couldn't render template: %v", err)
log.Criticalf(c, "couldn't render template: %v", err)
return
}
if err := postGerritMessage(c, com, msg.String()); err != nil {
c.Errorf("couldn't post to gerrit: %v", err)
log.Errorf(c, "couldn't post to gerrit: %v", err)
}
}
// postGerritMessage posts a message to the code review thread for the given
// Commit.
func postGerritMessage(c appengine.Context, com *Commit, message string) error {
func postGerritMessage(c context.Context, com *Commit, message string) error {
if appengine.IsDevAppServer() {
c.Infof("Skiping update of Gerrit review for %v with message: %v", com, message)
log.Infof(c, "Skiping update of Gerrit review for %v with message: %v", com, message)
return nil
}
// Get change ID using commit hash.
@ -203,7 +205,7 @@ func init() {
}
// MUST be called from inside a transaction.
func sendPerfFailMail(c appengine.Context, builder string, res *PerfResult) error {
func sendPerfFailMail(c context.Context, builder string, res *PerfResult) error {
com := &Commit{Hash: res.CommitHash}
if err := datastore.Get(c, com.Key(c), com); err != nil {
return err
@ -224,18 +226,18 @@ func sendPerfFailMail(c appengine.Context, builder string, res *PerfResult) erro
// commonNotify MUST!!! be called from within a transaction inside which
// the provided Commit entity was retrieved from the datastore.
func commonNotify(c appengine.Context, com *Commit, builder, logHash string) error {
func commonNotify(c context.Context, com *Commit, builder, logHash string) error {
if com.Num == 0 || com.Desc == "" {
stk := make([]byte, 10000)
n := runtime.Stack(stk, false)
stk = stk[:n]
c.Errorf("refusing to notify with com=%+v\n%s", *com, string(stk))
log.Errorf(c, "refusing to notify with com=%+v\n%s", *com, string(stk))
return fmt.Errorf("misuse of commonNotify")
}
if com.FailNotificationSent {
return nil
}
c.Infof("%s is broken commit; notifying", com.Hash)
log.Infof(c, "%s is broken commit; notifying", com.Hash)
notifyLater.Call(c, com, builder, logHash) // add task to queue
com.FailNotificationSent = true
return putCommit(c, com)
@ -277,11 +279,11 @@ var (
sendPerfMailTmpl = template.Must(
template.New("perf_notify.txt").
Funcs(template.FuncMap(tmplFuncs)).
ParseFiles("build/perf_notify.txt"),
ParseFiles("perf_notify.txt"),
)
)
func sendPerfMailFunc(c appengine.Context, com *Commit, prevCommitHash, builder string, changes []*PerfChange) {
func sendPerfMailFunc(c context.Context, com *Commit, prevCommitHash, builder string, changes []*PerfChange) {
// Sort the changes into the right order.
var benchmarks []*PerfChangeBenchmark
for _, ch := range changes {
@ -312,11 +314,11 @@ func sendPerfMailFunc(c appengine.Context, com *Commit, prevCommitHash, builder
"Builder": builder, "Hostname": domain, "Url": u, "Benchmarks": benchmarks,
})
if err != nil {
c.Errorf("rendering perf mail template: %v", err)
log.Errorf(c, "rendering perf mail template: %v", err)
return
}
if err := postGerritMessage(c, com, body.String()); err != nil {
c.Errorf("posting to gerrit: %v", err)
log.Errorf(c, "posting to gerrit: %v", err)
}
}

Просмотреть файл

Просмотреть файл

@ -7,13 +7,14 @@
package build
import (
"context"
"fmt"
"sort"
"strconv"
"strings"
"appengine"
"appengine/datastore"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/log"
)
var knownTags = map[string]string{
@ -32,7 +33,7 @@ func splitBench(benchProcs string) (string, int) {
return ss[0], procs
}
func dashPerfCommits(c appengine.Context, page int) ([]*Commit, error) {
func dashPerfCommits(c context.Context, page int) ([]*Commit, error) {
q := datastore.NewQuery("Commit").
Ancestor((&Package{}).Key(c)).
Order("-Num").
@ -76,13 +77,13 @@ func isPerfFailed(res *PerfResult, builder string) bool {
// without lots of duplicate accesses to datastore.
// It allows to iterate over newer or older results for some base commit.
type PerfResultCache struct {
c appengine.Context
c context.Context
newer bool
iter *datastore.Iterator
results map[int]*PerfResult
}
func MakePerfResultCache(c appengine.Context, com *Commit, newer bool) *PerfResultCache {
func MakePerfResultCache(c context.Context, com *Commit, newer bool) *PerfResultCache {
p := &Package{}
q := datastore.NewQuery("PerfResult").Ancestor(p.Key(c)).Limit(100)
if newer {
@ -128,7 +129,7 @@ func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) {
return nil, fmt.Errorf("fetching perf results: %v", err)
}
if (rc.newer && res.CommitNum < commitNum) || (!rc.newer && res.CommitNum > commitNum) {
rc.c.Errorf("PerfResultCache.Next: bad commit num")
log.Errorf(rc.c, "PerfResultCache.Next: bad commit num")
}
rc.results[res.CommitNum] = res
return res, nil

Просмотреть файл

@ -14,8 +14,8 @@ import (
"sort"
"strconv"
"appengine"
"appengine/datastore"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
)
func init() {
@ -221,7 +221,7 @@ type uiPerfConfigElem struct {
}
var perfChangesTemplate = template.Must(
template.New("perf_changes.html").Funcs(tmplFuncs).ParseFiles("build/perf_changes.html"),
template.New("perf_changes.html").Funcs(tmplFuncs).ParseFiles("perf_changes.html"),
)
type perfChangesData struct {

Просмотреть файл

Просмотреть файл

@ -15,8 +15,8 @@ import (
"strconv"
"strings"
"appengine"
"appengine/datastore"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
)
func init() {
@ -215,5 +215,5 @@ func (l uiPerfDetailMetrics) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l uiPerfDetailMetrics) Less(i, j int) bool { return l[i].Name < l[j].Name }
var uiPerfDetailTemplate = template.Must(
template.New("perf_detail.html").Funcs(tmplFuncs).ParseFiles("build/perf_detail.html"),
template.New("perf_detail.html").Funcs(tmplFuncs).ParseFiles("perf_detail.html"),
)

Просмотреть файл

Просмотреть файл

@ -13,8 +13,8 @@ import (
"net/http"
"strconv"
"appengine"
"appengine/datastore"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
)
func init() {
@ -244,7 +244,7 @@ func perfGraphHandler(w http.ResponseWriter, r *http.Request) {
}
var perfGraphTemplate = template.Must(
template.New("perf_graph.html").ParseFiles("build/perf_graph.html"),
template.New("perf_graph.html").ParseFiles("perf_graph.html"),
)
type perfGraphData struct {

Просмотреть файл

Просмотреть файл

@ -13,8 +13,9 @@ import (
"net/http"
"sort"
"appengine"
"appengine/datastore"
"golang.org/x/net/context"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
)
func init() {
@ -135,7 +136,7 @@ func perfLearnHandler(w http.ResponseWriter, r *http.Request) {
for k, v := range noise {
noiseLevels = append(noiseLevels, k+v)
}
tx := func(c appengine.Context) error {
tx := func(c context.Context) error {
pc, err := GetPerfConfig(c, r)
if err != nil {
return err
@ -162,7 +163,7 @@ func perfLearnHandler(w http.ResponseWriter, r *http.Request) {
}
var perfLearnTemplate = template.Must(
template.New("perf_learn.html").Funcs(tmplFuncs).ParseFiles("build/perf_learn.html"),
template.New("perf_learn.html").Funcs(tmplFuncs).ParseFiles("perf_learn.html"),
)
type perfLearnData struct {

Просмотреть файл

Просмотреть файл

Просмотреть файл

До

Ширина:  |  Высота:  |  Размер: 570 B

После

Ширина:  |  Высота:  |  Размер: 570 B

Просмотреть файл

До

Ширина:  |  Высота:  |  Размер: 328 B

После

Ширина:  |  Высота:  |  Размер: 328 B

Просмотреть файл

Просмотреть файл

@ -11,6 +11,7 @@ package build
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
@ -21,8 +22,9 @@ import (
"strings"
"time"
"appengine"
"appengine/datastore"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/log"
)
func init() {
@ -243,7 +245,7 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
*r = origReq
}()
for i, t := range testRequests {
c.Infof("running test %d %s vals='%q' req='%q' res='%q'", i, t.path, t.vals, t.req, t.res)
log.Infof(c, "running test %d %s vals='%q' req='%q' res='%q'", i, t.path, t.vals, t.req, t.res)
errorf := func(format string, args ...interface{}) {
fmt.Fprintf(w, "%d %s: ", i, t.path)
fmt.Fprintf(w, format, args...)
@ -280,7 +282,7 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
errorf(rec.Body.String())
return
}
c.Infof("response='%v'", rec.Body.String())
log.Infof(c, "response='%v'", rec.Body.String())
resp := new(dashResponse)
// If we're expecting a *Todo value,
@ -356,7 +358,7 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "PASS\nYou should see only one mail notification (for 0003/linux-386) in the dev_appserver logs.")
}
func nukeEntities(c appengine.Context, kinds []string) error {
func nukeEntities(c context.Context, kinds []string) error {
if !appengine.IsDevAppServer() {
return errors.New("can't nuke production data")
}

Просмотреть файл

@ -11,6 +11,7 @@ package build
import (
"bytes"
"context"
"encoding/json"
"fmt"
"html/template"
@ -22,11 +23,12 @@ import (
"golang.org/x/build/types"
"cache"
"golang.org/x/build/app/cache"
"appengine"
"appengine/datastore"
"appengine/memcache"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/log"
"google.golang.org/appengine/memcache"
)
func init() {
@ -174,11 +176,11 @@ func uiHandler(w http.ResponseWriter, r *http.Request) {
buf.WriteTo(w)
}
func listBranches(c appengine.Context) (branches []string) {
func listBranches(c context.Context) (branches []string) {
var commits []*Commit
_, err := datastore.NewQuery("Commit").Distinct().Project("Branch").GetAll(c, &commits)
if err != nil {
c.Errorf("listBranches: %v", err)
log.Errorf(c, "listBranches: %v", err)
return
}
for _, c := range commits {
@ -279,7 +281,7 @@ type Pagination struct {
// dashCommits gets a slice of the latest Commits to the current dashboard.
// If page > 0 it paginates by commitsPerPage.
func dashCommits(c appengine.Context, pkg *Package, page int, branch string) ([]*Commit, error) {
func dashCommits(c context.Context, pkg *Package, page int, branch string) ([]*Commit, error) {
offset := page * commitsPerPage
q := datastore.NewQuery("Commit").
Ancestor(pkg.Key(c)).
@ -296,7 +298,7 @@ func dashCommits(c appengine.Context, pkg *Package, page int, branch string) ([]
}
// fetchCommits gets a slice of the specific commit hashes
func fetchCommits(c appengine.Context, pkg *Package, hashes []string) ([]*Commit, error) {
func fetchCommits(c context.Context, pkg *Package, hashes []string) ([]*Commit, error) {
var out []*Commit
var keys []*datastore.Key
for _, hash := range hashes {
@ -412,7 +414,7 @@ type PackageState struct {
// GetTagState fetches the results for all Go subrepos at the specified Tag.
// (Kind is "tip" or "release"; name is like "release-branch.go1.4".)
func GetTagState(c appengine.Context, kind, name string) (*TagState, error) {
func GetTagState(c context.Context, kind, name string) (*TagState, error) {
tag, err := GetTag(c, kind, name)
if err != nil {
return nil, err
@ -425,7 +427,7 @@ func GetTagState(c appengine.Context, kind, name string) (*TagState, error) {
for _, pkg := range pkgs {
com, err := pkg.LastCommit(c)
if err != nil {
c.Warningf("%v: no Commit found: %v", pkg, err)
log.Warningf(c, "%v: no Commit found: %v", pkg, err)
continue
}
st.Packages = append(st.Packages, &PackageState{pkg, com})
@ -456,7 +458,7 @@ func buildingKey(hash, goHash, builder string) string {
// populateBuildingURLs populates each commit in Commits' buildingURLs map with the
// URLs of builds which are currently in progress.
func (td *uiTemplateData) populateBuildingURLs(ctx appengine.Context) {
func (td *uiTemplateData) populateBuildingURLs(ctx context.Context) {
// need are memcache keys: "building|<hash>|<gohash>|<builder>"
// The hash is of the main "go" repo, or the subrepo commit hash.
// The gohash is empty for the main repo, else it's the Go hash.
@ -495,7 +497,7 @@ func (td *uiTemplateData) populateBuildingURLs(ctx appengine.Context) {
m, err := memcache.GetMulti(ctx, need)
if err != nil {
// oh well. this is a cute non-critical feature anyway.
ctx.Debugf("GetMulti of building keys: %v", err)
log.Debugf(ctx, "GetMulti of building keys: %v", err)
return
}
for k, it := range m {
@ -519,7 +521,7 @@ func (td *uiTemplateData) populateBuildingURLs(ctx appengine.Context) {
}
var uiTemplate = template.Must(
template.New("ui.html").Funcs(tmplFuncs).ParseFiles("build/ui.html"),
template.New("ui.html").Funcs(tmplFuncs).ParseFiles("ui.html"),
)
var tmplFuncs = template.FuncMap{

Просмотреть файл

Просмотреть файл

@ -11,8 +11,9 @@ import (
"fmt"
"net/http"
"appengine"
"appengine/datastore"
"golang.org/x/net/context"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
)
func init() {
@ -45,7 +46,7 @@ func updateBenchmark(w http.ResponseWriter, r *http.Request) {
ncommit := 0
nrun := 0
tx := func(c appengine.Context) error {
tx := func(c context.Context) error {
var cr *CommitRun
for _, hash := range hashes {
// Update Commit.

25
app/cache/cache.go поставляемый
Просмотреть файл

@ -9,13 +9,14 @@ package cache
import (
"bytes"
"compress/gzip"
"context"
"encoding/gob"
"fmt"
"net/http"
"time"
"appengine"
"appengine/memcache"
"google.golang.org/appengine/log"
"google.golang.org/appengine/memcache"
)
// TimeKey specifies the memcache entity that keeps the logical datastore time.
@ -29,10 +30,10 @@ const (
func newTime() uint64 { return uint64(time.Now().Unix()) << 32 }
// Now returns the current logical datastore time to use for cache lookups.
func Now(c appengine.Context) uint64 {
func Now(c context.Context) uint64 {
t, err := memcache.Increment(c, TimeKey, 0, newTime())
if err != nil {
c.Errorf("cache.Now: %v", err)
log.Errorf(c, "cache.Now: %v", err)
return 0
}
return t
@ -40,10 +41,10 @@ func Now(c appengine.Context) uint64 {
// Tick sets the current logical datastore time to a never-before-used time
// and returns that time. It should be called to invalidate the cache.
func Tick(c appengine.Context) uint64 {
func Tick(c context.Context) uint64 {
t, err := memcache.Increment(c, TimeKey, 1, newTime())
if err != nil {
c.Errorf("cache.Tick: %v", err)
log.Errorf(c, "cache.Tick: %v", err)
return 0
}
return t
@ -52,26 +53,26 @@ func Tick(c appengine.Context) uint64 {
// Get fetches data for name at time now from memcache and unmarshals it into
// value. It reports whether it found the cache record and logs any errors to
// the admin console.
func Get(c appengine.Context, r *http.Request, now uint64, name string, value interface{}) bool {
func Get(c context.Context, r *http.Request, now uint64, name string, value interface{}) bool {
if now == 0 || r.FormValue(nocache) != "" {
return false
}
key := fmt.Sprintf("%s.%d", name, now)
_, err := gzipGobCodec.Get(c, key, value)
if err == nil {
c.Debugf("cache hit %q", key)
log.Debugf(c, "cache hit %q", key)
return true
}
c.Debugf("cache miss %q", key)
log.Debugf(c, "cache miss %q", key)
if err != memcache.ErrCacheMiss {
c.Errorf("get cache %q: %v", key, err)
log.Errorf(c, "get cache %q: %v", key, err)
}
return false
}
// Set puts value into memcache under name at time now.
// It logs any errors to the admin console.
func Set(c appengine.Context, r *http.Request, now uint64, name string, value interface{}) {
func Set(c context.Context, r *http.Request, now uint64, name string, value interface{}) {
if now == 0 || r.FormValue(nocache) != "" {
return
}
@ -82,7 +83,7 @@ func Set(c appengine.Context, r *http.Request, now uint64, name string, value in
Expiration: expiry,
})
if err != nil {
c.Errorf("set cache %q: %v", key, err)
log.Errorf(c, "set cache %q: %v", key, err)
}
}

Просмотреть файл

@ -7,10 +7,11 @@
package key
import (
"context"
"sync"
"appengine"
"appengine/datastore"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
)
var theKey struct {
@ -22,11 +23,11 @@ type builderKey struct {
Secret string
}
func (k *builderKey) Key(c appengine.Context) *datastore.Key {
func (k *builderKey) Key(c context.Context) *datastore.Key {
return datastore.NewKey(c, "BuilderKey", "root", 0, nil)
}
func Secret(c appengine.Context) string {
func Secret(c context.Context) string {
// check with rlock
theKey.RLock()
k := theKey.Secret