go
/
build
зеркало из https://github.com/golang/build.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

cmd/coordinator, internal/buildgo: create buildgo package 

This package contains the BuilderRev type moved from cmd/coordinator.

The rest of the CL is simply updating coordinator with the new
exported names of the type and its fields.

This refactoring is in preparation for moving the benchmark building and running
code into a separate package.

(Most of the diff could have been avoided with a type alias, but I
assume we'd rather not do that.)

Updates golang/go#19871

Change-Id: Ib6ce49431c8529d6b4e72725d3cd652b9d0160db
Reviewed-on: https://go-review.googlesource.com/44175
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
This commit is contained in:
Quentin Smith 2017-05-24 18:04:24 -04:00
Родитель 2ec582fede
Коммит 9a67679eab
6 изменённых файлов: 224 добавлений и 201 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

22
cmd/coordinator/benchmarks.go
Просмотреть файл

@ -6,6 +6,7 @@ package main
import ( import (
"bytes" "bytes"
"context"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -16,6 +17,7 @@ import (
"golang.org/x/build/buildlet" "golang.org/x/build/buildlet"
"golang.org/x/build/cmd/coordinator/spanlog" "golang.org/x/build/cmd/coordinator/spanlog"
"golang.org/x/build/dashboard" "golang.org/x/build/dashboard"
"golang.org/x/build/internal/buildgo"
) )
// benchRuns is the number of times to run each benchmark binary // benchRuns is the number of times to run each benchmark binary
@ -224,8 +226,8 @@ func runOneBenchBinary(conf dashboard.BuildConfig, bc *buildlet.Client, w io.Wri
} }
// parentRev returns the parent of this build's commit (but only if this build comes from a trySet). // parentRev returns the parent of this build's commit (but only if this build comes from a trySet).
func (st *buildStatus) parentRev() (pbr builderRev, err error) { func (st *buildStatus) parentRev() (pbr buildgo.BuilderRev, err error) {
pbr = st.builderRev // copy pbr = st.BuilderRev // copy
rev := st.trySet.ci.Revisions[st.trySet.ci.CurrentRevision] rev := st.trySet.ci.Revisions[st.trySet.ci.CurrentRevision]
if rev.Commit == nil { if rev.Commit == nil {
err = fmt.Errorf("commit information missing for revision %q", st.trySet.ci.CurrentRevision) err = fmt.Errorf("commit information missing for revision %q", st.trySet.ci.CurrentRevision)
@ -236,25 +238,25 @@ func (st *buildStatus) parentRev() (pbr builderRev, err error) {
err = errors.New("commit has no parent") err = errors.New("commit has no parent")
return return
} }
pbr.rev = rev.Commit.Parents[0].CommitID pbr.Rev = rev.Commit.Parents[0].CommitID
return return
} }
func (st *buildStatus) buildRev(sl spanlog.Logger, conf dashboard.BuildConfig, bc *buildlet.Client, w io.Writer, goroot string, br builderRev) error { func (st *buildStatus) buildRev(sl spanlog.Logger, conf dashboard.BuildConfig, bc *buildlet.Client, w io.Writer, goroot string, br buildgo.BuilderRev) error {
if br.snapshotExists() { if br.SnapshotExists(context.TODO(), buildEnv) {
return bc.PutTarFromURL(br.snapshotURL(), "go-parent") return bc.PutTarFromURL(br.SnapshotURL(buildEnv), goroot)
} }
if err := bc.PutTar(versionTgz(br.rev), "go-parent"); err != nil { if err := bc.PutTar(versionTgz(br.Rev), goroot); err != nil {
return err return err
} }
srcTar, err := getSourceTgz(sl, "go", br.rev) srcTar, err := getSourceTgz(sl, "go", br.Rev)
if err != nil { if err != nil {
return err return err
} }
if err := bc.PutTar(srcTar, "go-parent"); err != nil { if err := bc.PutTar(srcTar, goroot); err != nil {
return err return err
} }
remoteErr, err := st.runMake(bc, "go-parent", w) remoteErr, err := st.runMake(bc, goroot, w)
if err != nil { if err != nil {
return err return err
} }

291
cmd/coordinator/coordinator.go
Просмотреть файл

@ -54,6 +54,7 @@ import (
"golang.org/x/build/cmd/coordinator/spanlog" "golang.org/x/build/cmd/coordinator/spanlog"
"golang.org/x/build/dashboard" "golang.org/x/build/dashboard"
"golang.org/x/build/gerrit" "golang.org/x/build/gerrit"
"golang.org/x/build/internal/buildgo"
"golang.org/x/build/internal/lru" "golang.org/x/build/internal/lru"
"golang.org/x/build/internal/singleflight" "golang.org/x/build/internal/singleflight"
"golang.org/x/build/livelog" "golang.org/x/build/livelog"
@ -111,7 +112,7 @@ var (
var ( var (
statusMu sync.Mutex // guards the following four structures; see LOCK ORDER comment above statusMu sync.Mutex // guards the following four structures; see LOCK ORDER comment above
status = map[builderRev]*buildStatus{} status = map[buildgo.BuilderRev]*buildStatus{}
statusDone []*buildStatus // finished recently, capped to maxStatusDone statusDone []*buildStatus // finished recently, capped to maxStatusDone
tries = map[tryKey]*trySet{} // trybot builds tries = map[tryKey]*trySet{} // trybot builds
tryList []tryKey tryList []tryKey
@ -299,7 +300,7 @@ func main() {
} }
}() }()
workc := make(chan builderRev) workc := make(chan buildgo.BuilderRev)
if *mode == "dev" { if *mode == "dev" {
// TODO(crawshaw): do more in dev mode // TODO(crawshaw): do more in dev mode
@ -329,7 +330,7 @@ func main() {
case work := <-workc: case work := <-workc:
if !mayBuildRev(work) { if !mayBuildRev(work) {
if inStaging { if inStaging {
if _, ok := dashboard.Builders[work.name]; ok && logCantBuildStaging.Allow() { if _, ok := dashboard.Builders[work.Name]; ok && logCantBuildStaging.Allow() {
log.Printf("may not build %v; skipping", work) log.Printf("may not build %v; skipping", work)
} }
} }
@ -406,14 +407,14 @@ func numCurrentBuildsOfType(typ string) (n int) {
statusMu.Lock() statusMu.Lock()
defer statusMu.Unlock() defer statusMu.Unlock()
for rev := range status { for rev := range status {
if rev.name == typ { if rev.Name == typ {
n++ n++
} }
} }
return return
} }
func isBuilding(work builderRev) bool { func isBuilding(work buildgo.BuilderRev) bool {
statusMu.Lock() statusMu.Lock()
defer statusMu.Unlock() defer statusMu.Unlock()
_, building := status[work] _, building := status[work]
@ -428,21 +429,21 @@ var (
// mayBuildRev reports whether the build type & revision should be started. // mayBuildRev reports whether the build type & revision should be started.
// It returns true if it's not already building, and if a reverse buildlet is // It returns true if it's not already building, and if a reverse buildlet is
// required, if an appropriate machine is registered. // required, if an appropriate machine is registered.
func mayBuildRev(rev builderRev) bool { func mayBuildRev(rev buildgo.BuilderRev) bool {
if isBuilding(rev) { if isBuilding(rev) {
return false return false
} }
if buildEnv.MaxBuilds > 0 && numCurrentBuilds() >= buildEnv.MaxBuilds { if buildEnv.MaxBuilds > 0 && numCurrentBuilds() >= buildEnv.MaxBuilds {
return false return false
} }
buildConf, ok := dashboard.Builders[rev.name] buildConf, ok := dashboard.Builders[rev.Name]
if !ok { if !ok {
if logUnknownBuilder.Allow() { if logUnknownBuilder.Allow() {
log.Printf("unknown builder %q", rev.name) log.Printf("unknown builder %q", rev.Name)
} }
return false return false
} }
if buildConf.MaxAtOnce > 0 && numCurrentBuildsOfType(rev.name) >= buildConf.MaxAtOnce { if buildConf.MaxAtOnce > 0 && numCurrentBuildsOfType(rev.Name) >= buildConf.MaxAtOnce {
return false return false
} }
if buildConf.IsReverse() && !reversePool.CanBuild(buildConf.HostType) { if buildConf.IsReverse() && !reversePool.CanBuild(buildConf.HostType) {
@ -454,7 +455,7 @@ func mayBuildRev(rev builderRev) bool {
return true return true
} }
func setStatus(work builderRev, st *buildStatus) { func setStatus(work buildgo.BuilderRev, st *buildStatus) {
statusMu.Lock() statusMu.Lock()
defer statusMu.Unlock() defer statusMu.Unlock()
// TODO: panic if status[work] already exists. audit all callers. // TODO: panic if status[work] already exists. audit all callers.
@ -465,7 +466,7 @@ func setStatus(work builderRev, st *buildStatus) {
status[work] = st status[work] = st
} }
func markDone(work builderRev) { func markDone(work buildgo.BuilderRev) {
statusMu.Lock() statusMu.Lock()
defer statusMu.Unlock() defer statusMu.Unlock()
st, ok := status[work] st, ok := status[work]
@ -483,7 +484,7 @@ func markDone(work builderRev) {
// statusPtrStr disambiguates which status to return if there are // statusPtrStr disambiguates which status to return if there are
// multiple in the history (e.g. recent failures where the build // multiple in the history (e.g. recent failures where the build
// didn't finish for reasons outside of all.bash failing) // didn't finish for reasons outside of all.bash failing)
func getStatus(work builderRev, statusPtrStr string) *buildStatus { func getStatus(work buildgo.BuilderRev, statusPtrStr string) *buildStatus {
statusMu.Lock() statusMu.Lock()
defer statusMu.Unlock() defer statusMu.Unlock()
match := func(st *buildStatus) bool { match := func(st *buildStatus) bool {
@ -493,15 +494,15 @@ func getStatus(work builderRev, statusPtrStr string) *buildStatus {
return st return st
} }
for _, st := range statusDone { for _, st := range statusDone {
if st.builderRev == work && match(st) { if st.BuilderRev == work && match(st) {
return st return st
} }
} }
for k, ts := range tries { for k, ts := range tries {
if k.Commit == work.rev { if k.Commit == work.Rev {
ts.mu.Lock() ts.mu.Lock()
for _, st := range ts.builds { for _, st := range ts.builds {
if st.builderRev == work && match(st) { if st.BuilderRev == work && match(st) {
ts.mu.Unlock() ts.mu.Unlock()
return st return st
} }
@ -549,7 +550,7 @@ func handleTryStatus(w http.ResponseWriter, r *http.Request) {
} }
bs.mu.Unlock() bs.mu.Unlock()
fmt.Fprintf(w, "<tr valign=top><td align=left>%s</td><td align=center>%s</td><td><pre>%s</pre></td></tr>\n", fmt.Fprintf(w, "<tr valign=top><td align=left>%s</td><td align=center>%s</td><td><pre>%s</pre></td></tr>\n",
bs.name, bs.Name,
status, status,
bs.HTMLStatusLine()) bs.HTMLStatusLine())
} }
@ -571,11 +572,11 @@ func trySetOfCommitPrefix(commitPrefix string) *trySet {
} }
func handleLogs(w http.ResponseWriter, r *http.Request) { func handleLogs(w http.ResponseWriter, r *http.Request) {
br := builderRev{ br := buildgo.BuilderRev{
name: r.FormValue("name"), Name: r.FormValue("name"),
rev: r.FormValue("rev"), Rev: r.FormValue("rev"),
subName: r.FormValue("subName"), // may be empty SubName: r.FormValue("subName"), // may be empty
subRev: r.FormValue("subRev"), // may be empty SubRev: r.FormValue("subRev"), // may be empty
} }
st := getStatus(br, r.FormValue("st")) st := getStatus(br, r.FormValue("st"))
if st == nil { if st == nil {
@ -630,8 +631,8 @@ func handleDebugGoroutines(w http.ResponseWriter, r *http.Request) {
func writeStatusHeader(w http.ResponseWriter, st *buildStatus) { func writeStatusHeader(w http.ResponseWriter, st *buildStatus) {
st.mu.Lock() st.mu.Lock()
defer st.mu.Unlock() defer st.mu.Unlock()
fmt.Fprintf(w, " builder: %s\n", st.name) fmt.Fprintf(w, " builder: %s\n", st.Name)
fmt.Fprintf(w, " rev: %s\n", st.rev) fmt.Fprintf(w, " rev: %s\n", st.Rev)
workaroundFlush(w) workaroundFlush(w)
fmt.Fprintf(w, " buildlet: %s\n", st.bc) fmt.Fprintf(w, " buildlet: %s\n", st.bc)
fmt.Fprintf(w, " started: %v\n", st.startTime) fmt.Fprintf(w, " started: %v\n", st.startTime)
@ -662,10 +663,10 @@ func workaroundFlush(w http.ResponseWriter) {
// findWorkLoop polls https://build.golang.org/?mode=json looking for new work // findWorkLoop polls https://build.golang.org/?mode=json looking for new work
// for the main dashboard. It does not support gccgo. // for the main dashboard. It does not support gccgo.
// TODO(bradfitz): it also currently does not support subrepos. // TODO(bradfitz): it also currently does not support subrepos.
func findWorkLoop(work chan<- builderRev) { func findWorkLoop(work chan<- buildgo.BuilderRev) {
// Useful for debugging a single run: // Useful for debugging a single run:
if inStaging && false { if inStaging && false {
//work <- builderRev{name: "linux-arm", rev: "c9778ec302b2e0e0d6027e1e0fca892e428d9657", subName: "tools", subRev: "ac303766f5f240c1796eeea3dc9bf34f1261aa35"} //work <- buildgo.BuilderRev{name: "linux-arm", rev: "c9778ec302b2e0e0d6027e1e0fca892e428d9657", subName: "tools", subRev: "ac303766f5f240c1796eeea3dc9bf34f1261aa35"}
const debugArm = false const debugArm = false
if debugArm { if debugArm {
for !reversePool.CanBuild("host-linux-arm") { for !reversePool.CanBuild("host-linux-arm") {
@ -673,14 +674,14 @@ func findWorkLoop(work chan<- builderRev) {
time.Sleep(time.Second) time.Sleep(time.Second)
} }
log.Printf("ARM machine(s) registered.") log.Printf("ARM machine(s) registered.")
work <- builderRev{name: "linux-arm", rev: "3129c67db76bc8ee13a1edc38a6c25f9eddcbc6c"} work <- buildgo.BuilderRev{Name: "linux-arm", Rev: "3129c67db76bc8ee13a1edc38a6c25f9eddcbc6c"}
} else { } else {
work <- builderRev{name: "windows-amd64-2008", rev: "3129c67db76bc8ee13a1edc38a6c25f9eddcbc6c"} work <- buildgo.BuilderRev{Name: "windows-amd64-2008", Rev: "3129c67db76bc8ee13a1edc38a6c25f9eddcbc6c"}
work <- builderRev{name: "windows-386-gce", rev: "3129c67db76bc8ee13a1edc38a6c25f9eddcbc6c"} work <- buildgo.BuilderRev{Name: "windows-386-gce", Rev: "3129c67db76bc8ee13a1edc38a6c25f9eddcbc6c"}
} }
// Still run findWork but ignore what it does. // Still run findWork but ignore what it does.
ignore := make(chan builderRev) ignore := make(chan buildgo.BuilderRev)
go func() { go func() {
for range ignore { for range ignore {
} }
@ -696,7 +697,7 @@ func findWorkLoop(work chan<- builderRev) {
} }
} }
func findWork(work chan<- builderRev) error { func findWork(work chan<- buildgo.BuilderRev) error {
var bs types.BuildStatus var bs types.BuildStatus
if err := dash("GET", "", url.Values{"mode": {"json"}}, nil, &bs); err != nil { if err := dash("GET", "", url.Values{"mode": {"json"}}, nil, &bs); err != nil {
return err return err
@ -766,24 +767,24 @@ func findWork(work chan<- builderRev) error {
continue continue
} }
var rev builderRev var rev buildgo.BuilderRev
if br.Repo == "go" { if br.Repo == "go" {
rev = builderRev{ rev = buildgo.BuilderRev{
name: bs.Builders[i], Name: bs.Builders[i],
rev: br.Revision, Rev: br.Revision,
} }
} else { } else {
rev = builderRev{ rev = buildgo.BuilderRev{
name: bs.Builders[i], Name: bs.Builders[i],
rev: br.GoRevision, Rev: br.GoRevision,
subName: br.Repo, SubName: br.Repo,
subRev: br.Revision, SubRev: br.Revision,
} }
if awaitSnapshot && !rev.snapshotExists() { if awaitSnapshot && !rev.SnapshotExists(context.TODO(), buildEnv) {
continue continue
} }
} }
if rev.skipBuild() { if skipBuild(rev) {
continue continue
} }
if !isBuilding(rev) { if !isBuilding(rev) {
@ -802,8 +803,8 @@ func findWork(work chan<- builderRev) error {
continue continue
} }
for _, rev := range goRevisions { for _, rev := range goRevisions {
br := builderRev{name: b, rev: rev} br := buildgo.BuilderRev{Name: b, Rev: rev}
if !br.skipBuild() && !isBuilding(br) { if !skipBuild(br) && !isBuilding(br) {
work <- br work <- br
} }
} }
@ -987,18 +988,18 @@ func newTrySet(key tryKey, ci *gerrit.ChangeInfo) (*trySet, error) {
return ts, nil return ts, nil
} }
func tryKeyToBuilderRev(builder string, key tryKey) builderRev { func tryKeyToBuilderRev(builder string, key tryKey) buildgo.BuilderRev {
if key.Repo == "go" { if key.Repo == "go" {
return builderRev{ return buildgo.BuilderRev{
name: builder, Name: builder,
rev: key.Commit, Rev: key.Commit,
} }
} }
return builderRev{ return buildgo.BuilderRev{
name: builder, Name: builder,
rev: getRepoHead("go"), Rev: getRepoHead("go"),
subName: key.Repo, SubName: key.Repo,
subRev: key.Commit, SubRev: key.Commit,
} }
} }
@ -1108,7 +1109,7 @@ func (ts *trySet) noteBuildComplete(bconf dashboard.BuildConfig, bs *buildStatus
ts.mu.Lock() ts.mu.Lock()
if hasBenchResults { if hasBenchResults {
ts.benchResults = append(ts.benchResults, bs.name) ts.benchResults = append(ts.benchResults, bs.Name)
} }
ts.remain-- ts.remain--
remain := ts.remain remain := ts.remain
@ -1122,7 +1123,7 @@ func (ts *trySet) noteBuildComplete(bconf dashboard.BuildConfig, bs *buildStatus
if !succeeded { if !succeeded {
s1 := sha1.New() s1 := sha1.New()
io.WriteString(s1, buildLog) io.WriteString(s1, buildLog)
objName := fmt.Sprintf("%s/%s_%x.log", bs.rev[:8], bs.name, s1.Sum(nil)[:4]) objName := fmt.Sprintf("%s/%s_%x.log", bs.Rev[:8], bs.Name, s1.Sum(nil)[:4])
wr, failLogURL := newFailureLogBlob(objName) wr, failLogURL := newFailureLogBlob(objName)
if _, err := io.WriteString(wr, buildLog); err != nil { if _, err := io.WriteString(wr, buildLog); err != nil {
log.Printf("Failed to write to GCS: %v", err) log.Printf("Failed to write to GCS: %v", err)
@ -1137,7 +1138,7 @@ func (ts *trySet) noteBuildComplete(bconf dashboard.BuildConfig, bs *buildStatus
bs.failURL = failLogURL bs.failURL = failLogURL
bs.mu.Unlock() bs.mu.Unlock()
ts.mu.Lock() ts.mu.Lock()
fmt.Fprintf(&ts.errMsg, "Failed on %s: %s\n", bs.name, failLogURL) fmt.Fprintf(&ts.errMsg, "Failed on %s: %s\n", bs.Name, failLogURL)
ts.mu.Unlock() ts.mu.Unlock()
if numFail == 1 && remain > 0 { if numFail == 1 && remain > 0 {
@ -1147,7 +1148,7 @@ func (ts *trySet) noteBuildComplete(bconf dashboard.BuildConfig, bs *buildStatus
"This change failed on %s:\n"+ "This change failed on %s:\n"+
"See %s\n\n"+ "See %s\n\n"+
"Consult https://build.golang.org/ to see whether it's a new failure. Other builds still in progress; subsequent failure notices suppressed until final report.", "Consult https://build.golang.org/ to see whether it's a new failure. Other builds still in progress; subsequent failure notices suppressed until final report.",
bs.name, failLogURL), bs.Name, failLogURL),
}); err != nil { }); err != nil {
log.Printf("Failed to call Gerrit: %v", err) log.Printf("Failed to call Gerrit: %v", err)
return return
@ -1179,28 +1180,18 @@ func (ts *trySet) noteBuildComplete(bconf dashboard.BuildConfig, bs *buildStatus
} }
} }
// builderRev is a build configuration type and a revision. func skipBuild(br buildgo.BuilderRev) bool {
type builderRev struct { if strings.HasPrefix(br.Name, "netbsd-386") {
name string // e.g. "linux-amd64-race"
rev string // lowercase hex core repo git hash
// optional sub-repository details (both must be present)
subName string // e.g. "net"
subRev string // lowercase hex sub-repo git hash
}
func (br builderRev) skipBuild() bool {
if strings.HasPrefix(br.name, "netbsd-386") {
// Hangs during make.bash. TODO: remove once Issue 19339 is fixed. // Hangs during make.bash. TODO: remove once Issue 19339 is fixed.
return true return true
} }
if strings.HasPrefix(br.name, "netbsd-amd64") { if strings.HasPrefix(br.Name, "netbsd-amd64") {
// Broken and unloved. Wasting resources. // Broken and unloved. Wasting resources.
// Still available via gomote, but not building for now. // Still available via gomote, but not building for now.
// TODO: remove once Issue 19652 is fixed. // TODO: remove once Issue 19652 is fixed.
return true return true
} }
switch br.subName { switch br.SubName {
case "build", // has external deps case "build", // has external deps
"exp", // always broken, depends on mobile which is broken "exp", // always broken, depends on mobile which is broken
"mobile", // always broken (gl, etc). doesn't compile. "mobile", // always broken (gl, etc). doesn't compile.
@ -1208,13 +1199,13 @@ func (br builderRev) skipBuild() bool {
"oauth2": // has external deps "oauth2": // has external deps
return true return true
case "perf": case "perf":
if br.name == "linux-amd64-nocgo" { if br.Name == "linux-amd64-nocgo" {
// The "perf" repo requires sqlite, which // The "perf" repo requires sqlite, which
// requires cgo. Skip the no-cgo builder. // requires cgo. Skip the no-cgo builder.
return true return true
} }
case "net": case "net":
if br.name == "darwin-amd64-10_8" || br.name == "darwin-386-10_8" { if br.Name == "darwin-amd64-10_8" || br.Name == "darwin-386-10_8" {
// One of the tests seems to panic the kernel // One of the tests seems to panic the kernel
// and kill our buildlet which goes in a loop. // and kill our buildlet which goes in a loop.
return true return true
@ -1223,24 +1214,6 @@ func (br builderRev) skipBuild() bool {
return false return false
} }
func (br builderRev) isSubrepo() bool {
return br.subName != ""
}
func (br builderRev) subRevOrGoRev() string {
if br.subRev != "" {
return br.subRev
}
return br.rev
}
func (br builderRev) repoOrGo() string {
if br.subName == "" {
return "go"
}
return br.subName
}
type eventTimeLogger interface { type eventTimeLogger interface {
LogEventTime(event string, optText ...string) LogEventTime(event string, optText ...string)
} }
@ -1331,18 +1304,18 @@ func poolForConf(conf dashboard.BuildConfig) BuildletPool {
} }
} }
func newBuild(rev builderRev) (*buildStatus, error) { func newBuild(rev buildgo.BuilderRev) (*buildStatus, error) {
// Note: can't acquire statusMu in newBuild, as this is called // Note: can't acquire statusMu in newBuild, as this is called
// from findTryWork -> newTrySet, which holds statusMu. // from findTryWork -> newTrySet, which holds statusMu.
conf, ok := dashboard.Builders[rev.name] conf, ok := dashboard.Builders[rev.Name]
if !ok { if !ok {
return nil, fmt.Errorf("unknown builder type %q", rev.name) return nil, fmt.Errorf("unknown builder type %q", rev.Name)
} }
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
return &buildStatus{ return &buildStatus{
buildID: "B" + randHex(9), buildID: "B" + randHex(9),
builderRev: rev, BuilderRev: rev,
conf: conf, conf: conf,
startTime: time.Now(), startTime: time.Now(),
ctx: ctx, ctx: ctx,
@ -1354,7 +1327,7 @@ func newBuild(rev builderRev) (*buildStatus, error) {
// The buildStatus's context is closed when the build is complete, // The buildStatus's context is closed when the build is complete,
// successfully or not. // successfully or not.
func (st *buildStatus) start() { func (st *buildStatus) start() {
setStatus(st.builderRev, st) setStatus(st.BuilderRev, st)
go func() { go func() {
err := st.build() err := st.build()
if err == errSkipBuildDueToDeps { if err == errSkipBuildDueToDeps {
@ -1362,12 +1335,12 @@ func (st *buildStatus) start() {
} else { } else {
if err != nil { if err != nil {
fmt.Fprintf(st, "\n\nError: %v\n", err) fmt.Fprintf(st, "\n\nError: %v\n", err)
log.Println(st.builderRev, "failed:", err) log.Println(st.BuilderRev, "failed:", err)
} }
st.setDone(err == nil) st.setDone(err == nil)
putBuildRecord(st.buildRecord()) putBuildRecord(st.buildRecord())
} }
markDone(st.builderRev) markDone(st.BuilderRev)
}() }()
} }
@ -1424,7 +1397,7 @@ func (st *buildStatus) expectedBuildletStartDuration() time.Duration {
// ready, such that they're ready when make.bash is done. But we don't // ready, such that they're ready when make.bash is done. But we don't
// want to start too early, lest we waste idle resources during make.bash. // want to start too early, lest we waste idle resources during make.bash.
func (st *buildStatus) getHelpersReadySoon() { func (st *buildStatus) getHelpersReadySoon() {
if st.isSubrepo() || st.conf.NumTestHelpers(st.isTry()) == 0 || st.conf.IsReverse() { if st.IsSubrepo() || st.conf.NumTestHelpers(st.isTry()) == 0 || st.conf.IsReverse() {
return return
} }
time.AfterFunc(st.expectedMakeBashDuration()-st.expectedBuildletStartDuration(), time.AfterFunc(st.expectedMakeBashDuration()-st.expectedBuildletStartDuration(),
@ -1458,7 +1431,7 @@ func (st *buildStatus) useSnapshot() bool {
if st.useSnapshotMemo != nil { if st.useSnapshotMemo != nil {
return *st.useSnapshotMemo return *st.useSnapshotMemo
} }
b := st.conf.SplitMakeRun() && st.builderRev.snapshotExists() b := st.conf.SplitMakeRun() && st.BuilderRev.SnapshotExists(context.TODO(), buildEnv)
st.useSnapshotMemo = &b st.useSnapshotMemo = &b
return b return b
} }
@ -1474,7 +1447,7 @@ func (st *buildStatus) getCrossCompileConfig() *crossCompileConfig {
if kubeErr != nil { if kubeErr != nil {
return nil return nil
} }
config := crossCompileConfigs[st.name] config := crossCompileConfigs[st.Name]
if config == nil { if config == nil {
return nil return nil
} }
@ -1494,7 +1467,7 @@ func (st *buildStatus) checkDep(ctx context.Context, dep string) (have bool, err
for { for {
tries++ tries++
res, err := maintnerClient.HasAncestor(ctx, &apipb.HasAncestorRequest{ res, err := maintnerClient.HasAncestor(ctx, &apipb.HasAncestorRequest{
Commit: st.rev, Commit: st.Rev,
Ancestor: dep, Ancestor: dep,
}) })
if err != nil { if err != nil {
@ -1526,12 +1499,12 @@ func (st *buildStatus) build() error {
for _, dep := range deps { for _, dep := range deps {
has, err := st.checkDep(ctx, dep) has, err := st.checkDep(ctx, dep)
if err != nil { if err != nil {
fmt.Fprintf(st, "Error checking whether commit %s includes ancestor %s: %v\n", st.rev, dep, err) fmt.Fprintf(st, "Error checking whether commit %s includes ancestor %s: %v\n", st.Rev, dep, err)
return err return err
} }
if !has { if !has {
st.LogEventTime(eventSkipBuildMissingDep) st.LogEventTime(eventSkipBuildMissingDep)
fmt.Fprintf(st, "skipping build; commit %s lacks ancestor %s\n", st.rev, dep) fmt.Fprintf(st, "skipping build; commit %s lacks ancestor %s\n", st.Rev, dep)
return errSkipBuildDueToDeps return errSkipBuildDueToDeps
} }
} }
@ -1542,7 +1515,7 @@ func (st *buildStatus) build() error {
sp := st.CreateSpan("checking_for_snapshot") sp := st.CreateSpan("checking_for_snapshot")
if inStaging { if inStaging {
err := storageClient.Bucket(buildEnv.SnapBucket).Object(st.snapshotObjectName()).Delete(context.Background()) err := storageClient.Bucket(buildEnv.SnapBucket).Object(st.SnapshotObjectName()).Delete(context.Background())
st.LogEventTime("deleted_snapshot", fmt.Sprint(err)) st.LogEventTime("deleted_snapshot", fmt.Sprint(err))
} }
snapshotExists := st.useSnapshot() snapshotExists := st.useSnapshot()
@ -1572,7 +1545,7 @@ func (st *buildStatus) build() error {
if st.useSnapshot() { if st.useSnapshot() {
sp := st.CreateSpan("write_snapshot_tar") sp := st.CreateSpan("write_snapshot_tar")
if err := bc.PutTarFromURL(st.snapshotURL(), "go"); err != nil { if err := bc.PutTarFromURL(st.SnapshotURL(buildEnv), "go"); err != nil {
return sp.Done(fmt.Errorf("failed to put snapshot to buildlet: %v", err)) return sp.Done(fmt.Errorf("failed to put snapshot to buildlet: %v", err))
} }
sp.Done(nil) sp.Done(nil)
@ -1587,9 +1560,9 @@ func (st *buildStatus) build() error {
} }
execStartTime := time.Now() execStartTime := time.Now()
fmt.Fprintf(st, "%s at %v", st.name, st.rev) fmt.Fprintf(st, "%s at %v", st.Name, st.Rev)
if st.isSubrepo() { if st.IsSubrepo() {
fmt.Fprintf(st, " building %v at %v", st.subName, st.subRev) fmt.Fprintf(st, " building %v at %v", st.SubName, st.SubRev)
} }
fmt.Fprint(st, "\n\n") fmt.Fprint(st, "\n\n")
@ -1639,7 +1612,7 @@ func (st *buildStatus) build() error {
buildLog += "\n" + remoteErr.Error() buildLog += "\n" + remoteErr.Error()
} }
} }
if err := recordResult(st.builderRev, remoteErr == nil, buildLog, time.Since(execStartTime)); err != nil { if err := recordResult(st.BuilderRev, remoteErr == nil, buildLog, time.Since(execStartTime)); err != nil {
if remoteErr != nil { if remoteErr != nil {
return fmt.Errorf("Remote error was %q but failed to report it to the dashboard: %v", remoteErr, err) return fmt.Errorf("Remote error was %q but failed to report it to the dashboard: %v", remoteErr, err)
} }
@ -1660,10 +1633,10 @@ func (st *buildStatus) buildRecord() *types.BuildRecord {
ProcessID: processID, ProcessID: processID,
StartTime: st.startTime, StartTime: st.startTime,
IsTry: st.isTry(), IsTry: st.isTry(),
GoRev: st.rev, GoRev: st.Rev,
Rev: st.subRevOrGoRev(), Rev: st.SubRevOrGoRev(),
Repo: st.repoOrGo(), Repo: st.RepoOrGo(),
Builder: st.name, Builder: st.Name,
OS: st.conf.GOOS(), OS: st.conf.GOOS(),
Arch: st.conf.GOARCH(), Arch: st.conf.GOARCH(),
} }
@ -1688,10 +1661,10 @@ func (st *buildStatus) spanRecord(sp *span, err error) *types.SpanRecord {
rec := &types.SpanRecord{ rec := &types.SpanRecord{
BuildID: st.buildID, BuildID: st.buildID,
IsTry: st.isTry(), IsTry: st.isTry(),
GoRev: st.rev, GoRev: st.Rev,
Rev: st.subRevOrGoRev(), Rev: st.SubRevOrGoRev(),
Repo: st.repoOrGo(), Repo: st.RepoOrGo(),
Builder: st.name, Builder: st.Name,
OS: st.conf.GOOS(), OS: st.conf.GOOS(),
Arch: st.conf.GOARCH(), Arch: st.conf.GOARCH(),
@ -1712,7 +1685,7 @@ func (st *buildStatus) shouldBench() bool {
if !*shouldRunBench { if !*shouldRunBench {
return false return false
} }
return st.isTry() && !st.isSubrepo() && st.conf.RunBench return st.isTry() && !st.IsSubrepo() && st.conf.RunBench
} }
// runAllSharded runs make.bash and then shards the test execution. // runAllSharded runs make.bash and then shards the test execution.
@ -1741,7 +1714,7 @@ func (st *buildStatus) runAllSharded() (remoteErr, err error) {
return nil, err return nil, err
} }
if st.isSubrepo() { if st.IsSubrepo() {
remoteErr, err = st.runSubrepoTests() remoteErr, err = st.runSubrepoTests()
} else { } else {
remoteErr, err = st.runTests(st.getHelpers()) remoteErr, err = st.runTests(st.getHelpers())
@ -1886,7 +1859,7 @@ func (st *buildStatus) runMake(bc *buildlet.Client, goroot string, w io.Writer)
sp.Done(nil) sp.Done(nil)
} }
if st.name == "linux-amd64-racecompile" { if st.Name == "linux-amd64-racecompile" {
return st.runConcurrentGoBuildStdCmd(bc) return st.runConcurrentGoBuildStdCmd(bc)
} }
@ -1962,24 +1935,6 @@ func (st *buildStatus) doSnapshot(bc *buildlet.Client) error {
return nil return nil
} }
// snapshotExists reports whether the snapshot exists in storage.
// It returns potentially false negatives on network errors.
// Callers must not depend on this as more than an optimization.
func (br *builderRev) snapshotExists() bool {
req, err := http.NewRequest("HEAD", br.snapshotURL(), nil)
if err != nil {
panic(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
res, err := http.DefaultClient.Do(req.WithContext(ctx))
if err != nil {
log.Printf("snapshotExists check: %v", err)
return false
}
return res.StatusCode == http.StatusOK
}
func (st *buildStatus) writeGoSource() error { func (st *buildStatus) writeGoSource() error {
return st.writeGoSourceTo(st.bc) return st.writeGoSourceTo(st.bc)
} }
@ -1987,11 +1942,11 @@ func (st *buildStatus) writeGoSource() error {
func (st *buildStatus) writeGoSourceTo(bc *buildlet.Client) error { func (st *buildStatus) writeGoSourceTo(bc *buildlet.Client) error {
// Write the VERSION file. // Write the VERSION file.
sp := st.CreateSpan("write_version_tar") sp := st.CreateSpan("write_version_tar")
if err := bc.PutTar(versionTgz(st.rev), "go"); err != nil { if err := bc.PutTar(versionTgz(st.Rev), "go"); err != nil {
return sp.Done(fmt.Errorf("writing VERSION tgz: %v", err)) return sp.Done(fmt.Errorf("writing VERSION tgz: %v", err))
} }
srcTar, err := getSourceTgz(st, "go", st.rev) srcTar, err := getSourceTgz(st, "go", st.Rev)
if err != nil { if err != nil {
return err return err
} }
@ -2020,18 +1975,6 @@ func (st *buildStatus) cleanForSnapshot(bc *buildlet.Client) error {
)) ))
} }
// snapshotObjectName is the cloud storage object name of the
// built Go tree for this builder and Go rev (not the sub-repo).
// The entries inside this tarball do not begin with "go/".
func (br *builderRev) snapshotObjectName() string {
return fmt.Sprintf("%v/%v/%v.tar.gz", "go", br.name, br.rev)
}
// snapshotURL is the absolute URL of the snapshot object (see above).
func (br *builderRev) snapshotURL() string {
return buildEnv.SnapshotURL(br.name, br.rev)
}
func (st *buildStatus) writeSnapshot(bc *buildlet.Client) (err error) { func (st *buildStatus) writeSnapshot(bc *buildlet.Client) (err error) {
sp := st.CreateSpan("write_snapshot_to_gcs") sp := st.CreateSpan("write_snapshot_to_gcs")
defer func() { sp.Done(err) }() defer func() { sp.Done(err) }()
@ -2050,7 +1993,7 @@ func (st *buildStatus) writeSnapshot(bc *buildlet.Client) (err error) {
} }
defer tgz.Close() defer tgz.Close()
wr := storageClient.Bucket(buildEnv.SnapBucket).Object(st.snapshotObjectName()).NewWriter(ctx) wr := storageClient.Bucket(buildEnv.SnapBucket).Object(st.SnapshotObjectName()).NewWriter(ctx)
wr.ContentType = "application/octet-stream" wr.ContentType = "application/octet-stream"
wr.ACL = append(wr.ACL, storage.ACLRule{Entity: storage.AllUsers, Role: storage.RoleReader}) wr.ACL = append(wr.ACL, storage.ACLRule{Entity: storage.AllUsers, Role: storage.RoleReader})
if _, err := io.Copy(wr, tgz); err != nil { if _, err := io.Copy(wr, tgz); err != nil {
@ -2107,7 +2050,7 @@ func (st *buildStatus) distTestList() (names []string, remoteErr, err error) {
// only do this for slow builders running redundant tests. (That is, // only do this for slow builders running redundant tests. (That is,
// tests which have identical behavior across different ports) // tests which have identical behavior across different ports)
func (st *buildStatus) shouldSkipTest(testName string) bool { func (st *buildStatus) shouldSkipTest(testName string) bool {
if inStaging && st.name == "linux-arm" && false { if inStaging && st.Name == "linux-arm" && false {
if strings.HasPrefix(testName, "go_test:") && testName < "go_test:runtime" { if strings.HasPrefix(testName, "go_test:") && testName < "go_test:runtime" {
return true return true
} }
@ -2117,7 +2060,7 @@ func (st *buildStatus) shouldSkipTest(testName string) bool {
// Old vetall test name, before the sharding in CL 37572. // Old vetall test name, before the sharding in CL 37572.
return true return true
case "api": case "api":
return st.isTry() && st.name != "linux-amd64" return st.isTry() && st.Name != "linux-amd64"
} }
return false return false
} }
@ -2130,7 +2073,7 @@ func (st *buildStatus) newTestSet(names []string, benchmarks []*benchmarkItem) *
set.items = append(set.items, &testItem{ set.items = append(set.items, &testItem{
set: set, set: set,
name: name, name: name,
duration: testDuration(st.builderRev.name, name), duration: testDuration(st.BuilderRev.Name, name),
take: make(chan token, 1), take: make(chan token, 1),
done: make(chan token), done: make(chan token),
}) })
@ -2141,7 +2084,7 @@ func (st *buildStatus) newTestSet(names []string, benchmarks []*benchmarkItem) *
set: set, set: set,
name: name, name: name,
bench: bench, bench: bench,
duration: testDuration(st.builderRev.name, name), duration: testDuration(st.BuilderRev.Name, name),
take: make(chan token, 1), take: make(chan token, 1),
done: make(chan token), done: make(chan token),
}) })
@ -2534,7 +2477,7 @@ func fetchSubrepo(sl spanlog.Logger, bc *buildlet.Client, repo, rev string) erro
} }
func (st *buildStatus) runSubrepoTests() (remoteErr, err error) { func (st *buildStatus) runSubrepoTests() (remoteErr, err error) {
st.LogEventTime("fetching_subrepo", st.subName) st.LogEventTime("fetching_subrepo", st.SubName)
workDir, err := st.bc.WorkDir() workDir, err := st.bc.WorkDir()
if err != nil { if err != nil {
@ -2545,7 +2488,7 @@ func (st *buildStatus) runSubrepoTests() (remoteErr, err error) {
gopath := st.conf.FilePathJoin(workDir, "gopath") gopath := st.conf.FilePathJoin(workDir, "gopath")
fetched := map[string]bool{} fetched := map[string]bool{}
toFetch := []string{st.subName} toFetch := []string{st.SubName}
// fetch checks out the provided sub-repo to the buildlet's workspace. // fetch checks out the provided sub-repo to the buildlet's workspace.
fetch := func(repo, rev string) error { fetch := func(repo, rev string) error {
@ -2602,7 +2545,7 @@ func (st *buildStatus) runSubrepoTests() (remoteErr, err error) {
} }
// For the repo under test, choose that specific revision. // For the repo under test, choose that specific revision.
if i == 0 { if i == 0 {
rev = st.subRev rev = st.SubRev
} }
if err := fetch(repo, rev); err != nil { if err := fetch(repo, rev); err != nil {
return nil, err return nil, err
@ -2616,7 +2559,7 @@ func (st *buildStatus) runSubrepoTests() (remoteErr, err error) {
} }
} }
sp := st.CreateSpan("running_subrepo_tests", st.subName) sp := st.CreateSpan("running_subrepo_tests", st.SubName)
defer func() { sp.Done(err) }() defer func() { sp.Done(err) }()
return st.bc.Exec(path.Join("go", "bin", "go"), buildlet.ExecOpts{ return st.bc.Exec(path.Join("go", "bin", "go"), buildlet.ExecOpts{
Output: st, Output: st,
@ -2626,7 +2569,7 @@ func (st *buildStatus) runSubrepoTests() (remoteErr, err error) {
"GOPATH="+gopath, "GOPATH="+gopath,
"GO15VENDOREXPERIMENT=1"), "GO15VENDOREXPERIMENT=1"),
Path: []string{"$WORKDIR/go/bin", "$PATH"}, Path: []string{"$WORKDIR/go/bin", "$PATH"},
Args: []string{"test", "-short", subrepoPrefix + st.subName + "/..."}, Args: []string{"test", "-short", subrepoPrefix + st.SubName + "/..."},
}) })
} }
@ -2700,7 +2643,7 @@ func (st *buildStatus) runTests(helpers <-chan *buildlet.Client) (remoteErr, err
defer st.LogEventTime("DEV_HELPER_SLEEP", bc.Name()) defer st.LogEventTime("DEV_HELPER_SLEEP", bc.Name())
} }
st.LogEventTime("got_empty_test_helper", bc.String()) st.LogEventTime("got_empty_test_helper", bc.String())
if err := bc.PutTarFromURL(st.snapshotURL(), "go"); err != nil { if err := bc.PutTarFromURL(st.SnapshotURL(buildEnv), "go"); err != nil {
log.Printf("failed to extract snapshot for helper %s: %v", bc.Name(), err) log.Printf("failed to extract snapshot for helper %s: %v", bc.Name(), err)
return return
} }
@ -2846,14 +2789,14 @@ func (st *buildStatus) benchFiles() []*benchFile {
} }
fmt.Fprintf(&benchFiles[0].out, "cl: %d\nps: %d\ntry: %s\nbuildlet: %s\nbranch: %s\nrepo: https://go.googlesource.com/%s\n", fmt.Fprintf(&benchFiles[0].out, "cl: %d\nps: %d\ntry: %s\nbuildlet: %s\nbranch: %s\nrepo: https://go.googlesource.com/%s\n",
st.trySet.ci.ChangeNumber, ps, st.trySet.tryID, st.trySet.ci.ChangeNumber, ps, st.trySet.tryID,
st.name, st.trySet.ci.Branch, st.trySet.ci.Project, st.Name, st.trySet.ci.Branch, st.trySet.ci.Project,
) )
if inStaging { if inStaging {
benchFiles[0].out.WriteString("staging: true\n") benchFiles[0].out.WriteString("staging: true\n")
} }
benchFiles[1].out.Write(benchFiles[0].out.Bytes()) benchFiles[1].out.Write(benchFiles[0].out.Bytes())
fmt.Fprintf(&benchFiles[0].out, "commit: %s\n", rev.Commit.Parents[0].CommitID) fmt.Fprintf(&benchFiles[0].out, "commit: %s\n", rev.Commit.Parents[0].CommitID)
fmt.Fprintf(&benchFiles[1].out, "commit: %s\n", st.builderRev.rev) fmt.Fprintf(&benchFiles[1].out, "commit: %s\n", st.BuilderRev.Rev)
return benchFiles return benchFiles
} }
@ -3037,7 +2980,7 @@ func (s *testSet) initInOrder() {
// First do the go_test:* ones. partitionGoTests // First do the go_test:* ones. partitionGoTests
// only returns those, which are the ones we merge together. // only returns those, which are the ones we merge together.
stdSets := partitionGoTests(s.st.builderRev.name, names) stdSets := partitionGoTests(s.st.BuilderRev.Name, names)
for _, set := range stdSets { for _, set := range stdSets {
tis := make([]*testItem, len(set)) tis := make([]*testItem, len(set))
for i, name := range set { for i, name := range set {
@ -3135,7 +3078,7 @@ type eventAndTime struct {
// buildStatus is the status of a build. // buildStatus is the status of a build.
type buildStatus struct { type buildStatus struct {
// Immutable: // Immutable:
builderRev buildgo.BuilderRev
buildID string // "B" + 9 random hex buildID string // "B" + 9 random hex
conf dashboard.BuildConfig conf dashboard.BuildConfig
startTime time.Time // actually time of newBuild (~same thing); TODO(bradfitz): rename this createTime startTime time.Time // actually time of newBuild (~same thing); TODO(bradfitz): rename this createTime
@ -3179,7 +3122,7 @@ func (st *buildStatus) isRunning() bool {
func (st *buildStatus) isRunningLocked() bool { return st.done.IsZero() } func (st *buildStatus) isRunningLocked() bool { return st.done.IsZero() }
func (st *buildStatus) logf(format string, args ...interface{}) { func (st *buildStatus) logf(format string, args ...interface{}) {
log.Printf("[build %s %s]: %s", st.name, st.rev, fmt.Sprintf(format, args...)) log.Printf("[build %s %s]: %s", st.Name, st.Rev, fmt.Sprintf(format, args...))
} }
// span is an event covering a region of time. // span is an event covering a region of time.
@ -3292,10 +3235,10 @@ func (st *buildStatus) htmlStatusLine(full bool) template.HTML {
var buf bytes.Buffer var buf bytes.Buffer
fmt.Fprintf(&buf, "<a href='https://github.com/golang/go/wiki/DashboardBuilders'>%s</a> rev <a href='%s%s'>%s</a>", fmt.Fprintf(&buf, "<a href='https://github.com/golang/go/wiki/DashboardBuilders'>%s</a> rev <a href='%s%s'>%s</a>",
st.name, urlPrefix, st.rev, st.rev[:8]) st.Name, urlPrefix, st.Rev, st.Rev[:8])
if st.isSubrepo() { if st.IsSubrepo() {
fmt.Fprintf(&buf, " (sub-repo %s rev <a href='%s%s'>%s</a>)", fmt.Fprintf(&buf, " (sub-repo %s rev <a href='%s%s'>%s</a>)",
st.subName, urlPrefix, st.subRev, st.subRev[:8]) st.SubName, urlPrefix, st.SubRev, st.SubRev[:8])
} }
if ts := st.trySet; ts != nil { if ts := st.trySet; ts != nil {
fmt.Fprintf(&buf, " (<a href='/try?commit=%v'>trybot set</a> for <a href='https://go-review.googlesource.com/#/q/%s'>%s</a>)", fmt.Fprintf(&buf, " (<a href='/try?commit=%v'>trybot set</a> for <a href='https://go-review.googlesource.com/#/q/%s'>%s</a>)",
@ -3339,9 +3282,9 @@ func (st *buildStatus) logsURLLocked() string {
if *mode == "dev" { if *mode == "dev" {
urlPrefix = "https://localhost:8119" urlPrefix = "https://localhost:8119"
} }
u := fmt.Sprintf("%v/temporarylogs?name=%s&rev=%s&st=%p", urlPrefix, st.name, st.rev, st) u := fmt.Sprintf("%v/temporarylogs?name=%s&rev=%s&st=%p", urlPrefix, st.Name, st.Rev, st)
if st.isSubrepo() { if st.IsSubrepo() {
u += fmt.Sprintf("&subName=%v&subRev=%v", st.subName, st.subRev) u += fmt.Sprintf("&subName=%v&subRev=%v", st.SubName, st.SubRev)
} }
return u return u
} }

30
cmd/coordinator/dash.go
Просмотреть файл

@ -22,6 +22,8 @@ import (
"sync" "sync"
"time" "time"
"golang.org/x/build/internal/buildgo"
"cloud.google.com/go/compute/metadata" "cloud.google.com/go/compute/metadata"
) )
@ -90,22 +92,22 @@ func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
// recordResult sends build results to the dashboard. // recordResult sends build results to the dashboard.
// This is not used for trybot failures; only failures after commit. // This is not used for trybot failures; only failures after commit.
// The URLs end up looking like https://build.golang.org/log/$HEXDIGEST // The URLs end up looking like https://build.golang.org/log/$HEXDIGEST
func recordResult(br builderRev, ok bool, buildLog string, runTime time.Duration) error { func recordResult(br buildgo.BuilderRev, ok bool, buildLog string, runTime time.Duration) error {
req := map[string]interface{}{ req := map[string]interface{}{
"Builder": br.name, "Builder": br.Name,
"PackagePath": "", "PackagePath": "",
"Hash": br.rev, "Hash": br.Rev,
"GoHash": "", "GoHash": "",
"OK": ok, "OK": ok,
"Log": buildLog, "Log": buildLog,
"RunTime": runTime, "RunTime": runTime,
} }
if br.isSubrepo() { if br.IsSubrepo() {
req["PackagePath"] = subrepoPrefix + br.subName req["PackagePath"] = subrepoPrefix + br.SubName
req["Hash"] = br.subRev req["Hash"] = br.SubRev
req["GoHash"] = br.rev req["GoHash"] = br.Rev
} }
args := url.Values{"key": {builderKey(br.name)}, "builder": {br.name}} args := url.Values{"key": {builderKey(br.Name)}, "builder": {br.Name}}
if *mode == "dev" { if *mode == "dev" {
log.Printf("In dev mode, not recording result: %v", req) log.Printf("In dev mode, not recording result: %v", req)
return nil return nil
@ -138,14 +140,14 @@ func (st *buildStatus) pingDashboard() {
logsURL := st.logsURLLocked() logsURL := st.logsURLLocked()
st.mu.Unlock() st.mu.Unlock()
args := url.Values{ args := url.Values{
"builder": []string{st.name}, "builder": []string{st.Name},
"key": []string{builderKey(st.name)}, "key": []string{builderKey(st.Name)},
"hash": []string{st.rev}, "hash": []string{st.Rev},
"url": []string{logsURL}, "url": []string{logsURL},
} }
if st.isSubrepo() { if st.IsSubrepo() {
args.Set("hash", st.subRev) args.Set("hash", st.SubRev)
args.Set("gohash", st.rev) args.Set("gohash", st.Rev)
} }
u := buildEnv.DashBase() + "building?" + args.Encode() u := buildEnv.DashBase() + "building?" + args.Encode()
for { for {

5
cmd/coordinator/debug.go
Просмотреть файл

@ -15,13 +15,14 @@ import (
"strings" "strings"
"text/template" "text/template"
"golang.org/x/build/internal/buildgo"
"golang.org/x/build/types" "golang.org/x/build/types"
) )
// handleDoSomeWork adds the last committed CL as work to do. // handleDoSomeWork adds the last committed CL as work to do.
// //
// Only available in dev mode. // Only available in dev mode.
func handleDoSomeWork(work chan<- builderRev) func(w http.ResponseWriter, r *http.Request) { func handleDoSomeWork(work chan<- buildgo.BuilderRev) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" { if r.Method == "GET" {
w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Header().Set("Content-Type", "text/html; charset=utf-8")
@ -63,7 +64,7 @@ func handleDoSomeWork(work chan<- builderRev) func(w http.ResponseWriter, r *htt
} }
fmt.Fprintf(w, "found work: %v\n", revs) fmt.Fprintf(w, "found work: %v\n", revs)
for _, rev := range revs { for _, rev := range revs {
work <- builderRev{name: mode, rev: rev} work <- buildgo.BuilderRev{Name: mode, Rev: rev}
} }
} }
} }

2
cmd/coordinator/status.go
Просмотреть файл

@ -56,7 +56,7 @@ func handleStatus(w http.ResponseWriter, r *http.Request) {
key.ChangeTriple(), key.Commit, key.Commit[:8]) key.ChangeTriple(), key.Commit, key.Commit[:8])
fmt.Fprintf(&buf, " Remain: %d, fails: %v\n", state.remain, state.failed) fmt.Fprintf(&buf, " Remain: %d, fails: %v\n", state.remain, state.failed)
for _, bs := range ts.builds { for _, bs := range ts.builds {
fmt.Fprintf(&buf, " %s: running=%v\n", bs.name, bs.isRunning()) fmt.Fprintf(&buf, " %s: running=%v\n", bs.Name, bs.isRunning())
} }
} }
} }

75
internal/buildgo/buildgo.go Normal file
Просмотреть файл

@ -0,0 +1,75 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package buildgo provides tools for pushing and building the Go
// distribution on buildlets.
package buildgo
import (
"context"
"fmt"
"log"
"net/http"
"time"
"golang.org/x/build/buildenv"
)
// BuilderRev is a build configuration type and a revision.
type BuilderRev struct {
Name string // e.g. "linux-amd64-race"
Rev string // lowercase hex core repo git hash
// optional sub-repository details (both must be present)
SubName string // e.g. "net"
SubRev string // lowercase hex sub-repo git hash
}
func (br BuilderRev) IsSubrepo() bool {
return br.SubName != ""
}
func (br BuilderRev) SubRevOrGoRev() string {
if br.SubRev != "" {
return br.SubRev
}
return br.Rev
}
func (br BuilderRev) RepoOrGo() string {
if br.SubName == "" {
return "go"
}
return br.SubName
}
// SnapshotObjectName is the cloud storage object name of the
// built Go tree for this builder and Go rev (not the sub-repo).
// The entries inside this tarball do not begin with "go/".
func (br *BuilderRev) SnapshotObjectName() string {
return fmt.Sprintf("%v/%v/%v.tar.gz", "go", br.Name, br.Rev)
}
// SnapshotURL is the absolute URL of the snapshot object (see above).
func (br *BuilderRev) SnapshotURL(buildEnv *buildenv.Environment) string {
return buildEnv.SnapshotURL(br.Name, br.Rev)
}
// snapshotExists reports whether the snapshot exists in storage.
// It returns potentially false negatives on network errors.
// Callers must not depend on this as more than an optimization.
func (br *BuilderRev) SnapshotExists(ctx context.Context, buildEnv *buildenv.Environment) bool {
req, err := http.NewRequest("HEAD", br.SnapshotURL(buildEnv), nil)
if err != nil {
panic(err)
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
res, err := http.DefaultClient.Do(req.WithContext(ctx))
if err != nil {
log.Printf("SnapshotExists check: %v", err)
return false
}
return res.StatusCode == http.StatusOK
}