cmd/coordinator: increase snapshot timeout from 1 to 5 minutes

Change-Id: I6230fdc5bbbfdaac9a42bad03350055fc6dbb97b
Reviewed-on: https://go-review.googlesource.com/37468
Reviewed-by: Kevin Burke <kev@inburke.com>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

cmd/coordinator/coordinator.go

@@ -1930,7 +1930,11 @@ func (br *builderRev) snapshotURL() string {
 func (st *buildStatus) writeSnapshot(bc *buildlet.Client) (err error) {
 	sp := st.createSpan("write_snapshot_to_gcs")
 	defer func() { sp.done(err) }()
-	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	// This should happen in 15 seconds or so, but I saw timeouts
+	// a couple times at 1 minute. Some buildlets might be far
+	// away on the network, so be more lenient. The timeout mostly
+	// is here to prevent infinite hangs.
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
 
 	tsp := st.createSpan("fetch_snapshot_reader_from_buildlet")