dashboard/app: eliminate more flakes

Currently we still see some flakes on perf dashboard (e.g. sys-stack is quite frequent).
I am planning to run real perf builder with 5 different values of GOMAXPROCS,
so we can require 3 builders to agree on a change instead of 2.
This will provide 20x improvement in flake detection.
At the same time lower noise bar from 1.2 to 1.1, as I see some real perf changes gets ignored as noise.

All these magic numbers affect only representation of data, but not the data stored in DB.
So we can continue freely tuning them later. There are no significant risks here.

LGTM=adg
R=adg
CC=golang-codereviews
https://golang.org/cl/127520044

app/build/perf.go

@@ -114,14 +114,12 @@ func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) {
 			}
 		}
 	}
-	//rc.c.Errorf("PerfResultCache.Next: num=%v next=%v", commitNum, next)
 	if next != -1 {
 		return rc.results[next], nil
 	}
 	// Fetch next result from datastore.
 	res := new(PerfResult)
 	_, err := rc.iter.Next(res)
-	//rc.c.Errorf("PerfResultCache.Next: fetched %v %+v", err, res)
 	if err == datastore.Done {
 		return nil, nil
 	}
@@ -218,25 +216,28 @@ func significantPerfChanges(pc *PerfConfig, builder string, prevRes, res *PerfRe
 		}
 	}
 	// Then, strip non-repeatable changes (flakes).
-	// The hypothesis is that a real change must show up with at least
-	// 2 different values of GOMAXPROCS.
+	// The hypothesis is that a real change must show up with the majority of GOMAXPROCS values.
+	majority := len(pc.ProcList(builder))/2 + 1
 	cnt := make(map[string]int)
 	for _, ch := range changes {
 		b, _ := splitBench(ch.bench)
 		name := b + "|" + ch.metric
-		inc := 1
 		if ch.diff < 0 {
-			inc = -1
+			name += "--"
 		}
-		cnt[name] = cnt[name] + inc
+		cnt[name] = cnt[name] + 1
 	}
 	for i := 0; i < len(changes); i++ {
 		ch := changes[i]
 		b, _ := splitBench(ch.bench)
 		name := b + "|" + ch.metric
-		if n := cnt[name]; n <= -2 || n >= 2 {
+		if cnt[name] >= majority {
 			continue
 		}
+		if cnt[name+"--"] >= majority {
+			continue
+		}
+		// Remove flake.
 		last := len(changes) - 1
 		changes[i] = changes[last]
 		changes = changes[:last]

app/build/perf_learn.go

@@ -23,7 +23,7 @@ func init() {
 
 const (
 	learnPercentile       = 0.95
-	learnSignalMultiplier = 1.2
+	learnSignalMultiplier = 1.1
 	learnMinSignal        = 0.5
 )