Decrease noise by 50% by removing outliners. Also increased change to see real regression, by decreasing the noise multiplier

detector/regression_detector.py

@@ -29,6 +29,23 @@ def regressed(score):
   if abs(change) <= score.noise():
     return False
 
+  # Don't report outliner
+  if score.next() is None:
+    return None
+  prevs = score.avg_prevs()
+  nexts = score.next().avg_nexts()
+  if prevs is None or nexts is None:
+    return None
+  if abs(prevs-nexts) <= score.noise():
+    return False
+  if score.prev() is not None:
+    prevs = score.prev().avg_prevs()
+    nexts = score.avg_nexts()
+    if prevs is None or nexts is None:
+      return None
+    if abs(prevs-nexts) <= score.noise():
+      return False
+
   # average change over multiple runs.
   change = score.avg_change()
 

detector/tables.py

@@ -424,6 +424,14 @@ class RegressionTools(DBTable):
         nexts.append(point)
     return nexts
 
+  def avg_prevs(self):
+    avg_prevs, _ = self.avg_prevs_nexts()
+    return avg_prevs
+
+  def avg_nexts(self):
+    _, avg_nexts = self.avg_prevs_nexts()
+    return avg_nexts
+
   def avg_prevs_nexts(self):
     self.initialize()
     if "avg_prevs" not in self.cached:
@@ -538,7 +546,7 @@ class Score(RegressionTools):
     noise = RegressionScoreNoise(self.get('build').get('run').get('machine'),
                                  self.get('suite_version'),
                                  self.get('build').get('mode')).get('noise')
-    return 2.2*noise
+    return 2.0*noise
 
   @classmethod
   def first(class_, machine, suite, mode):
@@ -657,7 +665,7 @@ class Breakdown(RegressionTools):
     noise = RegressionBreakdownNoise(self.get('build').get('run').get('machine'),
                                      self.get('suite_test'),
                                      self.get('build').get('mode')).get('noise')
-    return 2.2*noise
+    return 2.0*noise
 
   def dump(self):
     import datetime

detector/tests.py

@@ -17,7 +17,7 @@ def testRuns():
              WHERE stamp > 1428624000 AND                                       \
                    status = 1 AND                                               \
                    detector = 1 AND                                             \
-                   machine in (28)")
+                   machine in (28, 29)")
   runs = []
   for row in c.fetchall():
     runs.append(tables.Run(row[0]))
@@ -53,6 +53,8 @@ for run in testRuns():
             status_now = "regressed"
     
         key = status_db+"-"+status_now
+        changes["db_"+status_db] += 1
+        changes["now_"+status_now] += 1
         changes[key] += 1  
         
         if key == "regressed-noregression" or key == "noregression-regressed":
@@ -70,5 +72,5 @@ print "Lost detections!:", changes["regressed-noregression"]
 print "Over active detection: ", changes["noregression-regressed"]
 print "% less detections: ", 1.0*int(changes["marked noregression-noregression"])/(int(changes["marked noregression-regressed"])+int(changes["marked noregression-noregression"]))
 
-
-
+print "Db regressions: ", int(changes["db_marked noregression"]) + int(changes["db_regressed"])
+print "Now regressions: ", int(changes["now_regressed"])