Bug 858735 (4/4) - Correctly process t-tests for series with zero variance [r=catlee]

treeherder/perfalert/perfalert/analyze.py

@@ -14,11 +14,14 @@ def calc_t(w1, w2):
 
     s1 = analyze(w1)
     s2 = analyze(w2)
+    delta_s = s2['avg'] - s1['avg']
 
-    if s1['variance'] == 0 and s2['variance'] == 0:
+    if delta_s == 0:
         return 0
+    if s1['variance'] == 0 and s2['variance'] == 0:
+        return float('inf')
 
-    return (s2['avg'] - s1['avg']) / (((s1['variance'] / s1['n']) + (s2['variance'] / s2['n'])) ** 0.5)
+    return delta_s / (((s1['variance'] / s1['n']) + (s2['variance'] / s2['n'])) ** 0.5)
 
 class PerfDatum(object):
     __slots__ = ('testrun_id', 'machine_id', 'timestamp', 'value', 'buildid',

treeherder/perfalert/perfalert/test_analyze.py

@@ -15,6 +15,8 @@ class TestAnalyze(unittest.TestCase):
 
     def test_calc_t(self):
         self.assertEqual(calc_t([0.0, 0.0], [1.0, 2.0]), 3.0)
+        self.assertEqual(calc_t([0.0, 0.0], [0.0, 0.0]), 0.0)
+        self.assertEqual(calc_t([0.0, 0.0], [1.0, 1.0]), float('inf'))
 
 class TestTalosAnalyzer(unittest.TestCase):
     def get_data(self):
@@ -38,7 +40,7 @@ class TestTalosAnalyzer(unittest.TestCase):
             (5, 'good'),
             (6, 'regression'),
             (7, 'regression'),
-            (8, 'good'),
+            (8, 'regression'),
             (9, 'regression'),
             (10, 'regression'),
             (11, 'good')])