Bug 863061 - Ignore percentage_threshold for libxul linker memory [r=catlee]

treeherder/perfalert/perfalert/analysis.cfg.template

@@ -20,6 +20,9 @@ threshold = 9
 # What percentage difference is required to consider a change significant
 percentage_threshold = 2
 
+# Ignore percentage_threshold for these tests (comma-separated list of regexps)
+ignore_percentage_tests = LibXUL Memory during link
+
 # What t score is required to consider a machine to be bad
 machine_threshold = 15
 

treeherder/perfalert/perfalert/analyze_talos.py

@@ -384,13 +384,19 @@ class AnalysisRunner:
                 self.bug_cache[bug_num] = bug
                 return bug
 
-    def isTestReversed(self, test_name):
-        reversed_tests = []
-        if self.config.has_option('main', 'reverse_tests'):
-            for i in self.config.get('main', 'reverse_tests').split(','):
-                reversed_tests.append(i.strip())
+    def ignorePercentageForTest(self, test_name):
+        return self.testMatchesOption(test_name, 'ignore_percentage_tests')
 
-        for text_exp in reversed_tests:
+    def isTestReversed(self, test_name):
+        return self.testMatchesOption(test_name, 'reverse_tests')
+
+    def testMatchesOption(self, test_name, option):
+        patterns = []
+        if self.config.has_option('main', option):
+            for i in self.config.get('main', option).split(','):
+                patterns.append(i.strip())
+
+        for text_exp in patterns:
             if re.search(text_exp, test_name, re.I):
                 return True
         return False
@@ -572,17 +578,24 @@ class AnalysisRunner:
             self.output.write("\n")
             self.output.flush()
 
+    def shouldSendWarning(self, d, test_name):
+        # Don't email if the percentage change is under the threshold
+        initial_value = d.historical_stats['avg']
+        new_value = d.forward_stats['avg']
+        if self.config.has_option('main', 'percentage_threshold') and \
+                initial_value != 0 and \
+                not self.ignorePercentageForTest(test_name):
+            change = 100.0 * abs(new_value - initial_value) / float(initial_value)
+            if change < self.config.getfloat('main', 'percentage_threshold'):
+                return False
+        return True
+
     def emailWarning(self, series, d, state, last_good):
         addresses = []
         branch = series.branch_name
 
-        # Don't email if the percentage change is under the threshold
-        initial_value = d.historical_stats['avg']
-        new_value = d.forward_stats['avg']
-        if self.config.has_option('main', 'percentage_threshold') and initial_value != 0:
-            change = 100.0 * abs(new_value - initial_value) / float(initial_value)
-            if change < self.config.getfloat('main', 'percentage_threshold'):
-                return
+        if not self.shouldSendWarning(d, series.test_name):
+            return
 
         if state == 'regression':
             if self.config.has_option(branch, 'regression_emails'):

treeherder/perfalert/perfalert/test_analyze_talos.py

@@ -8,12 +8,8 @@ from analyze_talos import *
 from ConfigParser import RawConfigParser
 from time import time
 
-TEST_CONFIG = """
-base_hg_url = http://example.com
-"""
-
 class TestAnalysisRunner(unittest.TestCase):
-    def get_config(self):
+    def create_runner(self):
         options, args = parse_options(['--start-time', '0'])
         config = get_config(options)
         config.set('main', 'fore_window', '5')
@@ -22,7 +18,9 @@ class TestAnalysisRunner(unittest.TestCase):
         config.set('main', 'percentage_threshold', '9')
         config.set('main', 'machine_threshold', '9')
         config.set('main', 'machine_history_size', '0')
-        return options, config
+        config.set('main', 'ignore_percentage_tests', 'LibXUL Memory.*')
+        config.set('main', 'reverse_tests', 'Dromaeo.*, V8 version 7.*')
+        return AnalysisRunner(options, config)
 
     def get_data(self):
         return [
@@ -37,8 +35,7 @@ class TestAnalysisRunner(unittest.TestCase):
         ]
 
     def test_processSeries(self):
-        options, config = self.get_config()
-        runner = AnalysisRunner(options, config)
+        runner = self.create_runner()
 
         data = self.get_data()
         results = runner.processSeries(data, [])
@@ -51,6 +48,51 @@ class TestAnalysisRunner(unittest.TestCase):
         self.assertEqual(results[4], (data[4], False, data[3]))
         self.assertEqual(results[5], (data[5], False, data[5]))
 
+    def test_isTestReversed(self):
+        runner = self.create_runner()
+
+        self.assertTrue(runner.isTestReversed('x Dromaeo'))
+        self.assertTrue(runner.isTestReversed('Dromaeo x'))
+        self.assertTrue(runner.isTestReversed('V8 version 7'))
+        self.assertTrue(runner.isTestReversed('V8 version 7.1'))
+
+        self.assertFalse(runner.isTestReversed('V8'))
+        self.assertFalse(runner.isTestReversed('some other test'))
+
+    def test_ignorePercentageForTest(self):
+        runner = self.create_runner()
+
+        self.assertTrue(runner.ignorePercentageForTest('LibXUL Memory during link'))
+        self.assertTrue(runner.ignorePercentageForTest('LibXUL Memory'))
+
+        self.assertFalse(runner.ignorePercentageForTest('LibXUL something else'))
+        self.assertFalse(runner.ignorePercentageForTest('V8'))
+
+    def test_shouldSendWarning(self):
+        runner = self.create_runner()
+        d = PerfDatum(0, 0, time() + 0, 0.0, 0, 0)
+        d.historical_stats = { 'avg': 100.0 }
+
+        # 1% increase
+        d.forward_stats = { 'avg': 101.0 }
+        self.assertFalse(runner.shouldSendWarning(d, 'some test'))
+
+        # 10% increase
+        d.forward_stats = { 'avg': 110.0 }
+        self.assertTrue(runner.shouldSendWarning(d, 'some test'))
+
+        # 1% decrease
+        d.forward_stats = { 'avg': 99.0 }
+        self.assertFalse(runner.shouldSendWarning(d, 'some test'))
+
+        # 10% decrease
+        d.forward_stats = { 'avg': 90.0 }
+        self.assertTrue(runner.shouldSendWarning(d, 'some test'))
+
+        # 1% increase, ignore percentage
+        d.forward_stats = { 'avg': 101.0 }
+        self.assertTrue(runner.shouldSendWarning(d, 'LibXUL Memory during link'))
+
 
 if __name__ == '__main__':
     unittest.main()