diff --git a/treeherder/perfalert/perfalert/analysis.cfg.template b/treeherder/perfalert/perfalert/analysis.cfg.template index b1d1ad5da..313367c40 100644 --- a/treeherder/perfalert/perfalert/analysis.cfg.template +++ b/treeherder/perfalert/perfalert/analysis.cfg.template @@ -20,6 +20,9 @@ threshold = 9 # What percentage difference is required to consider a change significant percentage_threshold = 2 +# Ignore percentage_threshold for these tests (comma-separated list of regexps) +ignore_percentage_tests = LibXUL Memory during link + # What t score is required to consider a machine to be bad machine_threshold = 15 diff --git a/treeherder/perfalert/perfalert/analyze_talos.py b/treeherder/perfalert/perfalert/analyze_talos.py index 773c15f72..f5b044fea 100644 --- a/treeherder/perfalert/perfalert/analyze_talos.py +++ b/treeherder/perfalert/perfalert/analyze_talos.py @@ -384,13 +384,19 @@ class AnalysisRunner: self.bug_cache[bug_num] = bug return bug - def isTestReversed(self, test_name): - reversed_tests = [] - if self.config.has_option('main', 'reverse_tests'): - for i in self.config.get('main', 'reverse_tests').split(','): - reversed_tests.append(i.strip()) + def ignorePercentageForTest(self, test_name): + return self.testMatchesOption(test_name, 'ignore_percentage_tests') - for text_exp in reversed_tests: + def isTestReversed(self, test_name): + return self.testMatchesOption(test_name, 'reverse_tests') + + def testMatchesOption(self, test_name, option): + patterns = [] + if self.config.has_option('main', option): + for i in self.config.get('main', option).split(','): + patterns.append(i.strip()) + + for text_exp in patterns: if re.search(text_exp, test_name, re.I): return True return False @@ -572,17 +578,24 @@ class AnalysisRunner: self.output.write("\n") self.output.flush() + def shouldSendWarning(self, d, test_name): + # Don't email if the percentage change is under the threshold + initial_value = d.historical_stats['avg'] + new_value = d.forward_stats['avg'] + if self.config.has_option('main', 'percentage_threshold') and \ + initial_value != 0 and \ + not self.ignorePercentageForTest(test_name): + change = 100.0 * abs(new_value - initial_value) / float(initial_value) + if change < self.config.getfloat('main', 'percentage_threshold'): + return False + return True + def emailWarning(self, series, d, state, last_good): addresses = [] branch = series.branch_name - # Don't email if the percentage change is under the threshold - initial_value = d.historical_stats['avg'] - new_value = d.forward_stats['avg'] - if self.config.has_option('main', 'percentage_threshold') and initial_value != 0: - change = 100.0 * abs(new_value - initial_value) / float(initial_value) - if change < self.config.getfloat('main', 'percentage_threshold'): - return + if not self.shouldSendWarning(d, series.test_name): + return if state == 'regression': if self.config.has_option(branch, 'regression_emails'): diff --git a/treeherder/perfalert/perfalert/test_analyze_talos.py b/treeherder/perfalert/perfalert/test_analyze_talos.py index 43353018d..012604549 100644 --- a/treeherder/perfalert/perfalert/test_analyze_talos.py +++ b/treeherder/perfalert/perfalert/test_analyze_talos.py @@ -8,12 +8,8 @@ from analyze_talos import * from ConfigParser import RawConfigParser from time import time -TEST_CONFIG = """ -base_hg_url = http://example.com -""" - class TestAnalysisRunner(unittest.TestCase): - def get_config(self): + def create_runner(self): options, args = parse_options(['--start-time', '0']) config = get_config(options) config.set('main', 'fore_window', '5') @@ -22,7 +18,9 @@ class TestAnalysisRunner(unittest.TestCase): config.set('main', 'percentage_threshold', '9') config.set('main', 'machine_threshold', '9') config.set('main', 'machine_history_size', '0') - return options, config + config.set('main', 'ignore_percentage_tests', 'LibXUL Memory.*') + config.set('main', 'reverse_tests', 'Dromaeo.*, V8 version 7.*') + return AnalysisRunner(options, config) def get_data(self): return [ @@ -37,8 +35,7 @@ class TestAnalysisRunner(unittest.TestCase): ] def test_processSeries(self): - options, config = self.get_config() - runner = AnalysisRunner(options, config) + runner = self.create_runner() data = self.get_data() results = runner.processSeries(data, []) @@ -51,6 +48,51 @@ class TestAnalysisRunner(unittest.TestCase): self.assertEqual(results[4], (data[4], False, data[3])) self.assertEqual(results[5], (data[5], False, data[5])) + def test_isTestReversed(self): + runner = self.create_runner() + + self.assertTrue(runner.isTestReversed('x Dromaeo')) + self.assertTrue(runner.isTestReversed('Dromaeo x')) + self.assertTrue(runner.isTestReversed('V8 version 7')) + self.assertTrue(runner.isTestReversed('V8 version 7.1')) + + self.assertFalse(runner.isTestReversed('V8')) + self.assertFalse(runner.isTestReversed('some other test')) + + def test_ignorePercentageForTest(self): + runner = self.create_runner() + + self.assertTrue(runner.ignorePercentageForTest('LibXUL Memory during link')) + self.assertTrue(runner.ignorePercentageForTest('LibXUL Memory')) + + self.assertFalse(runner.ignorePercentageForTest('LibXUL something else')) + self.assertFalse(runner.ignorePercentageForTest('V8')) + + def test_shouldSendWarning(self): + runner = self.create_runner() + d = PerfDatum(0, 0, time() + 0, 0.0, 0, 0) + d.historical_stats = { 'avg': 100.0 } + + # 1% increase + d.forward_stats = { 'avg': 101.0 } + self.assertFalse(runner.shouldSendWarning(d, 'some test')) + + # 10% increase + d.forward_stats = { 'avg': 110.0 } + self.assertTrue(runner.shouldSendWarning(d, 'some test')) + + # 1% decrease + d.forward_stats = { 'avg': 99.0 } + self.assertFalse(runner.shouldSendWarning(d, 'some test')) + + # 10% decrease + d.forward_stats = { 'avg': 90.0 } + self.assertTrue(runner.shouldSendWarning(d, 'some test')) + + # 1% increase, ignore percentage + d.forward_stats = { 'avg': 101.0 } + self.assertTrue(runner.shouldSendWarning(d, 'LibXUL Memory during link')) + if __name__ == '__main__': unittest.main()