diff --git a/android/pylib/perf/setup.py b/android/pylib/perf/setup.py index 74bf79fdd..39e50980d 100644 --- a/android/pylib/perf/setup.py +++ b/android/pylib/perf/setup.py @@ -10,6 +10,7 @@ import logging import os import psutil import signal +import shutil import time from pylib import android_commands @@ -58,8 +59,9 @@ def Setup(test_options): Returns: A tuple of (TestRunnerFactory, tests). """ - if not os.path.exists(constants.PERF_OUTPUT_DIR): - os.makedirs(constants.PERF_OUTPUT_DIR) + if os.path.exists(constants.PERF_OUTPUT_DIR): + shutil.rmtree(constants.PERF_OUTPUT_DIR) + os.makedirs(constants.PERF_OUTPUT_DIR) # Before running the tests, kill any leftover server. _KillPendingServers() diff --git a/android/pylib/perf/test_runner.py b/android/pylib/perf/test_runner.py index 10a8da2d4..e3648f74a 100644 --- a/android/pylib/perf/test_runner.py +++ b/android/pylib/perf/test_runner.py @@ -2,7 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Runs a perf test on a single device. +"""Runs perf tests. Our buildbot infrastructure requires each slave to run steps serially. This is sub-optimal for android, where these steps can run independently on @@ -93,11 +93,24 @@ class TestRunner(base_test_runner.BaseTestRunner): self._tests = tests self._flaky_tests = flaky_tests + @staticmethod + def _IsBetter(result): + if result['actual_exit_code'] == 0: + return True + pickled = os.path.join(constants.PERF_OUTPUT_DIR, + result['name']) + if not os.path.exists(pickled): + return True + with file(pickled, 'r') as f: + previous = pickle.loads(f.read()) + return result['actual_exit_code'] < previous['actual_exit_code'] + @staticmethod def _SaveResult(result): - with file(os.path.join(constants.PERF_OUTPUT_DIR, - result['name']), 'w') as f: - f.write(pickle.dumps(result)) + if TestRunner._IsBetter(result): + with file(os.path.join(constants.PERF_OUTPUT_DIR, + result['name']), 'w') as f: + f.write(pickle.dumps(result)) def _LaunchPerfTest(self, test_name): """Runs a perf test. @@ -133,6 +146,7 @@ class TestRunner(base_test_runner.BaseTestRunner): result_type = base_test_result.ResultType.FAIL if exit_code == 0: result_type = base_test_result.ResultType.PASS + actual_exit_code = exit_code if test_name in self._flaky_tests: # The exit_code is used at the second stage when printing the # test output. If the test is flaky, force to "0" to get that step green @@ -144,6 +158,7 @@ class TestRunner(base_test_runner.BaseTestRunner): 'name': test_name, 'output': output, 'exit_code': exit_code, + 'actual_exit_code': actual_exit_code, 'result_type': result_type, 'total_time': (end_time - start_time).seconds, 'device': self.device,