From 1e6a6f874f4b768b19dabbad53b6fdcd0f23dc99 Mon Sep 17 00:00:00 2001 From: "frankf@chromium.org" Date: Tue, 30 Jul 2013 20:14:36 +0000 Subject: [PATCH] [Android] Dedup content_browsertests test setup from gtests. - This also changes the command for running content_browsertests to be in line with other gtest targets - Additional cleanup: get rid of --exe option and fall back to using exe if apk is not found TEST=test_runner.py gtest -s content_browsertests BUG=261950 R=bulach@chromium.org, craigdh@chromium.org Review URL: https://codereview.chromium.org/20649002 git-svn-id: http://src.chromium.org/svn/trunk/src/build@214421 4ff67af0-8c30-449e-8e8b-ad334ec8d88c --- android/buildbot/bb_device_steps.py | 27 ++----- android/pylib/android_commands.py | 4 + android/pylib/browsertests/__init__.py | 4 - android/pylib/browsertests/setup.py | 88 ---------------------- android/pylib/constants.py | 1 - android/pylib/gtest/gtest_config.py | 43 +++++------ android/pylib/gtest/setup.py | 82 +++++++-------------- android/pylib/gtest/test_package.py | 60 ++++++++------- android/pylib/gtest/test_package_apk.py | 98 +++++++++++-------------- android/pylib/gtest/test_package_exe.py | 65 ++++++++-------- android/pylib/gtest/test_runner.py | 71 ++++++------------ android/run_browser_tests.py | 2 +- android/test_runner.py | 84 ++++----------------- 13 files changed, 194 insertions(+), 435 deletions(-) delete mode 100644 android/pylib/browsertests/__init__.py delete mode 100644 android/pylib/browsertests/setup.py diff --git a/android/buildbot/bb_device_steps.py b/android/buildbot/bb_device_steps.py index 9ec3fe3dc..a1b5f93b2 100755 --- a/android/buildbot/bb_device_steps.py +++ b/android/buildbot/bb_device_steps.py @@ -105,7 +105,7 @@ def RunTestSuites(options, suites): Args: options: options object. - suites: List of suites to run. + suites: List of suite names to run. """ args = ['--verbose'] if options.target == 'Release': @@ -113,26 +113,12 @@ def RunTestSuites(options, suites): if options.asan: args.append('--tool=asan') for suite in suites: - bb_annotations.PrintNamedStep(suite.name) - cmd = ['build/android/test_runner.py', 'gtest', '-s', suite.name] + args - if suite.is_suite_exe: - cmd.append('--exe') + bb_annotations.PrintNamedStep(suite) + cmd = ['build/android/test_runner.py', 'gtest', '-s', suite] + args + if suite == 'content_browsertests': + cmd.append('--num_retries=1') RunCmd(cmd) -def RunBrowserTestSuite(options): - """Manages an invocation of test_runner.py for content_browsertests. - - Args: - options: options object. - """ - args = ['--verbose', '--num_retries=1'] - if options.target == 'Release': - args.append('--release') - if options.asan: - args.append('--tool=asan') - bb_annotations.PrintNamedStep(constants.BROWSERTEST_SUITE_NAME) - RunCmd(['build/android/test_runner.py', 'content_browsertests'] + args) - def RunChromeDriverTests(_): """Run all the steps for running chromedriver tests.""" bb_annotations.PrintNamedStep('chromedriver_annotation') @@ -287,7 +273,7 @@ def RunInstrumentationTests(options): def RunWebkitTests(options): - RunTestSuites(options, [gtest_config.Apk('webkit_unit_tests')]) + RunTestSuites(options, ['webkit_unit_tests']) RunWebkitLint(options.target) @@ -344,7 +330,6 @@ def MainTestWrapper(options): if options.experimental: RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES) - RunBrowserTestSuite(options) # Run all post test steps for _, cmd in GetPostTestStepCmds(): diff --git a/android/pylib/android_commands.py b/android/pylib/android_commands.py index 73d0c2fb5..4cf070d07 100644 --- a/android/pylib/android_commands.py +++ b/android/pylib/android_commands.py @@ -262,6 +262,10 @@ class AndroidCommands(object): """Returns our AdbInterface to avoid us wrapping all its methods.""" return self._adb + def GetDevice(self): + """Returns the device serial.""" + return self._device + def IsOnline(self): """Checks whether the device is online. diff --git a/android/pylib/browsertests/__init__.py b/android/pylib/browsertests/__init__.py deleted file mode 100644 index af9943712..000000000 --- a/android/pylib/browsertests/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - diff --git a/android/pylib/browsertests/setup.py b/android/pylib/browsertests/setup.py deleted file mode 100644 index 82c6b1131..000000000 --- a/android/pylib/browsertests/setup.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Generate test runner factory and tests for content_browsertests.""" - -import logging -import os -import sys - -from pylib import android_commands -from pylib import cmd_helper -from pylib import constants -from pylib import ports -from pylib.base import base_test_result -from pylib.gtest import setup as gtest_setup -from pylib.gtest import test_runner -from pylib.utils import report_results - - -def Setup(test_arguments, timeout, cleanup_test_files, tool, build_type, - push_deps, gtest_filter): - """Create the test runner factory and tests. - - Args: - test_arguments: Additional arguments to pass to the test binary. - timeout: Timeout for each test. - cleanup_test_files: Whether or not to cleanup test files on device. - tool: Name of the Valgrind tool. - build_type: 'Release' or 'Debug'. - push_deps: If True, push all dependencies to the device. - gtest_filter: filter for tests. - - Returns: - A tuple of (TestRunnerFactory, tests). - """ - - if not ports.ResetTestServerPortAllocation(): - raise Exception('Failed to reset test server port.') - - suite_path = os.path.join(cmd_helper.OutDirectory.get(), build_type, 'apks', - constants.BROWSERTEST_SUITE_NAME + '.apk') - - gtest_setup._GenerateDepsDirUsingIsolate( - constants.BROWSERTEST_SUITE_NAME, build_type) - - # Constructs a new TestRunner with the current options. - def TestRunnerFactory(device, shard_index): - return test_runner.TestRunner( - device, - suite_path, - test_arguments, - timeout, - cleanup_test_files, - tool, - build_type, - push_deps, - constants.BROWSERTEST_TEST_PACKAGE_NAME, - constants.BROWSERTEST_TEST_ACTIVITY_NAME, - constants.BROWSERTEST_COMMAND_LINE_FILE) - - # TODO(gkanwar): This breaks the abstraction of having test_dispatcher.py deal - # entirely with the devices. Can we do this another way? - attached_devices = android_commands.GetAttachedDevices() - - all_tests = gtest_setup.GetTestsFiltered( - constants.BROWSERTEST_SUITE_NAME, gtest_filter, TestRunnerFactory, - attached_devices) - - return (TestRunnerFactory, all_tests) - - -def _FilterTests(all_enabled_tests): - """Filters out tests and fixtures starting with PRE_ and MANUAL_.""" - return [t for t in all_enabled_tests if _ShouldRunOnBot(t)] - - -def _ShouldRunOnBot(test): - fixture, case = test.split('.', 1) - if _StartsWith(fixture, case, 'PRE_'): - return False - if _StartsWith(fixture, case, 'MANUAL_'): - return False - return True - - -def _StartsWith(a, b, prefix): - return a.startswith(prefix) or b.startswith(prefix) diff --git a/android/pylib/constants.py b/android/pylib/constants.py index 681853e44..6ca9a31fd 100644 --- a/android/pylib/constants.py +++ b/android/pylib/constants.py @@ -40,7 +40,6 @@ GTEST_TEST_PACKAGE_NAME = 'org.chromium.native_test' GTEST_TEST_ACTIVITY_NAME = 'org.chromium.native_test.ChromeNativeTestActivity' GTEST_COMMAND_LINE_FILE = 'chrome-native-tests-command-line' -BROWSERTEST_SUITE_NAME = 'content_browsertests' BROWSERTEST_TEST_PACKAGE_NAME = 'org.chromium.content_browsertests_apk' BROWSERTEST_TEST_ACTIVITY_NAME = ( 'org.chromium.content_browsertests_apk.ContentBrowserTestsActivity') diff --git a/android/pylib/gtest/gtest_config.py b/android/pylib/gtest/gtest_config.py index 892b9df3d..d74c03296 100644 --- a/android/pylib/gtest/gtest_config.py +++ b/android/pylib/gtest/gtest_config.py @@ -4,37 +4,30 @@ """Configuration file for android gtest suites.""" -import collections - - -Suite = collections.namedtuple('Suite', ['is_suite_exe', 'name']) -Exe = lambda name : Suite(True, name) -Apk = lambda name : Suite(False, name) - - # Add new suites here before upgrading them to the stable list below. EXPERIMENTAL_TEST_SUITES = [ + 'content_browsertests', ] # Do not modify this list without approval of an android owner. # This list determines which suites are run by default, both for local # testing and on android trybots running on commit-queue. STABLE_TEST_SUITES = [ - Apk('android_webview_unittests'), - Apk('base_unittests'), - Apk('cc_unittests'), - Apk('components_unittests'), - Apk('content_unittests'), - Apk('gpu_unittests'), - Apk('ipc_tests'), - Apk('media_unittests'), - Apk('net_unittests'), - Apk('sql_unittests'), - Apk('sync_unit_tests'), - Apk('ui_unittests'), - Apk('unit_tests'), - Apk('webkit_compositor_bindings_unittests'), - Apk('webkit_unit_tests'), - Exe('breakpad_unittests'), - Exe('sandbox_linux_unittests'), + 'android_webview_unittests', + 'base_unittests', + 'cc_unittests', + 'components_unittests', + 'content_unittests', + 'gpu_unittests', + 'ipc_tests', + 'media_unittests', + 'net_unittests', + 'sql_unittests', + 'sync_unit_tests', + 'ui_unittests', + 'unit_tests', + 'webkit_compositor_bindings_unittests', + 'webkit_unit_tests', + 'breakpad_unittests', + 'sandbox_linux_unittests', ] diff --git a/android/pylib/gtest/setup.py b/android/pylib/gtest/setup.py index a2468338a..a7271e587 100644 --- a/android/pylib/gtest/setup.py +++ b/android/pylib/gtest/setup.py @@ -18,6 +18,8 @@ from pylib import ports from pylib.base import base_test_result import gtest_config +import test_package_apk +import test_package_exe import test_runner sys.path.insert(0, @@ -148,38 +150,6 @@ def _GenerateDepsDirUsingIsolate(suite_name, build_type): os.rmdir(os.path.join(constants.ISOLATE_DEPS_DIR, 'out')) -def _GetSuitePath(use_exe_test_runner, suite_name, build_type): - """Get the absolute path to the test suite. - - Args: - use_exe_test_runner: If True, use the executable-based test runner. - suite_name: The suite name specified on the command line. - build_type: 'Release' or 'Debug'. - - Returns: - The absolute path of the given suite. - Ex. '/tmp/chrome/src/out/Debug/content_unittests_apk/' - 'content_unittests-debug.apk' - - Raises: - Exception: If test suite not found. - """ - if use_exe_test_runner: - relpath = suite_name - else: - relpath = os.path.join(suite_name + '_apk', suite_name + '-debug.apk') - suite_path = os.path.join(cmd_helper.OutDirectory.get(), build_type, relpath) - - if not os.path.exists(suite_path): - raise Exception('Test suite %s not found in %s.\n' - 'Supported test suites:\n %s\n' - 'Ensure it has been built.\n' % - (suite_name, suite_path, - [s.name for s in gtest_config.STABLE_TEST_SUITES])) - - return suite_path - - def _GetDisabledTestsFilterFromFile(suite_name): """Returns a gtest filter based on the *_disabled file. @@ -220,9 +190,7 @@ def _GetTestsFromDevice(runner_factory, devices): for device in devices: try: logging.info('Obtaining tests from %s', device) - runner = runner_factory(device, 0) - runner.test_package.Install() - return runner.test_package.GetAllTests() + return runner_factory(device, 0).GetAllTests() except (android_commands.errors.WaitForResponseTimedOutError, android_commands.errors.DeviceUnresponsiveError), e: logging.warning('Failed obtaining tests from %s with exception: %s', @@ -235,8 +203,8 @@ def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False): Args: all_tests: List of tests to filter. - pre: If True, include tests with _PRE prefix. - manual: If True, include tests with _MANUAL prefix. + pre: If True, include tests with PRE_ prefix. + manual: If True, include tests with MANUAL_ prefix. Returns: List of tests remaining. @@ -258,7 +226,7 @@ def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False): return filtered_tests -def GetTestsFiltered(suite_name, gtest_filter, runner_factory, devices): +def _GetTestsFiltered(suite_name, gtest_filter, runner_factory, devices): """Get all tests in the suite and filter them. Obtains a list of tests from the test package on the device, and @@ -289,13 +257,12 @@ def GetTestsFiltered(suite_name, gtest_filter, runner_factory, devices): return tests -def Setup(use_exe_test_runner, suite_name, test_arguments, timeout, +def Setup(suite_name, test_arguments, timeout, cleanup_test_files, tool, build_type, push_deps, gtest_filter): """Create the test runner factory and tests. Args: - use_exe_test_runner: If True, use the executable-based test runner. suite_name: The suite name specified on the command line. test_arguments: Additional arguments to pass to the test binary. timeout: Timeout for each test. @@ -312,33 +279,36 @@ def Setup(use_exe_test_runner, suite_name, test_arguments, timeout, if not ports.ResetTestServerPortAllocation(): raise Exception('Failed to reset test server port.') - suite_path = _GetSuitePath(use_exe_test_runner, suite_name, build_type) + test_package = test_package_apk.TestPackageApk(suite_name, build_type) + if not os.path.exists(test_package.suite_path): + test_package = test_package_exe.TestPackageExecutable( + suite_name, build_type) + if not os.path.exists(test_package.suite_path): + raise Exception( + 'Did not find %s target. Ensure it has been built.' % suite_name) + logging.warning('Found target %s', test_package.suite_path) - # TODO(gkanwar): This breaks the abstraction of having test_dispatcher.py deal - # entirely with the devices. Can we do this another way? - attached_devices = android_commands.GetAttachedDevices() + _GenerateDepsDirUsingIsolate(suite_name, build_type) - deps_dir = _GenerateDepsDirUsingIsolate(suite_name, build_type) # Constructs a new TestRunner with the current options. def TestRunnerFactory(device, shard_index): return test_runner.TestRunner( device, - suite_path, + test_package, test_arguments, timeout, cleanup_test_files, tool, build_type, - push_deps, - constants.GTEST_TEST_PACKAGE_NAME, - constants.GTEST_TEST_ACTIVITY_NAME, - constants.GTEST_COMMAND_LINE_FILE) + push_deps) - # Get tests and split them up based on the number of devices. - tests = GetTestsFiltered(suite_name, gtest_filter, - TestRunnerFactory, attached_devices) - num_devices = len(attached_devices) - tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)] - tests = [t for t in tests if t] + attached_devices = android_commands.GetAttachedDevices() + tests = _GetTestsFiltered(suite_name, gtest_filter, + TestRunnerFactory, attached_devices) + # Coalesce unit tests into a single test per device + if suite_name != 'content_browsertests': + num_devices = len(attached_devices) + tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)] + tests = [t for t in tests if t] return (TestRunnerFactory, tests) diff --git a/android/pylib/gtest/test_package.py b/android/pylib/gtest/test_package.py index 945b4fde9..ce07f4fb9 100644 --- a/android/pylib/gtest/test_package.py +++ b/android/pylib/gtest/test_package.py @@ -4,67 +4,65 @@ """Base class representing GTest test packages.""" -import os - -from pylib import constants +import logging class TestPackage(object): + """A helper base class for both APK and stand-alone executables. Args: - adb: ADB interface the tests are using. - device: Device to run the tests. - suite_path_full: Absolute path to a specific test suite to run, - empty to run all. - Ex: '/foo/bar/base_unittests-debug.apk', for which - self.suite_path_full = '/foo/bar/base_unittests-debug.apk' - self.suite_path = '/foo/bar/base_unittests-debug' - self.suite_basename = 'base_unittests' - self.suite_dirname = '/foo/bar' - tool: Name of the Valgrind tool. + suite_name: Name of the test suite (e.g. base_unittests). """ + def __init__(self, suite_name): + self.suite_name = suite_name - def __init__(self, adb, device, suite_path_full, tool): - self.adb = adb - self.device = device - self.suite_path_full = suite_path_full - self.suite_path = os.path.splitext(suite_path_full)[0] - self.suite_basename = self._GetTestSuiteBaseName() - self.suite_dirname = os.path.dirname( - self.suite_path.split(self.suite_basename)[0]) - self.tool = tool + def ClearApplicationState(self, adb): + """Clears the application state. - def ClearApplicationState(self): - """Clears the application state.""" + Args: + adb: Instance of AndroidCommands. + """ raise NotImplementedError('Method must be overriden.') - def CreateCommandLineFileOnDevice(self, test_filter, test_arguments): + def CreateCommandLineFileOnDevice(self, adb, test_filter, test_arguments): """Creates a test runner script and pushes to the device. Args: + adb: Instance of AndroidCommands. test_filter: A test_filter flag. test_arguments: Additional arguments to pass to the test binary. """ raise NotImplementedError('Method must be overriden.') - def GetAllTests(self): - """Returns a list of all tests available in the test suite.""" + def GetAllTests(self, adb): + """Returns a list of all tests available in the test suite. + + Args: + adb: Instance of AndroidCommands. + """ raise NotImplementedError('Method must be overriden.') - def GetGTestReturnCode(self): + def GetGTestReturnCode(self, adb): return None - def SpawnTestProcess(self): + def SpawnTestProcess(self, adb): """Spawn the test process. + Args: + adb: Instance of AndroidCommands. + Returns: An instance of pexpect spawn class. """ raise NotImplementedError('Method must be overriden.') - def Install(self): - """Install the test package to the device.""" + def Install(self, adb): + """Install the test package to the device. + + Args: + adb: Instance of AndroidCommands. + """ raise NotImplementedError('Method must be overriden.') def _ParseGTestListTests(self, raw_list): diff --git a/android/pylib/gtest/test_package_apk.py b/android/pylib/gtest/test_package_apk.py index 1ea0f8892..84dd7bd8e 100644 --- a/android/pylib/gtest/test_package_apk.py +++ b/android/pylib/gtest/test_package_apk.py @@ -12,6 +12,7 @@ import tempfile import time from pylib import android_commands +from pylib import cmd_helper from pylib import constants from pylib import pexpect from pylib.android_commands import errors @@ -22,41 +23,36 @@ from test_package import TestPackage class TestPackageApk(TestPackage): """A helper class for running APK-based native tests.""" - def __init__(self, adb, device, suite_path_full, tool, test_apk_package_name, - test_activity_name, command_line_file): + def __init__(self, suite_name, build_type): """ Args: - adb: ADB interface the tests are using. - device: Device to run the tests. - suite_path_full: Absolute path to a specific test suite to run, - empty to run all. - Ex: '/foo/bar/base_unittests-debug.apk', for which - self.suite_path_full = '/foo/bar/base_unittests-debug.apk' - self.suite_path = '/foo/bar/base_unittests-debug' - self.suite_basename = 'base_unittests' - self.suite_dirname = '/foo/bar' - tool: Name of the Valgrind tool. - test_apk_package_name: Apk package name for tests running in APKs. - test_activity_name: Test activity to invoke for APK tests. - command_line_file: Filename to use to pass arguments to tests. + suite_name: Name of the test suite (e.g. base_unittests). + build_type: 'Release' or 'Debug'. """ - TestPackage.__init__(self, adb, device, suite_path_full, tool) - self._test_apk_package_name = test_apk_package_name - self._test_activity_name = test_activity_name - self._command_line_file = command_line_file + TestPackage.__init__(self, suite_name) + product_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type) + if suite_name == 'content_browsertests': + self.suite_path = os.path.join( + product_dir, 'apks', '%s.apk' % suite_name) + self._test_apk_package_name = constants.BROWSERTEST_TEST_PACKAGE_NAME + self._test_activity_name = constants.BROWSERTEST_TEST_ACTIVITY_NAME + self._command_line_file = constants.BROWSERTEST_COMMAND_LINE_FILE + else: + self.suite_path = os.path.join( + product_dir, '%s_apk' % suite_name, '%s-debug.apk' % suite_name) + self._test_apk_package_name = constants.GTEST_TEST_PACKAGE_NAME + self._test_activity_name = constants.GTEST_TEST_ACTIVITY_NAME + self._command_line_file = constants.GTEST_COMMAND_LINE_FILE - def _CreateCommandLineFileOnDevice(self, options): + def _CreateCommandLineFileOnDevice(self, adb, options): command_line_file = tempfile.NamedTemporaryFile() # GTest expects argv[0] to be the executable path. - command_line_file.write(self.suite_basename + ' ' + options) + command_line_file.write(self.suite_name + ' ' + options) command_line_file.flush() - self.adb.PushIfNeeded(command_line_file.name, + adb.PushIfNeeded(command_line_file.name, constants.TEST_EXECUTABLE_DIR + '/' + self._command_line_file) - def _GetGTestReturnCode(self): - return None - def _GetFifo(self): # The test.fifo path is determined by: # testing/android/java/src/org/chromium/native_test/ @@ -64,54 +60,49 @@ class TestPackageApk(TestPackage): # testing/android/native_test_launcher.cc return '/data/data/' + self._test_apk_package_name + '/files/test.fifo' - def _ClearFifo(self): - self.adb.RunShellCommand('rm -f ' + self._GetFifo()) + def _ClearFifo(self, adb): + adb.RunShellCommand('rm -f ' + self._GetFifo()) - def _WatchFifo(self, timeout, logfile=None): + def _WatchFifo(self, adb, timeout, logfile=None): for i in range(10): - if self.adb.FileExistsOnDevice(self._GetFifo()): + if adb.FileExistsOnDevice(self._GetFifo()): logging.info('Fifo created.') break time.sleep(i) else: raise errors.DeviceUnresponsiveError( 'Unable to find fifo on device %s ' % self._GetFifo()) - args = shlex.split(self.adb.Adb()._target_arg) + args = shlex.split(adb.Adb()._target_arg) args += ['shell', 'cat', self._GetFifo()] return pexpect.spawn('adb', args, timeout=timeout, logfile=logfile) - def _StartActivity(self): - self.adb.StartActivity( + def _StartActivity(self, adb): + adb.StartActivity( self._test_apk_package_name, self._test_activity_name, wait_for_completion=True, action='android.intent.action.MAIN', force_stop=True) - def _GetTestSuiteBaseName(self): - """Returns the base name of the test suite.""" - # APK test suite names end with '-debug.apk' - return os.path.basename(self.suite_path).rsplit('-debug', 1)[0] + #override + def ClearApplicationState(self, adb): + adb.ClearApplicationState(self._test_apk_package_name) #override - def ClearApplicationState(self): - self.adb.ClearApplicationState(self._test_apk_package_name) - - #override - def CreateCommandLineFileOnDevice(self, test_filter, test_arguments): + def CreateCommandLineFileOnDevice(self, adb, test_filter, test_arguments): self._CreateCommandLineFileOnDevice( - '--gtest_filter=%s %s' % (test_filter, test_arguments)) + adb, '--gtest_filter=%s %s' % (test_filter, test_arguments)) #override - def GetAllTests(self): - self._CreateCommandLineFileOnDevice('--gtest_list_tests') + def GetAllTests(self, adb): + self._CreateCommandLineFileOnDevice(adb, '--gtest_list_tests') try: self.tool.SetupEnvironment() # Clear and start monitoring logcat. - self._ClearFifo() - self._StartActivity() + self._ClearFifo(adb) + self._StartActivity(adb) # Wait for native test to complete. - p = self._WatchFifo(timeout=30 * self.tool.GetTimeoutScale()) + p = self._WatchFifo(adb, timeout=30 * self.tool.GetTimeoutScale()) p.expect('< %s' % (constants.TEST_EXECUTABLE_DIR, - self._AddNativeCoverageExports(), + self._AddNativeCoverageExports(adb), tool_wrapper, constants.TEST_EXECUTABLE_DIR, - self.suite_basename, + self.suite_name, test_filter, test_arguments, TestPackageExecutable._TEST_RUNNER_RET_VAL_FILE)) sh_script_file.flush() cmd_helper.RunCmd(['chmod', '+x', sh_script_file.name]) - self.adb.PushIfNeeded( + adb.PushIfNeeded( sh_script_file.name, constants.TEST_EXECUTABLE_DIR + '/chrome_test_runner.sh') logging.info('Conents of the test runner script: ') @@ -109,27 +103,27 @@ class TestPackageExecutable(TestPackage): logging.info(' ' + line.rstrip()) #override - def GetAllTests(self): - all_tests = self.adb.RunShellCommand( + def GetAllTests(self, adb): + all_tests = adb.RunShellCommand( '%s %s/%s --gtest_list_tests' % (self.tool.GetTestWrapper(), constants.TEST_EXECUTABLE_DIR, - self.suite_basename)) + self.suite_name)) return self._ParseGTestListTests(all_tests) #override - def SpawnTestProcess(self): - args = ['adb', '-s', self.device, 'shell', 'sh', + def SpawnTestProcess(self, adb): + args = ['adb', '-s', adb.GetDevice(), 'shell', 'sh', constants.TEST_EXECUTABLE_DIR + '/chrome_test_runner.sh'] logging.info(args) return pexpect.spawn(args[0], args[1:], logfile=sys.stdout) #override - def Install(self): + def Install(self, adb): if self.tool.NeedsDebugInfo(): target_name = self.suite_path else: - target_name = self.suite_path + '_' + self.device + '_stripped' + target_name = self.suite_path + '_' + adb.GetDevice() + '_stripped' should_strip = True if os.path.isfile(target_name): logging.info('Found target file %s' % target_name) @@ -145,11 +139,10 @@ class TestPackageExecutable(TestPackage): 'new one (%s).' % target_name) # Whenever we generate a stripped binary, copy to the symbols dir. If we # aren't stripping a new binary, assume it's there. - if self.symbols_dir: - if not os.path.exists(self.symbols_dir): - os.makedirs(self.symbols_dir) - shutil.copy(self.suite_path, self.symbols_dir) + if not os.path.exists(self._symbols_dir): + os.makedirs(self._symbols_dir) + shutil.copy(self.suite_path, self._symbols_dir) strip = os.environ['STRIP'] cmd_helper.RunCmd([strip, self.suite_path, '-o', target_name]) - test_binary = constants.TEST_EXECUTABLE_DIR + '/' + self.suite_basename - self.adb.PushIfNeeded(target_name, test_binary) + test_binary = constants.TEST_EXECUTABLE_DIR + '/' + self.suite_name + adb.PushIfNeeded(target_name, test_binary) diff --git a/android/pylib/gtest/test_runner.py b/android/pylib/gtest/test_runner.py index da422c38e..2a93e3753 100644 --- a/android/pylib/gtest/test_runner.py +++ b/android/pylib/gtest/test_runner.py @@ -11,76 +11,52 @@ from pylib import constants from pylib import pexpect from pylib.base import base_test_result from pylib.base import base_test_runner -from pylib.utils import run_tests_helper - -import test_package_apk -import test_package_exe -def _TestSuiteRequiresMockTestServer(suite_basename): +def _TestSuiteRequiresMockTestServer(suite_name): """Returns True if the test suite requires mock test server.""" tests_require_net_test_server = ['unit_tests', 'net_unittests', 'content_unittests', 'content_browsertests'] - return (suite_basename in + return (suite_name in tests_require_net_test_server) class TestRunner(base_test_runner.BaseTestRunner): - def __init__(self, device, suite_name, test_arguments, timeout, - cleanup_test_files, tool_name, build_type, - push_deps, test_apk_package_name=None, - test_activity_name=None, command_line_file=None): + def __init__(self, device, test_package, test_arguments, timeout, + cleanup_test_files, tool_name, build_type, push_deps): """Single test suite attached to a single device. Args: device: Device to run the tests. - suite_name: A specific test suite to run, empty to run all. + test_package: An instance of TestPackage class. test_arguments: Additional arguments to pass to the test binary. timeout: Timeout for each test. cleanup_test_files: Whether or not to cleanup test files on device. tool_name: Name of the Valgrind tool. build_type: 'Release' or 'Debug'. push_deps: If True, push all dependencies to the device. - test_apk_package_name: Apk package name for tests running in APKs. - test_activity_name: Test activity to invoke for APK tests. - command_line_file: Filename to use to pass arguments to tests. """ super(TestRunner, self).__init__(device, tool_name, build_type, push_deps, cleanup_test_files) + self.test_package = test_package + self.test_package.tool = self.tool self._test_arguments = test_arguments if timeout == 0: timeout = 60 # On a VM (e.g. chromium buildbots), this timeout is way too small. if os.environ.get('BUILDBOT_SLAVENAME'): timeout = timeout * 2 - self.timeout = timeout * self.tool.GetTimeoutScale() - - logging.warning('Test suite: ' + str(suite_name)) - if os.path.splitext(suite_name)[1] == '.apk': - self.test_package = test_package_apk.TestPackageApk( - self.adb, - device, - suite_name, - self.tool, - test_apk_package_name, - test_activity_name, - command_line_file) - else: - # Put a copy into the android out/target directory, to allow stack trace - # generation. - symbols_dir = os.path.join(constants.DIR_SOURCE_ROOT, 'out', build_type, - 'lib.target') - self.test_package = test_package_exe.TestPackageExecutable( - self.adb, - device, - suite_name, - self.tool, - symbols_dir) + self._timeout = timeout * self.tool.GetTimeoutScale() #override def InstallTestPackage(self): - self.test_package.Install() + self.test_package.Install(self.adb) + + def GetAllTests(self): + """Install test package and get a list of all tests.""" + self.test_package.Install(self.adb) + return self.test_package.GetAllTests(self.adb) #override def PushDataDeps(self): @@ -90,7 +66,7 @@ class TestRunner(base_test_runner.BaseTestRunner): device_dir = self.adb.GetExternalStorage() # TODO(frankf): linux_dumper_unittest_helper needs to be in the same dir # as breakpad_unittests exe. Find a better way to do this. - if self.test_package.suite_basename == 'breakpad_unittests': + if self.test_package.suite_name == 'breakpad_unittests': device_dir = constants.TEST_EXECUTABLE_DIR for p in os.listdir(constants.ISOLATE_DEPS_DIR): self.adb.PushIfNeeded( @@ -125,14 +101,14 @@ class TestRunner(base_test_runner.BaseTestRunner): while True: full_test_name = None found = p.expect([re_run, re_passed, re_runner_fail], - timeout=self.timeout) + timeout=self._timeout) if found == 1: # re_passed break elif found == 2: # re_runner_fail break else: # re_run full_test_name = p.match.group(1).replace('\r', '') - found = p.expect([re_ok, re_fail, re_crash], timeout=self.timeout) + found = p.expect([re_ok, re_fail, re_crash], timeout=self._timeout) log = p.before.replace('\r', '') if found == 0: # re_ok if full_test_name == p.match.group(1).replace('\r', ''): @@ -160,7 +136,7 @@ class TestRunner(base_test_runner.BaseTestRunner): log=p.before.replace('\r', ''))) except pexpect.TIMEOUT: logging.error('Test terminated after %d second timeout.', - self.timeout) + self._timeout) if full_test_name: results.AddResult(base_test_result.BaseTestResult( full_test_name, base_test_result.ResultType.TIMEOUT, @@ -168,7 +144,7 @@ class TestRunner(base_test_runner.BaseTestRunner): finally: p.close() - ret_code = self.test_package.GetGTestReturnCode() + ret_code = self.test_package.GetGTestReturnCode(self.adb) if ret_code: logging.critical( 'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s', @@ -183,10 +159,11 @@ class TestRunner(base_test_runner.BaseTestRunner): return test_results, None try: - self.test_package.ClearApplicationState() + self.test_package.ClearApplicationState(self.adb) self.test_package.CreateCommandLineFileOnDevice( - test, self._test_arguments) - test_results = self._ParseTestOutput(self.test_package.SpawnTestProcess()) + self.adb, test, self._test_arguments) + test_results = self._ParseTestOutput( + self.test_package.SpawnTestProcess(self.adb)) finally: self.CleanupSpawningServerState() # Calculate unknown test results. @@ -203,7 +180,7 @@ class TestRunner(base_test_runner.BaseTestRunner): def SetUp(self): """Sets up necessary test enviroment for the test suite.""" super(TestRunner, self).SetUp() - if _TestSuiteRequiresMockTestServer(self.test_package.suite_basename): + if _TestSuiteRequiresMockTestServer(self.test_package.suite_name): self.LaunchChromeTestServerSpawner() self.tool.SetupEnvironment() diff --git a/android/run_browser_tests.py b/android/run_browser_tests.py index 045894da4..6f6003dcb 100755 --- a/android/run_browser_tests.py +++ b/android/run_browser_tests.py @@ -16,7 +16,7 @@ from pylib import cmd_helper if __name__ == '__main__': args = ['python', os.path.join(os.path.dirname(__file__), 'test_runner.py'), - 'content_browsertests'] + sys.argv[1:] + 'gtest', '-s', 'content_browsertests'] + sys.argv[1:] logging.warning('*' * 80) logging.warning('This script is deprecated and will be removed soon.') logging.warning('Use the following instead: %s', ' '.join(args)) diff --git a/android/test_runner.py b/android/test_runner.py index 38cdc5884..b51effb23 100755 --- a/android/test_runner.py +++ b/android/test_runner.py @@ -21,7 +21,6 @@ from pylib import constants from pylib import ports from pylib.base import base_test_result from pylib.base import test_dispatcher -from pylib.browsertests import setup as browsertests_setup from pylib.gtest import setup as gtest_setup from pylib.gtest import gtest_config from pylib.host_driven import run_python_tests as python_dispatch @@ -97,35 +96,6 @@ def ProcessCommonOptions(options): run_tests_helper.SetLogLevel(options.verbose_count) -def AddCoreGTestOptions(option_parser): - """Add options specific to the gtest framework to |option_parser|.""" - - # TODO(gkanwar): Consolidate and clean up test filtering for gtests and - # content_browsertests. - option_parser.add_option('-f', '--gtest_filter', dest='test_filter', - help='googletest-style filter string.') - option_parser.add_option('-a', '--test_arguments', dest='test_arguments', - help='Additional arguments to pass to the test.') - option_parser.add_option('--exe', action='store_true', - help='If set, use the exe test runner instead of ' - 'the APK.') - option_parser.add_option('-t', dest='timeout', - help='Timeout to wait for each test', - type='int', - default=60) - - -def AddContentBrowserTestOptions(option_parser): - """Adds Content Browser test options to |option_parser|.""" - - option_parser.usage = '%prog content_browsertests [options]' - option_parser.command_list = [] - option_parser.example = '%prog content_browsertests' - - AddCoreGTestOptions(option_parser) - AddCommonOptions(option_parser) - - def AddGTestOptions(option_parser): """Adds gtest options to |option_parser|.""" @@ -137,7 +107,14 @@ def AddGTestOptions(option_parser): option_parser.add_option('-s', '--suite', dest='suite_name', help=('Executable name of the test suite to run ' '(use -s help to list them).')) - AddCoreGTestOptions(option_parser) + option_parser.add_option('-f', '--gtest_filter', dest='test_filter', + help='googletest-style filter string.') + option_parser.add_option('-a', '--test_arguments', dest='test_arguments', + help='Additional arguments to pass to the test.') + option_parser.add_option('-t', dest='timeout', + help='Timeout to wait for each test', + type='int', + default=60) # TODO(gkanwar): Move these to Common Options once we have the plumbing # in our other test types to handle these commands AddCommonOptions(option_parser) @@ -154,8 +131,9 @@ def ProcessGTestOptions(options): """ if options.suite_name == 'help': print 'Available test suites are:' - for test_suite in gtest_config.STABLE_TEST_SUITES: - print test_suite.name + for test_suite in (gtest_config.STABLE_TEST_SUITES + + gtest_config.EXPERIMENTAL_TEST_SUITES): + print test_suite return False # Convert to a list, assuming all test suites if nothing was specified. @@ -163,8 +141,7 @@ def ProcessGTestOptions(options): if options.suite_name: options.suite_name = [options.suite_name] else: - options.suite_name = [suite.name - for suite in gtest_config.STABLE_TEST_SUITES] + options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES] return True @@ -350,7 +327,7 @@ def _RunGTests(options, error_func): exit_code = 0 for suite_name in options.suite_name: runner_factory, tests = gtest_setup.Setup( - options.exe, suite_name, options.test_arguments, + suite_name, options.test_arguments, options.timeout, options.cleanup_test_files, options.tool, options.build_type, options.push_deps, options.test_filter) @@ -377,37 +354,6 @@ def _RunGTests(options, error_func): return exit_code -def _RunContentBrowserTests(options, error_func): - """Subcommand of RunTestsCommands which runs content_browsertests.""" - runner_factory, tests = browsertests_setup.Setup( - options.test_arguments, options.timeout, options.cleanup_test_files, - options.tool, options.build_type, options.push_deps, - options.test_filter) - - # TODO(nileshagrawal): remove this abnormally long setup timeout once fewer - # files are pushed to the devices for content_browsertests: crbug.com/138275 - setup_timeout = 20 * 60 # 20 minutes - results, exit_code = test_dispatcher.RunTests( - tests, runner_factory, False, options.test_device, - shard=True, - build_type=options.build_type, - test_timeout=None, - setup_timeout=setup_timeout, - num_retries=options.num_retries) - - report_results.LogFull( - results=results, - test_type='Unit test', - test_package=constants.BROWSERTEST_SUITE_NAME, - build_type=options.build_type, - flakiness_server=options.flakiness_dashboard_server) - - if os.path.isdir(constants.ISOLATE_DEPS_DIR): - shutil.rmtree(constants.ISOLATE_DEPS_DIR) - - return exit_code - - def _RunInstrumentationTests(options, error_func): """Subcommand of RunTestsCommands which runs instrumentation tests.""" ProcessInstrumentationOptions(options, error_func) @@ -527,8 +473,6 @@ def RunTestsCommand(command, options, args, option_parser): if command == 'gtest': return _RunGTests(options, option_parser.error) - elif command == 'content_browsertests': - return _RunContentBrowserTests(options, option_parser.error) elif command == 'instrumentation': return _RunInstrumentationTests(options, option_parser.error) elif command == 'uiautomator': @@ -587,8 +531,6 @@ CommandFunctionTuple = collections.namedtuple( 'CommandFunctionTuple', ['add_options_func', 'run_command_func']) VALID_COMMANDS = { 'gtest': CommandFunctionTuple(AddGTestOptions, RunTestsCommand), - 'content_browsertests': CommandFunctionTuple( - AddContentBrowserTestOptions, RunTestsCommand), 'instrumentation': CommandFunctionTuple( AddInstrumentationTestOptions, RunTestsCommand), 'uiautomator': CommandFunctionTuple(