diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py index eac1bfa130cf..50b2a3b47ddd 100644 --- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -157,6 +157,12 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): "default": False, "help": "Use Raptor to measure memory usage.", }], + [["--cpu-test"], { + "dest": "cpu_test", + "action": "store_true", + "default": False, + "help": "Use Raptor to measure CPU usage" + }], [["--debug-mode"], { "dest": "debug_mode", "action": "store_true", @@ -254,6 +260,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): self.host = os.environ['HOST_IP'] self.power_test = self.config.get('power_test') self.memory_test = self.config.get('memory_test') + self.cpu_test = self.config.get('cpu_test') self.is_release_build = self.config.get('is_release_build') self.debug_mode = self.config.get('debug_mode', False) self.firefox_android_browsers = ["fennec", "geckoview", "refbrow", "fenix"] @@ -389,6 +396,8 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): options.extend(['--power-test']) if self.config.get('memory_test', False): options.extend(['--memory-test']) + if self.config.get('cpu_test', False): + options.extend(['--cpu-test']) for key, value in kw_options.items(): options.extend(['--%s' % key, value]) @@ -520,6 +529,8 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): expected_perfherder += 1 if self.config.get('memory_test', None): expected_perfherder += 1 + if self.config.get('cpu_test', None): + expected_perfherder += 1 if len(parser.found_perf_data) != expected_perfherder: self.critical("PERFHERDER_DATA was seen %d times, expected %d." % (len(parser.found_perf_data), expected_perfherder)) @@ -659,6 +670,10 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-memory.json') self._artifact_perf_data(src, dest) + if self.cpu_test: + src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-cpu.json') + self._artifact_perf_data(src, dest) + src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'screenshots.html') if os.path.exists(src): dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'screenshots.html') diff --git a/testing/raptor/mach_commands.py b/testing/raptor/mach_commands.py index 2ee1695044a2..a927ca00257f 100644 --- a/testing/raptor/mach_commands.py +++ b/testing/raptor/mach_commands.py @@ -56,6 +56,7 @@ class RaptorRunner(MozbuildObject): self.is_release_build = kwargs['is_release_build'] self.memory_test = kwargs['memory_test'] self.power_test = kwargs['power_test'] + self.cpu_test = kwargs['cpu_test'] if Conditions.is_android(self) or kwargs["app"] in FIREFOX_ANDROID_BROWSERS: self.binary_path = None @@ -152,6 +153,7 @@ class RaptorRunner(MozbuildObject): 'host': self.host, 'power_test': self.power_test, 'memory_test': self.memory_test, + 'cpu_test': self.cpu_test, 'is_release_build': self.is_release_build, } diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index 579d596dd701..ab164845b735 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -89,6 +89,8 @@ def create_parser(mach_interface=False): "The host ip address must be specified via the --host command line argument.") add_arg('--memory-test', dest="memory_test", action="store_true", help="Use Raptor to measure memory usage.") + add_arg('--cpu-test', dest="cpu_test", action="store_true", + help="Use Raptor to measure CPU usage. Currently supported for Android only.") add_arg('--is-release-build', dest="is_release_build", default=False, action='store_true', help="Whether the build is a release build which requires workarounds " diff --git a/testing/raptor/raptor/cpu.py b/testing/raptor/raptor/cpu.py new file mode 100644 index 000000000000..7cf8163f0312 --- /dev/null +++ b/testing/raptor/raptor/cpu.py @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + + +def get_app_cpu_usage(raptor): + # If we don't find the browser running, we default to 0 usage + cpu_usage = 0 + app_name = raptor.config['binary'] + verbose = raptor.device._verbose + raptor.device._verbose = False + + ''' + There are two ways to get CPU usage information: + + 1. By using the 'top' command and parsing details + 2. By using 'dumpsys cpuinfo' and parsing the details + + 'top' is our first choice if it is available but the + parameters we use are only available in Android 8 or + greater, otherwise we fall back to using dumpsys + ''' + if raptor.device.version >= 8: + cpuinfo = raptor.device.shell_output("top -O %CPU -n 1").split("\n") + raptor.device._verbose = verbose + for line in cpuinfo: + # 14781 u0_a83 0 92.8 12.4 64:53.04 org.mozilla.geckoview_example + data = line.split() + if data[-1] == app_name: + cpu_usage = float(data[3]) + else: + cpuinfo = raptor.device.shell_output("dumpsys cpuinfo | grep %s" % app_name).split("\n") + for line in cpuinfo: + # 34% 14781/org.mozilla.geckoview_example: 26% user + 7.5% kernel + data = line.split() + cpu_usage = float(data[0].strip('%')) + + return cpu_usage + + +def generate_android_cpu_profile(raptor, test_name): + if not raptor.device or not raptor.config['cpu_test']: + return + + result = get_app_cpu_usage(raptor) + + cpuinfo_data = { + u'type': u'cpu', + u'test': test_name, + u'unit': u'%', + u'values': { + u'browser_cpu_usage': result + } + } + raptor.control_server.submit_supporting_data(cpuinfo_data) diff --git a/testing/raptor/raptor/raptor.py b/testing/raptor/raptor/raptor.py index 2298c5d6366e..408edfaabe7c 100644 --- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -58,6 +58,7 @@ from memory import generate_android_memory_profile from power import init_android_power_test, finish_android_power_test from results import RaptorResultsHandler from utils import view_gecko_profile +from cpu import generate_android_cpu_profile LOG = RaptorLogger(component='raptor-main') @@ -82,7 +83,7 @@ class Raptor(object): def __init__(self, app, binary, run_local=False, obj_path=None, profile_class=None, gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None, - symbols_path=None, host=None, power_test=False, memory_test=False, + symbols_path=None, host=None, power_test=False, cpu_test=False, memory_test=False, is_release_build=False, debug_mode=False, post_startup_delay=None, interrupt_handler=None, e10s=True, **kwargs): @@ -104,6 +105,7 @@ class Raptor(object): 'host': host, 'power_test': power_test, 'memory_test': memory_test, + 'cpu_test': cpu_test, 'is_release_build': is_release_build, 'enable_control_server_wait': memory_test, 'e10s': e10s, @@ -1001,7 +1003,6 @@ class RaptorAndroid(Raptor): finally: if self.config['power_test']: finish_android_power_test(self, test['name']) - self.run_test_teardown() def run_test_cold(self, test, timeout=None): @@ -1081,6 +1082,10 @@ class RaptorAndroid(Raptor): # now start the browser/app under test self.launch_firefox_android_app(test['name']) + # If we are measuring CPU, let's grab a snapshot + if self.config['cpu_test']: + generate_android_cpu_profile(self, test['name']) + # set our control server flag to indicate we are running the browser/app self.control_server._finished = False @@ -1118,6 +1123,10 @@ class RaptorAndroid(Raptor): # now start the browser/app under test self.launch_firefox_android_app(test['name']) + # If we are collecting CPU info, let's grab the details + if self.config['cpu_test']: + generate_android_cpu_profile(self, test['name']) + # set our control server flag to indicate we are running the browser/app self.control_server._finished = False @@ -1200,6 +1209,7 @@ def main(args=sys.argv[1:]): symbols_path=args.symbols_path, host=args.host, power_test=args.power_test, + cpu_test=args.cpu_test, memory_test=args.memory_test, is_release_build=args.is_release_build, debug_mode=args.debug_mode, diff --git a/testing/raptor/test/files/top-info.txt b/testing/raptor/test/files/top-info.txt new file mode 100644 index 000000000000..91e413ccbee9 --- /dev/null +++ b/testing/raptor/test/files/top-info.txt @@ -0,0 +1,41 @@ +Tasks: 142 total, 1 running, 140 sleeping, 0 stopped, 1 zombie +Mem: 1548824k total, 1234756k used, 314068k free, 37080k buffers +Swap: 0k total, 0k used, 0k free, 552360k cached +200%cpu 122%user 9%nice 50%sys 13%idle 0%iow 0%irq 6%sirq 0%host + PID USER [%CPU]%CPU %MEM TIME+ ARGS +17504 u0_a83 93.7 93.7 14.2 0:12.12 org.mozilla.geckoview_example +17529 u0_a83 43.7 43.7 19.3 0:11.80 org.mozilla.geckoview_example:tab + 7030 u0_a54 28.1 28.1 5.6 0:05.47 com.google.android.tts + 1598 root 9.3 9.3 0.1 0:13.73 dhcpclient -i eth0 + 1667 system 6.2 6.2 9.6 16:10.78 system_server + 1400 system 6.2 6.2 0.2 8:15.20 android.hardware.sensors@1.0-service +17729 shell 3.1 3.1 0.1 0:00.02 top -O %CPU -n 1 + 1411 system 3.1 3.1 0.7 23:06.11 surfaceflinger +17497 shell 0.0 0.0 0.1 0:00.01 sh - +17321 root 0.0 0.0 0.0 0:00.13 [kworker/0:1] +17320 root 0.0 0.0 0.0 0:00.15 [kworker/u4:1] +17306 root 0.0 0.0 0.0 0:00.21 [kworker/u5:1] +16545 root 0.0 0.0 0.0 0:00.17 [kworker/0:0] +16543 root 0.0 0.0 0.0 0:00.15 [kworker/u4:2] +16411 root 0.0 0.0 0.0 0:00.41 [kworker/u5:2] +15827 root 0.0 0.0 0.0 0:00.04 [kworker/1:2] +14998 root 0.0 0.0 0.0 0:00.03 [kworker/1:1] +14996 root 0.0 0.0 0.0 0:00.38 [kworker/0:2] +14790 root 0.0 0.0 0.0 0:01.04 [kworker/u5:0] +14167 root 0.0 0.0 0.0 0:01.32 [kworker/u4:0] +11922 u0_a50 0.0 0.0 6.9 0:00.80 com.google.android.apps.docs +11906 u0_a67 0.0 0.0 5.0 0:00.25 com.google.android.apps.photos +11887 u0_a11 0.0 0.0 4.3 0:00.25 com.android.documentsui +11864 u0_a6 0.0 0.0 3.3 0:00.19 com.android.defcontainer +10866 u0_a15 0.0 0.0 3.3 0:00.04 com.google.android.partnersetup + 8956 u0_a1 0.0 0.0 3.7 0:00.40 com.android.providers.calendar + 8070 u0_a10 0.0 0.0 6.7 0:01.21 com.google.android.gms.unstable + 6638 u0_a10 0.0 0.0 7.4 0:12.89 com.google.android.gms + 2291 u0_a30 0.0 0.0 9.0 5:45.93 com.google.android.googlequicksearchbox:search + 2230 u0_a10 0.0 0.0 3.9 0:02.00 com.google.process.gapps + 2213 u0_a22 0.0 0.0 7.2 4:12.95 com.google.android.apps.nexuslauncher + 2195 u0_a30 0.0 0.0 4.1 0:00.37 com.google.android.googlequicksearchbox:interactor + 2163 u0_a10 0.0 0.0 8.2 1:49.32 com.google.android.gms.persistent + 1882 radio 0.0 0.0 5.1 0:53.61 com.android.phone + 1875 wifi 0.0 0.0 0.4 0:02.25 wpa_supplicant -Dnl80211 -iwlan0 -c/vendor/etc/wifi/wpa_supplicant.conf -g@android:wpa_wla+ + 1828 webview_zyg+ 0.0 0.0 3.0 0:00.45 webview_zygote32 \ No newline at end of file diff --git a/testing/raptor/test/python.ini b/testing/raptor/test/python.ini index 7b022ef63b04..3c0919eb0991 100644 --- a/testing/raptor/test/python.ini +++ b/testing/raptor/test/python.ini @@ -9,3 +9,4 @@ skip-if = python == 3 [test_playback.py] [test_print_tests.py] [test_raptor.py] +[test_cpu.py] diff --git a/testing/raptor/test/test_cmdline.py b/testing/raptor/test/test_cmdline.py index 82c51144320a..8833b198d8b1 100644 --- a/testing/raptor/test/test_cmdline.py +++ b/testing/raptor/test/test_cmdline.py @@ -17,6 +17,7 @@ def test_verify_options(filedir): page_timeout=60000, debug='True', power_test=False, + cpu_test=False, memory_test=False) parser = ArgumentParser() @@ -34,6 +35,7 @@ def test_verify_options(filedir): is_release_build=False, host='sophie', power_test=False, + cpu_test=False, memory_test=False) verify_options(parser, args) # assert no exception @@ -45,6 +47,7 @@ def test_verify_options(filedir): is_release_build=False, host='sophie', power_test=False, + cpu_test=False, memory_test=False) verify_options(parser, args) # assert no exception @@ -56,6 +59,19 @@ def test_verify_options(filedir): is_release_build=False, host='sophie', power_test=False, + cpu_test=False, + memory_test=False) + verify_options(parser, args) # assert no exception + + args = Namespace(app='geckoview', + binary='org.mozilla.geckoview_example', + activity='org.mozilla.geckoview_example.GeckoViewActivity', + intent='android.intent.action.MAIN', + gecko_profile='False', + is_release_build=False, + host='sophie', + power_test=False, + cpu_test=True, memory_test=False) verify_options(parser, args) # assert no exception @@ -67,6 +83,7 @@ def test_verify_options(filedir): is_release_build=False, host='sophie', power_test=False, + cpu_test=False, memory_test=False) parser = ArgumentParser() diff --git a/testing/raptor/test/test_cpu.py b/testing/raptor/test/test_cpu.py new file mode 100644 index 000000000000..8bf7d46a61b1 --- /dev/null +++ b/testing/raptor/test/test_cpu.py @@ -0,0 +1,125 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, unicode_literals + +import mozunit +import os +import mock + +from raptor import cpu +from raptor.raptor import RaptorAndroid + + +def test_no_device(): + raptor = RaptorAndroid('geckoview', 'org.mozilla.org.mozilla.geckoview_example', cpu_test=True) + raptor.device = None + resp = cpu.generate_android_cpu_profile(raptor, 'no_control_server_device') + + assert resp is None + + +def test_usage_with_invalid_data_returns_zero(): + with mock.patch('mozdevice.adb.ADBDevice') as device: + with mock.patch('raptor.raptor.RaptorControlServer') as control_server: + # Create a device that returns invalid data + device.shell_output.return_value = 'geckoview' + device.version = 8 + device._verbose = True + + # Create a control server + control_server.cpu_test = True + control_server.device = device + raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example', cpu_test=True) + raptor.config['cpu_test'] = True + raptor.control_server = control_server + raptor.device = device + + # Verify the call to submit data was made + cpuinfo_data = { + 'type': 'cpu', + 'test': 'usage_with_invalid_data_returns_zero', + 'unit': '%', + 'values': { + 'browser_cpu_usage': float(0) + } + } + cpu.generate_android_cpu_profile( + raptor, + "usage_with_invalid_data_returns_zero") + control_server.submit_supporting_data.assert_called_once_with(cpuinfo_data) + + +def test_usage_with_output(): + with mock.patch('mozdevice.adb.ADBDevice') as device: + with mock.patch('raptor.raptor.RaptorControlServer') as control_server: + # Override the shell output with sample CPU usage details + filepath = os.path.abspath(os.path.dirname(__file__)) + '/files/' + f = open(filepath + 'top-info.txt', 'r') + device.shell_output.return_value = f.read() + device._verbose = True + device.version = 8 + + # Create a control server + control_server.cpu_test = True + control_server.test_name = 'cpuunittest' + control_server.device = device + control_server.app_name = 'org.mozilla.geckoview_example' + raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example', cpu_test=True) + raptor.device = device + raptor.config['cpu_test'] = True + raptor.control_server = control_server + + # Verify the response contains our expected CPU % of 93.7 + cpuinfo_data = { + u'type': u'cpu', + u'test': u'usage_with_integer_cpu_info_output', + u'unit': u'%', + u'values': { + u'browser_cpu_usage': float(93.7) + } + } + cpu.generate_android_cpu_profile( + raptor, + "usage_with_integer_cpu_info_output") + control_server.submit_supporting_data.assert_called_once_with(cpuinfo_data) + + +def test_usage_with_fallback(): + with mock.patch('mozdevice.adb.ADBDevice') as device: + with mock.patch('raptor.raptor.RaptorControlServer') as control_server: + # We set the version to be less than Android 8 + device.version = 7 + device._verbose = True + + # Return what our shell call to dumpsys would give us + shell_output = ' 34% 14781/org.mozilla.geckoview_example: 26% user + 7.5% kernel' + device.shell_output.return_value = shell_output + + # Create a control server + control_server.cpu_test = True + control_server.test_name = 'cpuunittest' + control_server.device = device + control_server.app_name = 'org.mozilla.geckoview_example' + raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example', cpu_test=True) + raptor.device = device + raptor.config['cpu_test'] = True + raptor.control_server = control_server + + # Verify the response contains our expected CPU % of 34 + cpuinfo_data = { + u'type': u'cpu', + u'test': u'usage_with_fallback', + u'unit': u'%', + u'values': { + u'browser_cpu_usage': float(34) + } + } + cpu.generate_android_cpu_profile( + raptor, + "usage_with_fallback") + control_server.submit_supporting_data.assert_called_once_with(cpuinfo_data) + + +if __name__ == '__main__': + mozunit.main()