diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py index f29c7d3e8516..969b5698f80e 100644 --- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -140,10 +140,18 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): }], [["--power-test"], { "dest": "power_test", + "action": "store_true", + "default": False, "help": "Use Raptor to measure power usage. Currently only supported for Geckoview. " "The host ip address must be specified either via the --host command line " "argument.", }], + [["--memory-test"], { + "dest": "memory_test", + "action": "store_true", + "default": False, + "help": "Use Raptor to measure memory usage.", + }], [["--debug-mode"], { "dest": "debug_mode", "action": "store_true", @@ -225,6 +233,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): if self.host == 'HOST_IP': self.host = os.environ['HOST_IP'] self.power_test = self.config.get('power_test') + self.memory_test = self.config.get('memory_test') self.is_release_build = self.config.get('is_release_build') self.debug_mode = self.config.get('debug_mode', False) self.firefox_android_browsers = ["fennec", "geckoview", "refbrow", "fenix"] @@ -362,6 +371,8 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): options.extend(['--is-release-build']) if self.config.get('power_test', False): options.extend(['--power-test']) + if self.config.get('memory_test', False): + options.extend(['--memory-test']) for key, value in kw_options.items(): options.extend(['--%s' % key, value]) @@ -473,6 +484,8 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): expected_perfherder = 1 if self.config.get('power_test', None): expected_perfherder += 1 + if self.config.get('memory_test', None): + expected_perfherder += 1 if len(parser.found_perf_data) != expected_perfherder: self.critical("PERFHERDER_DATA was seen %d times, expected %d." % (len(parser.found_perf_data), expected_perfherder)) @@ -608,6 +621,10 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-power.json') self._artifact_perf_data(src, dest) + if self.memory_test: + src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-memory.json') + self._artifact_perf_data(src, dest) + src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'screenshots.html') if os.path.exists(src): dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'screenshots.html') diff --git a/testing/raptor/mach_commands.py b/testing/raptor/mach_commands.py index 0702e30341a9..a59b50944b58 100644 --- a/testing/raptor/mach_commands.py +++ b/testing/raptor/mach_commands.py @@ -60,6 +60,7 @@ class RaptorRunner(MozbuildObject): kwargs['host'] = os.environ['HOST_IP'] self.host = kwargs['host'] self.power_test = kwargs['power_test'] + self.memory_test = kwargs['memory_test'] self.is_release_build = kwargs['is_release_build'] def setup_benchmarks(self): @@ -139,6 +140,7 @@ class RaptorRunner(MozbuildObject): 'raptor_cmd_line_args': self.raptor_args, 'host': self.host, 'power_test': self.power_test, + 'memory_test': self.memory_test, 'is_release_build': self.is_release_build, } diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index 099169f282d5..0a0ed4152eac 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -69,6 +69,8 @@ def create_parser(mach_interface=False): add_arg('--power-test', dest="power_test", action="store_true", help="Use Raptor to measure power usage. Currently supported for Geckoview. " "The host ip address must be specified via the --host command line argument.") + add_arg('--memory-test', dest="memory_test", action="store_true", + help="Use Raptor to measure memory usage.") add_arg('--is-release-build', dest="is_release_build", default=False, action='store_true', help="Whether the build is a release build which requires work arounds " diff --git a/testing/raptor/raptor/control_server.py b/testing/raptor/raptor/control_server.py index ed08ab47ecda..0e2174b2f684 100644 --- a/testing/raptor/raptor/control_server.py +++ b/testing/raptor/raptor/control_server.py @@ -7,10 +7,12 @@ from __future__ import absolute_import import BaseHTTPServer +import datetime import json import os import socket import threading +import time from mozlog import get_proxy_logger @@ -22,6 +24,81 @@ here = os.path.abspath(os.path.dirname(__file__)) def MakeCustomHandlerClass(results_handler, shutdown_browser, write_raw_gecko_profile): class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler, object): + """ + Control server expects messages of the form + {'type': 'messagetype', 'data':...} + + Each message is given a key which is calculated as + + If type is 'webext_status', then + the key is data['type']/data['data'] + otherwise + the key is data['type']. + + The contol server can be forced to wait before performing an + action requested via POST by sending a special message + + {'type': 'wait-set', 'data': key} + + where key is the key of the message control server should + perform a wait before processing. The handler will store + this key in the wait_after_messages dict as a True value. + + wait_after_messages[key] = True + + For subsequent requests the handler will check the key of + the incoming message against wait_for_messages and if it is + found and its value is True, the handler will assign the key + to waiting_in_state and will loop until the key is removed + or until its value is changed to False. + + Control server will stop waiting for a state to be continued + or cleared after wait_timeout seconds after which the wait + will be removed and the control server will finish processing + the current POST request. wait_timeout defaults to 60 seconds + but can be set globally for all wait states by sending the + message + + {'type': 'wait-timeout', 'data': timeout} + + The value of waiting_in_state can be retrieved by sending the + message + + {'type': 'wait-get', 'data': ''} + + which will return the value of waiting_in_state in the + content of the response. If the value returned is not + 'None', then the control server has received a message whose + key is recorded in wait_after_messages and is waiting before + completing the request. + + The control server can be told to stop waiting and to finish + processing the current request while keeping the wait for + subsequent requests by sending + + {'type': 'wait-continue', 'data': ''} + + The control server can be told to stop waiting and to finish + processing the current request while removing the wait for + subsequent requests by sending + + {'type': 'wait-clear', 'data': key} + + if key is the empty string '' + the key in waiting_in_state is removed from wait_after_messages + waiting_in_state is set to None + else if key is 'all' + all keys in wait_after_messages are removed + else key is not in wait_after_messages + the message is ignored + else + the key is removed from wait_after messages + if the key matches the value in waiting_in_state, + then waiting_in_state is set to None + """ + wait_after_messages = {} + waiting_in_state = None + wait_timeout = 60 def __init__(self, *args, **kwargs): self.results_handler = results_handler @@ -65,6 +142,32 @@ def MakeCustomHandlerClass(results_handler, shutdown_browser, write_raw_gecko_pr # could have received a status update or test results data = json.loads(post_body) + if data['type'] == 'webext_status': + wait_key = "%s/%s" % (data['type'], data['data']) + else: + wait_key = data['type'] + + if MyHandler.wait_after_messages.get(wait_key, None): + LOG.info("Waiting in %s" % wait_key) + MyHandler.waiting_in_state = wait_key + start_time = datetime.datetime.now() + + while MyHandler.wait_after_messages.get(wait_key, None): + time.sleep(1) + elapsed_time = datetime.datetime.now() - start_time + if elapsed_time > datetime.timedelta(seconds=MyHandler.wait_timeout): + del MyHandler.wait_after_messages[wait_key] + MyHandler.waiting_in_state = None + LOG.error("TEST-UNEXPECTED-ERROR | " + "ControlServer wait %s exceeded %s seconds" % + (wait_key, MyHandler.wait_timeout)) + + if MyHandler.wait_after_messages.get(wait_key, None) is not None: + # If the wait is False, it was continued and we just set it back + # to True for the next time. If it was removed by clear, we + # leave it alone so it will not cause a wait any more. + MyHandler.wait_after_messages[wait_key] = True + if data['type'] == "webext_gecko_profile": # received gecko profiling results _test = str(data['data'][0]) @@ -84,7 +187,7 @@ def MakeCustomHandlerClass(results_handler, shutdown_browser, write_raw_gecko_pr self.results_handler.add_page_timeout(str(data['data'][0]), str(data['data'][1]), dict(data['data'][2])) - elif data['data'] == "__raptor_shutdownBrowser": + elif data['type'] == 'webext_status' and data['data'] == "__raptor_shutdownBrowser": LOG.info("received " + data['type'] + ": " + str(data['data'])) # webext is telling us it's done, and time to shutdown the browser self.shutdown_browser() @@ -93,6 +196,39 @@ def MakeCustomHandlerClass(results_handler, shutdown_browser, write_raw_gecko_pr self.results_handler.add_image(str(data['data'][0]), str(data['data'][1]), str(data['data'][2])) + elif data['type'] == 'webext_status': + LOG.info("received " + data['type'] + ": " + str(data['data'])) + elif data['type'] == 'wait-set': + LOG.info("received " + data['type'] + ": " + str(data['data'])) + MyHandler.wait_after_messages[str(data['data'])] = True + elif data['type'] == 'wait-timeout': + LOG.info("received " + data['type'] + ": " + str(data['data'])) + MyHandler.wait_timeout = data['data'] + elif data['type'] == 'wait-get': + self.wfile.write(MyHandler.waiting_in_state) + elif data['type'] == 'wait-continue': + LOG.info("received " + data['type'] + ": " + str(data['data'])) + if MyHandler.waiting_in_state: + MyHandler.wait_after_messages[MyHandler.waiting_in_state] = False + MyHandler.waiting_in_state = None + elif data['type'] == 'wait-clear': + LOG.info("received " + data['type'] + ": " + str(data['data'])) + clear_key = str(data['data']) + if clear_key == '': + if MyHandler.waiting_in_state: + del MyHandler.wait_after_messages[MyHandler.waiting_in_state] + MyHandler.waiting_in_state = None + else: + pass + elif clear_key == 'all': + MyHandler.wait_after_messages = {} + MyHandler.waiting_in_state = None + elif clear_key not in MyHandler.wait_after_messages: + pass + else: + del MyHandler.wait_after_messages[clear_key] + if MyHandler.waiting_in_state == clear_key: + MyHandler.waiting_in_state = None else: LOG.info("received " + data['type'] + ": " + str(data['data'])) diff --git a/testing/raptor/raptor/memory.py b/testing/raptor/raptor/memory.py new file mode 100644 index 000000000000..2102d253ad7a --- /dev/null +++ b/testing/raptor/raptor/memory.py @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import re + + +def get_app_memory_usage(raptor): + app_name = raptor.config['binary'] + total = 0 + re_total_memory = re.compile(r'TOTAL:\s+(\d+)') + verbose = raptor.device._verbose + raptor.device._verbose = False + meminfo = raptor.device.shell_output("dumpsys meminfo %s" % app_name).split('\n') + raptor.device._verbose = verbose + for line in meminfo: + match = re_total_memory.search(line) + if match: + total = int(match.group(1)) + break + return total + + +def generate_android_memory_profile(raptor, test_name): + if not raptor.device or not raptor.config['memory_test']: + return + foreground = get_app_memory_usage(raptor) + # put app into background + verbose = raptor.device._verbose + raptor.device._verbose = False + raptor.device.shell_output("am start -a android.intent.action.MAIN " + "-c android.intent.category.HOME") + raptor.device._verbose = verbose + background = get_app_memory_usage(raptor) + meminfo_data = {'type': 'memory', + 'test': test_name, + 'unit': 'KB', + 'values': { + 'foreground': foreground, + 'background': background + }} + raptor.control_server.submit_supporting_data(meminfo_data) diff --git a/testing/raptor/raptor/raptor.py b/testing/raptor/raptor/raptor.py index 4967efb49dc8..a7faf11d3ccf 100644 --- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -13,6 +13,8 @@ import sys import tempfile import time +import requests + import mozcrash import mozinfo from mozdevice import ADBDevice @@ -57,6 +59,7 @@ from mozproxy import get_playback from results import RaptorResultsHandler from gecko_profile import GeckoProfile from power import init_android_power_test, finish_android_power_test +from memory import generate_android_memory_profile from utils import view_gecko_profile @@ -65,8 +68,8 @@ class Raptor(object): def __init__(self, app, binary, run_local=False, obj_path=None, gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None, - symbols_path=None, host=None, power_test=False, is_release_build=False, - debug_mode=False, post_startup_delay=None, activity=None): + symbols_path=None, host=None, power_test=False, memory_test=False, + is_release_build=False, debug_mode=False, post_startup_delay=None, activity=None): # Override the magic --host HOST_IP with the value of the environment variable. if host == 'HOST_IP': @@ -84,7 +87,9 @@ class Raptor(object): self.config['symbols_path'] = symbols_path self.config['host'] = host self.config['power_test'] = power_test + self.config['memory_test'] = memory_test self.config['is_release_build'] = is_release_build + self.config['enable_control_server_wait'] = memory_test self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv') self.log = get_default_logger(component='raptor-main') self.control_server = None @@ -187,6 +192,9 @@ class Raptor(object): self.control_server = RaptorControlServer(self.results_handler, self.debug_mode) self.control_server.start() + if self.config['enable_control_server_wait']: + self.control_server_wait_set('webext_status/__raptor_shutdownBrowser') + # for android we must make the control server available to the device if self.config['app'] in self.firefox_android_apps and \ self.config['host'] in ('localhost', '127.0.0.1'): @@ -336,6 +344,12 @@ class Raptor(object): elapsed_time = 0 while not self.control_server._finished: + if self.config['enable_control_server_wait']: + response = self.control_server_wait_get() + if response == 'webext_status/__raptor_shutdownBrowser': + if self.config['memory_test']: + generate_android_memory_profile(self, test['name']) + self.control_server_wait_continue() time.sleep(1) # we only want to force browser-shutdown on timeout if not in debug mode; # in debug-mode we leave the browser running (require manual shutdown) @@ -365,6 +379,9 @@ class Raptor(object): return self.results_handler.page_timeout_list def clean_up(self): + if self.config['enable_control_server_wait']: + self.control_server_wait_clear('all') + self.control_server.stop() if self.config['app'] not in self.firefox_android_apps: self.runner.stop() @@ -375,15 +392,40 @@ class Raptor(object): pass self.log.info("finished") + def control_server_wait_set(self, state): + response = requests.post("http://127.0.0.1:%s/" % self.control_server.port, + json={"type": "wait-set", "data": state}) + return response.content + + def control_server_wait_timeout(self, timeout): + response = requests.post("http://127.0.0.1:%s/" % self.control_server.port, + json={"type": "wait-timeout", "data": timeout}) + return response.content + + def control_server_wait_get(self): + response = requests.post("http://127.0.0.1:%s/" % self.control_server.port, + json={"type": "wait-get", "data": ""}) + return response.content + + def control_server_wait_continue(self): + response = requests.post("http://127.0.0.1:%s/" % self.control_server.port, + json={"type": "wait-continue", "data": ""}) + return response.content + + def control_server_wait_clear(self, state): + response = requests.post("http://127.0.0.1:%s/" % self.control_server.port, + json={"type": "wait-clear", "data": state}) + return response.content + class RaptorDesktop(Raptor): def __init__(self, app, binary, run_local=False, obj_path=None, gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None, - symbols_path=None, host=None, power_test=False, is_release_build=False, - debug_mode=False, post_startup_delay=None, activity=None): + symbols_path=None, host=None, power_test=False, memory_test=False, + is_release_build=False, debug_mode=False, post_startup_delay=None, activity=None): Raptor.__init__(self, app, binary, run_local, obj_path, gecko_profile, gecko_profile_interval, gecko_profile_entries, symbols_path, - host, power_test, is_release_build, debug_mode, + host, power_test, memory_test, is_release_build, debug_mode, post_startup_delay) def create_browser_handler(self): @@ -446,11 +488,11 @@ class RaptorDesktop(Raptor): class RaptorDesktopFirefox(RaptorDesktop): def __init__(self, app, binary, run_local=False, obj_path=None, gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None, - symbols_path=None, host=None, power_test=False, is_release_build=False, - debug_mode=False, post_startup_delay=None, activity=None): + symbols_path=None, host=None, power_test=False, memory_test=False, + is_release_build=False, debug_mode=False, post_startup_delay=None, activity=None): RaptorDesktop.__init__(self, app, binary, run_local, obj_path, gecko_profile, gecko_profile_interval, gecko_profile_entries, symbols_path, - host, power_test, is_release_build, debug_mode, + host, power_test, memory_test, is_release_build, debug_mode, post_startup_delay) def disable_non_local_connections(self): @@ -491,11 +533,11 @@ class RaptorDesktopFirefox(RaptorDesktop): class RaptorDesktopChrome(RaptorDesktop): def __init__(self, app, binary, run_local=False, obj_path=None, gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None, - symbols_path=None, host=None, power_test=False, is_release_build=False, - debug_mode=False, post_startup_delay=None, activity=None): + symbols_path=None, host=None, power_test=False, memory_test=False, + is_release_build=False, debug_mode=False, post_startup_delay=None, activity=None): RaptorDesktop.__init__(self, app, binary, run_local, obj_path, gecko_profile, gecko_profile_interval, gecko_profile_entries, symbols_path, - host, power_test, is_release_build, debug_mode, + host, power_test, memory_test, is_release_build, debug_mode, post_startup_delay) def setup_chrome_desktop_for_playback(self): @@ -530,11 +572,11 @@ class RaptorDesktopChrome(RaptorDesktop): class RaptorAndroid(Raptor): def __init__(self, app, binary, run_local=False, obj_path=None, gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None, - symbols_path=None, host=None, power_test=False, is_release_build=False, - debug_mode=False, post_startup_delay=None, activity=None): + symbols_path=None, host=None, power_test=False, memory_test=False, + is_release_build=False, debug_mode=False, post_startup_delay=None, activity=None): Raptor.__init__(self, app, binary, run_local, obj_path, gecko_profile, gecko_profile_interval, gecko_profile_entries, symbols_path, host, - power_test, is_release_build, debug_mode, post_startup_delay) + power_test, memory_test, is_release_build, debug_mode, post_startup_delay) # on android, when creating the browser profile, we want to use a 'firefox' type profile self.profile_class = "firefox" @@ -847,6 +889,7 @@ def main(args=sys.argv[1:]): symbols_path=args.symbols_path, host=args.host, power_test=args.power_test, + memory_test=args.memory_test, is_release_build=args.is_release_build, debug_mode=args.debug_mode, post_startup_delay=args.post_startup_delay, diff --git a/testing/raptor/test/test_cmdline.py b/testing/raptor/test/test_cmdline.py index 11ae58386f0b..b5457ef20d97 100644 --- a/testing/raptor/test/test_cmdline.py +++ b/testing/raptor/test/test_cmdline.py @@ -16,7 +16,8 @@ def test_verify_options(filedir): page_cycles=1, page_timeout=60000, debug='True', - power_test=False) + power_test=False, + memory_test=False) parser = ArgumentParser() with pytest.raises(SystemExit): @@ -31,7 +32,8 @@ def test_verify_options(filedir): gecko_profile='False', is_release_build=False, host='sophie', - power_test=False) + power_test=False, + memory_test=False) verify_options(parser, args) # assert no exception args = Namespace(app='refbrow', @@ -40,7 +42,8 @@ def test_verify_options(filedir): gecko_profile='False', is_release_build=False, host='sophie', - power_test=False) + power_test=False, + memory_test=False) verify_options(parser, args) # assert no exception args = Namespace(app='fenix', @@ -49,7 +52,8 @@ def test_verify_options(filedir): gecko_profile='False', is_release_build=False, host='sophie', - power_test=False) + power_test=False, + memory_test=False) verify_options(parser, args) # assert no exception args = Namespace(app='refbrow', @@ -58,7 +62,8 @@ def test_verify_options(filedir): gecko_profile='False', is_release_build=False, host='sophie', - power_test=False) + power_test=False, + memory_test=False) parser = ArgumentParser() verify_options(parser, args) # also will work as uses default activity diff --git a/testing/raptor/test/test_raptor.py b/testing/raptor/test/test_raptor.py index 12e8c731623e..6f5a37280bb4 100644 --- a/testing/raptor/test/test_raptor.py +++ b/testing/raptor/test/test_raptor.py @@ -91,6 +91,67 @@ def test_start_and_stop_server(raptor): assert not raptor.control_server._server_thread.is_alive() +def test_server_wait_states(raptor): + import datetime + + import requests + + def post_state(): + requests.post("http://127.0.0.1:%s/" % raptor.control_server.port, + json={"type": "webext_status", + "data": "test status"}) + + assert raptor.control_server is None + + raptor.create_browser_profile() + raptor.create_browser_handler() + raptor.start_control_server() + + wait_time = 5 + message_state = 'webext_status/test status' + rhc = raptor.control_server.server.RequestHandlerClass + + # Test initial state + assert rhc.wait_after_messages == {} + assert rhc.waiting_in_state is None + assert rhc.wait_timeout == 60 + assert raptor.control_server_wait_get() == 'None' + + # Test setting a state + assert raptor.control_server_wait_set(message_state) == '' + assert message_state in rhc.wait_after_messages + assert rhc.wait_after_messages[message_state] + + # Test clearing a non-existent state + assert raptor.control_server_wait_clear('nothing') == '' + assert message_state in rhc.wait_after_messages + + # Test clearing a state + assert raptor.control_server_wait_clear(message_state) == '' + assert message_state not in rhc.wait_after_messages + + # Test clearing all states + assert raptor.control_server_wait_set(message_state) == '' + assert message_state in rhc.wait_after_messages + assert raptor.control_server_wait_clear('all') == '' + assert rhc.wait_after_messages == {} + + # Test wait timeout + # Block on post request + assert raptor.control_server_wait_set(message_state) == '' + assert rhc.wait_after_messages[message_state] + assert raptor.control_server_wait_timeout(wait_time) == '' + assert rhc.wait_timeout == wait_time + start = datetime.datetime.now() + post_state() + assert datetime.datetime.now() - start < datetime.timedelta(seconds=wait_time+2) + assert raptor.control_server_wait_get() == 'None' + assert message_state not in rhc.wait_after_messages + + raptor.clean_up() + assert not raptor.control_server._server_thread.is_alive() + + @pytest.mark.parametrize('app', [ 'firefox', pytest.mark.xfail('chrome'),