Bug 1533848 - Implement Android memory test using control server waits, r=rwood.

Depends on D26973

Differential Revision: https://phabricator.services.mozilla.com/D26974

--HG--
extra : moz-landing-system : lando

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -140,10 +140,18 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
         }],
         [["--power-test"], {
             "dest": "power_test",
+            "action": "store_true",
+            "default": False,
             "help": "Use Raptor to measure power usage. Currently only supported for Geckoview. "
                     "The host ip address must be specified either via the --host command line "
                     "argument.",
         }],
+        [["--memory-test"], {
+            "dest": "memory_test",
+            "action": "store_true",
+            "default": False,
+            "help": "Use Raptor to measure memory usage.",
+        }],
         [["--debug-mode"], {
             "dest": "debug_mode",
             "action": "store_true",
@@ -225,6 +233,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
         if self.host == 'HOST_IP':
             self.host = os.environ['HOST_IP']
         self.power_test = self.config.get('power_test')
+        self.memory_test = self.config.get('memory_test')
         self.is_release_build = self.config.get('is_release_build')
         self.debug_mode = self.config.get('debug_mode', False)
         self.firefox_android_browsers = ["fennec", "geckoview", "refbrow", "fenix"]
@@ -362,6 +371,8 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
             options.extend(['--is-release-build'])
         if self.config.get('power_test', False):
             options.extend(['--power-test'])
+        if self.config.get('memory_test', False):
+            options.extend(['--memory-test'])
         for key, value in kw_options.items():
             options.extend(['--%s' % key, value])
 
@@ -473,6 +484,8 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
         expected_perfherder = 1
         if self.config.get('power_test', None):
             expected_perfherder += 1
+        if self.config.get('memory_test', None):
+            expected_perfherder += 1
         if len(parser.found_perf_data) != expected_perfherder:
             self.critical("PERFHERDER_DATA was seen %d times, expected %d."
                           % (len(parser.found_perf_data), expected_perfherder))
@@ -608,6 +621,10 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
                     src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-power.json')
                     self._artifact_perf_data(src, dest)
 
+                if self.memory_test:
+                    src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-memory.json')
+                    self._artifact_perf_data(src, dest)
+
                 src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'screenshots.html')
                 if os.path.exists(src):
                     dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'screenshots.html')

testing/raptor/mach_commands.py

@@ -60,6 +60,7 @@ class RaptorRunner(MozbuildObject):
             kwargs['host'] = os.environ['HOST_IP']
         self.host = kwargs['host']
         self.power_test = kwargs['power_test']
+        self.memory_test = kwargs['memory_test']
         self.is_release_build = kwargs['is_release_build']
 
     def setup_benchmarks(self):
@@ -139,6 +140,7 @@ class RaptorRunner(MozbuildObject):
             'raptor_cmd_line_args': self.raptor_args,
             'host': self.host,
             'power_test': self.power_test,
+            'memory_test': self.memory_test,
             'is_release_build': self.is_release_build,
         }
 

testing/raptor/raptor/cmdline.py

@@ -69,6 +69,8 @@ def create_parser(mach_interface=False):
     add_arg('--power-test', dest="power_test", action="store_true",
             help="Use Raptor to measure power usage. Currently supported for Geckoview. "
             "The host ip address must be specified via the --host command line argument.")
+    add_arg('--memory-test', dest="memory_test", action="store_true",
+            help="Use Raptor to measure memory usage.")
     add_arg('--is-release-build', dest="is_release_build", default=False,
             action='store_true',
             help="Whether the build is a release build which requires work arounds "

testing/raptor/raptor/control_server.py

@@ -7,10 +7,12 @@
 from __future__ import absolute_import
 
 import BaseHTTPServer
+import datetime
 import json
 import os
 import socket
 import threading
+import time
 
 from mozlog import get_proxy_logger
 
@@ -22,6 +24,81 @@ here = os.path.abspath(os.path.dirname(__file__))
 def MakeCustomHandlerClass(results_handler, shutdown_browser, write_raw_gecko_profile):
 
     class MyHandler(BaseHTTPServer.BaseHTTPRequestHandler, object):
+        """
+        Control server expects messages of the form
+        {'type': 'messagetype', 'data':...}
+
+        Each message is given a key which is calculated as
+
+           If type is 'webext_status', then
+              the key is data['type']/data['data']
+           otherwise
+              the key is data['type'].
+
+        The contol server can be forced to wait before performing an
+        action requested via POST by sending a special message
+
+        {'type': 'wait-set', 'data': key}
+
+        where key is the key of the message control server should
+        perform a wait before processing. The handler will store
+        this key in the wait_after_messages dict as a True value.
+
+        wait_after_messages[key] = True
+
+        For subsequent requests the handler will check the key of
+        the incoming message against wait_for_messages and if it is
+        found and its value is True, the handler will assign the key
+        to waiting_in_state and will loop until the key is removed
+        or until its value is changed to False.
+
+        Control server will stop waiting for a state to be continued
+        or cleared after wait_timeout seconds after which the wait
+        will be removed and the control server will finish processing
+        the current POST request. wait_timeout defaults to 60 seconds
+        but can be set globally for all wait states by sending the
+        message
+
+        {'type': 'wait-timeout', 'data': timeout}
+
+        The value of waiting_in_state can be retrieved by sending the
+        message
+
+        {'type': 'wait-get', 'data': ''}
+
+        which will return the value of waiting_in_state in the
+        content of the response. If the value returned is not
+        'None', then the control server has received a message whose
+        key is recorded in wait_after_messages and is waiting before
+        completing the request.
+
+        The control server can be told to stop waiting and to finish
+        processing the current request while keeping the wait for
+        subsequent requests by sending
+
+        {'type': 'wait-continue', 'data': ''}
+
+        The control server can be told to stop waiting and to finish
+        processing the current request while removing the wait for
+        subsequent requests by sending
+
+        {'type': 'wait-clear', 'data': key}
+
+            if key is the empty string ''
+                the key in waiting_in_state is removed from wait_after_messages
+                waiting_in_state is set to None
+            else if key is 'all'
+                 all keys in wait_after_messages are removed
+            else key is not in wait_after_messages
+                 the message is ignored
+            else
+                 the key is removed from wait_after messages
+                 if the key matches the value in waiting_in_state,
+                 then waiting_in_state is set to None
+        """
+        wait_after_messages = {}
+        waiting_in_state = None
+        wait_timeout = 60
 
         def __init__(self, *args, **kwargs):
             self.results_handler = results_handler
@@ -65,6 +142,32 @@ def MakeCustomHandlerClass(results_handler, shutdown_browser, write_raw_gecko_pr
             # could have received a status update or test results
             data = json.loads(post_body)
 
+            if data['type'] == 'webext_status':
+                wait_key = "%s/%s" % (data['type'], data['data'])
+            else:
+                wait_key = data['type']
+
+            if MyHandler.wait_after_messages.get(wait_key, None):
+                LOG.info("Waiting in %s" % wait_key)
+                MyHandler.waiting_in_state = wait_key
+                start_time = datetime.datetime.now()
+
+            while MyHandler.wait_after_messages.get(wait_key, None):
+                time.sleep(1)
+                elapsed_time = datetime.datetime.now() - start_time
+                if elapsed_time > datetime.timedelta(seconds=MyHandler.wait_timeout):
+                    del MyHandler.wait_after_messages[wait_key]
+                    MyHandler.waiting_in_state = None
+                    LOG.error("TEST-UNEXPECTED-ERROR | "
+                              "ControlServer wait %s exceeded %s seconds" %
+                              (wait_key, MyHandler.wait_timeout))
+
+            if MyHandler.wait_after_messages.get(wait_key, None) is not None:
+                # If the wait is False, it was continued and we just set it back
+                # to True for the next time. If it was removed by clear, we
+                # leave it alone so it will not cause a wait any more.
+                MyHandler.wait_after_messages[wait_key] = True
+
             if data['type'] == "webext_gecko_profile":
                 # received gecko profiling results
                 _test = str(data['data'][0])
@@ -84,7 +187,7 @@ def MakeCustomHandlerClass(results_handler, shutdown_browser, write_raw_gecko_pr
                 self.results_handler.add_page_timeout(str(data['data'][0]),
                                                       str(data['data'][1]),
                                                       dict(data['data'][2]))
-            elif data['data'] == "__raptor_shutdownBrowser":
+            elif data['type'] == 'webext_status' and data['data'] == "__raptor_shutdownBrowser":
                 LOG.info("received " + data['type'] + ": " + str(data['data']))
                 # webext is telling us it's done, and time to shutdown the browser
                 self.shutdown_browser()
@@ -93,6 +196,39 @@ def MakeCustomHandlerClass(results_handler, shutdown_browser, write_raw_gecko_pr
                 self.results_handler.add_image(str(data['data'][0]),
                                                str(data['data'][1]),
                                                str(data['data'][2]))
+            elif data['type'] == 'webext_status':
+                LOG.info("received " + data['type'] + ": " + str(data['data']))
+            elif data['type'] == 'wait-set':
+                LOG.info("received " + data['type'] + ": " + str(data['data']))
+                MyHandler.wait_after_messages[str(data['data'])] = True
+            elif data['type'] == 'wait-timeout':
+                LOG.info("received " + data['type'] + ": " + str(data['data']))
+                MyHandler.wait_timeout = data['data']
+            elif data['type'] == 'wait-get':
+                self.wfile.write(MyHandler.waiting_in_state)
+            elif data['type'] == 'wait-continue':
+                LOG.info("received " + data['type'] + ": " + str(data['data']))
+                if MyHandler.waiting_in_state:
+                    MyHandler.wait_after_messages[MyHandler.waiting_in_state] = False
+                    MyHandler.waiting_in_state = None
+            elif data['type'] == 'wait-clear':
+                LOG.info("received " + data['type'] + ": " + str(data['data']))
+                clear_key = str(data['data'])
+                if clear_key == '':
+                    if MyHandler.waiting_in_state:
+                        del MyHandler.wait_after_messages[MyHandler.waiting_in_state]
+                        MyHandler.waiting_in_state = None
+                    else:
+                        pass
+                elif clear_key == 'all':
+                    MyHandler.wait_after_messages = {}
+                    MyHandler.waiting_in_state = None
+                elif clear_key not in MyHandler.wait_after_messages:
+                    pass
+                else:
+                    del MyHandler.wait_after_messages[clear_key]
+                    if MyHandler.waiting_in_state == clear_key:
+                        MyHandler.waiting_in_state = None
             else:
                 LOG.info("received " + data['type'] + ": " + str(data['data']))
 

testing/raptor/raptor/memory.py

@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+import re
+
+
+def get_app_memory_usage(raptor):
+    app_name = raptor.config['binary']
+    total = 0
+    re_total_memory = re.compile(r'TOTAL:\s+(\d+)')
+    verbose = raptor.device._verbose
+    raptor.device._verbose = False
+    meminfo = raptor.device.shell_output("dumpsys meminfo %s" % app_name).split('\n')
+    raptor.device._verbose = verbose
+    for line in meminfo:
+        match = re_total_memory.search(line)
+        if match:
+            total = int(match.group(1))
+            break
+    return total
+
+
+def generate_android_memory_profile(raptor, test_name):
+    if not raptor.device or not raptor.config['memory_test']:
+        return
+    foreground = get_app_memory_usage(raptor)
+    # put app into background
+    verbose = raptor.device._verbose
+    raptor.device._verbose = False
+    raptor.device.shell_output("am start -a android.intent.action.MAIN "
+                               "-c android.intent.category.HOME")
+    raptor.device._verbose = verbose
+    background = get_app_memory_usage(raptor)
+    meminfo_data = {'type': 'memory',
+                    'test': test_name,
+                    'unit': 'KB',
+                    'values': {
+                        'foreground': foreground,
+                        'background': background
+                    }}
+    raptor.control_server.submit_supporting_data(meminfo_data)

testing/raptor/raptor/raptor.py

@@ -13,6 +13,8 @@ import sys
 import tempfile
 import time
 
+import requests
+
 import mozcrash
 import mozinfo
 from mozdevice import ADBDevice
@@ -57,6 +59,7 @@ from mozproxy import get_playback
 from results import RaptorResultsHandler
 from gecko_profile import GeckoProfile
 from power import init_android_power_test, finish_android_power_test
+from memory import generate_android_memory_profile
 from utils import view_gecko_profile
 
 
@@ -65,8 +68,8 @@ class Raptor(object):
 
     def __init__(self, app, binary, run_local=False, obj_path=None,
                  gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
-                 symbols_path=None, host=None, power_test=False, is_release_build=False,
-                 debug_mode=False, post_startup_delay=None, activity=None):
+                 symbols_path=None, host=None, power_test=False, memory_test=False,
+                 is_release_build=False, debug_mode=False, post_startup_delay=None, activity=None):
 
         # Override the magic --host HOST_IP with the value of the environment variable.
         if host == 'HOST_IP':
@@ -84,7 +87,9 @@ class Raptor(object):
         self.config['symbols_path'] = symbols_path
         self.config['host'] = host
         self.config['power_test'] = power_test
+        self.config['memory_test'] = memory_test
         self.config['is_release_build'] = is_release_build
+        self.config['enable_control_server_wait'] = memory_test
         self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
         self.log = get_default_logger(component='raptor-main')
         self.control_server = None
@@ -187,6 +192,9 @@ class Raptor(object):
         self.control_server = RaptorControlServer(self.results_handler, self.debug_mode)
         self.control_server.start()
 
+        if self.config['enable_control_server_wait']:
+            self.control_server_wait_set('webext_status/__raptor_shutdownBrowser')
+
         # for android we must make the control server available to the device
         if self.config['app'] in self.firefox_android_apps and \
                 self.config['host'] in ('localhost', '127.0.0.1'):
@@ -336,6 +344,12 @@ class Raptor(object):
 
         elapsed_time = 0
         while not self.control_server._finished:
+            if self.config['enable_control_server_wait']:
+                response = self.control_server_wait_get()
+                if response == 'webext_status/__raptor_shutdownBrowser':
+                    if self.config['memory_test']:
+                        generate_android_memory_profile(self, test['name'])
+                    self.control_server_wait_continue()
             time.sleep(1)
             # we only want to force browser-shutdown on timeout if not in debug mode;
             # in debug-mode we leave the browser running (require manual shutdown)
@@ -365,6 +379,9 @@ class Raptor(object):
         return self.results_handler.page_timeout_list
 
     def clean_up(self):
+        if self.config['enable_control_server_wait']:
+            self.control_server_wait_clear('all')
+
         self.control_server.stop()
         if self.config['app'] not in self.firefox_android_apps:
             self.runner.stop()
@@ -375,15 +392,40 @@ class Raptor(object):
             pass
         self.log.info("finished")
 
+    def control_server_wait_set(self, state):
+        response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
+                                 json={"type": "wait-set", "data": state})
+        return response.content
+
+    def control_server_wait_timeout(self, timeout):
+        response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
+                                 json={"type": "wait-timeout", "data": timeout})
+        return response.content
+
+    def control_server_wait_get(self):
+        response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
+                                 json={"type": "wait-get", "data": ""})
+        return response.content
+
+    def control_server_wait_continue(self):
+        response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
+                                 json={"type": "wait-continue", "data": ""})
+        return response.content
+
+    def control_server_wait_clear(self, state):
+        response = requests.post("http://127.0.0.1:%s/" % self.control_server.port,
+                                 json={"type": "wait-clear", "data": state})
+        return response.content
+
 
 class RaptorDesktop(Raptor):
     def __init__(self, app, binary, run_local=False, obj_path=None,
                  gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
-                 symbols_path=None, host=None, power_test=False, is_release_build=False,
-                 debug_mode=False, post_startup_delay=None, activity=None):
+                 symbols_path=None, host=None, power_test=False, memory_test=False,
+                 is_release_build=False, debug_mode=False, post_startup_delay=None, activity=None):
         Raptor.__init__(self, app, binary, run_local, obj_path, gecko_profile,
                         gecko_profile_interval, gecko_profile_entries, symbols_path,
-                        host, power_test, is_release_build, debug_mode,
+                        host, power_test, memory_test, is_release_build, debug_mode,
                         post_startup_delay)
 
     def create_browser_handler(self):
@@ -446,11 +488,11 @@ class RaptorDesktop(Raptor):
 class RaptorDesktopFirefox(RaptorDesktop):
     def __init__(self, app, binary, run_local=False, obj_path=None,
                  gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
-                 symbols_path=None, host=None, power_test=False, is_release_build=False,
-                 debug_mode=False, post_startup_delay=None, activity=None):
+                 symbols_path=None, host=None, power_test=False, memory_test=False,
+                 is_release_build=False, debug_mode=False, post_startup_delay=None, activity=None):
         RaptorDesktop.__init__(self, app, binary, run_local, obj_path, gecko_profile,
                                gecko_profile_interval, gecko_profile_entries, symbols_path,
-                               host, power_test, is_release_build, debug_mode,
+                               host, power_test, memory_test, is_release_build, debug_mode,
                                post_startup_delay)
 
     def disable_non_local_connections(self):
@@ -491,11 +533,11 @@ class RaptorDesktopFirefox(RaptorDesktop):
 class RaptorDesktopChrome(RaptorDesktop):
     def __init__(self, app, binary, run_local=False, obj_path=None,
                  gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
-                 symbols_path=None, host=None, power_test=False, is_release_build=False,
-                 debug_mode=False, post_startup_delay=None, activity=None):
+                 symbols_path=None, host=None, power_test=False, memory_test=False,
+                 is_release_build=False, debug_mode=False, post_startup_delay=None, activity=None):
         RaptorDesktop.__init__(self, app, binary, run_local, obj_path, gecko_profile,
                                gecko_profile_interval, gecko_profile_entries, symbols_path,
-                               host, power_test, is_release_build, debug_mode,
+                               host, power_test, memory_test, is_release_build, debug_mode,
                                post_startup_delay)
 
     def setup_chrome_desktop_for_playback(self):
@@ -530,11 +572,11 @@ class RaptorDesktopChrome(RaptorDesktop):
 class RaptorAndroid(Raptor):
     def __init__(self, app, binary, run_local=False, obj_path=None,
                  gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
-                 symbols_path=None, host=None, power_test=False, is_release_build=False,
-                 debug_mode=False, post_startup_delay=None, activity=None):
+                 symbols_path=None, host=None, power_test=False, memory_test=False,
+                 is_release_build=False, debug_mode=False, post_startup_delay=None, activity=None):
         Raptor.__init__(self, app, binary, run_local, obj_path, gecko_profile,
                         gecko_profile_interval, gecko_profile_entries, symbols_path, host,
-                        power_test, is_release_build, debug_mode, post_startup_delay)
+                        power_test, memory_test, is_release_build, debug_mode, post_startup_delay)
 
         # on android, when creating the browser profile, we want to use a 'firefox' type profile
         self.profile_class = "firefox"
@@ -847,6 +889,7 @@ def main(args=sys.argv[1:]):
                           symbols_path=args.symbols_path,
                           host=args.host,
                           power_test=args.power_test,
+                          memory_test=args.memory_test,
                           is_release_build=args.is_release_build,
                           debug_mode=args.debug_mode,
                           post_startup_delay=args.post_startup_delay,

testing/raptor/test/test_cmdline.py

@@ -16,7 +16,8 @@ def test_verify_options(filedir):
                      page_cycles=1,
                      page_timeout=60000,
                      debug='True',
-                     power_test=False)
+                     power_test=False,
+                     memory_test=False)
     parser = ArgumentParser()
 
     with pytest.raises(SystemExit):
@@ -31,7 +32,8 @@ def test_verify_options(filedir):
                      gecko_profile='False',
                      is_release_build=False,
                      host='sophie',
-                     power_test=False)
+                     power_test=False,
+                     memory_test=False)
     verify_options(parser, args)  # assert no exception
 
     args = Namespace(app='refbrow',
@@ -40,7 +42,8 @@ def test_verify_options(filedir):
                      gecko_profile='False',
                      is_release_build=False,
                      host='sophie',
-                     power_test=False)
+                     power_test=False,
+                     memory_test=False)
     verify_options(parser, args)  # assert no exception
 
     args = Namespace(app='fenix',
@@ -49,7 +52,8 @@ def test_verify_options(filedir):
                      gecko_profile='False',
                      is_release_build=False,
                      host='sophie',
-                     power_test=False)
+                     power_test=False,
+                     memory_test=False)
     verify_options(parser, args)  # assert no exception
 
     args = Namespace(app='refbrow',
@@ -58,7 +62,8 @@ def test_verify_options(filedir):
                      gecko_profile='False',
                      is_release_build=False,
                      host='sophie',
-                     power_test=False)
+                     power_test=False,
+                     memory_test=False)
     parser = ArgumentParser()
 
     verify_options(parser, args)  # also will work as uses default activity

testing/raptor/test/test_raptor.py

@@ -91,6 +91,67 @@ def test_start_and_stop_server(raptor):
     assert not raptor.control_server._server_thread.is_alive()
 
 
+def test_server_wait_states(raptor):
+    import datetime
+
+    import requests
+
+    def post_state():
+        requests.post("http://127.0.0.1:%s/" % raptor.control_server.port,
+                      json={"type": "webext_status",
+                            "data": "test status"})
+
+    assert raptor.control_server is None
+
+    raptor.create_browser_profile()
+    raptor.create_browser_handler()
+    raptor.start_control_server()
+
+    wait_time = 5
+    message_state = 'webext_status/test status'
+    rhc = raptor.control_server.server.RequestHandlerClass
+
+    # Test initial state
+    assert rhc.wait_after_messages == {}
+    assert rhc.waiting_in_state is None
+    assert rhc.wait_timeout == 60
+    assert raptor.control_server_wait_get() == 'None'
+
+    # Test setting a state
+    assert raptor.control_server_wait_set(message_state) == ''
+    assert message_state in rhc.wait_after_messages
+    assert rhc.wait_after_messages[message_state]
+
+    # Test clearing a non-existent state
+    assert raptor.control_server_wait_clear('nothing') == ''
+    assert message_state in rhc.wait_after_messages
+
+    # Test clearing a state
+    assert raptor.control_server_wait_clear(message_state) == ''
+    assert message_state not in rhc.wait_after_messages
+
+    # Test clearing all states
+    assert raptor.control_server_wait_set(message_state) == ''
+    assert message_state in rhc.wait_after_messages
+    assert raptor.control_server_wait_clear('all') == ''
+    assert rhc.wait_after_messages == {}
+
+    # Test wait timeout
+    # Block on post request
+    assert raptor.control_server_wait_set(message_state) == ''
+    assert rhc.wait_after_messages[message_state]
+    assert raptor.control_server_wait_timeout(wait_time) == ''
+    assert rhc.wait_timeout == wait_time
+    start = datetime.datetime.now()
+    post_state()
+    assert datetime.datetime.now() - start < datetime.timedelta(seconds=wait_time+2)
+    assert raptor.control_server_wait_get() == 'None'
+    assert message_state not in rhc.wait_after_messages
+
+    raptor.clean_up()
+    assert not raptor.control_server._server_thread.is_alive()
+
+
 @pytest.mark.parametrize('app', [
     'firefox',
     pytest.mark.xfail('chrome'),