From 1938144b4ec267aa42249dfb023c3b206af5f1af Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 4 Nov 2016 16:28:59 +0000 Subject: [PATCH] Bug 1318666 - Update to latest wptrunner, a=testonly MozReview-Commit-ID: 9HnoUTtAf4l --- testing/web-platform/harness/.travis.yml | 20 ++ testing/web-platform/harness/MANIFEST.in | 4 + testing/web-platform/harness/requirements.txt | 2 +- .../web-platform/harness/requirements_b2g.txt | 7 - testing/web-platform/harness/tox.ini | 15 ++ .../harness/wptrunner/browsers/__init__.py | 4 +- .../harness/wptrunner/browsers/b2g.py | 243 ------------------ .../harness/wptrunner/browsers/edge.py | 71 +++++ .../harness/wptrunner/browsers/servo.py | 7 +- .../harness/wptrunner/browsers/servodriver.py | 2 +- .../wptrunner/executors/executormarionette.py | 12 +- .../wptrunner/executors/executorselenium.py | 22 +- .../wptrunner/executors/executorservo.py | 106 +++++++- .../executors/executorservodriver.py | 3 +- .../executors/pytestrunner/fixtures.py | 88 ++++++- .../executors/pytestrunner/runner.py | 1 + .../executors/testharness_webdriver.js | 14 +- .../wptrunner/testharnessreport-servo.js | 7 +- .../harness/wptrunner/testloader.py | 1 + .../harness/wptrunner/testrunner.py | 9 +- .../harness/wptrunner/tests/test_chunker.py | 37 +-- .../harness/wptrunner/tests/test_hosts.py | 5 +- .../wptrunner/tests/test_testloader.py | 32 +++ .../harness/wptrunner/tests/test_update.py | 10 + .../harness/wptrunner/webdriver_server.py | 45 +++- .../harness/wptrunner/wptcommandline.py | 9 +- .../harness/wptrunner/wptmanifest/parser.py | 16 +- .../wptmanifest/tests/test_conditional.py | 3 - .../wptmanifest/tests/test_serializer.py | 4 + .../wptmanifest/tests/test_static.py | 3 - .../harness/wptrunner/wptrunner.py | 1 + .../web-platform/harness/wptrunner/wpttest.py | 13 +- 32 files changed, 464 insertions(+), 352 deletions(-) create mode 100644 testing/web-platform/harness/.travis.yml delete mode 100644 testing/web-platform/harness/requirements_b2g.txt create mode 100644 testing/web-platform/harness/tox.ini delete mode 100644 testing/web-platform/harness/wptrunner/browsers/b2g.py create mode 100644 testing/web-platform/harness/wptrunner/browsers/edge.py create mode 100644 testing/web-platform/harness/wptrunner/tests/test_testloader.py diff --git a/testing/web-platform/harness/.travis.yml b/testing/web-platform/harness/.travis.yml new file mode 100644 index 000000000000..add6efd12ca2 --- /dev/null +++ b/testing/web-platform/harness/.travis.yml @@ -0,0 +1,20 @@ +language: python +python: 2.7 + +sudo: false + +cache: + directories: + - $HOME/.cache/pip + +env: + - TOXENV="{py27,pypy}-base" + - TOXENV="{py27,pypy}-chrome" + - TOXENV="{py27,pypy}-firefox" + - TOXENV="{py27,pypy}-servo" + +install: + - pip install -U tox + +script: + - tox diff --git a/testing/web-platform/harness/MANIFEST.in b/testing/web-platform/harness/MANIFEST.in index ac7ad5cdc745..0c5e38bf2654 100644 --- a/testing/web-platform/harness/MANIFEST.in +++ b/testing/web-platform/harness/MANIFEST.in @@ -5,9 +5,13 @@ include wptrunner.default.ini include wptrunner/testharness_runner.html include wptrunner/testharnessreport.js include wptrunner/testharnessreport-servo.js +include wptrunner/testharnessreport-servodriver.js include wptrunner/executors/testharness_marionette.js +include wptrunner/executors/testharness_servodriver.js include wptrunner/executors/testharness_webdriver.js include wptrunner/executors/reftest.js include wptrunner/executors/reftest-wait.js +include wptrunner/executors/reftest-wait_servodriver.js +include wptrunner/executors/reftest-wait_webdriver.js include wptrunner/config.json include wptrunner/browsers/server-locations.txt \ No newline at end of file diff --git a/testing/web-platform/harness/requirements.txt b/testing/web-platform/harness/requirements.txt index 9553db449100..319a2ff984aa 100644 --- a/testing/web-platform/harness/requirements.txt +++ b/testing/web-platform/harness/requirements.txt @@ -1,4 +1,4 @@ html5lib >= 0.99 mozinfo >= 0.7 -mozlog >= 3.0 +mozlog >= 3.3 mozdebug >= 0.1 diff --git a/testing/web-platform/harness/requirements_b2g.txt b/testing/web-platform/harness/requirements_b2g.txt deleted file mode 100644 index 132e0a7ba660..000000000000 --- a/testing/web-platform/harness/requirements_b2g.txt +++ /dev/null @@ -1,7 +0,0 @@ -fxos_appgen >= 0.5 -mozdevice >= 0.41 -gaiatest >= 0.26 -marionette_client >= 0.7.10 -moznetwork >= 0.24 -mozprofile >= 0.21 -mozrunner >= 6.1 diff --git a/testing/web-platform/harness/tox.ini b/testing/web-platform/harness/tox.ini new file mode 100644 index 000000000000..844a8d05b20b --- /dev/null +++ b/testing/web-platform/harness/tox.ini @@ -0,0 +1,15 @@ +[pytest] +xfail_strict=true + +[tox] +envlist = {py27,pypy}-{base,b2g,chrome,firefox,servo} + +[testenv] +deps = + pytest>=2.9 + -r{toxinidir}/requirements.txt + chrome: -r{toxinidir}/requirements_chrome.txt + firefox: -r{toxinidir}/requirements_firefox.txt + servo: -r{toxinidir}/requirements_servo.txt + +commands = py.test [] diff --git a/testing/web-platform/harness/wptrunner/browsers/__init__.py b/testing/web-platform/harness/wptrunner/browsers/__init__.py index ffc5aedc8304..8b34cc3963fa 100644 --- a/testing/web-platform/harness/wptrunner/browsers/__init__.py +++ b/testing/web-platform/harness/wptrunner/browsers/__init__.py @@ -26,8 +26,8 @@ All classes and functions named in the above dict must be imported into the module global scope. """ -product_list = ["b2g", - "chrome", +product_list = ["chrome", + "edge", "firefox", "servo", "servodriver"] diff --git a/testing/web-platform/harness/wptrunner/browsers/b2g.py b/testing/web-platform/harness/wptrunner/browsers/b2g.py deleted file mode 100644 index bedb00a49432..000000000000 --- a/testing/web-platform/harness/wptrunner/browsers/b2g.py +++ /dev/null @@ -1,243 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import tempfile -import shutil -import subprocess - -import fxos_appgen -import gaiatest -import mozdevice -import moznetwork -import mozrunner -from marionette import expected -from marionette.by import By -from marionette.wait import Wait -from mozprofile import FirefoxProfile, Preferences - -from .base import get_free_port, BrowserError, Browser, ExecutorBrowser -from ..executors.executormarionette import MarionetteTestharnessExecutor -from ..hosts import HostsFile, HostsLine -from ..environment import hostnames - -here = os.path.split(__file__)[0] - -__wptrunner__ = {"product": "b2g", - "check_args": "check_args", - "browser": "B2GBrowser", - "executor": {"testharness": "B2GMarionetteTestharnessExecutor"}, - "browser_kwargs": "browser_kwargs", - "executor_kwargs": "executor_kwargs", - "env_options": "env_options"} - - -def check_args(**kwargs): - pass - - -def browser_kwargs(test_environment, **kwargs): - return {"prefs_root": kwargs["prefs_root"], - "no_backup": kwargs.get("b2g_no_backup", False)} - - -def executor_kwargs(test_type, server_config, cache_manager, run_info_data, - **kwargs): - timeout_multiplier = kwargs["timeout_multiplier"] - if timeout_multiplier is None: - timeout_multiplier = 2 - - executor_kwargs = {"server_config": server_config, - "timeout_multiplier": timeout_multiplier, - "close_after_done": False} - - if test_type == "reftest": - executor_kwargs["cache_manager"] = cache_manager - - return executor_kwargs - - -def env_options(): - return {"host": "web-platform.test", - "bind_hostname": "false", - "test_server_port": False} - - -class B2GBrowser(Browser): - used_ports = set() - init_timeout = 180 - - def __init__(self, logger, prefs_root, no_backup=False): - Browser.__init__(self, logger) - logger.info("Waiting for device") - subprocess.call(["adb", "wait-for-device"]) - self.device = mozdevice.DeviceManagerADB() - self.marionette_port = get_free_port(2828, exclude=self.used_ports) - self.used_ports.add(self.marionette_port) - self.cert_test_app = None - self.runner = None - self.prefs_root = prefs_root - - self.no_backup = no_backup - self.backup_path = None - self.backup_paths = [] - self.backup_dirs = [] - - def setup(self): - self.logger.info("Running B2G setup") - self.backup_path = tempfile.mkdtemp() - - self.logger.debug("Backing up device to %s" % (self.backup_path,)) - - if not self.no_backup: - self.backup_dirs = [("/data/local", os.path.join(self.backup_path, "local")), - ("/data/b2g/mozilla", os.path.join(self.backup_path, "profile"))] - - self.backup_paths = [("/system/etc/hosts", os.path.join(self.backup_path, "hosts"))] - - for remote, local in self.backup_dirs: - self.device.getDirectory(remote, local) - - for remote, local in self.backup_paths: - self.device.getFile(remote, local) - - self.setup_hosts() - - def start(self): - profile = FirefoxProfile() - - profile.set_preferences({"dom.disable_open_during_load": False, - "marionette.defaultPrefs.enabled": True}) - - self.logger.debug("Creating device runner") - self.runner = mozrunner.B2GDeviceRunner(profile=profile) - self.logger.debug("Starting device runner") - self.runner.start() - self.logger.debug("Device runner started") - - def setup_hosts(self): - host_ip = moznetwork.get_ip() - - temp_dir = tempfile.mkdtemp() - hosts_path = os.path.join(temp_dir, "hosts") - remote_path = "/system/etc/hosts" - try: - self.device.getFile("/system/etc/hosts", hosts_path) - - with open(hosts_path) as f: - hosts_file = HostsFile.from_file(f) - - for canonical_hostname in hostnames: - hosts_file.set_host(HostsLine(host_ip, canonical_hostname)) - - with open(hosts_path, "w") as f: - hosts_file.to_file(f) - - self.logger.info("Installing hosts file") - - self.device.remount() - self.device.removeFile(remote_path) - self.device.pushFile(hosts_path, remote_path) - finally: - os.unlink(hosts_path) - os.rmdir(temp_dir) - - def load_prefs(self): - prefs_path = os.path.join(self.prefs_root, "prefs_general.js") - if os.path.exists(prefs_path): - preferences = Preferences.read_prefs(prefs_path) - else: - self.logger.warning("Failed to find base prefs file in %s" % prefs_path) - preferences = [] - - return preferences - - def stop(self): - pass - - def on_output(self): - raise NotImplementedError - - def cleanup(self): - self.logger.debug("Running browser cleanup steps") - - self.device.remount() - - for remote, local in self.backup_dirs: - self.device.removeDir(remote) - self.device.pushDir(local, remote) - - for remote, local in self.backup_paths: - self.device.removeFile(remote) - self.device.pushFile(local, remote) - - shutil.rmtree(self.backup_path) - self.device.reboot(wait=True) - - def pid(self): - return None - - def is_alive(self): - return True - - def executor_browser(self): - return B2GExecutorBrowser, {"marionette_port": self.marionette_port} - - -class B2GExecutorBrowser(ExecutorBrowser): - # The following methods are called from a different process - def __init__(self, *args, **kwargs): - ExecutorBrowser.__init__(self, *args, **kwargs) - - import sys, subprocess - - self.device = mozdevice.ADBB2G() - self.device.forward("tcp:%s" % self.marionette_port, - "tcp:2828") - self.executor = None - self.marionette = None - self.gaia_device = None - self.gaia_apps = None - - def after_connect(self, executor): - self.executor = executor - self.marionette = executor.marionette - self.executor.logger.debug("Running browser.after_connect steps") - - self.gaia_apps = gaiatest.GaiaApps(marionette=executor.marionette) - - self.executor.logger.debug("Waiting for homescreen to load") - - # Moved out of gaia_test temporarily - self.executor.logger.info("Waiting for B2G to be ready") - self.wait_for_homescreen(timeout=60) - - self.install_cert_app() - self.use_cert_app() - - def install_cert_app(self): - """Install the container app used to run the tests""" - if fxos_appgen.is_installed("CertTest App"): - self.executor.logger.info("CertTest App is already installed") - return - self.executor.logger.info("Installing CertTest App") - app_path = os.path.join(here, "b2g_setup", "certtest_app.zip") - fxos_appgen.install_app("CertTest App", app_path, marionette=self.marionette) - self.executor.logger.debug("Install complete") - - def use_cert_app(self): - """Start the app used to run the tests""" - self.executor.logger.info("Homescreen loaded") - self.gaia_apps.launch("CertTest App") - - def wait_for_homescreen(self, timeout): - self.executor.logger.info("Waiting for home screen to load") - Wait(self.marionette, timeout).until(expected.element_present( - By.CSS_SELECTOR, '#homescreen[loading-state=false]')) - - -class B2GMarionetteTestharnessExecutor(MarionetteTestharnessExecutor): - def after_connect(self): - self.browser.after_connect(self) - MarionetteTestharnessExecutor.after_connect(self) diff --git a/testing/web-platform/harness/wptrunner/browsers/edge.py b/testing/web-platform/harness/wptrunner/browsers/edge.py new file mode 100644 index 000000000000..7c993fce1aca --- /dev/null +++ b/testing/web-platform/harness/wptrunner/browsers/edge.py @@ -0,0 +1,71 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from .base import Browser, ExecutorBrowser, require_arg +from ..webdriver_server import EdgeDriverServer +from ..executors import executor_kwargs as base_executor_kwargs +from ..executors.executorselenium import (SeleniumTestharnessExecutor, + SeleniumRefTestExecutor) + +__wptrunner__ = {"product": "edge", + "check_args": "check_args", + "browser": "EdgeBrowser", + "executor": {"testharness": "SeleniumTestharnessExecutor", + "reftest": "SeleniumRefTestExecutor"}, + "browser_kwargs": "browser_kwargs", + "executor_kwargs": "executor_kwargs", + "env_options": "env_options"} + + +def check_args(**kwargs): + require_arg(kwargs, "webdriver_binary") + +def browser_kwargs(**kwargs): + return {"webdriver_binary": kwargs["webdriver_binary"]} + +def executor_kwargs(test_type, server_config, cache_manager, run_info_data, + **kwargs): + from selenium.webdriver import DesiredCapabilities + + executor_kwargs = base_executor_kwargs(test_type, server_config, + cache_manager, **kwargs) + executor_kwargs["close_after_done"] = True + executor_kwargs["capabilities"] = dict(DesiredCapabilities.EDGE.items()) + return executor_kwargs + +def env_options(): + return {"host": "web-platform.test", + "bind_hostname": "true", + "supports_debugger": False} + +class EdgeBrowser(Browser): + used_ports = set() + + def __init__(self, logger, webdriver_binary): + Browser.__init__(self, logger) + self.server = EdgeDriverServer(self.logger, binary=webdriver_binary) + self.webdriver_host = "localhost" + self.webdriver_port = self.server.port + + def start(self): + print self.server.url + self.server.start() + + def stop(self): + self.server.stop() + + def pid(self): + return self.server.pid + + def is_alive(self): + # TODO(ato): This only indicates the server is alive, + # and doesn't say anything about whether a browser session + # is active. + return self.server.is_alive() + + def cleanup(self): + self.stop() + + def executor_browser(self): + return ExecutorBrowser, {"webdriver_url": self.server.url} diff --git a/testing/web-platform/harness/wptrunner/browsers/servo.py b/testing/web-platform/harness/wptrunner/browsers/servo.py index bc90cefcfc9e..2eeb5aaa158b 100644 --- a/testing/web-platform/harness/wptrunner/browsers/servo.py +++ b/testing/web-platform/harness/wptrunner/browsers/servo.py @@ -6,7 +6,7 @@ import os from .base import NullBrowser, ExecutorBrowser, require_arg from ..executors import executor_kwargs as base_executor_kwargs -from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor +from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor, ServoWdspecExecutor here = os.path.join(os.path.split(__file__)[0]) @@ -14,7 +14,8 @@ __wptrunner__ = {"product": "servo", "check_args": "check_args", "browser": "ServoBrowser", "executor": {"testharness": "ServoTestharnessExecutor", - "reftest": "ServoRefTestExecutor"}, + "reftest": "ServoRefTestExecutor", + "wdspec": "ServoWdspecExecutor"}, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", "env_options": "env_options", @@ -64,7 +65,7 @@ def render_arg(render_backend): class ServoBrowser(NullBrowser): def __init__(self, logger, binary, debug_info=None, binary_args=None, - user_stylesheets=None, render_backend="cpu"): + user_stylesheets=None, render_backend="webrender"): NullBrowser.__init__(self, logger) self.binary = binary self.debug_info = debug_info diff --git a/testing/web-platform/harness/wptrunner/browsers/servodriver.py b/testing/web-platform/harness/wptrunner/browsers/servodriver.py index 2c05a4dd5054..ab4dc7956b90 100644 --- a/testing/web-platform/harness/wptrunner/browsers/servodriver.py +++ b/testing/web-platform/harness/wptrunner/browsers/servodriver.py @@ -80,7 +80,7 @@ class ServoWebDriverBrowser(Browser): used_ports = set() def __init__(self, logger, binary, debug_info=None, webdriver_host="127.0.0.1", - user_stylesheets=None, render_backend="cpu"): + user_stylesheets=None, render_backend="webrender"): Browser.__init__(self, logger) self.binary = binary self.webdriver_host = webdriver_host diff --git a/testing/web-platform/harness/wptrunner/executors/executormarionette.py b/testing/web-platform/harness/wptrunner/executors/executormarionette.py index fb5aa58eba2d..21606ddef2f8 100644 --- a/testing/web-platform/harness/wptrunner/executors/executormarionette.py +++ b/testing/web-platform/harness/wptrunner/executors/executormarionette.py @@ -17,11 +17,11 @@ from ..wpttest import WdspecResult, WdspecSubtestResult errors = None marionette = None +pytestrunner = None webdriver = None here = os.path.join(os.path.split(__file__)[0]) -from . import pytestrunner from .base import (ExecutorException, Protocol, RefTestExecutor, @@ -41,7 +41,7 @@ extra_timeout = 5 # seconds def do_delayed_imports(): - global errors, marionette, webdriver + global errors, marionette # Marionette client used to be called marionette, recently it changed # to marionette_driver for unfathomable reasons @@ -51,8 +51,6 @@ def do_delayed_imports(): except ImportError: from marionette_driver import marionette, errors - import webdriver - class MarionetteProtocol(Protocol): def __init__(self, executor, browser): @@ -561,6 +559,7 @@ class WdspecRun(object): class MarionetteWdspecExecutor(WdspecExecutor): def __init__(self, browser, server_config, webdriver_binary, timeout_multiplier=1, close_after_done=True, debug_info=None): + self.do_delayed_imports() WdspecExecutor.__init__(self, browser, server_config, timeout_multiplier=timeout_multiplier, debug_info=debug_info) @@ -590,3 +589,8 @@ class MarionetteWdspecExecutor(WdspecExecutor): harness_result = ("OK", None) subtest_results = pytestrunner.run(path, session, timeout=timeout) return (harness_result, subtest_results) + + def do_delayed_imports(self): + global pytestrunner, webdriver + from . import pytestrunner + import webdriver diff --git a/testing/web-platform/harness/wptrunner/executors/executorselenium.py b/testing/web-platform/harness/wptrunner/executors/executorselenium.py index 587c49c7c92f..f5d65f499b0a 100644 --- a/testing/web-platform/harness/wptrunner/executors/executorselenium.py +++ b/testing/web-platform/harness/wptrunner/executors/executorselenium.py @@ -22,20 +22,21 @@ from .base import (ExecutorException, strip_server) from ..testrunner import Stop - here = os.path.join(os.path.split(__file__)[0]) webdriver = None exceptions = None +RemoteConnection = None extra_timeout = 5 def do_delayed_imports(): global webdriver global exceptions + global RemoteConnection from selenium import webdriver from selenium.common import exceptions - + from selenium.webdriver.remote.remote_connection import RemoteConnection class SeleniumProtocol(Protocol): def __init__(self, executor, browser, capabilities, **kwargs): @@ -53,8 +54,9 @@ class SeleniumProtocol(Protocol): session_started = False try: - self.webdriver = webdriver.Remote( - self.url, desired_capabilities=self.capabilities) + self.webdriver = webdriver.Remote(command_executor=RemoteConnection(self.url.strip("/"), + resolve_ip=False), + desired_capabilities=self.capabilities) except: self.logger.warning( "Connecting to Selenium failed:\n%s" % traceback.format_exc()) @@ -231,17 +233,7 @@ class SeleniumRefTestExecutor(RefTestExecutor): def do_test(self, test): self.logger.info("Test requires OS-level window focus") - if self.close_after_done and self.has_window: - self.protocol.webdriver.close() - self.protocol.webdriver.switch_to_window( - self.protocol.webdriver.window_handles[-1]) - self.has_window = False - - if not self.has_window: - self.protocol.webdriver.execute_script(self.script) - self.protocol.webdriver.switch_to_window( - self.protocol.webdriver.window_handles[-1]) - self.has_window = True + self.protocol.webdriver.set_window_size(600, 600) result = self.implementation.run_test(test) diff --git a/testing/web-platform/harness/wptrunner/executors/executorservo.py b/testing/web-platform/harness/wptrunner/executors/executorservo.py index 068061b958db..b627223a7df6 100644 --- a/testing/web-platform/harness/wptrunner/executors/executorservo.py +++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py @@ -4,11 +4,13 @@ import base64 import hashlib +import httplib import json import os import subprocess import tempfile import threading +import traceback import urlparse import uuid from collections import defaultdict @@ -19,11 +21,19 @@ from .base import (ExecutorException, Protocol, RefTestImplementation, testharness_result_converter, - reftest_result_converter) + reftest_result_converter, + WdspecExecutor) from .process import ProcessTestExecutor from ..browsers.base import browser_command -render_arg = None +from ..wpttest import WdspecResult, WdspecSubtestResult +from ..webdriver_server import ServoDriverServer +from .executormarionette import WdspecRun +pytestrunner = None +render_arg = None +webdriver = None + +extra_timeout = 5 # seconds def do_delayed_imports(): global render_arg @@ -205,21 +215,23 @@ class ServoRefTestExecutor(ProcessTestExecutor): self.binary, [render_arg(self.browser.render_backend), "--hard-fail", "--exit", "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates", - "--output=%s" % output_path, full_url], + "--output=%s" % output_path, full_url] + self.browser.binary_args, self.debug_info) for stylesheet in self.browser.user_stylesheets: command += ["--user-stylesheet", stylesheet] - for pref in test.environment.get('prefs', {}): - command += ["--pref", pref] + for pref, value in test.environment.get('prefs', {}).iteritems(): + command += ["--pref", "%s=%s" % (pref, value)] - if viewport_size: - command += ["--resolution", viewport_size] + command += ["--resolution", viewport_size or "800x600"] if dpi: command += ["--device-pixel-ratio", dpi] + # Run ref tests in headless mode + command += ["-z"] + self.command = debug_args + command env = os.environ.copy() @@ -273,3 +285,83 @@ class ServoRefTestExecutor(ProcessTestExecutor): self.logger.process_output(self.proc.pid, line, " ".join(self.command)) + +class ServoWdspecProtocol(Protocol): + def __init__(self, executor, browser): + self.do_delayed_imports() + Protocol.__init__(self, executor, browser) + self.session = None + self.server = None + + def setup(self, runner): + try: + self.server = ServoDriverServer(self.logger, binary=self.browser.binary, binary_args=self.browser.binary_args, render_backend=self.browser.render_backend) + self.server.start(block=False) + self.logger.info( + "WebDriver HTTP server listening at %s" % self.server.url) + + self.logger.info( + "Establishing new WebDriver session with %s" % self.server.url) + self.session = webdriver.Session( + self.server.host, self.server.port, self.server.base_path) + except Exception: + self.logger.error(traceback.format_exc()) + self.executor.runner.send_message("init_failed") + else: + self.executor.runner.send_message("init_succeeded") + + def teardown(self): + if self.server is not None: + try: + if self.session.session_id is not None: + self.session.end() + except Exception: + pass + if self.server.is_alive: + self.server.stop() + + @property + def is_alive(self): + conn = httplib.HTTPConnection(self.server.host, self.server.port) + conn.request("HEAD", self.server.base_path + "invalid") + res = conn.getresponse() + return res.status == 404 + + def do_delayed_imports(self): + global pytestrunner, webdriver + from . import pytestrunner + import webdriver + + +class ServoWdspecExecutor(WdspecExecutor): + def __init__(self, browser, server_config, + timeout_multiplier=1, close_after_done=True, debug_info=None, + **kwargs): + WdspecExecutor.__init__(self, browser, server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.protocol = ServoWdspecProtocol(self, browser) + + def is_alive(self): + return self.protocol.is_alive + + def on_environment_change(self, new_environment): + pass + + def do_test(self, test): + timeout = test.timeout * self.timeout_multiplier + extra_timeout + + success, data = WdspecRun(self.do_wdspec, + self.protocol.session, + test.path, + timeout).run() + + if success: + return self.convert_result(test, data) + + return (test.result_cls(*data), []) + + def do_wdspec(self, session, path, timeout): + harness_result = ("OK", None) + subtest_results = pytestrunner.run(path, session, timeout=timeout) + return (harness_result, subtest_results) diff --git a/testing/web-platform/harness/wptrunner/executors/executorservodriver.py b/testing/web-platform/harness/wptrunner/executors/executorservodriver.py index fceeb58fad2f..279658d0cd1f 100644 --- a/testing/web-platform/harness/wptrunner/executors/executorservodriver.py +++ b/testing/web-platform/harness/wptrunner/executors/executorservodriver.py @@ -14,7 +14,6 @@ from .base import (Protocol, RefTestImplementation, TestharnessExecutor, strip_server) -from .. import webdriver from ..testrunner import Stop webdriver = None @@ -26,7 +25,7 @@ extra_timeout = 5 def do_delayed_imports(): global webdriver - import webdriver + from tools import webdriver class ServoWebDriverProtocol(Protocol): diff --git a/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py b/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py index 77afb4a36842..81796d69883c 100644 --- a/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py +++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py @@ -3,6 +3,10 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. import pytest +import webdriver + +import contextlib +import httplib """pytest fixtures for use in Python-based WPT tests. @@ -17,7 +21,7 @@ class Session(object): in tests. The session is not created by default to enable testing of session - creation. However, a module-scoped session will be implicitly created + creation. However, a function-scoped session will be implicitly created at the first call to a WebDriver command. This means methods such as `session.send_command` and `session.session_id` are possible to use without having a session. @@ -45,14 +49,88 @@ class Session(object): def test_something(setup, session): assert session.url == "https://example.org" - The session is closed when the test module goes out of scope by an - implicit call to `session.end`. + When the test function goes out of scope, any remaining user prompts + and opened windows are closed, and the current browsing context is + switched back to the top-level browsing context. """ def __init__(self, client): self.client = client - @pytest.fixture(scope="module") + @pytest.fixture(scope="function") def session(self, request): - request.addfinalizer(self.client.end) + # finalisers are popped off a stack, + # making their ordering reverse + request.addfinalizer(self.switch_to_top_level_browsing_context) + request.addfinalizer(self.restore_windows) + request.addfinalizer(self.dismiss_user_prompts) + return self.client + + def dismiss_user_prompts(self): + """Dismisses any open user prompts in windows.""" + current_window = self.client.window_handle + + for window in self.windows(): + self.client.window_handle = window + try: + self.client.alert.dismiss() + except webdriver.NoSuchAlertException: + pass + + self.client.window_handle = current_window + + def restore_windows(self): + """Closes superfluous windows opened by the test without ending + the session implicitly by closing the last window. + """ + current_window = self.client.window_handle + + for window in self.windows(exclude=[current_window]): + self.client.window_handle = window + if len(self.client.window_handles) > 1: + self.client.close() + + self.client.window_handle = current_window + + def switch_to_top_level_browsing_context(self): + """If the current browsing context selected by WebDriver is a + `` or an `