mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1639321 - increase the coverage to 85 r=sparky 

Increase the coverage to 85%

Differential Revision: https://phabricator.services.mozilla.com/D76048
This commit is contained in:
Tarek Ziadé 2020-05-26 14:44:14 +00:00
Родитель f94f60a9c2
Коммит af3ea56177
7 изменённых файлов: 234 добавлений и 109 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
python/mozperftest/mozperftest/.coveragerc
Просмотреть файл

@ -7,4 +7,4 @@ include =
[report]
sort = Cover
show_missing = True
fail_under = 75
fail_under = 84

171
python/mozperftest/mozperftest/browser/browsertime/runner.py
Просмотреть файл

@ -8,9 +8,7 @@ import pathlib
import sys
import re
import shutil
from mozbuild.util import mkdir
import mozpack.path as mozpath
from pathlib import Path
from mozperftest.scriptinfo import ScriptInfo
from mozperftest.utils import install_package
@ -21,12 +19,32 @@ from mozperftest.browser.browsertime.setup import (
)
AUTOMATION = "MOZ_AUTOMATION" in os.environ
BROWSERTIME_SRC_ROOT = os.path.dirname(__file__)
BROWSERTIME_SRC_ROOT = Path(__file__).parent
PILLOW_VERSION = "6.0.0"
PYSSIM_VERSION = "0.4"
def matches(args, *flags):
"""Returns True if any argument matches any of the given flags
Maybe with an argument.
"""
for flag in flags:
if flag in args or any(arg.startswith(flag + "=") for arg in args):
return True
return False
def extract_browser_name(args):
"Extracts the browser name if any"
# These are BT arguments, it's BT job to check them
# here we just want to extract the browser name
res = re.findall(r"(--browser|-b)[= ]([\w]+)", " ".join(args))
if res == []:
return None
return res[0][-1]
class BrowsertimeRunner(NodeRunner):
"""Runs a browsertime test.
"""
@ -67,40 +85,37 @@ class BrowsertimeRunner(NodeRunner):
self.virtualenv_manager = mach_cmd.virtualenv_manager
self._created_dirs = []
self._test_info = {}
self._setup_helper = None
self.get_binary_path = mach_cmd.get_binary_path
@property
def setup_helper(self):
if self._setup_helper is not None:
return self._setup_helper
sys.path.append(str(Path(self.topsrcdir, "tools", "lint", "eslint")))
import setup_helper
self._setup_helper = setup_helper
return self._setup_helper
@property
def artifact_cache_path(self):
"""Downloaded artifacts will be kept here."""
# The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
return mozpath.join(self._mach_context.state_dir, "cache", "browsertime")
return Path(self._mach_context.state_dir, "cache", "browsertime")
@property
def state_path(self):
"""Unpacked artifacts will be kept here."""
# The convention is $MOZBUILD_STATE_PATH/$FEATURE.
res = mozpath.join(self._mach_context.state_dir, "browsertime")
mkdir(res)
res = Path(self._mach_context.state_dir, "browsertime")
os.makedirs(str(res), exist_ok=True)
return res
@property
def browsertime_js(self):
root = os.environ.get("BROWSERTIME", self.state_path)
return os.path.join(
root, "node_modules", "browsertime", "bin", "browsertime.js"
)
def _need_install(self, package):
from pip._internal.req.constructors import install_req_from_line
req = install_req_from_line("Pillow")
req.check_if_exists(use_user_site=False)
if req.satisfied_by is None:
return True
venv_site_lib = os.path.abspath(
os.path.join(self.virtualenv_manager.bin_path, "..", "lib")
)
site_packages = os.path.abspath(req.satisfied_by.location)
return not site_packages.startswith(venv_site_lib)
return Path(root, "node_modules", "browsertime", "bin", "browsertime.js")
def setup(self):
"""Install browsertime and visualmetrics.py prerequisites and the Node.js package.
@ -117,29 +132,29 @@ class BrowsertimeRunner(NodeRunner):
# check if the browsertime package has been deployed correctly
# for this we just check for the browsertime directory presence
if os.path.exists(self.browsertime_js):
if self.browsertime_js.exists():
return
if install_url is None:
system_prerequisites(
self.state_path, self.artifact_cache_path, self.log, self.info
str(self.state_path), str(self.artifact_cache_path), self.log, self.info
)
# preparing ~/.mozbuild/browsertime
for file in ("package.json", "package-lock.json"):
src = mozpath.join(BROWSERTIME_SRC_ROOT, file)
target = mozpath.join(self.state_path, file)
if not os.path.exists(target):
shutil.copyfile(src, target)
src = BROWSERTIME_SRC_ROOT / file
target = self.state_path / file
if not target.exists():
shutil.copyfile(str(src), str(target))
package_json_path = mozpath.join(self.state_path, "package.json")
package_json_path = self.state_path / "package.json"
if install_url is not None:
self.info(
"Updating browsertime node module version in {package_json_path} "
"to {install_url}",
install_url=install_url,
package_json_path=package_json_path,
package_json_path=str(package_json_path),
)
expr = r"/tarball/[a-f0-9]{40}$"
@ -150,24 +165,21 @@ class BrowsertimeRunner(NodeRunner):
)
)
with open(package_json_path) as f:
with package_json_path.open() as f:
existing_body = json.loads(
f.read(), object_pairs_hook=collections.OrderedDict
)
existing_body["devDependencies"]["browsertime"] = install_url
updated_body = json.dumps(existing_body)
with open(package_json_path, "w") as f:
with package_json_path.open("w") as f:
f.write(updated_body)
self._setup_node_packages(package_json_path)
def _setup_node_packages(self, package_json_path):
# Install the browsertime Node.js requirements.
sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint"))
import setup_helper
if not setup_helper.check_node_executables_valid():
if not self.setup_helper.check_node_executables_valid():
return
should_clobber = self.get_arg("clobber")
@ -175,89 +187,60 @@ class BrowsertimeRunner(NodeRunner):
# os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url)
# to an endpoint with binaries named like
# https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31.
if AUTOMATION:
automation = "MOZ_AUTOMATION" in os.environ
if automation:
os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true"
os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true"
self.info(
"Installing browsertime node module from {package_json}",
package_json=package_json_path,
package_json=str(package_json_path),
)
install_url = self.get_arg("install-url")
setup_helper.package_setup(
self.state_path,
self.setup_helper.package_setup(
str(self.state_path),
"browsertime",
should_update=install_url is not None,
should_clobber=should_clobber,
no_optional=install_url or AUTOMATION,
no_optional=install_url or automation,
)
def append_env(self, append_path=True):
env = super(BrowsertimeRunner, self).append_env(append_path)
return append_system_env(env, self.state_path, append_path)
return append_system_env(env, str(self.state_path), append_path)
def extra_default_args(self, args=[]):
# Add Mozilla-specific default arguments. This is tricky because browsertime is quite
# loose about arguments; repeat arguments are generally accepted but then produce
# difficult to interpret type errors.
def extract_browser_name(args):
"Extracts the browser name if any"
# These are BT arguments, it's BT job to check them
# here we just want to extract the browser name
res = re.findall(r"(--browser|-b)[= ]([\w]+)", " ".join(args))
if res == []:
return None
return res[0][-1]
def matches(args, *flags):
"""Returns True if any argument matches any of the given flags
Maybe with an argument.
"""
for flag in flags:
if flag in args or any(arg.startswith(flag + "=") for arg in args):
return True
return False
extra_args = []
# Default to Firefox. Override with `-b ...` or `--browser=...`.
specifies_browser = matches(args, "-b", "--browser")
if not specifies_browser:
if not matches(args, "-b", "--browser"):
extra_args.extend(("-b", "firefox"))
# Default to not collect HAR. Override with `--skipHar=false`.
specifies_har = matches(args, "--har", "--skipHar", "--gzipHar")
if not specifies_har:
if not matches(args, "--har", "--skipHar", "--gzipHar"):
extra_args.append("--skipHar")
if not matches(args, "--android"):
# If --firefox.binaryPath is not specified, default to the objdir binary
# Note: --firefox.release is not a real browsertime option, but it will
# silently ignore it instead and default to a release installation.
specifies_binaryPath = matches(
args,
"--firefox.binaryPath",
"--firefox.release",
"--firefox.nightly",
"--firefox.beta",
"--firefox.developer",
)
if not specifies_binaryPath:
specifies_binaryPath = extract_browser_name(args) == "chrome"
if not specifies_binaryPath:
try:
extra_args.extend(("--firefox.binaryPath", self.get_binary_path()))
except Exception:
print(
"Please run |./mach build| "
"or specify a Firefox binary with --firefox.binaryPath."
)
return 1
if (
not matches(
args,
"--firefox.binaryPath",
"--firefox.release",
"--firefox.nightly",
"--firefox.beta",
"--firefox.developer",
)
and extract_browser_name(args) != "chrome"
):
extra_args.extend(("--firefox.binaryPath", self.get_binary_path()))
geckodriver = self.get_arg("geckodriver")
if geckodriver is not None:
@ -271,11 +254,10 @@ class BrowsertimeRunner(NodeRunner):
return extra_args
def browsertime_android(self, metadata):
def _android_args(self, metadata):
app_name = self.get_arg("android-app-name")
args_list = [
# "--browser", "firefox",
"--android",
# Work around a `selenium-webdriver` issue where Browsertime
# fails to find a Firefox binary even though we're going to
@ -343,12 +325,11 @@ class BrowsertimeRunner(NodeRunner):
name, value = option
args += ["--" + name, value]
firefox_args = ["--firefox.developer"]
if self.get_arg("android"):
args.extend(self.browsertime_android(metadata))
args.extend(self._android_args(metadata))
extra = self.extra_default_args(args=firefox_args)
command = [self.browsertime_js] + extra + args
extra = self.extra_default_args(args=args)
command = [str(self.browsertime_js)] + extra + args
self.info("Running browsertime with this command %s" % " ".join(command))
self.node(command)

12
python/mozperftest/mozperftest/browser/browsertime/setup.py
Просмотреть файл

@ -19,6 +19,9 @@ from mozperftest.utils import host_platform
AUTOMATION = "MOZ_AUTOMATION" in os.environ
GECKO_RELEASES = (
"https://github.com/ncalexan/geckodriver/releases/download/v0.24.0-android"
)
# Map from `host_platform()` to a `fetch`-like syntax.
@ -26,7 +29,7 @@ host_fetches = {
"darwin": {
"ffmpeg": {
"type": "static-url",
"url": "https://github.com/ncalexan/geckodriver/releases/download/v0.24.0-android/ffmpeg-4.1.1-macos64-static.zip", # noqa
"url": GECKO_RELEASES + "/ffmpeg-4.1.1-macos64-static.zip", # noqa
# An extension to `fetch` syntax.
"path": "ffmpeg-4.1.1-macos64-static",
}
@ -34,7 +37,7 @@ host_fetches = {
"linux64": {
"ffmpeg": {
"type": "static-url",
"url": "https://github.com/ncalexan/geckodriver/releases/download/v0.24.0-android/ffmpeg-4.1.4-i686-static.tar.xz", # noqa
"url": GECKO_RELEASES + "/ffmpeg-4.1.4-i686-static.tar.xz", # noqa
# An extension to `fetch` syntax.
"path": "ffmpeg-4.1.4-i686-static",
},
@ -46,7 +49,7 @@ host_fetches = {
"win64": {
"ffmpeg": {
"type": "static-url",
"url": "https://github.com/ncalexan/geckodriver/releases/download/v0.24.0-android/ffmpeg-4.1.1-win64-static.zip", # noqa
"url": GECKO_RELEASES + "/ffmpeg-4.1.1-win64-static.zip", # noqa
# An extension to `fetch` syntax.
"path": "ffmpeg-4.1.1-win64-static",
},
@ -72,14 +75,13 @@ def system_prerequisites(state_path, artifact_cache_path, log, info):
for im_program in im_programs:
prog = which(im_program)
if not prog:
print(
raise Exception(
"Error: On Linux, ImageMagick must be on the PATH. "
"Install ImageMagick manually and try again (or update PATH). "
"On Ubuntu and Debian, try `sudo apt-get install imagemagick`. "
"On Fedora, try `sudo dnf install imagemagick`. "
"On CentOS, try `sudo yum install imagemagick`."
)
return 1
# Download the visualmetrics.py requirements.
artifact_cache = ArtifactCache(artifact_cache_path, log=log, skip_cache=False)

6
python/mozperftest/mozperftest/metrics/perfherder.py
Просмотреть файл

@ -44,8 +44,10 @@ class Perfherder(Layer):
"fenix",
"refbrow",
],
"help": "Shorthand name of application that is "
"being tested (used in perfherder data).",
"help": (
"Shorthand name of application that is "
"being tested (used in perfherder data)."
),
},
"metrics": {
"nargs": "*",

113
python/mozperftest/mozperftest/tests/test_browsertime.py
Просмотреть файл

@ -3,11 +3,16 @@ import os
import mozunit
import mock
import shutil
import string
import random
import pytest
from mozperftest.tests.support import get_running_env, EXAMPLE_TEST
from mozperftest.environment import BROWSER
from mozperftest.browser.browsertime import add_options
from mozperftest.utils import silence
from mozperftest.browser.browsertime.runner import matches, extract_browser_name
from mozperftest.utils import silence, temporary_env
HERE = os.path.dirname(__file__)
@ -27,7 +32,14 @@ def fetch(self, url):
new=lambda x, y: None,
)
def test_browser(*mocked):
mach_cmd, metadata, env = get_running_env(browsertime_geckodriver="GECKODRIVER")
mach_cmd, metadata, env = get_running_env(
android=True,
android_app_name="something",
browsertime_geckodriver="GECKODRIVER",
browsertime_iterations=1,
browsertime_extra_options="one=1,two=2",
)
browser = env.layers[BROWSER]
env.set_arg("tests", [EXAMPLE_TEST])
@ -38,21 +50,55 @@ def test_browser(*mocked):
shutil.rmtree(mach_cmd._mach_context.state_dir)
assert mach_cmd.run_process.call_count == 1
# XXX more checks
assert mach_cmd.run_process.call_args[0][-1][-1] == EXAMPLE_TEST
# Make sure all arguments are of type str
for option in mach_cmd.run_process.call_args[0][0]:
assert isinstance(option, str)
cmd = " ".join(mach_cmd.run_process.call_args[0][0])
assert EXAMPLE_TEST in cmd
assert "--firefox.geckodriverPath GECKODRIVER" in cmd
assert "--one 1" in cmd
assert "--two 2" in cmd
results = metadata.get_results()
assert len(results) == 1
assert set(list(results[0].keys())) - set(["name", "results"]) == set()
assert results[0]["name"] == "Example"
@mock.patch("mozperftest.browser.browsertime.runner.install_package")
@mock.patch(
"mozperftest.browser.noderunner.NodeRunner.verify_node_install", new=lambda x: True
)
@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
@mock.patch(
"mozperftest.browser.browsertime.runner.BrowsertimeRunner._setup_node_packages",
new=lambda x, y: None,
)
def test_browser_desktop(*mocked):
mach_cmd, metadata, env = get_running_env(
browsertime_iterations=1, browsertime_extra_options="one=1,two=2",
)
browser = env.layers[BROWSER]
env.set_arg("tests", [EXAMPLE_TEST])
try:
with browser as b, silence():
# just checking that the setup_helper property gets
# correctly initialized
browsertime = browser.layers[-1]
assert browsertime.setup_helper is not None
helper = browsertime.setup_helper
assert browsertime.setup_helper is helper
b(metadata)
finally:
shutil.rmtree(mach_cmd._mach_context.state_dir)
assert mach_cmd.run_process.call_count == 1
cmd = " ".join(mach_cmd.run_process.call_args[0][0])
# check that --firefox.binaryPath is set automatically
assert "--firefox.binaryPath" in cmd
def test_add_options():
mach_cmd, metadata, env = get_running_env()
@ -63,5 +109,62 @@ def test_add_options():
assert "two=2" in extra
@mock.patch("mozperftest.browser.browsertime.runner.install_package")
@mock.patch(
"mozperftest.browser.noderunner.NodeRunner.verify_node_install", new=lambda x: True
)
@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
@mock.patch("mozperftest.browser.browsertime.runner.BrowsertimeRunner.setup_helper")
def test_install_url(*mocked):
url = "https://here/tarball/" + "".join(
[random.choice(string.hexdigits[:-6]) for c in range(40)]
)
mach, metadata, env = get_running_env(browsertime_install_url=url)
browser = env.layers[BROWSER]
env.set_arg("tests", [EXAMPLE_TEST])
try:
with temporary_env(MOZ_AUTOMATION="1"), browser as b, silence():
b(metadata)
finally:
shutil.rmtree(mach._mach_context.state_dir)
assert mach.run_process.call_count == 1
@mock.patch("mozperftest.browser.browsertime.runner.install_package")
@mock.patch(
"mozperftest.browser.noderunner.NodeRunner.verify_node_install", new=lambda x: True
)
@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
@mock.patch(
"mozperftest.browser.browsertime.runner.BrowsertimeRunner._setup_node_packages",
new=lambda x, y: None,
)
def test_install_url_bad(*mocked):
mach, metadata, env = get_running_env(browsertime_install_url="meh")
browser = env.layers[BROWSER]
env.set_arg("tests", [EXAMPLE_TEST])
with pytest.raises(ValueError):
try:
with browser as b, silence():
b(metadata)
finally:
shutil.rmtree(mach._mach_context.state_dir)
def test_matches():
args = ["arg1=1", "--arg2=value2"]
assert matches(args, "arg1")
assert not matches(args, "arg3")
def test_extract_browser_name():
args = ["arg1=1", "--arg2=value2", "--browser=me", "--zome"]
assert extract_browser_name(args) == "me"
if __name__ == "__main__":
mozunit.main()

37
python/mozperftest/mozperftest/tests/test_mach_commands.py
Просмотреть файл

@ -60,6 +60,21 @@ def test_command(mocked_func):
# XXX add assertions
@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
@mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv")
@mock.patch("tryselect.push.push_to_try")
def test_push_command(*mocked):
with _get_command() as test, silence(test):
test.run_perftest(
tests=[EXAMPLE_TESTS_DIR],
flavor="script",
push_to_try=True,
try_platform="g5",
)
mocked[1].assert_called()
# XXX add assertions
@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
@mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv")
def test_doc_flavor(mocked_func):
@ -82,5 +97,27 @@ def test_test_runner(*mocked):
mach_commands.ON_TRY = old
@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
@mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv")
def test_run_python_script(*mocked):
with _get_command(PerftestTests) as test, silence(test) as captured:
test._run_python_script("lib2to3", *["--help"])
stdout, stderr = captured
stdout.seek(0)
assert stdout.read() == "=> lib2to3 [OK]\n"
@mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
@mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv")
def test_run_python_script_failed(*mocked):
with _get_command(PerftestTests) as test, silence(test) as captured:
test._run_python_script("nothing")
stdout, stderr = captured
stdout.seek(0)
assert stdout.read().endswith("[FAILED]\n")
if __name__ == "__main__":
mozunit.main()

2
python/mozperftest/mozperftest/utils.py
Просмотреть файл

@ -41,7 +41,7 @@ def silence(layer=None):
try:
sys.stdout, sys.stderr = StringIO(), StringIO()
sys.stdout.fileno = sys.stderr.fileno = lambda: -1
yield
yield sys.stdout, sys.stderr
finally:
sys.stdout, sys.stderr = oldout, olderr
for obj, meths in patched.items():