зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1609392 - Split Raptor and Browsertime classes from raptor.py r=tarek,AlexandruIonescu,octavian_negru,sparky,perftest-reviewers,alexandru.irimovici
Differential Revision: https://phabricator.services.mozilla.com/D60028 --HG-- rename : testing/raptor/raptor/raptor.py => testing/raptor/raptor/webextension/base.py extra : moz-landing-system : lando
This commit is contained in:
Florin Strugariu
Родитель
53c369d992
Коммит
007caefd70
|
|
@ -0,0 +1,10 @@
|
|||
# flake8: noqa
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from .base import Browsertime
|
||||
from .desktop import BrowsertimeDesktop
|
||||
from .android import BrowsertimeAndroid
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
|
||||
from mozdevice import ADBDevice
|
||||
|
||||
from logger.logger import RaptorLogger
|
||||
from performance_tuning import tune_performance
|
||||
from perftest import PerftestAndroid
|
||||
|
||||
from .base import Browsertime
|
||||
|
||||
LOG = RaptorLogger(component="raptor-browsertime-android")
|
||||
|
||||
|
||||
class BrowsertimeAndroid(PerftestAndroid, Browsertime):
|
||||
"""Android setup and configuration for browsertime
|
||||
|
||||
When running raptor-browsertime tests on android, we create the profile (and set the proxy
|
||||
prefs in the profile that is using playback) but we don't need to copy it onto the device
|
||||
because geckodriver takes care of that.
|
||||
We tell browsertime to use our profile (we pass it in with the firefox.profileTemplate arg);
|
||||
browsertime creates a copy of that and passes that into geckodriver. Geckodriver then takes
|
||||
the profile and copies it onto the mobile device's sdcard for us; and then it even writes
|
||||
the geckoview app config.yaml file onto the device, which points the app to the profile on
|
||||
the sdcard.
|
||||
Therefore, raptor doesn't have to copy the profile onto the scard (and create the config.yaml)
|
||||
file ourselves. Also note when using playback, the nss certificate db is created as usual when
|
||||
mitmproxy is started (and saved in the profile) so it is already included in the profile that
|
||||
browsertime/geckodriver copies onto the device.
|
||||
"""
|
||||
|
||||
def __init__(self, app, binary, activity=None, intent=None, **kwargs):
|
||||
super(BrowsertimeAndroid, self).__init__(
|
||||
app, binary, profile_class="firefox", **kwargs
|
||||
)
|
||||
|
||||
self.config.update({"activity": activity, "intent": intent})
|
||||
|
||||
self.remote_test_root = os.path.abspath(
|
||||
os.path.join(os.sep, "sdcard", "raptor")
|
||||
)
|
||||
self.remote_profile = os.path.join(self.remote_test_root, "profile")
|
||||
|
||||
@property
|
||||
def browsertime_args(self):
|
||||
args_list = [
|
||||
"--browser", "firefox",
|
||||
"--android",
|
||||
# Work around a `selenium-webdriver` issue where Browsertime
|
||||
# fails to find a Firefox binary even though we're going to
|
||||
# actually do things on an Android device.
|
||||
"--firefox.binaryPath", self.browsertime_node,
|
||||
"--firefox.android.package", self.config["binary"],
|
||||
"--firefox.android.activity", self.config["activity"],
|
||||
]
|
||||
|
||||
# if running on Fenix we must add the intent as we use a special non-default one there
|
||||
if self.config["app"] == "fenix" and self.config.get("intent") is not None:
|
||||
args_list.extend(["--firefox.android.intentArgument=-a"])
|
||||
args_list.extend(
|
||||
["--firefox.android.intentArgument", self.config["intent"]]
|
||||
)
|
||||
args_list.extend(["--firefox.android.intentArgument=-d"])
|
||||
args_list.extend(["--firefox.android.intentArgument", str("about:blank")])
|
||||
|
||||
return args_list
|
||||
|
||||
def build_browser_profile(self):
|
||||
super(BrowsertimeAndroid, self).build_browser_profile()
|
||||
|
||||
# Merge in the Android profile.
|
||||
path = os.path.join(self.profile_data_dir, "raptor-android")
|
||||
LOG.info("Merging profile: {}".format(path))
|
||||
self.profile.merge(path)
|
||||
self.profile.set_preferences(
|
||||
{"browser.tabs.remote.autostart": self.config["e10s"]}
|
||||
)
|
||||
|
||||
# There's no great way to have "after" advice in Python, so we do this
|
||||
# in super and then again here since the profile merging re-introduces
|
||||
# the "#MozRunner" delimiters.
|
||||
self.remove_mozprofile_delimiters_from_profile()
|
||||
|
||||
def setup_adb_device(self):
|
||||
if self.device is None:
|
||||
self.device = ADBDevice(verbose=True)
|
||||
tune_performance(self.device, log=LOG)
|
||||
|
||||
self.clear_app_data()
|
||||
self.set_debug_app_flag()
|
||||
|
||||
def run_test_setup(self, test):
|
||||
super(BrowsertimeAndroid, self).run_test_setup(test)
|
||||
|
||||
self.set_reverse_ports()
|
||||
self.turn_on_android_app_proxy()
|
||||
self.remove_mozprofile_delimiters_from_profile()
|
||||
|
||||
def run_tests(self, tests, test_names):
|
||||
self.setup_adb_device()
|
||||
|
||||
return super(BrowsertimeAndroid, self).run_tests(tests, test_names)
|
||||
|
||||
def run_test_teardown(self, test):
|
||||
LOG.info("removing reverse socket connections")
|
||||
self.device.remove_socket_connections("reverse")
|
||||
|
||||
super(BrowsertimeAndroid, self).run_test_teardown(test)
|
||||
|
|
@ -0,0 +1,300 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
|
||||
import os
|
||||
import json
|
||||
import six
|
||||
|
||||
import mozprocess
|
||||
from benchmark import Benchmark
|
||||
from logger.logger import RaptorLogger
|
||||
from perftest import Perftest
|
||||
from results import BrowsertimeResultsHandler
|
||||
|
||||
LOG = RaptorLogger(component="raptor-browsertime")
|
||||
DEFAULT_CHROMEVERSION = "77"
|
||||
|
||||
|
||||
class Browsertime(Perftest):
|
||||
"""Abstract base class for Browsertime"""
|
||||
|
||||
__metaclass__ = ABCMeta
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def browsertime_args(self):
|
||||
pass
|
||||
|
||||
def __init__(self, app, binary, process_handler=None, **kwargs):
|
||||
self.process_handler = process_handler or mozprocess.ProcessHandler
|
||||
for key in list(kwargs):
|
||||
if key.startswith("browsertime_"):
|
||||
value = kwargs.pop(key)
|
||||
setattr(self, key, value)
|
||||
|
||||
def klass(**config):
|
||||
root_results_dir = os.path.join(
|
||||
os.environ.get("MOZ_UPLOAD_DIR", os.getcwd()), "browsertime-results"
|
||||
)
|
||||
return BrowsertimeResultsHandler(config, root_results_dir=root_results_dir)
|
||||
|
||||
super(Browsertime, self).__init__(
|
||||
app, binary, results_handler_class=klass, **kwargs
|
||||
)
|
||||
LOG.info("cwd: '{}'".format(os.getcwd()))
|
||||
self.config["browsertime"] = True
|
||||
|
||||
# For debugging.
|
||||
for k in (
|
||||
"browsertime_node",
|
||||
"browsertime_browsertimejs",
|
||||
"browsertime_ffmpeg",
|
||||
"browsertime_geckodriver",
|
||||
"browsertime_chromedriver",
|
||||
):
|
||||
try:
|
||||
if not self.browsertime_video and k == "browsertime_ffmpeg":
|
||||
continue
|
||||
LOG.info("{}: {}".format(k, getattr(self, k)))
|
||||
LOG.info("{}: {}".format(k, os.stat(getattr(self, k))))
|
||||
except Exception as e:
|
||||
LOG.info("{}: {}".format(k, e))
|
||||
|
||||
def build_browser_profile(self):
|
||||
super(Browsertime, self).build_browser_profile()
|
||||
self.remove_mozprofile_delimiters_from_profile()
|
||||
|
||||
def remove_mozprofile_delimiters_from_profile(self):
|
||||
# Perftest.build_browser_profile uses mozprofile to create the profile and merge in prefs;
|
||||
# while merging, mozprofile adds in special delimiters; these delimiters are not recognized
|
||||
# by selenium-webdriver ultimately causing Firefox launch to fail. So we must remove these
|
||||
# delimiters from the browser profile before passing into btime via firefox.profileTemplate
|
||||
|
||||
LOG.info("Removing mozprofile delimiters from browser profile")
|
||||
userjspath = os.path.join(self.profile.profile, "user.js")
|
||||
try:
|
||||
with open(userjspath) as userjsfile:
|
||||
lines = userjsfile.readlines()
|
||||
lines = [line for line in lines if not line.startswith("#MozRunner")]
|
||||
with open(userjspath, "w") as userjsfile:
|
||||
userjsfile.writelines(lines)
|
||||
except Exception as e:
|
||||
LOG.critical("Exception {} while removing mozprofile delimiters".format(e))
|
||||
|
||||
def set_browser_test_prefs(self, raw_prefs):
|
||||
# add test specific preferences
|
||||
LOG.info("setting test-specific Firefox preferences")
|
||||
self.profile.set_preferences(json.loads(raw_prefs))
|
||||
self.remove_mozprofile_delimiters_from_profile()
|
||||
|
||||
def run_test_setup(self, test):
|
||||
super(Browsertime, self).run_test_setup(test)
|
||||
|
||||
if test.get("type") == "benchmark":
|
||||
# benchmark-type tests require the benchmark test to be served out
|
||||
self.benchmark = Benchmark(self.config, test)
|
||||
test["test_url"] = test["test_url"].replace("<host>", self.benchmark.host)
|
||||
test["test_url"] = test["test_url"].replace("<port>", self.benchmark.port)
|
||||
|
||||
if test.get("playback") is not None and self.playback is None:
|
||||
self.start_playback(test)
|
||||
|
||||
# TODO: geckodriver/chromedriver from tasks.
|
||||
self.driver_paths = []
|
||||
if self.browsertime_geckodriver:
|
||||
self.driver_paths.extend(
|
||||
["--firefox.geckodriverPath", self.browsertime_geckodriver]
|
||||
)
|
||||
if self.browsertime_chromedriver:
|
||||
if (
|
||||
not self.config.get("run_local", None)
|
||||
or "{}" in self.browsertime_chromedriver
|
||||
):
|
||||
if self.browser_version:
|
||||
bvers = str(self.browser_version)
|
||||
chromedriver_version = bvers.split(".")[0]
|
||||
else:
|
||||
chromedriver_version = DEFAULT_CHROMEVERSION
|
||||
|
||||
self.browsertime_chromedriver = self.browsertime_chromedriver.format(
|
||||
chromedriver_version
|
||||
)
|
||||
|
||||
self.driver_paths.extend(
|
||||
["--chrome.chromedriverPath", self.browsertime_chromedriver]
|
||||
)
|
||||
|
||||
LOG.info("test: {}".format(test))
|
||||
|
||||
def run_test_teardown(self, test):
|
||||
super(Browsertime, self).run_test_teardown(test)
|
||||
|
||||
# if we were using a playback tool, stop it
|
||||
if self.playback is not None:
|
||||
self.playback.stop()
|
||||
self.playback = None
|
||||
|
||||
def check_for_crashes(self):
|
||||
super(Browsertime, self).check_for_crashes()
|
||||
|
||||
def clean_up(self):
|
||||
super(Browsertime, self).clean_up()
|
||||
|
||||
def _compose_cmd(self, test, timeout):
|
||||
browsertime_script = [
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..",
|
||||
"..",
|
||||
"browsertime",
|
||||
"browsertime_pageload.js",
|
||||
)
|
||||
]
|
||||
|
||||
btime_args = self.browsertime_args
|
||||
if self.config["app"] in ("chrome", "chromium"):
|
||||
btime_args.extend(self.setup_chrome_args(test))
|
||||
|
||||
browsertime_script.extend(btime_args)
|
||||
|
||||
# pass a few extra options to the browsertime script
|
||||
# XXX maybe these should be in the browsertime_args() func
|
||||
browsertime_script.extend(
|
||||
["--browsertime.page_cycles", str(test.get("page_cycles", 1))]
|
||||
)
|
||||
browsertime_script.extend(["--browsertime.url", test["test_url"]])
|
||||
|
||||
# Raptor's `pageCycleDelay` delay (ms) between pageload cycles
|
||||
browsertime_script.extend(["--browsertime.page_cycle_delay", "1000"])
|
||||
|
||||
# Raptor's `post startup delay` is settle time after the browser has started
|
||||
browsertime_script.extend(
|
||||
["--browsertime.post_startup_delay", str(self.post_startup_delay)]
|
||||
)
|
||||
|
||||
browsertime_options = [
|
||||
"--firefox.profileTemplate", str(self.profile.profile),
|
||||
"--skipHar",
|
||||
"--video", self.browsertime_video and "true" or "false",
|
||||
"--visualMetrics", "false",
|
||||
# url load timeout (milliseconds)
|
||||
"--timeouts.pageLoad", str(timeout),
|
||||
# running browser scripts timeout (milliseconds)
|
||||
"--timeouts.script", str(timeout * int(test.get("page_cycles", 1))),
|
||||
"-vvv",
|
||||
"--resultDir", self.results_handler.result_dir_for_test(test),
|
||||
]
|
||||
|
||||
# have browsertime use our newly-created conditioned-profile path
|
||||
if not self.no_condprof:
|
||||
self.profile.profile = self.conditioned_profile_dir
|
||||
|
||||
if self.config["gecko_profile"]:
|
||||
self.config[
|
||||
"browsertime_result_dir"
|
||||
] = self.results_handler.result_dir_for_test(test)
|
||||
self._init_gecko_profiling(test)
|
||||
browsertime_options.append("--firefox.geckoProfiler")
|
||||
|
||||
for option, browser_time_option in (
|
||||
("gecko_profile_interval", "--firefox.geckoProfilerParams.interval"),
|
||||
("gecko_profile_entries", "--firefox.geckoProfilerParams.bufferSize"),
|
||||
):
|
||||
value = self.config.get(option)
|
||||
if value is None:
|
||||
value = test.get(option)
|
||||
if value is not None:
|
||||
browsertime_options.extend([browser_time_option, str(value)])
|
||||
|
||||
return (
|
||||
[self.browsertime_node, self.browsertime_browsertimejs]
|
||||
+ self.driver_paths
|
||||
+ browsertime_script
|
||||
+
|
||||
# -n option for the browsertime to restart the browser
|
||||
browsertime_options
|
||||
+ ["-n", str(test.get("browser_cycles", 1))]
|
||||
)
|
||||
|
||||
def _compute_process_timeout(self, test, timeout):
|
||||
# bt_timeout will be the overall browsertime cmd/session timeout (seconds)
|
||||
# browsertime deals with page cycles internally, so we need to give it a timeout
|
||||
# value that includes all page cycles
|
||||
bt_timeout = int(timeout / 1000) * int(test.get("page_cycles", 1))
|
||||
|
||||
# the post-startup-delay is a delay after the browser has started, to let it settle
|
||||
# it's handled within browsertime itself by loading a 'preUrl' (about:blank) and having a
|
||||
# delay after that ('preURLDelay') as the post-startup-delay, so we must add that in sec
|
||||
bt_timeout += int(self.post_startup_delay / 1000)
|
||||
|
||||
# add some time for browser startup, time for the browsertime measurement code
|
||||
# to be injected/invoked, and for exceptions to bubble up; be generous
|
||||
bt_timeout += 20
|
||||
|
||||
# browsertime also handles restarting the browser/running all of the browser cycles;
|
||||
# so we need to multiply our bt_timeout by the number of browser cycles
|
||||
bt_timeout = bt_timeout * int(test.get("browser_cycles", 1))
|
||||
|
||||
# if geckoProfile enabled, give browser more time for profiling
|
||||
if self.config["gecko_profile"] is True:
|
||||
bt_timeout += 5 * 60
|
||||
return bt_timeout
|
||||
|
||||
def run_test(self, test, timeout):
|
||||
self.run_test_setup(test)
|
||||
# timeout is a single page-load timeout value (ms) from the test INI
|
||||
# this will be used for btime --timeouts.pageLoad
|
||||
cmd = self._compose_cmd(test, timeout)
|
||||
|
||||
if test.get("type") == "benchmark":
|
||||
cmd.extend(
|
||||
[
|
||||
"--script",
|
||||
os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..",
|
||||
"..",
|
||||
"browsertime",
|
||||
"browsertime_benchmark.js",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
LOG.info("timeout (s): {}".format(timeout))
|
||||
LOG.info("browsertime cwd: {}".format(os.getcwd()))
|
||||
LOG.info("browsertime cmd: {}".format(" ".join(cmd)))
|
||||
if self.browsertime_video:
|
||||
LOG.info("browsertime_ffmpeg: {}".format(self.browsertime_ffmpeg))
|
||||
|
||||
# browsertime requires ffmpeg on the PATH for `--video=true`.
|
||||
# It's easier to configure the PATH here than at the TC level.
|
||||
env = dict(os.environ)
|
||||
if self.browsertime_video and self.browsertime_ffmpeg:
|
||||
ffmpeg_dir = os.path.dirname(os.path.abspath(self.browsertime_ffmpeg))
|
||||
old_path = env.setdefault("PATH", "")
|
||||
new_path = os.pathsep.join([ffmpeg_dir, old_path])
|
||||
if isinstance(new_path, six.text_type):
|
||||
# Python 2 doesn't like unicode in the environment.
|
||||
new_path = new_path.encode("utf-8", "strict")
|
||||
env["PATH"] = new_path
|
||||
|
||||
LOG.info("PATH: {}".format(env["PATH"]))
|
||||
|
||||
try:
|
||||
proc = self.process_handler(cmd, env=env)
|
||||
proc.run(
|
||||
timeout=self._compute_process_timeout(test, timeout),
|
||||
outputTimeout=2 * 60,
|
||||
)
|
||||
proc.wait()
|
||||
|
||||
except Exception as e:
|
||||
LOG.critical("Error while attempting to run browsertime: %s" % str(e))
|
||||
raise
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from logger.logger import RaptorLogger
|
||||
from perftest import PerftestDesktop
|
||||
|
||||
from .base import Browsertime
|
||||
|
||||
LOG = RaptorLogger(component="raptor-browsertime-desktop")
|
||||
|
||||
|
||||
class BrowsertimeDesktop(PerftestDesktop, Browsertime):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(BrowsertimeDesktop, self).__init__(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def browsertime_args(self):
|
||||
binary_path = self.config["binary"]
|
||||
LOG.info("binary_path: {}".format(binary_path))
|
||||
|
||||
if self.config["app"] == "chrome":
|
||||
return ["--browser", self.config["app"], "--chrome.binaryPath", binary_path]
|
||||
return ["--browser", self.config["app"], "--firefox.binaryPath", binary_path]
|
||||
|
||||
def setup_chrome_args(self, test):
|
||||
# Setup required chrome arguments
|
||||
chrome_args = self.desktop_chrome_args(test)
|
||||
|
||||
# Add this argument here, it's added by mozrunner
|
||||
# for raptor
|
||||
chrome_args.append("--no-first-run")
|
||||
|
||||
btime_chrome_args = []
|
||||
for arg in chrome_args:
|
||||
btime_chrome_args.extend(["--chrome.args=" + str(arg.replace("'", '"'))])
|
||||
|
||||
return btime_chrome_args
|
||||
|
|
@ -11,26 +11,38 @@ import os
|
|||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from abc import ABCMeta, abstractmethod
|
||||
|
||||
import mozinfo
|
||||
import mozprocess
|
||||
import mozversion
|
||||
from cmdline import FIREFOX_ANDROID_APPS
|
||||
from condprof.client import get_profile, ProfileNotFoundError
|
||||
from condprof.util import get_current_platform
|
||||
from logger.logger import RaptorLogger
|
||||
from mozprofile import create_profile
|
||||
from mozproxy import get_playback
|
||||
|
||||
|
||||
# need this so raptor imports work both from /raptor and via mach
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
paths = [here]
|
||||
|
||||
webext_dir = os.path.join(here, "..", "webext")
|
||||
paths.append(webext_dir)
|
||||
|
||||
for path in paths:
|
||||
if not os.path.exists(path):
|
||||
raise IOError("%s does not exist. " % path)
|
||||
sys.path.insert(0, path)
|
||||
|
||||
|
||||
from cmdline import FIREFOX_ANDROID_APPS
|
||||
from condprof.client import get_profile, ProfileNotFoundError
|
||||
from condprof.util import get_current_platform
|
||||
from logger.logger import RaptorLogger
|
||||
from gecko_profile import GeckoProfile
|
||||
from results import RaptorResultsHandler
|
||||
|
||||
LOG = RaptorLogger(component='raptor-perftest')
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
LOG = RaptorLogger(component="raptor-perftest")
|
||||
|
||||
try:
|
||||
from mozbuild.base import MozbuildObject
|
||||
|
|
@ -46,63 +58,85 @@ either Raptor or browsertime."""
|
|||
|
||||
__metaclass__ = ABCMeta
|
||||
|
||||
def __init__(self, app, binary, run_local=False, noinstall=False,
|
||||
obj_path=None, profile_class=None, installerpath=None,
|
||||
gecko_profile=False, gecko_profile_interval=None, gecko_profile_entries=None,
|
||||
symbols_path=None, host=None, power_test=False, cpu_test=False, memory_test=False,
|
||||
is_release_build=False, debug_mode=False, post_startup_delay=None,
|
||||
interrupt_handler=None, e10s=True, enable_webrender=False,
|
||||
results_handler_class=RaptorResultsHandler, no_conditioned_profile=False,
|
||||
device_name=None, extra_prefs={}, **kwargs):
|
||||
def __init__(
|
||||
self,
|
||||
app,
|
||||
binary,
|
||||
run_local=False,
|
||||
noinstall=False,
|
||||
obj_path=None,
|
||||
profile_class=None,
|
||||
installerpath=None,
|
||||
gecko_profile=False,
|
||||
gecko_profile_interval=None,
|
||||
gecko_profile_entries=None,
|
||||
symbols_path=None,
|
||||
host=None,
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
is_release_build=False,
|
||||
debug_mode=False,
|
||||
post_startup_delay=None,
|
||||
interrupt_handler=None,
|
||||
e10s=True,
|
||||
enable_webrender=False,
|
||||
results_handler_class=RaptorResultsHandler,
|
||||
no_conditioned_profile=False,
|
||||
device_name=None,
|
||||
extra_prefs={},
|
||||
**kwargs
|
||||
):
|
||||
|
||||
# Override the magic --host HOST_IP with the value of the environment variable.
|
||||
if host == 'HOST_IP':
|
||||
host = os.environ['HOST_IP']
|
||||
if host == "HOST_IP":
|
||||
host = os.environ["HOST_IP"]
|
||||
|
||||
self.config = {
|
||||
'app': app,
|
||||
'binary': binary,
|
||||
'platform': mozinfo.os,
|
||||
'processor': mozinfo.processor,
|
||||
'run_local': run_local,
|
||||
'obj_path': obj_path,
|
||||
'gecko_profile': gecko_profile,
|
||||
'gecko_profile_interval': gecko_profile_interval,
|
||||
'gecko_profile_entries': gecko_profile_entries,
|
||||
'symbols_path': symbols_path,
|
||||
'host': host,
|
||||
'power_test': power_test,
|
||||
'memory_test': memory_test,
|
||||
'cpu_test': cpu_test,
|
||||
'is_release_build': is_release_build,
|
||||
'enable_control_server_wait': memory_test or cpu_test,
|
||||
'e10s': e10s,
|
||||
'enable_webrender': enable_webrender,
|
||||
'no_conditioned_profile': no_conditioned_profile,
|
||||
'device_name': device_name,
|
||||
'enable_fission': extra_prefs.get('fission.autostart', False),
|
||||
'extra_prefs': extra_prefs
|
||||
"app": app,
|
||||
"binary": binary,
|
||||
"platform": mozinfo.os,
|
||||
"processor": mozinfo.processor,
|
||||
"run_local": run_local,
|
||||
"obj_path": obj_path,
|
||||
"gecko_profile": gecko_profile,
|
||||
"gecko_profile_interval": gecko_profile_interval,
|
||||
"gecko_profile_entries": gecko_profile_entries,
|
||||
"symbols_path": symbols_path,
|
||||
"host": host,
|
||||
"power_test": power_test,
|
||||
"memory_test": memory_test,
|
||||
"cpu_test": cpu_test,
|
||||
"is_release_build": is_release_build,
|
||||
"enable_control_server_wait": memory_test or cpu_test,
|
||||
"e10s": e10s,
|
||||
"enable_webrender": enable_webrender,
|
||||
"no_conditioned_profile": no_conditioned_profile,
|
||||
"device_name": device_name,
|
||||
"enable_fission": extra_prefs.get("fission.autostart", False),
|
||||
"extra_prefs": extra_prefs,
|
||||
}
|
||||
|
||||
self.firefox_android_apps = FIREFOX_ANDROID_APPS
|
||||
# See bugs 1582757, 1606199, and 1606767; until we support win10-aarch64,
|
||||
# fennec_aurora, and reference browser conditioned-profile builds,
|
||||
# fall back to mozrunner-created profiles
|
||||
self.no_condprof = ((self.config['platform'] == 'win'
|
||||
and self.config['processor'] == 'aarch64') or
|
||||
self.config['binary'] == 'org.mozilla.fennec_aurora' or
|
||||
self.config['binary'] == 'org.mozilla.reference.browser.raptor' or
|
||||
self.config['no_conditioned_profile'])
|
||||
self.no_condprof = (
|
||||
(self.config["platform"] == "win" and self.config["processor"] == "aarch64")
|
||||
or self.config["binary"] == "org.mozilla.fennec_aurora"
|
||||
or self.config["binary"] == "org.mozilla.reference.browser.raptor"
|
||||
or self.config["no_conditioned_profile"]
|
||||
)
|
||||
LOG.info("self.no_condprof is: {}".format(self.no_condprof))
|
||||
|
||||
# We can never use e10s on fennec
|
||||
if self.config['app'] == 'fennec':
|
||||
self.config['e10s'] = False
|
||||
if self.config["app"] == "fennec":
|
||||
self.config["e10s"] = False
|
||||
|
||||
self.browser_name = None
|
||||
self.browser_version = None
|
||||
|
||||
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
|
||||
self.raptor_venv = os.path.join(os.getcwd(), "raptor-venv")
|
||||
self.installerpath = installerpath
|
||||
self.playback = None
|
||||
self.benchmark = None
|
||||
|
|
@ -118,23 +152,25 @@ either Raptor or browsertime."""
|
|||
self.browser_name, self.browser_version = self.get_browser_meta()
|
||||
|
||||
browser_name, browser_version = self.get_browser_meta()
|
||||
self.results_handler.add_browser_meta(self.config['app'], browser_version)
|
||||
self.results_handler.add_browser_meta(self.config["app"], browser_version)
|
||||
|
||||
# debug mode is currently only supported when running locally
|
||||
self.debug_mode = debug_mode if self.config['run_local'] else False
|
||||
self.debug_mode = debug_mode if self.config["run_local"] else False
|
||||
|
||||
# if running debug-mode reduce the pause after browser startup
|
||||
if self.debug_mode:
|
||||
self.post_startup_delay = min(self.post_startup_delay, 3000)
|
||||
LOG.info("debug-mode enabled, reducing post-browser startup pause to %d ms"
|
||||
% self.post_startup_delay)
|
||||
LOG.info(
|
||||
"debug-mode enabled, reducing post-browser startup pause to %d ms"
|
||||
% self.post_startup_delay
|
||||
)
|
||||
LOG.info("main raptor init, config is: %s" % str(self.config))
|
||||
|
||||
self.build_browser_profile()
|
||||
|
||||
@property
|
||||
def is_localhost(self):
|
||||
return self.config.get('host') in ('localhost', '127.0.0.1')
|
||||
return self.config.get("host") in ("localhost", "127.0.0.1")
|
||||
|
||||
def get_conditioned_profile(self):
|
||||
"""Downloads a platform-specific conditioned profile, using the
|
||||
|
|
@ -142,8 +178,11 @@ either Raptor or browsertime."""
|
|||
|
||||
# create a temp file to help ensure uniqueness
|
||||
temp_download_dir = tempfile.mkdtemp()
|
||||
LOG.info("Making temp_download_dir from inside get_conditioned_profile {}"
|
||||
.format(temp_download_dir))
|
||||
LOG.info(
|
||||
"Making temp_download_dir from inside get_conditioned_profile {}".format(
|
||||
temp_download_dir
|
||||
)
|
||||
)
|
||||
# call condprof's client API to yield our platform-specific
|
||||
# conditioned-profile binary
|
||||
if isinstance(self, PerftestAndroid):
|
||||
|
|
@ -158,18 +197,27 @@ either Raptor or browsertime."""
|
|||
cond_prof_target_dir = get_profile(temp_download_dir, platform, "settled")
|
||||
except ProfileNotFoundError:
|
||||
# If we can't find the profile on mozilla-central, we look on try
|
||||
cond_prof_target_dir = get_profile(temp_download_dir, platform,
|
||||
"settled", repo="try")
|
||||
cond_prof_target_dir = get_profile(
|
||||
temp_download_dir, platform, "settled", repo="try"
|
||||
)
|
||||
# now get the full directory path to our fetched conditioned profile
|
||||
self.conditioned_profile_dir = os.path.join(temp_download_dir, cond_prof_target_dir)
|
||||
self.conditioned_profile_dir = os.path.join(
|
||||
temp_download_dir, cond_prof_target_dir
|
||||
)
|
||||
if not os.path.exists(cond_prof_target_dir):
|
||||
LOG.critical("Can't find target_dir {}, from get_profile()"
|
||||
"temp_download_dir {}, platform {}, settled"
|
||||
.format(cond_prof_target_dir, temp_download_dir, platform))
|
||||
LOG.critical(
|
||||
"Can't find target_dir {}, from get_profile()"
|
||||
"temp_download_dir {}, platform {}, settled".format(
|
||||
cond_prof_target_dir, temp_download_dir, platform
|
||||
)
|
||||
)
|
||||
raise OSError
|
||||
|
||||
LOG.info("self.conditioned_profile_dir is now set: {}"
|
||||
.format(self.conditioned_profile_dir))
|
||||
LOG.info(
|
||||
"self.conditioned_profile_dir is now set: {}".format(
|
||||
self.conditioned_profile_dir
|
||||
)
|
||||
)
|
||||
shutil.rmtree(temp_download_dir)
|
||||
|
||||
return self.conditioned_profile_dir
|
||||
|
|
@ -180,58 +228,66 @@ either Raptor or browsertime."""
|
|||
else:
|
||||
self.get_conditioned_profile()
|
||||
# use mozprofile to create a profile for us, from our conditioned profile's path
|
||||
self.profile = create_profile(self.profile_class, profile=self.conditioned_profile_dir)
|
||||
self.profile = create_profile(
|
||||
self.profile_class, profile=self.conditioned_profile_dir
|
||||
)
|
||||
# Merge extra profile data from testing/profiles
|
||||
with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
|
||||
base_profiles = json.load(fh)['raptor']
|
||||
with open(os.path.join(self.profile_data_dir, "profiles.json"), "r") as fh:
|
||||
base_profiles = json.load(fh)["raptor"]
|
||||
|
||||
for profile in base_profiles:
|
||||
path = os.path.join(self.profile_data_dir, profile)
|
||||
LOG.info("Merging profile: {}".format(path))
|
||||
self.profile.merge(path)
|
||||
|
||||
if self.config['extra_prefs'].get('fission.autostart', False):
|
||||
LOG.info('Enabling fission via browser preferences')
|
||||
LOG.info('Browser preferences: {}'.format(self.config['extra_prefs']))
|
||||
self.profile.set_preferences(self.config['extra_prefs'])
|
||||
if self.config["extra_prefs"].get("fission.autostart", False):
|
||||
LOG.info("Enabling fission via browser preferences")
|
||||
LOG.info("Browser preferences: {}".format(self.config["extra_prefs"]))
|
||||
self.profile.set_preferences(self.config["extra_prefs"])
|
||||
|
||||
# share the profile dir with the config and the control server
|
||||
self.config['local_profile_dir'] = self.profile.profile
|
||||
LOG.info('Local browser profile: {}'.format(self.profile.profile))
|
||||
self.config["local_profile_dir"] = self.profile.profile
|
||||
LOG.info("Local browser profile: {}".format(self.profile.profile))
|
||||
|
||||
@property
|
||||
def profile_data_dir(self):
|
||||
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
|
||||
return os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'], 'testing', 'profiles')
|
||||
if "MOZ_DEVELOPER_REPO_DIR" in os.environ:
|
||||
return os.path.join(
|
||||
os.environ["MOZ_DEVELOPER_REPO_DIR"], "testing", "profiles"
|
||||
)
|
||||
if build:
|
||||
return os.path.join(build.topsrcdir, 'testing', 'profiles')
|
||||
return os.path.join(here, 'profile_data')
|
||||
return os.path.join(build.topsrcdir, "testing", "profiles")
|
||||
return os.path.join(here, "profile_data")
|
||||
|
||||
@property
|
||||
def artifact_dir(self):
|
||||
artifact_dir = os.getcwd()
|
||||
if self.config.get('run_local', False):
|
||||
if 'MOZ_DEVELOPER_REPO_DIR' in os.environ:
|
||||
artifact_dir = os.path.join(os.environ['MOZ_DEVELOPER_REPO_DIR'],
|
||||
'testing', 'mozharness', 'build')
|
||||
if self.config.get("run_local", False):
|
||||
if "MOZ_DEVELOPER_REPO_DIR" in os.environ:
|
||||
artifact_dir = os.path.join(
|
||||
os.environ["MOZ_DEVELOPER_REPO_DIR"],
|
||||
"testing",
|
||||
"mozharness",
|
||||
"build",
|
||||
)
|
||||
else:
|
||||
artifact_dir = here
|
||||
elif os.getenv('MOZ_UPLOAD_DIR'):
|
||||
artifact_dir = os.getenv('MOZ_UPLOAD_DIR')
|
||||
elif os.getenv("MOZ_UPLOAD_DIR"):
|
||||
artifact_dir = os.getenv("MOZ_UPLOAD_DIR")
|
||||
return artifact_dir
|
||||
|
||||
@abstractmethod
|
||||
def run_test_setup(self, test):
|
||||
LOG.info("starting test: %s" % test['name'])
|
||||
LOG.info("starting test: %s" % test["name"])
|
||||
|
||||
# if 'alert_on' was provided in the test INI, add to our config for results/output
|
||||
self.config['subtest_alert_on'] = test.get('alert_on')
|
||||
self.config["subtest_alert_on"] = test.get("alert_on")
|
||||
|
||||
if test.get('playback') is not None and self.playback is None:
|
||||
if test.get("playback") is not None and self.playback is None:
|
||||
self.start_playback(test)
|
||||
|
||||
if test.get("preferences") is not None:
|
||||
self.set_browser_test_prefs(test['preferences'])
|
||||
self.set_browser_test_prefs(test["preferences"])
|
||||
|
||||
@abstractmethod
|
||||
def setup_chrome_args(self):
|
||||
|
|
@ -245,7 +301,7 @@ either Raptor or browsertime."""
|
|||
try:
|
||||
for test in tests:
|
||||
try:
|
||||
self.run_test(test, timeout=int(test.get('page_timeout')))
|
||||
self.run_test(test, timeout=int(test.get("page_timeout")))
|
||||
except RuntimeError as e:
|
||||
LOG.critical("Tests failed to finish! Application timed out.")
|
||||
LOG.error(e)
|
||||
|
|
@ -264,7 +320,7 @@ either Raptor or browsertime."""
|
|||
self.check_for_crashes()
|
||||
|
||||
# gecko profiling symbolication
|
||||
if self.config['gecko_profile']:
|
||||
if self.config["gecko_profile"]:
|
||||
self.gecko_profiler.symbolicate()
|
||||
# clean up the temp gecko profiling folders
|
||||
LOG.info("cleaning up after gecko profiling")
|
||||
|
|
@ -273,12 +329,12 @@ either Raptor or browsertime."""
|
|||
def process_results(self, tests, test_names):
|
||||
# when running locally output results in build/raptor.json; when running
|
||||
# in production output to a local.json to be turned into tc job artifact
|
||||
raptor_json_path = os.path.join(self.artifact_dir, 'raptor.json')
|
||||
if not self.config.get('run_local', False):
|
||||
raptor_json_path = os.path.join(os.getcwd(), 'local.json')
|
||||
raptor_json_path = os.path.join(self.artifact_dir, "raptor.json")
|
||||
if not self.config.get("run_local", False):
|
||||
raptor_json_path = os.path.join(os.getcwd(), "local.json")
|
||||
|
||||
self.config['raptor_json_path'] = raptor_json_path
|
||||
self.config['artifact_dir'] = self.artifact_dir
|
||||
self.config["raptor_json_path"] = raptor_json_path
|
||||
self.config["artifact_dir"] = self.artifact_dir
|
||||
return self.results_handler.summarize_and_output(self.config, tests, test_names)
|
||||
|
||||
@abstractmethod
|
||||
|
|
@ -313,52 +369,59 @@ either Raptor or browsertime."""
|
|||
def log_recording_dates(self, test):
|
||||
_recording_paths = self.get_recording_paths(test)
|
||||
if _recording_paths is None:
|
||||
LOG.info("No playback recordings specified in the test; so not getting recording info")
|
||||
LOG.info(
|
||||
"No playback recordings specified in the test; so not getting recording info"
|
||||
)
|
||||
return
|
||||
|
||||
for r in _recording_paths:
|
||||
json_path = '{}.json'.format(r.split('.')[0])
|
||||
json_path = "{}.json".format(r.split(".")[0])
|
||||
|
||||
if os.path.exists(json_path):
|
||||
with open(json_path) as f:
|
||||
recording_date = json.loads(f.read()).get('recording_date')
|
||||
recording_date = json.loads(f.read()).get("recording_date")
|
||||
|
||||
if recording_date is not None:
|
||||
LOG.info('Playback recording date: {} '.
|
||||
format(recording_date.split(' ')[0]))
|
||||
LOG.info(
|
||||
"Playback recording date: {} ".format(
|
||||
recording_date.split(" ")[0]
|
||||
)
|
||||
)
|
||||
else:
|
||||
LOG.info('Playback recording date not available')
|
||||
LOG.info("Playback recording date not available")
|
||||
else:
|
||||
LOG.info('Playback recording information not available')
|
||||
LOG.info("Playback recording information not available")
|
||||
|
||||
def get_playback_config(self, test):
|
||||
platform = self.config['platform']
|
||||
playback_dir = os.path.join(here, 'playback')
|
||||
platform = self.config["platform"]
|
||||
playback_dir = os.path.join(here, "playback")
|
||||
|
||||
self.config.update({
|
||||
'playback_tool': test.get('playback'),
|
||||
'playback_version': test.get('playback_version', "4.0.4"),
|
||||
'playback_binary_zip': test.get('playback_binary_zip_%s' % platform),
|
||||
'playback_pageset_zip': test.get('playback_pageset_zip_%s' % platform),
|
||||
'playback_binary_manifest': test.get('playback_binary_manifest'),
|
||||
'playback_pageset_manifest': test.get('playback_pageset_manifest'),
|
||||
})
|
||||
self.config.update(
|
||||
{
|
||||
"playback_tool": test.get("playback"),
|
||||
"playback_version": test.get("playback_version", "4.0.4"),
|
||||
"playback_binary_zip": test.get("playback_binary_zip_%s" % platform),
|
||||
"playback_pageset_zip": test.get("playback_pageset_zip_%s" % platform),
|
||||
"playback_binary_manifest": test.get("playback_binary_manifest"),
|
||||
"playback_pageset_manifest": test.get("playback_pageset_manifest"),
|
||||
}
|
||||
)
|
||||
|
||||
for key in ('playback_pageset_manifest', 'playback_pageset_zip'):
|
||||
for key in ("playback_pageset_manifest", "playback_pageset_zip"):
|
||||
if self.config.get(key) is None:
|
||||
continue
|
||||
self.config[key] = os.path.join(playback_dir, self.config[key])
|
||||
|
||||
LOG.info("test uses playback tool: %s " % self.config['playback_tool'])
|
||||
LOG.info("test uses playback tool: %s " % self.config["playback_tool"])
|
||||
|
||||
def delete_proxy_settings_from_profile(self):
|
||||
# Must delete the proxy settings from the profile if running
|
||||
# the test with a host different from localhost.
|
||||
userjspath = os.path.join(self.profile.profile, 'user.js')
|
||||
userjspath = os.path.join(self.profile.profile, "user.js")
|
||||
with open(userjspath) as userjsfile:
|
||||
prefs = userjsfile.readlines()
|
||||
prefs = [pref for pref in prefs if 'network.proxy' not in pref]
|
||||
with open(userjspath, 'w') as userjsfile:
|
||||
prefs = [pref for pref in prefs if "network.proxy" not in pref]
|
||||
with open(userjspath, "w") as userjsfile:
|
||||
userjsfile.writelines(prefs)
|
||||
|
||||
def start_playback(self, test):
|
||||
|
|
@ -366,7 +429,7 @@ either Raptor or browsertime."""
|
|||
self.get_playback_config(test)
|
||||
self.playback = get_playback(self.config)
|
||||
|
||||
self.playback.config['playback_files'] = self.get_recording_paths(test)
|
||||
self.playback.config["playback_files"] = self.get_recording_paths(test)
|
||||
|
||||
# let's start it!
|
||||
self.playback.start()
|
||||
|
|
@ -375,46 +438,44 @@ either Raptor or browsertime."""
|
|||
|
||||
def _init_gecko_profiling(self, test):
|
||||
LOG.info("initializing gecko profiler")
|
||||
upload_dir = os.getenv('MOZ_UPLOAD_DIR')
|
||||
upload_dir = os.getenv("MOZ_UPLOAD_DIR")
|
||||
if not upload_dir:
|
||||
LOG.critical("Profiling ignored because MOZ_UPLOAD_DIR was not set")
|
||||
else:
|
||||
self.gecko_profiler = GeckoProfile(upload_dir,
|
||||
self.config,
|
||||
test)
|
||||
self.gecko_profiler = GeckoProfile(upload_dir, self.config, test)
|
||||
|
||||
|
||||
class PerftestAndroid(Perftest):
|
||||
"""Mixin class for Android-specific Perftest subclasses."""
|
||||
|
||||
def setup_chrome_args(self, test):
|
||||
'''Sets up chrome/chromium cmd-line arguments.
|
||||
"""Sets up chrome/chromium cmd-line arguments.
|
||||
|
||||
Needs to be "implemented" here to deal with Python 2
|
||||
unittest failures.
|
||||
'''
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_browser_meta(self):
|
||||
'''Returns the browser name and version in a tuple (name, version).
|
||||
"""Returns the browser name and version in a tuple (name, version).
|
||||
|
||||
Uses mozversion as the primary method to get this meta data and for
|
||||
android this is the only method which exists to get this data. With android,
|
||||
we use the installerpath attribute to determine this and this only works
|
||||
with Firefox browsers.
|
||||
'''
|
||||
"""
|
||||
browser_name = None
|
||||
browser_version = None
|
||||
|
||||
if self.config['app'] in self.firefox_android_apps:
|
||||
if self.config["app"] in self.firefox_android_apps:
|
||||
try:
|
||||
meta = mozversion.get_version(binary=self.installerpath)
|
||||
browser_name = meta.get('application_name')
|
||||
browser_version = meta.get('application_version')
|
||||
browser_name = meta.get("application_name")
|
||||
browser_version = meta.get("application_version")
|
||||
except Exception as e:
|
||||
LOG.warning(
|
||||
"Failed to get android browser meta data through mozversion: %s-%s" %
|
||||
(e.__class__.__name__, e)
|
||||
"Failed to get android browser meta data through mozversion: %s-%s"
|
||||
% (e.__class__.__name__, e)
|
||||
)
|
||||
|
||||
if not browser_name:
|
||||
|
|
@ -431,13 +492,13 @@ class PerftestAndroid(Perftest):
|
|||
|
||||
def set_reverse_port(self, port):
|
||||
tcp_port = "tcp:{}".format(port)
|
||||
self.device.create_socket_connection('reverse', tcp_port, tcp_port)
|
||||
self.device.create_socket_connection("reverse", tcp_port, tcp_port)
|
||||
|
||||
def set_reverse_ports(self):
|
||||
if self.is_localhost:
|
||||
|
||||
# only raptor-webext uses the control server
|
||||
if self.config.get('browsertime', False) is False:
|
||||
if self.config.get("browsertime", False) is False:
|
||||
LOG.info("making the raptor control server port available to device")
|
||||
self.set_reverse_port(self.control_server.port)
|
||||
|
||||
|
|
@ -455,24 +516,26 @@ class PerftestAndroid(Perftest):
|
|||
super(PerftestAndroid, self).build_browser_profile()
|
||||
|
||||
# Merge in the Android profile.
|
||||
path = os.path.join(self.profile_data_dir, 'raptor-android')
|
||||
path = os.path.join(self.profile_data_dir, "raptor-android")
|
||||
LOG.info("Merging profile: {}".format(path))
|
||||
self.profile.merge(path)
|
||||
self.profile.set_preferences({'browser.tabs.remote.autostart': self.config['e10s']})
|
||||
self.profile.set_preferences(
|
||||
{"browser.tabs.remote.autostart": self.config["e10s"]}
|
||||
)
|
||||
|
||||
def clear_app_data(self):
|
||||
LOG.info("clearing %s app data" % self.config['binary'])
|
||||
self.device.shell("pm clear %s" % self.config['binary'])
|
||||
LOG.info("clearing %s app data" % self.config["binary"])
|
||||
self.device.shell("pm clear %s" % self.config["binary"])
|
||||
|
||||
def set_debug_app_flag(self):
|
||||
# required so release apks will read the android config.yml file
|
||||
LOG.info("setting debug-app flag for %s" % self.config['binary'])
|
||||
self.device.shell("am set-debug-app --persistent %s" % self.config['binary'])
|
||||
LOG.info("setting debug-app flag for %s" % self.config["binary"])
|
||||
self.device.shell("am set-debug-app --persistent %s" % self.config["binary"])
|
||||
|
||||
def copy_profile_to_device(self):
|
||||
"""Copy the profile to the device, and update permissions of all files."""
|
||||
if not self.device.is_app_installed(self.config['binary']):
|
||||
raise Exception('%s is not installed' % self.config['binary'])
|
||||
if not self.device.is_app_installed(self.config["binary"]):
|
||||
raise Exception("%s is not installed" % self.config["binary"])
|
||||
|
||||
try:
|
||||
LOG.info("copying profile to device: %s" % self.remote_profile)
|
||||
|
|
@ -495,9 +558,13 @@ class PerftestAndroid(Perftest):
|
|||
proxy_prefs["network.proxy.http_port"] = self.playback.port
|
||||
proxy_prefs["network.proxy.ssl"] = self.playback.host
|
||||
proxy_prefs["network.proxy.ssl_port"] = self.playback.port
|
||||
proxy_prefs["network.proxy.no_proxies_on"] = self.config['host']
|
||||
proxy_prefs["network.proxy.no_proxies_on"] = self.config["host"]
|
||||
|
||||
LOG.info("setting profile prefs to turn on the android app proxy: {}".format(proxy_prefs))
|
||||
LOG.info(
|
||||
"setting profile prefs to turn on the android app proxy: {}".format(
|
||||
proxy_prefs
|
||||
)
|
||||
)
|
||||
self.profile.set_preferences(proxy_prefs)
|
||||
|
||||
|
||||
|
|
@ -505,65 +572,65 @@ class PerftestDesktop(Perftest):
|
|||
"""Mixin class for Desktop-specific Perftest subclasses"""
|
||||
|
||||
def setup_chrome_args(self, test):
|
||||
'''Sets up chrome/chromium cmd-line arguments.
|
||||
"""Sets up chrome/chromium cmd-line arguments.
|
||||
|
||||
Needs to be "implemented" here to deal with Python 2
|
||||
unittest failures.
|
||||
'''
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def desktop_chrome_args(self, test):
|
||||
'''Returns cmd line options required to run pageload tests on Desktop Chrome
|
||||
"""Returns cmd line options required to run pageload tests on Desktop Chrome
|
||||
and Chromium. Also add the cmd line options to turn on the proxy and
|
||||
ignore security certificate errors if using host localhost, 127.0.0.1.
|
||||
'''
|
||||
chrome_args = [
|
||||
'--use-mock-keychain',
|
||||
'--no-default-browser-check',
|
||||
]
|
||||
"""
|
||||
chrome_args = ["--use-mock-keychain", "--no-default-browser-check"]
|
||||
|
||||
if test.get('playback', False):
|
||||
if test.get("playback", False):
|
||||
pb_args = [
|
||||
'--proxy-server=%s:%d' % (self.playback.host, self.playback.port),
|
||||
'--proxy-bypass-list=localhost;127.0.0.1',
|
||||
'--ignore-certificate-errors',
|
||||
"--proxy-server=%s:%d" % (self.playback.host, self.playback.port),
|
||||
"--proxy-bypass-list=localhost;127.0.0.1",
|
||||
"--ignore-certificate-errors",
|
||||
]
|
||||
|
||||
if not self.is_localhost:
|
||||
pb_args[0] = pb_args[0].replace('127.0.0.1', self.config['host'])
|
||||
pb_args[0] = pb_args[0].replace("127.0.0.1", self.config["host"])
|
||||
|
||||
chrome_args.extend(pb_args)
|
||||
|
||||
if self.debug_mode:
|
||||
chrome_args.extend(['--auto-open-devtools-for-tabs'])
|
||||
chrome_args.extend(["--auto-open-devtools-for-tabs"])
|
||||
|
||||
return chrome_args
|
||||
|
||||
def get_browser_meta(self):
|
||||
'''Returns the browser name and version in a tuple (name, version).
|
||||
"""Returns the browser name and version in a tuple (name, version).
|
||||
|
||||
On desktop, we use mozversion but a fallback method also exists
|
||||
for non-firefox browsers, where mozversion is known to fail. The
|
||||
methods are OS-specific, with windows being the outlier.
|
||||
'''
|
||||
"""
|
||||
browser_name = None
|
||||
browser_version = None
|
||||
|
||||
try:
|
||||
meta = mozversion.get_version(binary=self.config['binary'])
|
||||
browser_name = meta.get('application_name')
|
||||
browser_version = meta.get('application_version')
|
||||
meta = mozversion.get_version(binary=self.config["binary"])
|
||||
browser_name = meta.get("application_name")
|
||||
browser_version = meta.get("application_version")
|
||||
except Exception as e:
|
||||
LOG.warning(
|
||||
"Failed to get browser meta data through mozversion: %s-%s" %
|
||||
(e.__class__.__name__, e)
|
||||
"Failed to get browser meta data through mozversion: %s-%s"
|
||||
% (e.__class__.__name__, e)
|
||||
)
|
||||
LOG.info("Attempting to get version through fallback method...")
|
||||
|
||||
# Fall-back method to get browser version on desktop
|
||||
try:
|
||||
if 'linux' in self.config['platform'] or 'mac' in self.config['platform']:
|
||||
command = [self.config['binary'], '--version']
|
||||
if (
|
||||
"linux" in self.config["platform"]
|
||||
or "mac" in self.config["platform"]
|
||||
):
|
||||
command = [self.config["binary"], "--version"]
|
||||
proc = mozprocess.ProcessHandler(command)
|
||||
proc.run(timeout=10, outputTimeout=10)
|
||||
proc.wait()
|
||||
|
|
@ -573,28 +640,28 @@ class PerftestDesktop(Perftest):
|
|||
if len(bmeta) != 0:
|
||||
match = meta_re.match(bmeta[0])
|
||||
if match:
|
||||
browser_name = self.config['app']
|
||||
browser_name = self.config["app"]
|
||||
browser_version = match.group(2)
|
||||
else:
|
||||
LOG.info("Couldn't get browser version and name")
|
||||
else:
|
||||
# On windows we need to use wimc to get the version
|
||||
command = r'wmic datafile where name="{0}"'.format(
|
||||
self.config['binary'].replace('\\', r"\\")
|
||||
self.config["binary"].replace("\\", r"\\")
|
||||
)
|
||||
bmeta = subprocess.check_output(command)
|
||||
|
||||
meta_re = re.compile(r"\s+([\d.a-z]+)\s+")
|
||||
match = meta_re.findall(bmeta)
|
||||
if len(match) > 0:
|
||||
browser_name = self.config['app']
|
||||
browser_name = self.config["app"]
|
||||
browser_version = match[-1]
|
||||
else:
|
||||
LOG.info("Couldn't get browser version and name")
|
||||
except Exception as e:
|
||||
LOG.warning(
|
||||
"Failed to get browser meta data through fallback method: %s-%s" %
|
||||
(e.__class__.__name__, e)
|
||||
"Failed to get browser meta data through fallback method: %s-%s"
|
||||
% (e.__class__.__name__, e)
|
||||
)
|
||||
|
||||
if not browser_name:
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -0,0 +1,9 @@
|
|||
# flake8: noqa
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from .desktop import WebExtensionFirefox, WebExtensionDesktopChrome
|
||||
from .android import WebExtensionAndroid
|
||||
|
|
@ -0,0 +1,403 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import posixpath
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import mozcrash
|
||||
from cpu import start_android_cpu_profiler
|
||||
from logger.logger import RaptorLogger
|
||||
from mozdevice import ADBDevice
|
||||
from performance_tuning import tune_performance
|
||||
from perftest import PerftestAndroid
|
||||
from power import init_android_power_test, finish_android_power_test
|
||||
from signal_handler import SignalHandlerException
|
||||
from utils import write_yml_file
|
||||
from webextension.base import WebExtension
|
||||
|
||||
LOG = RaptorLogger(component="raptor-webext-android")
|
||||
|
||||
|
||||
class WebExtensionAndroid(PerftestAndroid, WebExtension):
|
||||
def __init__(self, app, binary, activity=None, intent=None, **kwargs):
|
||||
super(WebExtensionAndroid, self).__init__(
|
||||
app, binary, profile_class="firefox", **kwargs
|
||||
)
|
||||
|
||||
self.config.update({"activity": activity, "intent": intent})
|
||||
|
||||
self.remote_test_root = os.path.abspath(
|
||||
os.path.join(os.sep, "sdcard", "raptor")
|
||||
)
|
||||
self.remote_profile = os.path.join(self.remote_test_root, "profile")
|
||||
self.os_baseline_data = None
|
||||
self.power_test_time = None
|
||||
self.screen_off_timeout = 0
|
||||
self.screen_brightness = 127
|
||||
self.app_launched = False
|
||||
|
||||
def setup_adb_device(self):
|
||||
if self.device is None:
|
||||
self.device = ADBDevice(verbose=True)
|
||||
tune_performance(self.device, log=LOG)
|
||||
|
||||
LOG.info("creating remote root folder for raptor: %s" % self.remote_test_root)
|
||||
self.device.rm(self.remote_test_root, force=True, recursive=True)
|
||||
self.device.mkdir(self.remote_test_root)
|
||||
self.device.chmod(self.remote_test_root, recursive=True, root=True)
|
||||
|
||||
self.clear_app_data()
|
||||
self.set_debug_app_flag()
|
||||
|
||||
def write_android_app_config(self):
|
||||
# geckoview supports having a local on-device config file; use this file
|
||||
# to tell the app to use the specified browser profile, as well as other opts
|
||||
# on-device: /data/local/tmp/com.yourcompany.yourapp-geckoview-config.yaml
|
||||
# https://mozilla.github.io/geckoview/tutorials/automation.html#configuration-file-format
|
||||
|
||||
# only supported for geckoview apps
|
||||
if self.config["app"] == "fennec":
|
||||
return
|
||||
LOG.info("creating android app config.yml")
|
||||
|
||||
yml_config_data = dict(
|
||||
args=[
|
||||
"--profile",
|
||||
self.remote_profile,
|
||||
"use_multiprocess",
|
||||
self.config["e10s"],
|
||||
],
|
||||
env=dict(
|
||||
LOG_VERBOSE=1,
|
||||
R_LOG_LEVEL=6,
|
||||
MOZ_WEBRENDER=int(self.config["enable_webrender"]),
|
||||
),
|
||||
)
|
||||
|
||||
yml_name = "%s-geckoview-config.yaml" % self.config["binary"]
|
||||
yml_on_host = os.path.join(tempfile.mkdtemp(), yml_name)
|
||||
write_yml_file(yml_on_host, yml_config_data)
|
||||
yml_on_device = os.path.join("/data", "local", "tmp", yml_name)
|
||||
|
||||
try:
|
||||
LOG.info("copying %s to device: %s" % (yml_on_host, yml_on_device))
|
||||
self.device.rm(yml_on_device, force=True, recursive=True)
|
||||
self.device.push(yml_on_host, yml_on_device)
|
||||
|
||||
except Exception:
|
||||
LOG.critical("failed to push %s to device!" % yml_on_device)
|
||||
raise
|
||||
|
||||
def log_android_device_temperature(self):
|
||||
try:
|
||||
# retrieve and log the android device temperature
|
||||
thermal_zone0 = self.device.shell_output(
|
||||
"cat sys/class/thermal/thermal_zone0/temp"
|
||||
)
|
||||
thermal_zone0 = float(thermal_zone0)
|
||||
zone_type = self.device.shell_output(
|
||||
"cat sys/class/thermal/thermal_zone0/type"
|
||||
)
|
||||
LOG.info(
|
||||
"(thermal_zone0) device temperature: %.3f zone type: %s"
|
||||
% (thermal_zone0 / 1000, zone_type)
|
||||
)
|
||||
except Exception as exc:
|
||||
LOG.warning("Unexpected error: {} - {}".format(exc.__class__.__name__, exc))
|
||||
|
||||
def launch_firefox_android_app(self, test_name):
|
||||
LOG.info("starting %s" % self.config["app"])
|
||||
|
||||
extra_args = [
|
||||
"-profile", self.remote_profile,
|
||||
"--es", "env0",
|
||||
"LOG_VERBOSE=1",
|
||||
"--es", "env1",
|
||||
"R_LOG_LEVEL=6",
|
||||
"--es", "env2",
|
||||
"MOZ_WEBRENDER=%d" % self.config["enable_webrender"],
|
||||
]
|
||||
|
||||
try:
|
||||
# make sure the android app is not already running
|
||||
self.device.stop_application(self.config["binary"])
|
||||
|
||||
if self.config["app"] == "fennec":
|
||||
self.device.launch_fennec(
|
||||
self.config["binary"],
|
||||
extra_args=extra_args,
|
||||
url="about:blank",
|
||||
fail_if_running=False,
|
||||
)
|
||||
else:
|
||||
|
||||
# command line 'extra' args not used with geckoview apps; instead we use
|
||||
# an on-device config.yml file (see write_android_app_config)
|
||||
|
||||
self.device.launch_application(
|
||||
self.config["binary"],
|
||||
self.config["activity"],
|
||||
self.config["intent"],
|
||||
extras=None,
|
||||
url="about:blank",
|
||||
fail_if_running=False,
|
||||
)
|
||||
|
||||
# Check if app has started and it's running
|
||||
if not self.device.process_exist(self.config["binary"]):
|
||||
raise Exception(
|
||||
"Error launching %s. App did not start properly!"
|
||||
% self.config["binary"]
|
||||
)
|
||||
self.app_launched = True
|
||||
except Exception as e:
|
||||
LOG.error("Exception launching %s" % self.config["binary"])
|
||||
LOG.error("Exception: %s %s" % (type(e).__name__, str(e)))
|
||||
if self.config["power_test"]:
|
||||
finish_android_power_test(self, test_name)
|
||||
raise
|
||||
|
||||
# give our control server the device and app info
|
||||
self.control_server.device = self.device
|
||||
self.control_server.app_name = self.config["binary"]
|
||||
|
||||
def copy_cert_db(self, source_dir, target_dir):
|
||||
# copy browser cert db (that was previously created via certutil) from source to target
|
||||
cert_db_files = ["pkcs11.txt", "key4.db", "cert9.db"]
|
||||
for next_file in cert_db_files:
|
||||
_source = os.path.join(source_dir, next_file)
|
||||
_dest = os.path.join(target_dir, next_file)
|
||||
if os.path.exists(_source):
|
||||
LOG.info("copying %s to %s" % (_source, _dest))
|
||||
shutil.copyfile(_source, _dest)
|
||||
else:
|
||||
LOG.critical("unable to find ssl cert db file: %s" % _source)
|
||||
|
||||
def run_tests(self, tests, test_names):
|
||||
self.setup_adb_device()
|
||||
|
||||
return super(WebExtensionAndroid, self).run_tests(tests, test_names)
|
||||
|
||||
def run_test_setup(self, test):
|
||||
super(WebExtensionAndroid, self).run_test_setup(test)
|
||||
self.set_reverse_ports()
|
||||
|
||||
def run_test_teardown(self, test):
|
||||
LOG.info("removing reverse socket connections")
|
||||
self.device.remove_socket_connections("reverse")
|
||||
|
||||
super(WebExtensionAndroid, self).run_test_teardown(test)
|
||||
|
||||
def run_test(self, test, timeout):
|
||||
# tests will be run warm (i.e. NO browser restart between page-cycles)
|
||||
# unless otheriwse specified in the test INI by using 'cold = true'
|
||||
try:
|
||||
|
||||
if self.config["power_test"]:
|
||||
# gather OS baseline data
|
||||
init_android_power_test(self)
|
||||
LOG.info("Running OS baseline, pausing for 1 minute...")
|
||||
time.sleep(60)
|
||||
finish_android_power_test(self, "os-baseline", os_baseline=True)
|
||||
|
||||
# initialize for the test
|
||||
init_android_power_test(self)
|
||||
|
||||
if test.get("cold", False) is True:
|
||||
self.__run_test_cold(test, timeout)
|
||||
else:
|
||||
self.__run_test_warm(test, timeout)
|
||||
|
||||
except SignalHandlerException:
|
||||
self.device.stop_application(self.config["binary"])
|
||||
|
||||
finally:
|
||||
if self.config["power_test"]:
|
||||
finish_android_power_test(self, test["name"])
|
||||
|
||||
def __run_test_cold(self, test, timeout):
|
||||
"""
|
||||
Run the Raptor test but restart the entire browser app between page-cycles.
|
||||
|
||||
Note: For page-load tests, playback will only be started once - at the beginning of all
|
||||
browser cycles, and then stopped after all cycles are finished. The proxy is set via prefs
|
||||
in the browser profile so those will need to be set again in each new profile/cycle.
|
||||
Note that instead of using the certutil tool each time to create a db and import the
|
||||
mitmproxy SSL cert (it's done in mozbase/mozproxy) we will simply copy the existing
|
||||
cert db from the first cycle's browser profile into the new clean profile; this way
|
||||
we don't have to re-create the cert db on each browser cycle.
|
||||
|
||||
Since we're running in cold-mode, before this point (in manifest.py) the
|
||||
'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
|
||||
and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
|
||||
browser restart.
|
||||
|
||||
The 'browser-cycle' value is the current overall browser start iteration. The control
|
||||
server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
|
||||
each results set received; and will pass that on as part of the results so that the
|
||||
results processing will know results for multiple browser cycles are being received.
|
||||
|
||||
The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
|
||||
"""
|
||||
LOG.info(
|
||||
"test %s is running in cold mode; browser WILL be restarted between "
|
||||
"page cycles" % test["name"]
|
||||
)
|
||||
|
||||
for test["browser_cycle"] in range(1, test["expected_browser_cycles"] + 1):
|
||||
|
||||
LOG.info(
|
||||
"begin browser cycle %d of %d for test %s"
|
||||
% (test["browser_cycle"], test["expected_browser_cycles"], test["name"])
|
||||
)
|
||||
|
||||
self.run_test_setup(test)
|
||||
|
||||
self.clear_app_data()
|
||||
self.set_debug_app_flag()
|
||||
|
||||
if test["browser_cycle"] == 1:
|
||||
if test.get("playback") is not None:
|
||||
# an ssl cert db has now been created in the profile; copy it out so we
|
||||
# can use the same cert db in future test cycles / browser restarts
|
||||
local_cert_db_dir = tempfile.mkdtemp()
|
||||
LOG.info(
|
||||
"backing up browser ssl cert db that was created via certutil"
|
||||
)
|
||||
self.copy_cert_db(
|
||||
self.config["local_profile_dir"], local_cert_db_dir
|
||||
)
|
||||
|
||||
if not self.is_localhost:
|
||||
self.delete_proxy_settings_from_profile()
|
||||
|
||||
else:
|
||||
# double-check to ensure app has been shutdown
|
||||
self.device.stop_application(self.config["binary"])
|
||||
|
||||
# initial browser profile was already created before run_test was called;
|
||||
# now additional browser cycles we want to create a new one each time
|
||||
self.build_browser_profile()
|
||||
|
||||
if test.get("playback") is not None:
|
||||
# get cert db from previous cycle profile and copy into new clean profile
|
||||
# this saves us from having to start playback again / recreate cert db etc.
|
||||
LOG.info("copying existing ssl cert db into new browser profile")
|
||||
self.copy_cert_db(
|
||||
local_cert_db_dir, self.config["local_profile_dir"]
|
||||
)
|
||||
|
||||
self.run_test_setup(test)
|
||||
|
||||
if test.get("playback") is not None:
|
||||
self.turn_on_android_app_proxy()
|
||||
|
||||
self.copy_profile_to_device()
|
||||
self.log_android_device_temperature()
|
||||
|
||||
# write android app config.yml
|
||||
self.write_android_app_config()
|
||||
|
||||
# now start the browser/app under test
|
||||
self.launch_firefox_android_app(test["name"])
|
||||
|
||||
# set our control server flag to indicate we are running the browser/app
|
||||
self.control_server._finished = False
|
||||
|
||||
if self.config["cpu_test"]:
|
||||
# start measuring CPU usage
|
||||
self.cpu_profiler = start_android_cpu_profiler(self)
|
||||
|
||||
self.wait_for_test_finish(test, timeout)
|
||||
|
||||
# in debug mode, and running locally, leave the browser running
|
||||
if self.debug_mode and self.config["run_local"]:
|
||||
LOG.info(
|
||||
"* debug-mode enabled - please shutdown the browser manually..."
|
||||
)
|
||||
self.runner.wait(timeout=None)
|
||||
|
||||
# break test execution if a exception is present
|
||||
if len(self.results_handler.page_timeout_list) > 0:
|
||||
break
|
||||
|
||||
def __run_test_warm(self, test, timeout):
|
||||
LOG.info(
|
||||
"test %s is running in warm mode; browser will NOT be restarted between "
|
||||
"page cycles" % test["name"]
|
||||
)
|
||||
|
||||
self.run_test_setup(test)
|
||||
|
||||
if not self.is_localhost:
|
||||
self.delete_proxy_settings_from_profile()
|
||||
|
||||
if test.get("playback") is not None:
|
||||
self.turn_on_android_app_proxy()
|
||||
|
||||
self.clear_app_data()
|
||||
self.set_debug_app_flag()
|
||||
self.copy_profile_to_device()
|
||||
self.log_android_device_temperature()
|
||||
|
||||
# write android app config.yml
|
||||
self.write_android_app_config()
|
||||
|
||||
# now start the browser/app under test
|
||||
self.launch_firefox_android_app(test["name"])
|
||||
|
||||
# set our control server flag to indicate we are running the browser/app
|
||||
self.control_server._finished = False
|
||||
|
||||
if self.config["cpu_test"]:
|
||||
# start measuring CPU usage
|
||||
self.cpu_profiler = start_android_cpu_profiler(self)
|
||||
|
||||
self.wait_for_test_finish(test, timeout)
|
||||
|
||||
# in debug mode, and running locally, leave the browser running
|
||||
if self.debug_mode and self.config["run_local"]:
|
||||
LOG.info("* debug-mode enabled - please shutdown the browser manually...")
|
||||
|
||||
def check_for_crashes(self):
|
||||
super(WebExtensionAndroid, self).check_for_crashes()
|
||||
|
||||
if not self.app_launched:
|
||||
LOG.info("skipping check_for_crashes: application has not been launched")
|
||||
return
|
||||
self.app_launched = False
|
||||
|
||||
# Turn off verbose to prevent logcat from being inserted into the main log.
|
||||
verbose = self.device._verbose
|
||||
self.device._verbose = False
|
||||
logcat = self.device.get_logcat()
|
||||
self.device._verbose = verbose
|
||||
if logcat:
|
||||
if mozcrash.check_for_java_exception(logcat, "raptor"):
|
||||
return
|
||||
try:
|
||||
dump_dir = tempfile.mkdtemp()
|
||||
remote_dir = posixpath.join(self.remote_profile, "minidumps")
|
||||
if not self.device.is_dir(remote_dir):
|
||||
return
|
||||
self.device.pull(remote_dir, dump_dir)
|
||||
mozcrash.log_crashes(LOG, dump_dir, self.config["symbols_path"])
|
||||
finally:
|
||||
try:
|
||||
shutil.rmtree(dump_dir)
|
||||
except Exception:
|
||||
LOG.warning("unable to remove directory: %s" % dump_dir)
|
||||
|
||||
def clean_up(self):
|
||||
LOG.info("removing test folder for raptor: %s" % self.remote_test_root)
|
||||
self.device.rm(self.remote_test_root, force=True, recursive=True)
|
||||
|
||||
super(WebExtensionAndroid, self).clean_up()
|
||||
|
|
@ -0,0 +1,231 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
import time
|
||||
|
||||
from benchmark import Benchmark
|
||||
from cmdline import CHROMIUM_DISTROS
|
||||
from control_server import RaptorControlServer
|
||||
from gen_test_config import gen_test_config
|
||||
from logger.logger import RaptorLogger
|
||||
from memory import generate_android_memory_profile
|
||||
from perftest import Perftest
|
||||
from results import RaptorResultsHandler
|
||||
|
||||
LOG = RaptorLogger(component="raptor-webext")
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
webext_dir = os.path.join(here, "..", "..", "webext")
|
||||
|
||||
|
||||
class WebExtension(Perftest):
|
||||
"""Container class for WebExtension"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.raptor_webext = None
|
||||
self.control_server = None
|
||||
self.cpu_profiler = None
|
||||
|
||||
super(WebExtension, self).__init__(*args, **kwargs)
|
||||
|
||||
# set up the results handler
|
||||
self.results_handler = RaptorResultsHandler(
|
||||
gecko_profile=self.config.get("gecko_profile"),
|
||||
power_test=self.config.get("power_test"),
|
||||
cpu_test=self.config.get("cpu_test"),
|
||||
memory_test=self.config.get("memory_test"),
|
||||
no_conditioned_profile=self.config["no_conditioned_profile"],
|
||||
extra_prefs=self.config.get("extra_prefs"),
|
||||
)
|
||||
browser_name, browser_version = self.get_browser_meta()
|
||||
self.results_handler.add_browser_meta(self.config["app"], browser_version)
|
||||
|
||||
self.start_control_server()
|
||||
|
||||
def run_test_setup(self, test):
|
||||
super(WebExtension, self).run_test_setup(test)
|
||||
|
||||
LOG.info("starting web extension test: %s" % test["name"])
|
||||
LOG.info("test settings: %s" % str(test))
|
||||
LOG.info("web extension config: %s" % str(self.config))
|
||||
|
||||
if test.get("type") == "benchmark":
|
||||
self.serve_benchmark_source(test)
|
||||
|
||||
gen_test_config(
|
||||
self.config["app"],
|
||||
test["name"],
|
||||
self.control_server.port,
|
||||
self.post_startup_delay,
|
||||
host=self.config["host"],
|
||||
b_port=self.benchmark_port,
|
||||
debug_mode=1 if self.debug_mode else 0,
|
||||
browser_cycle=test.get("browser_cycle", 1),
|
||||
)
|
||||
|
||||
self.install_raptor_webext()
|
||||
|
||||
def wait_for_test_finish(self, test, timeout):
|
||||
# this is a 'back-stop' i.e. if for some reason Raptor doesn't finish for some
|
||||
# serious problem; i.e. the test was unable to send a 'page-timeout' to the control
|
||||
# server, etc. Therefore since this is a 'back-stop' we want to be generous here;
|
||||
# we don't want this timeout occurring unless abosultely necessary
|
||||
|
||||
# convert timeout to seconds and account for page cycles
|
||||
timeout = int(timeout / 1000) * int(test.get("page_cycles", 1))
|
||||
# account for the pause the raptor webext runner takes after browser startup
|
||||
# and the time an exception is propagated through the framework
|
||||
timeout += int(self.post_startup_delay / 1000) + 10
|
||||
|
||||
# for page-load tests we don't start the page-timeout timer until the pageload.js content
|
||||
# is successfully injected and invoked; which differs per site being tested; therefore we
|
||||
# need to be generous here - let's add 10 seconds extra per page-cycle
|
||||
if test.get("type") == "pageload":
|
||||
timeout += 10 * int(test.get("page_cycles", 1))
|
||||
|
||||
# if geckoProfile enabled, give browser more time for profiling
|
||||
if self.config["gecko_profile"] is True:
|
||||
timeout += 5 * 60
|
||||
|
||||
# we also need to give time for results processing, not just page/browser cycles!
|
||||
timeout += 60
|
||||
|
||||
elapsed_time = 0
|
||||
while not self.control_server._finished:
|
||||
if self.config["enable_control_server_wait"]:
|
||||
response = self.control_server_wait_get()
|
||||
if response == "webext_shutdownBrowser":
|
||||
if self.config["memory_test"]:
|
||||
generate_android_memory_profile(self, test["name"])
|
||||
if self.cpu_profiler:
|
||||
self.cpu_profiler.generate_android_cpu_profile(test["name"])
|
||||
|
||||
self.control_server_wait_continue()
|
||||
time.sleep(1)
|
||||
# we only want to force browser-shutdown on timeout if not in debug mode;
|
||||
# in debug-mode we leave the browser running (require manual shutdown)
|
||||
if not self.debug_mode:
|
||||
elapsed_time += 1
|
||||
if elapsed_time > (timeout) - 5: # stop 5 seconds early
|
||||
self.control_server.wait_for_quit()
|
||||
raise RuntimeError(
|
||||
"Test failed to finish. "
|
||||
"Application timed out after {} seconds".format(timeout)
|
||||
)
|
||||
|
||||
def run_test_teardown(self, test):
|
||||
super(WebExtension, self).run_test_teardown(test)
|
||||
|
||||
if self.playback is not None:
|
||||
self.playback.stop()
|
||||
self.playback = None
|
||||
|
||||
self.remove_raptor_webext()
|
||||
|
||||
def set_browser_test_prefs(self, raw_prefs):
|
||||
# add test specific preferences
|
||||
LOG.info("setting test-specific Firefox preferences")
|
||||
self.profile.set_preferences(json.loads(raw_prefs))
|
||||
|
||||
def build_browser_profile(self):
|
||||
super(WebExtension, self).build_browser_profile()
|
||||
|
||||
if self.control_server:
|
||||
# The control server and the browser profile are not well factored
|
||||
# at this time, so the start-up process overlaps. Accommodate.
|
||||
self.control_server.user_profile = self.profile
|
||||
|
||||
def start_control_server(self):
|
||||
self.control_server = RaptorControlServer(self.results_handler, self.debug_mode)
|
||||
self.control_server.user_profile = self.profile
|
||||
self.control_server.start()
|
||||
|
||||
if self.config["enable_control_server_wait"]:
|
||||
self.control_server_wait_set("webext_shutdownBrowser")
|
||||
|
||||
def serve_benchmark_source(self, test):
|
||||
# benchmark-type tests require the benchmark test to be served out
|
||||
self.benchmark = Benchmark(self.config, test)
|
||||
self.benchmark_port = int(self.benchmark.port)
|
||||
|
||||
def install_raptor_webext(self):
|
||||
# must intall raptor addon each time because we dynamically update some content
|
||||
# the webext is installed into the browser profile
|
||||
# note: for chrome the addon is just a list of paths that ultimately are added
|
||||
# to the chromium command line '--load-extension' argument
|
||||
self.raptor_webext = os.path.join(webext_dir, "raptor")
|
||||
LOG.info("installing webext %s" % self.raptor_webext)
|
||||
self.profile.addons.install(self.raptor_webext)
|
||||
|
||||
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
|
||||
try:
|
||||
self.webext_id = self.profile.addons.addon_details(self.raptor_webext)["id"]
|
||||
except AttributeError:
|
||||
self.webext_id = None
|
||||
|
||||
def remove_raptor_webext(self):
|
||||
# remove the raptor webext; as it must be reloaded with each subtest anyway
|
||||
if not self.raptor_webext:
|
||||
LOG.info("raptor webext not installed - not attempting removal")
|
||||
return
|
||||
|
||||
LOG.info("removing webext %s" % self.raptor_webext)
|
||||
if self.config["app"] in ["firefox", "geckoview", "fennec", "refbrow", "fenix"]:
|
||||
self.profile.addons.remove_addon(self.webext_id)
|
||||
|
||||
# for chrome the addon is just a list (appended to cmd line)
|
||||
chrome_apps = CHROMIUM_DISTROS + ["chrome-android", "chromium-android"]
|
||||
if self.config["app"] in chrome_apps:
|
||||
self.profile.addons.remove(self.raptor_webext)
|
||||
|
||||
def clean_up(self):
|
||||
super(WebExtension, self).clean_up()
|
||||
|
||||
if self.config["enable_control_server_wait"]:
|
||||
self.control_server_wait_clear("all")
|
||||
|
||||
self.control_server.stop()
|
||||
LOG.info("finished")
|
||||
|
||||
def control_server_wait_set(self, state):
|
||||
response = requests.post(
|
||||
"http://127.0.0.1:%s/" % self.control_server.port,
|
||||
json={"type": "wait-set", "data": state},
|
||||
)
|
||||
return response.text
|
||||
|
||||
def control_server_wait_timeout(self, timeout):
|
||||
response = requests.post(
|
||||
"http://127.0.0.1:%s/" % self.control_server.port,
|
||||
json={"type": "wait-timeout", "data": timeout},
|
||||
)
|
||||
return response.text
|
||||
|
||||
def control_server_wait_get(self):
|
||||
response = requests.post(
|
||||
"http://127.0.0.1:%s/" % self.control_server.port,
|
||||
json={"type": "wait-get", "data": ""},
|
||||
)
|
||||
return response.text
|
||||
|
||||
def control_server_wait_continue(self):
|
||||
response = requests.post(
|
||||
"http://127.0.0.1:%s/" % self.control_server.port,
|
||||
json={"type": "wait-continue", "data": ""},
|
||||
)
|
||||
return response.text
|
||||
|
||||
def control_server_wait_clear(self, state):
|
||||
response = requests.post(
|
||||
"http://127.0.0.1:%s/" % self.control_server.port,
|
||||
json={"type": "wait-clear", "data": state},
|
||||
)
|
||||
return response.text
|
||||
|
|
@ -0,0 +1,270 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from mozpower import MozPower
|
||||
from mozrunner import runners
|
||||
|
||||
from logger.logger import RaptorLogger
|
||||
from outputhandler import OutputHandler
|
||||
from perftest import PerftestDesktop
|
||||
from .base import WebExtension
|
||||
|
||||
LOG = RaptorLogger(component="raptor-webext-desktop")
|
||||
|
||||
|
||||
class WebExtensionDesktop(PerftestDesktop, WebExtension):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(WebExtensionDesktop, self).__init__(*args, **kwargs)
|
||||
|
||||
# create the desktop browser runner
|
||||
LOG.info("creating browser runner using mozrunner")
|
||||
self.output_handler = OutputHandler()
|
||||
process_args = {"processOutputLine": [self.output_handler]}
|
||||
runner_cls = runners[self.config["app"]]
|
||||
self.runner = runner_cls(
|
||||
self.config["binary"],
|
||||
profile=self.profile,
|
||||
process_args=process_args,
|
||||
symbols_path=self.config["symbols_path"],
|
||||
)
|
||||
|
||||
if self.config["enable_webrender"]:
|
||||
self.runner.env["MOZ_WEBRENDER"] = "1"
|
||||
self.runner.env["MOZ_ACCELERATED"] = "1"
|
||||
else:
|
||||
self.runner.env["MOZ_WEBRENDER"] = "0"
|
||||
|
||||
def launch_desktop_browser(self, test):
|
||||
raise NotImplementedError
|
||||
|
||||
def start_runner_proc(self):
|
||||
# launch the browser via our previously-created runner
|
||||
self.runner.start()
|
||||
|
||||
proc = self.runner.process_handler
|
||||
self.output_handler.proc = proc
|
||||
|
||||
# give our control server the browser process so it can shut it down later
|
||||
self.control_server.browser_proc = proc
|
||||
|
||||
def run_test(self, test, timeout):
|
||||
# tests will be run warm (i.e. NO browser restart between page-cycles)
|
||||
# unless otheriwse specified in the test INI by using 'cold = true'
|
||||
mozpower_measurer = None
|
||||
if self.config.get("power_test", False):
|
||||
powertest_name = test["name"].replace("/", "-").replace("\\", "-")
|
||||
output_dir = os.path.join(
|
||||
self.artifact_dir, "power-measurements-%s" % powertest_name
|
||||
)
|
||||
test_dir = os.path.join(output_dir, powertest_name)
|
||||
|
||||
try:
|
||||
if not os.path.exists(output_dir):
|
||||
os.mkdir(output_dir)
|
||||
if not os.path.exists(test_dir):
|
||||
os.mkdir(test_dir)
|
||||
except Exception:
|
||||
LOG.critical(
|
||||
"Could not create directories to store power testing data."
|
||||
)
|
||||
raise
|
||||
|
||||
# Start power measurements with IPG creating a power usage log
|
||||
# every 30 seconds with 1 data point per second (or a 1000 milli-
|
||||
# second sampling rate).
|
||||
mozpower_measurer = MozPower(
|
||||
ipg_measure_duration=30,
|
||||
sampling_rate=1000,
|
||||
output_file_path=os.path.join(test_dir, "power-usage"),
|
||||
)
|
||||
mozpower_measurer.initialize_power_measurements()
|
||||
|
||||
if test.get("cold", False) is True:
|
||||
self.__run_test_cold(test, timeout)
|
||||
else:
|
||||
self.__run_test_warm(test, timeout)
|
||||
|
||||
if mozpower_measurer:
|
||||
mozpower_measurer.finalize_power_measurements(test_name=test["name"])
|
||||
perfherder_data = mozpower_measurer.get_perfherder_data()
|
||||
|
||||
if not self.config.get("run_local", False):
|
||||
# when not running locally, zip the data and delete the folder which
|
||||
# was placed in the zip
|
||||
powertest_name = test["name"].replace("/", "-").replace("\\", "-")
|
||||
power_data_path = os.path.join(
|
||||
self.artifact_dir, "power-measurements-%s" % powertest_name
|
||||
)
|
||||
shutil.make_archive(power_data_path, "zip", power_data_path)
|
||||
shutil.rmtree(power_data_path)
|
||||
|
||||
for data_type in perfherder_data:
|
||||
self.control_server.submit_supporting_data(perfherder_data[data_type])
|
||||
|
||||
def __run_test_cold(self, test, timeout):
|
||||
"""
|
||||
Run the Raptor test but restart the entire browser app between page-cycles.
|
||||
|
||||
Note: For page-load tests, playback will only be started once - at the beginning of all
|
||||
browser cycles, and then stopped after all cycles are finished. That includes the import
|
||||
of the mozproxy ssl cert and turning on the browser proxy.
|
||||
|
||||
Since we're running in cold-mode, before this point (in manifest.py) the
|
||||
'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
|
||||
and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
|
||||
browser restart.
|
||||
|
||||
The 'browser-cycle' value is the current overall browser start iteration. The control
|
||||
server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
|
||||
each results set received; and will pass that on as part of the results so that the
|
||||
results processing will know results for multiple browser cycles are being received.
|
||||
|
||||
The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
|
||||
"""
|
||||
LOG.info(
|
||||
"test %s is running in cold mode; browser WILL be restarted between "
|
||||
"page cycles" % test["name"]
|
||||
)
|
||||
|
||||
for test["browser_cycle"] in range(1, test["expected_browser_cycles"] + 1):
|
||||
|
||||
LOG.info(
|
||||
"begin browser cycle %d of %d for test %s"
|
||||
% (test["browser_cycle"], test["expected_browser_cycles"], test["name"])
|
||||
)
|
||||
|
||||
self.run_test_setup(test)
|
||||
|
||||
if test["browser_cycle"] == 1:
|
||||
|
||||
if not self.is_localhost:
|
||||
self.delete_proxy_settings_from_profile()
|
||||
|
||||
else:
|
||||
# initial browser profile was already created before run_test was called;
|
||||
# now additional browser cycles we want to create a new one each time
|
||||
self.build_browser_profile()
|
||||
|
||||
# Update runner profile
|
||||
self.runner.profile = self.profile
|
||||
|
||||
self.run_test_setup(test)
|
||||
|
||||
# now start the browser/app under test
|
||||
self.launch_desktop_browser(test)
|
||||
|
||||
# set our control server flag to indicate we are running the browser/app
|
||||
self.control_server._finished = False
|
||||
|
||||
self.wait_for_test_finish(test, timeout)
|
||||
|
||||
def __run_test_warm(self, test, timeout):
|
||||
self.run_test_setup(test)
|
||||
|
||||
if not self.is_localhost:
|
||||
self.delete_proxy_settings_from_profile()
|
||||
|
||||
# start the browser/app under test
|
||||
self.launch_desktop_browser(test)
|
||||
|
||||
# set our control server flag to indicate we are running the browser/app
|
||||
self.control_server._finished = False
|
||||
|
||||
self.wait_for_test_finish(test, timeout)
|
||||
|
||||
def run_test_teardown(self, test):
|
||||
# browser should be closed by now but this is a backup-shutdown (if not in debug-mode)
|
||||
if not self.debug_mode:
|
||||
if self.runner.is_running():
|
||||
self.runner.stop()
|
||||
else:
|
||||
# in debug mode, and running locally, leave the browser running
|
||||
if self.config["run_local"]:
|
||||
LOG.info(
|
||||
"* debug-mode enabled - please shutdown the browser manually..."
|
||||
)
|
||||
self.runner.wait(timeout=None)
|
||||
|
||||
super(WebExtensionDesktop, self).run_test_teardown(test)
|
||||
|
||||
def check_for_crashes(self):
|
||||
super(WebExtensionDesktop, self).check_for_crashes()
|
||||
|
||||
try:
|
||||
self.runner.check_for_crashes()
|
||||
except NotImplementedError: # not implemented for Chrome
|
||||
pass
|
||||
|
||||
def clean_up(self):
|
||||
self.runner.stop()
|
||||
|
||||
super(WebExtensionDesktop, self).clean_up()
|
||||
|
||||
|
||||
class WebExtensionFirefox(WebExtensionDesktop):
|
||||
def disable_non_local_connections(self):
|
||||
# For Firefox we need to set MOZ_DISABLE_NONLOCAL_CONNECTIONS=1 env var before startup
|
||||
# when testing release builds from mozilla-beta/release. This is because of restrictions
|
||||
# on release builds that require webextensions to be signed unless this env var is set
|
||||
LOG.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=1")
|
||||
os.environ["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
|
||||
|
||||
def enable_non_local_connections(self):
|
||||
# pageload tests need to be able to access non-local connections via mitmproxy
|
||||
LOG.info("setting MOZ_DISABLE_NONLOCAL_CONNECTIONS=0")
|
||||
os.environ["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "0"
|
||||
|
||||
def launch_desktop_browser(self, test):
|
||||
LOG.info("starting %s" % self.config["app"])
|
||||
if self.config["is_release_build"]:
|
||||
self.disable_non_local_connections()
|
||||
|
||||
# if running debug-mode, tell Firefox to open the browser console on startup
|
||||
if self.debug_mode:
|
||||
self.runner.cmdargs.extend(["-jsconsole"])
|
||||
|
||||
self.start_runner_proc()
|
||||
|
||||
if self.config["is_release_build"] and test.get("playback") is not None:
|
||||
self.enable_non_local_connections()
|
||||
|
||||
# if geckoProfile is enabled, initialize it
|
||||
if self.config["gecko_profile"] is True:
|
||||
self._init_gecko_profiling(test)
|
||||
# tell the control server the gecko_profile dir; the control server
|
||||
# will receive the filename of the stored gecko profile from the web
|
||||
# extension, and will move it out of the browser user profile to
|
||||
# this directory; where it is picked-up by gecko_profile.symbolicate
|
||||
self.control_server.gecko_profile_dir = (
|
||||
self.gecko_profiler.gecko_profile_dir
|
||||
)
|
||||
|
||||
|
||||
class WebExtensionDesktopChrome(WebExtensionDesktop):
|
||||
def setup_chrome_args(self, test):
|
||||
# Setup chrome args and add them to the runner's args
|
||||
chrome_args = self.desktop_chrome_args(test)
|
||||
if " ".join(chrome_args) not in " ".join(self.runner.cmdargs):
|
||||
self.runner.cmdargs.extend(chrome_args)
|
||||
|
||||
def launch_desktop_browser(self, test):
|
||||
LOG.info("starting %s" % self.config["app"])
|
||||
|
||||
# Setup chrome/chromium specific arguments then start the runner
|
||||
self.setup_chrome_args(test)
|
||||
self.start_runner_proc()
|
||||
|
||||
def set_browser_test_prefs(self, raw_prefs):
|
||||
# add test-specific preferences
|
||||
LOG.info(
|
||||
"preferences were configured for the test, however \
|
||||
we currently do not install them on non-Firefox browsers."
|
||||
)
|
||||
|
|
@ -2,66 +2,69 @@ from __future__ import absolute_import
|
|||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from argparse import Namespace
|
||||
|
||||
# need this so raptor imports work both from /raptor and via mach
|
||||
# need this so the raptor unit tests can find raptor/raptor classes
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
raptor_dir = os.path.join(os.path.dirname(here), "raptor")
|
||||
sys.path.insert(0, raptor_dir)
|
||||
|
||||
from raptor.raptor import Perftest, RaptorDesktopFirefox, Browsertime
|
||||
from perftest import Perftest
|
||||
from webextension import WebExtensionFirefox
|
||||
from browsertime import Browsertime
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def options(request):
|
||||
opts = {
|
||||
'app': 'firefox',
|
||||
'binary': 'path/to/dummy/browser',
|
||||
'no_conditioned_profile': True
|
||||
"app": "firefox",
|
||||
"binary": "path/to/dummy/browser",
|
||||
"no_conditioned_profile": True,
|
||||
}
|
||||
|
||||
if hasattr(request.module, 'OPTIONS'):
|
||||
if hasattr(request.module, "OPTIONS"):
|
||||
opts.update(request.module.OPTIONS)
|
||||
return opts
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def browsertime_options(options):
|
||||
options['browsertime_node'] = 'browsertime_node'
|
||||
options['browsertime_browsertimejs'] = 'browsertime_browsertimejs'
|
||||
options['browsertime_ffmpeg'] = 'browsertime_ffmpeg'
|
||||
options['browsertime_geckodriver'] = 'browsertime_geckodriver'
|
||||
options['browsertime_chromedriver'] = 'browsertime_chromedriver'
|
||||
options['browsertime_video'] = 'browsertime_video'
|
||||
options["browsertime_node"] = "browsertime_node"
|
||||
options["browsertime_browsertimejs"] = "browsertime_browsertimejs"
|
||||
options["browsertime_ffmpeg"] = "browsertime_ffmpeg"
|
||||
options["browsertime_geckodriver"] = "browsertime_geckodriver"
|
||||
options["browsertime_chromedriver"] = "browsertime_chromedriver"
|
||||
options["browsertime_video"] = "browsertime_video"
|
||||
return options
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def raptor(options):
|
||||
return RaptorDesktopFirefox(**options)
|
||||
return WebExtensionFirefox(**options)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_test():
|
||||
return {
|
||||
'name': 'raptor-firefox-tp6',
|
||||
'test_url': '/dummy/url',
|
||||
}
|
||||
return {"name": "raptor-firefox-tp6", "test_url": "/dummy/url"}
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
@pytest.fixture(scope="session")
|
||||
def get_prefs():
|
||||
def _inner(browser):
|
||||
import raptor
|
||||
prefs_dir = os.path.join(raptor.__file__, 'preferences')
|
||||
with open(os.path.join(prefs_dir, '{}.json'.format(browser)), 'r') as fh:
|
||||
|
||||
prefs_dir = os.path.join(raptor.__file__, "preferences")
|
||||
with open(os.path.join(prefs_dir, "{}.json".format(browser)), "r") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
@pytest.fixture(scope="session")
|
||||
def filedir():
|
||||
return os.path.join(here, 'files')
|
||||
return os.path.join(here, "files")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -69,7 +72,7 @@ def get_binary():
|
|||
from moztest.selftest import fixtures
|
||||
|
||||
def inner(app):
|
||||
if app != 'firefox':
|
||||
if app != "firefox":
|
||||
pytest.xfail(reason="{} support not implemented".format(app))
|
||||
|
||||
binary = fixtures.binary()
|
||||
|
|
@ -82,18 +85,20 @@ def get_binary():
|
|||
|
||||
@pytest.fixture
|
||||
def create_args():
|
||||
args = Namespace(app='firefox',
|
||||
test='raptor-tp6-unittest',
|
||||
binary='path/to/binary',
|
||||
gecko_profile=False,
|
||||
debug_mode=False,
|
||||
page_cycles=None,
|
||||
page_timeout=None,
|
||||
test_url_params=None,
|
||||
host=None,
|
||||
run_local=True,
|
||||
browsertime=True,
|
||||
cold=False)
|
||||
args = Namespace(
|
||||
app="firefox",
|
||||
test="raptor-tp6-unittest",
|
||||
binary="path/to/binary",
|
||||
gecko_profile=False,
|
||||
debug_mode=False,
|
||||
page_cycles=None,
|
||||
page_timeout=None,
|
||||
test_url_params=None,
|
||||
host=None,
|
||||
run_local=True,
|
||||
browsertime=True,
|
||||
cold=False,
|
||||
)
|
||||
|
||||
def inner(**kwargs):
|
||||
for next_arg in kwargs:
|
||||
|
|
@ -105,7 +110,7 @@ def create_args():
|
|||
return inner
|
||||
|
||||
|
||||
@pytest.fixture(scope='module')
|
||||
@pytest.fixture(scope="module")
|
||||
def ConcretePerftest():
|
||||
class PerftestImplementation(Perftest):
|
||||
def check_for_crashes(self):
|
||||
|
|
@ -135,7 +140,7 @@ def ConcretePerftest():
|
|||
return PerftestImplementation
|
||||
|
||||
|
||||
@pytest.fixture(scope='module')
|
||||
@pytest.fixture(scope="module")
|
||||
def ConcreteBrowsertime():
|
||||
class BrowsertimeImplementation(Browsertime):
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -1,100 +1,119 @@
|
|||
from __future__ import absolute_import, unicode_literals
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
import mozunit
|
||||
|
||||
# need this so the raptor unit tests can find raptor/raptor classes
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
raptor_dir = os.path.join(os.path.dirname(here), "raptor")
|
||||
sys.path.insert(0, raptor_dir)
|
||||
|
||||
from argparse import ArgumentParser, Namespace
|
||||
from raptor.cmdline import verify_options
|
||||
from cmdline import verify_options
|
||||
|
||||
|
||||
def test_verify_options(filedir):
|
||||
args = Namespace(app='firefox',
|
||||
binary='invalid/path',
|
||||
gecko_profile='False',
|
||||
page_cycles=1,
|
||||
page_timeout=60000,
|
||||
debug='True',
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
enable_webrender=False)
|
||||
args = Namespace(
|
||||
app="firefox",
|
||||
binary="invalid/path",
|
||||
gecko_profile="False",
|
||||
page_cycles=1,
|
||||
page_timeout=60000,
|
||||
debug="True",
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
enable_webrender=False,
|
||||
)
|
||||
parser = ArgumentParser()
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
verify_options(parser, args)
|
||||
|
||||
args.binary = os.path.join(filedir, 'fake_binary.exe')
|
||||
args.binary = os.path.join(filedir, "fake_binary.exe")
|
||||
verify_options(parser, args) # assert no exception
|
||||
|
||||
args = Namespace(app='geckoview',
|
||||
binary='org.mozilla.geckoview_example',
|
||||
activity='org.mozilla.geckoview_example.GeckoViewActivity',
|
||||
intent='android.intent.action.MAIN',
|
||||
gecko_profile='False',
|
||||
is_release_build=False,
|
||||
host='sophie',
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
enable_webrender=False)
|
||||
args = Namespace(
|
||||
app="geckoview",
|
||||
binary="org.mozilla.geckoview_example",
|
||||
activity="org.mozilla.geckoview_example.GeckoViewActivity",
|
||||
intent="android.intent.action.MAIN",
|
||||
gecko_profile="False",
|
||||
is_release_build=False,
|
||||
host="sophie",
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
enable_webrender=False,
|
||||
)
|
||||
verify_options(parser, args) # assert no exception
|
||||
|
||||
args = Namespace(app='refbrow',
|
||||
binary='org.mozilla.reference.browser',
|
||||
activity='org.mozilla.reference.browser.BrowserTestActivity',
|
||||
intent='android.intent.action.MAIN',
|
||||
gecko_profile='False',
|
||||
is_release_build=False,
|
||||
host='sophie',
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
enable_webrender=False)
|
||||
args = Namespace(
|
||||
app="refbrow",
|
||||
binary="org.mozilla.reference.browser",
|
||||
activity="org.mozilla.reference.browser.BrowserTestActivity",
|
||||
intent="android.intent.action.MAIN",
|
||||
gecko_profile="False",
|
||||
is_release_build=False,
|
||||
host="sophie",
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
enable_webrender=False,
|
||||
)
|
||||
verify_options(parser, args) # assert no exception
|
||||
|
||||
args = Namespace(app='fenix',
|
||||
binary='org.mozilla.fenix.browser',
|
||||
activity='org.mozilla.fenix.browser.BrowserPerformanceTestActivity',
|
||||
intent='android.intent.action.VIEW',
|
||||
gecko_profile='False',
|
||||
is_release_build=False,
|
||||
host='sophie',
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
enable_webrender=False)
|
||||
args = Namespace(
|
||||
app="fenix",
|
||||
binary="org.mozilla.fenix.browser",
|
||||
activity="org.mozilla.fenix.browser.BrowserPerformanceTestActivity",
|
||||
intent="android.intent.action.VIEW",
|
||||
gecko_profile="False",
|
||||
is_release_build=False,
|
||||
host="sophie",
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
enable_webrender=False,
|
||||
)
|
||||
verify_options(parser, args) # assert no exception
|
||||
|
||||
args = Namespace(app='geckoview',
|
||||
binary='org.mozilla.geckoview_example',
|
||||
activity='org.mozilla.geckoview_example.GeckoViewActivity',
|
||||
intent='android.intent.action.MAIN',
|
||||
gecko_profile='False',
|
||||
is_release_build=False,
|
||||
host='sophie',
|
||||
power_test=False,
|
||||
cpu_test=True,
|
||||
memory_test=False,
|
||||
enable_webrender=False)
|
||||
args = Namespace(
|
||||
app="geckoview",
|
||||
binary="org.mozilla.geckoview_example",
|
||||
activity="org.mozilla.geckoview_example.GeckoViewActivity",
|
||||
intent="android.intent.action.MAIN",
|
||||
gecko_profile="False",
|
||||
is_release_build=False,
|
||||
host="sophie",
|
||||
power_test=False,
|
||||
cpu_test=True,
|
||||
memory_test=False,
|
||||
enable_webrender=False,
|
||||
)
|
||||
verify_options(parser, args) # assert no exception
|
||||
|
||||
args = Namespace(app='refbrow',
|
||||
binary='org.mozilla.reference.browser',
|
||||
activity=None,
|
||||
intent='android.intent.action.MAIN',
|
||||
gecko_profile='False',
|
||||
is_release_build=False,
|
||||
host='sophie',
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
enable_webrender=False)
|
||||
args = Namespace(
|
||||
app="refbrow",
|
||||
binary="org.mozilla.reference.browser",
|
||||
activity=None,
|
||||
intent="android.intent.action.MAIN",
|
||||
gecko_profile="False",
|
||||
is_release_build=False,
|
||||
host="sophie",
|
||||
power_test=False,
|
||||
cpu_test=False,
|
||||
memory_test=False,
|
||||
enable_webrender=False,
|
||||
)
|
||||
parser = ArgumentParser()
|
||||
|
||||
verify_options(parser, args) # also will work as uses default activity
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
mozunit.main()
|
||||
|
|
|
|||
|
|
@ -3,34 +3,45 @@
|
|||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
from __future__ import absolute_import, unicode_literals
|
||||
|
||||
import mozunit
|
||||
import os
|
||||
import mock
|
||||
|
||||
from raptor import cpu
|
||||
from raptor.raptor import RaptorAndroid
|
||||
import os
|
||||
import sys
|
||||
import mock
|
||||
import mozunit
|
||||
|
||||
# need this so the raptor unit tests can find output & filter classes
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
raptor_dir = os.path.join(os.path.dirname(here), "raptor")
|
||||
sys.path.insert(0, raptor_dir)
|
||||
|
||||
import cpu
|
||||
from webextension import WebExtensionAndroid
|
||||
|
||||
|
||||
def test_no_device():
|
||||
raptor = RaptorAndroid('geckoview', 'org.mozilla.org.mozilla.geckoview_example',
|
||||
cpu_test=True, no_conditioned_profile=False)
|
||||
raptor = WebExtensionAndroid(
|
||||
"geckoview",
|
||||
"org.mozilla.org.mozilla.geckoview_example",
|
||||
cpu_test=True,
|
||||
no_conditioned_profile=False,
|
||||
)
|
||||
raptor.device = None
|
||||
resp = cpu.start_android_cpu_profiler(raptor)
|
||||
assert resp is None
|
||||
|
||||
|
||||
def test_usage_with_invalid_data_returns_zero():
|
||||
with mock.patch('mozdevice.adb.ADBDevice') as device:
|
||||
with mock.patch('raptor.raptor.RaptorControlServer') as control_server:
|
||||
with mock.patch("mozdevice.adb.ADBDevice") as device:
|
||||
with mock.patch("control_server.RaptorControlServer") as control_server:
|
||||
# Create a device that returns invalid data
|
||||
device.shell_output.side_effect = ['8.0.0', 'geckoview']
|
||||
device.shell_output.side_effect = ["8.0.0", "geckoview"]
|
||||
device._verbose = True
|
||||
|
||||
# Create a control server
|
||||
control_server.cpu_test = True
|
||||
control_server.device = device
|
||||
raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example')
|
||||
raptor.config['cpu_test'] = True
|
||||
raptor = WebExtensionAndroid("geckoview", "org.mozilla.geckoview_example")
|
||||
raptor.config["cpu_test"] = True
|
||||
raptor.control_server = control_server
|
||||
raptor.device = device
|
||||
|
||||
|
|
@ -39,56 +50,54 @@ def test_usage_with_invalid_data_returns_zero():
|
|||
|
||||
# Verify the call to submit data was made
|
||||
avg_cpuinfo_data = {
|
||||
u'type': u'cpu',
|
||||
u'test': u'usage_with_invalid_data_returns_zero-avg',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'avg': 0
|
||||
}
|
||||
"type": "cpu",
|
||||
"test": "usage_with_invalid_data_returns_zero-avg",
|
||||
"unit": "%",
|
||||
"values": {"avg": 0},
|
||||
}
|
||||
min_cpuinfo_data = {
|
||||
u'type': u'cpu',
|
||||
u'test': u'usage_with_invalid_data_returns_zero-min',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'min': 0
|
||||
}
|
||||
"type": "cpu",
|
||||
"test": "usage_with_invalid_data_returns_zero-min",
|
||||
"unit": "%",
|
||||
"values": {"min": 0},
|
||||
}
|
||||
max_cpuinfo_data = {
|
||||
u'type': u'cpu',
|
||||
u'test': u'usage_with_invalid_data_returns_zero-max',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'max': 0
|
||||
}
|
||||
"type": "cpu",
|
||||
"test": "usage_with_invalid_data_returns_zero-max",
|
||||
"unit": "%",
|
||||
"values": {"max": 0},
|
||||
}
|
||||
|
||||
cpu_profiler.generate_android_cpu_profile("usage_with_invalid_data_returns_zero")
|
||||
control_server.submit_supporting_data.assert_has_calls([
|
||||
mock.call(avg_cpuinfo_data),
|
||||
mock.call(min_cpuinfo_data),
|
||||
mock.call(max_cpuinfo_data)
|
||||
])
|
||||
cpu_profiler.generate_android_cpu_profile(
|
||||
"usage_with_invalid_data_returns_zero"
|
||||
)
|
||||
control_server.submit_supporting_data.assert_has_calls(
|
||||
[
|
||||
mock.call(avg_cpuinfo_data),
|
||||
mock.call(min_cpuinfo_data),
|
||||
mock.call(max_cpuinfo_data),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_usage_with_output():
|
||||
with mock.patch('mozdevice.adb.ADBDevice') as device:
|
||||
with mock.patch('raptor.raptor.RaptorControlServer') as control_server:
|
||||
with mock.patch("mozdevice.adb.ADBDevice") as device:
|
||||
with mock.patch("control_server.RaptorControlServer") as control_server:
|
||||
# Override the shell output with sample CPU usage details
|
||||
filepath = os.path.abspath(os.path.dirname(__file__)) + '/files/'
|
||||
with open(filepath + 'top-info.txt', 'r') as f:
|
||||
filepath = os.path.abspath(os.path.dirname(__file__)) + "/files/"
|
||||
with open(filepath + "top-info.txt", "r") as f:
|
||||
test_data = f.read()
|
||||
device.shell_output.side_effect = ['8.0.0', test_data]
|
||||
device.shell_output.side_effect = ["8.0.0", test_data]
|
||||
device._verbose = True
|
||||
|
||||
# Create a control server
|
||||
control_server.cpu_test = True
|
||||
control_server.test_name = 'cpuunittest'
|
||||
control_server.test_name = "cpuunittest"
|
||||
control_server.device = device
|
||||
control_server.app_name = 'org.mozilla.geckoview_example'
|
||||
raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example')
|
||||
control_server.app_name = "org.mozilla.geckoview_example"
|
||||
raptor = WebExtensionAndroid("geckoview", "org.mozilla.geckoview_example")
|
||||
raptor.device = device
|
||||
raptor.config['cpu_test'] = True
|
||||
raptor.config["cpu_test"] = True
|
||||
raptor.control_server = control_server
|
||||
|
||||
cpu_profiler = cpu.AndroidCPUProfiler(raptor)
|
||||
|
|
@ -97,58 +106,58 @@ def test_usage_with_output():
|
|||
|
||||
# Verify the response contains our expected CPU % of 93.7
|
||||
avg_cpuinfo_data = {
|
||||
u'type': u'cpu',
|
||||
u'test': u'usage_with_integer_cpu_info_output-avg',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'avg': 93.7/2
|
||||
}
|
||||
"type": "cpu",
|
||||
"test": "usage_with_integer_cpu_info_output-avg",
|
||||
"unit": "%",
|
||||
"values": {"avg": 93.7 / 2},
|
||||
}
|
||||
min_cpuinfo_data = {
|
||||
u'type': u'cpu',
|
||||
u'test': u'usage_with_integer_cpu_info_output-min',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'min': 0
|
||||
}
|
||||
"type": "cpu",
|
||||
"test": "usage_with_integer_cpu_info_output-min",
|
||||
"unit": "%",
|
||||
"values": {"min": 0},
|
||||
}
|
||||
max_cpuinfo_data = {
|
||||
u'type': u'cpu',
|
||||
u'test': u'usage_with_integer_cpu_info_output-max',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'max': 93.7
|
||||
}
|
||||
"type": "cpu",
|
||||
"test": "usage_with_integer_cpu_info_output-max",
|
||||
"unit": "%",
|
||||
"values": {"max": 93.7},
|
||||
}
|
||||
|
||||
cpu_profiler.generate_android_cpu_profile("usage_with_integer_cpu_info_output")
|
||||
control_server.submit_supporting_data.assert_has_calls([
|
||||
mock.call(avg_cpuinfo_data),
|
||||
mock.call(min_cpuinfo_data),
|
||||
mock.call(max_cpuinfo_data)
|
||||
])
|
||||
cpu_profiler.generate_android_cpu_profile(
|
||||
"usage_with_integer_cpu_info_output"
|
||||
)
|
||||
control_server.submit_supporting_data.assert_has_calls(
|
||||
[
|
||||
mock.call(avg_cpuinfo_data),
|
||||
mock.call(min_cpuinfo_data),
|
||||
mock.call(max_cpuinfo_data),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_usage_with_fallback():
|
||||
with mock.patch('mozdevice.adb.ADBDevice') as device:
|
||||
with mock.patch('raptor.raptor.RaptorControlServer') as control_server:
|
||||
with mock.patch("mozdevice.adb.ADBDevice") as device:
|
||||
with mock.patch("control_server.RaptorControlServer") as control_server:
|
||||
device._verbose = True
|
||||
|
||||
# Return what our shell call to dumpsys would give us
|
||||
shell_output = ' 31093 u0_a196 10 -10 8% S ' + \
|
||||
'66 1392100K 137012K fg org.mozilla.geckoview_example'
|
||||
shell_output = (
|
||||
" 31093 u0_a196 10 -10 8% S "
|
||||
+ "66 1392100K 137012K fg org.mozilla.geckoview_example"
|
||||
)
|
||||
|
||||
# We set the version to be less than Android 8
|
||||
device.shell_output.side_effect = ['7.0.0', shell_output]
|
||||
device.shell_output.side_effect = ["7.0.0", shell_output]
|
||||
|
||||
# Create a control server
|
||||
control_server.cpu_test = True
|
||||
control_server.test_name = 'cpuunittest'
|
||||
control_server.test_name = "cpuunittest"
|
||||
control_server.device = device
|
||||
control_server.app_name = 'org.mozilla.geckoview_example'
|
||||
raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example')
|
||||
control_server.app_name = "org.mozilla.geckoview_example"
|
||||
raptor = WebExtensionAndroid("geckoview", "org.mozilla.geckoview_example")
|
||||
raptor.device = device
|
||||
raptor.config['cpu_test'] = True
|
||||
raptor.config["cpu_test"] = True
|
||||
raptor.control_server = control_server
|
||||
|
||||
cpu_profiler = cpu.AndroidCPUProfiler(raptor)
|
||||
|
|
@ -157,37 +166,33 @@ def test_usage_with_fallback():
|
|||
|
||||
# Verify the response contains our expected CPU % of 8
|
||||
avg_cpuinfo_data = {
|
||||
u'type': u'cpu',
|
||||
u'test': u'usage_with_fallback-avg',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'avg': 8/2
|
||||
}
|
||||
"type": "cpu",
|
||||
"test": "usage_with_fallback-avg",
|
||||
"unit": "%",
|
||||
"values": {"avg": 8 / 2},
|
||||
}
|
||||
min_cpuinfo_data = {
|
||||
u'type': u'cpu',
|
||||
u'test': u'usage_with_fallback-min',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'min': 0
|
||||
}
|
||||
"type": "cpu",
|
||||
"test": "usage_with_fallback-min",
|
||||
"unit": "%",
|
||||
"values": {"min": 0},
|
||||
}
|
||||
max_cpuinfo_data = {
|
||||
u'type': u'cpu',
|
||||
u'test': u'usage_with_fallback-max',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'max': 8
|
||||
}
|
||||
"type": "cpu",
|
||||
"test": "usage_with_fallback-max",
|
||||
"unit": "%",
|
||||
"values": {"max": 8},
|
||||
}
|
||||
|
||||
cpu_profiler.generate_android_cpu_profile("usage_with_fallback")
|
||||
control_server.submit_supporting_data.assert_has_calls([
|
||||
mock.call(avg_cpuinfo_data),
|
||||
mock.call(min_cpuinfo_data),
|
||||
mock.call(max_cpuinfo_data)
|
||||
])
|
||||
control_server.submit_supporting_data.assert_has_calls(
|
||||
[
|
||||
mock.call(avg_cpuinfo_data),
|
||||
mock.call(min_cpuinfo_data),
|
||||
mock.call(max_cpuinfo_data),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
mozunit.main()
|
||||
|
|
|
|||
|
|
@ -11,8 +11,9 @@ import tarfile
|
|||
import sys
|
||||
|
||||
# need this so raptor imports work both from /raptor and via mach
|
||||
HERE = os.path.abspath(os.path.dirname(__file__))
|
||||
raptor_dir = os.path.join(os.path.dirname(HERE), 'raptor')
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
raptor_dir = os.path.join(os.path.dirname(here), "raptor")
|
||||
sys.path.insert(0, raptor_dir)
|
||||
|
||||
from gecko_profile import GeckoProfile
|
||||
|
|
@ -21,21 +22,24 @@ from gecko_profile import GeckoProfile
|
|||
def test_browsertime_profiling():
|
||||
result_dir = tempfile.mkdtemp()
|
||||
# untar geckoProfile.tar
|
||||
with tarfile.open(os.path.join(HERE, "geckoProfile.tar")) as f:
|
||||
with tarfile.open(os.path.join(here, "geckoProfile.tar")) as f:
|
||||
f.extractall(path=result_dir)
|
||||
|
||||
# Makes sure we can run the profile process against a browsertime-generated
|
||||
# profile (geckoProfile-1.json in this test dir)
|
||||
upload_dir = tempfile.mkdtemp()
|
||||
symbols_path = tempfile.mkdtemp()
|
||||
raptor_config = {'symbols_path': symbols_path, 'browsertime': True,
|
||||
'browsertime_result_dir': result_dir}
|
||||
test_config = {'name': 'tp6'}
|
||||
raptor_config = {
|
||||
"symbols_path": symbols_path,
|
||||
"browsertime": True,
|
||||
"browsertime_result_dir": result_dir,
|
||||
}
|
||||
test_config = {"name": "tp6"}
|
||||
try:
|
||||
profile = GeckoProfile(upload_dir, raptor_config, test_config)
|
||||
profile.symbolicate()
|
||||
profile.clean()
|
||||
arcname = os.environ['RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE']
|
||||
arcname = os.environ["RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE"]
|
||||
assert os.stat(arcname).st_size > 1000000, "We got a 1mb+ zip"
|
||||
finally:
|
||||
shutil.rmtree(upload_dir)
|
||||
|
|
@ -43,5 +47,5 @@ def test_browsertime_profiling():
|
|||
shutil.rmtree(result_dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
mozunit.main()
|
||||
|
|
|
|||
|
|
@ -8,7 +8,10 @@ import mozunit
|
|||
|
||||
from mozlog.structuredlog import set_default_logger, StructuredLogger
|
||||
|
||||
set_default_logger(StructuredLogger('test_playback'))
|
||||
# need this so raptor imports work both from /raptor and via mach
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
set_default_logger(StructuredLogger("test_playback"))
|
||||
|
||||
from mozproxy import get_playback
|
||||
from mozproxy.backends.mitm import MitmproxyDesktop
|
||||
|
|
@ -16,32 +19,36 @@ from mozproxy.backends.mitm import MitmproxyDesktop
|
|||
config = {}
|
||||
|
||||
run_local = True
|
||||
if os.environ.get('TOOLTOOLCACHE') is None:
|
||||
if os.environ.get("TOOLTOOLCACHE") is None:
|
||||
run_local = False
|
||||
|
||||
|
||||
def test_get_playback(get_binary):
|
||||
config['platform'] = mozinfo.os
|
||||
if 'win' in config['platform']:
|
||||
config["platform"] = mozinfo.os
|
||||
if "win" in config["platform"]:
|
||||
# this test is not yet supported on windows
|
||||
assert True
|
||||
return
|
||||
config['obj_path'] = os.path.dirname(get_binary('firefox'))
|
||||
config['playback_tool'] = 'mitmproxy'
|
||||
config['playback_version'] = '4.0.4'
|
||||
config['playback_binary_manifest'] = 'mitmproxy-rel-bin-4.0.4-{platform}.manifest'
|
||||
config['playback_pageset_manifest'] = os.path.join(
|
||||
os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "raptor", "playback",
|
||||
'mitm4-linux-firefox-amazon.manifest')
|
||||
config['playback_recordings'] = 'amazon.mp'
|
||||
config['binary'] = get_binary('firefox')
|
||||
config['run_local'] = run_local
|
||||
config['app'] = 'firefox'
|
||||
config['host'] = 'https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1'
|
||||
config["obj_path"] = os.path.dirname(get_binary("firefox"))
|
||||
config["playback_tool"] = "mitmproxy"
|
||||
config["playback_version"] = "4.0.4"
|
||||
config["playback_binary_manifest"] = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
|
||||
config["playback_pageset_manifest"] = os.path.join(
|
||||
os.path.dirname(os.path.abspath(os.path.dirname(__file__))),
|
||||
"raptor",
|
||||
"playback",
|
||||
"mitm4-linux-firefox-amazon.manifest",
|
||||
)
|
||||
config["playback_recordings"] = "amazon.mp"
|
||||
config["binary"] = get_binary("firefox")
|
||||
config["run_local"] = run_local
|
||||
config["app"] = "firefox"
|
||||
config["host"] = "127.0.0.1"
|
||||
|
||||
playback = get_playback(config)
|
||||
playback.config['playback_files'] = [os.path.join(playback.mozproxy_dir,
|
||||
config['playback_recordings'])]
|
||||
playback.config["playback_files"] = [
|
||||
os.path.join(playback.mozproxy_dir, config["playback_recordings"])
|
||||
]
|
||||
assert isinstance(playback, MitmproxyDesktop)
|
||||
playback.start()
|
||||
time.sleep(1)
|
||||
|
|
@ -49,7 +56,7 @@ def test_get_playback(get_binary):
|
|||
|
||||
|
||||
def test_get_unsupported_playback():
|
||||
config['playback_tool'] = 'unsupported'
|
||||
config["playback_tool"] = "unsupported"
|
||||
playback = get_playback(config)
|
||||
assert playback is None
|
||||
|
||||
|
|
@ -59,5 +66,5 @@ def test_get_playback_missing_tool_name():
|
|||
assert playback is None
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
mozunit.main()
|
||||
|
|
|
|||
|
|
@ -3,24 +3,33 @@
|
|||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
from __future__ import absolute_import, unicode_literals
|
||||
|
||||
import mozunit
|
||||
import sys
|
||||
|
||||
import os
|
||||
import mock
|
||||
import tempfile
|
||||
|
||||
from raptor import power
|
||||
from raptor.raptor import RaptorAndroid
|
||||
import mozunit
|
||||
|
||||
# need this so raptor imports work both from /raptor and via mach
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
raptor_dir = os.path.join(os.path.dirname(here), "raptor")
|
||||
sys.path.insert(0, raptor_dir)
|
||||
|
||||
import power
|
||||
from webextension import WebExtensionAndroid
|
||||
|
||||
|
||||
def test_android7_power():
|
||||
if not os.getenv('MOZ_UPLOAD_DIR'):
|
||||
os.environ['MOZ_UPLOAD_DIR'] = tempfile.mkdtemp()
|
||||
if not os.getenv("MOZ_UPLOAD_DIR"):
|
||||
os.environ["MOZ_UPLOAD_DIR"] = tempfile.mkdtemp()
|
||||
|
||||
with mock.patch('mozdevice.adb.ADBDevice') as device:
|
||||
with mock.patch('raptor.raptor.RaptorControlServer') as control_server:
|
||||
with mock.patch("mozdevice.adb.ADBDevice") as device:
|
||||
with mock.patch("control_server.RaptorControlServer") as control_server:
|
||||
# Override the shell output with sample CPU usage details
|
||||
filepath = os.path.abspath(os.path.dirname(__file__)) + '/files/'
|
||||
f = open(filepath + 'batterystats-android-7.txt', 'r')
|
||||
filepath = os.path.abspath(os.path.dirname(__file__)) + "/files/"
|
||||
f = open(filepath + "batterystats-android-7.txt", "r")
|
||||
batterystats_return_value = f.read()
|
||||
|
||||
# Multiple shell output calls are performed
|
||||
|
|
@ -29,10 +38,10 @@ def test_android7_power():
|
|||
device.shell_output.side_effect = [
|
||||
None,
|
||||
None,
|
||||
'Test value',
|
||||
'Test value',
|
||||
"Test value",
|
||||
"Test value",
|
||||
batterystats_return_value,
|
||||
'7.0.0',
|
||||
"7.0.0",
|
||||
]
|
||||
|
||||
device._verbose = True
|
||||
|
|
@ -40,70 +49,67 @@ def test_android7_power():
|
|||
|
||||
# Create a control server
|
||||
control_server.power_test = True
|
||||
control_server.test_name = 'gve-pytest'
|
||||
control_server.test_name = "gve-pytest"
|
||||
control_server.device = device
|
||||
control_server.app_name = 'org.mozilla.geckoview_example'
|
||||
raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example', power_test=True)
|
||||
raptor.device = device
|
||||
raptor.config['power_test'] = True
|
||||
raptor.control_server = control_server
|
||||
raptor.power_test_time = 20 # minutes
|
||||
raptor.os_baseline_data = {
|
||||
u'type': u'power',
|
||||
u'test': u'gve-pytest',
|
||||
u'unit': u'mAh',
|
||||
u'values': {
|
||||
u'cpu': float(5),
|
||||
u'wifi': float(5),
|
||||
u'screen': float(5)
|
||||
}
|
||||
control_server.app_name = "org.mozilla.geckoview_example"
|
||||
web_extension = WebExtensionAndroid(
|
||||
"geckoview", "org.mozilla.geckoview_example", power_test=True
|
||||
)
|
||||
web_extension.device = device
|
||||
web_extension.config["power_test"] = True
|
||||
web_extension.control_server = control_server
|
||||
web_extension.power_test_time = 20 # minutes
|
||||
web_extension.os_baseline_data = {
|
||||
"type": "power",
|
||||
"test": "gve-pytest",
|
||||
"unit": "mAh",
|
||||
"values": {"cpu": float(5), "wifi": float(5), "screen": float(5)},
|
||||
}
|
||||
|
||||
# Verify the response contains our expected calculations
|
||||
# (no proportional measure on android 7)
|
||||
power_data = {
|
||||
u'type': u'power',
|
||||
u'test': u'gve-pytest',
|
||||
u'unit': u'mAh',
|
||||
u'values': {
|
||||
u'cpu': float(14.5),
|
||||
u'wifi': float(0.132),
|
||||
u'screen': float(70.7)
|
||||
}
|
||||
"type": "power",
|
||||
"test": "gve-pytest",
|
||||
"unit": "mAh",
|
||||
"values": {
|
||||
"cpu": float(14.5),
|
||||
"wifi": float(0.132),
|
||||
"screen": float(70.7),
|
||||
},
|
||||
}
|
||||
|
||||
pc_data = {
|
||||
u'type': u'power',
|
||||
u'test': u'gve-pytest-%change',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'cpu': float(14.5),
|
||||
u'wifi': float(0.132000000000005),
|
||||
u'screen': float(70.70000000000002)
|
||||
}
|
||||
"type": "power",
|
||||
"test": "gve-pytest-%change",
|
||||
"unit": "%",
|
||||
"values": {
|
||||
"cpu": float(14.5),
|
||||
"wifi": float(0.132000000000005),
|
||||
"screen": float(70.70000000000002),
|
||||
},
|
||||
}
|
||||
|
||||
power.finish_android_power_test(
|
||||
raptor,
|
||||
'gve-pytest'
|
||||
)
|
||||
power.finish_android_power_test(web_extension, "gve-pytest")
|
||||
|
||||
control_server.submit_supporting_data.assert_has_calls([
|
||||
control_server.submit_supporting_data.assert_has_calls(
|
||||
[
|
||||
mock.call(power_data),
|
||||
mock.call(pc_data),
|
||||
mock.call(raptor.os_baseline_data)
|
||||
])
|
||||
mock.call(web_extension.os_baseline_data),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_android8_power():
|
||||
if not os.getenv('MOZ_UPLOAD_DIR'):
|
||||
os.environ['MOZ_UPLOAD_DIR'] = tempfile.mkdtemp()
|
||||
if not os.getenv("MOZ_UPLOAD_DIR"):
|
||||
os.environ["MOZ_UPLOAD_DIR"] = tempfile.mkdtemp()
|
||||
|
||||
with mock.patch('mozdevice.adb.ADBDevice') as device:
|
||||
with mock.patch('raptor.raptor.RaptorControlServer') as control_server:
|
||||
with mock.patch("mozdevice.adb.ADBDevice") as device:
|
||||
with mock.patch("control_server.RaptorControlServer") as control_server:
|
||||
# Override the shell output with sample CPU usage details
|
||||
filepath = os.path.abspath(os.path.dirname(__file__)) + '/files/'
|
||||
f = open(filepath + 'batterystats-android-8.txt', 'r')
|
||||
filepath = os.path.abspath(os.path.dirname(__file__)) + "/files/"
|
||||
f = open(filepath + "batterystats-android-8.txt", "r")
|
||||
batterystats_return_value = f.read()
|
||||
print(type(batterystats_return_value))
|
||||
|
||||
|
|
@ -113,10 +119,10 @@ def test_android8_power():
|
|||
device.shell_output.side_effect = [
|
||||
None,
|
||||
None,
|
||||
'Test value',
|
||||
'Test value',
|
||||
"Test value",
|
||||
"Test value",
|
||||
batterystats_return_value,
|
||||
'8.0.0',
|
||||
"8.0.0",
|
||||
]
|
||||
|
||||
device._verbose = True
|
||||
|
|
@ -124,72 +130,73 @@ def test_android8_power():
|
|||
|
||||
# Create a control server
|
||||
control_server.power_test = True
|
||||
control_server.test_name = 'gve-pytest'
|
||||
control_server.test_name = "gve-pytest"
|
||||
control_server.device = device
|
||||
control_server.app_name = 'org.mozilla.geckoview_example'
|
||||
raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example', power_test=True)
|
||||
raptor.device = device
|
||||
raptor.config['power_test'] = True
|
||||
raptor.control_server = control_server
|
||||
raptor.power_test_time = 20 # minutes
|
||||
raptor.os_baseline_data = {
|
||||
u'type': u'power',
|
||||
u'test': u'gve-pytest',
|
||||
u'unit': u'mAh',
|
||||
u'values': {
|
||||
u'cpu': float(5),
|
||||
u'wifi': float(5),
|
||||
u'screen': float(5),
|
||||
u'proportional': float(5)
|
||||
}
|
||||
control_server.app_name = "org.mozilla.geckoview_example"
|
||||
web_extension = WebExtensionAndroid(
|
||||
"geckoview", "org.mozilla.geckoview_example", power_test=True
|
||||
)
|
||||
web_extension.device = device
|
||||
web_extension.config["power_test"] = True
|
||||
web_extension.control_server = control_server
|
||||
web_extension.power_test_time = 20 # minutes
|
||||
web_extension.os_baseline_data = {
|
||||
"type": "power",
|
||||
"test": "gve-pytest",
|
||||
"unit": "mAh",
|
||||
"values": {
|
||||
"cpu": float(5),
|
||||
"wifi": float(5),
|
||||
"screen": float(5),
|
||||
"proportional": float(5),
|
||||
},
|
||||
}
|
||||
|
||||
# Verify the response contains our expected calculations
|
||||
power_data = {
|
||||
u'type': u'power',
|
||||
u'test': u'gve-pytest',
|
||||
u'unit': u'mAh',
|
||||
u'values': {
|
||||
u'cpu': float(4.7),
|
||||
u'wifi': float(0.000556),
|
||||
u'screen': float(51.5),
|
||||
u'proportional': float(11.2)
|
||||
}
|
||||
"type": "power",
|
||||
"test": "gve-pytest",
|
||||
"unit": "mAh",
|
||||
"values": {
|
||||
"cpu": float(4.7),
|
||||
"wifi": float(0.000556),
|
||||
"screen": float(51.5),
|
||||
"proportional": float(11.2),
|
||||
},
|
||||
}
|
||||
|
||||
pc_data = {
|
||||
u'type': u'power',
|
||||
u'test': u'gve-pytest-%change',
|
||||
u'unit': u'%',
|
||||
u'values': {
|
||||
u'cpu': float(4.700000000000017),
|
||||
u'wifi': float(0.0005559999999888987),
|
||||
u'screen': float(51.5),
|
||||
u'proportional': float(11.199999999999989)
|
||||
}
|
||||
"type": "power",
|
||||
"test": "gve-pytest-%change",
|
||||
"unit": "%",
|
||||
"values": {
|
||||
"cpu": float(4.700000000000017),
|
||||
"wifi": float(0.0005559999999888987),
|
||||
"screen": float(51.5),
|
||||
"proportional": float(11.199999999999989),
|
||||
},
|
||||
}
|
||||
|
||||
power.finish_android_power_test(
|
||||
raptor,
|
||||
'gve-pytest'
|
||||
)
|
||||
power.finish_android_power_test(web_extension, "gve-pytest")
|
||||
|
||||
control_server.submit_supporting_data.assert_has_calls([
|
||||
control_server.submit_supporting_data.assert_has_calls(
|
||||
[
|
||||
mock.call(power_data),
|
||||
mock.call(pc_data),
|
||||
mock.call(raptor.os_baseline_data)
|
||||
])
|
||||
mock.call(web_extension.os_baseline_data),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_androidos_baseline_power():
|
||||
if not os.getenv('MOZ_UPLOAD_DIR'):
|
||||
os.environ['MOZ_UPLOAD_DIR'] = tempfile.mkdtemp()
|
||||
if not os.getenv("MOZ_UPLOAD_DIR"):
|
||||
os.environ["MOZ_UPLOAD_DIR"] = tempfile.mkdtemp()
|
||||
|
||||
with mock.patch('mozdevice.adb.ADBDevice') as device:
|
||||
with mock.patch('raptor.raptor.RaptorControlServer') as control_server:
|
||||
with mock.patch("mozdevice.adb.ADBDevice") as device:
|
||||
with mock.patch("control_server.RaptorControlServer") as control_server:
|
||||
# Override the shell output with sample CPU usage details
|
||||
filepath = os.path.abspath(os.path.dirname(__file__)) + '/files/'
|
||||
f = open(filepath + 'batterystats-android-8.txt', 'r')
|
||||
filepath = os.path.abspath(os.path.dirname(__file__)) + "/files/"
|
||||
f = open(filepath + "batterystats-android-8.txt", "r")
|
||||
batterystats_return_value = f.read()
|
||||
|
||||
# Multiple shell output calls are performed
|
||||
|
|
@ -198,10 +205,10 @@ def test_androidos_baseline_power():
|
|||
device.shell_output.side_effect = [
|
||||
None,
|
||||
None,
|
||||
'Test value',
|
||||
'Test value',
|
||||
"Test value",
|
||||
"Test value",
|
||||
batterystats_return_value,
|
||||
'8.0.0',
|
||||
"8.0.0",
|
||||
]
|
||||
|
||||
device._verbose = True
|
||||
|
|
@ -209,36 +216,36 @@ def test_androidos_baseline_power():
|
|||
|
||||
# Create a control server
|
||||
control_server.power_test = True
|
||||
control_server.test_name = 'gve-pytest'
|
||||
control_server.test_name = "gve-pytest"
|
||||
control_server.device = device
|
||||
control_server.app_name = 'org.mozilla.geckoview_example'
|
||||
raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example', power_test=True)
|
||||
raptor.device = device
|
||||
raptor.config['power_test'] = True
|
||||
raptor.control_server = control_server
|
||||
control_server.app_name = "org.mozilla.geckoview_example"
|
||||
web_extension = WebExtensionAndroid(
|
||||
"geckoview", "org.mozilla.geckoview_example", power_test=True
|
||||
)
|
||||
web_extension.device = device
|
||||
web_extension.config["power_test"] = True
|
||||
web_extension.control_server = control_server
|
||||
|
||||
# Expected OS baseline calculation result
|
||||
os_baseline_data = {
|
||||
u'type': u'power',
|
||||
u'test': u'gve-pytest',
|
||||
u'unit': u'mAh',
|
||||
u'values': {
|
||||
u'cpu': float(10.786654),
|
||||
u'wifi': float(2.26132),
|
||||
u'screen': float(51.66),
|
||||
u'proportional': float(11.294805199999999)
|
||||
}
|
||||
"type": "power",
|
||||
"test": "gve-pytest",
|
||||
"unit": "mAh",
|
||||
"values": {
|
||||
"cpu": float(10.786654),
|
||||
"wifi": float(2.26132),
|
||||
"screen": float(51.66),
|
||||
"proportional": float(11.294805199999999),
|
||||
},
|
||||
}
|
||||
|
||||
# Verify the response contains our expected calculations
|
||||
power.finish_android_power_test(
|
||||
raptor,
|
||||
'gve-pytest',
|
||||
os_baseline=True
|
||||
web_extension, "gve-pytest", os_baseline=True
|
||||
)
|
||||
|
||||
assert raptor.os_baseline_data == os_baseline_data
|
||||
assert web_extension.os_baseline_data == os_baseline_data
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
mozunit.main()
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ import os
|
|||
import mozunit
|
||||
import pytest
|
||||
|
||||
# need this so raptor imports work both from /raptor and via mach
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
from raptor import cmdline
|
||||
|
||||
|
||||
|
|
@ -12,9 +14,11 @@ def test_pageload_subtests(capsys, monkeypatch, tmpdir):
|
|||
# respect the --test path that would be much better.
|
||||
def mock(path):
|
||||
return str(tmpdir)
|
||||
|
||||
monkeypatch.setattr(os.path, "dirname", mock)
|
||||
manifest = tmpdir.join("raptor.ini")
|
||||
manifest.write("""
|
||||
manifest.write(
|
||||
"""
|
||||
[DEFAULT]
|
||||
type = pageload
|
||||
apps = firefox
|
||||
|
|
@ -23,11 +27,14 @@ apps = firefox
|
|||
measure = foo, bar
|
||||
|
||||
[raptor-subtest-2]
|
||||
""")
|
||||
"""
|
||||
)
|
||||
with pytest.raises(SystemExit):
|
||||
cmdline.parse_args(["--print-tests"])
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == """
|
||||
assert (
|
||||
captured.out
|
||||
== """
|
||||
Raptor Tests Available for Firefox Desktop
|
||||
==========================================
|
||||
|
||||
|
|
@ -39,7 +46,8 @@ raptor
|
|||
|
||||
Done.
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
mozunit.main()
|
||||
|
|
|
|||
|
|
@ -1,27 +1,30 @@
|
|||
from __future__ import absolute_import, unicode_literals
|
||||
from six import reraise
|
||||
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import mock
|
||||
import mozunit
|
||||
import pytest
|
||||
from mock import Mock
|
||||
from six import reraise
|
||||
|
||||
import mozunit
|
||||
from mozprofile import BaseProfile
|
||||
from mozrunner.errors import RunnerNotStartedError
|
||||
|
||||
# need this so raptor imports work both from /raptor and via mach
|
||||
# need this so the raptor unit tests can find output & filter classes
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
raptor_dir = os.path.join(os.path.dirname(here), "raptor")
|
||||
sys.path.insert(0, raptor_dir)
|
||||
|
||||
from raptor.raptor import (BrowsertimeDesktop,
|
||||
BrowsertimeAndroid,
|
||||
RaptorDesktopFirefox,
|
||||
RaptorDesktopChrome,
|
||||
RaptorAndroid)
|
||||
|
||||
from browsertime import BrowsertimeDesktop, BrowsertimeAndroid
|
||||
from webextension import (
|
||||
WebExtensionFirefox,
|
||||
WebExtensionDesktopChrome,
|
||||
WebExtensionAndroid,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_TIMEOUT = 125
|
||||
|
|
@ -43,23 +46,26 @@ class TestBrowserThread(threading.Thread):
|
|||
|
||||
|
||||
# Perftest tests
|
||||
@pytest.mark.parametrize("perftest_class, app_name", [
|
||||
[RaptorDesktopFirefox, "firefox"],
|
||||
[RaptorDesktopChrome, "chrome"],
|
||||
[RaptorDesktopChrome, "chromium"],
|
||||
[RaptorAndroid, "fennec"],
|
||||
[RaptorAndroid, "geckoview"],
|
||||
[BrowsertimeDesktop, "firefox"],
|
||||
[BrowsertimeDesktop, "chrome"],
|
||||
[BrowsertimeDesktop, "chromium"],
|
||||
[BrowsertimeAndroid, "fennec"],
|
||||
[BrowsertimeAndroid, "geckoview"],
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"perftest_class, app_name",
|
||||
[
|
||||
[WebExtensionFirefox, "firefox"],
|
||||
[WebExtensionDesktopChrome, "chrome"],
|
||||
[WebExtensionDesktopChrome, "chromium"],
|
||||
[WebExtensionAndroid, "fennec"],
|
||||
[WebExtensionAndroid, "geckoview"],
|
||||
[BrowsertimeDesktop, "firefox"],
|
||||
[BrowsertimeDesktop, "chrome"],
|
||||
[BrowsertimeDesktop, "chromium"],
|
||||
[BrowsertimeAndroid, "fennec"],
|
||||
[BrowsertimeAndroid, "geckoview"],
|
||||
],
|
||||
)
|
||||
def test_build_profile(options, perftest_class, app_name, get_prefs):
|
||||
options['app'] = app_name
|
||||
options["app"] = app_name
|
||||
perftest_instance = perftest_class(**options)
|
||||
assert isinstance(perftest_instance.profile, BaseProfile)
|
||||
if app_name != 'firefox':
|
||||
if app_name != "firefox":
|
||||
return
|
||||
|
||||
# These prefs are set in mozprofile
|
||||
|
|
@ -67,13 +73,13 @@ def test_build_profile(options, perftest_class, app_name, get_prefs):
|
|||
'user_pref("app.update.checkInstallTime", false);',
|
||||
'user_pref("app.update.disabledForTesting", true);',
|
||||
'user_pref("'
|
||||
'security.turn_off_all_security_so_that_viruses_can_take_over_this_computer", true);'
|
||||
'security.turn_off_all_security_so_that_viruses_can_take_over_this_computer", true);',
|
||||
]
|
||||
# This pref is set in raptor
|
||||
raptor_pref = 'user_pref("security.enable_java", false);'
|
||||
|
||||
prefs_file = os.path.join(perftest_instance.profile.profile, 'user.js')
|
||||
with open(prefs_file, 'r') as fh:
|
||||
prefs_file = os.path.join(perftest_instance.profile.profile, "user.js")
|
||||
with open(prefs_file, "r") as fh:
|
||||
prefs = fh.read()
|
||||
for firefox_pref in firefox_prefs:
|
||||
assert firefox_pref in prefs
|
||||
|
|
@ -81,57 +87,58 @@ def test_build_profile(options, perftest_class, app_name, get_prefs):
|
|||
|
||||
|
||||
def test_perftest_host_ip(ConcretePerftest, options, get_prefs):
|
||||
os.environ['HOST_IP'] = 'some_dummy_host_ip'
|
||||
options['host'] = 'HOST_IP'
|
||||
os.environ["HOST_IP"] = "some_dummy_host_ip"
|
||||
options["host"] = "HOST_IP"
|
||||
|
||||
perftest = ConcretePerftest(**options)
|
||||
|
||||
assert perftest.config['host'] == os.environ['HOST_IP']
|
||||
assert perftest.config["host"] == os.environ["HOST_IP"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('app_name, expected_e10s_flag', [
|
||||
['firefox', True],
|
||||
['fennec', False],
|
||||
['geckoview', True],
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"app_name, expected_e10s_flag",
|
||||
[["firefox", True], ["fennec", False], ["geckoview", True]],
|
||||
)
|
||||
def test_e10s_enabling(ConcretePerftest, options, app_name, expected_e10s_flag):
|
||||
options['app'] = app_name
|
||||
perftest = ConcretePerftest(profile_class='firefox', **options)
|
||||
assert perftest.config['e10s'] == expected_e10s_flag
|
||||
options["app"] = app_name
|
||||
perftest = ConcretePerftest(profile_class="firefox", **options)
|
||||
assert perftest.config["e10s"] == expected_e10s_flag
|
||||
|
||||
|
||||
def test_profile_was_provided_locally(ConcretePerftest, options):
|
||||
perftest = ConcretePerftest(**options)
|
||||
assert os.path.isdir(perftest.config['local_profile_dir'])
|
||||
assert os.path.isdir(perftest.config["local_profile_dir"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize('profile_class, app, expected_profile', [
|
||||
['firefox', 'firefox', 'firefox'],
|
||||
[None, 'firefox', 'firefox'],
|
||||
['firefox', None, 'firefox'],
|
||||
['firefox', 'fennec', 'firefox'],
|
||||
])
|
||||
def test_profile_class_assignation(ConcretePerftest,
|
||||
options,
|
||||
profile_class,
|
||||
app,
|
||||
expected_profile):
|
||||
options['app'] = app
|
||||
@pytest.mark.parametrize(
|
||||
"profile_class, app, expected_profile",
|
||||
[
|
||||
["firefox", "firefox", "firefox"],
|
||||
[None, "firefox", "firefox"],
|
||||
["firefox", None, "firefox"],
|
||||
["firefox", "fennec", "firefox"],
|
||||
],
|
||||
)
|
||||
def test_profile_class_assignation(
|
||||
ConcretePerftest, options, profile_class, app, expected_profile
|
||||
):
|
||||
options["app"] = app
|
||||
perftest = ConcretePerftest(profile_class=profile_class, **options)
|
||||
assert perftest.profile_class == expected_profile
|
||||
|
||||
|
||||
def test_raptor_venv(ConcretePerftest, options):
|
||||
perftest = ConcretePerftest(**options)
|
||||
assert perftest.raptor_venv.endswith('raptor-venv')
|
||||
assert perftest.raptor_venv.endswith("raptor-venv")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'run_local,'
|
||||
'debug_mode,'
|
||||
'post_startup_delay,'
|
||||
'expected_post_startup_delay,'
|
||||
'expected_debug_mode', [
|
||||
"run_local,"
|
||||
"debug_mode,"
|
||||
"post_startup_delay,"
|
||||
"expected_post_startup_delay,"
|
||||
"expected_debug_mode",
|
||||
[
|
||||
[True, True, 1234, 1234, True],
|
||||
[True, True, 12345, 3000, True],
|
||||
[False, False, 1234, 1234, False],
|
||||
|
|
@ -140,14 +147,17 @@ def test_raptor_venv(ConcretePerftest, options):
|
|||
[True, False, 12345, 12345, False],
|
||||
[False, True, 1234, 1234, False],
|
||||
[False, True, 12345, 12345, False],
|
||||
])
|
||||
def test_post_startup_delay(ConcretePerftest,
|
||||
options,
|
||||
run_local,
|
||||
debug_mode,
|
||||
post_startup_delay,
|
||||
expected_post_startup_delay,
|
||||
expected_debug_mode):
|
||||
],
|
||||
)
|
||||
def test_post_startup_delay(
|
||||
ConcretePerftest,
|
||||
options,
|
||||
run_local,
|
||||
debug_mode,
|
||||
post_startup_delay,
|
||||
expected_post_startup_delay,
|
||||
expected_debug_mode,
|
||||
):
|
||||
perftest = ConcretePerftest(
|
||||
run_local=run_local,
|
||||
debug_mode=debug_mode,
|
||||
|
|
@ -158,36 +168,32 @@ def test_post_startup_delay(ConcretePerftest,
|
|||
assert perftest.debug_mode == expected_debug_mode
|
||||
|
||||
|
||||
@pytest.mark.parametrize('alert, expected_alert', [
|
||||
['test_to_alert_on', 'test_to_alert_on'],
|
||||
[None, None],
|
||||
])
|
||||
def test_perftest_run_test_setup(ConcretePerftest, options, mock_test, alert, expected_alert):
|
||||
@pytest.mark.parametrize(
|
||||
"alert, expected_alert", [["test_to_alert_on", "test_to_alert_on"], [None, None]]
|
||||
)
|
||||
def test_perftest_run_test_setup(
|
||||
ConcretePerftest, options, mock_test, alert, expected_alert
|
||||
):
|
||||
perftest = ConcretePerftest(**options)
|
||||
mock_test['alert_on'] = alert
|
||||
mock_test["alert_on"] = alert
|
||||
|
||||
perftest.run_test_setup(mock_test)
|
||||
|
||||
assert perftest.config['subtest_alert_on'] == expected_alert
|
||||
assert perftest.config["subtest_alert_on"] == expected_alert
|
||||
|
||||
|
||||
# Raptor tests
|
||||
@pytest.mark.parametrize('app', [
|
||||
'firefox',
|
||||
pytest.mark.xfail('chrome'),
|
||||
pytest.mark.xfail('chromium'),
|
||||
])
|
||||
# WebExtension tests
|
||||
@pytest.mark.parametrize(
|
||||
"app", ["firefox", pytest.mark.xfail("chrome"), pytest.mark.xfail("chromium")]
|
||||
)
|
||||
def test_start_browser(get_binary, app):
|
||||
binary = get_binary(app)
|
||||
assert binary
|
||||
|
||||
raptor = RaptorDesktopFirefox(app, binary, post_startup_delay=0)
|
||||
raptor = WebExtensionFirefox(app, binary, post_startup_delay=0)
|
||||
|
||||
tests = [{
|
||||
'name': 'raptor-{}-tp6'.format(app),
|
||||
'page_timeout': 1000
|
||||
}]
|
||||
test_names = [test['name'] for test in tests]
|
||||
tests = [{"name": "raptor-{}-tp6".format(app), "page_timeout": 1000}]
|
||||
test_names = [test["name"] for test in tests]
|
||||
|
||||
thread = TestBrowserThread(raptor, tests, test_names)
|
||||
thread.start()
|
||||
|
|
@ -217,27 +223,26 @@ def test_start_browser(get_binary, app):
|
|||
# Browsertime tests
|
||||
def test_cmd_arguments(ConcreteBrowsertime, browsertime_options, mock_test):
|
||||
expected_cmd = {
|
||||
browsertime_options['browsertime_node'],
|
||||
browsertime_options['browsertime_browsertimejs'],
|
||||
'--firefox.geckodriverPath', browsertime_options['browsertime_geckodriver'],
|
||||
'--chrome.chromedriverPath', browsertime_options['browsertime_chromedriver'],
|
||||
'--browsertime.page_cycles', '1',
|
||||
'--browsertime.url', mock_test['test_url'],
|
||||
'--browsertime.page_cycle_delay', '1000',
|
||||
'--browsertime.post_startup_delay', str(DEFAULT_TIMEOUT),
|
||||
'--firefox.profileTemplate',
|
||||
'--skipHar',
|
||||
'--video', 'true',
|
||||
'--visualMetrics', 'false',
|
||||
'--timeouts.pageLoad', str(DEFAULT_TIMEOUT),
|
||||
'--timeouts.script', str(DEFAULT_TIMEOUT),
|
||||
'-vvv',
|
||||
'--resultDir',
|
||||
'-n', '1',
|
||||
browsertime_options["browsertime_node"],
|
||||
browsertime_options["browsertime_browsertimejs"],
|
||||
"--firefox.geckodriverPath", browsertime_options["browsertime_geckodriver"],
|
||||
"--chrome.chromedriverPath", browsertime_options["browsertime_chromedriver"],
|
||||
"--browsertime.page_cycles", "1",
|
||||
"--browsertime.url", mock_test["test_url"],
|
||||
"--browsertime.page_cycle_delay", "1000",
|
||||
"--browsertime.post_startup_delay", str(DEFAULT_TIMEOUT),
|
||||
"--firefox.profileTemplate",
|
||||
"--skipHar",
|
||||
"--video", "true",
|
||||
"--visualMetrics", "false",
|
||||
"--timeouts.pageLoad", str(DEFAULT_TIMEOUT),
|
||||
"--timeouts.script", str(DEFAULT_TIMEOUT),
|
||||
"-vvv",
|
||||
"--resultDir",
|
||||
"-n", "1",
|
||||
}
|
||||
browsertime = ConcreteBrowsertime(
|
||||
post_startup_delay=DEFAULT_TIMEOUT,
|
||||
**browsertime_options
|
||||
post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
|
||||
)
|
||||
browsertime.run_test_setup(mock_test)
|
||||
cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT)
|
||||
|
|
@ -250,23 +255,32 @@ def extract_arg_value(cmd, arg):
|
|||
return cmd[param_index]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('arg_to_test, expected, test_patch, options_patch', [
|
||||
['-n', '1', {}, {'browser_cycles': None}],
|
||||
['-n', '123', {'browser_cycles': 123}, {}],
|
||||
['--video', 'false', {}, {'browsertime_video': None}],
|
||||
['--video', 'true', {}, {'browsertime_video': 'dummy_value'}],
|
||||
['--timeouts.script', str(DEFAULT_TIMEOUT), {}, {'page_cycles': None}],
|
||||
['--timeouts.script', str(123*DEFAULT_TIMEOUT), {'page_cycles': 123}, {}],
|
||||
['--browsertime.page_cycles', '1', {}, {'page_cycles': None}],
|
||||
['--browsertime.page_cycles', '123', {'page_cycles': 123}, {}],
|
||||
])
|
||||
def test_browsertime_arguments(ConcreteBrowsertime, browsertime_options,
|
||||
mock_test, arg_to_test, expected, test_patch, options_patch):
|
||||
@pytest.mark.parametrize(
|
||||
"arg_to_test, expected, test_patch, options_patch",
|
||||
[
|
||||
["-n", "1", {}, {"browser_cycles": None}],
|
||||
["-n", "123", {"browser_cycles": 123}, {}],
|
||||
["--video", "false", {}, {"browsertime_video": None}],
|
||||
["--video", "true", {}, {"browsertime_video": "dummy_value"}],
|
||||
["--timeouts.script", str(DEFAULT_TIMEOUT), {}, {"page_cycles": None}],
|
||||
["--timeouts.script", str(123 * DEFAULT_TIMEOUT), {"page_cycles": 123}, {}],
|
||||
["--browsertime.page_cycles", "1", {}, {"page_cycles": None}],
|
||||
["--browsertime.page_cycles", "123", {"page_cycles": 123}, {}],
|
||||
],
|
||||
)
|
||||
def test_browsertime_arguments(
|
||||
ConcreteBrowsertime,
|
||||
browsertime_options,
|
||||
mock_test,
|
||||
arg_to_test,
|
||||
expected,
|
||||
test_patch,
|
||||
options_patch,
|
||||
):
|
||||
mock_test.update(test_patch)
|
||||
browsertime_options.update(options_patch)
|
||||
browsertime = ConcreteBrowsertime(
|
||||
post_startup_delay=DEFAULT_TIMEOUT,
|
||||
**browsertime_options
|
||||
post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
|
||||
)
|
||||
browsertime.run_test_setup(mock_test)
|
||||
cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT)
|
||||
|
|
@ -275,37 +289,49 @@ def test_browsertime_arguments(ConcreteBrowsertime, browsertime_options,
|
|||
assert expected == param_value
|
||||
|
||||
|
||||
@pytest.mark.parametrize('timeout, expected_timeout, test_patch, options_patch', [
|
||||
[0, 20, {}, {}],
|
||||
[0, 20, {}, {'gecko_profile': False}],
|
||||
[1000, 321, {}, {'gecko_profile': True}],
|
||||
])
|
||||
def test_compute_process_timeout(ConcreteBrowsertime, browsertime_options,
|
||||
mock_test, timeout, expected_timeout, test_patch, options_patch):
|
||||
@pytest.mark.parametrize(
|
||||
"timeout, expected_timeout, test_patch, options_patch",
|
||||
[
|
||||
[0, 20, {}, {}],
|
||||
[0, 20, {}, {"gecko_profile": False}],
|
||||
[1000, 321, {}, {"gecko_profile": True}],
|
||||
],
|
||||
)
|
||||
def test_compute_process_timeout(
|
||||
ConcreteBrowsertime,
|
||||
browsertime_options,
|
||||
mock_test,
|
||||
timeout,
|
||||
expected_timeout,
|
||||
test_patch,
|
||||
options_patch,
|
||||
):
|
||||
mock_test.update(test_patch)
|
||||
browsertime_options.update(options_patch)
|
||||
browsertime = ConcreteBrowsertime(
|
||||
post_startup_delay=DEFAULT_TIMEOUT,
|
||||
**browsertime_options
|
||||
post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
|
||||
)
|
||||
bt_timeout = browsertime._compute_process_timeout(mock_test, timeout)
|
||||
assert bt_timeout == expected_timeout
|
||||
|
||||
|
||||
@pytest.mark.parametrize('host, playback, benchmark', [
|
||||
["127.0.0.1", True, False],
|
||||
["localhost", False, True],
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"host, playback, benchmark",
|
||||
[["127.0.0.1", True, False], ["localhost", False, True]],
|
||||
)
|
||||
def test_android_reverse_ports(host, playback, benchmark):
|
||||
raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example',
|
||||
host=host)
|
||||
raptor = WebExtensionAndroid(
|
||||
"geckoview", "org.mozilla.geckoview_example", host=host
|
||||
)
|
||||
if benchmark:
|
||||
benchmark_mock = mock.patch("raptor.raptor.benchmark.Benchmark")
|
||||
raptor.benchmark = benchmark_mock
|
||||
raptor.benchmark_port = 1234
|
||||
|
||||
if playback:
|
||||
playback_mock = mock.patch("mozbase.mozproxy.mozproxy.backends.mitm.mitm.MitmproxyAndroid")
|
||||
playback_mock = mock.patch(
|
||||
"mozbase.mozproxy.mozproxy.backends.mitm.mitm.MitmproxyAndroid"
|
||||
)
|
||||
playback_mock.port = 4321
|
||||
raptor.playback = playback_mock
|
||||
|
||||
|
|
@ -321,8 +347,9 @@ def test_android_reverse_ports(host, playback, benchmark):
|
|||
|
||||
|
||||
def test_android_reverse_ports_non_local_host():
|
||||
raptor = RaptorAndroid('geckoview', 'org.mozilla.geckoview_example',
|
||||
host="192.168.100.10")
|
||||
raptor = WebExtensionAndroid(
|
||||
"geckoview", "org.mozilla.geckoview_example", host="192.168.100.10"
|
||||
)
|
||||
|
||||
raptor.set_reverse_port = Mock()
|
||||
raptor.set_reverse_ports()
|
||||
|
|
@ -330,5 +357,5 @@ def test_android_reverse_ports_non_local_host():
|
|||
raptor.set_reverse_port.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
mozunit.main()
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче