зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1455107 - Integrate raptor into mach; r=gbrown
MozReview-Commit-ID: 84vIqU2NWkE --HG-- extra : rebase_source : ac5454192271d19b4da448e6be2f97dbb3420828
This commit is contained in:
Rob Wood
Родитель
fa42a6873c
Коммит
8b76a9ff3c
|
|
@ -54,6 +54,7 @@ MACH_MODULES = [
|
|||
'testing/marionette/mach_commands.py',
|
||||
'testing/mochitest/mach_commands.py',
|
||||
'testing/mozharness/mach_commands.py',
|
||||
'testing/raptor/mach_commands.py',
|
||||
'testing/talos/mach_commands.py',
|
||||
'testing/web-platform/mach_commands.py',
|
||||
'testing/xpcshell/mach_commands.py',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
PYTHON = sys.executable
|
||||
VENV_PATH = '%s/build/venv' % os.getcwd()
|
||||
|
||||
TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
|
||||
MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
|
||||
|
||||
exes = {
|
||||
'python': PYTHON,
|
||||
}
|
||||
ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
|
||||
INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
|
||||
|
||||
config = {
|
||||
"log_name": "raptor",
|
||||
"buildbot_json_path": "buildprops.json",
|
||||
"installer_path": INSTALLER_PATH,
|
||||
"virtualenv_path": VENV_PATH,
|
||||
"find_links": [
|
||||
"http://pypi.pvt.build.mozilla.org/pub",
|
||||
"http://pypi.pub.build.mozilla.org/pub",
|
||||
],
|
||||
"pip_index": False,
|
||||
"exes": exes,
|
||||
"title": os.uname()[1].lower().split('.')[0],
|
||||
"default_actions": [
|
||||
"clobber",
|
||||
"read-buildbot-config",
|
||||
"download-and-extract",
|
||||
"populate-webroot",
|
||||
"create-virtualenv",
|
||||
"install",
|
||||
"run-tests",
|
||||
],
|
||||
"default_blob_upload_servers": [
|
||||
"https://blobupload.elasticbeanstalk.com",
|
||||
],
|
||||
"blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
|
||||
"download_minidump_stackwalk": True,
|
||||
"minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
|
||||
"minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
|
||||
"tooltool_cache": "/builds/worker/tooltool-cache",
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
import os
|
||||
import platform
|
||||
|
||||
VENV_PATH = '%s/build/venv' % os.getcwd()
|
||||
if platform.architecture()[0] == '64bit':
|
||||
TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
|
||||
MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
|
||||
else:
|
||||
TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
|
||||
MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
|
||||
|
||||
config = {
|
||||
"log_name": "raptor",
|
||||
"buildbot_json_path": "buildprops.json",
|
||||
"installer_path": "installer.exe",
|
||||
"virtualenv_path": VENV_PATH,
|
||||
"find_links": [
|
||||
"http://pypi.pvt.build.mozilla.org/pub",
|
||||
"http://pypi.pub.build.mozilla.org/pub",
|
||||
],
|
||||
"pip_index": False,
|
||||
"title": os.uname()[1].lower().split('.')[0],
|
||||
"default_actions": [
|
||||
"clobber",
|
||||
"read-buildbot-config",
|
||||
"download-and-extract",
|
||||
"populate-webroot",
|
||||
"create-virtualenv",
|
||||
"install",
|
||||
"setup-mitmproxy",
|
||||
"run-tests",
|
||||
],
|
||||
"default_blob_upload_servers": [
|
||||
"https://blobupload.elasticbeanstalk.com",
|
||||
],
|
||||
"blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
|
||||
"download_minidump_stackwalk": True,
|
||||
"minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
|
||||
"minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
|
||||
"tooltool_cache": "/builds/tooltool_cache",
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
ENABLE_SCREEN_RESOLUTION_CHECK = True
|
||||
|
||||
SCREEN_RESOLUTION_CHECK = {
|
||||
"name": "check_screen_resolution",
|
||||
"cmd": ["bash", "-c", "screenresolution get && screenresolution list && system_profiler SPDisplaysDataType"],
|
||||
"architectures": ["32bit", "64bit"],
|
||||
"halt_on_failure": False,
|
||||
"enabled": ENABLE_SCREEN_RESOLUTION_CHECK
|
||||
}
|
||||
|
||||
import os
|
||||
|
||||
VENV_PATH = '%s/build/venv' % os.getcwd()
|
||||
|
||||
config = {
|
||||
"log_name": "raptor",
|
||||
"buildbot_json_path": "buildprops.json",
|
||||
"installer_path": "installer.exe",
|
||||
"virtualenv_path": VENV_PATH,
|
||||
"find_links": [
|
||||
"http://pypi.pvt.build.mozilla.org/pub",
|
||||
"http://pypi.pub.build.mozilla.org/pub",
|
||||
],
|
||||
"pip_index": False,
|
||||
"title": os.uname()[1].lower().split('.')[0],
|
||||
"default_actions": [
|
||||
"clobber",
|
||||
"read-buildbot-config",
|
||||
"download-and-extract",
|
||||
"populate-webroot",
|
||||
"create-virtualenv",
|
||||
"install",
|
||||
"run-tests",
|
||||
],
|
||||
"run_cmd_checks_enabled": True,
|
||||
"preflight_run_cmd_suites": [
|
||||
SCREEN_RESOLUTION_CHECK,
|
||||
],
|
||||
"postflight_run_cmd_suites": [
|
||||
SCREEN_RESOLUTION_CHECK,
|
||||
],
|
||||
"default_blob_upload_servers": [
|
||||
"https://blobupload.elasticbeanstalk.com",
|
||||
],
|
||||
"blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
|
||||
"download_minidump_stackwalk": True,
|
||||
"minidump_stackwalk_path": "macosx64-minidump_stackwalk",
|
||||
"minidump_tooltool_manifest_path": "config/tooltool-manifests/macosx64/releng.manifest",
|
||||
"tooltool_cache": "/builds/tooltool_cache",
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
import os
|
||||
import socket
|
||||
import sys
|
||||
|
||||
PYTHON = sys.executable
|
||||
PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
|
||||
VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
|
||||
|
||||
config = {
|
||||
"log_name": "raptor",
|
||||
"buildbot_json_path": "buildprops.json",
|
||||
"installer_path": "installer.exe",
|
||||
"virtualenv_path": VENV_PATH,
|
||||
"pip_index": False,
|
||||
"find_links": [
|
||||
"http://pypi.pvt.build.mozilla.org/pub",
|
||||
"http://pypi.pub.build.mozilla.org/pub",
|
||||
],
|
||||
"virtualenv_modules": ['pywin32', 'raptor', 'mozinstall'],
|
||||
"exes": {
|
||||
'python': PYTHON,
|
||||
'easy_install': ['%s/scripts/python' % VENV_PATH,
|
||||
'%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
|
||||
'mozinstall': ['%s/scripts/python' % VENV_PATH,
|
||||
'%s/scripts/mozinstall-script.py' % VENV_PATH],
|
||||
'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
|
||||
'tooltool.py': [PYTHON, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
|
||||
},
|
||||
"title": socket.gethostname().split('.')[0],
|
||||
"default_actions": [
|
||||
"clobber",
|
||||
"read-buildbot-config",
|
||||
"download-and-extract",
|
||||
"populate-webroot",
|
||||
"create-virtualenv",
|
||||
"install",
|
||||
"run-tests",
|
||||
],
|
||||
"default_blob_upload_servers": [
|
||||
"https://blobupload.elasticbeanstalk.com",
|
||||
],
|
||||
"blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
|
||||
"metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
|
||||
"download_minidump_stackwalk": True,
|
||||
"tooltool_cache": os.path.join('c:\\', 'build', 'tooltool_cache'),
|
||||
"minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
|
||||
"minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
|
||||
"python3_manifest": {
|
||||
"win32": "python3.manifest",
|
||||
"win64": "python3_x64.manifest",
|
||||
},
|
||||
"env": {
|
||||
# python3 requires C runtime, found in firefox installation; see bug 1361732
|
||||
"PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
import os
|
||||
import socket
|
||||
import sys
|
||||
|
||||
PYTHON = sys.executable
|
||||
PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
|
||||
VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
|
||||
|
||||
config = {
|
||||
"log_name": "raptor",
|
||||
"buildbot_json_path": "buildprops.json",
|
||||
"installer_path": "installer.exe",
|
||||
"virtualenv_path": VENV_PATH,
|
||||
"pip_index": False,
|
||||
"find_links": [
|
||||
"http://pypi.pvt.build.mozilla.org/pub",
|
||||
"http://pypi.pub.build.mozilla.org/pub",
|
||||
],
|
||||
"virtualenv_modules": ['pywin32', 'raptor', 'mozinstall'],
|
||||
"exes": {
|
||||
'python': PYTHON,
|
||||
'easy_install': ['%s/scripts/python' % VENV_PATH,
|
||||
'%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
|
||||
'mozinstall': ['%s/scripts/python' % VENV_PATH,
|
||||
'%s/scripts/mozinstall-script.py' % VENV_PATH],
|
||||
'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
|
||||
},
|
||||
"title": socket.gethostname().split('.')[0],
|
||||
"default_actions": [
|
||||
"clobber",
|
||||
"read-buildbot-config",
|
||||
"download-and-extract",
|
||||
"populate-webroot",
|
||||
"create-virtualenv",
|
||||
"install",
|
||||
"run-tests",
|
||||
],
|
||||
"default_blob_upload_servers": [
|
||||
"https://blobupload.elasticbeanstalk.com",
|
||||
],
|
||||
"blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
|
||||
"metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
|
||||
"download_minidump_stackwalk": True,
|
||||
"tooltool_cache": os.path.join('c:\\', 'build', 'tooltool_cache'),
|
||||
"minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
|
||||
"minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
|
||||
"python3_manifest": {
|
||||
"win32": "python3.manifest",
|
||||
"win64": "python3_x64.manifest",
|
||||
},
|
||||
"env": {
|
||||
# python3 requires C runtime, found in firefox installation; see bug 1361732
|
||||
"PATH": "%(PATH)s;c:\\slave\\test\\build\\application\\firefox;"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,401 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
import mozharness
|
||||
|
||||
from mozharness.base.config import parse_config_file
|
||||
from mozharness.base.errors import PythonErrorList
|
||||
from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO, WARNING
|
||||
from mozharness.base.python import Python3Virtualenv
|
||||
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
|
||||
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
|
||||
from mozharness.mozilla.tooltool import TooltoolMixin
|
||||
from mozharness.base.vcs.vcsbase import MercurialScript
|
||||
from mozharness.mozilla.testing.codecoverage import (
|
||||
CodeCoverageMixin,
|
||||
code_coverage_config_options
|
||||
)
|
||||
|
||||
scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
|
||||
external_tools_path = os.path.join(scripts_path, 'external_tools')
|
||||
|
||||
RaptorErrorList = PythonErrorList + [
|
||||
{'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
|
||||
{'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
|
||||
{'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
|
||||
{'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
|
||||
{'substr': r'''raptorError''', 'level': CRITICAL},
|
||||
{'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
|
||||
{'substr': r"""No such file or directory: 'browser_output.txt'""",
|
||||
'level': CRITICAL,
|
||||
'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
|
||||
]
|
||||
|
||||
class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin):
|
||||
"""
|
||||
install and run raptor tests
|
||||
"""
|
||||
config_options = [
|
||||
[["--test"],
|
||||
{"action": "store",
|
||||
"dest": "test",
|
||||
"help": "Raptor test to run"
|
||||
}],
|
||||
[["--branch-name"],
|
||||
{"action": "store",
|
||||
"dest": "branch",
|
||||
"help": "branch running against"
|
||||
}],
|
||||
[["--add-option"],
|
||||
{"action": "extend",
|
||||
"dest": "raptor_extra_options",
|
||||
"default": None,
|
||||
"help": "extra options to raptor"
|
||||
}],
|
||||
] + testing_config_options + copy.deepcopy(blobupload_config_options) \
|
||||
+ copy.deepcopy(code_coverage_config_options)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
kwargs.setdefault('config_options', self.config_options)
|
||||
kwargs.setdefault('all_actions', ['clobber',
|
||||
'read-buildbot-config',
|
||||
'download-and-extract',
|
||||
'populate-webroot',
|
||||
'create-virtualenv',
|
||||
'install',
|
||||
'run-tests',
|
||||
])
|
||||
kwargs.setdefault('default_actions', ['clobber',
|
||||
'download-and-extract',
|
||||
'populate-webroot',
|
||||
'create-virtualenv',
|
||||
'install',
|
||||
'run-tests',
|
||||
])
|
||||
kwargs.setdefault('config', {})
|
||||
super(Raptor, self).__init__(**kwargs)
|
||||
|
||||
self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience
|
||||
|
||||
self.run_local = self.config.get('run_local')
|
||||
self.installer_url = self.config.get("installer_url")
|
||||
self.raptor_json_url = self.config.get("raptor_json_url")
|
||||
self.raptor_json = self.config.get("raptor_json")
|
||||
self.raptor_json_config = self.config.get("raptor_json_config")
|
||||
self.repo_path = self.config.get("repo_path")
|
||||
self.obj_path = self.config.get("obj_path")
|
||||
self.tests = None
|
||||
self.gecko_profile = self.config.get('gecko_profile')
|
||||
self.gecko_profile_interval = self.config.get('gecko_profile_interval')
|
||||
self.mitmproxy_rel_bin = None # some platforms download a mitmproxy release binary
|
||||
self.mitmproxy_pageset = None # zip file found on tooltool that contains all of the mitmproxy recordings
|
||||
self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) # files inside the recording set
|
||||
self.mitmdump = None # path to mitmdump tool itself, in py3 venv
|
||||
|
||||
# We accept some configuration options from the try commit message in the format mozharness: <options>
|
||||
# Example try commit message:
|
||||
# mozharness: --geckoProfile try: <stuff>
|
||||
def query_gecko_profile_options(self):
|
||||
gecko_results = []
|
||||
if self.buildbot_config:
|
||||
# this is inside automation
|
||||
# now let's see if we added GeckoProfile specs in the commit message
|
||||
try:
|
||||
junk, junk, opts = self.buildbot_config['sourcestamp']['changes'][-1]['comments'].partition('mozharness:')
|
||||
except IndexError:
|
||||
# when we don't have comments on changes (bug 1255187)
|
||||
opts = None
|
||||
|
||||
if opts:
|
||||
# In the case of a multi-line commit message, only examine
|
||||
# the first line for mozharness options
|
||||
opts = opts.split('\n')[0]
|
||||
opts = re.sub(r'\w+:.*', '', opts).strip().split(' ')
|
||||
if "--geckoProfile" in opts:
|
||||
# overwrite whatever was set here.
|
||||
self.gecko_profile = True
|
||||
try:
|
||||
idx = opts.index('--geckoProfileInterval')
|
||||
if len(opts) > idx + 1:
|
||||
self.gecko_profile_interval = opts[idx + 1]
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
# no opts, check for '--geckoProfile' in try message text directly
|
||||
if self.try_message_has_flag('geckoProfile'):
|
||||
self.gecko_profile = True
|
||||
|
||||
# finally, if gecko_profile is set, we add that to the raptor options
|
||||
if self.gecko_profile:
|
||||
gecko_results.append('--geckoProfile')
|
||||
if self.gecko_profile_interval:
|
||||
gecko_results.extend(
|
||||
['--geckoProfileInterval', str(self.gecko_profile_interval)]
|
||||
)
|
||||
return gecko_results
|
||||
|
||||
def query_abs_dirs(self):
|
||||
if self.abs_dirs:
|
||||
return self.abs_dirs
|
||||
abs_dirs = super(Raptor, self).query_abs_dirs()
|
||||
abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
|
||||
abs_dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
|
||||
self.abs_dirs = abs_dirs
|
||||
return self.abs_dirs
|
||||
|
||||
def raptor_options(self, args=None, **kw):
|
||||
"""return options to raptor"""
|
||||
# binary path
|
||||
binary_path = self.binary_path or self.config.get('binary_path')
|
||||
if not binary_path:
|
||||
self.fatal("Raptor requires a path to the binary. You can specify binary_path or add download-and-extract to your action list.")
|
||||
# raptor options
|
||||
if binary_path.endswith('.exe'):
|
||||
binary_path = binary_path[:-4]
|
||||
options = []
|
||||
kw_options = {'binary': binary_path}
|
||||
# options overwritten from **kw
|
||||
if 'suite' in self.config:
|
||||
kw_options['suite'] = self.config['suite']
|
||||
if self.config.get('branch'):
|
||||
kw_options['branchName'] = self.config['branch']
|
||||
if self.symbols_path:
|
||||
kw_options['symbolsPath'] = self.symbols_path
|
||||
kw_options.update(kw)
|
||||
# configure profiling options
|
||||
options.extend(self.query_gecko_profile_options())
|
||||
# extra arguments
|
||||
if args is not None:
|
||||
options += args
|
||||
if 'raptor_extra_options' in self.config:
|
||||
options += self.config['raptor_extra_options']
|
||||
if self.config.get('code_coverage', False):
|
||||
options.extend(['--code-coverage'])
|
||||
for key, value in kw_options.items():
|
||||
options.extend(['--%s' % key, value])
|
||||
return options
|
||||
|
||||
def populate_webroot(self):
|
||||
"""Populate the production test slaves' webroots"""
|
||||
self.raptor_path = os.path.join(
|
||||
self.query_abs_dirs()['abs_test_install_dir'], 'raptor'
|
||||
)
|
||||
|
||||
if self.config.get('run_local'):
|
||||
# raptor initiated locally, get and verify test from cmd line
|
||||
self.raptor_path = os.path.join(self.repo_path, 'testing', 'raptor')
|
||||
if 'raptor_extra_options' in self.config:
|
||||
if '--test' in self.config['raptor_extra_options']:
|
||||
# --test specified, get test from cmd line and ensure is valid
|
||||
test_name_index = self.config['raptor_extra_options'].index('--test') + 1
|
||||
if test_name_index < len(self.config['raptor_extra_options']):
|
||||
self.test = self.config['raptor_extra_options'][test_name_index]
|
||||
else:
|
||||
self.fatal("Test name not provided")
|
||||
else:
|
||||
# raptor initiated in production via mozharness
|
||||
self.test = self.config['test']
|
||||
|
||||
# Action methods. {{{1
|
||||
# clobber defined in BaseScript
|
||||
# read_buildbot_config defined in BuildbotMixin
|
||||
|
||||
def download_and_extract(self, extract_dirs=None, suite_categories=None):
|
||||
return super(Raptor, self).download_and_extract(
|
||||
suite_categories=['common', 'raptor']
|
||||
)
|
||||
|
||||
def create_virtualenv(self, **kwargs):
|
||||
"""VirtualenvMixin.create_virtualenv() assuemes we're using
|
||||
self.config['virtualenv_modules']. Since we are installing
|
||||
raptor from its source, we have to wrap that method here."""
|
||||
# if virtualenv already exists, just add to path and don't re-install, need it
|
||||
# in path so can import jsonschema later when validating output for perfherder
|
||||
_virtualenv_path = self.config.get("virtualenv_path")
|
||||
|
||||
if self.run_local and os.path.exists(_virtualenv_path):
|
||||
self.info("Virtualenv already exists, skipping creation")
|
||||
_python_interp = self.config.get('exes')['python']
|
||||
|
||||
if 'win' in self.platform_name():
|
||||
_path = os.path.join(_virtualenv_path,
|
||||
'Lib',
|
||||
'site-packages')
|
||||
else:
|
||||
_path = os.path.join(_virtualenv_path,
|
||||
'lib',
|
||||
os.path.basename(_python_interp),
|
||||
'site-packages')
|
||||
sys.path.append(_path)
|
||||
return
|
||||
|
||||
# virtualenv doesn't already exist so create it
|
||||
# install mozbase first, so we use in-tree versions
|
||||
if not self.run_local:
|
||||
mozbase_requirements = os.path.join(
|
||||
self.query_abs_dirs()['abs_test_install_dir'],
|
||||
'config',
|
||||
'mozbase_requirements.txt'
|
||||
)
|
||||
else:
|
||||
mozbase_requirements = os.path.join(
|
||||
os.path.dirname(self.raptor_path),
|
||||
'config',
|
||||
'mozbase_source_requirements.txt'
|
||||
)
|
||||
self.register_virtualenv_module(
|
||||
requirements=[mozbase_requirements],
|
||||
two_pass=True,
|
||||
editable=True,
|
||||
)
|
||||
# require pip >= 1.5 so pip will prefer .whl files to install
|
||||
super(Raptor, self).create_virtualenv(
|
||||
modules=['pip>=1.5']
|
||||
)
|
||||
# raptor in harness requires what else is
|
||||
# listed in raptor requirements.txt file.
|
||||
self.install_module(
|
||||
requirements=[os.path.join(self.raptor_path,
|
||||
'requirements.txt')]
|
||||
)
|
||||
|
||||
def _validate_treeherder_data(self, parser):
|
||||
# late import is required, because install is done in create_virtualenv
|
||||
import jsonschema
|
||||
|
||||
if len(parser.found_perf_data) != 1:
|
||||
self.critical("PERFHERDER_DATA was seen %d times, expected 1."
|
||||
% len(parser.found_perf_data))
|
||||
return
|
||||
|
||||
schema_path = os.path.join(external_tools_path,
|
||||
'performance-artifact-schema.json')
|
||||
self.info("Validating PERFHERDER_DATA against %s" % schema_path)
|
||||
try:
|
||||
with open(schema_path) as f:
|
||||
schema = json.load(f)
|
||||
data = json.loads(parser.found_perf_data[0])
|
||||
jsonschema.validate(data, schema)
|
||||
except:
|
||||
self.exception("Error while validating PERFHERDER_DATA")
|
||||
|
||||
def _artifact_perf_data(self, dest):
|
||||
src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'local.json')
|
||||
try:
|
||||
shutil.copyfile(src, dest)
|
||||
except:
|
||||
self.critical("Error copying results %s to upload dir %s" % (src, dest))
|
||||
|
||||
def run_tests(self, args=None, **kw):
|
||||
"""run raptor tests"""
|
||||
|
||||
# get raptor options
|
||||
options = self.raptor_options(args=args, **kw)
|
||||
|
||||
# python version check
|
||||
python = self.query_python_path()
|
||||
self.run_command([python, "--version"])
|
||||
parser = RaptorOutputParser(config=self.config, log_obj=self.log_obj,
|
||||
error_list=RaptorErrorList)
|
||||
env = {}
|
||||
env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
|
||||
if not self.run_local:
|
||||
env['MINIDUMP_STACKWALK'] = self.query_minidump_stackwalk()
|
||||
env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
|
||||
env['RUST_BACKTRACE'] = 'full'
|
||||
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
|
||||
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
|
||||
env = self.query_env(partial_env=env, log_level=INFO)
|
||||
# adjust PYTHONPATH to be able to use raptor as a python package
|
||||
if 'PYTHONPATH' in env:
|
||||
env['PYTHONPATH'] = self.raptor_path + os.pathsep + env['PYTHONPATH']
|
||||
else:
|
||||
env['PYTHONPATH'] = self.raptor_path
|
||||
|
||||
# mitmproxy needs path to mozharness when installing the cert
|
||||
env['SCRIPTSPATH'] = scripts_path
|
||||
|
||||
if self.repo_path is not None:
|
||||
env['MOZ_DEVELOPER_REPO_DIR'] = self.repo_path
|
||||
if self.obj_path is not None:
|
||||
env['MOZ_DEVELOPER_OBJ_DIR'] = self.obj_path
|
||||
|
||||
# sets a timeout for how long raptor should run without output
|
||||
output_timeout = self.config.get('raptor_output_timeout', 3600)
|
||||
# run raptor tests
|
||||
run_tests = os.path.join(self.raptor_path, 'raptor', 'raptor.py')
|
||||
|
||||
mozlog_opts = ['--log-tbpl-level=debug']
|
||||
if not self.run_local and 'suite' in self.config:
|
||||
fname_pattern = '%s_%%s.log' % self.config['test']
|
||||
mozlog_opts.append('--log-errorsummary=%s'
|
||||
% os.path.join(env['MOZ_UPLOAD_DIR'],
|
||||
fname_pattern % 'errorsummary'))
|
||||
mozlog_opts.append('--log-raw=%s'
|
||||
% os.path.join(env['MOZ_UPLOAD_DIR'],
|
||||
fname_pattern % 'raw'))
|
||||
|
||||
def launch_in_debug_mode(cmdline):
|
||||
cmdline = set(cmdline)
|
||||
debug_opts = {'--debug', '--debugger', '--debugger_args'}
|
||||
|
||||
return bool(debug_opts.intersection(cmdline))
|
||||
|
||||
command = [python, run_tests] + options + mozlog_opts
|
||||
if launch_in_debug_mode(command):
|
||||
raptor_process = subprocess.Popen(command, cwd=self.workdir, env=env)
|
||||
raptor_process.wait()
|
||||
else:
|
||||
self.return_code = self.run_command(command, cwd=self.workdir,
|
||||
output_timeout=output_timeout,
|
||||
output_parser=parser,
|
||||
env=env)
|
||||
if parser.minidump_output:
|
||||
self.info("Looking at the minidump files for debugging purposes...")
|
||||
for item in parser.minidump_output:
|
||||
self.run_command(["ls", "-l", item])
|
||||
|
||||
if self.return_code not in [0]:
|
||||
# update the worst log level
|
||||
log_level = ERROR
|
||||
if self.return_code == 1:
|
||||
log_level = WARNING
|
||||
if self.return_code == 4:
|
||||
log_level = WARNING
|
||||
|
||||
elif '--no-upload-results' not in options:
|
||||
if not self.gecko_profile:
|
||||
self._validate_treeherder_data(parser)
|
||||
if not self.run_local:
|
||||
# copy results to upload dir so they are included as an artifact
|
||||
dest = os.path.join(env['MOZ_UPLOAD_DIR'], 'perfherder-data.json')
|
||||
self._artifact_perf_data(dest)
|
||||
|
||||
|
||||
class RaptorOutputParser(OutputParser):
|
||||
minidump_regex = re.compile(r'''raptorError: "error executing: '(\S+) (\S+) (\S+)'"''')
|
||||
RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super(RaptorOutputParser, self).__init__(**kwargs)
|
||||
self.minidump_output = None
|
||||
self.found_perf_data = []
|
||||
|
||||
def parse_single_line(self, line):
|
||||
m = self.minidump_regex.search(line)
|
||||
if m:
|
||||
self.minidump_output = (m.group(1), m.group(2), m.group(3))
|
||||
|
||||
m = self.RE_PERF_DATA.match(line)
|
||||
if m:
|
||||
self.found_perf_data.append(m.group(1))
|
||||
super(RaptorOutputParser, self).parse_single_line(line)
|
||||
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""raptor
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
# load modules from parent dir
|
||||
sys.path.insert(1, os.path.dirname(sys.path[0]))
|
||||
|
||||
from mozharness.mozilla.testing.raptor import Raptor
|
||||
|
||||
if __name__ == '__main__':
|
||||
raptor = Raptor()
|
||||
raptor.run_and_exit()
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# Originally taken from /talos/mach_commands.py
|
||||
|
||||
# Integrates raptor mozharness with mach
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import socket
|
||||
|
||||
from mozbuild.base import MozbuildObject, MachCommandBase
|
||||
from mach.decorators import CommandProvider, Command
|
||||
|
||||
HERE = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
|
||||
class RaptorRunner(MozbuildObject):
|
||||
def run_test(self, raptor_args):
|
||||
"""
|
||||
We want to do couple of things before running raptor
|
||||
1. Clone mozharness
|
||||
2. Make config for raptor mozharness
|
||||
3. Run mozharness
|
||||
"""
|
||||
|
||||
self.init_variables(raptor_args)
|
||||
self.make_config()
|
||||
self.write_config()
|
||||
self.make_args()
|
||||
return self.run_mozharness()
|
||||
|
||||
def init_variables(self, raptor_args):
|
||||
self.raptor_dir = os.path.join(self.topsrcdir, 'testing', 'raptor')
|
||||
self.mozharness_dir = os.path.join(self.topsrcdir, 'testing',
|
||||
'mozharness')
|
||||
self.config_file_path = os.path.join(self._topobjdir, 'testing',
|
||||
'raptor-in_tree_conf.json')
|
||||
self.binary_path = self.get_binary_path()
|
||||
self.virtualenv_script = os.path.join(self.topsrcdir, 'third_party', 'python',
|
||||
'virtualenv', 'virtualenv.py')
|
||||
self.virtualenv_path = os.path.join(self._topobjdir, 'testing',
|
||||
'raptor-venv')
|
||||
self.python_interp = sys.executable
|
||||
self.raptor_args = raptor_args
|
||||
|
||||
def make_config(self):
|
||||
default_actions = ['populate-webroot', 'create-virtualenv', 'run-tests']
|
||||
self.config = {
|
||||
'run_local': True,
|
||||
'binary_path': self.binary_path,
|
||||
'repo_path': self.topsrcdir,
|
||||
'raptor_path': self.raptor_dir,
|
||||
'obj_path': self.topobjdir,
|
||||
'log_name': 'raptor',
|
||||
'virtualenv_path': self.virtualenv_path,
|
||||
'pypi_url': 'http://pypi.python.org/simple',
|
||||
'base_work_dir': self.mozharness_dir,
|
||||
'exes': {
|
||||
'python': self.python_interp,
|
||||
'virtualenv': [self.python_interp, self.virtualenv_script],
|
||||
},
|
||||
'title': socket.gethostname(),
|
||||
'default_actions': default_actions,
|
||||
'raptor_extra_options': self.raptor_args,
|
||||
'python3_manifest': {
|
||||
'win32': 'python3.manifest',
|
||||
'win64': 'python3_x64.manifest',
|
||||
}
|
||||
}
|
||||
|
||||
def make_args(self):
|
||||
self.args = {
|
||||
'config': {},
|
||||
'initial_config_file': self.config_file_path,
|
||||
}
|
||||
|
||||
def write_config(self):
|
||||
try:
|
||||
config_file = open(self.config_file_path, 'wb')
|
||||
config_file.write(json.dumps(self.config))
|
||||
config_file.close()
|
||||
except IOError as e:
|
||||
err_str = "Error writing to Raptor Mozharness config file {0}:{1}"
|
||||
print(err_str.format(self.config_file_path, str(e)))
|
||||
raise e
|
||||
|
||||
def run_mozharness(self):
|
||||
sys.path.insert(0, self.mozharness_dir)
|
||||
from mozharness.mozilla.testing.raptor import Raptor
|
||||
raptor_mh = Raptor(config=self.args['config'],
|
||||
initial_config_file=self.args['initial_config_file'])
|
||||
return raptor_mh.run()
|
||||
|
||||
|
||||
def create_parser():
|
||||
sys.path.insert(0, HERE) # allow to import the raptor package
|
||||
from raptor.cmdline import create_parser
|
||||
return create_parser(mach_interface=True)
|
||||
|
||||
|
||||
@CommandProvider
|
||||
class MachRaptor(MachCommandBase):
|
||||
@Command('raptor-test', category='testing',
|
||||
description='Run raptor performance tests.',
|
||||
parser=create_parser)
|
||||
def run_raptor_test(self, **kwargs):
|
||||
raptor = self._spawn(RaptorRunner)
|
||||
|
||||
try:
|
||||
return raptor.run_test(sys.argv[2:])
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
return 1
|
||||
|
|
@ -13,13 +13,17 @@ def create_parser(mach_interface=False):
|
|||
parser = argparse.ArgumentParser()
|
||||
add_arg = parser.add_argument
|
||||
|
||||
add_arg('-t', '--test', default=None, dest="test",
|
||||
if not mach_interface:
|
||||
add_arg('--app', default='firefox', dest='app',
|
||||
help="name of the application we are testing (default: firefox)",
|
||||
choices=['firefox', 'chrome'])
|
||||
add_arg('-b', '--binary', required=True, dest='binary',
|
||||
help="path to the browser executable that we are testing")
|
||||
|
||||
# remaining arg is test name
|
||||
add_arg("test",
|
||||
nargs="*",
|
||||
help="name of raptor test to run")
|
||||
add_arg('--app', default='firefox', dest='app',
|
||||
help="name of the application we are testing (default: firefox)",
|
||||
choices=['firefox', 'chrome'])
|
||||
add_arg('-b', '--binary', required=True,
|
||||
help="path to the browser executable that we are testing")
|
||||
|
||||
add_logging_group(parser)
|
||||
return parser
|
||||
|
|
|
|||
|
|
@ -2,13 +2,14 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# simple local server on port 8000, to demonstrate
|
||||
# receiving hero element timing results from a web extension
|
||||
# control server for raptor performance framework
|
||||
# communicates with the raptor browser webextension
|
||||
from __future__ import absolute_import
|
||||
|
||||
import BaseHTTPServer
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import threading
|
||||
|
||||
from mozlog import get_proxy_logger
|
||||
|
|
@ -69,11 +70,18 @@ class RaptorControlServer():
|
|||
self.raptor_venv = os.path.join(os.getcwd(), 'raptor-venv')
|
||||
self.server = None
|
||||
self._server_thread = None
|
||||
self.port = None
|
||||
|
||||
def start(self):
|
||||
config_dir = os.path.join(here, 'tests')
|
||||
os.chdir(config_dir)
|
||||
server_address = ('', 8000)
|
||||
|
||||
# pick a free port
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.bind(('', 0))
|
||||
self.port = sock.getsockname()[1]
|
||||
sock.close()
|
||||
server_address = ('', self.port)
|
||||
|
||||
server_class = BaseHTTPServer.HTTPServer
|
||||
handler_class = MyHandler
|
||||
|
|
@ -83,7 +91,7 @@ class RaptorControlServer():
|
|||
self._server_thread = threading.Thread(target=httpd.serve_forever)
|
||||
self._server_thread.setDaemon(True) # don't hang on exit
|
||||
self._server_thread.start()
|
||||
LOG.info("raptor control server running on port 8000...")
|
||||
LOG.info("raptor control server running on port %d..." % self.port)
|
||||
self.server = httpd
|
||||
|
||||
def stop(self):
|
||||
|
|
|
|||
|
|
@ -13,15 +13,15 @@ webext_dir = os.path.join(os.path.dirname(here), 'webext', 'raptor')
|
|||
LOG = get_proxy_logger(component="gen_test_url")
|
||||
|
||||
|
||||
def gen_test_config(browser, test):
|
||||
def gen_test_config(browser, test, cs_port):
|
||||
LOG.info("writing test settings url background js, so webext can get it")
|
||||
|
||||
data = """// this file is auto-generated by raptor, do not edit directly
|
||||
function getTestConfig() {
|
||||
return {"browser": "%s", "test_settings_url": "http://localhost:8000/%s.json"};
|
||||
return {"browser": "%s", "test_settings_url": "http://localhost:%d/%s.json"};
|
||||
}
|
||||
|
||||
""" % (browser, test)
|
||||
""" % (browser, cs_port, test)
|
||||
|
||||
webext_background_script = (os.path.join(webext_dir, "auto_gen_test_config.js"))
|
||||
|
||||
|
|
|
|||
|
|
@ -66,7 +66,8 @@ def get_raptor_test_list(args):
|
|||
# get a list of available raptor tests, for the browser we're testing on
|
||||
available_tests = get_browser_test_list(args.app)
|
||||
tests_to_run = []
|
||||
|
||||
# currently only support one test name on cmd line
|
||||
args.test = args.test[0]
|
||||
# if test name not provided on command line, run all available raptor tests for this browser;
|
||||
# if test name provided on command line, make sure it exists, and then only include that one
|
||||
if args.test is not None:
|
||||
|
|
|
|||
|
|
@ -16,15 +16,17 @@ from mozlog import commandline, get_default_logger
|
|||
from mozprofile import create_profile
|
||||
from mozrunner import runners
|
||||
|
||||
from raptor.cmdline import parse_args
|
||||
from raptor.control_server import RaptorControlServer
|
||||
from raptor.gen_test_config import gen_test_config
|
||||
from raptor.outputhandler import OutputHandler
|
||||
from raptor.playback import get_playback
|
||||
from raptor.manifest import get_raptor_test_list
|
||||
|
||||
# need this so raptor imports work both from /raptor and via mach
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
webext_dir = os.path.join(os.path.dirname(here), 'webext')
|
||||
sys.path.insert(0, here)
|
||||
|
||||
from cmdline import parse_args
|
||||
from control_server import RaptorControlServer
|
||||
from gen_test_config import gen_test_config
|
||||
from outputhandler import OutputHandler
|
||||
from playback import get_playback
|
||||
from manifest import get_raptor_test_list
|
||||
|
||||
|
||||
class Raptor(object):
|
||||
|
|
@ -79,7 +81,7 @@ class Raptor(object):
|
|||
|
||||
def run_test(self, test, timeout=None):
|
||||
self.log.info("starting raptor test: %s" % test['name'])
|
||||
gen_test_config(self.config['app'], test['name'])
|
||||
gen_test_config(self.config['app'], test['name'], self.control_server.port)
|
||||
|
||||
self.profile.addons.install(os.path.join(webext_dir, 'raptor'))
|
||||
|
||||
|
|
|
|||
|
|
@ -1,22 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# raptor tp7 chrome
|
||||
|
||||
[DEFAULT]
|
||||
apps = chrome
|
||||
type = pageload
|
||||
playback = mitmproxy
|
||||
release_bin_mac = mitmproxy-2.0.2-osx.tar.gz
|
||||
page_cycles = 25
|
||||
|
||||
[raptor-chrome-tp7]
|
||||
test_url = http://localhost:8081/heroes
|
||||
measure =
|
||||
fcp
|
||||
hero
|
||||
hero =
|
||||
mugshot
|
||||
title
|
||||
anime
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# raptor tp6 firefox
|
||||
# raptor tp6 on firefox
|
||||
|
||||
[DEFAULT]
|
||||
apps = firefox
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# raptor speedometer
|
||||
|
||||
[raptor-speedometer]
|
||||
apps =
|
||||
firefox
|
||||
chrome
|
||||
type = benchmark
|
||||
test_url = http://localhost:8081/Speedometer/index.html?raptor
|
||||
page_cycles = 1
|
||||
page_timeout = 120000
|
||||
|
|
@ -10,7 +10,6 @@ import pytest
|
|||
from mozprofile import BaseProfile
|
||||
from mozrunner.errors import RunnerNotStartedError
|
||||
|
||||
from raptor.control_server import RaptorControlServer
|
||||
from raptor.raptor import Raptor
|
||||
|
||||
|
||||
|
|
@ -36,12 +35,16 @@ def test_create_profile(options, app, get_prefs):
|
|||
|
||||
|
||||
def test_start_and_stop_server(raptor):
|
||||
print("*RW* control server is now:")
|
||||
print(str(raptor.control_server))
|
||||
assert raptor.control_server is None
|
||||
|
||||
raptor.start_control_server()
|
||||
assert isinstance(raptor.control_server, RaptorControlServer)
|
||||
|
||||
assert raptor.control_server._server_thread.is_alive()
|
||||
assert raptor.control_server.port is not None
|
||||
assert raptor.control_server.server is not None
|
||||
|
||||
raptor.clean_up()
|
||||
assert not raptor.control_server._server_thread.is_alive()
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче