зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1230279 - Develop new mozharness script for media-tests for Jenkins; r=maja_zf r=whimboo
--HG-- extra : transplant_source : %B7%DB%14%10c%C7%BE%83%B4%00%18%DB%D6%5D%0B%9F%F3%2A%D4%F4
This commit is contained in:
Syd Polk
Родитель
749cfba4bb
Коммит
dc9a970c40
|
|
@ -41,10 +41,7 @@ config = {
|
|||
|
||||
"firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
|
||||
"firefox_media_branch": 'master',
|
||||
"firefox_media_rev": '49b500b30b80372a6c678ec7d0a2b074844f5e84',
|
||||
"firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
|
||||
"firefox_ui_branch": 'mozilla-central',
|
||||
"firefox_ui_rev": '32be49d74e1d10c6bf087235b1d6753c1b840bc4',
|
||||
"firefox_media_rev": '0830e972e4b95fef3507207fc6bce028da27f2d3',
|
||||
|
||||
"suite_definitions": {
|
||||
"media-tests": {
|
||||
|
|
|
|||
|
|
@ -52,10 +52,7 @@ config = {
|
|||
|
||||
"firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git',
|
||||
"firefox_media_branch": 'master',
|
||||
"firefox_media_rev": '49b500b30b80372a6c678ec7d0a2b074844f5e84',
|
||||
"firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git',
|
||||
"firefox_ui_branch": 'mozilla-central',
|
||||
"firefox_ui_rev": '32be49d74e1d10c6bf087235b1d6753c1b840bc4',
|
||||
"firefox_media_rev": '0830e972e4b95fef3507207fc6bce028da27f2d3',
|
||||
|
||||
"suite_definitions": {
|
||||
"media-tests": {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
# Default configuration as used by Mozmill CI (Jenkins)
|
||||
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
|
||||
import mozharness
|
||||
|
||||
|
||||
external_tools_path = os.path.join(
|
||||
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
|
||||
'external_tools',
|
||||
)
|
||||
|
||||
config = {
|
||||
'env': {
|
||||
'PIP_TRUSTED_HOST': 'pypi.pub.build.mozilla.org',
|
||||
},
|
||||
|
||||
# PIP
|
||||
'find_links': ['http://pypi.pub.build.mozilla.org/pub'],
|
||||
'pip_index': False,
|
||||
|
||||
# mozcrash support
|
||||
'download_minidump_stackwalk': True,
|
||||
'download_symbols': 'ondemand',
|
||||
'download_tooltool': True,
|
||||
|
||||
# Version control information
|
||||
'firefox_media_repo': 'https://github.com/mjzffr/firefox-media-tests.git',
|
||||
'firefox_media_branch': 'master',
|
||||
|
||||
# Default test suite
|
||||
'test_suite': 'media-tests',
|
||||
|
||||
'suite_definitions': {
|
||||
'media-tests': {
|
||||
'options': [],
|
||||
},
|
||||
'media-youtube-tests': {
|
||||
'options': [
|
||||
'%(test_manifest)s'
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
'default_actions': [
|
||||
'clobber',
|
||||
'checkout',
|
||||
'download-and-extract',
|
||||
'create-virtualenv',
|
||||
'install',
|
||||
'run-media-tests',
|
||||
],
|
||||
|
||||
}
|
||||
|
||||
# General local variable overwrite
|
||||
# Bug 1227079 - Python executable eeded to get it executed on Windows
|
||||
if platform.system() == 'windows':
|
||||
gittool = [
|
||||
sys.executable,
|
||||
os.path.join(external_tools_path, 'gittool.py')
|
||||
]
|
||||
else:
|
||||
gittool = os.path.join(external_tools_path, 'gittool.py')
|
||||
|
||||
exes = {
|
||||
'gittool.py' : gittool,
|
||||
}
|
||||
|
||||
config['exes'] = exes
|
||||
|
|
@ -4,13 +4,12 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
# ***** BEGIN LICENSE BLOCK *****
|
||||
"""firefox_media_tests.py
|
||||
|
||||
Author: Maja Frydrychowicz
|
||||
"""
|
||||
import copy
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import urlparse
|
||||
|
||||
from mozharness.base.log import ERROR, WARNING
|
||||
from mozharness.base.script import PreScriptAction
|
||||
|
|
@ -72,23 +71,14 @@ media_test_config_options = [
|
|||
'dest': 'firefox_media_rev',
|
||||
'help': 'which firefox_media_tests revision to use',
|
||||
}],
|
||||
[['--firefox-ui-repo'], {
|
||||
'dest': 'firefox_ui_repo',
|
||||
'default': 'https://github.com/mozilla/firefox-ui-tests.git',
|
||||
'help': 'which firefox_ui_tests repo to use',
|
||||
}],
|
||||
[['--firefox-ui-branch'], {
|
||||
'dest': 'firefox_ui_branch',
|
||||
'default': 'master',
|
||||
'help': 'which branch to use for firefox_ui_tests',
|
||||
}],
|
||||
[['--firefox-ui-rev'], {
|
||||
'dest': 'firefox_ui_rev',
|
||||
'help': 'which firefox_ui_tests revision to use',
|
||||
}],
|
||||
[["--suite"],
|
||||
{"action": "store",
|
||||
"dest": "test_suite",
|
||||
"default": "media-tests",
|
||||
"help": "suite name",
|
||||
}],
|
||||
] + (copy.deepcopy(testing_config_options))
|
||||
|
||||
|
||||
class JobResultParser(TestSummaryOutputParserHelper):
|
||||
""" Parses test output to determine overall result."""
|
||||
def __init__(self, **kwargs):
|
||||
|
|
@ -138,7 +128,9 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
|
|||
actions = [
|
||||
'clobber',
|
||||
'checkout',
|
||||
'download-and-extract',
|
||||
'create-virtualenv',
|
||||
'install',
|
||||
'run-media-tests',
|
||||
]
|
||||
super(FirefoxMediaTestsBase, self).__init__(
|
||||
|
|
@ -148,23 +140,45 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
|
|||
**kwargs
|
||||
)
|
||||
c = self.config
|
||||
|
||||
self.media_urls = c.get('media_urls')
|
||||
self.profile = c.get('profile')
|
||||
self.test_timeout = int(c.get('test_timeout'))
|
||||
self.tests = c.get('tests')
|
||||
self.e10s = c.get('e10s')
|
||||
self.installer_url = c.get('installer_url')
|
||||
self.installer_path = c.get('installer_path')
|
||||
self.binary_path = c.get('binary_path')
|
||||
self.test_packages_url = c.get('test_packages_url')
|
||||
self.test_url = c.get('test_url')
|
||||
|
||||
@PreScriptAction('create-virtualenv')
|
||||
def _pre_create_virtualenv(self, action):
|
||||
dirs = self.query_abs_dirs()
|
||||
requirements_file = os.path.join(dirs['firefox_media_dir'],
|
||||
marionette_requirements = os.path.join(dirs['abs_test_install_dir'],
|
||||
'config',
|
||||
'marionette_requirements.txt')
|
||||
if os.access(marionette_requirements, os.F_OK):
|
||||
self.register_virtualenv_module(requirements=[marionette_requirements],
|
||||
two_pass=True)
|
||||
|
||||
media_tests_requirements = os.path.join(dirs['firefox_media_dir'],
|
||||
'requirements.txt')
|
||||
if os.path.isfile(requirements_file):
|
||||
self.register_virtualenv_module(requirements=[requirements_file])
|
||||
self.register_virtualenv_module(name='firefox-ui-tests',
|
||||
url=dirs['firefox_ui_dir'])
|
||||
self.register_virtualenv_module(name='firefox-media-tests',
|
||||
url=dirs['firefox_media_dir'])
|
||||
|
||||
if os.access(media_tests_requirements, os.F_OK):
|
||||
self.register_virtualenv_module(requirements=[media_tests_requirements],
|
||||
two_pass=True)
|
||||
|
||||
def download_and_extract(self):
|
||||
"""Overriding method from TestingMixin until firefox-media-tests are in tree.
|
||||
|
||||
Right now we only care about the installer and symbolds.
|
||||
|
||||
"""
|
||||
self._download_installer()
|
||||
|
||||
if self.config.get('download_symbols'):
|
||||
self._download_and_extract_symbols()
|
||||
|
||||
def query_abs_dirs(self):
|
||||
if self.abs_dirs:
|
||||
|
|
@ -174,8 +188,8 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
|
|||
'firefox_media_dir': os.path.join(abs_dirs['abs_work_dir'],
|
||||
'firefox-media-tests')
|
||||
}
|
||||
dirs['firefox_ui_dir'] = os.path.join(dirs['firefox_media_dir'],
|
||||
'firefox-ui-tests')
|
||||
dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'],
|
||||
'tests')
|
||||
abs_dirs.update(dirs)
|
||||
self.abs_dirs = abs_dirs
|
||||
return self.abs_dirs
|
||||
|
|
@ -188,20 +202,13 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
|
|||
self.firefox_media_vc = {
|
||||
'branch': c['firefox_media_branch'],
|
||||
'repo': c['firefox_media_repo'],
|
||||
'revision': c['firefox_media_rev'],
|
||||
'dest': dirs['firefox_media_dir'],
|
||||
}
|
||||
self.firefox_ui_vc = {
|
||||
'branch': c['firefox_ui_branch'],
|
||||
'repo': c['firefox_ui_repo'],
|
||||
'revision': c['firefox_ui_rev'],
|
||||
'dest': dirs['firefox_ui_dir']
|
||||
}
|
||||
if 'firefox-media-rev' in c:
|
||||
self.firefox_media_vc['revision'] = c['firefox_media_rev']
|
||||
|
||||
def checkout(self):
|
||||
revision = self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
|
||||
if revision:
|
||||
self.vcs_checkout(vcs='gittool', **self.firefox_ui_vc)
|
||||
self.vcs_checkout(vcs='gittool', **self.firefox_media_vc)
|
||||
|
||||
def _query_cmd(self):
|
||||
""" Determine how to call firefox-media-tests """
|
||||
|
|
@ -226,8 +233,48 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
|
|||
if self.e10s:
|
||||
cmd.append('--e10s')
|
||||
|
||||
test_suite = self.config.get('test_suite')
|
||||
if test_suite not in self.config["suite_definitions"]:
|
||||
self.fatal("%s is not defined in the config!" % test_suite)
|
||||
|
||||
test_manifest = None if test_suite != 'media-youtube-tests' else \
|
||||
os.path.join(dirs['firefox_media_dir'],
|
||||
'firefox_media_tests',
|
||||
'playback', 'youtube', 'manifest.ini')
|
||||
config_fmt_args = {
|
||||
'test_manifest': test_manifest,
|
||||
}
|
||||
|
||||
for s in self.config["suite_definitions"][test_suite]["options"]:
|
||||
cmd.append(s % config_fmt_args)
|
||||
|
||||
return cmd
|
||||
|
||||
def query_minidump_stackwalk(self):
|
||||
"""We don't have an extracted test package available to get the manifest file.
|
||||
|
||||
So we have to explicitely download the latest version of the manifest from the
|
||||
mozilla-central repository and feed it into the query_minidump_stackwalk() method.
|
||||
|
||||
We can remove this whole method once our tests are part of the tree.
|
||||
|
||||
"""
|
||||
manifest_path = None
|
||||
|
||||
if os.environ.get('MINIDUMP_STACKWALK') or self.config.get('download_minidump_stackwalk'):
|
||||
tooltool_manifest = self.query_minidump_tooltool_manifest()
|
||||
url_base = 'https://hg.mozilla.org/mozilla-central/raw-file/default/testing/'
|
||||
|
||||
dirs = self.query_abs_dirs()
|
||||
manifest_path = os.path.join(dirs['abs_work_dir'], 'releng.manifest')
|
||||
try:
|
||||
self.download_file(urlparse.urljoin(url_base, tooltool_manifest),
|
||||
manifest_path)
|
||||
except Exception as e:
|
||||
self.fatal('Download of tooltool manifest file failed: %s' % e.message)
|
||||
|
||||
return super(FirefoxMediaTestsBase, self).query_minidump_stackwalk(manifest=manifest_path)
|
||||
|
||||
def run_media_tests(self):
|
||||
cmd = self._query_cmd()
|
||||
self.job_result_parser = JobResultParser(
|
||||
|
|
@ -237,8 +284,7 @@ class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
|
|||
)
|
||||
|
||||
env = self.query_env()
|
||||
if (not os.environ.get('MINIDUMP_STACKWALK') and
|
||||
self.query_minidump_stackwalk()):
|
||||
if self.query_minidump_stackwalk():
|
||||
env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
|
||||
|
||||
return_code = self.run_command(
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ import sys
|
|||
|
||||
sys.path.insert(1, os.path.dirname(sys.path[0]))
|
||||
|
||||
from mozharness.base.log import ERROR, DEBUG, INFO
|
||||
from mozharness.base.script import PreScriptAction, PostScriptAction
|
||||
from mozharness.base.log import DEBUG, ERROR, INFO
|
||||
from mozharness.base.script import PostScriptAction
|
||||
from mozharness.mozilla.blob_upload import (
|
||||
BlobUploadMixin,
|
||||
blobupload_config_options
|
||||
|
|
@ -25,25 +25,16 @@ from mozharness.mozilla.buildbot import (
|
|||
TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
|
||||
)
|
||||
from mozharness.mozilla.testing.firefox_media_tests import (
|
||||
FirefoxMediaTestsBase, BUSTED, TESTFAILED, UNKNOWN, EXCEPTION, SUCCESS
|
||||
FirefoxMediaTestsBase, TESTFAILED, SUCCESS
|
||||
)
|
||||
|
||||
buildbot_media_test_options = [
|
||||
[["--suite"],
|
||||
{"action": "store",
|
||||
"dest": "test_suite",
|
||||
"default": "media-tests",
|
||||
"help": "suite name",
|
||||
}],
|
||||
]
|
||||
|
||||
|
||||
class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
|
||||
|
||||
def __init__(self):
|
||||
config_options = copy.deepcopy(blobupload_config_options)
|
||||
super(FirefoxMediaTestsBuildbot, self).__init__(
|
||||
config_options=config_options + buildbot_media_test_options,
|
||||
config_options=config_options,
|
||||
all_actions=['clobber',
|
||||
'read-buildbot-config',
|
||||
'checkout',
|
||||
|
|
@ -53,62 +44,6 @@ class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
|
|||
'run-media-tests',
|
||||
],
|
||||
)
|
||||
c = self.config
|
||||
self.installer_url = c.get('installer_url')
|
||||
self.installer_path = c.get('installer_path')
|
||||
self.binary_path = c.get('binary_path')
|
||||
self.test_packages_url = c.get('test_packages_url')
|
||||
|
||||
@PreScriptAction('create-virtualenv')
|
||||
def _pre_create_virtualenv(self, action):
|
||||
dirs = self.query_abs_dirs()
|
||||
requirements = os.path.join(dirs['abs_test_install_dir'],
|
||||
'config',
|
||||
'marionette_requirements.txt')
|
||||
if os.access(requirements, os.F_OK):
|
||||
self.register_virtualenv_module(requirements=[requirements],
|
||||
two_pass=True)
|
||||
super(FirefoxMediaTestsBuildbot, self)._pre_create_virtualenv(action)
|
||||
|
||||
def query_abs_dirs(self):
|
||||
if self.abs_dirs:
|
||||
return self.abs_dirs
|
||||
dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
|
||||
dirs['abs_blob_upload_dir'] = os.path.join(dirs['abs_work_dir'],
|
||||
'blobber_upload_dir')
|
||||
dirs['abs_test_install_dir'] = os.path.join(dirs['abs_work_dir'],
|
||||
'tests')
|
||||
self.abs_dirs = dirs
|
||||
return self.abs_dirs
|
||||
|
||||
def _query_cmd(self):
|
||||
""" Determine how to call firefox-media-tests """
|
||||
cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
|
||||
# configure logging
|
||||
dirs = self.query_abs_dirs()
|
||||
blob_upload_dir = dirs.get('abs_blob_upload_dir')
|
||||
cmd += ['--gecko-log', os.path.join(blob_upload_dir,
|
||||
'gecko.log')]
|
||||
cmd += ['--log-html', os.path.join(blob_upload_dir,
|
||||
'media_tests.html')]
|
||||
cmd += ['--log-mach', os.path.join(blob_upload_dir,
|
||||
'media_tests_mach.log')]
|
||||
|
||||
test_suite = self.config.get('test_suite')
|
||||
test_manifest = None if test_suite != 'media-youtube-tests' else \
|
||||
os.path.join(dirs['firefox_media_dir'],
|
||||
'firefox_media_tests',
|
||||
'playback', 'youtube', 'manifest.ini')
|
||||
config_fmt_args = {
|
||||
'test_manifest': test_manifest,
|
||||
}
|
||||
|
||||
if test_suite not in self.config["suite_definitions"]:
|
||||
self.fatal("%s is not defined in the config!" % test_suite)
|
||||
for s in self.config["suite_definitions"][test_suite]["options"]:
|
||||
cmd.append(s % config_fmt_args)
|
||||
|
||||
return cmd
|
||||
|
||||
def run_media_tests(self):
|
||||
status = super(FirefoxMediaTestsBuildbot, self).run_media_tests()
|
||||
|
|
@ -120,6 +55,28 @@ class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
|
|||
tbpl_status = TBPL_WARNING
|
||||
self.buildbot_status(tbpl_status)
|
||||
|
||||
def query_abs_dirs(self):
|
||||
if self.abs_dirs:
|
||||
return self.abs_dirs
|
||||
abs_dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
|
||||
dirs = {
|
||||
'abs_blob_upload_dir': os.path.join(abs_dirs['abs_work_dir'],
|
||||
'blobber_upload_dir')
|
||||
}
|
||||
abs_dirs.update(dirs)
|
||||
self.abs_dirs = abs_dirs
|
||||
return self.abs_dirs
|
||||
|
||||
def _query_cmd(self):
|
||||
cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
|
||||
dirs = self.query_abs_dirs()
|
||||
# configure logging
|
||||
blob_upload_dir = dirs.get('abs_blob_upload_dir')
|
||||
cmd += ['--gecko-log', os.path.join(blob_upload_dir, 'gecko.log')]
|
||||
cmd += ['--log-html', os.path.join(blob_upload_dir, 'media_tests.html')]
|
||||
cmd += ['--log-mach', os.path.join(blob_upload_dir, 'media_tests_mach.log')]
|
||||
return cmd
|
||||
|
||||
@PostScriptAction('run-media-tests')
|
||||
def _collect_uploads(self, action, success=None):
|
||||
""" Copy extra (log) files to blob upload dir. """
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
#!/usr/bin/env python
|
||||
# ***** BEGIN LICENSE BLOCK *****
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
# ***** BEGIN LICENSE BLOCK *****
|
||||
"""firefox_media_tests_jenkins.py
|
||||
|
||||
Author: Syd Polk
|
||||
"""
|
||||
import copy
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(1, os.path.dirname(sys.path[0]))
|
||||
|
||||
from mozharness.base.script import PreScriptAction
|
||||
from mozharness.mozilla.testing.firefox_media_tests import (
|
||||
FirefoxMediaTestsBase
|
||||
)
|
||||
|
||||
|
||||
class FirefoxMediaTestsJenkins(FirefoxMediaTestsBase):
|
||||
|
||||
def __init__(self):
|
||||
super(FirefoxMediaTestsJenkins, self).__init__(
|
||||
all_actions=['clobber',
|
||||
'checkout',
|
||||
'download-and-extract',
|
||||
'create-virtualenv',
|
||||
'install',
|
||||
'run-media-tests',
|
||||
],
|
||||
)
|
||||
|
||||
def _query_cmd(self):
|
||||
cmd = super(FirefoxMediaTestsJenkins, self)._query_cmd()
|
||||
|
||||
dirs = self.query_abs_dirs()
|
||||
|
||||
# configure logging
|
||||
log_dir = dirs.get('abs_log_dir')
|
||||
cmd += ['--gecko-log', os.path.join(log_dir, 'gecko.log')]
|
||||
cmd += ['--log-html', os.path.join(log_dir, 'media_tests.html')]
|
||||
cmd += ['--log-mach', os.path.join(log_dir, 'media_tests_mach.log')]
|
||||
|
||||
return cmd
|
||||
|
||||
if __name__ == '__main__':
|
||||
media_test = FirefoxMediaTestsJenkins()
|
||||
media_test.run_and_exit()
|
||||
Загрузка…
Ссылка в новой задаче