зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1335873 - Convert marionette harness unittests to standard python unittests, r=maja_zf
This formats the marionette-harness python tests to be a regular |mach python-test| suite. Though we add subsuite=marionette, this is just for automation purposes. The new preferred way to run the marionette harness tests locally is: ./mach python-test testing/marionette They will also run if running the full suite. The mozbase packages.txt file modifies mozlog to use 'setup.py' instead of 'pth'. The reason for this is that the marionette-harness tests use the pytest_mozlog pytest plugin for formatting their results (converts pytest format into something resembling the standard tbpl logging format). In order for this plugin to get picked up however, mozlog's setup.py file needs to be processed. MozReview-Commit-ID: Ata99evHxbd --HG-- extra : rebase_source : 16ed70edd38a53c3279d8632d7cba3df4d5216c3
This commit is contained in:
Andrew Halberstadt
Родитель
640509a3a9
Коммит
0fa9f19bdf
|
|
@ -57,11 +57,6 @@ class MachCommands(MachCommandBase):
|
|||
default=False,
|
||||
action='store_true',
|
||||
help='Stop running tests after the first error or failure.')
|
||||
@CommandArgument('--path-only',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help=('Collect all tests under given path instead of default '
|
||||
'test resolution. Supports pytest-style tests.'))
|
||||
@CommandArgument('-j', '--jobs',
|
||||
default=1,
|
||||
type=int,
|
||||
|
|
@ -79,7 +74,6 @@ class MachCommands(MachCommandBase):
|
|||
test_objects=None,
|
||||
subsuite=None,
|
||||
verbose=False,
|
||||
path_only=False,
|
||||
stop=False,
|
||||
jobs=1):
|
||||
self._activate_virtualenv()
|
||||
|
|
@ -109,30 +103,19 @@ class MachCommands(MachCommandBase):
|
|||
# which produces output in the format Mozilla infrastructure expects.
|
||||
# Some tests are run via pytest.
|
||||
if test_objects is None:
|
||||
# If we're not being called from `mach test`, do our own
|
||||
# test resolution.
|
||||
if path_only:
|
||||
if tests:
|
||||
test_objects = [{'path': p} for p in find_tests_by_path()]
|
||||
else:
|
||||
self.log(logging.WARN, 'python-test', {},
|
||||
'TEST-UNEXPECTED-FAIL | No tests specified')
|
||||
test_objects = []
|
||||
from mozbuild.testing import TestResolver
|
||||
resolver = self._spawn(TestResolver)
|
||||
if tests:
|
||||
# If we were given test paths, try to find tests matching them.
|
||||
test_objects = resolver.resolve_tests(paths=tests,
|
||||
flavor='python')
|
||||
else:
|
||||
from mozbuild.testing import TestResolver
|
||||
resolver = self._spawn(TestResolver)
|
||||
if tests:
|
||||
# If we were given test paths, try to find tests matching them.
|
||||
test_objects = resolver.resolve_tests(paths=tests,
|
||||
flavor='python')
|
||||
else:
|
||||
# Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
|
||||
test_objects = resolver.resolve_tests(flavor='python')
|
||||
# Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
|
||||
test_objects = resolver.resolve_tests(flavor='python')
|
||||
|
||||
if not test_objects:
|
||||
message = 'TEST-UNEXPECTED-FAIL | No tests collected'
|
||||
if not path_only:
|
||||
message += ' (Not in PYTHON_UNITTEST_MANIFESTS? Try --path-only?)'
|
||||
message = 'TEST-UNEXPECTED-FAIL | No tests collected ' + \
|
||||
'(Not in PYTHON_UNITTEST_MANIFESTS?)'
|
||||
self.log(logging.WARN, 'python-test', {}, message)
|
||||
return 1
|
||||
|
||||
|
|
|
|||
|
|
@ -1,50 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst
|
||||
|
||||
implementation: taskgraph.task.transform:TransformTask
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.marionette_harness:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
# NOTE: this task should be refactored so that it is invoked as a job either
|
||||
# with a run.using of "mozharness", and combined with the source-check kind.
|
||||
|
||||
jobs:
|
||||
marionette-harness/opt:
|
||||
description: "Marionette harness unit test"
|
||||
attributes:
|
||||
build_platform: marionette-harness
|
||||
build_type: opt
|
||||
treeherder:
|
||||
platform: linux64/opt
|
||||
kind: test
|
||||
tier: 2
|
||||
symbol: tc(Mn-h)
|
||||
worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
docker-image: {in-tree: desktop-build} # NOTE: better to use the lint image
|
||||
env:
|
||||
JOB_SCRIPT: "taskcluster/scripts/tester/harness-test-linux.sh"
|
||||
MOZHARNESS_SCRIPT: "testing/mozharness/scripts/marionette_harness_tests.py"
|
||||
TOOLS_DISABLE: "true"
|
||||
artifacts:
|
||||
- name: public/logs/
|
||||
path: /home/worker/workspace/mozharness_workspace/upload/logs/
|
||||
type: directory
|
||||
command:
|
||||
- "bash"
|
||||
- "/home/worker/bin/build.sh"
|
||||
- "--tests=testing/marionette/harness/marionette_harness/tests/harness_unit"
|
||||
- "--work-dir=mozharness_workspace"
|
||||
max-run-time: 1800
|
||||
when:
|
||||
files-changed:
|
||||
- "testing/marionette/harness/**"
|
||||
- "testing/mozbase/mozlog/mozlog/pytest_mozlog/**"
|
||||
- "testing/mozharness/scripts/marionette_harness_tests.py"
|
||||
- "testing/config/marionette_harness_test_requirements.txt"
|
||||
|
|
@ -22,6 +22,35 @@ taskgraph-tests/opt:
|
|||
- 'config/mozunit.py'
|
||||
- 'python/mach/**/*.py'
|
||||
|
||||
marionette-harness/opt:
|
||||
description: testing/marionette/harness unit tests
|
||||
platforms:
|
||||
- linux64/opt
|
||||
treeherder:
|
||||
symbol: py(mnh)
|
||||
kind: test
|
||||
tier: 2
|
||||
worker-type:
|
||||
by-platform:
|
||||
linux64.*: aws-provisioner-v1/b2gtest
|
||||
worker:
|
||||
by-platform:
|
||||
linux64.*:
|
||||
implementation: docker-worker
|
||||
docker-image: {in-tree: "lint"}
|
||||
max-run-time: 3600
|
||||
run:
|
||||
using: mach
|
||||
mach: python-test --subsuite marionette-harness
|
||||
run-on-projects:
|
||||
- integration
|
||||
- release
|
||||
when:
|
||||
files-changed:
|
||||
- 'testing/marionette/harness/**'
|
||||
- 'testing/mozbase/mozlog/mozlog/pytest_mozlog/**'
|
||||
- 'python/mach_commands.py'
|
||||
|
||||
mozbase/opt:
|
||||
description: testing/mozbase unit tests
|
||||
platforms:
|
||||
|
|
|
|||
|
|
@ -90,11 +90,6 @@ spidermonkey
|
|||
Spidermonkey tasks check out the full gecko source tree, then compile only the
|
||||
spidermonkey portion. Each task runs specific tests after the build.
|
||||
|
||||
marionette-harness
|
||||
------------------
|
||||
|
||||
TBD (Maja)
|
||||
|
||||
Tests
|
||||
-----
|
||||
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
#! /bin/bash -vex
|
||||
|
||||
set -x -e
|
||||
|
||||
echo "running as" $(id)
|
||||
|
||||
####
|
||||
# Taskcluster friendly wrapper for running a script in
|
||||
# testing/mozharness/scripts in a source checkout (no build).
|
||||
# Example use: Python-only harness unit tests
|
||||
####
|
||||
|
||||
: WORKSPACE ${WORKSPACE:=/home/worker/workspace}
|
||||
: SRC_ROOT ${SRC_ROOT:=$WORKSPACE/build/src}
|
||||
# These paths should be relative to $SRC_ROOT
|
||||
: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT}
|
||||
: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG}
|
||||
: mozharness args "${@}"
|
||||
|
||||
set -v
|
||||
cd $WORKSPACE
|
||||
|
||||
fail() {
|
||||
echo # make sure error message is on a new line
|
||||
echo "[harness-test-linux.sh:error]" "${@}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
|
||||
|
||||
# support multiple, space delimited, config files
|
||||
config_cmds=""
|
||||
for cfg in $MOZHARNESS_CONFIG; do
|
||||
config_cmds="${config_cmds} --config-file ${SRC_ROOT}/${cfg}"
|
||||
done
|
||||
|
||||
python2.7 $SRC_ROOT/${MOZHARNESS_SCRIPT} ${config_cmds} "${@}"
|
||||
|
||||
|
||||
|
||||
|
|
@ -37,7 +37,6 @@ BUILD_KINDS = set([
|
|||
JOB_KINDS = set([
|
||||
'source-test',
|
||||
'toolchain',
|
||||
'marionette-harness',
|
||||
'android-stuff',
|
||||
])
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
-r mozbase_requirements.txt
|
||||
|
||||
# TODO - if we structure common.tests.zip to match the in-tree structure of the
|
||||
# testing directory, we could use ./marionette_requirements.txt instead
|
||||
../web-platform/tests/tools/wptserve
|
||||
../marionette/client
|
||||
../marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py
|
||||
../marionette/harness
|
||||
|
||||
# pytest
|
||||
../../python/py
|
||||
../../python/pytest
|
||||
../../python/mock-1.0.0
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
[DEFAULT]
|
||||
subsuite = marionette-harness
|
||||
|
||||
[test_httpd.py]
|
||||
[test_marionette_arguments.py]
|
||||
[test_marionette_harness.py]
|
||||
[test_marionette_runner.py]
|
||||
[test_marionette_test_result.py]
|
||||
[test_serve.py]
|
||||
|
|
@ -87,4 +87,5 @@ def test_handler(server):
|
|||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.exit(pytest.main(["--verbose", __file__]))
|
||||
sys.exit(pytest.main(
|
||||
['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
|
||||
|
|
|
|||
|
|
@ -29,4 +29,5 @@ def test_parse_arg_socket_timeout(socket_timeout):
|
|||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
sys.exit(pytest.main(['--verbose', __file__]))
|
||||
sys.exit(pytest.main(
|
||||
['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
|
||||
|
|
|
|||
|
|
@ -105,4 +105,5 @@ def test_harness_sets_up_default_test_handlers(mach_parsed_kwargs):
|
|||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
sys.exit(pytest.main(['--verbose', __file__]))
|
||||
sys.exit(pytest.main(
|
||||
['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
|
||||
|
|
|
|||
|
|
@ -439,4 +439,5 @@ def test_e10s_option_clash_raises(mock_runner):
|
|||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
sys.exit(pytest.main(['--verbose', __file__]))
|
||||
sys.exit(pytest.main(
|
||||
['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
|
||||
|
|
|
|||
|
|
@ -51,4 +51,5 @@ def test_crash_is_recorded_as_error(empty_marionette_test,
|
|||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
sys.exit(pytest.main(['--verbose', __file__]))
|
||||
sys.exit(pytest.main(
|
||||
['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
|
||||
|
|
|
|||
|
|
@ -64,4 +64,5 @@ def test_where_is():
|
|||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.exit(pytest.main(["-s", "--verbose", __file__]))
|
||||
sys.exit(pytest.main(
|
||||
['-s', '-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ mozhttpd.pth:testing/mozbase/mozhttpd
|
|||
mozinfo.pth:testing/mozbase/mozinfo
|
||||
mozinstall.pth:testing/mozbase/mozinstall
|
||||
mozleak.pth:testing/mozbase/mozleak
|
||||
mozlog.pth:testing/mozbase/mozlog
|
||||
setup.py:testing/mozbase/mozlog:develop
|
||||
moznetwork.pth:testing/mozbase/moznetwork
|
||||
mozprocess.pth:testing/mozbase/mozprocess
|
||||
mozprofile.pth:testing/mozbase/mozprofile
|
||||
|
|
|
|||
|
|
@ -1,141 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
import copy
|
||||
import os
|
||||
import sys
|
||||
|
||||
# load modules from parent dir
|
||||
sys.path.insert(1, os.path.dirname(sys.path[0]))
|
||||
|
||||
from mozharness.base.python import PreScriptAction
|
||||
from mozharness.base.python import (
|
||||
VirtualenvMixin,
|
||||
virtualenv_config_options,
|
||||
)
|
||||
from mozharness.base.script import BaseScript
|
||||
from mozharness.mozilla.buildbot import (
|
||||
BuildbotMixin, TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE,
|
||||
TBPL_EXCEPTION
|
||||
)
|
||||
|
||||
marionette_harness_tests_config_options = [
|
||||
[['--tests'], {
|
||||
'dest': 'test_path',
|
||||
'default': None,
|
||||
'help': 'Path to test_*.py or directory relative to src root.',
|
||||
}],
|
||||
[['--src-dir'], {
|
||||
'dest': 'rel_src_dir',
|
||||
'default': None,
|
||||
'help': 'Path to hg.mo source checkout relative to work dir.',
|
||||
}],
|
||||
|
||||
] + copy.deepcopy(virtualenv_config_options)
|
||||
|
||||
marionette_harness_tests_config = {
|
||||
"find_links": [
|
||||
"http://pypi.pub.build.mozilla.org/pub",
|
||||
],
|
||||
"pip_index": False,
|
||||
# relative to workspace
|
||||
"rel_src_dir": os.path.join("build", "src"),
|
||||
}
|
||||
|
||||
class MarionetteHarnessTests(VirtualenvMixin, BuildbotMixin, BaseScript):
|
||||
|
||||
def __init__(self, config_options=None,
|
||||
all_actions=None, default_actions=None,
|
||||
*args, **kwargs):
|
||||
config_options = config_options or marionette_harness_tests_config_options
|
||||
actions = [
|
||||
'clobber',
|
||||
'create-virtualenv',
|
||||
'run-tests',
|
||||
]
|
||||
super(MarionetteHarnessTests, self).__init__(
|
||||
config_options=config_options,
|
||||
all_actions=all_actions or actions,
|
||||
default_actions=default_actions or actions,
|
||||
config=marionette_harness_tests_config,
|
||||
*args, **kwargs)
|
||||
|
||||
@PreScriptAction('create-virtualenv')
|
||||
def _pre_create_virtualenv(self, action):
|
||||
dirs = self.query_abs_dirs()
|
||||
c = self.config
|
||||
requirements = os.path.join(
|
||||
dirs['abs_src_dir'],
|
||||
'testing', 'config',
|
||||
'marionette_harness_test_requirements.txt'
|
||||
)
|
||||
self.register_virtualenv_module(
|
||||
requirements=[requirements],
|
||||
two_pass=True
|
||||
)
|
||||
|
||||
def query_abs_dirs(self):
|
||||
if self.abs_dirs:
|
||||
return self.abs_dirs
|
||||
c = self.config
|
||||
abs_dirs = super(MarionetteHarnessTests, self).query_abs_dirs()
|
||||
dirs = {
|
||||
'abs_src_dir': os.path.abspath(
|
||||
os.path.join(abs_dirs['base_work_dir'], c['rel_src_dir'])
|
||||
),
|
||||
}
|
||||
|
||||
for key in dirs:
|
||||
if key not in abs_dirs:
|
||||
abs_dirs[key] = dirs[key]
|
||||
self.abs_dirs = abs_dirs
|
||||
|
||||
return self.abs_dirs
|
||||
|
||||
def _get_pytest_status(self, code):
|
||||
"""
|
||||
Translate pytest exit code to TH status
|
||||
|
||||
Based on https://github.com/pytest-dev/pytest/blob/master/_pytest/main.py#L21-L26
|
||||
"""
|
||||
if code == 0:
|
||||
return TBPL_SUCCESS
|
||||
elif code == 1:
|
||||
return TBPL_WARNING
|
||||
elif 1 < code < 6:
|
||||
self.error("pytest returned exit code: %s" % code)
|
||||
return TBPL_FAILURE
|
||||
else:
|
||||
return TBPL_EXCEPTION
|
||||
|
||||
def run_tests(self):
|
||||
"""Run all the tests"""
|
||||
dirs = self.query_abs_dirs()
|
||||
test_relpath = self.config.get(
|
||||
'test_path',
|
||||
os.path.join('testing', 'marionette',
|
||||
'harness', 'marionette_harness', 'tests',
|
||||
'harness_unit')
|
||||
)
|
||||
test_path = os.path.join(dirs['abs_src_dir'], test_relpath)
|
||||
self.activate_virtualenv()
|
||||
import pytest
|
||||
command = ['-p', 'no:terminalreporter', # disable pytest logging
|
||||
test_path]
|
||||
logs = {}
|
||||
for fmt in ['tbpl', 'mach', 'raw']:
|
||||
logs[fmt] = os.path.join(dirs['abs_log_dir'],
|
||||
'mn-harness_{}.log'.format(fmt))
|
||||
command.extend(['--log-'+fmt, logs[fmt]])
|
||||
self.info('Calling pytest.main with the following arguments: %s' % command)
|
||||
status = self._get_pytest_status(pytest.main(command))
|
||||
self.read_from_file(logs['tbpl'])
|
||||
for log in logs.values():
|
||||
self.copy_to_upload_dir(log, dest='logs/')
|
||||
self.buildbot_status(status)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
script = MarionetteHarnessTests()
|
||||
script.run_and_exit()
|
||||
Загрузка…
Ссылка в новой задаче