зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1646813 - Part 2. write tests for taskgraph/utils/chunking.py r=ahal
Changes: - added tests that exercise manifest loading, mozinfo guessing and the overall process of chunking. - tests added for both web-platform and traditional mochitest/xpcshell suites. Differential Revision: https://phabricator.services.mozilla.com/D80985
This commit is contained in:
Edwin Takahashi
Родитель
4a20f4f8c0
Коммит
5e8e090932
|
|
@ -4,6 +4,8 @@
|
||||||
|
|
||||||
from __future__ import absolute_import, division, print_function, unicode_literals
|
from __future__ import absolute_import, division, print_function, unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
from itertools import combinations
|
from itertools import combinations
|
||||||
from six.moves import range
|
from six.moves import range
|
||||||
|
|
||||||
|
|
@ -60,6 +62,122 @@ def unchunked_manifests():
|
||||||
return inner
|
return inner
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='module')
|
||||||
|
def mock_task_definition():
|
||||||
|
"""Builds a mock task definition for use in testing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
platform (str): represents the build platform.
|
||||||
|
bits (int): software bits.
|
||||||
|
build_type (str): opt or debug.
|
||||||
|
suite (str): name of the unittest suite.
|
||||||
|
test_platform (str, optional): full name of the platform and major version.
|
||||||
|
variant (str, optional): specify fission or vanilla.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict: mocked task definition.
|
||||||
|
"""
|
||||||
|
def inner(platform, bits, build_type, suite, test_platform='', variant=''):
|
||||||
|
bits = str(bits)
|
||||||
|
test_variant = [suite, 'e10s']
|
||||||
|
if 'fis' in variant:
|
||||||
|
test_variant.insert(1, 'fis')
|
||||||
|
output = {
|
||||||
|
'build-attributes': {
|
||||||
|
'build_platform': platform + bits,
|
||||||
|
'build_type': build_type,
|
||||||
|
|
||||||
|
},
|
||||||
|
'attributes': {
|
||||||
|
'e10s': True,
|
||||||
|
'unittest_variant': variant
|
||||||
|
},
|
||||||
|
'test-name': '-'.join(test_variant),
|
||||||
|
'test-platform': ''.join([test_platform, '-', bits, '/', build_type])
|
||||||
|
}
|
||||||
|
return output
|
||||||
|
return inner
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='module')
|
||||||
|
def mock_mozinfo():
|
||||||
|
"""Returns a mocked mozinfo object, similar to guess_mozinfo_from_task().
|
||||||
|
|
||||||
|
Args:
|
||||||
|
os (str): typically one of 'win, linux, mac, android'.
|
||||||
|
processor (str): processor architecture.
|
||||||
|
asan (bool, optional): addressanitizer build.
|
||||||
|
bits (int, optional): defaults to 64.
|
||||||
|
ccov (bool, optional): code coverage build.
|
||||||
|
debug (bool, optional): debug type build.
|
||||||
|
fission (bool, optional): process fission.
|
||||||
|
headless (bool, optional): headless browser testing without displays.
|
||||||
|
tsan (bool, optional): threadsanitizer build.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict: Dictionary mimickign the results from guess_mozinfo_from_task.
|
||||||
|
"""
|
||||||
|
def inner(os, processor, asan=False, bits=64, ccov=False, debug=False,
|
||||||
|
fission=False, headless=False, tsan=False):
|
||||||
|
return {
|
||||||
|
'os': os,
|
||||||
|
'processor': processor,
|
||||||
|
'toolkit': '',
|
||||||
|
'asan': asan,
|
||||||
|
'bits': bits,
|
||||||
|
'ccov': ccov,
|
||||||
|
'debug': debug,
|
||||||
|
'e10s': True,
|
||||||
|
'fission': fission,
|
||||||
|
'headless': headless,
|
||||||
|
'tsan': tsan,
|
||||||
|
'webrender': False,
|
||||||
|
'appname': 'firefox',
|
||||||
|
}
|
||||||
|
return inner
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('params,exception', [
|
||||||
|
[('win', 32, 'opt', 'web-platform-tests', '', ''), None],
|
||||||
|
[('win', 64, 'opt', 'web-platform-tests', '', ''), None],
|
||||||
|
[('linux', 64, 'debug', 'mochitest-plain', '', ''), None],
|
||||||
|
[('mac', 64, 'debug', 'mochitest-plain', '', ''), None],
|
||||||
|
[('mac', 64, 'opt', 'mochitest-plain-headless', '', ''), None],
|
||||||
|
[('android', 64, 'debug', 'xpcshell', '', ''), None],
|
||||||
|
[('and', 64, 'debug', 'awsy', '', ''), ValueError],
|
||||||
|
[('', 64, 'opt', '', '', ''), ValueError],
|
||||||
|
[('linux-ccov', 64, 'opt', '', '', ''), None],
|
||||||
|
[('linux-asan', 64, 'opt', '', '', ''), None],
|
||||||
|
[('win-tsan', 64, 'opt', '', '', ''), None],
|
||||||
|
[('mac-ccov', 64, 'opt', '', '', ''), None],
|
||||||
|
[('android', 64, 'opt', 'crashtest', 'arm64', 'fission'), None],
|
||||||
|
[('win-aarch64', 64, 'opt', 'crashtest', '', ''), None],
|
||||||
|
])
|
||||||
|
def test_guess_mozinfo_from_task(params, exception, mock_task_definition):
|
||||||
|
"""Tests the mozinfo guessing process.
|
||||||
|
"""
|
||||||
|
# Set up a mocked task object.
|
||||||
|
task = mock_task_definition(*params)
|
||||||
|
|
||||||
|
if exception:
|
||||||
|
with pytest.raises(exception):
|
||||||
|
result = chunking.guess_mozinfo_from_task(task)
|
||||||
|
else:
|
||||||
|
result = chunking.guess_mozinfo_from_task(task)
|
||||||
|
|
||||||
|
assert result['bits'] == (32 if '32' in task['test-platform'] else 64)
|
||||||
|
assert result['os'] in ('android', 'linux', 'mac', 'win')
|
||||||
|
|
||||||
|
# Ensure the outcome of special build variants being present match what
|
||||||
|
# guess_mozinfo_from_task method returns for these attributes.
|
||||||
|
assert ('asan' in task['build-attributes']['build_platform']) == result['asan']
|
||||||
|
assert ('tsan' in task['build-attributes']['build_platform']) == result['tsan']
|
||||||
|
assert ('ccov' in task['build-attributes']['build_platform']) == result['ccov']
|
||||||
|
|
||||||
|
assert result['fission'] == any(task['attributes']['unittest_variant'])
|
||||||
|
assert result['e10s']
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('platform', ['unix', 'windows', 'android'])
|
@pytest.mark.parametrize('platform', ['unix', 'windows', 'android'])
|
||||||
@pytest.mark.parametrize('suite', ['crashtest', 'reftest', 'web-platform-tests', 'xpcshell'])
|
@pytest.mark.parametrize('suite', ['crashtest', 'reftest', 'web-platform-tests', 'xpcshell'])
|
||||||
def test_get_runtimes(platform, suite):
|
def test_get_runtimes(platform, suite):
|
||||||
|
|
@ -68,29 +186,28 @@ def test_get_runtimes(platform, suite):
|
||||||
assert chunking.get_runtimes(platform, suite)
|
assert chunking.get_runtimes(platform, suite)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('test_cases', [
|
@pytest.mark.parametrize('platform,suite,exception', [
|
||||||
('nonexistent_platform', 'nonexistent_suite', KeyError),
|
('nonexistent_platform', 'nonexistent_suite', KeyError),
|
||||||
('unix', 'nonexistent_suite', KeyError),
|
('unix', 'nonexistent_suite', KeyError),
|
||||||
('unix', '', TypeError),
|
('unix', '', TypeError),
|
||||||
('', '', TypeError),
|
('', '', TypeError),
|
||||||
('', 'nonexistent_suite', TypeError)
|
('', 'nonexistent_suite', TypeError),
|
||||||
])
|
])
|
||||||
def test_get_runtimes_invalid(test_cases):
|
def test_get_runtimes_invalid(platform, suite, exception):
|
||||||
"""Ensure get_runtimes() method raises an exception if improper request is made.
|
"""Ensure get_runtimes() method raises an exception if improper request is made.
|
||||||
"""
|
"""
|
||||||
platform = test_cases[0]
|
with pytest.raises(exception):
|
||||||
suite = test_cases[1]
|
|
||||||
expected = test_cases[2]
|
|
||||||
|
|
||||||
try:
|
|
||||||
chunking.get_runtimes(platform, suite)
|
chunking.get_runtimes(platform, suite)
|
||||||
except Exception as e:
|
|
||||||
assert isinstance(e, expected)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('suite', ['web-platform-tests', 'web-platform-tests-reftests'])
|
@pytest.mark.parametrize('suite', [
|
||||||
|
'web-platform-tests',
|
||||||
|
'web-platform-tests-reftest',
|
||||||
|
'web-platform-tests-wdspec',
|
||||||
|
'web-platform-tests-crashtest',
|
||||||
|
])
|
||||||
@pytest.mark.parametrize('chunks', [1, 3, 6, 20])
|
@pytest.mark.parametrize('chunks', [1, 3, 6, 20])
|
||||||
def test_chunk_manifests_wpt(mock_manifest_runtimes, unchunked_manifests, suite, chunks):
|
def test_mock_chunk_manifests_wpt(unchunked_manifests, suite, chunks):
|
||||||
"""Tests web-platform-tests and its subsuites chunking process.
|
"""Tests web-platform-tests and its subsuites chunking process.
|
||||||
"""
|
"""
|
||||||
# Setup.
|
# Setup.
|
||||||
|
|
@ -123,5 +240,106 @@ def test_chunk_manifests_wpt(mock_manifest_runtimes, unchunked_manifests, suite,
|
||||||
assert expected == chunked_manifests
|
assert expected == chunked_manifests
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('suite', [
|
||||||
|
'mochitest-devtools-chrome',
|
||||||
|
'mochitest-browser-chrome',
|
||||||
|
'mochitest-plain',
|
||||||
|
'mochitest-chrome',
|
||||||
|
'xpcshell',
|
||||||
|
])
|
||||||
|
@pytest.mark.parametrize('chunks', [1, 3, 6, 20])
|
||||||
|
def test_mock_chunk_manifests(mock_manifest_runtimes, unchunked_manifests, suite, chunks):
|
||||||
|
"""Tests non-WPT tests and its subsuites chunking process.
|
||||||
|
"""
|
||||||
|
# Setup.
|
||||||
|
manifests = unchunked_manifests(suite)
|
||||||
|
|
||||||
|
# Call the method under test on unchunked manifests.
|
||||||
|
chunked_manifests = chunking.chunk_manifests(suite, 'unix', chunks, manifests)
|
||||||
|
|
||||||
|
# Assertions and end test.
|
||||||
|
assert chunked_manifests
|
||||||
|
if chunks > len(manifests):
|
||||||
|
# If chunk count exceeds number of manifests, not all chunks will have
|
||||||
|
# manifests.
|
||||||
|
with pytest.raises(AssertionError):
|
||||||
|
assert all(chunked_manifests)
|
||||||
|
else:
|
||||||
|
assert all(chunked_manifests)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('suite', [
|
||||||
|
'web-platform-tests',
|
||||||
|
'web-platform-tests-reftest',
|
||||||
|
'xpcshell',
|
||||||
|
'mochitest-plain',
|
||||||
|
'mochitest-devtools-chrome',
|
||||||
|
'mochitest-browser-chrome',
|
||||||
|
'mochitest-chrome',
|
||||||
|
])
|
||||||
|
@pytest.mark.parametrize('platform', [
|
||||||
|
('mac', 'x86_64'),
|
||||||
|
('win', 'x86_64'),
|
||||||
|
('win', 'x86'),
|
||||||
|
('win', 'aarch64'),
|
||||||
|
('linux', 'x86_64'),
|
||||||
|
('linux', 'x86'),
|
||||||
|
])
|
||||||
|
def test_get_manifests(suite, platform, mock_mozinfo):
|
||||||
|
"""Tests the DefaultLoader class' ability to load manifests.
|
||||||
|
"""
|
||||||
|
mozinfo = mock_mozinfo(*platform)
|
||||||
|
|
||||||
|
loader = chunking.DefaultLoader([])
|
||||||
|
manifests = loader.get_manifests(suite, frozenset(mozinfo.items()))
|
||||||
|
|
||||||
|
assert manifests
|
||||||
|
assert manifests['active']
|
||||||
|
if 'web-platform' in suite:
|
||||||
|
assert manifests['skipped'] == []
|
||||||
|
else:
|
||||||
|
assert manifests['skipped']
|
||||||
|
|
||||||
|
items = manifests['active']
|
||||||
|
if suite == 'xpcshell':
|
||||||
|
assert all([re.search(r'xpcshell(.*)?.ini', m) for m in items])
|
||||||
|
if 'mochitest' in suite:
|
||||||
|
assert all([re.search(r'(mochitest|chrome|browser).*.ini', m) for m in items])
|
||||||
|
if 'web-platform' in suite:
|
||||||
|
assert all([m.startswith('/') and m.count('/') <= 4 for m in items])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('suite', [
|
||||||
|
'mochitest-devtools-chrome',
|
||||||
|
'mochitest-browser-chrome',
|
||||||
|
'mochitest-plain',
|
||||||
|
'mochitest-chrome',
|
||||||
|
'web-platform-tests',
|
||||||
|
'web-platform-tests-reftest',
|
||||||
|
'xpcshell',
|
||||||
|
])
|
||||||
|
@pytest.mark.parametrize('platform', [
|
||||||
|
('mac', 'x86_64'),
|
||||||
|
('win', 'x86_64'),
|
||||||
|
('linux', 'x86_64'),
|
||||||
|
])
|
||||||
|
@pytest.mark.parametrize('chunks', [1, 3, 6, 20])
|
||||||
|
def test_chunk_manifests(suite, platform, chunks, mock_mozinfo):
|
||||||
|
"""Tests chunking with real manifests.
|
||||||
|
"""
|
||||||
|
mozinfo = mock_mozinfo(*platform)
|
||||||
|
|
||||||
|
loader = chunking.DefaultLoader([])
|
||||||
|
manifests = loader.get_manifests(suite, frozenset(mozinfo.items()))
|
||||||
|
|
||||||
|
chunked_manifests = chunking.chunk_manifests(suite, platform, chunks,
|
||||||
|
manifests['active'])
|
||||||
|
|
||||||
|
# Assertions and end test.
|
||||||
|
assert chunked_manifests
|
||||||
|
assert len(chunked_manifests) == chunks
|
||||||
|
assert all(chunked_manifests)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
||||||
|
|
|
||||||
Загрузка…
Ссылка в новой задаче