зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1632870 - [ci] Create a schema for test settings and associated test to validate it, r=jmaher
This also adds way to bypass the 'check_schema' call since this schema's keys contains strings from `variants.yml` which sometimes have underscoes in them. Differential Revision: https://phabricator.services.mozilla.com/D131282
This commit is contained in:
Родитель
1011b4fe5b
Коммит
f63dacad27
|
@ -125,3 +125,6 @@ Test ``settings`` are available in the ``task.extra.test-setting`` object in
|
|||
all test tasks. They are defined by the
|
||||
:py:func:`~gecko_taskgraph.transforms.tests.set_test_setting` transform
|
||||
function.
|
||||
|
||||
The full schema is defined in the
|
||||
:py:data:`~gecko_taskgraph.transforms.tests.test_setting_description_schema`.
|
||||
|
|
|
@ -1441,6 +1441,39 @@ def split_e10s(config, tasks):
|
|||
yield task
|
||||
|
||||
|
||||
test_setting_description_schema = Schema(
|
||||
{
|
||||
"platform": {
|
||||
Required("arch"): Any("32", "64", "aarch64", "arm7", "x86_64"),
|
||||
Required("os"): {
|
||||
Required("name"): Any("android", "linux", "macosx", "windows"),
|
||||
Required("version"): str,
|
||||
Optional("build"): str,
|
||||
},
|
||||
Optional("device"): str,
|
||||
Optional("machine"): Any("ref-hw-2017"),
|
||||
},
|
||||
"build": {
|
||||
Required("type"): Any("opt", "debug", "debug-isolated-process"),
|
||||
Any(
|
||||
"asan",
|
||||
"ccov",
|
||||
"clang-trunk",
|
||||
"devedition",
|
||||
"lite",
|
||||
"mingwclang",
|
||||
"shippable",
|
||||
"tsan",
|
||||
): bool,
|
||||
},
|
||||
"runtime": {Any(*list(TEST_VARIANTS.keys()) + ["1proc"]): bool},
|
||||
},
|
||||
check=False,
|
||||
)
|
||||
"""Schema test settings must conform to. Validated by
|
||||
:py:func:`~test.test_mozilla_central.test_test_setting`"""
|
||||
|
||||
|
||||
@transforms.add
|
||||
def set_test_setting(config, tasks):
|
||||
"""A test ``setting`` is the set of configuration that uniquely
|
||||
|
|
|
@ -209,13 +209,17 @@ class Schema(voluptuous.Schema):
|
|||
in the process.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
def __init__(self, *args, check=True, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
if not gecko_taskgraph.fast:
|
||||
|
||||
self.check = check
|
||||
if not gecko_taskgraph.fast and self.check:
|
||||
check_schema(self)
|
||||
|
||||
def extend(self, *args, **kwargs):
|
||||
schema = super().extend(*args, **kwargs)
|
||||
|
||||
if self.check:
|
||||
check_schema(schema)
|
||||
# We want twice extend schema to be checked too.
|
||||
schema.__class__ = Schema
|
||||
|
|
|
@ -35,5 +35,37 @@ def test_tasks_are_scheduled(optimized_task_graph, filter_tasks, func, min_expec
|
|||
assert len(tasks) >= min_expected
|
||||
|
||||
|
||||
def test_test_setting(full_task_graph, filter_tasks):
|
||||
"""Verify that all test tasks' ``test-setting`` object conforms to the schema."""
|
||||
from gecko_taskgraph.transforms.tests import test_setting_description_schema
|
||||
from gecko_taskgraph.util.schema import validate_schema
|
||||
|
||||
tasks = filter_tasks(full_task_graph, lambda t: t.kind == "test")
|
||||
|
||||
failures = []
|
||||
for task in tasks:
|
||||
try:
|
||||
validate_schema(
|
||||
test_setting_description_schema,
|
||||
task.task["extra"]["test-setting"],
|
||||
task.label,
|
||||
)
|
||||
except Exception as e:
|
||||
failures.append(e)
|
||||
|
||||
if failures:
|
||||
more = None
|
||||
# Only display a few failures at once.
|
||||
if len(failures) > 10:
|
||||
more = len(failures) - 10
|
||||
failures = failures[:10]
|
||||
|
||||
failstr = "\n\n" + "\n\n".join(str(e) for e in failures)
|
||||
if more:
|
||||
failstr += "\n\n" + f"... and {more} more"
|
||||
|
||||
pytest.fail(f"Task validation errors:{failstr}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
Загрузка…
Ссылка в новой задаче