From a5e0666632dc73ebf069d0464939cf0a78675319 Mon Sep 17 00:00:00 2001 From: Rob Wood Date: Fri, 28 Dec 2018 20:01:10 +0000 Subject: [PATCH] Bug 1491097 - Add manifest unit test; r=ahal Differential Revision: https://phabricator.services.mozilla.com/D13139 --HG-- extra : moz-landing-system : lando --- testing/raptor/test/conftest.py | 23 ++++ testing/raptor/test/python.ini | 1 + testing/raptor/test/test_manifest.py | 164 +++++++++++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 testing/raptor/test/test_manifest.py diff --git a/testing/raptor/test/conftest.py b/testing/raptor/test/conftest.py index 9df9f56f0656..694120f7a3e0 100644 --- a/testing/raptor/test/conftest.py +++ b/testing/raptor/test/conftest.py @@ -6,6 +6,8 @@ import sys import pytest +from argparse import Namespace + # need this so raptor imports work both from /raptor and via mach here = os.path.abspath(os.path.dirname(__file__)) if os.environ.get('SCRIPTSPATH', None) is not None: @@ -64,3 +66,24 @@ def get_binary(): return binary return inner + + +@pytest.fixture +def create_args(): + args = Namespace(app='firefox', + test='raptor-tp6-1', + binary='path/to/binary', + gecko_profile=False, + debug_mode=False, + page_cycles=None, + page_timeout=None, + host=None) + + def inner(**kwargs): + for next_arg in kwargs: + print next_arg + print kwargs[next_arg] + setattr(args, next_arg, kwargs[next_arg]) + return args + + return inner diff --git a/testing/raptor/test/python.ini b/testing/raptor/test/python.ini index 28abaca9a4de..64094a625f9c 100644 --- a/testing/raptor/test/python.ini +++ b/testing/raptor/test/python.ini @@ -3,6 +3,7 @@ subsuite = raptor skip-if = python == 3 [test_cmdline.py] +[test_manifest.py] [test_control_server.py] [test_playback.py] [test_raptor.py] diff --git a/testing/raptor/test/test_manifest.py b/testing/raptor/test/test_manifest.py new file mode 100644 index 000000000000..2b816f6deb1d --- /dev/null +++ b/testing/raptor/test/test_manifest.py @@ -0,0 +1,164 @@ +from __future__ import absolute_import, unicode_literals + +import mozinfo +import mozunit +import os +import pytest +import sys + +# need this so raptor imports work both from /raptor and via mach +here = os.path.abspath(os.path.dirname(__file__)) +if os.environ.get('SCRIPTSPATH', None) is not None: + # in production it is env SCRIPTS_PATH + mozharness_dir = os.environ['SCRIPTSPATH'] +else: + # locally it's in source tree + mozharness_dir = os.path.join(here, '../../mozharness') +sys.path.insert(1, mozharness_dir) + +raptor_dir = os.path.join(os.path.dirname(here), 'raptor') +sys.path.insert(0, raptor_dir) + +from manifest import get_browser_test_list, validate_test_ini, get_raptor_test_list + + +# some test details (test INIs) +VALID_MANIFESTS = [{'apps': 'firefox', + 'type': 'pageload', + 'page_cycles': 25, + 'test_url': 'http://www.test-url/goes/here', + 'measure': 'fnbpaint, dcf', + 'unit': 'ms', + 'lower_is_better': True, + 'alert_threshold': 2.0, + 'playback': 'mitmproxy', + 'playback_binary_manifest': 'binary.manifest', + 'playback_pageset_manifest': 'pageset.manifest', + 'playback_recordings': 'recorded_site.mp', + 'python3_win_manifest': 'py3.manifest', + 'manifest': 'valid_details_1'}, + {'apps': 'chrome', + 'type': 'benchmark', + 'page_cycles': 5, + 'test_url': 'http://www.test-url/goes/here', + 'unit': 'score', + 'lower_is_better': False, + 'alert_threshold': 2.0, + 'manifest': 'valid_details_2'}] + +INVALID_MANIFESTS = [{'apps': 'firefox', + 'type': 'pageload', + 'page_cycles': 25, + 'test_url': 'http://www.test-url/goes/here', + 'unit': 'ms', + 'lower_is_better': True, + 'alert_threshold': 2.0, + 'playback': 'mitmproxy', + 'playback_binary_manifest': 'binary.manifest', + 'playback_pageset_manifest': 'pageset.manifest', + 'playback_recordings': 'recorded_site.mp', + 'python3_win_manifest': 'py3.manifest', + 'manifest': 'invalid_details_1'}, + {'apps': 'chrome', + 'type': 'pageload', + 'page_cycles': 25, + 'test_url': 'http://www.test-url/goes/here', + 'measure': 'fnbpaint, dcf', + 'unit': 'ms', + 'lower_is_better': True, + 'alert_threshold': 2.0, + 'playback': 'mitmproxy', + 'manifest': 'invalid_details_2'}] + + +@pytest.mark.parametrize('app', ['firefox', 'chrome', 'geckoview']) +def test_get_browser_test_list(app): + test_list = get_browser_test_list(app) + assert len(test_list) > 0 + + +@pytest.mark.parametrize('test_details', VALID_MANIFESTS) +def test_validate_test_ini_valid(test_details): + assert(validate_test_ini(test_details)) + + +@pytest.mark.parametrize('test_details', INVALID_MANIFESTS) +def test_validate_test_ini_invalid(test_details): + assert not (validate_test_ini(test_details)) + + +def test_get_raptor_test_list_firefox(create_args): + args = create_args() + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 4 + + subtests = ['raptor-tp6-google-firefox', 'raptor-tp6-amazon-firefox', + 'raptor-tp6-facebook-firefox', 'raptor-tp6-youtube-firefox'] + + for next_subtest in test_list: + assert next_subtest['name'] in subtests + + +def test_get_raptor_test_list_chrome(create_args): + args = create_args(app="chrome", + test="raptor-speedometer") + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]['name'] == 'raptor-speedometer-chrome' + + +def test_get_raptor_test_list_geckoview(create_args): + args = create_args(app="geckoview", + test="raptor-unity-webgl") + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]['name'] == 'raptor-unity-webgl-geckoview' + + +def test_get_raptor_test_list_gecko_profiling(create_args): + args = create_args(test="raptor-tp6-google-firefox", + gecko_profile=True) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]['name'] == 'raptor-tp6-google-firefox' + assert test_list[0]['gecko_profile'] is True + assert test_list[0]['page_cycles'] == 3 + + +def test_get_raptor_test_list_debug_mode(create_args): + args = create_args(test="raptor-tp6-google-firefox", + debug_mode=True) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]['name'] == 'raptor-tp6-google-firefox' + assert test_list[0]['debug_mode'] is True + assert test_list[0]['page_cycles'] == 2 + + +def test_get_raptor_test_list_override_page_cycles(create_args): + args = create_args(test="raptor-tp6-google-firefox", + page_cycles=99) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]['name'] == 'raptor-tp6-google-firefox' + assert test_list[0]['page_cycles'] == 99 + + +def test_get_raptor_test_list_override_page_timeout(create_args): + args = create_args(test="raptor-tp6-google-firefox", + page_timeout=9999) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]['name'] == 'raptor-tp6-google-firefox' + assert test_list[0]['page_timeout'] == 9999 + + +if __name__ == '__main__': + mozunit.main()