Bug 1621095 - add reftest-qr to run focused webrender reftests on android phones. r=bc

add reftest-qr to run focused webrender reftests on android phones. Differential Revision: https://phabricator.services.mozilla.com/D72312
2020-04-24 12:35:12 +00:00 · 2020-04-24 12:35:12 +00:00 · b9a6e9d31c
--- a/layout/reftests/reftest-qr.list
+++ b/layout/reftests/reftest-qr.list
@ -0,0 +1,26 @@
+# tests that are more likely to be affected by WebRender changes
+# as per bug 1621095 this is created to reduce load on android phones
+
+# gfx
+include ../../gfx/tests/reftest/reftest.list
+
+include async-scrolling/reftest.list
+
+# bugs/
+include bugs/reftest.list
+
+# css animations
+include css-animations/reftest.list
+
+# invalidation
+include invalidation/reftest.list
+
+# sticky positioning
+include position-sticky/reftest.list
+
+# svg/
+include svg/reftest.list
+
+# 3d transforms
+include transform-3d/reftest.list
+
--- a/moz.build
+++ b/moz.build
@ -78,6 +78,10 @@ with Files("**/reftest.list"):
    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
    SCHEDULES.exclusive = ['reftest']

+with Files("**/reftest-qr.list"):
+    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
+    SCHEDULES.exclusive = ['reftest']
+
 with Files("**/crashtest*.list"):
    SCHEDULES.inclusive += ['test-verify', 'test-verify-gpu']
    SCHEDULES.exclusive = ['crashtest']
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@ -461,6 +461,7 @@ ARCHIVE_FILES = {
            'base': '',
            'manifests': [
                'layout/reftests/reftest.list',
+                'layout/reftests/reftest-qr.list',
                'testing/crashtest/crashtests.list',
            ],
            'dest': 'reftest/tests',
--- a/taskcluster/ci/test/reftest.yml
+++ b/taskcluster/ci/test/reftest.yml
@ -146,6 +146,18 @@ reftest:
            linux1804-64-qr/debug: ['trunk']
            default: []

+reftest-qr:
+    description: "Reftest webrender run"
+    treeherder-symbol: R(R)
+    chunks: 2
+    e10s: true
+    run-on-projects: built-projects
+    max-run-time: 2400
+    mozharness:
+        chunked: true
+    tier: 2
+
+
 reftest-gpu:
    description: "Reftest GPU run"
    treeherder-symbol: R(Rg)
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@ -604,7 +604,7 @@ android-hw-aarch64-debug-unittests:

 android-hw-aarch64-qr:
    - crashtest
-    - reftest
+    - reftest-qr

 android-hw-arm7-raptor:
    - raptor-speedometer-geckoview
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@ -1280,6 +1280,7 @@ CHUNK_SUITES_BLACKLIST = (
    'mochitest-webgl2-ext',
    'raptor',
    'reftest',
+    'reftest-qr',
    'reftest-gpu',
    'reftest-no-accel',
    'talos',
@ -1344,10 +1345,11 @@ def split_chunks(config, tests):
            if chunked_manifests is not None:
                manifests = sorted(chunked_manifests[i])
                if not manifests:
+                    print(chunked_manifests)
                    raise Exception(
                        'Chunking algorithm yielded no manifests for chunk {} of {} on {}'.format(
                            this_chunk, test['test-name'], test['test-platform']))
-                chunked['test-manifests'] = manifests
+                    chunked['test-manifests'] = manifests

            if test['chunks'] > 1:
                # add the chunk number to the TH symbol
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@ -189,6 +189,14 @@ TEST_SUITES = {
        'task_regex': ['(opt|debug)-reftest($|.*(-1|[^0-9])$)',
                       'test-verify-gpu($|.*(-1|[^0-9])$)'],
    },
+    'reftest-qr': {
+        'aliases': ('rr',),
+        'build_flavor': 'reftest',
+        'mach_command': 'reftest',
+        'kwargs': {'tests': None},
+        'task_regex': ['(opt|debug)-reftest-qr($|.*(-1|[^0-9])$)',
+                       'test-verify-gpu($|.*(-1|[^0-9])$)'],
+    },
    'robocop': {
        'mach_command': 'robocop',
        'kwargs': {'test_paths': None},
--- a/testing/mozharness/configs/android/android_common.py
+++ b/testing/mozharness/configs/android/android_common.py
@ -159,6 +159,31 @@ config = {
            ],
            "tests": ["tests/layout/reftests/reftest.list",],
        },
+        "reftest-qr": {
+            "run_filename": "remotereftest.py",
+            "testsdir": "reftest",
+            "options": [
+                "--app=%(app)s",
+                "--ignore-window-size",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--httpd-path", "%(modules_dir)s",
+                "--symbols-path=%(symbols_path)s",
+                "--extra-profile-file=fonts",
+                "--extra-profile-file=hyphenation",
+                "--suite=reftest",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--deviceSerial=%(device_serial)s",
+                "--topsrcdir=tests",
+            ],
+            "tests": ["tests/layout/reftests/reftest-qr.list",],
+        },
        "crashtest": {
            "run_filename": "remotereftest.py",
            "testsdir": "reftest",
--- a/testing/mozharness/mozharness/mozilla/testing/errors.py
+++ b/testing/mozharness/mozharness/mozilla/testing/errors.py
@ -25,6 +25,13 @@ _mochitest_summary = {
    'known_fail_group': "Todo",
 }

+_reftest_summary = {
+    'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
+    'pass_group': "Successful",
+    'fail_group': "Unexpected",
+    'known_fail_group': "Known problems",
+}
+
 TinderBoxPrintRe = {
    "mochitest-chrome_summary": _mochitest_summary,
    "mochitest-webgl1-core_summary": _mochitest_summary,
@ -42,30 +49,16 @@ TinderBoxPrintRe = {
        'fail_group': "failed",
        'known_fail_group': "todo",
    },
-    "reftest_summary": {
-        'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
-        'pass_group': "Successful",
-        'fail_group': "Unexpected",
-        'known_fail_group': "Known problems",
-    },
-    "crashtest_summary": {
-        'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
-        'pass_group': "Successful",
-        'fail_group': "Unexpected",
-        'known_fail_group': "Known problems",
-    },
+    "reftest_summary": _reftest_summary,
+    "reftest-qr_summary": _reftest_summary,
+    "crashtest_summary": _reftest_summary,
    "xpcshell_summary": {
        'regex': re.compile(r'''INFO \| (Passed|Failed|Todo): (\d+)'''),
        'pass_group': "Passed",
        'fail_group': "Failed",
        'known_fail_group': "Todo",
    },
-    "jsreftest_summary": {
-        'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),  # NOQA: E501
-        'pass_group': "Successful",
-        'fail_group': "Unexpected",
-        'known_fail_group': "Known problems",
-    },
+    "jsreftest_summary": _reftest_summary,
    "instrumentation_summary": _mochitest_summary,
    "cppunittest_summary": {
        'regex': re.compile(r'''cppunittests INFO \| (Passed|Failed): (\d+)'''),
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@ -320,6 +320,7 @@ You can set this by specifying --test-url URL
            'mochitest-webgpu': 'mochitest',
            'geckoview': 'mochitest',
            'geckoview-junit': 'mochitest',
+            'reftest-qr': 'reftest',
            'jsreftest': 'reftest',
            'crashtest': 'reftest',
            'reftest-debug': 'reftest',
--- a/tools/tryselect/test/test_mozharness_integration.py
+++ b/tools/tryselect/test/test_mozharness_integration.py
@ -25,6 +25,7 @@ MOZHARNESS_SCRIPTS = {
            'geckoview-junit',
            'jittest',
            'jsreftest',
+            'reftest-qr',
        ],
    },
    'desktop_unittest': {
@ -45,7 +46,6 @@ MOZHARNESS_SCRIPTS = {
            'mochitest-valgrind-plain',
            'reftest-gpu',
            'reftest-no-accel',
-            'reftest-qr',
        ],
    },
 }