Bug 1621483 - set up separate enviornment for linux1804 marionette and web-platform-tests-wdspec r=jmaher,marionette-reviewers,whimboo

Changes:

As documented in bug 1621483, `marionette` experiences a non-trivial amount of issues when run on ubuntu1804 docker image with GTK/GNOME desktop environment enabled.

GTK/GNOME has a higher degree of asynciness when manipulating window size/position and this leads to `marionette` and the derived suite `web-platform-tests-wdspec` reporting intermittent oranges for a number of tests.

While attempts were made to incorporate a fix for the marionette driver itself, the best attempts have only been able to achieve a ~50% reliability in green runs.

This patch reintroduces the use of bare `compiz` window manager exclusively for these two problematic test suites so that at least the tests are running on non-legacy software.

Differential Revision: https://phabricator.services.mozilla.com/D66482

--HG--
extra : moz-landing-system : lando

taskcluster/ci/test/test-platforms.yml

@@ -18,6 +18,7 @@
 
 ##
 # Linux platforms (matching /linux.*/)
+# Linux1804-64 uses Ubuntu 18.04 and is run on docker
 
 linux1804-32-shippable/opt:
     build-platform: linux-shippable/opt
@@ -31,13 +32,13 @@ linux1804-64/opt:
         - desktop-screenshot-capture
         - linux1804-tests
         - mochitest-headless
-        - web-platform-tests-1804
+        - web-platform-tests
 
 linux1804-64/debug:
     build-platform: linux64/debug
     test-sets:
         - linux1804-tests
-        - web-platform-tests-1804
+        - web-platform-tests
 
 linux1804-64-shippable/opt:
     build-platform: linux64-shippable/opt
@@ -47,7 +48,7 @@ linux1804-64-shippable/opt:
         - linux1804-tests
         - marionette-headless
         - mochitest-headless
-        - web-platform-tests-1804
+        - web-platform-tests
         - web-platform-tests-wdspec-headless
 
 linux1804-64-devedition/opt:
@@ -55,33 +56,33 @@ linux1804-64-devedition/opt:
     test-sets:
         - desktop-screenshot-capture
         - linux1804-tests
-        - web-platform-tests-1804
+        - web-platform-tests
 
 linux1804-64-asan/opt:
     build-platform: linux64-asan/opt
     test-sets:
         - linux1804-tests
-        - web-platform-tests-1804
+        - web-platform-tests
 
 linux1804-64-qr/opt:
     build-platform: linux64/opt
     test-sets:
         - awsy
         - linux1804-qr-tests
-        - web-platform-tests-1804
+        - web-platform-tests
 
 linux1804-64-shippable-qr/opt:
     build-platform: linux64-shippable/opt
     test-sets:
         - awsy
         - linux1804-qr-tests
-        - web-platform-tests-1804
+        - web-platform-tests
 
 linux1804-64-qr/debug:
     build-platform: linux64/debug
     test-sets:
         - linux1804-qr-tests
-        - web-platform-tests-1804
+        - web-platform-tests
 
 linux1804-64-asan-qr/opt:
     build-platform: linux64-asan/opt
@@ -92,21 +93,15 @@ linux1804-64-ccov/opt:
     build-platform: linux64-ccov/opt
     test-sets:
         - awsy
-        - linux1804-ccov-tests
-        - web-platform-tests-1804
+        - linux1804-tests
+        - web-platform-tests
         - ccov-code-coverage-tests
 
-linux64/debug:
-    build-platform: linux64/debug
-    test-sets:
-        - linux1604-legacy-tests
-        - web-platform-tests-1604
+# Linux64 uses Ubuntu 16.04 and is run on hardware
 
 linux64/opt:
     build-platform: linux64/opt
     test-sets:
-        - linux1604-legacy-tests
-        - web-platform-tests-1604
         - talos
         - linux-talos-flex
         - raptor-firefox
@@ -116,8 +111,6 @@ linux64/opt:
 linux64-shippable/opt:
     build-platform: linux64-shippable/opt
     test-sets:
-        - linux1604-legacy-tests
-        - web-platform-tests-1604
         - talos
         - linux-talos-flex
         - raptor-chrome
@@ -130,19 +123,6 @@ linux64-shippable/opt:
         - raptor-fetch-profiling
         - talos-profiling
 
-linux64-devedition/opt:
-    build-platform: linux64-devedition/opt
-    test-sets:
-        - linux1604-legacy-tests
-        - web-platform-tests-1604
-
-# TODO: use 'asan' label here, instead of -asan/opt
-linux64-asan/opt:
-    build-platform: linux64-asan/opt
-    test-sets:
-        - linux1604-legacy-tests
-        - web-platform-tests-1604
-
 linux64-tsan/opt:
     build-platform: linux64-tsan/opt
     test-sets:
@@ -153,7 +133,6 @@ linux64-qr/opt:
     build-platform: linux64/opt
     test-sets:
         - talos
-        - web-platform-tests-1604
         - raptor-firefox
         - browsertime
         - raptor-fetch-firefox
@@ -162,21 +141,13 @@ linux64-shippable-qr/opt:
     build-platform: linux64-shippable/opt
     test-sets:
         - talos
-        - web-platform-tests-1604
         - raptor-firefox
         - browsertime
         - raptor-fetch-firefox
 
-linux64-qr/debug:
-    build-platform: linux64/debug
-    test-sets:
-        - web-platform-tests-1604
-
 linux64-ccov/opt:
     build-platform: linux64-ccov/opt
     test-sets:
-        - linux1604-legacy-tests
-        - web-platform-tests-1604
         - talos
 
 ##

taskcluster/ci/test/test-sets.yml

@@ -21,14 +21,6 @@ web-platform-tests:
     - web-platform-tests-reftests
     - web-platform-tests-wdspec
 
-web-platform-tests-1804:
-    - web-platform-tests
-    - web-platform-tests-crashtests
-    - web-platform-tests-reftests
-
-web-platform-tests-1604:
-    - web-platform-tests-wdspec
-
 web-platform-tests-wdspec-headless:
     - web-platform-tests-wdspec-headless
 
@@ -335,17 +327,15 @@ desktop-screenshot-capture:
 linux32-tests:
     - web-platform-tests
 
-linux1604-legacy-tests:
-    - marionette
-
 linux1804-tests:
     - cppunit
     - crashtest
     - firefox-ui-functional-local
     - firefox-ui-functional-remote
     - gtest
-    - jsreftest
     - jittest
+    - jsreftest
+    - marionette
     - mochitest-plain
     - mochitest-a11y
     - mochitest-browser-chrome
@@ -354,11 +344,11 @@ linux1804-tests:
     - mochitest-gpu
     - mochitest-media
     - mochitest-remote
-    - mochitest-webgpu
     - mochitest-webgl1-core
     - mochitest-webgl1-ext
     - mochitest-webgl2-core
     - mochitest-webgl2-ext
+    - mochitest-webgpu
     - reftest
     - reftest-no-accel
     - telemetry-tests-client
@@ -386,35 +376,6 @@ linux1804-qr-tests:
     - reftest
     - xpcshell
 
-linux1804-ccov-tests:
-    - cppunit
-    - crashtest
-    - firefox-ui-functional-local
-    - firefox-ui-functional-remote
-    - gtest
-    - jittest
-    - jsreftest
-    - mochitest-plain
-    - mochitest-a11y
-    - mochitest-browser-chrome
-    - mochitest-chrome
-    - mochitest-devtools-chrome
-    - mochitest-gpu
-    - mochitest-media
-    - mochitest-remote
-    - mochitest-webgl1-core
-    - mochitest-webgl1-ext
-    - mochitest-webgl2-core
-    - mochitest-webgl2-ext
-    - mochitest-webgpu
-    - reftest
-    - reftest-no-accel
-    - telemetry-tests-client
-    - test-verify
-    - test-verify-gpu
-    - test-verify-wpt
-    - xpcshell
-
 linux-qr-smoketests:
     - crashtest
     - reftest

taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh

@@ -19,6 +19,7 @@ apt_packages+=('bluez-cups')
 apt_packages+=('build-essential')
 apt_packages+=('ca-certificates')
 apt_packages+=('ccache')
+apt_packages+=('compiz')
 apt_packages+=('curl')
 apt_packages+=('fonts-kacst')
 apt_packages+=('fonts-kacst-one')

taskcluster/scripts/tester/test-linux.sh

@@ -38,6 +38,7 @@ fi
 : NEED_XVFB                     ${NEED_XVFB:=true}
 : NEED_WINDOW_MANAGER           ${NEED_WINDOW_MANAGER:=false}
 : NEED_PULSEAUDIO               ${NEED_PULSEAUDIO:=false}
+: NEED_COMPIZ                   ${NEED_COPMPIZ:=false}
 : START_VNC                     ${START_VNC:=false}
 : TASKCLUSTER_INTERACTIVE       ${TASKCLUSTER_INTERACTIVE:=false}
 : mozharness args               "${@}"
@@ -178,8 +179,10 @@ if $NEED_WINDOW_MANAGER; then
     eval `echo '' | /usr/bin/gnome-keyring-daemon -r -d --unlock --components=secrets`
 fi
 
-if [ $DISTRIBUTION == "Ubuntu" ] && [ $RELEASE == "16.04" ]; then
+if [ $NEED_COMPIZ == "true" ]  && [ $RELEASE == "16.04" ]; then
     compiz 2>&1 &
+elif [ $NEED_COMPIZ == "true" ] && [ $RELEASE == "18.04" ]; then
+    compiz --replace 2>&1 &
 fi
 
 maybe_start_pulse

taskcluster/taskgraph/transforms/job/mozharness_test.py

@@ -111,11 +111,25 @@ def mozharness_test_on_docker(config, job, taskdesc):
         'MOZILLA_BUILD_URL': {'task-reference': installer_url},
         'NEED_PULSEAUDIO': 'true',
         'NEED_WINDOW_MANAGER': 'true',
-        'NEED_COMPIZ': 'true',
         'ENABLE_E10S': text_type(bool(test.get('e10s'))).lower(),
         'WORKING_DIR': '/builds/worker',
     })
 
+    # Legacy linux64 tests rely on compiz.
+    if test.get('docker-image', {}).get('in-tree') == 'desktop1604-test':
+        env.update({
+            'NEED_COMPIZ': 'true'
+        })
+
+    # Bug 1602701/1601828 - use compiz on ubuntu1804 due to GTK asynchiness
+    # when manipulating windows.
+    if test.get('docker-image', {}).get('in-tree') == 'ubuntu1804-test':
+        if ('wdspec' in job['run']['test']['suite'] or
+            ('marionette' in job['run']['test']['suite'] and 'headless' not in job['label'])):
+            env.update({
+                'NEED_COMPIZ': 'true'
+            })
+
     if mozharness.get('mochitest-flavor'):
         env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']
 

testing/web-platform/meta/webdriver/tests/close_window/close.py.ini

@@ -0,0 +1,3 @@
+[close.py]
+  expected:
+    if os == 'linux' and os_version == '18.04': ["OK", "TIMEOUT"]