Bug 1464234 [wpt PR 11150] - [testharness.js] Extend test suite with "variants", a=testonly

Automatic update from web-platform-tests[testharness.js] Extend test suite with "variants"

Feature description (from modification to `README.md` file):

> `testharness.js` is expected to function consistently in a number of
> distinct environments. In order to verify this expectation, each functional
> test may be executed under a number of distinct conditions. These conditions
> are applied using WPT's "test variants" pattern. The available variants are
> defined in the `variants.js` file; this file must be included before
> `testharness.js`. Every test must specify at least one variant.

Extend test harness for `testharness.js` with additional validation to
ensure that each test is correctly formatted. Introduce a minimal UI to
help future contributors discover this new functionality and its
motivation.

--

wpt-commits: 08995880734011eed359c4e79420db29626016ff
wpt-pr: 11150

testing/web-platform/tests/resources/test/README.md

@@ -81,3 +81,10 @@ must include a summary of the expected results as a JSON string within a
       "type": "complete"
     }
     </script>
+
+`testharness.js` is expected to function consistently in a number of
+distinct environments. In order to verify this expectation, each functional
+test may be executed under a number of distinct conditions. These conditions
+are applied using WPT's "test variants" pattern. The available variants are
+defined in the `variants.js` file; this file must be included before
+`testharness.js`. Every test must specify at least one variant.

testing/web-platform/tests/resources/test/conftest.py

@@ -12,6 +12,7 @@ ENC = 'utf8'
 HERE = os.path.dirname(os.path.abspath(__file__))
 WPT_ROOT = os.path.normpath(os.path.join(HERE, '..', '..'))
 HARNESS = os.path.join(HERE, 'harness.html')
+TEST_TYPES = ('functional', 'unit')
 
 def pytest_addoption(parser):
     parser.addoption("--binary", action="store", default=None, help="path to browser binary")
@@ -32,12 +33,23 @@ def pytest_configure(config):
     config.add_cleanup(config.server.stop)
     config.add_cleanup(config.driver.quit)
 
+def resolve_uri(context, uri):
+    if uri.startswith('/'):
+        base = WPT_ROOT
+        path = uri[1:]
+    else:
+        base = os.path.dirname(context)
+        path = uri
+
+    return os.path.exists(os.path.join(base, path))
+
 class HTMLItem(pytest.Item, pytest.Collector):
     def __init__(self, filename, test_type, parent):
         self.filename = filename
         self.type = test_type
+        self.variants = []
 
-        if test_type not in ('functional', 'unit'):
+        if test_type not in TEST_TYPES:
             raise ValueError('Unrecognized test type: "%s"' % test_type)
 
         with io.open(filename, encoding=ENC) as f:
@@ -45,20 +57,36 @@ class HTMLItem(pytest.Item, pytest.Collector):
 
         parsed = html5lib.parse(markup, namespaceHTMLElements=False)
         name = None
+        includes_variants_script = False
         self.expected = None
 
         for element in parsed.getiterator():
             if not name and element.tag == 'title':
                 name = element.text
                 continue
-            if element.attrib.get('id') == 'expected':
-                self.expected = json.loads(unicode(element.text))
+            if element.tag == 'meta' and element.attrib.get('name') == 'variant':
+                self.variants.append(element.attrib.get('content'))
                 continue
+            if element.tag == 'script':
+                if element.attrib.get('id') == 'expected':
+                    self.expected = json.loads(unicode(element.text))
+
+                src = element.attrib.get('src', '')
+
+                if 'variants.js' in src:
+                    includes_variants_script = True
+                    if not resolve_uri(filename, src):
+                        raise ValueError('Could not resolve path "%s" from %s' % (src, filename))
 
         if not name:
             raise ValueError('No name found in file: %s' % filename)
-        elif self.type == 'functional' and not self.expected:
-            raise ValueError('Functional tests must specify expected report data')
+        elif self.type == 'functional':
+            if not self.expected:
+                raise ValueError('Functional tests must specify expected report data')
+            if not includes_variants_script:
+                raise ValueError('No variants script found in file: %s' % filename)
+            if len(self.variants) == 0:
+                raise ValueError('No test variants specified in file %s' % filename)
         elif self.type == 'unit' and self.expected:
             raise ValueError('Unit tests must not specify expected report data')
 
@@ -95,12 +123,17 @@ class HTMLItem(pytest.Item, pytest.Collector):
             assert test[u'status_string'] == u'PASS', msg
 
     def _run_functional_test(self):
+        for variant in self.variants:
+            self._run_functional_test_variant(variant)
+
+    def _run_functional_test_variant(self, variant):
         driver = self.session.config.driver
         server = self.session.config.server
 
         driver.get(server.url(HARNESS))
 
-        actual = driver.execute_async_script('runTest("%s", "foo", arguments[0])' % server.url(str(self.filename)))
+        test_url = server.url(str(self.filename) + variant)
+        actual = driver.execute_async_script('runTest("%s", "foo", arguments[0])' % test_url)
 
         # Test object ordering is not guaranteed. This weak assertion verifies
         # that the indices are unique and sequential

testing/web-platform/tests/resources/test/tests/functional/add_cleanup.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Test#add_cleanup</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>

testing/web-platform/tests/resources/test/tests/functional/add_cleanup_count.html

@@ -1,7 +1,9 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="?keep-promise">
 <title>Test#add_cleanup reported count</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>

testing/web-platform/tests/resources/test/tests/functional/add_cleanup_err.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Test#add_cleanup: error</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>

testing/web-platform/tests/resources/test/tests/functional/add_cleanup_err_multi.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Test#add_cleanup: multiple functions with one in error</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>

testing/web-platform/tests/resources/test/tests/functional/api-tests-1.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Sample HTML5 API Tests</title>
+<script src="../../variants.js"></script>
 <meta name="timeout" content="6000">
 </head>
 <body onload="load_test_attr.done()">

testing/web-platform/tests/resources/test/tests/functional/api-tests-2.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Sample HTML5 API Tests</title>
+<script src="../../variants.js"></script>
 </head>
 <body>
 <h1>Sample HTML5 API Tests</h1>

testing/web-platform/tests/resources/test/tests/functional/api-tests-3.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Sample HTML5 API Tests</title>
+<script src="../../variants.js"></script>
 </head>
 <script src="/resources/testharness.js"></script>
 

testing/web-platform/tests/resources/test/tests/functional/force_timeout.html

@@ -1,7 +1,9 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="?keep-promise">
 <title>Test#force_timeout</title>
+<script src="../../variants.js"></script>
 </head>
 <body>
 <h1>Test#force_timeout</h1>

testing/web-platform/tests/resources/test/tests/functional/generate-callback.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Sample for using generate_tests to create a series of tests that share the same callback.</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>

testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlDictionary/test_partial_interface_of.html

@@ -3,7 +3,10 @@
 
 <head>
   <meta charset="utf-8">
+  <meta name="variant" content="">
+  <meta name="variant" content="?keep-promise">
   <title>idlharness: Partial dictionary</title>
+  <script src="/resources/test/variants.js"></script>
   <script src="/resources/testharness.js"></script>
   <script src="/resources/testharnessreport.js"></script>
   <script src="/resources/WebIDLParser.js"></script>

testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_immutable_prototype.html

@@ -2,7 +2,10 @@
 <html>
 <head>
   <meta charset="utf-8">
+  <meta name="variant" content="">
+  <meta name="variant" content="?keep-promise">
   <title>idlharness: Immutable prototypes</title>
+  <script src="../../../../variants.js"></script>
   <script src="/resources/testharness.js"></script>
   <script src="/resources/testharnessreport.js"></script>
   <script src="/resources/WebIDLParser.js"></script>

testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_partial_interface_of.html

@@ -3,7 +3,10 @@
 
 <head>
   <meta charset="utf-8">
+  <meta name="variant" content="">
+  <meta name="variant" content="?keep-promise">
   <title>idlharness: Partail interface</title>
+  <script src="/resources/test/variants.js"></script>
   <script src="/resources/testharness.js"></script>
   <script src="/resources/testharnessreport.js"></script>
   <script src="/resources/WebIDLParser.js"></script>

testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_primary_interface_of.html

@@ -2,7 +2,10 @@
 <html>
 <head>
   <meta charset="utf-8">
+  <meta name="variant" content="">
+  <meta name="variant" content="?keep-promise">
   <title>idlharness: Primary interface</title>
+  <script src="../../../../variants.js"></script>
   <script src="/resources/testharness.js"></script>
   <script src="/resources/testharnessreport.js"></script>
   <script src="/resources/WebIDLParser.js"></script>

testing/web-platform/tests/resources/test/tests/functional/idlharness/IdlInterface/test_to_json_operation.html

@@ -2,7 +2,10 @@
 <html>
 <head>
   <meta charset="utf-8">
+  <meta name="variant" content="">
+  <meta name="variant" content="?keep-promise">
   <title>IdlInterface.prototype.test_to_json_operation()</title>
+  <script src="../../../../variants.js"></script>
   <script src="/resources/testharness.js"></script>
   <script src="/resources/testharnessreport.js"></script>
   <script src="/resources/WebIDLParser.js"></script>

testing/web-platform/tests/resources/test/tests/functional/iframe-callback.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with iframe that notifies containing document via callbacks</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>

testing/web-platform/tests/resources/test/tests/functional/iframe-consolidate-errors.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with iframe that consolidates errors via fetch_tests_from_window</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>

testing/web-platform/tests/resources/test/tests/functional/iframe-consolidate-tests.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with iframe that consolidates tests via fetch_tests_from_window</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>
@@ -16,7 +19,7 @@ tests from an <tt>iframe</tt> into the primary document.</p>
 executing</p>
 <div id="log"></div>
 
-<iframe id="childContext" src="promise-async.html" style="display:none"></iframe>
+<iframe id="childContext" src="promise-async.html?keep-promise" style="display:none"></iframe>
 <!-- promise-async.html has async tests with promises -->
 
 <script>

testing/web-platform/tests/resources/test/tests/functional/iframe-msg.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with iframe that notifies containing document via cross document messaging</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>

testing/web-platform/tests/resources/test/tests/functional/order.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Ordering</title>
+<script src="../../variants.js"></script>
 <meta name="timeout" content="6000">
 </head>
 <body>

testing/web-platform/tests/resources/test/tests/functional/promise-async.html

@@ -1,7 +1,9 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="?keep-promise">
 <title>Async Tests and Promises</title>
+<script src="../../variants.js"></script>
 </head>
 <body>
 <h1>Async Tests and Promises</h1>

testing/web-platform/tests/resources/test/tests/functional/promise.html

@@ -1,7 +1,9 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="?keep-promise">
 <title>Promise Tests</title>
+<script src="../../variants.js"></script>
 </head>
 <body>
 <h1>Promise Tests</h1>

testing/web-platform/tests/resources/test/tests/functional/single-page-test-fail.html

@@ -1,5 +1,8 @@
 <!DOCTYPE HTML>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with file_is_test (should fail)</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>

testing/web-platform/tests/resources/test/tests/functional/single-page-test-no-assertions.html

@@ -1,5 +1,8 @@
 <!DOCTYPE HTML>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example single page test with no asserts</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>

testing/web-platform/tests/resources/test/tests/functional/single-page-test-no-body.html

@@ -1,5 +1,8 @@
 <!DOCTYPE HTML>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example single page test with no body</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>

testing/web-platform/tests/resources/test/tests/functional/single-page-test-pass.html

@@ -1,5 +1,8 @@
 <!DOCTYPE HTML>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with file_is_test</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 <script>

testing/web-platform/tests/resources/test/tests/functional/uncaught-exception-handle.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Harness Handling Uncaught Exception</title>
+<script src="../../variants.js"></script>
 </head>
 <script src="/resources/testharness.js"></script>
 

testing/web-platform/tests/resources/test/tests/functional/uncaught-exception-ignore.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Harness Ignoring Uncaught Exception</title>
+<script src="../../variants.js"></script>
 </head>
 <script src="/resources/testharness.js"></script>
 

testing/web-platform/tests/resources/test/tests/functional/worker-dedicated.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Dedicated Worker Tests</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>

testing/web-platform/tests/resources/test/tests/functional/worker-service.html

@@ -1,7 +1,9 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="?keep-promise">
 <title>Example with a service worker</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>

testing/web-platform/tests/resources/test/tests/functional/worker-shared.html

@@ -1,7 +1,10 @@
 <!DOCTYPE HTML>
 <html>
 <head>
+<meta name="variant" content="">
+<meta name="variant" content="?keep-promise">
 <title>Example with a shared worker</title>
+<script src="../../variants.js"></script>
 <script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>
 </head>

testing/web-platform/tests/resources/test/variants.js

@@ -0,0 +1,57 @@
+(function() {
+  'use strict';
+  var variants = {
+    /**
+     * Tests are executed in the absence of the global Promise constructor by
+     * default in order to verify support for the Server browser engine.
+     *
+     * https://github.com/w3c/web-platform-tests/issues/6266
+     */
+    'default': {
+      description: 'Global Promise constructor removed.',
+      apply: function() {
+        delete window.Promise;
+      }
+    },
+    /**
+     * This variant allows for testing functionality that is fundamentally
+     * dependent on Promise support, e.g. the `promise_test` function
+     */
+    'keep-promise': {
+      description: 'No modification of global environment.',
+      apply: function() {}
+    }
+  };
+  var match = window.location.search.match(/\?(.*)$/);
+  var variantName = (match && match[1]) || 'default';
+
+  if (!Object.hasOwnProperty.call(variants, variantName)) {
+    window.location = 'javascript:"Unrecognized variant: ' + variantName + '";';
+    document.close();
+    return;
+  }
+
+  if (typeof test === 'function') {
+    test(function() {
+      assert_unreached('variants.js must be included before testharness.js');
+    });
+  }
+  var variant = variants[variantName];
+
+  var variantNode = document.createElement('div');
+  variantNode.innerHTML = '<p>This testharness.js test was executed with ' +
+    'the variant named, "' + variantName + '". ' + variant.description +
+    '</p><p>Refer to the test harness README file for more information.</p>';
+  function onReady() {
+    if (document.readyState !== 'complete') {
+      return;
+    }
+
+    document.body.insertBefore(variantNode, document.body.firstChild);
+  }
+
+  onReady();
+  document.addEventListener('readystatechange', onReady);
+
+  variant.apply();
+}());