From b37213dc9f58efe61d1e2ffd33993c46d1d1906a Mon Sep 17 00:00:00 2001
From: jugglinmike <mike@mikepennisi.com>
Date: Tue, 26 Nov 2019 11:31:04 +0000
Subject: [PATCH] Bug 1595207 [wpt PR 20187] - [testharness.js] Implement
 `promise_setup`, a=testonly

Automatic update from web-platform-tests
[testharness.js] Implement `promise_setup`

This enacts the change specified by WPT RFC 32

https://github.com/web-platform-tests/rfcs/blob/master/rfcs/asynchronous_setup.md
--

wpt-commits: 783959beb60fc589e897cc11d08c23f621259f62
wpt-pr: 20187
---
 .../docs/writing-tests/testharness-api.md     |  34 +-
 testing/web-platform/tests/lint.whitelist     |   1 +
 .../tests/unit/promise_setup-timeout.html     |  28 ++
 .../test/tests/unit/promise_setup.html        | 333 ++++++++++++++++++
 .../tests/resources/testharness.js            |  65 +++-
 5 files changed, 450 insertions(+), 11 deletions(-)
 create mode 100644 testing/web-platform/tests/resources/test/tests/unit/promise_setup-timeout.html
 create mode 100644 testing/web-platform/tests/resources/test/tests/unit/promise_setup.html

diff --git a/testing/web-platform/tests/docs/writing-tests/testharness-api.md b/testing/web-platform/tests/docs/writing-tests/testharness-api.md
index 1bdf06613f99..a17f5795bc33 100644
--- a/testing/web-platform/tests/docs/writing-tests/testharness-api.md
+++ b/testing/web-platform/tests/docs/writing-tests/testharness-api.md
@@ -430,18 +430,38 @@ not possible to make the test reliable in some other way.
 
 ## Setup ##
 
-Sometimes tests require non-trivial setup that may fail. For this purpose
-there is a `setup()` function, that may be called with one or two arguments.
-The two argument version is:
+Sometimes tests require non-trivial setup that may fail. testharness.js
+provides two global functions for this purpose, `setup` and `promise_setup`.
+
+`setup()` may be called with one or two arguments. The two argument version is:
 
 ```js
 setup(func, properties)
 ```
 
-The one argument versions may omit either argument.
-func is a function to be run synchronously. `setup()` becomes a no-op once
-any tests have returned results. Properties are global properties of the test
-harness. Currently recognised properties are:
+The one argument version may omit either argument. `func` is a function to be
+run synchronously. `setup()` becomes a no-op once any tests have returned
+results. `properties` is an object which specifies global properties of the
+test harness (enumerated in the following section).
+
+`promise_setup()` allows authors to pause testing until some asynchronous
+operation has completed. It has the following signature:
+
+```js
+promise_setup(func, properties)
+```
+
+Here, the `func` argument is required. This argument must be a function which
+returns an object with a `then` method (e.g. an ECMAScript Promise instance);
+the harness will wait for the fulfillment of this value before executing any
+additional subtests defined with the `promise_test` function. If the value is
+rejected, the harness will report an error and cancel the remaining tests.
+`properties` may optionally be provided as an object which specifies global
+properties of the test harness (enumerated in the following section).
+
+### Setup properties ##
+
+Both setup functions recognize the following properties:
 
 `explicit_done` - Wait for an explicit call to done() before declaring all
 tests complete (see below; implicitly true for single page tests)
diff --git a/testing/web-platform/tests/lint.whitelist b/testing/web-platform/tests/lint.whitelist
index cec2237f4d37..bd0861ccd6c4 100644
--- a/testing/web-platform/tests/lint.whitelist
+++ b/testing/web-platform/tests/lint.whitelist
@@ -305,6 +305,7 @@ SET TIMEOUT: resources/test/tests/functional/worker.js
 SET TIMEOUT: resources/test/tests/functional/worker-uncaught-allow.js
 SET TIMEOUT: resources/test/tests/unit/exceptional-cases.html
 SET TIMEOUT: resources/test/tests/unit/exceptional-cases-timeouts.html
+SET TIMEOUT: resources/test/tests/unit/promise_setup.html
 SET TIMEOUT: resources/testharness.js
 
 # setTimeout use in reftests
diff --git a/testing/web-platform/tests/resources/test/tests/unit/promise_setup-timeout.html b/testing/web-platform/tests/resources/test/tests/unit/promise_setup-timeout.html
new file mode 100644
index 000000000000..c4947feef42a
--- /dev/null
+++ b/testing/web-platform/tests/resources/test/tests/unit/promise_setup-timeout.html
@@ -0,0 +1,28 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="timeout" content="long">
+  <script src="/resources/testharness.js"></script>
+  <script src="../../nested-testharness.js"></script>
+  <title>promise_setup - timeout</title>
+</head>
+<body>
+<script>
+'use strict';
+promise_test(() => {
+  return makeTest(
+      () => {
+        test(() => {}, 'before');
+        promise_setup(() => new Promise(() => {}));
+        promise_test(() => Promise.resolve(), 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'TIMEOUT');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'NOTRUN');
+    });
+}, 'timeout when returned promise does not settle');
+</script>
+</body>
+</html>
diff --git a/testing/web-platform/tests/resources/test/tests/unit/promise_setup.html b/testing/web-platform/tests/resources/test/tests/unit/promise_setup.html
new file mode 100644
index 000000000000..2abb10a476e6
--- /dev/null
+++ b/testing/web-platform/tests/resources/test/tests/unit/promise_setup.html
@@ -0,0 +1,333 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <meta charset="utf-8">
+  <script src="/resources/testharness.js"></script>
+  <script src="../../nested-testharness.js"></script>
+  <title>promise_setup</title>
+</head>
+<body>
+<script>
+'use strict';
+promise_test(() => {
+  return makeTest(
+      () => {
+        // Ensure that the harness error is the result of explicit error
+        // handling
+        setup({ allow_uncaught_exception: true });
+
+        test(() => {}, 'before');
+        promise_setup({});
+        promise_test(() => Promise.resolve(), 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, undefined);
+    });
+}, 'Error when no function provided');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        test(() => {}, 'before');
+        promise_setup(() => Promise.resolve(), {});
+        promise_test(() => Promise.resolve(), 'after');
+        throw new Error('this error is expected');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'PASS');
+    });
+}, 'Does not apply unspecified configuration properties');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        var properties = {
+          allow_uncaught_exception: true
+        };
+        test(() => {}, 'before');
+        promise_setup(() => Promise.resolve(), properties);
+        promise_test(() => Promise.resolve(), 'after');
+        throw new Error('this error is expected');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'PASS');
+    });
+}, 'Ignores configuration properties when some tests have already run');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        var properties = {
+          allow_uncaught_exception: true
+        };
+        promise_setup(() => Promise.resolve(), properties);
+        promise_test(() => {
+          return new Promise((resolve) => {
+            setTimeout(() => {
+              resolve();
+              throw new Error('this error is expected');
+            });
+          });
+        }, 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'OK');
+      assert_equals(tests.after, 'PASS');
+    });
+}, 'Honors configuration properties');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        // Ensure that the harness error is the result of explicit error
+        // handling
+        setup({ allow_uncaught_exception: true });
+
+        test(() => {}, 'before');
+        promise_setup(() => { throw new Error('this error is expected'); });
+        promise_test(() => Promise.resolve(), 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'NOTRUN');
+    });
+}, 'Error for synchronous exceptions');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        // Ensure that the harness error is the result of explicit error
+        // handling
+        setup({ allow_uncaught_exception: true });
+
+        test(() => {}, 'before');
+        promise_setup(() => undefined);
+        promise_test(() => Promise.resolve(), 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'NOTRUN');
+    });
+}, 'Error for missing return value');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        // Ensure that the harness error is the result of explicit error
+        // handling
+        setup({ allow_uncaught_exception: true });
+
+        test(() => {}, 'before');
+        var noThen = Promise.resolve();
+        noThen.then = undefined;
+        promise_setup(() => noThen);
+        promise_test(() => Promise.resolve(), 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'NOTRUN');
+    });
+}, 'Error for non-thenable return value');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        // Ensure that the harness error is the result of explicit error
+        // handling
+        setup({ allow_uncaught_exception: true });
+
+        test(() => {}, 'before');
+        var poisonedThen = {
+          get then() {
+            throw new Error('this error is expected');
+          }
+        };
+        promise_setup(() => poisonedThen);
+        promise_test(() => Promise.resolve(), 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'NOTRUN');
+    });
+}, 'Error for "poisoned" `then` property');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        // Ensure that the harness error is the result of explicit error
+        // handling
+        setup({ allow_uncaught_exception: true });
+
+        test(() => {}, 'before');
+        var badThen = {
+          then() {
+            throw new Error('this error is expected');
+          }
+        };
+        promise_setup(() => badThen);
+        promise_test(() => Promise.resolve(), 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'NOTRUN');
+    });
+}, 'Error for synchronous error from `then` method');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        // Ensure that the harness error is the result of explicit error
+        // handling
+        setup({ allow_uncaught_exception: true });
+
+        test(() => {}, 'before');
+        promise_setup(() => Promise.resolve());
+        test(() => {}, 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, undefined);
+    });
+}, 'Error for subsequent invocation of `test`');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        // Ensure that the harness error is the result of explicit error
+        // handling
+        setup({ allow_uncaught_exception: true });
+
+        test(() => {}, 'before');
+        promise_setup(() => Promise.resolve());
+        async_test((t) => t.done(), 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, undefined);
+    });
+}, 'Error for subsequent invocation of `async_test`');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        // Ensure that the harness error is the result of explicit error
+        // handling
+        setup({ allow_uncaught_exception: true });
+
+        test(() => {}, 'before');
+        promise_setup(() => Promise.reject());
+        promise_test(() => Promise.resolve(), 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'NOTRUN');
+    });
+}, 'Error for rejected promise');
+
+promise_test(() => {
+  var expected_sequence = [
+    'test body',
+    'promise_setup begin',
+    'promise_setup end',
+    'promise_test body'
+  ];
+  var actual_sequence = window.actual_sequence = [];
+
+  return makeTest(
+      () => {
+        test(() => { parent.actual_sequence.push('test body'); }, 'before');
+        promise_setup(() => {
+          parent.actual_sequence.push('promise_setup begin');
+
+          return Promise.resolve()
+            .then(() => new Promise((resolve) => setTimeout(resolve, 300)))
+            .then(() => parent.actual_sequence.push('promise_setup end'));
+        });
+        promise_test(() => {
+          parent.actual_sequence.push('promise_test body');
+          return Promise.resolve();
+        }, 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'OK');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'PASS');
+      assert_array_equals(actual_sequence, expected_sequence);
+    });
+}, 'Waits for promise to settle');
+
+promise_test(() => {
+  var expected_sequence = [
+    'promise_test 1 begin',
+    'promise_test 1 end',
+    'promise_setup begin',
+    'promise_setup end',
+    'promise_test 2 body'
+  ];
+  var actual_sequence = window.actual_sequence = [];
+
+  return makeTest(
+      () => {
+        promise_test((t) => {
+          parent.actual_sequence.push('promise_test 1 begin');
+
+          return Promise.resolve()
+            .then(() => new Promise((resolve) => t.step_timeout(resolve, 300)))
+            .then(() => parent.actual_sequence.push('promise_test 1 end'));
+        }, 'before');
+        promise_setup(() => {
+          parent.actual_sequence.push('promise_setup begin');
+
+          return Promise.resolve()
+            .then(() => new Promise((resolve) => setTimeout(resolve, 300)))
+            .then(() => parent.actual_sequence.push('promise_setup end'));
+        });
+        promise_test(() => {
+          parent.actual_sequence.push('promise_test 2 body');
+          return Promise.resolve();
+        }, 'after');
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'OK');
+      assert_equals(tests.before, 'PASS');
+      assert_equals(tests.after, 'PASS');
+      assert_array_equals(actual_sequence, expected_sequence);
+    });
+}, 'Waits for existing promise_test to complete');
+
+promise_test(() => {
+  return makeTest(
+      () => {
+        var properties = { allow_uncaught_exception: true };
+        promise_test(() => {
+          return new Promise((resolve) => {
+            setTimeout(() => {
+              resolve();
+              throw new Error('this error is expected');
+            });
+          });
+        }, 'before');
+        promise_setup(() => Promise.resolve(), properties);
+      }
+    ).then(({harness, tests}) => {
+      assert_equals(harness, 'ERROR');
+      assert_equals(tests.before, 'PASS');
+    });
+}, 'Defers application of setup properties');
+</script>
+</body>
+</html>
diff --git a/testing/web-platform/tests/resources/testharness.js b/testing/web-platform/tests/resources/testharness.js
index 93809664ea34..f289e7797201 100644
--- a/testing/web-platform/tests/resources/testharness.js
+++ b/testing/web-platform/tests/resources/testharness.js
@@ -538,8 +538,12 @@ policies and contribution forms [3].
      */
     function test(func, name, properties)
     {
+        if (tests.promise_setup_called) {
+            tests.status.status = tests.status.ERROR;
+            tests.status.message = '`test` invoked after `promise_setup`';
+            tests.complete();
+        }
         var test_name = name ? name : test_environment.next_default_test_name();
-        properties = properties ? properties : {};
         var test_obj = new Test(test_name, properties);
         var value = test_obj.step(func, test_obj, test_obj);
 
@@ -564,13 +568,17 @@ policies and contribution forms [3].
 
     function async_test(func, name, properties)
     {
+        if (tests.promise_setup_called) {
+            tests.status.status = tests.status.ERROR;
+            tests.status.message = '`async_test` invoked after `promise_setup`';
+            tests.complete();
+        }
         if (typeof func !== "function") {
             properties = name;
             name = func;
             func = null;
         }
         var test_name = name ? name : test_environment.next_default_test_name();
-        properties = properties ? properties : {};
         var test_obj = new Test(test_name, properties);
         if (func) {
             test_obj.step(func, test_obj, test_obj);
@@ -579,7 +587,13 @@ policies and contribution forms [3].
     }
 
     function promise_test(func, name, properties) {
-        var test = async_test(name, properties);
+        if (typeof func !== "function") {
+            properties = name;
+            name = func;
+            func = null;
+        }
+        var test_name = name ? name : test_environment.next_default_test_name();
+        var test = new Test(test_name, properties);
         test._is_promise_test = true;
 
         // If there is no promise tests queue make one.
@@ -790,6 +804,44 @@ policies and contribution forms [3].
         test_environment.on_new_harness_properties(properties);
     }
 
+    function promise_setup(func, maybe_properties)
+    {
+        if (typeof func !== "function") {
+            tests.set_status(tests.status.ERROR,
+                             "promise_test invoked without a function");
+            tests.complete();
+            return;
+        }
+        tests.promise_setup_called = true;
+
+        if (!tests.promise_tests) {
+            tests.promise_tests = Promise.resolve();
+        }
+
+        tests.promise_tests = tests.promise_tests
+            .then(function()
+                  {
+                      var properties = maybe_properties || {};
+                      var result;
+
+                      tests.setup(null, properties);
+                      result = func();
+                      test_environment.on_new_harness_properties(properties);
+
+                      if (!result || typeof result.then !== "function") {
+                          throw "Non-thenable returned by function passed to `promise_setup`";
+                      }
+                      return result;
+                  })
+            .catch(function(e)
+                   {
+                       tests.set_status(tests.status.ERROR,
+                                        String(e),
+                                        e && e.stack);
+                       tests.complete();
+                   });
+    }
+
     function done() {
         if (tests.tests.length === 0) {
             // `done` is invoked after handling uncaught exceptions, so if the
@@ -852,6 +904,7 @@ policies and contribution forms [3].
     expose(promise_rejects_exactly, 'promise_rejects_exactly');
     expose(generate_tests, 'generate_tests');
     expose(setup, 'setup');
+    expose(promise_setup, 'promise_setup');
     expose(done, 'done');
     expose(on_event, 'on_event');
     expose(step_timeout, 'step_timeout');
@@ -1842,7 +1895,7 @@ policies and contribution forms [3].
         this.timeout_id = null;
         this.index = null;
 
-        this.properties = properties;
+        this.properties = properties || {};
         this.timeout_length = settings.test_timeout;
         if (this.timeout_length !== null) {
             this.timeout_length *= tests.timeout_multiplier;
@@ -2450,6 +2503,10 @@ policies and contribution forms [3].
         this.allow_uncaught_exception = false;
 
         this.file_is_test = false;
+        // This value is lazily initialized in order to avoid introducing a
+        // dependency on ECMAScript 2015 Promises to all tests.
+        this.promise_tests = null;
+        this.promise_setup_called = false;
 
         this.timeout_multiplier = 1;
         this.timeout_length = test_environment.test_timeout();