Bug 1822891 [wpt PR 39036] - [wptrunner] Downgrade URL validation in content shell driver, a=testonly

Automatic update from web-platform-tests
[wptrunner] Downgrade URL validation in content shell driver (#39036)

See the comment in `testharnessreport-content-shell.js`. URL mismatches
will no longer be treated as a harness `ERROR`.

For the URLs to match as much as possible, this change partially undoes
web-platform-tests/wpt#39007, which broke validation for final URLs with
fragments. Instead, we use the standard `URL(...)` instead of `<a>` to
parse the original `location.href`.
--

wpt-commits: 071134cb4d44b95f8068828830498a19c60ad120
wpt-pr: 39036

testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executorcontentshell.py

@@ -264,6 +264,15 @@ class ContentShellTestharnessExecutor(TestharnessExecutor):
             if not text:
                 return (test.result_cls("ERROR", errors), [])
 
-            return self.convert_result(test, json.loads(text))
+            result_url, status, message, stack, subtest_results = json.loads(text)
+            if result_url != test.url:
+                # Suppress `convert_result`'s URL validation.
+                # See `testharnessreport-content-shell.js` for details.
+                self.logger.warning('Got results from %s, expected %s' % (result_url, test.url))
+                self.logger.warning('URL mismatch may be a false positive '
+                                    'if the test navigates')
+                result_url = test.url
+            raw_result = result_url, status, message, stack, subtest_results
+            return self.convert_result(test, raw_result)
         except BaseException as exception:
             return _convert_exception(test, exception, self.protocol.content_shell_errors.read_errors())

testing/web-platform/tests/tools/wptrunner/wptrunner/testharnessreport-content-shell.js

@@ -1,23 +1,45 @@
-var props = {output:%(output)d, debug: %(debug)s};
-setup(props);
+(function() {
+    var props = {output:%(output)d, debug: %(debug)s};
+    setup(props);
 
-testRunner.dumpAsText();
-testRunner.waitUntilDone();
-testRunner.setPopupBlockingEnabled(false);
-testRunner.setDumpJavaScriptDialogs(false);
+    // Some tests navigate away from the original URL as part of the
+    // functionality they exercise. In that case, `add_completion_callback(...)`
+    // uses the final `window.location` to report the test ID, which may not be
+    // correct [1].
+    //
+    // Persisting the original `window.location` with standard web platform APIs
+    // (e.g., `localStorage`) could interfere with the their tests, so this must
+    // be avoided. Unfortunately, there doesn't appear to be anything in content
+    // shell's protocol mode or Blink-specific `window.testRunner` or
+    // `window.internals` [2] that could help with this. As such, the driver
+    // simply downgrades a mismatched test ID to a logged warning instead of a
+    // harness error.
+    //
+    // [1] crbug.com/1418753
+    // [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/docs/testing/writing_web_tests.md#Relying-on-Blink_Specific-Testing-APIs
+    const url = new URL(location.href);
 
-add_completion_callback(function (tests, harness_status) {
-    var id = decodeURIComponent(location.pathname) + decodeURIComponent(location.search) + decodeURIComponent(location.hash);
-    var result_string = JSON.stringify([
-        id,
-        harness_status.status,
-        harness_status.message,
-        harness_status.stack,
-        tests.map(function(t) {
-            return [t.name, t.status, t.message, t.stack]
-        }),
-    ]);
+    testRunner.dumpAsText();
+    testRunner.waitUntilDone();
+    testRunner.setPopupBlockingEnabled(false);
+    testRunner.setDumpJavaScriptDialogs(false);
+    // Show `CONSOLE MESSAGE:` and `CONSOLE ERROR:` in stderr.
+    if (props.debug) {
+        testRunner.setDumpConsoleMessages(true);
+    }
 
-    testRunner.setCustomTextOutput(result_string);
-    testRunner.notifyDone();
-});
+    add_completion_callback(function (tests, harness_status) {
+        const test_id = decodeURIComponent(url.pathname) + decodeURIComponent(url.search) + decodeURIComponent(url.hash);
+        const result_string = JSON.stringify([
+            test_id,
+            harness_status.status,
+            harness_status.message,
+            harness_status.stack,
+            tests.map(function(t) {
+                return [t.name, t.status, t.message, t.stack]
+            }),
+        ]);
+        testRunner.setCustomTextOutput(result_string);
+        testRunner.notifyDone();
+    });
+})();