Bug 1461925 [wpt PR 11028] - Generalize subsetTest() in common/ and use for WebCryptoAPI tests, a=testonly

Automatic update from web-platform-testsGeneralize subsetTest() in common/ and use for WebCryptoAPI tests (#11028)

Fixes #10890.
--

wpt-commits: e9c44b8cf4d6b3d7c7f2d5d629e83c616ea903e1
wpt-pr: 11028

testing/web-platform/meta/MANIFEST.json

@@ -193262,6 +193262,11 @@
      {}
     ]
    ],
+   "common/subset-tests.js": [
+    [
+     {}
+    ]
+   ],
    "common/test-setting-immutable-prototype.js": [
     [
      {}
@@ -405748,47 +405753,47 @@
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes.js": [
-   "db8091eaf59ea342ae7c8f64e6548c976b5f0b36",
+   "f0f112ddcf43a46cfccc0e7cf29b46208f87bb5f",
    "support"
   ],
   "WebCryptoAPI/generateKey/successes_AES-CBC.https.any.js": [
-   "71f7d2beac74da25795366722aea79fb88c197ab",
+   "49ce28516303c7525f74d29fbbe36b578e9e0d08",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_AES-CTR.https.any.js": [
-   "57f1bff81966f8fc20b7870ed1a5ecf939f1c4d1",
+   "377e8f0ec969f36949c9dc08ae935063d25c08d8",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_AES-GCM.https.any.js": [
-   "faa70a7dda91b0602987ec8e7ef0787c4ed163e3",
+   "a65a71f8f83b017fe9eb8fccbd8da3489ed1f9f0",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_AES-KW.https.any.js": [
-   "23b4567c82e12847f41fb6aa8a6a2cb6bd219a94",
+   "165159903403f67cc27d29f78d46369eb598811c",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_ECDH.https.any.js": [
-   "51f5990d71b6d0008163c5550dfe3ca88fe56356",
+   "525b874eff610c6db8cca7f50ba0c074b8975001",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_ECDSA.https.any.js": [
-   "bfce352ef583a31aaae4a5a303a8b98737cfc22f",
+   "2a63f8d0d12987ca2e4c41e5eb28a53741a091f5",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_HMAC.https.any.js": [
-   "64b1846740147585fc4d497ba8d20a4b5bdef315",
+   "af064fd9cc422c35718d1dcf2f6c861aab7385cc",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_RSA-OAEP.https.any.js": [
-   "ccb63d91473cb7ee6554ad80f0ee2be004e75c91",
+   "f82ce4307fea94a097ccdf470b766f110d876b87",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_RSA-PSS.https.any.js": [
-   "9549e735a6d5d2a570f2b6ec4b09c49fd6a88f51",
+   "8ff33e42d855b7482455ebb3caab5ac311e7cbf5",
    "testharness"
   ],
   "WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js": [
-   "11aff41d669c5a44c5929ac8754d680b675f3c8c",
+   "fbbbef333461a7845ea1b3cba2169435c7f82761",
    "testharness"
   ],
   "WebCryptoAPI/getRandomValues.any.js": [
@@ -410595,6 +410600,10 @@
    "e3593850f8098d3f3ff82c042deab15f51c66a52",
    "support"
   ],
+  "common/subset-tests.js": [
+   "ee72de5b5f2302d111a0f131eddf5081ad02b789",
+   "support"
+  ],
   "common/test-setting-immutable-prototype.js": [
    "2cafc5c2b867e0fd6f738b1fbeaa503761c400b7",
    "support"
@@ -556388,7 +556397,7 @@
    "support"
   ],
   "docs/_writing-tests/testharness.md": [
-   "6d581e1021a2401b34c0c024990b5e85dbef1fa5",
+   "0a0d88c5ddde91514a51d607405c2b7dedee4776",
    "support"
   ],
   "docs/_writing-tests/visual.md": [

testing/web-platform/tests/WebCryptoAPI/generateKey/successes.js

@@ -70,21 +70,6 @@ function run_test(algorithmNames, slowTest) {
         }, testTag + ": generateKey" + parameterString(algorithm, extractable, usages));
     }
 
-    // Only test a subset of tests with, e.g., ?1-10 in the URL
-    var subTestStart = 0;
-    var subTestEnd = Infinity;
-    var match;
-    if (location.search) {
-        match = /^\?(\d+)-(\d+|last)$/.exec(location.search);
-        if (match) {
-          subTestStart = match[1];
-          if (match[2] !== "last") {
-              subTestEnd = match[2];
-          }
-        }
-    }
-    var currentSubTest = 0;
-
     // Test all valid sets of parameters for successful
     // key generation.
     testVectors.forEach(function(vector) {
@@ -92,10 +77,7 @@ function run_test(algorithmNames, slowTest) {
             allAlgorithmSpecifiersFor(name).forEach(function(algorithm) {
                 allValidUsages(vector.usages, false, vector.mandatoryUsages).forEach(function(usages) {
                     [false, true].forEach(function(extractable) {
-                        currentSubTest++;
-                        if (currentSubTest >= subTestStart && currentSubTest <= subTestEnd) {
-                            testSuccess(algorithm, extractable, usages, vector.resultType, "Success");
-                        }
+                        subsetTest(testSuccess, algorithm, extractable, usages, vector.resultType, "Success");
                     });
                 });
             });

testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-CBC.https.any.js

@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["AES-CBC"]);

testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-CTR.https.any.js

@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["AES-CTR"]);

testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-GCM.https.any.js

@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["AES-GCM"]);

testing/web-platform/tests/WebCryptoAPI/generateKey/successes_AES-KW.https.any.js

@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["AES-KW"]);

testing/web-platform/tests/WebCryptoAPI/generateKey/successes_ECDH.https.any.js

@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["ECDH"]);

testing/web-platform/tests/WebCryptoAPI/generateKey/successes_ECDSA.https.any.js

@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["ECDSA"]);

testing/web-platform/tests/WebCryptoAPI/generateKey/successes_HMAC.https.any.js

@@ -1,4 +1,5 @@
 // META: timeout=long
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["HMAC"]);

testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSA-OAEP.https.any.js

@@ -16,5 +16,6 @@
 // META: variant=?141-150
 // META: variant=?151-last
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["RSA-OAEP"]);

testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSA-PSS.https.any.js

@@ -4,5 +4,6 @@
 // META: variant=?21-30
 // META: variant=?31-last
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["RSA-PSS"]);

testing/web-platform/tests/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js

@@ -4,5 +4,6 @@
 // META: variant=?21-30
 // META: variant=?31-last
 // META: script=../util/helpers.js
+// META: script=/common/subset-tests.js
 // META: script=successes.js
 run_test(["RSASSA-PKCS1-v1_5"]);

testing/web-platform/tests/common/subset-tests.js

@@ -0,0 +1,33 @@
+// Only test a subset of tests with, e.g., ?1-10 in the URL.
+// Can be used together with <meta name="variant" content="...">
+// Sample usage:
+// for (const test of tests) {
+//   subsetTest(async_test, test.fn, test.name);
+// }
+(function() {
+  var subTestStart = 0;
+  var subTestEnd = Infinity;
+  var match;
+  if (location.search) {
+      match = /(?:^\?|&)(\d+)-(\d+|last)(?:&|$)/.exec(location.search);
+      if (match) {
+        subTestStart = parseInt(match[1], 10);
+        if (match[2] !== "last") {
+            subTestEnd = parseInt(match[2], 10);
+        }
+      }
+  }
+  function shouldRunSubTest(currentSubTest) {
+    return currentSubTest >= subTestStart && currentSubTest <= subTestEnd;
+  }
+  var currentSubTest = 0;
+  function subsetTest(testFunc, ...args) {
+    currentSubTest++;
+    if (shouldRunSubTest(currentSubTest)) {
+      return testFunc(...args);
+    }
+    return null;
+  }
+  self.shouldRunSubTest = shouldRunSubTest;
+  self.subsetTest = subsetTest;
+})();

testing/web-platform/tests/docs/_writing-tests/testharness.md

@@ -19,6 +19,44 @@ documented in two sections:
 See [server features][] for advanced testing features that are commonly used
 with testharness.js. See also the [general guidelines][] for all test types.
 
+## Variants
+
+A test file can have multiple variants by including `meta` elements,
+for example:
+
+```
+<meta name="variant" content="">
+<meta name="variant" content="?wss">
+```
+
+The test can then do different things based on the URL.
+
+There is a utility script in `/common/subset-tests.js` that works
+well together with variants, where a test that would otherwise have
+too many tests to be useful can be split up in ranges of subtests.
+For example:
+
+```
+<!doctype html>
+<title>Testing variants</title>
+<meta name="variant" content="?1-1000">
+<meta name="variant" content="?1001-2000">
+<meta name="variant" content="?2001-last">
+<script src="/resources/testharness.js">
+<script src="/resources/testharnessreport.js">
+<script src="/common/subset-tests.js">
+<script>
+ const tests = [
+                 { fn: t => { ... }, name: "..." },
+                 ... lots of tests ...
+               ];
+ for (const test of tests) {
+   subsetTest(async_test, test.fn, test.name);
+ }
+</script>
+```
+
+
 ## Auto-generated test boilerplate
 
 While most JavaScript tests require a certain amount of HTML
@@ -121,7 +159,7 @@ can be used to include both the global and a local `utils.js` in a test.
 
 Use `// META: timeout=long` at the beginning of the resource.
 
-### Specifying test variants in auto-generated boilerplate tests
+### Specifying test [variants](#variants) in auto-generated boilerplate tests
 
 Use `// META: variant=url-suffix` at the beginning of the resource. For example,