chore: Update CI config to include nodejs v20 (#3014)

* chore: Update CI config to include nodejs v20
* chore: Remove nodejs 16 mention from the README.md
* chore: tweak fake-amo-server to listen on ipv6 localhost on windows
* chore: Remove custom babel-loader (used to workaround a quibble bug on windows)
* chore: make unit test failure non blocking on nodejs experimental ci job
* chore: bump nodejs version range to > v18 in package.json

.circleci/config.yml

@@ -24,11 +24,13 @@ references:
   # set of nodejs versions that should be tested.
   # The nodejs version set as `nodejs_current` is the one used to
   # release the package on npm.
+  #
+  # See https://nodejs.org/en/about/previous-releases for updates to nodejs versions.
   nodejs_versions:
     # nvm-windows wants a full Node version, not just `<major>.<minor>`.
-    - &nodejs_current "16.14.2"
+    - &nodejs_current "18.19.0"
-    - &nodejs_next "18.15"
+    - &nodejs_next "20.11.0"
-    - &nodejs_experimental "19.3"
+    - &nodejs_experimental "21.5"
 
   nodejs_enum: &nodejs_enum
     type: enum
@@ -173,13 +175,24 @@ jobs:
           node_env: test
       ## Skip code coverage and the additional legacy bundling tests on jobs
       ## running on the next nodejs versions.
-      - unless:
+      - when:
           condition:
-            equal: [*nodejs_current, << parameters.nodejs >>]
+            equal: [*nodejs_next, << parameters.nodejs >>]
           steps:
             - run:
-                name: run linting checks and unit tests
+                  name: run linting checks and unit tests
-                command: npm run test
+                  command: npm run test
+            - run_functional_tests
+      ## Allow npm run test to fail when running on nodejs experimental.
+      # TODO(https://github.com/mozilla/web-ext/issues/3015): change this to do not
+      # allow failures on nodejs 21 once fixed by a testdouble dependency update.
+      - when:
+          condition:
+            equal: [*nodejs_experimental, << parameters.nodejs >>]
+          steps:
+            - run:
+                  name: run linting checks and unit tests (but allow failure)
+                  command: npm run test || echo "NOTE - Unit tests failed, but allowed to fail on nodejs experimental"
             - run_functional_tests
       ## Steps only executed in jobs running on the current nodejs version.
       - when:

README.md

@@ -76,7 +76,7 @@ version on the command line with this:
 
 You'll need:
 
-- [Node.js](https://nodejs.org/en/), 16.0.0 or higher
+- [Node.js](https://nodejs.org/en/) (current [LTS](https://github.com/nodejs/LTS))
 - [npm](https://www.npmjs.com/), 8.0.0 or higher is recommended
 
 Optionally, you may like:

package-lock.json

@@ -74,7 +74,7 @@
         "prettyjson": "1.2.5",
         "shelljs": "0.8.5",
         "sinon": "17.0.1",
-        "testdouble": "3.16.8",
+        "testdouble": "3.20.1",
         "yauzl": "2.10.0"
       },
       "engines": {
@@ -10004,17 +10004,16 @@
       ]
     },
     "node_modules/quibble": {
-      "version": "0.6.14",
+      "version": "0.9.1",
-      "resolved": "https://registry.npmjs.org/quibble/-/quibble-0.6.14.tgz",
+      "resolved": "https://registry.npmjs.org/quibble/-/quibble-0.9.1.tgz",
-      "integrity": "sha512-r5noQhWx61qMOjaMQ48ePOKc9MKXzXFKUNj4S7/wIB9rzht3yyyf/Ms3BhXEVEPJtUvTNNnQxnT/6sHzcbkRoA==",
+      "integrity": "sha512-2EkLLm3CsBhbHfYEgBWHSJZZRpVHUZLeuJVEQoU/lsCqxcOvVkgVlF4nWv2ACWKkb0lgxgMh3m8vq9rhx9LTIg==",
       "dev": true,
       "dependencies": {
         "lodash": "^4.17.21",
-        "resolve": "^1.20.0"
+        "resolve": "^1.22.8"
       },
       "engines": {
-        "iojs": ">= 1.0.0",
+        "node": ">= 0.14.0"
-        "node": ">= 0.12.0"
       }
     },
     "node_modules/quick-format-unescaped": {
@@ -11276,18 +11275,18 @@
       }
     },
     "node_modules/testdouble": {
-      "version": "3.16.8",
+      "version": "3.20.1",
-      "resolved": "https://registry.npmjs.org/testdouble/-/testdouble-3.16.8.tgz",
+      "resolved": "https://registry.npmjs.org/testdouble/-/testdouble-3.20.1.tgz",
-      "integrity": "sha512-jOKYRJ9mfgDxwuUOj84sl9DWiP1+KpHcgnhjlSHC8h1ZxJT3KD1FAAFVqnqmmyrzc/+0DRbI/U5xo1/K3PLi8w==",
+      "integrity": "sha512-D9Or6ayxr16dPPEkmXyGb8ow7VcQjUzuYFUxPTkx2FdSkn5Z6EC6cxQHwEGhedmE30FAJOYiAW+r7XXg6FmYOQ==",
       "dev": true,
       "dependencies": {
         "lodash": "^4.17.21",
-        "quibble": "^0.6.14",
+        "quibble": "^0.9.1",
         "stringify-object-es5": "^2.5.0",
         "theredoc": "^1.0.0"
       },
       "engines": {
-        "node": ">= 4.0.0"
+        "node": ">= 16"
       }
     },
     "node_modules/text-extensions": {
@@ -19424,13 +19423,13 @@
       "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="
     },
     "quibble": {
-      "version": "0.6.14",
+      "version": "0.9.1",
-      "resolved": "https://registry.npmjs.org/quibble/-/quibble-0.6.14.tgz",
+      "resolved": "https://registry.npmjs.org/quibble/-/quibble-0.9.1.tgz",
-      "integrity": "sha512-r5noQhWx61qMOjaMQ48ePOKc9MKXzXFKUNj4S7/wIB9rzht3yyyf/Ms3BhXEVEPJtUvTNNnQxnT/6sHzcbkRoA==",
+      "integrity": "sha512-2EkLLm3CsBhbHfYEgBWHSJZZRpVHUZLeuJVEQoU/lsCqxcOvVkgVlF4nWv2ACWKkb0lgxgMh3m8vq9rhx9LTIg==",
       "dev": true,
       "requires": {
         "lodash": "^4.17.21",
-        "resolve": "^1.20.0"
+        "resolve": "^1.22.8"
       }
     },
     "quick-format-unescaped": {
@@ -20377,13 +20376,13 @@
       }
     },
     "testdouble": {
-      "version": "3.16.8",
+      "version": "3.20.1",
-      "resolved": "https://registry.npmjs.org/testdouble/-/testdouble-3.16.8.tgz",
+      "resolved": "https://registry.npmjs.org/testdouble/-/testdouble-3.20.1.tgz",
-      "integrity": "sha512-jOKYRJ9mfgDxwuUOj84sl9DWiP1+KpHcgnhjlSHC8h1ZxJT3KD1FAAFVqnqmmyrzc/+0DRbI/U5xo1/K3PLi8w==",
+      "integrity": "sha512-D9Or6ayxr16dPPEkmXyGb8ow7VcQjUzuYFUxPTkx2FdSkn5Z6EC6cxQHwEGhedmE30FAJOYiAW+r7XXg6FmYOQ==",
       "dev": true,
       "requires": {
         "lodash": "^4.17.21",
-        "quibble": "^0.6.14",
+        "quibble": "^0.9.1",
         "stringify-object-es5": "^2.5.0",
         "theredoc": "^1.0.0"
       }

package.json

@@ -14,7 +14,7 @@
     "lib/**"
   ],
   "engines": {
-    "node": ">=16.0.0",
+    "node": ">=18.0.0",
     "npm": ">=8.0.0"
   },
   "engine-strict": true,
@@ -120,7 +120,7 @@
     "prettyjson": "1.2.5",
     "shelljs": "0.8.5",
     "sinon": "17.0.1",
-    "testdouble": "3.16.8",
+    "testdouble": "3.20.1",
     "yauzl": "2.10.0"
   },
   "author": "Kumar McMillan",

scripts/lib/mocha.js

@@ -20,17 +20,16 @@ const runMocha = (args, execMochaOptions = {}, coverageEnabled) => {
     shell.echo(`\nSetting mocha timeout from env var: ${MOCHA_TIMEOUT}\n`);
   }
 
-  // Pass custom babel-loader node loader to transpile on the fly
+  // Pass testdouble node loader to support ESM module mocking and
-  // the tests modules.
+  // transpiling on the fly the tests modules.
-  binArgs.push('-n="loader=./tests/babel-loader.js"');
+  binArgs.push('-n="loader=testdouble"');
 
   const res = spawnSync(binPath, binArgs, {
     ...execMochaOptions,
     env: {
       ...process.env,
       // Make sure NODE_ENV is set to test (which also enable babel
-      // install plugin for all modules transpiled on the fly by the
+      // install plugin for all modules transpiled on the fly).
-      // tests/babel-loader.js).
       NODE_ENV: 'test',
     },
     stdio: 'inherit',

tests/babel-loader.js

@@ -1,110 +0,0 @@
-/**
- * NOTE: This ESM module is being used as a node loader (https://nodejs.org/api/esm.html#esm_experimental_loaders)
- * while running mocha tests, e.g.:
- *
- *     npx mocha -n "loader=./tests/babel-loader.js" -r tests/setup.js tests/unit/test.config.js
- *
- * It is responsible for the transpiling on the fly of the imported tests modules using babel.
- *
- * The simplified transformSource function that follows has been derived from the existing node ESM loader:
- *
- * - https://github.com/giltayar/babel-register-esm
- *
- * COMPATIBILITY NOTES:
- *
- * nodejs module loader API is experimental and so different nodejs versions expects
- * a different set of hooks:
- *
- * - in nodejs < 16: `getSource` and `transformSource`
- * - in nodejs >= 16: `resolve` and `load`
- */
-
-import path from 'path';
-import { fileURLToPath, pathToFileURL } from 'url';
-
-import babel from '@babel/core';
-import * as td from 'testdouble';
-
-const MODULE_TYPES = ['module', 'commonjs'];
-const TESTS_BASE_URL = path.dirname(import.meta.url);
-const SRC_BASE_URL = pathToFileURL(
-  path.resolve(path.join(fileURLToPath(TESTS_BASE_URL), '..', 'src')),
-).href;
-
-global.__webextMocks = new Set();
-
-// Re-export testdouble nodeloader (used to mock ESM modules).
-export const resolve = td.resolve;
-export const getSource = td.getSource;
-
-const isTestModule = (url) => url.startsWith(TESTS_BASE_URL);
-const isSrcModule = (url) => url.startsWith(SRC_BASE_URL);
-const needsTranspile = (url) => isTestModule(url) || isSrcModule(url);
-
-function hasMock(url) {
-  if (!isSrcModule(url)) {
-    return false;
-  }
-  const cleanURL = pathToFileURL(fileURLToPath(url)).href;
-  return global.__webextMocks.has(cleanURL);
-}
-
-/**
- * @param {string} url
- * @param {{ format: string, url: string }} context
- * @param {Function} [defaultLoad]
- * @returns {Promise<{ source: (string | SharedArrayBuffer | Uint8Array) }>}
- */
-export async function load(url, context, defaultLoad) {
-  if (hasMock(url) || !needsTranspile(url)) {
-    return td.load(url, context, defaultLoad);
-  }
-
-  const { source: rawSource } = await defaultLoad(url, { format: 'module' });
-
-  const result = {
-    format: 'module',
-    ...(await transformSource(rawSource, { url, format: 'module' })),
-  };
-  return result;
-}
-
-/**
- * @param {string | SharedArrayBuffer | Uint8Array} source
- * @param {{ format: string, url: string }} context
- * @param {Function} [defaultTransformSource]
- * @returns {Promise<{ source: (string | SharedArrayBuffer | Uint8Array) }>}
- */
-export async function transformSource(source, context, defaultTransformSource) {
-  const { url, format } = context;
-  if (!MODULE_TYPES.includes(format) || !needsTranspile(url) || hasMock(url)) {
-    if (defaultTransformSource) {
-      return defaultTransformSource(source, context, defaultTransformSource);
-    } else {
-      return { source };
-    }
-  }
-
-  // Transpile tests-related modules on the fly using babel.
-  const stringSource =
-    typeof source === 'string'
-      ? source
-      : Buffer.isBuffer(source)
-        ? source.toString('utf-8')
-        : Buffer.from(source).toString('utf-8');
-
-  let sourceCode = await babel.transformAsync(stringSource, {
-    sourceType: 'module',
-    filename: fileURLToPath(url),
-  });
-
-  sourceCode = sourceCode ? sourceCode.code : undefined;
-
-  if (!sourceCode) {
-    throw new Error(
-      `tests/babel-loader.js: undefined babel transform result for ${url}`,
-    );
-  }
-
-  return { source: sourceCode };
-}

tests/functional/fake-amo-server.js

@@ -89,7 +89,7 @@ http
       process.exit(1);
     }
   })
-  .listen(8989, '127.0.0.1', () => {
+  .listen(8989, 'localhost', () => {
     process.stdout.write('listening');
     process.stdout.uncork();
   });

tests/unit/helpers.js

@@ -331,10 +331,8 @@ export function mockModule({
   ).href;
 
   td.replaceEsm(fullModuleURL, namedExports, defaultExport);
-  global.__webextMocks?.add(fullModuleURL);
 }
 
 export function resetMockModules() {
   td.reset();
-  global.__webextMocks?.clear();
 }

tests/unit/test-util/test.submit-addon.js

@@ -28,8 +28,15 @@ class JSONResponse extends Response {
 }
 
 const mockNodeFetch = (nodeFetchStub, url, method, responses) => {
+  // Trust us... You don't want to know why... but if you really do like nightmares
+  // take a look to the details and links kindly provided in this comment
+  // that helped investigating this:
+  // https://github.com/mozilla/web-ext/issues/2917#issuecomment-1766000545
+  const urlMatch = url instanceof URL ? url.href : url;
   const stubMatcher = nodeFetchStub.withArgs(
-    url instanceof URL ? url : new URL(url),
+    sinon.match(
+      (urlArg) => urlMatch === (urlArg instanceof URL ? urlArg.href : urlArg),
+    ),
     sinon.match.has('method', method),
   );
   for (let i = 0; i < responses.length; i++) {