Upgrade node-fetch to v3

2024-09-05 16:55:10 +02:00 · 2024-09-05 16:55:10 +02:00 · d0c9849ff0
--- a/extensions/ql-vscode/package-lock.json
+++ b/extensions/ql-vscode/package-lock.json
@ -27,7 +27,7 @@
        "js-yaml": "^4.1.0",
        "msw": "^2.2.13",
        "nanoid": "^5.0.7",
-        "node-fetch": "^2.6.7",
+        "node-fetch": "^3.3.2",
        "p-queue": "^8.0.1",
        "react": "^18.3.1",
        "react-dom": "^18.3.1",
@ -82,7 +82,6 @@
        "@types/js-yaml": "^4.0.6",
        "@types/nanoid": "^3.0.0",
        "@types/node": "20.14.*",
-        "@types/node-fetch": "^2.5.2",
        "@types/react": "^18.3.1",
        "@types/react-dom": "^18.3.0",
        "@types/sarif": "^2.1.2",
@ -6558,16 +6557,6 @@
        "undici-types": "~5.26.4"
      }
    },
-    "node_modules/@types/node-fetch": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.9.tgz",
-      "integrity": "sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==",
-      "dev": true,
-      "dependencies": {
-        "@types/node": "*",
-        "form-data": "^4.0.0"
-      }
-    },
    "node_modules/@types/parse-json": {
      "version": "4.0.2",
      "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz",
@ -10526,6 +10515,14 @@
      "integrity": "sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==",
      "dev": true
    },
+    "node_modules/data-uri-to-buffer": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
+      "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
+      "engines": {
+        "node": ">= 12"
+      }
+    },
    "node_modules/data-urls": {
      "version": "3.0.2",
      "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz",
@ -13050,6 +13047,28 @@
        "pend": "~1.2.0"
      }
    },
+    "node_modules/fetch-blob": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
+      "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/jimmywarting"
+        },
+        {
+          "type": "paypal",
+          "url": "https://paypal.me/jimmywarting"
+        }
+      ],
+      "dependencies": {
+        "node-domexception": "^1.0.0",
+        "web-streams-polyfill": "^3.0.3"
+      },
+      "engines": {
+        "node": "^12.20 || >= 14.13"
+      }
+    },
    "node_modules/figures": {
      "version": "3.2.0",
      "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
@ -13391,6 +13410,17 @@
        "node": ">= 6"
      }
    },
+    "node_modules/formdata-polyfill": {
+      "version": "4.0.10",
+      "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
+      "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
+      "dependencies": {
+        "fetch-blob": "^3.1.2"
+      },
+      "engines": {
+        "node": ">=12.20.0"
+      }
+    },
    "node_modules/forwarded": {
      "version": "0.2.0",
      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@ -19813,23 +19843,39 @@
        "node": ">= 0.10.5"
      }
    },
+    "node_modules/node-domexception": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
+      "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/jimmywarting"
+        },
+        {
+          "type": "github",
+          "url": "https://paypal.me/jimmywarting"
+        }
+      ],
+      "engines": {
+        "node": ">=10.5.0"
+      }
+    },
    "node_modules/node-fetch": {
-      "version": "2.7.0",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
-      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+      "version": "3.3.2",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
+      "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
      "dependencies": {
-        "whatwg-url": "^5.0.0"
+        "data-uri-to-buffer": "^4.0.0",
+        "fetch-blob": "^3.1.4",
+        "formdata-polyfill": "^4.0.10"
      },
      "engines": {
-        "node": "4.x || >=6.0.0"
+        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
      },
-      "peerDependencies": {
-        "encoding": "^0.1.0"
-      },
-      "peerDependenciesMeta": {
-        "encoding": {
-          "optional": true
-        }
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/node-fetch"
      }
    },
    "node_modules/node-fetch-native": {
@ -19838,25 +19884,6 @@
      "integrity": "sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ==",
      "dev": true
    },
-    "node_modules/node-fetch/node_modules/tr46": {
-      "version": "0.0.3",
-      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
-    },
-    "node_modules/node-fetch/node_modules/webidl-conversions": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
-    },
-    "node_modules/node-fetch/node_modules/whatwg-url": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
-      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
-      "dependencies": {
-        "tr46": "~0.0.3",
-        "webidl-conversions": "^3.0.0"
-      }
-    },
    "node_modules/node-int64": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
@ -25416,6 +25443,14 @@
        "defaults": "^1.0.3"
      }
    },
+    "node_modules/web-streams-polyfill": {
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
+      "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
    "node_modules/webidl-conversions": {
      "version": "7.0.0",
      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
--- a/extensions/ql-vscode/package.json
+++ b/extensions/ql-vscode/package.json
@ -1984,7 +1984,7 @@
    "js-yaml": "^4.1.0",
    "msw": "^2.2.13",
    "nanoid": "^5.0.7",
-    "node-fetch": "^2.6.7",
+    "node-fetch": "^3.3.2",
    "p-queue": "^8.0.1",
    "react": "^18.3.1",
    "react-dom": "^18.3.1",
@ -2039,7 +2039,6 @@
    "@types/js-yaml": "^4.0.6",
    "@types/nanoid": "^3.0.0",
    "@types/node": "20.14.*",
-    "@types/node-fetch": "^2.5.2",
    "@types/react": "^18.3.1",
    "@types/react-dom": "^18.3.0",
    "@types/sarif": "^2.1.2",
--- a/extensions/ql-vscode/src/codeql-cli/distribution.ts
+++ b/extensions/ql-vscode/src/codeql-cli/distribution.ts
@ -404,6 +404,11 @@ class ExtensionSpecificDistributionManager {
          signal,
        );

+      const body = assetStream.body;
+      if (!body) {
+        throw new Error("No body in asset stream");
+      }
+
      const archivePath = join(tmpDirectory, "distributionDownload.zip");
      archiveFile = createWriteStream(archivePath);

@ -412,26 +417,23 @@ class ExtensionSpecificDistributionManager {
        ? parseInt(contentLength, 10)
        : undefined;
      reportStreamProgress(
-        assetStream.body,
+        body,
        `Downloading CodeQL CLI ${release.name}…`,
        totalNumBytes,
        progressCallback,
      );

-      assetStream.body.on("data", onData);
+      body.on("data", onData);

      await new Promise((resolve, reject) => {
        if (!archiveFile) {
          throw new Error("Invariant violation: archiveFile not set");
        }

-        assetStream.body
-          .pipe(archiveFile)
-          .on("finish", resolve)
-          .on("error", reject);
+        body.pipe(archiveFile).on("finish", resolve).on("error", reject);

        // If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error).
-        assetStream.body.on("error", reject);
+        body.on("error", reject);
      });

      disposeTimeout();
--- a/extensions/ql-vscode/src/codeql-cli/distribution/releases-api-consumer.ts
+++ b/extensions/ql-vscode/src/codeql-cli/distribution/releases-api-consumer.ts
@ -34,9 +34,9 @@ export class ReleasesApiConsumer {
    additionalCompatibilityCheck?: (release: GithubRelease) => boolean,
  ): Promise<Release> {
    const apiPath = `/repos/${this.repositoryNwo}/releases`;
-    const allReleases: GithubRelease[] = await (
+    const allReleases = (await (
      await this.makeApiCall(apiPath)
-    ).json();
+    ).json()) as GithubRelease[];
    const compatibleReleases = allReleases.filter((release) => {
      if (release.prerelease && !includePrerelease) {
        return false;
--- a/extensions/ql-vscode/src/common/mock-gh-api/gh-api-request.ts
+++ b/extensions/ql-vscode/src/common/mock-gh-api/gh-api-request.ts
@ -69,7 +69,7 @@ export interface GetVariantAnalysisRepoResultRequest {
  };
  response: {
    status: number;
-    body?: Buffer | string;
+    body?: ArrayBuffer | string;
    contentType: string;
  };
 }
--- a/extensions/ql-vscode/src/common/mock-gh-api/recorder.ts
+++ b/extensions/ql-vscode/src/common/mock-gh-api/recorder.ts
@ -91,7 +91,14 @@ export class Recorder extends DisposableObject {

        let bodyFileLink = undefined;
        if (writtenRequest.response.body) {
-          await writeFile(bodyFilePath, writtenRequest.response.body);
+          if (typeof writtenRequest.response.body === "string") {
+            await writeFile(bodyFilePath, writtenRequest.response.body);
+          } else {
+            await writeFile(
+              bodyFilePath,
+              Buffer.from(writtenRequest.response.body),
+            );
+          }
          bodyFileLink = `file:${bodyFileName}`;
        }

@ -226,7 +233,7 @@ async function createGitHubApiRequest(
        "x-vscode-codeql-msw-bypass": "true",
      },
    });
-    const responseBuffer = await response.buffer();
+    const responseBuffer = await response.arrayBuffer();

    return {
      request: {
--- a/extensions/ql-vscode/src/databases/database-fetcher.ts
+++ b/extensions/ql-vscode/src/databases/database-fetcher.ts
@ -545,30 +545,27 @@ export class DatabaseFetcher {
      throw e;
    }

+    const body = response.body;
+    if (!body) {
+      throw new Error("No response body found");
+    }
+
    const archiveFileStream = createWriteStream(archivePath);

    const contentLength = response.headers.get("content-length");
    const totalNumBytes = contentLength
      ? parseInt(contentLength, 10)
      : undefined;
-    reportStreamProgress(
-      response.body,
-      "Downloading database",
-      totalNumBytes,
-      progress,
-    );
+    reportStreamProgress(body, "Downloading database", totalNumBytes, progress);

-    response.body.on("data", onData);
+    body.on("data", onData);

    try {
      await new Promise((resolve, reject) => {
-        response.body
-          .pipe(archiveFileStream)
-          .on("finish", resolve)
-          .on("error", reject);
+        body.pipe(archiveFileStream).on("finish", resolve).on("error", reject);

        // If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error).
-        response.body.on("error", reject);
+        body.on("error", reject);
      });
    } catch (e) {
      // Close and remove the file if an error occurs
--- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts
+++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts
@ -99,6 +99,10 @@ export class VariantAnalysisResultsManager extends DisposableObject {
      responseSize = response.size;
    }

+    if (!response.body) {
+      throw new Error("No response body found");
+    }
+
    let amountDownloaded = 0;
    for await (const chunk of response.body) {
      await appendFile(zipFilePath, Buffer.from(chunk));
--- a/extensions/ql-vscode/test/jest-config.ts
+++ b/extensions/ql-vscode/test/jest-config.ts
@ -5,10 +5,14 @@ const transformPackages = [
  "@vscode/webview-ui-toolkit",
  "before-after-hook",
  "d3",
+  "data-uri-to-buffer",
  "delaunator",
  "exenv-es6",
+  "fetch-blob",
+  "formdata-polyfill",
  "internmap",
  "nanoid",
+  "node-fetch",
  "p-queue",
  "p-timeout",
  "robust-predicates",
--- a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-manager.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-manager.test.ts
@ -12,7 +12,6 @@ import {
  remove,
 } from "fs-extra";
 import { join } from "path";
-import { Readable } from "stream";
 import * as fetchModule from "node-fetch";
 import { Response } from "node-fetch";

@ -227,8 +226,7 @@ describe("Variant Analysis Manager", () => {
          "data/variant-analysis-results.zip",
        );
        const fileContents = await readFile(sourceFilePath);
-        const response = new Response(Readable.from(fileContents));
-        response.size = fileContents.length;
+        const response = new Response(fileContents);
        getVariantAnalysisRepoResultStub.mockResolvedValue(response);
      });

--- a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-results-manager.test.ts
+++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-results-manager.test.ts
@ -105,9 +105,9 @@ describe(VariantAnalysisResultsManager.name, () => {

        getVariantAnalysisRepoResultStub = jest
          .spyOn(fetchModule, "default")
-          .mockImplementation((url: RequestInfo, _init?: RequestInit) => {
+          .mockImplementation((url: URL | RequestInfo, _init?: RequestInit) => {
            if (url === dummyRepoTask.artifactUrl) {
-              return Promise.resolve(new Response(Readable.from(fileContents)));
+              return Promise.resolve(new Response(fileContents));
            }
            return Promise.reject(new Error("Unexpected artifact URL"));
          });
@ -162,7 +162,7 @@ describe(VariantAnalysisResultsManager.name, () => {
        }

        getVariantAnalysisRepoResultStub.mockImplementation(
-          (url: RequestInfo, _init?: RequestInit) => {
+          (url: URL | RequestInfo, _init?: RequestInit) => {
            if (url === dummyRepoTask.artifactUrl) {
              const response = new Response(Readable.from(generateInParts()));
              response.headers.set(
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts
@ -23,6 +23,10 @@ beforeAll(async () => {

    await new Promise((resolve, reject) => {
      return fetch(DB_URL).then((response) => {
+        if (!response.body) {
+          throw new Error("No response body found");
+        }
+
        const dest = createWriteStream(dbLoc);
        response.body.pipe(dest);

--- a/extensions/ql-vscode/test/vscode-tests/ensureCli.ts
+++ b/extensions/ql-vscode/test/vscode-tests/ensureCli.ts
@ -5,7 +5,6 @@ import {
  codeQlLauncherName,
 } from "../../src/common/distribution";
 import { unzipToDirectorySequentially } from "../../src/common/unzip";
-import fetch from "node-fetch";
 import supportedCliVersions from "../../supported_cli_versions.json";

 /**
@ -112,26 +111,33 @@ async function downloadWithProgress(url: string, filePath: string) {
  const contentLength = Number(assetStream.headers.get("content-length") || 0);
  console.log("Total content size", Math.round(contentLength / _1MB), "MB");
  const archiveFile = createWriteStream(filePath);
-  const body = assetStream.body;
-  await new Promise<void>((resolve, reject) => {
-    let numBytesDownloaded = 0;
-    let lastMessage = 0;
-    body.on("data", (data) => {
-      numBytesDownloaded += data.length;
-      if (numBytesDownloaded - lastMessage > _10MB) {
-        console.log("Downloaded", Math.round(numBytesDownloaded / _1MB), "MB");
-        lastMessage = numBytesDownloaded;
-      }
-      archiveFile.write(data);
-    });
-    body.on("finish", () => {
-      archiveFile.end(() => {
-        console.log("Finished download into", filePath);
-        resolve();
+  const body = assetStream.body?.getReader();
+  if (!body) {
+    throw new Error("No response body found");
+  }
+
+  let numBytesDownloaded = 0;
+  let lastMessage = 0;
+
+  // eslint-disable-next-line no-constant-condition -- This is a loop that reads from a stream
+  while (true) {
+    const { done, value } = await body.read();
+    if (done) {
+      return new Promise((resolve) => {
+        archiveFile.end(() => {
+          console.log("Finished download into", filePath);
+          resolve(undefined);
+        });
      });
-    });
-    body.on("error", reject);
-  });
+    }
+
+    numBytesDownloaded += value.length;
+    if (numBytesDownloaded - lastMessage > _10MB) {
+      console.log("Downloaded", Math.round(numBytesDownloaded / _1MB), "MB");
+      lastMessage = numBytesDownloaded;
+    }
+    archiveFile.write(value);
+  }
 }

 async function unzipWithProgress(