diff --git a/extensions/ql-vscode/package-lock.json b/extensions/ql-vscode/package-lock.json index ce443b0f3..4a086c9e7 100644 --- a/extensions/ql-vscode/package-lock.json +++ b/extensions/ql-vscode/package-lock.json @@ -27,7 +27,7 @@ "js-yaml": "^4.1.0", "msw": "^2.2.13", "nanoid": "^5.0.7", - "node-fetch": "^2.6.7", + "node-fetch": "^3.3.2", "p-queue": "^8.0.1", "react": "^18.3.1", "react-dom": "^18.3.1", @@ -82,7 +82,6 @@ "@types/js-yaml": "^4.0.6", "@types/nanoid": "^3.0.0", "@types/node": "20.14.*", - "@types/node-fetch": "^2.5.2", "@types/react": "^18.3.1", "@types/react-dom": "^18.3.0", "@types/sarif": "^2.1.2", @@ -6558,16 +6557,6 @@ "undici-types": "~5.26.4" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.9.tgz", - "integrity": "sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==", - "dev": true, - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.0" - } - }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -10526,6 +10515,14 @@ "integrity": "sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==", "dev": true }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, "node_modules/data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -13050,6 +13047,28 @@ "pend": "~1.2.0" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -13391,6 +13410,17 @@ "node": ">= 6" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -19813,23 +19843,39 @@ "node": ">= 0.10.5" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", "dependencies": { - "whatwg-url": "^5.0.0" + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" }, "engines": { - "node": "4.x || >=6.0.0" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" } }, "node_modules/node-fetch-native": { @@ -19838,25 +19884,6 @@ "integrity": "sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ==", "dev": true }, - "node_modules/node-fetch/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "node_modules/node-fetch/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/node-fetch/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -25416,6 +25443,14 @@ "defaults": "^1.0.3" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json index e36b45160..7d9eb7e9e 100644 --- a/extensions/ql-vscode/package.json +++ b/extensions/ql-vscode/package.json @@ -1984,7 +1984,7 @@ "js-yaml": "^4.1.0", "msw": "^2.2.13", "nanoid": "^5.0.7", - "node-fetch": "^2.6.7", + "node-fetch": "^3.3.2", "p-queue": "^8.0.1", "react": "^18.3.1", "react-dom": "^18.3.1", @@ -2039,7 +2039,6 @@ "@types/js-yaml": "^4.0.6", "@types/nanoid": "^3.0.0", "@types/node": "20.14.*", - "@types/node-fetch": "^2.5.2", "@types/react": "^18.3.1", "@types/react-dom": "^18.3.0", "@types/sarif": "^2.1.2", diff --git a/extensions/ql-vscode/src/codeql-cli/distribution.ts b/extensions/ql-vscode/src/codeql-cli/distribution.ts index 223bf5fe5..f0da0530e 100644 --- a/extensions/ql-vscode/src/codeql-cli/distribution.ts +++ b/extensions/ql-vscode/src/codeql-cli/distribution.ts @@ -404,6 +404,11 @@ class ExtensionSpecificDistributionManager { signal, ); + const body = assetStream.body; + if (!body) { + throw new Error("No body in asset stream"); + } + const archivePath = join(tmpDirectory, "distributionDownload.zip"); archiveFile = createWriteStream(archivePath); @@ -412,26 +417,23 @@ class ExtensionSpecificDistributionManager { ? parseInt(contentLength, 10) : undefined; reportStreamProgress( - assetStream.body, + body, `Downloading CodeQL CLI ${release.name}…`, totalNumBytes, progressCallback, ); - assetStream.body.on("data", onData); + body.on("data", onData); await new Promise((resolve, reject) => { if (!archiveFile) { throw new Error("Invariant violation: archiveFile not set"); } - assetStream.body - .pipe(archiveFile) - .on("finish", resolve) - .on("error", reject); + body.pipe(archiveFile).on("finish", resolve).on("error", reject); // If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error). - assetStream.body.on("error", reject); + body.on("error", reject); }); disposeTimeout(); diff --git a/extensions/ql-vscode/src/codeql-cli/distribution/releases-api-consumer.ts b/extensions/ql-vscode/src/codeql-cli/distribution/releases-api-consumer.ts index ff482a7b2..6be8a4671 100644 --- a/extensions/ql-vscode/src/codeql-cli/distribution/releases-api-consumer.ts +++ b/extensions/ql-vscode/src/codeql-cli/distribution/releases-api-consumer.ts @@ -34,9 +34,9 @@ export class ReleasesApiConsumer { additionalCompatibilityCheck?: (release: GithubRelease) => boolean, ): Promise { const apiPath = `/repos/${this.repositoryNwo}/releases`; - const allReleases: GithubRelease[] = await ( + const allReleases = (await ( await this.makeApiCall(apiPath) - ).json(); + ).json()) as GithubRelease[]; const compatibleReleases = allReleases.filter((release) => { if (release.prerelease && !includePrerelease) { return false; diff --git a/extensions/ql-vscode/src/common/mock-gh-api/gh-api-request.ts b/extensions/ql-vscode/src/common/mock-gh-api/gh-api-request.ts index b9c08955c..bf77bb11f 100644 --- a/extensions/ql-vscode/src/common/mock-gh-api/gh-api-request.ts +++ b/extensions/ql-vscode/src/common/mock-gh-api/gh-api-request.ts @@ -69,7 +69,7 @@ export interface GetVariantAnalysisRepoResultRequest { }; response: { status: number; - body?: Buffer | string; + body?: ArrayBuffer | string; contentType: string; }; } diff --git a/extensions/ql-vscode/src/common/mock-gh-api/recorder.ts b/extensions/ql-vscode/src/common/mock-gh-api/recorder.ts index 234e42a79..a7773aeb1 100644 --- a/extensions/ql-vscode/src/common/mock-gh-api/recorder.ts +++ b/extensions/ql-vscode/src/common/mock-gh-api/recorder.ts @@ -91,7 +91,14 @@ export class Recorder extends DisposableObject { let bodyFileLink = undefined; if (writtenRequest.response.body) { - await writeFile(bodyFilePath, writtenRequest.response.body); + if (typeof writtenRequest.response.body === "string") { + await writeFile(bodyFilePath, writtenRequest.response.body); + } else { + await writeFile( + bodyFilePath, + Buffer.from(writtenRequest.response.body), + ); + } bodyFileLink = `file:${bodyFileName}`; } @@ -226,7 +233,7 @@ async function createGitHubApiRequest( "x-vscode-codeql-msw-bypass": "true", }, }); - const responseBuffer = await response.buffer(); + const responseBuffer = await response.arrayBuffer(); return { request: { diff --git a/extensions/ql-vscode/src/databases/database-fetcher.ts b/extensions/ql-vscode/src/databases/database-fetcher.ts index d6ce79c5e..473788880 100644 --- a/extensions/ql-vscode/src/databases/database-fetcher.ts +++ b/extensions/ql-vscode/src/databases/database-fetcher.ts @@ -545,30 +545,27 @@ export class DatabaseFetcher { throw e; } + const body = response.body; + if (!body) { + throw new Error("No response body found"); + } + const archiveFileStream = createWriteStream(archivePath); const contentLength = response.headers.get("content-length"); const totalNumBytes = contentLength ? parseInt(contentLength, 10) : undefined; - reportStreamProgress( - response.body, - "Downloading database", - totalNumBytes, - progress, - ); + reportStreamProgress(body, "Downloading database", totalNumBytes, progress); - response.body.on("data", onData); + body.on("data", onData); try { await new Promise((resolve, reject) => { - response.body - .pipe(archiveFileStream) - .on("finish", resolve) - .on("error", reject); + body.pipe(archiveFileStream).on("finish", resolve).on("error", reject); // If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error). - response.body.on("error", reject); + body.on("error", reject); }); } catch (e) { // Close and remove the file if an error occurs diff --git a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts index e7856ba59..9723ef76d 100644 --- a/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts +++ b/extensions/ql-vscode/src/variant-analysis/variant-analysis-results-manager.ts @@ -99,6 +99,10 @@ export class VariantAnalysisResultsManager extends DisposableObject { responseSize = response.size; } + if (!response.body) { + throw new Error("No response body found"); + } + let amountDownloaded = 0; for await (const chunk of response.body) { await appendFile(zipFilePath, Buffer.from(chunk)); diff --git a/extensions/ql-vscode/test/jest-config.ts b/extensions/ql-vscode/test/jest-config.ts index ab0b72243..fa3dd63d2 100644 --- a/extensions/ql-vscode/test/jest-config.ts +++ b/extensions/ql-vscode/test/jest-config.ts @@ -5,10 +5,14 @@ const transformPackages = [ "@vscode/webview-ui-toolkit", "before-after-hook", "d3", + "data-uri-to-buffer", "delaunator", "exenv-es6", + "fetch-blob", + "formdata-polyfill", "internmap", "nanoid", + "node-fetch", "p-queue", "p-timeout", "robust-predicates", diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-manager.test.ts b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-manager.test.ts index 137478710..35d7ec595 100644 --- a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-manager.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-manager.test.ts @@ -12,7 +12,6 @@ import { remove, } from "fs-extra"; import { join } from "path"; -import { Readable } from "stream"; import * as fetchModule from "node-fetch"; import { Response } from "node-fetch"; @@ -227,8 +226,7 @@ describe("Variant Analysis Manager", () => { "data/variant-analysis-results.zip", ); const fileContents = await readFile(sourceFilePath); - const response = new Response(Readable.from(fileContents)); - response.size = fileContents.length; + const response = new Response(fileContents); getVariantAnalysisRepoResultStub.mockResolvedValue(response); }); diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-results-manager.test.ts b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-results-manager.test.ts index 9fa66973e..6be349c85 100644 --- a/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-results-manager.test.ts +++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/variant-analysis/variant-analysis-results-manager.test.ts @@ -105,9 +105,9 @@ describe(VariantAnalysisResultsManager.name, () => { getVariantAnalysisRepoResultStub = jest .spyOn(fetchModule, "default") - .mockImplementation((url: RequestInfo, _init?: RequestInit) => { + .mockImplementation((url: URL | RequestInfo, _init?: RequestInit) => { if (url === dummyRepoTask.artifactUrl) { - return Promise.resolve(new Response(Readable.from(fileContents))); + return Promise.resolve(new Response(fileContents)); } return Promise.reject(new Error("Unexpected artifact URL")); }); @@ -162,7 +162,7 @@ describe(VariantAnalysisResultsManager.name, () => { } getVariantAnalysisRepoResultStub.mockImplementation( - (url: RequestInfo, _init?: RequestInit) => { + (url: URL | RequestInfo, _init?: RequestInit) => { if (url === dummyRepoTask.artifactUrl) { const response = new Response(Readable.from(generateInParts())); response.headers.set( diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts index c3990282a..41d62e61a 100644 --- a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts +++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.setup.ts @@ -23,6 +23,10 @@ beforeAll(async () => { await new Promise((resolve, reject) => { return fetch(DB_URL).then((response) => { + if (!response.body) { + throw new Error("No response body found"); + } + const dest = createWriteStream(dbLoc); response.body.pipe(dest); diff --git a/extensions/ql-vscode/test/vscode-tests/ensureCli.ts b/extensions/ql-vscode/test/vscode-tests/ensureCli.ts index 15e88c284..4ddad1afb 100644 --- a/extensions/ql-vscode/test/vscode-tests/ensureCli.ts +++ b/extensions/ql-vscode/test/vscode-tests/ensureCli.ts @@ -5,7 +5,6 @@ import { codeQlLauncherName, } from "../../src/common/distribution"; import { unzipToDirectorySequentially } from "../../src/common/unzip"; -import fetch from "node-fetch"; import supportedCliVersions from "../../supported_cli_versions.json"; /** @@ -112,26 +111,33 @@ async function downloadWithProgress(url: string, filePath: string) { const contentLength = Number(assetStream.headers.get("content-length") || 0); console.log("Total content size", Math.round(contentLength / _1MB), "MB"); const archiveFile = createWriteStream(filePath); - const body = assetStream.body; - await new Promise((resolve, reject) => { - let numBytesDownloaded = 0; - let lastMessage = 0; - body.on("data", (data) => { - numBytesDownloaded += data.length; - if (numBytesDownloaded - lastMessage > _10MB) { - console.log("Downloaded", Math.round(numBytesDownloaded / _1MB), "MB"); - lastMessage = numBytesDownloaded; - } - archiveFile.write(data); - }); - body.on("finish", () => { - archiveFile.end(() => { - console.log("Finished download into", filePath); - resolve(); + const body = assetStream.body?.getReader(); + if (!body) { + throw new Error("No response body found"); + } + + let numBytesDownloaded = 0; + let lastMessage = 0; + + // eslint-disable-next-line no-constant-condition -- This is a loop that reads from a stream + while (true) { + const { done, value } = await body.read(); + if (done) { + return new Promise((resolve) => { + archiveFile.end(() => { + console.log("Finished download into", filePath); + resolve(undefined); + }); }); - }); - body.on("error", reject); - }); + } + + numBytesDownloaded += value.length; + if (numBytesDownloaded - lastMessage > _10MB) { + console.log("Downloaded", Math.round(numBytesDownloaded / _1MB), "MB"); + lastMessage = numBytesDownloaded; + } + archiveFile.write(value); + } } async function unzipWithProgress(