Merge pull request #3706 from github/koesie10/upgrade-node-fetch
Upgrade node-fetch to v3
This commit is contained in:
Koen Vlaswinkel
GitHub
Коммит
282bd221fe
|
|
@ -27,7 +27,7 @@
|
|||
"js-yaml": "^4.1.0",
|
||||
"msw": "^2.2.13",
|
||||
"nanoid": "^5.0.7",
|
||||
"node-fetch": "^2.6.7",
|
||||
"node-fetch": "^3.3.2",
|
||||
"p-queue": "^8.0.1",
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
|
|
@ -82,7 +82,6 @@
|
|||
"@types/js-yaml": "^4.0.6",
|
||||
"@types/nanoid": "^3.0.0",
|
||||
"@types/node": "20.15.*",
|
||||
"@types/node-fetch": "^2.5.2",
|
||||
"@types/react": "^18.3.1",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"@types/sarif": "^2.1.2",
|
||||
|
|
@ -6558,16 +6557,6 @@
|
|||
"undici-types": "~6.13.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node-fetch": {
|
||||
"version": "2.6.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.9.tgz",
|
||||
"integrity": "sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@types/node": "*",
|
||||
"form-data": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node/node_modules/undici-types": {
|
||||
"version": "6.13.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.13.0.tgz",
|
||||
|
|
@ -10531,6 +10520,14 @@
|
|||
"integrity": "sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/data-uri-to-buffer": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
|
||||
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
|
||||
"engines": {
|
||||
"node": ">= 12"
|
||||
}
|
||||
},
|
||||
"node_modules/data-urls": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz",
|
||||
|
|
@ -13055,6 +13052,28 @@
|
|||
"pend": "~1.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fetch-blob": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
|
||||
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "paypal",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"dependencies": {
|
||||
"node-domexception": "^1.0.0",
|
||||
"web-streams-polyfill": "^3.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20 || >= 14.13"
|
||||
}
|
||||
},
|
||||
"node_modules/figures": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
|
||||
|
|
@ -13396,6 +13415,17 @@
|
|||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/formdata-polyfill": {
|
||||
"version": "4.0.10",
|
||||
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
|
||||
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
|
||||
"dependencies": {
|
||||
"fetch-blob": "^3.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/forwarded": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
|
||||
|
|
@ -19818,23 +19848,39 @@
|
|||
"node": ">= 0.10.5"
|
||||
}
|
||||
},
|
||||
"node_modules/node-domexception": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
|
||||
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=10.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"version": "3.3.2",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
|
||||
"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
"data-uri-to-buffer": "^4.0.0",
|
||||
"fetch-blob": "^3.1.4",
|
||||
"formdata-polyfill": "^4.0.10"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/node-fetch"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch-native": {
|
||||
|
|
@ -19843,25 +19889,6 @@
|
|||
"integrity": "sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/node-fetch/node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
|
||||
},
|
||||
"node_modules/node-fetch/node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
|
||||
},
|
||||
"node_modules/node-fetch/node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-int64": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
|
||||
|
|
@ -25422,6 +25449,14 @@
|
|||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/web-streams-polyfill": {
|
||||
"version": "3.3.3",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
|
||||
"integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
|
||||
|
|
|
|||
|
|
@ -1984,7 +1984,7 @@
|
|||
"js-yaml": "^4.1.0",
|
||||
"msw": "^2.2.13",
|
||||
"nanoid": "^5.0.7",
|
||||
"node-fetch": "^2.6.7",
|
||||
"node-fetch": "^3.3.2",
|
||||
"p-queue": "^8.0.1",
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
|
|
@ -2039,7 +2039,6 @@
|
|||
"@types/js-yaml": "^4.0.6",
|
||||
"@types/nanoid": "^3.0.0",
|
||||
"@types/node": "20.15.*",
|
||||
"@types/node-fetch": "^2.5.2",
|
||||
"@types/react": "^18.3.1",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"@types/sarif": "^2.1.2",
|
||||
|
|
|
|||
|
|
@ -404,6 +404,11 @@ class ExtensionSpecificDistributionManager {
|
|||
signal,
|
||||
);
|
||||
|
||||
const body = assetStream.body;
|
||||
if (!body) {
|
||||
throw new Error("No body in asset stream");
|
||||
}
|
||||
|
||||
const archivePath = join(tmpDirectory, "distributionDownload.zip");
|
||||
archiveFile = createWriteStream(archivePath);
|
||||
|
||||
|
|
@ -412,26 +417,23 @@ class ExtensionSpecificDistributionManager {
|
|||
? parseInt(contentLength, 10)
|
||||
: undefined;
|
||||
reportStreamProgress(
|
||||
assetStream.body,
|
||||
body,
|
||||
`Downloading CodeQL CLI ${release.name}…`,
|
||||
totalNumBytes,
|
||||
progressCallback,
|
||||
);
|
||||
|
||||
assetStream.body.on("data", onData);
|
||||
body.on("data", onData);
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
if (!archiveFile) {
|
||||
throw new Error("Invariant violation: archiveFile not set");
|
||||
}
|
||||
|
||||
assetStream.body
|
||||
.pipe(archiveFile)
|
||||
.on("finish", resolve)
|
||||
.on("error", reject);
|
||||
body.pipe(archiveFile).on("finish", resolve).on("error", reject);
|
||||
|
||||
// If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error).
|
||||
assetStream.body.on("error", reject);
|
||||
body.on("error", reject);
|
||||
});
|
||||
|
||||
disposeTimeout();
|
||||
|
|
|
|||
|
|
@ -34,9 +34,9 @@ export class ReleasesApiConsumer {
|
|||
additionalCompatibilityCheck?: (release: GithubRelease) => boolean,
|
||||
): Promise<Release> {
|
||||
const apiPath = `/repos/${this.repositoryNwo}/releases`;
|
||||
const allReleases: GithubRelease[] = await (
|
||||
const allReleases = (await (
|
||||
await this.makeApiCall(apiPath)
|
||||
).json();
|
||||
).json()) as GithubRelease[];
|
||||
const compatibleReleases = allReleases.filter((release) => {
|
||||
if (release.prerelease && !includePrerelease) {
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ export interface GetVariantAnalysisRepoResultRequest {
|
|||
};
|
||||
response: {
|
||||
status: number;
|
||||
body?: Buffer | string;
|
||||
body?: ArrayBuffer | string;
|
||||
contentType: string;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,7 +91,14 @@ export class Recorder extends DisposableObject {
|
|||
|
||||
let bodyFileLink = undefined;
|
||||
if (writtenRequest.response.body) {
|
||||
await writeFile(bodyFilePath, writtenRequest.response.body);
|
||||
if (typeof writtenRequest.response.body === "string") {
|
||||
await writeFile(bodyFilePath, writtenRequest.response.body);
|
||||
} else {
|
||||
await writeFile(
|
||||
bodyFilePath,
|
||||
Buffer.from(writtenRequest.response.body),
|
||||
);
|
||||
}
|
||||
bodyFileLink = `file:${bodyFileName}`;
|
||||
}
|
||||
|
||||
|
|
@ -226,7 +233,7 @@ async function createGitHubApiRequest(
|
|||
"x-vscode-codeql-msw-bypass": "true",
|
||||
},
|
||||
});
|
||||
const responseBuffer = await response.buffer();
|
||||
const responseBuffer = await response.arrayBuffer();
|
||||
|
||||
return {
|
||||
request: {
|
||||
|
|
|
|||
|
|
@ -545,30 +545,27 @@ export class DatabaseFetcher {
|
|||
throw e;
|
||||
}
|
||||
|
||||
const body = response.body;
|
||||
if (!body) {
|
||||
throw new Error("No response body found");
|
||||
}
|
||||
|
||||
const archiveFileStream = createWriteStream(archivePath);
|
||||
|
||||
const contentLength = response.headers.get("content-length");
|
||||
const totalNumBytes = contentLength
|
||||
? parseInt(contentLength, 10)
|
||||
: undefined;
|
||||
reportStreamProgress(
|
||||
response.body,
|
||||
"Downloading database",
|
||||
totalNumBytes,
|
||||
progress,
|
||||
);
|
||||
reportStreamProgress(body, "Downloading database", totalNumBytes, progress);
|
||||
|
||||
response.body.on("data", onData);
|
||||
body.on("data", onData);
|
||||
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
response.body
|
||||
.pipe(archiveFileStream)
|
||||
.on("finish", resolve)
|
||||
.on("error", reject);
|
||||
body.pipe(archiveFileStream).on("finish", resolve).on("error", reject);
|
||||
|
||||
// If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error).
|
||||
response.body.on("error", reject);
|
||||
body.on("error", reject);
|
||||
});
|
||||
} catch (e) {
|
||||
// Close and remove the file if an error occurs
|
||||
|
|
|
|||
|
|
@ -99,6 +99,10 @@ export class VariantAnalysisResultsManager extends DisposableObject {
|
|||
responseSize = response.size;
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
throw new Error("No response body found");
|
||||
}
|
||||
|
||||
let amountDownloaded = 0;
|
||||
for await (const chunk of response.body) {
|
||||
await appendFile(zipFilePath, Buffer.from(chunk));
|
||||
|
|
|
|||
|
|
@ -5,10 +5,14 @@ const transformPackages = [
|
|||
"@vscode/webview-ui-toolkit",
|
||||
"before-after-hook",
|
||||
"d3",
|
||||
"data-uri-to-buffer",
|
||||
"delaunator",
|
||||
"exenv-es6",
|
||||
"fetch-blob",
|
||||
"formdata-polyfill",
|
||||
"internmap",
|
||||
"nanoid",
|
||||
"node-fetch",
|
||||
"p-queue",
|
||||
"p-timeout",
|
||||
"robust-predicates",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ import {
|
|||
remove,
|
||||
} from "fs-extra";
|
||||
import { join } from "path";
|
||||
import { Readable } from "stream";
|
||||
import * as fetchModule from "node-fetch";
|
||||
import { Response } from "node-fetch";
|
||||
|
||||
|
|
@ -227,8 +226,7 @@ describe("Variant Analysis Manager", () => {
|
|||
"data/variant-analysis-results.zip",
|
||||
);
|
||||
const fileContents = await readFile(sourceFilePath);
|
||||
const response = new Response(Readable.from(fileContents));
|
||||
response.size = fileContents.length;
|
||||
const response = new Response(fileContents);
|
||||
getVariantAnalysisRepoResultStub.mockResolvedValue(response);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -105,9 +105,9 @@ describe(VariantAnalysisResultsManager.name, () => {
|
|||
|
||||
getVariantAnalysisRepoResultStub = jest
|
||||
.spyOn(fetchModule, "default")
|
||||
.mockImplementation((url: RequestInfo, _init?: RequestInit) => {
|
||||
.mockImplementation((url: URL | RequestInfo, _init?: RequestInit) => {
|
||||
if (url === dummyRepoTask.artifactUrl) {
|
||||
return Promise.resolve(new Response(Readable.from(fileContents)));
|
||||
return Promise.resolve(new Response(fileContents));
|
||||
}
|
||||
return Promise.reject(new Error("Unexpected artifact URL"));
|
||||
});
|
||||
|
|
@ -162,7 +162,7 @@ describe(VariantAnalysisResultsManager.name, () => {
|
|||
}
|
||||
|
||||
getVariantAnalysisRepoResultStub.mockImplementation(
|
||||
(url: RequestInfo, _init?: RequestInit) => {
|
||||
(url: URL | RequestInfo, _init?: RequestInit) => {
|
||||
if (url === dummyRepoTask.artifactUrl) {
|
||||
const response = new Response(Readable.from(generateInParts()));
|
||||
response.headers.set(
|
||||
|
|
|
|||
|
|
@ -23,6 +23,10 @@ beforeAll(async () => {
|
|||
|
||||
await new Promise((resolve, reject) => {
|
||||
return fetch(DB_URL).then((response) => {
|
||||
if (!response.body) {
|
||||
throw new Error("No response body found");
|
||||
}
|
||||
|
||||
const dest = createWriteStream(dbLoc);
|
||||
response.body.pipe(dest);
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import {
|
|||
codeQlLauncherName,
|
||||
} from "../../src/common/distribution";
|
||||
import { unzipToDirectorySequentially } from "../../src/common/unzip";
|
||||
import fetch from "node-fetch";
|
||||
import supportedCliVersions from "../../supported_cli_versions.json";
|
||||
|
||||
/**
|
||||
|
|
@ -112,26 +111,33 @@ async function downloadWithProgress(url: string, filePath: string) {
|
|||
const contentLength = Number(assetStream.headers.get("content-length") || 0);
|
||||
console.log("Total content size", Math.round(contentLength / _1MB), "MB");
|
||||
const archiveFile = createWriteStream(filePath);
|
||||
const body = assetStream.body;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let numBytesDownloaded = 0;
|
||||
let lastMessage = 0;
|
||||
body.on("data", (data) => {
|
||||
numBytesDownloaded += data.length;
|
||||
if (numBytesDownloaded - lastMessage > _10MB) {
|
||||
console.log("Downloaded", Math.round(numBytesDownloaded / _1MB), "MB");
|
||||
lastMessage = numBytesDownloaded;
|
||||
}
|
||||
archiveFile.write(data);
|
||||
});
|
||||
body.on("finish", () => {
|
||||
archiveFile.end(() => {
|
||||
console.log("Finished download into", filePath);
|
||||
resolve();
|
||||
const body = assetStream.body?.getReader();
|
||||
if (!body) {
|
||||
throw new Error("No response body found");
|
||||
}
|
||||
|
||||
let numBytesDownloaded = 0;
|
||||
let lastMessage = 0;
|
||||
|
||||
// eslint-disable-next-line no-constant-condition -- This is a loop that reads from a stream
|
||||
while (true) {
|
||||
const { done, value } = await body.read();
|
||||
if (done) {
|
||||
return new Promise((resolve) => {
|
||||
archiveFile.end(() => {
|
||||
console.log("Finished download into", filePath);
|
||||
resolve(undefined);
|
||||
});
|
||||
});
|
||||
});
|
||||
body.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
numBytesDownloaded += value.length;
|
||||
if (numBytesDownloaded - lastMessage > _10MB) {
|
||||
console.log("Downloaded", Math.round(numBytesDownloaded / _1MB), "MB");
|
||||
lastMessage = numBytesDownloaded;
|
||||
}
|
||||
archiveFile.write(value);
|
||||
}
|
||||
}
|
||||
|
||||
async function unzipWithProgress(
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче