Merge pull request #3750 from github/koesie10/switch-to-native-fetch
Switch from node-fetch to native Node.js fetch
This commit is contained in:
Koen Vlaswinkel
GitHub
Коммит
4e342889d9
|
|
@ -3,6 +3,7 @@
|
|||
## [UNRELEASED]
|
||||
|
||||
- Increase the required version of VS Code to 1.90.0. [#3737](https://github.com/github/vscode-codeql/pull/3737)
|
||||
- Fix a bug where some variant analysis results failed to download. [#3750](https://github.com/github/vscode-codeql/pull/3750)
|
||||
|
||||
## 1.15.0 - 26 September 2024
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@
|
|||
"js-yaml": "^4.1.0",
|
||||
"msw": "^2.2.13",
|
||||
"nanoid": "^5.0.7",
|
||||
"node-fetch": "^3.3.2",
|
||||
"p-queue": "^8.0.1",
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
|
|
@ -10098,14 +10097,6 @@
|
|||
"integrity": "sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/data-uri-to-buffer": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
|
||||
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
|
||||
"engines": {
|
||||
"node": ">= 12"
|
||||
}
|
||||
},
|
||||
"node_modules/data-urls": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz",
|
||||
|
|
@ -12456,28 +12447,6 @@
|
|||
"pend": "~1.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fetch-blob": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
|
||||
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "paypal",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"dependencies": {
|
||||
"node-domexception": "^1.0.0",
|
||||
"web-streams-polyfill": "^3.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20 || >= 14.13"
|
||||
}
|
||||
},
|
||||
"node_modules/figures": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
|
||||
|
|
@ -12779,17 +12748,6 @@
|
|||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/formdata-polyfill": {
|
||||
"version": "4.0.10",
|
||||
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
|
||||
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
|
||||
"dependencies": {
|
||||
"fetch-blob": "^3.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/forwarded": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
|
||||
|
|
@ -18503,41 +18461,6 @@
|
|||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/node-domexception": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
|
||||
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=10.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "3.3.2",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
|
||||
"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
|
||||
"dependencies": {
|
||||
"data-uri-to-buffer": "^4.0.0",
|
||||
"fetch-blob": "^3.1.4",
|
||||
"formdata-polyfill": "^4.0.10"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/node-fetch"
|
||||
}
|
||||
},
|
||||
"node_modules/node-int64": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
|
||||
|
|
@ -23642,14 +23565,6 @@
|
|||
"makeerror": "1.0.12"
|
||||
}
|
||||
},
|
||||
"node_modules/web-streams-polyfill": {
|
||||
"version": "3.3.3",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
|
||||
"integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
|
||||
|
|
|
|||
|
|
@ -1985,7 +1985,6 @@
|
|||
"js-yaml": "^4.1.0",
|
||||
"msw": "^2.2.13",
|
||||
"nanoid": "^5.0.7",
|
||||
"node-fetch": "^3.3.2",
|
||||
"p-queue": "^8.0.1",
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import { reportUnzipProgress } from "../common/vscode/unzip-progress";
|
|||
import type { Release } from "./distribution/release";
|
||||
import { ReleasesApiConsumer } from "./distribution/releases-api-consumer";
|
||||
import { createTimeoutSignal } from "../common/fetch-stream";
|
||||
import { AbortError } from "node-fetch";
|
||||
|
||||
/**
|
||||
* distribution.ts
|
||||
|
|
@ -416,24 +415,40 @@ class ExtensionSpecificDistributionManager {
|
|||
const totalNumBytes = contentLength
|
||||
? parseInt(contentLength, 10)
|
||||
: undefined;
|
||||
reportStreamProgress(
|
||||
body,
|
||||
|
||||
const reportProgress = reportStreamProgress(
|
||||
`Downloading CodeQL CLI ${release.name}…`,
|
||||
totalNumBytes,
|
||||
progressCallback,
|
||||
);
|
||||
|
||||
body.on("data", onData);
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
if (!archiveFile) {
|
||||
throw new Error("Invariant violation: archiveFile not set");
|
||||
const reader = body.getReader();
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
|
||||
body.pipe(archiveFile).on("finish", resolve).on("error", reject);
|
||||
onData();
|
||||
reportProgress(value?.length ?? 0);
|
||||
|
||||
// If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error).
|
||||
body.on("error", reject);
|
||||
await new Promise((resolve, reject) => {
|
||||
archiveFile?.write(value, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
resolve(undefined);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
archiveFile?.close((err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
resolve(undefined);
|
||||
});
|
||||
});
|
||||
|
||||
disposeTimeout();
|
||||
|
|
@ -454,8 +469,8 @@ class ExtensionSpecificDistributionManager {
|
|||
: undefined,
|
||||
);
|
||||
} catch (e) {
|
||||
if (e instanceof AbortError) {
|
||||
const thrownError = new AbortError("The download timed out.");
|
||||
if (e instanceof DOMException && e.name === "AbortError") {
|
||||
const thrownError = new Error("The download timed out.");
|
||||
thrownError.stack = e.stack;
|
||||
throw thrownError;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
import type { Response } from "node-fetch";
|
||||
import { default as fetch } from "node-fetch";
|
||||
import type { Range } from "semver";
|
||||
import { compare, parse, satisfies } from "semver";
|
||||
import { URL } from "url";
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { join, resolve } from "path";
|
|||
import { pathExists } from "fs-extra";
|
||||
import type { SetupServer } from "msw/node";
|
||||
import { setupServer } from "msw/node";
|
||||
import type { UnhandledRequestStrategy } from "msw/lib/core/utils/request/onUnhandledRequest";
|
||||
|
||||
import { DisposableObject } from "../disposable-object";
|
||||
|
||||
|
|
@ -26,12 +27,14 @@ export class MockGitHubApiServer extends DisposableObject {
|
|||
this.recorder = this.push(new Recorder(this.server));
|
||||
}
|
||||
|
||||
public startServer(): void {
|
||||
public startServer(
|
||||
onUnhandledRequest: UnhandledRequestStrategy = "bypass",
|
||||
): void {
|
||||
if (this._isListening) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.server.listen({ onUnhandledRequest: "bypass" });
|
||||
this.server.listen({ onUnhandledRequest });
|
||||
this._isListening = true;
|
||||
}
|
||||
|
||||
|
|
@ -54,8 +57,7 @@ export class MockGitHubApiServer extends DisposableObject {
|
|||
const scenarioPath = join(scenariosPath, scenarioName);
|
||||
|
||||
const handlers = await createRequestHandlers(scenarioPath);
|
||||
this.server.resetHandlers();
|
||||
this.server.use(...handlers);
|
||||
this.server.resetHandlers(...handlers);
|
||||
}
|
||||
|
||||
public async saveScenario(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import { ensureDir, writeFile } from "fs-extra";
|
||||
import { join } from "path";
|
||||
|
||||
import fetch from "node-fetch";
|
||||
import type { SetupServer } from "msw/node";
|
||||
|
||||
import { DisposableObject } from "../disposable-object";
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
import { Octokit } from "@octokit/rest";
|
||||
import { retry } from "@octokit/plugin-retry";
|
||||
import fetch from "node-fetch";
|
||||
|
||||
export const AppOctokit = Octokit.defaults({
|
||||
request: {
|
||||
fetch,
|
||||
// MSW replaces the global fetch object, so we can't just pass a reference to the
|
||||
// fetch object at initialization time. Instead, we pass a function that will
|
||||
// always call the global fetch object.
|
||||
fetch: (input: string | URL | Request, init?: RequestInit) =>
|
||||
fetch(input, init),
|
||||
},
|
||||
retry,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -97,17 +97,15 @@ export function withProgress<R>(
|
|||
* Displays a progress monitor that indicates how much progess has been made
|
||||
* reading from a stream.
|
||||
*
|
||||
* @param readable The stream to read progress from
|
||||
* @param messagePrefix A prefix for displaying the message
|
||||
* @param totalNumBytes Total number of bytes in this stream
|
||||
* @param progress The progress callback used to set messages
|
||||
*/
|
||||
export function reportStreamProgress(
|
||||
readable: NodeJS.ReadableStream,
|
||||
messagePrefix: string,
|
||||
totalNumBytes?: number,
|
||||
progress?: ProgressCallback,
|
||||
) {
|
||||
): (bytesRead: number) => void {
|
||||
if (progress && totalNumBytes) {
|
||||
let numBytesDownloaded = 0;
|
||||
const updateProgress = () => {
|
||||
|
|
@ -123,10 +121,10 @@ export function reportStreamProgress(
|
|||
// Display the progress straight away rather than waiting for the first chunk.
|
||||
updateProgress();
|
||||
|
||||
readable.on("data", (data) => {
|
||||
numBytesDownloaded += data.length;
|
||||
return (bytesRead: number) => {
|
||||
numBytesDownloaded += bytesRead;
|
||||
updateProgress();
|
||||
});
|
||||
};
|
||||
} else if (progress) {
|
||||
progress({
|
||||
step: 1,
|
||||
|
|
@ -134,4 +132,6 @@ export function reportStreamProgress(
|
|||
message: `${messagePrefix} (Size unknown)`,
|
||||
});
|
||||
}
|
||||
|
||||
return () => {};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
import type { Response } from "node-fetch";
|
||||
import fetch, { AbortError } from "node-fetch";
|
||||
import type { InputBoxOptions } from "vscode";
|
||||
import { Uri, window } from "vscode";
|
||||
import type { CodeQLCliServer } from "../codeql-cli/cli";
|
||||
|
|
@ -536,8 +534,8 @@ export class DatabaseFetcher {
|
|||
} catch (e) {
|
||||
disposeTimeout();
|
||||
|
||||
if (e instanceof AbortError) {
|
||||
const thrownError = new AbortError("The request timed out.");
|
||||
if (e instanceof DOMException && e.name === "AbortError") {
|
||||
const thrownError = new Error("The request timed out.");
|
||||
thrownError.stack = e.stack;
|
||||
throw thrownError;
|
||||
}
|
||||
|
|
@ -556,16 +554,41 @@ export class DatabaseFetcher {
|
|||
const totalNumBytes = contentLength
|
||||
? parseInt(contentLength, 10)
|
||||
: undefined;
|
||||
reportStreamProgress(body, "Downloading database", totalNumBytes, progress);
|
||||
|
||||
body.on("data", onData);
|
||||
const reportProgress = reportStreamProgress(
|
||||
"Downloading database",
|
||||
totalNumBytes,
|
||||
progress,
|
||||
);
|
||||
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
body.pipe(archiveFileStream).on("finish", resolve).on("error", reject);
|
||||
const reader = body.getReader();
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
|
||||
// If an error occurs on the body, we also want to reject the promise (e.g. during a timeout error).
|
||||
body.on("error", reject);
|
||||
onData();
|
||||
reportProgress(value?.length ?? 0);
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
archiveFileStream.write(value, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
resolve(undefined);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
archiveFileStream.close((err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
resolve(undefined);
|
||||
});
|
||||
});
|
||||
} catch (e) {
|
||||
// Close and remove the file if an error occurs
|
||||
|
|
@ -573,8 +596,8 @@ export class DatabaseFetcher {
|
|||
void remove(archivePath);
|
||||
});
|
||||
|
||||
if (e instanceof AbortError) {
|
||||
const thrownError = new AbortError("The download timed out.");
|
||||
if (e instanceof DOMException && e.name === "AbortError") {
|
||||
const thrownError = new Error("The download timed out.");
|
||||
thrownError.stack = e.stack;
|
||||
throw thrownError;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,7 +78,6 @@ import {
|
|||
REPO_STATES_FILENAME,
|
||||
writeRepoStates,
|
||||
} from "./repo-states-store";
|
||||
import { FetchError } from "node-fetch";
|
||||
import {
|
||||
showAndLogExceptionWithTelemetry,
|
||||
showAndLogInformationMessage,
|
||||
|
|
@ -859,7 +858,9 @@ export class VariantAnalysisManager
|
|||
} catch (e) {
|
||||
if (
|
||||
retry++ < maxRetryCount &&
|
||||
e instanceof FetchError &&
|
||||
e &&
|
||||
typeof e === "object" &&
|
||||
"code" in e &&
|
||||
(e.code === "ETIMEDOUT" || e.code === "ECONNRESET")
|
||||
) {
|
||||
void this.app.logger.log(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import { appendFile, pathExists, rm } from "fs-extra";
|
||||
import fetch from "node-fetch";
|
||||
import { EOL } from "os";
|
||||
import { join } from "path";
|
||||
|
||||
|
|
@ -94,17 +93,23 @@ export class VariantAnalysisResultsManager extends DisposableObject {
|
|||
|
||||
const response = await fetch(repoTask.artifactUrl);
|
||||
|
||||
let responseSize = parseInt(response.headers.get("content-length") || "0");
|
||||
if (responseSize === 0 && response.size > 0) {
|
||||
responseSize = response.size;
|
||||
}
|
||||
const responseSize = parseInt(
|
||||
response.headers.get("content-length") || "1",
|
||||
);
|
||||
|
||||
if (!response.body) {
|
||||
throw new Error("No response body found");
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
|
||||
let amountDownloaded = 0;
|
||||
for await (const chunk of response.body) {
|
||||
for (;;) {
|
||||
const { value: chunk, done } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
|
||||
await appendFile(zipFilePath, Buffer.from(chunk));
|
||||
amountDownloaded += chunk.length;
|
||||
await onDownloadPercentageChanged(
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ import type { VariantAnalysisRepoTask } from "../../../../src/variant-analysis/g
|
|||
import { VariantAnalysisRepoStatus } from "../../../../src/variant-analysis/shared/variant-analysis";
|
||||
import { createMockRepository } from "./repository";
|
||||
|
||||
export function createMockVariantAnalysisRepoTask(): VariantAnalysisRepoTask {
|
||||
export function createMockVariantAnalysisRepoTask(): VariantAnalysisRepoTask &
|
||||
Required<Pick<VariantAnalysisRepoTask, "artifact_url">> {
|
||||
return {
|
||||
repository: {
|
||||
...createMockRepository(),
|
||||
|
|
@ -12,6 +13,6 @@ export function createMockVariantAnalysisRepoTask(): VariantAnalysisRepoTask {
|
|||
analysis_status: VariantAnalysisRepoStatus.Succeeded,
|
||||
result_count: faker.number.int(),
|
||||
artifact_size_in_bytes: faker.number.int(),
|
||||
artifact_url: "https://www.pickles.com",
|
||||
artifact_url: faker.internet.url(),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ import { createMockRepositoryWithMetadata } from "./repository";
|
|||
|
||||
export function createMockVariantAnalysisRepositoryTask(
|
||||
data?: Partial<VariantAnalysisRepositoryTask>,
|
||||
): VariantAnalysisRepositoryTask {
|
||||
): VariantAnalysisRepositoryTask &
|
||||
Required<Pick<VariantAnalysisRepositoryTask, "artifactUrl">> {
|
||||
return {
|
||||
repository: createMockRepositoryWithMetadata(),
|
||||
analysisStatus: VariantAnalysisRepoStatus.Pending,
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ const transformPackages = [
|
|||
"formdata-polyfill",
|
||||
"internmap",
|
||||
"nanoid",
|
||||
"node-fetch",
|
||||
"p-queue",
|
||||
"p-timeout",
|
||||
"robust-predicates",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import { Response } from "node-fetch";
|
||||
import { Range } from "semver";
|
||||
|
||||
import type { GithubRelease } from "../../../../src/codeql-cli/distribution/releases-api-consumer";
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import { response as variantAnalysisRepoJson_response } from "../../../../src/co
|
|||
import { testCredentialsWithRealOctokit } from "../../../factories/authentication";
|
||||
|
||||
const mockServer = new MockGitHubApiServer();
|
||||
beforeAll(() => mockServer.startServer());
|
||||
beforeAll(() => mockServer.startServer("error"));
|
||||
afterEach(() => mockServer.unloadScenario());
|
||||
afterAll(() => mockServer.stopServer());
|
||||
|
||||
|
|
|
|||
|
|
@ -12,8 +12,6 @@ import {
|
|||
remove,
|
||||
} from "fs-extra";
|
||||
import { join } from "path";
|
||||
import * as fetchModule from "node-fetch";
|
||||
import { Response } from "node-fetch";
|
||||
|
||||
import { VariantAnalysisManager } from "../../../../src/variant-analysis/variant-analysis-manager";
|
||||
import type { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
|
||||
|
|
@ -49,10 +47,35 @@ import {
|
|||
} from "../../../../src/variant-analysis/repo-states-store";
|
||||
import { permissiveFilterSortState } from "../../../unit-tests/variant-analysis-filter-sort.test";
|
||||
import { createMockVariantAnalysisConfig } from "../../../factories/config";
|
||||
import { setupServer } from "msw/node";
|
||||
import type { RequestHandler } from "msw";
|
||||
import { http } from "msw";
|
||||
|
||||
// up to 3 minutes per test
|
||||
jest.setTimeout(3 * 60 * 1000);
|
||||
|
||||
const server = setupServer();
|
||||
|
||||
beforeAll(() =>
|
||||
server.listen({
|
||||
onUnhandledRequest: "error",
|
||||
}),
|
||||
);
|
||||
afterEach(() => server.resetHandlers());
|
||||
afterAll(() => server.close());
|
||||
|
||||
let requests: Request[] = [];
|
||||
|
||||
beforeAll(() => {
|
||||
server.events.on("request:start", ({ request }) => {
|
||||
requests.push(request);
|
||||
});
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
requests = [];
|
||||
});
|
||||
|
||||
describe("Variant Analysis Manager", () => {
|
||||
let app: App;
|
||||
let variantAnalysisManager: VariantAnalysisManager;
|
||||
|
|
@ -175,9 +198,6 @@ describe("Variant Analysis Manager", () => {
|
|||
let getVariantAnalysisRepoStub: jest.SpiedFunction<
|
||||
typeof ghApiClient.getVariantAnalysisRepo
|
||||
>;
|
||||
let getVariantAnalysisRepoResultStub: jest.SpiedFunction<
|
||||
typeof fetchModule.default
|
||||
>;
|
||||
|
||||
let repoStatesPath: string;
|
||||
|
||||
|
|
@ -186,7 +206,6 @@ describe("Variant Analysis Manager", () => {
|
|||
ghApiClient,
|
||||
"getVariantAnalysisRepo",
|
||||
);
|
||||
getVariantAnalysisRepoResultStub = jest.spyOn(fetchModule, "default");
|
||||
|
||||
repoStatesPath = join(
|
||||
storagePath,
|
||||
|
|
@ -197,7 +216,8 @@ describe("Variant Analysis Manager", () => {
|
|||
|
||||
describe("when the artifact_url is missing", () => {
|
||||
beforeEach(async () => {
|
||||
const dummyRepoTask = createMockVariantAnalysisRepoTask();
|
||||
const dummyRepoTask: VariantAnalysisRepoTask =
|
||||
createMockVariantAnalysisRepoTask();
|
||||
delete dummyRepoTask.artifact_url;
|
||||
|
||||
getVariantAnalysisRepoStub.mockResolvedValue(dummyRepoTask);
|
||||
|
|
@ -209,25 +229,30 @@ describe("Variant Analysis Manager", () => {
|
|||
variantAnalysis,
|
||||
);
|
||||
|
||||
expect(getVariantAnalysisRepoResultStub).not.toHaveBeenCalled();
|
||||
expect(requests).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("when the artifact_url is present", () => {
|
||||
let dummyRepoTask: VariantAnalysisRepoTask;
|
||||
let dummyRepoTask: ReturnType<typeof createMockVariantAnalysisRepoTask>;
|
||||
let handlers: RequestHandler[];
|
||||
|
||||
beforeEach(async () => {
|
||||
dummyRepoTask = createMockVariantAnalysisRepoTask();
|
||||
|
||||
getVariantAnalysisRepoStub.mockResolvedValue(dummyRepoTask);
|
||||
|
||||
handlers = [
|
||||
http.get(dummyRepoTask.artifact_url, async () => {
|
||||
const sourceFilePath = join(
|
||||
__dirname,
|
||||
"data/variant-analysis-results.zip",
|
||||
);
|
||||
const fileContents = await readFile(sourceFilePath);
|
||||
const response = new Response(fileContents);
|
||||
getVariantAnalysisRepoResultStub.mockResolvedValue(response);
|
||||
return new Response(fileContents);
|
||||
}),
|
||||
];
|
||||
server.resetHandlers(...handlers);
|
||||
});
|
||||
|
||||
it("should fetch a repo task", async () => {
|
||||
|
|
@ -245,7 +270,7 @@ describe("Variant Analysis Manager", () => {
|
|||
variantAnalysis,
|
||||
);
|
||||
|
||||
expect(getVariantAnalysisRepoResultStub).toHaveBeenCalled();
|
||||
expect(requests).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("should skip the download if the repository has already been downloaded", async () => {
|
||||
|
|
@ -281,8 +306,10 @@ describe("Variant Analysis Manager", () => {
|
|||
});
|
||||
|
||||
it("should not write the repo state when the download fails", async () => {
|
||||
getVariantAnalysisRepoResultStub.mockRejectedValue(
|
||||
new Error("Failed to download"),
|
||||
server.resetHandlers(
|
||||
http.get(dummyRepoTask.artifact_url, async () => {
|
||||
return new Response(JSON.stringify({}), { status: 500 });
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
|
|
@ -329,8 +356,10 @@ describe("Variant Analysis Manager", () => {
|
|||
});
|
||||
|
||||
it("should have a failed repo state when the download fails", async () => {
|
||||
getVariantAnalysisRepoResultStub.mockRejectedValueOnce(
|
||||
new Error("Failed to download"),
|
||||
server.resetHandlers(
|
||||
http.get(dummyRepoTask.artifact_url, async () => {
|
||||
return new Response(JSON.stringify({}), { status: 500 });
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
|
|
@ -342,6 +371,8 @@ describe("Variant Analysis Manager", () => {
|
|||
|
||||
await expect(pathExists(repoStatesPath)).resolves.toBe(false);
|
||||
|
||||
server.resetHandlers(...handlers);
|
||||
|
||||
await variantAnalysisManager.autoDownloadVariantAnalysisResult(
|
||||
scannedRepos[1],
|
||||
variantAnalysis,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,6 @@
|
|||
import { extLogger } from "../../../../src/common/logging/vscode";
|
||||
import { readFile, pathExists, remove, outputJson, readJson } from "fs-extra";
|
||||
import { outputJson, pathExists, readFile, readJson, remove } from "fs-extra";
|
||||
import { join, resolve } from "path";
|
||||
import { Readable } from "stream";
|
||||
import * as fetchModule from "node-fetch";
|
||||
import type { RequestInfo, RequestInit } from "node-fetch";
|
||||
import { Response } from "node-fetch";
|
||||
|
||||
import { VariantAnalysisResultsManager } from "../../../../src/variant-analysis/variant-analysis-results-manager";
|
||||
import type { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
|
||||
|
|
@ -17,9 +13,21 @@ import type {
|
|||
} from "../../../../src/variant-analysis/shared/variant-analysis";
|
||||
import { mockedObject } from "../../utils/mocking.helpers";
|
||||
import { createMockVariantAnalysisConfig } from "../../../factories/config";
|
||||
import { setupServer } from "msw/node";
|
||||
import { http } from "msw";
|
||||
|
||||
jest.setTimeout(10_000);
|
||||
|
||||
const server = setupServer();
|
||||
|
||||
beforeAll(() =>
|
||||
server.listen({
|
||||
onUnhandledRequest: "error",
|
||||
}),
|
||||
);
|
||||
afterEach(() => server.resetHandlers());
|
||||
afterAll(() => server.close());
|
||||
|
||||
describe(VariantAnalysisResultsManager.name, () => {
|
||||
let variantAnalysisId: number;
|
||||
let variantAnalysisResultsManager: VariantAnalysisResultsManager;
|
||||
|
|
@ -37,7 +45,9 @@ describe(VariantAnalysisResultsManager.name, () => {
|
|||
});
|
||||
|
||||
describe("download", () => {
|
||||
let dummyRepoTask: VariantAnalysisRepositoryTask;
|
||||
let dummyRepoTask: ReturnType<
|
||||
typeof createMockVariantAnalysisRepositoryTask
|
||||
>;
|
||||
let variantAnalysisStoragePath: string;
|
||||
let repoTaskStorageDirectory: string;
|
||||
|
||||
|
|
@ -76,7 +86,8 @@ describe(VariantAnalysisResultsManager.name, () => {
|
|||
|
||||
describe("when the artifact_url is missing", () => {
|
||||
it("should not try to download the result", async () => {
|
||||
const dummyRepoTask = createMockVariantAnalysisRepositoryTask();
|
||||
const dummyRepoTask: VariantAnalysisRepositoryTask =
|
||||
createMockVariantAnalysisRepositoryTask();
|
||||
delete dummyRepoTask.artifactUrl;
|
||||
|
||||
await expect(
|
||||
|
|
@ -91,10 +102,8 @@ describe(VariantAnalysisResultsManager.name, () => {
|
|||
});
|
||||
|
||||
describe("when the artifact_url is present", () => {
|
||||
let getVariantAnalysisRepoResultStub: jest.SpiedFunction<
|
||||
typeof fetchModule.default
|
||||
>;
|
||||
let fileContents: Buffer;
|
||||
let artifactRequest: Request | undefined;
|
||||
|
||||
beforeEach(async () => {
|
||||
const sourceFilePath = join(
|
||||
|
|
@ -103,14 +112,19 @@ describe(VariantAnalysisResultsManager.name, () => {
|
|||
);
|
||||
fileContents = await readFile(sourceFilePath);
|
||||
|
||||
getVariantAnalysisRepoResultStub = jest
|
||||
.spyOn(fetchModule, "default")
|
||||
.mockImplementation((url: URL | RequestInfo, _init?: RequestInit) => {
|
||||
if (url === dummyRepoTask.artifactUrl) {
|
||||
return Promise.resolve(new Response(fileContents));
|
||||
artifactRequest = undefined;
|
||||
|
||||
server.resetHandlers(
|
||||
http.get(dummyRepoTask.artifactUrl, ({ request }) => {
|
||||
if (artifactRequest) {
|
||||
throw new Error("Unexpected artifact request");
|
||||
}
|
||||
return Promise.reject(new Error("Unexpected artifact URL"));
|
||||
});
|
||||
|
||||
artifactRequest = request;
|
||||
|
||||
return new Response(fileContents);
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("should call the API to download the results", async () => {
|
||||
|
|
@ -121,7 +135,7 @@ describe(VariantAnalysisResultsManager.name, () => {
|
|||
() => Promise.resolve(),
|
||||
);
|
||||
|
||||
expect(getVariantAnalysisRepoResultStub).toHaveBeenCalledTimes(1);
|
||||
expect(artifactRequest).not.toBeUndefined();
|
||||
});
|
||||
|
||||
it("should save the results zip file to disk", async () => {
|
||||
|
|
@ -151,28 +165,29 @@ describe(VariantAnalysisResultsManager.name, () => {
|
|||
});
|
||||
|
||||
it("should report download progress", async () => {
|
||||
server.resetHandlers(
|
||||
http.get(dummyRepoTask.artifactUrl, () => {
|
||||
// This generates a "fake" stream which "downloads" the file in 5 chunks,
|
||||
// rather than in 1 chunk. This is used for testing that we actually get
|
||||
// multiple progress reports.
|
||||
async function* generateInParts() {
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
const partLength = fileContents.length / 5;
|
||||
for (let i = 0; i < 5; i++) {
|
||||
yield fileContents.subarray(i * partLength, (i + 1) * partLength);
|
||||
}
|
||||
}
|
||||
|
||||
getVariantAnalysisRepoResultStub.mockImplementation(
|
||||
(url: URL | RequestInfo, _init?: RequestInit) => {
|
||||
if (url === dummyRepoTask.artifactUrl) {
|
||||
const response = new Response(Readable.from(generateInParts()));
|
||||
response.headers.set(
|
||||
"Content-Length",
|
||||
fileContents.length.toString(),
|
||||
controller.enqueue(
|
||||
fileContents.subarray(i * partLength, (i + 1) * partLength),
|
||||
);
|
||||
return Promise.resolve(response);
|
||||
}
|
||||
return Promise.reject(new Error("Unexpected artifact URL"));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(stream, {
|
||||
headers: {
|
||||
"Content-Length": fileContents.length.toString(),
|
||||
},
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
const downloadPercentageChanged = jest
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import {
|
|||
import { createWriteStream, existsSync, mkdirpSync } from "fs-extra";
|
||||
import { dirname, join } from "path";
|
||||
import { DB_URL, dbLoc, testprojLoc } from "../global.helper";
|
||||
import fetch from "node-fetch";
|
||||
import { renameSync } from "fs";
|
||||
import { unzipToDirectoryConcurrently } from "../../../src/common/unzip-concurrently";
|
||||
import { platform } from "os";
|
||||
|
|
@ -21,24 +20,44 @@ beforeAll(async () => {
|
|||
if (!existsSync(dbLoc)) {
|
||||
console.log(`Downloading test database to ${dbLoc}`);
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
return fetch(DB_URL).then((response) => {
|
||||
const response = await fetch(DB_URL);
|
||||
if (!response.body) {
|
||||
throw new Error("No response body found");
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download test database: ${response.status}`);
|
||||
}
|
||||
|
||||
const dest = createWriteStream(dbLoc);
|
||||
response.body.pipe(dest);
|
||||
|
||||
response.body.on("error", reject);
|
||||
dest.on("error", reject);
|
||||
dest.on("close", () => {
|
||||
resolve(dbLoc);
|
||||
});
|
||||
const reader = response.body.getReader();
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
dest.write(value, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
resolve(undefined);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
await new Promise((resolve, reject) =>
|
||||
dest.close((err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
resolve(undefined);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// unzip the database from dbLoc to testprojLoc
|
||||
if (!existsSync(testprojLoc)) {
|
||||
console.log(`Unzipping test database to ${testprojLoc}`);
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import { createVSCodeCommandManager } from "../../../../src/common/vscode/comman
|
|||
import type { AllCommands } from "../../../../src/common/commands";
|
||||
|
||||
const mockServer = new MockGitHubApiServer();
|
||||
beforeAll(() => mockServer.startServer());
|
||||
beforeAll(() => mockServer.startServer("bypass"));
|
||||
afterEach(() => mockServer.unloadScenario());
|
||||
afterAll(() => mockServer.stopServer());
|
||||
|
||||
|
|
@ -23,7 +23,8 @@ async function showQlDocument(name: string): Promise<TextDocument> {
|
|||
return document;
|
||||
}
|
||||
|
||||
describe("Variant Analysis Submission Integration", () => {
|
||||
// MSW can't intercept fetch requests made in VS Code, so we are skipping these tests for now
|
||||
describe.skip("Variant Analysis Submission Integration", () => {
|
||||
const commandManager = createVSCodeCommandManager<AllCommands>();
|
||||
let quickPickSpy: jest.SpiedFunction<typeof window.showQuickPick>;
|
||||
let executeCommandSpy: jest.SpiedFunction<typeof commands.executeCommand>;
|
||||
|
|
|
|||
|
|
@ -3,19 +3,15 @@ import { reportStreamProgress } from "../../../../../src/common/vscode/progress"
|
|||
describe("helpers", () => {
|
||||
it("should report stream progress", () => {
|
||||
const progressSpy = jest.fn();
|
||||
const mockReadable = {
|
||||
on: jest.fn(),
|
||||
};
|
||||
const max = 1024 * 1024 * 4;
|
||||
const firstStep = 1024 * 1024 + 1024 * 600;
|
||||
const secondStep = 1024 * 1024 * 2;
|
||||
|
||||
(reportStreamProgress as any)(mockReadable, "My prefix", max, progressSpy);
|
||||
const reportProgress = reportStreamProgress("My prefix", max, progressSpy);
|
||||
|
||||
// now pretend that we have received some messages
|
||||
const listener = mockReadable.on.mock.calls[0][1] as (data: any) => void;
|
||||
listener({ length: firstStep });
|
||||
listener({ length: secondStep });
|
||||
reportProgress(firstStep);
|
||||
reportProgress(secondStep);
|
||||
|
||||
expect(progressSpy).toHaveBeenCalledTimes(3);
|
||||
expect(progressSpy).toHaveBeenCalledWith({
|
||||
|
|
@ -37,18 +33,14 @@ describe("helpers", () => {
|
|||
|
||||
it("should report stream progress when total bytes unknown", () => {
|
||||
const progressSpy = jest.fn();
|
||||
const mockReadable = {
|
||||
on: jest.fn(),
|
||||
};
|
||||
(reportStreamProgress as any)(
|
||||
mockReadable,
|
||||
const reportProgress = reportStreamProgress(
|
||||
"My prefix",
|
||||
undefined,
|
||||
progressSpy,
|
||||
);
|
||||
|
||||
// There are no listeners registered to this readable
|
||||
expect(mockReadable.on).not.toHaveBeenCalled();
|
||||
// It should not report progress when calling the callback
|
||||
reportProgress(100);
|
||||
|
||||
expect(progressSpy).toHaveBeenCalledTimes(1);
|
||||
expect(progressSpy).toHaveBeenCalledWith({
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче