Import testproj databases into workspace storage

If a folder that ends with `.testproj` is encountered, assume it is a
database created by a codeql test. When the user wants to import this
database, copy it into workspace storage. The database can be
re-imported, which first removes the old version before importing it
again.
This commit is contained in:
Andrew Eisenberg 2024-02-25 10:58:55 -08:00
Родитель 8063d6c46b
Коммит ca21ed18d0
10 изменённых файлов: 322 добавлений и 28 удалений

141
extensions/ql-vscode/package-lock.json сгенерированный
Просмотреть файл

@ -140,7 +140,8 @@
"ts-loader": "^9.4.2",
"ts-node": "^10.7.0",
"ts-unused-exports": "^10.0.0",
"typescript": "^5.0.2"
"typescript": "^5.0.2",
"unzipper": "^0.10.14"
},
"engines": {
"node": "^18.17.1",
@ -12242,6 +12243,19 @@
"node": "*"
}
},
"node_modules/binary": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
"integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==",
"dev": true,
"dependencies": {
"buffers": "~0.1.1",
"chainsaw": "~0.1.0"
},
"engines": {
"node": "*"
}
},
"node_modules/binary-extensions": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@ -12295,6 +12309,12 @@
"node": ">= 6"
}
},
"node_modules/bluebird": {
"version": "3.4.7",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz",
"integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==",
"dev": true
},
"node_modules/body-parser": {
"version": "1.20.1",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
@ -12520,6 +12540,24 @@
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
},
"node_modules/buffer-indexof-polyfill": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz",
"integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==",
"dev": true,
"engines": {
"node": ">=0.10"
}
},
"node_modules/buffers": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==",
"dev": true,
"engines": {
"node": ">=0.2.0"
}
},
"node_modules/bundle-name": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-3.0.0.tgz",
@ -12643,6 +12681,18 @@
"node": ">=4"
}
},
"node_modules/chainsaw": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
"integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==",
"dev": true,
"dependencies": {
"traverse": ">=0.3.0 <0.4"
},
"engines": {
"node": "*"
}
},
"node_modules/chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@ -15003,6 +15053,15 @@
"node": ">=12"
}
},
"node_modules/duplexer2": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz",
"integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==",
"dev": true,
"dependencies": {
"readable-stream": "^2.0.2"
}
},
"node_modules/duplexify": {
"version": "3.7.1",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
@ -17818,6 +17877,53 @@
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/fstream": {
"version": "1.0.12",
"resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz",
"integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==",
"dev": true,
"dependencies": {
"graceful-fs": "^4.1.2",
"inherits": "~2.0.0",
"mkdirp": ">=0.5 0",
"rimraf": "2"
},
"engines": {
"node": ">=0.6"
}
},
"node_modules/fstream/node_modules/glob": {
"version": "7.2.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
"dev": true,
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.1.1",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
},
"engines": {
"node": "*"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/fstream/node_modules/rimraf": {
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
"dev": true,
"dependencies": {
"glob": "^7.1.3"
},
"bin": {
"rimraf": "bin.js"
}
},
"node_modules/function-bind": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
@ -23787,6 +23893,12 @@
"node": ">= 8"
}
},
"node_modules/listenercount": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz",
"integrity": "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==",
"dev": true
},
"node_modules/listr2": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/listr2/-/listr2-8.0.0.tgz",
@ -30690,6 +30802,15 @@
"node": ">=12"
}
},
"node_modules/traverse": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
"integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==",
"dev": true,
"engines": {
"node": "*"
}
},
"node_modules/tree-kill": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
@ -31632,6 +31753,24 @@
"node": ">=8"
}
},
"node_modules/unzipper": {
"version": "0.10.14",
"resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.14.tgz",
"integrity": "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==",
"dev": true,
"dependencies": {
"big-integer": "^1.6.17",
"binary": "~0.3.0",
"bluebird": "~3.4.1",
"buffer-indexof-polyfill": "~1.0.0",
"duplexer2": "~0.1.4",
"fstream": "^1.0.12",
"graceful-fs": "^4.2.2",
"listenercount": "~1.0.1",
"readable-stream": "~2.3.6",
"setimmediate": "~1.0.4"
}
},
"node_modules/upath": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz",

Просмотреть файл

@ -738,6 +738,10 @@
"command": "codeQL.setCurrentDatabase",
"title": "CodeQL: Set Current Database"
},
{
"command": "codeQL.importTestDatabase",
"title": "CodeQL: (Re-)Import Test Database"
},
{
"command": "codeQL.getCurrentDatabase",
"title": "CodeQL: Get Current Database"
@ -1322,7 +1326,12 @@
{
"command": "codeQL.setCurrentDatabase",
"group": "9_qlCommands",
"when": "resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zip"
"when": "resourceExtname != .testproj && (resourceScheme == codeql-zip-archive || explorerResourceIsFolder || resourceExtname == .zipz)"
},
{
"command": "codeQL.importTestDatabase",
"group": "9_qlCommands",
"when": "explorerResourceIsFolder && resourceExtname == .testproj"
},
{
"command": "codeQL.viewAstContextExplorer",
@ -1476,6 +1485,10 @@
"command": "codeQL.setCurrentDatabase",
"when": "false"
},
{
"command": "codeQL.importTestDatabase",
"when": "false"
},
{
"command": "codeQL.getCurrentDatabase",
"when": "false"
@ -2068,7 +2081,8 @@
"ts-loader": "^9.4.2",
"ts-node": "^10.7.0",
"ts-unused-exports": "^10.0.0",
"typescript": "^5.0.2"
"typescript": "^5.0.2",
"unzipper": "^0.10.14"
},
"lint-staged": {
"./**/*.{json,css,scss}": [

Просмотреть файл

@ -220,6 +220,7 @@ export type LocalDatabasesCommands = {
// Explorer context menu
"codeQL.setCurrentDatabase": (uri: Uri) => Promise<void>;
"codeQL.importTestDatabase": (uri: Uri) => Promise<void>;
// Database panel view title commands
"codeQLDatabases.chooseDatabaseFolder": () => Promise<void>;

Просмотреть файл

@ -12,6 +12,7 @@ import {
remove,
stat,
readdir,
copy,
} from "fs-extra";
import { basename, join } from "path";
import type { Octokit } from "@octokit/rest";
@ -61,7 +62,7 @@ export async function promptImportInternetDatabase(
validateUrl(databaseUrl);
const item = await databaseArchiveFetcher(
const item = await fetchDatabaseToWorkspaceStorage(
databaseUrl,
{},
databaseManager,
@ -254,7 +255,7 @@ export async function downloadGitHubDatabaseFromUrl(
* We only need the actual token string.
*/
const octokitToken = ((await octokit.auth()) as { token: string })?.token;
return await databaseArchiveFetcher(
return await fetchDatabaseToWorkspaceStorage(
databaseUrl,
{
Accept: "application/zip",
@ -278,14 +279,15 @@ export async function downloadGitHubDatabaseFromUrl(
}
/**
* Imports a database from a local archive.
* Imports a database from a local archive or a test database that is in a folder
* ending with `.testproj`.
*
* @param databaseUrl the file url of the archive to import
* @param databaseUrl the file url of the archive or directory to import
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
* @param cli the CodeQL CLI server
*/
export async function importArchiveDatabase(
export async function importLocalDatabase(
commandManager: AppCommandManager,
databaseUrl: string,
databaseManager: DatabaseManager,
@ -294,16 +296,18 @@ export async function importArchiveDatabase(
cli: CodeQLCliServer,
): Promise<DatabaseItem | undefined> {
try {
const item = await databaseArchiveFetcher(
const origin: DatabaseOrigin = {
type: databaseUrl.endsWith(".testproj") ? "testproj" : "archive",
// TODO validate that archive origins can use a file path, not a URI
path: Uri.parse(databaseUrl).fsPath,
};
const item = await fetchDatabaseToWorkspaceStorage(
databaseUrl,
{},
databaseManager,
storagePath,
undefined,
{
type: "archive",
path: databaseUrl,
},
origin,
progress,
cli,
);
@ -328,10 +332,10 @@ export async function importArchiveDatabase(
}
/**
* Fetches an archive database. The database might be on the internet
* Fetches a database into workspace storage. The database might be on the internet
* or in the local filesystem.
*
* @param databaseUrl URL from which to grab the database
* @param databaseUrl URL from which to grab the database. This could be a local archive file, a local directory, or a remote URL.
* @param requestHeaders Headers to send with the request
* @param databaseManager the DatabaseManager
* @param storagePath where to store the unzipped database.
@ -342,7 +346,7 @@ export async function importArchiveDatabase(
* @param makeSelected make the new database selected in the databases panel (default: true)
* @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
*/
async function databaseArchiveFetcher(
async function fetchDatabaseToWorkspaceStorage(
databaseUrl: string,
requestHeaders: { [key: string]: string },
databaseManager: DatabaseManager,
@ -366,7 +370,11 @@ async function databaseArchiveFetcher(
const unzipPath = await getStorageFolder(storagePath, databaseUrl);
if (isFile(databaseUrl)) {
await readAndUnzip(databaseUrl, unzipPath, cli, progress);
if (origin.type == "testproj") {
await copyDatabase(origin.path, unzipPath, progress);
} else {
await readAndUnzip(databaseUrl, unzipPath, cli, progress);
}
} else {
await fetchAndUnzip(databaseUrl, requestHeaders, unzipPath, cli, progress);
}
@ -416,6 +424,8 @@ async function getStorageFolder(storagePath: string, urlStr: string) {
let lastName = basename(url.path).substring(0, 250);
if (lastName.endsWith(".zip")) {
lastName = lastName.substring(0, lastName.length - 4);
} else if (lastName.endsWith(".testproj")) {
lastName = lastName.substring(0, lastName.length - 9);
}
const realpath = await fs_realpath(storagePath);
@ -446,6 +456,26 @@ function validateUrl(databaseUrl: string) {
}
}
/**
* Copies a database folder from the file system into the workspace storage.
* @param scrDir the original location of the database
* @param destDir the location to copy the database to. This should be a folder in the workspace storage.
* @param progress callback to send progress messages to
*/
async function copyDatabase(
scrDir: string,
destDir: string,
progress?: ProgressCallback,
) {
progress?.({
maxStep: 10,
step: 9,
message: `Copying database ${basename(destDir)} into the workspace`,
});
await ensureDir(destDir);
await copy(scrDir, destDir);
}
async function readAndUnzip(
zipUrl: string,
unzipPath: string,

Просмотреть файл

@ -43,7 +43,7 @@ import {
showAndLogErrorMessage,
} from "../common/logging";
import {
importArchiveDatabase,
importLocalDatabase,
promptImportGithubDatabase,
promptImportInternetDatabase,
} from "./database-fetcher";
@ -282,6 +282,7 @@ export class DatabaseUI extends DisposableObject {
this.handleChooseDatabaseInternet.bind(this),
"codeQL.chooseDatabaseGithub": this.handleChooseDatabaseGithub.bind(this),
"codeQL.setCurrentDatabase": this.handleSetCurrentDatabase.bind(this),
"codeQL.importTestDatabase": this.handleImportTestDatabase.bind(this),
"codeQL.setDefaultTourDatabase":
this.handleSetDefaultTourDatabase.bind(this),
"codeQL.upgradeCurrentDatabase":
@ -716,7 +717,7 @@ export class DatabaseUI extends DisposableObject {
try {
// Assume user has selected an archive if the file has a .zip extension
if (uri.path.endsWith(".zip")) {
await importArchiveDatabase(
await importLocalDatabase(
this.app.commands,
uri.toString(true),
this.databaseManager,
@ -744,6 +745,54 @@ export class DatabaseUI extends DisposableObject {
);
}
private async handleImportTestDatabase(uri: Uri): Promise<void> {
return withProgress(
async (progress) => {
try {
// Assume user has selected an archive if the file has a .zip extension
if (!uri.path.endsWith(".testproj")) {
throw new Error(
"Please select a valid test database to import. Test databases end with `.testproj`.",
);
}
// Check if the database is already in the workspace. If
// so, delete it first before importing the new one.
const existingItem = this.databaseManager.findTestDatabase(uri);
if (existingItem !== undefined) {
progress({
maxStep: 9,
step: 1,
message: `Removing existing test database ${basename(
uri.fsPath,
)}`,
});
await this.databaseManager.removeDatabaseItem(existingItem);
}
await importLocalDatabase(
this.app.commands,
uri.toString(true),
this.databaseManager,
this.storagePath,
progress,
this.queryServer.cliServer,
);
} catch (e) {
// rethrow and let this be handled by default error handling.
throw new Error(
`Could not set database to ${basename(
uri.fsPath,
)}. Reason: ${getErrorMessage(e)}`,
);
}
},
{
title: "(Re-)importing test database from directory",
},
);
}
private async handleRemoveDatabase(
databaseItems: DatabaseItem[],
): Promise<void> {
@ -963,7 +1012,7 @@ export class DatabaseUI extends DisposableObject {
} else {
// we are selecting a database archive. Must unzip into a workspace-controlled area
// before importing.
return await importArchiveDatabase(
return await importLocalDatabase(
this.app.commands,
uri.toString(true),
this.databaseManager,

Просмотреть файл

@ -159,6 +159,23 @@ export class DatabaseManager extends DisposableObject {
);
}
/**
* Finds a test database that was originally imported from `uri`.
* A test database is creeated by the `codeql test run` command
* and ends with `.testproj`.
* @param uri The original location of the database
* @returns The first database item found that matches the uri
*/
public findTestDatabase(uri: vscode.Uri): DatabaseItem | undefined {
const originPath = uri.fsPath;
for (const item of this._databaseItems) {
if (item.origin?.type === "testproj" && item.origin.path === originPath) {
return item
}
}
return undefined;
}
/**
* Adds a {@link DatabaseItem} to the list of open databases, if that database is not already on
* the list.

Просмотреть файл

@ -24,9 +24,15 @@ interface DatabaseOriginDebugger {
type: "debugger";
}
export interface DatabaseOriginTestProj {
type: "testproj";
path: string;
}
export type DatabaseOrigin =
| DatabaseOriginFolder
| DatabaseOriginArchive
| DatabaseOriginGitHub
| DatabaseOriginInternet
| DatabaseOriginDebugger;
| DatabaseOriginDebugger
| DatabaseOriginTestProj;

Просмотреть файл

@ -4,7 +4,7 @@ import { Uri, window } from "vscode";
import type { CodeQLCliServer } from "../../../../src/codeql-cli/cli";
import type { DatabaseManager } from "../../../../src/databases/local-databases";
import {
importArchiveDatabase,
importLocalDatabase,
promptImportInternetDatabase,
} from "../../../../src/databases/database-fetcher";
import {
@ -13,6 +13,7 @@ import {
DB_URL,
getActivatedExtension,
storagePath,
testprojLoc,
} from "../../global.helper";
import { createMockCommandManager } from "../../../__mocks__/commandsMock";
import { remove } from "fs-extra";
@ -46,10 +47,10 @@ describe("database-fetcher", () => {
await remove(storagePath);
});
describe("importArchiveDatabase", () => {
it("should add a database from a folder", async () => {
describe("importLocalDatabase", () => {
it("should add a database from an archive", async () => {
const uri = Uri.file(dbLoc);
let dbItem = await importArchiveDatabase(
let dbItem = await importLocalDatabase(
createMockCommandManager(),
uri.toString(true),
databaseManager,
@ -64,6 +65,23 @@ describe("database-fetcher", () => {
expect(dbItem.name).toBe("db");
expect(dbItem.databaseUri.fsPath).toBe(join(storagePath, "db", "db"));
});
it("should import a testproj database", async () => {
let dbItem = await importLocalDatabase(
createMockCommandManager(),
Uri.file(testprojLoc).toString(true),
databaseManager,
storagePath,
progressCallback,
cli,
);
expect(dbItem).toBe(databaseManager.currentDatabaseItem);
expect(dbItem).toBe(databaseManager.databaseItems[0]);
expect(dbItem).toBeDefined();
dbItem = dbItem!;
expect(dbItem.name).toBe("db");
expect(dbItem.databaseUri.fsPath).toBe(join(storagePath, "db", "db"));
});
});
describe("promptImportInternetDatabase", () => {

Просмотреть файл

@ -7,8 +7,10 @@ import {
} from "../jest.activated-extension.setup";
import { createWriteStream, existsSync, mkdirpSync } from "fs-extra";
import { dirname } from "path";
import { DB_URL, dbLoc } from "../global.helper";
import { DB_URL, dbLoc, testprojLoc } from "../global.helper";
import fetch from "node-fetch";
import { createReadStream, renameSync } from "fs";
import { Extract } from "unzipper";
beforeAll(async () => {
// ensure the test database is downloaded
@ -28,6 +30,18 @@ beforeAll(async () => {
});
});
});
// unzip the database from dbLoc to testprojLoc
if (!existsSync(testprojLoc)) {
console.log(`Unzipping test database to ${testprojLoc}`);
const dbDir = dirname(testprojLoc);
mkdirpSync(dbDir);
console.log(`Unzipping test database to ${testprojLoc}`);
createReadStream(dbLoc)
.pipe(Extract({ path: dirname(dbDir) }))
.on("close", () => console.log("Unzip completed."));
}
renameSync(dbLoc, testprojLoc);
}
await beforeAllAction();

Просмотреть файл

@ -7,7 +7,7 @@ import type {
} from "../../src/databases/local-databases";
import type { CodeQLCliServer } from "../../src/codeql-cli/cli";
import type { CodeQLExtensionInterface } from "../../src/extension";
import { importArchiveDatabase } from "../../src/databases/database-fetcher";
import { importLocalDatabase } from "../../src/databases/database-fetcher";
import { createMockCommandManager } from "../__mocks__/commandsMock";
// This file contains helpers shared between tests that work with an activated extension.
@ -21,6 +21,12 @@ export const dbLoc = join(
realpathSync(join(__dirname, "../../../")),
"build/tests/db.zip",
);
export const testprojLoc = join(
realpathSync(join(__dirname, "../../../")),
"build/tests/db.zip",
);
// eslint-disable-next-line import/no-mutable-exports
export let storagePath: string;
@ -34,7 +40,7 @@ export async function ensureTestDatabase(
// Add a database, but make sure the database manager is empty first
await cleanDatabases(databaseManager);
const uri = Uri.file(dbLoc);
const maybeDbItem = await importArchiveDatabase(
const maybeDbItem = await importLocalDatabase(
createMockCommandManager(),
uri.toString(true),
databaseManager,