Save query history across restarts
Successfully completed queries will be stored on disk and available across restarts. - The query results are contained in global storage. - Metadata and a summary about a query are stored in workspace storage. - There is a job that runs every 2 hours to determine if any queries are old enough to be deleted.
This commit is contained in:
Andrew Eisenberg
Родитель
b7dafc31bb
Коммит
29c29f9e3a
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
- Fix a bug where invoking _View AST_ from the file explorer would not view the selected file. Instead it would view the active editor. Also, prevent the _View AST_ from appearing if the current selection includes a directory or multiple files. [#1113](https://github.com/github/vscode-codeql/pull/1113)
|
||||
- Add query history items as soon as a query is run, including new icons for each history item. [#1094](https://github.com/github/vscode-codeql/pull/1094)
|
||||
- Save query history items across restarts. Items will be saved for 30 days and can be overwritten by setting the `codeQL.queryHistory.ttl` configuration setting. [???]
|
||||
|
||||
## 1.5.10 - 25 January 2022
|
||||
|
||||
|
|
|
|||
|
|
@ -224,6 +224,12 @@
|
|||
"default": "%q on %d - %s, %r result count [%t]",
|
||||
"markdownDescription": "Default string for how to label query history items.\n* %t is the time of the query\n* %q is the human-readable query name\n* %f is the query file name\n* %d is the database name\n* %r is the number of results\n* %s is a status string"
|
||||
},
|
||||
"codeQL.queryHistory.ttl": {
|
||||
"type": "number",
|
||||
"default": 30,
|
||||
"description": "Number of days to retain queries in the query history before being automatically deleted.",
|
||||
"scope": "machine"
|
||||
},
|
||||
"codeQL.runningTests.additionalTestArguments": {
|
||||
"scope": "window",
|
||||
"type": "array",
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ import { workspace, Event, EventEmitter, ConfigurationChangeEvent, Configuration
|
|||
import { DistributionManager } from './distribution';
|
||||
import { logger } from './logging';
|
||||
|
||||
const ONE_DAY_IN_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/** Helper class to look up a labelled (and possibly nested) setting. */
|
||||
export class Setting {
|
||||
name: string;
|
||||
|
|
@ -54,8 +56,11 @@ const DISTRIBUTION_SETTING = new Setting('cli', ROOT_SETTING);
|
|||
export const CUSTOM_CODEQL_PATH_SETTING = new Setting('executablePath', DISTRIBUTION_SETTING);
|
||||
const INCLUDE_PRERELEASE_SETTING = new Setting('includePrerelease', DISTRIBUTION_SETTING);
|
||||
const PERSONAL_ACCESS_TOKEN_SETTING = new Setting('personalAccessToken', DISTRIBUTION_SETTING);
|
||||
|
||||
// Query History configuration
|
||||
const QUERY_HISTORY_SETTING = new Setting('queryHistory', ROOT_SETTING);
|
||||
const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING);
|
||||
const QUERY_HISTORY_TTL = new Setting('format', QUERY_HISTORY_SETTING);
|
||||
|
||||
/** When these settings change, the distribution should be updated. */
|
||||
const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];
|
||||
|
|
@ -71,7 +76,6 @@ export interface DistributionConfig {
|
|||
}
|
||||
|
||||
// Query server configuration
|
||||
|
||||
const RUNNING_QUERIES_SETTING = new Setting('runningQueries', ROOT_SETTING);
|
||||
const NUMBER_OF_THREADS_SETTING = new Setting('numberOfThreads', RUNNING_QUERIES_SETTING);
|
||||
const SAVE_CACHE_SETTING = new Setting('saveCache', RUNNING_QUERIES_SETTING);
|
||||
|
|
@ -106,10 +110,11 @@ export interface QueryServerConfig {
|
|||
}
|
||||
|
||||
/** When these settings change, the query history should be refreshed. */
|
||||
const QUERY_HISTORY_SETTINGS = [QUERY_HISTORY_FORMAT_SETTING];
|
||||
const QUERY_HISTORY_SETTINGS = [QUERY_HISTORY_FORMAT_SETTING, QUERY_HISTORY_TTL];
|
||||
|
||||
export interface QueryHistoryConfig {
|
||||
format: string;
|
||||
ttlInMillis: number;
|
||||
onDidChangeConfiguration: Event<void>;
|
||||
}
|
||||
|
||||
|
|
@ -251,6 +256,13 @@ export class QueryHistoryConfigListener extends ConfigListener implements QueryH
|
|||
public get format(): string {
|
||||
return QUERY_HISTORY_FORMAT_SETTING.getValue<string>();
|
||||
}
|
||||
|
||||
/**
|
||||
* The configuration value is in days, but return the value in milliseconds.
|
||||
*/
|
||||
public get ttlInMillis(): number {
|
||||
return (QUERY_HISTORY_TTL.getValue<number>() || 30) * ONE_DAY_IN_MS;
|
||||
}
|
||||
}
|
||||
|
||||
export class CliConfigListener extends ConfigListener implements CliConfig {
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ export async function getLocationsForUriString(
|
|||
dbm: DatabaseManager,
|
||||
uriString: string,
|
||||
keyType: KeyType,
|
||||
queryStorageLocation: string,
|
||||
progress: ProgressCallback,
|
||||
token: CancellationToken,
|
||||
filter: (src: string, dest: string) => boolean
|
||||
|
|
@ -69,6 +70,7 @@ export async function getLocationsForUriString(
|
|||
qs,
|
||||
db,
|
||||
initialInfo,
|
||||
queryStorageLocation,
|
||||
progress,
|
||||
token,
|
||||
templates
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
|
|||
private cli: CodeQLCliServer,
|
||||
private qs: QueryServerClient,
|
||||
private dbm: DatabaseManager,
|
||||
private queryStorageLocation: string,
|
||||
) {
|
||||
this.cache = new CachedOperation<LocationLink[]>(this.getDefinitions.bind(this));
|
||||
}
|
||||
|
|
@ -69,6 +70,7 @@ export class TemplateQueryDefinitionProvider implements DefinitionProvider {
|
|||
this.dbm,
|
||||
uriString,
|
||||
KeyType.DefinitionQuery,
|
||||
this.queryStorageLocation,
|
||||
progress,
|
||||
token,
|
||||
(src, _dest) => src === uriString
|
||||
|
|
@ -84,6 +86,7 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
|
|||
private cli: CodeQLCliServer,
|
||||
private qs: QueryServerClient,
|
||||
private dbm: DatabaseManager,
|
||||
private queryStorageLocation: string,
|
||||
) {
|
||||
this.cache = new CachedOperation<FullLocationLink[]>(this.getReferences.bind(this));
|
||||
}
|
||||
|
|
@ -116,6 +119,7 @@ export class TemplateQueryReferenceProvider implements ReferenceProvider {
|
|||
this.dbm,
|
||||
uriString,
|
||||
KeyType.DefinitionQuery,
|
||||
this.queryStorageLocation,
|
||||
progress,
|
||||
token,
|
||||
(src, _dest) => src === uriString
|
||||
|
|
@ -136,6 +140,7 @@ export class TemplatePrintAstProvider {
|
|||
private cli: CodeQLCliServer,
|
||||
private qs: QueryServerClient,
|
||||
private dbm: DatabaseManager,
|
||||
private queryStorageLocation: string,
|
||||
) {
|
||||
this.cache = new CachedOperation<QueryWithDb>(this.getAst.bind(this));
|
||||
}
|
||||
|
|
@ -216,6 +221,7 @@ export class TemplatePrintAstProvider {
|
|||
this.qs,
|
||||
db,
|
||||
initialInfo,
|
||||
this.queryStorageLocation,
|
||||
progress,
|
||||
token,
|
||||
templates
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import {
|
|||
} from 'vscode';
|
||||
import { LanguageClient } from 'vscode-languageclient';
|
||||
import * as os from 'os';
|
||||
import * as fs from 'fs-extra';
|
||||
import * as path from 'path';
|
||||
import * as tmp from 'tmp-promise';
|
||||
import { testExplorerExtensionId, TestHub } from 'vscode-test-adapter-api';
|
||||
|
|
@ -435,16 +436,21 @@ async function activateWithInstalledDistribution(
|
|||
ctx.subscriptions.push(queryHistoryConfigurationListener);
|
||||
const showResults = async (item: FullCompletedQueryInfo) =>
|
||||
showResultsForCompletedQuery(item, WebviewReveal.Forced);
|
||||
const queryStorageLocation = path.join(ctx.globalStorageUri.fsPath, 'queries');
|
||||
await fs.ensureDir(queryStorageLocation);
|
||||
|
||||
const qhm = new QueryHistoryManager(
|
||||
qs,
|
||||
dbm,
|
||||
ctx.extensionPath,
|
||||
queryStorageLocation,
|
||||
ctx,
|
||||
queryHistoryConfigurationListener,
|
||||
showResults,
|
||||
async (from: FullCompletedQueryInfo, to: FullCompletedQueryInfo) =>
|
||||
showResultsForComparison(from, to),
|
||||
);
|
||||
await qhm.readQueryHistory();
|
||||
|
||||
ctx.subscriptions.push(qhm);
|
||||
void logger.log('Initializing results panel interface.');
|
||||
const intm = new InterfaceManager(ctx, dbm, cliServer, queryServerLogger);
|
||||
|
|
@ -513,10 +519,12 @@ async function activateWithInstalledDistribution(
|
|||
qs,
|
||||
databaseItem,
|
||||
initialInfo,
|
||||
queryStorageLocation,
|
||||
progress,
|
||||
source.token,
|
||||
);
|
||||
item.completeThisQuery(completedQueryInfo);
|
||||
await qhm.writeQueryHistory();
|
||||
await showResultsForCompletedQuery(item as FullCompletedQueryInfo, WebviewReveal.NotForced);
|
||||
// Note we must update the query history view after showing results as the
|
||||
// display and sorting might depend on the number of results
|
||||
|
|
@ -988,16 +996,16 @@ async function activateWithInstalledDistribution(
|
|||
void logger.log('Registering jump-to-definition handlers.');
|
||||
languages.registerDefinitionProvider(
|
||||
{ scheme: archiveFilesystemProvider.zipArchiveScheme },
|
||||
new TemplateQueryDefinitionProvider(cliServer, qs, dbm)
|
||||
new TemplateQueryDefinitionProvider(cliServer, qs, dbm, queryStorageLocation)
|
||||
);
|
||||
|
||||
languages.registerReferenceProvider(
|
||||
{ scheme: archiveFilesystemProvider.zipArchiveScheme },
|
||||
new TemplateQueryReferenceProvider(cliServer, qs, dbm)
|
||||
new TemplateQueryReferenceProvider(cliServer, qs, dbm, queryStorageLocation)
|
||||
);
|
||||
|
||||
const astViewer = new AstViewer();
|
||||
const templateProvider = new TemplatePrintAstProvider(cliServer, qs, dbm);
|
||||
const templateProvider = new TemplatePrintAstProvider(cliServer, qs, dbm, queryStorageLocation);
|
||||
|
||||
ctx.subscriptions.push(astViewer);
|
||||
ctx.subscriptions.push(commandRunnerWithProgress('codeQL.viewAst', async (
|
||||
|
|
@ -1036,7 +1044,7 @@ async function activateWithInstalledDistribution(
|
|||
}
|
||||
|
||||
function getContextStoragePath(ctx: ExtensionContext) {
|
||||
return ctx.storagePath || ctx.globalStoragePath;
|
||||
return ctx.storageUri?.fsPath || ctx.globalStorageUri.fsPath;
|
||||
}
|
||||
|
||||
async function initializeLogging(ctx: ExtensionContext): Promise<void> {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
import * as path from 'path';
|
||||
import * as fs from 'fs-extra';
|
||||
import {
|
||||
commands,
|
||||
Disposable,
|
||||
env,
|
||||
Event,
|
||||
EventEmitter,
|
||||
ExtensionContext,
|
||||
ProviderResult,
|
||||
Range,
|
||||
ThemeIcon,
|
||||
|
|
@ -80,6 +83,9 @@ export enum SortOrder {
|
|||
CountDesc = 'CountDesc',
|
||||
}
|
||||
|
||||
const ONE_HOUR_IN_MS = 1000 * 60 * 60;
|
||||
const TWO_HOURS_IN_MS = 1000 * 60 * 60 * 2;
|
||||
|
||||
/**
|
||||
* Tree data provider for the query history view.
|
||||
*/
|
||||
|
|
@ -118,6 +124,7 @@ export class HistoryTreeDataProvider extends DisposableObject {
|
|||
title: 'Query History Item',
|
||||
command: 'codeQLQueryHistory.itemClicked',
|
||||
arguments: [element],
|
||||
tooltip: element.failureReason || element.label
|
||||
};
|
||||
|
||||
// Populate the icon and the context value. We use the context value to
|
||||
|
|
@ -217,6 +224,12 @@ export class HistoryTreeDataProvider extends DisposableObject {
|
|||
return this.history;
|
||||
}
|
||||
|
||||
set allHistory(history: FullQueryInfo[]) {
|
||||
this.history = history;
|
||||
this.current = history[0];
|
||||
this.refresh();
|
||||
}
|
||||
|
||||
refresh() {
|
||||
this._onDidChangeTreeData.fire(undefined);
|
||||
}
|
||||
|
|
@ -240,16 +253,20 @@ const DOUBLE_CLICK_TIME = 500;
|
|||
const NO_QUERY_SELECTED = 'No query selected. Select a query history item you have already run and try again.';
|
||||
|
||||
export class QueryHistoryManager extends DisposableObject {
|
||||
|
||||
treeDataProvider: HistoryTreeDataProvider;
|
||||
treeView: TreeView<FullQueryInfo>;
|
||||
lastItemClick: { time: Date; item: FullQueryInfo } | undefined;
|
||||
compareWithItem: FullQueryInfo | undefined;
|
||||
queryHistoryScrubber: Disposable;
|
||||
private queryMetadataStorageLocation;
|
||||
|
||||
constructor(
|
||||
private qs: QueryServerClient,
|
||||
private dbm: DatabaseManager,
|
||||
extensionPath: string,
|
||||
queryHistoryConfigListener: QueryHistoryConfig,
|
||||
private queryStorageLocation: string,
|
||||
ctx: ExtensionContext,
|
||||
private queryHistoryConfigListener: QueryHistoryConfig,
|
||||
private selectedCallback: (item: FullCompletedQueryInfo) => Promise<void>,
|
||||
private doCompareCallback: (
|
||||
from: FullCompletedQueryInfo,
|
||||
|
|
@ -258,8 +275,10 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
) {
|
||||
super();
|
||||
|
||||
this.queryMetadataStorageLocation = path.join((ctx.storageUri || ctx.globalStorageUri).fsPath, 'query-history.json');
|
||||
|
||||
this.treeDataProvider = this.push(new HistoryTreeDataProvider(
|
||||
extensionPath
|
||||
ctx.extensionPath
|
||||
));
|
||||
this.treeView = this.push(window.createTreeView('codeQLQueryHistory', {
|
||||
treeDataProvider: this.treeDataProvider,
|
||||
|
|
@ -381,6 +400,16 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
this.push(
|
||||
queryHistoryConfigListener.onDidChangeConfiguration(() => {
|
||||
this.treeDataProvider.refresh();
|
||||
// recreate the history scrubber
|
||||
this.queryHistoryScrubber.dispose();
|
||||
this.queryHistoryScrubber = this.push(
|
||||
registerQueryHistoryScubber(
|
||||
ONE_HOUR_IN_MS, TWO_HOURS_IN_MS,
|
||||
queryHistoryConfigListener.ttlInMillis,
|
||||
this.queryStorageLocation,
|
||||
ctx
|
||||
)
|
||||
);
|
||||
})
|
||||
);
|
||||
|
||||
|
|
@ -398,6 +427,27 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
);
|
||||
},
|
||||
}));
|
||||
|
||||
// Register the query history scrubber
|
||||
// Every hour check if we need to re-run the query history scrubber.
|
||||
this.queryHistoryScrubber = this.push(
|
||||
registerQueryHistoryScubber(
|
||||
ONE_HOUR_IN_MS, TWO_HOURS_IN_MS,
|
||||
queryHistoryConfigListener.ttlInMillis,
|
||||
path.join(ctx.globalStorageUri.fsPath, 'queries'),
|
||||
ctx
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async readQueryHistory(): Promise<void> {
|
||||
const history = await FullQueryInfo.slurp(this.queryMetadataStorageLocation, this.queryHistoryConfigListener);
|
||||
this.treeDataProvider.allHistory = history;
|
||||
}
|
||||
|
||||
async writeQueryHistory(): Promise<void> {
|
||||
const toSave = this.treeDataProvider.allHistory.filter(q => q.isCompleted());
|
||||
await FullQueryInfo.splat(toSave, this.queryMetadataStorageLocation);
|
||||
}
|
||||
|
||||
async invokeCallbackOn(queryHistoryItem: FullQueryInfo) {
|
||||
|
|
@ -445,14 +495,16 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
multiSelect: FullQueryInfo[]
|
||||
) {
|
||||
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
|
||||
|
||||
(finalMultiSelect || [finalSingleItem]).forEach((item) => {
|
||||
// Removing in progress queries is not supported yet
|
||||
const toDelete = (finalMultiSelect || [finalSingleItem]);
|
||||
await Promise.all(toDelete.map(async (item) => {
|
||||
// Removing in progress queries is not supported. They must be cancelled first.
|
||||
if (item.status !== QueryStatus.InProgress) {
|
||||
this.treeDataProvider.remove(item);
|
||||
item.completedQuery?.dispose();
|
||||
await item.completedQuery?.query.cleanUp();
|
||||
}
|
||||
});
|
||||
}));
|
||||
await this.writeQueryHistory();
|
||||
const current = this.treeDataProvider.getCurrent();
|
||||
if (current !== undefined) {
|
||||
await this.treeView.reveal(current, { select: true });
|
||||
|
|
@ -488,19 +540,21 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
singleItem: FullQueryInfo,
|
||||
multiSelect: FullQueryInfo[]
|
||||
): Promise<void> {
|
||||
if (!this.assertSingleQuery(multiSelect)) {
|
||||
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
|
||||
|
||||
if (!this.assertSingleQuery(finalMultiSelect)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const response = await window.showInputBox({
|
||||
prompt: 'Label:',
|
||||
placeHolder: '(use default)',
|
||||
value: singleItem.label,
|
||||
value: finalSingleItem.label,
|
||||
});
|
||||
// undefined response means the user cancelled the dialog; don't change anything
|
||||
if (response !== undefined) {
|
||||
// Interpret empty string response as 'go back to using default'
|
||||
singleItem.initialInfo.userSpecifiedLabel = response === '' ? undefined : response;
|
||||
finalSingleItem.initialInfo.userSpecifiedLabel = response === '' ? undefined : response;
|
||||
this.treeDataProvider.refresh();
|
||||
}
|
||||
}
|
||||
|
|
@ -509,13 +563,15 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
singleItem: FullQueryInfo,
|
||||
multiSelect: FullQueryInfo[]
|
||||
) {
|
||||
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
|
||||
|
||||
try {
|
||||
if (!singleItem.completedQuery?.didRunSuccessfully) {
|
||||
if (!finalSingleItem.completedQuery?.didRunSuccessfully) {
|
||||
throw new Error('Please select a successful query.');
|
||||
}
|
||||
|
||||
const from = this.compareWithItem || singleItem;
|
||||
const to = await this.findOtherQueryToCompare(from, multiSelect);
|
||||
const to = await this.findOtherQueryToCompare(from, finalMultiSelect);
|
||||
|
||||
if (from.isCompleted() && to?.isCompleted()) {
|
||||
await this.doCompareCallback(from as FullCompletedQueryInfo, to as FullCompletedQueryInfo);
|
||||
|
|
@ -593,20 +649,22 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
singleItem: FullQueryInfo,
|
||||
multiSelect: FullQueryInfo[]
|
||||
) {
|
||||
if (!this.assertSingleQuery(multiSelect)) {
|
||||
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
|
||||
|
||||
if (!this.assertSingleQuery(finalMultiSelect)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!singleItem) {
|
||||
if (!finalSingleItem) {
|
||||
throw new Error(NO_QUERY_SELECTED);
|
||||
}
|
||||
|
||||
const params = new URLSearchParams({
|
||||
isQuickEval: String(!!singleItem.initialInfo.quickEvalPosition),
|
||||
queryText: encodeURIComponent(await this.getQueryText(singleItem)),
|
||||
isQuickEval: String(!!finalSingleItem.initialInfo.quickEvalPosition),
|
||||
queryText: encodeURIComponent(await this.getQueryText(finalSingleItem)),
|
||||
});
|
||||
const uri = Uri.parse(
|
||||
`codeql:${singleItem.initialInfo.id}?${params.toString()}`, true
|
||||
`codeql:${finalSingleItem.initialInfo.id}?${params.toString()}`, true
|
||||
);
|
||||
const doc = await workspace.openTextDocument(uri);
|
||||
await window.showTextDocument(doc, { preview: false });
|
||||
|
|
@ -616,17 +674,20 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
singleItem: FullQueryInfo,
|
||||
multiSelect: FullQueryInfo[]
|
||||
) {
|
||||
if (!this.assertSingleQuery(multiSelect) || !singleItem.completedQuery) {
|
||||
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
|
||||
|
||||
if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem.completedQuery) {
|
||||
return;
|
||||
}
|
||||
const query = singleItem.completedQuery.query;
|
||||
|
||||
const query = finalSingleItem.completedQuery.query;
|
||||
const hasInterpretedResults = query.canHaveInterpretedResults();
|
||||
if (hasInterpretedResults) {
|
||||
await this.tryOpenExternalFile(
|
||||
query.resultsPaths.interpretedResultsPath
|
||||
);
|
||||
} else {
|
||||
const label = singleItem.label;
|
||||
const label = finalSingleItem.label;
|
||||
void showAndLogInformationMessage(
|
||||
`Query ${label} has no interpreted results.`
|
||||
);
|
||||
|
|
@ -637,13 +698,15 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
singleItem: FullQueryInfo,
|
||||
multiSelect: FullQueryInfo[]
|
||||
) {
|
||||
if (!this.assertSingleQuery(multiSelect)) {
|
||||
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
|
||||
|
||||
if (!this.assertSingleQuery(finalMultiSelect)) {
|
||||
return;
|
||||
}
|
||||
if (!singleItem.completedQuery) {
|
||||
if (!finalSingleItem.completedQuery) {
|
||||
return;
|
||||
}
|
||||
const query = singleItem.completedQuery.query;
|
||||
const query = finalSingleItem.completedQuery.query;
|
||||
if (await query.hasCsv()) {
|
||||
void this.tryOpenExternalFile(query.csvPath);
|
||||
return;
|
||||
|
|
@ -659,12 +722,14 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
singleItem: FullQueryInfo,
|
||||
multiSelect: FullQueryInfo[]
|
||||
) {
|
||||
if (!this.assertSingleQuery(multiSelect) || !singleItem.completedQuery) {
|
||||
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
|
||||
|
||||
if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem.completedQuery) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.tryOpenExternalFile(
|
||||
await singleItem.completedQuery.query.ensureCsvProduced(this.qs, this.dbm)
|
||||
await finalSingleItem.completedQuery.query.ensureCsvProduced(this.qs, this.dbm)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -672,15 +737,17 @@ export class QueryHistoryManager extends DisposableObject {
|
|||
singleItem: FullQueryInfo,
|
||||
multiSelect: FullQueryInfo[],
|
||||
) {
|
||||
if (!this.assertSingleQuery(multiSelect)) {
|
||||
const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
|
||||
|
||||
if (!this.assertSingleQuery(finalMultiSelect)) {
|
||||
return;
|
||||
}
|
||||
if (!singleItem.completedQuery) {
|
||||
if (!finalSingleItem.completedQuery) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.tryOpenExternalFile(
|
||||
await singleItem.completedQuery.query.ensureDilPath(this.qs)
|
||||
await finalSingleItem.completedQuery.query.ensureDilPath(this.qs)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -884,3 +951,106 @@ the file in the file explorer and dragging it into the workspace.`
|
|||
this.treeDataProvider.refresh();
|
||||
}
|
||||
}
|
||||
|
||||
const LAST_SCRUB_TIME_KEY = 'lastScrubTime';
|
||||
|
||||
/**
|
||||
* Registers an interval timer that will periodically check for queries old enought
|
||||
* to be deleted.
|
||||
*
|
||||
* Note that this scrubber will clean all queries from all workspaces. It should not
|
||||
* run too often and it should only run from one workspace at a time.
|
||||
*
|
||||
* Generally, `wakeInterval` should be significantly shorter than `throttleTime`.
|
||||
*
|
||||
* @param wakeInterval How often to check to see if the job should run.
|
||||
* @param throttleTime How often to actually run the job.
|
||||
* @param maxQueryTime The maximum age of a query before is ready for deletion.
|
||||
* @param queryDirectory The directory containing all queries.
|
||||
* @param ctx The extension context.
|
||||
*/
|
||||
export function registerQueryHistoryScubber(
|
||||
wakeInterval: number,
|
||||
throttleTime: number,
|
||||
maxQueryTime: number,
|
||||
queryDirectory: string,
|
||||
ctx: ExtensionContext,
|
||||
|
||||
// optional counter to keep track of how many times the scrubber has run
|
||||
counter?: {
|
||||
increment: () => void;
|
||||
}
|
||||
): Disposable {
|
||||
const deregister = setInterval(async () => {
|
||||
const lastScrubTime = ctx.globalState.get<number>(LAST_SCRUB_TIME_KEY);
|
||||
const now = Date.now();
|
||||
if (lastScrubTime === undefined || now - lastScrubTime >= throttleTime) {
|
||||
let scrubCount = 0;
|
||||
try {
|
||||
counter?.increment();
|
||||
void logger.log('Scrubbing query directory. Removing old queries.');
|
||||
// do a scrub
|
||||
if (!(await fs.pathExists(queryDirectory))) {
|
||||
void logger.log(`Query directory does not exist: ${queryDirectory}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const baseNames = await fs.readdir(queryDirectory);
|
||||
const errors: string[] = [];
|
||||
for (const baseName of baseNames) {
|
||||
const dir = path.join(queryDirectory, baseName);
|
||||
const timestampFile = path.join(dir, 'timestamp');
|
||||
try {
|
||||
if (!(await fs.stat(dir)).isDirectory()) {
|
||||
void logger.log(` ${dir} is not a directory. Deleting.`);
|
||||
await fs.remove(dir);
|
||||
scrubCount++;
|
||||
} else if (!(await fs.pathExists(timestampFile))) {
|
||||
void logger.log(` ${dir} has no timestamp file. Deleting.`);
|
||||
await fs.remove(dir);
|
||||
scrubCount++;
|
||||
} else if (!(await fs.stat(timestampFile)).isFile()) {
|
||||
void logger.log(` ${timestampFile} is not a file. Deleting.`);
|
||||
await fs.remove(dir);
|
||||
scrubCount++;
|
||||
} else {
|
||||
const timestampText = await fs.readFile(timestampFile, 'utf8');
|
||||
const timestamp = parseInt(timestampText, 10);
|
||||
|
||||
if (Number.isNaN(timestamp)) {
|
||||
void logger.log(` ${dir} has invalid timestamp '${timestampText}'. Deleting.`);
|
||||
await fs.remove(dir);
|
||||
scrubCount++;
|
||||
} else if (now - timestamp > maxQueryTime) {
|
||||
void logger.log(` ${dir} is older than ${maxQueryTime / 1000} seconds. Deleting.`);
|
||||
await fs.remove(dir);
|
||||
scrubCount++;
|
||||
} else {
|
||||
void logger.log(` ${dir} is not older than ${maxQueryTime / 1000} seconds. Keeping.`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push(` Could not delete '${dir}': ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
throw new Error('\n' + errors.join('\n'));
|
||||
}
|
||||
} catch (e) {
|
||||
void logger.log(`Error while scrubbing query directory: ${e}`);
|
||||
} finally {
|
||||
|
||||
// keep track of when we last scrubbed
|
||||
await ctx.globalState.update(LAST_SCRUB_TIME_KEY, now);
|
||||
void logger.log(`Scrubbed ${scrubCount} queries.`);
|
||||
}
|
||||
}
|
||||
}, wakeInterval);
|
||||
|
||||
return {
|
||||
dispose: () => {
|
||||
clearInterval(deregister);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import {
|
|||
import { QueryHistoryConfig } from './config';
|
||||
import { DatabaseInfo } from './pure/interface-types';
|
||||
import { showAndLogErrorMessage } from './helpers';
|
||||
import { asyncFilter } from './pure/helpers-pure';
|
||||
|
||||
/**
|
||||
* A description of the information about a query
|
||||
|
|
@ -190,9 +191,13 @@ export class FullQueryInfo {
|
|||
|
||||
static async slurp(fsPath: string, config: QueryHistoryConfig): Promise<FullQueryInfo[]> {
|
||||
try {
|
||||
if (!(await fs.pathExists(fsPath))) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const data = await fs.readFile(fsPath, 'utf8');
|
||||
const queries = JSON.parse(data);
|
||||
return queries.map((q: FullQueryInfo) => {
|
||||
const parsedQueries = queries.map((q: FullQueryInfo) => {
|
||||
|
||||
// Need to explicitly set prototype since reading in from JSON will not
|
||||
// do this automatically. Note that we can't call the constructor here since
|
||||
|
|
@ -215,6 +220,14 @@ export class FullQueryInfo {
|
|||
}
|
||||
return q;
|
||||
});
|
||||
|
||||
// filter out queries that have been deleted on disk
|
||||
// most likely another workspace has deleted them because the
|
||||
// queries aged out.
|
||||
return asyncFilter(parsedQueries, async (q) => {
|
||||
const resultsPath = q.completedQuery?.query.resultsPaths.resultsPath;
|
||||
return !!resultsPath && await fs.pathExists(resultsPath);
|
||||
});
|
||||
} catch (e) {
|
||||
void showAndLogErrorMessage('Error loading query history.', {
|
||||
fullMessage: ['Error loading query history.', e.stack].join('\n'),
|
||||
|
|
@ -234,8 +247,12 @@ export class FullQueryInfo {
|
|||
*/
|
||||
static async splat(queries: FullQueryInfo[], fsPath: string): Promise<void> {
|
||||
try {
|
||||
const data = JSON.stringify(queries, null, 2);
|
||||
await fs.mkdirp(path.dirname(fsPath));
|
||||
if (!(await fs.pathExists(fsPath))) {
|
||||
await fs.mkdir(path.dirname(fsPath), { recursive: true });
|
||||
}
|
||||
// remove incomplete queries since they cannot be recreated on restart
|
||||
const filteredQueries = queries.filter(q => q.completedQuery !== undefined);
|
||||
const data = JSON.stringify(filteredQueries, null, 2);
|
||||
await fs.writeFile(fsPath, data);
|
||||
} catch (e) {
|
||||
throw new Error(`Error saving query history to ${fsPath}: ${e.message}`);
|
||||
|
|
@ -253,13 +270,15 @@ export class FullQueryInfo {
|
|||
constructor(
|
||||
public readonly initialInfo: InitialQueryInfo,
|
||||
config: QueryHistoryConfig,
|
||||
private readonly source?: CancellationTokenSource
|
||||
private source?: CancellationTokenSource // used to cancel in progress queries
|
||||
) {
|
||||
this.setConfig(config);
|
||||
}
|
||||
|
||||
cancel() {
|
||||
this.source?.cancel();
|
||||
// query is no longer in progress, can delete the cancellation token source
|
||||
delete this.source;
|
||||
}
|
||||
|
||||
get startTime() {
|
||||
|
|
@ -342,6 +361,7 @@ export class FullQueryInfo {
|
|||
|
||||
completeThisQuery(info: QueryWithResults) {
|
||||
this.completedQuery = new CompletedQueryInfo(info);
|
||||
delete this.source;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -47,9 +47,6 @@ export const tmpDirDisposal = {
|
|||
}
|
||||
};
|
||||
|
||||
// exported for testing
|
||||
export const queriesDir = path.join(tmpDir.name, 'queries');
|
||||
|
||||
/**
|
||||
* A collection of evaluation-time information about a query,
|
||||
* including the query itself, and where we have decided to put
|
||||
|
|
@ -57,14 +54,13 @@ export const queriesDir = path.join(tmpDir.name, 'queries');
|
|||
* output and results.
|
||||
*/
|
||||
export class QueryEvaluationInfo {
|
||||
readonly querySaveDir: string;
|
||||
|
||||
/**
|
||||
* Note that in the {@link FullQueryInfo.slurp} method, we create a QueryEvaluationInfo instance
|
||||
* by explicitly setting the prototype in order to avoid calling this constructor.
|
||||
*/
|
||||
constructor(
|
||||
public readonly id: string,
|
||||
private readonly querySaveDir: string,
|
||||
public readonly dbItemPath: string,
|
||||
private readonly databaseHasMetadataFile: boolean,
|
||||
public readonly queryDbscheme: string, // the dbscheme file the query expects, based on library path resolution
|
||||
|
|
@ -72,7 +68,7 @@ export class QueryEvaluationInfo {
|
|||
public readonly metadata?: QueryMetadata,
|
||||
public readonly templates?: messages.TemplateDefinitions
|
||||
) {
|
||||
this.querySaveDir = path.join(queriesDir, this.id);
|
||||
/**/
|
||||
}
|
||||
|
||||
get dilPath() {
|
||||
|
|
@ -98,6 +94,16 @@ export class QueryEvaluationInfo {
|
|||
return path.join(this.querySaveDir, `sortedResults-${resultSetName}.bqrs`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a file in the query directory that indicates when this query was created.
|
||||
* This is important for keeping track of when queries should be removed.
|
||||
*/
|
||||
async createTimestampFile() {
|
||||
const timestampPath = path.join(this.querySaveDir, 'timestamp');
|
||||
await fs.ensureDir(this.querySaveDir);
|
||||
await fs.writeFile(timestampPath, Date.now().toString(), 'utf8');
|
||||
}
|
||||
|
||||
async run(
|
||||
qs: qsClient.QueryServerClient,
|
||||
upgradeQlo: string | undefined,
|
||||
|
|
@ -291,6 +297,10 @@ export class QueryEvaluationInfo {
|
|||
await qs.cliServer.generateResultsCsv(ensureMetadataIsComplete(this.metadata), this.resultsPaths.resultsPath, this.csvPath, sourceInfo);
|
||||
return this.csvPath;
|
||||
}
|
||||
|
||||
async cleanUp(): Promise<void> {
|
||||
await fs.remove(this.querySaveDir);
|
||||
}
|
||||
}
|
||||
|
||||
export interface QueryWithResults {
|
||||
|
|
@ -588,6 +598,7 @@ export async function compileAndRunQueryAgainstDatabase(
|
|||
qs: qsClient.QueryServerClient,
|
||||
dbItem: DatabaseItem,
|
||||
initialInfo: InitialQueryInfo,
|
||||
queryStorageLocation: string,
|
||||
progress: ProgressCallback,
|
||||
token: CancellationToken,
|
||||
templates?: messages.TemplateDefinitions,
|
||||
|
|
@ -635,10 +646,12 @@ export async function compileAndRunQueryAgainstDatabase(
|
|||
// The `capabilities.untrustedWorkspaces.restrictedConfigurations` entry in package.json doesn't
|
||||
// work with hidden settings, so we manually check that the workspace is trusted before looking at
|
||||
// whether the `shouldInsecurelyLoadMlModelsFromPacks` setting is enabled.
|
||||
if (workspace.isTrusted &&
|
||||
if (
|
||||
workspace.isTrusted &&
|
||||
config.isCanary() &&
|
||||
config.shouldInsecurelyLoadMlModelsFromPacks() &&
|
||||
await cliServer.cliConstraints.supportsResolveMlModels()) {
|
||||
await cliServer.cliConstraints.supportsResolveMlModels()
|
||||
) {
|
||||
try {
|
||||
availableMlModels = (await cliServer.resolveMlModels(diskWorkspaceFolders)).models;
|
||||
void logger.log(`Found available ML models at the following paths: ${availableMlModels.map(x => `'${x.path}'`).join(', ')}.`);
|
||||
|
|
@ -651,7 +664,7 @@ export async function compileAndRunQueryAgainstDatabase(
|
|||
|
||||
const hasMetadataFile = (await dbItem.hasMetadataFile());
|
||||
const query = new QueryEvaluationInfo(
|
||||
initialInfo.id,
|
||||
path.join(queryStorageLocation, initialInfo.id),
|
||||
dbItem.databaseUri.fsPath,
|
||||
hasMetadataFile,
|
||||
packConfig.dbscheme,
|
||||
|
|
@ -659,11 +672,13 @@ export async function compileAndRunQueryAgainstDatabase(
|
|||
metadata,
|
||||
templates
|
||||
);
|
||||
await query.createTimestampFile();
|
||||
|
||||
const upgradeDir = await tmp.dir({ dir: upgradesTmpDir.name, unsafeCleanup: true });
|
||||
let upgradeDir: tmp.DirectoryResult | undefined;
|
||||
try {
|
||||
let upgradeQlo;
|
||||
if (await hasNondestructiveUpgradeCapabilities(qs)) {
|
||||
upgradeDir = await tmp.dir({ dir: upgradesTmpDir.name, unsafeCleanup: true });
|
||||
upgradeQlo = await compileNonDestructiveUpgrade(qs, upgradeDir, query, qlProgram, dbItem, progress, token);
|
||||
} else {
|
||||
await checkDbschemeCompatibility(cliServer, qs, query, qlProgram, dbItem, progress, token);
|
||||
|
|
@ -722,7 +737,7 @@ export async function compileAndRunQueryAgainstDatabase(
|
|||
}
|
||||
} finally {
|
||||
try {
|
||||
await upgradeDir.cleanup();
|
||||
await upgradeDir?.cleanup();
|
||||
} catch (e) {
|
||||
void qs.logger.log(`Could not clean up the upgrades dir. Reason: ${e.message || e}`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import { DatabaseItem, DatabaseManager } from '../../databases';
|
|||
import { CodeQLExtensionInterface } from '../../extension';
|
||||
import { dbLoc, storagePath } from './global.helper';
|
||||
import { importArchiveDatabase } from '../../databaseFetcher';
|
||||
import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo } from '../../run-queries';
|
||||
import { compileAndRunQueryAgainstDatabase, createInitialQueryInfo, tmpDir } from '../../run-queries';
|
||||
import { CodeQLCliServer } from '../../cli';
|
||||
import { QueryServerClient } from '../../queryserver-client';
|
||||
import { skipIfNoCodeQL } from '../ensureCli';
|
||||
|
|
@ -97,6 +97,7 @@ describe('Queries', function() {
|
|||
qs,
|
||||
dbItem,
|
||||
await mockInitialQueryInfo(queryPath),
|
||||
path.join(tmpDir.name, 'mock-storage-path'),
|
||||
progress,
|
||||
token
|
||||
);
|
||||
|
|
@ -119,6 +120,7 @@ describe('Queries', function() {
|
|||
qs,
|
||||
dbItem,
|
||||
await mockInitialQueryInfo(queryPath),
|
||||
path.join(tmpDir.name, 'mock-storage-path'),
|
||||
progress,
|
||||
token
|
||||
);
|
||||
|
|
|
|||
|
|
@ -15,6 +15,9 @@ describe('launching with a minimal workspace', async () => {
|
|||
assert(ext);
|
||||
});
|
||||
|
||||
// Note, this test will only pass in pristine workspaces. This means that when run locally and you
|
||||
// reuse an existing workspace that starts with an open ql file, this test will fail. There is
|
||||
// no need to make any changes since this will still pass on CI.
|
||||
it('should not activate the extension at first', () => {
|
||||
assert(ext!.isActive === false);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,17 +1,21 @@
|
|||
import * as fs from 'fs-extra';
|
||||
import * as path from 'path';
|
||||
import * as chai from 'chai';
|
||||
import 'mocha';
|
||||
import 'sinon-chai';
|
||||
import * as vscode from 'vscode';
|
||||
import * as sinon from 'sinon';
|
||||
|
||||
import * as chaiAsPromised from 'chai-as-promised';
|
||||
import { logger } from '../../logging';
|
||||
import { QueryHistoryManager, HistoryTreeDataProvider, SortOrder } from '../../query-history';
|
||||
import { QueryEvaluationInfo, QueryWithResults } from '../../run-queries';
|
||||
import { QueryHistoryManager, HistoryTreeDataProvider, SortOrder, registerQueryHistoryScubber } from '../../query-history';
|
||||
import { QueryEvaluationInfo, QueryWithResults, tmpDir } from '../../run-queries';
|
||||
import { QueryHistoryConfigListener } from '../../config';
|
||||
import * as messages from '../../pure/messages';
|
||||
import { QueryServerClient } from '../../queryserver-client';
|
||||
import { FullQueryInfo, InitialQueryInfo } from '../../query-results';
|
||||
import { DatabaseManager } from '../../databases';
|
||||
import * as tmp from 'tmp-promise';
|
||||
|
||||
chai.use(chaiAsPromised);
|
||||
const expect = chai.expect;
|
||||
|
|
@ -19,6 +23,7 @@ const assert = chai.assert;
|
|||
|
||||
|
||||
describe('query-history', () => {
|
||||
const mockExtensionLocation = path.join(tmpDir.name, 'mock-extension-location');
|
||||
let configListener: QueryHistoryConfigListener;
|
||||
let showTextDocumentSpy: sinon.SinonStub;
|
||||
let showInformationMessageSpy: sinon.SinonStub;
|
||||
|
|
@ -312,7 +317,7 @@ describe('query-history', () => {
|
|||
describe('HistoryTreeDataProvider', () => {
|
||||
let historyTreeDataProvider: HistoryTreeDataProvider;
|
||||
beforeEach(() => {
|
||||
historyTreeDataProvider = new HistoryTreeDataProvider(vscode.Uri.file('/a/b/c').fsPath);
|
||||
historyTreeDataProvider = new HistoryTreeDataProvider(vscode.Uri.file(mockExtensionLocation).fsPath);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
|
@ -327,29 +332,30 @@ describe('query-history', () => {
|
|||
title: 'Query History Item',
|
||||
command: 'codeQLQueryHistory.itemClicked',
|
||||
arguments: [mockQuery],
|
||||
tooltip: mockQuery.label,
|
||||
});
|
||||
expect(treeItem.label).to.contain('hucairz');
|
||||
expect(treeItem.contextValue).to.eq('rawResultsItem');
|
||||
expect(treeItem.iconPath).to.deep.eq(vscode.Uri.file('/a/b/c/media/drive.svg').fsPath);
|
||||
expect(treeItem.iconPath).to.deep.eq(vscode.Uri.file(mockExtensionLocation + '/media/drive.svg').fsPath);
|
||||
});
|
||||
|
||||
it('should get a tree item with interpreted results', async () => {
|
||||
const mockQuery = createMockFullQueryInfo('a', createMockQueryWithResults(true, /* interpreted results */ true));
|
||||
const treeItem = await historyTreeDataProvider.getTreeItem(mockQuery);
|
||||
expect(treeItem.contextValue).to.eq('interpretedResultsItem');
|
||||
expect(treeItem.iconPath).to.deep.eq(vscode.Uri.file('/a/b/c/media/drive.svg').fsPath);
|
||||
expect(treeItem.iconPath).to.deep.eq(vscode.Uri.file(mockExtensionLocation + '/media/drive.svg').fsPath);
|
||||
});
|
||||
|
||||
it('should get a tree item that did not complete successfully', async () => {
|
||||
const mockQuery = createMockFullQueryInfo('a', createMockQueryWithResults(false), false);
|
||||
const treeItem = await historyTreeDataProvider.getTreeItem(mockQuery);
|
||||
expect(treeItem.iconPath).to.eq(vscode.Uri.file('/a/b/c/media/red-x.svg').fsPath);
|
||||
expect(treeItem.iconPath).to.eq(vscode.Uri.file(mockExtensionLocation + '/media/red-x.svg').fsPath);
|
||||
});
|
||||
|
||||
it('should get a tree item that failed before creating any results', async () => {
|
||||
const mockQuery = createMockFullQueryInfo('a', undefined, true);
|
||||
const treeItem = await historyTreeDataProvider.getTreeItem(mockQuery);
|
||||
expect(treeItem.iconPath).to.eq(vscode.Uri.file('/a/b/c/media/red-x.svg').fsPath);
|
||||
expect(treeItem.iconPath).to.eq(vscode.Uri.file(mockExtensionLocation + '/media/red-x.svg').fsPath);
|
||||
});
|
||||
|
||||
it('should get a tree item that is in progress', async () => {
|
||||
|
|
@ -537,17 +543,180 @@ describe('query-history', () => {
|
|||
return fqi;
|
||||
}
|
||||
|
||||
describe('query history scrubber', () => {
|
||||
let clock: sinon.SinonFakeTimers;
|
||||
let deregister: vscode.Disposable | undefined;
|
||||
let mockCtx: vscode.ExtensionContext;
|
||||
let runCount = 0;
|
||||
|
||||
const ONE_HOUR_IN_MS = 60 * 60 * 1000;
|
||||
const TWO_HOURS_IN_MS = 2 * ONE_HOUR_IN_MS;
|
||||
const ONE_DAY_IN_MS = 24 * ONE_HOUR_IN_MS;
|
||||
// We don't want our times to align exactly with the hour,
|
||||
// so we can better mimic real life
|
||||
const LESS_THAN_ONE_DAY = ONE_DAY_IN_MS - 1000;
|
||||
const tmpDir = tmp.dirSync({
|
||||
unsafeCleanup: true
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
clock = sandbox.useFakeTimers({
|
||||
toFake: ['setInterval', 'Date']
|
||||
});
|
||||
mockCtx = {
|
||||
globalState: {
|
||||
lastScrubTime: Date.now(),
|
||||
get(key: string) {
|
||||
if (key !== 'lastScrubTime') {
|
||||
throw new Error(`Unexpected key: ${key}`);
|
||||
}
|
||||
return this.lastScrubTime;
|
||||
},
|
||||
async update(key: string, value: any) {
|
||||
if (key !== 'lastScrubTime') {
|
||||
throw new Error(`Unexpected key: ${key}`);
|
||||
}
|
||||
this.lastScrubTime = value;
|
||||
}
|
||||
}
|
||||
} as any as vscode.ExtensionContext;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clock.restore();
|
||||
if (deregister) {
|
||||
deregister.dispose();
|
||||
deregister = undefined;
|
||||
}
|
||||
});
|
||||
|
||||
it('should not throw an error when the query directory does not exist', async function() {
|
||||
// because of the waits, we need to have a higher timeout on this test.
|
||||
this.timeout(5000);
|
||||
registerScrubber('idontexist');
|
||||
|
||||
clock.tick(ONE_HOUR_IN_MS);
|
||||
await wait();
|
||||
expect(runCount, 'Should not have called the scrubber').to.eq(0);
|
||||
|
||||
clock.tick(ONE_HOUR_IN_MS - 1);
|
||||
await wait();
|
||||
expect(runCount, 'Should not have called the scrubber').to.eq(0);
|
||||
|
||||
clock.tick(1);
|
||||
await wait();
|
||||
expect(runCount, 'Should have called the scrubber once').to.eq(1);
|
||||
|
||||
clock.tick(TWO_HOURS_IN_MS);
|
||||
await wait();
|
||||
expect(runCount, 'Should have called the scrubber a second time').to.eq(2);
|
||||
|
||||
expect((mockCtx.globalState as any).lastScrubTime).to.eq(TWO_HOURS_IN_MS * 2, 'Should have scrubbed the last time at 4 hours.');
|
||||
});
|
||||
|
||||
it('should scrub directories', async () => {
|
||||
// create two query directories that are right around the cut off time
|
||||
const queryDir = createMockQueryDir(ONE_HOUR_IN_MS, TWO_HOURS_IN_MS);
|
||||
registerScrubber(queryDir);
|
||||
|
||||
clock.tick(TWO_HOURS_IN_MS);
|
||||
await wait();
|
||||
|
||||
// should have deleted only the invalid locations
|
||||
expectDirectories(
|
||||
queryDir,
|
||||
toQueryDirName(ONE_HOUR_IN_MS),
|
||||
toQueryDirName(TWO_HOURS_IN_MS)
|
||||
);
|
||||
|
||||
clock.tick(LESS_THAN_ONE_DAY);
|
||||
await wait();
|
||||
|
||||
// should have deleted the older directory
|
||||
expectDirectories(
|
||||
queryDir,
|
||||
toQueryDirName(TWO_HOURS_IN_MS)
|
||||
);
|
||||
|
||||
// Wait two more hours and the final query will be deleted
|
||||
clock.tick(TWO_HOURS_IN_MS);
|
||||
await wait();
|
||||
|
||||
// should have deleted everything
|
||||
expectDirectories(
|
||||
queryDir
|
||||
);
|
||||
});
|
||||
|
||||
function expectDirectories(queryDir: string, ...dirNames: string[]) {
|
||||
const files = fs.readdirSync(queryDir);
|
||||
expect(files.sort()).to.deep.eq(dirNames.sort());
|
||||
}
|
||||
|
||||
function createMockQueryDir(...timestamps: number[]) {
|
||||
const dir = tmpDir.name;
|
||||
const queryDir = path.join(dir, 'query');
|
||||
// create qyuery directory and fill it with some query directories
|
||||
fs.mkdirSync(queryDir);
|
||||
|
||||
// create an invalid file
|
||||
const invalidFile = path.join(queryDir, 'invalid.txt');
|
||||
fs.writeFileSync(invalidFile, 'invalid');
|
||||
|
||||
// create a directory without a timestamp file
|
||||
const noTimestampDir = path.join(queryDir, 'noTimestampDir');
|
||||
fs.mkdirSync(noTimestampDir);
|
||||
fs.writeFileSync(path.join(noTimestampDir, 'invalid.txt'), 'invalid');
|
||||
|
||||
// create a directory with a timestamp file, but is invalid
|
||||
const invalidTimestampDir = path.join(queryDir, 'invalidTimestampDir');
|
||||
fs.mkdirSync(invalidTimestampDir);
|
||||
fs.writeFileSync(path.join(invalidTimestampDir, 'timestamp'), 'invalid');
|
||||
|
||||
// create a directories with a valid timestamp files from the args
|
||||
timestamps.forEach((timestamp) => {
|
||||
const dir = path.join(queryDir, toQueryDirName(timestamp));
|
||||
fs.mkdirSync(dir);
|
||||
fs.writeFileSync(path.join(dir, 'timestamp'), `${Date.now() + timestamp}`);
|
||||
});
|
||||
|
||||
return queryDir;
|
||||
}
|
||||
|
||||
function toQueryDirName(timestamp: number) {
|
||||
return `query-${timestamp}`;
|
||||
}
|
||||
|
||||
function registerScrubber(dir: string) {
|
||||
deregister = registerQueryHistoryScubber(
|
||||
ONE_HOUR_IN_MS,
|
||||
TWO_HOURS_IN_MS,
|
||||
LESS_THAN_ONE_DAY,
|
||||
dir,
|
||||
mockCtx,
|
||||
{
|
||||
increment: () => runCount++
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async function wait(ms = 500) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
});
|
||||
|
||||
function createMockQueryWithResults(didRunSuccessfully = true, hasInterpretedResults = true): QueryWithResults {
|
||||
return {
|
||||
query: {
|
||||
hasInterpretedResults: () => Promise.resolve(hasInterpretedResults)
|
||||
} as QueryEvaluationInfo,
|
||||
hasInterpretedResults: () => Promise.resolve(hasInterpretedResults),
|
||||
cleanUp: sandbox.stub(),
|
||||
} as unknown as QueryEvaluationInfo,
|
||||
result: {
|
||||
resultType: didRunSuccessfully
|
||||
? messages.QueryResultType.SUCCESS
|
||||
: messages.QueryResultType.OTHER_ERROR
|
||||
} as messages.EvaluationResult,
|
||||
dispose: sandbox.spy(),
|
||||
dispose: sandbox.spy()
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -556,6 +725,10 @@ describe('query-history', () => {
|
|||
{} as QueryServerClient,
|
||||
{} as DatabaseManager,
|
||||
'xxx',
|
||||
{
|
||||
globalStorageUri: vscode.Uri.file(mockExtensionLocation),
|
||||
extensionPath: vscode.Uri.file('/x/y/z').fsPath,
|
||||
} as vscode.ExtensionContext,
|
||||
configListener,
|
||||
selectedCallback,
|
||||
doCompareCallback
|
||||
|
|
|
|||
|
|
@ -6,13 +6,12 @@ import 'sinon-chai';
|
|||
import * as sinon from 'sinon';
|
||||
import * as chaiAsPromised from 'chai-as-promised';
|
||||
import { FullQueryInfo, InitialQueryInfo, interpretResults } from '../../query-results';
|
||||
import { queriesDir, QueryEvaluationInfo, QueryWithResults, tmpDir } from '../../run-queries';
|
||||
import { QueryEvaluationInfo, QueryWithResults, tmpDir } from '../../run-queries';
|
||||
import { QueryHistoryConfig } from '../../config';
|
||||
import { EvaluationResult, QueryResultType } from '../../pure/messages';
|
||||
import { DatabaseInfo, SortDirection, SortedResultSetInfo } from '../../pure/interface-types';
|
||||
import { CodeQLCliServer, SourceInfo } from '../../cli';
|
||||
import { env } from 'process';
|
||||
import { CancellationTokenSource, Uri } from 'vscode';
|
||||
import { CancellationTokenSource, Uri, env } from 'vscode';
|
||||
|
||||
chai.use(chaiAsPromised);
|
||||
const expect = chai.expect;
|
||||
|
|
@ -22,12 +21,15 @@ describe('query-results', () => {
|
|||
let onDidChangeQueryHistoryConfigurationSpy: sinon.SinonSpy;
|
||||
let mockConfig: QueryHistoryConfig;
|
||||
let sandbox: sinon.SinonSandbox;
|
||||
let queryPath: string;
|
||||
let cnt = 0;
|
||||
|
||||
beforeEach(() => {
|
||||
sandbox = sinon.createSandbox();
|
||||
disposeSpy = sandbox.spy();
|
||||
onDidChangeQueryHistoryConfigurationSpy = sandbox.spy();
|
||||
mockConfig = mockQueryHistoryConfig();
|
||||
queryPath = path.join(Uri.file(tmpDir.name).fsPath, `query-${cnt++}`);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
|
@ -40,6 +42,7 @@ describe('query-results', () => {
|
|||
const date = new Date('2022-01-01T00:00:00.000Z');
|
||||
const dateStr = date.toLocaleString(env.language);
|
||||
(fqi.initialInfo as any).start = date;
|
||||
|
||||
expect(fqi.interpolate('xxx')).to.eq('xxx');
|
||||
expect(fqi.interpolate('%t %q %d %s %%')).to.eq(`${dateStr} hucairz a in progress %`);
|
||||
expect(fqi.interpolate('%t %q %d %s %%::%t %q %d %s %%')).to.eq(`${dateStr} hucairz a in progress %::${dateStr} hucairz a in progress %`);
|
||||
|
|
@ -51,7 +54,7 @@ describe('query-results', () => {
|
|||
// from the query path
|
||||
expect(fqi.getQueryName()).to.eq('hucairz');
|
||||
|
||||
fqi.completeThisQuery(createMockQueryWithResults());
|
||||
fqi.completeThisQuery(createMockQueryWithResults(queryPath));
|
||||
|
||||
// from the metadata
|
||||
expect(fqi.getQueryName()).to.eq('vwx');
|
||||
|
|
@ -92,7 +95,7 @@ describe('query-results', () => {
|
|||
|
||||
// the %q from the config is now replaced by the name of the query
|
||||
// in the metadata
|
||||
fqi.completeThisQuery(createMockQueryWithResults());
|
||||
fqi.completeThisQuery(createMockQueryWithResults(queryPath));
|
||||
expect(fqi.label).to.eq('from config vwx');
|
||||
|
||||
// replace the config with a user specified label
|
||||
|
|
@ -102,9 +105,10 @@ describe('query-results', () => {
|
|||
});
|
||||
|
||||
it('should get the getResultsPath', () => {
|
||||
const fqi = createMockFullQueryInfo('a', createMockQueryWithResults());
|
||||
const query = createMockQueryWithResults(queryPath);
|
||||
const fqi = createMockFullQueryInfo('a', query);
|
||||
const completedQuery = fqi.completedQuery!;
|
||||
const expectedResultsPath = path.join(queriesDir, 'some-id/results.bqrs');
|
||||
const expectedResultsPath = path.join(queryPath, 'results.bqrs');
|
||||
|
||||
// from results path
|
||||
expect(completedQuery.getResultsPath('zxa', false)).to.eq(expectedResultsPath);
|
||||
|
|
@ -121,7 +125,7 @@ describe('query-results', () => {
|
|||
});
|
||||
|
||||
it('should get the statusString', () => {
|
||||
const fqi = createMockFullQueryInfo('a', createMockQueryWithResults(false));
|
||||
const fqi = createMockFullQueryInfo('a', createMockQueryWithResults(queryPath, false));
|
||||
const completedQuery = fqi.completedQuery!;
|
||||
|
||||
completedQuery.result.message = 'Tremendously';
|
||||
|
|
@ -146,7 +150,7 @@ describe('query-results', () => {
|
|||
|
||||
it('should updateSortState', async () => {
|
||||
// setup
|
||||
const fqi = createMockFullQueryInfo('a', createMockQueryWithResults());
|
||||
const fqi = createMockFullQueryInfo('a', createMockQueryWithResults(queryPath));
|
||||
const completedQuery = fqi.completedQuery!;
|
||||
|
||||
const spy = sandbox.spy();
|
||||
|
|
@ -162,8 +166,8 @@ describe('query-results', () => {
|
|||
await completedQuery.updateSortState(mockServer, 'a-result-set-name', sortState);
|
||||
|
||||
// verify
|
||||
const expectedResultsPath = path.join(queriesDir, 'some-id/results.bqrs');
|
||||
const expectedSortedResultsPath = path.join(queriesDir, 'some-id/sortedResults-a-result-set-name.bqrs');
|
||||
const expectedResultsPath = path.join(queryPath, 'results.bqrs');
|
||||
const expectedSortedResultsPath = path.join(queryPath, 'sortedResults-a-result-set-name.bqrs');
|
||||
expect(spy).to.have.been.calledWith(
|
||||
expectedResultsPath,
|
||||
expectedSortedResultsPath,
|
||||
|
|
@ -248,12 +252,11 @@ describe('query-results', () => {
|
|||
});
|
||||
|
||||
describe('splat and slurp', () => {
|
||||
// TODO also add a test for round trip starting from file
|
||||
it('should splat and slurp query history', async () => {
|
||||
const infoSuccessRaw = createMockFullQueryInfo('a', createMockQueryWithResults(false, false, '/a/b/c/a', false));
|
||||
const infoSuccessInterpreted = createMockFullQueryInfo('b', createMockQueryWithResults(true, true, '/a/b/c/b', false));
|
||||
const infoSuccessRaw = createMockFullQueryInfo('a', createMockQueryWithResults(`${queryPath}-a`, false, false, '/a/b/c/a', false));
|
||||
const infoSuccessInterpreted = createMockFullQueryInfo('b', createMockQueryWithResults(`${queryPath}-b`, true, true, '/a/b/c/b', false));
|
||||
const infoEarlyFailure = createMockFullQueryInfo('c', undefined, true);
|
||||
const infoLateFailure = createMockFullQueryInfo('d', createMockQueryWithResults(false, false, '/a/b/c/d', false));
|
||||
const infoLateFailure = createMockFullQueryInfo('d', createMockQueryWithResults(`${queryPath}-c`, false, false, '/a/b/c/d', false));
|
||||
const infoInprogress = createMockFullQueryInfo('e');
|
||||
const allHistory = [
|
||||
infoSuccessRaw,
|
||||
|
|
@ -263,7 +266,16 @@ describe('query-results', () => {
|
|||
infoInprogress
|
||||
];
|
||||
|
||||
const allHistoryPath = path.join(queriesDir, 'all-history.json');
|
||||
// the expected results only contains the history with completed queries
|
||||
const expectedHistory = [
|
||||
infoSuccessRaw,
|
||||
infoSuccessInterpreted,
|
||||
infoLateFailure,
|
||||
];
|
||||
|
||||
const allHistoryPath = path.join(tmpDir.name, 'all-history.json');
|
||||
|
||||
// splat and slurp
|
||||
await FullQueryInfo.splat(allHistory, allHistoryPath);
|
||||
const allHistoryActual = await FullQueryInfo.slurp(allHistoryPath, mockConfig);
|
||||
|
||||
|
|
@ -287,7 +299,7 @@ describe('query-results', () => {
|
|||
}
|
||||
}
|
||||
});
|
||||
allHistory.forEach(info => {
|
||||
expectedHistory.forEach(info => {
|
||||
if (info.completedQuery) {
|
||||
(info.completedQuery as any).dispose = undefined;
|
||||
}
|
||||
|
|
@ -295,16 +307,27 @@ describe('query-results', () => {
|
|||
|
||||
// make the diffs somewhat sane by comparing each element directly
|
||||
for (let i = 0; i < allHistoryActual.length; i++) {
|
||||
expect(allHistoryActual[i]).to.deep.eq(allHistory[i]);
|
||||
expect(allHistoryActual[i]).to.deep.eq(expectedHistory[i]);
|
||||
}
|
||||
expect(allHistoryActual.length).to.deep.eq(allHistory.length);
|
||||
expect(allHistoryActual.length).to.deep.eq(expectedHistory.length);
|
||||
});
|
||||
});
|
||||
|
||||
function createMockQueryWithResults(
|
||||
queryPath: string,
|
||||
didRunSuccessfully = true,
|
||||
hasInterpretedResults = true,
|
||||
dbPath = '/a/b/c',
|
||||
includeSpies = true
|
||||
): QueryWithResults {
|
||||
// pretend that the results path exists
|
||||
const resultsPath = path.join(queryPath, 'results.bqrs');
|
||||
fs.mkdirpSync(queryPath);
|
||||
fs.writeFileSync(resultsPath, '', 'utf8');
|
||||
|
||||
function createMockQueryWithResults(didRunSuccessfully = true, hasInterpretedResults = true, dbPath = '/a/b/c', includeSpies = true): QueryWithResults {
|
||||
const query = new QueryEvaluationInfo('some-id',
|
||||
Uri.file(dbPath).fsPath, // parse the Uri to make sure it is platform-independent
|
||||
const query = new QueryEvaluationInfo(
|
||||
queryPath,
|
||||
Uri.file(dbPath).fsPath,
|
||||
true,
|
||||
'queryDbscheme',
|
||||
undefined,
|
||||
|
|
@ -361,6 +384,7 @@ describe('query-results', () => {
|
|||
function mockQueryHistoryConfig(): QueryHistoryConfig {
|
||||
return {
|
||||
onDidChangeConfiguration: onDidChangeQueryHistoryConfigurationSpy,
|
||||
ttlInMillis: 999999,
|
||||
format: 'from config %q'
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import 'sinon-chai';
|
|||
import * as sinon from 'sinon';
|
||||
import * as chaiAsPromised from 'chai-as-promised';
|
||||
|
||||
import { QueryEvaluationInfo, queriesDir } from '../../run-queries';
|
||||
import { QueryEvaluationInfo } from '../../run-queries';
|
||||
import { Severity, compileQuery } from '../../pure/messages';
|
||||
import { Uri } from 'vscode';
|
||||
|
||||
|
|
@ -14,13 +14,13 @@ const expect = chai.expect;
|
|||
|
||||
describe('run-queries', () => {
|
||||
it('should create a QueryEvaluationInfo', () => {
|
||||
const info = createMockQueryInfo();
|
||||
const saveDir = 'query-save-dir';
|
||||
const info = createMockQueryInfo(true, saveDir);
|
||||
|
||||
const queryId = info.id;
|
||||
expect(info.compiledQueryPath).to.eq(path.join(queriesDir, queryId, 'compiledQuery.qlo'));
|
||||
expect(info.dilPath).to.eq(path.join(queriesDir, queryId, 'results.dil'));
|
||||
expect(info.resultsPaths.resultsPath).to.eq(path.join(queriesDir, queryId, 'results.bqrs'));
|
||||
expect(info.resultsPaths.interpretedResultsPath).to.eq(path.join(queriesDir, queryId, 'interpretedResults.sarif'));
|
||||
expect(info.compiledQueryPath).to.eq(path.join(saveDir, 'compiledQuery.qlo'));
|
||||
expect(info.dilPath).to.eq(path.join(saveDir, 'results.dil'));
|
||||
expect(info.resultsPaths.resultsPath).to.eq(path.join(saveDir, 'results.bqrs'));
|
||||
expect(info.resultsPaths.interpretedResultsPath).to.eq(path.join(saveDir, 'interpretedResults.sarif'));
|
||||
expect(info.dbItemPath).to.eq(Uri.file('/abc').fsPath);
|
||||
});
|
||||
|
||||
|
|
@ -90,9 +90,9 @@ describe('run-queries', () => {
|
|||
});
|
||||
|
||||
let queryNum = 0;
|
||||
function createMockQueryInfo(databaseHasMetadataFile = true) {
|
||||
function createMockQueryInfo(databaseHasMetadataFile = true, saveDir = `save-dir${queryNum++}`) {
|
||||
return new QueryEvaluationInfo(
|
||||
`save-dir${queryNum++}`,
|
||||
saveDir,
|
||||
Uri.parse('file:///abc').fsPath,
|
||||
databaseHasMetadataFile,
|
||||
'my-scheme', // queryDbscheme,
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче