Merge pull request #1142 from github/aeisenberg/remote-queries-history

Store remote query artifacts in global storage

extensions/ql-vscode/src/extension.ts

@@ -815,7 +815,7 @@ async function activateWithInstalledDistribution(
   );
 
   void logger.log('Initializing remote queries interface.');
-  const rqm = new RemoteQueriesManager(ctx, cliServer, logger);
+  const rqm = new RemoteQueriesManager(ctx, cliServer, queryStorageDir, logger);
 
   registerRemoteQueryTextProvider();
 
@@ -862,7 +862,7 @@ async function activateWithInstalledDistribution(
 
   ctx.subscriptions.push(
     commandRunner('codeQL.showFakeRemoteQueryResults', async () => {
-      const analysisResultsManager = new AnalysesResultsManager(ctx, logger);
+      const analysisResultsManager = new AnalysesResultsManager(ctx, queryStorageDir, logger);
       const rqim = new RemoteQueriesInterfaceManager(ctx, logger, analysisResultsManager);
       await rqim.showResults(sampleData.sampleRemoteQuery, sampleData.sampleRemoteQueryResult);
 

extensions/ql-vscode/src/helpers.ts

@@ -545,3 +545,16 @@ export async function tryGetQueryMetadata(cliServer: CodeQLCliServer, queryPath:
     return;
   }
 }
+
+/**
+ * Creates a file in the query directory that indicates when this query was created.
+ * This is important for keeping track of when queries should be removed.
+ *
+ * @param queryPath The directory that will containt all files relevant to a query result.
+ * It does not need to exist.
+ */
+export async function createTimestampFile(storagePath: string) {
+  const timestampPath = path.join(storagePath, 'timestamp');
+  await fs.ensureDir(storagePath);
+  await fs.writeFile(timestampPath, Date.now().toString(), 'utf8');
+}

extensions/ql-vscode/src/query-history.ts

@@ -303,7 +303,7 @@ export class QueryHistoryManager extends DisposableObject {
         } else {
           this.treeDataProvider.setCurrentItem(ev.selection[0]);
         }
-        this.updateCompareWith(ev.selection);
+        this.updateCompareWith([...ev.selection]);
       })
     );
 
@@ -929,14 +929,17 @@ the file in the file explorer and dragging it into the workspace.`
   private determineSelection(
     singleItem: FullQueryInfo,
     multiSelect: FullQueryInfo[]
-  ): { finalSingleItem: FullQueryInfo; finalMultiSelect: FullQueryInfo[] } {
+  ): {
+    finalSingleItem: FullQueryInfo;
+    finalMultiSelect: FullQueryInfo[]
+  } {
     if (!singleItem && !multiSelect?.[0]) {
       const selection = this.treeView.selection;
       const current = this.treeDataProvider.getCurrent();
       if (selection?.length) {
         return {
           finalSingleItem: selection[0],
-          finalMultiSelect: selection
+          finalMultiSelect: [...selection]
         };
       } else if (current) {
         return {

extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts

@@ -15,6 +15,7 @@ export class AnalysesResultsManager {
 
   constructor(
     private readonly ctx: ExtensionContext,
+    readonly storagePath: string,
     private readonly logger: Logger,
   ) {
     this.analysesResults = [];
@@ -97,7 +98,7 @@ export class AnalysesResultsManager {
 
     let artifactPath;
     try {
-      artifactPath = await downloadArtifactFromLink(credentials, analysis.downloadLink);
+      artifactPath = await downloadArtifactFromLink(credentials, this.storagePath, analysis.downloadLink);
     }
     catch (e) {
       throw new Error(`Could not download the analysis results for ${analysis.nwo}: ${e.message}`);

extensions/ql-vscode/src/remote-queries/download-link.ts

@@ -1,15 +1,15 @@
 /**
- * Represents a link to an artifact to be downloaded. 
+ * Represents a link to an artifact to be downloaded.
  */
 export interface DownloadLink {
   /**
-   *  A unique id of the artifact being downloaded. 
+   *  A unique id of the artifact being downloaded.
    */
   id: string;
 
   /**
    * The URL path to use against the GitHub API to download the
-   * linked artifact. 
+   * linked artifact.
    */
   urlPath: string;
 
@@ -17,4 +17,9 @@ export interface DownloadLink {
    * An optional path to follow inside the downloaded archive containing the artifact.
    */
   innerFilePath?: string;
+
+  /**
+   * A unique id of the remote query run. This is used to determine where to store artifacts and data from the run.
+   */
+  queryId: string;
 }

extensions/ql-vscode/src/remote-queries/gh-actions-api-client.ts

@@ -54,23 +54,23 @@ export async function getRemoteQueryIndex(
 
 export async function downloadArtifactFromLink(
   credentials: Credentials,
+  storagePath: string,
   downloadLink: DownloadLink
 ): Promise<string> {
+
   const octokit = await credentials.getOctokit();
 
   // Download the zipped artifact.
   const response = await octokit.request(`GET ${downloadLink.urlPath}/zip`, {});
 
-  const zipFilePath = path.join(tmpDir.name, `${downloadLink.id}.zip`);
+  const zipFilePath = path.join(storagePath, downloadLink.queryId, `${downloadLink.id}.zip`);
   await saveFile(`${zipFilePath}`, response.data as ArrayBuffer);
 
   // Extract the zipped artifact.
-  const extractedPath = path.join(tmpDir.name, downloadLink.id);
+  const extractedPath = path.join(storagePath, downloadLink.queryId, downloadLink.id);
   await unzipFile(zipFilePath, extractedPath);
 
-  return downloadLink.innerFilePath
-    ? path.join(extractedPath, downloadLink.innerFilePath)
-    : extractedPath;
+  return path.join(extractedPath, downloadLink.innerFilePath || '');
 }
 
 /**

extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts

@@ -24,7 +24,7 @@ import { AnalysisSummary, RemoteQueryResult } from './remote-query-result';
 import { RemoteQuery } from './remote-query';
 import { RemoteQueryResult as RemoteQueryResultViewModel } from './shared/remote-query-result';
 import { AnalysisSummary as AnalysisResultViewModel } from './shared/remote-query-result';
-import { showAndLogWarningMessage, tmpDir } from '../helpers';
+import { showAndLogWarningMessage } from '../helpers';
 import { URLSearchParams } from 'url';
 import { SHOW_QUERY_TEXT_MSG } from '../query-history';
 import { AnalysesResultsManager } from './analyses-results-manager';
@@ -98,7 +98,7 @@ export class RemoteQueriesInterfaceManager {
           enableFindWidget: true,
           retainContextWhenHidden: true,
           localResourceRoots: [
-            Uri.file(tmpDir.name),
+            Uri.file(this.analysesResultsManager.storagePath),
             Uri.file(path.join(this.ctx.extensionPath, 'out')),
           ],
         }
@@ -224,7 +224,7 @@ export class RemoteQueriesInterfaceManager {
 
   private async viewAnalysisResults(msg: RemoteQueryViewAnalysisResultsMessage): Promise<void> {
     const downloadLink = msg.analysisSummary.downloadLink;
-    const filePath = path.join(tmpDir.name, downloadLink.id, downloadLink.innerFilePath || '');
+    const filePath = path.join(this.analysesResultsManager.storagePath, downloadLink.queryId, downloadLink.id, downloadLink.innerFilePath || '');
 
     const sarifViewerExtensionId = 'MS-SarifVSCode.sarif-viewer';
 

extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts

@@ -1,8 +1,12 @@
 import { CancellationToken, commands, ExtensionContext, Uri, window } from 'vscode';
+import { nanoid } from 'nanoid';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+
 import { Credentials } from '../authentication';
 import { CodeQLCliServer } from '../cli';
 import { ProgressCallback } from '../commandRunner';
-import { showAndLogErrorMessage, showInformationMessageWithAction } from '../helpers';
+import { createTimestampFile, showAndLogErrorMessage, showInformationMessageWithAction } from '../helpers';
 import { Logger } from '../logging';
 import { runRemoteQuery } from './run-remote-query';
 import { RemoteQueriesInterfaceManager } from './remote-queries-interface';
@@ -13,6 +17,7 @@ import { RemoteQueryResultIndex } from './remote-query-result-index';
 import { RemoteQueryResult } from './remote-query-result';
 import { DownloadLink } from './download-link';
 import { AnalysesResultsManager } from './analyses-results-manager';
+import { assertNever } from '../pure/helpers-pure';
 
 const autoDownloadMaxSize = 300 * 1024;
 const autoDownloadMaxCount = 100;
@@ -25,9 +30,10 @@ export class RemoteQueriesManager {
   constructor(
     private readonly ctx: ExtensionContext,
     private readonly cliServer: CodeQLCliServer,
+    private readonly storagePath: string,
     logger: Logger,
   ) {
-    this.analysesResultsManager = new AnalysesResultsManager(ctx, logger);
+    this.analysesResultsManager = new AnalysesResultsManager(ctx, storagePath, logger);
     this.interfaceManager = new RemoteQueriesInterfaceManager(ctx, logger, this.analysesResultsManager);
     this.remoteQueriesMonitor = new RemoteQueriesMonitor(ctx, logger);
   }
@@ -57,18 +63,24 @@ export class RemoteQueriesManager {
   ): Promise<void> {
     const credentials = await Credentials.initialize(this.ctx);
 
-    const queryResult = await this.remoteQueriesMonitor.monitorQuery(query, cancellationToken);
+    const queryWorkflowResult = await this.remoteQueriesMonitor.monitorQuery(query, cancellationToken);
 
     const executionEndTime = new Date();
 
-    if (queryResult.status === 'CompletedSuccessfully') {
+    if (queryWorkflowResult.status === 'CompletedSuccessfully') {
       const resultIndex = await getRemoteQueryIndex(credentials, query);
       if (!resultIndex) {
         void showAndLogErrorMessage(`There was an issue retrieving the result for the query ${query.queryName}`);
         return;
       }
 
-      const queryResult = this.mapQueryResult(executionEndTime, resultIndex);
+      const queryId = this.createQueryId(query.queryName);
+      await this.prepareStorageDirectory(queryId);
+      const queryResult = this.mapQueryResult(executionEndTime, resultIndex, queryId);
+
+      // Write the query result to the storage directory.
+      const queryResultFilePath = path.join(this.storagePath, queryId, 'query-result.json');
+      await fs.writeFile(queryResultFilePath, JSON.stringify(queryResult, null, 2), 'utf8');
 
       // Kick off auto-download of results.
       void commands.executeCommand('codeQL.autoDownloadRemoteQueryResults', queryResult);
@@ -79,13 +91,19 @@ export class RemoteQueriesManager {
       const shouldOpenView = await showInformationMessageWithAction(message, 'View');
       if (shouldOpenView) {
         await this.interfaceManager.showResults(query, queryResult);
-
       }
-    } else if (queryResult.status === 'CompletedUnsuccessfully') {
-      await showAndLogErrorMessage(`Remote query execution failed. Error: ${queryResult.error}`);
-      return;
-    } else if (queryResult.status === 'Cancelled') {
+    } else if (queryWorkflowResult.status === 'CompletedUnsuccessfully') {
+      await showAndLogErrorMessage(`Remote query execution failed. Error: ${queryWorkflowResult.error}`);
+
+    } else if (queryWorkflowResult.status === 'Cancelled') {
       await showAndLogErrorMessage('Remote query monitoring was cancelled');
+
+    } else if (queryWorkflowResult.status === 'InProgress') {
+      // Should not get here
+      await showAndLogErrorMessage(`Unexpected status: ${queryWorkflowResult.status}`);
+    } else {
+      // Ensure all cases are covered
+      assertNever(queryWorkflowResult.status);
     }
   }
 
@@ -109,7 +127,7 @@ export class RemoteQueriesManager {
       results => this.interfaceManager.setAnalysisResults(results));
   }
 
-  private mapQueryResult(executionEndTime: Date, resultIndex: RemoteQueryResultIndex): RemoteQueryResult {
+  private mapQueryResult(executionEndTime: Date, resultIndex: RemoteQueryResultIndex, queryId: string): RemoteQueryResult {
     const analysisSummaries = resultIndex.items.map(item => ({
       nwo: item.nwo,
       resultCount: item.resultCount,
@@ -117,13 +135,36 @@ export class RemoteQueriesManager {
       downloadLink: {
         id: item.artifactId.toString(),
         urlPath: `${resultIndex.artifactsUrlPath}/${item.artifactId}`,
-        innerFilePath: item.sarifFileSize ? 'results.sarif' : 'results.bqrs'
+        innerFilePath: item.sarifFileSize ? 'results.sarif' : 'results.bqrs',
+        queryId,
       } as DownloadLink
     }));
 
     return {
       executionEndTime,
-      analysisSummaries
+      analysisSummaries,
     };
   }
+
+  /**
+   * Generates a unique id for this query, suitable for determining the storage location for the downloaded query artifacts.
+   * @param queryName
+   * @returns
+   */
+  private createQueryId(queryName: string): string {
+    return `${queryName}-${nanoid()}`;
+
+  }
+
+  /**
+   * Prepares a directory for storing analysis results for a single query run.
+   * This directory contains a timestamp file, which will be
+   * used by the query history manager to determine when the directory
+   * should be deleted.
+   *
+   * @param queryName The name of the query that was run.
+   */
+  private async prepareStorageDirectory(queryId: string): Promise<void> {
+    await createTimestampFile(path.join(this.storagePath, queryId));
+  }
 }

extensions/ql-vscode/src/remote-queries/sample-data.ts

@@ -46,7 +46,8 @@ export const sampleRemoteQueryResult: RemoteQueryResult = {
       downloadLink: {
         id: '137697017',
         urlPath: '/repos/big-corp/controller-repo/actions/artifacts/137697017',
-        innerFilePath: 'results.sarif'
+        innerFilePath: 'results.sarif',
+        queryId: 'query.ql-123-xyz'
       }
     },
     {
@@ -56,7 +57,8 @@ export const sampleRemoteQueryResult: RemoteQueryResult = {
       downloadLink: {
         id: '137697018',
         urlPath: '/repos/big-corp/controller-repo/actions/artifacts/137697018',
-        innerFilePath: 'results.sarif'
+        innerFilePath: 'results.sarif',
+        queryId: 'query.ql-123-xyz'
       }
     },
     {
@@ -66,7 +68,8 @@ export const sampleRemoteQueryResult: RemoteQueryResult = {
       downloadLink: {
         id: '137697019',
         urlPath: '/repos/big-corp/controller-repo/actions/artifacts/137697019',
-        innerFilePath: 'results.sarif'
+        innerFilePath: 'results.sarif',
+        queryId: 'query.ql-123-xyz'
       }
     },
     {
@@ -76,7 +79,8 @@ export const sampleRemoteQueryResult: RemoteQueryResult = {
       downloadLink: {
         id: '137697020',
         urlPath: '/repos/big-corp/controller-repo/actions/artifacts/137697020',
-        innerFilePath: 'results.sarif'
+        innerFilePath: 'results.sarif',
+        queryId: 'query.ql-123-xyz'
       }
     }
   ]

extensions/ql-vscode/src/run-queries.ts

@@ -17,7 +17,7 @@ import { ErrorCodes, ResponseError } from 'vscode-languageclient';
 import * as cli from './cli';
 import * as config from './config';
 import { DatabaseItem, DatabaseManager } from './databases';
-import { getOnDiskWorkspaceFolders, showAndLogErrorMessage, tryGetQueryMetadata, upgradesTmpDir } from './helpers';
+import { createTimestampFile, getOnDiskWorkspaceFolders, showAndLogErrorMessage, tryGetQueryMetadata, upgradesTmpDir } from './helpers';
 import { ProgressCallback, UserCancellationException } from './commandRunner';
 import { DatabaseInfo, QueryMetadata } from './pure/interface-types';
 import { logger } from './logging';
@@ -99,9 +99,7 @@ export class QueryEvaluationInfo {
    * This is important for keeping track of when queries should be removed.
    */
   async createTimestampFile() {
-    const timestampPath = path.join(this.querySaveDir, 'timestamp');
-    await fs.ensureDir(this.querySaveDir);
-    await fs.writeFile(timestampPath, Date.now().toString(), 'utf8');
+    await createTimestampFile(this.querySaveDir);
   }
 
   async run(