Add support for remote queries raw results (#1198)

extensions/ql-vscode/src/extension.ts

@@ -890,7 +890,7 @@ async function activateWithInstalledDistribution(
 
   ctx.subscriptions.push(
     commandRunner('codeQL.showFakeRemoteQueryResults', async () => {
-      const analysisResultsManager = new AnalysesResultsManager(ctx, queryStorageDir, logger);
+      const analysisResultsManager = new AnalysesResultsManager(ctx, cliServer, queryStorageDir, logger);
       const rqim = new RemoteQueriesInterfaceManager(ctx, logger, analysisResultsManager);
       await rqim.showResults(sampleData.sampleRemoteQuery, sampleData.sampleRemoteQueryResult);
 

extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts

@@ -6,10 +6,12 @@ import { Credentials } from '../authentication';
 import { Logger } from '../logging';
 import { downloadArtifactFromLink } from './gh-actions-api-client';
 import { AnalysisSummary } from './shared/remote-query-result';
-import { AnalysisResults, AnalysisAlert } from './shared/analysis-result';
+import { AnalysisResults, AnalysisAlert, AnalysisRawResults } from './shared/analysis-result';
 import { UserCancellationException } from '../commandRunner';
 import { sarifParser } from '../sarif-parser';
 import { extractAnalysisAlerts } from './sarif-processing';
+import { CodeQLCliServer } from '../cli';
+import { extractRawResults } from './bqrs-processing';
 
 export class AnalysesResultsManager {
   // Store for the results of various analyses for each remote query.
@@ -18,6 +20,7 @@ export class AnalysesResultsManager {
 
   constructor(
     private readonly ctx: ExtensionContext,
+    private readonly cliServer: CodeQLCliServer,
     readonly storagePath: string,
     private readonly logger: Logger,
   ) {
@@ -119,15 +122,23 @@ export class AnalysesResultsManager {
     }
 
     let newAnaysisResults: AnalysisResults;
-    if (path.extname(artifactPath) === '.sarif') {
-      const queryResults = await this.readResults(artifactPath);
+    const fileExtension = path.extname(artifactPath);
+    if (fileExtension === '.sarif') {
+      const queryResults = await this.readSarifResults(artifactPath);
       newAnaysisResults = {
         ...analysisResults,
         interpretedResults: queryResults,
         status: 'Completed'
       };
+    } else if (fileExtension === '.bqrs') {
+      const queryResults = await this.readBqrsResults(artifactPath);
+      newAnaysisResults = {
+        ...analysisResults,
+        rawResults: queryResults,
+        status: 'Completed'
+      };
     } else {
-      void this.logger.log('Cannot download results. Only alert and path queries are fully supported.');
+      void this.logger.log(`Cannot download results. File type '${fileExtension}' not supported.`);
       newAnaysisResults = {
         ...analysisResults,
         status: 'Failed'
@@ -137,7 +148,11 @@ export class AnalysesResultsManager {
     void publishResults([...resultsForQuery]);
   }
 
-  private async readResults(filePath: string): Promise<AnalysisAlert[]> {
+  private async readBqrsResults(filePath: string): Promise<AnalysisRawResults> {
+    return await extractRawResults(this.cliServer, this.logger, filePath);
+  }
+
+  private async readSarifResults(filePath: string): Promise<AnalysisAlert[]> {
     const sarifLog = await sarifParser(filePath);
 
     const processedSarif = extractAnalysisAlerts(sarifLog);

extensions/ql-vscode/src/remote-queries/bqrs-processing.ts

@@ -0,0 +1,34 @@
+import { CodeQLCliServer } from '../cli';
+import { Logger } from '../logging';
+import { transformBqrsResultSet } from '../pure/bqrs-cli-types';
+import { AnalysisRawResults } from './shared/analysis-result';
+import { MAX_RAW_RESULTS } from './shared/result-limits';
+
+export async function extractRawResults(
+  cliServer: CodeQLCliServer,
+  logger: Logger,
+  filePath: string
+): Promise<AnalysisRawResults> {
+  const bqrsInfo = await cliServer.bqrsInfo(filePath);
+  const resultSets = bqrsInfo['result-sets'];
+
+  if (resultSets.length < 1) {
+    throw new Error('No result sets found in results file.');
+  }
+  if (resultSets.length > 1) {
+    void logger.log('Multiple result sets found in results file. Only the first one will be used.');
+  }
+
+  const schema = resultSets[0];
+
+  const chunk = await cliServer.bqrsDecode(
+    filePath,
+    schema.name,
+    { pageSize: MAX_RAW_RESULTS });
+
+  const resultSet = transformBqrsResultSet(schema, chunk);
+
+  const capped = !!chunk.next;
+
+  return { schema, resultSet, capped };
+}

extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts

@@ -41,7 +41,7 @@ export class RemoteQueriesManager extends DisposableObject {
     logger: Logger,
   ) {
     super();
-    this.analysesResultsManager = new AnalysesResultsManager(ctx, storagePath, logger);
+    this.analysesResultsManager = new AnalysesResultsManager(ctx, cliServer, storagePath, logger);
     this.interfaceManager = new RemoteQueriesInterfaceManager(ctx, logger, this.analysesResultsManager);
     this.remoteQueriesMonitor = new RemoteQueriesMonitor(ctx, logger);
 

extensions/ql-vscode/src/remote-queries/shared/analysis-result.ts

@@ -1,9 +1,18 @@
+import { RawResultSet, ResultSetSchema } from '../../pure/bqrs-cli-types';
+
 export type AnalysisResultStatus = 'InProgress' | 'Completed' | 'Failed';
 
 export interface AnalysisResults {
   nwo: string;
   status: AnalysisResultStatus;
   interpretedResults: AnalysisAlert[];
+  rawResults?: AnalysisRawResults;
+}
+
+export interface AnalysisRawResults {
+  schema: ResultSetSchema,
+  resultSet: RawResultSet,
+  capped: boolean;
 }
 
 export interface AnalysisAlert {

extensions/ql-vscode/src/remote-queries/shared/result-limits.ts

@@ -0,0 +1,5 @@
+// The maximum number of raw results to read from a BQRS file.
+// This is used to avoid reading the entire result set into memory
+// and trying to render it on screen. Users will be warned if the
+// results are capped.
+export const MAX_RAW_RESULTS = 500;

extensions/ql-vscode/src/remote-queries/view/RawResultsTable.tsx

@@ -0,0 +1,82 @@
+import * as React from 'react';
+import { Box, Link } from '@primer/react';
+import { CellValue, RawResultSet, ResultSetSchema } from '../../pure/bqrs-cli-types';
+import { useState } from 'react';
+import TextButton from './TextButton';
+
+const numOfResultsInContractedMode = 5;
+
+const Row = ({
+  row
+}: {
+  row: CellValue[]
+}) => (
+  <>
+    {row.map((cell, cellIndex) => (
+      <Box key={cellIndex}
+        borderColor="border.default"
+        borderStyle="solid"
+        justifyContent="center"
+        alignItems="center"
+        p={2}>
+        <Cell value={cell} />
+      </Box>
+    ))}
+  </>
+);
+
+const Cell = ({
+  value
+}: {
+  value: CellValue
+}) => {
+  switch (typeof value) {
+    case 'string':
+    case 'number':
+    case 'boolean':
+      return <span>{value.toString()}</span>;
+    case 'object':
+      // TODO: This will be converted to a proper link once there
+      // is support for populating link URLs.
+      return <Link>{value.label}</Link>;
+  }
+};
+
+const RawResultsTable = ({
+  schema,
+  results
+}: {
+  schema: ResultSetSchema,
+  results: RawResultSet
+}) => {
+  const [tableExpanded, setTableExpanded] = useState(false);
+  const numOfResultsToShow = tableExpanded ? results.rows.length : numOfResultsInContractedMode;
+  const showButton = results.rows.length > numOfResultsInContractedMode;
+
+  // Create n equal size columns. We use minmax(0, 1fr) because the
+  // minimum width of 1fr is auto, not 0.
+  // https://css-tricks.com/equal-width-columns-in-css-grid-are-kinda-weird/
+  const gridTemplateColumns = `repeat(${schema.columns.length}, minmax(0, 1fr))`;
+
+  return (
+    <>
+      <Box
+        display="grid"
+        gridTemplateColumns={gridTemplateColumns}
+        maxWidth="45rem"
+        p={2}>
+        {results.rows.slice(0, numOfResultsToShow).map((row, rowIndex) => (
+          <Row key={rowIndex} row={row} />
+        ))}
+      </Box>
+      {
+        showButton &&
+        <TextButton size='x-small' onClick={() => setTableExpanded(!tableExpanded)}>
+          {tableExpanded ? (<span>View less</span>) : (<span>View all</span>)}
+        </TextButton>
+      }
+    </>
+  );
+};
+
+export default RawResultsTable;

extensions/ql-vscode/src/remote-queries/view/RemoteQueries.tsx

@@ -4,7 +4,7 @@ import * as Rdom from 'react-dom';
 import { Flash, ThemeProvider } from '@primer/react';
 import { ToRemoteQueriesMessage } from '../../pure/interface-types';
 import { AnalysisSummary, RemoteQueryResult } from '../shared/remote-query-result';
-
+import { MAX_RAW_RESULTS } from '../shared/result-limits';
 import { vscode } from '../../view/vscode-api';
 
 import SectionTitle from './SectionTitle';
@@ -18,6 +18,7 @@ import DownloadSpinner from './DownloadSpinner';
 import CollapsibleItem from './CollapsibleItem';
 import { AlertIcon, CodeSquareIcon, FileCodeIcon, FileSymlinkFileIcon, RepoIcon, TerminalIcon } from '@primer/octicons-react';
 import AnalysisAlertResult from './AnalysisAlertResult';
+import RawResultsTable from './RawResultsTable';
 
 const numOfReposInContractedMode = 10;
 
@@ -72,8 +73,13 @@ const openQueryTextVirtualFile = (queryResult: RemoteQueryResult) => {
   });
 };
 
+const getAnalysisResultCount = (analysisResults: AnalysisResults): number => {
+  const rawResultCount = analysisResults.rawResults?.resultSet.rows.length || 0;
+  return analysisResults.interpretedResults.length + rawResultCount;
+};
+
 const sumAnalysesResults = (analysesResults: AnalysisResults[]) =>
-  analysesResults.reduce((acc, curr) => acc + curr.interpretedResults.length, 0);
+  analysesResults.reduce((acc, curr) => acc + getAnalysisResultCount(curr), 0);
 
 const QueryInfo = (queryResult: RemoteQueryResult) => (
   <>
@@ -249,22 +255,41 @@ const AnalysesResultsTitle = ({ totalAnalysesResults, totalResults }: { totalAna
   return <SectionTitle>{totalAnalysesResults}/{totalResults} results</SectionTitle>;
 };
 
-const AnalysesResultsDescription = ({ totalAnalysesResults, totalResults }: { totalAnalysesResults: number, totalResults: number }) => {
-  if (totalAnalysesResults < totalResults) {
-    return <>
-      <VerticalSpace size={1} />
-      Some results haven&apos;t been downloaded automatically because of their size or because enough were downloaded already.
-      Download them manually from the list above if you want to see them here.
-    </>;
-  }
+const AnalysesResultsDescription = ({
+  queryResult,
+  analysesResults,
+}: {
+  queryResult: RemoteQueryResult
+  analysesResults: AnalysisResults[],
+}) => {
+  const showDownloadsMessage = queryResult.analysisSummaries.some(
+    s => !analysesResults.some(a => a.nwo === s.nwo && a.status === 'Completed'));
+  const downloadsMessage = <>
+    <VerticalSpace size={1} />
+    Some results haven&apos;t been downloaded automatically because of their size or because enough were downloaded already.
+    Download them manually from the list above if you want to see them here.
+  </>;
 
-  return <></>;
+  const showMaxResultsMessage = analysesResults.some(a => a.rawResults?.capped);
+  const maxRawResultsMessage = <>
+    <VerticalSpace size={1} />
+    Some repositories have more than {MAX_RAW_RESULTS} results. We will only show you up to {MAX_RAW_RESULTS}
+    results for each repository.
+  </>;
+
+  return (
+    <>
+      {showDownloadsMessage && downloadsMessage}
+      {showMaxResultsMessage && maxRawResultsMessage}
+    </>
+  );
 };
 
 const RepoAnalysisResults = (analysisResults: AnalysisResults) => {
+  const numOfResults = getAnalysisResultCount(analysisResults);
   const title = <>
     {analysisResults.nwo}
-    <Badge text={analysisResults.interpretedResults.length.toString()} />
+    <Badge text={numOfResults.toString()} />
   </>;
 
   return (
@@ -276,11 +301,24 @@ const RepoAnalysisResults = (analysisResults: AnalysisResults) => {
             <VerticalSpace size={2} />
           </li>)}
       </ul>
+      {analysisResults.rawResults &&
+        <RawResultsTable
+          schema={analysisResults.rawResults.schema}
+          results={analysisResults.rawResults.resultSet} />
+      }
     </CollapsibleItem>
   );
 };
 
-const AnalysesResults = ({ analysesResults, totalResults }: { analysesResults: AnalysisResults[], totalResults: number }) => {
+const AnalysesResults = ({
+  queryResult,
+  analysesResults,
+  totalResults
+}: {
+  queryResult: RemoteQueryResult,
+  analysesResults: AnalysisResults[],
+  totalResults: number
+}) => {
   const totalAnalysesResults = sumAnalysesResults(analysesResults);
 
   if (totalResults === 0) {
@@ -294,10 +332,10 @@ const AnalysesResults = ({ analysesResults, totalResults }: { analysesResults: A
         totalAnalysesResults={totalAnalysesResults}
         totalResults={totalResults} />
       <AnalysesResultsDescription
-        totalAnalysesResults={totalAnalysesResults}
-        totalResults={totalResults} />
+        queryResult={queryResult}
+        analysesResults={analysesResults} />
       <ul className="vscode-codeql__flat-list">
-        {analysesResults.filter(a => a.interpretedResults.length > 0).map(r =>
+        {analysesResults.filter(a => a.interpretedResults.length > 0 || a.rawResults).map(r =>
           <li key={r.nwo} className="vscode-codeql__analyses-results-list-item">
             <RepoAnalysisResults {...r} />
           </li>)}
@@ -340,7 +378,11 @@ export function RemoteQueries(): JSX.Element {
         <QueryInfo {...queryResult} />
         <Failures {...queryResult} />
         <Summary queryResult={queryResult} analysesResults={analysesResults} />
-        {showAnalysesResults && <AnalysesResults analysesResults={analysesResults} totalResults={queryResult.totalResultCount} />}
+        {showAnalysesResults &&
+          <AnalysesResults
+            queryResult={queryResult}
+            analysesResults={analysesResults}
+            totalResults={queryResult.totalResultCount} />}
       </ThemeProvider>
     </div>;
   } catch (err) {

extensions/ql-vscode/src/remote-queries/view/TextButton.tsx

@@ -0,0 +1,30 @@
+import * as React from 'react';
+import styled from 'styled-components';
+
+type Size = 'x-small' | 'small' | 'medium' | 'large' | 'x-large';
+
+const StyledButton = styled.button<{ size: Size }>`
+  background: none;
+  color: var(--vscode-textLink-foreground);
+  border: none;
+  cursor: pointer;
+  font-size: ${props => props.size};
+`;
+
+const TextButton = ({
+  size,
+  onClick,
+  children
+}: {
+  size: Size,
+  onClick: () => void,
+  children: React.ReactNode
+}) => (
+  <StyledButton
+    size={size}
+    onClick={onClick}>
+    {children}
+  </StyledButton>
+);
+
+export default TextButton;

extensions/ql-vscode/src/vscode-tests/no-workspace/remote-query-history.test.ts

@@ -176,6 +176,7 @@ describe('Remote queries and query history manager', function() {
     let mockCredentials: any;
     let mockOctokit: any;
     let mockLogger: any;
+    let mockCliServer: any;
     let arm: AnalysesResultsManager;
 
     beforeEach(() => {
@@ -188,10 +189,15 @@ describe('Remote queries and query history manager', function() {
       mockLogger = {
         log: sandbox.spy()
       };
+      mockCliServer = {
+        bqrsInfo: sandbox.spy(),
+        bqrsDecode: sandbox.spy()
+      };
       sandbox.stub(Credentials, 'initialize').resolves(mockCredentials);
 
       arm = new AnalysesResultsManager(
         {} as ExtensionContext,
+        mockCliServer,
         path.join(STORAGE_DIR, 'queries'),
         mockLogger
       );