github
/
vscode-codeql
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #1004 from marcnjaramillo/fix-large-sarif-handling 

Fix large sarif handling
This commit is contained in:
Andrew Eisenberg 2021-11-22 12:24:17 -08:00 коммит произвёл GitHub
Родитель 30d01cb0e0 ad81127267
Коммит 16c0bea799
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
10 изменённых файлов: 305 добавлений и 20 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
extensions/ql-vscode/CHANGELOG.md
Просмотреть файл

@ -2,7 +2,8 @@
## [UNRELEASED]
- Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979)
- Fix the _CodeQL: Open Referenced File_ command for Windows systems. [#979](https://github.com/github/vscode-codeql/pull/979)
- Support large SARIF results files (>4GB) without crashing VS Code. [#1004](https://github.com/github/vscode-codeql/pull/1004)
- Fix a bug that shows 'Set current database' when hovering over the currently selected database in the databases view. [#976](https://github.com/github/vscode-codeql/pull/976)
- Fix a bug with importing large databases. Databases over 4GB can now be imported directly from LGTM or from a zip file. This functionality is only available when using CodeQL CLI version 2.6.0 or later. [#971](https://github.com/github/vscode-codeql/pull/971)
- Replace certain control codes (`U+0000` - `U+001F`) with their corresponding control labels (`U+2400` - `U+241F`) in the results view. [#963](https://github.com/github/vscode-codeql/pull/963)

96
extensions/ql-vscode/package-lock.json сгенерированный
Просмотреть файл

@ -21,6 +21,9 @@
"react": "^16.8.6",
"react-dom": "^16.8.6",
"semver": "~7.3.2",
"stream": "^0.0.2",
"stream-chain": "~2.2.4",
"stream-json": "~1.7.3",
"tmp": "^0.1.0",
"tmp-promise": "~3.0.2",
"tree-kill": "~1.2.2",
@ -55,6 +58,8 @@
"@types/semver": "~7.2.0",
"@types/sinon": "~7.5.2",
"@types/sinon-chai": "~3.2.3",
"@types/stream-chain": "~2.0.1",
"@types/stream-json": "~1.7.1",
"@types/through2": "^2.0.36",
"@types/tmp": "^0.1.0",
"@types/unzipper": "~0.10.1",
@ -852,6 +857,25 @@
"integrity": "sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA==",
"dev": true
},
"node_modules/@types/stream-chain": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.1.tgz",
"integrity": "sha512-D+Id9XpcBpampptkegH7WMsEk6fUdf9LlCIX7UhLydILsqDin4L0QT7ryJR0oycwC7OqohIzdfcMHVZ34ezNGg==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/stream-json": {
"version": "1.7.1",
"resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.1.tgz",
"integrity": "sha512-BNIK/ix6iJvWvoXbDVVJhw5LNG1wie/rXcUo7jw4hBqY3FhIrg0e+RMXFN5UreKclBIStl9FDEHNSDLuuQ9/MQ==",
"dev": true,
"dependencies": {
"@types/node": "*",
"@types/stream-chain": "*"
}
},
"node_modules/@types/tapable": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.6.tgz",
@ -3543,6 +3567,11 @@
"integrity": "sha512-EGuiJW4yBPOTj2NtWGZcX93ZE8IGj33HJAx4d3ouE2zOfW2trbWU+t1e0yzLr1qQIw81++txbM3BH52QwSRE6Q==",
"dev": true
},
"node_modules/emitter-component": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/emitter-component/-/emitter-component-1.1.1.tgz",
"integrity": "sha1-Bl4tvtaVm/RwZ57avq95gdEAOrY="
},
"node_modules/emitter-listener": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz",
@ -4715,6 +4744,7 @@
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz",
"integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==",
"dev": true,
"hasInstallScript": true,
"optional": true,
"os": [
"darwin"
@ -9720,12 +9750,33 @@
"node": ">=0.10.0"
}
},
"node_modules/stream": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/stream/-/stream-0.0.2.tgz",
"integrity": "sha1-f1Nj8Ff2WSxVlfALyAon9c7B8O8=",
"dependencies": {
"emitter-component": "^1.1.1"
}
},
"node_modules/stream-chain": {
"version": "2.2.4",
"resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.4.tgz",
"integrity": "sha512-9lsl3YM53V5N/I1C2uJtc3Kavyi3kNYN83VkKb/bMWRk7D9imiFyUPYa0PoZbLohSVOX1mYE9YsmwObZUsth6Q=="
},
"node_modules/stream-exhaust": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/stream-exhaust/-/stream-exhaust-1.0.2.tgz",
"integrity": "sha512-b/qaq/GlBK5xaq1yrK9/zFcyRSTNxmcZwFLGSTG0mXgZl/4Z6GgiyYOXOvY7N3eEvFRAG1bkDRz5EPGSvPYQlw==",
"dev": true
},
"node_modules/stream-json": {
"version": "1.7.3",
"resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.7.3.tgz",
"integrity": "sha512-Y6dXn9KKWSwxOqnvHGcdZy1PK+J+7alBwHCeU3W9oRqm4ilLRA0XSPmd1tWwhg7tv9EIxJTMWh7KF15tYelKJg==",
"dependencies": {
"stream-chain": "^2.2.4"
}
},
"node_modules/stream-shift": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
@ -12526,6 +12577,25 @@
"integrity": "sha512-K5K+yml8LTo9bWJI/rECfIPrGgxdpeNbj+d53lwN4QjW1MCwlkhUms+gtdzigTeUyBr09+u8BwOIY3MXvHdcsA==",
"dev": true
},
"@types/stream-chain": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.1.tgz",
"integrity": "sha512-D+Id9XpcBpampptkegH7WMsEk6fUdf9LlCIX7UhLydILsqDin4L0QT7ryJR0oycwC7OqohIzdfcMHVZ34ezNGg==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/stream-json": {
"version": "1.7.1",
"resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.1.tgz",
"integrity": "sha512-BNIK/ix6iJvWvoXbDVVJhw5LNG1wie/rXcUo7jw4hBqY3FhIrg0e+RMXFN5UreKclBIStl9FDEHNSDLuuQ9/MQ==",
"dev": true,
"requires": {
"@types/node": "*",
"@types/stream-chain": "*"
}
},
"@types/tapable": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.6.tgz",
@ -14728,6 +14798,11 @@
"integrity": "sha512-EGuiJW4yBPOTj2NtWGZcX93ZE8IGj33HJAx4d3ouE2zOfW2trbWU+t1e0yzLr1qQIw81++txbM3BH52QwSRE6Q==",
"dev": true
},
"emitter-component": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/emitter-component/-/emitter-component-1.1.1.tgz",
"integrity": "sha1-Bl4tvtaVm/RwZ57avq95gdEAOrY="
},
"emitter-listener": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz",
@ -19683,12 +19758,33 @@
}
}
},
"stream": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/stream/-/stream-0.0.2.tgz",
"integrity": "sha1-f1Nj8Ff2WSxVlfALyAon9c7B8O8=",
"requires": {
"emitter-component": "^1.1.1"
}
},
"stream-chain": {
"version": "2.2.4",
"resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.4.tgz",
"integrity": "sha512-9lsl3YM53V5N/I1C2uJtc3Kavyi3kNYN83VkKb/bMWRk7D9imiFyUPYa0PoZbLohSVOX1mYE9YsmwObZUsth6Q=="
},
"stream-exhaust": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/stream-exhaust/-/stream-exhaust-1.0.2.tgz",
"integrity": "sha512-b/qaq/GlBK5xaq1yrK9/zFcyRSTNxmcZwFLGSTG0mXgZl/4Z6GgiyYOXOvY7N3eEvFRAG1bkDRz5EPGSvPYQlw==",
"dev": true
},
"stream-json": {
"version": "1.7.3",
"resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.7.3.tgz",
"integrity": "sha512-Y6dXn9KKWSwxOqnvHGcdZy1PK+J+7alBwHCeU3W9oRqm4ilLRA0XSPmd1tWwhg7tv9EIxJTMWh7KF15tYelKJg==",
"requires": {
"stream-chain": "^2.2.4"
}
},
"stream-shift": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",

5
extensions/ql-vscode/package.json
Просмотреть файл

@ -992,6 +992,9 @@
"react": "^16.8.6",
"react-dom": "^16.8.6",
"semver": "~7.3.2",
"stream": "^0.0.2",
"stream-chain": "~2.2.4",
"stream-json": "~1.7.3",
"tmp": "^0.1.0",
"tmp-promise": "~3.0.2",
"tree-kill": "~1.2.2",
@ -1026,6 +1029,8 @@
"@types/semver": "~7.2.0",
"@types/sinon": "~7.5.2",
"@types/sinon-chai": "~3.2.3",
"@types/stream-chain": "~2.0.1",
"@types/stream-json": "~1.7.1",
"@types/through2": "^2.0.36",
"@types/tmp": "^0.1.0",
"@types/unzipper": "~0.10.1",

21
extensions/ql-vscode/src/cli.ts
Просмотреть файл

@ -1,6 +1,5 @@
import * as cpp from 'child-process-promise';
import * as child_process from 'child_process';
import * as fs from 'fs-extra';
import * as path from 'path';
import * as sarif from 'sarif';
import { SemVer } from 'semver';
@ -17,6 +16,7 @@ import { assertNever } from './pure/helpers-pure';
import { QueryMetadata, SortDirection } from './pure/interface-types';
import { Logger, ProgressReporter } from './logging';
import { CompilationMessage } from './pure/messages';
import { sarifParser } from './sarif-parser';
import { dbSchemeToLanguage } from './helpers';
/**
@ -696,22 +696,7 @@ export class CodeQLCliServer implements Disposable {
async interpretBqrs(metadata: QueryMetadata, resultsPath: string, interpretedResultsPath: string, sourceInfo?: SourceInfo): Promise<sarif.Log> {
await this.runInterpretCommand(SARIF_FORMAT, metadata, resultsPath, interpretedResultsPath, sourceInfo);
let output: string;
try {
output = await fs.readFile(interpretedResultsPath, 'utf8');
} catch (e) {
const rawMessage = e.stderr || e.message;
const errorMessage = rawMessage.startsWith('Cannot create a string')
? `SARIF too large. ${rawMessage}`
: rawMessage;
throw new Error(`Reading output of interpretation failed: ${errorMessage}`);
}
try {
return JSON.parse(output) as sarif.Log;
} catch (err) {
throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
}
return await sarifParser(interpretedResultsPath);
}
async generateResultsCsv(metadata: QueryMetadata, resultsPath: string, csvPath: string, sourceInfo?: SourceInfo): Promise<void> {
@ -1157,7 +1142,7 @@ export class CliVersionConstraint {
/**
* CLI version where database registration was introduced
*/
*/
public static CLI_VERSION_WITH_DB_REGISTRATION = new SemVer('2.4.1');
/**

48
extensions/ql-vscode/src/sarif-parser.ts Normal file
Просмотреть файл

@ -0,0 +1,48 @@
import * as Sarif from 'sarif';
import * as fs from 'fs-extra';
import { parser } from 'stream-json';
import { pick } from 'stream-json/filters/Pick';
import Assembler = require('stream-json/Assembler');
import { chain } from 'stream-chain';
const DUMMY_TOOL : Sarif.Tool = {driver: {name: ''}};
export async function sarifParser(interpretedResultsPath: string) : Promise<Sarif.Log> {
try {
// Parse the SARIF file into token streams, filtering out only the results array.
const p = parser();
const pipeline = chain([
fs.createReadStream(interpretedResultsPath),
p,
pick({filter: 'runs.0.results'})
]);
// Creates JavaScript objects from the token stream
const asm = Assembler.connectTo(pipeline);
// Returns a constructed Log object with the results or an empty array if no results were found.
// If the parser fails for any reason, it will reject the promise.
return await new Promise((resolve, reject) => {
pipeline.on('error', (error) => {
reject(error);
});
asm.on('done', (asm) => {
const log : Sarif.Log = {
version: '2.1.0',
runs: [
{
tool: DUMMY_TOOL,
results: asm.current ?? []
}
]
};
resolve(log);
});
});
} catch (err) {
throw new Error(`Parsing output of interpretation failed: ${err.stderr || err}`);
}
}

34
extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/emptyResultsSarif.sarif Normal file
Просмотреть файл

@ -0,0 +1,34 @@
{
"version": "2.1.0",
"$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4",
"runs": [
{
"tool": {
"driver": {
"name": "ESLint",
"informationUri": "https://eslint.org",
"rules": [
{
"id": "no-unused-vars",
"shortDescription": {
"text": "disallow unused variables"
},
"helpUri": "https://eslint.org/docs/rules/no-unused-vars",
"properties": {
"category": "Variables"
}
}
]
}
},
"artifacts": [
{
"location": {
"uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js"
}
}
],
"results": []
}
]
}

33
extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/invalidSarif.sarif Normal file
Просмотреть файл

@ -0,0 +1,33 @@
{
"version": "2.1.0",
"$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4",
"runs": [
{
"tool": {
"driver": {
"name": "ESLint",
"informationUri": "https://eslint.org",
"rules": [
{
"id": "no-unused-vars",
"shortDescription": {
"text": "disallow unused variables"
},
"helpUri": "https://eslint.org/docs/rules/no-unused-vars",
"properties": {
"category": "Variables"
}
}
]
}
},
"artifacts": [
{
"location": {
"uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js"
}
}
]
}
]
}

57
extensions/ql-vscode/src/vscode-tests/no-workspace/data/sarif/validSarif.sarif Normal file
Просмотреть файл

@ -0,0 +1,57 @@
{
"version": "2.1.0",
"$schema": "http://json.schemastore.org/sarif-2.1.0-rtm.4",
"runs": [
{
"tool": {
"driver": {
"name": "ESLint",
"informationUri": "https://eslint.org",
"rules": [
{
"id": "no-unused-vars",
"shortDescription": {
"text": "disallow unused variables"
},
"helpUri": "https://eslint.org/docs/rules/no-unused-vars",
"properties": {
"category": "Variables"
}
}
]
}
},
"artifacts": [
{
"location": {
"uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js"
}
}
],
"results": [
{
"level": "error",
"message": {
"text": "'x' is assigned a value but never used."
},
"locations": [
{
"physicalLocation": {
"artifactLocation": {
"uri": "file:///C:/dev/sarif/sarif-tutorials/samples/Introduction/simple-example.js",
"index": 0
},
"region": {
"startLine": 1,
"startColumn": 5
}
}
}
],
"ruleId": "no-unused-vars",
"ruleIndex": 0
}
]
}
]
}

25
extensions/ql-vscode/src/vscode-tests/no-workspace/sarifParser.test.ts Normal file
Просмотреть файл

@ -0,0 +1,25 @@
import * as path from 'path';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import { sarifParser } from '../../sarif-parser';
chai.use(chaiAsPromised);
const expect = chai.expect;
describe.only('sarif parser', function() {
const sarifDir = path.join(__dirname, 'data/sarif');
it('should parse a valid SARIF file', async () => {
const result = await sarifParser(path.join(sarifDir, 'validSarif.sarif'));
expect(result.version).to.exist;
expect(result.runs).to.exist;
expect(result.runs[0].tool).to.exist;
expect(result.runs[0].tool.driver).to.exist;
expect(result.runs.length).to.be.at.least(1);
});
it('should return an empty array if there are no results', async () => {
const result = await sarifParser(path.join(sarifDir, 'emptyResultsSarif.sarif'));
expect(result.runs[0].results).to.be.empty;
});
});

3
extensions/ql-vscode/test/pure-tests/sarif-utils.test.ts
Просмотреть файл

@ -6,11 +6,12 @@ import {
getPathRelativeToSourceLocationPrefix,
parseSarifLocation,
parseSarifPlainTextMessage,
unescapeSarifText
unescapeSarifText,
} from '../../src/pure/sarif-utils';
describe('parsing sarif', () => {
it('should be able to parse a simple message from the spec', async function() {
const message = 'Tainted data was used. The data came from [here](3).';
const results = parseSarifPlainTextMessage(message);