Parse summary of evaluator logs into data model (#1405)
Co-authored-by: Aditya Sharad <6874315+adityasharad@users.noreply.github.com> Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
This commit is contained in:
Angela P Wen
GitHub
Родитель
e57bbcb711
Коммит
c9d895ea42
|
|
@ -689,6 +689,23 @@ export class CodeQLCliServer implements Disposable {
|
|||
return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating log summary');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a JSON summary of an evaluation log.
|
||||
* @param inputPath The path of an evaluation event log.
|
||||
* @param outputPath The path to write a JSON summary of it to.
|
||||
*/
|
||||
async generateJsonLogSummary(
|
||||
inputPath: string,
|
||||
outputPath: string,
|
||||
): Promise<string> {
|
||||
const subcommandArgs = [
|
||||
'--format=predicates',
|
||||
inputPath,
|
||||
outputPath
|
||||
];
|
||||
return await this.runCodeQlCliCommand(['generate', 'log-summary'], subcommandArgs, 'Generating JSON log summary');
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the results from a bqrs.
|
||||
* @param bqrsPath The path to the bqrs.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
import * as os from 'os';
|
||||
|
||||
// TODO(angelapwen): Only load in necessary information and
|
||||
// location in bytes for this log to save memory.
|
||||
export interface EvaluatorLogData {
|
||||
queryCausingWork: string;
|
||||
predicateName: string;
|
||||
millis: number;
|
||||
resultSize: number;
|
||||
ra: Pipelines;
|
||||
}
|
||||
|
||||
interface Pipelines {
|
||||
// Key: pipeline identifier; Value: array of pipeline steps
|
||||
pipelineNamesToSteps: Map<string, string[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A pure method that parses a string of evaluator log summaries into
|
||||
* an array of EvaluatorLogData objects.
|
||||
*
|
||||
*/
|
||||
export function parseVisualizerData(logSummary: string): EvaluatorLogData[] {
|
||||
// Remove newline delimiters because summary is in .jsonl format.
|
||||
const jsonSummaryObjects: string[] = logSummary.split(os.EOL + os.EOL);
|
||||
const visualizerData: EvaluatorLogData[] = [];
|
||||
|
||||
for (const obj of jsonSummaryObjects) {
|
||||
const jsonObj = JSON.parse(obj);
|
||||
|
||||
// Only convert log items that have an RA and millis field
|
||||
if (jsonObj.ra !== undefined && jsonObj.millis !== undefined) {
|
||||
const newLogData: EvaluatorLogData = {
|
||||
queryCausingWork: jsonObj.queryCausingWork,
|
||||
predicateName: jsonObj.predicateName,
|
||||
millis: jsonObj.millis,
|
||||
resultSize: jsonObj.resultSize,
|
||||
ra: jsonObj.ra
|
||||
};
|
||||
visualizerData.push(newLogData);
|
||||
}
|
||||
}
|
||||
return visualizerData;
|
||||
}
|
||||
|
|
@ -267,6 +267,10 @@ export function findQueryEvalLogSummaryFile(resultPath: string): string {
|
|||
return path.join(resultPath, 'evaluator-log.summary');
|
||||
}
|
||||
|
||||
export function findJsonQueryEvalLogSummaryFile(resultPath: string): string {
|
||||
return path.join(resultPath, 'evaluator-log.summary.jsonl');
|
||||
}
|
||||
|
||||
export function findQueryEvalLogEndSummaryFile(resultPath: string): string {
|
||||
return path.join(resultPath, 'evaluator-log-end.summary');
|
||||
}
|
||||
|
|
@ -37,6 +37,7 @@ import { ensureMetadataIsComplete } from './query-results';
|
|||
import { SELECT_QUERY_NAME } from './contextual/locationFinder';
|
||||
import { DecodedBqrsChunk } from './pure/bqrs-cli-types';
|
||||
import { getErrorMessage } from './pure/helpers-pure';
|
||||
import { parseVisualizerData } from './pure/log-summary-parser';
|
||||
|
||||
/**
|
||||
* run-queries.ts
|
||||
|
|
@ -103,6 +104,10 @@ export class QueryEvaluationInfo {
|
|||
return qsClient.findQueryEvalLogSummaryFile(this.querySaveDir);
|
||||
}
|
||||
|
||||
get jsonEvalLogSummaryPath() {
|
||||
return qsClient.findJsonQueryEvalLogSummaryFile(this.querySaveDir);
|
||||
}
|
||||
|
||||
get evalLogEndSummaryPath() {
|
||||
return qsClient.findQueryEvalLogEndSummaryFile(this.querySaveDir);
|
||||
}
|
||||
|
|
@ -198,22 +203,10 @@ export class QueryEvaluationInfo {
|
|||
logPath: this.evalLogPath,
|
||||
});
|
||||
if (await this.hasEvalLog()) {
|
||||
queryInfo.evalLogLocation = this.evalLogPath;
|
||||
void qs.cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath)
|
||||
.then(() => {
|
||||
queryInfo.evalLogSummaryLocation = this.evalLogSummaryPath;
|
||||
fs.readFile(this.evalLogEndSummaryPath, (err, buffer) => {
|
||||
if (err) {
|
||||
throw new Error(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`);
|
||||
}
|
||||
void qs.logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath });
|
||||
void qs.logger.log(buffer.toString(), { additionalLogLocation: this.logPath });
|
||||
});
|
||||
})
|
||||
|
||||
.catch(err => {
|
||||
void showAndLogWarningMessage(`Failed to generate structured evaluator log summary. Reason: ${err.message}`);
|
||||
});
|
||||
this.displayHumanReadableLogSummary(queryInfo, qs);
|
||||
if (config.isCanary()) {
|
||||
this.parseJsonLogSummary(qs.cliServer);
|
||||
}
|
||||
} else {
|
||||
void showAndLogWarningMessage(`Failed to write structured evaluator log to ${this.evalLogPath}.`);
|
||||
}
|
||||
|
|
@ -338,6 +331,48 @@ export class QueryEvaluationInfo {
|
|||
return fs.pathExists(this.evalLogPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the appropriate CLI command to generate a human-readable log summary
|
||||
* and logs to the Query Server console and query log file.
|
||||
*/
|
||||
displayHumanReadableLogSummary(queryInfo: LocalQueryInfo, qs: qsClient.QueryServerClient): void {
|
||||
queryInfo.evalLogLocation = this.evalLogPath;
|
||||
void qs.cliServer.generateLogSummary(this.evalLogPath, this.evalLogSummaryPath, this.evalLogEndSummaryPath)
|
||||
.then(() => {
|
||||
queryInfo.evalLogSummaryLocation = this.evalLogSummaryPath;
|
||||
fs.readFile(this.evalLogEndSummaryPath, (err, buffer) => {
|
||||
if (err) {
|
||||
throw new Error(`Could not read structured evaluator log end of summary file at ${this.evalLogEndSummaryPath}.`);
|
||||
}
|
||||
void qs.logger.log(' --- Evaluator Log Summary --- ', { additionalLogLocation: this.logPath });
|
||||
void qs.logger.log(buffer.toString(), { additionalLogLocation: this.logPath });
|
||||
});
|
||||
})
|
||||
.catch(err => {
|
||||
void showAndLogWarningMessage(`Failed to generate human-readable structured evaluator log summary. Reason: ${err.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the appropriate CLI command to generate a JSON log summary and parse it
|
||||
* into the appropriate data model for the log visualizer.
|
||||
*/
|
||||
parseJsonLogSummary(cliServer: cli.CodeQLCliServer): void {
|
||||
void cliServer.generateJsonLogSummary(this.evalLogPath, this.jsonEvalLogSummaryPath)
|
||||
.then(() => {
|
||||
// TODO(angelapwen): Stream the file in.
|
||||
fs.readFile(this.jsonEvalLogSummaryPath, (err, buffer) => {
|
||||
if (err) {
|
||||
throw new Error(`Could not read structured evaluator log summary JSON file at ${this.jsonEvalLogSummaryPath}.`);
|
||||
}
|
||||
parseVisualizerData(buffer.toString()); // Eventually this return value will feed into the tree visualizer.
|
||||
});
|
||||
})
|
||||
.catch(err => {
|
||||
void showAndLogWarningMessage(`Failed to generate JSON structured evaluator log summary. Reason: ${err.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the CSV file containing the results of this query. This will only be called if the query
|
||||
* does not have interpreted results and the CSV file does not already exist.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"summaryLogVersion" : "0.3.0",
|
||||
"codeqlVersion" : "2.9.0+202204201304plus",
|
||||
"startTime" : "2022-06-23T14:02:41.607Z"
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"completionTime" : "2022-06-23T14:02:42.019Z",
|
||||
"raHash" : "8caddafufc3it9svjph9m8d43me",
|
||||
"predicateName" : "function_instantiation",
|
||||
"appearsAs" : {
|
||||
"function_instantiation" : {
|
||||
"uboot-taint.ql" : [ 3, 9, 10, 11, 13, 14, 15, 17, 18, 20, 21, 22, 23, 24 ]
|
||||
}
|
||||
},
|
||||
"queryCausingWork" : "uboot-taint.ql",
|
||||
"evaluationStrategy" : "EXTENSIONAL",
|
||||
"millis" : 0,
|
||||
"resultSize" : 0
|
||||
}
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
{
|
||||
"completionTime" : "2022-06-23T14:02:42.007Z",
|
||||
"raHash" : "e77dfaa9ciimqv7gb3imoesdb11",
|
||||
"predicateName" : "query::ClassPointerType::getBaseType#dispred#f0820431#ff",
|
||||
"appearsAs" : {
|
||||
"query::ClassPointerType::getBaseType#dispred#f0820431#ff" : {
|
||||
"uboot-taint.ql" : [ 3 ]
|
||||
}
|
||||
},
|
||||
"queryCausingWork" : "uboot-taint.ql",
|
||||
"evaluationStrategy" : "COMPUTE_SIMPLE",
|
||||
"millis" : 11,
|
||||
"resultSize" : 1413,
|
||||
"dependencies" : {
|
||||
"derivedtypes_2013#join_rhs" : "0da8f1fqbiin9i0mcitjedvlbc7",
|
||||
"query::Class#class#f0820431#f" : "2e5d24rnudi1l99mvtse9u567b7",
|
||||
"derivedtypes" : "070e120iu2i3dhj6pt13oqnbi66"
|
||||
},
|
||||
"ra" : {
|
||||
"pipeline" : [
|
||||
" {1} r1 = CONSTANT(unique int)[1]",
|
||||
" {3} r2 = JOIN r1 WITH derivedtypes_2013#join_rhs ON FIRST 1 OUTPUT Rhs.3, Rhs.1, Rhs.2",
|
||||
" {4} r3 = JOIN r2 WITH query::Class#class#f0820431#f ON FIRST 1 OUTPUT Lhs.1, Lhs.2, 1, Lhs.0",
|
||||
" {2} r4 = JOIN r3 WITH derivedtypes ON FIRST 4 OUTPUT Lhs.0, Lhs.3",
|
||||
" return r4"
|
||||
]
|
||||
},
|
||||
"pipelineRuns" : [ {
|
||||
"raReference" : "pipeline"
|
||||
} ]
|
||||
}
|
||||
|
||||
{
|
||||
"completionTime" : "2022-06-23T14:02:42.072Z",
|
||||
"raHash" : "0a0e3cicgtsru1m0cun896qhi61",
|
||||
"predicateName" : "query::DefinedMemberFunction#class#f0820431#f",
|
||||
"appearsAs" : {
|
||||
"query::DefinedMemberFunction#class#f0820431#f" : {
|
||||
"uboot-taint.ql" : [ 3 ]
|
||||
}
|
||||
},
|
||||
"queryCausingWork" : "uboot-taint.ql",
|
||||
"evaluationStrategy" : "COMPUTE_SIMPLE",
|
||||
"millis" : 20,
|
||||
"resultSize" : 8740,
|
||||
"dependencies" : {
|
||||
"fun_decls" : "e6e2e6vn02fcrrtp1ks1f75cd01",
|
||||
"fun_def" : "6bec314lcq8sm3skg9mpecscp02",
|
||||
"function_instantiation" : "8caddafufc3it9svjph9m8d43me",
|
||||
"fun_decls_10#join_rhs" : "16474d6lehg7ssk83gnv2q7r8t8"
|
||||
},
|
||||
"ra" : {
|
||||
"pipeline" : [
|
||||
" {2} r1 = SCAN fun_decls OUTPUT In.0, In.1",
|
||||
" {2} r2 = STREAM DEDUP r1",
|
||||
" {1} r3 = JOIN r2 WITH fun_def ON FIRST 1 OUTPUT Lhs.1",
|
||||
"",
|
||||
" {2} r4 = SCAN function_instantiation OUTPUT In.1, In.0",
|
||||
" {2} r5 = JOIN r4 WITH fun_decls_10#join_rhs ON FIRST 1 OUTPUT Rhs.1, Lhs.1",
|
||||
" {1} r6 = JOIN r5 WITH fun_def ON FIRST 1 OUTPUT Lhs.1",
|
||||
"",
|
||||
" {1} r7 = r3 UNION r6",
|
||||
" return r7"
|
||||
]
|
||||
},
|
||||
"pipelineRuns" : [ {
|
||||
"raReference" : "pipeline"
|
||||
} ]
|
||||
}
|
||||
|
||||
{
|
||||
"completionTime" : "2022-06-23T14:02:43.799Z",
|
||||
"raHash" : "1d0c6wplpr6bnlnii51r7f6lh85",
|
||||
"predicateName" : "QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff",
|
||||
"appearsAs" : {
|
||||
"QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff" : {
|
||||
"uboot-taint.ql" : [ 10 ]
|
||||
}
|
||||
},
|
||||
"queryCausingWork" : "uboot-taint.ql",
|
||||
"evaluationStrategy" : "COMPUTE_RECURSIVE",
|
||||
"millis" : 2,
|
||||
"predicateIterationMillis" : [ 1, 0 ],
|
||||
"deltaSizes" : [ 1, 0 ],
|
||||
"resultSize" : 1,
|
||||
"dependencies" : {
|
||||
"namespaces" : "bf72dcmerq68kur5k2uttmjoco4",
|
||||
"namespacembrs_1#antijoin_rhs" : "3c47bbkgae024k8hgf148mgeqi2",
|
||||
"QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#join_rhs#1" : "d55d698lva8n15u4v7055ma1vl9",
|
||||
"namespacembrs" : "08a148i70tonoa8fp0mb5eb44r1",
|
||||
"QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#join_rhs" : "77cd9dha843p3v12qso4qpe4qoe"
|
||||
},
|
||||
"layerSize" : 1,
|
||||
"ra" : {
|
||||
"base" : [
|
||||
" {2} r1 = namespaces AND NOT namespacembrs_1#antijoin_rhs(Lhs.0)",
|
||||
" return r1"
|
||||
],
|
||||
"standard" : [
|
||||
" {2} r1 = JOIN QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev_delta WITH QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#join_rhs#1 ON FIRST 1 OUTPUT Rhs.1, Lhs.1",
|
||||
"",
|
||||
" {2} r2 = JOIN QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev_delta WITH QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#join_rhs ON FIRST 1 OUTPUT Rhs.1, (Lhs.1 ++ \"::\" ++ Rhs.2)",
|
||||
"",
|
||||
" {2} r3 = r1 UNION r2",
|
||||
" {2} r4 = r3 AND NOT QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev(Lhs.0, Lhs.1)",
|
||||
" return r4"
|
||||
],
|
||||
"order_500000" : [
|
||||
" {2} r1 = JOIN QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev_delta WITH QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#join_rhs#1 ON FIRST 1 OUTPUT Rhs.1, Lhs.1",
|
||||
"",
|
||||
" {2} r2 = JOIN QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev_delta WITH namespacembrs ON FIRST 1 OUTPUT Rhs.1, Lhs.1",
|
||||
" {2} r3 = JOIN r2 WITH namespaces ON FIRST 1 OUTPUT Lhs.0, (Lhs.1 ++ \"::\" ++ Rhs.1)",
|
||||
"",
|
||||
" {2} r4 = r1 UNION r3",
|
||||
" {2} r5 = r4 AND NOT QualifiedName::Namespace::getAQualifierForMembers#f0820431#ff#prev(Lhs.0, Lhs.1)",
|
||||
" return r5"
|
||||
]
|
||||
},
|
||||
"pipelineRuns" : [ {
|
||||
"raReference" : "base"
|
||||
}, {
|
||||
"raReference" : "order_500000"
|
||||
} ]
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import { expect } from 'chai';
|
||||
import * as fs from 'fs-extra';
|
||||
import * as path from 'path';
|
||||
import 'mocha';
|
||||
|
||||
import { parseVisualizerData } from '../../src/pure/log-summary-parser';
|
||||
|
||||
describe('Evaluator log summary tests', async function () {
|
||||
describe('for a valid summary text', async function () {
|
||||
it('should return only valid EvaluatorLogData objects', async function () {
|
||||
const validSummaryText = await fs.readFile(path.join(__dirname, 'evaluator-log-summaries/valid-summary.jsonl'), 'utf8');
|
||||
const logDataItems = parseVisualizerData(validSummaryText.toString());
|
||||
expect(logDataItems).to.not.be.undefined;
|
||||
expect (logDataItems.length).to.eq(3);
|
||||
for (const item of logDataItems) {
|
||||
expect(item.queryCausingWork).to.not.be.empty;
|
||||
expect(item.predicateName).to.not.be.empty;
|
||||
expect(item.millis).to.be.a('number');
|
||||
expect(item.resultSize).to.be.a('number');
|
||||
expect(item.ra).to.not.be.undefined;
|
||||
expect(item.ra).to.not.be.empty;
|
||||
for (const [pipeline, steps] of Object.entries(item.ra)) {
|
||||
expect (pipeline).to.not.be.empty;
|
||||
expect (steps).to.not.be.undefined;
|
||||
expect (steps.length).to.be.greaterThan(0);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('should not parse a summary header object', async function () {
|
||||
const invalidHeaderText = await fs.readFile(path.join(__dirname, 'evaluator-log-summaries/invalid-header.jsonl'), 'utf8');
|
||||
const logDataItems = parseVisualizerData(invalidHeaderText);
|
||||
expect (logDataItems.length).to.eq(0);
|
||||
});
|
||||
|
||||
it('should not parse a log event missing RA or millis fields', async function () {
|
||||
const invalidSummaryText = await fs.readFile(path.join(__dirname, 'evaluator-log-summaries/invalid-summary.jsonl'), 'utf8');
|
||||
const logDataItems = parseVisualizerData(invalidSummaryText);
|
||||
expect (logDataItems.length).to.eq(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
Загрузка…
Ссылка в новой задаче