From b20f8fc8c9efe88282e542187aebe00c84d78e58 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Mon, 14 Nov 2022 12:27:54 +0000 Subject: [PATCH 1/3] Kotlin: Add total number of diagnostics to telemetry --- .../src/main/kotlin/utils/Logger.kt | 11 +++++-- java/ql/src/Telemetry/ExtractorInformation.ql | 31 ++++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/java/kotlin-extractor/src/main/kotlin/utils/Logger.kt b/java/kotlin-extractor/src/main/kotlin/utils/Logger.kt index 43c6ee4cc10..6ec568e769a 100644 --- a/java/kotlin-extractor/src/main/kotlin/utils/Logger.kt +++ b/java/kotlin-extractor/src/main/kotlin/utils/Logger.kt @@ -138,6 +138,10 @@ open class LoggerBase(val logCounter: LogCounter) { fullMsgBuilder.append(suffix) val fullMsg = fullMsgBuilder.toString() + emitDiagnostic(tw, severity, diagnosticLocStr, msg, fullMsg, locationString, mkLocationId) + } + + fun emitDiagnostic(tw: TrapWriter, severity: Severity, diagnosticLocStr: String, msg: String, fullMsg: String, locationString: String? = null, mkLocationId: () -> Label = { tw.unknownLocation }) { val locStr = if (locationString == null) "" else "At " + locationString + ": " val kind = if (severity <= Severity.WarnHigh) "WARN" else "ERROR" val logMessage = LogMessage(kind, "Diagnostic($diagnosticLocStr): $locStr$fullMsg") @@ -190,9 +194,10 @@ open class LoggerBase(val logCounter: LogCounter) { // We don't know if this location relates to an error // or a warning, so we just declare hitting the limit // to be an error regardless. - val logMessage = LogMessage("ERROR", "Total of $count diagnostics from $caller.") - tw.writeComment(logMessage.toText()) - logStream.write(logMessage.toJsonLine()) + val message = "Total of $count diagnostics (reached limit of ${logCounter.diagnosticLimit}) from $caller." + if (verbosity >= 1) { + emitDiagnostic(tw, Severity.Error, "Limit", message, message) + } } } } diff --git a/java/ql/src/Telemetry/ExtractorInformation.ql b/java/ql/src/Telemetry/ExtractorInformation.ql index 0eb420ba651..861cde5c661 100644 --- a/java/ql/src/Telemetry/ExtractorInformation.ql +++ b/java/ql/src/Telemetry/ExtractorInformation.ql @@ -53,6 +53,34 @@ predicate extractorDiagnostics(string key, int value) { ) } +/* + * Just counting the diagnostics doesn't give the full picture, as + * CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT means that some diagnostics + * will be suppressed. In that case, we need to look for the + * suppression message, uncount those that did get emitted, uncount the + * suppression message itself, and then add on the full count. + */ + +predicate extractorTotalDiagnostics(string key, int value) { + exists(string extractor | + key = "Total number of diagnostics from " + extractor and + value = + strictcount(Diagnostic d | d.getGeneratedBy() = extractor) + + sum(Diagnostic d | + d.getGeneratedBy() = extractor + | + d.getMessage() + .regexpCapture("Total of ([0-9]+) diagnostics \\(reached limit of ([0-9]+)\\).*", + 1) + .toInt() - + d.getMessage() + .regexpCapture("Total of ([0-9]+) diagnostics \\(reached limit of ([0-9]+)\\).*", + 2) + .toInt() - 1 + ) + ) +} + from string key, int value where fileCount(key, value) or @@ -61,5 +89,6 @@ where numberOfLinesOfCode(key, value) or totalNumberOfLinesByExtension(key, value) or numberOfLinesOfCodeByExtension(key, value) or - extractorDiagnostics(key, value) + extractorDiagnostics(key, value) or + extractorTotalDiagnostics(key, value) select key, value From 847ecd1eec06213714ef9907fdc461f333e77e59 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Mon, 14 Nov 2022 13:09:49 +0000 Subject: [PATCH 2/3] Java/Kotlin: Small refactoring of ExtractorInformation --- java/ql/src/Telemetry/ExtractorInformation.ql | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/java/ql/src/Telemetry/ExtractorInformation.ql b/java/ql/src/Telemetry/ExtractorInformation.ql index 861cde5c661..eaf0eac1fa9 100644 --- a/java/ql/src/Telemetry/ExtractorInformation.ql +++ b/java/ql/src/Telemetry/ExtractorInformation.ql @@ -62,21 +62,16 @@ predicate extractorDiagnostics(string key, int value) { */ predicate extractorTotalDiagnostics(string key, int value) { - exists(string extractor | + exists(string extractor, string limitRegex | + limitRegex = "Total of ([0-9]+) diagnostics \\(reached limit of ([0-9]+)\\).*" and key = "Total number of diagnostics from " + extractor and value = strictcount(Diagnostic d | d.getGeneratedBy() = extractor) + sum(Diagnostic d | d.getGeneratedBy() = extractor | - d.getMessage() - .regexpCapture("Total of ([0-9]+) diagnostics \\(reached limit of ([0-9]+)\\).*", - 1) - .toInt() - - d.getMessage() - .regexpCapture("Total of ([0-9]+) diagnostics \\(reached limit of ([0-9]+)\\).*", - 2) - .toInt() - 1 + d.getMessage().regexpCapture(limitRegex, 1).toInt() - + d.getMessage().regexpCapture(limitRegex, 2).toInt() - 1 ) ) } From fab2d30f381154f01abd35ff844cdb5a9228d8b0 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Mon, 14 Nov 2022 13:53:16 +0000 Subject: [PATCH 3/3] Kotlin: Make emitDiagnostic private --- java/kotlin-extractor/src/main/kotlin/utils/Logger.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/kotlin-extractor/src/main/kotlin/utils/Logger.kt b/java/kotlin-extractor/src/main/kotlin/utils/Logger.kt index 6ec568e769a..36ebda9dd1c 100644 --- a/java/kotlin-extractor/src/main/kotlin/utils/Logger.kt +++ b/java/kotlin-extractor/src/main/kotlin/utils/Logger.kt @@ -141,7 +141,7 @@ open class LoggerBase(val logCounter: LogCounter) { emitDiagnostic(tw, severity, diagnosticLocStr, msg, fullMsg, locationString, mkLocationId) } - fun emitDiagnostic(tw: TrapWriter, severity: Severity, diagnosticLocStr: String, msg: String, fullMsg: String, locationString: String? = null, mkLocationId: () -> Label = { tw.unknownLocation }) { + private fun emitDiagnostic(tw: TrapWriter, severity: Severity, diagnosticLocStr: String, msg: String, fullMsg: String, locationString: String? = null, mkLocationId: () -> Label = { tw.unknownLocation }) { val locStr = if (locationString == null) "" else "At " + locationString + ": " val kind = if (severity <= Severity.WarnHigh) "WARN" else "ERROR" val logMessage = LogMessage(kind, "Diagnostic($diagnosticLocStr): $locStr$fullMsg")