JS: add QL classes for the extraction metrics

javascript/ql/src/meta/extraction-metrics/FileData.ql

@@ -0,0 +1,24 @@
+/**
+ * @name File correlations
+ * @description
+ * @kind table
+ * @id js/meta/extraction/file-data
+ */
+
+import javascript
+import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
+
+FileWithExtractionMetrics getACacheMember(string cacheFile) { cacheFile = result.getCacheFile() }
+
+FileWithExtractionMetrics getACacheHit(FileWithExtractionMetrics f) {
+  result = getACacheMember(f.getCacheFile()) and
+  result.isFromCache()
+}
+
+from FileWithExtractionMetrics file, boolean fromCache
+where (if file.isFromCache() then fromCache = true else fromCache = false)
+select file.getAbsolutePath() as FILE, file.getCpuTime() as CPU_NANO,
+  file.getNumberOfLines() as LINES, count(Locatable n | n.getFile() = file) as LOCATABLES,
+  count(TypeAnnotation n | n.getFile() = file) as TYPES, file.getLength() as LENGTH,
+  fromCache as FROM_CACHE, count(getACacheMember(file.getCacheFile())) as CACHE_MEMBERS,
+  count(getACacheHit(file)) as CACHE_HITS, file.getCacheFile() as CACHE_FILE

javascript/ql/src/meta/extraction-metrics/MissingMetrics.ql

@@ -0,0 +1,16 @@
+/**
+ * @name File with missing extraction metrics
+ * @description A file missing extraction metrics is indicative of a faulty extractor.
+ * @kind table
+ * @problem.severity error
+ * @id js/meta/extraction/missing-metrics
+ */
+
+import javascript
+import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
+
+from File f, string cause
+where not extraction_data(f, _, _, _) and cause = "No extraction_data for this file"
+      or
+      not extraction_time(f, _,_, _) and cause =  "No extraction_time for this file"
+select f, cause

javascript/ql/src/meta/extraction-metrics/PhaseTimings.ql

@@ -0,0 +1,14 @@
+/**
+ * @name Extractor phase timings
+ * @description An overview of how time was spent during extraction
+ * @kind table
+ * @id js/meta/extraction/phase-timings
+ */
+
+import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
+
+from PhaseName phaseName, float cpuTime, int cpuPerc
+where
+  cpuTime = Aggregated::getCpuTime(phaseName) and
+  cpuPerc = ((cpuTime / Aggregated::getCpuTime()) * 100).floor()
+select phaseName, cpuTime as CPU_NANO, cpuPerc as CPU_PERC

javascript/ql/src/semmle/javascript/meta/ExtractionMetrics.qll

@@ -0,0 +1,134 @@
+import javascript
+
+/**
+ * INTERNAL: Do not use in ordinary queries.
+ *
+ * Extraction metrics for profiling extraction behaviours.
+ */
+module ExtractionMetrics {
+  /**
+   * A file with extraction metrics.
+   */
+  class FileWithExtractionMetrics extends File {
+
+    FileWithExtractionMetrics() { extraction_data(this, _, _, _) and extraction_time(this, _, _, _)}
+
+    /**
+     * Gets the CPU time in nanoseconds it took to extract this file.
+     */
+    float getCpuTime() { result = strictsum(getTime(_, 0)) }
+
+    /**
+     * Gets the wall-clock time in nanoseconds it took to extract this file.
+     */
+    float getWallclockTime() { result = strictsum(getTime(_, 1)) }
+
+    /**
+     * Gets the CPU time in nanoseconds it took to process phase `phaseName` during the extraction this file.
+     */
+    float getCpuTime(PhaseName phaseName) { result = getTime(phaseName, 0) }
+
+    /**
+     * Gets the wall-clock time in nanoseconds it took to process phase `phaseName` during the extraction this file.
+     */
+    float getWallclockTime(PhaseName phaseName) { result = getTime(phaseName, 1) }
+
+    /**
+     * Holds if this file was extracted from the trap cache.
+     */
+    predicate isFromCache() { extraction_data(this, _, true, _) }
+
+    /**
+     * Gets the path to the cache file used for extraction of this file.
+     */
+    string getCacheFile() { extraction_data(this, result, _, _) }
+
+    /**
+     * Gets the number of characters in this file.
+     */
+    int getLength() { extraction_data(this, _, _, result) }
+
+    private float getTime(PhaseName phaseName, int timerKind) {
+      // note that we use strictsum to make it clear if data is missing because it comes from an upgraded database.
+      strictsum(int phaseId, float r |
+        phaseName = getExtractionPhaseName(phaseId) and
+        extraction_time(this, phaseId, timerKind, r)
+      |
+        r
+      ) = result
+    }
+  }
+
+  /**
+   * Converts database ids to human-readable names.
+   */
+  private string getExtractionPhaseName(int phaseId) {
+    // these names ought to match the names used in
+    // `com.semmle.js.extractor.ExtractionTimer.ExtractionPhase`
+    "ASTExtractor_extract" = result and 0 = phaseId
+    or
+    "CFGExtractor_extract" = result and 1 = phaseId
+    or
+    "FileExtractor_extractContents" = result and 2 = phaseId
+    or
+    "JSExtractor_extract" = result and 3 = phaseId
+    or
+    "JSParser_parse" = result and 4 = phaseId
+    or
+    "LexicalExtractor_extractLines" = result and 5 = phaseId
+    or
+    "LexicalExtractor_extractTokens" = result and 6 = phaseId
+    or
+    "TypeScriptASTConverter_convertAST" = result and 7 = phaseId
+    or
+    "TypeScriptParser_talkToParserWrapper" = result and 8 = phaseId
+  }
+
+
+  /**
+   * The name of a phase of the extraction.
+   */
+  class PhaseName extends string {
+    bindingset[this]
+    PhaseName() { this = getExtractionPhaseName(_) }
+  }
+
+  /**
+   * Utilities for aggregating metrics for multiple files.
+   */
+  module Aggregated {
+    /**
+     * Gets the total CPU time spent on extraction.
+     */
+    float getCpuTime() { result = strictsum(any(FileWithExtractionMetrics f).getCpuTime()) }
+
+    /**
+     * Gets the total wallclock time spent on extraction.
+     */
+    float getWallclockTime() { result = strictsum(any(FileWithExtractionMetrics f).getWallclockTime()) }
+
+    /**
+     * Gets the total CPU time spent in phase `phaseName` of the extraction.
+     */
+    float getCpuTime(PhaseName phaseName) {
+      /* bind */ phaseName = getExtractionPhaseName(_) and
+      result = strictsum(any(FileWithExtractionMetrics f).getCpuTime(phaseName))
+    }
+
+    /**
+     * Gets the total wallclock time spent in phase `phaseName` of the extraction.
+     */
+    float getWallclockTime(PhaseName phaseName) {
+      /* bind */ phaseName = getExtractionPhaseName(_) and
+      result = strictsum(any(FileWithExtractionMetrics f).getWallclockTime(phaseName))
+    }
+  }
+
+  /**
+   * Gets `nanoseconds` formatted as a whole number of milliseconds.
+   */
+  bindingset[nanoSeconds]
+  string formatAsMilliSeconds(float nanoSeconds) {
+    result = (nanoSeconds / (1000 * 1000)).ceil() + ""
+  }
+}

javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics.expected

@@ -0,0 +1,4 @@
+| tst2.js:0:0:0:0 | tst2.js |
+| tst.html:0:0:0:0 | tst.html |
+| tst.js:0:0:0:0 | tst.js |
+| tst.ts:0:0:0:0 | tst.ts |

javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics.ql

@@ -0,0 +1,4 @@
+import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
+
+from FileWithExtractionMetrics f
+select f

javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics_length.expected

@@ -0,0 +1,4 @@
+| tst2.js:0:0:0:0 | tst2.js | 26 |
+| tst.html:0:0:0:0 | tst.html | 127 |
+| tst.js:0:0:0:0 | tst.js | 26 |
+| tst.ts:0:0:0:0 | tst.ts | 31 |

javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics_length.ql

@@ -0,0 +1,4 @@
+import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
+
+from FileWithExtractionMetrics f
+select f, f.getLength()

javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics_phases.expected

@@ -0,0 +1,36 @@
+| tst2.js:0:0:0:0 | tst2.js | ASTExtractor_extract |
+| tst2.js:0:0:0:0 | tst2.js | CFGExtractor_extract |
+| tst2.js:0:0:0:0 | tst2.js | FileExtractor_extractContents |
+| tst2.js:0:0:0:0 | tst2.js | JSExtractor_extract |
+| tst2.js:0:0:0:0 | tst2.js | JSParser_parse |
+| tst2.js:0:0:0:0 | tst2.js | LexicalExtractor_extractLines |
+| tst2.js:0:0:0:0 | tst2.js | LexicalExtractor_extractTokens |
+| tst2.js:0:0:0:0 | tst2.js | TypeScriptASTConverter_convertAST |
+| tst2.js:0:0:0:0 | tst2.js | TypeScriptParser_talkToParserWrapper |
+| tst.html:0:0:0:0 | tst.html | ASTExtractor_extract |
+| tst.html:0:0:0:0 | tst.html | CFGExtractor_extract |
+| tst.html:0:0:0:0 | tst.html | FileExtractor_extractContents |
+| tst.html:0:0:0:0 | tst.html | JSExtractor_extract |
+| tst.html:0:0:0:0 | tst.html | JSParser_parse |
+| tst.html:0:0:0:0 | tst.html | LexicalExtractor_extractLines |
+| tst.html:0:0:0:0 | tst.html | LexicalExtractor_extractTokens |
+| tst.html:0:0:0:0 | tst.html | TypeScriptASTConverter_convertAST |
+| tst.html:0:0:0:0 | tst.html | TypeScriptParser_talkToParserWrapper |
+| tst.js:0:0:0:0 | tst.js | ASTExtractor_extract |
+| tst.js:0:0:0:0 | tst.js | CFGExtractor_extract |
+| tst.js:0:0:0:0 | tst.js | FileExtractor_extractContents |
+| tst.js:0:0:0:0 | tst.js | JSExtractor_extract |
+| tst.js:0:0:0:0 | tst.js | JSParser_parse |
+| tst.js:0:0:0:0 | tst.js | LexicalExtractor_extractLines |
+| tst.js:0:0:0:0 | tst.js | LexicalExtractor_extractTokens |
+| tst.js:0:0:0:0 | tst.js | TypeScriptASTConverter_convertAST |
+| tst.js:0:0:0:0 | tst.js | TypeScriptParser_talkToParserWrapper |
+| tst.ts:0:0:0:0 | tst.ts | ASTExtractor_extract |
+| tst.ts:0:0:0:0 | tst.ts | CFGExtractor_extract |
+| tst.ts:0:0:0:0 | tst.ts | FileExtractor_extractContents |
+| tst.ts:0:0:0:0 | tst.ts | JSExtractor_extract |
+| tst.ts:0:0:0:0 | tst.ts | JSParser_parse |
+| tst.ts:0:0:0:0 | tst.ts | LexicalExtractor_extractLines |
+| tst.ts:0:0:0:0 | tst.ts | LexicalExtractor_extractTokens |
+| tst.ts:0:0:0:0 | tst.ts | TypeScriptASTConverter_convertAST |
+| tst.ts:0:0:0:0 | tst.ts | TypeScriptParser_talkToParserWrapper |

javascript/ql/test/library-tests/meta/Extraction/FileWithExtractionMetrics_phases.ql

@@ -0,0 +1,7 @@
+import semmle.javascript.meta.ExtractionMetrics::ExtractionMetrics
+
+from FileWithExtractionMetrics f, PhaseName phase
+where
+  exists(f.getCpuTime(phase)) and
+  exists(f.getWallclockTime(phase))
+select f, phase

javascript/ql/test/library-tests/meta/Extraction/tst.html

@@ -0,0 +1,7 @@
+<script>
+  console.log("extract me")
+</script>
+<script>
+  console.log("extract me")
+</script>
+<script src="./tst.js"></script>

javascript/ql/test/library-tests/meta/Extraction/tst.js

@@ -0,0 +1 @@
+console.log("extract me")

javascript/ql/test/library-tests/meta/Extraction/tst.ts

@@ -0,0 +1 @@
+console.log("extract me too");

javascript/ql/test/library-tests/meta/Extraction/tst2.js

@@ -0,0 +1 @@
+console.log("extract me")