Merge pull request #3924 from RasmusWL/python-metrics-queries-for-dist-compare

Approved by tausbn

python/ql/src/meta/MetaMetrics.qll

@@ -0,0 +1,26 @@
+/**
+ * Helpers for generating meta metrics, that is, metrics about the CodeQL analysis and extractor.
+ */
+
+import python
+private import semmle.python.filters.GeneratedCode
+private import semmle.python.filters.Tests
+
+/**
+ * Gets the root folder of the snapshot.
+ *
+ * This is selected as the location for project-wide metrics.
+ */
+Folder projectRoot() { result.getRelativePath() = "" }
+
+/** A file we ignore because it is a test file, part of a third-party library, or compiled/generated/bundled code. */
+class IgnoredFile extends File {
+  IgnoredFile() {
+    any(TestScope ts).getLocation().getFile() = this
+    or
+    this instanceof GeneratedFile
+    or
+    // outside source root (inspired by `Scope.inSource`)
+    not exists(this.getRelativePath())
+  }
+}

python/ql/src/meta/analysis-quality/CallGraphQuality.qll

@@ -0,0 +1,65 @@
+/**
+ * Provides predicates for measuring the quality of the call graph, that is,
+ * the number of calls that could be resolved to a callee.
+ */
+
+import python
+import meta.MetaMetrics
+
+/**
+ * A call that is (possibly) relevant for analysis quality.
+ * See `IgnoredFile` for details on what is excluded.
+ */
+class RelevantCall extends Call {
+  RelevantCall() { not this.getLocation().getFile() instanceof IgnoredFile }
+}
+
+/** Provides classes for call-graph resolution by using points-to. */
+module PointsToBasedCallGraph {
+  /** A call that can be resolved by points-to. */
+  class ResolvableCall extends RelevantCall {
+    Value callee;
+
+    ResolvableCall() { callee.getACall() = this.getAFlowNode() }
+
+    /** Gets a resolved callee of this call. */
+    Value getCallee() { result = callee }
+  }
+
+  /** A call that cannot be resolved by points-to. */
+  class UnresolvableCall extends RelevantCall {
+    UnresolvableCall() { not this instanceof ResolvableCall }
+  }
+
+  /**
+   * A call that can be resolved by points-to, where the resolved callee is relevant.
+   * Relevant callees include:
+   * - builtins
+   * - standard library
+   * - source code of the project
+   */
+  class ResolvableCallRelevantCallee extends ResolvableCall {
+    ResolvableCallRelevantCallee() {
+      callee.isBuiltin()
+      or
+      exists(File file |
+        file = callee.(CallableValue).getScope().getLocation().getFile()
+        or
+        file = callee.(ClassValue).getScope().getLocation().getFile()
+      |
+        file.inStdlib()
+        or
+        // part of the source code of the project
+        exists(file.getRelativePath())
+      )
+    }
+  }
+
+  /**
+   * A call that can be resolved by points-to, where the resolved callee is not considered relevant.
+   * See `ResolvableCallRelevantCallee` for the definition of relevance.
+   */
+  class ResolvableCallIrrelevantCallee extends ResolvableCall {
+    ResolvableCallIrrelevantCallee() { not this instanceof ResolvableCallRelevantCallee }
+  }
+}

python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql

@@ -0,0 +1,15 @@
+/**
+ * @name Ratio of resolvable call by points-to
+ * @description The percentage of (relevant) calls that can be resolved to a callee.
+ * @kind metric
+ * @metricType project
+ * @metricAggregate sum min max avg
+ * @tags meta
+ * @id py/meta/points-to-resolvable-call-ratio
+ */
+
+import python
+import CallGraphQuality
+
+select projectRoot(),
+  100.0 * count(PointsToBasedCallGraph::ResolvableCall call) / count(RelevantCall call).(float)

python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql

@@ -0,0 +1,14 @@
+/**
+ * @name Resolvable calls by points-to
+ * @description The number of (relevant) calls that can be resolved to a callee.
+ * @kind metric
+ * @metricType project
+ * @metricAggregate sum
+ * @tags meta
+ * @id py/meta/points-to-resolvable-calls
+ */
+
+import python
+import CallGraphQuality
+
+select projectRoot(), count(PointsToBasedCallGraph::ResolvableCall call)

python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql

@@ -0,0 +1,14 @@
+/**
+ * @name Resolvable calls by points-to, to relevant callee
+ * @description The number of (relevant) calls that could be resolved to a callee that is relevant.
+ * @kind metric
+ * @metricType project
+ * @metricAggregate sum
+ * @tags meta
+ * @id py/meta/points-to-resolvable-calls-relevant-callee
+ */
+
+import python
+import CallGraphQuality
+
+select projectRoot(), count(PointsToBasedCallGraph::ResolvableCallRelevantCallee call)

python/ql/src/meta/analysis-quality/ResolvableCallCandidates.ql

@@ -0,0 +1,14 @@
+/**
+ * @name Resolvable call candidates
+ * @description The number of (relevant) calls in the program.
+ * @kind metric
+ * @metricType project
+ * @metricAggregate sum
+ * @tags meta
+ * @id py/meta/resolvable-call-candidates
+ */
+
+import python
+import CallGraphQuality
+
+select projectRoot(), count(RelevantCall call)