From 32219e58c04d8f3aa2d196a6255f9d719465c6cb Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 7 Jul 2020 18:50:09 +0200 Subject: [PATCH 1/6] Python: Add basic call-graph metric queries For use with dist-compare --- python/ql/src/meta/MetaMetrics.qll | 27 ++++++++ .../analysis-quality/CallGraphQuality.qll | 65 +++++++++++++++++++ .../PointsToResolvableCallRatio.ql | 14 ++++ .../PointsToResolvableCalls.ql | 14 ++++ .../PointsToResolvableCallsRelevantTarget.ql | 14 ++++ .../ResolvableCallCandidates.ql | 14 ++++ 6 files changed, 148 insertions(+) create mode 100644 python/ql/src/meta/MetaMetrics.qll create mode 100644 python/ql/src/meta/analysis-quality/CallGraphQuality.qll create mode 100644 python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql create mode 100644 python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql create mode 100644 python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql create mode 100644 python/ql/src/meta/analysis-quality/ResolvableCallCandidates.ql diff --git a/python/ql/src/meta/MetaMetrics.qll b/python/ql/src/meta/MetaMetrics.qll new file mode 100644 index 00000000000..83ec23f1b52 --- /dev/null +++ b/python/ql/src/meta/MetaMetrics.qll @@ -0,0 +1,27 @@ +/** + * Helpers to generating meta metrics, that is, metrics about the CodeQL analysis and extractor. + */ + +import python + +private import semmle.python.filters.GeneratedCode +private import semmle.python.filters.Tests + +/** + * Gets the root folder of the snapshot. + * + * This is selected as the location for project-wide metrics. + */ +Folder projectRoot() { result.getRelativePath() = "" } + +/** A file we ignore because it is a test file, part of a third-part library, or compiled/generated/bundled code. */ +class IgnoredFile extends File { + IgnoredFile() { + any(TestScope ts).getLocation().getFile() = this + or + this instanceof GeneratedFile + or + // outside source root (inspired by `Scope.inSource`) + not exists(this.getRelativePath()) + } +} diff --git a/python/ql/src/meta/analysis-quality/CallGraphQuality.qll b/python/ql/src/meta/analysis-quality/CallGraphQuality.qll new file mode 100644 index 00000000000..a0a5e42cc83 --- /dev/null +++ b/python/ql/src/meta/analysis-quality/CallGraphQuality.qll @@ -0,0 +1,65 @@ +/** + * Provides predicates for measuring the quality of the call graph, that is, + * the number of calls that could be resolved to a callee. + */ + +import python +import meta.MetaMetrics + +/** + * A call that is (possibly) relevant for analysis quality. + * See `IgnoredFile` for details on what is excluded. + */ +class RelevantCall extends Call { + RelevantCall() { not this.getLocation().getFile() instanceof IgnoredFile } +} + +/** Provides classes for call-graph resolution by using points-to */ +module PointsTo { + /** A call that can be resolved by points-to. */ + class ResolvableCall extends RelevantCall { + Value target; + + ResolvableCall() { target.getACall() = this.getAFlowNode() } + + /** Gets a resolved target of this call */ + Value getTarget() { result = target } + } + + /** A call that cannot be resolved by points-to. */ + class UnresolvableCall extends RelevantCall { + UnresolvableCall() { not this instanceof ResolvableCall } + } + + /** + * A call that can be resolved by points-to, where the resolved target is relevant. + * Relevant targets include: + * - builtins + * - standard library + * - source code of the project + */ + class ResolvableCallRelevantTarget extends ResolvableCall { + ResolvableCallRelevantTarget() { + target.isBuiltin() + or + exists(File file | + file = target.(CallableValue).getScope().getLocation().getFile() + or + file = target.(ClassValue).getScope().getLocation().getFile() + | + file.inStdlib() + or + // part of the source code of the project + exists(file.getRelativePath()) + ) + } + } + + /** + * A call that can be resolved by points-to, where resolved target is not considered relevant. + * See `ResolvableCallRelevantTarget` for definition of relevance. + */ + class ResolvableCallIrrelevantTarget extends ResolvableCall { + ResolvableCallIrrelevantTarget() { not this instanceof ResolvableCallRelevantTarget } + } +} diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql new file mode 100644 index 00000000000..118b0c64df1 --- /dev/null +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql @@ -0,0 +1,14 @@ +/** + * @name Ratio of resolvable call by points-to + * @description The percentage (relevant) calls that can be resolved to a target. + * @kind metric + * @metricType project + * @metricAggregate sum min max avg + * @tags meta + * @id py/meta/points-to-resolvable-call-ratio + */ + +import python +import CallGraphQuality + +select projectRoot(), 100.0 * count(PointsTo::ResolvableCall call) / count(RelevantCall call).(float) diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql new file mode 100644 index 00000000000..e236f7577b6 --- /dev/null +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql @@ -0,0 +1,14 @@ +/** + * @name Resolvable calls by points-to + * @description The number of (relevant) calls that could be resolved to its target. + * @kind metric + * @metricType project + * @metricAggregate sum + * @tags meta + * @id py/meta/points-to-resolvable-calls + */ + +import python +import CallGraphQuality + +select projectRoot(), count(PointsTo::ResolvableCall call) diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql new file mode 100644 index 00000000000..9a04085cc7d --- /dev/null +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql @@ -0,0 +1,14 @@ +/** + * @name Resolvable calls by points-to, to relevant target + * @description The number of (relevant) calls that could be resolved to its target that is relevant. + * @kind metric + * @metricType project + * @metricAggregate sum + * @tags meta + * @id py/meta/points-to-resolvable-calls-relevant-target + */ + +import python +import CallGraphQuality + +select projectRoot(), count(PointsTo::ResolvableCallRelevantTarget call) diff --git a/python/ql/src/meta/analysis-quality/ResolvableCallCandidates.ql b/python/ql/src/meta/analysis-quality/ResolvableCallCandidates.ql new file mode 100644 index 00000000000..c93ffba86cb --- /dev/null +++ b/python/ql/src/meta/analysis-quality/ResolvableCallCandidates.ql @@ -0,0 +1,14 @@ +/** + * @name Resolvable call candidates + * @description The number (relevant) calls in the program. + * @kind metric + * @metricType project + * @metricAggregate sum + * @tags meta + * @id py/meta/resolvable-call-candidates + */ + +import python +import CallGraphQuality + +select projectRoot(), count(RelevantCall call) From e7c89dc24ba38b18ac73c3b0307b65ebf4346140 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Thu, 9 Jul 2020 10:39:58 +0200 Subject: [PATCH 2/6] Python: Fix grammar Co-authored-by: intrigus-lgtm <60750685+intrigus-lgtm@users.noreply.github.com> --- python/ql/src/meta/MetaMetrics.qll | 2 +- python/ql/src/meta/analysis-quality/CallGraphQuality.qll | 6 +++--- .../meta/analysis-quality/PointsToResolvableCallRatio.ql | 2 +- .../src/meta/analysis-quality/ResolvableCallCandidates.ql | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/ql/src/meta/MetaMetrics.qll b/python/ql/src/meta/MetaMetrics.qll index 83ec23f1b52..8a79edb2e90 100644 --- a/python/ql/src/meta/MetaMetrics.qll +++ b/python/ql/src/meta/MetaMetrics.qll @@ -14,7 +14,7 @@ private import semmle.python.filters.Tests */ Folder projectRoot() { result.getRelativePath() = "" } -/** A file we ignore because it is a test file, part of a third-part library, or compiled/generated/bundled code. */ +/** A file we ignore because it is a test file, part of a third-party library, or compiled/generated/bundled code. */ class IgnoredFile extends File { IgnoredFile() { any(TestScope ts).getLocation().getFile() = this diff --git a/python/ql/src/meta/analysis-quality/CallGraphQuality.qll b/python/ql/src/meta/analysis-quality/CallGraphQuality.qll index a0a5e42cc83..32277d3e183 100644 --- a/python/ql/src/meta/analysis-quality/CallGraphQuality.qll +++ b/python/ql/src/meta/analysis-quality/CallGraphQuality.qll @@ -14,7 +14,7 @@ class RelevantCall extends Call { RelevantCall() { not this.getLocation().getFile() instanceof IgnoredFile } } -/** Provides classes for call-graph resolution by using points-to */ +/** Provides classes for call-graph resolution by using points-to. */ module PointsTo { /** A call that can be resolved by points-to. */ class ResolvableCall extends RelevantCall { @@ -22,7 +22,7 @@ module PointsTo { ResolvableCall() { target.getACall() = this.getAFlowNode() } - /** Gets a resolved target of this call */ + /** Gets a resolved target of this call. */ Value getTarget() { result = target } } @@ -56,7 +56,7 @@ module PointsTo { } /** - * A call that can be resolved by points-to, where resolved target is not considered relevant. + * A call that can be resolved by points-to, where the resolved target is not considered relevant. * See `ResolvableCallRelevantTarget` for definition of relevance. */ class ResolvableCallIrrelevantTarget extends ResolvableCall { diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql index 118b0c64df1..b316804311d 100644 --- a/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql @@ -1,6 +1,6 @@ /** * @name Ratio of resolvable call by points-to - * @description The percentage (relevant) calls that can be resolved to a target. + * @description The percentage of (relevant) calls that can be resolved to a target. * @kind metric * @metricType project * @metricAggregate sum min max avg diff --git a/python/ql/src/meta/analysis-quality/ResolvableCallCandidates.ql b/python/ql/src/meta/analysis-quality/ResolvableCallCandidates.ql index c93ffba86cb..7fa078800ec 100644 --- a/python/ql/src/meta/analysis-quality/ResolvableCallCandidates.ql +++ b/python/ql/src/meta/analysis-quality/ResolvableCallCandidates.ql @@ -1,6 +1,6 @@ /** * @name Resolvable call candidates - * @description The number (relevant) calls in the program. + * @description The number of (relevant) calls in the program. * @kind metric * @metricType project * @metricAggregate sum From a7d23063ded9718720ee421230359fbc6ca52265 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 13 Jul 2020 10:44:19 +0200 Subject: [PATCH 3/6] Python: Fix grammar Co-authored-by: Taus --- python/ql/src/meta/MetaMetrics.qll | 2 +- python/ql/src/meta/analysis-quality/CallGraphQuality.qll | 2 +- python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql | 2 +- .../analysis-quality/PointsToResolvableCallsRelevantTarget.ql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ql/src/meta/MetaMetrics.qll b/python/ql/src/meta/MetaMetrics.qll index 8a79edb2e90..5b8af0973b4 100644 --- a/python/ql/src/meta/MetaMetrics.qll +++ b/python/ql/src/meta/MetaMetrics.qll @@ -1,5 +1,5 @@ /** - * Helpers to generating meta metrics, that is, metrics about the CodeQL analysis and extractor. + * Helpers for generating meta metrics, that is, metrics about the CodeQL analysis and extractor. */ import python diff --git a/python/ql/src/meta/analysis-quality/CallGraphQuality.qll b/python/ql/src/meta/analysis-quality/CallGraphQuality.qll index 32277d3e183..bd10f87865f 100644 --- a/python/ql/src/meta/analysis-quality/CallGraphQuality.qll +++ b/python/ql/src/meta/analysis-quality/CallGraphQuality.qll @@ -57,7 +57,7 @@ module PointsTo { /** * A call that can be resolved by points-to, where the resolved target is not considered relevant. - * See `ResolvableCallRelevantTarget` for definition of relevance. + * See `ResolvableCallRelevantTarget` for the definition of relevance. */ class ResolvableCallIrrelevantTarget extends ResolvableCall { ResolvableCallIrrelevantTarget() { not this instanceof ResolvableCallRelevantTarget } diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql index e236f7577b6..5a3be2cb6fc 100644 --- a/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql @@ -1,6 +1,6 @@ /** * @name Resolvable calls by points-to - * @description The number of (relevant) calls that could be resolved to its target. + * @description The number of (relevant) calls that can be resolved to a target. * @kind metric * @metricType project * @metricAggregate sum diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql index 9a04085cc7d..902053fe880 100644 --- a/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql @@ -1,6 +1,6 @@ /** * @name Resolvable calls by points-to, to relevant target - * @description The number of (relevant) calls that could be resolved to its target that is relevant. + * @description The number of (relevant) calls that could be resolved to a target that is relevant. * @kind metric * @metricType project * @metricAggregate sum From 0b6c3ff99d8909f1c37d2eb5f1375772f03c3e3f Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Mon, 13 Jul 2020 10:46:03 +0200 Subject: [PATCH 4/6] Python: Don't use PointsTo module name in metrics query To avoid confusion with the normal PointsTo module in python/ql/src/semmle/python/pointsto/PointsTo.qll --- python/ql/src/meta/analysis-quality/CallGraphQuality.qll | 2 +- .../src/meta/analysis-quality/PointsToResolvableCallRatio.ql | 3 ++- python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql | 2 +- .../analysis-quality/PointsToResolvableCallsRelevantTarget.ql | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/ql/src/meta/analysis-quality/CallGraphQuality.qll b/python/ql/src/meta/analysis-quality/CallGraphQuality.qll index bd10f87865f..095f7b008a0 100644 --- a/python/ql/src/meta/analysis-quality/CallGraphQuality.qll +++ b/python/ql/src/meta/analysis-quality/CallGraphQuality.qll @@ -15,7 +15,7 @@ class RelevantCall extends Call { } /** Provides classes for call-graph resolution by using points-to. */ -module PointsTo { +module PointsToBasedCallGraph { /** A call that can be resolved by points-to. */ class ResolvableCall extends RelevantCall { Value target; diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql index b316804311d..8a00d986d86 100644 --- a/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql @@ -11,4 +11,5 @@ import python import CallGraphQuality -select projectRoot(), 100.0 * count(PointsTo::ResolvableCall call) / count(RelevantCall call).(float) +select projectRoot(), + 100.0 * count(PointsToBasedCallGraph::ResolvableCall call) / count(RelevantCall call).(float) diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql index 5a3be2cb6fc..31fa9b3a684 100644 --- a/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql @@ -11,4 +11,4 @@ import python import CallGraphQuality -select projectRoot(), count(PointsTo::ResolvableCall call) +select projectRoot(), count(PointsToBasedCallGraph::ResolvableCall call) diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql index 902053fe880..efcd9a0fedf 100644 --- a/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql @@ -11,4 +11,4 @@ import python import CallGraphQuality -select projectRoot(), count(PointsTo::ResolvableCallRelevantTarget call) +select projectRoot(), count(PointsToBasedCallGraph::ResolvableCallRelevantTarget call) From d913d332892d3b381c15563a01c662338376e106 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 14 Jul 2020 11:21:55 +0200 Subject: [PATCH 5/6] Python: Autoformat --- python/ql/src/meta/MetaMetrics.qll | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ql/src/meta/MetaMetrics.qll b/python/ql/src/meta/MetaMetrics.qll index 5b8af0973b4..a345a1f7eff 100644 --- a/python/ql/src/meta/MetaMetrics.qll +++ b/python/ql/src/meta/MetaMetrics.qll @@ -3,7 +3,6 @@ */ import python - private import semmle.python.filters.GeneratedCode private import semmle.python.filters.Tests From ee42d0839e6a366cfe449f42eebb1023098e2222 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 14 Jul 2020 11:26:05 +0200 Subject: [PATCH 6/6] Python: Rename target => callee To use a standardised naming :) --- .../analysis-quality/CallGraphQuality.qll | 30 +++++++++---------- .../PointsToResolvableCallRatio.ql | 2 +- .../PointsToResolvableCalls.ql | 2 +- .../PointsToResolvableCallsRelevantTarget.ql | 8 ++--- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/python/ql/src/meta/analysis-quality/CallGraphQuality.qll b/python/ql/src/meta/analysis-quality/CallGraphQuality.qll index 095f7b008a0..46f384b89ad 100644 --- a/python/ql/src/meta/analysis-quality/CallGraphQuality.qll +++ b/python/ql/src/meta/analysis-quality/CallGraphQuality.qll @@ -18,12 +18,12 @@ class RelevantCall extends Call { module PointsToBasedCallGraph { /** A call that can be resolved by points-to. */ class ResolvableCall extends RelevantCall { - Value target; + Value callee; - ResolvableCall() { target.getACall() = this.getAFlowNode() } + ResolvableCall() { callee.getACall() = this.getAFlowNode() } - /** Gets a resolved target of this call. */ - Value getTarget() { result = target } + /** Gets a resolved callee of this call. */ + Value getCallee() { result = callee } } /** A call that cannot be resolved by points-to. */ @@ -32,20 +32,20 @@ module PointsToBasedCallGraph { } /** - * A call that can be resolved by points-to, where the resolved target is relevant. - * Relevant targets include: + * A call that can be resolved by points-to, where the resolved callee is relevant. + * Relevant callees include: * - builtins * - standard library * - source code of the project */ - class ResolvableCallRelevantTarget extends ResolvableCall { - ResolvableCallRelevantTarget() { - target.isBuiltin() + class ResolvableCallRelevantCallee extends ResolvableCall { + ResolvableCallRelevantCallee() { + callee.isBuiltin() or exists(File file | - file = target.(CallableValue).getScope().getLocation().getFile() + file = callee.(CallableValue).getScope().getLocation().getFile() or - file = target.(ClassValue).getScope().getLocation().getFile() + file = callee.(ClassValue).getScope().getLocation().getFile() | file.inStdlib() or @@ -56,10 +56,10 @@ module PointsToBasedCallGraph { } /** - * A call that can be resolved by points-to, where the resolved target is not considered relevant. - * See `ResolvableCallRelevantTarget` for the definition of relevance. + * A call that can be resolved by points-to, where the resolved callee is not considered relevant. + * See `ResolvableCallRelevantCallee` for the definition of relevance. */ - class ResolvableCallIrrelevantTarget extends ResolvableCall { - ResolvableCallIrrelevantTarget() { not this instanceof ResolvableCallRelevantTarget } + class ResolvableCallIrrelevantCallee extends ResolvableCall { + ResolvableCallIrrelevantCallee() { not this instanceof ResolvableCallRelevantCallee } } } diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql index 8a00d986d86..c0d2df8bcf1 100644 --- a/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql @@ -1,6 +1,6 @@ /** * @name Ratio of resolvable call by points-to - * @description The percentage of (relevant) calls that can be resolved to a target. + * @description The percentage of (relevant) calls that can be resolved to a callee. * @kind metric * @metricType project * @metricAggregate sum min max avg diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql index 31fa9b3a684..e055eebcb89 100644 --- a/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql @@ -1,6 +1,6 @@ /** * @name Resolvable calls by points-to - * @description The number of (relevant) calls that can be resolved to a target. + * @description The number of (relevant) calls that can be resolved to a callee. * @kind metric * @metricType project * @metricAggregate sum diff --git a/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql b/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql index efcd9a0fedf..0e9c47023c3 100644 --- a/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql +++ b/python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql @@ -1,14 +1,14 @@ /** - * @name Resolvable calls by points-to, to relevant target - * @description The number of (relevant) calls that could be resolved to a target that is relevant. + * @name Resolvable calls by points-to, to relevant callee + * @description The number of (relevant) calls that could be resolved to a callee that is relevant. * @kind metric * @metricType project * @metricAggregate sum * @tags meta - * @id py/meta/points-to-resolvable-calls-relevant-target + * @id py/meta/points-to-resolvable-calls-relevant-callee */ import python import CallGraphQuality -select projectRoot(), count(PointsToBasedCallGraph::ResolvableCallRelevantTarget call) +select projectRoot(), count(PointsToBasedCallGraph::ResolvableCallRelevantCallee call)