From 6f1399be9b02b4725741f1578b39e16accd40252 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 25 Jun 2019 11:32:58 +0100 Subject: [PATCH] Python: Better handle calls on edge of context. --- .../src/semmle/python/objects/Callables.qll | 14 +++-- .../ql/src/semmle/python/objects/Classes.qll | 3 + .../src/semmle/python/objects/Constants.qll | 2 + .../src/semmle/python/objects/Descriptors.qll | 6 ++ .../src/semmle/python/objects/Instances.qll | 6 ++ .../ql/src/semmle/python/objects/Modules.qll | 4 ++ .../semmle/python/objects/ObjectInternal.qll | 9 +++ .../src/semmle/python/objects/Sequences.qll | 2 + .../src/semmle/python/pointsto/PointsTo.qll | 1 + .../PointsTo/decorators/Test.expected | 49 ++++++++++++++++ .../library-tests/PointsTo/decorators/Test.ql | 8 +-- .../PointsTo/decorators/Values.expected | 58 +++++++++++++++++++ .../PointsTo/decorators/Values.ql | 10 ++++ .../library-tests/PointsTo/decorators/test.py | 34 ++++++++++- 14 files changed, 197 insertions(+), 9 deletions(-) create mode 100644 python/ql/test/library-tests/PointsTo/decorators/Values.expected create mode 100644 python/ql/test/library-tests/PointsTo/decorators/Values.ql diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index aac3dfcdc5c..1373f95dd88 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -156,6 +156,8 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti function = this and offset = 0 } + override predicate contextSensitiveCallee() { any() } + } @@ -277,6 +279,8 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc function = this and offset = 0 } + override predicate contextSensitiveCallee() { none() } + } /** Class representing methods of built-in classes (otherwise known as method-descriptors) such as `list.append`. @@ -367,6 +371,8 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod function = this and offset = 0 } + override predicate contextSensitiveCallee() { none() } + } /** Class representing bound-methods. @@ -422,7 +428,6 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { result = this.getFunction().getName() } - override Function getScope() { result = this.getFunction().getScope() } @@ -453,8 +458,9 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { function = this.getFunction() and offset = 1 } + override predicate contextSensitiveCallee() { + this.getFunction().contextSensitiveCallee() + } + } - - - diff --git a/python/ql/src/semmle/python/objects/Classes.qll b/python/ql/src/semmle/python/objects/Classes.qll index 66ea28ab764..917d893aec2 100644 --- a/python/ql/src/semmle/python/objects/Classes.qll +++ b/python/ql/src/semmle/python/objects/Classes.qll @@ -89,6 +89,9 @@ abstract class ClassObjectInternal extends ObjectInternal { } override predicate subscriptUnknown() { none() } + + override predicate contextSensitiveCallee() { none() } + } /** Class representing Python source classes */ diff --git a/python/ql/src/semmle/python/objects/Constants.qll b/python/ql/src/semmle/python/objects/Constants.qll index eeddfbe559c..8336c54e81b 100644 --- a/python/ql/src/semmle/python/objects/Constants.qll +++ b/python/ql/src/semmle/python/objects/Constants.qll @@ -69,6 +69,8 @@ abstract class ConstantObjectInternal extends ObjectInternal { override string getName() { none() } + override predicate contextSensitiveCallee() { none() } + } private abstract class BooleanObjectInternal extends ConstantObjectInternal { diff --git a/python/ql/src/semmle/python/objects/Descriptors.qll b/python/ql/src/semmle/python/objects/Descriptors.qll index 3dd31ce7e23..cde240ce41b 100644 --- a/python/ql/src/semmle/python/objects/Descriptors.qll +++ b/python/ql/src/semmle/python/objects/Descriptors.qll @@ -91,6 +91,8 @@ class PropertyInternal extends ObjectInternal, TProperty { ) } + override predicate contextSensitiveCallee() { none() } + } /** A class representing classmethods in Python */ @@ -176,6 +178,8 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod { result = this.getFunction().getName() } + override predicate contextSensitiveCallee() { none() } + } class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { @@ -247,4 +251,6 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod { result = this.getFunction().getName() } + override predicate contextSensitiveCallee() { none() } + } diff --git a/python/ql/src/semmle/python/objects/Instances.qll b/python/ql/src/semmle/python/objects/Instances.qll index b2dc7eb07d0..f54f0848ac0 100644 --- a/python/ql/src/semmle/python/objects/Instances.qll +++ b/python/ql/src/semmle/python/objects/Instances.qll @@ -51,6 +51,8 @@ abstract class InstanceObject extends ObjectInternal { override string getName() { none() } + override predicate contextSensitiveCallee() { none() } + } private predicate self_variable_reaching_init_exit(EssaVariable self) { @@ -366,6 +368,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal { override string getName() { none() } + override predicate contextSensitiveCallee() { none() } + } private int lengthFromClass(ClassObjectInternal cls) { @@ -472,5 +476,7 @@ class SuperInstance extends TSuperInstance, ObjectInternal { override string getName() { none() } + override predicate contextSensitiveCallee() { none() } + } diff --git a/python/ql/src/semmle/python/objects/Modules.qll b/python/ql/src/semmle/python/objects/Modules.qll index c3e3d6f19ef..11094bd77e8 100644 --- a/python/ql/src/semmle/python/objects/Modules.qll +++ b/python/ql/src/semmle/python/objects/Modules.qll @@ -52,6 +52,8 @@ abstract class ModuleObjectInternal extends ObjectInternal { any(PackageObjectInternal package).getInitModule() = this } + override predicate contextSensitiveCallee() { none() } + } /** A class representing built-in modules */ @@ -408,5 +410,7 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA /* We know what this is called, but not its innate name */ override string getName() { none() } + override predicate contextSensitiveCallee() { none() } + } diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index b5cf48eab04..7d2ea2ccada 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -167,6 +167,8 @@ class ObjectInternal extends TObject { */ abstract string getName(); + abstract predicate contextSensitiveCallee(); + } @@ -249,6 +251,9 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject { override string getName() { result = this.getBuiltin().getName() } + + override predicate contextSensitiveCallee() { none() } + } @@ -326,6 +331,8 @@ class UnknownInternal extends ObjectInternal, TUnknown { override string getName() { none() } + override predicate contextSensitiveCallee() { none() } + } class UndefinedInternal extends ObjectInternal, TUndefined { @@ -404,6 +411,8 @@ class UndefinedInternal extends ObjectInternal, TUndefined { override string getName() { none() } + override predicate contextSensitiveCallee() { none() } + } module ObjectInternal { diff --git a/python/ql/src/semmle/python/objects/Sequences.qll b/python/ql/src/semmle/python/objects/Sequences.qll index 7c72c7ade2f..1e0d7cd1baf 100644 --- a/python/ql/src/semmle/python/objects/Sequences.qll +++ b/python/ql/src/semmle/python/objects/Sequences.qll @@ -32,6 +32,8 @@ abstract class SequenceObjectInternal extends ObjectInternal { override string getName() { none() } + override predicate contextSensitiveCallee() { none() } + } abstract class TupleObjectInternal extends SequenceObjectInternal { diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index cd87ee8be8f..589d57ae746 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -839,6 +839,7 @@ module InterProceduralPointsTo { ) or context.untrackableCall(f) and + func.contextSensitiveCallee() and value = ObjectInternal::unknown() and origin = f or exists(CfgOrigin orig | diff --git a/python/ql/test/library-tests/PointsTo/decorators/Test.expected b/python/ql/test/library-tests/PointsTo/decorators/Test.expected index 136adce143f..83a6576bf00 100644 --- a/python/ql/test/library-tests/PointsTo/decorators/Test.expected +++ b/python/ql/test/library-tests/PointsTo/decorators/Test.expected @@ -1,3 +1,52 @@ +| 1 | ControlFlowNode for functools | Module functools | test.py:1 | +| 3 | ControlFlowNode for annotate | Function annotate | test.py:3 | +| 4 | ControlFlowNode for inner | Function inner | test.py:4 | +| 5 | ControlFlowNode for func | Function func1 | test.py:23 | +| 6 | ControlFlowNode for func | Function func1 | test.py:23 | +| 7 | ControlFlowNode for inner | Function inner | test.py:4 | +| 9 | ControlFlowNode for wraps1 | Function wraps1 | test.py:9 | +| 10 | ControlFlowNode for args | args | test.py:10 | +| 10 | ControlFlowNode for wrapper | Function wrapper | test.py:10 | +| 11 | ControlFlowNode for args | args | test.py:10 | +| 13 | ControlFlowNode for wrapper | Function wrapper | test.py:10 | +| 15 | ControlFlowNode for wraps2 | Function wraps2 | test.py:15 | +| 16 | ControlFlowNode for func | Function func3 | test.py:31 | +| 16 | ControlFlowNode for functools | Module functools | test.py:1 | +| 17 | ControlFlowNode for args | args | test.py:17 | +| 17 | ControlFlowNode for wrapper | Function wrapper | test.py:17 | +| 18 | ControlFlowNode for args | args | test.py:17 | +| 20 | ControlFlowNode for wrapper | Function wrapper | test.py:17 | +| 22 | ControlFlowNode for annotate | Function annotate | test.py:3 | +| 23 | ControlFlowNode for func1 | Function func1 | test.py:23 | +| 26 | ControlFlowNode for wraps1 | Function wraps1 | test.py:9 | +| 27 | ControlFlowNode for func2 | Function wrapper | test.py:10 | +| 30 | ControlFlowNode for wraps2 | Function wraps2 | test.py:15 | +| 31 | ControlFlowNode for func3 | Function wrapper | test.py:17 | | 41 | ControlFlowNode for func1 | Function func1 | test.py:23 | | 42 | ControlFlowNode for func2 | Function wrapper | test.py:10 | | 43 | ControlFlowNode for func3 | Function wrapper | test.py:17 | +| 48 | ControlFlowNode for None | NoneType None | test.py:48 | +| 48 | ControlFlowNode for register | Function register | test.py:48 | +| 49 | ControlFlowNode for decorator | Function decorator | test.py:49 | +| 50 | ControlFlowNode for callable | Builtin-function callable | test.py:50 | +| 50 | ControlFlowNode for func | Function baz | test.py:72 | +| 50 | ControlFlowNode for func | Function foo | test.py:60 | +| 51 | ControlFlowNode for ValueError | builtin-class ValueError | test.py:51 | +| 52 | ControlFlowNode for func | Function baz | test.py:72 | +| 52 | ControlFlowNode for func | Function foo | test.py:60 | +| 54 | ControlFlowNode for callable | Builtin-function callable | test.py:54 | +| 54 | ControlFlowNode for name | Function bar | test.py:66 | +| 54 | ControlFlowNode for name | NoneType None | test.py:48 | +| 54 | ControlFlowNode for name | int 17 | test.py:59 | +| 55 | ControlFlowNode for decorator | Function decorator | test.py:49 | +| 55 | ControlFlowNode for name | Function bar | test.py:66 | +| 57 | ControlFlowNode for decorator | Function decorator | test.py:49 | +| 59 | ControlFlowNode for register | Function register | test.py:48 | +| 60 | ControlFlowNode for foo | Function foo | test.py:60 | +| 63 | ControlFlowNode for foo | Function foo | test.py:60 | +| 65 | ControlFlowNode for register | Function register | test.py:48 | +| 66 | ControlFlowNode for bar | Function bar | test.py:66 | +| 69 | ControlFlowNode for bar | Function bar | test.py:66 | +| 71 | ControlFlowNode for register | Function register | test.py:48 | +| 72 | ControlFlowNode for baz | Function baz | test.py:72 | +| 75 | ControlFlowNode for baz | Function baz | test.py:72 | diff --git a/python/ql/test/library-tests/PointsTo/decorators/Test.ql b/python/ql/test/library-tests/PointsTo/decorators/Test.ql index c873feb9b48..af274849a26 100644 --- a/python/ql/test/library-tests/PointsTo/decorators/Test.ql +++ b/python/ql/test/library-tests/PointsTo/decorators/Test.ql @@ -1,11 +1,11 @@ import python -from ControlFlowNode f, Object o, ControlFlowNode x, int line +// We don't care about the internals of functools which vary from +// version to version, just the end result. +from NameNode f, Object o, ControlFlowNode x, int line where f.refersTo(o, x) and f.getLocation().getFile().getBaseName() = "test.py" and -// We don't care about the internals of functools which vary from -// version to version, just the end result. -line = f.getLocation().getStartLine() and line > 40 +line = f.getLocation().getStartLine() select line, f.toString(), o.toString(), x.getLocation().toString() diff --git a/python/ql/test/library-tests/PointsTo/decorators/Values.expected b/python/ql/test/library-tests/PointsTo/decorators/Values.expected new file mode 100644 index 00000000000..fd150267f72 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/decorators/Values.expected @@ -0,0 +1,58 @@ +| test.py:5:9:5:12 | ControlFlowNode for func | runtime | Unknown value | +| test.py:5:9:5:12 | ControlFlowNode for func | test.py:22 from import | Function func1 | +| test.py:6:16:6:19 | ControlFlowNode for func | runtime | Unknown value | +| test.py:6:16:6:19 | ControlFlowNode for func | test.py:22 from import | Function func1 | +| test.py:7:12:7:16 | ControlFlowNode for inner | runtime | Function annotate.inner | +| test.py:7:12:7:16 | ControlFlowNode for inner | test.py:22 from import | Function annotate.inner | +| test.py:11:21:11:24 | ControlFlowNode for args | runtime | instance of tuple | +| test.py:13:12:13:18 | ControlFlowNode for wrapper | runtime | Function wraps1.wrapper | +| test.py:13:12:13:18 | ControlFlowNode for wrapper | test.py:26 from import | Function wraps1.wrapper | +| test.py:16:6:16:14 | ControlFlowNode for functools | runtime | Module functools | +| test.py:16:6:16:14 | ControlFlowNode for functools | test.py:30 from import | Module functools | +| test.py:16:22:16:25 | ControlFlowNode for func | runtime | Unknown value | +| test.py:16:22:16:25 | ControlFlowNode for func | test.py:30 from import | Function func3 | +| test.py:18:21:18:24 | ControlFlowNode for args | runtime | instance of tuple | +| test.py:20:12:20:18 | ControlFlowNode for wrapper | test.py:30 from import | Function wraps2.wrapper | +| test.py:22:2:22:9 | ControlFlowNode for annotate | import | Function annotate | +| test.py:26:2:26:7 | ControlFlowNode for wraps1 | import | Function wraps1 | +| test.py:30:2:30:7 | ControlFlowNode for wraps2 | import | Function wraps2 | +| test.py:41:1:41:5 | ControlFlowNode for func1 | import | Function func1 | +| test.py:42:1:42:5 | ControlFlowNode for func2 | import | Function wraps1.wrapper | +| test.py:43:1:43:5 | ControlFlowNode for func3 | import | Function wraps2.wrapper | +| test.py:48:19:48:22 | ControlFlowNode for None | import | None | +| test.py:50:16:50:23 | ControlFlowNode for callable | runtime | Builtin-function callable | +| test.py:50:16:50:23 | ControlFlowNode for callable | test.py:55 from runtime | Builtin-function callable | +| test.py:50:16:50:23 | ControlFlowNode for callable | test.py:59 from import | Builtin-function callable | +| test.py:50:16:50:23 | ControlFlowNode for callable | test.py:71 from import | Builtin-function callable | +| test.py:50:25:50:28 | ControlFlowNode for func | runtime | Unknown value | +| test.py:50:25:50:28 | ControlFlowNode for func | test.py:55 from runtime | Unknown value | +| test.py:50:25:50:28 | ControlFlowNode for func | test.py:59 from import | Function foo | +| test.py:50:25:50:28 | ControlFlowNode for func | test.py:71 from import | Function baz | +| test.py:51:19:51:28 | ControlFlowNode for ValueError | runtime | builtin-class ValueError | +| test.py:51:19:51:28 | ControlFlowNode for ValueError | test.py:55 from runtime | builtin-class ValueError | +| test.py:52:16:52:19 | ControlFlowNode for func | runtime | Unknown value | +| test.py:52:16:52:19 | ControlFlowNode for func | test.py:55 from runtime | Unknown value | +| test.py:52:16:52:19 | ControlFlowNode for func | test.py:59 from import | Function foo | +| test.py:52:16:52:19 | ControlFlowNode for func | test.py:71 from import | Function baz | +| test.py:54:8:54:15 | ControlFlowNode for callable | runtime | Builtin-function callable | +| test.py:54:8:54:15 | ControlFlowNode for callable | test.py:59 from import | Builtin-function callable | +| test.py:54:8:54:15 | ControlFlowNode for callable | test.py:65 from import | Builtin-function callable | +| test.py:54:8:54:15 | ControlFlowNode for callable | test.py:71 from import | Builtin-function callable | +| test.py:54:17:54:20 | ControlFlowNode for name | runtime | None | +| test.py:54:17:54:20 | ControlFlowNode for name | runtime | Unknown value | +| test.py:54:17:54:20 | ControlFlowNode for name | test.py:59 from import | int 17 | +| test.py:54:17:54:20 | ControlFlowNode for name | test.py:65 from import | Function bar | +| test.py:54:17:54:20 | ControlFlowNode for name | test.py:71 from import | None | +| test.py:55:16:55:24 | ControlFlowNode for decorator | runtime | Function register.decorator | +| test.py:55:16:55:24 | ControlFlowNode for decorator | test.py:65 from import | Function register.decorator | +| test.py:55:26:55:29 | ControlFlowNode for name | runtime | Unknown value | +| test.py:55:26:55:29 | ControlFlowNode for name | test.py:65 from import | Function bar | +| test.py:57:16:57:24 | ControlFlowNode for decorator | runtime | Function register.decorator | +| test.py:57:16:57:24 | ControlFlowNode for decorator | test.py:59 from import | Function register.decorator | +| test.py:57:16:57:24 | ControlFlowNode for decorator | test.py:71 from import | Function register.decorator | +| test.py:59:2:59:9 | ControlFlowNode for register | import | Function register | +| test.py:63:1:63:3 | ControlFlowNode for foo | import | Function foo | +| test.py:65:2:65:9 | ControlFlowNode for register | import | Function register | +| test.py:69:1:69:3 | ControlFlowNode for bar | import | Function bar | +| test.py:71:2:71:9 | ControlFlowNode for register | import | Function register | +| test.py:75:1:75:3 | ControlFlowNode for baz | import | Function baz | diff --git a/python/ql/test/library-tests/PointsTo/decorators/Values.ql b/python/ql/test/library-tests/PointsTo/decorators/Values.ql new file mode 100644 index 00000000000..2e752466c55 --- /dev/null +++ b/python/ql/test/library-tests/PointsTo/decorators/Values.ql @@ -0,0 +1,10 @@ + +import python +import semmle.python.pointsto.PointsTo +import semmle.python.objects.ObjectInternal + +from NameNode f, Context ctx, ObjectInternal v +where + f.getLocation().getFile().getBaseName() = "test.py" and + PointsTo::pointsTo(f, ctx, v, _) +select f, ctx, v diff --git a/python/ql/test/library-tests/PointsTo/decorators/test.py b/python/ql/test/library-tests/PointsTo/decorators/test.py index 1c83d17fef4..4e5585946d7 100644 --- a/python/ql/test/library-tests/PointsTo/decorators/test.py +++ b/python/ql/test/library-tests/PointsTo/decorators/test.py @@ -40,4 +40,36 @@ def func3(): func1 func2 -func3 \ No newline at end of file +func3 + + +#Fancy decorators + +def register(name=None): + def decorator(func): + if not callable(func): + raise ValueError("not a callable") + return func + + if callable(name): + return decorator(name) + else: + return decorator + +@register(17) +def foo(): + pass + +foo + +@register +def bar(): + pass + +bar() + +@register() +def baz(): + pass + +baz()