From 8570b4117fc5b1a48e82b3b9d04067eb8037ef61 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 27 Jun 2019 12:21:28 +0100 Subject: [PATCH] Python: Add opaque 'decorated function' for complex decorated functions. Allows finding calls in taint-tracking without contaminating points-to results. --- python/ql/src/semmle/python/Flow.qll | 14 ++- .../src/semmle/python/objects/Callables.qll | 3 +- .../semmle/python/objects/ObjectInternal.qll | 87 +++++++++++++++++++ .../ql/src/semmle/python/objects/TObject.qll | 5 ++ .../src/semmle/python/pointsto/PointsTo.qll | 18 ++-- .../PointsTo/decorators/Test.expected | 12 +-- .../PointsTo/decorators/Values.expected | 6 +- .../PointsTo/general/GlobalPointsTo.expected | 6 +- .../PointsTo/general/LocalPointsTo.expected | 6 +- .../general/LocalPointsToType.expected | 3 - 10 files changed, 128 insertions(+), 32 deletions(-) diff --git a/python/ql/src/semmle/python/Flow.qll b/python/ql/src/semmle/python/Flow.qll index a3a5907af80..78ddb86610c 100755 --- a/python/ql/src/semmle/python/Flow.qll +++ b/python/ql/src/semmle/python/Flow.qll @@ -480,15 +480,23 @@ class CallNode extends ControlFlowNode { override Call getNode() { result = super.getNode() } predicate isDecoratorCall() { - exists(FunctionExpr func | - this.getNode() = func.getADecoratorCall() - ) + this.isClassDecoratorCall() or + this.isFunctionDecoratorCall() + } + + predicate isClassDecoratorCall() { exists(ClassExpr cls | this.getNode() = cls.getADecoratorCall() ) } + predicate isFunctionDecoratorCall() { + exists(FunctionExpr func | + this.getNode() = func.getADecoratorCall() + ) + } + /** Gets the tuple (*) argument of this call, provided there is exactly one. */ ControlFlowNode getStarArg() { result.getNode() = this.getNode().getStarArg() and diff --git a/python/ql/src/semmle/python/objects/Callables.qll b/python/ql/src/semmle/python/objects/Callables.qll index 50be0d63089..fbdfd946d07 100644 --- a/python/ql/src/semmle/python/objects/Callables.qll +++ b/python/ql/src/semmle/python/objects/Callables.qll @@ -465,5 +465,4 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod { this.getFunction().contextSensitiveCallee() } -} - +} \ No newline at end of file diff --git a/python/ql/src/semmle/python/objects/ObjectInternal.qll b/python/ql/src/semmle/python/objects/ObjectInternal.qll index 57f3f428b84..7b86e939cd6 100644 --- a/python/ql/src/semmle/python/objects/ObjectInternal.qll +++ b/python/ql/src/semmle/python/objects/ObjectInternal.qll @@ -523,6 +523,93 @@ module ObjectInternal { } +class DecoratedFunction extends ObjectInternal, TDecoratedFunction { + + + CallNode getDecoratorCall() { + this = TDecoratedFunction(result) + } + + override Builtin getBuiltin() { + none() + } + + private ObjectInternal decoratedObject() { + PointsTo::pointsTo(this.getDecoratorCall().getArg(0), _, result, _) + } + + override string getName() { + result = this.decoratedObject().getName() + } + + override string toString() { + result = "Decorated " + this.decoratedObject().toString() + } + + override boolean booleanValue() { result = true } + + override ClassDecl getClassDeclaration() { + none() + } + + override boolean isClass() { result = false } + + override ObjectInternal getClass() { result = TUnknownClass() } + + override predicate introducedAt(ControlFlowNode node, PointsToContext context) { + none() + } + + override predicate notTestableForEquality() { none() } + + override predicate callResult(PointsToContext callee, ObjectInternal obj, CfgOrigin origin) { + none() + } + + override predicate callResult(ObjectInternal obj, CfgOrigin origin) { + obj = ObjectInternal::unknown() and origin = CfgOrigin::unknown() + } + + override ControlFlowNode getOrigin() { + result = this.getDecoratorCall() + } + + override int intValue() { + none() + } + + override string strValue() { + none() + } + + override predicate calleeAndOffset(Function scope, int paramOffset) { + none() + } + + override predicate attribute(string name, ObjectInternal value, CfgOrigin origin) { + none() + } + + override predicate attributesUnknown() { none() } + + override predicate subscriptUnknown() { none() } + + override boolean isDescriptor() { result = false } + + pragma [noinline] override predicate descriptorGetClass(ObjectInternal cls, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate descriptorGetInstance(ObjectInternal instance, ObjectInternal value, CfgOrigin origin) { none() } + + pragma [noinline] override predicate binds(ObjectInternal instance, string name, ObjectInternal descriptor) { none() } + + override int length() { none() } + + override ObjectInternal getIterNext() { none() } + + override predicate contextSensitiveCallee() { none() } + +} + /** Helper for boolean predicates returning both `true` and `false` */ boolean maybe() { result = true or result = false diff --git a/python/ql/src/semmle/python/objects/TObject.qll b/python/ql/src/semmle/python/objects/TObject.qll index 27f09e5f3ac..bfe8f2784de 100644 --- a/python/ql/src/semmle/python/objects/TObject.qll +++ b/python/ql/src/semmle/python/objects/TObject.qll @@ -222,6 +222,11 @@ cached newtype TObject = not common_module_name(modname + "." + attrname) ) } + or + /* Opaque object representing the result of calling a decorator on a function that we don't understand */ + TDecoratedFunction(CallNode call) { + call.isFunctionDecoratorCall() + } private predicate is_power_2(int n) { n = 1 or diff --git a/python/ql/src/semmle/python/pointsto/PointsTo.qll b/python/ql/src/semmle/python/pointsto/PointsTo.qll index 078e55a29da..13f2a00fdbd 100644 --- a/python/ql/src/semmle/python/pointsto/PointsTo.qll +++ b/python/ql/src/semmle/python/pointsto/PointsTo.qll @@ -833,20 +833,20 @@ module InterProceduralPointsTo { (value != ObjectInternal::unknown() or not f.isDecoratorCall()) and call_points_to_from_callee(f, context, value, origin) or - call_result_is_first_argument(f, context) and + f.isFunctionDecoratorCall() and + call_points_to_from_callee(f, context, ObjectInternal::unknown(), _) and + value = TDecoratedFunction(f) and origin = f + or + f.isClassDecoratorCall() and + call_points_to_from_callee(f, context, ObjectInternal::unknown(), _) and + PointsToInternal::pointsTo(f.getArg(0), context, value, origin) + or + Types::six_add_metaclass(f, context, _, _) and PointsToInternal::pointsTo(f.getArg(0), context, value, origin) or Expressions::typeCallPointsTo(f, context, value, origin, _, _) } - /** Helper for call_points_to to improve join-order */ - private predicate call_result_is_first_argument(CallNode f, PointsToContext context) { - Types::six_add_metaclass(f, context, _, _) - or - /* A decorator and we don't understand it. Use the original, undecorated value */ - f.isDecoratorCall() and call_points_to_from_callee(f, context, ObjectInternal::unknown(), _) - } - /** Helper for call_points_to to improve join-order */ pragma [noinline] private predicate call_points_to_from_callee(CallNode f, PointsToContext context, ObjectInternal value, ControlFlowNode origin) { diff --git a/python/ql/test/library-tests/PointsTo/decorators/Test.expected b/python/ql/test/library-tests/PointsTo/decorators/Test.expected index 83a6576bf00..4352b429ca0 100644 --- a/python/ql/test/library-tests/PointsTo/decorators/Test.expected +++ b/python/ql/test/library-tests/PointsTo/decorators/Test.expected @@ -13,18 +13,18 @@ | 16 | ControlFlowNode for func | Function func3 | test.py:31 | | 16 | ControlFlowNode for functools | Module functools | test.py:1 | | 17 | ControlFlowNode for args | args | test.py:17 | -| 17 | ControlFlowNode for wrapper | Function wrapper | test.py:17 | +| 17 | ControlFlowNode for wrapper | Attribute()() | test.py:16 | | 18 | ControlFlowNode for args | args | test.py:17 | -| 20 | ControlFlowNode for wrapper | Function wrapper | test.py:17 | +| 20 | ControlFlowNode for wrapper | Attribute()() | test.py:16 | | 22 | ControlFlowNode for annotate | Function annotate | test.py:3 | | 23 | ControlFlowNode for func1 | Function func1 | test.py:23 | | 26 | ControlFlowNode for wraps1 | Function wraps1 | test.py:9 | | 27 | ControlFlowNode for func2 | Function wrapper | test.py:10 | | 30 | ControlFlowNode for wraps2 | Function wraps2 | test.py:15 | -| 31 | ControlFlowNode for func3 | Function wrapper | test.py:17 | +| 31 | ControlFlowNode for func3 | Attribute()() | test.py:16 | | 41 | ControlFlowNode for func1 | Function func1 | test.py:23 | | 42 | ControlFlowNode for func2 | Function wrapper | test.py:10 | -| 43 | ControlFlowNode for func3 | Function wrapper | test.py:17 | +| 43 | ControlFlowNode for func3 | Attribute()() | test.py:16 | | 48 | ControlFlowNode for None | NoneType None | test.py:48 | | 48 | ControlFlowNode for register | Function register | test.py:48 | | 49 | ControlFlowNode for decorator | Function decorator | test.py:49 | @@ -45,8 +45,8 @@ | 60 | ControlFlowNode for foo | Function foo | test.py:60 | | 63 | ControlFlowNode for foo | Function foo | test.py:60 | | 65 | ControlFlowNode for register | Function register | test.py:48 | -| 66 | ControlFlowNode for bar | Function bar | test.py:66 | -| 69 | ControlFlowNode for bar | Function bar | test.py:66 | +| 66 | ControlFlowNode for bar | register() | test.py:65 | +| 69 | ControlFlowNode for bar | register() | test.py:65 | | 71 | ControlFlowNode for register | Function register | test.py:48 | | 72 | ControlFlowNode for baz | Function baz | test.py:72 | | 75 | ControlFlowNode for baz | Function baz | test.py:72 | diff --git a/python/ql/test/library-tests/PointsTo/decorators/Values.expected b/python/ql/test/library-tests/PointsTo/decorators/Values.expected index fd150267f72..67695e0b8ad 100644 --- a/python/ql/test/library-tests/PointsTo/decorators/Values.expected +++ b/python/ql/test/library-tests/PointsTo/decorators/Values.expected @@ -12,13 +12,13 @@ | test.py:16:22:16:25 | ControlFlowNode for func | runtime | Unknown value | | test.py:16:22:16:25 | ControlFlowNode for func | test.py:30 from import | Function func3 | | test.py:18:21:18:24 | ControlFlowNode for args | runtime | instance of tuple | -| test.py:20:12:20:18 | ControlFlowNode for wrapper | test.py:30 from import | Function wraps2.wrapper | +| test.py:20:12:20:18 | ControlFlowNode for wrapper | test.py:30 from import | Decorated Function wraps2.wrapper | | test.py:22:2:22:9 | ControlFlowNode for annotate | import | Function annotate | | test.py:26:2:26:7 | ControlFlowNode for wraps1 | import | Function wraps1 | | test.py:30:2:30:7 | ControlFlowNode for wraps2 | import | Function wraps2 | | test.py:41:1:41:5 | ControlFlowNode for func1 | import | Function func1 | | test.py:42:1:42:5 | ControlFlowNode for func2 | import | Function wraps1.wrapper | -| test.py:43:1:43:5 | ControlFlowNode for func3 | import | Function wraps2.wrapper | +| test.py:43:1:43:5 | ControlFlowNode for func3 | import | Decorated Function wraps2.wrapper | | test.py:48:19:48:22 | ControlFlowNode for None | import | None | | test.py:50:16:50:23 | ControlFlowNode for callable | runtime | Builtin-function callable | | test.py:50:16:50:23 | ControlFlowNode for callable | test.py:55 from runtime | Builtin-function callable | @@ -53,6 +53,6 @@ | test.py:59:2:59:9 | ControlFlowNode for register | import | Function register | | test.py:63:1:63:3 | ControlFlowNode for foo | import | Function foo | | test.py:65:2:65:9 | ControlFlowNode for register | import | Function register | -| test.py:69:1:69:3 | ControlFlowNode for bar | import | Function bar | +| test.py:69:1:69:3 | ControlFlowNode for bar | import | Decorated Function bar | | test.py:71:2:71:9 | ControlFlowNode for register | import | Function register | | test.py:75:1:75:3 | ControlFlowNode for baz | import | Function baz | diff --git a/python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected b/python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected index 806114fee02..4287a695927 100644 --- a/python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/general/GlobalPointsTo.expected @@ -32,9 +32,9 @@ | Class X | 36 | ControlFlowNode for classmethod() | classmethod() | | Class X | 37 | ControlFlowNode for FunctionExpr | Function method1 | | Class X | 37 | ControlFlowNode for method1 | classmethod() | -| Class X | 40 | ControlFlowNode for deco() | Function method2 | +| Class X | 40 | ControlFlowNode for deco() | deco() | | Class X | 41 | ControlFlowNode for FunctionExpr | Function method2 | -| Class X | 41 | ControlFlowNode for method2 | Function method2 | +| Class X | 41 | ControlFlowNode for method2 | deco() | | Module pointsto_test | 17 | ControlFlowNode for Attribute | list object | | Module pointsto_test | 17 | ControlFlowNode for Compare | bool False | | Module pointsto_test | 17 | ControlFlowNode for Compare | bool True | @@ -85,7 +85,7 @@ | Module pointsto_test | 66 | ControlFlowNode for tuple | builtin-class tuple | | Module pointsto_test | 69 | ControlFlowNode for Attribute | Attribute | | Module pointsto_test | 69 | ControlFlowNode for X | class X | -| Module pointsto_test | 70 | ControlFlowNode for Attribute | Function method2 | +| Module pointsto_test | 70 | ControlFlowNode for Attribute | deco() | | Module pointsto_test | 70 | ControlFlowNode for X | class X | | Module pointsto_test | 72 | ControlFlowNode for ImportExpr | Module abc | | Module pointsto_test | 72 | ControlFlowNode for ImportMember | Function abstractmethod | diff --git a/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected b/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected index 787c36bded4..5f310036e85 100644 --- a/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected +++ b/python/ql/test/library-tests/PointsTo/general/LocalPointsTo.expected @@ -49,9 +49,9 @@ | 36 | ControlFlowNode for classmethod() | classmethod() | | 37 | ControlFlowNode for FunctionExpr | Function method1 | | 37 | ControlFlowNode for method1 | classmethod() | -| 40 | ControlFlowNode for deco() | Function method2 | +| 40 | ControlFlowNode for deco() | deco() | | 41 | ControlFlowNode for FunctionExpr | Function method2 | -| 41 | ControlFlowNode for method2 | Function method2 | +| 41 | ControlFlowNode for method2 | deco() | | 44 | ControlFlowNode for FunctionExpr | Function deco | | 44 | ControlFlowNode for deco | Function deco | | 47 | ControlFlowNode for v1 | class C | @@ -93,7 +93,7 @@ | 66 | ControlFlowNode for tuple | builtin-class tuple | | 69 | ControlFlowNode for Attribute | Attribute | | 69 | ControlFlowNode for X | class X | -| 70 | ControlFlowNode for Attribute | Function method2 | +| 70 | ControlFlowNode for Attribute | deco() | | 70 | ControlFlowNode for X | class X | | 72 | ControlFlowNode for ImportExpr | Module abc | | 72 | ControlFlowNode for ImportMember | Function abstractmethod | diff --git a/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected b/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected index 504263e7c3a..836b06f3eef 100644 --- a/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected +++ b/python/ql/test/library-tests/PointsTo/general/LocalPointsToType.expected @@ -51,9 +51,7 @@ | 36 | ControlFlowNode for classmethod() | classmethod() | builtin-class classmethod | | 37 | ControlFlowNode for FunctionExpr | Function method1 | builtin-class function | | 37 | ControlFlowNode for method1 | classmethod() | builtin-class classmethod | -| 40 | ControlFlowNode for deco() | Function method2 | builtin-class function | | 41 | ControlFlowNode for FunctionExpr | Function method2 | builtin-class function | -| 41 | ControlFlowNode for method2 | Function method2 | builtin-class function | | 44 | ControlFlowNode for FunctionExpr | Function deco | builtin-class function | | 44 | ControlFlowNode for deco | Function deco | builtin-class function | | 47 | ControlFlowNode for v1 | class C | builtin-class type | @@ -96,7 +94,6 @@ | 66 | ControlFlowNode for tuple | builtin-class tuple | builtin-class type | | 69 | ControlFlowNode for Attribute | Attribute | builtin-class method | | 69 | ControlFlowNode for X | class X | builtin-class type | -| 70 | ControlFlowNode for Attribute | Function method2 | builtin-class function | | 70 | ControlFlowNode for X | class X | builtin-class type | | 72 | ControlFlowNode for ImportExpr | Module abc | builtin-class module | | 72 | ControlFlowNode for ImportMember | Function abstractmethod | builtin-class function |