Merge pull request #1494 from markshannon/python-better-handling-calls-on-edge-of-context

Python: better handling calls on edge of context

python/ql/src/semmle/python/objects/Callables.qll

@@ -159,6 +159,8 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
         function = this and offset = 0
     }
 
+    override predicate contextSensitiveCallee() { any() }
+
 }
 
 
@@ -280,6 +282,8 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc
         function = this and offset = 0
     }
 
+    override predicate contextSensitiveCallee() { none() }
+
 }
 
 /** Class representing methods of built-in classes (otherwise known as method-descriptors) such as `list.append`.
@@ -370,6 +374,8 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod
         function = this and offset = 0
     }
 
+    override predicate contextSensitiveCallee() { none() }
+
 }
 
 /** Class representing bound-methods.
@@ -425,7 +431,6 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
         result = this.getFunction().getName()
     }
 
-
     override Function getScope() { 
         result = this.getFunction().getScope()
     }
@@ -456,8 +461,9 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
         function = this.getFunction() and offset = 1
     }
 
+    override predicate contextSensitiveCallee() {
+        this.getFunction().contextSensitiveCallee()
+    }
+
 }
 
-
-
-

python/ql/src/semmle/python/objects/Classes.qll

@@ -90,6 +90,8 @@ abstract class ClassObjectInternal extends ObjectInternal {
 
     override predicate subscriptUnknown() { none() }
 
+    override predicate contextSensitiveCallee() { none() }
+
     /* Classes aren't usually iterable, but can e.g. Enums */
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 

python/ql/src/semmle/python/objects/Constants.qll

@@ -69,6 +69,8 @@ abstract class ConstantObjectInternal extends ObjectInternal {
 
     override string getName() { none() }
 
+    override predicate contextSensitiveCallee() { none() }
+
 }
 
 private abstract class BooleanObjectInternal extends ConstantObjectInternal {

python/ql/src/semmle/python/objects/Descriptors.qll

@@ -91,6 +91,8 @@ class PropertyInternal extends ObjectInternal, TProperty {
         )
     }
 
+    override predicate contextSensitiveCallee() { none() }
+
     /* Properties aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
@@ -179,6 +181,8 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
         result = this.getFunction().getName()
     }
 
+    override predicate contextSensitiveCallee() { none() }
+
     /* Classmethods aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
@@ -253,6 +257,8 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
         result = this.getFunction().getName()
     }
 
+    override predicate contextSensitiveCallee() { none() }
+
     /* Staticmethods aren't iterable */
     override ObjectInternal getIterNext() { none() }
 

python/ql/src/semmle/python/objects/Instances.qll

@@ -51,6 +51,8 @@ abstract class InstanceObject extends ObjectInternal {
 
     override string getName() { none() }
 
+    override predicate contextSensitiveCallee() { none() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }
@@ -368,6 +370,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
 
     override string getName() { none() }
 
+    override predicate contextSensitiveCallee() { none() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }
@@ -476,6 +480,8 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
 
     override string getName() { none() }
 
+    override predicate contextSensitiveCallee() { none() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }

python/ql/src/semmle/python/objects/Modules.qll

@@ -52,6 +52,8 @@ abstract class ModuleObjectInternal extends ObjectInternal {
         any(PackageObjectInternal package).getInitModule() = this
     }
 
+    override predicate contextSensitiveCallee() { none() }
+
     /* Modules aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
@@ -411,6 +413,8 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA
     /* We know what this is called, but not its innate name */
     override string getName() { none() }
 
+    override predicate contextSensitiveCallee() { none() }
+
     /* Modules aren't iterable */
     override ObjectInternal getIterNext() { none() }
 

python/ql/src/semmle/python/objects/ObjectInternal.qll

@@ -167,6 +167,8 @@ class ObjectInternal extends TObject {
      */
     abstract string getName();
 
+    abstract predicate contextSensitiveCallee();
+
     /** Gets the 'object' resulting from iterating over this object.
      * Used in the context `for i in this:`. The result is the 'object'
      * assigned to `i`.
@@ -256,6 +258,8 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
         result = this.getBuiltin().getName()
     }
 
+    override predicate contextSensitiveCallee() { none() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }
@@ -335,6 +339,8 @@ class UnknownInternal extends ObjectInternal, TUnknown {
 
     override string getName() { none() }
 
+    override predicate contextSensitiveCallee() { none() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }
@@ -415,6 +421,10 @@ class UndefinedInternal extends ObjectInternal, TUndefined {
 
     override string getName() { none() }
 
+    /** Holds if this object requires context to determine the object resulting from a call to it.
+     * True for most callables. */
+    override predicate contextSensitiveCallee() { none() }
+
     override ObjectInternal getIterNext() { none() }
 
 }

python/ql/src/semmle/python/objects/Sequences.qll

@@ -32,6 +32,8 @@ abstract class SequenceObjectInternal extends ObjectInternal {
 
     override string getName() { none() }
 
+    override predicate contextSensitiveCallee() { none() }
+
     override ObjectInternal getIterNext() { result = this.getItem(_) }
 
 }

python/ql/src/semmle/python/pointsto/PointsTo.qll

@@ -860,6 +860,7 @@ module InterProceduralPointsTo {
             )
             or
             context.untrackableCall(f) and
+            func.contextSensitiveCallee() and
             value = ObjectInternal::unknown() and origin = f
             or
             exists(CfgOrigin orig |

python/ql/test/library-tests/PointsTo/decorators/Test.expected

@@ -1,3 +1,52 @@
+| 1 | ControlFlowNode for functools | Module functools | test.py:1 |
+| 3 | ControlFlowNode for annotate | Function annotate | test.py:3 |
+| 4 | ControlFlowNode for inner | Function inner | test.py:4 |
+| 5 | ControlFlowNode for func | Function func1 | test.py:23 |
+| 6 | ControlFlowNode for func | Function func1 | test.py:23 |
+| 7 | ControlFlowNode for inner | Function inner | test.py:4 |
+| 9 | ControlFlowNode for wraps1 | Function wraps1 | test.py:9 |
+| 10 | ControlFlowNode for args | args | test.py:10 |
+| 10 | ControlFlowNode for wrapper | Function wrapper | test.py:10 |
+| 11 | ControlFlowNode for args | args | test.py:10 |
+| 13 | ControlFlowNode for wrapper | Function wrapper | test.py:10 |
+| 15 | ControlFlowNode for wraps2 | Function wraps2 | test.py:15 |
+| 16 | ControlFlowNode for func | Function func3 | test.py:31 |
+| 16 | ControlFlowNode for functools | Module functools | test.py:1 |
+| 17 | ControlFlowNode for args | args | test.py:17 |
+| 17 | ControlFlowNode for wrapper | Function wrapper | test.py:17 |
+| 18 | ControlFlowNode for args | args | test.py:17 |
+| 20 | ControlFlowNode for wrapper | Function wrapper | test.py:17 |
+| 22 | ControlFlowNode for annotate | Function annotate | test.py:3 |
+| 23 | ControlFlowNode for func1 | Function func1 | test.py:23 |
+| 26 | ControlFlowNode for wraps1 | Function wraps1 | test.py:9 |
+| 27 | ControlFlowNode for func2 | Function wrapper | test.py:10 |
+| 30 | ControlFlowNode for wraps2 | Function wraps2 | test.py:15 |
+| 31 | ControlFlowNode for func3 | Function wrapper | test.py:17 |
 | 41 | ControlFlowNode for func1 | Function func1 | test.py:23 |
 | 42 | ControlFlowNode for func2 | Function wrapper | test.py:10 |
 | 43 | ControlFlowNode for func3 | Function wrapper | test.py:17 |
+| 48 | ControlFlowNode for None | NoneType None | test.py:48 |
+| 48 | ControlFlowNode for register | Function register | test.py:48 |
+| 49 | ControlFlowNode for decorator | Function decorator | test.py:49 |
+| 50 | ControlFlowNode for callable | Builtin-function callable | test.py:50 |
+| 50 | ControlFlowNode for func | Function baz | test.py:72 |
+| 50 | ControlFlowNode for func | Function foo | test.py:60 |
+| 51 | ControlFlowNode for ValueError | builtin-class ValueError | test.py:51 |
+| 52 | ControlFlowNode for func | Function baz | test.py:72 |
+| 52 | ControlFlowNode for func | Function foo | test.py:60 |
+| 54 | ControlFlowNode for callable | Builtin-function callable | test.py:54 |
+| 54 | ControlFlowNode for name | Function bar | test.py:66 |
+| 54 | ControlFlowNode for name | NoneType None | test.py:48 |
+| 54 | ControlFlowNode for name | int 17 | test.py:59 |
+| 55 | ControlFlowNode for decorator | Function decorator | test.py:49 |
+| 55 | ControlFlowNode for name | Function bar | test.py:66 |
+| 57 | ControlFlowNode for decorator | Function decorator | test.py:49 |
+| 59 | ControlFlowNode for register | Function register | test.py:48 |
+| 60 | ControlFlowNode for foo | Function foo | test.py:60 |
+| 63 | ControlFlowNode for foo | Function foo | test.py:60 |
+| 65 | ControlFlowNode for register | Function register | test.py:48 |
+| 66 | ControlFlowNode for bar | Function bar | test.py:66 |
+| 69 | ControlFlowNode for bar | Function bar | test.py:66 |
+| 71 | ControlFlowNode for register | Function register | test.py:48 |
+| 72 | ControlFlowNode for baz | Function baz | test.py:72 |
+| 75 | ControlFlowNode for baz | Function baz | test.py:72 |

python/ql/test/library-tests/PointsTo/decorators/Test.ql

@@ -1,11 +1,11 @@
 import python
 
-from ControlFlowNode f, Object o, ControlFlowNode x, int line
+// We don't care about the internals of functools which vary from
+// version to version, just the end result.
+from NameNode f, Object o, ControlFlowNode x, int line
 
 where f.refersTo(o, x) and
 f.getLocation().getFile().getBaseName() = "test.py" and
-// We don't care about the internals of functools which vary from
-// version to version, just the end result.
-line = f.getLocation().getStartLine() and line > 40
+line = f.getLocation().getStartLine()
 
 select line, f.toString(), o.toString(), x.getLocation().toString()

python/ql/test/library-tests/PointsTo/decorators/Values.expected

@@ -0,0 +1,58 @@
+| test.py:5:9:5:12 | ControlFlowNode for func | runtime | Unknown value |
+| test.py:5:9:5:12 | ControlFlowNode for func | test.py:22 from import | Function func1 |
+| test.py:6:16:6:19 | ControlFlowNode for func | runtime | Unknown value |
+| test.py:6:16:6:19 | ControlFlowNode for func | test.py:22 from import | Function func1 |
+| test.py:7:12:7:16 | ControlFlowNode for inner | runtime | Function annotate.inner |
+| test.py:7:12:7:16 | ControlFlowNode for inner | test.py:22 from import | Function annotate.inner |
+| test.py:11:21:11:24 | ControlFlowNode for args | runtime | instance of tuple |
+| test.py:13:12:13:18 | ControlFlowNode for wrapper | runtime | Function wraps1.wrapper |
+| test.py:13:12:13:18 | ControlFlowNode for wrapper | test.py:26 from import | Function wraps1.wrapper |
+| test.py:16:6:16:14 | ControlFlowNode for functools | runtime | Module functools |
+| test.py:16:6:16:14 | ControlFlowNode for functools | test.py:30 from import | Module functools |
+| test.py:16:22:16:25 | ControlFlowNode for func | runtime | Unknown value |
+| test.py:16:22:16:25 | ControlFlowNode for func | test.py:30 from import | Function func3 |
+| test.py:18:21:18:24 | ControlFlowNode for args | runtime | instance of tuple |
+| test.py:20:12:20:18 | ControlFlowNode for wrapper | test.py:30 from import | Function wraps2.wrapper |
+| test.py:22:2:22:9 | ControlFlowNode for annotate | import | Function annotate |
+| test.py:26:2:26:7 | ControlFlowNode for wraps1 | import | Function wraps1 |
+| test.py:30:2:30:7 | ControlFlowNode for wraps2 | import | Function wraps2 |
+| test.py:41:1:41:5 | ControlFlowNode for func1 | import | Function func1 |
+| test.py:42:1:42:5 | ControlFlowNode for func2 | import | Function wraps1.wrapper |
+| test.py:43:1:43:5 | ControlFlowNode for func3 | import | Function wraps2.wrapper |
+| test.py:48:19:48:22 | ControlFlowNode for None | import | None |
+| test.py:50:16:50:23 | ControlFlowNode for callable | runtime | Builtin-function callable |
+| test.py:50:16:50:23 | ControlFlowNode for callable | test.py:55 from runtime | Builtin-function callable |
+| test.py:50:16:50:23 | ControlFlowNode for callable | test.py:59 from import | Builtin-function callable |
+| test.py:50:16:50:23 | ControlFlowNode for callable | test.py:71 from import | Builtin-function callable |
+| test.py:50:25:50:28 | ControlFlowNode for func | runtime | Unknown value |
+| test.py:50:25:50:28 | ControlFlowNode for func | test.py:55 from runtime | Unknown value |
+| test.py:50:25:50:28 | ControlFlowNode for func | test.py:59 from import | Function foo |
+| test.py:50:25:50:28 | ControlFlowNode for func | test.py:71 from import | Function baz |
+| test.py:51:19:51:28 | ControlFlowNode for ValueError | runtime | builtin-class ValueError |
+| test.py:51:19:51:28 | ControlFlowNode for ValueError | test.py:55 from runtime | builtin-class ValueError |
+| test.py:52:16:52:19 | ControlFlowNode for func | runtime | Unknown value |
+| test.py:52:16:52:19 | ControlFlowNode for func | test.py:55 from runtime | Unknown value |
+| test.py:52:16:52:19 | ControlFlowNode for func | test.py:59 from import | Function foo |
+| test.py:52:16:52:19 | ControlFlowNode for func | test.py:71 from import | Function baz |
+| test.py:54:8:54:15 | ControlFlowNode for callable | runtime | Builtin-function callable |
+| test.py:54:8:54:15 | ControlFlowNode for callable | test.py:59 from import | Builtin-function callable |
+| test.py:54:8:54:15 | ControlFlowNode for callable | test.py:65 from import | Builtin-function callable |
+| test.py:54:8:54:15 | ControlFlowNode for callable | test.py:71 from import | Builtin-function callable |
+| test.py:54:17:54:20 | ControlFlowNode for name | runtime | None |
+| test.py:54:17:54:20 | ControlFlowNode for name | runtime | Unknown value |
+| test.py:54:17:54:20 | ControlFlowNode for name | test.py:59 from import | int 17 |
+| test.py:54:17:54:20 | ControlFlowNode for name | test.py:65 from import | Function bar |
+| test.py:54:17:54:20 | ControlFlowNode for name | test.py:71 from import | None |
+| test.py:55:16:55:24 | ControlFlowNode for decorator | runtime | Function register.decorator |
+| test.py:55:16:55:24 | ControlFlowNode for decorator | test.py:65 from import | Function register.decorator |
+| test.py:55:26:55:29 | ControlFlowNode for name | runtime | Unknown value |
+| test.py:55:26:55:29 | ControlFlowNode for name | test.py:65 from import | Function bar |
+| test.py:57:16:57:24 | ControlFlowNode for decorator | runtime | Function register.decorator |
+| test.py:57:16:57:24 | ControlFlowNode for decorator | test.py:59 from import | Function register.decorator |
+| test.py:57:16:57:24 | ControlFlowNode for decorator | test.py:71 from import | Function register.decorator |
+| test.py:59:2:59:9 | ControlFlowNode for register | import | Function register |
+| test.py:63:1:63:3 | ControlFlowNode for foo | import | Function foo |
+| test.py:65:2:65:9 | ControlFlowNode for register | import | Function register |
+| test.py:69:1:69:3 | ControlFlowNode for bar | import | Function bar |
+| test.py:71:2:71:9 | ControlFlowNode for register | import | Function register |
+| test.py:75:1:75:3 | ControlFlowNode for baz | import | Function baz |

python/ql/test/library-tests/PointsTo/decorators/Values.ql

@@ -0,0 +1,10 @@
+
+import python
+import semmle.python.pointsto.PointsTo
+import semmle.python.objects.ObjectInternal
+
+from NameNode f, Context ctx, ObjectInternal v
+where
+  f.getLocation().getFile().getBaseName() = "test.py" and
+  PointsTo::pointsTo(f, ctx, v, _)
+select f, ctx, v

python/ql/test/library-tests/PointsTo/decorators/test.py

@@ -40,4 +40,36 @@ def func3():
 
 func1
 func2
-func3
+func3
+
+
+#Fancy decorators
+
+def register(name=None):
+    def decorator(func):
+        if not callable(func):
+            raise ValueError("not a callable")
+        return func
+
+    if callable(name):
+        return decorator(name)
+    else:
+        return decorator
+
+@register(17)
+def foo():
+    pass
+
+foo
+
+@register
+def bar():
+    pass
+
+bar()
+
+@register()
+def baz():
+    pass
+
+baz()