Python: Internal points-to extension enhancement. Use it handle re module constants.

change-notes/1.20/analysis-python.md

@@ -5,6 +5,10 @@
 
  > Changes that affect alerts in many files or from many queries
 > For example, changes to file classification
+
+The constants `MULTILINE` and `VERBOSE` in `re` module, are now understood for Python 3.6 and upward. 
+Removes false positives seen when using Python 3.6, but not when using earlier versions.
+
  ## New queries
 
  | **Query**                   | **Tags**  | **Purpose**                                                        |

python/ql/src/semmle/python/pointsto/PointsTo.qll

@@ -266,7 +266,6 @@ module PointsTo {
             SSA::ssa_definition_points_to(var.getDefinition(), context, value, cls, origin)
         }
 
-
     }
 
     predicate name_maybe_imported_from(ModuleObject mod, string name) {
@@ -635,6 +634,11 @@ module PointsTo {
         )
         or
         points_to(f.getObject(), context, unknownValue(), theUnknownType(), origin) and value = unknownValue() and cls = theUnknownType()
+        or
+        exists(CustomPointsToAttribute object, string name |
+            points_to(f.getObject(name), context, object, _, _) and
+            object.attributePointsTo(name, value, cls, origin)
+        )
     }
 
     /** Holds if `f` is an expression node `tval if cond else fval` and points to `(value, cls, origin)`. */

python/ql/src/semmle/python/regex.qll

@@ -40,7 +40,7 @@ string mode_from_mode_object(Object obj) {
         result = "MULTILINE" or result = "DOTALL" or result = "UNICODE" or
         result = "VERBOSE"
     ) and
-    ModuleObject::named("re").getAttribute(result) = obj
+    ModuleObject::named("sre_constants").getAttribute("SRE_FLAG_" + result) = obj
     or
     exists(BinaryExpr be, Object sub | obj.getOrigin() = be |
         be.getOp() instanceof BitOr and

python/ql/src/semmle/python/types/Extensions.qll

@@ -34,6 +34,13 @@ abstract class CustomPointsToOriginFact extends CustomPointsToFact {
 
 }
 
+/** INTERNAL -- Do not use */
+abstract class CustomPointsToAttribute extends Object {
+
+    abstract predicate attributePointsTo(string name, Object value, ClassObject cls, ControlFlowNode origin);
+
+}
+
 /* An example */
 
 /** Any variable iterating over range or xrange must be an integer */
@@ -56,4 +63,21 @@ class RangeIterationVariableFact extends CustomPointsToFact {
     }
 }
 
+/* Python 3.6+ regex module constants */
+
+class ReModulePointToExtension extends CustomPointsToAttribute {
+
+    ReModulePointToExtension() {
+        this.(ModuleObject).getName() = "re"
+    }
+
+    override predicate attributePointsTo(string name, Object value, ClassObject cls, ControlFlowNode origin) {
+        exists(ModuleObject sre_constants |
+            sre_constants.getName() = "sre_constants" and
+            sre_constants.attributeRefersTo("SRE_FLAG_" + name, value, cls, origin)
+        )
+    }
+}
+
+
 

python/ql/test/library-tests/PointsTo/extensions/Extend.expected

@@ -0,0 +1,7 @@
+| test.py:4:1:4:3 | ControlFlowNode for one | int 1 |
+| test.py:5:1:5:3 | ControlFlowNode for two | int 2 |
+| test.py:8:1:8:1 | ControlFlowNode for IntegerLiteral | int 1 |
+| test.py:8:1:8:7 | ControlFlowNode for Tuple | Tuple |
+| test.py:8:3:8:3 | ControlFlowNode for IntegerLiteral | int 2 |
+| test.py:8:5:8:5 | ControlFlowNode for IntegerLiteral | int 3 |
+| test.py:8:7:8:7 | ControlFlowNode for IntegerLiteral | int 4 |

python/ql/test/library-tests/PointsTo/extensions/Extend.ql

@@ -0,0 +1,30 @@
+
+
+import python
+
+private import semmle.python.types.Extensions
+
+
+class CfgExtension  extends CustomPointsToOriginFact {
+
+    CfgExtension() {
+        this.(NameNode).getId() = "one"
+        or
+        this.(NameNode).getId() = "two"
+    }
+
+    override predicate pointsTo(Object value, ClassObject cls) {
+        cls = theIntType() and
+        (
+            this.(NameNode).getId() = "one" and value.(NumericObject).intValue() = 1
+            or
+            this.(NameNode).getId() = "two" and value.(NumericObject).intValue() = 2
+        )
+    }
+}
+
+from ControlFlowNode f, Object o
+where f.getLocation().getFile().getBaseName() = "test.py" and f.refersTo(o)
+select f, o.toString()
+
+

python/ql/test/library-tests/PointsTo/extensions/test.py

@@ -0,0 +1,8 @@
+
+#Magical values
+
+one
+two
+
+#Make sure values exist in DB
+1,2,3,4

python/ql/test/library-tests/regex/Characters.ql

@@ -8,7 +8,7 @@ import python
 import semmle.python.regex
 
 from Regex r, int start, int end
-where r.character(start, end)
+where r.character(start, end) and r.getLocation().getFile().getBaseName() = "test.py"
 select r.getText(), start, end
 
 

python/ql/test/library-tests/regex/FirstLast.ql

@@ -10,5 +10,5 @@ predicate part(Regex r, int start, int end, string kind) {
 }
 
 from Regex r, int start, int end, string kind
-where part(r, start, end, kind)
+where part(r, start, end, kind) and r.getLocation().getFile().getBaseName() = "test.py"
 select r.getText(), kind, start, end

python/ql/test/library-tests/regex/Regex.ql

@@ -22,5 +22,5 @@ predicate part(Regex r, int start, int end, string kind) {
 }
 
 from Regex r, int start, int end, string kind
-where part(r, start, end, kind)
+where part(r, start, end, kind) and r.getLocation().getFile().getBaseName() = "test.py"
 select r.getText(), kind, start, end