Merge pull request #1844 from asger-semmle/more-type-info

Approved by xiemaisi

change-notes/1.23/analysis-javascript.md

@@ -2,6 +2,9 @@
 
 ## General improvements
 
+* Support for the following frameworks and libraries has been improved:
+  - [firebase](https://www.npmjs.com/package/firebase)
+
 ## New queries
 
 | **Query**                                                                 | **Tags**                                                          | **Purpose**                                                                                                                                                                            |

javascript/extractor/lib/typescript/src/main.ts

@@ -271,6 +271,15 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
         getCurrentDirectory: () => basePath,
     });
 
+    for (let typeRoot of typeRoots || []) {
+        traverseTypeRoot(typeRoot, "");
+    }
+
+    for (let sourceFile of program.getSourceFiles()) {
+        addModuleBindingsFromModuleDeclarations(sourceFile);
+        addModuleBindingsFromFilePath(sourceFile);
+    }
+
     /** Concatenates two imports paths. These always use `/` as path separator. */
     function joinModulePath(prefix: string, suffix: string) {
         if (prefix.length === 0) return suffix;
@@ -300,16 +309,25 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
             if (sourceFile == null) {
                 continue;
             }
+            addModuleBindingFromRelativePath(sourceFile, importPrefix, child);
+        }
+    }
+
+    /**
+     * Emits module bindings for a module with relative path `folder/baseName`.
+     */
+    function addModuleBindingFromRelativePath(sourceFile: ts.SourceFile, folder: string, baseName: string) {
         let symbol = typeChecker.getSymbolAtLocation(sourceFile);
-            if (symbol == null) continue; // Happens if the source file is not a module.
+        if (symbol == null) return; // Happens if the source file is not a module.
+
+        let stem = getStem(baseName);
+        let importPath = (stem === "index")
+            ? folder
+            : joinModulePath(folder, stem);
 
         let canonicalSymbol = getEffectiveExportTarget(symbol); // Follow `export = X` declarations.
         let symbolId = state.typeTable.getSymbolId(canonicalSymbol);
 
-            let importPath = (child === "index.d.ts")
-                ? importPrefix
-                : joinModulePath(importPrefix, pathlib.basename(child, ".d.ts"));
-
         // Associate the module name with this symbol.
         state.typeTable.addModuleMapping(symbolId, importPath);
 
@@ -322,14 +340,43 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
             });
         }
     }
+
+    /**
+     * Returns the basename of `file` without its extension, while treating `.d.ts` as a
+     * single extension.
+     */
+    function getStem(file: string) {
+        if (file.endsWith(".d.ts")) {
+            return pathlib.basename(file, ".d.ts");
         }
-    for (let typeRoot of typeRoots || []) {
+        let base = pathlib.basename(file);
-        traverseTypeRoot(typeRoot, "");
+        let dot = base.lastIndexOf('.');
+        return dot === -1 || dot === 0 ? base : base.substring(0, dot);
     }
 
-    // Emit module name bindings for external module declarations, i.e: `declare module 'X' {..}`
+    /**
-    // These can generally occur anywhere; they may or may not be on the type root path.
+     * Emits module bindings for a module based on its file path.
-    for (let sourceFile of program.getSourceFiles()) {
+     *
+     * This looks for enclosing `node_modules` folders to determine the module name.
+     * This is needed for modules that ship their own type definitions as opposed to having
+     * type definitions loaded from a type root (conventionally named `@types/xxx`).
+     */
+    function addModuleBindingsFromFilePath(sourceFile: ts.SourceFile) {
+        let fullPath = sourceFile.fileName;
+        let index = fullPath.lastIndexOf('/node_modules/');
+        if (index === -1) return;
+        let relativePath = fullPath.substring(index + '/node_modules/'.length);
+        // Ignore node_modules/@types folders here as they are typically handled as type roots.
+        if (relativePath.startsWith("@types/")) return;
+        let { dir, base } = pathlib.parse(relativePath);
+        addModuleBindingFromRelativePath(sourceFile, dir, base);
+    }
+
+    /**
+     * Emit module name bindings for external module declarations, i.e: `declare module 'X' {..}`
+     * These can generally occur anywhere; they may or may not be on the type root path.
+     */
+    function addModuleBindingsFromModuleDeclarations(sourceFile: ts.SourceFile) {
         for (let stmt of sourceFile.statements) {
             if (ts.isModuleDeclaration(stmt) && ts.isStringLiteral(stmt.name)) {
                 let symbol = (stmt as any).symbol;

javascript/ql/src/semmle/javascript/CanonicalNames.qll

@@ -37,7 +37,14 @@ class CanonicalName extends @symbol {
   /**
    * Gets the name of the external module represented by this canonical name, if any.
    */
-  string getExternalModuleName() { symbol_module(this, result) }
+  string getExternalModuleName() {
+    symbol_module(this, result)
+    or
+    exists(PackageJSON pkg |
+      getModule() = pkg.getMainModule() and
+      result = pkg.getPackageName()
+    )
+  }
 
   /**
    * Gets the name of the global variable represented by this canonical name, if any.

javascript/ql/src/semmle/javascript/JSON.qll

@@ -32,6 +32,14 @@ class JSONValue extends @json_value, Locatable {
   predicate isTopLevel() { not exists(getParent()) }
 
   override string toString() { json(this, _, _, _, result) }
+
+  /** Gets the JSON file containing this value. */
+  File getJsonFile() {
+    exists(Location loc |
+      json_locations(this, loc) and
+      result = loc.getFile()
+    )
+  }
 }
 
 /**

javascript/ql/src/semmle/javascript/NPM.qll

@@ -8,7 +8,7 @@ private import NodeModuleResolutionImpl
 /** A `package.json` configuration object. */
 class PackageJSON extends JSONObject {
   PackageJSON() {
-    getFile().getBaseName() = "package.json" and
+    getJsonFile().getBaseName() = "package.json" and
     isTopLevel()
   }
 
@@ -274,7 +274,7 @@ class NPMPackage extends @folder {
   /** The `package.json` file of this package. */
   PackageJSON pkg;
 
-  NPMPackage() { pkg.getFile().getParentContainer() = this }
+  NPMPackage() { pkg.getJsonFile().getParentContainer() = this }
 
   /** Gets a textual representation of this package. */
   string toString() { result = this.(Folder).toString() }

javascript/ql/src/semmle/javascript/TypeScript.qll

@@ -1550,14 +1550,20 @@ class ReferenceImport extends LineComment {
   string getAttributeName() { result = attribute }
 
   /**
+   * DEPRECATED. This is no longer supported.
+   *
    * Gets the file referenced by this import.
    */
-  File getImportedFile() { none() } // Overridden in subtypes.
+  deprecated
+  File getImportedFile() { none() }
 
   /**
+   * DEPRECATED. This is no longer supported.
+   *
    * Gets the top-level of the referenced file.
    */
-  TopLevel getImportedTopLevel() { result.getFile() = getImportedFile() }
+  deprecated
+  TopLevel getImportedTopLevel() { none() }
 }
 
 /**
@@ -1568,24 +1574,6 @@ class ReferenceImport extends LineComment {
  */
 class ReferencePathImport extends ReferenceImport {
   ReferencePathImport() { attribute = "path" }
-
-  override File getImportedFile() { result = this.(PathExpr).resolve() }
-}
-
-/**
- * Treats reference imports comments as path expressions without exposing
- * the methods from `PathExpr` on `ReferenceImport`.
- */
-private class ReferenceImportAsPathExpr extends PathExpr {
-  ReferenceImport reference;
-
-  ReferenceImportAsPathExpr() { this = reference }
-
-  override string getValue() { result = reference.getAttributeValue() }
-
-  override Folder getSearchRoot(int priority) {
-    result = reference.getFile().getParentContainer() and priority = 0
-  }
 }
 
 /**
@@ -1596,14 +1584,6 @@ private class ReferenceImportAsPathExpr extends PathExpr {
  */
 class ReferenceTypesImport extends ReferenceImport {
   ReferenceTypesImport() { attribute = "types" }
-
-  override File getImportedFile() {
-    result = min(Folder nodeModules, int distance |
-        findNodeModulesFolder(getFile().getParentContainer(), nodeModules, distance)
-      |
-        nodeModules.getFolder("@types").getFolder(value).getFile("index.d.ts") order by distance
-      )
-  }
 }
 
 /**

javascript/ql/src/semmle/javascript/dataflow/DataFlow.qll

@@ -193,6 +193,62 @@ module DataFlow {
         not fun.getExit().isJoin() // can only reach exit by the return statement
       )
     }
+
+    /**
+     * Gets the static type of this node as determined by the TypeScript type system.
+     */
+    private Type getType() {
+      exists(AST::ValueNode node |
+        this = TValueNode(node) and
+        ast_node_type(node, result)
+      )
+      or
+      exists(BindingPattern pattern |
+        this = lvalueNode(pattern) and
+        ast_node_type(pattern, result)
+      )
+      or
+      exists(MethodDefinition def |
+        this = TThisNode(def.getInit()) and
+        ast_node_type(def.getDeclaringClass(), result)
+      )
+    }
+
+    /**
+     * Gets the type annotation describing the type of this node,
+     * provided that a static type could not be found.
+     *
+     * Doesn't take field types and function return types into account.
+     */
+    private JSDocTypeExpr getFallbackTypeAnnotation() {
+      exists(BindingPattern pattern |
+        this = lvalueNode(pattern) and
+        not ast_node_type(pattern, _) and
+        result = pattern.getTypeAnnotation()
+      )
+      or
+      result = getAPredecessor().getFallbackTypeAnnotation()
+    }
+
+    /**
+     * Holds if this node is annotated with the given named type,
+     * or is declared as a subtype thereof, or is a union or intersection containing such a type.
+     */
+    predicate hasUnderlyingType(string globalName) {
+      getType().hasUnderlyingType(globalName)
+      or
+      getFallbackTypeAnnotation().getAnUnderlyingType().hasQualifiedName(globalName)
+    }
+
+    /**
+     * Holds if this node is annotated with the given named type,
+     * or is declared as a subtype thereof, or is a union or intersection containing such a type.
+     */
+    predicate hasUnderlyingType(string moduleName, string typeName) {
+      getType().hasUnderlyingType(moduleName, typeName)
+      or
+      getFallbackTypeAnnotation().getAnUnderlyingType().hasQualifiedName(moduleName, typeName)
+    }
   }
 
   /**

javascript/ql/src/semmle/javascript/frameworks/Babel.qll

@@ -11,7 +11,7 @@ module Babel {
    */
   class Config extends JSONObject {
     Config() {
-      isTopLevel() and getFile().getBaseName().matches(".babelrc%")
+      isTopLevel() and getJsonFile().getBaseName().matches(".babelrc%")
       or
       this = any(PackageJSON pkg).getPropValue("babel")
     }
@@ -34,12 +34,12 @@ module Babel {
      * Gets a file affected by this Babel configuration.
      */
     Container getAContainerInScope() {
-      result = getFile().getParentContainer()
+      result = getJsonFile().getParentContainer()
       or
       result = getAContainerInScope().getAChildContainer() and
       // File-relative .babelrc search stops at any package.json or .babelrc file.
-      not result.getAChildContainer() = any(PackageJSON pkg).getFile() and
+      not result.getAChildContainer() = any(PackageJSON pkg).getJsonFile() and
-      not result.getAChildContainer() = any(Config pkg).getFile()
+      not result.getAChildContainer() = any(Config pkg).getJsonFile()
     }
 
     /**
@@ -133,7 +133,7 @@ module Babel {
     /**
      * Gets the folder in which this configuration is located.
      */
-    Folder getFolder() { result = getFile().getParentContainer() }
+    Folder getFolder() { result = getJsonFile().getParentContainer() }
   }
 
   /**

javascript/ql/src/semmle/javascript/frameworks/Firebase.qll

@@ -27,7 +27,11 @@ module Firebase {
 
   /** Gets a reference to a Firebase app created with `initializeApp`. */
   private DataFlow::SourceNode initApp(DataFlow::TypeTracker t) {
-    result = firebase().getAMethodCall("initializeApp") and t.start()
+    t.start() and
+    result = firebase().getAMethodCall("initializeApp")
+    or
+    t.start() and
+    result.hasUnderlyingType("firebase", "app.App")
     or
     exists (DataFlow::TypeTracker t2 |
       result = initApp(t2).track(t2, t)
@@ -48,6 +52,9 @@ module Firebase {
     private DataFlow::SourceNode database(DataFlow::TypeTracker t) {
       result = app().getAMethodCall("database") and t.start()
       or
+      t.start() and
+      result.hasUnderlyingType("firebase", "database.Database")
+      or
       exists (DataFlow::TypeTracker t2 |
         result = database(t2).track(t2, t)
       )
@@ -78,6 +85,8 @@ module Firebase {
         )
         or
         result = snapshot().getAPropertyRead("ref")
+        or
+        result.hasUnderlyingType("firebase", "database.Reference")
       )
       or
       exists (DataFlow::TypeTracker t2 |
@@ -102,6 +111,8 @@ module Firebase {
           name = "orderBy" + any(string s) or
           name = "startAt"
         )
+        or
+        result.hasUnderlyingType("firebase", "database.Query")
       )
       or
       exists (DataFlow::TypeTracker t2 |
@@ -293,6 +304,8 @@ module Firebase {
         prop = "before" or // only defined on Change objects
         prop = "after"
       )
+      or
+      result.hasUnderlyingType("firebase", "database.DataSnapshot")
     )
     or
     promiseTaintStep(snapshot(t), result)

javascript/ql/src/semmle/javascript/frameworks/Vue.qll

@@ -352,7 +352,8 @@ module Vue {
     }
 
     private Module getModule() {
-      exists(HTML::ScriptElement elem | elem.getFile() = file |
+      exists(HTML::ScriptElement elem |
+        xmlElements(elem, _, _, _, file) and // Avoid materializing all of Locatable.getFile()
         result.getTopLevel() = elem.getScript()
       )
     }

javascript/ql/test/library-tests/TypeScript/DeclarationFiles/ReferenceImports.expected

@@ -1,4 +1,4 @@
-| client1.ts:1:1:1:40 | /// <re ... .ts" /> | path | framework1.d.ts | framework1.d.ts:0:0:0:0 | framework1.d.ts |
+| client1.ts:1:1:1:40 | /// <re ... .ts" /> | path | framework1.d.ts |
-| client2.ts:1:1:1:36 | /// <re ... rk2" /> | types | framework2 | node_modules/@types/framework2/index.d.ts:0:0:0:0 | node_modules/@types/framework2/index.d.ts |
+| client2.ts:1:1:1:36 | /// <re ... rk2" /> | types | framework2 |
-| declare-module-client2.ts:1:1:1:45 | /// <re ... d.ts"/> | path | ./declare-module.d.ts | declare-module.d.ts:0:0:0:0 | declare-module.d.ts |
+| declare-module-client2.ts:1:1:1:45 | /// <re ... d.ts"/> | path | ./declare-module.d.ts |
-| declare-module-client.ts:1:1:1:43 | /// <re ... d.ts"/> | path | declare-module.d.ts | declare-module.d.ts:0:0:0:0 | declare-module.d.ts |
+| declare-module-client.ts:1:1:1:43 | /// <re ... d.ts"/> | path | declare-module.d.ts |

javascript/ql/test/library-tests/TypeScript/DeclarationFiles/ReferenceImports.ql

@@ -1,4 +1,4 @@
 import javascript
 
 from ReferenceImport comment
-select comment, comment.getAttributeName(), comment.getAttributeValue(), comment.getImportedFile()
+select comment, comment.getAttributeName(), comment.getAttributeValue()

javascript/ql/test/library-tests/TypeScript/ImportOwnPackage/TypeNames.expected

@@ -0,0 +1,8 @@
+| bar/client.ts:3:5:3:5 | f | my-awesome-package | Foo |
+| bar/client.ts:3:9:3:17 | new Foo() | my-awesome-package | Foo |
+| bar/client.ts:4:5:4:5 | b | my-awesome-package | Bar |
+| bar/client.ts:4:9:4:9 | f | my-awesome-package | Foo |
+| bar/client.ts:4:9:4:15 | f.bar() | my-awesome-package | Bar |
+| foo/index.ts:1:14:1:16 | Foo | my-awesome-package | Foo |
+| foo/index.ts:2:23:2:31 | new Bar() | my-awesome-package | Bar |
+| foo/index.ts:5:14:5:16 | Bar | my-awesome-package | Bar |

javascript/ql/test/library-tests/TypeScript/ImportOwnPackage/TypeNames.ql

@@ -0,0 +1,5 @@
+import javascript
+
+from Expr e, string mod, string name
+where e.getType().(TypeReference).hasQualifiedName(mod, name)
+select e, mod, name

javascript/ql/test/library-tests/TypeScript/ImportOwnPackage/bar/client.ts

@@ -0,0 +1,4 @@
+import { Foo } from "../foo";
+
+let f = new Foo();
+let b = f.bar();

javascript/ql/test/library-tests/TypeScript/ImportOwnPackage/foo/index.ts

@@ -0,0 +1,5 @@
+export class Foo {
+  bar(): Bar { return new Bar() }
+}
+
+export class Bar {}

javascript/ql/test/library-tests/TypeScript/ImportOwnPackage/foo/package.json

@@ -0,0 +1,3 @@
+{
+  "name": "my-awesome-package"
+}

javascript/ql/test/library-tests/TypeScript/ImportOwnPackage/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "include": ["foo", "bar"]
+}

javascript/ql/test/library-tests/frameworks/Firebase/FirebaseRef.expected

@@ -1,3 +1,6 @@
+| src/import_assign.ts:4:3:4:17 | db.ref("hello") |
+| src/import_named.ts:4:3:4:17 | db.ref("hello") |
+| src/import_star.ts:4:3:4:17 | db.ref("hello") |
 | tst.js:5:1:5:22 | fb.data ... ef('x') |
 | tst.js:7:3:7:7 | x.ref |
 | tst.js:7:3:7:14 | x.ref.parent |

javascript/ql/test/library-tests/frameworks/Firebase/node_modules/firebase/index.d.ts

@@ -0,0 +1,10 @@
+// Greatly simplified version of the Firebase d.ts file.
+
+declare namespace firebase.database {
+  interface Database {
+    ref(name: string): any;
+  }
+}
+
+export = firebase;
+export as namespace firebase;

javascript/ql/test/library-tests/frameworks/Firebase/src/import_assign.ts

@@ -0,0 +1,5 @@
+import firebase = require("firebase");
+
+function test(db: firebase.database.Database) {
+  db.ref("hello");
+}

javascript/ql/test/library-tests/frameworks/Firebase/src/import_named.ts

@@ -0,0 +1,5 @@
+import { database } from "firebase";
+
+function test(db: database.Database) {
+  db.ref("hello");
+}

javascript/ql/test/library-tests/frameworks/Firebase/src/import_star.ts

@@ -0,0 +1,5 @@
+import * as firebase from "firebase";
+
+function test(db: firebase.database.Database) {
+  db.ref("hello");
+}

javascript/ql/test/library-tests/frameworks/Firebase/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "include": ["src"]
+}