Merge pull request #4096 from erik-krogh/qlMod

Approved by esbena
2020-08-20 10:05:30 +01:00 · 2020-08-20 10:05:30 +01:00 · 6adedac337
--- a/change-notes/1.26/analysis-javascript.md
+++ b/change-notes/1.26/analysis-javascript.md
@ -14,6 +14,8 @@
  - [pretty-format](https://www.npmjs.com/package/pretty-format)
  - [stringify-object](https://www.npmjs.com/package/stringify-object)

+* Analyzing files with the ".cjs" extension is now supported.
+
 ## New queries

 | **Query**                                                                       | **Tags**                                                          | **Purpose**                                                                                                                                                                            |
--- a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
@ -144,7 +144,7 @@ import com.semmle.util.trap.TrapWriter;
 *
 * <ul>
 *   <li>All JavaScript files, that is, files with one of the extensions supported by {@link
- *       FileType#JS} (currently ".js", ".jsx", ".mjs", ".es6", ".es").
+ *       FileType#JS} (currently ".js", ".jsx", ".mjs", ".cjs", ".es6", ".es").
 *   <li>All HTML files, that is, files with with one of the extensions supported by {@link
 *       FileType#HTML} (currently ".htm", ".html", ".xhtm", ".xhtml", ".vue").
 *   <li>All YAML files, that is, files with one of the extensions supported by {@link
--- a/javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java
@ -58,7 +58,7 @@ public class FileExtractor {
      }
    },

-    JS(".js", ".jsx", ".mjs", ".es6", ".es") {
+    JS(".js", ".jsx", ".mjs", ".cjs", ".es6", ".es") {
      @Override
      public IExtractor mkExtractor(ExtractorConfig config, ExtractorState state) {
        return new ScriptExtractor(config);
--- a/javascript/extractor/src/com/semmle/js/extractor/Main.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/Main.java
@ -43,7 +43,7 @@ public class Main {
   * A version identifier that should be updated every time the extractor changes in such a way that
   * it may produce different tuples for the same file under the same {@link ExtractorConfig}.
   */
-  public static final String EXTRACTOR_VERSION = "2020-08-18";
+  public static final String EXTRACTOR_VERSION = "2020-08-19";

  public static final Pattern NEWLINE = Pattern.compile("\n");

--- a/javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java
+++ b/javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java
@ -19,6 +19,11 @@ public class ScriptExtractor implements IExtractor {
    return extension.equals(".mjs") || extension.equals(".es6") || extension.equals(".es");
  }

+  /** True if files with the given extension should always be treated as CommonJS modules. */
+  private boolean isAlwaysCommonJSModule(String extension) {
+    return extension.equals(".cjs");
+  }
+
  @Override
  public LoCInfo extract(TextualExtractor textualExtractor) {
    LocationManager locationManager = textualExtractor.getLocationManager();
@ -45,9 +50,13 @@ public class ScriptExtractor implements IExtractor {
    }

    // Some file extensions are interpreted as modules by default.
-    if (isAlwaysModule(locationManager.getSourceFileExtension())) {
-      if (config.getSourceType() == SourceType.AUTO)
+    if (config.getSourceType() == SourceType.AUTO) {
+      if (isAlwaysModule(locationManager.getSourceFileExtension())) {
        config = config.withSourceType(SourceType.MODULE);
+      }
+      if (isAlwaysCommonJSModule(locationManager.getSourceFileExtension())) {
+        config = config.withSourceType(SourceType.COMMONJS_MODULE);
+      }
    }

    ScopeManager scopeManager =
--- a/javascript/extractor/tests/extensions/input/tst4.cjs
+++ b/javascript/extractor/tests/extensions/input/tst4.cjs
@ -0,0 +1 @@
+console.log("Hello CommonJS");
--- a/javascript/extractor/tests/extensions/output/trap/tst4.cjs.trap
+++ b/javascript/extractor/tests/extensions/output/trap/tst4.cjs.trap
@ -0,0 +1,141 @@
+#10000=@"/tst4.cjs;sourcefile"
+files(#10000,"/tst4.cjs","tst4","cjs",0)
+#10001=@"/;folder"
+folders(#10001,"/","")
+containerparent(#10001,#10000)
+#10002=@"loc,{#10000},0,0,0,0"
+locations_default(#10002,#10000,0,0,0,0)
+hasLocation(#10000,#10002)
+#20000=@"global_scope"
+scopes(#20000,0)
+#20001=@"script;{#10000},1,1"
+#20002=*
+lines(#20002,#20001,"console.log(""Hello CommonJS"");","
+")
+#20003=@"loc,{#10000},1,1,1,30"
+locations_default(#20003,#10000,1,1,1,30)
+hasLocation(#20002,#20003)
+numlines(#20001,1,1,0)
+#20004=*
+tokeninfo(#20004,6,#20001,0,"console")
+#20005=@"loc,{#10000},1,1,1,7"
+locations_default(#20005,#10000,1,1,1,7)
+hasLocation(#20004,#20005)
+#20006=*
+tokeninfo(#20006,8,#20001,1,".")
+#20007=@"loc,{#10000},1,8,1,8"
+locations_default(#20007,#10000,1,8,1,8)
+hasLocation(#20006,#20007)
+#20008=*
+tokeninfo(#20008,6,#20001,2,"log")
+#20009=@"loc,{#10000},1,9,1,11"
+locations_default(#20009,#10000,1,9,1,11)
+hasLocation(#20008,#20009)
+#20010=*
+tokeninfo(#20010,8,#20001,3,"(")
+#20011=@"loc,{#10000},1,12,1,12"
+locations_default(#20011,#10000,1,12,1,12)
+hasLocation(#20010,#20011)
+#20012=*
+tokeninfo(#20012,4,#20001,4,"""Hello CommonJS""")
+#20013=@"loc,{#10000},1,13,1,28"
+locations_default(#20013,#10000,1,13,1,28)
+hasLocation(#20012,#20013)
+#20014=*
+tokeninfo(#20014,8,#20001,5,")")
+#20015=@"loc,{#10000},1,29,1,29"
+locations_default(#20015,#10000,1,29,1,29)
+hasLocation(#20014,#20015)
+#20016=*
+tokeninfo(#20016,8,#20001,6,";")
+#20017=@"loc,{#10000},1,30,1,30"
+locations_default(#20017,#10000,1,30,1,30)
+hasLocation(#20016,#20017)
+#20018=*
+tokeninfo(#20018,0,#20001,7,"")
+#20019=@"loc,{#10000},2,1,2,0"
+locations_default(#20019,#10000,2,1,2,0)
+hasLocation(#20018,#20019)
+toplevels(#20001,0)
+#20020=@"loc,{#10000},1,1,2,0"
+locations_default(#20020,#10000,1,1,2,0)
+hasLocation(#20001,#20020)
+#20021=@"module;{#10000},1,1"
+scopes(#20021,3)
+scopenodes(#20001,#20021)
+scopenesting(#20021,#20000)
+#20022=@"var;{require};{#20021}"
+variables(#20022,"require",#20021)
+#20023=@"var;{module};{#20021}"
+variables(#20023,"module",#20021)
+#20024=@"var;{exports};{#20021}"
+variables(#20024,"exports",#20021)
+#20025=@"var;{__filename};{#20021}"
+variables(#20025,"__filename",#20021)
+#20026=@"var;{__dirname};{#20021}"
+variables(#20026,"__dirname",#20021)
+#20027=@"var;{arguments};{#20021}"
+variables(#20027,"arguments",#20021)
+isModule(#20001)
+#20028=*
+stmts(#20028,2,#20001,0,"console ... onJS"");")
+hasLocation(#20028,#20003)
+stmtContainers(#20028,#20001)
+#20029=*
+exprs(#20029,13,#20028,0,"console ... monJS"")")
+#20030=@"loc,{#10000},1,1,1,29"
+locations_default(#20030,#10000,1,1,1,29)
+hasLocation(#20029,#20030)
+enclosingStmt(#20029,#20028)
+exprContainers(#20029,#20001)
+#20031=*
+exprs(#20031,14,#20029,-1,"console.log")
+#20032=@"loc,{#10000},1,1,1,11"
+locations_default(#20032,#10000,1,1,1,11)
+hasLocation(#20031,#20032)
+enclosingStmt(#20031,#20028)
+exprContainers(#20031,#20001)
+#20033=*
+exprs(#20033,79,#20031,0,"console")
+hasLocation(#20033,#20005)
+enclosingStmt(#20033,#20028)
+exprContainers(#20033,#20001)
+literals("console","console",#20033)
+#20034=@"var;{console};{#20000}"
+variables(#20034,"console",#20000)
+bind(#20033,#20034)
+#20035=*
+exprs(#20035,0,#20031,1,"log")
+hasLocation(#20035,#20009)
+enclosingStmt(#20035,#20028)
+exprContainers(#20035,#20001)
+literals("log","log",#20035)
+#20036=*
+exprs(#20036,4,#20029,0,"""Hello CommonJS""")
+hasLocation(#20036,#20013)
+enclosingStmt(#20036,#20028)
+exprContainers(#20036,#20001)
+literals("Hello CommonJS","""Hello CommonJS""",#20036)
+#20037=*
+regexpterm(#20037,14,#20036,0,"Hello CommonJS")
+#20038=@"loc,{#10000},1,14,1,27"
+locations_default(#20038,#10000,1,14,1,27)
+hasLocation(#20037,#20038)
+regexpConstValue(#20037,"Hello CommonJS")
+#20039=*
+entry_cfg_node(#20039,#20001)
+#20040=@"loc,{#10000},1,1,1,0"
+locations_default(#20040,#10000,1,1,1,0)
+hasLocation(#20039,#20040)
+#20041=*
+exit_cfg_node(#20041,#20001)
+hasLocation(#20041,#20019)
+successor(#20028,#20033)
+successor(#20036,#20029)
+successor(#20035,#20031)
+successor(#20033,#20035)
+successor(#20031,#20036)
+successor(#20029,#20041)
+successor(#20039,#20028)
+numlines(#10000,1,1,0)
+filetype(#10000,"javascript")
--- a/javascript/ql/src/semmle/javascript/NodeJS.qll
+++ b/javascript/ql/src/semmle/javascript/NodeJS.qll
@ -221,7 +221,7 @@ class Require extends CallExpr, Import {
   *
   * <ul>
   * <li> the file `c/p`;
-   * <li> the file `c/p.{tsx,ts,jsx,es6,es,mjs}`;
+   * <li> the file `c/p.{tsx,ts,jsx,es6,es,mjs,cjs}`;
   * <li> the file `c/p.js`;
   * <li> the file `c/p.json`;
   * <li> the file `c/p.node`;
@ -230,12 +230,12 @@ class Require extends CallExpr, Import {
   *      <li> if `c/p/package.json` exists and specifies a `main` module `m`:
   *        <ul>
   *        <li> the file `c/p/m`;
-   *        <li> the file `c/p/m.{tsx,ts,jsx,es6,es,mjs}`;
+   *        <li> the file `c/p/m.{tsx,ts,jsx,es6,es,mjs,cjs}`;
   *        <li> the file `c/p/m.js`;
   *        <li> the file `c/p/m.json`;
   *        <li> the file `c/p/m.node`;
   *        </ul>
-   *      <li> the file `c/p/index.{tsx,ts,jsx,es6,es,mjs}`;
+   *      <li> the file `c/p/index.{tsx,ts,jsx,es6,es,mjs,cjs}`;
   *      <li> the file `c/p/index.js`;
   *      <li> the file `c/p/index.json`;
   *      <li> the file `c/p/index.node`.
--- a/javascript/ql/src/semmle/javascript/NodeModuleResolutionImpl.qll
+++ b/javascript/ql/src/semmle/javascript/NodeModuleResolutionImpl.qll
@ -26,11 +26,13 @@ int getFileExtensionPriority(string ext) {
  or
  ext = "mjs" and result = 5
  or
-  ext = "js" and result = 6
+  ext = "cjs" and result = 6
  or
-  ext = "json" and result = 7
+  ext = "js" and result = 7
  or
-  ext = "node" and result = 8
+  ext = "json" and result = 8
+  or
+  ext = "node" and result = 9
 }

 int prioritiesPerCandidate() { result = 3 * (numberOfExtensions() + 1) }
--- a/javascript/ql/test/library-tests/ModuleTypes/commonjs.cjs
+++ b/javascript/ql/test/library-tests/ModuleTypes/commonjs.cjs
@ -0,0 +1,3 @@
+var fs = require("fs");
+console.log("I'm a .cjs file!");
+console.log(fs);
--- a/javascript/ql/test/library-tests/ModuleTypes/import.js
+++ b/javascript/ql/test/library-tests/ModuleTypes/import.js
@ -0,0 +1,5 @@
+import * as fs from "fs";
+
+export default function (x) {
+    return fs.readFileSync(x);
+};
--- a/javascript/ql/test/library-tests/ModuleTypes/mjs.mjs
+++ b/javascript/ql/test/library-tests/ModuleTypes/mjs.mjs
@ -0,0 +1 @@
+console.log("I'm a .mjs file!");
--- a/javascript/ql/test/library-tests/ModuleTypes/require.js
+++ b/javascript/ql/test/library-tests/ModuleTypes/require.js
@ -0,0 +1,7 @@
+var fs = require("fs");
+
+module.exports = {
+    foo: function (x) {
+        return fs.readFileSync(x);
+    }
+}
--- a/javascript/ql/test/library-tests/ModuleTypes/script.js
+++ b/javascript/ql/test/library-tests/ModuleTypes/script.js
@ -0,0 +1 @@
+console.log("I'm a plain script!");
--- a/javascript/ql/test/library-tests/ModuleTypes/tests.expected
+++ b/javascript/ql/test/library-tests/ModuleTypes/tests.expected
@ -0,0 +1,5 @@
+| commonjs.cjs:1:1:3:16 | <toplevel> | node |
+| import.js:1:1:5:2 | <toplevel> | es2015 |
+| mjs.mjs:1:1:1:32 | <toplevel> | es2015 |
+| require.js:1:1:7:1 | <toplevel> | node |
+| script.js:1:1:1:35 | <toplevel> | non-module |
--- a/javascript/ql/test/library-tests/ModuleTypes/tests.ql
+++ b/javascript/ql/test/library-tests/ModuleTypes/tests.ql
@ -0,0 +1,18 @@
+import javascript
+
+query string getModuleType(TopLevel top) {
+  not top.isExterns() and
+  (
+    not top instanceof Module and
+    result = "non-module"
+    or
+    top instanceof NodeModule and
+    result = "node"
+    or
+    top instanceof ES2015Module and
+    result = "es2015"
+    or
+    top instanceof AmdModule and
+    result = "amd"
+  )
+}