Bug 1447996 - Don't GC when hitting overrecursion in RegExpCompiler; make the static analysis detect this. r=sfink

--HG--
extra : rebase_source : 09b91fd21fbb615aba2564ef92e8033b76bbf3ed

js/src/irregexp/RegExpEngine.cpp

@@ -30,6 +30,7 @@
 
 #include "irregexp/RegExpEngine.h"
 
+#include "gc/GC.h"
 #include "irregexp/NativeRegExpMacroAssembler.h"
 #include "irregexp/RegExpCharacters.h"
 #include "irregexp/RegExpMacroAssembler.h"
@@ -1694,6 +1695,7 @@ RegExpCompiler::Assemble(JSContext* cx,
 
     if (reg_exp_too_big_) {
         code.destroy();
+        js::gc::AutoSuppressGC suppress(cx);
         JS_ReportErrorASCII(cx, "regexp too big");
         return RegExpCode();
     }
@@ -1812,6 +1814,11 @@ irregexp::CompilePattern(JSContext* cx, HandleRegExpShared shared, RegExpCompile
         return RegExpCode();
     }
 
+    // We should not GC when we have a jit::MacroAssembler on the stack. Check
+    // this here because the static analysis does not understand the
+    // Maybe<NativeRegExpMacroAssembler> below.
+    JS::AutoCheckCannotGC nogc(cx);
+
     Maybe<jit::JitContext> ctx;
     Maybe<NativeRegExpMacroAssembler> native_assembler;
     Maybe<InterpretedRegExpMacroAssembler> interpreted_assembler;
@@ -2537,7 +2544,7 @@ BoyerMooreLookahead::EmitSkipInstructions(RegExpMacroAssembler* masm)
 bool
 RegExpCompiler::CheckOverRecursed()
 {
-    if (!CheckRecursionLimit(cx())) {
+    if (!CheckRecursionLimitDontReport(cx())) {
         SetRegExpTooBig();
         return false;
     }
@@ -3903,6 +3910,9 @@ TextNode::TextEmitPass(RegExpCompiler* compiler,
                     break;
                 }
                 if (emit_function != nullptr) {
+                    // emit_function is a function pointer. Suppress static
+                    // analysis false positives.
+                    JS::AutoSuppressGCAnalysis suppress;
                     bool bound_checked = emit_function(compiler,
                                                        quarks[j],
                                                        backtrack,

js/src/jit-test/tests/basic/bug1447996.js

@@ -0,0 +1,10 @@
+if (!('stackTest' in this))
+    quit();
+var x = 0;
+function f() {
+    var s = "abcdef(((((((a|b)a|b)a|b)a|b)a|b)a|b)a|b)" + x;
+    res = "abcdefa".match(new RegExp(s));
+    x++;
+}
+f();
+stackTest(f, true);