Bug 1382449 - irregexp: Do not assert for stack overflow exceptions. r=bhackett

js/src/irregexp/RegExpEngine.cpp

@@ -1788,7 +1788,7 @@ irregexp::CompilePattern(JSContext* cx, HandleRegExpShared shared, RegExpCompile
     }
 
     if (compiler.isRegExpTooBig()) {
-        MOZ_ASSERT(compiler.cx()->isExceptionPending()); // over recursed
+        // This might erase the over-recurse error, if any.
         JS_ReportErrorASCII(cx, "regexp too big");
         return RegExpCode();
     }
@@ -1912,7 +1912,7 @@ RegExpDisjunction::ToNode(RegExpCompiler* compiler, RegExpNode* on_success)
     const RegExpTreeVector& alternatives = this->alternatives();
     size_t length = alternatives.length();
     ChoiceNode* result = compiler->alloc()->newInfallible<ChoiceNode>(compiler->alloc(), length);
-    for (size_t i = 0; i < length; i++) {
+    for (size_t i = 0; i < length && !compiler->isRegExpTooBig(); i++) {
         GuardedAlternative alternative(alternatives[i]->ToNode(compiler, on_success));
         result->AddAlternative(alternative);
     }
@@ -2263,7 +2263,7 @@ RegExpAlternative::ToNode(RegExpCompiler* compiler, RegExpNode* on_success)
 
     const RegExpTreeVector& children = nodes();
     RegExpNode* current = on_success;
-    for (int i = children.length() - 1; i >= 0; i--)
+    for (int i = children.length() - 1; i >= 0 && !compiler->isRegExpTooBig(); i--)
         current = children[i]->ToNode(compiler, current);
     return current;
 }

js/src/jit-test/tests/regexp/huge-01.js

@@ -0,0 +1,19 @@
+
+function g(N, p) {
+    var prefix = p.repeat(N);
+    var str = prefix + "[AB]";
+
+    try {
+        var re = new RegExp(str);
+        re.exec(prefix + "A");
+    } catch(e) {
+        // 1. Regexp too big is raised by the lack of the 64k virtual registers
+        // reserved for the regexp evaluation.
+        // 2. The stack overflow can occur during the analysis of the regexp
+        assertEq(e.message.includes("regexp too big") || e.message.includes("Stack overflow"), true);
+    }
+}
+
+var prefix = "/(?=k)ok/";
+for (var i = 0; i < 18; i++)
+    g(Math.pow(2, i), prefix)