Bug 1636495: Add JS::CheckRegExpSyntax r=evilpie

To make sure that `<input>` elements with `pattern` attributes update their validation state (`:invalid`) properly, nsContentUtils::IsPatternMatching needs to be able to distinguish between parsing errors caused by an invalid pattern, vs parsing errors caused by OOM/overrecursion.

This patch also fixes up the places inside the new regexp engine where we can throw over-recursed to make sure that we set the right flag on the context, then fixes regexp/huge-01.js (and the binast variants) to accept a different error message.

Differential Revision: https://phabricator.services.mozilla.com/D74499

js/public/RegExp.h

@@ -79,6 +79,16 @@ extern JS_PUBLIC_API RegExpFlags GetRegExpFlags(JSContext* cx,
  */
 extern JS_PUBLIC_API JSString* GetRegExpSource(JSContext* cx,
                                                Handle<JSObject*> obj);
+/**
+ * Check whether the given source is a valid regexp. If the regexp parses
+ * successfully, returns true and sets |error| to undefined. If the regexp
+ * has a syntax error, returns true, sets |error| to that error object, and
+ * clears the exception. Returns false on OOM or over-recursion.
+ */
+extern JS_PUBLIC_API bool CheckRegExpSyntax(JSContext* cx,
+                                            const char16_t* chars,
+                                            size_t length, RegExpFlags flags,
+                                            MutableHandle<Value> error);
 
 }  // namespace JS
 

js/src/jit-test/tests/regexp/huge-01.js

@@ -10,7 +10,9 @@ function g(N, p) {
         // 1. Regexp too big is raised by the lack of the 64k virtual registers
         // reserved for the regexp evaluation.
         // 2. The stack overflow can occur during the analysis of the regexp
-        assertEq(e.message.includes("regexp too big") || e.message.includes("Stack overflow"), true);
+        assertEq(e.message.includes("regexp too big") ||
+		 e.message.includes("Stack overflow") ||
+		 e.message.includes("too much recursion"), true);
     }
 }
 

js/src/new-regexp/RegExpAPI.cpp

@@ -153,6 +153,11 @@ static void ReportSyntaxError(TokenStreamAnyChars& ts,
   gc::AutoSuppressGC suppressGC(ts.context());
   uint32_t errorNumber = ErrorNumber(result.error);
 
+  if (errorNumber == JSMSG_OVER_RECURSED) {
+    ReportOverRecursed(ts.context());
+    return;
+  }
+
   uint32_t offset = std::max(result.error_pos, 0);
   MOZ_ASSERT(offset <= length);
 
@@ -427,7 +432,7 @@ bool CompilePattern(JSContext* cx, MutableHandleRegExpShared re,
   data.error = AnalyzeRegExp(cx->isolate, isLatin1, data.node);
   if (data.error != RegExpError::kNone) {
     MOZ_ASSERT(data.error == RegExpError::kAnalysisStackOverflow);
-    JS_ReportErrorASCII(cx, "Stack overflow");
+    ReportOverRecursed(cx);
     return false;
   }
 

js/src/vm/RegExpObject.cpp

@@ -1758,3 +1758,36 @@ JS_PUBLIC_API JSString* JS::GetRegExpSource(JSContext* cx, HandleObject obj) {
   }
   return shared->getSource();
 }
+
+JS_PUBLIC_API bool JS::CheckRegExpSyntax(JSContext* cx, const char16_t* chars,
+                                         size_t length, RegExpFlags flags,
+                                         MutableHandleValue error) {
+  AssertHeapIsIdle();
+  CHECK_THREAD(cx);
+
+  CompileOptions dummyOptions(cx);
+  frontend::DummyTokenStream dummyTokenStream(cx, dummyOptions);
+
+  LifoAllocScope allocScope(&cx->tempLifoAlloc());
+
+  mozilla::Range<const char16_t> source(chars, length);
+#ifdef ENABLE_NEW_REGEXP
+  bool success =
+      irregexp::CheckPatternSyntax(cx, dummyTokenStream, source, flags);
+#else
+  bool success = irregexp::ParsePatternSyntax(
+      dummyTokenStream, allocScope.alloc(), source, flags.unicode());
+#endif
+  error.set(UndefinedValue());
+  if (!success) {
+    // We can fail because of OOM or over-recursion even if the syntax is valid.
+    if (cx->isThrowingOutOfMemory() || cx->isThrowingOverRecursed()) {
+      return false;
+    }
+    if (!cx->getPendingException(error)) {
+      return false;
+    }
+    cx->clearPendingException();
+  }
+  return true;
+}