Bug 1331606 - Avoid OOM crashes when we reach the executable code limit. r=luke

js/src/irregexp/RegExpEngine.cpp

@@ -33,6 +33,7 @@
 #include "irregexp/NativeRegExpMacroAssembler.h"
 #include "irregexp/RegExpCharacters.h"
 #include "irregexp/RegExpMacroAssembler.h"
+#include "jit/ExecutableAllocator.h"
 #include "jit/JitCommon.h"
 
 #include "irregexp/RegExpCharacters-inl.h"
@@ -1777,7 +1778,10 @@ irregexp::CompilePattern(JSContext* cx, RegExpShared* shared, RegExpCompileData*
     Maybe<InterpretedRegExpMacroAssembler> interpreted_assembler;
 
     RegExpMacroAssembler* assembler;
-    if (IsNativeRegExpEnabled(cx) && !force_bytecode) {
+    if (IsNativeRegExpEnabled(cx) &&
+        !force_bytecode &&
+        jit::CanLikelyAllocateMoreExecutableMemory())
+    {
         NativeRegExpMacroAssembler::Mode mode =
             is_ascii ? NativeRegExpMacroAssembler::ASCII
                      : NativeRegExpMacroAssembler::CHAR16;

js/src/jit/BaselineJIT.cpp

@@ -321,12 +321,17 @@ CanEnterBaselineJIT(JSContext* cx, HandleScript script, InterpreterFrame* osrFra
     if (script->nslots() > BaselineScript::MAX_JSSCRIPT_SLOTS)
         return Method_CantCompile;
 
-    if (!cx->compartment()->ensureJitCompartmentExists(cx))
-        return Method_Error;
-
     if (script->hasBaselineScript())
         return Method_Compiled;
 
+    // Check this before calling ensureJitCompartmentExists, so we're less
+    // likely to report OOM in JSRuntime::createJitRuntime.
+    if (!CanLikelyAllocateMoreExecutableMemory())
+        return Method_Skipped;
+
+    if (!cx->compartment()->ensureJitCompartmentExists(cx))
+        return Method_Error;
+
     // Check script warm-up counter.
     if (script->incWarmUpCounter() <= JitOptions.baselineWarmUpThreshold)
         return Method_Skipped;

js/src/jit/ExecutableAllocator.cpp

@@ -413,9 +413,9 @@ ExecutableAllocator::poisonCode(JSRuntime* rt, JitPoisonRangeVector& ranges)
 // Limit on the number of bytes of executable memory to prevent JIT spraying
 // attacks.
 #if JS_BITS_PER_WORD == 32
-static const size_t MaxCodeBytesPerProcess = 128 * 1024 * 1024;
+static const size_t MaxCodeBytesPerProcess = 160 * 1024 * 1024;
 #else
-static const size_t MaxCodeBytesPerProcess = 512 * 1024 * 1024;
+static const size_t MaxCodeBytesPerProcess = 640 * 1024 * 1024;
 #endif
 
 static mozilla::Atomic<size_t> allocatedExecutableBytes(0);
@@ -453,3 +453,13 @@ js::jit::AssertAllocatedExecutableBytesIsZero()
 {
     MOZ_ASSERT(allocatedExecutableBytes == 0);
 }
+
+bool
+js::jit::CanLikelyAllocateMoreExecutableMemory()
+{
+    // Use a 16 MB buffer.
+    static const size_t BufferSize = 16 * 1024 * 1024;
+
+    MOZ_ASSERT(allocatedExecutableBytes <= MaxCodeBytesPerProcess);
+    return allocatedExecutableBytes + BufferSize <= MaxCodeBytesPerProcess;
+}

js/src/jit/ExecutableAllocator.h

@@ -353,6 +353,15 @@ SubAllocatedExecutableBytes(size_t bytes);
 extern void
 AssertAllocatedExecutableBytesIsZero();
 
+// Returns true if we can allocate a few more MB of executable code without
+// hitting our code limit. This function can be used to stop compiling things
+// that are optional (like Baseline and Ion code) when we're about to reach the
+// limit, so we are less likely to OOM or crash. Note that the limit is
+// per-process, so other threads can also allocate code after we call this
+// function.
+extern bool
+CanLikelyAllocateMoreExecutableMemory();
+
 } // namespace jit
 } // namespace js
 

js/src/jit/Ion.cpp

@@ -2502,6 +2502,11 @@ Compile(JSContext* cx, HandleScript script, BaselineFrame* osrFrame, jsbytecode*
     if (optimizationLevel == OptimizationLevel::DontCompile)
         return Method_Skipped;
 
+    if (!CanLikelyAllocateMoreExecutableMemory()) {
+        script->resetWarmUpCounter();
+        return Method_Skipped;
+    }
+
     if (script->hasIonScript()) {
         IonScript* scriptIon = script->ionScript();
         if (!scriptIon->method())

js/src/jit/IonAnalysis.cpp

@@ -4177,6 +4177,9 @@ jit::AnalyzeNewScriptDefiniteProperties(JSContext* cx, HandleFunction fun,
     TempAllocator temp(&alloc);
     JitContext jctx(cx, &temp);
 
+    if (!jit::CanLikelyAllocateMoreExecutableMemory())
+        return true;
+
     if (!cx->compartment()->ensureJitCompartmentExists(cx))
         return false;
 
@@ -4422,6 +4425,9 @@ jit::AnalyzeArgumentsUsage(JSContext* cx, JSScript* scriptArg)
     TempAllocator temp(&alloc);
     JitContext jctx(cx, &temp);
 
+    if (!jit::CanLikelyAllocateMoreExecutableMemory())
+        return true;
+
     if (!cx->compartment()->ensureJitCompartmentExists(cx))
         return false;
 

js/src/jscompartment.cpp

@@ -165,6 +165,12 @@ JSRuntime::createJitRuntime(JSContext* cx)
 
     MOZ_ASSERT(!jitRuntime_);
 
+    if (!CanLikelyAllocateMoreExecutableMemory()) {
+        // Report OOM instead of potentially hitting the MOZ_CRASH below.
+        ReportOutOfMemory(cx);
+        return nullptr;
+    }
+
     jit::JitRuntime* jrt = cx->new_<jit::JitRuntime>(cx->runtime());
     if (!jrt)
         return nullptr;

js/src/vm/UnboxedObject.cpp

@@ -7,6 +7,7 @@
 #include "vm/UnboxedObject-inl.h"
 
 #include "jit/BaselineIC.h"
+#include "jit/ExecutableAllocator.h"
 #include "jit/JitCommon.h"
 #include "jit/Linker.h"
 
@@ -706,7 +707,8 @@ UnboxedPlainObject::createWithProperties(ExclusiveContext* cx, HandleObjectGroup
     if (cx->isJSContext() &&
         !group->unknownProperties() &&
         !layout.constructorCode() &&
-        cx->asJSContext()->runtime()->jitSupportsFloatingPoint)
+        cx->asJSContext()->runtime()->jitSupportsFloatingPoint &&
+        jit::CanLikelyAllocateMoreExecutableMemory())
     {
         if (!UnboxedLayout::makeConstructorCode(cx->asJSContext(), group))
             return nullptr;