Bug 627475 - Clean up methodjit OOM handling (r=dmandelin)

js/src/methodjit/Compiler.cpp

@@ -71,10 +71,8 @@ using namespace js::mjit::ic;
 
 #define RETURN_IF_OOM(retval)                                   \
     JS_BEGIN_MACRO                                              \
-        if (oomInVector || masm.oom() || stubcc.masm.oom()) {   \
-            js_ReportOutOfMemory(cx);                           \
+        if (oomInVector || masm.oom() || stubcc.masm.oom())     \
             return retval;                                      \
-        }                                                       \
     JS_END_MACRO
 
 #if defined(JS_METHODJIT_SPEW)
@@ -151,11 +149,14 @@ mjit::Compiler::compile()
     return status;
 }
 
-#define CHECK_STATUS(expr)              \
-    JS_BEGIN_MACRO                      \
-        CompileStatus status_ = (expr); \
-        if (status_ != Compile_Okay)    \
-            return status_;             \
+#define CHECK_STATUS(expr)                                           \
+    JS_BEGIN_MACRO                                                   \
+        CompileStatus status_ = (expr);                              \
+        if (status_ != Compile_Okay) {                               \
+            if (oomInVector || masm.oom() || stubcc.masm.oom())      \
+                js_ReportOutOfMemory(cx);                            \
+            return status_;                                          \
+        }                                                            \
     JS_END_MACRO
 
 CompileStatus
@@ -169,8 +170,10 @@ mjit::Compiler::performCompilation(JITScript **jitp)
 
     analysis.analyze(cx, script);
 
-    if (analysis.OOM())
+    if (analysis.OOM()) {
+        js_ReportOutOfMemory(cx);
         return Compile_Error;
+    }
     if (analysis.failed()) {
         JaegerSpew(JSpew_Abort, "couldn't analyze bytecode; probably switchX or OOM\n");
         return Compile_Abort;
@@ -178,12 +181,16 @@ mjit::Compiler::performCompilation(JITScript **jitp)
 
     this->analysis = &analysis;
 
-    if (!frame.init())
-        return Compile_Abort;
+    if (!frame.init()) {
+        js_ReportOutOfMemory(cx);
+        return Compile_Error;
+    }
 
     jumpMap = (Label *)cx->malloc(sizeof(Label) * script->length);
-    if (!jumpMap)
+    if (!jumpMap) {
+        js_ReportOutOfMemory(cx);
         return Compile_Error;
+    }
 #ifdef DEBUG
     for (uint32 i = 0; i < script->length; i++)
         jumpMap[i] = Label();
@@ -401,10 +408,17 @@ mjit::Compiler::finishThisUp(JITScript **jitp)
                        jumpTableOffsets.length() * sizeof(void *);
 
     JSC::ExecutablePool *execPool = getExecPool(script, totalSize);
-    if (!execPool)
-        return Compile_Abort;
+    if (!execPool) {
+        js_ReportOutOfMemory(cx);
+        return Compile_Error;
+    }
 
     uint8 *result = (uint8 *)execPool->alloc(totalSize);
+    if (!result) {
+        execPool->release();
+        js_ReportOutOfMemory(cx);
+        return Compile_Error;
+    }
     JSC::ExecutableAllocator::makeWritable(result, totalSize);
     masm.executableCopy(result);
     stubcc.masm.executableCopy(result + masm.size());
@@ -438,6 +452,7 @@ mjit::Compiler::finishThisUp(JITScript **jitp)
     uint8 *cursor = (uint8 *)cx->calloc(totalBytes);
     if (!cursor) {
         execPool->release();
+        js_ReportOutOfMemory(cx);
         return Compile_Error;
     }
 
@@ -1471,7 +1486,8 @@ mjit::Compiler::generateMethod()
 
             masm.jump(Registers::ReturnReg);
 #else
-            jsop_tableswitch(PC);
+            if (!jsop_tableswitch(PC))
+                return Compile_Error;
 #endif
             PC += js_GetVariableBytecodeLength(PC);
             break;
@@ -4808,7 +4824,7 @@ mjit::Compiler::constructThis()
     return true;
 }
 
-void
+bool
 mjit::Compiler::jsop_tableswitch(jsbytecode *pc)
 {
 #if defined JS_CPU_ARM
@@ -4832,7 +4848,7 @@ mjit::Compiler::jsop_tableswitch(jsbytecode *pc)
      */
     if (numJumps == 0) {
         frame.pop();
-        return;
+        return true;
     }
 
     FrameEntry *fe = frame.peek(-1);
@@ -4844,7 +4860,7 @@ mjit::Compiler::jsop_tableswitch(jsbytecode *pc)
         INLINE_STUBCALL(stubs::TableSwitch);
         frame.pop();
         masm.jump(Registers::ReturnReg);
-        return;
+        return true;
     }
 
     RegisterID dataReg;
@@ -4890,7 +4906,7 @@ mjit::Compiler::jsop_tableswitch(jsbytecode *pc)
         stubcc.masm.jump(Registers::ReturnReg);
     }
     frame.pop();
-    jumpAndTrace(defaultCase, originalPC + defaultTarget);
+    return jumpAndTrace(defaultCase, originalPC + defaultTarget);
 #endif
 }
 

js/src/methodjit/Compiler.h

@@ -461,7 +461,7 @@ class Compiler : public BaseCompiler
     void leaveBlock();
     void emitEval(uint32 argc);
     void jsop_arguments();
-    void jsop_tableswitch(jsbytecode *pc);
+    bool jsop_tableswitch(jsbytecode *pc);
     void jsop_forprop(JSAtom *atom);
     void jsop_forname(JSAtom *atom);
     void jsop_forgname(JSAtom *atom);