Bug 1127777 - Split ARM simulator construction into two phases to handle OOM. r=jandem

js/src/jit/arm/Simulator-arm.cpp

@@ -363,7 +363,7 @@ Simulator::Create()
     if (!sim)
         return nullptr;
 
-    if (!sim->icache_.init()) {
+    if (!sim->init()) {
         js_delete(sim);
         return false;
     }
@@ -1031,16 +1031,11 @@ Simulator::Simulator()
     // Set up simulator support first. Some of this information is needed to
     // setup the architecture state.
 
-    // Allocate 2MB for the stack. Note that we will only use 1MB, see below.
+    // Note, allocation and anything that depends on allocated memory is
-    static const size_t stackSize = 2 * 1024*1024;
+    // deferred until init(), in order to handle OOM properly.
-    stack_ = reinterpret_cast<char*>(js_malloc(stackSize));
+
-    if (!stack_) {
+    stack_ = nullptr;
-        MOZ_ReportAssertionFailure("[unhandlable oom] Simulator stack", __FILE__, __LINE__);
+    stackLimit_ = 0;
-        MOZ_CRASH();
-    }
-    // Leave a safety margin of 1MB to prevent overrunning the stack when
-    // pushing values (total stack size is 2MB).
-    stackLimit_ = reinterpret_cast<uintptr_t>(stack_) + 1024 * 1024;
     pc_modified_ = false;
     icount_ = 0L;
     resume_pc_ = 0;
@@ -1077,11 +1072,6 @@ Simulator::Simulator()
     underflow_vfp_flag_ = false;
     inexact_vfp_flag_ = false;
 
-    // The sp is initialized to point to the bottom (high address) of the
-    // allocated stack area. To be safe in potential stack underflows we leave
-    // some buffer below.
-    registers_[sp] = reinterpret_cast<int32_t>(stack_) + stackSize - 64;
-
     // The lr and pc are initialized to a known bad value that will cause an
     // access violation if the simulator ever tries to execute it.
     registers_[pc] = bad_lr;
@@ -1092,6 +1082,30 @@ Simulator::Simulator()
     redirection_ = nullptr;
 }
 
+bool
+Simulator::init()
+{
+    if (!icache_.init())
+        return false;
+
+    // Allocate 2MB for the stack. Note that we will only use 1MB, see below.
+    static const size_t stackSize = 2 * 1024*1024;
+    stack_ = reinterpret_cast<char*>(js_malloc(stackSize));
+    if (!stack_)
+        return false;
+
+    // Leave a safety margin of 1MB to prevent overrunning the stack when
+    // pushing values (total stack size is 2MB).
+    stackLimit_ = reinterpret_cast<uintptr_t>(stack_) + 1024 * 1024;
+
+    // The sp is initialized to point to the bottom (high address) of the
+    // allocated stack area. To be safe in potential stack underflows we leave
+    // some buffer below.
+    registers_[sp] = reinterpret_cast<int32_t>(stack_) + stackSize - 64;
+
+    return true;
+}
+
 // When the generated code calls a VM function (masm.callWithABI) we need to
 // call that function instead of trying to execute it with the simulator
 // (because it's x86 code instead of arm code). We do that by redirecting the VM

js/src/jit/arm/Simulator-arm.h

@@ -94,7 +94,9 @@ class Simulator
         num_q_registers = 16
     };
 
+    // Returns nullptr on OOM.
     static Simulator *Create();
+
     static void Destroy(Simulator *simulator);
 
     // Constructor/destructor are for internal use only; use the static methods above.
@@ -193,6 +195,8 @@ class Simulator
         end_sim_pc = -2
     };
 
+    bool init();
+
     // Checks if the current instruction should be executed based on its
     // condition bits.
     inline bool conditionallyExecute(SimInstruction* instr);