Bug 1208259 - Handle OOM in the ARM64 VIXL Simulator. r=sstangl

Make the Simulator creation robust against OOM stress tests. Now passes
jsapitests testNewRuntime.

js/src/jit/arm64/vixl/Instrument-vixl.cpp

@@ -136,8 +136,8 @@ Instrument::Instrument(const char* datafile, uint64_t sample_period)
 
   // Construct Counter objects from counter description array.
   for (int i = 0; i < num_counters; i++) {
-    Counter* counter = js_new<Counter>(kCounterList[i].name, kCounterList[i].type);
-    counters_.append(counter);
+    if (Counter* counter = js_new<Counter>(kCounterList[i].name, kCounterList[i].type))
+      counters_.append(counter);
   }
 
   DumpCounterNames();

js/src/jit/arm64/vixl/MozSimulator-vixl.cpp

@@ -37,26 +37,19 @@ namespace vixl {
 using mozilla::DebugOnly;
 using js::jit::ABIFunctionType;
 
-
-Simulator::Simulator() {
-  decoder_ = js_new<Decoder>();
-  if (!decoder_) {
-    MOZ_ReportAssertionFailure("[unhandlable oom] Decoder", __FILE__, __LINE__);
-    MOZ_CRASH();
-  }
-
-  // FIXME: This just leaks the Decoder object for now, which is probably OK.
-  // FIXME: We should free it at some point.
-  // FIXME: Note that it can't be stored in the SimulatorRuntime due to lifetime conflicts.
-  this->init(decoder_, stdout);
+Simulator::Simulator(Decoder* decoder, FILE* stream)
+  : stream_(nullptr)
+  , print_disasm_(nullptr)
+  , instrumentation_(nullptr)
+  , stack_(nullptr)
+  , stack_limit_(nullptr)
+  , decoder_(nullptr)
+  , oom_(false)
+  , lock_(nullptr)
+{
+    this->init(decoder, stream);
 }
 
-
-Simulator::Simulator(Decoder* decoder, FILE* stream) {
-  this->init(decoder, stream);
-}
-
-
 void Simulator::ResetState() {
   // Reset the system registers.
   nzcv_ = SimSystemRegister::DefaultValueFor(NZCV);
@@ -92,6 +85,10 @@ void Simulator::init(Decoder* decoder, FILE* stream) {
 
   stream_ = stream;
   print_disasm_ = js_new<PrintDisassembler>(stream_);
+  if (!print_disasm_) {
+    oom_ = true;
+    return;
+  }
   set_coloured_trace(false);
   trace_parameters_ = LOG_NONE;
 
@@ -99,6 +96,10 @@ void Simulator::init(Decoder* decoder, FILE* stream) {
 
   // Allocate and set up the simulator stack.
   stack_ = (byte*)js_malloc(stack_size_);
+  if (!stack_) {
+    oom_ = true;
+    return;
+  }
   stack_limit_ = stack_ + stack_protection_size_;
   // Configure the starting stack pointer.
   //  - Find the top of the stack.
@@ -111,6 +112,10 @@ void Simulator::init(Decoder* decoder, FILE* stream) {
 
   // Set the sample period to 10, as the VIXL examples and tests are short.
   instrumentation_ = js_new<Instrument>("vixl_stats.csv", 10);
+  if (!instrumentation_) {
+    oom_ = true;
+    return;
+  }
 
   // Print a warning about exclusive-access instructions, but only the first
   // time they are encountered. This warning can be silenced using
@@ -118,8 +123,10 @@ void Simulator::init(Decoder* decoder, FILE* stream) {
   print_exclusive_access_warning_ = true;
 
   lock_ = PR_NewLock();
-  if (!lock_)
-    MOZ_CRASH("Could not allocate simulator lock.");
+  if (!lock_) {
+    oom_ = true;
+    return;
+  }
 #ifdef DEBUG
   lockOwner_ = nullptr;
 #endif
@@ -133,30 +140,24 @@ Simulator* Simulator::Current() {
 
 
 Simulator* Simulator::Create() {
-  Decoder* decoder = js_new<vixl::Decoder>();
-  if (!decoder) {
-    MOZ_ReportAssertionFailure("[unhandlable oom] Decoder", __FILE__, __LINE__);
-    MOZ_CRASH();
-  }
+  Decoder *decoder = js_new<vixl::Decoder>();
+  if (!decoder)
+    return nullptr;
 
   // FIXME: This just leaks the Decoder object for now, which is probably OK.
   // FIXME: We should free it at some point.
   // FIXME: Note that it can't be stored in the SimulatorRuntime due to lifetime conflicts.
-  if (getenv("USE_DEBUGGER") != nullptr) {
-    Debugger* debugger = js_new<Debugger>(decoder, stdout);
-    if (!debugger) {
-      MOZ_ReportAssertionFailure("[unhandlable oom] Decoder", __FILE__, __LINE__);
-      MOZ_CRASH();
-    }
-    return debugger;
-  }
+  Simulator *sim;
+  if (getenv("USE_DEBUGGER") != nullptr)
+    sim = js_new<Debugger>(decoder, stdout);
+  else
+    sim = js_new<Simulator>(decoder, stdout);
 
-  Simulator* sim = js_new<Simulator>();
-  if (!sim) {
-    MOZ_CRASH("NEED SIMULATOR");
+  // Check if Simulator:init ran out of memory.
+  if (sim && sim->oom()) {
+    js_delete(sim);
     return nullptr;
   }
-  sim->init(decoder, stdout);
 
   return sim;
 }

js/src/jit/arm64/vixl/Simulator-vixl.cpp

@@ -70,11 +70,15 @@ SimSystemRegister SimSystemRegister::DefaultValueFor(SystemRegister id) {
 Simulator::~Simulator() {
   js_free(stack_);
   // The decoder may outlive the simulator.
-  decoder_->RemoveVisitor(print_disasm_);
-  js_delete(print_disasm_);
+  if (print_disasm_) {
+    decoder_->RemoveVisitor(print_disasm_);
+    js_delete(print_disasm_);
+  }
 
-  decoder_->RemoveVisitor(instrumentation_);
-  js_delete(instrumentation_);
+  if (instrumentation_) {
+    decoder_->RemoveVisitor(instrumentation_);
+    js_delete(instrumentation_);
+  }
 }
 
 

js/src/jit/arm64/vixl/Simulator-vixl.h

@@ -322,7 +322,6 @@ class Simulator : public DecoderVisitor {
   ~Simulator();
 
   // Moz changes.
-  explicit Simulator();
   void init(Decoder* decoder, FILE* stream);
   static Simulator* Current();
   static Simulator* Create();
@@ -976,6 +975,14 @@ class Simulator : public DecoderVisitor {
   bool print_exclusive_access_warning_;
   void PrintExclusiveAccessWarning();
 
+  // Indicates that the simulator ran out of memory at some point.
+  // Data structures may not be fully allocated.
+  bool oom_;
+
+ public:
+  // True if the simulator ran out of memory during or after construction.
+  bool oom() const { return oom_; }
+
  protected:
   // Moz: Synchronizes access between main thread and compilation threads.
   PRLock* lock_;