Bug 837835 - Reserve VM space at startup and free it when a crash happens, in the hope that writing a minidump will then succeed. r=ted

toolkit/crashreporter/nsExceptionHandler.cpp

@@ -714,15 +714,44 @@ bool MinidumpCallback(
 }
 
 #ifdef XP_WIN
+static void* gBreakpadReservedVM;
+
+/**
+ * Reserve some VM space. In the event that we crash because VM space is
+ * being leaked without leaking memory, freeing this space before taking
+ * the minidump will allow us to collect a minidump.
+ */
+static const SIZE_T kReserveSize = 0xc00000; // 12 MB
+
+static void
+ReserveBreakpadVM()
+{
+  if (!gBreakpadReservedVM) {
+    gBreakpadReservedVM = VirtualAlloc(NULL, kReserveSize, MEM_RESERVE, 0);
+  }
+}
+
+static void
+FreeBreakpadVM()
+{
+  if (gBreakpadReservedVM) {
+    VirtualFree(gBreakpadReservedVM, kReserveSize, MEM_RELEASE);
+  }
+}
+
 /**
  * Filters out floating point exceptions which are handled by nsSigHandlers.cpp
  * and should not be handled as crashes.
+ *
+ * Also calls FreeBreakpadVM if appropriate.
  */
 static bool FPEFilter(void* context, EXCEPTION_POINTERS* exinfo,
                       MDRawAssertionInfo* assertion)
 {
-  if (!exinfo)
+  if (!exinfo) {
+    FreeBreakpadVM();
     return true;
+  }
 
   PEXCEPTION_RECORD e = (PEXCEPTION_RECORD)exinfo->ExceptionRecord;
   switch (e->ExceptionCode) {
@@ -737,6 +766,7 @@ static bool FPEFilter(void* context, EXCEPTION_POINTERS* exinfo,
     case STATUS_FLOAT_MULTIPLE_TRAPS:
       return false; // Don't write minidump, continue exception search
   }
+  FreeBreakpadVM();
   return true;
 }
 #endif // XP_WIN
@@ -901,6 +931,8 @@ nsresult SetExceptionHandler(nsIFile* aXREDirectory,
 #endif
 
 #ifdef XP_WIN32
+  ReserveBreakpadVM();
+
   MINIDUMP_TYPE minidump_type = MiniDumpNormal;
 
   // Try to determine what version of dbghelp.dll we're using.