Bug 733493 - Improve JS shell OOM testing code, r=jorendorff

js/public/Utility.h

@@ -45,6 +45,11 @@
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef JS_OOM_DO_BACKTRACES
+#include <stdio.h>
+#include <execinfo.h>
+#endif
+
 #include "jstypes.h"
 
 #ifdef __cplusplus
@@ -112,10 +117,43 @@ extern JS_PUBLIC_API(void) JS_Abort(void);
  */
 extern JS_PUBLIC_DATA(uint32_t) OOM_maxAllocations; /* set from shell/js.cpp */
 extern JS_PUBLIC_DATA(uint32_t) OOM_counter; /* data race, who cares. */
+
+#define JS_OOM_BACKTRACE_SIZE 32
+static JS_ALWAYS_INLINE void
+PrintBacktrace()
+{
+    void* OOM_trace[JS_OOM_BACKTRACE_SIZE];
+    char** OOM_traceSymbols = NULL;
+    int32_t OOM_traceSize = 0;
+    int32_t OOM_traceIdx = 0;
+    OOM_traceSize = backtrace(OOM_trace, JS_OOM_BACKTRACE_SIZE);
+    OOM_traceSymbols = backtrace_symbols(OOM_trace, OOM_traceSize);
+    
+    if (!OOM_traceSymbols)
+        return;
+
+    for (OOM_traceIdx = 0; OOM_traceIdx < OOM_traceSize; ++OOM_traceIdx) {
+        fprintf(stderr, "#%d %s\n", OOM_traceIdx, OOM_traceSymbols[OOM_traceIdx]);
+    }
+
+    free(OOM_traceSymbols);
+}
+
+#ifdef JS_OOM_DO_BACKTRACES
+#define JS_OOM_EMIT_BACKTRACE() \
+    do {\
+        fprintf(stderr, "Forcing artificial memory allocation function failure:\n");\
+	PrintBacktrace();\
+    } while (0)
+# else
+#  define JS_OOM_EMIT_BACKTRACE() do {} while(0)
+#endif /* JS_OOM_DO_BACKTRACES */
+
 #  define JS_OOM_POSSIBLY_FAIL() \
     do \
     { \
         if (++OOM_counter > OOM_maxAllocations) { \
+            JS_OOM_EMIT_BACKTRACE();\
             return NULL; \
         } \
     } while (0)

js/src/configure.in

@@ -4527,6 +4527,18 @@ if test -n "$JS_MORE_DETERMINISTIC"; then
     AC_DEFINE(JS_MORE_DETERMINISTIC)
 fi
 
+dnl ========================================================
+dnl Enable output of backtraces on artificial OOMs
+dnl ========================================================
+MOZ_ARG_ENABLE_BOOL(oom-backtrace,
+[  --enable-oom-backtrace
+                          Enable output of backtraces on artificial OOMs (-A)],
+    JS_OOM_DO_BACKTRACES=1,
+    JS_OOM_DO_BACKTRACES= )
+if test -n "$JS_OOM_DO_BACKTRACES"; then
+    AC_DEFINE(JS_OOM_DO_BACKTRACES)
+fi
+
 dnl ======================================================
 dnl = Enable compiling with ccache
 dnl ======================================================

js/src/ds/LifoAlloc.h

@@ -221,6 +221,8 @@ class LifoAlloc
 
     JS_ALWAYS_INLINE
     void *alloc(size_t n) {
+        JS_OOM_POSSIBLY_FAIL();
+
         void *result;
         if (latest && (result = latest->tryAlloc(n)))
             return result;

js/src/jsgcinlines.h

@@ -407,6 +407,9 @@ NewGCThing(JSContext *cx, js::gc::AllocKind kind, size_t thingSize)
     JS_ASSERT(!cx->runtime->gcRunning);
     JS_ASSERT(!cx->runtime->noGCOrAllocationCheck);
 
+    /* For testing out of memory conditions */
+    JS_OOM_POSSIBLY_FAIL();
+
 #ifdef JS_GC_ZEAL
     if (cx->runtime->needZealousGC())
         js::gc::RunDebugGC(cx);