Bug 1236801: Don't check for simulated OOM in a way that invalidates AddPtrs for no discernable reason. (Revised to fix uninitialized var, r=sfink) r=jonco

--HG--
extra : rebase_source : 23222e831fb516bc3cc2672b4c6807564d56bce1

js/public/HashTable.h

@@ -1657,7 +1657,7 @@ class HashTable : private AllocPolicy
             RebuildStatus status = checkOverloaded();
             if (status == RehashFailed)
                 return false;
-            if (!this->checkSimulatedOOM())
+            if (status == NotOverloaded && !this->checkSimulatedOOM())
                 return false;
             if (status == Rehashed)
                 p.entry_ = &findFreeEntry(p.keyHash);

js/src/jsapi-tests/testHashTable.cpp

@@ -3,6 +3,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "js/HashTable.h"
+#include "js/Utility.h"
 #include "jsapi-tests/tests.h"
 
 //#define FUZZ
@@ -343,3 +344,51 @@ BEGIN_TEST(testHashSetOfMoveOnlyType)
     return true;
 }
 END_TEST(testHashSetOfMoveOnlyType)
+
+#if defined(DEBUG)
+
+// Add entries to a HashMap using lookupWithDefault until either we get an OOM,
+// or the table has been resized a few times.
+static bool
+LookupWithDefaultUntilResize() {
+    IntMap m;
+
+    if (!m.init())
+        return false;
+
+    // Add entries until we've resized the table four times.
+    size_t lastCapacity = m.capacity();
+    size_t resizes = 0;
+    uint32_t key = 0;
+    while (resizes < 4) {
+        if (!m.lookupWithDefault(key++, 0))
+            return false;
+
+        size_t capacity = m.capacity();
+        if (capacity != lastCapacity) {
+            resizes++;
+            lastCapacity = capacity;
+        }
+    }
+
+    return true;
+}
+
+BEGIN_TEST(testHashMapLookupWithDefaultOOM)
+{
+    js::oom::targetThread = js::oom::THREAD_TYPE_MAIN;
+    uint32_t timeToFail;
+    for (timeToFail = 1; timeToFail < 1000; timeToFail++) {
+        OOM_maxAllocations = OOM_counter + timeToFail;
+        OOM_failAlways = false;
+
+        LookupWithDefaultUntilResize();
+    }
+
+    js::oom::targetThread = js::oom::THREAD_TYPE_NONE;
+
+    return true;
+}
+
+END_TEST(testHashMapLookupWithDefaultOOM)
+#endif // defined(DEBUG)