Bug 1300904 - Reset the ObjectElements Frozen flag on OOM. r=ekleog

--HG--
extra : rebase_source : 881e72c188977dfc1656e5f132de5671459095f5

js/src/jit-test/tests/basic/bug1300904.js

@@ -0,0 +1,7 @@
+if (!('oomTest' in this))
+    quit();
+Object.getOwnPropertyNames(this);
+oomTest(function() {
+    this[0] = null;
+    Object.freeze(this);
+});

js/src/jsobj.cpp

@@ -2622,8 +2622,15 @@ js::PreventExtensions(JSContext* cx, HandleObject obj, ObjectOpResult& result, I
         }
     }
 
-    if (!obj->setFlags(cx, BaseShape::NOT_EXTENSIBLE, JSObject::GENERATE_SHAPE))
+    if (!obj->setFlags(cx, BaseShape::NOT_EXTENSIBLE, JSObject::GENERATE_SHAPE)) {
+        // We failed to mark the object non-extensible, so reset the frozen
+        // flag on the elements.
+        MOZ_ASSERT(obj->nonProxyIsExtensible());
+        if (obj->isNative() && obj->as<NativeObject>().getElementsHeader()->isFrozen())
+            obj->as<NativeObject>().getElementsHeader()->markNotFrozen();
         return false;
+    }
+
     return result.succeed();
 }
 

js/src/vm/NativeObject.h

@@ -296,8 +296,14 @@ class ObjectElements
     }
     void freeze() {
         MOZ_ASSERT(!isFrozen());
+        MOZ_ASSERT(!isCopyOnWrite());
         flags |= FROZEN;
     }
+    void markNotFrozen() {
+        MOZ_ASSERT(isFrozen());
+        MOZ_ASSERT(!isCopyOnWrite());
+        flags &= ~FROZEN;
+    }
 
     // This is enough slots to store an object of this class. See the static
     // assertion below.