Bug 734196 - Updating the private pointer should not recurse when marking; r=billm

The private pointer is the only pointer update during GC marking that can invoke
unrelated code.  If this code triggers a write barrier, then this will recurse.
This patch adds a way to update private pointers without triggering a barrier,
explicitly for use by the GC during marking.

js/src/jit-test/tests/basic/bug734196.js

@@ -0,0 +1,4 @@
+var x = new ArrayBuffer(2);
+gczeal(4);
+actual = [].concat(x).toString();
+var count2 = countHeap();

js/src/jsapi.cpp

@@ -4271,7 +4271,7 @@ prop_iter_trace(JSTracer *trc, JSObject *obj)
          */
         Shape *tmp = (Shape *)pdata;
         MarkShapeUnbarriered(trc, &tmp, "prop iter shape");
-        JS_ASSERT(tmp == pdata);
+        obj->setPrivateUnbarriered(tmp);
     } else {
         /* Non-native case: mark each id in the JSIdArray private. */
         JSIdArray *ida = (JSIdArray *) pdata;

js/src/jsobj.h

@@ -774,6 +774,7 @@ struct JSObject : public js::ObjectImpl
     inline bool hasPrivate() const;
     inline void *getPrivate() const;
     inline void setPrivate(void *data);
+    inline void setPrivateUnbarriered(void *data);
     inline void initPrivate(void *data);
 
     /* Access private data for an object with a known number of fixed slots. */

js/src/jsobjinlines.h

@@ -121,6 +121,13 @@ JSObject::setPrivate(void *data)
     privateWriteBarrierPost(pprivate);
 }
 
+inline void
+JSObject::setPrivateUnbarriered(void *data)
+{
+    void **pprivate = &privateRef(numFixedSlots());
+    *pprivate = data;
+}
+
 inline void
 JSObject::initPrivate(void *data)
 {

js/src/jstypedarray.cpp

@@ -332,7 +332,7 @@ ArrayBuffer::obj_trace(JSTracer *trc, JSObject *obj)
     JSObject *delegate = static_cast<JSObject*>(obj->getPrivate());
     if (delegate) {
         MarkObjectUnbarriered(trc, &delegate, "arraybuffer.delegate");
-        obj->setPrivate(delegate);
+        obj->setPrivateUnbarriered(delegate);
     }
 }
 

js/src/vm/Debugger.cpp

@@ -3214,7 +3214,7 @@ DebuggerObject_trace(JSTracer *trc, JSObject *obj)
          */
         if (JSObject *referent = (JSObject *) obj->getPrivate()) {
             MarkObjectUnbarriered(trc, &referent, "Debugger.Object referent");
-            obj->setPrivate(referent);
+            obj->setPrivateUnbarriered(referent);
         }
     }
 }
@@ -3858,7 +3858,7 @@ DebuggerEnv_trace(JSTracer *trc, JSObject *obj)
          */
         if (Env *referent = (JSObject *) obj->getPrivate()) {
             MarkObjectUnbarriered(trc, &referent, "Debugger.Environment referent");
-            obj->setPrivate(referent);
+            obj->setPrivateUnbarriered(referent);
         }
     }
 }