Bug 926012 - Part 1: Clean up __proto__ setting semantics on native objects. (r=Waldo)

js/src/js.msg

@@ -433,3 +433,4 @@ MSG_DEF(JSMSG_NO_EXPORT_NAME,           378, 0, JSEXN_SYNTAXERR, "missing export
 MSG_DEF(JSMSG_DECLARATION_AFTER_EXPORT, 379, 0, JSEXN_SYNTAXERR, "missing declaration after 'export' keyword")
 MSG_DEF(JSMSG_INVALID_PROTOTYPE,        380, 0, JSEXN_TYPEERR, "prototype field is not an object")
 MSG_DEF(JSMSG_TYPEDOBJECT_HANDLE_TO_UNSIZED, 381, 0, JSEXN_TYPEERR, "cannot create a handle to an unsized type")
+MSG_DEF(JSMSG_SETPROTOTYPEOF_FAIL,      382, 0, JSEXN_TYPEERR, "[[SetPrototypeOf]] failed")

js/src/jsapi.cpp

@@ -2374,7 +2374,16 @@ JS_SetPrototype(JSContext *cx, JS::Handle<JSObject*> obj, JS::Handle<JSObject*>
     CHECK_REQUEST(cx);
     assertSameCompartment(cx, obj, proto);
 
-    return SetClassAndProto(cx, obj, obj->getClass(), proto, false);
+    bool succeeded;
+    if (!JSObject::setProto(cx, obj, proto, &succeeded))
+        return false;
+
+    if (!succeeded) {
+        JS_ReportErrorNumber(cx, js_GetErrorMessage, NULL, JSMSG_SETPROTOTYPEOF_FAIL);
+        return false;
+    }
+
+    return true;
 }
 
 JS_PUBLIC_API(JSObject *)

js/src/jsobj.cpp

@@ -1888,9 +1888,10 @@ JSObject::ReserveForTradeGuts(JSContext *cx, JSObject *aArg, JSObject *bArg,
     const Class *bClass = b->getClass();
     Rooted<TaggedProto> aProto(cx, a->getTaggedProto());
     Rooted<TaggedProto> bProto(cx, b->getTaggedProto());
-    if (!SetClassAndProto(cx, a, bClass, bProto, false))
+    bool success;
+    if (!SetClassAndProto(cx, a, bClass, bProto, &success) || !success)
         return false;
-    if (!SetClassAndProto(cx, b, aClass, aProto, false))
+    if (!SetClassAndProto(cx, b, aClass, aProto, &success) || !success)
         return false;
 
     if (a->tenuredSizeOfThis() == b->tenuredSizeOfThis())
@@ -2898,10 +2899,9 @@ static const ClassInitializerOp lazy_prototype_init[JSProto_LIMIT] = {
 
 bool
 js::SetClassAndProto(JSContext *cx, HandleObject obj,
-                     const Class *clasp, Handle<js::TaggedProto> proto, bool checkForCycles)
+                     const Class *clasp, Handle<js::TaggedProto> proto,
+                     bool *succeeded)
 {
-    JS_ASSERT_IF(!checkForCycles, obj.get() != proto.raw());
-
     /*
      * Regenerate shapes for all of the scopes along the old prototype chain,
      * in case any entries were filled by looking up through obj. Stop when a
@@ -2921,6 +2921,7 @@ js::SetClassAndProto(JSContext *cx, HandleObject obj,
      *
      * :XXX: bug 707717 make this code less brittle.
      */
+    *succeeded = false;
     RootedObject oldproto(cx, obj);
     while (oldproto && oldproto->isNative()) {
         if (oldproto->hasSingletonType()) {
@@ -2933,21 +2934,6 @@ js::SetClassAndProto(JSContext *cx, HandleObject obj,
         oldproto = oldproto->getProto();
     }
 
-    if (checkForCycles) {
-        JS_ASSERT(!proto.isLazy());
-        RootedObject obj2(cx);
-        for (obj2 = proto.toObjectOrNull(); obj2; ) {
-            if (obj2 == obj) {
-                JS_ReportErrorNumber(cx, js_GetErrorMessage, nullptr, JSMSG_CYCLIC_VALUE,
-                                     js_proto_str);
-                return false;
-            }
-
-            if (!JSObject::getProto(cx, obj2, &obj2))
-                return false;
-        }
-    }
-
     if (obj->hasSingletonType()) {
         /*
          * Just splice the prototype, but mark the properties as unknown for
@@ -2956,6 +2942,7 @@ js::SetClassAndProto(JSContext *cx, HandleObject obj,
         if (!obj->splicePrototype(cx, clasp, proto))
             return false;
         MarkTypeObjectUnknownProperties(cx, obj->type());
+        *succeeded = true;
         return true;
     }
 
@@ -2981,6 +2968,7 @@ js::SetClassAndProto(JSContext *cx, HandleObject obj,
     MarkTypeObjectUnknownProperties(cx, type, true);
 
     obj->setType(type);
+    *succeeded = true;
     return true;
 }
 

js/src/jsobj.h

@@ -478,6 +478,9 @@ class JSObject : public js::ObjectImpl
     }
     static inline bool getProto(JSContext *cx, js::HandleObject obj,
                                 js::MutableHandleObject protop);
+    // Returns false on error, success of operation in outparam.
+    static inline bool setProto(JSContext *cx, JS::HandleObject obj,
+                                JS::HandleObject proto, bool *succeeded);
 
     // uninlinedSetType() is the same as setType(), but not inlined.
     inline void setType(js::types::TypeObject *newType);
@@ -1600,7 +1603,7 @@ GetClassPrototypePure(GlobalObject *global, JSProtoKey protoKey);
 
 extern bool
 SetClassAndProto(JSContext *cx, HandleObject obj,
-                 const Class *clasp, Handle<TaggedProto> proto, bool checkForCycles);
+                 const Class *clasp, Handle<TaggedProto> proto, bool *succeeded);
 
 extern JSObject *
 NonNullObject(JSContext *cx, const Value &v);

js/src/jsobjinlines.h

@@ -21,6 +21,8 @@
 #include "jsgcinlines.h"
 #include "jsinferinlines.h"
 
+#include "vm/ObjectImpl-inl.h"
+
 /* static */ inline bool
 JSObject::setGenericAttributes(JSContext *cx, js::HandleObject obj,
                                js::HandleId id, unsigned *attrsp)
@@ -408,6 +410,33 @@ JSObject::getProto(JSContext *cx, js::HandleObject obj, js::MutableHandleObject
     }
 }
 
+/* static */ inline bool
+JSObject::setProto(JSContext *cx, JS::HandleObject obj, JS::HandleObject proto, bool *succeeded)
+{
+    /* ES6 9.1.2 step 5 forbids changing [[Prototype]] if not [[Extensible]]. */
+    bool extensible;
+    if (!JSObject::isExtensible(cx, obj, &extensible))
+        return false;
+    if (!extensible) {
+        *succeeded = false;
+        return true;
+    }
+
+    /* ES6 9.1.2 step 6 forbids generating cyclical prototype chains. */
+    js::RootedObject obj2(cx);
+    for (obj2 = proto; obj2; ) {
+        if (obj2 == obj) {
+            *succeeded = false;
+            return true;
+        }
+
+        if (!JSObject::getProto(cx, obj2, &obj2))
+            return false;
+    }
+
+    return SetClassAndProto(cx, obj, obj->getClass(), proto, succeeded);
+}
+
 inline bool JSObject::isVarObj()
 {
     if (is<js::DebugScopeObject>())

js/src/vm/GlobalObject.cpp

@@ -137,15 +137,6 @@ ProtoSetterImpl(JSContext *cx, CallArgs args)
 
     Rooted<JSObject*> obj(cx, &args.thisv().toObject());
 
-    /* ES5 8.6.2 forbids changing [[Prototype]] if not [[Extensible]]. */
-    bool extensible;
-    if (!JSObject::isExtensible(cx, obj, &extensible))
-        return false;
-    if (!extensible) {
-        obj->reportNotExtensible(cx);
-        return false;
-    }
-
     /*
      * Disallow mutating the [[Prototype]] of a proxy that wasn't simply
      * wrapping some other object.  Also disallow it on ArrayBuffer objects,
@@ -173,9 +164,15 @@ ProtoSetterImpl(JSContext *cx, CallArgs args)
     if (!CheckAccess(cx, obj, nid, JSAccessMode(JSACC_PROTO | JSACC_WRITE), &v, &dummy))
         return false;
 
-    if (!SetClassAndProto(cx, obj, obj->getClass(), newProto, true))
+    bool success;
+    if (!JSObject::setProto(cx, obj, newProto, &success))
         return false;
 
+    if (!success) {
+        JS_ReportErrorNumber(cx, js_GetErrorMessage, NULL, JSMSG_SETPROTOTYPEOF_FAIL);
+        return false;
+    }
+
     args.rval().setUndefined();
     return true;
 }