Bug 1264613 - Allow object-to-nonobject serialization. r=baku

Fix the Structured Clone API to avoid assigning a numeric back-reference
id to objects that are serialized as non-objects; in particular, this
fixes incorrect serialization and crashes upon deserialization when a
DedicatedWorkerGlobalScope is serialized twice in the same serialization
packet.

dom/base/Console.cpp

@@ -546,6 +546,10 @@ protected:
       return true;
     }
 
+    if (!JS_ObjectNotWritten(aWriter, aObj)) {
+      return false;
+    }
+
     JS::Rooted<JS::Value> value(aCx, JS::ObjectOrNullValue(aObj));
     JS::Rooted<JSString*> jsString(aCx, JS::ToString(aCx, value));
     if (NS_WARN_IF(!jsString)) {

js/public/StructuredClone.h

@@ -281,4 +281,7 @@ JS_WriteString(JSStructuredCloneWriter* w, JS::HandleString str);
 JS_PUBLIC_API(bool)
 JS_WriteTypedArray(JSStructuredCloneWriter* w, JS::HandleValue v);
 
+JS_PUBLIC_API(bool)
+JS_ObjectNotWritten(JSStructuredCloneWriter* w, JS::HandleObject obj);
+
 #endif  /* js_StructuredClone_h */

js/src/vm/StructuredClone.cpp

@@ -356,6 +356,7 @@ struct JSStructuredCloneWriter {
 
     friend bool JS_WriteString(JSStructuredCloneWriter* w, HandleString str);
     friend bool JS_WriteTypedArray(JSStructuredCloneWriter* w, HandleValue v);
+    friend bool JS_ObjectNotWritten(JSStructuredCloneWriter* w, HandleObject obj);
 };
 
 JS_FRIEND_API(uint64_t)
@@ -1763,7 +1764,7 @@ JSStructuredCloneReader::startRead(MutableHandleValue vp)
       }
 
       case SCTAG_BACK_REFERENCE_OBJECT: {
-        if (data >= allObjs.length()) {
+        if (data >= allObjs.length() || !allObjs[data].isObject()) {
             JS_ReportErrorNumber(context(), GetErrorMessage, nullptr,
                                  JSMSG_SC_BAD_SERIALIZED_DATA,
                                  "invalid back reference in input");
@@ -2422,3 +2423,11 @@ JS_WriteTypedArray(JSStructuredCloneWriter* w, HandleValue v)
     RootedObject obj(w->context(), &v.toObject());
     return w->writeTypedArray(obj);
 }
+
+JS_PUBLIC_API(bool)
+JS_ObjectNotWritten(JSStructuredCloneWriter* w, HandleObject obj)
+{
+    w->memory.remove(w->memory.lookup(obj));
+
+    return true;
+}