Bug 523769 - Don't call into plugin hooks (specifically NPObject.deallocate) while mid-GC, r=jst+mrbkap

js/src/xpconnect/idl/nsIJSRuntimeService.idl

@@ -41,11 +41,20 @@
 #include "nsISupports.idl"
 
 [ptr] native JSRuntime(JSRuntime);
+native JSGCCallback(JSGCCallback);
+
 interface nsIXPCScriptable;
 
-[uuid(e7d09265-4c23-4028-b1b0-c99e02aa78f8)]
+[uuid(364bcec3-7034-4a4e-bff5-b3f796ca9771)]
 interface nsIJSRuntimeService : nsISupports
 {
     readonly attribute JSRuntime        runtime;
     readonly attribute nsIXPCScriptable backstagePass;
+
+    /**
+     * Register additional GC callback which will run after the
+     * standard XPConnect callback.
+     */
+    [noscript, notxpcom] void registerGCCallback(in JSGCCallback func);
+    [noscript, notxpcom] void unregisterGCCallback(in JSGCCallback func);
 };

js/src/xpconnect/src/nsXPConnect.cpp

@@ -2546,6 +2546,20 @@ nsXPConnect::GetBackstagePass(nsIXPCScriptable **bsp)
     return NS_OK;
 }
 
+/* [noscript, notxpcom] void registerGCCallback(in JSGCCallback func); */
+NS_IMETHODIMP_(void)
+nsXPConnect::RegisterGCCallback(JSGCCallback func)
+{
+    mRuntime->AddGCCallback(func);
+}
+
+/* [noscript, notxpcom] void unregisterGCCallback(in JSGCCallback func); */
+NS_IMETHODIMP_(void)
+nsXPConnect::UnregisterGCCallback(JSGCCallback func)
+{
+    mRuntime->RemoveGCCallback(func);
+}
+
 //  nsIJSContextStack and nsIThreadJSContextStack implementations
 
 /* readonly attribute PRInt32 Count; */

js/src/xpconnect/src/xpcjsruntime.cpp

@@ -767,6 +767,12 @@ JSBool XPCJSRuntime::GCCallback(JSContext *cx, JSGCStatus status)
         }
     }
 
+    nsTArray<JSGCCallback> callbacks(self->extraGCCallbacks);
+    for (PRInt32 i = 0; i < callbacks.Length(); ++i) {
+        if (!callbacks[i](cx, status))
+            return JS_FALSE;
+    }
+
     return JS_TRUE;
 }
 
@@ -1320,3 +1326,20 @@ XPCRootSetElem::RemoveFromRootSet(JSRuntime* rt)
     mNext = nsnull;
 #endif
 }
+
+void
+XPCJSRuntime::AddGCCallback(JSGCCallback cb)
+{
+    NS_ASSERTION(cb, "null callback");
+    extraGCCallbacks.AppendElement(cb);
+}
+
+void
+XPCJSRuntime::RemoveGCCallback(JSGCCallback cb)
+{
+    NS_ASSERTION(cb, "null callback");
+    PRBool found = extraGCCallbacks.RemoveElement(cb);
+    if (!found) {
+        NS_ERROR("Removing a callback which was never added.");
+    }
+}

js/src/xpconnect/src/xpcprivate.h

@@ -757,6 +757,9 @@ private:
 public:
 #endif
 
+    void AddGCCallback(JSGCCallback cb);
+    void RemoveGCCallback(JSGCCallback cb);
+
 private:
     XPCJSRuntime(); // no implementation
     XPCJSRuntime(nsXPConnect* aXPConnect);
@@ -794,6 +797,7 @@ private:
     uintN mUnrootedGlobalCount;
     PRCondVar *mWatchdogWakeup;
     PRThread *mWatchdogThread;
+    nsTArray<JSGCCallback> extraGCCallbacks;
 };
 
 /***************************************************************************/

modules/plugin/base/src/nsJSNPRuntime.cpp

@@ -78,6 +78,7 @@ static JSRuntime *sJSRuntime;
 // while executing JS on the context.
 static nsIJSContextStack *sContextStack;
 
+static nsTArray<NPObject*>* sDelayedReleases;
 
 // Helper class that reports any JS exceptions that were thrown while
 // the plugin executed JS.
@@ -197,6 +198,27 @@ static JSClass sNPObjectMemberClass =
     nsnull, nsnull, nsnull, NPObjectMember_Mark, nsnull
   };
 
+static void
+OnWrapperDestroyed();
+
+static JSBool
+DelayedReleaseGCCallback(JSContext* cx, JSGCStatus status)
+{
+  if (JSGC_END == status) {
+    if (sDelayedReleases) {
+      for (PRInt32 i = 0; i < sDelayedReleases->Length(); ++i) {
+        NPObject* obj = (*sDelayedReleases)[i];
+        if (obj)
+          _releaseobject(obj);
+        OnWrapperDestroyed();
+      }
+      delete sDelayedReleases;
+      sDelayedReleases = NULL;
+    }
+  }
+  return JS_TRUE;
+}
+
 static void
 OnWrapperCreated()
 {
@@ -209,6 +231,10 @@ OnWrapperCreated()
     rtsvc->GetRuntime(&sJSRuntime);
     NS_ASSERTION(sJSRuntime != nsnull, "no JSRuntime?!");
 
+    // Register our GC callback to perform delayed destruction of finalized
+    // NPObjects. Leave this callback around and don't ever unregister it.
+    rtsvc->RegisterGCCallback(DelayedReleaseGCCallback);
+
     CallGetService("@mozilla.org/js/xpc/ContextStack;1", &sContextStack);
   }
 }
@@ -1628,17 +1654,15 @@ static void
 NPObjWrapper_Finalize(JSContext *cx, JSObject *obj)
 {
   NPObject *npobj = (NPObject *)::JS_GetPrivate(cx, obj);
-
   if (npobj) {
     if (sNPObjWrappers.ops) {
       PL_DHashTableOperate(&sNPObjWrappers, npobj, PL_DHASH_REMOVE);
     }
-
-    // Let go of our NPObject
-    _releaseobject(npobj);
   }
 
-  OnWrapperDestroyed();
+  if (!sDelayedReleases)
+    sDelayedReleases = new nsTArray<NPObject*>;
+  sDelayedReleases->AppendElement(npobj);
 }
 
 static JSBool