Bug 1107775 - Make deferred finalization more re-entrant safe, r=mccr8

--HG--
extra : rebase_source : 74e300e96b50b470dbc89cdbbcfb9dbc7b75e629

xpcom/base/CycleCollectedJSRuntime.cpp

@@ -57,6 +57,7 @@
 #include "mozilla/CycleCollectedJSRuntime.h"
 #include <algorithm>
 #include "mozilla/ArrayUtils.h"
+#include "mozilla/AutoRestore.h"
 #include "mozilla/MemoryReporting.h"
 #include "mozilla/dom/BindingUtils.h"
 #include "mozilla/dom/DOMJSClass.h"
@@ -96,6 +97,7 @@ class IncrementalFinalizeRunnable : public nsRunnable
   nsTArray<nsISupports*> mSupports;
   DeferredFinalizeArray mDeferredFinalizeFunctions;
   uint32_t mFinalizeFunctionToRun;
+  bool mReleasing;
 
   static const PRTime SliceMillis = 10; /* ms */
 
@@ -1124,6 +1126,7 @@ IncrementalFinalizeRunnable::IncrementalFinalizeRunnable(CycleCollectedJSRuntime
                                                          DeferredFinalizerTable& aFinalizers)
   : mRuntime(aRt)
   , mFinalizeFunctionToRun(0)
+  , mReleasing(false)
 {
   this->mSupports.SwapElements(aSupports);
   DeferredFinalizeFunctionHolder* function =
@@ -1143,7 +1146,12 @@ IncrementalFinalizeRunnable::~IncrementalFinalizeRunnable()
 void
 IncrementalFinalizeRunnable::ReleaseNow(bool aLimited)
 {
-  //MOZ_ASSERT(NS_IsMainThread());
+  if (mReleasing) {
+    MOZ_ASSERT(false, "Try to avoid re-entering ReleaseNow!");
+    return;
+  }
+  mozilla::AutoRestore<bool> ar(mReleasing);
+  mReleasing = true;
   MOZ_ASSERT(mDeferredFinalizeFunctions.Length() != 0,
              "We should have at least ReleaseSliceNow to run");
   MOZ_ASSERT(mFinalizeFunctionToRun < mDeferredFinalizeFunctions.Length(),
@@ -1210,7 +1218,21 @@ IncrementalFinalizeRunnable::Run()
 void
 CycleCollectedJSRuntime::FinalizeDeferredThings(DeferredFinalizeType aType)
 {
-  MOZ_ASSERT(!mFinalizeRunnable);
+  /*
+   * If the previous GC created a runnable to finalize objects
+   * incrementally, and if it hasn't finished yet, finish it now. We
+   * don't want these to build up. We also don't want to allow any
+   * existing incremental finalize runnables to run after a
+   * non-incremental GC, since they are often used to detect leaks.
+   */
+  if (mFinalizeRunnable) {
+    mFinalizeRunnable->ReleaseNow(false);
+    if (mFinalizeRunnable) {
+      // If we re-entered ReleaseNow, we couldn't delete mFinalizeRunnable and
+      // we need to just continue processing it.
+      return;
+    }
+  }
   mFinalizeRunnable = new IncrementalFinalizeRunnable(this,
                                                       mDeferredSupports,
                                                       mDeferredFinalizerTable);
@@ -1261,17 +1283,6 @@ CycleCollectedJSRuntime::OnGC(JSGCStatus aStatus)
       }
 #endif
 
-      /*
-       * If the previous GC created a runnable to finalize objects
-       * incrementally, and if it hasn't finished yet, finish it now. We
-       * don't want these to build up. We also don't want to allow any
-       * existing incremental finalize runnables to run after a
-       * non-incremental GC, since they are often used to detect leaks.
-       */
-      if (mFinalizeRunnable) {
-        mFinalizeRunnable->ReleaseNow(false);
-      }
-
       // Do any deferred finalization of native objects.
       FinalizeDeferredThings(JS::WasIncrementalGC(mJSRuntime) ? FinalizeIncrementally :
                                                                 FinalizeNow);