Bug 449129 - abort a plugin process if a plugin attempts to spin an event loop while painting, r=jmathies

dom/plugins/PluginInstanceChild.cpp

@@ -615,6 +615,8 @@ PluginInstanceChild::AnswerNPP_HandleEvent_Shmem(const NPRemoteEvent& event,
     PLUGIN_LOG_DEBUG_FUNCTION;
     AssertPluginThread();
 
+    PaintTracker pt;
+
     NPCocoaEvent evcopy = event.event;
 
     if (evcopy.type == NPCocoaEventDrawRect) {
@@ -686,6 +688,8 @@ PluginInstanceChild::AnswerNPP_HandleEvent_IOSurface(const NPRemoteEvent& event,
     PLUGIN_LOG_DEBUG_FUNCTION;
     AssertPluginThread();
 
+    PaintTracker pt;
+
     NPCocoaEvent evcopy = event.event;
     nsIOSurface* surf = nsIOSurface::LookupSurface(surfaceid);
     if (!surf) {

dom/plugins/PluginInstanceChild.h

@@ -56,6 +56,7 @@
 #include "ChildTimer.h"
 #include "nsRect.h"
 #include "nsTHashtable.h"
+#include "mozilla/PaintTracker.h"
 
 namespace mozilla {
 namespace plugins {
@@ -100,6 +101,7 @@ protected:
     virtual bool
     AnswerPaint(const NPRemoteEvent& event, int16_t* handled)
     {
+        PaintTracker pt;
         return AnswerNPP_HandleEvent(event, handled);
     }
 

ipc/glue/WindowsMessageLoop.cpp

@@ -47,6 +47,7 @@
 #include "nsIXULAppInfo.h"
 
 #include "mozilla/Mutex.h"
+#include "mozilla/PaintTracker.h"
 
 using mozilla::ipc::SyncChannel;
 using mozilla::ipc::RPCChannel;
@@ -623,6 +624,10 @@ RPCChannel::ProcessNativeEventsInRPCCall()
 void
 RPCChannel::SpinInternalEventLoop()
 {
+  if (mozilla::PaintTracker::IsPainting()) {
+    NS_RUNTIMEABORT("Don't spin an event loop while painting.");
+  }
+
   NS_ASSERTION(mTopFrame && mTopFrame->mSpinNestedEvents,
                "Spinning incorrectly");
 

layout/base/Makefile.in

@@ -58,6 +58,8 @@ XPIDLSRCS	= \
 		nsIStyleSheetService.idl	\
 		$(NULL)
 
+EXPORTS_NAMESPACES = mozilla
+
 EXPORTS		= \
 		FrameLayerBuilder.h \
 		FramePropertyTable.h \
@@ -88,6 +90,10 @@ EXPORTS		= \
 		nsStyleConsts.h \
 		$(NULL)
 
+EXPORTS_mozilla = \
+  PaintTracker.h \
+  $(NULL)
+
 CPPSRCS		= \
 		FrameLayerBuilder.cpp \
 		FramePropertyTable.cpp \
@@ -115,6 +121,7 @@ CPPSRCS		= \
 		nsRefreshDriver.cpp \
 		nsStyleChangeList.cpp \
 		nsStyleSheetService.cpp \
+		PaintTracker.cpp \
 		$(NULL)
 
 ifndef MOZ_XUL

layout/base/PaintTracker.cpp

@@ -0,0 +1,7 @@
+#include "mozilla/PaintTracker.h"
+
+namespace mozilla {
+
+int PaintTracker::gPaintTracker;
+
+} // namespace mozilla

layout/base/PaintTracker.h

@@ -0,0 +1,29 @@
+#ifndef mozilla_PaintTracker_h
+#define mozilla_PaintTracker_h
+
+#include "nscore.h"
+#include "nsDebug.h"
+
+namespace mozilla {
+
+class NS_STACK_CLASS PaintTracker
+{
+public:
+  PaintTracker() {
+    ++gPaintTracker;
+  }
+  ~PaintTracker() {
+    NS_ASSERTION(gPaintTracker > 0, "Mismatched constructor/destructor");
+  }
+
+  static bool IsPainting() {
+    return !!gPaintTracker;
+  }
+
+private:
+  static int gPaintTracker;
+};
+
+} // namespace mozilla
+
+#endif // mozilla_PaintTracker_h