Bug 611689 - Mask ClearType settings from flash to prevent paint buffer alpha corruption. r=roc, a=betaN

dom/plugins/PluginInstanceChild.cpp

@@ -161,6 +161,7 @@ PluginInstanceChild::PluginInstanceChild(const NPPluginFuncs* aPluginIface)
 #endif // OS_WIN
 #if defined(OS_WIN)
     InitPopupMenuHook();
+    HookSystemParametersInfo();
 #endif // OS_WIN
 #ifdef MOZ_X11
     // Maemo flash can render plugin with any provided rectangle and not require this quirk.
@@ -1210,6 +1211,47 @@ PluginInstanceChild::PluginWindowProc(HWND hWnd,
     return res;
 }
 
+/* system parameters info hook for flash */
+
+typedef BOOL (WINAPI *User32SystemParametersInfoW)(UINT uiAction,
+                                                   UINT uiParam,
+                                                   PVOID pvParam,
+                                                   UINT fWinIni);
+
+static User32SystemParametersInfoW sUser32SystemParametersInfoWStub = NULL;
+
+static BOOL WINAPI User32SystemParametersInfoHook(UINT uiAction,
+                                                  UINT uiParam,
+                                                  PVOID pvParam,
+                                                  UINT fWinIni)
+{
+  if (!sUser32SystemParametersInfoWStub) {
+      NS_NOTREACHED("sUser32SystemParametersInfoWStub not set??");
+      return FALSE;
+  }
+
+  // Tell them cleartype is disabled, so they don't mess with
+  // the alpha channel in our buffers.
+  if (uiAction == SPI_GETFONTSMOOTHINGTYPE && pvParam) {
+      *((UINT*)(pvParam)) = FE_FONTSMOOTHINGSTANDARD;
+      return TRUE;
+  }
+
+  return sUser32SystemParametersInfoWStub(uiAction, uiParam, pvParam, fWinIni);
+}
+
+void
+PluginInstanceChild::HookSystemParametersInfo()
+{
+    if (!(GetQuirks() & PluginModuleChild::QUIRK_FLASH_MASK_CLEARTYPE_SETTINGS))
+        return;
+    if (sUser32SystemParametersInfoWStub)
+        return;
+    sUser32Intercept.Init("gdi32.dll");
+    sUser32Intercept.AddHook("SystemParametersInfoW", User32SystemParametersInfoHook,
+                             (void**) &sUser32SystemParametersInfoWStub);
+}
+
 /* set window long ptr hook for flash */
 
 /*

dom/plugins/PluginInstanceChild.h

@@ -279,6 +279,7 @@ private:
                                           int nIndex,
                                           LONG newLong);
 #endif
+    void HookSystemParametersInfo();
 
     class FlashThrottleAsyncMsg : public ChildAsyncCall
     {

dom/plugins/PluginModuleChild.cpp

@@ -1788,7 +1788,7 @@ PluginModuleChild::AllocPPluginInstance(const nsCString& aMimeType,
     InitQuirksModes(aMimeType);
 
 #ifdef XP_WIN
-    if (mQuirks & QUIRK_FLASH_HOOK_GETWINDOINFO) {
+    if (mQuirks & QUIRK_FLASH_HOOK_GETWINDOWINFO) {
         sUser32Intercept.Init("user32.dll");
         sUser32Intercept.AddHook("GetWindowInfo", PMCGetWindowInfoHook,
                                  (void**) &sGetWindowInfoPtrStub);
@@ -1827,7 +1827,8 @@ PluginModuleChild::InitQuirksModes(const nsCString& aMimeType)
         mQuirks |= QUIRK_WINLESS_TRACKPOPUP_HOOK;
         mQuirks |= QUIRK_FLASH_THROTTLE_WMUSER_EVENTS; 
         mQuirks |= QUIRK_FLASH_HOOK_SETLONGPTR;
-        mQuirks |= QUIRK_FLASH_HOOK_GETWINDOINFO;
+        mQuirks |= QUIRK_FLASH_HOOK_GETWINDOWINFO;
+        mQuirks |= QUIRK_FLASH_MASK_CLEARTYPE_SETTINGS;
     }
 #endif
 }

dom/plugins/PluginModuleChild.h

@@ -234,7 +234,10 @@ public:
         // Win32: Catch get window info calls on the browser and tweak the
         // results so mouse input works when flash is displaying it's settings
         // window.
-        QUIRK_FLASH_HOOK_GETWINDOINFO                   = 1 << 5,
+        QUIRK_FLASH_HOOK_GETWINDOWINFO                  = 1 << 5,
+        // Win: Flash trashes the alpha channel in our buffers when cleartype
+        // is enabled. Mask this setting so they don't know it's enabled.
+        QUIRK_FLASH_MASK_CLEARTYPE_SETTINGS             = 1 << 6,
     };
 
     int GetQuirks() { return mQuirks; }