Bug 1781747 - Don't use shadow class for non-menupopup popup windows. r=Jamie,handyman

These don't get styles early enough to make a shadow decision before Show().
This also matches the previous behavior (nothing would set
nsWidgetInitData::mDropShadow for these windows), so this is the less risky
fix.

It seems somewhat sketchy to use a popup window for these to begin with (Linux
for example maps them to a toplevel Window, at least on Wayland...), but let's
not change too much right now.

The hbrBackground change is a no-op because we can't have a brush before
Create() is called, we update it in SetBackgroundColor().

Differential Revision: https://phabricator.services.mozilla.com/D153095

layout/xul/nsMenuPopupFrame.cpp

@@ -329,6 +329,7 @@ nsresult nsMenuPopupFrame::CreateWidgetForView(nsView* aView) {
   nsWidgetInitData widgetData;
   widgetData.mWindowType = eWindowType_popup;
   widgetData.mBorderStyle = eBorderStyle_default;
+  widgetData.mForMenupopupFrame = true;
   widgetData.mClipSiblings = true;
   widgetData.mPopupHint = mPopupType;
   widgetData.mNoAutoHide = IsNoAutoHide();

widget/nsWidgetInitData.h

@@ -97,6 +97,7 @@ struct nsWidgetInitData {
   // when painting exclude area occupied by child windows and sibling windows
   bool mClipChildren = false;
   bool mClipSiblings = false;
+  bool mForMenupopupFrame = false;
   bool mRTL = false;
   bool mNoAutoHide = false;   // true for noautohide panels
   bool mIsDragPopup = false;  // true for drag feedback panels

widget/windows/nsWindow.cpp

@@ -979,6 +979,7 @@ nsresult nsWindow::Create(nsIWidget* aParent, nsNativeWidget aNativeParent,
   }
 
   mIsRTL = aInitData->mRTL;
+  mForMenupopupFrame = aInitData->mForMenupopupFrame;
   mOpeningAnimationSuppressed = aInitData->mIsAnimationSuppressed;
   mAlwaysOnTop = aInitData->mAlwaysOnTop;
   mResizable = aInitData->mResizable;
@@ -1019,7 +1020,8 @@ nsresult nsWindow::Create(nsIWidget* aParent, nsNativeWidget aNativeParent,
     }
   }
 
-  const wchar_t* className = GetWindowClass();
+  const wchar_t* className = ChooseWindowClass(mWindowType, mForMenupopupFrame);
+
   if (aInitData->mWindowType == eWindowType_toplevel && !aParent &&
       !sFirstTopLevelWindowCreated) {
     sFirstTopLevelWindowCreated = true;
@@ -1314,9 +1316,9 @@ void nsWindow::Destroy() {
  *
  **************************************************************/
 
+/* static */
 const wchar_t* nsWindow::RegisterWindowClass(const wchar_t* aClassName,
-                                             UINT aExtraStyle,
-                                             LPWSTR aIconID) const {
+                                             UINT aExtraStyle, LPWSTR aIconID) {
   WNDCLASSW wc;
   if (::GetClassInfoW(nsToolkit::mDllInstance, aClassName, &wc)) {
     // already registered
@@ -1331,7 +1333,7 @@ const wchar_t* nsWindow::RegisterWindowClass(const wchar_t* aClassName,
   wc.hIcon =
       aIconID ? ::LoadIconW(::GetModuleHandleW(nullptr), aIconID) : nullptr;
   wc.hCursor = nullptr;
-  wc.hbrBackground = mBrush;
+  wc.hbrBackground = nullptr;
   wc.lpszMenuName = nullptr;
   wc.lpszClassName = aClassName;
 
@@ -1346,16 +1348,21 @@ const wchar_t* nsWindow::RegisterWindowClass(const wchar_t* aClassName,
 
 static LPWSTR const gStockApplicationIcon = MAKEINTRESOURCEW(32512);
 
-// Return the proper window class for everything except popups.
-const wchar_t* nsWindow::GetWindowClass() const {
-  switch (mWindowType) {
+/* static */
+const wchar_t* nsWindow::ChooseWindowClass(nsWindowType aWindowType,
+                                           bool aForMenupopupFrame) {
+  MOZ_ASSERT_IF(aForMenupopupFrame, aWindowType == eWindowType_popup);
+  switch (aWindowType) {
     case eWindowType_invisible:
       return RegisterWindowClass(kClassNameHidden, 0, gStockApplicationIcon);
     case eWindowType_dialog:
       return RegisterWindowClass(kClassNameDialog, 0, 0);
     case eWindowType_popup:
-      return RegisterWindowClass(kClassNameDropShadow, CS_DROPSHADOW,
-                                 gStockApplicationIcon);
+      if (aForMenupopupFrame) {
+        return RegisterWindowClass(kClassNameDropShadow, CS_DROPSHADOW,
+                                   gStockApplicationIcon);
+      }
+      [[fallthrough]];
     default:
       return RegisterWindowClass(GetMainWindowClass(), 0,
                                  gStockApplicationIcon);
@@ -1674,7 +1681,9 @@ void nsWindow::Show(bool bState) {
 #endif  // defined(ACCESSIBILITY)
   }
 
-  if (mWindowType == eWindowType_popup) {
+  if (mForMenupopupFrame) {
+    MOZ_ASSERT(ChooseWindowClass(mWindowType, mForMenupopupFrame) ==
+               kClassNameDropShadow);
     const bool shouldUseDropShadow = [&] {
       if (mTransparencyMode == eTransparencyTransparent) {
         return false;
@@ -3751,9 +3760,10 @@ void* nsWindow::GetNativeData(uint32_t aDataType) {
   switch (aDataType) {
     case NS_NATIVE_TMP_WINDOW:
       return (void*)::CreateWindowExW(
-          mIsRTL ? WS_EX_LAYOUTRTL : 0, GetWindowClass(), L"", WS_CHILD,
-          CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, mWnd,
-          nullptr, nsToolkit::mDllInstance, nullptr);
+          mIsRTL ? WS_EX_LAYOUTRTL : 0,
+          ChooseWindowClass(mWindowType, /* aForMenupopupFrame = */ false), L"",
+          WS_CHILD, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT,
+          mWnd, nullptr, nsToolkit::mDllInstance, nullptr);
     case NS_NATIVE_WIDGET:
     case NS_NATIVE_WINDOW:
     case NS_NATIVE_WINDOW_WEBRTC_DEVICE_ID:

widget/windows/nsWindow.h

@@ -604,14 +604,14 @@ class nsWindow final : public nsBaseWidget {
   void UserActivity();
 
   int32_t GetHeight(int32_t aProposedHeight);
-  const wchar_t* GetWindowClass() const;
-  const wchar_t* GetWindowPopupClass() const;
+
   DWORD WindowStyle();
   DWORD WindowExStyle();
 
+  static const wchar_t* ChooseWindowClass(nsWindowType, bool aForMenupopupFrame);
   // This method registers the given window class, and returns the class name.
-  const wchar_t* RegisterWindowClass(const wchar_t* aClassName,
-                                     UINT aExtraStyle, LPWSTR aIconID) const;
+  static const wchar_t* RegisterWindowClass(const wchar_t* aClassName,
+                                            UINT aExtraStyle, LPWSTR aIconID);
 
   /**
    * XP and Vista theming support for windows with rounded edges
@@ -768,6 +768,7 @@ class nsWindow final : public nsBaseWidget {
   bool mIsEarlyBlankWindow = false;
   bool mIsShowingPreXULSkeletonUI = false;
   bool mResizable = false;
+  bool mForMenupopupFrame = false;
   DWORD_PTR mOldStyle = 0;
   DWORD_PTR mOldExStyle = 0;
   nsNativeDragTarget* mNativeDragTarget = nullptr;