Bug 1760276 [Wayland] When we check for popup resize workaround, consider toplevel window instead of parent one, r=emilio

Differential Revision: https://phabricator.services.mozilla.com/D141600

widget/gtk/nsWindow.cpp

@@ -2006,36 +2006,31 @@ void nsWindow::WaylandPopupSetDirectPosition() {
   }
 }
 
-bool nsWindow::WaylandPopupFitsParentWindow(const GdkRectangle& aSize) {
-  GtkWindow* parentGtkWindow = gtk_window_get_transient_for(GTK_WINDOW(mShell));
-  nsWindow* parentWindow =
-      get_window_for_gtk_widget(GTK_WIDGET(parentGtkWindow));
+bool nsWindow::WaylandPopupFitsToplevelWindow() {
+  LOG("nsWindow::WaylandPopupFitsToplevelWindow()");
 
-  // Don't try to fiddle with popup position when our parent is also popup.
-  // Use layout setup in such case.
-  if (parentWindow->IsPopup()) {
+  GtkWindow* parent = gtk_window_get_transient_for(GTK_WINDOW(mShell));
+  GtkWindow* tmp = parent;
+  while (tmp = gtk_window_get_transient_for(GTK_WINDOW(parent))) {
+    parent = tmp;
+  }
+  GdkWindow* toplevelGdkWindow = gtk_widget_get_window(GTK_WIDGET(parent));
+
+  if (!toplevelGdkWindow) {
+    NS_WARNING("Toplevel widget without GdkWindow?");
     return false;
   }
 
-  // Use MozContainer to get visible area without decorations.
-  GdkWindow* parentGdkWindow =
-      gtk_widget_get_window(GTK_WIDGET(parentWindow->GetMozContainer()));
+  int parentWidth = gdk_window_get_width(toplevelGdkWindow);
+  int parentHeight = gdk_window_get_height(toplevelGdkWindow);
+  LOG("  parent size %d x %d", parentWidth, parentHeight);
 
-  // x,y are offsets of mozcontainer, i.e. size of CSD decorations size.
-  int x, y;
-  gdk_window_get_position(parentGdkWindow, &x, &y);
+  GdkRectangle rect = DevicePixelsToGdkRectRoundOut(mBounds);
+  int fits = rect.x >= 0 && rect.y >= 0 && rect.x + rect.width <= parentWidth &&
+             rect.y + rect.height <= parentHeight;
 
-  // We use parent mozcontainer size plus left/top CSD decorations sizes as
-  // this coordinates are used by mBounds.
-  int parentWidth = gdk_window_get_width(parentGdkWindow) + x;
-  int parentHeight = gdk_window_get_height(parentGdkWindow) + y;
-  int popupWidth = aSize.width;
-  int popupHeight = aSize.height;
-
-  auto popupBounds = DevicePixelsToGdkRectRoundOut(mBounds);
-  return popupBounds.x >= 0 && popupBounds.y >= 0 &&
-         popupBounds.x + popupWidth <= parentWidth &&
-         popupBounds.y + popupHeight <= parentHeight;
+  LOG("  fits %d", fits);
+  return fits;
 }
 
 void nsWindow::NativeMoveResizeWaylandPopup(bool aMove, bool aResize) {
@@ -2089,7 +2084,7 @@ void nsWindow::NativeMoveResizeWaylandPopup(bool aMove, bool aResize) {
     gtk_window_resize(GTK_WINDOW(mShell), rect.width, rect.height);
   }
 
-  if (!aMove && WaylandPopupFitsParentWindow(rect)) {
+  if (!aMove && WaylandPopupFitsToplevelWindow()) {
     // Popup position has not been changed and its position/size fits
     // parent window so no need to reposition the window.
     LOG("  fits parent window size, just resize\n");
@@ -2362,6 +2357,8 @@ void nsWindow::WaylandPopupMove() {
     return;
   }
 
+  // See https://gitlab.gnome.org/GNOME/gtk/-/issues/1986
+  // We're likely fail to reposition already visible widget.
   if (gtk_widget_is_visible(mShell)) {
     NS_WARNING(
         "Positioning visible popup under Wayland, position may be wrong!");

widget/gtk/nsWindow.h

@@ -793,7 +793,7 @@ class nsWindow final : public nsBaseWidget {
   void WaylandPopupMarkAsClosed();
   void WaylandPopupRemoveClosedPopups();
   void WaylandPopupSetDirectPosition();
-  bool WaylandPopupFitsParentWindow(const GdkRectangle& aSize);
+  bool WaylandPopupFitsToplevelWindow();
   const WaylandPopupMoveToRectParams WaylandPopupGetPositionFromLayout();
   nsWindow* WaylandPopupFindLast(nsWindow* aPopup);
   GtkWindow* GetCurrentTopmostWindow();
@@ -807,9 +807,11 @@ class nsWindow final : public nsBaseWidget {
 
   // mPopupPosition is the original popup position from layout, set by
   // nsWindow::Move() or nsWindow::Resize().
+  // Popup position is relative to main (toplevel) window.
   GdkPoint mPopupPosition{};
 
-  // mRelativePopupPosition is popup position calculated against parent window.
+  // mRelativePopupPosition is popup position calculated against
+  // recent popup parent window.
   GdkPoint mRelativePopupPosition{};
 
   // Toplevel window (first element) of linked list of Wayland popups. It's null