Bug 1469716 allow transparency mode to be set appropriately according to whether drawing a titlebar r=stransky

The early return for non-popup windows was to workaround
https://bugzilla.mozilla.org/show_bug.cgi?id=1344839 which involved problems
with CleanLayerManagerRecursive().  In cases where layer manager configuration
does not change, there is no need to clean the layer manager and so no need to
return early.

Differential Revision: https://phabricator.services.mozilla.com/D40145

--HG--
extra : moz-landing-system : lando

widget/gtk/nsWindow.cpp

@@ -3700,14 +3700,6 @@ nsresult nsWindow::Create(nsIWidget* aParent, nsNativeWidget aNativeParent,
         }
       }
 
-      // We have a toplevel window with transparency. Mark it as transparent
-      // now as nsWindow::SetTransparencyMode() can't be called after
-      // nsWindow is created (Bug 1344839).
-      if (mWindowType == eWindowType_toplevel &&
-          (mHasAlphaVisual || mTransparencyBitmapForTitlebar)) {
-        mIsTransparent = true;
-      }
-
       // We only move a general managed toplevel window if someone has
       // actually placed the window somewhere.  If no placement has taken
       // place, we just let the window manager Do The Right Thing.
@@ -4441,7 +4433,12 @@ void nsWindow::SetTransparencyMode(nsTransparencyMode aMode) {
 
   if (mIsTransparent == isTransparent) {
     return;
-  } else if (mWindowType != eWindowType_popup) {
+  }
+  if (mWindowType != eWindowType_popup && !mHasAlphaVisual &&
+      !mTransparencyBitmapForTitlebar) {
+    // https://bugzilla.mozilla.org/show_bug.cgi?id=1344839 reported
+    // problems cleaning the layer manager for toplevel windows, so ignore
+    // the request to workaround that.
     NS_WARNING("Cannot set transparency mode on non-popup windows.");
     return;
   }
@@ -4453,9 +4450,12 @@ void nsWindow::SetTransparencyMode(nsTransparencyMode aMode) {
 
   mIsTransparent = isTransparent;
 
-  // Need to clean our LayerManager up while still alive because
-  // we don't want to use layers acceleration on shaped windows
-  CleanLayerManagerRecursive();
+  if (!mHasAlphaVisual && !mTransparencyBitmapForTitlebar) {
+    // The choice of layer manager depends on
+    // GtkCompositorWidgetInitData::Shaped(), which will need to change, so
+    // clean out the old layer manager.
+    CleanLayerManagerRecursive();
+  }
 }
 
 nsTransparencyMode nsWindow::GetTransparencyMode() {