Bug 1590550 - Don't apply contain: paint inappropriately. r=mattwoodrow

This fixes css/css-contain/contain-paint-{002,012,024}.html when not using the
fast path (i.e., with the following patch).

Also invert the check in IsStackingContext as IsFrameOfType is a virtual method,
and IsContain* is just a bitflag.

Differential Revision: https://phabricator.services.mozilla.com/D51804

--HG--
extra : moz-landing-system : lando

layout/generic/nsFrame.cpp

@@ -10889,9 +10889,9 @@ bool nsIFrame::IsStackingContext(const nsStyleDisplay* aStyleDisplay,
                                  bool aIsPositioned) {
   return HasOpacity(aStyleDisplay, aStyleEffects, nullptr) ||
          IsTransformed(aStyleDisplay) ||
-         (IsFrameOfType(eSupportsContainLayoutAndPaint) &&
-          (aStyleDisplay->IsContainPaint() ||
-           aStyleDisplay->IsContainLayout())) ||
+         ((aStyleDisplay->IsContainPaint() ||
+           aStyleDisplay->IsContainLayout()) &&
+          IsFrameOfType(eSupportsContainLayoutAndPaint)) ||
          // strictly speaking, 'perspective' doesn't require visual atomicity,
          // but the spec says it acts like the rest of these
          ChildrenHavePerspective(aStyleDisplay) ||

layout/generic/nsFrame.h

@@ -615,6 +615,7 @@ class nsFrame : public nsBox {
    */
   static bool ShouldApplyOverflowClipping(const nsIFrame* aFrame,
                                           const nsStyleDisplay* aDisp) {
+    MOZ_ASSERT(aDisp == aFrame->StyleDisplay(), "Wong display struct");
     // clip overflow:-moz-hidden-unscrollable, except for nsListControlFrame,
     // which is an nsHTMLScrollFrame.
     if (MOZ_UNLIKELY(aDisp->mOverflowX ==
@@ -628,7 +629,8 @@ class nsFrame : public nsBox {
     // clipping, because the scrollable frame will already clip overflowing
     // content, and because contain:paint should prevent all means of escaping
     // that clipping (e.g. because it forms a fixed-pos containing block).
-    if (aDisp->IsContainPaint() && !aFrame->IsScrollFrame()) {
+    if (aDisp->IsContainPaint() && !aFrame->IsScrollFrame() &&
+        aFrame->IsFrameOfType(eSupportsContainLayoutAndPaint)) {
       return true;
     }