Bug 1241078 - Only treat items as backface hidden if they are 3D-transformed. r=tlee

Add helper function nsIFrame::In3DContextAndBackfaceIsHidden() which
checks both if a frame is backface-hidden and whether it is within a
3D-transform context.

In FrameLayerBuilder, check this function rather than BackfaceIsHidden()
to determine whether a frame needs a backface-hidden layer. This will
avoid creating unnecessary extra layers for non-3d-transformed items
which for some reason have backface-hidden set.

layout/base/FrameLayerBuilder.cpp

@@ -3519,7 +3519,7 @@ ContainerState::NewPaintedLayerData(nsDisplayItem* aItem,
   data.mAnimatedGeometryRootOffset = aTopLeft;
   data.mReferenceFrame = aItem->ReferenceFrame();
   data.mSingleItemFixedToViewport = aShouldFixToViewport;
-  data.mBackfaceHidden = aItem->Frame()->BackfaceIsHidden();
+  data.mBackfaceHidden = aItem->Frame()->In3DContextAndBackfaceIsHidden();
   data.mIsCaret = aItem->GetType() == nsDisplayItem::TYPE_CARET;
 
   data.mNewChildLayersIndex = mNewChildLayers.Length();
@@ -4138,7 +4138,7 @@ ContainerState::ProcessDisplayItems(nsDisplayList* aList)
       PaintedLayerData* paintedLayerData =
         mPaintedLayerDataTree.FindPaintedLayerFor(animatedGeometryRoot, agrScrollClip,
                                                   itemVisibleRect, false,
-                                                  item->Frame()->BackfaceIsHidden(),
+                                                  item->Frame()->In3DContextAndBackfaceIsHidden(),
                                                   avoidCreatingLayer,
                                                   [&]() {
           layerCount++;

layout/generic/nsFrame.cpp

@@ -1164,6 +1164,12 @@ nsIFrame::Combines3DTransformWithAncestors() const
   return StyleDisplay()->HasTransform(this) || StyleDisplay()->BackfaceIsHidden();
 }
 
+bool
+nsIFrame::In3DContextAndBackfaceIsHidden() const
+{
+  return Combines3DTransformWithAncestors() && StyleDisplay()->BackfaceIsHidden();
+}
+
 bool
 nsIFrame::HasPerspective() const
 {

layout/generic/nsIFrame.h

@@ -1276,6 +1276,13 @@ public:
    */
   bool Combines3DTransformWithAncestors() const;
 
+  /**
+   * Returns whether this frame has a hidden backface and has a parent that
+   * Extend3DContext(). This is useful because in some cases the hidden
+   * backface can safely be ignored if it could not be visible anyway.
+   */
+  bool In3DContextAndBackfaceIsHidden() const;
+
   bool IsPreserve3DLeaf() const {
     return Combines3DTransformWithAncestors() && !Extend3DContext();
   }