Bug 1228033. When not painting to a window (and hence not retained) only have one AGR, the root AGR. r=mattwoodrow

The document structure of when printing is as follows:

ViewportFrame
  nsSimplePageSequenceFrame
    nsPageFrame
      (the content of the page)
    nsPageFrame
      (the content of the page)
    (further pages)

The nsSimplePageSequenceFrame calls nsLayoutUtils::PaintFrame with each nsPageFrame as the frame to paint.

nsPageFrame::BuildDisplayList will build display lists inside sibling nsPageFrames if it needs to descend into those sibling page frames to find placeholder frames for out of flow frames that are on the actual current page frame. Since these are sibling page frames they are not in the original page frame subtree, and hence we can't find an AGR for them that is in the original page frames subtree.

However, when printing we don't retain painted content, so there is no point in having more than one AGR. So if we aren't retaining (IsPaintingToWindow is necessary for that) then just have the root AGR.

This patch also fixes two other bugs.

1) We should always set mAnimatedGeometryRootForChildren for nsDisplayTransform items.

2) If the AGR choosen for a nsDisplayTransform is already the root AGR, then that is it's AGR. Don't try to set the AGR to the parent (which is null).

layout/base/nsDisplayList.cpp

@@ -698,7 +698,7 @@ nsDisplayListBuilder::WrapAGRForFrame(nsIFrame* aAnimatedGeometryRoot,
 
   AnimatedGeometryRoot* result = nullptr;
   if (!mFrameToAnimatedGeometryRootMap.Get(aAnimatedGeometryRoot, &result)) {
-    MOZ_ASSERT(aAnimatedGeometryRoot != RootReferenceFrame());
+    MOZ_ASSERT(nsLayoutUtils::IsAncestorFrameCrossDoc(RootReferenceFrame(), aAnimatedGeometryRoot));
     AnimatedGeometryRoot* parent = aParent;
     if (!parent) {
       nsIFrame* parentFrame = nsLayoutUtils::GetCrossDocParentFrame(aAnimatedGeometryRoot);
@@ -717,7 +717,7 @@ nsDisplayListBuilder::WrapAGRForFrame(nsIFrame* aAnimatedGeometryRoot,
 AnimatedGeometryRoot*
 nsDisplayListBuilder::FindAnimatedGeometryRootFor(nsIFrame* aFrame)
 {
-  if (!IsForPainting()) {
+  if (!IsPaintingToWindow()) {
     return &mRootAGR;
   }
   if (aFrame == mCurrentFrame) {
@@ -1091,8 +1091,16 @@ IsStickyFrameActive(nsDisplayListBuilder* aBuilder, nsIFrame* aFrame, nsIFrame*
 bool
 nsDisplayListBuilder::IsAnimatedGeometryRoot(nsIFrame* aFrame, nsIFrame** aParent)
 {
-  if (aFrame == mReferenceFrame)
+  if (aFrame == mReferenceFrame) {
     return true;
+  }
+  if (!IsPaintingToWindow()) {
+    if (aParent) {
+      *aParent = nsLayoutUtils::GetCrossDocParentFrame(aFrame);
+    }
+    return false;
+  }
+
   if (nsLayoutUtils::IsPopup(aFrame))
     return true;
   if (ActiveLayerTracker::IsOffsetOrMarginStyleAnimated(aFrame))
@@ -1145,6 +1153,7 @@ nsDisplayListBuilder::IsAnimatedGeometryRoot(nsIFrame* aFrame, nsIFrame** aParen
 nsIFrame*
 nsDisplayListBuilder::FindAnimatedGeometryRootFrameFor(nsIFrame* aFrame)
 {
+  MOZ_ASSERT(nsLayoutUtils::IsAncestorFrameCrossDoc(RootReferenceFrame(), aFrame));
   nsIFrame* cursor = aFrame;
   while (cursor != RootReferenceFrame()) {
     nsIFrame* next;
@@ -4883,6 +4892,7 @@ nsDisplayTransform::nsDisplayTransform(nsDisplayListBuilder* aBuilder,
 void
 nsDisplayTransform::SetReferenceFrameToAncestor(nsDisplayListBuilder* aBuilder)
 {
+  mAnimatedGeometryRootForChildren = mAnimatedGeometryRoot;
   if (mFrame == aBuilder->RootReferenceFrame()) {
     return;
   }
@@ -4890,7 +4900,6 @@ nsDisplayTransform::SetReferenceFrameToAncestor(nsDisplayListBuilder* aBuilder)
   mReferenceFrame =
     aBuilder->FindReferenceFrameFor(outerFrame);
   mToReferenceFrame = mFrame->GetOffsetToCrossDoc(mReferenceFrame);
-  mAnimatedGeometryRootForChildren = mAnimatedGeometryRoot;
   if (nsLayoutUtils::IsFixedPosFrameInDisplayPort(mFrame)) {
     // This is an odd special case. If we are both IsFixedPosFrameInDisplayPort
     // and transformed that we are our own AGR parent.
@@ -4898,7 +4907,7 @@ nsDisplayTransform::SetReferenceFrameToAncestor(nsDisplayListBuilder* aBuilder)
     // determine if we are inside a fixed pos subtree. If we use the outer AGR
     // from outside the fixed pos subtree FLB can't tell that we are fixed pos.
     mAnimatedGeometryRoot = mAnimatedGeometryRootForChildren;
-  } else {
+  } else if (mAnimatedGeometryRoot->mParentAGR) {
     mAnimatedGeometryRoot = mAnimatedGeometryRoot->mParentAGR;
   }
   mVisibleRect = aBuilder->GetDirtyRect() + mToReferenceFrame;