Bug 1173521 - Properly handle unapplying 3D projective transforms to rectangles in APZ code. r=botond

The untransformation is done using the newly added UntransformTo() functions
which call Matrix4x4::ProjectRectBounds(), which returns an empty rectangle
if the result is not at least partially on the positive side of the w = 0 plane.

UntransformTo() returnis the transformed rectangle if it's not empty, and
Nothing() otherwise, making callers check for this case.

The patch also adds some assertions to places where we apply transforms we
know should be 2D (and thus didn't switch to use UntransformTo()), checking
that the transforms are in fact 2D.

--HG--
extra : rebase_source : bbc226f4135c2cf229c4e00dfb017a6c9c078391
extra : source : c2732f62a3b9a5246801aa000ce572ee07f53481

gfx/layers/client/ClientTiledPaintedLayer.cpp

@@ -55,10 +55,10 @@ ClientTiledPaintedLayer::FillSpecificAttributes(SpecificLayerAttributes& aAttrs)
   aAttrs = PaintedLayerAttributes(GetValidRegion());
 }
 
-static LayerRect
-ApplyParentLayerToLayerTransform(const gfx::Matrix4x4& aTransform, const ParentLayerRect& aParentLayerRect)
+static Maybe<LayerRect>
+ApplyParentLayerToLayerTransform(const gfx::Matrix4x4& aTransform, const ParentLayerRect& aParentLayerRect, const LayerRect& aClip)
 {
-  return TransformTo<LayerPixel>(aTransform, aParentLayerRect);
+  return UntransformTo<LayerPixel>(aTransform, aParentLayerRect, aClip);
 }
 
 static gfx::Matrix4x4
@@ -124,10 +124,7 @@ ClientTiledPaintedLayer::GetAncestorLayers(LayerMetricsWrapper* aOutScrollAncest
 void
 ClientTiledPaintedLayer::BeginPaint()
 {
-  mPaintData.mLowPrecisionPaintCount = 0;
-  mPaintData.mPaintFinished = false;
-  mPaintData.mCompositionBounds.SetEmpty();
-  mPaintData.mCriticalDisplayPort.SetEmpty();
+  mPaintData.ResetPaintData();
 
   if (!GetBaseTransform().Is2D()) {
     // Give up if there is a complex CSS transform on the layer. We might
@@ -165,6 +162,8 @@ ClientTiledPaintedLayer::BeginPaint()
     GetTransformToAncestorsParentLayer(this, displayPortAncestor);
   transformDisplayPortToLayer.Invert();
 
+  LayerRect layerBounds = ViewAs<LayerPixel>(Rect(GetLayerBounds()));
+
   // Compute the critical display port that applies to this layer in the
   // LayoutDevice space of this layer, but only if there is no OMT animation
   // on this layer. If there is an OMT animation then we need to draw the whole
@@ -175,8 +174,13 @@ ClientTiledPaintedLayer::BeginPaint()
     ParentLayerRect criticalDisplayPort =
       (displayportMetrics.GetCriticalDisplayPort() * displayportMetrics.GetZoom())
       + displayportMetrics.GetCompositionBounds().TopLeft();
-    mPaintData.mCriticalDisplayPort = RoundedToInt(
-      ApplyParentLayerToLayerTransform(transformDisplayPortToLayer, criticalDisplayPort));
+    Maybe<LayerRect> criticalDisplayPortTransformed =
+      ApplyParentLayerToLayerTransform(transformDisplayPortToLayer, criticalDisplayPort, layerBounds);
+    if (!criticalDisplayPortTransformed) {
+      mPaintData.ResetPaintData();
+      return;
+    }
+    mPaintData.mCriticalDisplayPort = RoundedToInt(*criticalDisplayPortTransformed);
   }
   TILING_LOG("TILING %p: Critical displayport %s\n", this, Stringify(mPaintData.mCriticalDisplayPort).c_str());
 
@@ -190,8 +194,13 @@ ClientTiledPaintedLayer::BeginPaint()
     GetTransformToAncestorsParentLayer(this, scrollAncestor);
   gfx::Matrix4x4 transformToBounds = mPaintData.mTransformToCompBounds;
   transformToBounds.Invert();
-  mPaintData.mCompositionBounds = ApplyParentLayerToLayerTransform(
-    transformToBounds, scrollMetrics.GetCompositionBounds());
+  Maybe<LayerRect> compositionBoundsTransformed = ApplyParentLayerToLayerTransform(
+    transformToBounds, scrollMetrics.GetCompositionBounds(), layerBounds);
+  if (!compositionBoundsTransformed) {
+    mPaintData.ResetPaintData();
+    return;
+  }
+  mPaintData.mCompositionBounds = *compositionBoundsTransformed;
   TILING_LOG("TILING %p: Composition bounds %s\n", this, Stringify(mPaintData.mCompositionBounds).c_str());
 
   // Calculate the scroll offset since the last transaction

gfx/layers/client/TiledContentClient.cpp

@@ -1398,14 +1398,16 @@ ClientMultiTiledLayerBuffer::ValidateTile(TileClient& aTile,
  * transformed into the painted layer's LayerPixel coordinates, accounting
  * for the compositor state.
  */
-static LayerRect
+static Maybe<LayerRect>
 GetCompositorSideCompositionBounds(const LayerMetricsWrapper& aScrollAncestor,
                                    const Matrix4x4& aTransformToCompBounds,
-                                   const ViewTransform& aAPZTransform)
+                                   const ViewTransform& aAPZTransform,
+                                   const LayerRect& aClip)
 {
   Matrix4x4 transform = aTransformToCompBounds * Matrix4x4(aAPZTransform);
-  return TransformTo<LayerPixel>(transform.Inverse(),
-            aScrollAncestor.Metrics().GetCompositionBounds());
+
+  return UntransformTo<LayerPixel>(transform.Inverse(),
+    aScrollAncestor.Metrics().GetCompositionBounds(), aClip);
 }
 
 bool
@@ -1484,12 +1486,18 @@ ClientMultiTiledLayerBuffer::ComputeProgressiveUpdateRegion(const nsIntRegion& a
     }
   }
 
-  LayerRect transformedCompositionBounds =
+  Maybe<LayerRect> transformedCompositionBounds =
     GetCompositorSideCompositionBounds(scrollAncestor,
                                        aPaintData->mTransformToCompBounds,
-                                       viewTransform);
+                                       viewTransform,
+                                       ViewAs<LayerPixel>(Rect(mPaintedLayer->GetLayerBounds())));
 
-  TILING_LOG("TILING %p: Progressive update transformed compositor bounds %s\n", mPaintedLayer, Stringify(transformedCompositionBounds).c_str());
+  if (!transformedCompositionBounds) {
+    aPaintData->mPaintFinished = true;
+    return false;
+  }
+
+  TILING_LOG("TILING %p: Progressive update transformed compositor bounds %s\n", mPaintedLayer, Stringify(*transformedCompositionBounds).c_str());
 
   // Compute a "coherent update rect" that we should paint all at once in a
   // single transaction. This is to avoid rendering glitches on animated
@@ -1502,9 +1510,9 @@ ClientMultiTiledLayerBuffer::ComputeProgressiveUpdateRegion(const nsIntRegion& a
   // the browser, so we always use the entire user-visible area.
   IntRect coherentUpdateRect(LayerIntRect::ToUntyped(RoundedOut(
 #ifdef MOZ_WIDGET_ANDROID
-    transformedCompositionBounds.Intersect(aPaintData->mCompositionBounds)
+    transformedCompositionBounds->Intersect(aPaintData->mCompositionBounds)
 #else
-    transformedCompositionBounds
+    *transformedCompositionBounds
 #endif
   )));
 
@@ -1679,5 +1687,14 @@ TiledContentClient::Dump(std::stringstream& aStream,
   GetTiledBuffer()->Dump(aStream, aPrefix, aDumpHtml);
 }
 
+void
+BasicTiledLayerPaintData::ResetPaintData()
+{
+  mLowPrecisionPaintCount = 0;
+  mPaintFinished = false;
+  mCompositionBounds.SetEmpty();
+  mCriticalDisplayPort.SetEmpty();
+}
+
 } // namespace layers
 } // namespace mozilla

gfx/layers/client/TiledContentClient.h

@@ -352,6 +352,11 @@ struct BasicTiledLayerPaintData {
    * progressively.
    */
   bool mPaintFinished : 1;
+
+  /*
+   * Initializes/clears data to prepare for paint action.
+   */
+  void ResetPaintData();
 };
 
 class SharedFrameMetricsHelper

gfx/layers/composite/AsyncCompositionManager.cpp

@@ -158,6 +158,7 @@ static void
 TransformClipRect(Layer* aLayer,
                   const Matrix4x4& aTransform)
 {
+  MOZ_ASSERT(aTransform.Is2D());
   const Maybe<ParentLayerIntRect>& clipRect = aLayer->AsLayerComposite()->GetShadowClipRect();
   if (clipRect) {
     ParentLayerIntRect transformed = TransformTo<ParentLayerPixel>(aTransform, *clipRect);
@@ -663,6 +664,7 @@ AsyncCompositionManager::ApplyAsyncContentTransformToTree(Layer *aLayer)
     // using containerful scrolling, then the clip is not part of the scrolled
     // frame and should not be transformed.
     if (asyncClip && !metrics.UsesContainerScrolling()) {
+      MOZ_ASSERT(asyncTransform.Is2D());
       asyncClip = Some(TransformTo<ParentLayerPixel>(asyncTransform, *asyncClip));
     }
 

layout/base/UnitTransforms.h

@@ -180,6 +180,34 @@ static Maybe<gfx::IntPointTyped<TargetUnits>> UntransformTo(const gfx::Matrix4x4
   }
   return Some(RoundedToInt(ViewAs<TargetUnits>(point.As2DPoint())));
 }
+
+// The versions of UntransformTo() that take a rectangle also take a clip,
+// which represents the bounds within which the target must fall. The
+// result of the transform is intersected with this clip, and is considered
+// meaningful if the intersection is not empty.
+template <typename TargetUnits, typename SourceUnits>
+static Maybe<gfx::RectTyped<TargetUnits>> UntransformTo(const gfx::Matrix4x4& aTransform,
+                                                const gfx::RectTyped<SourceUnits>& aRect,
+                                                const gfx::RectTyped<TargetUnits>& aClip)
+{
+  gfx::Rect rect = aTransform.ProjectRectBounds(aRect.ToUnknownRect(), aClip.ToUnknownRect());
+  if (rect.IsEmpty()) {
+    return Nothing();
+  }
+  return Some(ViewAs<TargetUnits>(rect));
+}
+template <typename TargetUnits, typename SourceUnits>
+static Maybe<gfx::IntRectTyped<TargetUnits>> UntransformTo(const gfx::Matrix4x4& aTransform,
+                                                const gfx::IntRectTyped<SourceUnits>& aRect,
+                                                const gfx::IntRectTyped<TargetUnits>& aClip)
+{
+  gfx::Rect rect = aTransform.ProjectRectBounds(aRect.ToUnknownRect(), aClip.ToUnknownRect());
+  if (rect.IsEmpty()) {
+    return Nothing();
+  }
+  return Some(RoundedToInt(ViewAs<TargetUnits>(rect)));
+}
+
 template <typename TargetUnits, typename SourceUnits>
 static Maybe<gfx::PointTyped<TargetUnits>> UntransformVector(const gfx::Matrix4x4& aTransform,
                                                     const gfx::PointTyped<SourceUnits>& aVector,