diff --git a/gfx/layers/ipc/AsyncPanZoomController.cpp b/gfx/layers/ipc/AsyncPanZoomController.cpp
index 133935084522..f1c91c169398 100644
--- a/gfx/layers/ipc/AsyncPanZoomController.cpp
+++ b/gfx/layers/ipc/AsyncPanZoomController.cpp
@@ -1348,72 +1348,79 @@ void AsyncPanZoomController::ZoomToRect(const gfxRect& aRect) {
     gfx::Rect cssPageRect = mFrameMetrics.mScrollableRect;
     gfx::Point scrollOffset = mFrameMetrics.mScrollOffset;
     gfxSize resolution = CalculateResolution(mFrameMetrics);
+    gfxSize currentZoom = mFrameMetrics.mZoom;
+    float targetZoom;
+    gfxFloat targetResolution;
 
-    // If the rect is empty, treat it as a request to zoom out to the full page
-    // size.
-    if (zoomToRect.IsEmpty()) {
-      // composition bounds in CSS coordinates
-      nsIntRect cssCompositionBounds = compositionBounds;
-      cssCompositionBounds.ScaleInverseRoundIn(resolution.width,
-                                               resolution.height);
-      cssCompositionBounds.MoveBy(scrollOffset.x, scrollOffset.y);
-
-      float y = mFrameMetrics.mScrollOffset.y;
-      float newHeight =
-        cssCompositionBounds.height * cssPageRect.width / cssCompositionBounds.width;
-      float dh = cssCompositionBounds.height - newHeight;
-
-      zoomToRect = gfx::Rect(0.0f,
-                             y + dh/2,
-                             cssPageRect.width,
-                             y + dh/2 + newHeight);
-    }
-
-    gfxFloat targetResolution =
-      std::min(compositionBounds.width / zoomToRect.width,
-             compositionBounds.height / zoomToRect.height);
-
-    // Recalculate the zoom to rect using the new dimensions.
-    zoomToRect.width = compositionBounds.width / targetResolution;
-    zoomToRect.height = compositionBounds.height / targetResolution;
-
-    // Clamp the zoom to rect to the CSS rect to make sure it fits.
-    zoomToRect = zoomToRect.Intersect(cssPageRect);
-
-    // Do one final recalculation to get the resolution.
-    targetResolution = std::max(compositionBounds.width / zoomToRect.width,
-                              compositionBounds.height / zoomToRect.height);
-    float targetZoom = float(targetResolution / resolution.width) * mFrameMetrics.mZoom.width;
-
-    // If current zoom is equal to mMaxZoom,
-    // user still double-tapping it, just zoom-out to the full page size
-    if (mFrameMetrics.mZoom.width == mMaxZoom && targetZoom >= mMaxZoom) {
-      nsIntRect cssCompositionBounds = compositionBounds;
-      cssCompositionBounds.ScaleInverseRoundIn(resolution.width,
-                                               resolution.height);
-      cssCompositionBounds.MoveBy(scrollOffset.x, scrollOffset.y);
-
-      float y = mFrameMetrics.mScrollOffset.y;
-      float newHeight =
-        cssCompositionBounds.height * cssPageRect.width / cssCompositionBounds.width;
-      float dh = cssCompositionBounds.height - newHeight;
-
-      zoomToRect = gfx::Rect(0.0f,
-                             y + dh/2,
-                             cssPageRect.width,
-                             y + dh/2 + newHeight);
+    // The minimum zoom to prevent over-zoom-out.
+    // If the zoom factor is lower than this (i.e. we are zoomed more into the page),
+    // then the CSS content rect, in layers pixels, will be smaller than the
+    // composition bounds. If this happens, we can't fill the target composited
+    // area with this frame.
+    float localMinZoom;
+    gfx::Rect compositedRect = CalculateCompositedRectInCssPixels(mFrameMetrics);
+    localMinZoom =
+      std::max(currentZoom.width / (cssPageRect.width / compositedRect.width),
+               currentZoom.height / (cssPageRect.height / compositedRect.height));
+    localMinZoom = std::max(localMinZoom, mMinZoom);
 
+    if (!zoomToRect.IsEmpty()) {
+      // Intersect the zoom-to-rect to the CSS rect to make sure it fits.
       zoomToRect = zoomToRect.Intersect(cssPageRect);
-      // assign 1 to targetZoom is a shortcut
-      targetZoom = 1;
+      targetResolution =
+        std::min(compositionBounds.width / zoomToRect.width,
+                 compositionBounds.height / zoomToRect.height);
+      targetZoom = float(targetResolution / resolution.width) * currentZoom.width;
+    }
+    // 1. If the rect is empty, request received from browserElementScrolling.js
+    // 2. currentZoom is equal to mMaxZoom and user still double-tapping it
+    // 3. currentZoom is equal to localMinZoom and user still double-tapping it
+    // Treat these three cases as a request to zoom out as much as possible.
+    if (zoomToRect.IsEmpty() ||
+        (currentZoom.width == mMaxZoom && targetZoom >= mMaxZoom) ||
+        (currentZoom.width == localMinZoom && targetZoom <= localMinZoom)) {
+      nsIntRect cssCompositionBounds = compositionBounds;
+      cssCompositionBounds.ScaleInverseRoundIn(resolution.width,
+                                               resolution.height);
+
+      float y = scrollOffset.y;
+      float newHeight =
+        cssCompositionBounds.height * cssPageRect.width / cssCompositionBounds.width;
+      float dh = cssCompositionBounds.height - newHeight;
+
+      zoomToRect = gfx::Rect(0.0f,
+                             y + dh/2,
+                             cssPageRect.width,
+                             newHeight);
+      zoomToRect = zoomToRect.Intersect(cssPageRect);
+      targetResolution =
+        std::min(compositionBounds.width / zoomToRect.width,
+                 compositionBounds.height / zoomToRect.height);
+      targetZoom = float(targetResolution / resolution.width) * currentZoom.width;
     }
 
-    gfxFloat targetFinalZoom = clamped(targetZoom, mMinZoom, mMaxZoom);
-    mEndZoomToMetrics.mZoom = gfxSize(targetFinalZoom, targetFinalZoom);
+    targetZoom = clamped(targetZoom, localMinZoom, mMaxZoom);
+    mEndZoomToMetrics.mZoom = gfxSize(targetZoom, targetZoom);
+
+    // Adjust the zoomToRect to a sensible position to prevent overscrolling.
+    FrameMetrics metricsAfterZoom = mFrameMetrics;
+    metricsAfterZoom.mZoom = mEndZoomToMetrics.mZoom;
+    gfx::Rect rectAfterZoom
+      = CalculateCompositedRectInCssPixels(metricsAfterZoom);
+
+    // If either of these conditions are met, the page will be
+    // overscrolled after zoomed
+    if (zoomToRect.y + rectAfterZoom.height > cssPageRect.height) {
+      zoomToRect.y = cssPageRect.height - rectAfterZoom.height;
+      zoomToRect.y = zoomToRect.y > 0 ? zoomToRect.y : 0;
+    }
+    if (zoomToRect.x + rectAfterZoom.width > cssPageRect.width) {
+      zoomToRect.x = cssPageRect.width - rectAfterZoom.width;
+      zoomToRect.x = zoomToRect.x > 0 ? zoomToRect.x : 0;
+    }
 
     mStartZoomToMetrics = mFrameMetrics;
-    mEndZoomToMetrics.mScrollOffset =
-      gfx::Point(zoomToRect.x, zoomToRect.y);
+    mEndZoomToMetrics.mScrollOffset = gfx::Point(zoomToRect.x, zoomToRect.y);
 
     mAnimationStartTime = TimeStamp::Now();