diff --git a/layout/base/MobileViewportManager.cpp b/layout/base/MobileViewportManager.cpp
index 4d993d3dc81c..f47ba33a0d30 100644
--- a/layout/base/MobileViewportManager.cpp
+++ b/layout/base/MobileViewportManager.cpp
@@ -430,10 +430,6 @@ void MobileViewportManager::RefreshVisualViewportSize() {
     return;
   }
 
-  if (!gfxPrefs::APZAllowZooming()) {
-    return;
-  }
-
   ScreenIntSize displaySize = ViewAs<ScreenPixel>(
       mDisplaySize, PixelCastJustification::LayoutDeviceIsScreenForBounds);
 
@@ -513,6 +509,10 @@ void MobileViewportManager::RefreshViewportSize(bool aForceAdjustResolution) {
   if (gfxPrefs::APZAllowZooming()) {
     UpdateResolution(viewportInfo, displaySize, viewport,
                      displayWidthChangeRatio, UpdateType::ViewportSize);
+  } else {
+    // Even without zoom, we need to update that the visual viewport size
+    // has changed.
+    RefreshVisualViewportSize();
   }
   if (gfxPlatform::AsyncPanZoomEnabled()) {
     UpdateDisplayPortMargins();