diff --git a/gfx/layers/apz/src/AsyncPanZoomController.cpp b/gfx/layers/apz/src/AsyncPanZoomController.cpp
index d7d9d5852caa..de4cdc5632f7 100644
--- a/gfx/layers/apz/src/AsyncPanZoomController.cpp
+++ b/gfx/layers/apz/src/AsyncPanZoomController.cpp
@@ -1915,6 +1915,13 @@ void AsyncPanZoomController::HandlePanningWithTouchAction(double aAngle) {
   } else {
     SetState(NOTHING);
   }
+  if (!IsInPanningState()) {
+    // If we didn't enter a panning state because touch-action disallowed it,
+    // make sure to clear any leftover velocity from the pre-threshold
+    // touchmoves.
+    mX.SetVelocity(0);
+    mY.SetVelocity(0);
+  }
 }
 
 void AsyncPanZoomController::HandlePanning(double aAngle) {
@@ -1974,11 +1981,6 @@ nsEventStatus AsyncPanZoomController::StartPanning(const MultiTouchInput& aEvent
   float dx = mX.PanDistance(point.x);
   float dy = mY.PanDistance(point.y);
 
-  // When the touch move breaks through the pan threshold, reposition the touch down origin
-  // so the page won't jump when we start panning.
-  mX.StartTouch(point.x, aEvent.mTime);
-  mY.StartTouch(point.y, aEvent.mTime);
-
   double angle = atan2(dy, dx); // range [-pi, pi]
   angle = fabs(angle); // range [0, pi]
 
diff --git a/gfx/tests/gtest/TestAsyncPanZoomController.cpp b/gfx/tests/gtest/TestAsyncPanZoomController.cpp
index d53764a78dd8..111efdf1e7d0 100644
--- a/gfx/tests/gtest/TestAsyncPanZoomController.cpp
+++ b/gfx/tests/gtest/TestAsyncPanZoomController.cpp
@@ -509,7 +509,7 @@ Pan(const nsRefPtr<InputReceiver>& aTarget,
   gfxPrefs::SetAPZTouchStartTolerance(1.0f / 1000.0f);
   const int OVERCOME_TOUCH_TOLERANCE = 1;
 
-  const TimeDuration TIME_BETWEEN_TOUCH_EVENT = TimeDuration::FromMilliseconds(100);
+  const TimeDuration TIME_BETWEEN_TOUCH_EVENT = TimeDuration::FromMilliseconds(50);
 
   // Even if the caller doesn't care about the block id, we need it to set the
   // allowed touch behaviour below, so make sure aOutInputBlockId is non-null.