Bug 1778486 - Use Pointer event to record last user input position; r=smaug

In order to support record the last user input for touch given APZ won't synthesize mouse event for touch in parent process. Using pointer event could support pen input nicely, too. Differential Revision: https://phabricator.services.mozilla.com/D151282
2022-07-20 15:03:19 +00:00 · 2022-07-20 15:03:19 +00:00 · 4811778c31
--- a/dom/base/nsDOMWindowUtils.cpp
+++ b/dom/base/nsDOMWindowUtils.cpp
@ -284,8 +284,8 @@ CompositorBridgeChild* nsDOMWindowUtils::GetCompositorBridge() {
 }

 NS_IMETHODIMP
-nsDOMWindowUtils::GetLastOverWindowMouseLocationInCSSPixels(float* aX,
-                                                            float* aY) {
+nsDOMWindowUtils::GetLastOverWindowPointerLocationInCSSPixels(float* aX,
+                                                              float* aY) {
  const PresShell* presShell = GetPresShell();
  const nsPresContext* presContext = GetPresContext();

@ -293,18 +293,18 @@ nsDOMWindowUtils::GetLastOverWindowMouseLocationInCSSPixels(float* aX,
    return NS_ERROR_FAILURE;
  }

-  const nsPoint& lastOverWindowMouseLocation =
-      presShell->GetLastOverWindowMouseLocation();
+  const nsPoint& lastOverWindowPointerLocation =
+      presShell->GetLastOverWindowPointerLocation();

-  if (lastOverWindowMouseLocation.X() == NS_UNCONSTRAINEDSIZE &&
-      lastOverWindowMouseLocation.Y() == NS_UNCONSTRAINEDSIZE) {
+  if (lastOverWindowPointerLocation.X() == NS_UNCONSTRAINEDSIZE &&
+      lastOverWindowPointerLocation.Y() == NS_UNCONSTRAINEDSIZE) {
    *aX = 0;
    *aY = 0;
  } else {
-    const CSSPoint lastOverWindowMouseLocationInCSSPixels =
-        CSSPoint::FromAppUnits(lastOverWindowMouseLocation);
-    *aX = lastOverWindowMouseLocationInCSSPixels.X();
-    *aY = lastOverWindowMouseLocationInCSSPixels.Y();
+    const CSSPoint lastOverWindowPointerLocationInCSSPixels =
+        CSSPoint::FromAppUnits(lastOverWindowPointerLocation);
+    *aX = lastOverWindowPointerLocationInCSSPixels.X();
+    *aY = lastOverWindowPointerLocationInCSSPixels.Y();
  }

  return NS_OK;
--- a/dom/base/test/chrome.ini
+++ b/dom/base/test/chrome.ini
@ -27,6 +27,8 @@ support-files =
 [test_anchor_target_blank_referrer.html]
 [test_domrequesthelper.xhtml]
 [test_fragment_sanitization.xhtml]
+[test_getLastOverWindowPointerLocationInCSSPixels.html]
+support-files = !/gfx/layers/apz/test/mochitest/apz_test_utils.js
 [test_messagemanager_send_principal.html]
 [test_navigator_resolve_identity_xrays.xhtml]
 support-files = file_navigator_resolve_identity_xrays.xhtml
--- a/dom/base/test/test_getLastOverWindowPointerLocationInCSSPixels.html
+++ b/dom/base/test/test_getLastOverWindowPointerLocationInCSSPixels.html
@ -0,0 +1,83 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+<title>Test for Bug 1778486</title>
+<script src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script>
+<script src="chrome://mochikit/content/tests/SimpleTest/EventUtils.js"></script>
+<script src="chrome://mochikit/content/tests/SimpleTest/paint_listener.js"></script>
+<script src="chrome://mochitests/content/browser/gfx/layers/apz/test/mochitest/apz_test_utils.js"></script>
+<link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css" />
+<style>
+div {
+  width: 200px;
+  height: 200px;
+};
+</style>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1778486">Mozilla Bug 1778486</a>
+<p id="display"></p>
+<div id="target_mouse" style="background: red;"></div>
+<div id="target_touch" style="background: green;"></div>
+<script>
+
+function waitForEvent(aTarget, aEvent) {
+  return new Promise(resolve => {
+    aTarget.addEventListener(aEvent, resolve, { once: true });
+  });
+}
+
+function getLastOverWindowPointerLocationScreenOffset() {
+  let x = {};
+  let y = {};
+  let topWindow = window.browsingContext.topChromeWindow;
+  topWindow.windowUtils.getLastOverWindowPointerLocationInCSSPixels(x, y);
+  return {
+    x: x.value + topWindow.mozInnerScreenX,
+    y: y.value + topWindow.mozInnerScreenY,
+  };
+}
+
+function getElementScreenOffsetAtCentral(aElement) {
+  let rect = aElement.getBoundingClientRect();
+  return {
+    x: rect.width / 2 + rect.left + mozInnerScreenX,
+    y: rect.height / 2 + rect.top + mozInnerScreenY,
+  };
+}
+
+add_task(async function init() {
+  await SpecialPowers.pushPrefEnv({ set: [["test.events.async.enabled", true]] });
+  await waitUntilApzStable();
+});
+
+/** Test for Bug 1778486 **/
+add_task(async function test_mouse() {
+  let target = document.getElementById("target_mouse");
+  let promise = waitForEvent(target, "click");
+  synthesizeMouseAtCenter(target, {});
+  await promise;
+
+  isDeeply(
+    getLastOverWindowPointerLocationScreenOffset(),
+    getElementScreenOffsetAtCentral(target),
+    "check last pointer location");
+});
+
+add_task(async function test_touch() {
+  let target = document.getElementById("target_touch");
+  let promise = waitForEvent(target, "pointerup");
+  synthesizeTouchAtCenter(target, { type: "touchstart" });
+  synthesizeTouchAtCenter(target, { type: "touchend" });
+  await promise;
+
+  isDeeply(
+    getLastOverWindowPointerLocationScreenOffset(),
+    getElementScreenOffsetAtCentral(target),
+    "check last pointer location");
+});
+
+</script>
+</pre>
+</body>
+</html>
--- a/dom/events/test/browser_navigator_clipboard_readText.js
+++ b/dom/events/test/browser_navigator_clipboard_readText.js
@ -50,7 +50,7 @@ function promiseClickPasteButton() {
 function getMouseCoordsRelativeToScreenInDevicePixels() {
  let mouseXInCSSPixels = {};
  let mouseYInCSSPixels = {};
-  window.windowUtils.getLastOverWindowMouseLocationInCSSPixels(
+  window.windowUtils.getLastOverWindowPointerLocationInCSSPixels(
    mouseXInCSSPixels,
    mouseYInCSSPixels
  );
--- a/dom/interfaces/base/nsIDOMWindowUtils.idl
+++ b/dom/interfaces/base/nsIDOMWindowUtils.idl
@ -100,11 +100,8 @@ interface nsIDOMWindowUtils : nsISupports {
   *
   * @param aX 0, if there's no such location.
   * @param aY 0, if there's no such location.
-   *
-   * TODO: verify this is also the touch-position and rename the method
-   * accordingly (bug 1770358).
   */
-  void getLastOverWindowMouseLocationInCSSPixels(out float aX, out float aY);
+  void getLastOverWindowPointerLocationInCSSPixels(out float aX, out float aY);

  /**
   * Force a synchronous layer transaction for this window if necessary.
--- a/layout/base/PresShell.cpp
+++ b/layout/base/PresShell.cpp
@ -6665,13 +6665,30 @@ bool PresShell::MouseLocationWasSetBySynthesizedMouseEventForTests() const {
         rootPresShell->mMouseLocationWasSetBySynthesizedMouseEventForTests;
 }

-void PresShell::RecordMouseLocation(WidgetGUIEvent* aEvent) {
-  if (!mPresContext) return;
+nsPoint PresShell::GetEventLocation(const WidgetMouseEvent& aEvent) const {
+  nsIFrame* rootFrame = GetRootFrame();
+  if (rootFrame) {
+    RelativeTo relativeTo{rootFrame};
+    if (rootFrame->PresContext()->IsRootContentDocumentCrossProcess()) {
+      relativeTo.mViewportType = ViewportType::Visual;
+    }
+    return nsLayoutUtils::GetEventCoordinatesRelativeTo(&aEvent, relativeTo);
+  }
+
+  nsView* rootView = mViewManager->GetRootView();
+  return nsLayoutUtils::TranslateWidgetToView(mPresContext, aEvent.mWidget,
+                                              aEvent.mRefPoint, rootView);
+}
+
+void PresShell::RecordPointerLocation(WidgetGUIEvent* aEvent) {
+  if (!mPresContext) {
+    return;
+  }

  if (!mPresContext->IsRoot()) {
    PresShell* rootPresShell = GetRootPresShell();
    if (rootPresShell) {
-      rootPresShell->RecordMouseLocation(aEvent);
+      rootPresShell->RecordPointerLocation(aEvent);
    }
    return;
  }
@ -6680,25 +6697,8 @@ void PresShell::RecordMouseLocation(WidgetGUIEvent* aEvent) {
       aEvent->AsMouseEvent()->mReason == WidgetMouseEvent::eReal) ||
      aEvent->mMessage == eMouseEnterIntoWidget ||
      aEvent->mMessage == eMouseDown || aEvent->mMessage == eMouseUp) {
-    nsIFrame* rootFrame = GetRootFrame();
-    if (!rootFrame) {
-      nsView* rootView = mViewManager->GetRootView();
-      mMouseLocation = nsLayoutUtils::TranslateWidgetToView(
-          mPresContext, aEvent->mWidget, aEvent->mRefPoint, rootView);
-      // TODO: instead, encapsulate `mMouseLocation` and
-      // `mLastOverWindowMouseLocation` in a struct.
-      mLastOverWindowMouseLocation = mMouseLocation;
-      mMouseEventTargetGuid = InputAPZContext::GetTargetLayerGuid();
-    } else {
-      RelativeTo relativeTo{rootFrame};
-      if (rootFrame->PresContext()->IsRootContentDocumentCrossProcess()) {
-        relativeTo.mViewportType = ViewportType::Visual;
-      }
-      mMouseLocation =
-          nsLayoutUtils::GetEventCoordinatesRelativeTo(aEvent, relativeTo);
-      mLastOverWindowMouseLocation = mMouseLocation;
-      mMouseEventTargetGuid = InputAPZContext::GetTargetLayerGuid();
-    }
+    mMouseLocation = GetEventLocation(*aEvent->AsMouseEvent());
+    mMouseEventTargetGuid = InputAPZContext::GetTargetLayerGuid();
    mMouseLocationWasSetBySynthesizedMouseEventForTests =
        aEvent->mFlags.mIsSynthesizedForTests;
 #ifdef DEBUG_MOUSE_LOCATION
@ -6725,6 +6725,13 @@ void PresShell::RecordMouseLocation(WidgetGUIEvent* aEvent) {
    printf("[ps=%p]got mouse exit for %p\n", this, aEvent->mWidget);
    printf("[ps=%p]clearing mouse location\n", this);
 #endif
+  } else if ((aEvent->mMessage == ePointerMove &&
+              aEvent->AsMouseEvent()->mReason == WidgetMouseEvent::eReal) ||
+             aEvent->mMessage == ePointerDown ||
+             aEvent->mMessage == ePointerUp) {
+    // TODO: instead, encapsulate `mMouseLocation` and
+    // `mLastOverWindowPointerLocation` in a struct.
+    mLastOverWindowPointerLocation = GetEventLocation(*aEvent->AsMouseEvent());
  }
 }

@ -6874,7 +6881,7 @@ nsresult PresShell::EventHandler::HandleEvent(nsIFrame* aFrameForPresShell,
    return NS_OK;
  }

-  mPresShell->RecordMouseLocation(aGUIEvent);
+  mPresShell->RecordPointerLocation(aGUIEvent);

  if (MaybeHandleEventWithAccessibleCaret(aFrameForPresShell, aGUIEvent,
                                          aEventStatus)) {
@ -8077,6 +8084,9 @@ nsresult PresShell::EventHandler::HandleEventWithTarget(
 #endif
  NS_ENSURE_STATE(!aNewEventContent ||
                  aNewEventContent->GetComposedDoc() == GetDocument());
+  if (aEvent->mClass == ePointerEventClass) {
+    mPresShell->RecordPointerLocation(aEvent->AsMouseEvent());
+  }
  AutoPointerEventTargetUpdater updater(mPresShell, aEvent, aNewEventFrame,
                                        aTargetContent);
  AutoCurrentEventInfoSetter eventInfoSetter(*this, aNewEventFrame,
--- a/layout/base/PresShell.h
+++ b/layout/base/PresShell.h
@ -237,10 +237,10 @@ class PresShell final : public nsStubDocumentObserver,
 #endif  // #ifdef ACCESSIBILITY

  /**
-   * See `mLastOverWindowMouseLocation`.
+   * See `mLastOverWindowPointerLocation`.
   */
-  const nsPoint& GetLastOverWindowMouseLocation() const {
-    return mLastOverWindowMouseLocation;
+  const nsPoint& GetLastOverWindowPointerLocation() const {
+    return mLastOverWindowPointerLocation;
  }

  void Init(nsPresContext*, nsViewManager*);
@ -1979,9 +1979,15 @@ class PresShell final : public nsStubDocumentObserver,
    bool IsKeyPressEvent() override;
  };

+  /**
+   * return the nsPoint represents the location of the mouse event relative to
+   * the root document in visual coordinates
+   */
+  nsPoint GetEventLocation(const WidgetMouseEvent& aEvent) const;
+
  // Check if aEvent is a mouse event and record the mouse location for later
  // synth mouse moves.
-  void RecordMouseLocation(WidgetGUIEvent* aEvent);
+  void RecordPointerLocation(WidgetGUIEvent* aEvent);
  inline bool MouseLocationWasSetBySynthesizedMouseEventForTests() const;
  class nsSynthMouseMoveEvent final : public nsARefreshObserver {
   public:
@ -3005,8 +3011,9 @@ class PresShell final : public nsStubDocumentObserver,
  // NS_UNCONSTRAINEDSIZE) if the mouse isn't over our window or there is no
  // last observed mouse location for some reason.
  nsPoint mMouseLocation;
-  // The last mouse location (see `mMouseLocation`) which was over the window.
-  nsPoint mLastOverWindowMouseLocation;
+  // The last observed pointer location relative to that root document in visual
+  // coordinates.
+  nsPoint mLastOverWindowPointerLocation;
  // This is an APZ state variable that tracks the target guid for the last
  // mouse event that was processed (corresponding to mMouseLocation). This is
  // needed for the synthetic mouse events.
--- a/toolkit/actors/ClipboardReadTextPasteParent.jsm
+++ b/toolkit/actors/ClipboardReadTextPasteParent.jsm
@ -62,7 +62,7 @@ class ClipboardReadTextPasteParent extends JSWindowActorParent {

      let mouseXInCSSPixels = {};
      let mouseYInCSSPixels = {};
-      windowUtils.getLastOverWindowMouseLocationInCSSPixels(
+      windowUtils.getLastOverWindowPointerLocationInCSSPixels(
        mouseXInCSSPixels,
        mouseYInCSSPixels
      );