Bug 1454081 - Fix accessible coordinates in APZ viewports. r=surkov, r=yzen, r=jchen

accessible/generic/Accessible.cpp

@@ -561,6 +561,11 @@ Accessible::ChildAtPoint(int32_t aX, int32_t aY,
   nsRect screenRect = startFrame->GetScreenRectInAppUnits();
   nsPoint offset(presContext->DevPixelsToAppUnits(aX) - screenRect.X(),
                  presContext->DevPixelsToAppUnits(aY) - screenRect.Y());
+
+  // We need to take into account a non-1 resolution set on the presshell.
+  // This happens in mobile platforms with async pinch zooming.
+  offset = offset.RemoveResolution(presContext->PresShell()->GetResolution());
+
   nsIFrame* foundFrame = nsLayoutUtils::GetFrameForPoint(startFrame, offset);
 
   nsIContent* content = nullptr;
@@ -679,6 +684,10 @@ Accessible::Bounds() const
                      presContext->AppUnitsToDevPixels(unionRectTwips.Width()),
                      presContext->AppUnitsToDevPixels(unionRectTwips.Height()));
 
+  // We need to take into account a non-1 resolution set on the presshell.
+  // This happens in mobile platforms with async pinch zooming. Here we
+  // scale the bounds before adding the screen-relative offset.
+  screenRect.ScaleRoundOut(presContext->PresShell()->GetResolution());
   // We have the union of the rectangle, now we need to put it in absolute
   // screen coords.
   nsIntRect orgRectPixels = boundingFrame->GetScreenRectInAppUnits().

accessible/generic/HyperTextAccessible.cpp

@@ -1272,6 +1272,16 @@ HyperTextAccessible::TextBounds(int32_t aStartOffset, int32_t aEndOffset,
     offset1 = 0;
   }
 
+  // This document may have a resolution set, we will need to multiply
+  // the document-relative coordinates by that value and re-apply the doc's
+  // screen coordinates.
+  nsPresContext* presContext = mDoc->PresContext();
+  nsIFrame* rootFrame = presContext->PresShell()->GetRootFrame();
+  nsIntRect orgRectPixels = rootFrame->GetScreenRectInAppUnits().ToNearestPixels(presContext->AppUnitsPerDevPixel());
+  bounds.MoveBy(-orgRectPixels.X(), -orgRectPixels.Y());
+  bounds.ScaleRoundOut(presContext->PresShell()->GetResolution());
+  bounds.MoveBy(orgRectPixels.X(), orgRectPixels.Y());
+
   auto boundsX = bounds.X();
   auto boundsY = bounds.Y();
   nsAccUtils::ConvertScreenCoordsTo(&boundsX, &boundsY, aCoordType, this);

accessible/jsat/AccessFu.jsm

@@ -25,6 +25,7 @@ const GECKOVIEW_MESSAGE = {
   PREVIOUS: "GeckoView:AccessibilityPrevious",
   SCROLL_BACKWARD: "GeckoView:AccessibilityScrollBackward",
   SCROLL_FORWARD: "GeckoView:AccessibilityScrollForward",
+  EXPLORE_BY_TOUCH: "GeckoView:AccessibilityExploreByTouch"
 };
 
 var AccessFu = {
@@ -282,7 +283,7 @@ var AccessFu = {
         this.Input.activateCurrent(data);
         break;
       case GECKOVIEW_MESSAGE.LONG_PRESS:
-        this.Input.sendContextMenuMessage();
+        // XXX: Advertize long press on supported objects and implement action
         break;
       case GECKOVIEW_MESSAGE.SCROLL_FORWARD:
         this.Input.androidScroll("forward");
@@ -299,6 +300,9 @@ var AccessFu = {
       case GECKOVIEW_MESSAGE.BY_GRANULARITY:
         this.Input.moveByGranularity(data);
         break;
+      case GECKOVIEW_MESSAGE.EXPLORE_BY_TOUCH:
+        this.Input.moveToPoint("Simple", ...data.coordinates);
+        break;
     }
   },
 
@@ -375,30 +379,19 @@ var AccessFu = {
   _processedMessageManagers: [],
 
   /**
-   * Adjusts the given bounds relative to the given browser.
+   * Adjusts the given bounds that are defined in device display pixels
+   * to client-relative CSS pixels of the chrome window.
    * @param {Rect} aJsonBounds the bounds to adjust
-   * @param {browser} aBrowser the browser we want the bounds relative to
-   * @param {bool} aToCSSPixels whether to convert to CSS pixels (as opposed to
-   *               device pixels)
    */
-  adjustContentBounds(aJsonBounds, aBrowser, aToCSSPixels) {
+  screenToClientBounds(aJsonBounds) {
       let bounds = new Rect(aJsonBounds.left, aJsonBounds.top,
                             aJsonBounds.right - aJsonBounds.left,
                             aJsonBounds.bottom - aJsonBounds.top);
       let win = Utils.win;
       let dpr = win.devicePixelRatio;
-      let offset = { left: -win.mozInnerScreenX, top: -win.mozInnerScreenY };
-
-      // Add the offset; the offset is in CSS pixels, so multiply the
-      // devicePixelRatio back in before adding to preserve unit consistency.
-      bounds = bounds.translate(offset.left * dpr, offset.top * dpr);
-
-      // If we want to get to CSS pixels from device pixels, this needs to be
-      // further divided by the devicePixelRatio due to widget scaling.
-      if (aToCSSPixels) {
-        bounds = bounds.scale(1 / dpr, 1 / dpr);
-      }
 
+      bounds = bounds.scale(1 / dpr, 1 / dpr);
+      bounds = bounds.translate(-win.mozInnerScreenX, -win.mozInnerScreenY);
       return bounds.expandToIntegers();
     }
 };
@@ -517,7 +510,7 @@ var Output = {
         }
 
         let padding = aDetail.padding;
-        let r = AccessFu.adjustContentBounds(aDetail.bounds, aBrowser, true);
+        let r = AccessFu.screenToClientBounds(aDetail.bounds);
 
         // First hide it to avoid flickering when changing the style.
         highlightBox.classList.remove("show");
@@ -546,10 +539,6 @@ var Output = {
 
     for (let androidEvent of aDetails) {
       androidEvent.type = "GeckoView:AccessibilityEvent";
-      if (androidEvent.bounds) {
-        androidEvent.bounds = AccessFu.adjustContentBounds(
-          androidEvent.bounds, aBrowser);
-      }
 
       switch (androidEvent.eventType) {
         case ANDROID_VIEW_TEXT_CHANGED:
@@ -837,11 +826,6 @@ var Input = {
                         {offset, activateIfKey: aActivateIfKey});
   },
 
-  sendContextMenuMessage: function sendContextMenuMessage() {
-    let mm = Utils.getMessageManager(Utils.CurrentBrowser);
-    mm.sendAsyncMessage("AccessFu:ContextMenu", {});
-  },
-
   setEditState: function setEditState(aEditState) {
     Logger.debug(() => { return ["setEditState", JSON.stringify(aEditState)]; });
     this.editState = aEditState;
@@ -862,8 +846,7 @@ var Input = {
   doScroll: function doScroll(aDetails) {
     let horizontal = aDetails.horizontal;
     let page = aDetails.page;
-    let p = AccessFu.adjustContentBounds(
-      aDetails.bounds, Utils.CurrentBrowser, true).center();
+    let p = AccessFu.screenToClientBounds(aDetails.bounds).center();
     Utils.winUtils.sendWheelEvent(p.x, p.y,
       horizontal ? page : 0, horizontal ? 0 : page, 0,
       Utils.win.WheelEvent.DOM_DELTA_PAGE, 0, 0, 0, 0);

accessible/jsat/ContentControl.jsm

@@ -44,7 +44,6 @@ this.ContentControl.prototype = {
     for (let message of this.messagesOfInterest) {
       cs.addMessageListener(message, this);
     }
-    cs.addEventListener("mousemove", this);
   },
 
   stop: function cc_stop() {
@@ -52,7 +51,6 @@ this.ContentControl.prototype = {
     for (let message of this.messagesOfInterest) {
       cs.removeMessageListener(message, this);
     }
-    cs.removeEventListener("mousemove", this);
   },
 
   get document() {
@@ -106,7 +104,7 @@ this.ContentControl.prototype = {
     }
 
     this._contentScope.get().sendAsyncMessage("AccessFu:DoScroll",
-      { bounds: Utils.getBounds(position, true),
+      { bounds: Utils.getBounds(position),
         page: aMessage.json.direction === "forward" ? 1 : -1,
         horizontal: false });
   },
@@ -158,24 +156,11 @@ this.ContentControl.prototype = {
     }
   },
 
-  handleEvent: function cc_handleEvent(aEvent) {
-    if (aEvent.type === "mousemove") {
-      this.handleMoveToPoint(
-        { json: { x: aEvent.screenX, y: aEvent.screenY, rule: "Simple" } });
-    }
-    if (!Utils.getMessageManager(aEvent.target)) {
-      aEvent.preventDefault();
-    } else {
-      aEvent.target.focus();
-    }
-  },
-
   handleMoveToPoint: function cc_handleMoveToPoint(aMessage) {
     let [x, y] = [aMessage.json.x, aMessage.json.y];
     let rule = TraversalRules[aMessage.json.rule];
 
-    let dpr = this.window.devicePixelRatio;
-    this.vc.moveToPoint(rule, x * dpr, y * dpr, true);
+    this.vc.moveToPoint(rule, x, y, true);
   },
 
   handleClearCursor: function cc_handleClearCursor(aMessage) {

accessible/jsat/Utils.jsm

@@ -302,15 +302,11 @@ var Utils = { // jshint ignore:line
     return res.value;
   },
 
-  getBounds: function getBounds(aAccessible, aPreserveContentScale) {
+  getBounds: function getBounds(aAccessible) {
     let objX = {}, objY = {}, objW = {}, objH = {};
     aAccessible.getBounds(objX, objY, objW, objH);
 
-    let scale = aPreserveContentScale ? 1 :
-      this.getContentResolution(aAccessible);
-
-    return new Rect(objX.value, objY.value, objW.value, objH.value).scale(
-      scale, scale);
+    return new Rect(objX.value, objY.value, objW.value, objH.value);
   },
 
   getTextBounds: function getTextBounds(aAccessible, aStart, aEnd,
@@ -320,11 +316,7 @@ var Utils = { // jshint ignore:line
     accText.getRangeExtents(aStart, aEnd, objX, objY, objW, objH,
       Ci.nsIAccessibleCoordinateType.COORDTYPE_SCREEN_RELATIVE);
 
-    let scale = aPreserveContentScale ? 1 :
-      this.getContentResolution(aAccessible);
-
-    return new Rect(objX.value, objY.value, objW.value, objH.value).scale(
-      scale, scale);
+    return new Rect(objX.value, objY.value, objW.value, objH.value);
   },
 
   /**

accessible/jsat/content-script.js

@@ -62,19 +62,6 @@ function forwardToChild(aMessage, aListener, aVCPosition) {
   return true;
 }
 
-function activateContextMenu(aMessage) {
-  let position = Utils.getVirtualCursor(content.document).position;
-  if (!forwardToChild(aMessage, activateContextMenu, position)) {
-    let center = Utils.getBounds(position, true).center();
-
-    let evt = content.document.createEvent("HTMLEvents");
-    evt.initEvent("contextmenu", true, true);
-    evt.clientX = center.x;
-    evt.clientY = center.y;
-    position.DOMNode.dispatchEvent(evt);
-  }
-}
-
 function presentCaretChange(aText, aOldOffset, aNewOffset) {
   if (aOldOffset !== aNewOffset) {
     let msg = Presentation.textSelectionChanged(aText, aNewOffset, aNewOffset,
@@ -87,7 +74,7 @@ function scroll(aMessage) {
   let position = Utils.getVirtualCursor(content.document).position;
   if (!forwardToChild(aMessage, scroll, position)) {
     sendAsyncMessage("AccessFu:DoScroll",
-                     { bounds: Utils.getBounds(position, true),
+                     { bounds: Utils.getBounds(position),
                        page: aMessage.json.page,
                        horizontal: aMessage.json.horizontal });
   }
@@ -104,7 +91,6 @@ addMessageListener(
     if (m.json.buildApp)
       Utils.MozBuildApp = m.json.buildApp;
 
-    addMessageListener("AccessFu:ContextMenu", activateContextMenu);
     addMessageListener("AccessFu:Scroll", scroll);
 
     if (!contentControl) {
@@ -139,7 +125,6 @@ addMessageListener(
   function(m) {
     Logger.debug("AccessFu:Stop");
 
-    removeMessageListener("AccessFu:ContextMenu", activateContextMenu);
     removeMessageListener("AccessFu:Scroll", scroll);
 
     eventManager.stop();

accessible/tests/browser/bounds/browser.ini

@@ -6,6 +6,7 @@ support-files =
   !/accessible/tests/mochitest/*.js
   !/accessible/tests/mochitest/letters.gif
 
+[browser_test_resolution.js]
 [browser_test_zoom.js]
 [browser_test_zoom_text.js]
 skip-if = e10s && os == 'win' # bug 1372296

accessible/tests/browser/bounds/browser_test_resolution.js

@@ -0,0 +1,57 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/* import-globals-from ../../mochitest/layout.js */
+
+async function testScaledBounds(browser, accDoc, scale, id, type = "object") {
+  let acc = findAccessibleChildByID(accDoc, id);
+
+  // Get document offset
+  let [docX, docY] = getBounds(accDoc);
+
+  // Get the unscaled bounds of the accessible
+  let [x, y, width, height] = type == "text" ?
+    getRangeExtents(acc, 0, -1, COORDTYPE_SCREEN_RELATIVE) : getBounds(acc);
+
+  await ContentTask.spawn(browser, scale, _scale => {
+    setResolution(document, _scale);
+  });
+
+  let [scaledX, scaledY, scaledWidth, scaledHeight] = type == "text" ?
+    getRangeExtents(acc, 0, -1, COORDTYPE_SCREEN_RELATIVE) : getBounds(acc);
+
+  let name = prettyName(acc);
+  isWithin(scaledWidth, width * scale, 2, "Wrong scaled width of " + name);
+  isWithin(scaledHeight, height * scale, 2, "Wrong scaled height of " + name);
+  isWithin(scaledX - docX, (x - docX) * scale, 2, "Wrong scaled x of " + name);
+  isWithin(scaledY - docY, (y - docY) * scale, 2, "Wrong scaled y of " + name);
+
+  await ContentTask.spawn(browser, {}, () => {
+    setResolution(document, 1.0);
+  });
+}
+
+async function runTests(browser, accDoc) {
+  loadFrameScripts(browser, { name: "layout.js", dir: MOCHITESTS_DIR });
+
+  await testScaledBounds(browser, accDoc, 2.0, "p1");
+  await testScaledBounds(browser, accDoc, 0.5, "p2");
+  await testScaledBounds(browser, accDoc, 3.5, "b1");
+
+  await testScaledBounds(browser, accDoc, 2.0, "p1", "text");
+  await testScaledBounds(browser, accDoc, 0.75, "p2", "text");
+}
+
+/**
+ * Test accessible boundaries when page is zoomed
+ */
+addAccessibleTask(`
+<p id='p1' style='font-family: monospace;'>Tilimilitryamdiya</p>
+<p id="p2">para 2</p>
+<button id="b1">Hello</button>
+`,
+  runTests
+);

accessible/tests/mochitest/layout.js

@@ -68,6 +68,18 @@ function zoomDocument(aDocument, aZoom) {
   docViewer.fullZoom = aZoom;
 }
 
+/**
+ * Set the relative resolution of this document. This is what apz does.
+ * On non-mobile platforms you won't see a visible change.
+ */
+function setResolution(aDocument, aZoom) {
+  var windowUtils = aDocument.defaultView.
+    QueryInterface(Ci.nsIInterfaceRequestor).
+    getInterface(Ci.nsIDOMWindowUtils);
+
+  windowUtils.setResolutionAndScaleTo(aZoom);
+}
+
 /**
  * Return child accessible at the given point.
  *
@@ -196,6 +208,14 @@ function getBounds(aID) {
   return [x.value, y.value, width.value, height.value];
 }
 
+function getRangeExtents(aID, aStartOffset, aEndOffset, aCoordOrigin) {
+  var hyperText = getAccessible(aID, [nsIAccessibleText]);
+  var x = {}, y = {}, width = {}, height = {};
+  hyperText.getRangeExtents(aStartOffset, aEndOffset,
+                            x, y, width, height, aCoordOrigin);
+  return [x.value, y.value, width.value, height.value];
+}
+
 /**
  * Return DOM node coordinates relative the screen and its size in device
  * pixels.

mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/PanZoomController.java

@@ -228,6 +228,11 @@ public final class PanZoomController extends JNIObject {
         } else if ((action == MotionEvent.ACTION_HOVER_MOVE) ||
                    (action == MotionEvent.ACTION_HOVER_ENTER) ||
                    (action == MotionEvent.ACTION_HOVER_EXIT)) {
+            if (event.getSource() == InputDevice.SOURCE_TOUCHSCREEN) {
+                // A hover is not possible on a touchscreen unless via accessibility
+                // and we handle that elsewhere.
+                return false;
+            }
             return handleMouseEvent(event);
         } else {
             return false;

mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoView.java

@@ -516,15 +516,15 @@ public class GeckoView extends FrameLayout {
 
     @Override
     public boolean onHoverEvent(final MotionEvent event) {
-        // If we get a touchscreen hover event, and accessibility is not enabled, don't
-        // send it to Gecko.
-        if (event.getSource() == InputDevice.SOURCE_TOUCHSCREEN &&
-            !SessionAccessibility.Settings.isEnabled()) {
-            return false;
+        // A touchscreen hover event is a screen reader doing explore-by-touch
+        if (SessionAccessibility.Settings.isEnabled() &&
+            event.getSource() == InputDevice.SOURCE_TOUCHSCREEN &&
+            mSession != null) {
+            mSession.getAccessibility().onExploreByTouch(event);
+            return true;
         }
 
-        return mSession != null &&
-               mSession.getPanZoomController().onMotionEvent(event);
+        return false;
     }
 
     @Override

mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java

@@ -19,6 +19,7 @@ import android.graphics.Rect;
 import android.os.Build;
 import android.os.Bundle;
 import android.util.Log;
+import android.view.MotionEvent;
 import android.view.View;
 import android.view.ViewParent;
 import android.view.accessibility.AccessibilityEvent;
@@ -366,17 +367,17 @@ public class SessionAccessibility {
 
         final GeckoBundle bounds = message.getBundle("bounds");
         if (bounds != null) {
-            Rect relativeBounds = new Rect(bounds.getInt("left"), bounds.getInt("top"),
-                                           bounds.getInt("right"), bounds.getInt("bottom"));
-            node.setBoundsInParent(relativeBounds);
+            Rect screenBounds = new Rect(bounds.getInt("left"), bounds.getInt("top"),
+                                         bounds.getInt("right"), bounds.getInt("bottom"));
+            node.setBoundsInScreen(screenBounds);
 
             final Matrix matrix = new Matrix();
             final float[] origin = new float[2];
             mSession.getClientToScreenMatrix(matrix);
             matrix.mapPoints(origin);
 
-            relativeBounds.offset((int) origin[0], (int) origin[1]);
-            node.setBoundsInScreen(relativeBounds);
+            screenBounds.offset((int) -origin[0], (int) -origin[1]);
+            node.setBoundsInParent(screenBounds);
         }
 
     }
@@ -419,4 +420,10 @@ public class SessionAccessibility {
         populateEventFromJSON(accessibilityEvent, message);
         ((ViewParent) mView).requestSendAccessibilityEvent(mView, accessibilityEvent);
     }
+
+    public void onExploreByTouch(final MotionEvent event) {
+      final GeckoBundle data = new GeckoBundle(2);
+      data.putDoubleArray("coordinates", new double[] {event.getRawX(), event.getRawY()});
+      mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityExploreByTouch", data);
+    }
 }