Bug 1564549 - Implement move and explore by touch natively. r=geckoview-reviewers,yzen,snorp

Differential Revision: https://phabricator.services.mozilla.com/D45599

--HG--
extra : moz-landing-system : lando

accessible/android/AccessibleWrap.cpp

@@ -6,17 +6,22 @@
 #include "AccessibleWrap.h"
 
 #include "Accessible-inl.h"
+#include "AccEvent.h"
 #include "AndroidInputType.h"
 #include "DocAccessibleWrap.h"
 #include "IDSet.h"
 #include "JavaBuiltins.h"
 #include "SessionAccessibility.h"
+#include "TraversalRule.h"
+#include "Pivot.h"
 #include "nsAccessibilityService.h"
+#include "nsEventShell.h"
 #include "nsPersistentProperties.h"
 #include "nsIAccessibleAnnouncementEvent.h"
 #include "nsIStringBundle.h"
 #include "nsAccUtils.h"
 #include "nsTextEquivUtils.h"
+#include "RootAccessible.h"
 
 #include "mozilla/a11y/PDocAccessibleChild.h"
 #include "mozilla/jni/GeckoBundleUtils.h"
@@ -79,6 +84,27 @@ nsresult AccessibleWrap::HandleAccEvent(AccEvent* aEvent) {
         }
         break;
       }
+      case nsIAccessibleEvent::EVENT_TEXT_CARET_MOVED: {
+        if (accessible != aEvent->Document() && !aEvent->IsFromUserInput()) {
+          AccCaretMoveEvent* caretEvent = downcast_accEvent(aEvent);
+          if (IsHyperText()) {
+            DOMPoint point =
+                AsHyperText()->OffsetToDOMPoint(caretEvent->GetCaretOffset());
+            if (Accessible* newPos =
+                    doc->GetAccessibleOrContainer(point.node)) {
+              static_cast<AccessibleWrap*>(newPos)->Pivot(
+                  java::SessionAccessibility::HTML_GRANULARITY_DEFAULT, true,
+                  true);
+            }
+          }
+        }
+        break;
+      }
+      case nsIAccessibleEvent::EVENT_SCROLLING_START: {
+        accessible->Pivot(java::SessionAccessibility::HTML_GRANULARITY_DEFAULT,
+                          true, true);
+        break;
+      }
       default:
         break;
     }
@@ -123,15 +149,11 @@ nsresult AccessibleWrap::HandleAccEvent(AccEvent* aEvent) {
 
       RefPtr<AccessibleWrap> newPosition =
           static_cast<AccessibleWrap*>(vcEvent->NewAccessible());
-      auto oldPosition = static_cast<AccessibleWrap*>(vcEvent->OldAccessible());
-
       if (sessionAcc && newPosition) {
-        if (oldPosition != newPosition) {
-          if (vcEvent->Reason() == nsIAccessiblePivot::REASON_POINT) {
-            sessionAcc->SendHoverEnterEvent(newPosition);
-          } else {
-            sessionAcc->SendAccessibilityFocusedEvent(newPosition);
-          }
+        if (vcEvent->Reason() == nsIAccessiblePivot::REASON_POINT) {
+          sessionAcc->SendHoverEnterEvent(newPosition);
+        } else {
+          sessionAcc->SendAccessibilityFocusedEvent(newPosition);
         }
 
         if (vcEvent->BoundaryType() != nsIAccessiblePivot::NO_BOUNDARY) {
@@ -248,6 +270,37 @@ bool AccessibleWrap::GetSelectionBounds(int32_t* aStartOffset,
   return false;
 }
 
+void AccessibleWrap::Pivot(int32_t aGranularity, bool aForward,
+                           bool aInclusive) {
+  a11y::Pivot pivot(RootAccessible());
+  TraversalRule rule(aGranularity);
+  Accessible* result = aForward ? pivot.Next(this, rule, aInclusive)
+                                : pivot.Prev(this, rule, aInclusive);
+  if (result && (result != this || aInclusive)) {
+    PivotMoveReason reason = aForward ? nsIAccessiblePivot::REASON_NEXT
+                                      : nsIAccessiblePivot::REASON_PREV;
+    RefPtr<AccEvent> event = new AccVCChangeEvent(
+        result->Document(), this, -1, -1, result, -1, -1, reason,
+        nsIAccessiblePivot::NO_BOUNDARY, eFromUserInput);
+    nsEventShell::FireEvent(event);
+  }
+}
+
+void AccessibleWrap::ExploreByTouch(float aX, float aY) {
+  a11y::Pivot pivot(RootAccessible());
+  TraversalRule rule;
+
+  Accessible* result = pivot.AtPoint(aX, aY, rule);
+
+  if (result && result != this) {
+    RefPtr<AccEvent> event =
+        new AccVCChangeEvent(result->Document(), this, -1, -1, result, -1, -1,
+                             nsIAccessiblePivot::REASON_POINT,
+                             nsIAccessiblePivot::NO_BOUNDARY, eFromUserInput);
+    nsEventShell::FireEvent(event);
+  }
+}
+
 uint32_t AccessibleWrap::GetFlags(role aRole, uint64_t aState,
                                   uint8_t aActionCount) {
   uint32_t flags = 0;

accessible/android/AccessibleWrap.h

@@ -35,6 +35,10 @@ class AccessibleWrap : public Accessible {
 
   virtual bool GetSelectionBounds(int32_t* aStartOffset, int32_t* aEndOffset);
 
+  virtual void Pivot(int32_t aGranularity, bool aForward, bool aInclusive);
+
+  virtual void ExploreByTouch(float aX, float aY);
+
   mozilla::java::GeckoBundle::LocalRef ToBundle(bool aSmall = false);
 
   mozilla::java::GeckoBundle::LocalRef ToBundle(

accessible/android/Platform.cpp

@@ -130,13 +130,11 @@ void a11y::ProxyVirtualCursorChangeEvent(
     return;
   }
 
-  if (aOldPosition != aNewPosition) {
-    if (aReason == nsIAccessiblePivot::REASON_POINT) {
-      sessionAcc->SendHoverEnterEvent(WrapperFor(aNewPosition));
-    } else {
-      RefPtr<AccessibleWrap> wrapperForNewPosition = WrapperFor(aNewPosition);
-      sessionAcc->SendAccessibilityFocusedEvent(wrapperForNewPosition);
-    }
+  if (aReason == nsIAccessiblePivot::REASON_POINT) {
+    sessionAcc->SendHoverEnterEvent(WrapperFor(aNewPosition));
+  } else {
+    RefPtr<AccessibleWrap> wrapperForNewPosition = WrapperFor(aNewPosition);
+    sessionAcc->SendAccessibilityFocusedEvent(wrapperForNewPosition);
   }
 
   if (aBoundaryType != nsIAccessiblePivot::NO_BOUNDARY) {

accessible/android/ProxyAccessibleWrap.cpp

@@ -4,8 +4,11 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "ProxyAccessibleWrap.h"
+
 #include "nsPersistentProperties.h"
 
+#include "mozilla/a11y/DocAccessiblePlatformExtParent.h"
+
 using namespace mozilla::a11y;
 
 ProxyAccessibleWrap::ProxyAccessibleWrap(ProxyAccessible* aProxy)
@@ -106,6 +109,17 @@ bool ProxyAccessibleWrap::GetSelectionBounds(int32_t* aStartOffset,
   return Proxy()->SelectionBoundsAt(0, unused, aStartOffset, aEndOffset);
 }
 
+void ProxyAccessibleWrap::Pivot(int32_t aGranularity, bool aForward,
+                                bool aInclusive) {
+  Unused << Proxy()->Document()->GetPlatformExtension()->SendPivot(
+      Proxy()->ID(), aGranularity, aForward, aInclusive);
+}
+
+void ProxyAccessibleWrap::ExploreByTouch(float aX, float aY) {
+  Unused << Proxy()->Document()->GetPlatformExtension()->SendExploreByTouch(
+      Proxy()->ID(), aX, aY);
+}
+
 role ProxyAccessibleWrap::WrapperRole() { return Proxy()->Role(); }
 
 AccessibleWrap* ProxyAccessibleWrap::WrapperParent() {

accessible/android/ProxyAccessibleWrap.h

@@ -59,6 +59,11 @@ class ProxyAccessibleWrap : public AccessibleWrap {
   virtual bool GetSelectionBounds(int32_t* aStartOffset,
                                   int32_t* aEndOffset) override;
 
+  virtual void Pivot(int32_t aGranularity, bool aForward,
+                     bool aInclusive) override;
+
+  virtual void ExploreByTouch(float aX, float aY) override;
+
   virtual void WrapperDOMNodeID(nsString& aDOMNodeID) override;
 
  private:

accessible/android/SessionAccessibility.cpp

@@ -115,6 +115,15 @@ void SessionAccessibility::Click(int32_t aID) {
   FORWARD_ACTION_TO_ACCESSIBLE(DoAction, 0);
 }
 
+void SessionAccessibility::Pivot(int32_t aID, int32_t aGranularity,
+                                 bool aForward, bool aInclusive) {
+  FORWARD_ACTION_TO_ACCESSIBLE(Pivot, aGranularity, aForward, aInclusive);
+}
+
+void SessionAccessibility::ExploreByTouch(int32_t aID, float aX, float aY) {
+  FORWARD_ACTION_TO_ACCESSIBLE(ExploreByTouch, aX, aY);
+}
+
 SessionAccessibility* SessionAccessibility::GetInstanceFor(
     ProxyAccessible* aAccessible) {
   auto tab =

accessible/android/SessionAccessibility.h

@@ -54,6 +54,8 @@ class SessionAccessibility final
   jni::Object::LocalRef GetNodeInfo(int32_t aID);
   void SetText(int32_t aID, jni::String::Param aText);
   void Click(int32_t aID);
+  void Pivot(int32_t aID, int32_t aGranularity, bool aForward, bool aInclusive);
+  void ExploreByTouch(int32_t aID, float aX, float aY);
   void StartNativeAccessibility();
 
   // Event methods

accessible/ipc/extension/android/DocAccessiblePlatformExtChild.cpp

@@ -15,7 +15,7 @@ namespace a11y {
 mozilla::ipc::IPCResult DocAccessiblePlatformExtChild::RecvPivot(
     uint64_t aID, int32_t aGranularity, bool aForward, bool aInclusive) {
   if (auto acc = IdToAccessibleWrap(aID)) {
-    // XXX: Forward to appropriate wrapper method.
+    acc->Pivot(aGranularity, aForward, aInclusive);
   }
 
   return IPC_OK();
@@ -66,7 +66,9 @@ mozilla::ipc::IPCResult DocAccessiblePlatformExtChild::RecvPaste(uint64_t aID) {
 
 mozilla::ipc::IPCResult DocAccessiblePlatformExtChild::RecvExploreByTouch(
     uint64_t aID, float aX, float aY) {
-  // XXX: Forward to appropriate wrapper method.
+  if (auto acc = IdToAccessibleWrap(aID)) {
+    acc->ExploreByTouch(aX, aY);
+  }
 
   return IPC_OK();
 }

accessible/ipc/extension/android/PDocAccessiblePlatformExt.ipdl

@@ -27,7 +27,7 @@ child:
 
   async Paste(int32_t aID);
 
-  async ExploreByTouch(int32_t aID, float aX, float aY);
+  async ExploreByTouch(uint64_t aID, float aX, float aY);
   };
 }
 }

mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java

@@ -22,6 +22,7 @@ import android.os.Bundle;
 import android.support.annotation.NonNull;
 import android.support.annotation.Nullable;
 import android.support.annotation.UiThread;
+import android.text.TextUtils;
 import android.util.Log;
 import android.util.SparseArray;
 import android.view.InputDevice;
@@ -200,24 +201,11 @@ public class SessionAccessibility {
 
             switch (action) {
                 case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS:
-                    if (mAccessibilityFocusedNode == virtualViewId) {
-                        mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityClearCursor", null);
-                    }
                     sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED, virtualViewId, CLASSNAME_UNKNOWN, null);
                     return true;
                 case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS:
-                    if (virtualViewId == View.NO_ID) {
-                        sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, View.NO_ID, CLASSNAME_WEBVIEW, null);
-                    } else {
-                        if (mFocusedNode == virtualViewId && mHoveredOnNode != virtualViewId) {
-                            // If we are sending accessibility focus to the focused node, sync up the state with Gecko.
-                            // XXX: This is a stopgap for now until we remove the JS layer and manipulate the Gecko a11y virtual cursor directly
-                            // with the given virtualViewId
-                            mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityCursorToFocused", null);
-                        } else {
-                            sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, virtualViewId, CLASSNAME_UNKNOWN, null);
-                        }
-                    }
+                    sendEvent(AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED, virtualViewId,
+                            virtualViewId == View.NO_ID ? CLASSNAME_WEBVIEW : CLASSNAME_UNKNOWN, null);
                     return true;
                 case AccessibilityNodeInfo.ACTION_CLICK:
                     nativeProvider.click(virtualViewId);
@@ -255,16 +243,16 @@ public class SessionAccessibility {
                     nativeProvider.click(virtualViewId);
                     return true;
                 case AccessibilityNodeInfo.ACTION_NEXT_HTML_ELEMENT:
+                    requestViewFocus();
+                    nativeProvider.pivot(virtualViewId, arguments != null ?
+                            arguments.getString(AccessibilityNodeInfo.ACTION_ARGUMENT_HTML_ELEMENT_STRING) : "",
+                            true, false);
+                    return true;
                 case AccessibilityNodeInfo.ACTION_PREVIOUS_HTML_ELEMENT:
                     requestViewFocus();
-                    if (arguments != null) {
-                        data = new GeckoBundle(1);
-                        data.putString("rule", arguments.getString(AccessibilityNodeInfo.ACTION_ARGUMENT_HTML_ELEMENT_STRING));
-                    } else {
-                        data = null;
-                    }
-                    mSession.getEventDispatcher().dispatch(action == AccessibilityNodeInfo.ACTION_NEXT_HTML_ELEMENT ?
-                                                           "GeckoView:AccessibilityNext" : "GeckoView:AccessibilityPrevious", data);
+                    nativeProvider.pivot(virtualViewId, arguments != null ?
+                            arguments.getString(AccessibilityNodeInfo.ACTION_ARGUMENT_HTML_ELEMENT_STRING) : "",
+                            false, false);
                     return true;
                 case AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY:
                 case AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY:
@@ -496,11 +484,7 @@ public class SessionAccessibility {
                     // This is mostly here to let TalkBack know we are a legit "WebView".
                     bundle.putCharSequence(
                             "ACTION_ARGUMENT_HTML_ELEMENT_STRING_VALUES",
-                            "ARTICLE,BUTTON,CHECKBOX,COMBOBOX,CONTROL," +
-                                    "FOCUSABLE,FRAME,GRAPHIC,H1,H2,H3,H4,H5,H6," +
-                                    "HEADING,LANDMARK,LINK,LIST,LIST_ITEM,MAIN," +
-                                    "MEDIA,RADIO,SECTION,TABLE,TEXT_FIELD," +
-                                    "UNVISITED_LINK,VISITED_LINK");
+                            TextUtils.join(",", sHtmlGranularities));
                 }
 
 
@@ -564,8 +548,6 @@ public class SessionAccessibility {
     private int mAccessibilityFocusedNode = 0;
     // The current node with focus
     private int mFocusedNode = 0;
-    // A node with no accessibility focus that is currently being hovered.
-    private int mHoveredOnNode = 0;
     // Viewport cache
     final SparseArray<GeckoBundle> mViewportCache = new SparseArray<>();
     // Focus cache
@@ -702,9 +684,6 @@ public class SessionAccessibility {
             final GeckoBundle ret = new GeckoBundle(2);
             ret.putBoolean("touchEnabled", isTouchExplorationEnabled());
             ret.putBoolean("enabled", isEnabled());
-            // "GeckoView:AccessibilitySettings" is dispatched to the Gecko thread.
-            EventDispatcher.getInstance().dispatch("GeckoView:AccessibilitySettings", ret);
-            // "GeckoView:AccessibilityEnabled" is dispatched to the UI thread.
             EventDispatcher.getInstance().dispatch("GeckoView:AccessibilityEnabled", ret);
 
             if (GeckoThread.isStateAtLeast(GeckoThread.State.PROFILE_READY)) {
@@ -740,9 +719,10 @@ public class SessionAccessibility {
 
         requestViewFocus();
 
-        final GeckoBundle data = new GeckoBundle(2);
-        data.putDoubleArray("coordinates", new double[] {event.getRawX(), event.getRawY()});
-        mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityExploreByTouch", data);
+        nativeProvider.exploreByTouch(
+                mAccessibilityFocusedNode != 0 ? mAccessibilityFocusedNode : View.NO_ID,
+                event.getRawX(), event.getRawY());
+
         return true;
     }
 
@@ -826,12 +806,8 @@ public class SessionAccessibility {
                     mAccessibilityFocusedNode = 0;
                 }
                 break;
-            case AccessibilityEvent.TYPE_VIEW_HOVER_ENTER:
-                mHoveredOnNode = sourceId;
-                break;
             case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED:
                 mAccessibilityFocusedNode = sourceId;
-                mHoveredOnNode = 0;
                 break;
             case AccessibilityEvent.TYPE_VIEW_FOCUSED:
                 mFocusedNode = sourceId;
@@ -878,6 +854,16 @@ public class SessionAccessibility {
         @WrapForJNI(dispatchTo = "gecko")
         public native void click(int id);
 
+        public void pivot(final int id, final String granularity, final boolean forward, final boolean inclusive) {
+            pivotNative(id, java.util.Arrays.asList(sHtmlGranularities).indexOf(granularity), forward, inclusive);
+        }
+
+        @WrapForJNI(dispatchTo = "gecko", stubName = "Pivot")
+        public native void pivotNative(int id, int granularity, boolean forward, boolean inclusive);
+
+        @WrapForJNI(dispatchTo = "gecko")
+        public native void exploreByTouch(int id, float x, float y);
+
         @WrapForJNI(calledFrom = "gecko", stubName = "SendEvent")
         private void sendEventNative(final int eventType, final int sourceId, final int className, final GeckoBundle eventData) {
             ThreadUtils.postToUiThread(new Runnable() {