Bug 1137567 - Make nsWindow for Android use TextEventDispatcher; r=esawin r=rbarker r=masayuki r=snorp

Bug 1137567 - 1. Allow dispatching key events during composition; r=esawin

We potentially dispatch key events during composition to provide
compatibility for pages that only listen to key events.

Bug 1137567 - 2. Allow keyboard events in DispatchInputEvent when not on APZ thread; r=rbarker

We use nsIWidget::DispatchInputEvent to dispatch our keyboard events on
the Gecko thread, which on Android is not the APZ controller thread. We
should allow these events to pass instead of crashing.

Bug 1137567 - 3. Add GeckoEditableSupport class to support TextEventDispatcher; r=masayuki

Add a separate GeckoEditableSupport class, which implements
TextEventDispatcherListener and uses TextEventDispatcher for IME
operations. The new class is entirely separate from nsWindow to allow it
to be independently used in content processes as well.

Most of the code is copied from nsWindow::GeckoViewSupport, and adapted
to use TextEventDispatcher.

Bug 1137567 - 4. Make nsWindow::WindowPtr available for outside classes; r=snorp

Make nsWindow::WindowPtr available not just for classes inside nsWindow
but for outside classes as well. Also, add support for RefPtr native
objects to nsWindow::NativePtr.

Bug 1137567 - 5. Use GeckoEditableSupport in nsWindow; r=esawin

Use the new GeckoEditableSupport class in nsWindow to replace the
previous code in nsWindow::GeckoViewSupport. GeckoEditable native
methods now go to GeckoEditableSupport instead of GeckoViewSupport.

Several native methods in GeckoEditable are changed from
dispatchTo="proxy" to dispatchTo="gecko", because we no longer need the
special nsWindow::WindowEvent wrapper for our native calls.

mobile/android/app/mobile.js

@@ -920,3 +920,5 @@ pref("dom.audiochannel.mediaControl", true);
 pref("webchannel.allowObject.urlWhitelist", "https://accounts.firefox.com https://content.cdn.mozilla.net https://input.mozilla.org https://support.mozilla.org https://install.mozilla.org");
 
 pref("media.openUnsupportedTypeWithExternalApp", true);
+
+pref("dom.keyboardevent.dispatch_during_composition", true);

mobile/android/geckoview/src/main/java/org/mozilla/gecko/GeckoEditable.java

@@ -93,11 +93,11 @@ final class GeckoEditable extends JNIObject
     private static final int IME_RANGE_BACKCOLOR = 4;
     private static final int IME_RANGE_LINECOLOR = 8;
 
-    @WrapForJNI(dispatchTo = "proxy")
+    @WrapForJNI(dispatchTo = "proxy") // Dispatch a UI-activity event through proxy.
     private native void onKeyEvent(int action, int keyCode, int scanCode, int metaState,
-                                   long time, int unicodeChar, int baseUnicodeChar,
-                                   int domPrintableKeyValue, int repeatCount, int flags,
-                                   boolean isSynthesizedImeKey, KeyEvent event);
+                                   int keyPressMetaState, long time, int domPrintableKeyValue,
+                                   int repeatCount, int flags, boolean isSynthesizedImeKey,
+                                   KeyEvent event);
 
     private void onKeyEvent(KeyEvent event, int action, int savedMetaState,
                             boolean isSynthesizedImeKey) {
@@ -112,35 +112,40 @@ final class GeckoEditable extends JNIObject
         final int metaState = event.getMetaState() | savedMetaState;
         final int unmodifiedMetaState = metaState &
                 ~(KeyEvent.META_ALT_MASK | KeyEvent.META_CTRL_MASK | KeyEvent.META_META_MASK);
+
         final int unicodeChar = event.getUnicodeChar(metaState);
+        final int unmodifiedUnicodeChar = event.getUnicodeChar(unmodifiedMetaState);
         final int domPrintableKeyValue =
                 unicodeChar >= ' '               ? unicodeChar :
-                unmodifiedMetaState != metaState ? event.getUnicodeChar(unmodifiedMetaState) :
-                                                   0;
+                unmodifiedMetaState != metaState ? unmodifiedUnicodeChar : 0;
+
+        // If a modifier (e.g. meta key) caused a different character to be entered, we
+        // drop that modifier from the metastate for the generated keypress event.
+        final int keyPressMetaState = (unicodeChar >= ' ' &&
+                unicodeChar != unmodifiedUnicodeChar) ? unmodifiedMetaState : metaState;
+
         onKeyEvent(action, event.getKeyCode(), event.getScanCode(),
-                   metaState, event.getEventTime(), unicodeChar,
-                   // e.g. for Ctrl+A, Android returns 0 for unicodeChar,
-                   // but Gecko expects 'a', so we return that in baseUnicodeChar.
-                   event.getUnicodeChar(0), domPrintableKeyValue, event.getRepeatCount(),
-                   event.getFlags(), isSynthesizedImeKey, event);
+                   metaState, keyPressMetaState, event.getEventTime(),
+                   domPrintableKeyValue, event.getRepeatCount(), event.getFlags(),
+                   isSynthesizedImeKey, event);
     }
 
-    @WrapForJNI(dispatchTo = "proxy")
+    @WrapForJNI(dispatchTo = "gecko")
     private native void onImeSynchronize();
 
-    @WrapForJNI(dispatchTo = "proxy")
+    @WrapForJNI(dispatchTo = "proxy") // Dispatch a UI-activity event through proxy.
     private native void onImeReplaceText(int start, int end, String text);
 
-    @WrapForJNI(dispatchTo = "proxy")
+    @WrapForJNI(dispatchTo = "gecko")
     private native void onImeAddCompositionRange(int start, int end, int rangeType,
                                                  int rangeStyles, int rangeLineStyle,
                                                  boolean rangeBoldLine, int rangeForeColor,
                                                  int rangeBackColor, int rangeLineColor);
 
-    @WrapForJNI(dispatchTo = "proxy")
+    @WrapForJNI(dispatchTo = "proxy") // Dispatch a UI-activity event through proxy.
     private native void onImeUpdateComposition(int start, int end);
 
-    @WrapForJNI(dispatchTo = "proxy")
+    @WrapForJNI(dispatchTo = "gecko")
     private native void onImeRequestCursorUpdates(int requestMode);
 
     /**
@@ -582,7 +587,7 @@ final class GeckoEditable extends JNIObject
         onViewChange(v);
     }
 
-    @WrapForJNI(dispatchTo = "proxy") @Override
+    @WrapForJNI(dispatchTo = "gecko") @Override
     protected native void disposeNative();
 
     @WrapForJNI(calledFrom = "gecko")

widget/android/GeckoEditableSupport.h

@@ -0,0 +1,217 @@
+/* -*- Mode: c++; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_widget_GeckoEditableSupport_h
+#define mozilla_widget_GeckoEditableSupport_h
+
+#include "GeneratedJNIWrappers.h"
+#include "nsAppShell.h"
+#include "nsIWidget.h"
+#include "nsTArray.h"
+
+#include "mozilla/TextEventDispatcher.h"
+#include "mozilla/TextEventDispatcherListener.h"
+#include "mozilla/UniquePtr.h"
+
+class nsWindow;
+
+namespace mozilla {
+
+class TextComposition;
+
+namespace widget {
+
+class GeckoEditableSupport final
+    : public TextEventDispatcherListener
+    , public java::GeckoEditable::Natives<GeckoEditableSupport>
+{
+    /*
+        Rules for managing IME between Gecko and Java:
+
+        * Gecko controls the text content, and Java shadows the Gecko text
+           through text updates
+        * Gecko and Java maintain separate selections, and synchronize when
+           needed through selection updates and set-selection events
+        * Java controls the composition, and Gecko shadows the Java
+           composition through update composition events
+    */
+
+    using EditableBase = java::GeckoEditable::Natives<GeckoEditableSupport>;
+
+    // RAII helper class that automatically sends an event reply through
+    // OnImeSynchronize, as required by events like OnImeReplaceText.
+    class AutoIMESynchronize
+    {
+        GeckoEditableSupport* const mGES;
+    public:
+        AutoIMESynchronize(GeckoEditableSupport* ges) : mGES(ges) {}
+        ~AutoIMESynchronize() { mGES->OnImeSynchronize(); }
+    };
+
+    struct IMETextChange final {
+        int32_t mStart, mOldEnd, mNewEnd;
+
+        IMETextChange() :
+            mStart(-1), mOldEnd(-1), mNewEnd(-1) {}
+
+        IMETextChange(const IMENotification& aIMENotification)
+            : mStart(aIMENotification.mTextChangeData.mStartOffset)
+            , mOldEnd(aIMENotification.mTextChangeData.mRemovedEndOffset)
+            , mNewEnd(aIMENotification.mTextChangeData.mAddedEndOffset)
+        {
+            MOZ_ASSERT(aIMENotification.mMessage == NOTIFY_IME_OF_TEXT_CHANGE,
+                       "IMETextChange initialized with wrong notification");
+            MOZ_ASSERT(aIMENotification.mTextChangeData.IsValid(),
+                       "The text change notification isn't initialized");
+            MOZ_ASSERT(aIMENotification.mTextChangeData.IsInInt32Range(),
+                       "The text change notification is out of range");
+        }
+
+        bool IsEmpty() const { return mStart < 0; }
+    };
+
+    enum FlushChangesFlag
+    {
+        // Not retrying.
+        FLUSH_FLAG_NONE,
+        // Retrying due to IME text changes during flush.
+        FLUSH_FLAG_RETRY,
+        // Retrying due to IME sync exceptions during flush.
+        FLUSH_FLAG_RECOVER
+    };
+
+    enum RemoveCompositionFlag
+    {
+        CANCEL_IME_COMPOSITION,
+        COMMIT_IME_COMPOSITION
+    };
+
+    nsWindow::WindowPtr<GeckoEditableSupport> mWindow; // Parent only
+    RefPtr<TextEventDispatcher> mDispatcher;
+    java::GeckoEditable::GlobalRef mEditable;
+    InputContext mInputContext;
+    AutoTArray<UniquePtr<mozilla::WidgetEvent>, 4> mIMEKeyEvents;
+    AutoTArray<IMETextChange, 4> mIMETextChanges;
+    RefPtr<TextRangeArray> mIMERanges;
+    int32_t mIMEMaskEventsCount; // Mask events when > 0.
+    bool mIMEUpdatingContext;
+    bool mIMESelectionChanged;
+    bool mIMETextChangedDuringFlush;
+    bool mIMEMonitorCursor;
+
+    nsIWidget* GetWidget() const
+    {
+        return mDispatcher ? mDispatcher->GetWidget() : mWindow;
+    }
+
+    virtual ~GeckoEditableSupport() {}
+
+    RefPtr<TextComposition> GetComposition() const;
+    void RemoveComposition(
+            RemoveCompositionFlag aFlag = COMMIT_IME_COMPOSITION);
+    void SendIMEDummyKeyEvent(nsIWidget* aWidget, EventMessage msg);
+    void AddIMETextChange(const IMETextChange& aChange);
+    void PostFlushIMEChanges();
+    void FlushIMEChanges(FlushChangesFlag aFlags = FLUSH_FLAG_NONE);
+    void FlushIMEText(FlushChangesFlag aFlags = FLUSH_FLAG_NONE);
+    void AsyncNotifyIME(int32_t aNotification);
+    void UpdateCompositionRects();
+
+public:
+    template<typename Functor>
+    static void OnNativeCall(Functor&& aCall)
+    {
+        struct IMEEvent : nsAppShell::LambdaEvent<Functor>
+        {
+            using nsAppShell::LambdaEvent<Functor>::LambdaEvent;
+
+            nsAppShell::Event::Type ActivityType() const override
+            {
+                return nsAppShell::Event::Type::kUIActivity;
+            }
+        };
+        nsAppShell::PostEvent(mozilla::MakeUnique<IMEEvent>(
+                mozilla::Move(aCall)));
+    }
+
+    GeckoEditableSupport(nsWindow::NativePtr<GeckoEditableSupport>* aPtr,
+                         nsWindow* aWindow,
+                         java::GeckoEditable::Param aEditable)
+        : mWindow(aPtr, aWindow)
+        , mEditable(aEditable)
+        , mIMERanges(new TextRangeArray())
+        , mIMEMaskEventsCount(1) // Mask IME events since there's no focus yet
+        , mIMEUpdatingContext(false)
+        , mIMESelectionChanged(false)
+        , mIMETextChangedDuringFlush(false)
+        , mIMEMonitorCursor(false)
+    {}
+
+    NS_DECL_ISUPPORTS
+
+    // TextEventDispatcherListener methods
+    NS_IMETHOD NotifyIME(TextEventDispatcher* aTextEventDispatcher,
+                         const IMENotification& aNotification) override;
+
+    NS_IMETHOD_(void) OnRemovedFrom(
+            TextEventDispatcher* aTextEventDispatcher) override;
+
+    NS_IMETHOD_(void) WillDispatchKeyboardEvent(
+            TextEventDispatcher* aTextEventDispatcher,
+            WidgetKeyboardEvent& aKeyboardEvent,
+            uint32_t aIndexOfKeypress,
+            void* aData) override;
+
+    nsIMEUpdatePreference GetIMEUpdatePreference();
+
+    void SetInputContext(const InputContext& aContext,
+                         const InputContextAction& aAction);
+
+    InputContext GetInputContext();
+
+    // GeckoEditable methods
+    using EditableBase::AttachNative;
+    using EditableBase::DisposeNative;
+
+    void OnDetach() {
+        mEditable->OnViewChange(nullptr);
+    }
+
+    void OnViewChange(java::GeckoView::Param aView) {
+        mEditable->OnViewChange(aView);
+    }
+
+    // Handle an Android KeyEvent.
+    void OnKeyEvent(int32_t aAction, int32_t aKeyCode, int32_t aScanCode,
+                    int32_t aMetaState, int32_t aKeyPressMetaState,
+                    int64_t aTime, int32_t aDomPrintableKeyValue,
+                    int32_t aRepeatCount, int32_t aFlags,
+                    bool aIsSynthesizedImeKey,
+                    jni::Object::Param originalEvent);
+
+    // Synchronize Gecko thread with the InputConnection thread.
+    void OnImeSynchronize();
+
+    // Replace a range of text with new text.
+    void OnImeReplaceText(int32_t aStart, int32_t aEnd,
+                          jni::String::Param aText);
+
+    // Add styling for a range within the active composition.
+    void OnImeAddCompositionRange(int32_t aStart, int32_t aEnd,
+            int32_t aRangeType, int32_t aRangeStyle, int32_t aRangeLineStyle,
+            bool aRangeBoldLine, int32_t aRangeForeColor,
+            int32_t aRangeBackColor, int32_t aRangeLineColor);
+
+    // Update styling for the active composition using previous-added ranges.
+    void OnImeUpdateComposition(int32_t aStart, int32_t aEnd);
+
+    // Set cursor mode whether IME requests
+    void OnImeRequestCursorUpdates(int aRequestMode);
+};
+
+} // namespace widget
+} // namespace mozill
+
+#endif // mozilla_widget_GeckoEditableSupport_h

widget/android/GeneratedJNIWrappers.h

@@ -2112,7 +2112,7 @@ public:
         static const mozilla::jni::CallingThread callingThread =
                 mozilla::jni::CallingThread::ANY;
         static const mozilla::jni::DispatchTarget dispatchTarget =
-                mozilla::jni::DispatchTarget::PROXY;
+                mozilla::jni::DispatchTarget::GECKO;
     };
 
     struct NotifyIME_t {
@@ -2201,7 +2201,7 @@ public:
         static const mozilla::jni::CallingThread callingThread =
                 mozilla::jni::CallingThread::ANY;
         static const mozilla::jni::DispatchTarget dispatchTarget =
-                mozilla::jni::DispatchTarget::PROXY;
+                mozilla::jni::DispatchTarget::GECKO;
     };
 
     struct OnImeReplaceText_t {
@@ -2239,7 +2239,7 @@ public:
         static const mozilla::jni::CallingThread callingThread =
                 mozilla::jni::CallingThread::ANY;
         static const mozilla::jni::DispatchTarget dispatchTarget =
-                mozilla::jni::DispatchTarget::PROXY;
+                mozilla::jni::DispatchTarget::GECKO;
     };
 
     struct OnImeSynchronize_t {
@@ -2256,7 +2256,7 @@ public:
         static const mozilla::jni::CallingThread callingThread =
                 mozilla::jni::CallingThread::ANY;
         static const mozilla::jni::DispatchTarget dispatchTarget =
-                mozilla::jni::DispatchTarget::PROXY;
+                mozilla::jni::DispatchTarget::GECKO;
     };
 
     struct OnImeUpdateComposition_t {
@@ -2287,17 +2287,16 @@ public:
                 int32_t,
                 int32_t,
                 int32_t,
+                int32_t,
                 int64_t,
                 int32_t,
                 int32_t,
                 int32_t,
-                int32_t,
-                int32_t,
                 bool,
                 mozilla::jni::Object::Param> Args;
         static constexpr char name[] = "onKeyEvent";
         static constexpr char signature[] =
-                "(IIIIJIIIIIZLandroid/view/KeyEvent;)V";
+                "(IIIIIJIIIZLandroid/view/KeyEvent;)V";
         static const bool isStatic = false;
         static const mozilla::jni::ExceptionMode exceptionMode =
                 mozilla::jni::ExceptionMode::ABORT;

widget/android/moz.build

@@ -40,6 +40,7 @@ UNIFIED_SOURCES += [
     'AndroidUiThread.cpp',
     'ANRReporter.cpp',
     'EventDispatcher.cpp',
+    'GeckoEditableSupport.cpp',
     'GeneratedJNIWrappers.cpp',
     'GfxInfo.cpp',
     'nsAndroidProtocolHandler.cpp',

widget/android/nsWindow.h

@@ -23,7 +23,6 @@
 struct ANPEvent;
 
 namespace mozilla {
-    class TextComposition;
     class WidgetTouchEvent;
 
     namespace layers {
@@ -31,6 +30,10 @@ namespace mozilla {
         class LayerManager;
         class APZCTreeManager;
     }
+
+    namespace widget {
+        class GeckoEditableSupport;
+    } // namespace widget
 }
 
 class nsWindow : public nsBaseWidget
@@ -58,6 +61,7 @@ private:
              class Impl = typename Lambda::TargetClass>
     class WindowEvent;
 
+public:
     // Smart pointer for holding a pointer back to the nsWindow inside a native
     // object class. The nsWindow pointer is automatically cleared when the
     // nsWindow is destroyed, and a WindowPtr<Impl>::Locked class is provided
@@ -96,6 +100,59 @@ private:
         void Detach();
     };
 
+    template<class Impl>
+    class WindowPtr final
+    {
+        friend NativePtr<Impl>;
+
+        NativePtr<Impl>* mPtr;
+        nsWindow* mWindow;
+        mozilla::Mutex mWindowLock;
+
+    public:
+        class Locked final : private mozilla::MutexAutoLock
+        {
+            nsWindow* const mWindow;
+
+        public:
+            Locked(WindowPtr<Impl>& aPtr)
+                : mozilla::MutexAutoLock(aPtr.mWindowLock)
+                , mWindow(aPtr.mWindow)
+            {}
+
+            operator nsWindow*() const { return mWindow; }
+            nsWindow* operator->() const { return mWindow; }
+        };
+
+        WindowPtr(NativePtr<Impl>* aPtr, nsWindow* aWindow)
+            : mPtr(aPtr)
+            , mWindow(aWindow)
+            , mWindowLock(NativePtr<Impl>::sName)
+        {
+            MOZ_ASSERT(NS_IsMainThread());
+            mPtr->mPtr = this;
+        }
+
+        ~WindowPtr()
+        {
+            MOZ_ASSERT(NS_IsMainThread());
+            if (!mPtr) {
+                return;
+            }
+            mPtr->mPtr = nullptr;
+            mPtr->mImpl = nullptr;
+        }
+
+        operator nsWindow*() const
+        {
+            MOZ_ASSERT(NS_IsMainThread());
+            return mWindow;
+        }
+
+        nsWindow* operator->() const { return operator nsWindow*(); }
+    };
+
+private:
     class AndroidView final : public nsIAndroidView
     {
         virtual ~AndroidView() {}
@@ -126,6 +183,10 @@ private:
     // Owned by the Java NativePanZoomController instance.
     NativePtr<NPZCSupport> mNPZCSupport;
 
+    // Object that implements native GeckoEditable calls.
+    // Strong referenced by the Java instance.
+    NativePtr<mozilla::widget::GeckoEditableSupport> mEditableSupport;
+
     class GeckoViewSupport;
     // Object that implements native GeckoView calls and associated states.
     // nullptr for nsWindows that were not opened from GeckoView.
@@ -139,6 +200,9 @@ private:
 public:
     static nsWindow* TopWindow();
 
+    static mozilla::Modifiers GetModifiers(int32_t aMetaState);
+    static mozilla::TimeStamp GetEventTimeStamp(int64_t aEventTime);
+
     void OnSizeChanged(const mozilla::gfx::IntSize& aSize);
 
     void InitEvent(mozilla::WidgetGUIEvent& event,
@@ -202,6 +266,8 @@ public:
     virtual nsresult SetTitle(const nsAString& aTitle) override { return NS_OK; }
     virtual MOZ_MUST_USE nsresult GetAttention(int32_t aCycleCount) override { return NS_ERROR_NOT_IMPLEMENTED; }
 
+
+    TextEventDispatcherListener* GetNativeTextEventDispatcherListener() override;
     virtual void SetInputContext(const InputContext& aContext,
                                  const InputContextAction& aAction) override;
     virtual InputContext GetInputContext() override;
@@ -244,27 +310,20 @@ public:
 
     mozilla::jni::DependentRef<mozilla::java::GeckoLayerClient> GetLayerClient();
 
+    // Call this function when the users activity is the direct cause of an
+    // event (like a keypress or mouse click).
+    void UserActivity();
+
 protected:
     void BringToFront();
     nsWindow *FindTopLevel();
     bool IsTopLevel();
 
-    RefPtr<mozilla::TextComposition> GetIMEComposition();
-    enum RemoveIMECompositionFlag {
-        CANCEL_IME_COMPOSITION,
-        COMMIT_IME_COMPOSITION
-    };
-    void RemoveIMEComposition(RemoveIMECompositionFlag aFlag = COMMIT_IME_COMPOSITION);
-
     void ConfigureAPZControllerThread() override;
     void DispatchHitTest(const mozilla::WidgetTouchEvent& aEvent);
 
     already_AddRefed<GeckoContentController> CreateRootContentController() override;
 
-    // Call this function when the users activity is the direct cause of an
-    // event (like a keypress or mouse click).
-    void UserActivity();
-
     bool mIsVisible;
     nsTArray<nsWindow*> mChildren;
     nsWindow* mParent;
@@ -277,9 +336,6 @@ protected:
     bool mAwaitingFullScreen;
     bool mIsFullScreen;
 
-    virtual nsresult NotifyIMEInternal(
-                         const IMENotification& aIMENotification) override;
-
     bool UseExternalCompositingSurface() const override {
       return true;
     }

widget/nsBaseWidget.cpp

@@ -1223,7 +1223,8 @@ nsBaseWidget::DispatchInputEvent(WidgetInputEvent* aEvent)
       APZThreadUtils::RunOnControllerThread(r.forget());
       return nsEventStatus_eConsumeDoDefault;
     }
-    MOZ_CRASH();
+    // Allow dispatching keyboard events on Gecko thread.
+    MOZ_ASSERT(aEvent->AsKeyboardEvent());
   }
 
   nsEventStatus status;