From 685f2cf56fa9d6168b519504156a9e9f9034c38c Mon Sep 17 00:00:00 2001 From: Manuel Martin Date: Tue, 17 Mar 2020 11:43:40 +0100 Subject: [PATCH] Dual controller typing (#2936) * Dual hand typing * Fix for the keyboard dismiss when pressing two keys simultaneously * Fix for the key flicker when another key is pressed --- .../mozilla/vrbrowser/VRBrowserActivity.java | 12 +- .../vrbrowser/input/MotionEventGenerator.java | 69 ++++++--- .../ui/views/CustomKeyboardView.java | 33 ++++- .../vrbrowser/ui/widgets/KeyboardWidget.java | 133 +++++++++++++++++- .../mozilla/vrbrowser/ui/widgets/Widget.java | 1 + app/src/main/cpp/BrowserWorld.cpp | 15 +- app/src/main/cpp/ControllerContainer.cpp | 19 +-- app/src/main/cpp/VRBrowser.cpp | 6 +- app/src/main/cpp/VRBrowser.h | 2 +- .../main/res/layout/autocompletion_bar.xml | 2 +- app/src/main/res/layout/keyboard.xml | 45 ++++-- 11 files changed, 260 insertions(+), 77 deletions(-) diff --git a/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java b/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java index f589edd35..837c470cf 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java @@ -807,7 +807,7 @@ void dispatchCreateWidgetLayer(final int aHandle, final Surface aSurface, final @Keep @SuppressWarnings("unused") - void handleMotionEvent(final int aHandle, final int aDevice, final boolean aPressed, final float aX, final float aY) { + void handleMotionEvent(final int aHandle, final int aDevice, final boolean aFocused, final boolean aPressed, final float aX, final float aY) { runOnUiThread(() -> { Widget widget = mWidgets.get(aHandle); if (!isWidgetInputEnabled(widget)) { @@ -819,12 +819,14 @@ void handleMotionEvent(final int aHandle, final int aDevice, final boolean aPres final float y = aY / scale; if (widget == null) { - MotionEventGenerator.dispatch(mRootWidget, aDevice, aPressed, x, y); + MotionEventGenerator.dispatch(mRootWidget, aDevice, aFocused, aPressed, x, y); + } else if (widget.getBorderWidth() > 0) { final int border = widget.getBorderWidth(); - MotionEventGenerator.dispatch(widget, aDevice, aPressed, x - border, y - border); + MotionEventGenerator.dispatch(widget, aDevice, aFocused, aPressed, x - border, y - border); + } else { - MotionEventGenerator.dispatch(widget, aDevice, aPressed, x, y); + MotionEventGenerator.dispatch(widget, aDevice, aFocused, aPressed, x, y); } }); } @@ -839,7 +841,7 @@ void handleScrollEvent(final int aHandle, final int aDevice, final float aX, fin } if (widget != null) { float scrollDirection = mSettings.getScrollDirection() == 0 ? 1.0f : -1.0f; - MotionEventGenerator.dispatchScroll(widget, aDevice, aX * scrollDirection, aY * scrollDirection); + MotionEventGenerator.dispatchScroll(widget, aDevice, true,aX * scrollDirection, aY * scrollDirection); } else { Log.e(LOGTAG, "Failed to find widget for scroll event: " + aHandle); } diff --git a/app/src/common/shared/org/mozilla/vrbrowser/input/MotionEventGenerator.java b/app/src/common/shared/org/mozilla/vrbrowser/input/MotionEventGenerator.java index a3f000ece..3f1d64b62 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/input/MotionEventGenerator.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/input/MotionEventGenerator.java @@ -7,15 +7,14 @@ import android.os.SystemClock; import android.util.Log; -import android.view.MotionEvent; -import android.view.InputDevice; import android.util.SparseArray; +import android.view.InputDevice; +import android.view.MotionEvent; import org.mozilla.vrbrowser.ui.widgets.Widget; import org.mozilla.vrbrowser.utils.SystemUtils; import java.util.Arrays; -import java.util.List; public class MotionEventGenerator { static final String LOGTAG = SystemUtils.createLogtag(MotionEventGenerator.class); @@ -53,11 +52,11 @@ static class Device { private static SparseArray devices = new SparseArray<>(); - private static void generateEvent(Widget aWidget, Device aDevice, int aAction, boolean aGeneric) { - generateEvent(aWidget, aDevice, aAction, aGeneric, aDevice.mCoords); + private static void generateEvent(Widget aWidget, Device aDevice, boolean aFocused, int aAction, boolean aGeneric) { + generateEvent(aWidget, aDevice, aFocused, aAction, aGeneric, aDevice.mCoords); } - private static void generateEvent(Widget aWidget, Device aDevice, int aAction, boolean aGeneric, MotionEvent.PointerCoords[] aCoords) { + private static void generateEvent(Widget aWidget, Device aDevice, boolean aFocused, int aAction, boolean aGeneric, MotionEvent.PointerCoords[] aCoords) { MotionEvent event = MotionEvent.obtain( /*mDownTime*/ aDevice.mDownTime, /*eventTime*/ SystemClock.uptimeMillis(), @@ -69,19 +68,24 @@ private static void generateEvent(Widget aWidget, Device aDevice, int aAction, b /*buttonState*/ 0, /*xPrecision*/ 0, /*yPrecision*/ 0, - /*deviceId*/ 0, // aDevice.mDevice, + /*deviceId*/ aDevice.mDevice, /*edgeFlags*/ 0, /*source*/ InputDevice.SOURCE_TOUCHSCREEN, /*flags*/ 0); if (aGeneric) { - aWidget.handleHoverEvent(event); + if (aWidget.supportsMultipleInputDevices()) { + aWidget.handleHoverEvent(event); + + } else if (aFocused) { + aWidget.handleHoverEvent(event); + } } else { aWidget.handleTouchEvent(event); } event.recycle(); } - public static void dispatch(Widget aWidget, int aDevice, boolean aPressed, float aX, float aY) { + public static void dispatch(Widget aWidget, int aDevice, boolean aFocused, boolean aPressed, float aX, float aY) { Device device = devices.get(aDevice); if (device == null) { device = new Device(aDevice); @@ -99,10 +103,10 @@ public static void dispatch(Widget aWidget, int aDevice, boolean aPressed, float } if (!aPressed && (device.mPreviousWidget != null) && (device.mPreviousWidget != aWidget)) { if (device.mWasPressed) { - generateEvent(device.mPreviousWidget, device, MotionEvent.ACTION_CANCEL, false); + generateEvent(device.mPreviousWidget, device, aFocused, MotionEvent.ACTION_CANCEL, false); device.mWasPressed = false; } - generateEvent(device.mPreviousWidget, device, MotionEvent.ACTION_HOVER_EXIT, true, device.mMouseOutCoords); + generateEvent(device.mPreviousWidget, device, aFocused, MotionEvent.ACTION_HOVER_EXIT, true, device.mMouseOutCoords); device.mPreviousWidget = null; } if (aWidget == null) { @@ -110,22 +114,27 @@ public static void dispatch(Widget aWidget, int aDevice, boolean aPressed, float return; } if (aWidget != device.mPreviousWidget && !aPressed) { - generateEvent(aWidget, device, MotionEvent.ACTION_HOVER_ENTER, true); + generateEvent(aWidget, device, aFocused, MotionEvent.ACTION_HOVER_ENTER, true); } if (aPressed && !device.mWasPressed) { device.mDownTime = SystemClock.uptimeMillis(); device.mWasPressed = true; - generateEvent(aWidget, device, MotionEvent.ACTION_HOVER_EXIT, true); - generateEvent(aWidget, device, MotionEvent.ACTION_DOWN, false); + if (!isOtherDeviceDown(device.mDevice)) { + generateEvent(aWidget, device, aFocused, MotionEvent.ACTION_HOVER_EXIT, true); + generateEvent(aWidget, device, aFocused, MotionEvent.ACTION_DOWN, false); + } device.mTouchStartWidget = aWidget; } else if (!aPressed && device.mWasPressed) { device.mWasPressed = false; - generateEvent(device.mTouchStartWidget, device, MotionEvent.ACTION_UP, false); - generateEvent(aWidget, device, MotionEvent.ACTION_HOVER_ENTER, true); + if (!isOtherDeviceDown(device.mDevice)) { + generateEvent(device.mTouchStartWidget, device, aFocused, MotionEvent.ACTION_UP, false); + generateEvent(aWidget, device, aFocused, MotionEvent.ACTION_HOVER_ENTER, true); + } + device.mTouchStartWidget = null; } else if (moving && aPressed) { - generateEvent(aWidget, device, MotionEvent.ACTION_MOVE, false); + generateEvent(aWidget, device, aFocused, MotionEvent.ACTION_MOVE, false); } else if (moving) { - generateEvent(aWidget, device, MotionEvent.ACTION_HOVER_MOVE, true); + generateEvent(aWidget, device, aFocused, MotionEvent.ACTION_HOVER_MOVE, true); } else { Log.e("VRB", "Unknown touch event action"); return; @@ -133,7 +142,27 @@ public static void dispatch(Widget aWidget, int aDevice, boolean aPressed, float device.mPreviousWidget = aWidget; } - public static void dispatchScroll(Widget aWidget, int aDevice, float aX, float aY) { + /** + * Checks if any other device has an ongoing touch down event. + * Android throw away all previous state when starting a new touch gesture + * and this seem to make the previous touch to be sent up the view hierarchy. + * To avoid this we check if any other device has a button down before sending + * touch down/up event. + * @param deviceId Device Id to filter + * @return true if any other device has a button down, false otherwise + */ + private static boolean isOtherDeviceDown(int deviceId) { + boolean result = false; + for (int i=0; i dismiss()); + mCloseKeyboardButton.setOnHoverListener(new ControlButtonHoverListener()); mKeyboardMoveButton = findViewById(R.id.keyboardMoveButton); mKeyboardMoveButton.setOnTouchListener(new MoveTouchListener()); - mKeyboardMoveButton.setOnHoverListener(new MoveHoverListener()); + mKeyboardMoveButton.setOnHoverListener(new ControlButtonHoverListener()); mKeyWidth = getResources().getDimensionPixelSize(R.dimen.keyboard_key_width); mKeyboardPopupTopMargin = getResources().getDimensionPixelSize(R.dimen.keyboard_key_pressed_padding) * 2; @@ -255,6 +350,8 @@ private void initialize(Context aContext) { mKeyboardView.setVisibility(View.VISIBLE); mKeyboardNumericView.setKeyboard(mKeyboardNumeric); + mAutocompletionLayer.setOnClickListener(view -> hideOverlays()); + mKeyboardNumericLayer.setOnClickListener(view -> hideOverlays()); mPopupKeyboardLayer.setOnClickListener(view -> hideOverlays()); hideOverlays(); @@ -381,7 +478,9 @@ public void updateFocusedView(View aFocusedView) { } public void dismiss() { - if (mPopupKeyboardLayer.getVisibility() == VISIBLE) { + if (mPopupKeyboardLayer.getVisibility() == VISIBLE || + mAutocompletionLayer.getVisibility() == VISIBLE || + mKeyboardNumericLayer.getVisibility() == VISIBLE) { hideOverlays(); return; } @@ -420,8 +519,14 @@ public void proxifyLayerIfNeeded(ArrayList aWindows) { private void hideOverlays() { mPopupKeyboardView.setVisibility(View.GONE); mPopupKeyboardLayer.setVisibility(View.GONE); + mAutocompletionLayer.setVisibility(View.GONE); + mKeyboardNumericLayer.setVisibility(View.GONE); mLanguageSelectorView.setVisibility(View.GONE); mDomainSelectorView.setVisibility(View.GONE); + + mPopUpHoverDeviceId = -1; + mLanguageHoverDeviceId = -1; + mDomainHoverDeviceId = -1; } protected void onDismiss() { @@ -468,6 +573,9 @@ public void onLongPress(Keyboard.Key popupKey) { mPopupKeyboardView.setShifted(mIsCapsLock || mKeyboardView.isShifted()); mPopupKeyboardView.setVisibility(View.VISIBLE); mPopupKeyboardLayer.setVisibility(View.VISIBLE); + mKeyboardNumericLayer.setVisibility(View.VISIBLE); + mAutocompletionLayer.setVisibility(mAutoCompletionView.getVisibility()); + mPopUpHoverDeviceId = -1; mIsLongPress = true; @@ -647,7 +755,12 @@ private void handleShift(boolean isShifted) { } } } - mKeyboardView.setShifted(isShifted || mIsCapsLock); + + // setShifted trigger a full keyboard redraw. + // To avoid this we only call setShifted if it's state has changed. + if (mKeyboardView.isShifted() != isShifted) { + mKeyboardView.setShifted(isShifted || mIsCapsLock); + } } private void handleBackspace() { @@ -699,6 +812,9 @@ private void handleGlobeClick() { mLanguageSelectorView.setSelectedItem(mCurrentKeyboard); mLanguageSelectorView.setVisibility(View.VISIBLE); mPopupKeyboardLayer.setVisibility(View.VISIBLE); + mKeyboardNumericLayer.setVisibility(View.VISIBLE); + mAutocompletionLayer.setVisibility(mAutoCompletionView.getVisibility()); + mLanguageHoverDeviceId = -1; } private void handleEmojiInput() { @@ -716,6 +832,9 @@ private void handleDomain() { mDomainSelectorView.setVisibility(View.VISIBLE); mPopupKeyboardLayer.setVisibility(View.VISIBLE); + mKeyboardNumericLayer.setVisibility(View.VISIBLE); + mAutocompletionLayer.setVisibility(mAutoCompletionView.getVisibility()); + mDomainHoverDeviceId = -1; } private void handleLanguageChange(KeyboardInterface aKeyboard) { @@ -988,6 +1107,7 @@ private void updateSpaceBarLanguageLabel() { private void setAutoCompletionVisible(boolean aVisible) { mAutoCompletionView.setVisibility(aVisible ? View.VISIBLE : View.GONE); + mAutoCompleteHoverDeviceId = -1; } // Must be called in the input thread, see postInputCommand. @@ -1073,6 +1193,11 @@ public boolean dispatchKeyEvent(final KeyEvent event) { return false; } + @Override + public boolean supportsMultipleInputDevices() { + return true; + } + // GeckoSession.TextInputDelegate @Override diff --git a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/Widget.java b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/Widget.java index b812515c1..85332a408 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/Widget.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/Widget.java @@ -38,4 +38,5 @@ public interface Widget { default void detachFromWindow() {} default void attachToWindow(@NonNull WindowWidget window) {} int getBorderWidth(); + default boolean supportsMultipleInputDevices() { return false; } } diff --git a/app/src/main/cpp/BrowserWorld.cpp b/app/src/main/cpp/BrowserWorld.cpp index a30b5213f..0ed55a3b6 100644 --- a/app/src/main/cpp/BrowserWorld.cpp +++ b/app/src/main/cpp/BrowserWorld.cpp @@ -361,8 +361,6 @@ BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) { if (focusRequested) { ChangeControllerFocus(controller); } - controller.lastButtonState = controller.buttonState; - continue; } const vrb::Vector start = controller.StartPoint(); @@ -429,7 +427,7 @@ BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) { aRelayoutWidgets = true; } } - } else if (hitWidget && hitWidget->IsResizing()) { + } else if (hitWidget && hitWidget->IsResizing() && controller.focused) { bool aResized = false, aResizeEnded = false; hitWidget->HandleResize(hitPoint, pressed, aResized, aResizeEnded); @@ -464,8 +462,7 @@ BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) { controller.widget = handle; controller.pointerX = theX; controller.pointerY = theY; - VRBrowser::HandleMotionEvent(handle, controller.index, jboolean(pressed), - controller.pointerX, controller.pointerY); + VRBrowser::HandleMotionEvent(handle, controller.index, jboolean(controller.focused), jboolean(pressed), controller.pointerX, controller.pointerY); } if ((controller.scrollDeltaX != 0.0f) || controller.scrollDeltaY != 0.0f) { if (controller.scrollStart < 0.0) { @@ -498,18 +495,18 @@ BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) { } } } else if (controller.widget) { - VRBrowser::HandleMotionEvent(0, controller.index, (jboolean) pressed, 0.0f, 0.0f); + VRBrowser::HandleMotionEvent(0, controller.index, jboolean(controller.focused), (jboolean) pressed, 0.0f, 0.0f); controller.widget = 0; } else if (wasPressed != pressed) { - VRBrowser::HandleMotionEvent(0, controller.index, (jboolean) pressed, 0.0f, 0.0f); + VRBrowser::HandleMotionEvent(0, controller.index, jboolean(controller.focused), (jboolean) pressed, 0.0f, 0.0f); } else if (vrVideo != nullptr) { const bool togglePressed = controller.buttonState & ControllerDelegate::BUTTON_X || controller.buttonState & ControllerDelegate::BUTTON_A; const bool toggleWasPressed = controller.lastButtonState & ControllerDelegate::BUTTON_X || controller.lastButtonState & ControllerDelegate::BUTTON_A; if (togglePressed != toggleWasPressed) { - VRBrowser::HandleMotionEvent(0, controller.index, (jboolean) togglePressed, 0.0f, 0.0f); + VRBrowser::HandleMotionEvent(0, controller.index, jboolean(controller.focused), (jboolean) togglePressed, 0.0f, 0.0f); } } controller.lastButtonState = controller.buttonState; @@ -1142,7 +1139,7 @@ BrowserWorld::StartWidgetMove(int32_t aHandle, int32_t aMoveBehavour) { continue; } - if (controller.pointer && controller.pointer->GetHitWidget() == widget) { + if (controller.pointer && controller.focused && controller.pointer->GetHitWidget() == widget) { controllerIndex = controller.index; start = controller.StartPoint(); direction = controller.Direction(); diff --git a/app/src/main/cpp/ControllerContainer.cpp b/app/src/main/cpp/ControllerContainer.cpp index 60c47a2f0..2ca0e156f 100644 --- a/app/src/main/cpp/ControllerContainer.cpp +++ b/app/src/main/cpp/ControllerContainer.cpp @@ -200,9 +200,11 @@ ControllerContainer::CreateController(const int32_t aControllerIndex, const int3 CreationContextPtr create = m.context.lock(); controller.transform = Transform::Create(create); controller.pointer = Pointer::Create(create); + controller.pointer->SetVisible(true); if ((m.models.size() >= aModelIndex) && m.models[aModelIndex]) { controller.transform->AddNode(m.models[aModelIndex]); controller.beamToggle = vrb::Toggle::Create(create); + controller.beamToggle->ToggleAll(true); if (aBeamTransform.IsIdentity()) { controller.beamParent = controller.beamToggle; } else { @@ -212,7 +214,6 @@ ControllerContainer::CreateController(const int32_t aControllerIndex, const int3 controller.beamToggle->AddNode(beamTransform); } controller.transform->AddNode(controller.beamToggle); - controller.beamToggle->ToggleAll(false); if (m.beamModel && controller.beamParent) { controller.beamParent->AddNode(m.beamModel); } @@ -235,20 +236,7 @@ ControllerContainer::SetFocused(const int32_t aControllerIndex) { return; } for (Controller& controller: m.list) { - bool show = false; - if (controller.index == aControllerIndex) { - controller.focused = true; - show = true; - } else { - controller.focused = false; - } - - if (controller.beamToggle) { - controller.beamToggle->ToggleAll(show); - } - if (controller.pointer) { - controller.pointer->SetVisible(show); - } + controller.focused = controller.index == aControllerIndex; } } @@ -454,6 +442,7 @@ void ControllerContainer::SetPointerColor(const vrb::Color& aColor) const { void ControllerContainer::SetVisible(const bool aVisible) { + VRB_LOG("[ControllerContainer] SetVisible %d", aVisible) if (m.visible == aVisible) { return; } diff --git a/app/src/main/cpp/VRBrowser.cpp b/app/src/main/cpp/VRBrowser.cpp index 20ffd81c3..296c8f320 100644 --- a/app/src/main/cpp/VRBrowser.cpp +++ b/app/src/main/cpp/VRBrowser.cpp @@ -15,7 +15,7 @@ const char* kDispatchCreateWidgetSignature = "(ILandroid/graphics/SurfaceTexture const char* kDispatchCreateWidgetLayerName = "dispatchCreateWidgetLayer"; const char* kDispatchCreateWidgetLayerSignature = "(ILandroid/view/Surface;IIJ)V"; const char* kHandleMotionEventName = "handleMotionEvent"; -const char* kHandleMotionEventSignature = "(IIZFF)V"; +const char* kHandleMotionEventSignature = "(IIZZFF)V"; const char* kHandleScrollEventName = "handleScrollEvent"; const char* kHandleScrollEventSignature = "(IIFF)V"; const char* kHandleAudioPoseName = "handleAudioPose"; @@ -175,9 +175,9 @@ VRBrowser::DispatchCreateWidgetLayer(jint aWidgetHandle, jobject aSurface, jint void -VRBrowser::HandleMotionEvent(jint aWidgetHandle, jint aController, jboolean aPressed, jfloat aX, jfloat aY) { +VRBrowser::HandleMotionEvent(jint aWidgetHandle, jint aController, jboolean aFocused, jboolean aPressed, jfloat aX, jfloat aY) { if (!ValidateMethodID(sEnv, sActivity, sHandleMotionEvent, __FUNCTION__)) { return; } - sEnv->CallVoidMethod(sActivity, sHandleMotionEvent, aWidgetHandle, aController, aPressed, aX, aY); + sEnv->CallVoidMethod(sActivity, sHandleMotionEvent, aWidgetHandle, aController, aFocused, aPressed, aX, aY); CheckJNIException(sEnv, __FUNCTION__); } diff --git a/app/src/main/cpp/VRBrowser.h b/app/src/main/cpp/VRBrowser.h index 49e74d0e1..d45fc1021 100644 --- a/app/src/main/cpp/VRBrowser.h +++ b/app/src/main/cpp/VRBrowser.h @@ -20,7 +20,7 @@ void InitializeJava(JNIEnv* aEnv, jobject aActivity); void ShutdownJava(); void DispatchCreateWidget(jint aWidgetHandle, jobject aSurfaceTexture, jint aWidth, jint aHeight); void DispatchCreateWidgetLayer(jint aWidgetHandle, jobject aSurface, jint aWidth, jint aHeight, const std::function& aFirstCompositeCallback); -void HandleMotionEvent(jint aWidgetHandle, jint aController, jboolean aPressed, jfloat aX, jfloat aY); +void HandleMotionEvent(jint aWidgetHandle, jint aController, jboolean aFocused, jboolean aPressed, jfloat aX, jfloat aY); void HandleScrollEvent(jint aWidgetHandle, jint aController, jfloat aX, jfloat aY); void HandleAudioPose(jfloat qx, jfloat qy, jfloat qz, jfloat qw, jfloat px, jfloat py, jfloat pz); void HandleGesture(jint aType); diff --git a/app/src/main/res/layout/autocompletion_bar.xml b/app/src/main/res/layout/autocompletion_bar.xml index 669a8103b..684b694e0 100644 --- a/app/src/main/res/layout/autocompletion_bar.xml +++ b/app/src/main/res/layout/autocompletion_bar.xml @@ -6,7 +6,7 @@ android:gravity="center_horizontal" android:padding="0dp" android:orientation="vertical"> - + @@ -104,16 +105,29 @@ android:layout_height="0dp" android:layout_weight="1" /> - + android:layout_height="match_parent"> + + + @@ -122,9 +136,16 @@ android:layout_width="match_parent" android:layout_height="@dimen/autocompletion_widget_line_height" android:layout_marginStart="38dp" - android:visibility="gone"> - - + android:visibility="gone"/> +