From bbc241748b3621601c8ef46aa86fa6a06528ae03 Mon Sep 17 00:00:00 2001 From: mcfans Date: Tue, 17 Oct 2023 22:12:52 +0800 Subject: [PATCH 01/25] feat: support android keyboard input --- .../com/carriez/flutter_hbb/InputService.kt | 14 ++++++++++++++ .../com/carriez/flutter_hbb/MainService.kt | 6 ++++++ .../desktop/screen/desktop_remote_screen.dart | 6 +++--- flutter/lib/mobile/pages/remote_page.dart | 4 ++++ libs/scrap/src/android/ffi.rs | 19 +++++++++++++++++++ src/server/connection.rs | 13 +++++++++++-- 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 203558968..d46f084a3 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -10,8 +10,10 @@ import android.accessibilityservice.AccessibilityService import android.accessibilityservice.GestureDescription import android.graphics.Path import android.os.Build +import android.os.Bundle import android.util.Log import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo import androidx.annotation.RequiresApi import java.util.* import kotlin.math.abs @@ -252,6 +254,18 @@ class InputService : AccessibilityService() { } } + @RequiresApi(Build.VERSION_CODES.N) + fun onTextInput(str: String) { + findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + str + ) + it.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + } + } + override fun onServiceConnected() { super.onServiceConnected() ctx = this diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt index 535a3f8c3..e9b9c8ef0 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt @@ -94,6 +94,12 @@ class MainService : Service() { } } + @Keep + @RequiresApi(Build.VERSION_CODES.N) + fun rustInputString(input: String) { + InputService.ctx?.onTextInput(input) + } + @Keep fun rustGetByName(name: String): String { return when (name) { diff --git a/flutter/lib/desktop/screen/desktop_remote_screen.dart b/flutter/lib/desktop/screen/desktop_remote_screen.dart index 72f15dbfd..4427cf8a3 100644 --- a/flutter/lib/desktop/screen/desktop_remote_screen.dart +++ b/flutter/lib/desktop/screen/desktop_remote_screen.dart @@ -12,9 +12,9 @@ class DesktopRemoteScreen extends StatelessWidget { final Map params; DesktopRemoteScreen({Key? key, required this.params}) : super(key: key) { - if (!bind.mainStartGrabKeyboard()) { - stateGlobal.grabKeyboard = true; - } + if (!bind.mainStartGrabKeyboard()) { + stateGlobal.grabKeyboard = true; + } } @override diff --git a/flutter/lib/mobile/pages/remote_page.dart b/flutter/lib/mobile/pages/remote_page.dart index 249355012..3346e03de 100644 --- a/flutter/lib/mobile/pages/remote_page.dart +++ b/flutter/lib/mobile/pages/remote_page.dart @@ -364,6 +364,10 @@ class _RemotePageState extends State { ? [] : gFFI.ffiModel.isPeerAndroid ? [ + IconButton( + color: Colors.white, + icon: Icon(Icons.keyboard), + onPressed: openKeyboard), IconButton( color: Colors.white, icon: const Icon(Icons.build), diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index e9c60ef93..615b67b20 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -173,6 +173,25 @@ pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) -> } } +pub fn call_main_service_input_string(str: &str) -> JniResult<()> { + if let (Some(jvm), Some(ctx)) = ( + JVM.read().unwrap().as_ref(), + MAIN_SERVICE_CTX.read().unwrap().as_ref(), + ) { + let mut env = jvm.attach_current_thread_as_daemon()?; + let input_string = env.new_string(str)?; + env.call_method( + ctx, + "rustInputString", + "(Ljava/lang/String;)V", + &[JValue::Object(&JObject::from(input_string))], + )?; + return Ok(()); + } else { + return Err(JniError::ThrowFailed(-1)); + } +} + pub fn call_main_service_get_by_name(name: &str) -> JniResult { if let (Some(jvm), Some(ctx)) = ( JVM.read().unwrap().as_ref(), diff --git a/src/server/connection.rs b/src/server/connection.rs index 5f9502397..46e1e4298 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -41,7 +41,7 @@ use hbb_common::{ tokio_util::codec::{BytesCodec, Framed}, }; #[cfg(any(target_os = "android", target_os = "ios"))] -use scrap::android::call_main_service_pointer_input; +use scrap::android::{call_main_service_pointer_input, call_main_service_input_string}; use serde_json::{json, value::Value}; use sha2::{Digest, Sha256}; #[cfg(not(any(target_os = "android", target_os = "ios")))] @@ -1722,8 +1722,17 @@ impl Connection { } self.update_auto_disconnect_timer(); } - #[cfg(any(target_os = "android", target_os = "ios"))] + #[cfg(any(target_os = "ios"))] Some(message::Union::KeyEvent(..)) => {} + #[cfg(any(target_os = "android"))] + Some(message::Union::KeyEvent(me)) => { + // We can only use seq of key event, android device doesn't support abritrary key stroke. + let seq = me.seq(); + let result = call_main_service_input_string(seq); + if let Err(e) = result { + log::debug!("call_main_service_input_string fail:{}", e); + } + } #[cfg(not(any(target_os = "android", target_os = "ios")))] Some(message::Union::KeyEvent(me)) => { if self.peer_keyboard_enabled() { From 22165ec1a50f236b0ed4eec854e70e95979c9fb5 Mon Sep 17 00:00:00 2001 From: mcfans Date: Thu, 19 Oct 2023 00:16:22 +0800 Subject: [PATCH 02/25] feat: legacy mode android keyboard support --- flutter/android/app/build.gradle | 28 +++++ .../com/carriez/flutter_hbb/InputService.kt | 92 ++++++++++++-- .../flutter_hbb/KeyboardKeyEventMapper.kt | 115 ++++++++++++++++++ .../com/carriez/flutter_hbb/MainService.kt | 5 +- .../res/xml/accessibility_service_config.xml | 1 + libs/scrap/src/android/ffi.rs | 11 +- src/keyboard.rs | 6 +- src/server/connection.rs | 31 +++-- 8 files changed, 261 insertions(+), 28 deletions(-) create mode 100644 flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt diff --git a/flutter/android/app/build.gradle b/flutter/android/app/build.gradle index f4dc69e41..3793c1f4c 100644 --- a/flutter/android/app/build.gradle +++ b/flutter/android/app/build.gradle @@ -1,3 +1,8 @@ +import com.google.protobuf.gradle.* +plugins { + id "com.google.protobuf" version "0.9.4" +} + def keystoreProperties = new Properties() def keystorePropertiesFile = rootProject.file('key.properties') if (keystorePropertiesFile.exists()) { @@ -31,10 +36,33 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle" +dependencies { + implementation 'com.google.protobuf:protobuf-javalite:3.20.1' +} + +protobuf { + protoc { + artifact = 'com.google.protobuf:protoc:3.20.1' + } + + generateProtoTasks { + all().configureEach { task -> + task.builtins { + java { + option "lite" + } + } + } + } +} + android { compileSdkVersion 33 sourceSets { main.java.srcDirs += 'src/main/kotlin' + + main.proto.srcDirs += '../../../libs/hbb_common/protos' + main.proto.includes += "message.proto" } compileOptions { diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index d46f084a3..50445589c 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -11,13 +11,18 @@ import android.accessibilityservice.GestureDescription import android.graphics.Path import android.os.Build import android.os.Bundle +import android.os.Handler +import android.os.Looper import android.util.Log +import android.widget.EditText import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo import androidx.annotation.RequiresApi import java.util.* import kotlin.math.abs import kotlin.math.max +import hbb.MessageOuterClass.KeyEvent; +import hbb.KeyEventConverter; const val LIFT_DOWN = 9 const val LIFT_MOVE = 8 @@ -60,6 +65,8 @@ class InputService : AccessibilityService() { private var isWheelActionsPolling = false private var isWaitingLongPress = false + private var fakeEditTextForTextStateCalculation: EditText? = null + @RequiresApi(Build.VERSION_CODES.N) fun onMouseInput(mask: Int, _x: Int, _y: Int) { val x = max(0, _x) @@ -255,20 +262,87 @@ class InputService : AccessibilityService() { } @RequiresApi(Build.VERSION_CODES.N) - fun onTextInput(str: String) { - findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { - val arguments = Bundle() - arguments.putCharSequence( - AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, - str - ) - it.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + fun onKeyEvent(data: ByteArray) { + val keyEvent = KeyEvent.parseFrom(data); + val handler = Handler(Looper.getMainLooper()) + handler.post { + findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { node -> + val text = node.getText() + val isShowingHint = node.isShowingHintText() + + var textSelectionStart = node.getTextSelectionStart() + var textSelectionEnd = node.getTextSelectionEnd() + + if (text != null) { + if (textSelectionStart > text.length) { + textSelectionStart = text.length + } + if (textSelectionEnd > text.length) { + textSelectionEnd = text.length + } + if (textSelectionStart > textSelectionEnd) { + textSelectionStart = textSelectionEnd + } + } + + if (keyEvent.hasSeq()) { + val seq = keyEvent.getSeq() + + var newText = "" + + if ((textSelectionStart == -1) || (textSelectionEnd == -1)) { + newText = seq + } else { + newText = text.let { + it.substring(0, textSelectionStart) + seq + it.substring(textSelectionStart) + } + } + + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + newText + ) + node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + + } else { + KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> + Log.d(logTag, "event $event text $text start $textSelectionStar end $textSelectionEnd") + if (isShowingHint) { + this.fakeEditTextForTextStateCalculation?.setText(null) + } else { + this.fakeEditTextForTextStateCalculation?.setText(text) + } + if (textSelectionStart != -1 && textSelectionEnd != -1) { + this.fakeEditTextForTextStateCalculation?.setSelection( + textSelectionStart, + textSelectionEnd + ) + } + this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event) + + this.fakeEditTextForTextStateCalculation?.getText()?.let { newText -> + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + newText.toString() + ) + node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + } + } + } + } } } + + override fun onAccessibilityEvent(event: AccessibilityEvent) { + } + override fun onServiceConnected() { super.onServiceConnected() ctx = this + fakeEditTextForTextStateCalculation = EditText(this) Log.d(logTag, "onServiceConnected!") } @@ -277,7 +351,5 @@ class InputService : AccessibilityService() { super.onDestroy() } - override fun onAccessibilityEvent(event: AccessibilityEvent?) {} - override fun onInterrupt() {} } diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt new file mode 100644 index 000000000..f7273625e --- /dev/null +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt @@ -0,0 +1,115 @@ +package hbb; +import android.view.KeyEvent +import android.view.KeyCharacterMap +import hbb.MessageOuterClass.ControlKey + +object KeyEventConverter { + fun toAndroidKeyEvent(keyEventProto: hbb.MessageOuterClass.KeyEvent): KeyEvent { + var chrValue = 0 + var modifiers = 0 + + android.util.Log.d(tag, "proto: $keyEventProto") + + if (keyEventProto.hasUnicode()) { + chrValue = + } + + if (keyEventProto.hasChr()) { + chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int) + } else if (keyEventProto.hasControlKey()) { + chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey()) + } + + var modifiersList = keyEventProto.getModifiersList() + + if (modifiersList != null) { + for (modifier in keyEventProto.getModifiersList()) { + val modifierValue = convertModifier(modifier) + modifiers = modifiers and modifierValue + } + } + + var action = 0 + if (keyEventProto.getDown()) { + action = KeyEvent.ACTION_DOWN + } else { + action = KeyEvent.ACTION_UP + } + + return KeyEvent(0, 0, action, chrValue, 0, modifiers) + } + + private fun convertModifier(controlKey: hbb.MessageOuterClass.ControlKey): Int { + // Add logic to map ControlKey values to Android KeyEvent key codes. + // You'll need to provide the mapping for each key. + return when (controlKey) { + ControlKey.Alt -> KeyEvent.META_ALT_ON + ControlKey.Control -> KeyEvent.META_CTRL_ON + ControlKey.CapsLock -> KeyEvent.META_CAPS_LOCK_ON + ControlKey.Meta -> KeyEvent.META_META_ON + ControlKey.NumLock -> KeyEvent.META_NUM_LOCK_ON + ControlKey.RShift -> KeyEvent.META_SHIFT_RIGHT_ON + ControlKey.Shift -> KeyEvent.META_SHIFT_ON + ControlKey.RAlt -> KeyEvent.META_ALT_RIGHT_ON + ControlKey.RControl -> KeyEvent.META_CTRL_RIGHT_ON + else -> 0 // Default to unknown. + } + } + + private val tag = "KeyEventConverter" + + private fun convertUnicodeToKeyCode(unicode: Int): Int { + val charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD) + android.util.Log.d(tag, "unicode: $unicode") + val events = charMap.getEvents(charArrayOf(unicode.toChar())) + if (events != null && events.size > 0) { + android.util.Log.d(tag, "keycode ${events[0].keyCode}") + return events[0].keyCode + } + return 0 + } + + private fun convertControlKeyToKeyCode(controlKey: hbb.MessageOuterClass.ControlKey): Int { + // Add logic to map ControlKey values to Android KeyEvent key codes. + // You'll need to provide the mapping for each key. + return when (controlKey) { + ControlKey.Alt -> KeyEvent.KEYCODE_ALT_LEFT + ControlKey.Backspace -> KeyEvent.KEYCODE_DEL + ControlKey.Control -> KeyEvent.KEYCODE_CTRL_LEFT + ControlKey.CapsLock -> KeyEvent.KEYCODE_CAPS_LOCK + ControlKey.Meta -> KeyEvent.KEYCODE_META_LEFT + ControlKey.NumLock -> KeyEvent.KEYCODE_NUM_LOCK + ControlKey.RShift -> KeyEvent.KEYCODE_SHIFT_RIGHT + ControlKey.Shift -> KeyEvent.KEYCODE_SHIFT_LEFT + ControlKey.RAlt -> KeyEvent.KEYCODE_ALT_RIGHT + ControlKey.RControl -> KeyEvent.KEYCODE_CTRL_RIGHT + ControlKey.DownArrow -> KeyEvent.KEYCODE_DPAD_DOWN + ControlKey.LeftArrow -> KeyEvent.KEYCODE_DPAD_LEFT + ControlKey.RightArrow -> KeyEvent.KEYCODE_DPAD_RIGHT + ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP + ControlKey.End -> KeyEvent.KEYCODE_MOVE_END + ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME + ControlKey.Insert -> KeyEvent.KEYCODE_INSERT + ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE + ControlKey.F1 -> KeyEvent.KEYCODE_F1 + ControlKey.F2 -> KeyEvent.KEYCODE_F2 + ControlKey.F3 -> KeyEvent.KEYCODE_F3 + ControlKey.F4 -> KeyEvent.KEYCODE_F4 + ControlKey.F5 -> KeyEvent.KEYCODE_F5 + ControlKey.F6 -> KeyEvent.KEYCODE_F6 + ControlKey.F7 -> KeyEvent.KEYCODE_F7 + ControlKey.F8 -> KeyEvent.KEYCODE_F8 + ControlKey.F9 -> KeyEvent.KEYCODE_F9 + ControlKey.F10 -> KeyEvent.KEYCODE_F10 + ControlKey.F11 -> KeyEvent.KEYCODE_F11 + ControlKey.F12 -> KeyEvent.KEYCODE_F12 + ControlKey.Space -> KeyEvent.KEYCODE_SPACE + ControlKey.Tab -> KeyEvent.KEYCODE_TAB + ControlKey.Return -> KeyEvent.KEYCODE_ENTER + ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL + ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR + ControlKey.Pause -> KeyEvent.KEYCODE_BREAK + else -> 0 // Default to unknown. + } + } +} diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt index e9b9c8ef0..cc2e20e25 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt @@ -44,7 +44,6 @@ import java.nio.ByteBuffer import kotlin.math.max import kotlin.math.min - const val DEFAULT_NOTIFY_TITLE = "RustDesk" const val DEFAULT_NOTIFY_TEXT = "Service is running" const val DEFAULT_NOTIFY_ID = 1 @@ -96,8 +95,8 @@ class MainService : Service() { @Keep @RequiresApi(Build.VERSION_CODES.N) - fun rustInputString(input: String) { - InputService.ctx?.onTextInput(input) + fun rustKeyEventInput(input: ByteArray) { + InputService.ctx?.onKeyEvent(input) } @Keep diff --git a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml index fa9407128..90b57cd4e 100644 --- a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml +++ b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml @@ -1,5 +1,6 @@ } } -pub fn call_main_service_input_string(str: &str) -> JniResult<()> { +pub fn call_main_service_key_event(data: &[u8]) -> JniResult<()> { if let (Some(jvm), Some(ctx)) = ( JVM.read().unwrap().as_ref(), MAIN_SERVICE_CTX.read().unwrap().as_ref(), ) { let mut env = jvm.attach_current_thread_as_daemon()?; - let input_string = env.new_string(str)?; + let data = env.byte_array_from_slice(data)?; + env.call_method( ctx, - "rustInputString", - "(Ljava/lang/String;)V", - &[JValue::Object(&JObject::from(input_string))], + "rustKeyEventInput", + "([B)V", + &[JValue::Object(&JObject::from(data))], )?; return Ok(()); } else { diff --git a/src/keyboard.rs b/src/keyboard.rs index 9a6ac49d5..46d0c8f9d 100644 --- a/src/keyboard.rs +++ b/src/keyboard.rs @@ -877,7 +877,7 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) -> Some(key_event) } -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] fn try_fill_unicode(_peer: &str, event: &Event, key_event: &KeyEvent, events: &mut Vec) { match &event.unicode { Some(unicode_info) => { @@ -1046,11 +1046,11 @@ pub fn translate_keyboard_mode(peer: &str, event: &Event, key_event: KeyEvent) - events } -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] pub fn keycode_to_rdev_key(keycode: u32) -> Key { #[cfg(target_os = "windows")] return rdev::win_key_from_scancode(keycode); - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "android"))] return rdev::linux_key_from_code(keycode); #[cfg(target_os = "macos")] return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default()); diff --git a/src/server/connection.rs b/src/server/connection.rs index 46e1e4298..20e2c9e2e 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -41,7 +41,7 @@ use hbb_common::{ tokio_util::codec::{BytesCodec, Framed}, }; #[cfg(any(target_os = "android", target_os = "ios"))] -use scrap::android::{call_main_service_pointer_input, call_main_service_input_string}; +use scrap::android::{call_main_service_pointer_input, call_main_service_key_event}; use serde_json::{json, value::Value}; use sha2::{Digest, Sha256}; #[cfg(not(any(target_os = "android", target_os = "ios")))] @@ -1725,12 +1725,29 @@ impl Connection { #[cfg(any(target_os = "ios"))] Some(message::Union::KeyEvent(..)) => {} #[cfg(any(target_os = "android"))] - Some(message::Union::KeyEvent(me)) => { - // We can only use seq of key event, android device doesn't support abritrary key stroke. - let seq = me.seq(); - let result = call_main_service_input_string(seq); - if let Err(e) = result { - log::debug!("call_main_service_input_string fail:{}", e); + Some(message::Union::KeyEvent(mut me)) => { + let key = match me.mode.enum_value() { + Ok(KeyboardMode::Map) => { + Some(crate::keyboard::keycode_to_rdev_key(me.chr())) + } + Ok(KeyboardMode::Translate) => { + if let Some(key_event::Union::Chr(code)) = me.union { + Some(crate::keyboard::keycode_to_rdev_key(code & 0x0000FFFF)) + } else { + None + } + } + _ => None, + }; + let encode_result = me.write_to_bytes(); + + if let Ok(data) = encode_result { + let result = call_main_service_key_event(&data); + if let Err(e) = result { + log::debug!("call_main_service_key_event fail:{}", e); + } + } else { + log::debug!("encode key event fail:{}", encode_result.err().unwrap()); } } #[cfg(not(any(target_os = "android", target_os = "ios")))] From b3e2ab0f3b555e0f2656001fa8af7ac3f20f750e Mon Sep 17 00:00:00 2001 From: mcfans Date: Thu, 19 Oct 2023 13:33:26 +0800 Subject: [PATCH 03/25] feat: map mode and translate mode receive side --- .../com/carriez/flutter_hbb/InputService.kt | 2 +- .../flutter_hbb/KeyboardKeyEventMapper.kt | 13 +++++-- src/keyboard.rs | 19 ++++++++- src/server/connection.rs | 39 ++++++++++++++++--- 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 50445589c..8e0582d17 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -307,7 +307,7 @@ class InputService : AccessibilityService() { } else { KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> - Log.d(logTag, "event $event text $text start $textSelectionStar end $textSelectionEnd") + Log.d(logTag, "event $event text $text start $textSelectionStart end $textSelectionEnd") if (isShowingHint) { this.fakeEditTextForTextStateCalculation?.setText(null) } else { diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt index f7273625e..47b237eaf 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt @@ -1,6 +1,7 @@ package hbb; import android.view.KeyEvent import android.view.KeyCharacterMap +import hbb.MessageOuterClass.KeyboardMode import hbb.MessageOuterClass.ControlKey object KeyEventConverter { @@ -10,12 +11,14 @@ object KeyEventConverter { android.util.Log.d(tag, "proto: $keyEventProto") - if (keyEventProto.hasUnicode()) { - chrValue = - } + val keyboardMode = keyEventProto.getMode() if (keyEventProto.hasChr()) { - chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int) + if (keyboardMode == KeyboardMode.Map || keyboardMode == KeyboardMode.Translate) { + chrValue = keyEventProto.getChr() + } else { + chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int) + } } else if (keyEventProto.hasControlKey()) { chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey()) } @@ -29,6 +32,8 @@ object KeyEventConverter { } } + android.util.Log.d(tag, "modifiers: $modifiersList") + var action = 0 if (keyEventProto.getDown()) { action = KeyEvent.ACTION_DOWN diff --git a/src/keyboard.rs b/src/keyboard.rs index 46d0c8f9d..4a46d8934 100644 --- a/src/keyboard.rs +++ b/src/keyboard.rs @@ -163,6 +163,21 @@ pub mod client { } } + pub fn map_key_to_control_key(key: &rdev::Key) -> Option { + match key { + Key::Alt => Some(ControlKey::Alt), + Key::ShiftLeft => Some(ControlKey::Shift), + Key::ControlLeft => Some(ControlKey::Control), + Key::MetaLeft => Some(ControlKey::Meta), + Key::AltGr => Some(ControlKey::RAlt), + Key::ShiftRight => Some(ControlKey::RShift), + Key::ControlRight => Some(ControlKey::RControl), + Key::MetaRight => Some(ControlKey::RWin), + _ => None, + } + + } + pub fn event_lock_screen() -> KeyEvent { let mut key_event = KeyEvent::new(); key_event.set_control_key(ControlKey::LockScreen); @@ -355,7 +370,7 @@ pub fn get_keyboard_mode_enum(keyboard_mode: &str) -> KeyboardMode { } #[inline] -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] pub fn is_modifier(key: &rdev::Key) -> bool { matches!( key, @@ -1050,7 +1065,7 @@ pub fn translate_keyboard_mode(peer: &str, event: &Event, key_event: KeyEvent) - pub fn keycode_to_rdev_key(keycode: u32) -> Key { #[cfg(target_os = "windows")] return rdev::win_key_from_scancode(keycode); - #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg(any(target_os = "linux"))] return rdev::linux_key_from_code(keycode); #[cfg(target_os = "macos")] return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default()); diff --git a/src/server/connection.rs b/src/server/connection.rs index 20e2c9e2e..7f22bd8f9 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -38,10 +38,12 @@ use hbb_common::{ sync::mpsc, time::{self, Duration, Instant, Interval}, }, - tokio_util::codec::{BytesCodec, Framed}, + tokio_util::codec::{BytesCodec, Framed}, protobuf::EnumOrUnknown, }; #[cfg(any(target_os = "android", target_os = "ios"))] use scrap::android::{call_main_service_pointer_input, call_main_service_key_event}; +#[cfg(target_os = "android")] +use crate::keyboard::client::map_key_to_control_key; use serde_json::{json, value::Value}; use sha2::{Digest, Sha256}; #[cfg(not(any(target_os = "android", target_os = "ios")))] @@ -53,7 +55,7 @@ use std::{ #[cfg(not(any(target_os = "android", target_os = "ios")))] use system_shutdown; -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] use std::collections::HashSet; pub type Sender = mpsc::UnboundedSender<(Instant, Arc)>; @@ -211,7 +213,7 @@ pub struct Connection { voice_call_request_timestamp: Option, audio_input_device_before_voice_call: Option, options_in_login: Option, - #[cfg(not(any(target_os = "android", target_os = "ios")))] + #[cfg(not(any(target_os = "ios")))] pressed_modifiers: HashSet, #[cfg(all(target_os = "linux", feature = "linux_headless"))] #[cfg(not(any(feature = "flatpak", feature = "appimage")))] @@ -348,7 +350,7 @@ impl Connection { voice_call_request_timestamp: None, audio_input_device_before_voice_call: None, options_in_login: None, - #[cfg(not(any(target_os = "android", target_os = "ios")))] + #[cfg(not(any(target_os = "ios")))] pressed_modifiers: Default::default(), #[cfg(all(target_os = "linux", feature = "linux_headless"))] #[cfg(not(any(feature = "flatpak", feature = "appimage")))] @@ -1726,6 +1728,12 @@ impl Connection { Some(message::Union::KeyEvent(..)) => {} #[cfg(any(target_os = "android"))] Some(message::Union::KeyEvent(mut me)) => { + let is_press = if cfg!(target_os = "linux") { + (me.press || me.down) && !crate::is_modifier(&me) + } else { + me.press + }; + let key = match me.mode.enum_value() { Ok(KeyboardMode::Map) => { Some(crate::keyboard::keycode_to_rdev_key(me.chr())) @@ -1738,7 +1746,28 @@ impl Connection { } } _ => None, - }; + } + .filter(crate::keyboard::is_modifier); + log::debug!("key:{:?}", key); + + if let Some(key) = key { + if is_press { + self.pressed_modifiers.insert(key); + } else { + self.pressed_modifiers.remove(&key); + } + } + + let mut modifiers = vec![]; + + for key in self.pressed_modifiers.iter() { + if let Some(control_key) = map_key_to_control_key(key) { + modifiers.push(EnumOrUnknown::new(control_key)); + } + } + + me.modifiers = modifiers; + let encode_result = me.write_to_bytes(); if let Ok(data) = encode_result { From 28c11801f3fe37c0f92f6283c940c18686480980 Mon Sep 17 00:00:00 2001 From: mcfans Date: Thu, 19 Oct 2023 13:40:12 +0800 Subject: [PATCH 04/25] feat: keyboard map mode control side --- Cargo.lock | 28 ++++++++++++++++++++++++++-- Cargo.toml | 2 +- src/keyboard.rs | 5 +++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 519476e93..261f4a544 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1811,7 +1811,7 @@ dependencies = [ "log", "objc", "pkg-config", - "rdev", + "rdev 0.5.0-2 (git+https://github.com/fufesou/rdev)", "serde 1.0.163", "serde_derive", "tfc", @@ -4911,6 +4911,30 @@ dependencies = [ "x11 2.21.0", ] +[[package]] +name = "rdev" +version = "0.5.0-2" +source = "git+https://github.com/mcfans/rdev#d643ea4542e61931cb20e1ecd390d6a3c9569171" +dependencies = [ + "cocoa", + "core-foundation", + "core-foundation-sys 0.8.4", + "core-graphics", + "dispatch", + "enum-map", + "epoll", + "inotify", + "lazy_static", + "libc", + "log", + "mio", + "strum 0.24.1", + "strum_macros 0.24.3", + "widestring", + "winapi 0.3.9", + "x11 2.21.0", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -5196,7 +5220,7 @@ dependencies = [ "os-version", "pam", "parity-tokio-ipc", - "rdev", + "rdev 0.5.0-2 (git+https://github.com/mcfans/rdev)", "repng", "reqwest", "ringbuf", diff --git a/Cargo.toml b/Cargo.toml index f6c014483..2a45b074b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ default-net = "0.14" wol-rs = "1.0" flutter_rust_bridge = { version = "1.75", features = ["uuid"], optional = true} errno = "0.3" -rdev = { git = "https://github.com/fufesou/rdev" } +rdev = { git = "https://github.com/mcfans/rdev" } url = { version = "2.3", features = ["serde"] } crossbeam-queue = "0.3" hex = "0.4" diff --git a/src/keyboard.rs b/src/keyboard.rs index 4a46d8934..20368c9d6 100644 --- a/src/keyboard.rs +++ b/src/keyboard.rs @@ -28,6 +28,8 @@ const OS_LOWER_WINDOWS: &str = "windows"; const OS_LOWER_LINUX: &str = "linux"; #[allow(dead_code)] const OS_LOWER_MACOS: &str = "macos"; +#[allow(dead_code)] +const OS_LOWER_ANDROID: &str = "android"; #[cfg(any(target_os = "windows", target_os = "macos"))] static KEYBOARD_HOOKED: AtomicBool = AtomicBool::new(false); @@ -865,12 +867,14 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) -> rdev::win_scancode_to_macos_code(event.position_code)? } } + OS_LOWER_ANDROID => rdev::win_scancode_to_android_key_code(event.position_code)?, _ => rdev::win_scancode_to_linux_code(event.position_code)?, }; #[cfg(target_os = "macos")] let keycode = match _peer { OS_LOWER_WINDOWS => rdev::macos_code_to_win_scancode(event.platform_code as _)?, OS_LOWER_MACOS => event.platform_code as _, + OS_LOWER_ANDROID => rdev::macos_code_to_android_key_code(event.platform_code as _)?, _ => rdev::macos_code_to_linux_code(event.platform_code as _)?, }; #[cfg(target_os = "linux")] @@ -883,6 +887,7 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) -> rdev::linux_code_to_macos_code(event.position_code as _)? } } + OS_LOWER_ANDROID => rdev::linux_code_to_android_key_code(event.position_code as _)?, _ => event.position_code as _, }; #[cfg(any(target_os = "android", target_os = "ios"))] From 26e77ba2c38fad632967e826c397a43afd8e9a21 Mon Sep 17 00:00:00 2001 From: mcfans Date: Thu, 19 Oct 2023 14:19:19 +0800 Subject: [PATCH 05/25] feat: modifier key --- Cargo.lock | 2 +- .../com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt | 6 +----- src/keyboard.rs | 2 ++ src/server/connection.rs | 7 +------ 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 261f4a544..404ccf9dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4914,7 +4914,7 @@ dependencies = [ [[package]] name = "rdev" version = "0.5.0-2" -source = "git+https://github.com/mcfans/rdev#d643ea4542e61931cb20e1ecd390d6a3c9569171" +source = "git+https://github.com/mcfans/rdev#97f63d5bc51bb2387d3f2664d3127640e492c9e5" dependencies = [ "cocoa", "core-foundation", diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt index 47b237eaf..5686d30d9 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt @@ -9,8 +9,6 @@ object KeyEventConverter { var chrValue = 0 var modifiers = 0 - android.util.Log.d(tag, "proto: $keyEventProto") - val keyboardMode = keyEventProto.getMode() if (keyEventProto.hasChr()) { @@ -28,12 +26,10 @@ object KeyEventConverter { if (modifiersList != null) { for (modifier in keyEventProto.getModifiersList()) { val modifierValue = convertModifier(modifier) - modifiers = modifiers and modifierValue + modifiers = modifiers or modifierValue } } - android.util.Log.d(tag, "modifiers: $modifiersList") - var action = 0 if (keyEventProto.getDown()) { action = KeyEvent.ACTION_DOWN diff --git a/src/keyboard.rs b/src/keyboard.rs index 20368c9d6..3baeef2ec 100644 --- a/src/keyboard.rs +++ b/src/keyboard.rs @@ -1072,6 +1072,8 @@ pub fn keycode_to_rdev_key(keycode: u32) -> Key { return rdev::win_key_from_scancode(keycode); #[cfg(any(target_os = "linux"))] return rdev::linux_key_from_code(keycode); + #[cfg(any(target_os = "android"))] + return rdev::android_key_from_code(keycode); #[cfg(target_os = "macos")] return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default()); } diff --git a/src/server/connection.rs b/src/server/connection.rs index 7f22bd8f9..96395e570 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -1728,11 +1728,7 @@ impl Connection { Some(message::Union::KeyEvent(..)) => {} #[cfg(any(target_os = "android"))] Some(message::Union::KeyEvent(mut me)) => { - let is_press = if cfg!(target_os = "linux") { - (me.press || me.down) && !crate::is_modifier(&me) - } else { - me.press - }; + let is_press = (me.press || me.down) && !crate::is_modifier(&me); let key = match me.mode.enum_value() { Ok(KeyboardMode::Map) => { @@ -1748,7 +1744,6 @@ impl Connection { _ => None, } .filter(crate::keyboard::is_modifier); - log::debug!("key:{:?}", key); if let Some(key) = key { if is_press { From 9a903a1ca31a4fa50429a79f88474b601ee44cd6 Mon Sep 17 00:00:00 2001 From: mcfans Date: Fri, 20 Oct 2023 00:32:59 +0800 Subject: [PATCH 06/25] feat: use current input method to send key --- .../com/carriez/flutter_hbb/InputService.kt | 81 +++---------------- .../res/xml/accessibility_service_config.xml | 1 - 2 files changed, 10 insertions(+), 72 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 8e0582d17..dae86c3db 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -17,6 +17,8 @@ import android.util.Log import android.widget.EditText import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo +import android.accessibilityservice.AccessibilityServiceInfo +import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR import androidx.annotation.RequiresApi import java.util.* import kotlin.math.abs @@ -65,8 +67,6 @@ class InputService : AccessibilityService() { private var isWheelActionsPolling = false private var isWaitingLongPress = false - private var fakeEditTextForTextStateCalculation: EditText? = null - @RequiresApi(Build.VERSION_CODES.N) fun onMouseInput(mask: Int, _x: Int, _y: Int) { val x = max(0, _x) @@ -264,85 +264,24 @@ class InputService : AccessibilityService() { @RequiresApi(Build.VERSION_CODES.N) fun onKeyEvent(data: ByteArray) { val keyEvent = KeyEvent.parseFrom(data); - val handler = Handler(Looper.getMainLooper()) - handler.post { - findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { node -> - val text = node.getText() - val isShowingHint = node.isShowingHintText() - - var textSelectionStart = node.getTextSelectionStart() - var textSelectionEnd = node.getTextSelectionEnd() - - if (text != null) { - if (textSelectionStart > text.length) { - textSelectionStart = text.length - } - if (textSelectionEnd > text.length) { - textSelectionEnd = text.length - } - if (textSelectionStart > textSelectionEnd) { - textSelectionStart = textSelectionEnd - } - } - - if (keyEvent.hasSeq()) { - val seq = keyEvent.getSeq() - - var newText = "" - - if ((textSelectionStart == -1) || (textSelectionEnd == -1)) { - newText = seq - } else { - newText = text.let { - it.substring(0, textSelectionStart) + seq + it.substring(textSelectionStart) - } - } - - val arguments = Bundle() - arguments.putCharSequence( - AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, - newText - ) - node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) - - } else { - KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> - Log.d(logTag, "event $event text $text start $textSelectionStart end $textSelectionEnd") - if (isShowingHint) { - this.fakeEditTextForTextStateCalculation?.setText(null) - } else { - this.fakeEditTextForTextStateCalculation?.setText(text) - } - if (textSelectionStart != -1 && textSelectionEnd != -1) { - this.fakeEditTextForTextStateCalculation?.setSelection( - textSelectionStart, - textSelectionEnd - ) - } - this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event) - - this.fakeEditTextForTextStateCalculation?.getText()?.let { newText -> - val arguments = Bundle() - arguments.putCharSequence( - AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, - newText.toString() - ) - node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) - } - } + getInputMethod()?.let { inputMethod -> + inputMethod.getCurrentInputConnection()?.let { inputConnection -> + KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> + inputConnection.sendKeyEvent(event) } } } } - override fun onAccessibilityEvent(event: AccessibilityEvent) { - } + override fun onAccessibilityEvent(event: AccessibilityEvent) {} override fun onServiceConnected() { super.onServiceConnected() ctx = this - fakeEditTextForTextStateCalculation = EditText(this) + val info = AccessibilityServiceInfo() + info.flags = FLAG_INPUT_METHOD_EDITOR + setServiceInfo(info) Log.d(logTag, "onServiceConnected!") } diff --git a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml index 90b57cd4e..fa9407128 100644 --- a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml +++ b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml @@ -1,6 +1,5 @@ Date: Fri, 20 Oct 2023 21:12:02 +0800 Subject: [PATCH 07/25] fix: add old code back for old version --- .../com/carriez/flutter_hbb/InputService.kt | 86 +++++++++++++++++-- .../res/xml/accessibility_service_config.xml | 1 + 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index dae86c3db..9df9b1a48 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -67,6 +67,8 @@ class InputService : AccessibilityService() { private var isWheelActionsPolling = false private var isWaitingLongPress = false + private var fakeEditTextForTextStateCalculation: EditText? = null + @RequiresApi(Build.VERSION_CODES.N) fun onMouseInput(mask: Int, _x: Int, _y: Int) { val x = max(0, _x) @@ -264,17 +266,90 @@ class InputService : AccessibilityService() { @RequiresApi(Build.VERSION_CODES.N) fun onKeyEvent(data: ByteArray) { val keyEvent = KeyEvent.parseFrom(data); - getInputMethod()?.let { inputMethod -> - inputMethod.getCurrentInputConnection()?.let { inputConnection -> - KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> - inputConnection.sendKeyEvent(event) + if (Build.VERSION.SDK_INT >= 33) { + getInputMethod()?.let { inputMethod -> + inputMethod.getCurrentInputConnection()?.let { inputConnection -> + KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> + inputConnection.sendKeyEvent(event) + } + } + } + } else { + val handler = Handler(Looper.getMainLooper()) + handler.post { + findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { node -> + val text = node.getText() + val isShowingHint = node.isShowingHintText() + + var textSelectionStart = node.getTextSelectionStart() + var textSelectionEnd = node.getTextSelectionEnd() + + if (text != null) { + if (textSelectionStart > text.length) { + textSelectionStart = text.length + } + if (textSelectionEnd > text.length) { + textSelectionEnd = text.length + } + if (textSelectionStart > textSelectionEnd) { + textSelectionStart = textSelectionEnd + } + } + + if (keyEvent.hasSeq()) { + val seq = keyEvent.getSeq() + + var newText = "" + + if ((textSelectionStart == -1) || (textSelectionEnd == -1)) { + newText = seq + } else { + newText = text.let { + it.substring(0, textSelectionStart) + seq + it.substring(textSelectionStart) + } + } + + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + newText + ) + node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + + } else { + KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> + if (isShowingHint) { + this.fakeEditTextForTextStateCalculation?.setText(null) + } else { + this.fakeEditTextForTextStateCalculation?.setText(text) + } + if (textSelectionStart != -1 && textSelectionEnd != -1) { + this.fakeEditTextForTextStateCalculation?.setSelection( + textSelectionStart, + textSelectionEnd + ) + } + this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event) + + this.fakeEditTextForTextStateCalculation?.getText()?.let { newText -> + Log.d(logTag, "event $event text $text newText $newText") + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + newText.toString() + ) + node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + } + } + } } } } } - override fun onAccessibilityEvent(event: AccessibilityEvent) {} + override fun onAccessibilityEvent(event: AccessibilityEvent) { + } override fun onServiceConnected() { super.onServiceConnected() @@ -282,6 +357,7 @@ class InputService : AccessibilityService() { val info = AccessibilityServiceInfo() info.flags = FLAG_INPUT_METHOD_EDITOR setServiceInfo(info) + fakeEditTextForTextStateCalculation = EditText(this) Log.d(logTag, "onServiceConnected!") } diff --git a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml index fa9407128..90b57cd4e 100644 --- a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml +++ b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml @@ -1,5 +1,6 @@ Date: Tue, 24 Oct 2023 13:34:52 +0800 Subject: [PATCH 08/25] fix: use seq for translate mode and use commitText --- .../com/carriez/flutter_hbb/InputService.kt | 56 ++++++++++++++----- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 9df9b1a48..76c1ead6d 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -21,10 +21,12 @@ import android.accessibilityservice.AccessibilityServiceInfo import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR import androidx.annotation.RequiresApi import java.util.* +import java.lang.Character import kotlin.math.abs import kotlin.math.max -import hbb.MessageOuterClass.KeyEvent; -import hbb.KeyEventConverter; +import hbb.MessageOuterClass.KeyEvent +import hbb.MessageOuterClass.KeyboardMode +import hbb.KeyEventConverter const val LIFT_DOWN = 9 const val LIFT_MOVE = 8 @@ -265,12 +267,39 @@ class InputService : AccessibilityService() { @RequiresApi(Build.VERSION_CODES.N) fun onKeyEvent(data: ByteArray) { - val keyEvent = KeyEvent.parseFrom(data); + val keyEvent = KeyEvent.parseFrom(data) + val down = keyEvent.getDown() + val keyboardMode = keyEvent.getMode() + + var textToCommit: String? = null + + if (keyboardMode == KeyboardMode.Legacy) { + if (keyEvent.hasChr() && keyEvent.getDown()) { + val chr = keyEvent.getChr() + if (chr != null) { + textToCommit = String(Character.toChars(chr)) + } + } + } else if (keyboardMode == KeyboardMode.Translate) { + if (keyEvent.hasSeq() && keyEvent.getDown()) { + val seq = keyEvent.getSeq() + if (seq != null) { + textToCommit = seq + } + } + } + if (Build.VERSION.SDK_INT >= 33) { getInputMethod()?.let { inputMethod -> inputMethod.getCurrentInputConnection()?.let { inputConnection -> - KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> - inputConnection.sendKeyEvent(event) + if (textToCommit != null) { + textToCommit?.let { text -> + inputConnection.commitText(text, 1, null) + } + } else { + KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> + inputConnection.sendKeyEvent(event) + } } } } @@ -296,26 +325,24 @@ class InputService : AccessibilityService() { } } - if (keyEvent.hasSeq()) { - val seq = keyEvent.getSeq() - + if (textToCommit != null) { var newText = "" if ((textSelectionStart == -1) || (textSelectionEnd == -1)) { - newText = seq - } else { + newText = textToCommit + } else if (text != null) { + Log.d(logTag, "text selection start $textSelectionStart $textSelectionEnd") newText = text.let { - it.substring(0, textSelectionStart) + seq + it.substring(textSelectionStart) + it.substring(0, textSelectionStart) + textToCommit + it.substring(textSelectionStart) } } val arguments = Bundle() arguments.putCharSequence( - AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, - newText + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + newText ) node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) - } else { KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> if (isShowingHint) { @@ -332,7 +359,6 @@ class InputService : AccessibilityService() { this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event) this.fakeEditTextForTextStateCalculation?.getText()?.let { newText -> - Log.d(logTag, "event $event text $text newText $newText") val arguments = Bundle() arguments.putCharSequence( AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, From cbe9b9c455d55db032336029ecdd35052dbad5c2 Mon Sep 17 00:00:00 2001 From: mcfans Date: Tue, 24 Oct 2023 13:36:33 +0800 Subject: [PATCH 09/25] fix: hide map mode if peer is android --- src/flutter_ffi.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index ea6ae1180..8769106ef 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -364,6 +364,9 @@ pub fn session_get_custom_image_quality(session_id: SessionID) -> Option SyncReturn { if let Some(session) = sessions::get_session_by_session_id(&session_id) { if let Ok(mode) = KeyboardMode::from_str(&mode[..]) { + if session.peer_platform() == "Android" && mode == KeyboardMode::Map { + return SyncReturn(false); + } SyncReturn(is_keyboard_mode_supported( &mode, session.get_peer_version(), From 07bdf02af4d83fbd3797a0474ecb06e9ffeaf17f Mon Sep 17 00:00:00 2001 From: mcfans Date: Fri, 27 Oct 2023 13:17:49 +0800 Subject: [PATCH 10/25] chore: use updated old repo --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2a45b074b..f6c014483 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ default-net = "0.14" wol-rs = "1.0" flutter_rust_bridge = { version = "1.75", features = ["uuid"], optional = true} errno = "0.3" -rdev = { git = "https://github.com/mcfans/rdev" } +rdev = { git = "https://github.com/fufesou/rdev" } url = { version = "2.3", features = ["serde"] } crossbeam-queue = "0.3" hex = "0.4" From 67b2a433a89d577d93d6124ba6fba127b56eaca7 Mon Sep 17 00:00:00 2001 From: mcfans Date: Fri, 27 Oct 2023 13:18:35 +0800 Subject: [PATCH 11/25] fix: use enhanced accessibilty node find method --- .../com/carriez/flutter_hbb/InputService.kt | 230 +++++++++++++----- 1 file changed, 168 insertions(+), 62 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 76c1ead6d..86608e3c3 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -268,7 +268,6 @@ class InputService : AccessibilityService() { @RequiresApi(Build.VERSION_CODES.N) fun onKeyEvent(data: ByteArray) { val keyEvent = KeyEvent.parseFrom(data) - val down = keyEvent.getDown() val keyboardMode = keyEvent.getMode() var textToCommit: String? = null @@ -306,66 +305,12 @@ class InputService : AccessibilityService() { } else { val handler = Handler(Looper.getMainLooper()) handler.post { - findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { node -> - val text = node.getText() - val isShowingHint = node.isShowingHintText() - - var textSelectionStart = node.getTextSelectionStart() - var textSelectionEnd = node.getTextSelectionEnd() - - if (text != null) { - if (textSelectionStart > text.length) { - textSelectionStart = text.length - } - if (textSelectionEnd > text.length) { - textSelectionEnd = text.length - } - if (textSelectionStart > textSelectionEnd) { - textSelectionStart = textSelectionEnd - } - } - - if (textToCommit != null) { - var newText = "" - - if ((textSelectionStart == -1) || (textSelectionEnd == -1)) { - newText = textToCommit - } else if (text != null) { - Log.d(logTag, "text selection start $textSelectionStart $textSelectionEnd") - newText = text.let { - it.substring(0, textSelectionStart) + textToCommit + it.substring(textSelectionStart) - } - } - - val arguments = Bundle() - arguments.putCharSequence( - AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, - newText - ) - node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) - } else { - KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> - if (isShowingHint) { - this.fakeEditTextForTextStateCalculation?.setText(null) - } else { - this.fakeEditTextForTextStateCalculation?.setText(text) - } - if (textSelectionStart != -1 && textSelectionEnd != -1) { - this.fakeEditTextForTextStateCalculation?.setSelection( - textSelectionStart, - textSelectionEnd - ) - } - this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event) - - this.fakeEditTextForTextStateCalculation?.getText()?.let { newText -> - val arguments = Bundle() - arguments.putCharSequence( - AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, - newText.toString() - ) - node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) - } + KeyEventConverter.toAndroidKeyEvent(keyEvent)?.let { event -> + val possibleNodes = possibleAccessibiltyNodes() + for (item in possibleNodes) { + val success = trySendKeyEvent(event, item, textToCommit) + if (success) { + break } } } @@ -373,6 +318,165 @@ class InputService : AccessibilityService() { } } + private fun insertAccessibilityNode(list: LinkedList, node: AccessibilityNodeInfo) { + if (list.contains(node)) { + return + } + list.add(node) + } + + private fun findChildNode(node: AccessibilityNodeInfo?): AccessibilityNodeInfo? { + if (node == null) { + return null + } + if (node.isEditable() && node.isFocusable()) { + return node + } + val childCount = node.getChildCount() + for (i in 0 until childCount) { + val child = node.getChild(i) + if (child != null) { + if (child.isEditable() && child.isFocusable()) { + return child + } + if (Build.VERSION.SDK_INT < 33) { + child.recycle() + } + } + } + for (i in 0 until childCount) { + val child = node.getChild(i) + if (child != null) { + val result = findChildNode(child) + if (Build.VERSION.SDK_INT < 33) { + if (child != result) { + child.recycle() + } + } + if (result != null) { + return result + } + } + } + return null + } + + private fun possibleAccessibiltyNodes(): LinkedList { + val linkedList = LinkedList() + val latestList = LinkedList() + + val focusInput = findFocus(AccessibilityNodeInfo.FOCUS_INPUT) + var focusAccessibilityInput = findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY) + + val rootInActiveWindow = getRootInActiveWindow() + + if (focusInput != null) { + if (focusInput.isFocusable() && focusInput.isEditable()) { + insertAccessibilityNode(linkedList, focusInput) + } else { + insertAccessibilityNode(latestList, focusInput) + } + } + + if (focusAccessibilityInput != null) { + if (focusAccessibilityInput.isFocusable() && focusAccessibilityInput.isEditable()) { + insertAccessibilityNode(linkedList, focusAccessibilityInput) + } else { + insertAccessibilityNode(latestList, focusAccessibilityInput) + } + } + + val childFromFocusInput = findChildNode(focusInput) + + if (childFromFocusInput != null) { + insertAccessibilityNode(linkedList, childFromFocusInput) + } + + val childFromFocusAccessibilityInput = findChildNode(focusAccessibilityInput) + if (childFromFocusAccessibilityInput != null) { + insertAccessibilityNode(linkedList, childFromFocusAccessibilityInput) + } + + if (rootInActiveWindow != null) { + insertAccessibilityNode(linkedList, rootInActiveWindow) + } + + for (item in latestList) { + insertAccessibilityNode(linkedList, item) + } + + return linkedList + } + + private fun trySendKeyEvent(event: android.view.KeyEvent, node: AccessibilityNodeInfo, textToCommit: String?): Boolean { + node.refresh() + val text = node.getText() + var isShowingHint = false + if (Build.VERSION.SDK_INT >= 26) { + isShowingHint = node.isShowingHintText() + } + + var textSelectionStart = node.textSelectionStart + var textSelectionEnd = node.textSelectionEnd + + if (text != null) { + if (textSelectionStart > text.length) { + textSelectionStart = text.length + } + if (textSelectionEnd > text.length) { + textSelectionEnd = text.length + } + if (textSelectionStart > textSelectionEnd) { + textSelectionStart = textSelectionEnd + } + } + + var success = false + + if (textToCommit != null) { + var newText = "" + + if ((textSelectionStart == -1) || (textSelectionEnd == -1)) { + newText = textToCommit + } else if (text != null) { + Log.d(logTag, "text selection start $textSelectionStart $textSelectionEnd") + newText = text.let { + it.substring(0, textSelectionStart) + textToCommit + it.substring(textSelectionStart) + } + } + + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + newText + ) + success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + } else { + if (isShowingHint) { + this.fakeEditTextForTextStateCalculation?.setText(null) + } else { + this.fakeEditTextForTextStateCalculation?.setText(text) + } + if (textSelectionStart != -1 && textSelectionEnd != -1) { + this.fakeEditTextForTextStateCalculation?.setSelection( + textSelectionStart, + textSelectionEnd + ) + } + this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event) + + this.fakeEditTextForTextStateCalculation?.getText()?.let { newText -> + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + newText.toString() + ) + success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + } + } + return success + } + override fun onAccessibilityEvent(event: AccessibilityEvent) { } @@ -381,7 +485,9 @@ class InputService : AccessibilityService() { super.onServiceConnected() ctx = this val info = AccessibilityServiceInfo() - info.flags = FLAG_INPUT_METHOD_EDITOR + if (Build.VERSION.SDK_INT >= 33) { + info.flags = FLAG_INPUT_METHOD_EDITOR + } setServiceInfo(info) fakeEditTextForTextStateCalculation = EditText(this) Log.d(logTag, "onServiceConnected!") From e77edc56fdb066e26d02105852e7cf6c7853886e Mon Sep 17 00:00:00 2001 From: mcfans Date: Fri, 27 Oct 2023 16:42:24 +0800 Subject: [PATCH 12/25] chore: update lock --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 404ccf9dd..daa2a205b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4890,7 +4890,7 @@ dependencies = [ [[package]] name = "rdev" version = "0.5.0-2" -source = "git+https://github.com/fufesou/rdev#2e8221d653f4995c831ad52966e79a514516b1fa" +source = "git+https://github.com/fufesou/rdev?branch=master#339b2a334ba273afebb7e27fb76984e620fc76e5" dependencies = [ "cocoa", "core-foundation", @@ -4914,7 +4914,7 @@ dependencies = [ [[package]] name = "rdev" version = "0.5.0-2" -source = "git+https://github.com/mcfans/rdev#97f63d5bc51bb2387d3f2664d3127640e492c9e5" +source = "git+https://github.com/fufesou/rdev#2e8221d653f4995c831ad52966e79a514516b1fa" dependencies = [ "cocoa", "core-foundation", @@ -5220,7 +5220,7 @@ dependencies = [ "os-version", "pam", "parity-tokio-ipc", - "rdev 0.5.0-2 (git+https://github.com/mcfans/rdev)", + "rdev 0.5.0-2 (git+https://github.com/fufesou/rdev?branch=master)", "repng", "reqwest", "ringbuf", diff --git a/Cargo.toml b/Cargo.toml index f6c014483..3f1037f5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ default-net = "0.14" wol-rs = "1.0" flutter_rust_bridge = { version = "1.75", features = ["uuid"], optional = true} errno = "0.3" -rdev = { git = "https://github.com/fufesou/rdev" } +rdev = { git = "https://github.com/fufesou/rdev", branch = "master" } url = { version = "2.3", features = ["serde"] } crossbeam-queue = "0.3" hex = "0.4" From 9076f213e66e0a33d581e076ad8231bde798e606 Mon Sep 17 00:00:00 2001 From: mcfans Date: Fri, 27 Oct 2023 16:43:50 +0800 Subject: [PATCH 13/25] fix: hide map mode if peer is android --- src/ui_session_interface.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 0eea85173..2d24983f6 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -216,6 +216,11 @@ impl Session { pub fn get_keyboard_mode(&self) -> String { let mode = self.lc.read().unwrap().keyboard_mode.clone(); + if self.peer_platform() == crate::PLATFORM_ANDROID { + if mode == "map" { + return "translate".to_string(); + } + } if ["map", "translate", "legacy"].contains(&(&mode as &str)) { mode } else { From 70dd3f323ed44b69c5340ccea43dde9d8c2c1d9a Mon Sep 17 00:00:00 2001 From: mcfans Date: Mon, 30 Oct 2023 07:46:29 +0800 Subject: [PATCH 14/25] fix: update text selection for API level lower than 33 --- .../com/carriez/flutter_hbb/InputService.kt | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 86608e3c3..7637cb1bb 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -19,6 +19,7 @@ import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo import android.accessibilityservice.AccessibilityServiceInfo import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR +import android.view.inputmethod.EditorInfo import androidx.annotation.RequiresApi import java.util.* import java.lang.Character @@ -319,6 +320,9 @@ class InputService : AccessibilityService() { } private fun insertAccessibilityNode(list: LinkedList, node: AccessibilityNodeInfo) { + if (node == null) { + return + } if (list.contains(node)) { return } @@ -410,6 +414,8 @@ class InputService : AccessibilityService() { private fun trySendKeyEvent(event: android.view.KeyEvent, node: AccessibilityNodeInfo, textToCommit: String?): Boolean { node.refresh() + this.fakeEditTextForTextStateCalculation?.setSelection(0,0) + this.fakeEditTextForTextStateCalculation?.setText(null) val text = node.getText() var isShowingHint = false if (Build.VERSION.SDK_INT >= 26) { @@ -463,7 +469,13 @@ class InputService : AccessibilityService() { textSelectionEnd ) } - this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event) + + this.fakeEditTextForTextStateCalculation?.let { + val inputConnection = it.onCreateInputConnection(EditorInfo()) + if (inputConnection != null) { + success = inputConnection.sendKeyEvent(event) + } + } this.fakeEditTextForTextStateCalculation?.getText()?.let { newText -> val arguments = Bundle() @@ -473,6 +485,24 @@ class InputService : AccessibilityService() { ) success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) } + + if (success && this.fakeEditTextForTextStateCalculation != null) { + val selectionStart = this.fakeEditTextForTextStateCalculation?.selectionStart + val selectionEnd = this.fakeEditTextForTextStateCalculation?.selectionEnd + + if (selectionStart != null && selectionEnd != null) { + val arguments = Bundle() + arguments.putInt( + AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, + selectionStart + ) + arguments.putInt( + AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, + selectionEnd + ) + success = node.performAction(AccessibilityNodeInfo.ACTION_SET_SELECTION, arguments) + } + } } return success } From 6fdce633594945f40a26e319ff2a3ee5fc173e3d Mon Sep 17 00:00:00 2001 From: mcfans Date: Mon, 30 Oct 2023 15:34:01 +0800 Subject: [PATCH 15/25] fix: unified keyboard check logic in common.rs --- src/client.rs | 4 ++-- src/common.rs | 14 +++++++++---- src/flutter_ffi.rs | 4 +--- src/ui_session_interface.rs | 41 ++++++++++++++++++++++++------------- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/client.rs b/src/client.rs index 5a3c35467..f59ffe049 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1718,14 +1718,14 @@ impl LoginConfigHandler { crate::flutter::push_global_event(crate::flutter::APP_TYPE_MAIN, evt); } if config.keyboard_mode.is_empty() { - if is_keyboard_mode_supported(&KeyboardMode::Map, get_version_number(&pi.version)) { + if is_keyboard_mode_supported(&KeyboardMode::Map, get_version_number(&pi.version), &pi.platform) { config.keyboard_mode = KeyboardMode::Map.to_string(); } else { config.keyboard_mode = KeyboardMode::Legacy.to_string(); } } else { let keyboard_modes = - crate::get_supported_keyboard_modes(get_version_number(&pi.version)); + crate::get_supported_keyboard_modes(get_version_number(&pi.version), &pi.platform); let current_mode = &KeyboardMode::from_str(&config.keyboard_mode).unwrap_or_default(); if !keyboard_modes.contains(current_mode) { config.keyboard_mode = KeyboardMode::Legacy.to_string(); diff --git a/src/common.rs b/src/common.rs index cf75fab1b..1833d858b 100644 --- a/src/common.rs +++ b/src/common.rs @@ -977,18 +977,24 @@ pub fn make_privacy_mode_msg(state: back_notification::PrivacyModeState) -> Mess make_privacy_mode_msg_with_details(state, "".to_owned()) } -pub fn is_keyboard_mode_supported(keyboard_mode: &KeyboardMode, version_number: i64) -> bool { +pub fn is_keyboard_mode_supported(keyboard_mode: &KeyboardMode, version_number: i64, peer_platform: &str) -> bool { match keyboard_mode { KeyboardMode::Legacy => true, - KeyboardMode::Map => version_number >= hbb_common::get_version_number("1.2.0"), + KeyboardMode::Map => { + if peer_platform.to_lowercase() == crate::PLATFORM_ANDROID.to_lowercase() { + false + } else { + version_number >= hbb_common::get_version_number("1.2.0") + } + } KeyboardMode::Translate => version_number >= hbb_common::get_version_number("1.2.0"), KeyboardMode::Auto => version_number >= hbb_common::get_version_number("1.2.0"), } } -pub fn get_supported_keyboard_modes(version: i64) -> Vec { +pub fn get_supported_keyboard_modes(version: i64, peer_platform: &str) -> Vec { KeyboardMode::iter() - .filter(|&mode| is_keyboard_mode_supported(mode, version)) + .filter(|&mode| is_keyboard_mode_supported(mode, version, peer_platform)) .map(|&mode| mode) .collect::>() } diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index 0f692ee53..753848cc0 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -392,12 +392,10 @@ pub fn session_get_custom_image_quality(session_id: SessionID) -> Option SyncReturn { if let Some(session) = sessions::get_session_by_session_id(&session_id) { if let Ok(mode) = KeyboardMode::from_str(&mode[..]) { - if session.peer_platform() == "Android" && mode == KeyboardMode::Map { - return SyncReturn(false); - } SyncReturn(is_keyboard_mode_supported( &mode, session.get_peer_version(), + &session.peer_platform() )) } else { SyncReturn(false) diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index b9f30b0ef..fc765131e 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -1,4 +1,4 @@ -use crate::input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP, MOUSE_TYPE_WHEEL}; +use crate::{input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP, MOUSE_TYPE_WHEEL}, common::{is_keyboard_mode_supported, get_supported_keyboard_modes}}; use async_trait::async_trait; use bytes::Bytes; use rdev::{Event, EventType::*, KeyCode}; @@ -214,23 +214,36 @@ impl Session { self.lc.read().unwrap().version.clone() } + pub fn fallback_keyboard_mode(&self) -> String { + let peer_version = self.get_peer_version(); + let platform = self.peer_platform(); + + let supported_modes = get_supported_keyboard_modes(peer_version, &platform); + if let Some(mode) = supported_modes.first() { + return mode.to_string(); + } else { + if self.get_peer_version() >= get_version_number("1.2.0") { + return KeyboardMode::Map.to_string(); + } else { + return KeyboardMode::Legacy.to_string(); + } + } + } + pub fn get_keyboard_mode(&self) -> String { let mode = self.lc.read().unwrap().keyboard_mode.clone(); - if self.peer_platform() == crate::PLATFORM_ANDROID { - if mode == "map" { - return "translate".to_string(); + let keyboard_mode = KeyboardMode::from_str(&mode); + + let peer_version = self.get_peer_version(); + let platform = self.peer_platform(); + + // Saved keyboard mode still exists in this version. + if let Ok(mode) = keyboard_mode { + if is_keyboard_mode_supported(&mode, peer_version, &platform) { + return mode.to_string(); } } - if ["map", "translate", "legacy"].contains(&(&mode as &str)) { - mode - } else { - if self.get_peer_version() > hbb_common::get_version_number("1.2.0") { - "map" - } else { - "legacy" - } - .to_string() - } + self.fallback_keyboard_mode() } pub fn save_keyboard_mode(&self, value: String) { From 9521ac6adb97e7116de42b2f5efa820edd3a455c Mon Sep 17 00:00:00 2001 From: mcfans Date: Tue, 31 Oct 2023 21:14:32 +0800 Subject: [PATCH 16/25] chore: add some log --- .../main/kotlin/com/carriez/flutter_hbb/InputService.kt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 7637cb1bb..4141595ab 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -439,6 +439,8 @@ class InputService : AccessibilityService() { var success = false + Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd") + if (textToCommit != null) { var newText = "" @@ -451,6 +453,7 @@ class InputService : AccessibilityService() { } } + Log.d(logTag, "inserting text new text:$newText") val arguments = Bundle() arguments.putCharSequence( AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, @@ -464,6 +467,7 @@ class InputService : AccessibilityService() { this.fakeEditTextForTextStateCalculation?.setText(text) } if (textSelectionStart != -1 && textSelectionEnd != -1) { + Log.d(logTag, "setting selection $textSelectionStart $textSelectionEnd") this.fakeEditTextForTextStateCalculation?.setSelection( textSelectionStart, textSelectionEnd @@ -473,6 +477,7 @@ class InputService : AccessibilityService() { this.fakeEditTextForTextStateCalculation?.let { val inputConnection = it.onCreateInputConnection(EditorInfo()) if (inputConnection != null) { + Log.d(logTag, "sending keyevent $event") success = inputConnection.sendKeyEvent(event) } } @@ -484,6 +489,7 @@ class InputService : AccessibilityService() { newText.toString() ) success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + Log.d(logTag, "Update text to $newText success:$success") } if (success && this.fakeEditTextForTextStateCalculation != null) { @@ -501,6 +507,7 @@ class InputService : AccessibilityService() { selectionEnd ) success = node.performAction(AccessibilityNodeInfo.ACTION_SET_SELECTION, arguments) + Log.d(logTag, "Update selection to $selectionStart $selectionEnd success:$success") } } } From 47d57ddf70890ec3213a3ff3cb267442e29e275e Mon Sep 17 00:00:00 2001 From: mcfans Date: Wed, 1 Nov 2023 00:06:45 +0800 Subject: [PATCH 17/25] fix: set focus and FLAG_RETRIEVE_INTERACTIVE_WINDOWS --- .../com/carriez/flutter_hbb/InputService.kt | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 4141595ab..b70d01374 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -19,6 +19,7 @@ import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo import android.accessibilityservice.AccessibilityServiceInfo import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR +import android.accessibilityservice.AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS import android.view.inputmethod.EditorInfo import androidx.annotation.RequiresApi import java.util.* @@ -289,6 +290,8 @@ class InputService : AccessibilityService() { } } + Log.d(logTag, "onKeyEvent $keyEvent textToCommit:$textToCommit") + if (Build.VERSION.SDK_INT >= 33) { getInputMethod()?.let { inputMethod -> inputMethod.getCurrentInputConnection()?.let { inputConnection -> @@ -308,6 +311,7 @@ class InputService : AccessibilityService() { handler.post { KeyEventConverter.toAndroidKeyEvent(keyEvent)?.let { event -> val possibleNodes = possibleAccessibiltyNodes() + Log.d(logTag, "possibleNodes:$possibleNodes") for (item in possibleNodes) { val success = trySendKeyEvent(event, item, textToCommit) if (success) { @@ -374,6 +378,8 @@ class InputService : AccessibilityService() { val rootInActiveWindow = getRootInActiveWindow() + Log.d(logTag, "focusInput:$focusInput focusAccessibilityInput:$focusAccessibilityInput rootInActiveWindow:$rootInActiveWindow") + if (focusInput != null) { if (focusInput.isFocusable() && focusInput.isEditable()) { insertAccessibilityNode(linkedList, focusInput) @@ -391,6 +397,7 @@ class InputService : AccessibilityService() { } val childFromFocusInput = findChildNode(focusInput) + Log.d(logTag, "childFromFocusInput:$childFromFocusInput") if (childFromFocusInput != null) { insertAccessibilityNode(linkedList, childFromFocusInput) @@ -400,6 +407,7 @@ class InputService : AccessibilityService() { if (childFromFocusAccessibilityInput != null) { insertAccessibilityNode(linkedList, childFromFocusAccessibilityInput) } + Log.d(logTag, "childFromFocusAccessibilityInput:$childFromFocusAccessibilityInput") if (rootInActiveWindow != null) { insertAccessibilityNode(linkedList, rootInActiveWindow) @@ -439,7 +447,12 @@ class InputService : AccessibilityService() { var success = false - Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd") + val focused = this.fakeEditTextForTextStateCalculation?.isFocused + this.fakeEditTextForTextStateCalculation?.let { + it.requestFocus() + } + + Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd $focused") if (textToCommit != null) { var newText = "" @@ -511,6 +524,8 @@ class InputService : AccessibilityService() { } } } + fakeEditTextForTextStateCalculation?.setFocusableInTouchMode(false); + fakeEditTextForTextStateCalculation?.setFocusable(false); return success } @@ -525,8 +540,11 @@ class InputService : AccessibilityService() { if (Build.VERSION.SDK_INT >= 33) { info.flags = FLAG_INPUT_METHOD_EDITOR } + info.flags = FLAG_RETRIEVE_INTERACTIVE_WINDOWS setServiceInfo(info) fakeEditTextForTextStateCalculation = EditText(this) + fakeEditTextForTextStateCalculation?.setFocusable(false) + fakeEditTextForTextStateCalculation?.setFocusableInTouchMode(false) Log.d(logTag, "onServiceConnected!") } From 100967c57bd7d9480fb3aa7d1d7fa8a71b4f1458 Mon Sep 17 00:00:00 2001 From: mcfans Date: Thu, 2 Nov 2023 00:37:21 +0800 Subject: [PATCH 18/25] fix: set focusable before request focus --- .../main/kotlin/com/carriez/flutter_hbb/InputService.kt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index b70d01374..48f0ec9aa 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -424,6 +424,9 @@ class InputService : AccessibilityService() { node.refresh() this.fakeEditTextForTextStateCalculation?.setSelection(0,0) this.fakeEditTextForTextStateCalculation?.setText(null) + + this.fakeEditTextForTextStateCalculation?.setFocusable(true) + this.fakeEditTextForTextStateCalculation?.setFocusableInTouchMode(true) val text = node.getText() var isShowingHint = false if (Build.VERSION.SDK_INT >= 26) { @@ -447,12 +450,14 @@ class InputService : AccessibilityService() { var success = false - val focused = this.fakeEditTextForTextStateCalculation?.isFocused + val previousFocusedState = this.fakeEditTextForTextStateCalculation?.isFocused this.fakeEditTextForTextStateCalculation?.let { it.requestFocus() } - Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd $focused") + val nowFocusedState = this.fakeEditTextForTextStateCalculation?.isFocused + + Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd previous focus: $previousFocusedState now focus: $nowFocusedState") if (textToCommit != null) { var newText = "" From 0a94b7473d7607696232deb24c5847ce657670c6 Mon Sep 17 00:00:00 2001 From: mcfans Date: Fri, 3 Nov 2023 01:50:21 +0800 Subject: [PATCH 19/25] fix: use onKeyDown to dispatch key event --- .../com/carriez/flutter_hbb/InputService.kt | 111 +++++++++--------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 48f0ec9aa..598139567 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -16,6 +16,7 @@ import android.os.Looper import android.util.Log import android.widget.EditText import android.view.accessibility.AccessibilityEvent +import android.view.ViewGroup.LayoutParams import android.view.accessibility.AccessibilityNodeInfo import android.accessibilityservice.AccessibilityServiceInfo import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR @@ -425,8 +426,6 @@ class InputService : AccessibilityService() { this.fakeEditTextForTextStateCalculation?.setSelection(0,0) this.fakeEditTextForTextStateCalculation?.setText(null) - this.fakeEditTextForTextStateCalculation?.setFocusable(true) - this.fakeEditTextForTextStateCalculation?.setFocusableInTouchMode(true) val text = node.getText() var isShowingHint = false if (Build.VERSION.SDK_INT >= 26) { @@ -450,34 +449,18 @@ class InputService : AccessibilityService() { var success = false - val previousFocusedState = this.fakeEditTextForTextStateCalculation?.isFocused - this.fakeEditTextForTextStateCalculation?.let { - it.requestFocus() - } - - val nowFocusedState = this.fakeEditTextForTextStateCalculation?.isFocused - - Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd previous focus: $previousFocusedState now focus: $nowFocusedState") + Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd") if (textToCommit != null) { var newText = "" if ((textSelectionStart == -1) || (textSelectionEnd == -1)) { newText = textToCommit + success = updateTextForAccessibilityNode(node) } else if (text != null) { - Log.d(logTag, "text selection start $textSelectionStart $textSelectionEnd") - newText = text.let { - it.substring(0, textSelectionStart) + textToCommit + it.substring(textSelectionStart) - } + this.fakeEditTextForTextStateCalculation?.text?.insert(textSelectionStart, textToCommit) + success = updateTextAndSelectionForAccessibiltyNode(node) } - - Log.d(logTag, "inserting text new text:$newText") - val arguments = Bundle() - arguments.putCharSequence( - AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, - newText - ) - success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) } else { if (isShowingHint) { this.fakeEditTextForTextStateCalculation?.setText(null) @@ -493,44 +476,57 @@ class InputService : AccessibilityService() { } this.fakeEditTextForTextStateCalculation?.let { - val inputConnection = it.onCreateInputConnection(EditorInfo()) - if (inputConnection != null) { - Log.d(logTag, "sending keyevent $event") - success = inputConnection.sendKeyEvent(event) - } + // This is essiential to make sure layout object is created. OnKeyDown may not work if layout is not created. + it.onPreDraw() + if (event.action == android.view.KeyEvent.ACTION_DOWN) { + val succ = it.onKeyDown(event.getKeyCode(), event) + Log.d(logTag, "onKeyDown $succ") + } else if (event.action == android.view.KeyEvent.ACTION_UP) { + val success = it.onKeyUp(event.getKeyCode(), event) + Log.d(logTag, "keyup $success") + } else {} } - this.fakeEditTextForTextStateCalculation?.getText()?.let { newText -> + success = updateTextAndSelectionForAccessibiltyNode(node) + } + return success + } + + fun updateTextForAccessibilityNode(node: AccessibilityNodeInfo): Boolean { + var success = false + this.fakeEditTextForTextStateCalculation?.text?.let { + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + it.toString() + ) + success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + } + return success + } + + fun updateTextAndSelectionForAccessibiltyNode(node: AccessibilityNodeInfo): Boolean { + var success = updateTextForAccessibilityNode(node) + + if (success) { + val selectionStart = this.fakeEditTextForTextStateCalculation?.selectionStart + val selectionEnd = this.fakeEditTextForTextStateCalculation?.selectionEnd + + if (selectionStart != null && selectionEnd != null) { val arguments = Bundle() - arguments.putCharSequence( - AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, - newText.toString() + arguments.putInt( + AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, + selectionStart ) - success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) - Log.d(logTag, "Update text to $newText success:$success") - } - - if (success && this.fakeEditTextForTextStateCalculation != null) { - val selectionStart = this.fakeEditTextForTextStateCalculation?.selectionStart - val selectionEnd = this.fakeEditTextForTextStateCalculation?.selectionEnd - - if (selectionStart != null && selectionEnd != null) { - val arguments = Bundle() - arguments.putInt( - AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, - selectionStart - ) - arguments.putInt( - AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, - selectionEnd - ) - success = node.performAction(AccessibilityNodeInfo.ACTION_SET_SELECTION, arguments) - Log.d(logTag, "Update selection to $selectionStart $selectionEnd success:$success") - } + arguments.putInt( + AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, + selectionEnd + ) + success = node.performAction(AccessibilityNodeInfo.ACTION_SET_SELECTION, arguments) + Log.d(logTag, "Update selection to $selectionStart $selectionEnd success:$success") } } - fakeEditTextForTextStateCalculation?.setFocusableInTouchMode(false); - fakeEditTextForTextStateCalculation?.setFocusable(false); + return success } @@ -548,8 +544,11 @@ class InputService : AccessibilityService() { info.flags = FLAG_RETRIEVE_INTERACTIVE_WINDOWS setServiceInfo(info) fakeEditTextForTextStateCalculation = EditText(this) - fakeEditTextForTextStateCalculation?.setFocusable(false) - fakeEditTextForTextStateCalculation?.setFocusableInTouchMode(false) + // Size here doesn't matter, we won't show this view. + fakeEditTextForTextStateCalculation?.layoutParams = LayoutParams(100, 100) + fakeEditTextForTextStateCalculation?.onPreDraw() + val layout = fakeEditTextForTextStateCalculation?.getLayout() + Log.d(logTag, "fakeEditTextForTextStateCalculation layout:$layout") Log.d(logTag, "onServiceConnected!") } From c49853e7b4b4d41521e4a3579d7bb205069f2e3d Mon Sep 17 00:00:00 2001 From: mcfans Date: Fri, 3 Nov 2023 10:47:46 +0800 Subject: [PATCH 20/25] fix: set text before update accessibility node --- .../src/main/kotlin/com/carriez/flutter_hbb/InputService.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 598139567..adb888695 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -452,12 +452,12 @@ class InputService : AccessibilityService() { Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd") if (textToCommit != null) { - var newText = "" - if ((textSelectionStart == -1) || (textSelectionEnd == -1)) { - newText = textToCommit + val newText = textToCommit + this.fakeEditTextForTextStateCalculation?.setText(newText) success = updateTextForAccessibilityNode(node) } else if (text != null) { + this.fakeEditTextForTextStateCalculation?.setText(text) this.fakeEditTextForTextStateCalculation?.text?.insert(textSelectionStart, textToCommit) success = updateTextAndSelectionForAccessibiltyNode(node) } From b155cd9a5af2808bb08fc980e7b9260b68102460 Mon Sep 17 00:00:00 2001 From: mcfans Date: Sat, 4 Nov 2023 17:08:49 +0800 Subject: [PATCH 21/25] fix: set same rect for correct layout and navigation and set correct selection --- .../main/kotlin/com/carriez/flutter_hbb/InputService.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index adb888695..57a231975 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -18,6 +18,7 @@ import android.widget.EditText import android.view.accessibility.AccessibilityEvent import android.view.ViewGroup.LayoutParams import android.view.accessibility.AccessibilityNodeInfo +import android.graphics.Rect import android.accessibilityservice.AccessibilityServiceInfo import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR import android.accessibilityservice.AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS @@ -458,6 +459,10 @@ class InputService : AccessibilityService() { success = updateTextForAccessibilityNode(node) } else if (text != null) { this.fakeEditTextForTextStateCalculation?.setText(text) + this.fakeEditTextForTextStateCalculation?.setSelection( + textSelectionStart, + textSelectionEnd + ) this.fakeEditTextForTextStateCalculation?.text?.insert(textSelectionStart, textToCommit) success = updateTextAndSelectionForAccessibiltyNode(node) } @@ -477,6 +482,10 @@ class InputService : AccessibilityService() { this.fakeEditTextForTextStateCalculation?.let { // This is essiential to make sure layout object is created. OnKeyDown may not work if layout is not created. + val rect = Rect() + node.getBoundsInScreen(rect) + + it.layout(rect.left, rect.top, rect.right, rect.bottom) it.onPreDraw() if (event.action == android.view.KeyEvent.ACTION_DOWN) { val succ = it.onKeyDown(event.getKeyCode(), event) From 409d5b124a138858a02a2f3061270fd50fd5d9d4 Mon Sep 17 00:00:00 2001 From: mcfans Date: Sat, 4 Nov 2023 17:57:19 +0800 Subject: [PATCH 22/25] fix: add page up/down keymap --- .../kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt index 5686d30d9..effa3b2aa 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt @@ -90,6 +90,8 @@ object KeyEventConverter { ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP ControlKey.End -> KeyEvent.KEYCODE_MOVE_END ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME + ControlKey.PageUp -> KeyEvent.KEYCODE_PAGE_UP + ControlKey.PageDown -> KeyEvent.KEYCODE_PAGE_DOWN ControlKey.Insert -> KeyEvent.KEYCODE_INSERT ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE ControlKey.F1 -> KeyEvent.KEYCODE_F1 From 6d8272472a9cb11f67350ca0cf3807ab313ca1a5 Mon Sep 17 00:00:00 2001 From: mcfans Date: Sat, 4 Nov 2023 20:24:15 +0800 Subject: [PATCH 23/25] fix: set correct flag --- .../src/main/kotlin/com/carriez/flutter_hbb/InputService.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 57a231975..47c8f302c 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -548,9 +548,10 @@ class InputService : AccessibilityService() { ctx = this val info = AccessibilityServiceInfo() if (Build.VERSION.SDK_INT >= 33) { - info.flags = FLAG_INPUT_METHOD_EDITOR + info.flags = FLAG_INPUT_METHOD_EDITOR or FLAG_RETRIEVE_INTERACTIVE_WINDOWS + } else { + info.flags = FLAG_RETRIEVE_INTERACTIVE_WINDOWS } - info.flags = FLAG_RETRIEVE_INTERACTIVE_WINDOWS setServiceInfo(info) fakeEditTextForTextStateCalculation = EditText(this) // Size here doesn't matter, we won't show this view. From e474b595addbd11852742be414785c7b2006116a Mon Sep 17 00:00:00 2001 From: mcfans Date: Mon, 6 Nov 2023 01:04:53 +0800 Subject: [PATCH 24/25] fix: add proguard rules, avoid protobuf generated classes got obfuscated --- flutter/android/app/build.gradle | 1 + flutter/android/app/proguard-rules | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 flutter/android/app/proguard-rules diff --git a/flutter/android/app/build.gradle b/flutter/android/app/build.gradle index 3793c1f4c..9973c8f62 100644 --- a/flutter/android/app/build.gradle +++ b/flutter/android/app/build.gradle @@ -93,6 +93,7 @@ android { // TODO: Add your own signing config for the release build. // Signing with the debug keys for now, so `flutter run --release` works. signingConfig signingConfigs.release + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules' } } } diff --git a/flutter/android/app/proguard-rules b/flutter/android/app/proguard-rules new file mode 100644 index 000000000..0b12a6cda --- /dev/null +++ b/flutter/android/app/proguard-rules @@ -0,0 +1,4 @@ +# Keep class members from protobuf generated code. +-keepclassmembers class * extends com.google.protobuf.GeneratedMessageLite { + ; +} \ No newline at end of file From 32a29a55560f32dfdead2af087224ceae97c1ba5 Mon Sep 17 00:00:00 2001 From: mcfans Date: Tue, 7 Nov 2023 12:51:16 +0800 Subject: [PATCH 25/25] chore: use a match instead of unwrap error --- src/server/connection.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/server/connection.rs b/src/server/connection.rs index 29235ab73..174c2db9c 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -1806,13 +1806,16 @@ impl Connection { let encode_result = me.write_to_bytes(); - if let Ok(data) = encode_result { - let result = call_main_service_key_event(&data); - if let Err(e) = result { - log::debug!("call_main_service_key_event fail:{}", e); + match encode_result { + Ok(data) => { + let result = call_main_service_key_event(&data); + if let Err(e) = result { + log::debug!("call_main_service_key_event fail: {}", e); + } + } + Err(e) => { + log::debug!("encode key event fail: {}", e); } - } else { - log::debug!("encode key event fail:{}", encode_result.err().unwrap()); } } #[cfg(not(any(target_os = "android", target_os = "ios")))]