diff --git a/Cargo.lock b/Cargo.lock index 2d3de8743..9a79c58e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1796,7 +1796,7 @@ dependencies = [ "log", "objc", "pkg-config", - "rdev", + "rdev 0.5.0-2 (git+https://github.com/fufesou/rdev)", "serde 1.0.190", "serde_derive", "tfc", @@ -4881,6 +4881,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rdev" +version = "0.5.0-2" +source = "git+https://github.com/fufesou/rdev?branch=master#339b2a334ba273afebb7e27fb76984e620fc76e5" +dependencies = [ + "cocoa", + "core-foundation 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", + "core-foundation-sys 0.8.4", + "core-graphics 0.22.3", + "dispatch", + "enum-map", + "epoll", + "inotify", + "lazy_static", + "libc", + "log", + "mio", + "strum 0.24.1", + "strum_macros 0.24.3", + "widestring", + "winapi 0.3.9", + "x11 2.21.0", +] + [[package]] name = "rdev" version = "0.5.0-2" @@ -5226,7 +5250,7 @@ dependencies = [ "pam", "parity-tokio-ipc", "percent-encoding", - "rdev", + "rdev 0.5.0-2 (git+https://github.com/fufesou/rdev?branch=master)", "repng", "reqwest", "ringbuf", diff --git a/Cargo.toml b/Cargo.toml index 706d248b5..a18a3c984 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,7 +71,7 @@ default-net = "0.14" wol-rs = "1.0" flutter_rust_bridge = { version = "=1.80", features = ["uuid"], optional = true} errno = "0.3" -rdev = { git = "https://github.com/fufesou/rdev" } +rdev = { git = "https://github.com/fufesou/rdev", branch = "master" } url = { version = "2.3", features = ["serde"] } crossbeam-queue = "0.3" hex = "0.4" diff --git a/flutter/android/app/build.gradle b/flutter/android/app/build.gradle index a744d6324..9e32e163e 100644 --- a/flutter/android/app/build.gradle +++ b/flutter/android/app/build.gradle @@ -1,3 +1,8 @@ +import com.google.protobuf.gradle.* +plugins { + id "com.google.protobuf" version "0.9.4" +} + def keystoreProperties = new Properties() def keystorePropertiesFile = rootProject.file('key.properties') if (keystorePropertiesFile.exists()) { @@ -31,10 +36,33 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle" +dependencies { + implementation 'com.google.protobuf:protobuf-javalite:3.20.1' +} + +protobuf { + protoc { + artifact = 'com.google.protobuf:protoc:3.20.1' + } + + generateProtoTasks { + all().configureEach { task -> + task.builtins { + java { + option "lite" + } + } + } + } +} + android { compileSdkVersion 33 sourceSets { main.java.srcDirs += 'src/main/kotlin' + + main.proto.srcDirs += '../../../libs/hbb_common/protos' + main.proto.includes += "message.proto" } compileOptions { @@ -65,6 +93,7 @@ android { // TODO: Add your own signing config for the release build. // Signing with the debug keys for now, so `flutter run --release` works. signingConfig signingConfigs.release + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules' } } } diff --git a/flutter/android/app/proguard-rules b/flutter/android/app/proguard-rules new file mode 100644 index 000000000..0b12a6cda --- /dev/null +++ b/flutter/android/app/proguard-rules @@ -0,0 +1,4 @@ +# Keep class members from protobuf generated code. +-keepclassmembers class * extends com.google.protobuf.GeneratedMessageLite { + ; +} \ No newline at end of file diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 203558968..47c8f302c 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -10,12 +10,27 @@ import android.accessibilityservice.AccessibilityService import android.accessibilityservice.GestureDescription import android.graphics.Path import android.os.Build +import android.os.Bundle +import android.os.Handler +import android.os.Looper import android.util.Log +import android.widget.EditText import android.view.accessibility.AccessibilityEvent +import android.view.ViewGroup.LayoutParams +import android.view.accessibility.AccessibilityNodeInfo +import android.graphics.Rect +import android.accessibilityservice.AccessibilityServiceInfo +import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR +import android.accessibilityservice.AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS +import android.view.inputmethod.EditorInfo import androidx.annotation.RequiresApi import java.util.* +import java.lang.Character import kotlin.math.abs import kotlin.math.max +import hbb.MessageOuterClass.KeyEvent +import hbb.MessageOuterClass.KeyboardMode +import hbb.KeyEventConverter const val LIFT_DOWN = 9 const val LIFT_MOVE = 8 @@ -58,6 +73,8 @@ class InputService : AccessibilityService() { private var isWheelActionsPolling = false private var isWaitingLongPress = false + private var fakeEditTextForTextStateCalculation: EditText? = null + @RequiresApi(Build.VERSION_CODES.N) fun onMouseInput(mask: Int, _x: Int, _y: Int) { val x = max(0, _x) @@ -252,9 +269,296 @@ class InputService : AccessibilityService() { } } + @RequiresApi(Build.VERSION_CODES.N) + fun onKeyEvent(data: ByteArray) { + val keyEvent = KeyEvent.parseFrom(data) + val keyboardMode = keyEvent.getMode() + + var textToCommit: String? = null + + if (keyboardMode == KeyboardMode.Legacy) { + if (keyEvent.hasChr() && keyEvent.getDown()) { + val chr = keyEvent.getChr() + if (chr != null) { + textToCommit = String(Character.toChars(chr)) + } + } + } else if (keyboardMode == KeyboardMode.Translate) { + if (keyEvent.hasSeq() && keyEvent.getDown()) { + val seq = keyEvent.getSeq() + if (seq != null) { + textToCommit = seq + } + } + } + + Log.d(logTag, "onKeyEvent $keyEvent textToCommit:$textToCommit") + + if (Build.VERSION.SDK_INT >= 33) { + getInputMethod()?.let { inputMethod -> + inputMethod.getCurrentInputConnection()?.let { inputConnection -> + if (textToCommit != null) { + textToCommit?.let { text -> + inputConnection.commitText(text, 1, null) + } + } else { + KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> + inputConnection.sendKeyEvent(event) + } + } + } + } + } else { + val handler = Handler(Looper.getMainLooper()) + handler.post { + KeyEventConverter.toAndroidKeyEvent(keyEvent)?.let { event -> + val possibleNodes = possibleAccessibiltyNodes() + Log.d(logTag, "possibleNodes:$possibleNodes") + for (item in possibleNodes) { + val success = trySendKeyEvent(event, item, textToCommit) + if (success) { + break + } + } + } + } + } + } + + private fun insertAccessibilityNode(list: LinkedList, node: AccessibilityNodeInfo) { + if (node == null) { + return + } + if (list.contains(node)) { + return + } + list.add(node) + } + + private fun findChildNode(node: AccessibilityNodeInfo?): AccessibilityNodeInfo? { + if (node == null) { + return null + } + if (node.isEditable() && node.isFocusable()) { + return node + } + val childCount = node.getChildCount() + for (i in 0 until childCount) { + val child = node.getChild(i) + if (child != null) { + if (child.isEditable() && child.isFocusable()) { + return child + } + if (Build.VERSION.SDK_INT < 33) { + child.recycle() + } + } + } + for (i in 0 until childCount) { + val child = node.getChild(i) + if (child != null) { + val result = findChildNode(child) + if (Build.VERSION.SDK_INT < 33) { + if (child != result) { + child.recycle() + } + } + if (result != null) { + return result + } + } + } + return null + } + + private fun possibleAccessibiltyNodes(): LinkedList { + val linkedList = LinkedList() + val latestList = LinkedList() + + val focusInput = findFocus(AccessibilityNodeInfo.FOCUS_INPUT) + var focusAccessibilityInput = findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY) + + val rootInActiveWindow = getRootInActiveWindow() + + Log.d(logTag, "focusInput:$focusInput focusAccessibilityInput:$focusAccessibilityInput rootInActiveWindow:$rootInActiveWindow") + + if (focusInput != null) { + if (focusInput.isFocusable() && focusInput.isEditable()) { + insertAccessibilityNode(linkedList, focusInput) + } else { + insertAccessibilityNode(latestList, focusInput) + } + } + + if (focusAccessibilityInput != null) { + if (focusAccessibilityInput.isFocusable() && focusAccessibilityInput.isEditable()) { + insertAccessibilityNode(linkedList, focusAccessibilityInput) + } else { + insertAccessibilityNode(latestList, focusAccessibilityInput) + } + } + + val childFromFocusInput = findChildNode(focusInput) + Log.d(logTag, "childFromFocusInput:$childFromFocusInput") + + if (childFromFocusInput != null) { + insertAccessibilityNode(linkedList, childFromFocusInput) + } + + val childFromFocusAccessibilityInput = findChildNode(focusAccessibilityInput) + if (childFromFocusAccessibilityInput != null) { + insertAccessibilityNode(linkedList, childFromFocusAccessibilityInput) + } + Log.d(logTag, "childFromFocusAccessibilityInput:$childFromFocusAccessibilityInput") + + if (rootInActiveWindow != null) { + insertAccessibilityNode(linkedList, rootInActiveWindow) + } + + for (item in latestList) { + insertAccessibilityNode(linkedList, item) + } + + return linkedList + } + + private fun trySendKeyEvent(event: android.view.KeyEvent, node: AccessibilityNodeInfo, textToCommit: String?): Boolean { + node.refresh() + this.fakeEditTextForTextStateCalculation?.setSelection(0,0) + this.fakeEditTextForTextStateCalculation?.setText(null) + + val text = node.getText() + var isShowingHint = false + if (Build.VERSION.SDK_INT >= 26) { + isShowingHint = node.isShowingHintText() + } + + var textSelectionStart = node.textSelectionStart + var textSelectionEnd = node.textSelectionEnd + + if (text != null) { + if (textSelectionStart > text.length) { + textSelectionStart = text.length + } + if (textSelectionEnd > text.length) { + textSelectionEnd = text.length + } + if (textSelectionStart > textSelectionEnd) { + textSelectionStart = textSelectionEnd + } + } + + var success = false + + Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd") + + if (textToCommit != null) { + if ((textSelectionStart == -1) || (textSelectionEnd == -1)) { + val newText = textToCommit + this.fakeEditTextForTextStateCalculation?.setText(newText) + success = updateTextForAccessibilityNode(node) + } else if (text != null) { + this.fakeEditTextForTextStateCalculation?.setText(text) + this.fakeEditTextForTextStateCalculation?.setSelection( + textSelectionStart, + textSelectionEnd + ) + this.fakeEditTextForTextStateCalculation?.text?.insert(textSelectionStart, textToCommit) + success = updateTextAndSelectionForAccessibiltyNode(node) + } + } else { + if (isShowingHint) { + this.fakeEditTextForTextStateCalculation?.setText(null) + } else { + this.fakeEditTextForTextStateCalculation?.setText(text) + } + if (textSelectionStart != -1 && textSelectionEnd != -1) { + Log.d(logTag, "setting selection $textSelectionStart $textSelectionEnd") + this.fakeEditTextForTextStateCalculation?.setSelection( + textSelectionStart, + textSelectionEnd + ) + } + + this.fakeEditTextForTextStateCalculation?.let { + // This is essiential to make sure layout object is created. OnKeyDown may not work if layout is not created. + val rect = Rect() + node.getBoundsInScreen(rect) + + it.layout(rect.left, rect.top, rect.right, rect.bottom) + it.onPreDraw() + if (event.action == android.view.KeyEvent.ACTION_DOWN) { + val succ = it.onKeyDown(event.getKeyCode(), event) + Log.d(logTag, "onKeyDown $succ") + } else if (event.action == android.view.KeyEvent.ACTION_UP) { + val success = it.onKeyUp(event.getKeyCode(), event) + Log.d(logTag, "keyup $success") + } else {} + } + + success = updateTextAndSelectionForAccessibiltyNode(node) + } + return success + } + + fun updateTextForAccessibilityNode(node: AccessibilityNodeInfo): Boolean { + var success = false + this.fakeEditTextForTextStateCalculation?.text?.let { + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + it.toString() + ) + success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + } + return success + } + + fun updateTextAndSelectionForAccessibiltyNode(node: AccessibilityNodeInfo): Boolean { + var success = updateTextForAccessibilityNode(node) + + if (success) { + val selectionStart = this.fakeEditTextForTextStateCalculation?.selectionStart + val selectionEnd = this.fakeEditTextForTextStateCalculation?.selectionEnd + + if (selectionStart != null && selectionEnd != null) { + val arguments = Bundle() + arguments.putInt( + AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, + selectionStart + ) + arguments.putInt( + AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, + selectionEnd + ) + success = node.performAction(AccessibilityNodeInfo.ACTION_SET_SELECTION, arguments) + Log.d(logTag, "Update selection to $selectionStart $selectionEnd success:$success") + } + } + + return success + } + + + override fun onAccessibilityEvent(event: AccessibilityEvent) { + } + override fun onServiceConnected() { super.onServiceConnected() ctx = this + val info = AccessibilityServiceInfo() + if (Build.VERSION.SDK_INT >= 33) { + info.flags = FLAG_INPUT_METHOD_EDITOR or FLAG_RETRIEVE_INTERACTIVE_WINDOWS + } else { + info.flags = FLAG_RETRIEVE_INTERACTIVE_WINDOWS + } + setServiceInfo(info) + fakeEditTextForTextStateCalculation = EditText(this) + // Size here doesn't matter, we won't show this view. + fakeEditTextForTextStateCalculation?.layoutParams = LayoutParams(100, 100) + fakeEditTextForTextStateCalculation?.onPreDraw() + val layout = fakeEditTextForTextStateCalculation?.getLayout() + Log.d(logTag, "fakeEditTextForTextStateCalculation layout:$layout") Log.d(logTag, "onServiceConnected!") } @@ -263,7 +567,5 @@ class InputService : AccessibilityService() { super.onDestroy() } - override fun onAccessibilityEvent(event: AccessibilityEvent?) {} - override fun onInterrupt() {} } diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt new file mode 100644 index 000000000..effa3b2aa --- /dev/null +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt @@ -0,0 +1,118 @@ +package hbb; +import android.view.KeyEvent +import android.view.KeyCharacterMap +import hbb.MessageOuterClass.KeyboardMode +import hbb.MessageOuterClass.ControlKey + +object KeyEventConverter { + fun toAndroidKeyEvent(keyEventProto: hbb.MessageOuterClass.KeyEvent): KeyEvent { + var chrValue = 0 + var modifiers = 0 + + val keyboardMode = keyEventProto.getMode() + + if (keyEventProto.hasChr()) { + if (keyboardMode == KeyboardMode.Map || keyboardMode == KeyboardMode.Translate) { + chrValue = keyEventProto.getChr() + } else { + chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int) + } + } else if (keyEventProto.hasControlKey()) { + chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey()) + } + + var modifiersList = keyEventProto.getModifiersList() + + if (modifiersList != null) { + for (modifier in keyEventProto.getModifiersList()) { + val modifierValue = convertModifier(modifier) + modifiers = modifiers or modifierValue + } + } + + var action = 0 + if (keyEventProto.getDown()) { + action = KeyEvent.ACTION_DOWN + } else { + action = KeyEvent.ACTION_UP + } + + return KeyEvent(0, 0, action, chrValue, 0, modifiers) + } + + private fun convertModifier(controlKey: hbb.MessageOuterClass.ControlKey): Int { + // Add logic to map ControlKey values to Android KeyEvent key codes. + // You'll need to provide the mapping for each key. + return when (controlKey) { + ControlKey.Alt -> KeyEvent.META_ALT_ON + ControlKey.Control -> KeyEvent.META_CTRL_ON + ControlKey.CapsLock -> KeyEvent.META_CAPS_LOCK_ON + ControlKey.Meta -> KeyEvent.META_META_ON + ControlKey.NumLock -> KeyEvent.META_NUM_LOCK_ON + ControlKey.RShift -> KeyEvent.META_SHIFT_RIGHT_ON + ControlKey.Shift -> KeyEvent.META_SHIFT_ON + ControlKey.RAlt -> KeyEvent.META_ALT_RIGHT_ON + ControlKey.RControl -> KeyEvent.META_CTRL_RIGHT_ON + else -> 0 // Default to unknown. + } + } + + private val tag = "KeyEventConverter" + + private fun convertUnicodeToKeyCode(unicode: Int): Int { + val charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD) + android.util.Log.d(tag, "unicode: $unicode") + val events = charMap.getEvents(charArrayOf(unicode.toChar())) + if (events != null && events.size > 0) { + android.util.Log.d(tag, "keycode ${events[0].keyCode}") + return events[0].keyCode + } + return 0 + } + + private fun convertControlKeyToKeyCode(controlKey: hbb.MessageOuterClass.ControlKey): Int { + // Add logic to map ControlKey values to Android KeyEvent key codes. + // You'll need to provide the mapping for each key. + return when (controlKey) { + ControlKey.Alt -> KeyEvent.KEYCODE_ALT_LEFT + ControlKey.Backspace -> KeyEvent.KEYCODE_DEL + ControlKey.Control -> KeyEvent.KEYCODE_CTRL_LEFT + ControlKey.CapsLock -> KeyEvent.KEYCODE_CAPS_LOCK + ControlKey.Meta -> KeyEvent.KEYCODE_META_LEFT + ControlKey.NumLock -> KeyEvent.KEYCODE_NUM_LOCK + ControlKey.RShift -> KeyEvent.KEYCODE_SHIFT_RIGHT + ControlKey.Shift -> KeyEvent.KEYCODE_SHIFT_LEFT + ControlKey.RAlt -> KeyEvent.KEYCODE_ALT_RIGHT + ControlKey.RControl -> KeyEvent.KEYCODE_CTRL_RIGHT + ControlKey.DownArrow -> KeyEvent.KEYCODE_DPAD_DOWN + ControlKey.LeftArrow -> KeyEvent.KEYCODE_DPAD_LEFT + ControlKey.RightArrow -> KeyEvent.KEYCODE_DPAD_RIGHT + ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP + ControlKey.End -> KeyEvent.KEYCODE_MOVE_END + ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME + ControlKey.PageUp -> KeyEvent.KEYCODE_PAGE_UP + ControlKey.PageDown -> KeyEvent.KEYCODE_PAGE_DOWN + ControlKey.Insert -> KeyEvent.KEYCODE_INSERT + ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE + ControlKey.F1 -> KeyEvent.KEYCODE_F1 + ControlKey.F2 -> KeyEvent.KEYCODE_F2 + ControlKey.F3 -> KeyEvent.KEYCODE_F3 + ControlKey.F4 -> KeyEvent.KEYCODE_F4 + ControlKey.F5 -> KeyEvent.KEYCODE_F5 + ControlKey.F6 -> KeyEvent.KEYCODE_F6 + ControlKey.F7 -> KeyEvent.KEYCODE_F7 + ControlKey.F8 -> KeyEvent.KEYCODE_F8 + ControlKey.F9 -> KeyEvent.KEYCODE_F9 + ControlKey.F10 -> KeyEvent.KEYCODE_F10 + ControlKey.F11 -> KeyEvent.KEYCODE_F11 + ControlKey.F12 -> KeyEvent.KEYCODE_F12 + ControlKey.Space -> KeyEvent.KEYCODE_SPACE + ControlKey.Tab -> KeyEvent.KEYCODE_TAB + ControlKey.Return -> KeyEvent.KEYCODE_ENTER + ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL + ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR + ControlKey.Pause -> KeyEvent.KEYCODE_BREAK + else -> 0 // Default to unknown. + } + } +} diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt index 535a3f8c3..cc2e20e25 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt @@ -44,7 +44,6 @@ import java.nio.ByteBuffer import kotlin.math.max import kotlin.math.min - const val DEFAULT_NOTIFY_TITLE = "RustDesk" const val DEFAULT_NOTIFY_TEXT = "Service is running" const val DEFAULT_NOTIFY_ID = 1 @@ -94,6 +93,12 @@ class MainService : Service() { } } + @Keep + @RequiresApi(Build.VERSION_CODES.N) + fun rustKeyEventInput(input: ByteArray) { + InputService.ctx?.onKeyEvent(input) + } + @Keep fun rustGetByName(name: String): String { return when (name) { diff --git a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml index fa9407128..90b57cd4e 100644 --- a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml +++ b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml @@ -1,5 +1,6 @@ params; DesktopRemoteScreen({Key? key, required this.params}) : super(key: key) { - if (!bind.mainStartGrabKeyboard()) { - stateGlobal.grabKeyboard = true; - } + if (!bind.mainStartGrabKeyboard()) { + stateGlobal.grabKeyboard = true; + } } @override diff --git a/flutter/lib/mobile/pages/remote_page.dart b/flutter/lib/mobile/pages/remote_page.dart index 1353626e2..6afb58eac 100644 --- a/flutter/lib/mobile/pages/remote_page.dart +++ b/flutter/lib/mobile/pages/remote_page.dart @@ -364,6 +364,10 @@ class _RemotePageState extends State { ? [] : gFFI.ffiModel.isPeerAndroid ? [ + IconButton( + color: Colors.white, + icon: Icon(Icons.keyboard), + onPressed: openKeyboard), IconButton( color: Colors.white, icon: const Icon(Icons.build), diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index e9c60ef93..c07fe1fb1 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -173,6 +173,26 @@ pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) -> } } +pub fn call_main_service_key_event(data: &[u8]) -> JniResult<()> { + if let (Some(jvm), Some(ctx)) = ( + JVM.read().unwrap().as_ref(), + MAIN_SERVICE_CTX.read().unwrap().as_ref(), + ) { + let mut env = jvm.attach_current_thread_as_daemon()?; + let data = env.byte_array_from_slice(data)?; + + env.call_method( + ctx, + "rustKeyEventInput", + "([B)V", + &[JValue::Object(&JObject::from(data))], + )?; + return Ok(()); + } else { + return Err(JniError::ThrowFailed(-1)); + } +} + pub fn call_main_service_get_by_name(name: &str) -> JniResult { if let (Some(jvm), Some(ctx)) = ( JVM.read().unwrap().as_ref(), diff --git a/src/client.rs b/src/client.rs index 107aba9fa..14d0e62c4 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1795,14 +1795,14 @@ impl LoginConfigHandler { crate::flutter::push_global_event(crate::flutter::APP_TYPE_MAIN, evt); } if config.keyboard_mode.is_empty() { - if is_keyboard_mode_supported(&KeyboardMode::Map, get_version_number(&pi.version)) { + if is_keyboard_mode_supported(&KeyboardMode::Map, get_version_number(&pi.version), &pi.platform) { config.keyboard_mode = KeyboardMode::Map.to_string(); } else { config.keyboard_mode = KeyboardMode::Legacy.to_string(); } } else { let keyboard_modes = - crate::get_supported_keyboard_modes(get_version_number(&pi.version)); + crate::get_supported_keyboard_modes(get_version_number(&pi.version), &pi.platform); let current_mode = &KeyboardMode::from_str(&config.keyboard_mode).unwrap_or_default(); if !keyboard_modes.contains(current_mode) { config.keyboard_mode = KeyboardMode::Legacy.to_string(); diff --git a/src/common.rs b/src/common.rs index 27e32c98a..0957718e4 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1086,18 +1086,24 @@ pub fn make_privacy_mode_msg(state: back_notification::PrivacyModeState) -> Mess make_privacy_mode_msg_with_details(state, "".to_owned()) } -pub fn is_keyboard_mode_supported(keyboard_mode: &KeyboardMode, version_number: i64) -> bool { +pub fn is_keyboard_mode_supported(keyboard_mode: &KeyboardMode, version_number: i64, peer_platform: &str) -> bool { match keyboard_mode { KeyboardMode::Legacy => true, - KeyboardMode::Map => version_number >= hbb_common::get_version_number("1.2.0"), + KeyboardMode::Map => { + if peer_platform.to_lowercase() == crate::PLATFORM_ANDROID.to_lowercase() { + false + } else { + version_number >= hbb_common::get_version_number("1.2.0") + } + } KeyboardMode::Translate => version_number >= hbb_common::get_version_number("1.2.0"), KeyboardMode::Auto => version_number >= hbb_common::get_version_number("1.2.0"), } } -pub fn get_supported_keyboard_modes(version: i64) -> Vec { +pub fn get_supported_keyboard_modes(version: i64, peer_platform: &str) -> Vec { KeyboardMode::iter() - .filter(|&mode| is_keyboard_mode_supported(mode, version)) + .filter(|&mode| is_keyboard_mode_supported(mode, version, peer_platform)) .map(|&mode| mode) .collect::>() } diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index 433bc0a4d..436a95587 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -395,6 +395,7 @@ pub fn session_is_keyboard_mode_supported(session_id: SessionID, mode: String) - SyncReturn(is_keyboard_mode_supported( &mode, session.get_peer_version(), + &session.peer_platform() )) } else { SyncReturn(false) diff --git a/src/keyboard.rs b/src/keyboard.rs index 9a6ac49d5..3baeef2ec 100644 --- a/src/keyboard.rs +++ b/src/keyboard.rs @@ -28,6 +28,8 @@ const OS_LOWER_WINDOWS: &str = "windows"; const OS_LOWER_LINUX: &str = "linux"; #[allow(dead_code)] const OS_LOWER_MACOS: &str = "macos"; +#[allow(dead_code)] +const OS_LOWER_ANDROID: &str = "android"; #[cfg(any(target_os = "windows", target_os = "macos"))] static KEYBOARD_HOOKED: AtomicBool = AtomicBool::new(false); @@ -163,6 +165,21 @@ pub mod client { } } + pub fn map_key_to_control_key(key: &rdev::Key) -> Option { + match key { + Key::Alt => Some(ControlKey::Alt), + Key::ShiftLeft => Some(ControlKey::Shift), + Key::ControlLeft => Some(ControlKey::Control), + Key::MetaLeft => Some(ControlKey::Meta), + Key::AltGr => Some(ControlKey::RAlt), + Key::ShiftRight => Some(ControlKey::RShift), + Key::ControlRight => Some(ControlKey::RControl), + Key::MetaRight => Some(ControlKey::RWin), + _ => None, + } + + } + pub fn event_lock_screen() -> KeyEvent { let mut key_event = KeyEvent::new(); key_event.set_control_key(ControlKey::LockScreen); @@ -355,7 +372,7 @@ pub fn get_keyboard_mode_enum(keyboard_mode: &str) -> KeyboardMode { } #[inline] -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] pub fn is_modifier(key: &rdev::Key) -> bool { matches!( key, @@ -850,12 +867,14 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) -> rdev::win_scancode_to_macos_code(event.position_code)? } } + OS_LOWER_ANDROID => rdev::win_scancode_to_android_key_code(event.position_code)?, _ => rdev::win_scancode_to_linux_code(event.position_code)?, }; #[cfg(target_os = "macos")] let keycode = match _peer { OS_LOWER_WINDOWS => rdev::macos_code_to_win_scancode(event.platform_code as _)?, OS_LOWER_MACOS => event.platform_code as _, + OS_LOWER_ANDROID => rdev::macos_code_to_android_key_code(event.platform_code as _)?, _ => rdev::macos_code_to_linux_code(event.platform_code as _)?, }; #[cfg(target_os = "linux")] @@ -868,6 +887,7 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) -> rdev::linux_code_to_macos_code(event.position_code as _)? } } + OS_LOWER_ANDROID => rdev::linux_code_to_android_key_code(event.position_code as _)?, _ => event.position_code as _, }; #[cfg(any(target_os = "android", target_os = "ios"))] @@ -877,7 +897,7 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) -> Some(key_event) } -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] fn try_fill_unicode(_peer: &str, event: &Event, key_event: &KeyEvent, events: &mut Vec) { match &event.unicode { Some(unicode_info) => { @@ -1046,12 +1066,14 @@ pub fn translate_keyboard_mode(peer: &str, event: &Event, key_event: KeyEvent) - events } -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] pub fn keycode_to_rdev_key(keycode: u32) -> Key { #[cfg(target_os = "windows")] return rdev::win_key_from_scancode(keycode); - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux"))] return rdev::linux_key_from_code(keycode); + #[cfg(any(target_os = "android"))] + return rdev::android_key_from_code(keycode); #[cfg(target_os = "macos")] return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default()); } diff --git a/src/server/connection.rs b/src/server/connection.rs index 2b0829035..174c2db9c 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -38,10 +38,12 @@ use hbb_common::{ sync::mpsc, time::{self, Duration, Instant, Interval}, }, - tokio_util::codec::{BytesCodec, Framed}, + tokio_util::codec::{BytesCodec, Framed}, protobuf::EnumOrUnknown, }; #[cfg(any(target_os = "android", target_os = "ios"))] -use scrap::android::call_main_service_pointer_input; +use scrap::android::{call_main_service_pointer_input, call_main_service_key_event}; +#[cfg(target_os = "android")] +use crate::keyboard::client::map_key_to_control_key; use serde_derive::Serialize; use serde_json::{json, value::Value}; use sha2::{Digest, Sha256}; @@ -57,7 +59,7 @@ use system_shutdown; #[cfg(all(windows, feature = "virtual_display_driver"))] use crate::virtual_display_manager; -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] use std::collections::HashSet; pub type Sender = mpsc::UnboundedSender<(Instant, Arc)>; @@ -215,7 +217,7 @@ pub struct Connection { voice_call_request_timestamp: Option, audio_input_device_before_voice_call: Option, options_in_login: Option, - #[cfg(not(any(target_os = "android", target_os = "ios")))] + #[cfg(not(any(target_os = "ios")))] pressed_modifiers: HashSet, #[cfg(all(target_os = "linux", feature = "linux_headless"))] #[cfg(not(any(feature = "flatpak", feature = "appimage")))] @@ -354,7 +356,7 @@ impl Connection { voice_call_request_timestamp: None, audio_input_device_before_voice_call: None, options_in_login: None, - #[cfg(not(any(target_os = "android", target_os = "ios")))] + #[cfg(not(any(target_os = "ios")))] pressed_modifiers: Default::default(), #[cfg(all(target_os = "linux", feature = "linux_headless"))] #[cfg(not(any(feature = "flatpak", feature = "appimage")))] @@ -1763,8 +1765,59 @@ impl Connection { } self.update_auto_disconnect_timer(); } - #[cfg(any(target_os = "android", target_os = "ios"))] + #[cfg(any(target_os = "ios"))] Some(message::Union::KeyEvent(..)) => {} + #[cfg(any(target_os = "android"))] + Some(message::Union::KeyEvent(mut me)) => { + let is_press = (me.press || me.down) && !crate::is_modifier(&me); + + let key = match me.mode.enum_value() { + Ok(KeyboardMode::Map) => { + Some(crate::keyboard::keycode_to_rdev_key(me.chr())) + } + Ok(KeyboardMode::Translate) => { + if let Some(key_event::Union::Chr(code)) = me.union { + Some(crate::keyboard::keycode_to_rdev_key(code & 0x0000FFFF)) + } else { + None + } + } + _ => None, + } + .filter(crate::keyboard::is_modifier); + + if let Some(key) = key { + if is_press { + self.pressed_modifiers.insert(key); + } else { + self.pressed_modifiers.remove(&key); + } + } + + let mut modifiers = vec![]; + + for key in self.pressed_modifiers.iter() { + if let Some(control_key) = map_key_to_control_key(key) { + modifiers.push(EnumOrUnknown::new(control_key)); + } + } + + me.modifiers = modifiers; + + let encode_result = me.write_to_bytes(); + + match encode_result { + Ok(data) => { + let result = call_main_service_key_event(&data); + if let Err(e) = result { + log::debug!("call_main_service_key_event fail: {}", e); + } + } + Err(e) => { + log::debug!("encode key event fail: {}", e); + } + } + } #[cfg(not(any(target_os = "android", target_os = "ios")))] Some(message::Union::KeyEvent(me)) => { if self.peer_keyboard_enabled() { diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index b5ec529a4..c82b91029 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -1,4 +1,4 @@ -use crate::input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP, MOUSE_TYPE_WHEEL}; +use crate::{input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP, MOUSE_TYPE_WHEEL}, common::{is_keyboard_mode_supported, get_supported_keyboard_modes}}; use async_trait::async_trait; use bytes::Bytes; use rdev::{Event, EventType::*, KeyCode}; @@ -213,18 +213,36 @@ impl Session { self.lc.read().unwrap().version.clone() } + pub fn fallback_keyboard_mode(&self) -> String { + let peer_version = self.get_peer_version(); + let platform = self.peer_platform(); + + let supported_modes = get_supported_keyboard_modes(peer_version, &platform); + if let Some(mode) = supported_modes.first() { + return mode.to_string(); + } else { + if self.get_peer_version() >= get_version_number("1.2.0") { + return KeyboardMode::Map.to_string(); + } else { + return KeyboardMode::Legacy.to_string(); + } + } + } + pub fn get_keyboard_mode(&self) -> String { let mode = self.lc.read().unwrap().keyboard_mode.clone(); - if ["map", "translate", "legacy"].contains(&(&mode as &str)) { - mode - } else { - if self.get_peer_version() > hbb_common::get_version_number("1.2.0") { - "map" - } else { - "legacy" + let keyboard_mode = KeyboardMode::from_str(&mode); + + let peer_version = self.get_peer_version(); + let platform = self.peer_platform(); + + // Saved keyboard mode still exists in this version. + if let Ok(mode) = keyboard_mode { + if is_keyboard_mode_supported(&mode, peer_version, &platform) { + return mode.to_string(); } - .to_string() } + self.fallback_keyboard_mode() } pub fn save_keyboard_mode(&self, value: String) {