Merge pull request #6097 from mcfans/master

Physical keyboard to android support

flutter/android/app/build.gradle

@@ -1,3 +1,8 @@
+import com.google.protobuf.gradle.*
+plugins {
+    id "com.google.protobuf" version "0.9.4"
+}
+
 def keystoreProperties = new Properties()
 def keystorePropertiesFile = rootProject.file('key.properties')
 if (keystorePropertiesFile.exists()) {
@@ -31,10 +36,33 @@ apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
 apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"
 
+dependencies {
+    implementation 'com.google.protobuf:protobuf-javalite:3.20.1'
+}
+
+protobuf {
+    protoc {
+        artifact = 'com.google.protobuf:protoc:3.20.1'
+    }
+
+    generateProtoTasks {
+        all().configureEach { task ->
+            task.builtins {
+                java {
+                    option "lite"
+                }
+            }
+        }
+    }
+}
+
 android {
     compileSdkVersion 33
     sourceSets {
         main.java.srcDirs += 'src/main/kotlin'
+
+        main.proto.srcDirs += '../../../libs/hbb_common/protos'
+        main.proto.includes += "message.proto"
     }
 
     compileOptions {
@@ -65,6 +93,7 @@ android {
             // TODO: Add your own signing config for the release build.
             // Signing with the debug keys for now, so `flutter run --release` works.
             signingConfig signingConfigs.release
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules'
         }
     }
 }

flutter/android/app/proguard-rules

@@ -0,0 +1,4 @@
+# Keep class members from protobuf generated code.
+-keepclassmembers class * extends com.google.protobuf.GeneratedMessageLite {
+  <fields>;
+}

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -10,12 +10,27 @@ import android.accessibilityservice.AccessibilityService
 import android.accessibilityservice.GestureDescription
 import android.graphics.Path
 import android.os.Build
+import android.os.Bundle
+import android.os.Handler
+import android.os.Looper
 import android.util.Log
+import android.widget.EditText
 import android.view.accessibility.AccessibilityEvent
+import android.view.ViewGroup.LayoutParams
+import android.view.accessibility.AccessibilityNodeInfo
+import android.graphics.Rect
+import android.accessibilityservice.AccessibilityServiceInfo
+import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR
+import android.accessibilityservice.AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
+import android.view.inputmethod.EditorInfo
 import androidx.annotation.RequiresApi
 import java.util.*
+import java.lang.Character
 import kotlin.math.abs
 import kotlin.math.max
+import hbb.MessageOuterClass.KeyEvent
+import hbb.MessageOuterClass.KeyboardMode
+import hbb.KeyEventConverter
 
 const val LIFT_DOWN = 9
 const val LIFT_MOVE = 8
@@ -58,6 +73,8 @@ class InputService : AccessibilityService() {
     private var isWheelActionsPolling = false
     private var isWaitingLongPress = false
 
+    private var fakeEditTextForTextStateCalculation: EditText? = null
+
     @RequiresApi(Build.VERSION_CODES.N)
     fun onMouseInput(mask: Int, _x: Int, _y: Int) {
         val x = max(0, _x)
@@ -252,9 +269,296 @@ class InputService : AccessibilityService() {
         }
     }
 
+    @RequiresApi(Build.VERSION_CODES.N)
+    fun onKeyEvent(data: ByteArray) {
+        val keyEvent = KeyEvent.parseFrom(data)
+        val keyboardMode = keyEvent.getMode()
+
+        var textToCommit: String? = null
+
+        if (keyboardMode == KeyboardMode.Legacy) {
+            if (keyEvent.hasChr() && keyEvent.getDown()) {
+                val chr = keyEvent.getChr()
+                if (chr != null) {
+                    textToCommit = String(Character.toChars(chr))
+                }
+            }
+        } else if (keyboardMode == KeyboardMode.Translate) {
+            if (keyEvent.hasSeq() && keyEvent.getDown()) {
+                val seq = keyEvent.getSeq()
+                if (seq != null) {
+                    textToCommit = seq
+                }
+            }
+        }
+
+        Log.d(logTag, "onKeyEvent $keyEvent textToCommit:$textToCommit")
+
+        if (Build.VERSION.SDK_INT >= 33) {
+            getInputMethod()?.let { inputMethod ->
+                inputMethod.getCurrentInputConnection()?.let { inputConnection ->
+                    if (textToCommit != null) {
+                        textToCommit?.let { text ->
+                            inputConnection.commitText(text, 1, null)
+                        }
+                    } else {
+                        KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event ->
+                            inputConnection.sendKeyEvent(event)
+                        }
+                    }
+                }
+            }
+        } else {
+            val handler = Handler(Looper.getMainLooper())
+            handler.post {
+                KeyEventConverter.toAndroidKeyEvent(keyEvent)?.let { event ->
+                    val possibleNodes = possibleAccessibiltyNodes()
+                    Log.d(logTag, "possibleNodes:$possibleNodes")
+                    for (item in possibleNodes) {
+                        val success = trySendKeyEvent(event, item, textToCommit)
+                        if (success) {
+                            break
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private fun insertAccessibilityNode(list: LinkedList<AccessibilityNodeInfo>, node: AccessibilityNodeInfo) {
+        if (node == null) {
+            return
+        }
+        if (list.contains(node)) {
+            return
+        }
+        list.add(node)
+    }
+
+    private fun findChildNode(node: AccessibilityNodeInfo?): AccessibilityNodeInfo? {
+        if (node == null) {
+            return null
+        }
+        if (node.isEditable() && node.isFocusable()) {
+            return node
+        }
+        val childCount = node.getChildCount()
+        for (i in 0 until childCount) {
+            val child = node.getChild(i)
+            if (child != null) {
+                if (child.isEditable() && child.isFocusable()) {
+                    return child
+                }
+                if (Build.VERSION.SDK_INT < 33) {
+                    child.recycle()
+                }
+            }
+        }
+        for (i in 0 until childCount) {
+            val child = node.getChild(i)
+            if (child != null) {
+                val result = findChildNode(child)
+                if (Build.VERSION.SDK_INT < 33) {
+                    if (child != result) {
+                        child.recycle()
+                    }
+                }
+                if (result != null) {
+                    return result
+                }
+            }
+        }
+        return null
+    }
+
+    private fun possibleAccessibiltyNodes(): LinkedList<AccessibilityNodeInfo> {
+        val linkedList = LinkedList<AccessibilityNodeInfo>()
+        val latestList = LinkedList<AccessibilityNodeInfo>()
+
+        val focusInput = findFocus(AccessibilityNodeInfo.FOCUS_INPUT)
+        var focusAccessibilityInput = findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY)
+
+        val rootInActiveWindow = getRootInActiveWindow()
+
+        Log.d(logTag, "focusInput:$focusInput focusAccessibilityInput:$focusAccessibilityInput rootInActiveWindow:$rootInActiveWindow")
+
+        if (focusInput != null) {
+            if (focusInput.isFocusable() && focusInput.isEditable()) {
+                insertAccessibilityNode(linkedList, focusInput)
+            } else {
+                insertAccessibilityNode(latestList, focusInput)
+            }
+        }
+
+        if (focusAccessibilityInput != null) {
+            if (focusAccessibilityInput.isFocusable() && focusAccessibilityInput.isEditable()) {
+                insertAccessibilityNode(linkedList, focusAccessibilityInput)
+            } else {
+                insertAccessibilityNode(latestList, focusAccessibilityInput)
+            }
+        }
+
+        val childFromFocusInput = findChildNode(focusInput)
+        Log.d(logTag, "childFromFocusInput:$childFromFocusInput")
+
+        if (childFromFocusInput != null) {
+            insertAccessibilityNode(linkedList, childFromFocusInput)
+        }
+
+        val childFromFocusAccessibilityInput = findChildNode(focusAccessibilityInput)
+        if (childFromFocusAccessibilityInput != null) {
+            insertAccessibilityNode(linkedList, childFromFocusAccessibilityInput)
+        }
+        Log.d(logTag, "childFromFocusAccessibilityInput:$childFromFocusAccessibilityInput")
+
+        if (rootInActiveWindow != null) {
+            insertAccessibilityNode(linkedList, rootInActiveWindow)
+        }
+
+        for (item in latestList) {
+            insertAccessibilityNode(linkedList, item)
+        }
+
+        return linkedList
+    }
+
+    private fun trySendKeyEvent(event: android.view.KeyEvent, node: AccessibilityNodeInfo, textToCommit: String?): Boolean {
+        node.refresh()
+        this.fakeEditTextForTextStateCalculation?.setSelection(0,0)
+        this.fakeEditTextForTextStateCalculation?.setText(null)
+
+        val text = node.getText()
+        var isShowingHint = false
+        if (Build.VERSION.SDK_INT >= 26) {
+            isShowingHint = node.isShowingHintText()
+        }
+
+        var textSelectionStart = node.textSelectionStart
+        var textSelectionEnd = node.textSelectionEnd
+
+        if (text != null) {
+            if (textSelectionStart > text.length) {
+                textSelectionStart = text.length
+            }
+            if (textSelectionEnd > text.length) {
+                textSelectionEnd = text.length
+            }
+            if (textSelectionStart > textSelectionEnd) {
+                textSelectionStart = textSelectionEnd
+            }
+        }
+
+        var success = false
+
+        Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd")
+
+        if (textToCommit != null) {
+            if ((textSelectionStart == -1) || (textSelectionEnd == -1)) {
+                val newText = textToCommit
+                this.fakeEditTextForTextStateCalculation?.setText(newText)
+                success = updateTextForAccessibilityNode(node)
+            } else if (text != null) {
+                this.fakeEditTextForTextStateCalculation?.setText(text)
+                this.fakeEditTextForTextStateCalculation?.setSelection(
+                    textSelectionStart,
+                    textSelectionEnd
+                )
+                this.fakeEditTextForTextStateCalculation?.text?.insert(textSelectionStart, textToCommit)
+                success = updateTextAndSelectionForAccessibiltyNode(node)
+            }
+        } else {
+            if (isShowingHint) {
+                this.fakeEditTextForTextStateCalculation?.setText(null)
+            } else {
+                this.fakeEditTextForTextStateCalculation?.setText(text)
+            }
+            if (textSelectionStart != -1 && textSelectionEnd != -1) {
+                Log.d(logTag, "setting selection $textSelectionStart $textSelectionEnd")
+                this.fakeEditTextForTextStateCalculation?.setSelection(
+                    textSelectionStart,
+                    textSelectionEnd
+                )
+            }
+
+            this.fakeEditTextForTextStateCalculation?.let {
+                // This is essiential to make sure layout object is created. OnKeyDown may not work if layout is not created.
+                val rect = Rect()
+                node.getBoundsInScreen(rect)
+
+                it.layout(rect.left, rect.top, rect.right, rect.bottom)
+                it.onPreDraw()
+                if (event.action == android.view.KeyEvent.ACTION_DOWN) {
+                    val succ = it.onKeyDown(event.getKeyCode(), event)
+                    Log.d(logTag, "onKeyDown $succ")
+                } else if (event.action == android.view.KeyEvent.ACTION_UP) {
+                    val success = it.onKeyUp(event.getKeyCode(), event)
+                    Log.d(logTag, "keyup $success")
+                } else {}
+            }
+
+            success = updateTextAndSelectionForAccessibiltyNode(node)
+        }
+        return success
+    }
+
+    fun updateTextForAccessibilityNode(node: AccessibilityNodeInfo): Boolean {
+        var success = false
+        this.fakeEditTextForTextStateCalculation?.text?.let {
+            val arguments = Bundle()
+            arguments.putCharSequence(
+                AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
+                it.toString()
+            )
+            success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
+        }
+        return success
+    }
+
+    fun updateTextAndSelectionForAccessibiltyNode(node: AccessibilityNodeInfo): Boolean {
+        var success = updateTextForAccessibilityNode(node)
+
+        if (success) {
+            val selectionStart = this.fakeEditTextForTextStateCalculation?.selectionStart
+            val selectionEnd = this.fakeEditTextForTextStateCalculation?.selectionEnd
+
+            if (selectionStart != null && selectionEnd != null) {
+                val arguments = Bundle()
+                arguments.putInt(
+                    AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT,
+                    selectionStart
+                )
+                arguments.putInt(
+                    AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT,
+                    selectionEnd
+                )
+                success = node.performAction(AccessibilityNodeInfo.ACTION_SET_SELECTION, arguments)
+                Log.d(logTag, "Update selection to $selectionStart $selectionEnd success:$success")
+            }
+        }
+
+        return success
+    }
+
+
+    override fun onAccessibilityEvent(event: AccessibilityEvent) {
+    }
+
     override fun onServiceConnected() {
         super.onServiceConnected()
         ctx = this
+        val info = AccessibilityServiceInfo()
+        if (Build.VERSION.SDK_INT >= 33) {
+            info.flags = FLAG_INPUT_METHOD_EDITOR or FLAG_RETRIEVE_INTERACTIVE_WINDOWS
+        } else {
+            info.flags = FLAG_RETRIEVE_INTERACTIVE_WINDOWS
+        }
+        setServiceInfo(info)
+        fakeEditTextForTextStateCalculation = EditText(this)
+        // Size here doesn't matter, we won't show this view.
+        fakeEditTextForTextStateCalculation?.layoutParams = LayoutParams(100, 100)
+        fakeEditTextForTextStateCalculation?.onPreDraw()
+        val layout = fakeEditTextForTextStateCalculation?.getLayout()
+        Log.d(logTag, "fakeEditTextForTextStateCalculation layout:$layout")
         Log.d(logTag, "onServiceConnected!")
     }
 
@@ -263,7 +567,5 @@ class InputService : AccessibilityService() {
         super.onDestroy()
     }
 
-    override fun onAccessibilityEvent(event: AccessibilityEvent?) {}
-
     override fun onInterrupt() {}
 }

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt

@@ -0,0 +1,118 @@
+package hbb;
+import android.view.KeyEvent
+import android.view.KeyCharacterMap
+import hbb.MessageOuterClass.KeyboardMode
+import hbb.MessageOuterClass.ControlKey
+
+object KeyEventConverter {
+    fun toAndroidKeyEvent(keyEventProto: hbb.MessageOuterClass.KeyEvent): KeyEvent {
+        var chrValue = 0
+        var modifiers = 0
+
+        val keyboardMode = keyEventProto.getMode()
+
+        if (keyEventProto.hasChr()) {
+            if (keyboardMode == KeyboardMode.Map || keyboardMode == KeyboardMode.Translate) {
+                chrValue = keyEventProto.getChr()
+            } else {
+                chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int)
+            }
+        } else if (keyEventProto.hasControlKey()) {
+            chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey())
+        }
+
+        var modifiersList = keyEventProto.getModifiersList()
+
+        if (modifiersList != null) {
+            for (modifier in keyEventProto.getModifiersList()) {
+                val modifierValue = convertModifier(modifier)
+                modifiers = modifiers or modifierValue
+            }
+        }
+
+        var action = 0
+        if (keyEventProto.getDown()) {
+            action = KeyEvent.ACTION_DOWN
+        } else {
+            action = KeyEvent.ACTION_UP
+        }
+
+        return KeyEvent(0, 0, action, chrValue, 0, modifiers)
+    }
+
+    private fun convertModifier(controlKey: hbb.MessageOuterClass.ControlKey): Int {
+        // Add logic to map ControlKey values to Android KeyEvent key codes.
+        // You'll need to provide the mapping for each key.
+        return when (controlKey) {
+            ControlKey.Alt -> KeyEvent.META_ALT_ON
+            ControlKey.Control -> KeyEvent.META_CTRL_ON
+            ControlKey.CapsLock -> KeyEvent.META_CAPS_LOCK_ON
+            ControlKey.Meta -> KeyEvent.META_META_ON
+            ControlKey.NumLock -> KeyEvent.META_NUM_LOCK_ON
+            ControlKey.RShift -> KeyEvent.META_SHIFT_RIGHT_ON
+            ControlKey.Shift -> KeyEvent.META_SHIFT_ON
+            ControlKey.RAlt -> KeyEvent.META_ALT_RIGHT_ON
+            ControlKey.RControl -> KeyEvent.META_CTRL_RIGHT_ON
+            else -> 0 // Default to unknown.
+        }
+    }
+
+    private val tag = "KeyEventConverter"
+
+    private fun convertUnicodeToKeyCode(unicode: Int): Int {
+        val charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD)
+        android.util.Log.d(tag, "unicode: $unicode")
+        val events = charMap.getEvents(charArrayOf(unicode.toChar()))
+        if (events != null && events.size > 0) {
+            android.util.Log.d(tag, "keycode ${events[0].keyCode}")
+            return events[0].keyCode
+        }
+        return 0
+    }
+
+    private fun convertControlKeyToKeyCode(controlKey: hbb.MessageOuterClass.ControlKey): Int {
+        // Add logic to map ControlKey values to Android KeyEvent key codes.
+        // You'll need to provide the mapping for each key.
+        return when (controlKey) {
+            ControlKey.Alt -> KeyEvent.KEYCODE_ALT_LEFT
+            ControlKey.Backspace -> KeyEvent.KEYCODE_DEL
+            ControlKey.Control -> KeyEvent.KEYCODE_CTRL_LEFT
+            ControlKey.CapsLock -> KeyEvent.KEYCODE_CAPS_LOCK
+            ControlKey.Meta -> KeyEvent.KEYCODE_META_LEFT
+            ControlKey.NumLock -> KeyEvent.KEYCODE_NUM_LOCK
+            ControlKey.RShift -> KeyEvent.KEYCODE_SHIFT_RIGHT
+            ControlKey.Shift -> KeyEvent.KEYCODE_SHIFT_LEFT
+            ControlKey.RAlt -> KeyEvent.KEYCODE_ALT_RIGHT
+            ControlKey.RControl -> KeyEvent.KEYCODE_CTRL_RIGHT
+            ControlKey.DownArrow -> KeyEvent.KEYCODE_DPAD_DOWN
+            ControlKey.LeftArrow -> KeyEvent.KEYCODE_DPAD_LEFT
+            ControlKey.RightArrow -> KeyEvent.KEYCODE_DPAD_RIGHT
+            ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP
+            ControlKey.End -> KeyEvent.KEYCODE_MOVE_END
+            ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME
+            ControlKey.PageUp -> KeyEvent.KEYCODE_PAGE_UP
+            ControlKey.PageDown -> KeyEvent.KEYCODE_PAGE_DOWN
+            ControlKey.Insert -> KeyEvent.KEYCODE_INSERT
+            ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE
+            ControlKey.F1 -> KeyEvent.KEYCODE_F1
+            ControlKey.F2 -> KeyEvent.KEYCODE_F2
+            ControlKey.F3 -> KeyEvent.KEYCODE_F3
+            ControlKey.F4 -> KeyEvent.KEYCODE_F4
+            ControlKey.F5 -> KeyEvent.KEYCODE_F5
+            ControlKey.F6 -> KeyEvent.KEYCODE_F6
+            ControlKey.F7 -> KeyEvent.KEYCODE_F7
+            ControlKey.F8 -> KeyEvent.KEYCODE_F8
+            ControlKey.F9 -> KeyEvent.KEYCODE_F9
+            ControlKey.F10 -> KeyEvent.KEYCODE_F10
+            ControlKey.F11 -> KeyEvent.KEYCODE_F11
+            ControlKey.F12 -> KeyEvent.KEYCODE_F12
+            ControlKey.Space -> KeyEvent.KEYCODE_SPACE
+            ControlKey.Tab -> KeyEvent.KEYCODE_TAB
+            ControlKey.Return -> KeyEvent.KEYCODE_ENTER
+            ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL
+            ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR
+            ControlKey.Pause -> KeyEvent.KEYCODE_BREAK
+            else -> 0 // Default to unknown.
+        }
+    }
+}

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -44,7 +44,6 @@ import java.nio.ByteBuffer
 import kotlin.math.max
 import kotlin.math.min
 
-
 const val DEFAULT_NOTIFY_TITLE = "RustDesk"
 const val DEFAULT_NOTIFY_TEXT = "Service is running"
 const val DEFAULT_NOTIFY_ID = 1
@@ -94,6 +93,12 @@ class MainService : Service() {
         }
     }
 
+    @Keep
+    @RequiresApi(Build.VERSION_CODES.N)
+    fun rustKeyEventInput(input: ByteArray) {
+        InputService.ctx?.onKeyEvent(input)
+    }
+
     @Keep
     fun rustGetByName(name: String): String {
         return when (name) {

flutter/android/app/src/main/res/xml/accessibility_service_config.xml

@@ -1,5 +1,6 @@
 <accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
     android:accessibilityEventTypes="typeWindowsChanged"
+    android:canRetrieveWindowContent="true"
     android:accessibilityFlags="flagDefault"
     android:notificationTimeout="50"
     android:description="@string/accessibility_service_description"

flutter/lib/desktop/screen/desktop_remote_screen.dart

@@ -12,9 +12,9 @@ class DesktopRemoteScreen extends StatelessWidget {
   final Map<String, dynamic> params;
 
   DesktopRemoteScreen({Key? key, required this.params}) : super(key: key) {
-    if (!bind.mainStartGrabKeyboard()) {
-      stateGlobal.grabKeyboard = true;
-    }
+      if (!bind.mainStartGrabKeyboard()) {
+        stateGlobal.grabKeyboard = true;
+      }
   }
 
   @override

flutter/lib/mobile/pages/remote_page.dart

@@ -364,6 +364,10 @@ class _RemotePageState extends State<RemotePage> {
                       ? []
                       : gFFI.ffiModel.isPeerAndroid
                           ? [
+                              IconButton(
+                                  color: Colors.white,
+                                  icon: Icon(Icons.keyboard),
+                                  onPressed: openKeyboard),
                               IconButton(
                                 color: Colors.white,
                                 icon: const Icon(Icons.build),