Feat/android more actions (#8496)

* feat: android volume and power actions Signed-off-by: fufesou <linlong1266@gmail.com> * Add translations and refact action menus Signed-off-by: fufesou <linlong1266@gmail.com> * Remove divider Signed-off-by: fufesou <linlong1266@gmail.com> * fix: recover deleted translations Signed-off-by: fufesou <linlong1266@gmail.com> --------- Signed-off-by: fufesou <linlong1266@gmail.com>
2025-10-29 17:00:05 +00:00 · 2024-06-27 13:28:05 +08:00
parent b047730830
commit c5d3c7f390
54 changed files with 500 additions and 15 deletions
--- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt
+++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt
@@ -18,7 +18,9 @@ import android.widget.EditText
 import android.view.accessibility.AccessibilityEvent
 import android.view.ViewGroup.LayoutParams
 import android.view.accessibility.AccessibilityNodeInfo
+import android.view.KeyEvent as KeyEventAndroid
 import android.graphics.Rect
+import android.media.AudioManager
 import android.accessibilityservice.AccessibilityServiceInfo
 import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR
 import android.accessibilityservice.AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
@@ -75,6 +77,8 @@ class InputService : AccessibilityService() {

    private var fakeEditTextForTextStateCalculation: EditText? = null

+    private val volumeController: VolumeController by lazy { VolumeController(applicationContext.getSystemService(AUDIO_SERVICE) as AudioManager) }
+
    @RequiresApi(Build.VERSION_CODES.N)
    fun onMouseInput(mask: Int, _x: Int, _y: Int) {
        val x = max(0, _x)
@@ -294,6 +298,18 @@ class InputService : AccessibilityService() {

        Log.d(logTag, "onKeyEvent $keyEvent textToCommit:$textToCommit")

+        var ke: KeyEventAndroid? = null
+        if (Build.VERSION.SDK_INT < 33 || textToCommit == null) {
+            ke = KeyEventConverter.toAndroidKeyEvent(keyEvent)
+        }
+        ke?.let { event ->
+            if (tryHandleVolumeKeyEvent(event)) {
+                return
+            } else if (tryHandlePowerKeyEvent(event)) {
+                return
+            }
+        }
+
        if (Build.VERSION.SDK_INT >= 33) {
            getInputMethod()?.let { inputMethod ->
                inputMethod.getCurrentInputConnection()?.let { inputConnection ->
@@ -302,7 +318,7 @@ class InputService : AccessibilityService() {
                            inputConnection.commitText(text, 1, null)
                        }
                    } else {
-                        KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event ->
+                        ke?.let { event ->
                            inputConnection.sendKeyEvent(event)
                        }
                    }
@@ -311,7 +327,7 @@ class InputService : AccessibilityService() {
        } else {
            val handler = Handler(Looper.getMainLooper())
            handler.post {
-                KeyEventConverter.toAndroidKeyEvent(keyEvent)?.let { event ->
+                ke?.let { event ->
                    val possibleNodes = possibleAccessibiltyNodes()
                    Log.d(logTag, "possibleNodes:$possibleNodes")
                    for (item in possibleNodes) {
@@ -325,6 +341,43 @@ class InputService : AccessibilityService() {
        }
    }

+    private fun tryHandleVolumeKeyEvent(event: KeyEventAndroid): Boolean {
+        when (event.keyCode) {
+            KeyEventAndroid.KEYCODE_VOLUME_UP -> {
+                if (event.action == KeyEventAndroid.ACTION_DOWN) {
+                    volumeController.raiseVolume(null, true, AudioManager.STREAM_SYSTEM)
+                }
+                return true
+            }
+            KeyEventAndroid.KEYCODE_VOLUME_DOWN -> {
+                if (event.action == KeyEventAndroid.ACTION_DOWN) {
+                    volumeController.lowerVolume(null, true, AudioManager.STREAM_SYSTEM)
+                }
+                return true
+            }
+            KeyEventAndroid.KEYCODE_VOLUME_MUTE -> {
+                if (event.action == KeyEventAndroid.ACTION_DOWN) {
+                    volumeController.toggleMute(true, AudioManager.STREAM_SYSTEM)
+                }
+                return true
+            }
+            else -> {
+                return false
+            }
+        }
+    }
+
+    private fun tryHandlePowerKeyEvent(event: KeyEventAndroid): Boolean {
+        if (event.keyCode == KeyEventAndroid.KEYCODE_POWER) {
+            // Perform power dialog action when action is up
+            if (event.action == KeyEventAndroid.ACTION_UP) {
+                performGlobalAction(GLOBAL_ACTION_POWER_DIALOG);
+            }
+            return true
+        }
+        return false
+    }
+
    private fun insertAccessibilityNode(list: LinkedList<AccessibilityNodeInfo>, node: AccessibilityNodeInfo) {
        if (node == null) {
            return
@@ -422,7 +475,7 @@ class InputService : AccessibilityService() {
        return linkedList
    }

-    private fun trySendKeyEvent(event: android.view.KeyEvent, node: AccessibilityNodeInfo, textToCommit: String?): Boolean {
+    private fun trySendKeyEvent(event: KeyEventAndroid, node: AccessibilityNodeInfo, textToCommit: String?): Boolean {
        node.refresh()
        this.fakeEditTextForTextStateCalculation?.setSelection(0,0)
        this.fakeEditTextForTextStateCalculation?.setText(null)
@@ -487,10 +540,10 @@ class InputService : AccessibilityService() {

                it.layout(rect.left, rect.top, rect.right, rect.bottom)
                it.onPreDraw()
-                if (event.action == android.view.KeyEvent.ACTION_DOWN) {
+                if (event.action == KeyEventAndroid.ACTION_DOWN) {
                    val succ = it.onKeyDown(event.getKeyCode(), event)
                    Log.d(logTag, "onKeyDown $succ")
-                } else if (event.action == android.view.KeyEvent.ACTION_UP) {
+                } else if (event.action == KeyEventAndroid.ACTION_UP) {
                    val success = it.onKeyUp(event.getKeyCode(), event)
                    Log.d(logTag, "keyup $success")
                } else {}
--- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt
+++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt
@@ -37,6 +37,8 @@ object KeyEventConverter {
            action = KeyEvent.ACTION_UP
        }

+        // FIXME: The last parameter is the repeat count, not modifiers ?
+        // https://developer.android.com/reference/android/view/KeyEvent#KeyEvent(long,%20long,%20int,%20int,%20int)
        return KeyEvent(0, 0, action, chrValue, 0, modifiers)
    }

@@ -112,6 +114,10 @@ object KeyEventConverter {
            ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL
            ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR
            ControlKey.Pause -> KeyEvent.KEYCODE_BREAK
+            ControlKey.VolumeMute -> KeyEvent.KEYCODE_VOLUME_MUTE
+            ControlKey.VolumeUp -> KeyEvent.KEYCODE_VOLUME_UP
+            ControlKey.VolumeDown -> KeyEvent.KEYCODE_VOLUME_DOWN
+            ControlKey.Power -> KeyEvent.KEYCODE_POWER
            else -> 0 // Default to unknown.
        }
    }
--- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/VolumeController.kt
+++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/VolumeController.kt
@@ -0,0 +1,78 @@
+package com.carriez.flutter_hbb
+
+// Inspired by https://github.com/yosemiteyss/flutter_volume_controller/blob/main/android/src/main/kotlin/com/yosemiteyss/flutter_volume_controller/VolumeController.kt
+
+import android.media.AudioManager
+import android.os.Build
+import android.util.Log
+
+class VolumeController(private val audioManager: AudioManager) {
+    private val logTag = "volume controller"
+
+    fun getVolume(streamType: Int): Double {
+        val current = audioManager.getStreamVolume(streamType)
+        val max = audioManager.getStreamMaxVolume(streamType)
+        return current.toDouble() / max
+    }
+
+    fun setVolume(volume: Double, showSystemUI: Boolean, streamType: Int) {
+        val max = audioManager.getStreamMaxVolume(streamType)
+        audioManager.setStreamVolume(
+            streamType,
+            (max * volume).toInt(),
+            if (showSystemUI) AudioManager.FLAG_SHOW_UI else 0
+        )
+    }
+
+    fun raiseVolume(step: Double?, showSystemUI: Boolean, streamType: Int) {
+        if (step == null) {
+            audioManager.adjustStreamVolume(
+                streamType,
+                AudioManager.ADJUST_RAISE,
+                if (showSystemUI) AudioManager.FLAG_SHOW_UI else 0
+            )
+        } else {
+            val target = getVolume(streamType) + step
+            setVolume(target, showSystemUI, streamType)
+        }
+    }
+
+    fun lowerVolume(step: Double?, showSystemUI: Boolean, streamType: Int) {
+        if (step == null) {
+            audioManager.adjustStreamVolume(
+                streamType,
+                AudioManager.ADJUST_LOWER,
+                if (showSystemUI) AudioManager.FLAG_SHOW_UI else 0
+            )
+        } else {
+            val target = getVolume(streamType) - step
+            setVolume(target, showSystemUI, streamType)
+        }
+    }
+
+    fun getMute(streamType: Int): Boolean {
+        return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+            audioManager.isStreamMute(streamType)
+        } else {
+            audioManager.getStreamVolume(streamType) == 0
+        }
+    }
+
+    private fun setMute(isMuted: Boolean, showSystemUI: Boolean, streamType: Int) {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+            audioManager.adjustStreamVolume(
+                streamType,
+                if (isMuted) AudioManager.ADJUST_MUTE else AudioManager.ADJUST_UNMUTE,
+                if (showSystemUI) AudioManager.FLAG_SHOW_UI else 0
+            )
+        } else {
+            audioManager.setStreamMute(streamType, isMuted)
+        }
+    }
+
+    fun toggleMute(showSystemUI: Boolean, streamType: Int) {
+        val isMuted = getMute(streamType)
+        setMute(!isMuted, showSystemUI, streamType)
+    }
+}
+
--- a/flutter/lib/common.dart
+++ b/flutter/lib/common.dart
@@ -928,13 +928,9 @@ makeMobileActionsOverlayEntry(VoidCallback? onHide, {FFI? ffi}) {
      position: draggablePositions.mobileActions,
      width: overlayW,
      height: overlayH,
-      onBackPressed: () => session.inputModel.tap(MouseButtons.right),
-      onHomePressed: () => session.inputModel.tap(MouseButtons.wheel),
-      onRecentPressed: () async {
-        session.inputModel.sendMouse('down', MouseButtons.wheel);
-        await Future.delayed(const Duration(milliseconds: 500));
-        session.inputModel.sendMouse('up', MouseButtons.wheel);
-      },
+      onBackPressed: session.inputModel.onMobileBack,
+      onHomePressed: session.inputModel.onMobileHome,
+      onRecentPressed: session.inputModel.onMobileApps,
      onHidePressed: onHide,
    );
  }
--- a/flutter/lib/common/widgets/toolbar.dart
+++ b/flutter/lib/common/widgets/toolbar.dart
@@ -6,6 +6,7 @@ import 'package:flutter_hbb/common.dart';
 import 'package:flutter_hbb/common/shared_state.dart';
 import 'package:flutter_hbb/common/widgets/dialog.dart';
 import 'package:flutter_hbb/consts.dart';
+import 'package:flutter_hbb/models/input_model.dart';
 import 'package:flutter_hbb/models/model.dart';
 import 'package:flutter_hbb/models/platform_model.dart';
 import 'package:get/get.dart';
@@ -76,6 +77,36 @@ List<TTextMenu> toolbarControls(BuildContext context, String id, FFI ffi) {
  final sessionId = ffi.sessionId;

  List<TTextMenu> v = [];
+  if (isMobile &&
+      pi.platform == kPeerPlatformAndroid &&
+      perms['keyboard'] != false) {
+    v.addAll([
+      TTextMenu(
+        child: Text(translate('Back')),
+        onPressed: () => ffi.inputModel.onMobileBack(),
+      ),
+      TTextMenu(
+        child: Text(translate('Home')),
+        onPressed: () => ffi.inputModel.onMobileHome(),
+      ),
+      TTextMenu(
+        child: Text(translate('Apps')),
+        onPressed: () => ffi.inputModel.onMobileApps(),
+      ),
+      TTextMenu(
+        child: Text(translate('Volume up')),
+        onPressed: () => ffi.inputModel.onMobileVolumeUp(),
+      ),
+      TTextMenu(
+        child: Text(translate('Volume down')),
+        onPressed: () => ffi.inputModel.onMobileVolumeDown(),
+      ),
+      TTextMenu(
+        child: Text(translate('Power')),
+        onPressed: () => ffi.inputModel.onMobilePower(),
+      ),
+    ]);
+  }
  // elevation
  if (perms['keyboard'] != false && ffi.elevationModel.showRequestMenu) {
    v.add(
--- a/flutter/lib/consts.dart
+++ b/flutter/lib/consts.dart
@@ -155,6 +155,8 @@ const int kWindowMainId = 0;
 const String kPointerEventKindTouch = "touch";
 const String kPointerEventKindMouse = "mouse";

+const String kKeyFlutterKey = "flutter_key";
+
 const String kKeyShowDisplaysAsIndividualWindows =
    'displays_as_individual_windows';
 const String kKeyUseAllMyDisplaysForTheRemoteSession =
--- a/flutter/lib/desktop/widgets/remote_toolbar.dart
+++ b/flutter/lib/desktop/widgets/remote_toolbar.dart
@@ -6,6 +6,7 @@ import 'package:flutter/services.dart';
 import 'package:flutter_hbb/common/widgets/audio_input.dart';
 import 'package:flutter_hbb/common/widgets/toolbar.dart';
 import 'package:flutter_hbb/models/chat_model.dart';
+import 'package:flutter_hbb/models/input_model.dart';
 import 'package:flutter_hbb/models/state_model.dart';
 import 'package:flutter_hbb/consts.dart';
 import 'package:flutter_hbb/utils/multi_window_manager.dart';
@@ -1741,6 +1742,7 @@ class _KeyboardMenu extends StatelessWidget {
              viewMode(),
              Divider(),
              ...toolbarToggles(),
+              ...mobileActions(),
            ]);
  }

@@ -1877,6 +1879,39 @@ class _KeyboardMenu extends StatelessWidget {
        ffi: ffi,
        child: Text(translate('View Mode')));
  }
+
+  mobileActions() {
+    if (pi.platform != kPeerPlatformAndroid) return [];
+    final enabled = versionCmp(pi.version, '1.2.6') >= 0;
+    if (!enabled) return [];
+    return [
+      Divider(),
+      MenuButton(
+          child: Text(translate('Back')),
+          onPressed: () => ffi.inputModel.onMobileBack(),
+          ffi: ffi),
+      MenuButton(
+          child: Text(translate('Home')),
+          onPressed: () => ffi.inputModel.onMobileHome(),
+          ffi: ffi),
+      MenuButton(
+          child: Text(translate('Apps')),
+          onPressed: () => ffi.inputModel.onMobileApps(),
+          ffi: ffi),
+      MenuButton(
+          child: Text(translate('Volume up')),
+          onPressed: () => ffi.inputModel.onMobileVolumeUp(),
+          ffi: ffi),
+      MenuButton(
+          child: Text(translate('Volume down')),
+          onPressed: () => ffi.inputModel.onMobileVolumeDown(),
+          ffi: ffi),
+      MenuButton(
+          child: Text(translate('Power')),
+          onPressed: () => ffi.inputModel.onMobilePower(),
+          ffi: ffi),
+    ];
+  }
 }

 class _ChatMenu extends StatefulWidget {
--- a/flutter/lib/models/input_model.dart
+++ b/flutter/lib/models/input_model.dart
@@ -1152,4 +1152,27 @@ class InputModel {
      platformFFI.stopDesktopWebListener();
    }
  }
+
+  void onMobileBack() => tap(MouseButtons.right);
+  void onMobileHome() => tap(MouseButtons.wheel);
+  Future<void> onMobileApps() async {
+    sendMouse('down', MouseButtons.wheel);
+    await Future.delayed(const Duration(milliseconds: 500));
+    sendMouse('up', MouseButtons.wheel);
+  }
+
+  // Simulate a key press event.
+  // `usbHidUsage` is the USB HID usage code of the key.
+  Future<void> tapHidKey(int usbHidUsage) async {
+    inputRawKey(kKeyFlutterKey, usbHidUsage, 0, true);
+    await Future.delayed(Duration(milliseconds: 100));
+    inputRawKey(kKeyFlutterKey, usbHidUsage, 0, false);
+  }
+
+  Future<void> onMobileVolumeUp() async =>
+      await tapHidKey(PhysicalKeyboardKey.audioVolumeUp.usbHidUsage);
+  Future<void> onMobileVolumeDown() async =>
+      await tapHidKey(PhysicalKeyboardKey.audioVolumeDown.usbHidUsage);
+  Future<void> onMobilePower() async =>
+      await tapHidKey(PhysicalKeyboardKey.power.usbHidUsage);
 }
--- a/flutter/lib/models/model.dart
+++ b/flutter/lib/models/model.dart
@@ -195,7 +195,7 @@ class FfiModel with ChangeNotifier {
    if (desktopType == DesktopType.remote) {
      KeyboardEnabledState.find(id).value = _permissions['keyboard'] != false;
    }
-    debugPrint('$_permissions');
+    debugPrint('updatePermission: $_permissions');
    notifyListeners();
  }