Feat: Wayland flatpak input support | Remote desktop portal (#6675)

* autogen portal code

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* use remote desktop portal

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* remove clipboard portal in favour of #6586

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* remove clipboard portal

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* use select_devices for input capture

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* remove embedded cursor code as not being used | return session path for input capture

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* setup rdp input

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* remove simulate example

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* setup rdp input raw key events + mouse movements

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* fix rdp raw key input

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* refact rdp raw key inpuy & fix right meta key

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* refact and support rdp layout mode key input

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* support rdp mouse clicks

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* support rdp mouse scroll

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* support rdp key sequence input

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* use rdp input only when uinput is not available

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* combine rdp input and get_capturables into a single rdp request

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* rdp fix build

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* rdp fix build

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* fix rdp caps lock

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* format pipewire.rs

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* format rdp_input.rs

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* revert #6628 as rdp request state is now managed (better solution)

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* fix rdp crash on arch kde

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* rdp_input.rs improvements

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* refact request_remote_desktop

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* improve unwraps

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* remove unwraps

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

* use session references instead of clones

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

---------

Signed-off-by: Sahil Yeole <sahilyeole93@gmail.com>

src/server.rs

@@ -39,6 +39,8 @@ pub(crate) mod wayland;
 #[cfg(target_os = "linux")]
 pub mod uinput;
 #[cfg(target_os = "linux")]
+pub mod rdp_input;
+#[cfg(target_os = "linux")]
 pub mod dbus;
 pub mod input_service;
 } else {

src/server/connection.rs

@@ -5,6 +5,8 @@ use crate::clipboard_file::*;
 use crate::common::update_clipboard;
 #[cfg(target_os = "android")]
 use crate::keyboard::client::map_key_to_control_key;
+#[cfg(target_os = "linux")]
+use crate::platform::linux::is_x11;
 #[cfg(all(target_os = "linux", feature = "linux_headless"))]
 #[cfg(not(any(feature = "flatpak", feature = "appimage")))]
 use crate::platform::linux_desktop_manager;
@@ -12,8 +14,6 @@ use crate::platform::linux_desktop_manager;
 use crate::platform::WallPaperRemover;
 #[cfg(windows)]
 use crate::portable_service::client as portable_client;
-#[cfg(target_os = "linux")]
-use crate::platform::linux::is_x11;
 use crate::{
     client::{
         new_voice_call_request, new_voice_call_response, start_audio_thread, MediaData, MediaSender,
@@ -1172,7 +1172,21 @@ impl Connection {
             ..Default::default()
         })
         .into();
-        pi.resolutions = Self::get_supported_resolutions(self.display_idx).into();
+        #[cfg(not(any(target_os = "android", target_os = "ios")))]
+        {
+            pi.resolutions = Some(SupportedResolutions {
+                resolutions: display_service::try_get_displays()
+                    .map(|displays| {
+                        displays
+                            .get(self.display_idx)
+                            .map(|d| crate::platform::resolutions(&d.name()))
+                            .unwrap_or(vec![])
+                    })
+                    .unwrap_or(vec![]),
+                ..Default::default()
+            })
+            .into();
+        }
 
         let mut sub_service = false;
         if self.file_transfer.is_some() {
@@ -1194,6 +1208,14 @@ impl Connection {
                     pi.current_display = self.display_idx as _;
                     res.set_peer_info(pi);
                     sub_service = true;
+
+                    #[cfg(target_os = "linux")]
+                    {
+                        // use rdp_input when uinput is not available in wayland. Ex: flatpak
+                        if !is_x11() && !crate::is_server() {
+                            let _ = setup_rdp_input().await;
+                        }
+                    }
                 }
             }
             self.on_remote_authorized();
@@ -1236,31 +1258,6 @@ impl Connection {
         }
     }
 
-    fn get_supported_resolutions(display_idx: usize) -> Option<SupportedResolutions> {
-        #[cfg(any(target_os = "android", target_os = "ios"))]
-        return None;
-        #[cfg(not(any(target_os = "android", target_os = "ios")))]
-        {
-            #[cfg(target_os = "linux")]
-            {
-                if !is_x11() {
-                    return None;
-                }
-            }
-            Some(SupportedResolutions {
-                resolutions: display_service::try_get_displays()
-                    .map(|displays| {
-                        displays
-                            .get(display_idx)
-                            .map(|d| crate::platform::resolutions(&d.name()))
-                            .unwrap_or(vec![])
-                    })
-                    .unwrap_or(vec![]),
-                ..Default::default()
-            })
-        }
-    }
-
     fn on_remote_authorized(&self) {
         self.update_codec_on_login();
         #[cfg(any(target_os = "windows", target_os = "linux"))]

src/server/input_service.rs

@@ -5,6 +5,8 @@ use crate::input::*;
 #[cfg(target_os = "macos")]
 use dispatch::Queue;
 use enigo::{Enigo, Key, KeyboardControllable, MouseButton, MouseControllable};
+#[cfg(target_os = "linux")]
+use super::rdp_input::client::{RdpInputKeyboard, RdpInputMouse};
 use hbb_common::{
     get_time,
     message_proto::{pointer_device_event::Union::TouchEvent, touch_event::Union::ScaleUpdate},
@@ -13,6 +15,8 @@ use hbb_common::{
 use rdev::{self, EventType, Key as RdevKey, KeyCode, RawKey};
 #[cfg(target_os = "macos")]
 use rdev::{CGEventSourceStateID, CGEventTapLocation, VirtualInput};
+#[cfg(target_os = "linux")]
+use scrap::wayland::pipewire::RDP_RESPONSE;
 use std::{
     convert::TryFrom,
     ops::{Deref, DerefMut, Sub},
@@ -461,6 +465,25 @@ pub async fn setup_uinput(minx: i32, maxx: i32, miny: i32, maxy: i32) -> ResultT
     Ok(())
 }
 
+#[cfg(target_os = "linux")]
+pub async fn setup_rdp_input() -> ResultType<(), Box<dyn std::error::Error>> {
+    let mut en = ENIGO.lock()?;
+    let rdp_res_lock = RDP_RESPONSE.lock()?;
+    let rdp_res = rdp_res_lock.as_ref().ok_or("RDP response is None")?;
+
+    let keyboard = RdpInputKeyboard::new(rdp_res.conn.clone(), rdp_res.session.clone())?;
+    en.set_custom_keyboard(Box::new(keyboard));
+    log::info!("RdpInput keyboard created");
+
+    if let Some(stream) = rdp_res.streams.clone().into_iter().next() {
+        let mouse = RdpInputMouse::new(rdp_res.conn.clone(), rdp_res.session.clone(), stream)?;
+        en.set_custom_mouse(Box::new(mouse));
+        log::info!("RdpInput mouse created");
+    }
+
+    Ok(())
+}
+
 #[cfg(target_os = "linux")]
 pub async fn update_mouse_resolution(minx: i32, maxx: i32, miny: i32, maxy: i32) -> ResultType<()> {
     set_uinput_resolution(minx, maxx, miny, maxy).await?;

src/server/rdp_input.rs

@@ -0,0 +1,234 @@
+use crate::uinput::service::map_key;
+use dbus::{blocking::SyncConnection, Path};
+use enigo::{Key, KeyboardControllable, MouseButton, MouseControllable};
+use hbb_common::ResultType;
+use scrap::wayland::pipewire::{get_portal, PwStreamInfo};
+use scrap::wayland::remote_desktop_portal::OrgFreedesktopPortalRemoteDesktop as remote_desktop_portal;
+use std::collections::HashMap;
+use std::sync::Arc;
+
+pub mod client {
+    use super::*;
+
+    const EVDEV_MOUSE_LEFT: i32 = 272;
+    const EVDEV_MOUSE_RIGHT: i32 = 273;
+    const EVDEV_MOUSE_MIDDLE: i32 = 274;
+
+    const PRESSED_DOWN_STATE: u32 = 1;
+    const PRESSED_UP_STATE: u32 = 0;
+
+    pub struct RdpInputKeyboard {
+        conn: Arc<SyncConnection>,
+        session: Path<'static>,
+    }
+
+    impl RdpInputKeyboard {
+        pub fn new(conn: Arc<SyncConnection>, session: Path<'static>) -> ResultType<Self> {
+            Ok(Self { conn, session })
+        }
+    }
+
+    impl KeyboardControllable for RdpInputKeyboard {
+        fn as_any(&self) -> &dyn std::any::Any {
+            self
+        }
+
+        fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+            self
+        }
+
+        fn get_key_state(&mut self, _: Key) -> bool {
+            // no api for this
+            false
+        }
+
+        fn key_sequence(&mut self, s: &str) {
+            for c in s.chars() {
+                let key = Key::Layout(c);
+                let _ = handle_key(true, key, self.conn.clone(), &self.session);
+                let _ = handle_key(false, key, self.conn.clone(), &self.session);
+            }
+        }
+
+        fn key_down(&mut self, key: Key) -> enigo::ResultType {
+            handle_key(true, key, self.conn.clone(), &self.session)?;
+            Ok(())
+        }
+        fn key_up(&mut self, key: Key) {
+            let _ = handle_key(false, key, self.conn.clone(), &self.session);
+        }
+        fn key_click(&mut self, key: Key) {
+            let _ = handle_key(true, key, self.conn.clone(), &self.session);
+            let _ = handle_key(false, key, self.conn.clone(), &self.session);
+        }
+    }
+
+    pub struct RdpInputMouse {
+        conn: Arc<SyncConnection>,
+        session: Path<'static>,
+        stream: PwStreamInfo,
+    }
+
+    impl RdpInputMouse {
+        pub fn new(
+            conn: Arc<SyncConnection>,
+            session: Path<'static>,
+            stream: PwStreamInfo,
+        ) -> ResultType<Self> {
+            Ok(Self {
+                conn,
+                session,
+                stream,
+            })
+        }
+    }
+
+    impl MouseControllable for RdpInputMouse {
+        fn as_any(&self) -> &dyn std::any::Any {
+            self
+        }
+
+        fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+            self
+        }
+
+        fn mouse_move_to(&mut self, x: i32, y: i32) {
+            let portal = get_portal(&self.conn);
+            let _ = remote_desktop_portal::notify_pointer_motion_absolute(
+                &portal,
+                &self.session,
+                HashMap::new(),
+                self.stream.path as u32,
+                x as f64,
+                y as f64,
+            );
+        }
+        fn mouse_move_relative(&mut self, x: i32, y: i32) {
+            let portal = get_portal(&self.conn);
+            let _ = remote_desktop_portal::notify_pointer_motion(
+                &portal,
+                &self.session,
+                HashMap::new(),
+                x as f64,
+                y as f64,
+            );
+        }
+        fn mouse_down(&mut self, button: MouseButton) -> enigo::ResultType {
+            handle_mouse(true, button, self.conn.clone(), &self.session);
+            Ok(())
+        }
+        fn mouse_up(&mut self, button: MouseButton) {
+            handle_mouse(false, button, self.conn.clone(), &self.session);
+        }
+        fn mouse_click(&mut self, button: MouseButton) {
+            handle_mouse(true, button, self.conn.clone(), &self.session);
+            handle_mouse(false, button, self.conn.clone(), &self.session);
+        }
+        fn mouse_scroll_x(&mut self, length: i32) {
+            let portal = get_portal(&self.conn);
+            let _ = remote_desktop_portal::notify_pointer_axis(
+                &portal,
+                &self.session,
+                HashMap::new(),
+                length as f64,
+                0 as f64,
+            );
+        }
+        fn mouse_scroll_y(&mut self, length: i32) {
+            let portal = get_portal(&self.conn);
+            let _ = remote_desktop_portal::notify_pointer_axis(
+                &portal,
+                &self.session,
+                HashMap::new(),
+                0 as f64,
+                length as f64,
+            );
+        }
+    }
+
+    fn get_raw_evdev_keycode(key: u16) -> i32 {
+        // 8 is the offset between xkb and evdev
+        let mut key = key as i32 - 8;
+        // fix for right_meta key
+        if key == 126 {
+            key = 125;
+        }
+        key
+    }
+
+    fn handle_key(
+        down: bool,
+        key: Key,
+        conn: Arc<SyncConnection>,
+        session: &Path<'static>,
+    ) -> ResultType<()> {
+        let state: u32 = if down {
+            PRESSED_DOWN_STATE
+        } else {
+            PRESSED_UP_STATE
+        };
+        let portal = get_portal(&conn);
+        match key {
+            Key::Raw(key) => {
+                let key = get_raw_evdev_keycode(key);
+                remote_desktop_portal::notify_keyboard_keycode(
+                    &portal,
+                    &session,
+                    HashMap::new(),
+                    key,
+                    state,
+                )?;
+            }
+            _ => {
+                if let Ok((key, is_shift)) = map_key(&key) {
+                    if is_shift {
+                        remote_desktop_portal::notify_keyboard_keycode(
+                            &portal,
+                            &session,
+                            HashMap::new(),
+                            evdev::Key::KEY_LEFTSHIFT.code() as i32,
+                            state,
+                        )?;
+                    }
+                    remote_desktop_portal::notify_keyboard_keycode(
+                        &portal,
+                        &session,
+                        HashMap::new(),
+                        key.code() as i32,
+                        state,
+                    )?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn handle_mouse(
+        down: bool,
+        button: MouseButton,
+        conn: Arc<SyncConnection>,
+        session: &Path<'static>,
+    ) {
+        let portal = get_portal(&conn);
+        let but_key = match button {
+            MouseButton::Left => EVDEV_MOUSE_LEFT,
+            MouseButton::Right => EVDEV_MOUSE_RIGHT,
+            MouseButton::Middle => EVDEV_MOUSE_MIDDLE,
+            _ => {
+                return;
+            }
+        };
+        let state: u32 = if down {
+            PRESSED_DOWN_STATE
+        } else {
+            PRESSED_UP_STATE
+        };
+        let _ = remote_desktop_portal::notify_pointer_button(
+            &portal,
+            &session,
+            HashMap::new(),
+            but_key,
+            state,
+        );
+    }
+}

src/server/uinput.rs

@@ -382,7 +382,7 @@ pub mod service {
         Ok(keyboard)
     }
 
-    fn map_key(key: &enigo::Key) -> ResultType<(evdev::Key, bool)> {
+    pub fn map_key(key: &enigo::Key) -> ResultType<(evdev::Key, bool)> {
         if let Some(k) = KEY_MAP.get(&key) {
             log::trace!("mapkey {:?}, get {:?}", &key, &k);
             return Ok((k.clone(), false));