diff --git a/flutter/lib/common/widgets/remote_input.dart b/flutter/lib/common/widgets/remote_input.dart index 8c14bf26c..49c8879f3 100644 --- a/flutter/lib/common/widgets/remote_input.dart +++ b/flutter/lib/common/widgets/remote_input.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter/gestures.dart'; @@ -230,10 +232,18 @@ class _RawTouchGestureDetectorRegionState return; } if (isDesktop) { - // to-do + final scale = ((d.scale - _scale) * 1000).toInt(); + _scale = d.scale; + + if (scale != 0) { + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': scale} + })); + } } else { // mobile - // to-do: Is this correct? ffi.canvasModel.updateScale(d.scale / _scale); _scale = d.scale; ffi.canvasModel.panX(d.focalPointDelta.dx); @@ -246,14 +256,17 @@ class _RawTouchGestureDetectorRegionState return; } if (isDesktop) { - // to-do + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': 0} + })); } else { // mobile - // to-do: Is this correct? _scale = 1; bind.sessionSetViewStyle(sessionId: sessionId, value: ""); } - inputModel.sendMouse('up', MouseButtons.left); + inputModel.sendMouse('up', MouseButtons.left); } get onHoldDragCancel => null; diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 1032bbe9d..6b50aa37f 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -55,6 +55,8 @@ class InputModel { final _trackpadSpeed = 0.06; var _trackpadScrollUnsent = Offset.zero; + var _lastScale = 1.0; + // mouse final isPhysicalMouse = false.obs; int _lastButtons = 0; @@ -272,7 +274,7 @@ class InputModel { sendMouse('down', button); } - void tapUp(MouseButtons button) { + void tapUp(MouseButtons button) { sendMouse('up', button); } @@ -337,12 +339,24 @@ class InputModel { } void onPointerPanZoomStart(PointerPanZoomStartEvent e) { + _lastScale = 1.0; _stopFling = true; } // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures - // TODO(support zoom in/out) void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) { + final scale = ((e.scale - _lastScale) * 1000).toInt(); + _lastScale = e.scale; + + if (scale != 0) { + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': scale} + })); + return; + } + final delta = e.panDelta; _trackpadLastDelta = delta; @@ -422,6 +436,12 @@ class InputModel { } void onPointerPanZoomEnd(PointerPanZoomEndEvent e) { + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': 0} + })); + waitLastFlingDone(); _stopFling = false; diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index e67a5d3b1..2f732539c 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -111,6 +111,26 @@ message LoginResponse { } } +message TouchScaleUpdate { + // The delta scale factor relative to the previous scale. + // delta * 1000 + // 0 means scale end + int32 scale = 1; +} + +message TouchEvent { + oneof union { + TouchScaleUpdate scale_update = 1; + } +} + +message PointerDeviceEvent { + oneof union { + TouchEvent touch_event = 1; + } + repeated ControlKey modifiers = 2; +} + message MouseEvent { int32 mask = 1; sint32 x = 2; @@ -682,5 +702,6 @@ message Message { VoiceCallRequest voice_call_request = 23; VoiceCallResponse voice_call_response = 24; PeerInfo peer_info = 25; + PointerDeviceEvent pointer_device_event = 26; } } diff --git a/src/client.rs b/src/client.rs index 85047ee29..ccdae0512 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1109,7 +1109,8 @@ impl LoginConfigHandler { self.remember = !config.password.is_empty(); self.config = config; let mut sid = rand::random(); - if sid == 0 { // you won the lottery + if sid == 0 { + // you won the lottery sid = 1; } self.session_id = sid; @@ -1965,6 +1966,39 @@ pub fn send_mouse( interface.send(Data::Message(msg_out)); } +#[inline] +pub fn send_pointer_device_event( + mut evt: PointerDeviceEvent, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, + interface: &impl Interface, +) { + let mut msg_out = Message::new(); + if alt { + evt.modifiers.push(ControlKey::Alt.into()); + } + if shift { + evt.modifiers.push(ControlKey::Shift.into()); + } + if ctrl { + evt.modifiers.push(ControlKey::Control.into()); + } + if command { + evt.modifiers.push(ControlKey::Meta.into()); + } + #[cfg(all(target_os = "macos", not(feature = "flutter")))] + if check_scroll_on_mac(mask, x, y) { + let factor = 3; + mouse_event.mask = crate::input::MOUSE_TYPE_TRACKPAD; + mouse_event.x *= factor; + mouse_event.y *= factor; + } + msg_out.set_pointer_device_event(evt); + interface.send(Data::Message(msg_out)); +} + /// Activate OS by sending mouse movement. /// /// # Arguments diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index c8de03862..011c1b098 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1069,6 +1069,24 @@ pub fn main_start_dbus_server() { } } +pub fn session_send_pointer(session_id: SessionID, msg: String) { + if let Ok(m) = serde_json::from_str::>(&msg) { + let alt = m.get("alt").is_some(); + let ctrl = m.get("ctrl").is_some(); + let shift = m.get("shift").is_some(); + let command = m.get("command").is_some(); + if let Some(touch_event) = m.get("touch") { + if let Some(scale) = touch_event.get("scale") { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + if let Some(scale) = scale.as_i64() { + session.send_touch_scale(scale as _, alt, ctrl, shift, command); + } + } + } + } + } +} + pub fn session_send_mouse(session_id: SessionID, msg: String) { if let Ok(m) = serde_json::from_str::>(&msg) { let alt = m.get("alt").is_some(); diff --git a/src/ipc.rs b/src/ipc.rs index 526761c8b..9eaade320 100644 --- a/src/ipc.rs +++ b/src/ipc.rs @@ -152,6 +152,7 @@ pub enum DataPortableService { Pong, ConnCount(Option), Mouse((Vec, i32)), + Pointer((Vec, i32)), Key(Vec), RequestStart, WillClose, diff --git a/src/server/connection.rs b/src/server/connection.rs index 4224d670a..c91b813ff 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -115,6 +115,8 @@ enum MessageInput { Mouse((MouseEvent, i32)), #[cfg(not(any(target_os = "android", target_os = "ios")))] Key((KeyEvent, bool)), + #[cfg(not(any(target_os = "android", target_os = "ios")))] + Pointer((PointerDeviceEvent, i32)), BlockOn, BlockOff, #[cfg(all(feature = "flutter", feature = "plugin_framework"))] @@ -668,6 +670,9 @@ impl Connection { handle_key(&msg); } } + MessageInput::Pointer((msg, id)) => { + handle_pointer(&msg, id); + } MessageInput::BlockOn => { if crate::platform::block_input(true) { block_input_mode = true; @@ -1179,6 +1184,12 @@ impl Connection { self.tx_input.send(MessageInput::Mouse((msg, conn_id))).ok(); } + #[inline] + #[cfg(not(any(target_os = "android", target_os = "ios")))] + fn input_pointer(&self, msg: PointerDeviceEvent, conn_id: i32) { + self.tx_input.send(MessageInput::Pointer((msg, conn_id))).ok(); + } + #[inline] #[cfg(not(any(target_os = "android", target_os = "ios")))] fn input_key(&self, msg: KeyEvent, press: bool) { @@ -1577,6 +1588,13 @@ impl Connection { self.input_mouse(me, self.inner.id()); } } + Some(message::Union::PointerDeviceEvent(pde)) => { + #[cfg(not(any(target_os = "android", target_os = "ios")))] + if self.peer_keyboard_enabled() { + MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst); + self.input_pointer(pde, self.inner.id()); + } + } #[cfg(any(target_os = "android", target_os = "ios"))] Some(message::Union::KeyEvent(..)) => {} #[cfg(not(any(target_os = "android", target_os = "ios")))] diff --git a/src/server/input_service.rs b/src/server/input_service.rs index e71d166f5..8562ca3eb 100644 --- a/src/server/input_service.rs +++ b/src/server/input_service.rs @@ -1,13 +1,17 @@ use super::*; -use crate::input::*; #[cfg(target_os = "macos")] use crate::common::is_server; #[cfg(target_os = "linux")] use crate::common::IS_X11; +use crate::input::*; #[cfg(target_os = "macos")] use dispatch::Queue; use enigo::{Enigo, Key, KeyboardControllable, MouseButton, MouseControllable}; -use hbb_common::{get_time, protobuf::EnumOrUnknown}; +use hbb_common::{ + get_time, + message_proto::{pointer_device_event::Union::TouchEvent, touch_event::Union::ScaleUpdate}, + protobuf::EnumOrUnknown, +}; use rdev::{self, EventType, Key as RdevKey, KeyCode, RawKey}; #[cfg(target_os = "macos")] use rdev::{CGEventSourceStateID, CGEventTapLocation, VirtualInput}; @@ -523,6 +527,21 @@ pub fn handle_mouse(evt: &MouseEvent, conn: i32) { handle_mouse_(evt, conn); } +// to-do: merge handle_mouse and handle_pointer +pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) { + #[cfg(target_os = "macos")] + if !is_server() { + // having GUI, run main GUI thread, otherwise crash + let evt = evt.clone(); + QUEUE.exec_async(move || handle_pointer_(&evt, conn)); + return; + } + #[cfg(windows)] + crate::portable_service::client::handle_pointer(evt, conn); + #[cfg(not(windows))] + handle_pointer_(evt, conn); +} + pub fn fix_key_down_timeout_loop() { std::thread::spawn(move || loop { std::thread::sleep(std::time::Duration::from_millis(10_000)); @@ -743,6 +762,27 @@ fn active_mouse_(conn: i32) -> bool { } } +pub fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) { + if !active_mouse_(conn) { + return; + } + + if EXITING.load(Ordering::SeqCst) { + return; + } + + match &evt.union { + Some(TouchEvent(evt)) => match &evt.union { + Some(ScaleUpdate(_scale_evt)) => { + #[cfg(target_os = "windows")] + handle_scale(_scale_evt.scale); + } + _ => {} + }, + _ => {} + } +} + pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { if !active_mouse_(conn) { return; @@ -759,7 +799,7 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { let mut en = ENIGO.lock().unwrap(); #[cfg(not(target_os = "macos"))] let mut to_release = Vec::new(); - if evt_type == 1 { + if evt_type == MOUSE_TYPE_DOWN { fix_modifiers(&evt.modifiers[..], &mut en, 0); #[cfg(target_os = "macos")] en.reset_flag(); @@ -885,6 +925,18 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { } } +#[cfg(target_os = "windows")] +fn handle_scale(scale: i32) { + let mut en = ENIGO.lock().unwrap(); + if scale == 0 { + en.key_up(Key::Control); + } else { + if en.key_down(Key::Control).is_ok() { + en.mouse_scroll_y(scale); + } + } +} + pub fn is_enter(evt: &KeyEvent) -> bool { if let Some(key_event::Union::ControlKey(ck)) = evt.union { if ck.value() == ControlKey::Return.value() || ck.value() == ControlKey::NumpadEnter.value() diff --git a/src/server/portable_service.rs b/src/server/portable_service.rs index b794e9823..c8185b709 100644 --- a/src/server/portable_service.rs +++ b/src/server/portable_service.rs @@ -222,6 +222,8 @@ mod utils { // functions called in separate SYSTEM user process. pub mod server { + use hbb_common::message_proto::PointerDeviceEvent; + use super::*; lazy_static::lazy_static! { @@ -466,6 +468,11 @@ pub mod server { crate::input_service::handle_mouse_(&evt, conn); } } + Pointer((v, conn)) => { + if let Ok(evt) = PointerDeviceEvent::parse_from_bytes(&v) { + crate::input_service::handle_pointer_(&evt, conn); + } + } Key(v) => { if let Ok(evt) = KeyEvent::parse_from_bytes(&v) { crate::input_service::handle_key_(&evt); @@ -499,7 +506,7 @@ pub mod server { // functions called in main process. pub mod client { - use hbb_common::anyhow::Context; + use hbb_common::{anyhow::Context, message_proto::PointerDeviceEvent}; use super::*; @@ -864,6 +871,14 @@ pub mod client { )))) } + fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) -> ResultType<()> { + let mut v = vec![]; + evt.write_to_vec(&mut v)?; + ipc_send(Data::DataPortableService(DataPortableService::Pointer(( + v, conn, + )))) + } + fn handle_key_(evt: &KeyEvent) -> ResultType<()> { let mut v = vec![]; evt.write_to_vec(&mut v)?; @@ -910,6 +925,15 @@ pub mod client { } } + pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) { + if RUNNING.lock().unwrap().clone() { + crate::input_service::update_latest_input_cursor_time(conn); + handle_pointer_(evt, conn).ok(); + } else { + crate::input_service::handle_pointer_(evt, conn); + } + } + pub fn handle_key(evt: &KeyEvent) { if RUNNING.lock().unwrap().clone() { handle_key_(evt).ok(); diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index ed6052d39..a88a02edb 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -1,3 +1,4 @@ +use crate::input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP}; #[cfg(not(any(target_os = "android", target_os = "ios")))] use std::{collections::HashMap, sync::atomic::AtomicBool}; use std::{ @@ -34,8 +35,8 @@ use hbb_common::{ use crate::client::io_loop::Remote; use crate::client::{ check_if_retry, handle_hash, handle_login_error, handle_login_from_ui, handle_test_delay, - input_os_password, load_config, send_mouse, start_video_audio_threads, FileManager, Key, - LoginConfigHandler, QualityStatus, KEY_MAP, + input_os_password, load_config, send_mouse, send_pointer_device_event, + start_video_audio_threads, FileManager, Key, LoginConfigHandler, QualityStatus, KEY_MAP, }; #[cfg(not(any(target_os = "android", target_os = "ios")))] use crate::common::GrabState; @@ -50,7 +51,7 @@ const CHANGE_RESOLUTION_VALID_TIMEOUT_SECS: u64 = 15; #[derive(Clone, Default)] pub struct Session { pub session_id: SessionID, // different from the one in LoginConfigHandler, used for flutter UI message pass - pub id: String, // peer id + pub id: String, // peer id pub password: String, pub args: Vec, pub lc: Arc>, @@ -306,8 +307,7 @@ impl Session { } pub fn get_audit_server(&self, typ: String) -> String { - if LocalConfig::get_option("access_token").is_empty() - { + if LocalConfig::get_option("access_token").is_empty() { return "".to_owned(); } crate::get_audit_server( @@ -690,6 +690,18 @@ impl Session { self.send_key_event(&key_event); } + pub fn send_touch_scale(&self, scale: i32, alt: bool, ctrl: bool, shift: bool, command: bool) { + let scale_evt = TouchScaleUpdate { + scale, + ..Default::default() + }; + let mut touch_evt = TouchEvent::new(); + touch_evt.set_scale_update(scale_evt); + let mut evt = PointerDeviceEvent::new(); + evt.set_touch_event(touch_evt); + send_pointer_device_event(evt, alt, ctrl, shift, command, self); + } + pub fn send_mouse( &self, mask: i32, @@ -720,8 +732,20 @@ impl Session { if cfg!(target_os = "macos") { let buttons = mask >> 3; let evt_type = mask & 0x7; - if buttons == 1 && evt_type == 1 && ctrl && self.peer_platform() != "Mac OS" { - self.send_mouse((1 << 3 | 2) as _, x, y, alt, ctrl, shift, command); + if buttons == MOUSE_BUTTON_LEFT + && evt_type == MOUSE_TYPE_DOWN + && ctrl + && self.peer_platform() != "Mac OS" + { + self.send_mouse( + (MOUSE_BUTTON_LEFT << 3 | MOUSE_TYPE_UP) as _, + x, + y, + alt, + ctrl, + shift, + command, + ); } } }