Merge remote-tracking branch 'rustdesk/master' into flutter_desktop

# Conflicts:
#	.github/workflows/ci.yml
#	Cargo.lock
#	Cargo.toml
#	flutter/lib/common.dart
#	flutter/lib/mobile/pages/remote_page.dart
#	flutter/lib/mobile/pages/server_page.dart
#	flutter/lib/mobile/pages/settings_page.dart
#	flutter/lib/mobile/widgets/dialog.dart
#	flutter/lib/models/model.dart
#	flutter/lib/models/server_model.dart
#	src/client.rs
#	src/common.rs
#	src/ipc.rs
#	src/mobile_ffi.rs
#	src/rendezvous_mediator.rs
#	src/ui.rs

src/server/audio_service.rs

@@ -347,7 +347,7 @@ fn send_f32(data: &[f32], encoder: &mut Encoder, sp: &GenericService) {
                     Ok(data) => {
                         let mut msg_out = Message::new();
                         msg_out.set_audio_frame(AudioFrame {
-                            data,
+                            data: data.into(),
                             timestamp: crate::common::get_time(),
                             ..Default::default()
                         });
@@ -367,7 +367,7 @@ fn send_f32(data: &[f32], encoder: &mut Encoder, sp: &GenericService) {
         Ok(data) => {
             let mut msg_out = Message::new();
             msg_out.set_audio_frame(AudioFrame {
-                data,
+                data: data.into(),
                 timestamp: crate::common::get_time(),
                 ..Default::default()
             });

src/server/connection.rs

@@ -3,17 +3,18 @@ use super::{input_service::*, *};
 use crate::clipboard_file::*;
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 use crate::common::update_clipboard;
+use crate::video_service;
 #[cfg(any(target_os = "android", target_os = "ios"))]
 use crate::{common::MOBILE_INFO2, flutter::connection_manager::start_channel};
 use crate::{ipc, VERSION};
-use hbb_common::fs::can_enable_overwrite_detection;
 use hbb_common::{
     config::Config,
     fs,
+    fs::can_enable_overwrite_detection,
     futures::{SinkExt, StreamExt},
     get_version_number,
     message_proto::{option_message::BoolOption, permission_info::Permission},
-    sleep, timeout,
+    password_security as password, sleep, timeout,
     tokio::{
         net::TcpStream,
         sync::mpsc,
@@ -29,11 +30,14 @@ use std::sync::{
     atomic::{AtomicI64, Ordering},
     mpsc as std_mpsc,
 };
+#[cfg(not(any(target_os = "android", target_os = "ios")))]
+use system_shutdown;
 
 pub type Sender = mpsc::UnboundedSender<(Instant, Arc<Message>)>;
 
 lazy_static::lazy_static! {
     static ref LOGIN_FAILURES: Arc::<Mutex<HashMap<String, (i32, i32, i32)>>> = Default::default();
+    static ref SESSIONS: Arc::<Mutex<HashMap<String, Session>>> = Default::default();
 }
 pub static CLICK_TIME: AtomicI64 = AtomicI64::new(0);
 pub static MOUSE_MOVE_TIME: AtomicI64 = AtomicI64::new(0);
@@ -52,6 +56,14 @@ enum MessageInput {
     BlockOff,
 }
 
+#[derive(Clone, Debug)]
+struct Session {
+    name: String,
+    session_id: u64,
+    last_recv_time: Arc<Mutex<Instant>>,
+    random_password: String,
+}
+
 pub struct Connection {
     inner: ConnInner,
     stream: super::Stream,
@@ -68,8 +80,8 @@ pub struct Connection {
     clipboard: bool,
     audio: bool,
     file: bool,
+    restart: bool,
     last_test_delay: i64,
-    image_quality: i32,
     lock_after_session_end: bool,
     show_remote_cursor: bool, // by peer
     ip: String,
@@ -80,6 +92,8 @@ pub struct Connection {
     video_ack_required: bool,
     peer_info: (String, String),
     api_server: String,
+    lr: LoginRequest,
+    last_recv_time: Arc<Mutex<Instant>>,
 }
 
 impl Subscriber for ConnInner {
@@ -91,7 +105,7 @@ impl Subscriber for ConnInner {
     #[inline]
     fn send(&mut self, msg: Arc<Message>) {
         match &msg.union {
-            Some(message::Union::video_frame(_)) => {
+            Some(message::Union::VideoFrame(_)) => {
                 self.tx_video.as_mut().map(|tx| {
                     allow_err!(tx.send((Instant::now(), msg)));
                 });
@@ -105,12 +119,13 @@ impl Subscriber for ConnInner {
     }
 }
 
-const TEST_DELAY_TIMEOUT: Duration = Duration::from_secs(3);
+const TEST_DELAY_TIMEOUT: Duration = Duration::from_secs(1);
 const SEC30: Duration = Duration::from_secs(30);
 const H1: Duration = Duration::from_secs(3600);
 const MILLI1: Duration = Duration::from_millis(1);
 const SEND_TIMEOUT_VIDEO: u64 = 12_000;
 const SEND_TIMEOUT_OTHER: u64 = SEND_TIMEOUT_VIDEO * 10;
+const SESSION_TIMEOUT: Duration = Duration::from_secs(30);
 
 impl Connection {
     pub async fn start(
@@ -121,7 +136,7 @@ impl Connection {
     ) {
         let hash = Hash {
             salt: Config::get_salt(),
-            challenge: Config::get_auto_password(),
+            challenge: Config::get_auto_password(6),
             ..Default::default()
         };
         let (tx_from_cm_holder, mut rx_from_cm) = mpsc::unbounded_channel::<ipc::Data>();
@@ -153,8 +168,8 @@ impl Connection {
             clipboard: Config::get_option("enable-clipboard").is_empty(),
             audio: Config::get_option("enable-audio").is_empty(),
             file: Config::get_option("enable-file-transfer").is_empty(),
+            restart: Config::get_option("enable-remote-restart").is_empty(),
             last_test_delay: 0,
-            image_quality: ImageQuality::Balanced.value(),
             lock_after_session_end: false,
             show_remote_cursor: false,
             ip: "".to_owned(),
@@ -165,6 +180,8 @@ impl Connection {
             video_ack_required: false,
             peer_info: Default::default(),
             api_server: "".to_owned(),
+            lr: Default::default(),
+            last_recv_time: Arc::new(Mutex::new(Instant::now())),
         };
         #[cfg(not(any(target_os = "android", target_os = "ios")))]
         tokio::spawn(async move {
@@ -190,6 +207,9 @@ impl Connection {
         if !conn.file {
             conn.send_permission(Permission::File, false).await;
         }
+        if !conn.restart {
+            conn.send_permission(Permission::Restart, false).await;
+        }
         let mut test_delay_timer =
             time::interval_at(Instant::now() + TEST_DELAY_TIMEOUT, TEST_DELAY_TIMEOUT);
         let mut last_recv_time = Instant::now();
@@ -223,7 +243,8 @@ impl Connection {
                             let mut msg_out = Message::new();
                             msg_out.set_misc(misc);
                             conn.send(msg_out).await;
-                            conn.on_close("Close requested from connection manager", false);
+                            conn.on_close("Close requested from connection manager", false).await;
+                            SESSIONS.lock().unwrap().remove(&conn.lr.my_id);
                             break;
                         }
                         ipc::Data::ChatMessage{text} => {
@@ -266,6 +287,9 @@ impl Connection {
                                 conn.file = enabled;
                                 conn.send_permission(Permission::File, enabled).await;
                                 conn.send_to_cm(ipc::Data::ClipboardFileEnabled(conn.file_transfer_enabled()));
+                            } else if &name == "restart" {
+                                conn.restart = enabled;
+                                conn.send_permission(Permission::Restart, enabled).await;
                             }
                         }
                         ipc::Data::RawMessage(bytes) => {
@@ -282,24 +306,24 @@ impl Connection {
                                 ipc::PrivacyModeState::OffSucceeded => {
                                     video_service::set_privacy_mode_conn_id(0);
                                     crate::common::make_privacy_mode_msg(
-                                        back_notification::PrivacyModeState::OffSucceeded,
+                                        back_notification::PrivacyModeState::PrvOffSucceeded,
                                     )
                                 }
                                 ipc::PrivacyModeState::OffFailed => {
                                     crate::common::make_privacy_mode_msg(
-                                        back_notification::PrivacyModeState::OffFailed,
+                                        back_notification::PrivacyModeState::PrvOffFailed,
                                     )
                                 }
                                 ipc::PrivacyModeState::OffByPeer => {
                                     video_service::set_privacy_mode_conn_id(0);
                                     crate::common::make_privacy_mode_msg(
-                                        back_notification::PrivacyModeState::OffByPeer,
+                                        back_notification::PrivacyModeState::PrvOffByPeer,
                                     )
                                 }
                                 ipc::PrivacyModeState::OffUnknown => {
                                     video_service::set_privacy_mode_conn_id(0);
                                      crate::common::make_privacy_mode_msg(
-                                        back_notification::PrivacyModeState::OffUnknown,
+                                        back_notification::PrivacyModeState::PrvOffUnknown,
                                     )
                                 }
                             };
@@ -312,11 +336,12 @@ impl Connection {
                     if let Some(res) = res {
                         match res {
                             Err(err) => {
-                                conn.on_close(&err.to_string(), true);
+                                conn.on_close(&err.to_string(), true).await;
                                 break;
                             },
                             Ok(bytes) => {
                                 last_recv_time = Instant::now();
+                                *conn.last_recv_time.lock().unwrap() = Instant::now();
                                 if let Ok(msg_in) = Message::parse_from_bytes(&bytes) {
                                     if !conn.on_message(msg_in).await {
                                         break;
@@ -325,14 +350,14 @@ impl Connection {
                             }
                         }
                     } else {
-                        conn.on_close("Reset by the peer", true);
+                        conn.on_close("Reset by the peer", true).await;
                         break;
                     }
                 },
                 _ = conn.timer.tick() => {
                     if !conn.read_jobs.is_empty() {
                         if let Err(err) = fs::handle_read_jobs(&mut conn.read_jobs, &mut conn.stream).await {
-                            conn.on_close(&err.to_string(), false);
+                            conn.on_close(&err.to_string(), false).await;
                             break;
                         }
                     } else {
@@ -345,7 +370,7 @@ impl Connection {
                         video_service::notify_video_frame_feched(id, Some(instant.into()));
                     }
                     if let Err(err) = conn.stream.send(&value as &Message).await {
-                        conn.on_close(&err.to_string(), false);
+                        conn.on_close(&err.to_string(), false).await;
                         break;
                     }
                 },
@@ -355,7 +380,7 @@ impl Connection {
 
                     if latency > 1000 {
                         match &msg.union {
-                            Some(message::Union::audio_frame(_)) => {
+                            Some(message::Union::AudioFrame(_)) => {
                                 // log::info!("audio frame latency {}", instant.elapsed().as_secs_f32());
                                 continue;
                             }
@@ -363,21 +388,24 @@ impl Connection {
                         }
                     }
                     if let Err(err) = conn.stream.send(msg).await {
-                        conn.on_close(&err.to_string(), false);
+                        conn.on_close(&err.to_string(), false).await;
                         break;
                     }
                 },
                 _ = test_delay_timer.tick() => {
                     if last_recv_time.elapsed() >= SEC30 {
-                        conn.on_close("Timeout", true);
+                        conn.on_close("Timeout", true).await;
                         break;
                     }
                     let time = crate::get_time();
                     if time > 0 && conn.last_test_delay == 0 {
                         conn.last_test_delay = time;
                         let mut msg_out = Message::new();
+                        let qos = video_service::VIDEO_QOS.lock().unwrap();
                         msg_out.set_test_delay(TestDelay{
                             time,
+                            last_delay:qos.current_delay,
+                            target_bitrate:qos.target_bitrate,
                             ..Default::default()
                         });
                         conn.inner.send(msg_out.into());
@@ -394,10 +422,13 @@ impl Connection {
             let _ = privacy_mode::turn_off_privacy(0);
         }
         video_service::notify_video_frame_feched(id, None);
-        video_service::update_test_latency(id, 0);
-        video_service::update_image_quality(id, None);
+        scrap::codec::Encoder::update_video_encoder(id, scrap::codec::EncoderUpdate::Remove);
+        video_service::VIDEO_QOS.lock().unwrap().reset();
+        if conn.authorized {
+            password::update_temporary_password();
+        }
         if let Err(err) = conn.try_port_forward_loop(&mut rx_from_cm).await {
-            conn.on_close(&err.to_string(), false);
+            conn.on_close(&err.to_string(), false).await;
         }
 
         conn.post_audit(json!({
@@ -432,7 +463,7 @@ impl Connection {
                         } else {
                             Self::send_block_input_error(
                                 &tx,
-                                back_notification::BlockInputState::OnFailed,
+                                back_notification::BlockInputState::BlkOnFailed,
                             );
                         }
                     }
@@ -442,7 +473,7 @@ impl Connection {
                         } else {
                             Self::send_block_input_error(
                                 &tx,
-                                back_notification::BlockInputState::OffFailed,
+                                back_notification::BlockInputState::BlkOffFailed,
                             );
                         }
                     }
@@ -490,7 +521,7 @@ impl Connection {
                     res = self.stream.next() => {
                         if let Some(res) = res {
                             last_recv_time = Instant::now();
-                            timeout(SEND_TIMEOUT_OTHER, forward.send(res?.into())).await??;
+                            timeout(SEND_TIMEOUT_OTHER, forward.send(res?)).await??;
                         } else {
                             bail!("Stream reset by the peer");
                         }
@@ -569,7 +600,7 @@ impl Connection {
         let url = self.api_server.clone();
         let mut v = v;
         v["id"] = json!(Config::get_id());
-        v["uuid"] = json!(base64::encode(crate::get_uuid()));
+        v["uuid"] = json!(base64::encode(hbb_common::get_uuid()));
         v["Id"] = json!(self.inner.id);
         tokio::spawn(async move {
             allow_err!(Self::post_audit_async(url, v).await);
@@ -615,6 +646,16 @@ impl Connection {
             pi.hostname = MOBILE_INFO2.lock().unwrap().clone();
             pi.platform = "Android".into();
         }
+        #[cfg(feature = "hwcodec")]
+        {
+            let (h264, h265) = scrap::codec::Encoder::supported_encoding();
+            pi.encoding = Some(SupportedEncoding {
+                h264,
+                h265,
+                ..Default::default()
+            })
+            .into();
+        }
 
         if self.port_forward_socket.is_some() {
             let mut msg_out = Message::new();
@@ -626,9 +667,9 @@ impl Connection {
         #[cfg(target_os = "linux")]
         if !self.file_transfer.is_some() && !self.port_forward_socket.is_some() {
             let dtype = crate::platform::linux::get_display_server();
-            if dtype != "x11" {
+            if dtype != "x11" && dtype != "wayland" {
                 res.set_error(format!(
-                    "Unsupported display server type {}, x11 expected",
+                    "Unsupported display server type {}, x11 or wayland expected",
                     dtype
                 ));
                 let mut msg_out = Message::new();
@@ -664,7 +705,7 @@ impl Connection {
             res.set_peer_info(pi);
         } else {
             try_activate_screen();
-            match super::video_service::get_displays() {
+            match super::video_service::get_displays().await {
                 Err(err) => {
                     res.set_error(format!("X11 error: {}", err));
                 }
@@ -734,6 +775,7 @@ impl Connection {
             audio: self.audio,
             file: self.file,
             file_transfer_enabled: self.file_transfer_enabled(),
+            restart: self.restart,
         });
     }
 
@@ -776,17 +818,99 @@ impl Connection {
         self.tx_input.send(MessageInput::Key((msg, press))).ok();
     }
 
+    fn validate_one_password(&self, password: String) -> bool {
+        if password.len() == 0 {
+            return false;
+        }
+        let mut hasher = Sha256::new();
+        hasher.update(password);
+        hasher.update(&self.hash.salt);
+        let mut hasher2 = Sha256::new();
+        hasher2.update(&hasher.finalize()[..]);
+        hasher2.update(&self.hash.challenge);
+        hasher2.finalize()[..] == self.lr.password[..]
+    }
+
+    fn validate_password(&mut self) -> bool {
+        if password::temporary_enabled() {
+            let password = password::temporary_password();
+            if self.validate_one_password(password.clone()) {
+                SESSIONS.lock().unwrap().insert(
+                    self.lr.my_id.clone(),
+                    Session {
+                        name: self.lr.my_name.clone(),
+                        session_id: self.lr.session_id,
+                        last_recv_time: self.last_recv_time.clone(),
+                        random_password: password,
+                    },
+                );
+                return true;
+            }
+        }
+        if password::permanent_enabled() {
+            if self.validate_one_password(Config::get_permanent_password()) {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn is_of_recent_session(&mut self) -> bool {
+        let session = SESSIONS
+            .lock()
+            .unwrap()
+            .get(&self.lr.my_id)
+            .map(|s| s.to_owned());
+        if let Some(session) = session {
+            if session.name == self.lr.my_name
+                && session.session_id == self.lr.session_id
+                && !self.lr.password.is_empty()
+                && self.validate_one_password(session.random_password.clone())
+                && session.last_recv_time.lock().unwrap().elapsed() < SESSION_TIMEOUT
+            {
+                SESSIONS.lock().unwrap().insert(
+                    self.lr.my_id.clone(),
+                    Session {
+                        name: self.lr.my_name.clone(),
+                        session_id: self.lr.session_id,
+                        last_recv_time: self.last_recv_time.clone(),
+                        random_password: session.random_password,
+                    },
+                );
+                return true;
+            }
+        }
+        false
+    }
+
     async fn on_message(&mut self, msg: Message) -> bool {
-        if let Some(message::Union::login_request(lr)) = msg.union {
+        if let Some(message::Union::LoginRequest(lr)) = msg.union {
+            self.lr = lr.clone();
             if let Some(o) = lr.option.as_ref() {
                 self.update_option(o).await;
+                if let Some(q) = o.video_codec_state.clone().take() {
+                    scrap::codec::Encoder::update_video_encoder(
+                        self.inner.id(),
+                        scrap::codec::EncoderUpdate::State(q),
+                    );
+                } else {
+                    scrap::codec::Encoder::update_video_encoder(
+                        self.inner.id(),
+                        scrap::codec::EncoderUpdate::DisableHwIfNotExist,
+                    );
+                }
+            } else {
+                scrap::codec::Encoder::update_video_encoder(
+                    self.inner.id(),
+                    scrap::codec::EncoderUpdate::DisableHwIfNotExist,
+                );
             }
             self.video_ack_required = lr.video_ack_required;
             if self.authorized {
                 return true;
             }
             match lr.union {
-                Some(login_request::Union::file_transfer(ft)) => {
+                Some(login_request::Union::FileTransfer(ft)) => {
                     if !Config::get_option("enable-file-transfer").is_empty() {
                         self.send_login_error("No permission of file transfer")
                             .await;
@@ -795,7 +919,7 @@ impl Connection {
                     }
                     self.file_transfer = Some((ft.dir, ft.show_hidden));
                 }
-                Some(login_request::Union::port_forward(mut pf)) => {
+                Some(login_request::Union::PortForward(mut pf)) => {
                     if !Config::get_option("enable-tunnel").is_empty() {
                         self.send_login_error("No permission of IP tunneling").await;
                         sleep(1.).await;
@@ -832,15 +956,19 @@ impl Connection {
             }
             if !crate::is_ip(&lr.username) && lr.username != Config::get_id() {
                 self.send_login_error("Offline").await;
+            } else if self.is_of_recent_session() {
+                self.try_start_cm(lr.my_id, lr.my_name, true);
+                self.send_logon_response().await;
+                if self.port_forward_socket.is_some() {
+                    return false;
+                }
             } else if lr.password.is_empty() {
                 self.try_start_cm(lr.my_id, lr.my_name, false);
             } else {
-                let mut hasher = Sha256::new();
-                hasher.update(&Config::get_password());
-                hasher.update(&self.hash.salt);
-                let mut hasher2 = Sha256::new();
-                hasher2.update(&hasher.finalize()[..]);
-                hasher2.update(&self.hash.challenge);
+                if !password::has_valid_password() {
+                    self.send_login_error("Connection not allowed").await;
+                    return false;
+                }
                 let mut failure = LOGIN_FAILURES
                     .lock()
                     .unwrap()
@@ -853,7 +981,7 @@ impl Connection {
                         .await;
                 } else if time == failure.0 && failure.1 > 6 {
                     self.send_login_error("Please try 1 minute later").await;
-                } else if hasher2.finalize()[..] != lr.password[..] {
+                } else if !self.validate_password() {
                     if failure.0 == time {
                         failure.1 += 1;
                         failure.2 += 1;
@@ -879,21 +1007,22 @@ impl Connection {
                     }
                 }
             }
-        } else if let Some(message::Union::test_delay(t)) = msg.union {
+        } else if let Some(message::Union::TestDelay(t)) = msg.union {
             if t.from_client {
                 let mut msg_out = Message::new();
                 msg_out.set_test_delay(t);
                 self.inner.send(msg_out.into());
             } else {
                 self.last_test_delay = 0;
-                let latency = crate::get_time() - t.time;
-                if latency > 0 {
-                    super::video_service::update_test_latency(self.inner.id(), latency);
-                }
+                let new_delay = (crate::get_time() - t.time) as u32;
+                video_service::VIDEO_QOS
+                    .lock()
+                    .unwrap()
+                    .update_network_delay(new_delay);
             }
         } else if self.authorized {
             match msg.union {
-                Some(message::Union::mouse_event(me)) => {
+                Some(message::Union::MouseEvent(me)) => {
                     #[cfg(any(target_os = "android", target_os = "ios"))]
                     if let Err(e) = call_main_service_mouse_input(me.mask, me.x, me.y) {
                         log::debug!("call_main_service_mouse_input fail:{}", e);
@@ -908,7 +1037,7 @@ impl Connection {
                         self.input_mouse(me, self.inner.id());
                     }
                 }
-                Some(message::Union::key_event(me)) => {
+                Some(message::Union::KeyEvent(me)) => {
                     #[cfg(not(any(target_os = "android", target_os = "ios")))]
                     if self.keyboard {
                         if is_enter(&me) {
@@ -924,8 +1053,8 @@ impl Connection {
                         };
                         if is_press {
                             match me.union {
-                                Some(key_event::Union::unicode(_))
-                                | Some(key_event::Union::seq(_)) => {
+                                Some(key_event::Union::Unicode(_))
+                                | Some(key_event::Union::Seq(_)) => {
                                     self.input_key(me, false);
                                 }
                                 _ => {
@@ -937,14 +1066,14 @@ impl Connection {
                         }
                     }
                 }
-                Some(message::Union::clipboard(cb)) =>
+                Some(message::Union::Clipboard(cb)) =>
                 {
                     #[cfg(not(any(target_os = "android", target_os = "ios")))]
                     if self.clipboard {
                         update_clipboard(cb, None);
                     }
                 }
-                Some(message::Union::cliprdr(_clip)) => {
+                Some(message::Union::Cliprdr(_clip)) => {
                     if self.file_transfer_enabled() {
                         #[cfg(windows)]
                         if let Some(clip) = msg_2_clip(_clip) {
@@ -952,13 +1081,13 @@ impl Connection {
                         }
                     }
                 }
-                Some(message::Union::file_action(fa)) => {
+                Some(message::Union::FileAction(fa)) => {
                     if self.file_transfer.is_some() {
                         match fa.union {
-                            Some(file_action::Union::read_dir(rd)) => {
+                            Some(file_action::Union::ReadDir(rd)) => {
                                 self.read_dir(&rd.path, rd.include_hidden);
                             }
-                            Some(file_action::Union::all_files(f)) => {
+                            Some(file_action::Union::AllFiles(f)) => {
                                 match fs::get_recursive_files(&f.path, f.include_hidden) {
                                     Err(err) => {
                                         self.send(fs::new_error(f.id, err, -1)).await;
@@ -968,7 +1097,7 @@ impl Connection {
                                     }
                                 }
                             }
-                            Some(file_action::Union::send(s)) => {
+                            Some(file_action::Union::Send(s)) => {
                                 let id = s.id;
                                 let od =
                                     can_enable_overwrite_detection(get_version_number(VERSION));
@@ -993,7 +1122,7 @@ impl Connection {
                                     }
                                 }
                             }
-                            Some(file_action::Union::receive(r)) => {
+                            Some(file_action::Union::Receive(r)) => {
                                 self.send_fs(ipc::FS::NewWrite {
                                     path: r.path,
                                     id: r.id,
@@ -1006,31 +1135,31 @@ impl Connection {
                                         .collect(),
                                 });
                             }
-                            Some(file_action::Union::remove_dir(d)) => {
+                            Some(file_action::Union::RemoveDir(d)) => {
                                 self.send_fs(ipc::FS::RemoveDir {
                                     path: d.path,
                                     id: d.id,
                                     recursive: d.recursive,
                                 });
                             }
-                            Some(file_action::Union::remove_file(f)) => {
+                            Some(file_action::Union::RemoveFile(f)) => {
                                 self.send_fs(ipc::FS::RemoveFile {
                                     path: f.path,
                                     id: f.id,
                                     file_num: f.file_num,
                                 });
                             }
-                            Some(file_action::Union::create(c)) => {
+                            Some(file_action::Union::Create(c)) => {
                                 self.send_fs(ipc::FS::CreateDir {
                                     path: c.path,
                                     id: c.id,
                                 });
                             }
-                            Some(file_action::Union::cancel(c)) => {
+                            Some(file_action::Union::Cancel(c)) => {
                                 self.send_fs(ipc::FS::CancelWrite { id: c.id });
                                 fs::remove_job(c.id, &mut self.read_jobs);
                             }
-                            Some(file_action::Union::send_confirm(r)) => {
+                            Some(file_action::Union::SendConfirm(r)) => {
                                 if let Some(job) = fs::get_job(r.id, &mut self.read_jobs) {
                                     job.confirm(&r);
                                 }
@@ -1039,8 +1168,8 @@ impl Connection {
                         }
                     }
                 }
-                Some(message::Union::file_response(fr)) => match fr.union {
-                    Some(file_response::Union::block(block)) => {
+                Some(message::Union::FileResponse(fr)) => match fr.union {
+                    Some(file_response::Union::Block(block)) => {
                         self.send_fs(ipc::FS::WriteBlock {
                             id: block.id,
                             file_num: block.file_num,
@@ -1048,13 +1177,13 @@ impl Connection {
                             compressed: block.compressed,
                         });
                     }
-                    Some(file_response::Union::done(d)) => {
+                    Some(file_response::Union::Done(d)) => {
                         self.send_fs(ipc::FS::WriteDone {
                             id: d.id,
                             file_num: d.file_num,
                         });
                     }
-                    Some(file_response::Union::digest(d)) => self.send_fs(ipc::FS::CheckDigest {
+                    Some(file_response::Union::Digest(d)) => self.send_fs(ipc::FS::CheckDigest {
                         id: d.id,
                         file_num: d.file_num,
                         file_size: d.file_size,
@@ -1063,27 +1192,43 @@ impl Connection {
                     }),
                     _ => {}
                 },
-                Some(message::Union::misc(misc)) => match misc.union {
-                    Some(misc::Union::switch_display(s)) => {
-                        super::video_service::switch_display(s.display);
+                Some(message::Union::Misc(misc)) => match misc.union {
+                    Some(misc::Union::SwitchDisplay(s)) => {
+                        video_service::switch_display(s.display).await;
                     }
-                    Some(misc::Union::chat_message(c)) => {
+                    Some(misc::Union::ChatMessage(c)) => {
                         self.send_to_cm(ipc::Data::ChatMessage { text: c.text });
                     }
-                    Some(misc::Union::option(o)) => {
+                    Some(misc::Union::Option(o)) => {
                         self.update_option(&o).await;
                     }
-                    Some(misc::Union::refresh_video(r)) => {
+                    Some(misc::Union::RefreshVideo(r)) => {
                         if r {
                             super::video_service::refresh();
                         }
                     }
-                    Some(misc::Union::video_received(_)) => {
+                    Some(misc::Union::VideoReceived(_)) => {
                         video_service::notify_video_frame_feched(
                             self.inner.id,
                             Some(Instant::now().into()),
                         );
                     }
+                    Some(misc::Union::CloseReason(_)) => {
+                        self.on_close("Peer close", true).await;
+                        SESSIONS.lock().unwrap().remove(&self.lr.my_id);
+                        return false;
+                    }
+
+                    Some(misc::Union::RestartRemoteDevice(_)) =>
+                    {
+                        #[cfg(not(any(target_os = "android", target_os = "ios")))]
+                        if self.restart {
+                            match system_shutdown::reboot() {
+                                Ok(_) => log::info!("Restart by the peer"),
+                                Err(e) => log::error!("Failed to restart:{}", e),
+                            }
+                        }
+                    }
                     _ => {}
                 },
                 _ => {}
@@ -1095,13 +1240,20 @@ impl Connection {
     async fn update_option(&mut self, o: &OptionMessage) {
         log::info!("Option update: {:?}", o);
         if let Ok(q) = o.image_quality.enum_value() {
-            self.image_quality = q.value();
-            super::video_service::update_image_quality(self.inner.id(), Some(q.value()));
-        }
-        let q = o.custom_image_quality;
-        if q > 0 {
-            self.image_quality = q;
-            super::video_service::update_image_quality(self.inner.id(), Some(q));
+            let image_quality;
+            if let ImageQuality::NotSet = q {
+                if o.custom_image_quality > 0 {
+                    image_quality = o.custom_image_quality;
+                } else {
+                    image_quality = ImageQuality::Balanced.value();
+                }
+            } else {
+                image_quality = q.value();
+            }
+            video_service::VIDEO_QOS
+                .lock()
+                .unwrap()
+                .update_image_quality(image_quality);
         }
         if let Ok(q) = o.lock_after_session_end.enum_value() {
             if q != BoolOption::NotSet {
@@ -1164,7 +1316,7 @@ impl Connection {
                     BoolOption::Yes => {
                         let msg_out = if !video_service::is_privacy_mode_supported() {
                             crate::common::make_privacy_mode_msg(
-                                back_notification::PrivacyModeState::NotSupported,
+                                back_notification::PrivacyModeState::PrvNotSupported,
                             )
                         } else {
                             match privacy_mode::turn_on_privacy(self.inner.id) {
@@ -1172,7 +1324,7 @@ impl Connection {
                                     if video_service::test_create_capturer(self.inner.id, 5_000) {
                                         video_service::set_privacy_mode_conn_id(self.inner.id);
                                         crate::common::make_privacy_mode_msg(
-                                            back_notification::PrivacyModeState::OnSucceeded,
+                                            back_notification::PrivacyModeState::PrvOnSucceeded,
                                         )
                                     } else {
                                         log::error!(
@@ -1181,12 +1333,12 @@ impl Connection {
                                         video_service::set_privacy_mode_conn_id(0);
                                         let _ = privacy_mode::turn_off_privacy(self.inner.id);
                                         crate::common::make_privacy_mode_msg(
-                                            back_notification::PrivacyModeState::OnFailed,
+                                            back_notification::PrivacyModeState::PrvOnFailed,
                                         )
                                     }
                                 }
                                 Ok(false) => crate::common::make_privacy_mode_msg(
-                                    back_notification::PrivacyModeState::OnFailedPlugin,
+                                    back_notification::PrivacyModeState::PrvOnFailedPlugin,
                                 ),
                                 Err(e) => {
                                     log::error!("Failed to turn on privacy mode. {}", e);
@@ -1194,7 +1346,7 @@ impl Connection {
                                         let _ = privacy_mode::turn_off_privacy(0);
                                     }
                                     crate::common::make_privacy_mode_msg(
-                                        back_notification::PrivacyModeState::OnFailed,
+                                        back_notification::PrivacyModeState::PrvOnFailed,
                                     )
                                 }
                             }
@@ -1204,7 +1356,7 @@ impl Connection {
                     BoolOption::No => {
                         let msg_out = if !video_service::is_privacy_mode_supported() {
                             crate::common::make_privacy_mode_msg(
-                                back_notification::PrivacyModeState::NotSupported,
+                                back_notification::PrivacyModeState::PrvNotSupported,
                             )
                         } else {
                             video_service::set_privacy_mode_conn_id(0);
@@ -1229,16 +1381,22 @@ impl Connection {
                 }
             }
         }
+        if let Some(q) = o.video_codec_state.clone().take() {
+            scrap::codec::Encoder::update_video_encoder(
+                self.inner.id(),
+                scrap::codec::EncoderUpdate::State(q),
+            );
+        }
     }
 
-    fn on_close(&mut self, reason: &str, lock: bool) {
+    async fn on_close(&mut self, reason: &str, lock: bool) {
         if let Some(s) = self.server.upgrade() {
             s.write().unwrap().remove_connection(&self.inner);
         }
         log::info!("#{} Connection closed: {}", self.inner.id(), reason);
         if lock && self.lock_after_session_end && self.keyboard {
             #[cfg(not(any(target_os = "android", target_os = "ios")))]
-            lock_screen();
+            lock_screen().await;
         }
         self.tx_to_cm.send(ipc::Data::Close).ok();
         self.port_forward_socket.take();
@@ -1337,7 +1495,7 @@ async fn start_ipc(
                             file_num,
                             data,
                             compressed}) = data {
-                                stream.send(&Data::FS(ipc::FS::WriteBlock{id, file_num, data: Vec::new(), compressed})).await?;
+                                stream.send(&Data::FS(ipc::FS::WriteBlock{id, file_num, data: Bytes::new(), compressed})).await?;
                                 stream.send_raw(data).await?;
                         } else {
                             stream.send(&data).await?;
@@ -1373,19 +1531,19 @@ mod privacy_mode {
             let res = turn_off_privacy(_conn_id, None);
             match res {
                 Ok(_) => crate::common::make_privacy_mode_msg(
-                    back_notification::PrivacyModeState::OffSucceeded,
+                    back_notification::PrivacyModeState::PrvOffSucceeded,
                 ),
                 Err(e) => {
                     log::error!("Failed to turn off privacy mode {}", e);
                     crate::common::make_privacy_mode_msg(
-                        back_notification::PrivacyModeState::OffFailed,
+                        back_notification::PrivacyModeState::PrvOffFailed,
                     )
                 }
             }
         }
         #[cfg(not(windows))]
         {
-            crate::common::make_privacy_mode_msg(back_notification::PrivacyModeState::OffFailed)
+            crate::common::make_privacy_mode_msg(back_notification::PrivacyModeState::PrvOffFailed)
         }
     }
 

src/server/input_service.rs

@@ -2,7 +2,7 @@ use super::*;
 #[cfg(target_os = "macos")]
 use dispatch::Queue;
 use enigo::{Enigo, Key, KeyboardControllable, MouseButton, MouseControllable};
-use hbb_common::{config::COMPRESS_LEVEL, protobuf::ProtobufEnumOrUnknown};
+use hbb_common::{config::COMPRESS_LEVEL, protobuf::EnumOrUnknown};
 use std::{
     convert::TryFrom,
     sync::atomic::{AtomicBool, Ordering},
@@ -68,7 +68,7 @@ impl Subscriber for MouseCursorSub {
 
     #[inline]
     fn send(&mut self, msg: Arc<Message>) {
-        if let Some(message::Union::cursor_data(cd)) = &msg.union {
+        if let Some(message::Union::CursorData(cd)) = &msg.union {
             if let Some(msg) = self.cached.get(&cd.id) {
                 self.inner.send(msg.clone());
             } else {
@@ -145,7 +145,7 @@ fn run_cursor(sp: MouseCursorService, state: &mut StateCursor) -> ResultType<()>
                 msg = cached.clone();
             } else {
                 let mut data = crate::get_cursor_data(hcursor)?;
-                data.colors = hbb_common::compress::compress(&data.colors[..], COMPRESS_LEVEL);
+                data.colors = hbb_common::compress::compress(&data.colors[..], COMPRESS_LEVEL).into();
                 let mut tmp = Message::new();
                 tmp.set_cursor_data(data);
                 msg = Arc::new(tmp);
@@ -187,6 +187,26 @@ lazy_static::lazy_static! {
     static ref IS_SERVER: bool =  std::env::args().nth(1) == Some("--server".to_owned());
 }
 
+#[cfg(target_os = "linux")]
+pub async fn set_uinput() -> ResultType<()> {
+    // Keyboard and mouse both open /dev/uinput
+    // TODO: Make sure there's no race
+    let keyboard = super::uinput::client::UInputKeyboard::new().await?;
+    log::info!("UInput keyboard created");
+    let mouse = super::uinput::client::UInputMouse::new().await?;
+    log::info!("UInput mouse created");
+
+    let mut en = ENIGO.lock().unwrap();
+    en.set_uinput_keyboard(Some(Box::new(keyboard)));
+    en.set_uinput_mouse(Some(Box::new(mouse)));
+    Ok(())
+}
+
+#[cfg(target_os = "linux")]
+pub async fn set_uinput_resolution(minx: i32, maxx: i32, miny: i32, maxy: i32) -> ResultType<()> {
+    super::uinput::client::set_resolution(minx, maxx, miny, maxy).await
+}
+
 pub fn is_left_up(evt: &MouseEvent) -> bool {
     let buttons = evt.mask >> 3;
     let evt_type = evt.mask & 0x7;
@@ -299,12 +319,12 @@ fn fix_key_down_timeout(force: bool) {
 // e.g. current state of ctrl is down, but ctrl not in modifier, we should change ctrl to up, to make modifier state sync between remote and local
 #[inline]
 fn fix_modifier(
-    modifiers: &[ProtobufEnumOrUnknown<ControlKey>],
+    modifiers: &[EnumOrUnknown<ControlKey>],
     key0: ControlKey,
     key1: Key,
     en: &mut Enigo,
 ) {
-    if get_modifier_state(key1, en) && !modifiers.contains(&ProtobufEnumOrUnknown::new(key0)) {
+    if get_modifier_state(key1, en) && !modifiers.contains(&EnumOrUnknown::new(key0)) {
         #[cfg(windows)]
         if key0 == ControlKey::Control && get_modifier_state(Key::Alt, en) {
             // AltGr case
@@ -315,7 +335,7 @@ fn fix_modifier(
     }
 }
 
-fn fix_modifiers(modifiers: &[ProtobufEnumOrUnknown<ControlKey>], en: &mut Enigo, ck: i32) {
+fn fix_modifiers(modifiers: &[EnumOrUnknown<ControlKey>], en: &mut Enigo, ck: i32) {
     if ck != ControlKey::Shift.value() {
         fix_modifier(modifiers, ControlKey::Shift, Key::Shift, en);
     }
@@ -430,7 +450,7 @@ fn handle_mouse_(evt: &MouseEvent, conn: i32) {
 }
 
 pub fn is_enter(evt: &KeyEvent) -> bool {
-    if let Some(key_event::Union::control_key(ck)) = evt.union {
+    if let Some(key_event::Union::ControlKey(ck)) = evt.union {
         if ck.value() == ControlKey::Return.value() || ck.value() == ControlKey::NumpadEnter.value()
         {
             return true;
@@ -439,7 +459,7 @@ pub fn is_enter(evt: &KeyEvent) -> bool {
     return false;
 }
 
-pub fn lock_screen() {
+pub async fn lock_screen() {
     cfg_if::cfg_if! {
     if #[cfg(target_os = "linux")] {
         // xdg_screensaver lock not work on Linux from our service somehow
@@ -469,7 +489,7 @@ pub fn lock_screen() {
     crate::platform::lock_screen();
     }
     }
-    super::video_service::switch_to_primary();
+    super::video_service::switch_to_primary().await;
 }
 
 lazy_static::lazy_static! {
@@ -548,7 +568,6 @@ lazy_static::lazy_static! {
         (ControlKey::Equals, Key::Equals),
         (ControlKey::NumpadEnter, Key::NumpadEnter),
         (ControlKey::RAlt, Key::RightAlt),
-        (ControlKey::RWin, Key::RWin),
         (ControlKey::RControl, Key::RightControl),
         (ControlKey::RShift, Key::RightShift),
     ].iter().map(|(a, b)| (a.value(), b.clone())).collect();
@@ -598,7 +617,7 @@ fn handle_key_(evt: &KeyEvent) {
     #[cfg(windows)]
     let mut has_numlock = false;
     if evt.down {
-        let ck = if let Some(key_event::Union::control_key(ck)) = evt.union {
+        let ck = if let Some(key_event::Union::ControlKey(ck)) = evt.union {
             ck.value()
         } else {
             -1
@@ -653,7 +672,7 @@ fn handle_key_(evt: &KeyEvent) {
         }
     }
     match evt.union {
-        Some(key_event::Union::control_key(ck)) => {
+        Some(key_event::Union::ControlKey(ck)) => {
             if let Some(key) = KEY_MAP.get(&ck.value()) {
                 #[cfg(windows)]
                 if let Some(_) = NUMPAD_KEY_MAP.get(&ck.value()) {
@@ -679,10 +698,10 @@ fn handle_key_(evt: &KeyEvent) {
                     allow_err!(send_sas());
                 });
             } else if ck.value() == ControlKey::LockScreen.value() {
-                lock_screen();
+                lock_screen_2();
             }
         }
-        Some(key_event::Union::chr(chr)) => {
+        Some(key_event::Union::Chr(chr)) => {
             if evt.down {
                 if en.key_down(get_layout(chr)).is_ok() {
                     KEYS_DOWN
@@ -708,12 +727,12 @@ fn handle_key_(evt: &KeyEvent) {
                     .remove(&(chr as u64 + KEY_CHAR_START));
             }
         }
-        Some(key_event::Union::unicode(chr)) => {
+        Some(key_event::Union::Unicode(chr)) => {
             if let Ok(chr) = char::try_from(chr) {
                 en.key_sequence(&chr.to_string());
             }
         }
-        Some(key_event::Union::seq(ref seq)) => {
+        Some(key_event::Union::Seq(ref seq)) => {
             en.key_sequence(&seq);
         }
         _ => {}
@@ -729,6 +748,11 @@ fn handle_key_(evt: &KeyEvent) {
     }
 }
 
+#[tokio::main(flavor = "current_thread")]
+async fn lock_screen_2() {
+    lock_screen().await;
+}
+
 #[tokio::main(flavor = "current_thread")]
 async fn send_sas() -> ResultType<()> {
     let mut stream = crate::ipc::connect(1000, crate::POSTFIX_SERVICE).await?;

src/server/uinput.rs

@@ -0,0 +1,651 @@
+use crate::ipc::{self, new_listener, Connection, Data, DataKeyboard, DataMouse};
+use enigo::{Key, KeyboardControllable, MouseButton, MouseControllable};
+use evdev::{
+    uinput::{VirtualDevice, VirtualDeviceBuilder},
+    AttributeSet, EventType, InputEvent,
+};
+use hbb_common::{allow_err, bail, log, tokio, ResultType};
+
+static IPC_CONN_TIMEOUT: u64 = 1000;
+static IPC_REQUEST_TIMEOUT: u64 = 1000;
+static IPC_POSTFIX_KEYBOARD: &str = "_uinput_keyboard";
+static IPC_POSTFIX_MOUSE: &str = "_uinput_mouse";
+static IPC_POSTFIX_CONTROL: &str = "_uinput_control";
+
+pub mod client {
+    use super::*;
+
+    pub struct UInputKeyboard {
+        conn: Connection,
+    }
+
+    impl UInputKeyboard {
+        pub async fn new() -> ResultType<Self> {
+            let conn = ipc::connect(IPC_CONN_TIMEOUT, IPC_POSTFIX_KEYBOARD).await?;
+            Ok(Self { conn })
+        }
+
+        #[tokio::main(flavor = "current_thread")]
+        async fn send(&mut self, data: Data) -> ResultType<()> {
+            self.conn.send(&data).await
+        }
+
+        #[tokio::main(flavor = "current_thread")]
+        async fn send_get_key_state(&mut self, data: Data) -> ResultType<bool> {
+            self.conn.send(&data).await?;
+
+            match self.conn.next_timeout(IPC_REQUEST_TIMEOUT).await {
+                Ok(Some(Data::KeyboardResponse(ipc::DataKeyboardResponse::GetKeyState(state)))) => {
+                    Ok(state)
+                }
+                Ok(Some(resp)) => {
+                    // FATAL error!!!
+                    bail!(
+                        "FATAL error, wait keyboard result other response: {:?}",
+                        &resp
+                    );
+                }
+                Ok(None) => {
+                    // FATAL error!!!
+                    // Maybe wait later
+                    bail!("FATAL error, wait keyboard result, receive None",);
+                }
+                Err(e) => {
+                    // FATAL error!!!
+                    bail!(
+                        "FATAL error, wait keyboard result timeout {}, {}",
+                        &e,
+                        IPC_REQUEST_TIMEOUT
+                    );
+                }
+            }
+        }
+    }
+
+    impl KeyboardControllable for UInputKeyboard {
+        fn get_key_state(&mut self, key: Key) -> bool {
+            match self.send_get_key_state(Data::Keyboard(DataKeyboard::GetKeyState(key))) {
+                Ok(state) => state,
+                Err(e) => {
+                    // unreachable!()
+                    log::error!("Failed to get key state {}", &e);
+                    false
+                }
+            }
+        }
+
+        fn key_sequence(&mut self, sequence: &str) {
+            allow_err!(self.send(Data::Keyboard(DataKeyboard::Sequence(sequence.to_string()))));
+        }
+
+        // TODO: handle error???
+        fn key_down(&mut self, key: Key) -> enigo::ResultType {
+            allow_err!(self.send(Data::Keyboard(DataKeyboard::KeyDown(key))));
+            Ok(())
+        }
+        fn key_up(&mut self, key: Key) {
+            allow_err!(self.send(Data::Keyboard(DataKeyboard::KeyUp(key))));
+        }
+        fn key_click(&mut self, key: Key) {
+            allow_err!(self.send(Data::Keyboard(DataKeyboard::KeyClick(key))));
+        }
+    }
+
+    pub struct UInputMouse {
+        conn: Connection,
+    }
+
+    impl UInputMouse {
+        pub async fn new() -> ResultType<Self> {
+            let conn = ipc::connect(IPC_CONN_TIMEOUT, IPC_POSTFIX_MOUSE).await?;
+            Ok(Self { conn })
+        }
+
+        #[tokio::main(flavor = "current_thread")]
+        async fn send(&mut self, data: Data) -> ResultType<()> {
+            self.conn.send(&data).await
+        }
+    }
+
+    impl MouseControllable for UInputMouse {
+        fn mouse_move_to(&mut self, x: i32, y: i32) {
+            allow_err!(self.send(Data::Mouse(DataMouse::MoveTo(x, y))));
+        }
+        fn mouse_move_relative(&mut self, x: i32, y: i32) {
+            allow_err!(self.send(Data::Mouse(DataMouse::MoveRelative(x, y))));
+        }
+        // TODO: handle error???
+        fn mouse_down(&mut self, button: MouseButton) -> enigo::ResultType {
+            allow_err!(self.send(Data::Mouse(DataMouse::Down(button))));
+            Ok(())
+        }
+        fn mouse_up(&mut self, button: MouseButton) {
+            allow_err!(self.send(Data::Mouse(DataMouse::Up(button))));
+        }
+        fn mouse_click(&mut self, button: MouseButton) {
+            allow_err!(self.send(Data::Mouse(DataMouse::Click(button))));
+        }
+        fn mouse_scroll_x(&mut self, length: i32) {
+            allow_err!(self.send(Data::Mouse(DataMouse::ScrollX(length))));
+        }
+        fn mouse_scroll_y(&mut self, length: i32) {
+            allow_err!(self.send(Data::Mouse(DataMouse::ScrollY(length))));
+        }
+    }
+
+    pub async fn set_resolution(minx: i32, maxx: i32, miny: i32, maxy: i32) -> ResultType<()> {
+        let mut conn = ipc::connect(IPC_CONN_TIMEOUT, IPC_POSTFIX_CONTROL).await?;
+        conn.send(&Data::Control(ipc::DataControl::Resolution {
+            minx,
+            maxx,
+            miny,
+            maxy,
+        }))
+        .await?;
+        let _ = conn.next().await?;
+        Ok(())
+    }
+}
+
+pub mod service {
+    use super::*;
+    use hbb_common::lazy_static;
+    use mouce::MouseActions;
+    use std::{collections::HashMap, sync::Mutex};
+
+    lazy_static::lazy_static! {
+    static ref KEY_MAP: HashMap<enigo::Key, evdev::Key> = HashMap::from(
+        [
+            (enigo::Key::Alt, evdev::Key::KEY_LEFTALT),
+            (enigo::Key::Backspace, evdev::Key::KEY_BACKSPACE),
+            (enigo::Key::CapsLock, evdev::Key::KEY_CAPSLOCK),
+            (enigo::Key::Control, evdev::Key::KEY_LEFTCTRL),
+            (enigo::Key::Delete, evdev::Key::KEY_DELETE),
+            (enigo::Key::DownArrow, evdev::Key::KEY_DOWN),
+            (enigo::Key::End, evdev::Key::KEY_END),
+            (enigo::Key::Escape, evdev::Key::KEY_ESC),
+            (enigo::Key::F1, evdev::Key::KEY_F1),
+            (enigo::Key::F10, evdev::Key::KEY_F10),
+            (enigo::Key::F11, evdev::Key::KEY_F11),
+            (enigo::Key::F12, evdev::Key::KEY_F12),
+            (enigo::Key::F2, evdev::Key::KEY_F2),
+            (enigo::Key::F3, evdev::Key::KEY_F3),
+            (enigo::Key::F4, evdev::Key::KEY_F4),
+            (enigo::Key::F5, evdev::Key::KEY_F5),
+            (enigo::Key::F6, evdev::Key::KEY_F6),
+            (enigo::Key::F7, evdev::Key::KEY_F7),
+            (enigo::Key::F8, evdev::Key::KEY_F8),
+            (enigo::Key::F9, evdev::Key::KEY_F9),
+            (enigo::Key::Home, evdev::Key::KEY_HOME),
+            (enigo::Key::LeftArrow, evdev::Key::KEY_LEFT),
+            (enigo::Key::Meta, evdev::Key::KEY_LEFTMETA),
+            (enigo::Key::Option, evdev::Key::KEY_OPTION),
+            (enigo::Key::PageDown, evdev::Key::KEY_PAGEDOWN),
+            (enigo::Key::PageUp, evdev::Key::KEY_PAGEUP),
+            (enigo::Key::Return, evdev::Key::KEY_ENTER),
+            (enigo::Key::RightArrow, evdev::Key::KEY_RIGHT),
+            (enigo::Key::Shift, evdev::Key::KEY_LEFTSHIFT),
+            (enigo::Key::Space, evdev::Key::KEY_SPACE),
+            (enigo::Key::Tab, evdev::Key::KEY_TAB),
+            (enigo::Key::UpArrow, evdev::Key::KEY_UP),
+            (enigo::Key::Numpad0, evdev::Key::KEY_KP0),  // check if correct?
+            (enigo::Key::Numpad1, evdev::Key::KEY_KP1),
+            (enigo::Key::Numpad2, evdev::Key::KEY_KP2),
+            (enigo::Key::Numpad3, evdev::Key::KEY_KP3),
+            (enigo::Key::Numpad4, evdev::Key::KEY_KP4),
+            (enigo::Key::Numpad5, evdev::Key::KEY_KP5),
+            (enigo::Key::Numpad6, evdev::Key::KEY_KP6),
+            (enigo::Key::Numpad7, evdev::Key::KEY_KP7),
+            (enigo::Key::Numpad8, evdev::Key::KEY_KP8),
+            (enigo::Key::Numpad9, evdev::Key::KEY_KP9),
+            (enigo::Key::Cancel, evdev::Key::KEY_CANCEL),
+            (enigo::Key::Clear, evdev::Key::KEY_CLEAR),
+            (enigo::Key::Alt, evdev::Key::KEY_LEFTALT),
+            (enigo::Key::Pause, evdev::Key::KEY_PAUSE),
+            (enigo::Key::Kana, evdev::Key::KEY_KATAKANA),  // check if correct?
+            (enigo::Key::Hangul, evdev::Key::KEY_HANGEUL),  // check if correct?
+            // (enigo::Key::Junja, evdev::Key::KEY_JUNJA),     // map?
+            // (enigo::Key::Final, evdev::Key::KEY_FINAL),     // map?
+            (enigo::Key::Hanja, evdev::Key::KEY_HANJA),
+            // (enigo::Key::Kanji, evdev::Key::KEY_KANJI),      // map?
+            // (enigo::Key::Convert, evdev::Key::KEY_CONVERT),
+            (enigo::Key::Select, evdev::Key::KEY_SELECT),
+            (enigo::Key::Print, evdev::Key::KEY_PRINT),
+            // (enigo::Key::Execute, evdev::Key::KEY_EXECUTE),
+            // (enigo::Key::Snapshot, evdev::Key::KEY_SNAPSHOT),
+            (enigo::Key::Insert, evdev::Key::KEY_INSERT),
+            (enigo::Key::Help, evdev::Key::KEY_HELP),
+            (enigo::Key::Sleep, evdev::Key::KEY_SLEEP),
+            // (enigo::Key::Separator, evdev::Key::KEY_SEPARATOR),
+            (enigo::Key::Scroll, evdev::Key::KEY_SCROLLLOCK),
+            (enigo::Key::NumLock, evdev::Key::KEY_NUMLOCK),
+            (enigo::Key::RWin, evdev::Key::KEY_RIGHTMETA),
+            (enigo::Key::Apps, evdev::Key::KEY_CONTEXT_MENU),
+            (enigo::Key::Multiply, evdev::Key::KEY_KPASTERISK),
+            (enigo::Key::Add, evdev::Key::KEY_KPPLUS),
+            (enigo::Key::Subtract, evdev::Key::KEY_KPMINUS),
+            (enigo::Key::Decimal, evdev::Key::KEY_KPCOMMA),   // KEY_KPDOT and KEY_KPCOMMA are exchanged?
+            (enigo::Key::Divide, evdev::Key::KEY_KPSLASH),
+            (enigo::Key::Equals, evdev::Key::KEY_KPEQUAL),
+            (enigo::Key::NumpadEnter, evdev::Key::KEY_KPENTER),
+            (enigo::Key::RightAlt, evdev::Key::KEY_RIGHTALT),
+            (enigo::Key::RightControl, evdev::Key::KEY_RIGHTCTRL),
+            (enigo::Key::RightShift, evdev::Key::KEY_RIGHTSHIFT),
+        ]);
+
+        static ref KEY_MAP_LAYOUT: HashMap<char, evdev::Key> = HashMap::from(
+            [
+                ('a', evdev::Key::KEY_A),
+                ('b', evdev::Key::KEY_B),
+                ('c', evdev::Key::KEY_C),
+                ('d', evdev::Key::KEY_D),
+                ('e', evdev::Key::KEY_E),
+                ('f', evdev::Key::KEY_F),
+                ('g', evdev::Key::KEY_G),
+                ('h', evdev::Key::KEY_H),
+                ('i', evdev::Key::KEY_I),
+                ('j', evdev::Key::KEY_J),
+                ('k', evdev::Key::KEY_K),
+                ('l', evdev::Key::KEY_L),
+                ('m', evdev::Key::KEY_M),
+                ('n', evdev::Key::KEY_N),
+                ('o', evdev::Key::KEY_O),
+                ('p', evdev::Key::KEY_P),
+                ('q', evdev::Key::KEY_Q),
+                ('r', evdev::Key::KEY_R),
+                ('s', evdev::Key::KEY_S),
+                ('t', evdev::Key::KEY_T),
+                ('u', evdev::Key::KEY_U),
+                ('v', evdev::Key::KEY_V),
+                ('w', evdev::Key::KEY_W),
+                ('x', evdev::Key::KEY_X),
+                ('y', evdev::Key::KEY_Y),
+                ('z', evdev::Key::KEY_Z),
+                ('0', evdev::Key::KEY_0),
+                ('1', evdev::Key::KEY_1),
+                ('2', evdev::Key::KEY_2),
+                ('3', evdev::Key::KEY_3),
+                ('4', evdev::Key::KEY_4),
+                ('5', evdev::Key::KEY_5),
+                ('6', evdev::Key::KEY_6),
+                ('7', evdev::Key::KEY_7),
+                ('8', evdev::Key::KEY_8),
+                ('9', evdev::Key::KEY_9),
+                ('`', evdev::Key::KEY_GRAVE),
+                ('-', evdev::Key::KEY_MINUS),
+                ('=', evdev::Key::KEY_EQUAL),
+                ('[', evdev::Key::KEY_LEFTBRACE),
+                (']', evdev::Key::KEY_RIGHTBRACE),
+                ('\\', evdev::Key::KEY_BACKSLASH),
+                (',', evdev::Key::KEY_COMMA),
+                ('.', evdev::Key::KEY_DOT),
+                ('/', evdev::Key::KEY_SLASH),
+                (';', evdev::Key::KEY_SEMICOLON),
+                ('\'', evdev::Key::KEY_APOSTROPHE),
+            ]);
+
+        // ((minx, maxx), (miny, maxy))
+        static ref RESOLUTION: Mutex<((i32, i32), (i32, i32))> = Mutex::new(((0, 0), (0, 0)));
+    }
+
+    fn create_uinput_keyboard() -> ResultType<VirtualDevice> {
+        // TODO: ensure keys here
+        let mut keys = AttributeSet::<evdev::Key>::new();
+        for i in evdev::Key::KEY_ESC.code()..(evdev::Key::BTN_TRIGGER_HAPPY40.code() + 1) {
+            let key = evdev::Key::new(i);
+            if !format!("{:?}", &key).contains("unknown key") {
+                keys.insert(key);
+            }
+        }
+        let mut leds = AttributeSet::<evdev::LedType>::new();
+        leds.insert(evdev::LedType::LED_NUML);
+        leds.insert(evdev::LedType::LED_CAPSL);
+        leds.insert(evdev::LedType::LED_SCROLLL);
+        let mut miscs = AttributeSet::<evdev::MiscType>::new();
+        miscs.insert(evdev::MiscType::MSC_SCAN);
+        let keyboard = VirtualDeviceBuilder::new()?
+            .name("RustDesk UInput Keyboard")
+            .with_keys(&keys)?
+            .with_leds(&leds)?
+            .with_miscs(&miscs)?
+            .build()?;
+        Ok(keyboard)
+    }
+
+    fn map_key(key: &enigo::Key) -> ResultType<evdev::Key> {
+        if let Some(k) = KEY_MAP.get(&key) {
+            log::trace!("mapkey {:?}, get {:?}", &key, &k);
+            return Ok(k.clone());
+        } else {
+            match key {
+                enigo::Key::Layout(c) => {
+                    if let Some(k) = KEY_MAP_LAYOUT.get(&c) {
+                        log::trace!("mapkey {:?}, get {:?}", &key, k);
+                        return Ok(k.clone());
+                    }
+                }
+                // enigo::Key::Raw(c) => {
+                //     let k = evdev::Key::new(c);
+                //     if !format!("{:?}", &k).contains("unknown key") {
+                //         return Ok(k.clone());
+                //     }
+                // }
+                _ => {}
+            }
+        }
+        bail!("Failed to map key {:?}", &key);
+    }
+
+    async fn ipc_send_data(stream: &mut Connection, data: &Data) {
+        allow_err!(stream.send(data).await);
+    }
+
+    async fn handle_keyboard(
+        stream: &mut Connection,
+        keyboard: &mut VirtualDevice,
+        data: &DataKeyboard,
+    ) {
+        log::trace!("handle_keyboard {:?}", &data);
+        match data {
+            DataKeyboard::Sequence(_seq) => {
+                // ignore
+            }
+            DataKeyboard::KeyDown(key) => {
+                if let Ok(k) = map_key(key) {
+                    let down_event = InputEvent::new(EventType::KEY, k.code(), 1);
+                    allow_err!(keyboard.emit(&[down_event]));
+                }
+            }
+            DataKeyboard::KeyUp(key) => {
+                if let Ok(k) = map_key(key) {
+                    let up_event = InputEvent::new(EventType::KEY, k.code(), 0);
+                    allow_err!(keyboard.emit(&[up_event]));
+                }
+            }
+            DataKeyboard::KeyClick(key) => {
+                if let Ok(k) = map_key(key) {
+                    let down_event = InputEvent::new(EventType::KEY, k.code(), 1);
+                    let up_event = InputEvent::new(EventType::KEY, k.code(), 0);
+                    allow_err!(keyboard.emit(&[down_event, up_event]));
+                }
+            }
+            DataKeyboard::GetKeyState(key) => {
+                let key_state = if enigo::Key::CapsLock == *key {
+                    match keyboard.get_led_state() {
+                        Ok(leds) => leds.contains(evdev::LedType::LED_CAPSL),
+                        Err(_e) => {
+                            // log::debug!("Failed to get led state {}", &_e);
+                            false
+                        }
+                    }
+                } else {
+                    match keyboard.get_key_state() {
+                        Ok(keys) => match key {
+                            enigo::Key::Shift => {
+                                keys.contains(evdev::Key::KEY_LEFTSHIFT)
+                                    || keys.contains(evdev::Key::KEY_RIGHTSHIFT)
+                            }
+                            enigo::Key::Control => {
+                                keys.contains(evdev::Key::KEY_LEFTCTRL)
+                                    || keys.contains(evdev::Key::KEY_RIGHTCTRL)
+                            }
+                            enigo::Key::Alt => {
+                                keys.contains(evdev::Key::KEY_LEFTALT)
+                                    || keys.contains(evdev::Key::KEY_RIGHTALT)
+                            }
+                            enigo::Key::NumLock => keys.contains(evdev::Key::KEY_NUMLOCK),
+                            enigo::Key::Meta => {
+                                keys.contains(evdev::Key::KEY_LEFTMETA)
+                                    || keys.contains(evdev::Key::KEY_RIGHTMETA)
+                            }
+                            _ => false,
+                        },
+                        Err(_e) => {
+                            // log::debug!("Failed to get key state: {}", &_e);
+                            false
+                        }
+                    }
+                };
+                ipc_send_data(
+                    stream,
+                    &Data::KeyboardResponse(ipc::DataKeyboardResponse::GetKeyState(key_state)),
+                )
+                .await;
+            }
+        }
+    }
+
+    fn handle_mouse(mouse: &mut mouce::nix::UInputMouseManager, data: &DataMouse) {
+        log::trace!("handle_mouse {:?}", &data);
+        match data {
+            DataMouse::MoveTo(x, y) => {
+                allow_err!(mouse.move_to(*x as _, *y as _))
+            }
+            DataMouse::MoveRelative(x, y) => {
+                allow_err!(mouse.move_relative(*x, *y))
+            }
+            DataMouse::Down(button) => {
+                let btn = match button {
+                    enigo::MouseButton::Left => mouce::common::MouseButton::Left,
+                    enigo::MouseButton::Middle => mouce::common::MouseButton::Middle,
+                    enigo::MouseButton::Right => mouce::common::MouseButton::Right,
+                    _ => {
+                        return;
+                    }
+                };
+                allow_err!(mouse.press_button(&btn))
+            }
+            DataMouse::Up(button) => {
+                let btn = match button {
+                    enigo::MouseButton::Left => mouce::common::MouseButton::Left,
+                    enigo::MouseButton::Middle => mouce::common::MouseButton::Middle,
+                    enigo::MouseButton::Right => mouce::common::MouseButton::Right,
+                    _ => {
+                        return;
+                    }
+                };
+                allow_err!(mouse.release_button(&btn))
+            }
+            DataMouse::Click(button) => {
+                let btn = match button {
+                    enigo::MouseButton::Left => mouce::common::MouseButton::Left,
+                    enigo::MouseButton::Middle => mouce::common::MouseButton::Middle,
+                    enigo::MouseButton::Right => mouce::common::MouseButton::Right,
+                    _ => {
+                        return;
+                    }
+                };
+                allow_err!(mouse.click_button(&btn))
+            }
+            DataMouse::ScrollX(_length) => {
+                // TODO: not supported for now
+            }
+            DataMouse::ScrollY(length) => {
+                let mut length = *length;
+
+                let scroll = if length < 0 {
+                    mouce::common::ScrollDirection::Up
+                } else {
+                    mouce::common::ScrollDirection::Down
+                };
+
+                if length < 0 {
+                    length = -length;
+                }
+
+                for _ in 0..length {
+                    allow_err!(mouse.scroll_wheel(&scroll))
+                }
+            }
+        }
+    }
+
+    fn spawn_keyboard_handler(mut stream: Connection) {
+        tokio::spawn(async move {
+            let mut keyboard = match create_uinput_keyboard() {
+                Ok(keyboard) => keyboard,
+                Err(e) => {
+                    log::error!("Failed to create keyboard {}", e);
+                    return;
+                }
+            };
+            loop {
+                tokio::select! {
+                    res = stream.next() => {
+                        match res {
+                            Err(err) => {
+                                log::info!("UInput keyboard ipc connection closed: {}", err);
+                                break;
+                            }
+                            Ok(Some(data)) => {
+                                match data {
+                                    Data::Keyboard(data) => {
+                                        handle_keyboard(&mut stream, &mut keyboard, &data).await;
+                                    }
+                                    _ => {
+                                    }
+                                }
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+            }
+        });
+    }
+
+    fn spawn_mouse_handler(mut stream: ipc::Connection) {
+        let resolution = RESOLUTION.lock().unwrap();
+        if resolution.0 .0 == resolution.0 .1 || resolution.1 .0 == resolution.1 .1 {
+            return;
+        }
+        let rng_x = resolution.0.clone();
+        let rng_y = resolution.1.clone();
+        tokio::spawn(async move {
+            log::info!(
+                "Create uinput mouce with rng_x: ({}, {}), rng_y: ({}, {})",
+                rng_x.0,
+                rng_x.1,
+                rng_y.0,
+                rng_y.1
+            );
+            let mut mouse = match mouce::Mouse::new_uinput(rng_x, rng_y) {
+                Ok(mouse) => mouse,
+                Err(e) => {
+                    log::error!("Failed to create mouse, {}", e);
+                    return;
+                }
+            };
+            loop {
+                tokio::select! {
+                    res = stream.next() => {
+                        match res {
+                            Err(err) => {
+                                log::info!("UInput mouse ipc connection closed: {}", err);
+                                break;
+                            }
+                            Ok(Some(data)) => {
+                                match data {
+                                    Data::Mouse(data) => {
+                                        handle_mouse(&mut mouse, &data);
+                                    }
+                                    _ => {
+                                    }
+                                }
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+            }
+        });
+    }
+
+    fn spawn_controller_handler(mut stream: ipc::Connection) {
+        tokio::spawn(async move {
+            loop {
+                tokio::select! {
+                    res = stream.next() => {
+                        match res {
+                            Err(_err) => {
+                                // log::info!("UInput controller ipc connection closed: {}", err);
+                                break;
+                            }
+                            Ok(Some(data)) => {
+                                match data {
+                                    Data::Control(data) => match data {
+                                        ipc::DataControl::Resolution{
+                                            minx,
+                                            maxx,
+                                            miny,
+                                            maxy,
+                                        } => {
+                                            *RESOLUTION.lock().unwrap() = ((minx, maxx), (miny, maxy));
+                                            allow_err!(stream.send(&Data::Empty).await);
+                                        }
+                                    }
+                                    _ => {
+                                    }
+                                }
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+            }
+        });
+    }
+
+    /// Start uinput service.
+    async fn start_service<F: FnOnce(ipc::Connection) + Copy>(postfix: &str, handler: F) {
+        match new_listener(postfix).await {
+            Ok(mut incoming) => {
+                while let Some(result) = incoming.next().await {
+                    match result {
+                        Ok(stream) => {
+                            log::debug!("Got new connection of uinput ipc {}", postfix);
+                            handler(Connection::new(stream));
+                        }
+                        Err(err) => {
+                            log::error!("Couldn't get uinput mouse client: {:?}", err);
+                        }
+                    }
+                }
+            }
+            Err(err) => {
+                log::error!("Failed to start uinput mouse ipc service: {}", err);
+            }
+        }
+    }
+
+    /// Start uinput keyboard service.
+    #[tokio::main(flavor = "current_thread")]
+    pub async fn start_service_keyboard() {
+        log::info!("start uinput keyboard service");
+        start_service(IPC_POSTFIX_KEYBOARD, spawn_keyboard_handler).await;
+    }
+
+    /// Start uinput mouse service.
+    #[tokio::main(flavor = "current_thread")]
+    pub async fn start_service_mouse() {
+        log::info!("start uinput mouse service");
+        start_service(IPC_POSTFIX_MOUSE, spawn_mouse_handler).await;
+    }
+
+    /// Start uinput mouse service.
+    #[tokio::main(flavor = "current_thread")]
+    pub async fn start_service_control() {
+        log::info!("start uinput control service");
+        start_service(IPC_POSTFIX_CONTROL, spawn_controller_handler).await;
+    }
+
+    pub fn stop_service_keyboard() {
+        log::info!("stop uinput keyboard service");
+    }
+    pub fn stop_service_mouse() {
+        log::info!("stop uinput mouse service");
+    }
+    pub fn stop_service_control() {
+        log::info!("stop uinput control service");
+    }
+}

src/server/video_qos.rs

@@ -0,0 +1,218 @@
+use super::*;
+use std::time::Duration;
+const FPS: u8 = 30;
+trait Percent {
+    fn as_percent(&self) -> u32;
+}
+
+impl Percent for ImageQuality {
+    fn as_percent(&self) -> u32 {
+        match self {
+            ImageQuality::NotSet => 0,
+            ImageQuality::Low => 50,
+            ImageQuality::Balanced => 66,
+            ImageQuality::Best => 100,
+        }
+    }
+}
+
+pub struct VideoQoS {
+    width: u32,
+    height: u32,
+    user_image_quality: u32,
+    current_image_quality: u32,
+    enable_abr: bool,
+    pub current_delay: u32,
+    pub fps: u8,             // abr
+    pub target_bitrate: u32, // abr
+    updated: bool,
+    state: DelayState,
+    debounce_count: u32,
+}
+
+#[derive(PartialEq, Debug)]
+enum DelayState {
+    Normal = 0,
+    LowDelay = 200,
+    HighDelay = 500,
+    Broken = 1000,
+}
+
+impl DelayState {
+    fn from_delay(delay: u32) -> Self {
+        if delay > DelayState::Broken as u32 {
+            DelayState::Broken
+        } else if delay > DelayState::HighDelay as u32 {
+            DelayState::HighDelay
+        } else if delay > DelayState::LowDelay as u32 {
+            DelayState::LowDelay
+        } else {
+            DelayState::Normal
+        }
+    }
+}
+
+impl Default for VideoQoS {
+    fn default() -> Self {
+        VideoQoS {
+            fps: FPS,
+            user_image_quality: ImageQuality::Balanced.as_percent(),
+            current_image_quality: ImageQuality::Balanced.as_percent(),
+            enable_abr: false,
+            width: 0,
+            height: 0,
+            current_delay: 0,
+            target_bitrate: 0,
+            updated: false,
+            state: DelayState::Normal,
+            debounce_count: 0,
+        }
+    }
+}
+
+impl VideoQoS {
+    pub fn set_size(&mut self, width: u32, height: u32) {
+        if width == 0 || height == 0 {
+            return;
+        }
+        self.width = width;
+        self.height = height;
+    }
+
+    pub fn spf(&mut self) -> Duration {
+        if self.fps <= 0 {
+            self.fps = FPS;
+        }
+        Duration::from_secs_f32(1. / (self.fps as f32))
+    }
+
+    // update_network_delay periodically
+    // decrease the bitrate when the delay gets bigger
+    pub fn update_network_delay(&mut self, delay: u32) {
+        if self.current_delay.eq(&0) {
+            self.current_delay = delay;
+            return;
+        }
+
+        self.current_delay = delay / 2 + self.current_delay / 2;
+        log::trace!(
+            "VideoQoS update_network_delay:{}, {}, state:{:?}",
+            self.current_delay,
+            delay,
+            self.state,
+        );
+
+        // ABR
+        if !self.enable_abr {
+            return;
+        }
+        let current_state = DelayState::from_delay(self.current_delay);
+        if current_state != self.state && self.debounce_count > 5 {
+            log::debug!(
+                "VideoQoS state changed:{:?} -> {:?}",
+                self.state,
+                current_state
+            );
+            self.state = current_state;
+            self.debounce_count = 0;
+            self.refresh_quality();
+        } else {
+            self.debounce_count += 1;
+        }
+    }
+
+    fn refresh_quality(&mut self) {
+        match self.state {
+            DelayState::Normal => {
+                self.fps = FPS;
+                self.current_image_quality = self.user_image_quality;
+            }
+            DelayState::LowDelay => {
+                self.fps = FPS;
+                self.current_image_quality = std::cmp::min(self.user_image_quality, 50);
+            }
+            DelayState::HighDelay => {
+                self.fps = FPS / 2;
+                self.current_image_quality = std::cmp::min(self.user_image_quality, 25);
+            }
+            DelayState::Broken => {
+                self.fps = FPS / 4;
+                self.current_image_quality = 10;
+            }
+        }
+        let _ = self.generate_bitrate().ok();
+        self.updated = true;
+    }
+
+    // handle image_quality change from peer
+    pub fn update_image_quality(&mut self, image_quality: i32) {
+        let image_quality = Self::convert_quality(image_quality) as _;
+        if self.current_image_quality != image_quality {
+            self.current_image_quality = image_quality;
+            let _ = self.generate_bitrate().ok();
+            self.updated = true;
+        }
+
+        self.user_image_quality = self.current_image_quality;
+    }
+
+    pub fn generate_bitrate(&mut self) -> ResultType<u32> {
+        // https://www.nvidia.com/en-us/geforce/guides/broadcasting-guide/
+        if self.width == 0 || self.height == 0 {
+            bail!("Fail to generate_bitrate, width or height is not set");
+        }
+        if self.current_image_quality == 0 {
+            self.current_image_quality = ImageQuality::Balanced.as_percent();
+        }
+
+        let base_bitrate = ((self.width * self.height) / 800) as u32;
+
+        #[cfg(target_os = "android")]
+        {
+            // fix when andorid screen shrinks
+            let fix = scrap::Display::fix_quality() as u32;
+            log::debug!("Android screen, fix quality:{}", fix);
+            let base_bitrate = base_bitrate * fix;
+            self.target_bitrate = base_bitrate * self.current_image_quality / 100;
+            Ok(self.target_bitrate)
+        }
+        #[cfg(not(target_os = "android"))]
+        {
+            self.target_bitrate = base_bitrate * self.current_image_quality / 100;
+            Ok(self.target_bitrate)
+        }
+    }
+
+    pub fn check_if_updated(&mut self) -> bool {
+        if self.updated {
+            self.updated = false;
+            return true;
+        }
+        return false;
+    }
+
+    pub fn reset(&mut self) {
+        *self = Default::default();
+    }
+
+    pub fn check_abr_config(&mut self) -> bool {
+        self.enable_abr = if let Some(v) = Config2::get().options.get("enable-abr") {
+            v != "N"
+        } else {
+            true // default is true
+        };
+        self.enable_abr
+    }
+
+    pub fn convert_quality(q: i32) -> i32 {
+        if q == ImageQuality::Balanced.value() {
+            100 * 2 / 3
+        } else if q == ImageQuality::Low.value() {
+            100 / 2
+        } else if q == ImageQuality::Best.value() {
+            100
+        } else {
+            (q >> 8 & 0xFF) * 2
+        }
+    }
+}

src/server/video_service.rs

@@ -18,15 +18,20 @@
 // to-do:
 // https://slhck.info/video/2017/03/01/rate-control.html
 
-use super::*;
+use super::{video_qos::VideoQoS, *};
 use hbb_common::tokio::sync::{
     mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
     Mutex as TokioMutex,
 };
-use scrap::{Capturer, Config, Display, EncodeFrame, Encoder, Frame, VideoCodecId, STRIDE_ALIGN};
+use scrap::{
+    codec::{Encoder, EncoderCfg, HwEncoderConfig},
+    vpxcodec::{VpxEncoderConfig, VpxVideoCodecId},
+    Capturer, Display, TraitCapturer,
+};
 use std::{
     collections::HashSet,
-    io::{ErrorKind::WouldBlock, Result},
+    io::ErrorKind::WouldBlock,
+    ops::{Deref, DerefMut},
     time::{self, Duration, Instant},
 };
 #[cfg(windows)]
@@ -38,14 +43,13 @@ lazy_static::lazy_static! {
     static ref CURRENT_DISPLAY: Arc<Mutex<usize>> = Arc::new(Mutex::new(usize::MAX));
     static ref LAST_ACTIVE: Arc<Mutex<Instant>> = Arc::new(Mutex::new(Instant::now()));
     static ref SWITCH: Arc<Mutex<bool>> = Default::default();
-    static ref TEST_LATENCIES: Arc<Mutex<HashMap<i32, i64>>> = Default::default();
-    static ref IMAGE_QUALITIES: Arc<Mutex<HashMap<i32, i32>>> = Default::default();
     static ref FRAME_FETCHED_NOTIFIER: (UnboundedSender<(i32, Option<Instant>)>, Arc<TokioMutex<UnboundedReceiver<(i32, Option<Instant>)>>>) = {
         let (tx, rx) = unbounded_channel();
         (tx, Arc::new(TokioMutex::new(rx)))
     };
     static ref PRIVACY_MODE_CONN_ID: Mutex<i32> = Mutex::new(0);
     static ref IS_CAPTURER_MAGNIFIER_SUPPORTED: bool = is_capturer_mag_supported();
+    pub static ref VIDEO_QOS: Arc<Mutex<VideoQoS>> = Default::default();
 }
 
 fn is_capturer_mag_supported() -> bool {
@@ -124,46 +128,6 @@ impl VideoFrameController {
     }
 }
 
-trait TraitCapturer {
-    fn frame<'a>(&'a mut self, timeout_ms: u32) -> Result<Frame<'a>>;
-
-    #[cfg(windows)]
-    fn is_gdi(&self) -> bool;
-    #[cfg(windows)]
-    fn set_gdi(&mut self) -> bool;
-}
-
-impl TraitCapturer for Capturer {
-    fn frame<'a>(&'a mut self, timeout_ms: u32) -> Result<Frame<'a>> {
-        self.frame(timeout_ms)
-    }
-
-    #[cfg(windows)]
-    fn is_gdi(&self) -> bool {
-        self.is_gdi()
-    }
-
-    #[cfg(windows)]
-    fn set_gdi(&mut self) -> bool {
-        self.set_gdi()
-    }
-}
-
-#[cfg(windows)]
-impl TraitCapturer for scrap::CapturerMag {
-    fn frame<'a>(&'a mut self, _timeout_ms: u32) -> Result<Frame<'a>> {
-        self.frame(_timeout_ms)
-    }
-
-    fn is_gdi(&self) -> bool {
-        false
-    }
-
-    fn set_gdi(&mut self) -> bool {
-        false
-    }
-}
-
 pub fn new() -> GenericService {
     let sp = GenericService::new(NAME, true);
     sp.run(run);
@@ -176,6 +140,14 @@ fn check_display_changed(
     last_width: usize,
     last_hegiht: usize,
 ) -> bool {
+    #[cfg(target_os = "linux")]
+    {
+        // wayland do not support changing display for now
+        if !scrap::is_x11() {
+            return false;
+        }
+    }
+
     let displays = match try_get_displays() {
         Ok(d) => d,
         _ => return false,
@@ -201,9 +173,11 @@ fn check_display_changed(
 }
 
 // Capturer object is expensive, avoiding to create it frequently.
-fn create_capturer(privacy_mode_id: i32, display: Display) -> ResultType<Box<dyn TraitCapturer>> {
-    let use_yuv = true;
-
+fn create_capturer(
+    privacy_mode_id: i32,
+    display: Display,
+    use_yuv: bool,
+) -> ResultType<Box<dyn TraitCapturer>> {
     #[cfg(not(windows))]
     let c: Option<Box<dyn TraitCapturer>> = None;
     #[cfg(windows)]
@@ -288,11 +262,12 @@ fn ensure_close_virtual_device() -> ResultType<()> {
     Ok(())
 }
 
+// This function works on privacy mode. Windows only for now.
 pub fn test_create_capturer(privacy_mode_id: i32, timeout_millis: u64) -> bool {
     let test_begin = Instant::now();
     while test_begin.elapsed().as_millis() < timeout_millis as _ {
         if let Ok((_, _, display)) = get_current_display() {
-            if let Ok(_) = create_capturer(privacy_mode_id, display) {
+            if let Ok(_) = create_capturer(privacy_mode_id, display, true) {
                 return true;
             }
         }
@@ -316,13 +291,39 @@ fn check_uac_switch(privacy_mode_id: i32, captuerer_privacy_mode_id: i32) -> Res
     Ok(())
 }
 
-fn run(sp: GenericService) -> ResultType<()> {
-    #[cfg(windows)]
-    ensure_close_virtual_device()?;
+pub(super) struct CapturerInfo {
+    pub origin: (i32, i32),
+    pub width: usize,
+    pub height: usize,
+    pub ndisplay: usize,
+    pub current: usize,
+    pub privacy_mode_id: i32,
+    pub _captuerer_privacy_mode_id: i32,
+    pub capturer: Box<dyn TraitCapturer>,
+}
+
+impl Deref for CapturerInfo {
+    type Target = Box<dyn TraitCapturer>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.capturer
+    }
+}
+
+impl DerefMut for CapturerInfo {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.capturer
+    }
+}
+
+fn get_capturer(use_yuv: bool) -> ResultType<CapturerInfo> {
+    #[cfg(target_os = "linux")]
+    {
+        if !scrap::is_x11() {
+            return super::wayland::get_capturer();
+        }
+    }
 
-    let fps = 30;
-    let wait = 1000 / fps;
-    let spf = time::Duration::from_secs_f32(1. / (fps as f32));
     let (ndisplay, current, display) = get_current_display()?;
     let (origin, width, height) = (display.origin(), display.width(), display.height());
     log::debug!(
@@ -342,8 +343,10 @@ fn run(sp: GenericService) -> ResultType<()> {
     #[cfg(windows)]
     let mut captuerer_privacy_mode_id = privacy_mode_id;
     #[cfg(windows)]
-    if crate::ui::win_privacy::is_process_consent_running()? {
-        captuerer_privacy_mode_id = 0;
+    if captuerer_privacy_mode_id != 0 {
+        if crate::ui::win_privacy::is_process_consent_running()? {
+            captuerer_privacy_mode_id = 0;
+        }
     }
     log::debug!(
         "Try create capturer with captuerer privacy mode id {}",
@@ -355,36 +358,66 @@ fn run(sp: GenericService) -> ResultType<()> {
     } else {
         log::info!("In privacy mode, the peer side cannot watch the screen");
     }
-    let mut c = create_capturer(captuerer_privacy_mode_id, display)?;
+    let capturer = create_capturer(captuerer_privacy_mode_id, display, use_yuv)?;
+    Ok(CapturerInfo {
+        origin,
+        width,
+        height,
+        ndisplay,
+        current,
+        privacy_mode_id,
+        _captuerer_privacy_mode_id: captuerer_privacy_mode_id,
+        capturer,
+    })
+}
 
-    let q = get_image_quality();
-    let (bitrate, rc_min_quantizer, rc_max_quantizer, speed) = get_quality(width, height, q);
-    log::info!("bitrate={}, rc_min_quantizer={}", bitrate, rc_min_quantizer);
-    let cfg = Config {
-        width: width as _,
-        height: height as _,
-        timebase: [1, 1000], // Output timestamp precision
-        bitrate,
-        codec: VideoCodecId::VP9,
-        rc_min_quantizer,
-        rc_max_quantizer,
-        speed,
+fn run(sp: GenericService) -> ResultType<()> {
+    #[cfg(windows)]
+    ensure_close_virtual_device()?;
+
+    let mut c = get_capturer(true)?;
+
+    let mut video_qos = VIDEO_QOS.lock().unwrap();
+    video_qos.set_size(c.width as _, c.height as _);
+    let mut spf = video_qos.spf();
+    let bitrate = video_qos.generate_bitrate()?;
+    let abr = video_qos.check_abr_config();
+    drop(video_qos);
+    log::info!("init bitrate={}, abr enabled:{}", bitrate, abr);
+
+    let encoder_cfg = match Encoder::current_hw_encoder_name() {
+        Some(codec_name) => EncoderCfg::HW(HwEncoderConfig {
+            codec_name,
+            width: c.width,
+            height: c.height,
+            bitrate: bitrate as _,
+        }),
+        None => EncoderCfg::VPX(VpxEncoderConfig {
+            width: c.width as _,
+            height: c.height as _,
+            timebase: [1, 1000], // Output timestamp precision
+            bitrate,
+            codec: VpxVideoCodecId::VP9,
+            num_threads: (num_cpus::get() / 2) as _,
+        }),
     };
-    let mut vpx;
-    match Encoder::new(&cfg, (num_cpus::get() / 2) as _) {
-        Ok(x) => vpx = x,
+
+    let mut encoder;
+    match Encoder::new(encoder_cfg) {
+        Ok(x) => encoder = x,
         Err(err) => bail!("Failed to create encoder: {}", err),
     }
+    c.set_use_yuv(encoder.use_yuv());
 
     if *SWITCH.lock().unwrap() {
         log::debug!("Broadcasting display switch");
         let mut misc = Misc::new();
         misc.set_switch_display(SwitchDisplay {
-            display: current as _,
-            x: origin.0 as _,
-            y: origin.1 as _,
-            width: width as _,
-            height: height as _,
+            display: c.current as _,
+            x: c.origin.0 as _,
+            y: c.origin.1 as _,
+            width: c.width as _,
+            height: c.height as _,
             ..Default::default()
         });
         let mut msg_out = Message::new();
@@ -401,21 +434,35 @@ fn run(sp: GenericService) -> ResultType<()> {
     let mut try_gdi = 1;
     #[cfg(windows)]
     log::info!("gdi: {}", c.is_gdi());
+    let codec_name = Encoder::current_hw_encoder_name();
+
     while sp.ok() {
         #[cfg(windows)]
-        check_uac_switch(privacy_mode_id, captuerer_privacy_mode_id)?;
+        check_uac_switch(c.privacy_mode_id, c._captuerer_privacy_mode_id)?;
+
+        let mut video_qos = VIDEO_QOS.lock().unwrap();
+        if video_qos.check_if_updated() {
+            log::debug!(
+                "qos is updated, target_bitrate:{}, fps:{}",
+                video_qos.target_bitrate,
+                video_qos.fps
+            );
+            encoder.set_bitrate(video_qos.target_bitrate).unwrap();
+            spf = video_qos.spf();
+        }
+        drop(video_qos);
 
         if *SWITCH.lock().unwrap() {
             bail!("SWITCH");
         }
-        if current != *CURRENT_DISPLAY.lock().unwrap() {
+        if c.current != *CURRENT_DISPLAY.lock().unwrap() {
             *SWITCH.lock().unwrap() = true;
             bail!("SWITCH");
         }
-        check_privacy_mode_changed(&sp, privacy_mode_id)?;
-        if get_image_quality() != q {
+        if codec_name != Encoder::current_hw_encoder_name() {
             bail!("SWITCH");
         }
+        check_privacy_mode_changed(&sp, c.privacy_mode_id)?;
         #[cfg(windows)]
         {
             if crate::platform::windows::desktop_changed() {
@@ -425,7 +472,7 @@ fn run(sp: GenericService) -> ResultType<()> {
         let now = time::Instant::now();
         if last_check_displays.elapsed().as_millis() > 1000 {
             last_check_displays = now;
-            if ndisplay != get_display_num() {
+            if c.ndisplay != get_display_num() {
                 log::info!("Displays changed");
                 *SWITCH.lock().unwrap() = true;
                 bail!("SWITCH");
@@ -437,7 +484,7 @@ fn run(sp: GenericService) -> ResultType<()> {
         frame_controller.reset();
 
         #[cfg(any(target_os = "android", target_os = "ios"))]
-        let res = match (*c).frame(wait as _) {
+        let res = match c.frame(spf) {
             Ok(frame) => {
                 let time = now - start;
                 let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64;
@@ -448,7 +495,7 @@ fn run(sp: GenericService) -> ResultType<()> {
                     }
                     scrap::Frame::RAW(data) => {
                         if (data.len() != 0) {
-                            let send_conn_ids = handle_one_frame(&sp, data, ms, &mut vpx)?;
+                            let send_conn_ids = handle_one_frame(&sp, data, ms, &mut encoder)?;
                             frame_controller.set_send(now, send_conn_ids);
                         }
                     }
@@ -460,11 +507,11 @@ fn run(sp: GenericService) -> ResultType<()> {
         };
 
         #[cfg(not(any(target_os = "android", target_os = "ios")))]
-        let res = match (*c).frame(wait as _) {
+        let res = match c.frame(spf) {
             Ok(frame) => {
                 let time = now - start;
                 let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64;
-                let send_conn_ids = handle_one_frame(&sp, &frame, ms, &mut vpx)?;
+                let send_conn_ids = handle_one_frame(&sp, &frame, ms, &mut encoder)?;
                 frame_controller.set_send(now, send_conn_ids);
                 #[cfg(windows)]
                 {
@@ -489,7 +536,7 @@ fn run(sp: GenericService) -> ResultType<()> {
                 }
             }
             Err(err) => {
-                if check_display_changed(ndisplay, current, width, height) {
+                if check_display_changed(c.ndisplay, c.current, c.width, c.height) {
                     log::info!("Displays changed");
                     *SWITCH.lock().unwrap() = true;
                     bail!("SWITCH");
@@ -511,9 +558,9 @@ fn run(sp: GenericService) -> ResultType<()> {
         let timeout_millis = 3_000u64;
         let wait_begin = Instant::now();
         while wait_begin.elapsed().as_millis() < timeout_millis as _ {
-            check_privacy_mode_changed(&sp, privacy_mode_id)?;
+            check_privacy_mode_changed(&sp, c.privacy_mode_id)?;
             #[cfg(windows)]
-            check_uac_switch(privacy_mode_id, captuerer_privacy_mode_id)?;
+            check_uac_switch(c.privacy_mode_id, c._captuerer_privacy_mode_id)?;
             frame_controller.try_wait_next(&mut fetched_conn_ids, 300);
             // break if all connections have received current frame
             if fetched_conn_ids.len() >= frame_controller.send_conn_ids.len() {
@@ -531,13 +578,12 @@ fn run(sp: GenericService) -> ResultType<()> {
     Ok(())
 }
 
-#[inline]
 fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> ResultType<()> {
     let privacy_mode_id_2 = *PRIVACY_MODE_CONN_ID.lock().unwrap();
     if privacy_mode_id != privacy_mode_id_2 {
         if privacy_mode_id_2 != 0 {
             let msg_out = crate::common::make_privacy_mode_msg(
-                back_notification::PrivacyModeState::OnByOther,
+                back_notification::PrivacyModeState::PrvOnByOther,
             );
             sp.send_to_others(msg_out, privacy_mode_id_2);
         }
@@ -547,10 +593,11 @@ fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> Resu
 }
 
 #[inline]
-fn create_msg(vp9s: Vec<VP9>) -> Message {
+#[cfg(any(target_os = "android", target_os = "ios"))]
+fn create_msg(vp9s: Vec<EncodedVideoFrame>) -> Message {
     let mut msg_out = Message::new();
     let mut vf = VideoFrame::new();
-    vf.set_vp9s(VP9s {
+    vf.set_vp9s(EncodedVideoFrames {
         frames: vp9s.into(),
         ..Default::default()
     });
@@ -559,22 +606,12 @@ fn create_msg(vp9s: Vec<VP9>) -> Message {
     msg_out
 }
 
-#[inline]
-fn create_frame(frame: &EncodeFrame) -> VP9 {
-    VP9 {
-        data: frame.data.to_vec(),
-        key: frame.key,
-        pts: frame.pts,
-        ..Default::default()
-    }
-}
-
 #[inline]
 fn handle_one_frame(
     sp: &GenericService,
     frame: &[u8],
     ms: i64,
-    vpx: &mut Encoder,
+    encoder: &mut Encoder,
 ) -> ResultType<HashSet<i32>> {
     sp.snapshot(|sps| {
         // so that new sub and old sub share the same encoder after switch
@@ -585,20 +622,8 @@ fn handle_one_frame(
     })?;
 
     let mut send_conn_ids: HashSet<i32> = Default::default();
-    let mut frames = Vec::new();
-    for ref frame in vpx
-        .encode(ms, frame, STRIDE_ALIGN)
-        .with_context(|| "Failed to encode")?
-    {
-        frames.push(create_frame(frame));
-    }
-    for ref frame in vpx.flush().with_context(|| "Failed to flush")? {
-        frames.push(create_frame(frame));
-    }
-
-    // to-do: flush periodically, e.g. 1 second
-    if frames.len() > 0 {
-        send_conn_ids = sp.send_video_frame(create_msg(frames));
+    if let Ok(msg) = encoder.encode_to_message(frame, ms) {
+        send_conn_ids = sp.send_video_frame(msg);
     }
     Ok(send_conn_ids)
 }
@@ -618,8 +643,8 @@ pub fn handle_one_frame_encoded(
         Ok(())
     })?;
     let mut send_conn_ids: HashSet<i32> = Default::default();
-    let vp9_frame = VP9 {
-        data: frame.to_vec(),
+    let vp9_frame = EncodedVideoFrame {
+        data: frame.to_vec().into(),
         key: true,
         pts: ms,
         ..Default::default()
@@ -629,6 +654,17 @@ pub fn handle_one_frame_encoded(
 }
 
 fn get_display_num() -> usize {
+    #[cfg(target_os = "linux")]
+    {
+        if !scrap::is_x11() {
+            return if let Ok(n) = super::wayland::get_display_num() {
+                n
+            } else {
+                0
+            };
+        }
+    }
+
     if let Ok(d) = try_get_displays() {
         d.len()
     } else {
@@ -636,14 +672,10 @@ fn get_display_num() -> usize {
     }
 }
 
-pub fn get_displays() -> ResultType<(usize, Vec<DisplayInfo>)> {
-    // switch to primary display if long time (30 seconds) no users
-    if LAST_ACTIVE.lock().unwrap().elapsed().as_secs() >= 30 {
-        *CURRENT_DISPLAY.lock().unwrap() = usize::MAX;
-    }
+pub(super) fn get_displays_2(all: &Vec<Display>) -> (usize, Vec<DisplayInfo>) {
     let mut displays = Vec::new();
     let mut primary = 0;
-    for (i, d) in try_get_displays()?.iter().enumerate() {
+    for (i, d) in all.iter().enumerate() {
         if d.is_primary() {
             primary = i;
         }
@@ -661,12 +693,26 @@ pub fn get_displays() -> ResultType<(usize, Vec<DisplayInfo>)> {
     if *lock >= displays.len() {
         *lock = primary
     }
-    Ok((*lock, displays))
+    (*lock, displays)
 }
 
-pub fn switch_display(i: i32) {
+pub async fn get_displays() -> ResultType<(usize, Vec<DisplayInfo>)> {
+    #[cfg(target_os = "linux")]
+    {
+        if !scrap::is_x11() {
+            return super::wayland::get_displays().await;
+        }
+    }
+    // switch to primary display if long time (30 seconds) no users
+    if LAST_ACTIVE.lock().unwrap().elapsed().as_secs() >= 30 {
+        *CURRENT_DISPLAY.lock().unwrap() = usize::MAX;
+    }
+    Ok(get_displays_2(&try_get_displays()?))
+}
+
+pub async fn switch_display(i: i32) {
     let i = i as usize;
-    if let Ok((_, displays)) = get_displays() {
+    if let Ok((_, displays)) = get_displays().await {
         if i < displays.len() {
             *CURRENT_DISPLAY.lock().unwrap() = i;
         }
@@ -680,6 +726,16 @@ pub fn refresh() {
 }
 
 fn get_primary() -> usize {
+    #[cfg(target_os = "linux")]
+    {
+        if !scrap::is_x11() {
+            return match super::wayland::get_primary() {
+                Ok(n) => n,
+                Err(_) => 0,
+            };
+        }
+    }
+
     if let Ok(all) = try_get_displays() {
         for (i, d) in all.iter().enumerate() {
             if d.is_primary() {
@@ -690,8 +746,8 @@ fn get_primary() -> usize {
     0
 }
 
-pub fn switch_to_primary() {
-    switch_display(get_primary() as _);
+pub async fn switch_to_primary() {
+    switch_display(get_primary() as _).await;
 }
 
 #[cfg(not(windows))]
@@ -729,16 +785,15 @@ fn try_get_displays() -> ResultType<Vec<Display>> {
     Ok(displays)
 }
 
-fn get_current_display() -> ResultType<(usize, usize, Display)> {
+pub(super) fn get_current_display_2(mut all: Vec<Display>) -> ResultType<(usize, usize, Display)> {
     let mut current = *CURRENT_DISPLAY.lock().unwrap() as usize;
-    let mut displays = try_get_displays()?;
-    if displays.len() == 0 {
+    if all.len() == 0 {
         bail!("No displays");
     }
-    let n = displays.len();
+    let n = all.len();
     if current >= n {
         current = 0;
-        for (i, d) in displays.iter().enumerate() {
+        for (i, d) in all.iter().enumerate() {
             if d.is_primary() {
                 current = i;
                 break;
@@ -746,84 +801,9 @@ fn get_current_display() -> ResultType<(usize, usize, Display)> {
         }
         *CURRENT_DISPLAY.lock().unwrap() = current;
     }
-    return Ok((n, current, displays.remove(current)));
+    return Ok((n, current, all.remove(current)));
 }
 
-#[inline]
-fn update_latency(id: i32, latency: i64, latencies: &mut HashMap<i32, i64>) {
-    if latency <= 0 {
-        latencies.remove(&id);
-    } else {
-        latencies.insert(id, latency);
-    }
-}
-
-pub fn update_test_latency(id: i32, latency: i64) {
-    update_latency(id, latency, &mut *TEST_LATENCIES.lock().unwrap());
-}
-
-fn convert_quality(q: i32) -> i32 {
-    let q = {
-        if q == ImageQuality::Balanced.value() {
-            (100 * 2 / 3, 12)
-        } else if q == ImageQuality::Low.value() {
-            (100 / 2, 18)
-        } else if q == ImageQuality::Best.value() {
-            (100, 12)
-        } else {
-            let bitrate = q >> 8 & 0xFF;
-            let quantizer = q & 0xFF;
-            (bitrate * 2, (100 - quantizer) * 36 / 100)
-        }
-    };
-    if q.0 <= 0 {
-        0
-    } else {
-        q.0 << 8 | q.1
-    }
-}
-
-pub fn update_image_quality(id: i32, q: Option<i32>) {
-    match q {
-        Some(q) => {
-            let q = convert_quality(q);
-            if q > 0 {
-                IMAGE_QUALITIES.lock().unwrap().insert(id, q);
-            } else {
-                IMAGE_QUALITIES.lock().unwrap().remove(&id);
-            }
-        }
-        None => {
-            IMAGE_QUALITIES.lock().unwrap().remove(&id);
-        }
-    }
-}
-
-fn get_image_quality() -> i32 {
-    IMAGE_QUALITIES
-        .lock()
-        .unwrap()
-        .values()
-        .min()
-        .unwrap_or(&convert_quality(ImageQuality::Balanced.value()))
-        .clone()
-}
-
-#[inline]
-fn get_quality(w: usize, h: usize, q: i32) -> (u32, u32, u32, i32) {
-    // https://www.nvidia.com/en-us/geforce/guides/broadcasting-guide/
-    let bitrate = q >> 8 & 0xFF;
-    let quantizer = q & 0xFF;
-    let b = ((w * h) / 1000) as u32;
-
-    #[cfg(target_os = "android")]
-    {
-        // fix when andorid screen shrinks
-        let fix = Display::fix_quality() as u32;
-        log::debug!("Android screen, fix quality:{}", fix);
-        let b = b * fix;
-        return (bitrate as u32 * b / 100, quantizer as _, 56, 7);
-    }
-
-    (bitrate as u32 * b / 100, quantizer as _, 56, 7)
+fn get_current_display() -> ResultType<(usize, usize, Display)> {
+    get_current_display_2(try_get_displays()?)
 }

src/server/wayland.rs

@@ -0,0 +1,189 @@
+use super::*;
+use hbb_common::allow_err;
+use scrap::{Capturer, Display, Frame, TraitCapturer};
+use std::io::Result;
+
+lazy_static::lazy_static! {
+    static ref CAP_DISPLAY_INFO: RwLock<u64> = RwLock::new(0);
+}
+
+struct CapturerPtr(*mut Capturer);
+
+impl Clone for CapturerPtr {
+    fn clone(&self) -> Self {
+        Self(self.0)
+    }
+}
+
+impl TraitCapturer for CapturerPtr {
+    fn frame<'a>(&'a mut self, timeout: Duration) -> Result<Frame<'a>> {
+        unsafe { (*self.0).frame(timeout) }
+    }
+
+    fn set_use_yuv(&mut self, use_yuv: bool) {
+        unsafe {
+            (*self.0).set_use_yuv(use_yuv);
+        }
+    }
+}
+
+struct CapDisplayInfo {
+    rects: Vec<((i32, i32), usize, usize)>,
+    displays: Vec<DisplayInfo>,
+    num: usize,
+    primary: usize,
+    current: usize,
+    capturer: CapturerPtr,
+}
+
+async fn check_init() -> ResultType<()> {
+    if !scrap::is_x11() {
+        let mut minx = 0;
+        let mut maxx = 0;
+        let mut miny = 0;
+        let mut maxy = 0;
+
+        if *CAP_DISPLAY_INFO.read().unwrap() == 0 {
+            let mut lock = CAP_DISPLAY_INFO.write().unwrap();
+            if *lock == 0 {
+                let all = Display::all()?;
+                let num = all.len();
+                let (primary, displays) = super::video_service::get_displays_2(&all);
+
+                let mut rects: Vec<((i32, i32), usize, usize)> = Vec::new();
+                for d in &all {
+                    rects.push((d.origin(), d.width(), d.height()));
+                }
+
+                let (ndisplay, current, display) =
+                    super::video_service::get_current_display_2(all)?;
+                let (origin, width, height) = (display.origin(), display.width(), display.height());
+                log::debug!(
+                    "#displays={}, current={}, origin: {:?}, width={}, height={}, cpus={}/{}",
+                    ndisplay,
+                    current,
+                    &origin,
+                    width,
+                    height,
+                    num_cpus::get_physical(),
+                    num_cpus::get(),
+                );
+
+                minx = origin.0;
+                maxx = origin.0 + width as i32;
+                miny = origin.1;
+                maxy = origin.1 + height as i32;
+
+                let capturer = Box::into_raw(Box::new(
+                    Capturer::new(display, true).with_context(|| "Failed to create capturer")?,
+                ));
+                let capturer = CapturerPtr(capturer);
+                let cap_display_info = Box::into_raw(Box::new(CapDisplayInfo {
+                    rects,
+                    displays,
+                    num,
+                    primary,
+                    current,
+                    capturer,
+                }));
+                *lock = cap_display_info as _;
+            }
+        }
+
+        if minx != maxx && miny != maxy {
+            log::info!(
+                "send uinput resolution: ({}, {}), ({}, {})",
+                minx,
+                maxx,
+                miny,
+                maxy
+            );
+            allow_err!(input_service::set_uinput_resolution(minx, maxx, miny, maxy).await);
+            allow_err!(input_service::set_uinput().await);
+        }
+    }
+    Ok(())
+}
+
+pub fn clear() {
+    if scrap::is_x11() {
+        return;
+    }
+
+    let mut lock = CAP_DISPLAY_INFO.write().unwrap();
+    if *lock != 0 {
+        unsafe {
+            let cap_display_info = Box::from_raw(*lock as *mut CapDisplayInfo);
+            let _ = Box::from_raw(cap_display_info.capturer.0);
+        }
+        *lock = 0;
+    }
+}
+
+pub(super) async fn get_displays() -> ResultType<(usize, Vec<DisplayInfo>)> {
+    check_init().await?;
+    let addr = *CAP_DISPLAY_INFO.read().unwrap();
+    if addr != 0 {
+        let cap_display_info: *const CapDisplayInfo = addr as _;
+        unsafe {
+            let cap_display_info = &*cap_display_info;
+            let primary = cap_display_info.primary;
+            let displays = cap_display_info.displays.clone();
+            Ok((primary, displays))
+        }
+    } else {
+        bail!("Failed to get capturer display info");
+    }
+}
+
+pub(super) fn get_primary() -> ResultType<usize> {
+    let addr = *CAP_DISPLAY_INFO.read().unwrap();
+    if addr != 0 {
+        let cap_display_info: *const CapDisplayInfo = addr as _;
+        unsafe {
+            let cap_display_info = &*cap_display_info;
+            Ok(cap_display_info.primary)
+        }
+    } else {
+        bail!("Failed to get capturer display info");
+    }
+}
+
+pub(super) fn get_display_num() -> ResultType<usize> {
+    let addr = *CAP_DISPLAY_INFO.read().unwrap();
+    if addr != 0 {
+        let cap_display_info: *const CapDisplayInfo = addr as _;
+        unsafe {
+            let cap_display_info = &*cap_display_info;
+            Ok(cap_display_info.num)
+        }
+    } else {
+        bail!("Failed to get capturer display info");
+    }
+}
+
+pub(super) fn get_capturer() -> ResultType<super::video_service::CapturerInfo> {
+    if scrap::is_x11() {
+        bail!("Do not call this function if not wayland");
+    }
+    let addr = *CAP_DISPLAY_INFO.read().unwrap();
+    if addr != 0 {
+        let cap_display_info: *const CapDisplayInfo = addr as _;
+        unsafe {
+            let cap_display_info = &*cap_display_info;
+            let rect = cap_display_info.rects[cap_display_info.current];
+            Ok(super::video_service::CapturerInfo {
+                origin: rect.0,
+                width: rect.1,
+                height: rect.2,
+                ndisplay: cap_display_info.num,
+                current: cap_display_info.current,
+                privacy_mode_id: 0,
+                _captuerer_privacy_mode_id: 0,
+                capturer: Box::new(cap_display_info.capturer.clone()),
+            })
+        }
+    } else {
+        bail!("Failed to get capturer display info");
+    }
+}