diff --git a/src/client.rs b/src/client.rs index 41f290bf0..40630bb29 100644 --- a/src/client.rs +++ b/src/client.rs @@ -43,8 +43,7 @@ use hbb_common::{ rand, rendezvous_proto::*, socket_client, - sodiumoxide::base64, - sodiumoxide::crypto::sign, + sodiumoxide::{base64, crypto::sign}, tcp::FramedStream, timeout, tokio::time::Duration, @@ -2093,8 +2092,6 @@ pub type MediaSender = mpsc::Sender; struct VideoHandlerController { handler: VideoHandler, - count: u128, - duration: std::time::Duration, skip_beginning: u32, } @@ -2111,7 +2108,7 @@ pub fn start_video_audio_threads( MediaSender, MediaSender, Arc>>>, - Arc>>, + Arc>>, Arc>>, ) where @@ -2123,8 +2120,8 @@ where let video_queue_map_cloned = video_queue_map.clone(); let mut video_callback = video_callback; - let fps_map = Arc::new(RwLock::new(HashMap::new())); - let decode_fps_map = fps_map.clone(); + let fps = Arc::new(RwLock::new(None)); + let decode_fps_map = fps.clone(); let chroma = Arc::new(RwLock::new(None)); let chroma_cloned = chroma.clone(); let mut last_chroma = None; @@ -2134,9 +2131,8 @@ where sync_cpu_usage(); get_hwcodec_config(); let mut handler_controller_map = HashMap::new(); - // let mut count = Vec::new(); - // let mut duration = std::time::Duration::ZERO; - // let mut skip_beginning = Vec::new(); + let mut count = 0; + let mut duration = std::time::Duration::ZERO; loop { if let Ok(data) = video_receiver.recv() { match data { @@ -2169,8 +2165,6 @@ where display, VideoHandlerController { handler: VideoHandler::new(format, display), - count: 0, - duration: std::time::Duration::ZERO, skip_beginning: 0, }, ); @@ -2178,6 +2172,8 @@ where if let Some(handler_controller) = handler_controller_map.get_mut(&display) { let mut pixelbuffer = true; let mut tmp_chroma = None; + let format_changed = + handler_controller.handler.decoder.format() != format; match handler_controller.handler.handle_frame( vf, &mut pixelbuffer, @@ -2198,27 +2194,14 @@ where } // fps calculation - // The first frame will be very slow - if handler_controller.skip_beginning < 5 { - handler_controller.skip_beginning += 1; - continue; - } - - handler_controller.duration += start.elapsed(); - handler_controller.count += 1; - if handler_controller.count % 10 == 0 { - fps_map.write().unwrap().insert( - display, - (handler_controller.count * 1000 - / handler_controller.duration.as_millis()) - as usize, - ); - } - // Clear to get real-time fps - if handler_controller.count > 150 { - handler_controller.count = 0; - handler_controller.duration = Duration::ZERO; - } + fps_calculate( + handler_controller, + &fps, + format_changed, + start.elapsed(), + &mut count, + &mut duration, + ); } Err(e) => { // This is a simple workaround. @@ -2334,6 +2317,38 @@ pub fn start_audio_thread() -> MediaSender { audio_sender } +#[inline] +fn fps_calculate( + handler_controller: &mut VideoHandlerController, + fps: &Arc>>, + format_changed: bool, + elapsed: std::time::Duration, + count: &mut usize, + duration: &mut std::time::Duration, +) { + if format_changed { + *count = 0; + *duration = std::time::Duration::ZERO; + handler_controller.skip_beginning = 0; + } + // // The first frame will be very slow + if handler_controller.skip_beginning < 3 { + handler_controller.skip_beginning += 1; + return; + } + *duration += elapsed; + *count += 1; + let ms = duration.as_millis(); + if *count % 10 == 0 && ms > 0 { + *fps.write().unwrap() = Some((*count as usize) * 1000 / (ms as usize)); + } + // Clear to get real-time fps + if *count >= 30 { + *count = 0; + *duration = Duration::ZERO; + } +} + fn get_hwcodec_config() { // for sciter and unilink #[cfg(feature = "hwcodec")] diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs index 758ba8a13..33c39f359 100644 --- a/src/client/io_loop.rs +++ b/src/client/io_loop.rs @@ -71,8 +71,8 @@ pub struct Remote { frame_count_map: Arc>>, video_format: CodecFormat, elevation_requested: bool, - fps_control_map: HashMap, - decode_fps_map: Arc>>, + fps_control: FpsControl, + decode_fps: Arc>>, chroma: Arc>>, } @@ -85,7 +85,7 @@ impl Remote { receiver: mpsc::UnboundedReceiver, sender: mpsc::UnboundedSender, frame_count_map: Arc>>, - decode_fps: Arc>>, + decode_fps: Arc>>, chroma: Arc>>, ) -> Self { Self { @@ -110,8 +110,8 @@ impl Remote { stop_voice_call_sender: None, voice_call_request_timestamp: None, elevation_requested: false, - fps_control_map: Default::default(), - decode_fps_map: decode_fps, + fps_control: Default::default(), + decode_fps, chroma, } } @@ -971,69 +971,84 @@ impl Remote { if custom_fps < 5 || custom_fps > 120 { custom_fps = 30; } - let decode_fps_read = self.decode_fps_map.read().unwrap(); - for (display, decode_fps) in decode_fps_read.iter() { - let video_queue_map_read = self.video_queue_map.read().unwrap(); - let Some(video_queue) = video_queue_map_read.get(display) else { - continue; - }; - - if !self.fps_control_map.contains_key(display) { - self.fps_control_map.insert(*display, FpsControl::default()); + let ctl = &mut self.fps_control; + let len = self + .video_queue_map + .read() + .unwrap() + .iter() + .map(|v| v.1.len()) + .max() + .unwrap_or_default(); + let decode_fps = self.decode_fps.read().unwrap().clone(); + let Some(mut decode_fps) = decode_fps else { + return; + }; + if cfg!(feature = "flutter") { + let active_displays = ctl + .last_active_time + .iter() + .filter(|t| t.1.elapsed().as_secs() < 5) + .count(); + if active_displays > 1 { + decode_fps = decode_fps / active_displays; } - let Some(ctl) = self.fps_control_map.get_mut(display) else { - return; - }; + } + let mut limited_fps = if direct { + decode_fps * 9 / 10 // 30 got 27 + } else { + decode_fps * 4 / 5 // 30 got 24 + }; + if limited_fps > custom_fps { + limited_fps = custom_fps; + } + let should_decrease = (len > 1 + && ctl.last_auto_fps.clone().unwrap_or(custom_fps as _) > limited_fps) + || len > std::cmp::max(1, limited_fps / 2); - let len = video_queue.len(); - let decode_fps = *decode_fps; - let mut limited_fps = if direct { - decode_fps * 9 / 10 // 30 got 27 - } else { - decode_fps * 4 / 5 // 30 got 24 - }; - if limited_fps > custom_fps { - limited_fps = custom_fps; - } - let should_decrease = len > 1 && ctl.last_auto_fps.unwrap_or(0) > limited_fps as i32; - - // increase judgement - if len <= 1 { + // increase judgement + if len <= 1 { + if ctl.idle_counter < usize::MAX { ctl.idle_counter += 1; - } else { - ctl.idle_counter = 0; } - let mut should_increase = false; - if let Some(last_auto_fps) = ctl.last_auto_fps { - // ever set - if last_auto_fps + 3 <= limited_fps as i32 && ctl.idle_counter > 3 { - // limited_fps is 5 larger than last set, and idle time is more than 3 seconds - should_increase = true; - } + } else { + ctl.idle_counter = 0; + } + let mut should_increase = false; + if let Some(last_auto_fps) = ctl.last_auto_fps.clone() { + // ever set + if last_auto_fps + 3 <= limited_fps && ctl.idle_counter > 3 { + // limited_fps is 3 larger than last set, and idle time is more than 3 seconds + should_increase = true; } - if ctl.last_auto_fps.is_none() || should_decrease || should_increase { - // limited_fps to ensure decoding is faster than encoding - let mut auto_fps = limited_fps as i32; - if auto_fps < 1 { - auto_fps = 1; - } - // send custom fps - let mut misc = Misc::new(); - misc.set_option(OptionMessage { - custom_fps: auto_fps, - ..Default::default() - }); - let mut msg = Message::new(); - msg.set_misc(misc); - self.sender.send(Data::Message(msg)).ok(); - ctl.last_queue_size = len; - ctl.last_auto_fps = Some(auto_fps); + } + if ctl.last_auto_fps.is_none() || should_decrease || should_increase { + // limited_fps to ensure decoding is faster than encoding + let mut auto_fps = limited_fps; + if should_decrease && limited_fps < len { + auto_fps = limited_fps / 2; } - // send refresh + if auto_fps < 1 { + auto_fps = 1; + } + let mut misc = Misc::new(); + misc.set_option(OptionMessage { + custom_fps: auto_fps as _, + ..Default::default() + }); + let mut msg = Message::new(); + msg.set_misc(misc); + self.sender.send(Data::Message(msg)).ok(); + log::info!("Set fps to {}", auto_fps); + ctl.last_queue_size = len; + ctl.last_auto_fps = Some(auto_fps); + } + // send refresh + for (display, video_queue) in self.video_queue_map.read().unwrap().iter() { let tolerable = std::cmp::min(decode_fps, video_queue.capacity() / 2); - if ctl.refresh_times < 10 // enough - && (len > tolerable - && (ctl.refresh_times == 0 || ctl.last_refresh_instant.elapsed().as_secs() > 10)) + if ctl.refresh_times < 20 // enough + && (len > tolerable + && (ctl.refresh_times == 0 || ctl.last_refresh_instant.elapsed().as_secs() > 10)) { // Refresh causes client set_display, left frames cause flickering. while let Some(_) = video_queue.pop() {} @@ -1086,6 +1101,9 @@ impl Remote { } self.video_sender.send(MediaData::VideoQueue(display)).ok(); } + self.fps_control + .last_active_time + .insert(display, Instant::now()); } Some(message::Union::Hash(hash)) => { self.handler @@ -1840,8 +1858,9 @@ struct FpsControl { last_queue_size: usize, refresh_times: usize, last_refresh_instant: Instant, - last_auto_fps: Option, + last_auto_fps: Option, idle_counter: usize, + last_active_time: HashMap, } impl Default for FpsControl { @@ -1850,8 +1869,9 @@ impl Default for FpsControl { last_queue_size: Default::default(), refresh_times: Default::default(), last_refresh_instant: Instant::now(), - last_auto_fps: None, + last_auto_fps: Default::default(), idle_counter: 0, + last_active_time: Default::default(), } } } diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 1c56b97d1..b60711a0c 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -1692,7 +1692,7 @@ pub async fn io_loop(handler: Session, round: u32) { let frame_count_map: Arc>> = Default::default(); let frame_count_map_cl = frame_count_map.clone(); let ui_handler = handler.ui_handler.clone(); - let (video_sender, audio_sender, video_queue_map, decode_fps_map, chroma) = + let (video_sender, audio_sender, video_queue_map, decode_fps, chroma) = start_video_audio_threads( handler.clone(), move |display: usize, @@ -1720,7 +1720,7 @@ pub async fn io_loop(handler: Session, round: u32) { receiver, sender, frame_count_map, - decode_fps_map, + decode_fps, chroma, ); remote.io_loop(&key, &token, round).await;