weyne/rustdesk
1
0
Fork 0
mirror of https://github.com/weyne85/rustdesk.git synced 2025-10-29 17:00:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

codec set quality seperately and refactor network delay

Browse Source 
Signed-off-by: 21pages <pages21@163.com>
This commit is contained in:
21pages
2023-07-19 13:11:24 +08:00
parent 633c80d5e4
commit 2133f91089
9 changed files with 457 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/server/connection.rs
View File

@@ -196,6 +196,7 @@ pub struct Connection {
#[cfg(not(any(feature = "flatpak", feature = "appimage")))]
tx_desktop_ready: mpsc::Sender<()>,
closed: bool,
delay_response_instant: Instant,
}
impl ConnInner {
@@ -326,6 +327,7 @@ impl Connection {
#[cfg(not(any(feature = "flatpak", feature = "appimage")))]
tx_desktop_ready: _tx_desktop_ready,
closed: false,
delay_response_instant: Instant::now(),
};
if !conn.on_open(addr).await {
conn.closed = true;
@@ -593,18 +595,19 @@ impl Connection {
break;
}
let time = get_time();
let mut qos = video_service::VIDEO_QOS.lock().unwrap();
if time > 0 && conn.last_test_delay == 0 {
conn.last_test_delay = time;
let mut msg_out = Message::new();
let qos = video_service::VIDEO_QOS.lock().unwrap();
msg_out.set_test_delay(TestDelay{
time,
last_delay:conn.network_delay.unwrap_or_default(),
target_bitrate:qos.bitrate(),
target_bitrate: qos.bitrate(),
..Default::default()
});
conn.inner.send(msg_out.into());
}
qos.user_delay_response_elapsed(conn.inner.id(), conn.delay_response_instant.elapsed().as_millis());
}
}
}
@@ -1188,7 +1191,9 @@ impl Connection {
#[inline]
#[cfg(not(any(target_os = "android", target_os = "ios")))]
fn input_pointer(&self, msg: PointerDeviceEvent, conn_id: i32) {
self.tx_input.send(MessageInput::Pointer((msg, conn_id))).ok();
self.tx_input
.send(MessageInput::Pointer((msg, conn_id)))
.ok();
}
#[inline]
@@ -1553,6 +1558,7 @@ impl Connection {
.unwrap()
.user_network_delay(self.inner.id(), new_delay);
self.network_delay = Some(new_delay);
self.delay_response_instant = Instant::now();
}
} else if let Some(message::Union::SwitchSidesResponse(_s)) = msg.union {
#[cfg(feature = "flutter")]
@@ -1590,7 +1596,8 @@ impl Connection {
self.input_mouse(me, self.inner.id());
}
}
Some(message::Union::PointerDeviceEvent(pde)) => {
Some(message::Union::PointerDeviceEvent(pde)) =>
{
#[cfg(not(any(target_os = "android", target_os = "ios")))]
if self.peer_keyboard_enabled() {
MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst);

277
src/server/video_qos.rs
View File

@@ -1,4 +1,5 @@
use super::*;
use scrap::codec::Quality;
use std::time::Duration;
pub const FPS: u32 = 30;
pub const MIN_FPS: u32 = 1;
@@ -18,21 +19,29 @@ impl Percent for ImageQuality {
}
}
#[derive(Default, Debug)]
#[derive(Default, Debug, Copy, Clone)]
struct Delay {
state: DelayState,
staging_state: DelayState,
delay: u32,
counter: u32,
slower_than_old_state: Option<bool>,
}
#[derive(Default, Debug, Copy, Clone)]
struct UserData {
full_speed_fps: Option<u32>,
custom_fps: Option<u32>,
quality: Option<(i32, i64)>, // (quality, time)
delay: Option<(DelayState, u32, usize)>, // (state, ms, counter)
quality: Option<(i64, Quality)>, // (time, quality)
delay: Option<Delay>,
response_delayed: bool,
}
pub struct VideoQoS {
width: u32,
height: u32,
fps: u32,
target_bitrate: u32,
updated: bool,
quality: Quality,
users: HashMap<i32, UserData>,
bitrate_store: u32,
}
#[derive(PartialEq, Debug, Clone, Copy)]
@@ -43,6 +52,12 @@ enum DelayState {
Broken = 1000,
}
impl Default for DelayState {
fn default() -> Self {
DelayState::Normal
}
}
impl DelayState {
fn from_delay(delay: u32) -> Self {
if delay > DelayState::Broken as u32 {
@@ -61,24 +76,19 @@ impl Default for VideoQoS {
fn default() -> Self {
VideoQoS {
fps: FPS,
width: 0,
height: 0,
target_bitrate: 0,
updated: false,
quality: Default::default(),
users: Default::default(),
bitrate_store: 0,
}
}
}
impl VideoQoS {
pub fn set_size(&mut self, width: u32, height: u32) {
if width == 0 || height == 0 {
return;
}
self.width = width;
self.height = height;
}
#[derive(Debug, PartialEq, Eq)]
pub enum RefreshType {
SetImageQuality,
}
impl VideoQoS {
pub fn spf(&self) -> Duration {
Duration::from_secs_f32(1. / (self.fps as f32))
}
@@ -87,24 +97,23 @@ impl VideoQoS {
self.fps
}
pub fn bitrate(&self) -> u32 {
self.target_bitrate
pub fn store_bitrate(&mut self, bitrate: u32) {
self.bitrate_store = bitrate;
}
pub fn check_if_updated(&mut self) -> bool {
if self.updated {
self.updated = false;
return true;
}
return false;
pub fn bitrate(&self) -> u32 {
self.bitrate_store
}
pub fn quality(&self) -> Quality {
self.quality
}
pub fn abr_enabled() -> bool {
"N" != Config::get_option("enable-abr")
}
pub fn refresh(&mut self) {
let mut updated = false;
pub fn refresh(&mut self, typ: Option<RefreshType>) {
// fps
let user_fps = |u: &UserData| {
// full_speed_fps
@@ -117,13 +126,19 @@ impl VideoQoS {
}
// delay
if let Some(delay) = u.delay {
fps = match delay.0 {
fps = match delay.state {
DelayState::Normal => fps,
DelayState::LowDelay => fps,
DelayState::HighDelay => fps / 2,
DelayState::Broken => fps / 4,
}
}
// delay response
if u.response_delayed {
if fps > MIN_FPS + 2 {
fps = MIN_FPS + 2;
}
}
return fps;
};
let mut fps = self
@@ -136,64 +151,79 @@ impl VideoQoS {
if fps > MAX_FPS {
fps = MAX_FPS;
}
if fps != self.fps {
self.fps = fps;
updated = true;
}
self.fps = fps;
// quality
// latest image quality
let latest = self
let latest_quality = self
.users
.iter()
// .map(|(_, u)| u.quality)
.filter(|u| u.1.quality != None)
.max_by(|u1, u2| {
u1.1.quality
.unwrap_or_default()
.1
.cmp(&u2.1.quality.unwrap_or_default().1)
});
let quality = if let Some((id, data)) = latest {
let mut quality = data.quality.unwrap_or_default().0;
if quality <= 0 {
quality = ImageQuality::Balanced.as_percent() as _;
}
// use latest's delay for quality
if Self::abr_enabled() {
if let Some(Some((delay, _, _))) = self.users.get(id).map(|u| u.delay) {
quality = match delay {
DelayState::Normal => quality,
DelayState::LowDelay => std::cmp::min(quality, 50),
DelayState::HighDelay => std::cmp::min(quality, 25),
DelayState::Broken => 10,
};
.map(|(_, u)| u.quality)
.filter(|q| *q != None)
.max_by(|a, b| a.unwrap_or_default().0.cmp(&b.unwrap_or_default().0))
.unwrap_or_default()
.unwrap_or_default()
.1;
let mut quality = latest_quality;
// network delay
if Self::abr_enabled() && typ != Some(RefreshType::SetImageQuality) {
// max delay
let delay = self
.users
.iter()
.map(|u| u.1.delay)
.filter(|d| d.is_some())
.max_by(|a, b| {
(a.unwrap_or_default().state as u32).cmp(&(b.unwrap_or_default().state as u32))
});
let delay = delay.unwrap_or_default().unwrap_or_default().state;
if delay != DelayState::Normal {
match self.quality {
Quality::Best => {
quality = Quality::Balanced;
}
Quality::Balanced => {
quality = Quality::Low;
}
Quality::Low => {
quality = Quality::Low;
}
Quality::Custom(b) => match delay {
DelayState::LowDelay => {
quality =
Quality::Custom(if b >= 150 { 100 } else { std::cmp::min(50, b) });
}
DelayState::HighDelay => {
quality =
Quality::Custom(if b >= 100 { 50 } else { std::cmp::min(25, b) });
}
DelayState::Broken => {
quality =
Quality::Custom(if b >= 50 { 25 } else { std::cmp::min(10, b) });
}
DelayState::Normal => {}
},
}
} else {
match self.quality {
Quality::Low => {
if latest_quality == Quality::Best {
quality = Quality::Balanced;
}
}
Quality::Custom(current_b) => {
if let Quality::Custom(latest_b) = latest_quality {
if current_b < latest_b / 2 {
quality = Quality::Custom(latest_b / 2);
}
}
}
_ => {}
}
}
quality
} else {
ImageQuality::Balanced.as_percent() as _
};
// bitrate
#[allow(unused_mut)]
let mut base_bitrate = ((self.width * self.height) / 800) as u32;
if base_bitrate == 0 {
base_bitrate = 1920 * 1080 / 800;
}
#[cfg(target_os = "android")]
{
// fix when android screen shrinks
let fix = scrap::Display::fix_quality() as u32;
log::debug!("Android screen, fix quality:{}", fix);
base_bitrate = base_bitrate * fix;
}
let target_bitrate = base_bitrate * quality as u32 / 100;
if self.target_bitrate != target_bitrate {
self.target_bitrate = target_bitrate;
updated = true;
}
self.updated = updated;
self.quality = quality;
}
pub fn user_custom_fps(&mut self, id: i32, fps: u32) {
@@ -211,7 +241,7 @@ impl VideoQoS {
},
);
}
self.refresh();
self.refresh(None);
}
pub fn user_full_speed_fps(&mut self, id: i32, full_speed_fps: u32) {
@@ -226,23 +256,27 @@ impl VideoQoS {
},
);
}
self.refresh();
self.refresh(None);
}
pub fn user_image_quality(&mut self, id: i32, image_quality: i32) {
let convert_quality = |q: i32| -> i32 {
// https://github.com/rustdesk/rustdesk/blob/d716e2b40c38737f1aa3f16de0dec67394a6ac68/src/server/video_service.rs#L493
let convert_quality = |q: i32| {
if q == ImageQuality::Balanced.value() {
100 * 2 / 3
Quality::Balanced
} else if q == ImageQuality::Low.value() {
100 / 2
Quality::Low
} else if q == ImageQuality::Best.value() {
100
Quality::Best
} else {
(q >> 8 & 0xFF) * 2
let mut b = (q >> 8 & 0xFF) * 2;
b = std::cmp::max(b, 10);
b = std::cmp::min(b, 200);
Quality::Custom(b as u32)
}
};
let quality = Some((convert_quality(image_quality), hbb_common::get_time()));
let quality = Some((hbb_common::get_time(), convert_quality(image_quality)));
if let Some(user) = self.users.get_mut(&id) {
user.quality = quality;
} else {
@@ -254,43 +288,78 @@ impl VideoQoS {
},
);
}
self.refresh();
self.refresh(Some(RefreshType::SetImageQuality));
}
pub fn user_network_delay(&mut self, id: i32, delay: u32) {
let mut refresh = true;
let state = DelayState::from_delay(delay);
let debounce = 3;
if let Some(user) = self.users.get_mut(&id) {
if let Some((old_state, old_delay, mut counter)) = user.delay {
let new_delay = (delay + old_delay) / 2;
let new_state = DelayState::from_delay(new_delay);
if old_state == new_state {
counter += 1;
if let Some(d) = &mut user.delay {
d.delay = (delay + d.delay) / 2;
let new_state = DelayState::from_delay(d.delay);
let slower_than_old_state = new_state as i32 - d.staging_state as i32;
let slower_than_old_state = if slower_than_old_state > 0 {
Some(true)
} else if slower_than_old_state < 0 {
Some(false)
} else {
counter = 0;
None
};
if d.slower_than_old_state == slower_than_old_state {
let old_counter = d.counter;
d.counter += delay / 1000 + 1;
if old_counter < debounce && d.counter >= debounce {
d.counter = 0;
d.state = d.staging_state;
d.staging_state = new_state;
}
if d.counter % debounce == 0 {
self.refresh(None);
}
} else {
d.counter = 0;
d.staging_state = new_state;
d.slower_than_old_state = slower_than_old_state;
}
let debounce = 3;
refresh = counter == debounce;
user.delay = Some((new_state, new_delay, counter));
} else {
user.delay = Some((state, delay, 0));
user.delay = Some(Delay {
state: DelayState::Normal,
staging_state: state,
delay,
counter: 0,
slower_than_old_state: None,
});
}
} else {
self.users.insert(
id,
UserData {
delay: Some((state, delay, 0)),
delay: Some(Delay {
state: DelayState::Normal,
staging_state: state,
delay,
counter: 0,
slower_than_old_state: None,
}),
..Default::default()
},
);
}
if refresh {
self.refresh();
}
pub fn user_delay_response_elapsed(&mut self, id: i32, elapsed: u128) {
if let Some(user) = self.users.get_mut(&id) {
let old = user.response_delayed;
user.response_delayed = elapsed > 3000;
if old != user.response_delayed {
self.refresh(None);
}
}
}
pub fn on_connection_close(&mut self, id: i32) {
self.users.remove(&id);
self.refresh();
self.refresh(None);
}
}

32
src/server/video_service.rs
View File

@@ -514,13 +514,12 @@ fn run(sp: GenericService) -> ResultType<()> {
let mut c = get_capturer(true, last_portable_service_running)?;
let mut video_qos = VIDEO_QOS.lock().unwrap();
video_qos.set_size(c.width as _, c.height as _);
video_qos.refresh();
let mut spf = video_qos.spf();
let bitrate = video_qos.bitrate();
video_qos.refresh(None);
let mut spf;
let mut quality = video_qos.quality();
let abr = VideoQoS::abr_enabled();
drop(video_qos);
log::info!("init bitrate={}, abr enabled:{}", bitrate, abr);
log::info!("init quality={:?}, abr enabled:{}", quality, abr);
let encoder_cfg = match Encoder::negotiated_codec() {
scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => {
@@ -528,7 +527,7 @@ fn run(sp: GenericService) -> ResultType<()> {
name,
width: c.width,
height: c.height,
bitrate: bitrate as _,
quality,
})
}
name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => {
@@ -536,7 +535,7 @@ fn run(sp: GenericService) -> ResultType<()> {
width: c.width as _,
height: c.height as _,
timebase: [1, 1000], // Output timestamp precision
bitrate,
quality,
codec: if name == scrap::CodecName::VP8 {
VpxVideoCodecId::VP8
} else {
@@ -548,7 +547,7 @@ fn run(sp: GenericService) -> ResultType<()> {
scrap::CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig {
width: c.width as _,
height: c.height as _,
bitrate: bitrate as _,
quality,
}),
};
@@ -558,6 +557,7 @@ fn run(sp: GenericService) -> ResultType<()> {
Err(err) => bail!("Failed to create encoder: {}", err),
}
c.set_use_yuv(encoder.use_yuv());
VIDEO_QOS.lock().unwrap().store_bitrate(encoder.bitrate());
if *SWITCH.lock().unwrap() {
log::debug!("Broadcasting display switch");
@@ -611,16 +611,12 @@ fn run(sp: GenericService) -> ResultType<()> {
check_uac_switch(c.privacy_mode_id, c._capturer_privacy_mode_id)?;
let mut video_qos = VIDEO_QOS.lock().unwrap();
if video_qos.check_if_updated() {
log::debug!(
"qos is updated, target_bitrate:{}, fps:{}",
video_qos.bitrate(),
video_qos.fps()
);
if video_qos.bitrate() > 0 {
allow_err!(encoder.set_bitrate(video_qos.bitrate()));
}
spf = video_qos.spf();
spf = video_qos.spf();
if quality != video_qos.quality() {
log::debug!("quality: {:?} -> {:?}", quality, video_qos.quality());
quality = video_qos.quality();
allow_err!(encoder.set_quality(quality));
video_qos.store_bitrate(encoder.bitrate());
}
drop(video_qos);