weyne/rustdesk
1
0
Fork 0
mirror of https://github.com/weyne85/rustdesk.git synced 2025-10-29 17:00:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

codec set quality seperately and refactor network delay

Browse Source 
Signed-off-by: 21pages <pages21@163.com>
This commit is contained in:
21pages
2023-07-19 13:11:24 +08:00
parent 633c80d5e4
commit 2133f91089
9 changed files with 457 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
libs/scrap/examples/benchmark.rs
View File

@@ -2,7 +2,7 @@ use docopt::Docopt;
use hbb_common::env_logger::{init_from_env, Env, DEFAULT_FILTER_ENV};
use scrap::{
aom::{AomDecoder, AomDecoderConfig, AomEncoder, AomEncoderConfig},
codec::{EncoderApi, EncoderCfg},
codec::{EncoderApi, EncoderCfg, Quality as Q},
Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig,
VpxVideoCodecId::{self, *},
STRIDE_ALIGN,
@@ -15,13 +15,14 @@ const USAGE: &'static str = "
Codec benchmark.
Usage:
benchmark [--count=COUNT] [--bitrate=KBS] [--hw-pixfmt=PIXFMT]
benchmark [--count=COUNT] [--quality=QUALITY] [--hw-pixfmt=PIXFMT]
benchmark (-h | --help)
Options:
-h --help Show this screen.
--count=COUNT Capture frame count [default: 100].
--bitrate=KBS Video bitrate in kilobits per second [default: 5000].
--quality=QUALITY Video quality [default: Balanced].
Valid values: Best, Balanced, Low.
--hw-pixfmt=PIXFMT Hardware codec pixfmt. [default: i420]
Valid values: i420, nv12.
";
@@ -29,7 +30,7 @@ Options:
#[derive(Debug, serde::Deserialize)]
struct Args {
flag_count: usize,
flag_bitrate: usize,
flag_quality: Quality,
flag_hw_pixfmt: Pixfmt,
}
@@ -39,20 +40,32 @@ enum Pixfmt {
NV12,
}
#[derive(Debug, serde::Deserialize)]
enum Quality {
Best,
Balanced,
Low,
}
fn main() {
init_from_env(Env::default().filter_or(DEFAULT_FILTER_ENV, "info"));
let args: Args = Docopt::new(USAGE)
.and_then(|d| d.deserialize())
.unwrap_or_else(|e| e.exit());
let bitrate_k = args.flag_bitrate;
let quality = args.flag_quality;
let yuv_count = args.flag_count;
let (yuvs, width, height) = capture_yuv(yuv_count);
println!(
"benchmark {}x{} bitrate:{}k hw_pixfmt:{:?}",
width, height, bitrate_k, args.flag_hw_pixfmt
"benchmark {}x{} quality:{:?}k hw_pixfmt:{:?}",
width, height, quality, args.flag_hw_pixfmt
);
[VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, bitrate_k, yuv_count));
test_av1(&yuvs, width, height, bitrate_k, yuv_count);
let quality = match quality {
Quality::Best => Q::Best,
Quality::Balanced => Q::Balanced,
Quality::Low => Q::Low,
};
[VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, quality, yuv_count));
test_av1(&yuvs, width, height, quality, yuv_count);
#[cfg(feature = "hwcodec")]
{
use hwcodec::AVPixelFormat;
@@ -61,7 +74,7 @@ fn main() {
Pixfmt::NV12 => AVPixelFormat::AV_PIX_FMT_NV12,
};
let yuvs = hw::vpx_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt);
hw::test(&yuvs, width, height, bitrate_k, yuv_count, hw_pixfmt);
hw::test(&yuvs, width, height, quality, yuv_count, hw_pixfmt);
}
}
@@ -95,14 +108,14 @@ fn test_vpx(
yuvs: &Vec<Vec<u8>>,
width: usize,
height: usize,
bitrate_k: usize,
quality: Q,
yuv_count: usize,
) {
let config = EncoderCfg::VPX(VpxEncoderConfig {
width: width as _,
height: height as _,
timebase: [1, 1000],
bitrate: bitrate_k as _,
quality,
codec: codec_id,
num_threads: (num_cpus::get() / 2) as _,
});
@@ -148,11 +161,11 @@ fn test_vpx(
);
}
fn test_av1(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, bitrate_k: usize, yuv_count: usize) {
fn test_av1(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, quality: Q, yuv_count: usize) {
let config = EncoderCfg::AOM(AomEncoderConfig {
width: width as _,
height: height as _,
bitrate: bitrate_k as _,
quality,
});
let mut encoder = AomEncoder::new(config).unwrap();
let start = Instant::now();
@@ -208,17 +221,18 @@ mod hw {
yuvs: &Vec<Vec<u8>>,
width: usize,
height: usize,
bitrate_k: usize,
quality: Q,
yuv_count: usize,
pixfmt: AVPixelFormat,
) {
let bitrate = scrap::hwcodec::HwEncoder::convert_quality(quality);
let ctx = EncodeContext {
name: String::from(""),
width: width as _,
height: height as _,
pixfmt,
align: 0,
bitrate: (bitrate_k * 1000) as _,
bitrate: bitrate as i32 * 1000,
timebase: [1, 30],
gop: 60,
quality: Quality_Default,

36
libs/scrap/examples/record-screen.rs
View File

@@ -13,7 +13,7 @@ use std::time::{Duration, Instant};
use std::{io, thread};
use docopt::Docopt;
use scrap::codec::{EncoderApi, EncoderCfg};
use scrap::codec::{EncoderApi, EncoderCfg, Quality as Q};
use webm::mux;
use webm::mux::Track;
@@ -24,17 +24,18 @@ const USAGE: &'static str = "
Simple WebM screen capture.
Usage:
record-screen <path> [--time=<s>] [--fps=<fps>] [--bv=<kbps>] [--ba=<kbps>] [--codec CODEC]
record-screen <path> [--time=<s>] [--fps=<fps>] [--quality=<quality>] [--ba=<kbps>] [--codec CODEC]
record-screen (-h | --help)
Options:
-h --help Show this screen.
--time=<s> Recording duration in seconds.
--fps=<fps> Frames per second [default: 30].
--bv=<kbps> Video bitrate in kilobits per second [default: 5000].
--ba=<kbps> Audio bitrate in kilobits per second [default: 96].
--codec CODEC Configure the codec used. [default: vp9]
Valid values: vp8, vp9.
-h --help Show this screen.
--time=<s> Recording duration in seconds.
--fps=<fps> Frames per second [default: 30].
--quality=<quality> Video quality [default: Balanced].
Valid values: Best, Balanced, Low.
--ba=<kbps> Audio bitrate in kilobits per second [default: 96].
--codec CODEC Configure the codec used. [default: vp9]
Valid values: vp8, vp9.
";
#[derive(Debug, serde::Deserialize)]
@@ -43,7 +44,14 @@ struct Args {
flag_codec: Codec,
flag_time: Option<u64>,
flag_fps: u64,
flag_bv: u32,
flag_quality: Quality,
}
#[derive(Debug, serde::Deserialize)]
enum Quality {
Best,
Balanced,
Low,
}
#[derive(Debug, serde::Deserialize)]
@@ -97,12 +105,16 @@ fn main() -> io::Result<()> {
let mut vt = webm.add_video_track(width, height, None, mux_codec);
// Setup the encoder.
let quality = match args.flag_quality {
Quality::Best => Q::Best,
Quality::Balanced => Q::Balanced,
Quality::Low => Q::Low,
};
let mut vpx = vpx_encode::VpxEncoder::new(EncoderCfg::VPX(vpx_encode::VpxEncoderConfig {
width,
height,
timebase: [1, 1000],
bitrate: args.flag_bv,
quality,
codec: vpx_codec,
num_threads: 0,
}))

79
libs/scrap/src/common/aom.rs
View File

@@ -6,6 +6,7 @@
include!(concat!(env!("OUT_DIR"), "/aom_ffi.rs"));
use crate::codec::{base_bitrate, Quality};
use crate::{codec::EncoderApi, EncodeFrame, STRIDE_ALIGN};
use crate::{common::GoogleImage, generate_call_macro, generate_call_ptr_macro, Error, Result};
use hbb_common::{
@@ -43,7 +44,7 @@ impl Default for aom_image_t {
pub struct AomEncoderConfig {
pub width: u32,
pub height: u32,
pub bitrate: u32,
pub quality: Quality,
}
pub struct AomEncoder {
@@ -56,7 +57,6 @@ pub struct AomEncoder {
mod webrtc {
use super::*;
const kQpMin: u32 = 10;
const kUsageProfile: u32 = AOM_USAGE_REALTIME;
const kMinQindex: u32 = 145; // Min qindex threshold for QP scaling.
const kMaxQindex: u32 = 205; // Max qindex threshold for QP scaling.
@@ -65,7 +65,8 @@ mod webrtc {
const kRtpTicksPerSecond: i32 = 90000;
const kMinimumFrameRate: f64 = 1.0;
const kQpMax: u32 = 25; // to-do: webrtc use dynamic value, no more than 63
pub const DEFAULT_Q_MAX: u32 = 56; // no more than 63
pub const DEFAULT_Q_MIN: u32 = 12; // no more than 63, litter than q_max
fn number_of_threads(width: u32, height: u32, number_of_cores: usize) -> u32 {
// Keep the number of encoder threads equal to the possible number of
@@ -78,7 +79,7 @@ mod webrtc {
} else {
// Use 2 threads for low res on ARM.
#[cfg(any(target_arch = "arm", target_arch = "aarch64", target_os = "android"))]
if (width * height >= 320 * 180 && number_of_cores > 2) {
if width * height >= 320 * 180 && number_of_cores > 2 {
return 2;
}
// 1 thread less than VGA.
@@ -122,11 +123,23 @@ mod webrtc {
c.g_threads = number_of_threads(cfg.width, cfg.height, num_cpus::get());
c.g_timebase.num = 1;
c.g_timebase.den = kRtpTicksPerSecond;
c.rc_target_bitrate = cfg.bitrate; // kilobits/sec.
c.g_input_bit_depth = kBitDepth;
c.kf_mode = aom_kf_mode::AOM_KF_DISABLED;
c.rc_min_quantizer = kQpMin;
c.rc_max_quantizer = kQpMax;
let (q_min, q_max, b) = AomEncoder::convert_quality(cfg.quality);
if q_min > 0 && q_min < q_max && q_max < 64 {
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
} else {
c.rc_min_quantizer = DEFAULT_Q_MIN;
c.rc_max_quantizer = DEFAULT_Q_MAX;
}
let base_bitrate = base_bitrate(cfg.width as _, cfg.height as _);
let bitrate = base_bitrate * b / 100;
if bitrate > 0 {
c.rc_target_bitrate = bitrate;
} else {
c.rc_target_bitrate = base_bitrate;
}
c.rc_undershoot_pct = 50;
c.rc_overshoot_pct = 50;
c.rc_buf_initial_sz = 600;
@@ -259,11 +272,24 @@ impl EncoderApi for AomEncoder {
true
}
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() };
new_enc_cfg.rc_target_bitrate = bitrate;
call_aom!(aom_codec_enc_config_set(&mut self.ctx, &new_enc_cfg));
return Ok(());
fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
let mut c = unsafe { *self.ctx.config.enc.to_owned() };
let (q_min, q_max, b) = Self::convert_quality(quality);
if q_min > 0 && q_min < q_max && q_max < 64 {
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
}
let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
if bitrate > 0 {
c.rc_target_bitrate = bitrate;
}
call_aom!(aom_codec_enc_config_set(&mut self.ctx, &c));
Ok(())
}
fn bitrate(&self) -> u32 {
let c = unsafe { *self.ctx.config.enc.to_owned() };
c.rc_target_bitrate
}
}
@@ -319,6 +345,35 @@ impl AomEncoder {
..Default::default()
}
}
pub fn convert_quality(quality: Quality) -> (u32, u32, u32) {
// we can use lower bitrate for av1
match quality {
Quality::Best => (12, 25, 100),
Quality::Balanced => (12, 35, 100 * 2 / 3),
Quality::Low => (18, 45, 50),
Quality::Custom(b) => {
let (q_min, q_max) = Self::calc_q_values(b);
(q_min, q_max, b)
}
}
}
#[inline]
fn calc_q_values(b: u32) -> (u32, u32) {
let b = std::cmp::min(b, 200);
let q_min1: i32 = 24;
let q_min2 = 12;
let q_max1 = 45;
let q_max2 = 25;
let t = b as f32 / 200.0;
let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
(q_min, q_max)
}
}
impl Drop for AomEncoder {

36
libs/scrap/src/common/codec.rs
View File

@@ -42,7 +42,7 @@ pub struct HwEncoderConfig {
pub name: String,
pub width: usize,
pub height: usize,
pub bitrate: i32,
pub quality: Quality,
}
#[derive(Debug, Clone)]
@@ -61,7 +61,9 @@ pub trait EncoderApi {
fn use_yuv(&self) -> bool;
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>;
fn set_quality(&mut self, quality: Quality) -> ResultType<()>;
fn bitrate(&self) -> u32;
}
pub struct Encoder {
@@ -471,3 +473,33 @@ fn enable_hwcodec_option() -> bool {
}
return true; // default is true
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Quality {
Best,
Balanced,
Low,
Custom(u32),
}
impl Default for Quality {
fn default() -> Self {
Self::Balanced
}
}
pub fn base_bitrate(width: u32, height: u32) -> u32 {
#[allow(unused_mut)]
let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9
if base_bitrate == 0 {
base_bitrate = 1920 * 1080 / 1000;
}
#[cfg(target_os = "android")]
{
// fix when android screen shrinks
let fix = crate::Display::fix_quality() as u32;
log::debug!("Android screen, fix quality:{}", fix);
base_bitrate = base_bitrate * fix;
}
base_bitrate
}

39
libs/scrap/src/common/hwcodec.rs
View File

@@ -1,5 +1,5 @@
use crate::{
codec::{EncoderApi, EncoderCfg},
codec::{base_bitrate, EncoderApi, EncoderCfg},
hw, ImageFormat, ImageRgb, HW_STRIDE_ALIGN,
};
use hbb_common::{
@@ -34,6 +34,9 @@ pub struct HwEncoder {
yuv: Vec<u8>,
pub format: DataFormat,
pub pixfmt: AVPixelFormat,
width: u32,
height: u32,
bitrate: u32, //kbs
}
impl EncoderApi for HwEncoder {
@@ -43,13 +46,19 @@ impl EncoderApi for HwEncoder {
{
match cfg {
EncoderCfg::HW(config) => {
let b = Self::convert_quality(config.quality);
let base_bitrate = base_bitrate(config.width as _, config.height as _);
let mut bitrate = base_bitrate * b / 100;
if base_bitrate <= 0 {
bitrate = base_bitrate;
}
let ctx = EncodeContext {
name: config.name.clone(),
width: config.width as _,
height: config.height as _,
pixfmt: DEFAULT_PIXFMT,
align: HW_STRIDE_ALIGN as _,
bitrate: config.bitrate * 1000,
bitrate: bitrate as i32 * 1000,
timebase: DEFAULT_TIME_BASE,
gop: DEFAULT_GOP,
quality: DEFAULT_HW_QUALITY,
@@ -70,6 +79,9 @@ impl EncoderApi for HwEncoder {
yuv: vec![],
format,
pixfmt: ctx.pixfmt,
width: ctx.width as _,
height: ctx.height as _,
bitrate,
}),
Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
}
@@ -114,10 +126,19 @@ impl EncoderApi for HwEncoder {
false
}
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
self.encoder.set_bitrate((bitrate * 1000) as _).ok();
fn set_quality(&mut self, quality: crate::codec::Quality) -> ResultType<()> {
let b = Self::convert_quality(quality);
let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
if bitrate > 0 {
self.encoder.set_bitrate((bitrate * 1000) as _).ok();
self.bitrate = bitrate;
}
Ok(())
}
fn bitrate(&self) -> u32 {
self.bitrate
}
}
impl HwEncoder {
@@ -159,6 +180,16 @@ impl HwEncoder {
Err(_) => Ok(Vec::<EncodeFrame>::new()),
}
}
pub fn convert_quality(quality: crate::codec::Quality) -> u32 {
use crate::codec::Quality;
match quality {
Quality::Best => 150,
Quality::Balanced => 100,
Quality::Low => 50,
Quality::Custom(b) => b,
}
}
}
pub struct HwDecoder {

78
libs/scrap/src/common/vpxcodec.rs
View File

@@ -7,7 +7,7 @@ use hbb_common::log;
use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame};
use hbb_common::ResultType;
use crate::codec::EncoderApi;
use crate::codec::{base_bitrate, EncoderApi, Quality};
use crate::{GoogleImage, STRIDE_ALIGN};
use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *};
@@ -19,6 +19,9 @@ use std::{ptr, slice};
generate_call_macro!(call_vpx, false);
generate_call_ptr_macro!(call_vpx_ptr);
const DEFAULT_QP_MAX: u32 = 56; // no more than 63
const DEFAULT_QP_MIN: u32 = 12; // no more than 63
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum VpxVideoCodecId {
VP8,
@@ -64,8 +67,9 @@ impl EncoderApi for VpxEncoder {
c.g_h = config.height;
c.g_timebase.num = config.timebase[0];
c.g_timebase.den = config.timebase[1];
c.rc_target_bitrate = config.bitrate;
c.rc_undershoot_pct = 95;
// When the data buffer falls below this percentage of fullness, a dropped frame is indicated. Set the threshold to zero (0) to disable this feature.
// In dynamic scenes, low bitrate gets low fps while high bitrate gets high fps.
c.rc_dropframe_thresh = 25;
c.g_threads = if config.num_threads == 0 {
num_cpus::get() as _
@@ -79,6 +83,21 @@ impl EncoderApi for VpxEncoder {
// c.kf_min_dist = 0;
// c.kf_max_dist = 999999;
c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot
let (q_min, q_max, b) = Self::convert_quality(config.quality);
if q_min > 0 && q_min < q_max && q_max < 64 {
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
} else {
c.rc_min_quantizer = DEFAULT_QP_MIN;
c.rc_max_quantizer = DEFAULT_QP_MAX;
}
let base_bitrate = base_bitrate(config.width as _, config.height as _);
let bitrate = base_bitrate * b / 100;
if bitrate > 0 {
c.rc_target_bitrate = bitrate;
} else {
c.rc_target_bitrate = base_bitrate;
}
/*
The VPX encoder supports two-pass encoding for rate control purposes.
@@ -177,11 +196,24 @@ impl EncoderApi for VpxEncoder {
true
}
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() };
new_enc_cfg.rc_target_bitrate = bitrate;
call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &new_enc_cfg));
return Ok(());
fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
let mut c = unsafe { *self.ctx.config.enc.to_owned() };
let (q_min, q_max, b) = Self::convert_quality(quality);
if q_min > 0 && q_min < q_max && q_max < 64 {
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
}
let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
if bitrate > 0 {
c.rc_target_bitrate = bitrate;
}
call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &c));
Ok(())
}
fn bitrate(&self) -> u32 {
let c = unsafe { *self.ctx.config.enc.to_owned() };
c.rc_target_bitrate
}
}
@@ -258,6 +290,34 @@ impl VpxEncoder {
..Default::default()
}
}
fn convert_quality(quality: Quality) -> (u32, u32, u32) {
match quality {
Quality::Best => (6, 45, 150),
Quality::Balanced => (12, 56, 100 * 2 / 3),
Quality::Low => (18, 56, 50),
Quality::Custom(b) => {
let (q_min, q_max) = Self::calc_q_values(b);
(q_min, q_max, b)
}
}
}
#[inline]
fn calc_q_values(b: u32) -> (u32, u32) {
let b = std::cmp::min(b, 200);
let q_min1: i32 = 36;
let q_min2 = 12;
let q_max1 = 56;
let q_max2 = 37;
let t = b as f32 / 200.0;
let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
(q_min, q_max)
}
}
impl Drop for VpxEncoder {
@@ -289,8 +349,8 @@ pub struct VpxEncoderConfig {
pub height: c_uint,
/// The timebase numerator and denominator (in seconds).
pub timebase: [c_int; 2],
/// The target bitrate (in kilobits per second).
pub bitrate: c_uint,
/// The image quality
pub quality: Quality,
/// The codec
pub codec: VpxVideoCodecId,
pub num_threads: u32,