codec set quality seperately and refactor network delay

Signed-off-by: 21pages <pages21@163.com>

libs/scrap/src/common/aom.rs

@@ -6,6 +6,7 @@
 
 include!(concat!(env!("OUT_DIR"), "/aom_ffi.rs"));
 
+use crate::codec::{base_bitrate, Quality};
 use crate::{codec::EncoderApi, EncodeFrame, STRIDE_ALIGN};
 use crate::{common::GoogleImage, generate_call_macro, generate_call_ptr_macro, Error, Result};
 use hbb_common::{
@@ -43,7 +44,7 @@ impl Default for aom_image_t {
 pub struct AomEncoderConfig {
     pub width: u32,
     pub height: u32,
-    pub bitrate: u32,
+    pub quality: Quality,
 }
 
 pub struct AomEncoder {
@@ -56,7 +57,6 @@ pub struct AomEncoder {
 mod webrtc {
     use super::*;
 
-    const kQpMin: u32 = 10;
     const kUsageProfile: u32 = AOM_USAGE_REALTIME;
     const kMinQindex: u32 = 145; // Min qindex threshold for QP scaling.
     const kMaxQindex: u32 = 205; // Max qindex threshold for QP scaling.
@@ -65,7 +65,8 @@ mod webrtc {
     const kRtpTicksPerSecond: i32 = 90000;
     const kMinimumFrameRate: f64 = 1.0;
 
-    const kQpMax: u32 = 25; // to-do: webrtc use dynamic value, no more than 63
+    pub const DEFAULT_Q_MAX: u32 = 56; // no more than 63
+    pub const DEFAULT_Q_MIN: u32 = 12; // no more than 63, litter than q_max
 
     fn number_of_threads(width: u32, height: u32, number_of_cores: usize) -> u32 {
         // Keep the number of encoder threads equal to the possible number of
@@ -78,7 +79,7 @@ mod webrtc {
         } else {
             // Use 2 threads for low res on ARM.
             #[cfg(any(target_arch = "arm", target_arch = "aarch64", target_os = "android"))]
-            if (width * height >= 320 * 180 && number_of_cores > 2) {
+            if width * height >= 320 * 180 && number_of_cores > 2 {
                 return 2;
             }
             // 1 thread less than VGA.
@@ -122,11 +123,23 @@ mod webrtc {
         c.g_threads = number_of_threads(cfg.width, cfg.height, num_cpus::get());
         c.g_timebase.num = 1;
         c.g_timebase.den = kRtpTicksPerSecond;
-        c.rc_target_bitrate = cfg.bitrate; // kilobits/sec.
         c.g_input_bit_depth = kBitDepth;
         c.kf_mode = aom_kf_mode::AOM_KF_DISABLED;
-        c.rc_min_quantizer = kQpMin;
-        c.rc_max_quantizer = kQpMax;
+        let (q_min, q_max, b) = AomEncoder::convert_quality(cfg.quality);
+        if q_min > 0 && q_min < q_max && q_max < 64 {
+            c.rc_min_quantizer = q_min;
+            c.rc_max_quantizer = q_max;
+        } else {
+            c.rc_min_quantizer = DEFAULT_Q_MIN;
+            c.rc_max_quantizer = DEFAULT_Q_MAX;
+        }
+        let base_bitrate = base_bitrate(cfg.width as _, cfg.height as _);
+        let bitrate = base_bitrate * b / 100;
+        if bitrate > 0 {
+            c.rc_target_bitrate = bitrate;
+        } else {
+            c.rc_target_bitrate = base_bitrate;
+        }
         c.rc_undershoot_pct = 50;
         c.rc_overshoot_pct = 50;
         c.rc_buf_initial_sz = 600;
@@ -259,11 +272,24 @@ impl EncoderApi for AomEncoder {
         true
     }
 
-    fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
-        let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() };
-        new_enc_cfg.rc_target_bitrate = bitrate;
-        call_aom!(aom_codec_enc_config_set(&mut self.ctx, &new_enc_cfg));
-        return Ok(());
+    fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
+        let mut c = unsafe { *self.ctx.config.enc.to_owned() };
+        let (q_min, q_max, b) = Self::convert_quality(quality);
+        if q_min > 0 && q_min < q_max && q_max < 64 {
+            c.rc_min_quantizer = q_min;
+            c.rc_max_quantizer = q_max;
+        }
+        let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
+        if bitrate > 0 {
+            c.rc_target_bitrate = bitrate;
+        }
+        call_aom!(aom_codec_enc_config_set(&mut self.ctx, &c));
+        Ok(())
+    }
+
+    fn bitrate(&self) -> u32 {
+        let c = unsafe { *self.ctx.config.enc.to_owned() };
+        c.rc_target_bitrate
     }
 }
 
@@ -319,6 +345,35 @@ impl AomEncoder {
             ..Default::default()
         }
     }
+
+    pub fn convert_quality(quality: Quality) -> (u32, u32, u32) {
+        // we can use lower bitrate for av1
+        match quality {
+            Quality::Best => (12, 25, 100),
+            Quality::Balanced => (12, 35, 100 * 2 / 3),
+            Quality::Low => (18, 45, 50),
+            Quality::Custom(b) => {
+                let (q_min, q_max) = Self::calc_q_values(b);
+                (q_min, q_max, b)
+            }
+        }
+    }
+
+    #[inline]
+    fn calc_q_values(b: u32) -> (u32, u32) {
+        let b = std::cmp::min(b, 200);
+        let q_min1: i32 = 24;
+        let q_min2 = 12;
+        let q_max1 = 45;
+        let q_max2 = 25;
+
+        let t = b as f32 / 200.0;
+
+        let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
+        let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
+
+        (q_min, q_max)
+    }
 }
 
 impl Drop for AomEncoder {

libs/scrap/src/common/codec.rs

@@ -42,7 +42,7 @@ pub struct HwEncoderConfig {
     pub name: String,
     pub width: usize,
     pub height: usize,
-    pub bitrate: i32,
+    pub quality: Quality,
 }
 
 #[derive(Debug, Clone)]
@@ -61,7 +61,9 @@ pub trait EncoderApi {
 
     fn use_yuv(&self) -> bool;
 
-    fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>;
+    fn set_quality(&mut self, quality: Quality) -> ResultType<()>;
+
+    fn bitrate(&self) -> u32;
 }
 
 pub struct Encoder {
@@ -471,3 +473,33 @@ fn enable_hwcodec_option() -> bool {
     }
     return true; // default is true
 }
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Quality {
+    Best,
+    Balanced,
+    Low,
+    Custom(u32),
+}
+
+impl Default for Quality {
+    fn default() -> Self {
+        Self::Balanced
+    }
+}
+
+pub fn base_bitrate(width: u32, height: u32) -> u32 {
+    #[allow(unused_mut)]
+    let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9
+    if base_bitrate == 0 {
+        base_bitrate = 1920 * 1080 / 1000;
+    }
+    #[cfg(target_os = "android")]
+    {
+        // fix when android screen shrinks
+        let fix = crate::Display::fix_quality() as u32;
+        log::debug!("Android screen, fix quality:{}", fix);
+        base_bitrate = base_bitrate * fix;
+    }
+    base_bitrate
+}

libs/scrap/src/common/hwcodec.rs

@@ -1,5 +1,5 @@
 use crate::{
-    codec::{EncoderApi, EncoderCfg},
+    codec::{base_bitrate, EncoderApi, EncoderCfg},
     hw, ImageFormat, ImageRgb, HW_STRIDE_ALIGN,
 };
 use hbb_common::{
@@ -34,6 +34,9 @@ pub struct HwEncoder {
     yuv: Vec<u8>,
     pub format: DataFormat,
     pub pixfmt: AVPixelFormat,
+    width: u32,
+    height: u32,
+    bitrate: u32, //kbs
 }
 
 impl EncoderApi for HwEncoder {
@@ -43,13 +46,19 @@ impl EncoderApi for HwEncoder {
     {
         match cfg {
             EncoderCfg::HW(config) => {
+                let b = Self::convert_quality(config.quality);
+                let base_bitrate = base_bitrate(config.width as _, config.height as _);
+                let mut bitrate = base_bitrate * b / 100;
+                if base_bitrate <= 0 {
+                    bitrate = base_bitrate;
+                }
                 let ctx = EncodeContext {
                     name: config.name.clone(),
                     width: config.width as _,
                     height: config.height as _,
                     pixfmt: DEFAULT_PIXFMT,
                     align: HW_STRIDE_ALIGN as _,
-                    bitrate: config.bitrate * 1000,
+                    bitrate: bitrate as i32 * 1000,
                     timebase: DEFAULT_TIME_BASE,
                     gop: DEFAULT_GOP,
                     quality: DEFAULT_HW_QUALITY,
@@ -70,6 +79,9 @@ impl EncoderApi for HwEncoder {
                         yuv: vec![],
                         format,
                         pixfmt: ctx.pixfmt,
+                        width: ctx.width as _,
+                        height: ctx.height as _,
+                        bitrate,
                     }),
                     Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
                 }
@@ -114,10 +126,19 @@ impl EncoderApi for HwEncoder {
         false
     }
 
-    fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
-        self.encoder.set_bitrate((bitrate * 1000) as _).ok();
+    fn set_quality(&mut self, quality: crate::codec::Quality) -> ResultType<()> {
+        let b = Self::convert_quality(quality);
+        let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
+        if bitrate > 0 {
+            self.encoder.set_bitrate((bitrate * 1000) as _).ok();
+            self.bitrate = bitrate;
+        }
         Ok(())
     }
+
+    fn bitrate(&self) -> u32 {
+        self.bitrate
+    }
 }
 
 impl HwEncoder {
@@ -159,6 +180,16 @@ impl HwEncoder {
             Err(_) => Ok(Vec::<EncodeFrame>::new()),
         }
     }
+
+    pub fn convert_quality(quality: crate::codec::Quality) -> u32 {
+        use crate::codec::Quality;
+        match quality {
+            Quality::Best => 150,
+            Quality::Balanced => 100,
+            Quality::Low => 50,
+            Quality::Custom(b) => b,
+        }
+    }
 }
 
 pub struct HwDecoder {

libs/scrap/src/common/vpxcodec.rs

@@ -7,7 +7,7 @@ use hbb_common::log;
 use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame};
 use hbb_common::ResultType;
 
-use crate::codec::EncoderApi;
+use crate::codec::{base_bitrate, EncoderApi, Quality};
 use crate::{GoogleImage, STRIDE_ALIGN};
 
 use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *};
@@ -19,6 +19,9 @@ use std::{ptr, slice};
 generate_call_macro!(call_vpx, false);
 generate_call_ptr_macro!(call_vpx_ptr);
 
+const DEFAULT_QP_MAX: u32 = 56; // no more than 63
+const DEFAULT_QP_MIN: u32 = 12; // no more than 63
+
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum VpxVideoCodecId {
     VP8,
@@ -64,8 +67,9 @@ impl EncoderApi for VpxEncoder {
                 c.g_h = config.height;
                 c.g_timebase.num = config.timebase[0];
                 c.g_timebase.den = config.timebase[1];
-                c.rc_target_bitrate = config.bitrate;
                 c.rc_undershoot_pct = 95;
+                // When the data buffer falls below this percentage of fullness, a dropped frame is indicated. Set the threshold to zero (0) to disable this feature.
+                // In dynamic scenes, low bitrate gets low fps while high bitrate gets high fps.
                 c.rc_dropframe_thresh = 25;
                 c.g_threads = if config.num_threads == 0 {
                     num_cpus::get() as _
@@ -79,6 +83,21 @@ impl EncoderApi for VpxEncoder {
                 // c.kf_min_dist = 0;
                 // c.kf_max_dist = 999999;
                 c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot
+                let (q_min, q_max, b) = Self::convert_quality(config.quality);
+                if q_min > 0 && q_min < q_max && q_max < 64 {
+                    c.rc_min_quantizer = q_min;
+                    c.rc_max_quantizer = q_max;
+                } else {
+                    c.rc_min_quantizer = DEFAULT_QP_MIN;
+                    c.rc_max_quantizer = DEFAULT_QP_MAX;
+                }
+                let base_bitrate = base_bitrate(config.width as _, config.height as _);
+                let bitrate = base_bitrate * b / 100;
+                if bitrate > 0 {
+                    c.rc_target_bitrate = bitrate;
+                } else {
+                    c.rc_target_bitrate = base_bitrate;
+                }
 
                 /*
                 The VPX encoder supports two-pass encoding for rate control purposes.
@@ -177,11 +196,24 @@ impl EncoderApi for VpxEncoder {
         true
     }
 
-    fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
-        let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() };
-        new_enc_cfg.rc_target_bitrate = bitrate;
-        call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &new_enc_cfg));
-        return Ok(());
+    fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
+        let mut c = unsafe { *self.ctx.config.enc.to_owned() };
+        let (q_min, q_max, b) = Self::convert_quality(quality);
+        if q_min > 0 && q_min < q_max && q_max < 64 {
+            c.rc_min_quantizer = q_min;
+            c.rc_max_quantizer = q_max;
+        }
+        let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
+        if bitrate > 0 {
+            c.rc_target_bitrate = bitrate;
+        }
+        call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &c));
+        Ok(())
+    }
+
+    fn bitrate(&self) -> u32 {
+        let c = unsafe { *self.ctx.config.enc.to_owned() };
+        c.rc_target_bitrate
     }
 }
 
@@ -258,6 +290,34 @@ impl VpxEncoder {
             ..Default::default()
         }
     }
+
+    fn convert_quality(quality: Quality) -> (u32, u32, u32) {
+        match quality {
+            Quality::Best => (6, 45, 150),
+            Quality::Balanced => (12, 56, 100 * 2 / 3),
+            Quality::Low => (18, 56, 50),
+            Quality::Custom(b) => {
+                let (q_min, q_max) = Self::calc_q_values(b);
+                (q_min, q_max, b)
+            }
+        }
+    }
+
+    #[inline]
+    fn calc_q_values(b: u32) -> (u32, u32) {
+        let b = std::cmp::min(b, 200);
+        let q_min1: i32 = 36;
+        let q_min2 = 12;
+        let q_max1 = 56;
+        let q_max2 = 37;
+
+        let t = b as f32 / 200.0;
+
+        let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
+        let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
+
+        (q_min, q_max)
+    }
 }
 
 impl Drop for VpxEncoder {
@@ -289,8 +349,8 @@ pub struct VpxEncoderConfig {
     pub height: c_uint,
     /// The timebase numerator and denominator (in seconds).
     pub timebase: [c_int; 2],
-    /// The target bitrate (in kilobits per second).
-    pub bitrate: c_uint,
+    /// The image quality
+    pub quality: Quality,
     /// The codec
     pub codec: VpxVideoCodecId,
     pub num_threads: u32,