weyne/rustdesk
1
0
Fork 0
mirror of https://github.com/weyne85/rustdesk.git synced 2025-10-29 17:00:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

add ffmpeg mediacodec h264/h265 encode (#8028)

Browse Source 
* Check available when app start from kotlin via get codec info
* For latency free, repeat encode 10 frame at most when capture return WouldBlock
* For changing quality, kotlin support but jni doesn't support, rerun video service when quality is manualy
  changed
* 3 or 6 times bitrate for mediacodec because its quality is poor

Signed-off-by: 21pages <pages21@163.com>
This commit is contained in:
21pages
2024-05-13 12:39:04 +08:00
committed by GitHub
parent 4c99b8c70e
commit a7499c2de8
13 changed files with 380 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
libs/scrap/examples/benchmark.rs
View File

@@ -272,6 +272,7 @@ mod hw {
let mut encoder = HwRamEncoder::new(
EncoderCfg::HWRAM(HwRamEncoderConfig {
name: info.name.clone(),
mc_name: None,
width,
height,
quality,

46
libs/scrap/src/android/ffi.rs
View File

@@ -10,6 +10,7 @@ use jni::{
use jni::errors::{Error as JniError, Result as JniResult};
use lazy_static::lazy_static;
use serde::Deserialize;
use std::ops::Not;
use std::sync::atomic::{AtomicPtr, Ordering::SeqCst};
use std::sync::{Mutex, RwLock};
@@ -20,6 +21,7 @@ lazy_static! {
static ref VIDEO_RAW: Mutex<FrameRaw> = Mutex::new(FrameRaw::new("video", MAX_VIDEO_FRAME_TIMEOUT));
static ref AUDIO_RAW: Mutex<FrameRaw> = Mutex::new(FrameRaw::new("audio", MAX_AUDIO_FRAME_TIMEOUT));
static ref NDK_CONTEXT_INITED: Mutex<bool> = Default::default();
static ref MEDIA_CODEC_INFOS: RwLock<Option<MediaCodecInfos>> = RwLock::new(None);
}
const MAX_VIDEO_FRAME_TIMEOUT: Duration = Duration::from_millis(100);
@@ -154,6 +156,50 @@ pub extern "system" fn Java_ffi_FFI_init(env: JNIEnv, _class: JClass, ctx: JObje
}
}
#[derive(Debug, Deserialize, Clone)]
pub struct MediaCodecInfo {
pub name: String,
pub is_encoder: bool,
#[serde(default)]
pub hw: Option<bool>, // api 29+
pub mime_type: String,
pub surface: bool,
pub nv12: bool,
#[serde(default)]
pub low_latency: Option<bool>, // api 30+, decoder
pub min_bitrate: u32,
pub max_bitrate: u32,
pub min_width: usize,
pub max_width: usize,
pub min_height: usize,
pub max_height: usize,
}
#[derive(Debug, Deserialize, Clone)]
pub struct MediaCodecInfos {
pub version: usize,
pub codecs: Vec<MediaCodecInfo>,
}
#[no_mangle]
pub extern "system" fn Java_ffi_FFI_setCodecInfo(env: JNIEnv, _class: JClass, info: JString) {
let mut env = env;
if let Ok(info) = env.get_string(&info) {
let info: String = info.into();
if let Ok(infos) = serde_json::from_str::<MediaCodecInfos>(&info) {
*MEDIA_CODEC_INFOS.write().unwrap() = Some(infos);
}
}
}
pub fn get_codec_info() -> Option<MediaCodecInfos> {
MEDIA_CODEC_INFOS.read().unwrap().as_ref().cloned()
}
pub fn clear_codec_info() {
*MEDIA_CODEC_INFOS.write().unwrap() = None;
}
pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) -> JniResult<()> {
if let (Some(jvm), Some(ctx)) = (
JVM.read().unwrap().as_ref(),

8
libs/scrap/src/common/aom.rs
View File

@@ -296,6 +296,14 @@ impl EncoderApi for AomEncoder {
fn support_abr(&self) -> bool {
true
}
fn support_changing_quality(&self) -> bool {
true
}
fn latency_free(&self) -> bool {
true
}
}
impl AomEncoder {

27
libs/scrap/src/common/codec.rs
View File

@@ -70,6 +70,10 @@ pub trait EncoderApi {
fn bitrate(&self) -> u32;
fn support_abr(&self) -> bool;
fn support_changing_quality(&self) -> bool;
fn latency_free(&self) -> bool;
}
pub struct Encoder {
@@ -138,6 +142,9 @@ impl Encoder {
}),
Err(e) => {
log::error!("new hw encoder failed: {e:?}, clear config");
#[cfg(target_os = "android")]
crate::android::ffi::clear_codec_info();
#[cfg(not(target_os = "android"))]
hbb_common::config::HwCodecConfig::clear_ram();
Self::update(EncodingUpdate::Check);
*ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9;
@@ -346,7 +353,14 @@ impl Encoder {
EncoderCfg::AOM(_) => CodecFormat::AV1,
#[cfg(feature = "hwcodec")]
EncoderCfg::HWRAM(hw) => {
if hw.name.to_lowercase().contains("h264") {
let name = hw.name.to_lowercase();
if name.contains("vp8") {
CodecFormat::VP8
} else if name.contains("vp9") {
CodecFormat::VP9
} else if name.contains("av1") {
CodecFormat::AV1
} else if name.contains("h264") {
CodecFormat::H264
} else {
CodecFormat::H265
@@ -817,7 +831,7 @@ impl Decoder {
#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
pub fn enable_hwcodec_option() -> bool {
if cfg!(windows) || cfg!(target_os = "linux") || cfg!(feature = "mediacodec") {
if cfg!(windows) || cfg!(target_os = "linux") || cfg!(target_os = "android") {
if let Some(v) = Config2::get().options.get("enable-hwcodec") {
return v != "N";
}
@@ -847,6 +861,15 @@ impl Default for Quality {
}
}
impl Quality {
pub fn is_custom(&self) -> bool {
match self {
Quality::Custom(_) => true,
_ => false,
}
}
}
pub fn base_bitrate(width: u32, height: u32) -> u32 {
#[allow(unused_mut)]
let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9

145
libs/scrap/src/common/hwcodec.rs
View File

@@ -29,11 +29,15 @@ const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_NV12;
pub const DEFAULT_TIME_BASE: [i32; 2] = [1, 30];
const DEFAULT_GOP: i32 = i32::MAX;
const DEFAULT_HW_QUALITY: Quality = Quality_Default;
const DEFAULT_RC: RateControl = RC_DEFAULT;
#[cfg(target_os = "android")]
const DEFAULT_RC: RateControl = RC_VBR; // android cbr poor quality
#[cfg(not(target_os = "android"))]
const DEFAULT_RC: RateControl = RC_CBR;
#[derive(Debug, Clone)]
pub struct HwRamEncoderConfig {
pub name: String,
pub mc_name: Option<String>,
pub width: usize,
pub height: usize,
pub quality: Q,
@@ -57,20 +61,22 @@ impl EncoderApi for HwRamEncoder {
{
match cfg {
EncoderCfg::HWRAM(config) => {
let b = Self::convert_quality(config.quality);
let b = Self::convert_quality(&config.name, config.quality);
let base_bitrate = base_bitrate(config.width as _, config.height as _);
let mut bitrate = base_bitrate * b / 100;
if base_bitrate <= 0 {
bitrate = base_bitrate;
}
bitrate = Self::check_bitrate_range(&config.name, bitrate);
let gop = config.keyframe_interval.unwrap_or(DEFAULT_GOP as _) as i32;
let ctx = EncodeContext {
name: config.name.clone(),
mc_name: config.mc_name.clone(),
width: config.width as _,
height: config.height as _,
pixfmt: DEFAULT_PIXFMT,
align: HW_STRIDE_ALIGN as _,
bitrate: bitrate as i32 * 1000,
kbs: bitrate as i32,
timebase: DEFAULT_TIME_BASE,
gop,
quality: DEFAULT_HW_QUALITY,
@@ -166,10 +172,11 @@ impl EncoderApi for HwRamEncoder {
}
fn set_quality(&mut self, quality: crate::codec::Quality) -> ResultType<()> {
let b = Self::convert_quality(quality);
let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
let b = Self::convert_quality(&self.name, quality);
let mut bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
if bitrate > 0 {
self.encoder.set_bitrate((bitrate * 1000) as _).ok();
bitrate = Self::check_bitrate_range(&self.name, bitrate);
self.encoder.set_bitrate(bitrate as _).ok();
self.bitrate = bitrate;
}
Ok(())
@@ -180,7 +187,19 @@ impl EncoderApi for HwRamEncoder {
}
fn support_abr(&self) -> bool {
["qsv", "vaapi"].iter().all(|&x| !self.name.contains(x))
["qsv", "vaapi", "mediacodec"]
.iter()
.all(|&x| !self.name.contains(x))
}
fn support_changing_quality(&self) -> bool {
["vaapi", "mediacodec"]
.iter()
.all(|&x| !self.name.contains(x))
}
fn latency_free(&self) -> bool {
!self.name.contains("mediacodec")
}
}
@@ -217,14 +236,42 @@ impl HwRamEncoder {
}
}
pub fn convert_quality(quality: crate::codec::Quality) -> u32 {
pub fn convert_quality(name: &str, quality: crate::codec::Quality) -> u32 {
use crate::codec::Quality;
match quality {
let quality = match quality {
Quality::Best => 150,
Quality::Balanced => 100,
Quality::Low => 50,
Quality::Custom(b) => b,
};
let factor = if name.contains("mediacodec") {
if name.contains("h264") {
6
} else {
3
}
} else {
1
};
quality * factor
}
pub fn check_bitrate_range(name: &str, bitrate: u32) -> u32 {
#[cfg(target_os = "android")]
if name.contains("mediacodec") {
let info = crate::android::ffi::get_codec_info();
if let Some(info) = info {
if let Some(codec) = info.codecs.iter().find(|c| c.name == name && c.is_encoder) {
if bitrate > codec.max_bitrate {
return codec.max_bitrate;
}
if bitrate < codec.min_bitrate {
return codec.min_bitrate;
}
}
}
}
bitrate
}
}
@@ -285,7 +332,10 @@ impl HwRamDecoder {
match Decoder::new(ctx) {
Ok(decoder) => Ok(HwRamDecoder { decoder, info }),
Err(_) => {
HwCodecConfig::clear_ram();
#[cfg(target_os = "android")]
crate::android::ffi::clear_codec_info();
#[cfg(not(target_os = "android"))]
hbb_common::config::HwCodecConfig::clear_ram();
Err(anyhow!(format!("Failed to create decoder")))
}
}
@@ -363,20 +413,87 @@ struct Available {
}
fn get_config() -> ResultType<Available> {
match serde_json::from_str(&HwCodecConfig::load().ram) {
Ok(v) => Ok(v),
Err(e) => Err(anyhow!("Failed to get config:{e:?}")),
#[cfg(target_os = "android")]
{
let info = crate::android::ffi::get_codec_info();
struct T {
name_prefix: &'static str,
data_format: DataFormat,
}
let ts = vec![
T {
name_prefix: "h264",
data_format: DataFormat::H264,
},
T {
name_prefix: "hevc",
data_format: DataFormat::H265,
},
];
let mut e = vec![];
if let Some(info) = info {
ts.iter().for_each(|t| {
let codecs: Vec<_> = info
.codecs
.iter()
.filter(|c| {
c.is_encoder
&& c.mime_type.as_str() == get_mime_type(t.data_format)
&& c.nv12
})
.collect();
log::debug!("available {:?} encoders: {codecs:?}", t.data_format);
let mut best = None;
if let Some(c) = codecs.iter().find(|c| c.hw == Some(true)) {
best = Some(c.name.clone());
} else if let Some(c) = codecs.iter().find(|c| c.hw == None) {
best = Some(c.name.clone());
} else if let Some(c) = codecs.first() {
best = Some(c.name.clone());
}
if let Some(best) = best {
e.push(CodecInfo {
name: format!("{}_mediacodec", t.name_prefix),
mc_name: Some(best),
format: t.data_format,
hwdevice: hwcodec::ffmpeg::AVHWDeviceType::AV_HWDEVICE_TYPE_NONE,
priority: 0,
});
}
});
}
log::debug!("e: {e:?}");
Ok(Available { e, d: vec![] })
}
#[cfg(not(target_os = "android"))]
{
match serde_json::from_str(&HwCodecConfig::load().ram) {
Ok(v) => Ok(v),
Err(e) => Err(anyhow!("Failed to get config:{e:?}")),
}
}
}
#[cfg(target_os = "android")]
fn get_mime_type(codec: DataFormat) -> &'static str {
match codec {
DataFormat::VP8 => "video/x-vnd.on2.vp8",
DataFormat::VP9 => "video/x-vnd.on2.vp9",
DataFormat::AV1 => "video/av01",
DataFormat::H264 => "video/avc",
DataFormat::H265 => "video/hevc",
}
}
pub fn check_available_hwcodec() {
let ctx = EncodeContext {
name: String::from(""),
mc_name: None,
width: 1280,
height: 720,
pixfmt: DEFAULT_PIXFMT,
align: HW_STRIDE_ALIGN as _,
bitrate: 0,
kbs: 0,
timebase: DEFAULT_TIME_BASE,
gop: DEFAULT_GOP,
quality: DEFAULT_HW_QUALITY,

7
libs/scrap/src/common/vpxcodec.rs
View File

@@ -235,6 +235,13 @@ impl EncoderApi for VpxEncoder {
fn support_abr(&self) -> bool {
true
}
fn support_changing_quality(&self) -> bool {
true
}
fn latency_free(&self) -> bool {
true
}
}
impl VpxEncoder {

8
libs/scrap/src/common/vram.rs
View File

@@ -180,6 +180,14 @@ impl EncoderApi for VRamEncoder {
fn support_abr(&self) -> bool {
self.config.device.vendor_id != ADAPTER_VENDOR_INTEL as u32
}
fn support_changing_quality(&self) -> bool {
true
}
fn latency_free(&self) -> bool {
true
}
}
impl VRamEncoder {