weyne/rustdesk
1
0
Fork 0
mirror of https://github.com/weyne85/rustdesk.git synced 2025-10-29 17:00:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6275 from 21pages/opt_codec

Browse Source 
opt codec
This commit is contained in:
RustDesk
2023-11-03 15:06:46 +08:00
committed by GitHub
parent 429f44eb6a 0d01aa4eea
commit a2bc02b4c5
16 changed files with 266 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
libs/scrap/src/common/android.rs
View File

@@ -43,7 +43,7 @@ impl crate::TraitCapturer for Capturer {
unsafe {
std::ptr::copy_nonoverlapping(buf.as_ptr(), self.rgba.as_mut_ptr(), buf.len())
};
Ok(Frame::new(&self.rgba, self.height()))
Ok(Frame::new(&self.rgba, self.width(), self.height()))
} else {
return Err(io::ErrorKind::WouldBlock.into());
}
@@ -51,16 +51,23 @@ impl crate::TraitCapturer for Capturer {
}
pub struct Frame<'a> {
pub data: &'a [u8],
pub stride: Vec<usize>,
data: &'a [u8],
width: usize,
height: usize,
stride: Vec<usize>,
}
impl<'a> Frame<'a> {
pub fn new(data: &'a [u8], h: usize) -> Self {
let stride = data.len() / h;
let mut v = Vec::new();
v.push(stride);
Frame { data, stride: v }
pub fn new(data: &'a [u8], width: usize, height: usize) -> Self {
let stride0 = data.len() / height;
let mut stride = Vec::new();
stride.push(stride0);
Frame {
data,
width,
height,
stride,
}
}
}
@@ -69,6 +76,14 @@ impl<'a> crate::TraitFrame for Frame<'a> {
self.data
}
fn width(&self) -> usize {
self.width
}
fn height(&self) -> usize {
self.height
}
fn stride(&self) -> Vec<usize> {
self.stride.clone()
}

62
libs/scrap/src/common/aom.rs
View File

@@ -7,9 +7,9 @@
include!(concat!(env!("OUT_DIR"), "/aom_ffi.rs"));
use crate::codec::{base_bitrate, codec_thread_num, Quality};
use crate::Pixfmt;
use crate::{codec::EncoderApi, EncodeFrame, STRIDE_ALIGN};
use crate::{common::GoogleImage, generate_call_macro, generate_call_ptr_macro, Error, Result};
use crate::{EncodeYuvFormat, Pixfmt};
use hbb_common::{
anyhow::{anyhow, Context},
bytes::Bytes,
@@ -54,6 +54,7 @@ pub struct AomEncoder {
width: usize,
height: usize,
i444: bool,
yuvfmt: EncodeYuvFormat,
}
// https://webrtc.googlesource.com/src/+/refs/heads/main/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
@@ -241,6 +242,7 @@ impl EncoderApi for AomEncoder {
width: config.width as _,
height: config.height as _,
i444,
yuvfmt: Self::get_yuvfmt(config.width, config.height, i444),
})
}
_ => Err(anyhow!("encoder type mismatch")),
@@ -263,35 +265,7 @@ impl EncoderApi for AomEncoder {
}
fn yuvfmt(&self) -> crate::EncodeYuvFormat {
let mut img = Default::default();
let fmt = if self.i444 {
aom_img_fmt::AOM_IMG_FMT_I444
} else {
aom_img_fmt::AOM_IMG_FMT_I420
};
unsafe {
aom_img_wrap(
&mut img,
fmt,
self.width as _,
self.height as _,
crate::STRIDE_ALIGN as _,
0x1 as _,
);
}
let pixfmt = if self.i444 {
Pixfmt::I444
} else {
Pixfmt::I420
};
crate::EncodeYuvFormat {
pixfmt,
w: img.w as _,
h: img.h as _,
stride: img.stride.map(|s| s as usize).to_vec(),
u: img.planes[1] as usize - img.planes[0] as usize,
v: img.planes[2] as usize - img.planes[0] as usize,
}
self.yuvfmt.clone()
}
fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
@@ -400,6 +374,34 @@ impl AomEncoder {
(q_min, q_max)
}
fn get_yuvfmt(width: u32, height: u32, i444: bool) -> EncodeYuvFormat {
let mut img = Default::default();
let fmt = if i444 {
aom_img_fmt::AOM_IMG_FMT_I444
} else {
aom_img_fmt::AOM_IMG_FMT_I420
};
unsafe {
aom_img_wrap(
&mut img,
fmt,
width as _,
height as _,
crate::STRIDE_ALIGN as _,
0x1 as _,
);
}
let pixfmt = if i444 { Pixfmt::I444 } else { Pixfmt::I420 };
EncodeYuvFormat {
pixfmt,
w: img.w as _,
h: img.h as _,
stride: img.stride.map(|s| s as usize).to_vec(),
u: img.planes[1] as usize - img.planes[0] as usize,
v: img.planes[2] as usize - img.planes[0] as usize,
}
}
}
impl Drop for AomEncoder {

60
libs/scrap/src/common/convert.rs
View File

@@ -202,17 +202,31 @@ pub fn convert_to_yuv(
) -> ResultType<()> {
let src = captured.data();
let src_stride = captured.stride();
let captured_pixfmt = captured.pixfmt();
if captured_pixfmt == crate::Pixfmt::BGRA || captured_pixfmt == crate::Pixfmt::RGBA {
if src.len() < src_stride[0] * dst_fmt.h {
let src_pixfmt = captured.pixfmt();
let src_width = captured.width();
let src_height = captured.height();
if src_width > dst_fmt.w || src_height > dst_fmt.h {
bail!(
"src rect > dst rect: ({src_width}, {src_height}) > ({},{})",
dst_fmt.w,
dst_fmt.h
);
}
if src_pixfmt == crate::Pixfmt::BGRA || src_pixfmt == crate::Pixfmt::RGBA {
if src.len() < src_stride[0] * src_height {
bail!(
"length not enough: {} < {}",
"wrong src len, {} < {} * {}",
src.len(),
src_stride[0] * dst_fmt.h
src_stride[0],
src_height
);
}
}
match (captured_pixfmt, dst_fmt.pixfmt) {
let align = |x:usize| {
(x + 63) / 64 * 64
};
match (src_pixfmt, dst_fmt.pixfmt) {
(crate::Pixfmt::BGRA, crate::Pixfmt::I420) | (crate::Pixfmt::RGBA, crate::Pixfmt::I420) => {
let dst_stride_y = dst_fmt.stride[0];
let dst_stride_uv = dst_fmt.stride[1];
@@ -220,7 +234,7 @@ pub fn convert_to_yuv(
let dst_y = dst.as_mut_ptr();
let dst_u = dst[dst_fmt.u..].as_mut_ptr();
let dst_v = dst[dst_fmt.v..].as_mut_ptr();
let f = if captured_pixfmt == crate::Pixfmt::BGRA {
let f = if src_pixfmt == crate::Pixfmt::BGRA {
ARGBToI420
} else {
ABGRToI420
@@ -234,17 +248,20 @@ pub fn convert_to_yuv(
dst_stride_uv as _,
dst_v,
dst_stride_uv as _,
dst_fmt.w as _,
dst_fmt.h as _,
src_width as _,
src_height as _,
));
}
(crate::Pixfmt::BGRA, crate::Pixfmt::NV12) | (crate::Pixfmt::RGBA, crate::Pixfmt::NV12) => {
let dst_stride_y = dst_fmt.stride[0];
let dst_stride_uv = dst_fmt.stride[1];
dst.resize(dst_fmt.h * (dst_stride_y + dst_stride_uv / 2), 0);
dst.resize(
align(dst_fmt.h) * (align(dst_stride_y) + align(dst_stride_uv / 2)),
0,
);
let dst_y = dst.as_mut_ptr();
let dst_uv = dst[dst_fmt.u..].as_mut_ptr();
let f = if captured_pixfmt == crate::Pixfmt::BGRA {
let f = if src_pixfmt == crate::Pixfmt::BGRA {
ARGBToNV12
} else {
ABGRToNV12
@@ -256,19 +273,22 @@ pub fn convert_to_yuv(
dst_stride_y as _,
dst_uv,
dst_stride_uv as _,
dst_fmt.w as _,
dst_fmt.h as _,
src_width as _,
src_height as _,
));
}
(crate::Pixfmt::BGRA, crate::Pixfmt::I444) | (crate::Pixfmt::RGBA, crate::Pixfmt::I444) => {
let dst_stride_y = dst_fmt.stride[0];
let dst_stride_u = dst_fmt.stride[1];
let dst_stride_v = dst_fmt.stride[2];
dst.resize(dst_fmt.h * (dst_stride_y + dst_stride_u + dst_stride_v), 0);
dst.resize(
align(dst_fmt.h) * (align(dst_stride_y) + align(dst_stride_u) + align(dst_stride_v)),
0,
);
let dst_y = dst.as_mut_ptr();
let dst_u = dst[dst_fmt.u..].as_mut_ptr();
let dst_v = dst[dst_fmt.v..].as_mut_ptr();
let src = if captured_pixfmt == crate::Pixfmt::BGRA {
let src = if src_pixfmt == crate::Pixfmt::BGRA {
src
} else {
mid_data.resize(src.len(), 0);
@@ -277,8 +297,8 @@ pub fn convert_to_yuv(
src_stride[0] as _,
mid_data.as_mut_ptr(),
src_stride[0] as _,
dst_fmt.w as _,
dst_fmt.h as _,
src_width as _,
src_height as _,
));
mid_data
};
@@ -291,13 +311,13 @@ pub fn convert_to_yuv(
dst_stride_u as _,
dst_v,
dst_stride_v as _,
dst_fmt.w as _,
dst_fmt.h as _,
src_width as _,
src_height as _,
));
}
_ => {
bail!(
"convert not support, {captured_pixfmt:?} -> {:?}",
"convert not support, {src_pixfmt:?} -> {:?}",
dst_fmt.pixfmt
);
}

33
libs/scrap/src/common/dxgi.rs
View File

@@ -41,7 +41,7 @@ impl Capturer {
impl TraitCapturer for Capturer {
fn frame<'a>(&'a mut self, timeout: Duration) -> io::Result<Frame<'a>> {
match self.inner.frame(timeout.as_millis() as _) {
Ok(frame) => Ok(Frame::new(frame, self.height)),
Ok(frame) => Ok(Frame::new(frame, self.width, self.height)),
Err(ref error) if error.kind() == TimedOut => Err(WouldBlock.into()),
Err(error) => Err(error),
}
@@ -58,15 +58,22 @@ impl TraitCapturer for Capturer {
pub struct Frame<'a> {
data: &'a [u8],
width: usize,
height: usize,
stride: Vec<usize>,
}
impl<'a> Frame<'a> {
pub fn new(data: &'a [u8], h: usize) -> Self {
let stride = data.len() / h;
let mut v = Vec::new();
v.push(stride);
Frame { data, stride: v }
pub fn new(data: &'a [u8], width: usize, height: usize) -> Self {
let stride0 = data.len() / height;
let mut stride = Vec::new();
stride.push(stride0);
Frame {
data,
width,
height,
stride,
}
}
}
@@ -75,6 +82,14 @@ impl<'a> crate::TraitFrame for Frame<'a> {
self.data
}
fn width(&self) -> usize {
self.width
}
fn height(&self) -> usize {
self.height
}
fn stride(&self) -> Vec<usize> {
self.stride.clone()
}
@@ -167,7 +182,11 @@ impl CapturerMag {
impl TraitCapturer for CapturerMag {
fn frame<'a>(&'a mut self, _timeout_ms: Duration) -> io::Result<Frame<'a>> {
self.inner.frame(&mut self.data)?;
Ok(Frame::new(&self.data, self.inner.get_rect().2))
Ok(Frame::new(
&self.data,
self.inner.get_rect().1,
self.inner.get_rect().2,
))
}
fn is_gdi(&self) -> bool {

5
libs/scrap/src/common/mod.rs
View File

@@ -112,6 +112,10 @@ pub trait TraitCapturer {
pub trait TraitFrame {
fn data(&self) -> &[u8];
fn width(&self) -> usize;
fn height(&self) -> usize;
fn stride(&self) -> Vec<usize>;
fn pixfmt(&self) -> Pixfmt;
@@ -125,6 +129,7 @@ pub enum Pixfmt {
I444,
}
#[derive(Debug, Clone)]
pub struct EncodeYuvFormat {
pub pixfmt: Pixfmt,
pub w: usize,

26
libs/scrap/src/common/quartz.rs
View File

@@ -55,7 +55,12 @@ impl crate::TraitCapturer for Capturer {
Some(mut frame) => {
crate::would_block_if_equal(&mut self.saved_raw_data, frame.inner())?;
frame.surface_to_bgra(self.height());
Ok(Frame(frame, PhantomData))
Ok(Frame {
frame,
data: PhantomData,
width: self.width(),
height: self.height(),
})
}
None => Err(io::ErrorKind::WouldBlock.into()),
@@ -69,16 +74,29 @@ impl crate::TraitCapturer for Capturer {
}
}
pub struct Frame<'a>(pub quartz::Frame, PhantomData<&'a [u8]>);
pub struct Frame<'a> {
frame: quartz::Frame,
data: PhantomData<&'a [u8]>,
width: usize,
height: usize,
}
impl<'a> crate::TraitFrame for Frame<'a> {
fn data(&self) -> &[u8] {
&*self.0
&*self.frame
}
fn width(&self) -> usize {
self.width
}
fn height(&self) -> usize {
self.height
}
fn stride(&self) -> Vec<usize> {
let mut v = Vec::new();
v.push(self.0.stride());
v.push(self.frame.stride());
v
}

62
libs/scrap/src/common/vpxcodec.rs
View File

@@ -8,7 +8,7 @@ use hbb_common::message_proto::{Chroma, EncodedVideoFrame, EncodedVideoFrames, V
use hbb_common::ResultType;
use crate::codec::{base_bitrate, codec_thread_num, EncoderApi, Quality};
use crate::{GoogleImage, Pixfmt, STRIDE_ALIGN};
use crate::{EncodeYuvFormat, GoogleImage, Pixfmt, STRIDE_ALIGN};
use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *};
use crate::{generate_call_macro, generate_call_ptr_macro, Error, Result};
@@ -40,6 +40,7 @@ pub struct VpxEncoder {
height: usize,
id: VpxVideoCodecId,
i444: bool,
yuvfmt: EncodeYuvFormat,
}
pub struct VpxDecoder {
@@ -175,6 +176,7 @@ impl EncoderApi for VpxEncoder {
height: config.height as _,
id: config.codec,
i444,
yuvfmt: Self::get_yuvfmt(config.width, config.height, i444),
})
}
_ => Err(anyhow!("encoder type mismatch")),
@@ -202,35 +204,7 @@ impl EncoderApi for VpxEncoder {
}
fn yuvfmt(&self) -> crate::EncodeYuvFormat {
let mut img = Default::default();
let fmt = if self.i444 {
vpx_img_fmt::VPX_IMG_FMT_I444
} else {
vpx_img_fmt::VPX_IMG_FMT_I420
};
unsafe {
vpx_img_wrap(
&mut img,
fmt,
self.width as _,
self.height as _,
crate::STRIDE_ALIGN as _,
0x1 as _,
);
}
let pixfmt = if self.i444 {
Pixfmt::I444
} else {
Pixfmt::I420
};
crate::EncodeYuvFormat {
pixfmt,
w: img.w as _,
h: img.h as _,
stride: img.stride.map(|s| s as usize).to_vec(),
u: img.planes[1] as usize - img.planes[0] as usize,
v: img.planes[2] as usize - img.planes[0] as usize,
}
self.yuvfmt.clone()
}
fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
@@ -362,6 +336,34 @@ impl VpxEncoder {
(q_min, q_max)
}
fn get_yuvfmt(width: u32, height: u32, i444: bool) -> EncodeYuvFormat {
let mut img = Default::default();
let fmt = if i444 {
vpx_img_fmt::VPX_IMG_FMT_I444
} else {
vpx_img_fmt::VPX_IMG_FMT_I420
};
unsafe {
vpx_img_wrap(
&mut img,
fmt,
width as _,
height as _,
crate::STRIDE_ALIGN as _,
0x1 as _,
);
}
let pixfmt = if i444 { Pixfmt::I444 } else { Pixfmt::I420 };
EncodeYuvFormat {
pixfmt,
w: img.w as _,
h: img.h as _,
stride: img.stride.map(|s| s as usize).to_vec(),
u: img.planes[1] as usize - img.planes[0] as usize,
v: img.planes[2] as usize - img.planes[0] as usize,
}
}
}
impl Drop for VpxEncoder {

4
libs/scrap/src/common/wayland.rs
View File

@@ -62,8 +62,8 @@ impl Capturer {
impl TraitCapturer for Capturer {
fn frame<'a>(&'a mut self, timeout: Duration) -> io::Result<Frame<'a>> {
match self.1.capture(timeout.as_millis() as _).map_err(map_err)? {
PixelProvider::BGR0(_w, h, x) => Ok(Frame::new(x, crate::Pixfmt::BGRA, h)),
PixelProvider::RGB0(_w, h, x) => Ok(Frame::new(x, crate::Pixfmt::RGBA, h)),
PixelProvider::BGR0(w, h, x) => Ok(Frame::new(x, crate::Pixfmt::BGRA, w, h)),
PixelProvider::RGB0(w, h, x) => Ok(Frame::new(x, crate::Pixfmt::RGBA, w,h)),
PixelProvider::NONE => Err(std::io::ErrorKind::WouldBlock.into()),
_ => Err(map_err("Invalid data")),
}

38
libs/scrap/src/common/x11.rs
View File

@@ -1,4 +1,4 @@
use crate::{common::TraitCapturer, x11, TraitFrame, Pixfmt};
use crate::{common::TraitCapturer, x11, Pixfmt, TraitFrame};
use std::{io, time::Duration};
pub struct Capturer(x11::Capturer);
@@ -25,26 +25,42 @@ impl TraitCapturer for Capturer {
}
}
pub struct Frame<'a>{
pub struct Frame<'a> {
pub data: &'a [u8],
pub pixfmt:Pixfmt,
pub stride:Vec<usize>,
pub pixfmt: Pixfmt,
pub width: usize,
pub height: usize,
pub stride: Vec<usize>,
}
impl<'a> Frame<'a> {
pub fn new(data:&'a [u8], pixfmt:Pixfmt, h:usize) -> Self {
let stride = data.len() / h;
let mut v = Vec::new();
v.push(stride);
Self { data, pixfmt, stride: v }
impl<'a> Frame<'a> {
pub fn new(data: &'a [u8], pixfmt: Pixfmt, width: usize, height: usize) -> Self {
let stride0 = data.len() / height;
let mut stride = Vec::new();
stride.push(stride0);
Self {
data,
pixfmt,
width,
height,
stride,
}
}
}
impl<'a> TraitFrame for Frame<'a> {
impl<'a> TraitFrame for Frame<'a> {
fn data(&self) -> &[u8] {
self.data
}
fn width(&self) -> usize {
self.width
}
fn height(&self) -> usize {
self.height
}
fn stride(&self) -> Vec<usize> {
self.stride.clone()
}

2
libs/scrap/src/x11/capturer.rs
View File

@@ -102,7 +102,7 @@ impl Capturer {
let result = unsafe { slice::from_raw_parts(self.buffer, self.size) };
crate::would_block_if_equal(&mut self.saved_raw_data, result)?;
Ok(
Frame::new(result, crate::Pixfmt::BGRA, self.display.h())
Frame::new(result, crate::Pixfmt::BGRA, self.display.w(), self.display.h())
)
}
}