weyne/rustdesk
1
0
Fork 0
mirror of https://github.com/weyne85/rustdesk.git synced 2025-10-29 17:00:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

flutter texture render, mid commit

Browse Source 
Signed-off-by: fufesou <shuanglongchen@yeah.net>
This commit is contained in:
fufesou
2023-02-21 23:46:13 +08:00
parent 5acedecf0c
commit 77c4a14845
11 changed files with 322 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
libs/scrap/src/common/codec.rs
View File

@@ -11,7 +11,7 @@ use crate::hwcodec::*;
use crate::mediacodec::{
MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT,
};
use crate::vpxcodec::*;
use crate::{vpxcodec::*, ImageFormat};
use hbb_common::{
anyhow::anyhow,
@@ -306,16 +306,17 @@ impl Decoder {
pub fn handle_video_frame(
&mut self,
frame: &video_frame::Union,
fmt: ImageFormat,
rgb: &mut Vec<u8>,
) -> ResultType<bool> {
match frame {
video_frame::Union::Vp9s(vp9s) => {
Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, rgb)
Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, fmt, rgb)
}
#[cfg(feature = "hwcodec")]
video_frame::Union::H264s(h264s) => {
if let Some(decoder) = &mut self.hw.h264 {
Decoder::handle_hw_video_frame(decoder, h264s, rgb, &mut self.i420)
Decoder::handle_hw_video_frame(decoder, h264s, fmt, rgb, &mut self.i420)
} else {
Err(anyhow!("don't support h264!"))
}
@@ -323,7 +324,7 @@ impl Decoder {
#[cfg(feature = "hwcodec")]
video_frame::Union::H265s(h265s) => {
if let Some(decoder) = &mut self.hw.h265 {
Decoder::handle_hw_video_frame(decoder, h265s, rgb, &mut self.i420)
Decoder::handle_hw_video_frame(decoder, h265s, fmt, rgb, &mut self.i420)
} else {
Err(anyhow!("don't support h265!"))
}
@@ -331,7 +332,7 @@ impl Decoder {
#[cfg(feature = "mediacodec")]
video_frame::Union::H264s(h264s) => {
if let Some(decoder) = &mut self.media_codec.h264 {
Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb)
Decoder::handle_mediacodec_video_frame(decoder, h264s, fmt, rgb)
} else {
Err(anyhow!("don't support h264!"))
}
@@ -339,7 +340,7 @@ impl Decoder {
#[cfg(feature = "mediacodec")]
video_frame::Union::H265s(h265s) => {
if let Some(decoder) = &mut self.media_codec.h265 {
Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb)
Decoder::handle_mediacodec_video_frame(decoder, h265s, fmt, rgb)
} else {
Err(anyhow!("don't support h265!"))
}
@@ -351,6 +352,7 @@ impl Decoder {
fn handle_vp9s_video_frame(
decoder: &mut VpxDecoder,
vp9s: &EncodedVideoFrames,
fmt: ImageFormat,
rgb: &mut Vec<u8>,
) -> ResultType<bool> {
let mut last_frame = Image::new();
@@ -367,7 +369,7 @@ impl Decoder {
if last_frame.is_null() {
Ok(false)
} else {
last_frame.rgb(1, true, rgb);
last_frame.to(fmt, 1, rgb);
Ok(true)
}
}
@@ -376,14 +378,15 @@ impl Decoder {
fn handle_hw_video_frame(
decoder: &mut HwDecoder,
frames: &EncodedVideoFrames,
rgb: &mut Vec<u8>,
fmt: ImageFormat,
raw: &mut Vec<u8>,
i420: &mut Vec<u8>,
) -> ResultType<bool> {
let mut ret = false;
for h264 in frames.frames.iter() {
for image in decoder.decode(&h264.data)? {
// TODO: just process the last frame
if image.bgra(rgb, i420).is_ok() {
if image.to_fmt(fmt, raw, i420).is_ok() {
ret = true;
}
}
@@ -395,11 +398,12 @@ impl Decoder {
fn handle_mediacodec_video_frame(
decoder: &mut MediaCodecDecoder,
frames: &EncodedVideoFrames,
rgb: &mut Vec<u8>,
fmt: ImageFormat,
raw: &mut Vec<u8>,
) -> ResultType<bool> {
let mut ret = false;
for h264 in frames.frames.iter() {
return decoder.decode(&h264.data, rgb);
return decoder.decode(&h264.data, fmt, raw);
}
return Ok(false);
}

158
libs/scrap/src/common/convert.rs
View File

@@ -103,6 +103,19 @@ extern "C" {
height: c_int,
) -> c_int;
pub fn I420ToABGR(
src_y: *const u8,
src_stride_y: c_int,
src_u: *const u8,
src_stride_u: c_int,
src_v: *const u8,
src_stride_v: c_int,
dst_rgba: *mut u8,
dst_stride_rgba: c_int,
width: c_int,
height: c_int,
) -> c_int;
pub fn NV12ToARGB(
src_y: *const u8,
src_stride_y: c_int,
@@ -246,6 +259,7 @@ pub unsafe fn nv12_to_i420(
#[cfg(feature = "hwcodec")]
pub mod hw {
use hbb_common::{anyhow::anyhow, ResultType};
use crate::ImageFormat;
#[cfg(target_os = "windows")]
use hwcodec::{ffmpeg::ffmpeg_linesize_offset_length, AVPixelFormat};
@@ -315,7 +329,8 @@ pub mod hw {
}
#[cfg(target_os = "windows")]
pub fn hw_nv12_to_bgra(
pub fn hw_nv12_to(
fmt: ImageFormat,
width: usize,
height: usize,
src_y: &[u8],
@@ -355,18 +370,39 @@ pub mod hw {
width as _,
height as _,
);
super::I420ToARGB(
i420_offset_y,
i420_stride_y,
i420_offset_u,
i420_stride_u,
i420_offset_v,
i420_stride_v,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
);
match fmt {
ImageFormat::ARGB => {
super::I420ToARGB(
i420_offset_y,
i420_stride_y,
i420_offset_u,
i420_stride_u,
i420_offset_v,
i420_stride_v,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
);
}
ImageFormat::ABGR => {
super::I420ToABGR(
i420_offset_y,
i420_stride_y,
i420_offset_u,
i420_stride_u,
i420_offset_v,
i420_stride_v,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
);
}
_ => {
return Err(anyhow!("unsupported image format"));
}
}
return Ok(());
};
}
@@ -374,7 +410,8 @@ pub mod hw {
}
#[cfg(not(target_os = "windows"))]
pub fn hw_nv12_to_bgra(
pub fn hw_nv12_to(
fmt: ImageFormat,
width: usize,
height: usize,
src_y: &[u8],
@@ -387,23 +424,46 @@ pub mod hw {
) -> ResultType<()> {
dst.resize(width * height * 4, 0);
unsafe {
match super::NV12ToARGB(
src_y.as_ptr(),
src_stride_y as _,
src_uv.as_ptr(),
src_stride_uv as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
) {
0 => Ok(()),
_ => Err(anyhow!("NV12ToARGB failed")),
match fmt {
ImageFormat::ARGB => {
match super::NV12ToARGB(
src_y.as_ptr(),
src_stride_y as _,
src_uv.as_ptr(),
src_stride_uv as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
) {
0 => Ok(()),
_ => Err(anyhow!("NV12ToARGB failed")),
}
}
ImageFormat::ABGR => {
match super::NV12ToABGR(
src_y.as_ptr(),
src_stride_y as _,
src_uv.as_ptr(),
src_stride_uv as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
) {
0 => Ok(()),
_ => Err(anyhow!("NV12ToABGR failed")),
}
}
_ => {
Err(anyhow!("unsupported image format"));
}
}
}
}
pub fn hw_i420_to_bgra(
pub fn hw_i420_to(
fmt: ImageFormat,
width: usize,
height: usize,
src_y: &[u8],
@@ -419,18 +479,38 @@ pub mod hw {
let src_v = src_v.as_ptr();
dst.resize(width * height * 4, 0);
unsafe {
super::I420ToARGB(
src_y,
src_stride_y as _,
src_u,
src_stride_u as _,
src_v,
src_stride_v as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
);
match fmt {
ImageFormat::ARGB => {
super::I420ToARGB(
src_y,
src_stride_y as _,
src_u,
src_stride_u as _,
src_v,
src_stride_v as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
);
}
ImageFormat::ABGR => {
super::I420ToABGR(
src_y,
src_stride_y as _,
src_u,
src_stride_u as _,
src_v,
src_stride_v as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
);
}
_ => {
}
}
};
}
}

22
libs/scrap/src/common/hwcodec.rs
View File

@@ -1,6 +1,6 @@
use crate::{
codec::{EncoderApi, EncoderCfg},
hw, HW_STRIDE_ALIGN,
hw, ImageFormat, HW_STRIDE_ALIGN,
};
use hbb_common::{
anyhow::{anyhow, Context},
@@ -236,22 +236,24 @@ pub struct HwDecoderImage<'a> {
}
impl HwDecoderImage<'_> {
pub fn bgra(&self, bgra: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
pub fn to_fmt(&self, fmt: ImageFormat, fmt_data: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
let frame = self.frame;
match frame.pixfmt {
AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to_bgra(
AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to(
fmt,
frame.width as _,
frame.height as _,
&frame.data[0],
&frame.data[1],
frame.linesize[0] as _,
frame.linesize[1] as _,
bgra,
fmt_data,
i420,
HW_STRIDE_ALIGN,
),
AVPixelFormat::AV_PIX_FMT_YUV420P => {
hw::hw_i420_to_bgra(
hw::hw_i420_to(
fmt,
frame.width as _,
frame.height as _,
&frame.data[0],
@@ -260,12 +262,20 @@ impl HwDecoderImage<'_> {
frame.linesize[0] as _,
frame.linesize[1] as _,
frame.linesize[2] as _,
bgra,
fmt_data,
);
return Ok(());
}
}
}
pub fn bgra(&self, bgra: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
self.to_fmt(ImageFormat::ARGB, bgra, i420)
}
pub fn rgba(&self, rgba: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
self.to_fmt(ImageFormat::ABGR, rgba, i420)
}
}
fn get_config(k: &str) -> ResultType<CodecInfos> {

52
libs/scrap/src/common/mediacodec.rs
View File

@@ -8,9 +8,10 @@ use std::{
time::Duration,
};
use crate::ImageFormat;
use crate::{
codec::{EncoderApi, EncoderCfg},
I420ToARGB,
I420ToABGR, I420ToARGB,
};
/// MediaCodec mime type name
@@ -50,7 +51,7 @@ impl MediaCodecDecoder {
MediaCodecDecoders { h264, h265 }
}
pub fn decode(&mut self, data: &[u8], rgb: &mut Vec<u8>) -> ResultType<bool> {
pub fn decode(&mut self, data: &[u8], fmt: ImageFormat, raw: &mut Vec<u8>) -> ResultType<bool> {
match self.dequeue_input_buffer(Duration::from_millis(10))? {
Some(mut input_buffer) => {
let mut buf = input_buffer.buffer_mut();
@@ -83,23 +84,44 @@ impl MediaCodecDecoder {
let bps = 4;
let u = buf.len() * 2 / 3;
let v = buf.len() * 5 / 6;
rgb.resize(h * w * bps, 0);
raw.resize(h * w * bps, 0);
let y_ptr = buf.as_ptr();
let u_ptr = buf[u..].as_ptr();
let v_ptr = buf[v..].as_ptr();
unsafe {
I420ToARGB(
y_ptr,
stride,
u_ptr,
stride / 2,
v_ptr,
stride / 2,
rgb.as_mut_ptr(),
(w * bps) as _,
w as _,
h as _,
);
match fmt {
ImageFormat::ARGB => {
I420ToARGB(
y_ptr,
stride,
u_ptr,
stride / 2,
v_ptr,
stride / 2,
raw.as_mut_ptr(),
(w * bps) as _,
w as _,
h as _,
);
}
ImageFormat::ARGB => {
I420ToABGR(
y_ptr,
stride,
u_ptr,
stride / 2,
v_ptr,
stride / 2,
raw.as_mut_ptr(),
(w * bps) as _,
w as _,
h as _,
);
}
_ => {
bail!("Unsupported image format");
}
}
}
self.release_output_buffer(output_buffer, false)?;
Ok(true)

7
libs/scrap/src/common/mod.rs
View File

@@ -43,6 +43,13 @@ pub const HW_STRIDE_ALIGN: usize = 0; // recommended by av_frame_get_buffer
pub mod record;
mod vpx;
#[derive(Copy, Clone)]
pub enum ImageFormat {
Raw,
ABGR,
ARGB,
}
#[inline]
pub fn would_block_if_equal(old: &mut Vec<u8>, b: &[u8]) -> std::io::Result<()> {
// does this really help?

82
libs/scrap/src/common/vpxcodec.rs
View File

@@ -6,8 +6,8 @@ use hbb_common::anyhow::{anyhow, Context};
use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame};
use hbb_common::{get_time, ResultType};
use crate::codec::EncoderApi;
use crate::STRIDE_ALIGN;
use crate::{codec::EncoderApi, ImageFormat};
use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *};
use hbb_common::bytes::Bytes;
@@ -417,7 +417,7 @@ impl VpxDecoder {
Ok(Self { ctx })
}
pub fn decode2rgb(&mut self, data: &[u8], rgba: bool) -> Result<Vec<u8>> {
pub fn decode2rgb(&mut self, data: &[u8], fmt: ImageFormat) -> Result<Vec<u8>> {
let mut img = Image::new();
for frame in self.decode(data)? {
drop(img);
@@ -431,7 +431,7 @@ impl VpxDecoder {
Ok(Vec::new())
} else {
let mut out = Default::default();
img.rgb(1, rgba, &mut out);
img.to(fmt, 1, &mut out);
Ok(out)
}
}
@@ -539,40 +539,60 @@ impl Image {
self.inner().stride[iplane]
}
pub fn rgb(&self, stride_align: usize, rgba: bool, dst: &mut Vec<u8>) {
pub fn to(&self, fmt: ImageFormat, stride_align: usize, dst: &mut Vec<u8>) {
let h = self.height();
let mut w = self.width();
let bps = if rgba { 4 } else { 3 };
let bps = match fmt {
ImageFormat::Raw => 3,
ImageFormat::ARGB | ImageFormat::ABGR => 4,
};
w = (w + stride_align - 1) & !(stride_align - 1);
dst.resize(h * w * bps, 0);
let img = self.inner();
unsafe {
if rgba {
super::I420ToARGB(
img.planes[0],
img.stride[0],
img.planes[1],
img.stride[1],
img.planes[2],
img.stride[2],
dst.as_mut_ptr(),
(w * bps) as _,
self.width() as _,
self.height() as _,
);
} else {
super::I420ToRAW(
img.planes[0],
img.stride[0],
img.planes[1],
img.stride[1],
img.planes[2],
img.stride[2],
dst.as_mut_ptr(),
(w * bps) as _,
self.width() as _,
self.height() as _,
);
match fmt {
ImageFormat::Raw => {
super::I420ToRAW(
img.planes[0],
img.stride[0],
img.planes[1],
img.stride[1],
img.planes[2],
img.stride[2],
dst.as_mut_ptr(),
(w * bps) as _,
self.width() as _,
self.height() as _,
);
}
ImageFormat::ARGB => {
super::I420ToARGB(
img.planes[0],
img.stride[0],
img.planes[1],
img.stride[1],
img.planes[2],
img.stride[2],
dst.as_mut_ptr(),
(w * bps) as _,
self.width() as _,
self.height() as _,
);
}
ImageFormat::ABGR => {
super::I420ToABGR(
img.planes[0],
img.stride[0],
img.planes[1],
img.stride[1],
img.planes[2],
img.stride[2],
dst.as_mut_ptr(),
(w * bps) as _,
self.width() as _,
self.height() as _,
);
}
}
}
}