From 8919ea65e366d5dd0d4f12cd89cd0569263c90da Mon Sep 17 00:00:00 2001 From: 21pages Date: Thu, 30 May 2024 23:40:25 +0800 Subject: [PATCH] fix, mac hwcodec decoding align use `dst_align` (#8215) Signed-off-by: 21pages --- libs/scrap/src/common/convert.rs | 180 ------------------------------- libs/scrap/src/common/hwcodec.rs | 147 ++++++++++++++++--------- libs/scrap/src/common/mod.rs | 22 ++-- src/client.rs | 4 +- src/common.rs | 4 +- src/flutter.rs | 2 +- src/flutter_ffi.rs | 1 - 7 files changed, 114 insertions(+), 246 deletions(-) diff --git a/libs/scrap/src/common/convert.rs b/libs/scrap/src/common/convert.rs index c1b28b5d1..ba246c694 100644 --- a/libs/scrap/src/common/convert.rs +++ b/libs/scrap/src/common/convert.rs @@ -13,186 +13,6 @@ use hbb_common::{bail, log, ResultType}; generate_call_macro!(call_yuv, false); -#[cfg(feature = "hwcodec")] -pub mod hw { - use super::*; - use crate::ImageFormat; - #[cfg(target_os = "windows")] - use hwcodec::{ffmpeg::AVPixelFormat, ffmpeg_ram::ffmpeg_linesize_offset_length}; - - #[cfg(target_os = "windows")] - pub fn hw_nv12_to( - fmt: ImageFormat, - width: usize, - height: usize, - src_y: &[u8], - src_uv: &[u8], - src_stride_y: usize, - src_stride_uv: usize, - dst: &mut Vec, - i420: &mut Vec, - align: usize, - ) -> ResultType<()> { - let nv12_stride_y = src_stride_y; - let nv12_stride_uv = src_stride_uv; - if let Ok((linesize_i420, offset_i420, i420_len)) = - ffmpeg_linesize_offset_length(AVPixelFormat::AV_PIX_FMT_YUV420P, width, height, align) - { - dst.resize(width * height * 4, 0); - let i420_stride_y = linesize_i420[0]; - let i420_stride_u = linesize_i420[1]; - let i420_stride_v = linesize_i420[2]; - i420.resize(i420_len as _, 0); - - let i420_offset_y = unsafe { i420.as_ptr().add(0) as _ }; - let i420_offset_u = unsafe { i420.as_ptr().add(offset_i420[0] as _) as _ }; - let i420_offset_v = unsafe { i420.as_ptr().add(offset_i420[1] as _) as _ }; - call_yuv!(NV12ToI420( - src_y.as_ptr(), - nv12_stride_y as _, - src_uv.as_ptr(), - nv12_stride_uv as _, - i420_offset_y, - i420_stride_y, - i420_offset_u, - i420_stride_u, - i420_offset_v, - i420_stride_v, - width as _, - height as _, - )); - match fmt { - ImageFormat::ARGB => { - call_yuv!(I420ToARGB( - i420_offset_y, - i420_stride_y, - i420_offset_u, - i420_stride_u, - i420_offset_v, - i420_stride_v, - dst.as_mut_ptr(), - (width * 4) as _, - width as _, - height as _, - )); - } - ImageFormat::ABGR => { - call_yuv!(I420ToABGR( - i420_offset_y, - i420_stride_y, - i420_offset_u, - i420_stride_u, - i420_offset_v, - i420_stride_v, - dst.as_mut_ptr(), - (width * 4) as _, - width as _, - height as _, - )); - } - _ => { - bail!("unsupported image format"); - } - } - return Ok(()); - } - bail!("get linesize offset failed"); - } - - #[cfg(not(target_os = "windows"))] - pub fn hw_nv12_to( - fmt: ImageFormat, - width: usize, - height: usize, - src_y: &[u8], - src_uv: &[u8], - src_stride_y: usize, - src_stride_uv: usize, - dst: &mut Vec, - _i420: &mut Vec, - _align: usize, - ) -> ResultType<()> { - dst.resize(width * height * 4, 0); - match fmt { - ImageFormat::ARGB => { - call_yuv!(NV12ToARGB( - src_y.as_ptr(), - src_stride_y as _, - src_uv.as_ptr(), - src_stride_uv as _, - dst.as_mut_ptr(), - (width * 4) as _, - width as _, - height as _, - )); - } - ImageFormat::ABGR => { - call_yuv!(NV12ToABGR( - src_y.as_ptr(), - src_stride_y as _, - src_uv.as_ptr(), - src_stride_uv as _, - dst.as_mut_ptr(), - (width * 4) as _, - width as _, - height as _, - )); - } - _ => bail!("unsupported image format"), - } - Ok(()) - } - - pub fn hw_i420_to( - fmt: ImageFormat, - width: usize, - height: usize, - src_y: &[u8], - src_u: &[u8], - src_v: &[u8], - src_stride_y: usize, - src_stride_u: usize, - src_stride_v: usize, - dst: &mut Vec, - ) -> ResultType<()> { - let src_y = src_y.as_ptr(); - let src_u = src_u.as_ptr(); - let src_v = src_v.as_ptr(); - dst.resize(width * height * 4, 0); - match fmt { - ImageFormat::ARGB => { - call_yuv!(I420ToARGB( - src_y, - src_stride_y as _, - src_u, - src_stride_u as _, - src_v, - src_stride_v as _, - dst.as_mut_ptr(), - (width * 4) as _, - width as _, - height as _, - )); - } - ImageFormat::ABGR => { - call_yuv!(I420ToABGR( - src_y, - src_stride_y as _, - src_u, - src_stride_u as _, - src_v, - src_stride_v as _, - dst.as_mut_ptr(), - (width * 4) as _, - width as _, - height as _, - )); - } - _ => bail!("unsupported image format"), - }; - Ok(()) - } -} #[cfg(not(target_os = "ios"))] pub fn convert_to_yuv( captured: &PixelBuffer, diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index dcbe15692..8c50d12c7 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -2,7 +2,8 @@ use crate::{ codec::{ base_bitrate, codec_thread_num, enable_hwcodec_option, EncoderApi, EncoderCfg, Quality as Q, }, - hw, CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN, + convert::*, + CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN, }; use hbb_common::{ anyhow::{anyhow, bail, Context}, @@ -23,7 +24,7 @@ use hwcodec::{ ffmpeg_ram::{ decode::{DecodeContext, DecodeFrame, Decoder}, encode::{EncodeContext, EncodeFrame, Encoder}, - CodecInfo, + ffmpeg_linesize_offset_length, CodecInfo, }, }; @@ -32,6 +33,8 @@ pub const DEFAULT_TIME_BASE: [i32; 2] = [1, 30]; const DEFAULT_GOP: i32 = i32::MAX; const DEFAULT_HW_QUALITY: Quality = Quality_Default; +crate::generate_call_macro!(call_yuv, false); + #[derive(Debug, Clone)] pub struct HwRamEncoderConfig { pub name: String, @@ -237,9 +240,9 @@ impl HwRamEncoder { } } - fn rate_control(config: &HwRamEncoderConfig) -> RateControl { + fn rate_control(_config: &HwRamEncoderConfig) -> RateControl { #[cfg(target_os = "android")] - if config.name.contains("mediacodec") { + if _config.name.contains("mediacodec") { return RC_VBR; } RC_CBR @@ -262,15 +265,15 @@ impl HwRamEncoder { quality * factor } - pub fn check_bitrate_range(config: &HwRamEncoderConfig, bitrate: u32) -> u32 { + pub fn check_bitrate_range(_config: &HwRamEncoderConfig, bitrate: u32) -> u32 { #[cfg(target_os = "android")] - if config.name.contains("mediacodec") { + if _config.name.contains("mediacodec") { let info = crate::android::ffi::get_codec_info(); if let Some(info) = info { if let Some(codec) = info .codecs .iter() - .find(|c| Some(c.name.clone()) == config.mc_name && c.is_encoder) + .find(|c| Some(c.name.clone()) == _config.mc_name && c.is_encoder) { if codec.max_bitrate > codec.min_bitrate { if bitrate > codec.max_bitrate { @@ -368,54 +371,100 @@ impl HwRamDecoderImage<'_> { // rgb [in/out] fmt and stride must be set in ImageRgb pub fn to_fmt(&self, rgb: &mut ImageRgb, i420: &mut Vec) -> ResultType<()> { let frame = self.frame; - rgb.w = frame.width as _; - rgb.h = frame.height as _; - // take dst_stride into account when you convert - let dst_stride = rgb.stride(); + let width = frame.width; + let height = frame.height; + rgb.w = width as _; + rgb.h = height as _; + let dst_align = rgb.align(); + let bytes_per_row = (rgb.w * 4 + dst_align - 1) & !(dst_align - 1); + rgb.raw.resize(rgb.h * bytes_per_row, 0); match frame.pixfmt { - AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to( - rgb.fmt(), - frame.width as _, - frame.height as _, - &frame.data[0], - &frame.data[1], - frame.linesize[0] as _, - frame.linesize[1] as _, - &mut rgb.raw as _, - i420, - HW_STRIDE_ALIGN, - )?, + AVPixelFormat::AV_PIX_FMT_NV12 => { + // I420ToARGB is much faster than NV12ToARGB in tests on Windows + if cfg!(windows) { + let Ok((linesize_i420, offset_i420, len_i420)) = ffmpeg_linesize_offset_length( + AVPixelFormat::AV_PIX_FMT_YUV420P, + width as _, + height as _, + HW_STRIDE_ALIGN, + ) else { + bail!("failed to get i420 linesize, offset, length"); + }; + i420.resize(len_i420 as _, 0); + let i420_offset_y = unsafe { i420.as_ptr().add(0) as _ }; + let i420_offset_u = unsafe { i420.as_ptr().add(offset_i420[0] as _) as _ }; + let i420_offset_v = unsafe { i420.as_ptr().add(offset_i420[1] as _) as _ }; + call_yuv!(NV12ToI420( + frame.data[0].as_ptr(), + frame.linesize[0], + frame.data[1].as_ptr(), + frame.linesize[1], + i420_offset_y, + linesize_i420[0], + i420_offset_u, + linesize_i420[1], + i420_offset_v, + linesize_i420[2], + width, + height, + )); + let f = match rgb.fmt() { + ImageFormat::ARGB => I420ToARGB, + ImageFormat::ABGR => I420ToABGR, + _ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()), + }; + call_yuv!(f( + i420_offset_y, + linesize_i420[0], + i420_offset_u, + linesize_i420[1], + i420_offset_v, + linesize_i420[2], + rgb.raw.as_mut_ptr(), + bytes_per_row as _, + width, + height, + )); + } else { + let f = match rgb.fmt() { + ImageFormat::ARGB => NV12ToARGB, + ImageFormat::ABGR => NV12ToABGR, + _ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()), + }; + call_yuv!(f( + frame.data[0].as_ptr(), + frame.linesize[0], + frame.data[1].as_ptr(), + frame.linesize[1], + rgb.raw.as_mut_ptr(), + bytes_per_row as _, + width, + height, + )); + } + } AVPixelFormat::AV_PIX_FMT_YUV420P => { - hw::hw_i420_to( - rgb.fmt(), - frame.width as _, - frame.height as _, - &frame.data[0], - &frame.data[1], - &frame.data[2], - frame.linesize[0] as _, - frame.linesize[1] as _, - frame.linesize[2] as _, - &mut rgb.raw as _, - )?; + let f = match rgb.fmt() { + ImageFormat::ARGB => I420ToARGB, + ImageFormat::ABGR => I420ToABGR, + _ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()), + }; + call_yuv!(f( + frame.data[0].as_ptr(), + frame.linesize[0], + frame.data[1].as_ptr(), + frame.linesize[1], + frame.data[2].as_ptr(), + frame.linesize[2], + rgb.raw.as_mut_ptr(), + bytes_per_row as _, + width, + height, + )); } } Ok(()) } - - pub fn bgra(&self, bgra: &mut Vec, i420: &mut Vec) -> ResultType<()> { - let mut rgb = ImageRgb::new(ImageFormat::ARGB, 1); - self.to_fmt(&mut rgb, i420)?; - *bgra = rgb.raw; - Ok(()) - } - - pub fn rgba(&self, rgba: &mut Vec, i420: &mut Vec) -> ResultType<()> { - let mut rgb = ImageRgb::new(ImageFormat::ABGR, 1); - self.to_fmt(&mut rgb, i420)?; - *rgba = rgb.raw; - Ok(()) - } } #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index 91ca6b7af..13129db55 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -53,7 +53,7 @@ pub mod record; mod vpx; #[repr(usize)] -#[derive(Copy, Clone)] +#[derive(Debug, Copy, Clone)] pub enum ImageFormat { Raw, ABGR, @@ -65,17 +65,17 @@ pub struct ImageRgb { pub w: usize, pub h: usize, pub fmt: ImageFormat, - pub stride: usize, + pub align: usize, } impl ImageRgb { - pub fn new(fmt: ImageFormat, stride: usize) -> Self { + pub fn new(fmt: ImageFormat, align: usize) -> Self { Self { raw: Vec::new(), w: 0, h: 0, fmt, - stride, + align, } } @@ -85,13 +85,13 @@ impl ImageRgb { } #[inline] - pub fn stride(&self) -> usize { - self.stride + pub fn align(&self) -> usize { + self.align } #[inline] - pub fn set_stride(&mut self, stride: usize) { - self.stride = stride; + pub fn set_align(&mut self, align: usize) { + self.align = align; } } @@ -378,20 +378,20 @@ pub trait GoogleImage { fn stride(&self) -> Vec; fn planes(&self) -> Vec<*mut u8>; fn chroma(&self) -> Chroma; - fn get_bytes_per_row(w: usize, fmt: ImageFormat, stride: usize) -> usize { + fn get_bytes_per_row(w: usize, fmt: ImageFormat, align: usize) -> usize { let bytes_per_pixel = match fmt { ImageFormat::Raw => 3, ImageFormat::ARGB | ImageFormat::ABGR => 4, }; // https://github.com/lemenkov/libyuv/blob/6900494d90ae095d44405cd4cc3f346971fa69c9/source/convert_argb.cc#L128 // https://github.com/lemenkov/libyuv/blob/6900494d90ae095d44405cd4cc3f346971fa69c9/source/convert_argb.cc#L129 - (w * bytes_per_pixel + stride - 1) & !(stride - 1) + (w * bytes_per_pixel + align - 1) & !(align - 1) } // rgb [in/out] fmt and stride must be set in ImageRgb fn to(&self, rgb: &mut ImageRgb) { rgb.w = self.width(); rgb.h = self.height(); - let bytes_per_row = Self::get_bytes_per_row(rgb.w, rgb.fmt, rgb.stride()); + let bytes_per_row = Self::get_bytes_per_row(rgb.w, rgb.fmt, rgb.align()); rgb.raw.resize(rgb.h * bytes_per_row, 0); let stride = self.stride(); let planes = self.planes(); diff --git a/src/client.rs b/src/client.rs index 684b9cff0..0072935f1 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1052,7 +1052,7 @@ impl VideoHandler { log::info!("new video handler for display #{_display}, format: {format:?}, luid: {luid:?}"); VideoHandler { decoder: Decoder::new(format, luid), - rgb: ImageRgb::new(ImageFormat::ARGB, crate::get_dst_stride_rgba()), + rgb: ImageRgb::new(ImageFormat::ARGB, crate::get_dst_align_rgba()), texture: std::ptr::null_mut(), recorder: Default::default(), record: false, @@ -1105,7 +1105,7 @@ impl VideoHandler { /// Reset the decoder, change format if it is Some pub fn reset(&mut self, format: Option) { #[cfg(target_os = "macos")] - self.rgb.set_stride(crate::get_dst_stride_rgba()); + self.rgb.set_align(crate::get_dst_align_rgba()); let luid = Self::get_adapter_luid(); let format = format.unwrap_or(self.decoder.format()); self.decoder = Decoder::new(format, luid); diff --git a/src/common.rs b/src/common.rs index 5d5a1b7e7..18474b644 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1617,7 +1617,7 @@ fn read_custom_client_advanced_settings( #[inline] #[cfg(target_os = "macos")] -pub fn get_dst_stride_rgba() -> usize { +pub fn get_dst_align_rgba() -> usize { // https://developer.apple.com/forums/thread/712709 // Memory alignment should be multiple of 64. if crate::ui_interface::use_texture_render() { @@ -1629,7 +1629,7 @@ pub fn get_dst_stride_rgba() -> usize { #[inline] #[cfg(not(target_os = "macos"))] -pub fn get_dst_stride_rgba() -> usize { +pub fn get_dst_align_rgba() -> usize { 1 } diff --git a/src/flutter.rs b/src/flutter.rs index 25b7d3aeb..2d17944bc 100644 --- a/src/flutter.rs +++ b/src/flutter.rs @@ -445,7 +445,7 @@ impl VideoRenderer { rgba.raw.len() as _, rgba.w as _, rgba.h as _, - rgba.stride() as _, + rgba.align() as _, ) }; } diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index 20ff89b13..be55451e0 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -3,7 +3,6 @@ use crate::{ common::{is_keyboard_mode_supported, make_fd_to_json}, flutter::{ self, session_add, session_add_existed, session_start_, sessions, try_sync_peer_option, - FlutterHandler, }, input::*, ui_interface::{self, *},