video handler holds only one decoder of the current codec format (#6939)

1. For example: when receiving h264 video frames, only 1 decoder is created, vram > ram
2. For creation and decoding failed:
  * Remove real_supported_decodings, this will update real existing decoders, replace it with the "mark_unsupported" vector. After creating the decoder failure, marks the codec as unsupported and updates supported decoding to the controlled side
  *  Add `fail_counter` in the decoder. When decoding 10 consecutive frames failed, adding codec type to 'mark_unsupported' vector
  *  The controlled end always ignores the unavailability of VP9

Signed-off-by: 21pages <pages21@163.com>

libs/scrap/examples/benchmark.rs

@@ -240,7 +240,10 @@ fn test_av1(
 #[cfg(feature = "hwcodec")]
 mod hw {
     use hwcodec::ffmpeg::CodecInfo;
-    use scrap::hwcodec::{HwDecoder, HwEncoder, HwEncoderConfig};
+    use scrap::{
+        hwcodec::{HwDecoder, HwEncoder, HwEncoderConfig},
+        CodecFormat,
+    };
 
     use super::*;
 
@@ -254,13 +257,8 @@ mod hw {
         if let Some(info) = best.h265 {
             test_encoder(width, height, quality, info, c, yuv_count, &mut h265s);
         }
-        let best = HwDecoder::best();
-        if let Some(info) = best.h264 {
-            test_decoder(info, &h264s);
-        }
-        if let Some(info) = best.h265 {
-            test_decoder(info, &h265s);
-        }
+        test_decoder(CodecFormat::H264, &h264s);
+        test_decoder(CodecFormat::H265, &h265s);
     }
 
     fn test_encoder(
@@ -322,16 +320,21 @@ mod hw {
         );
     }
 
-    fn test_decoder(info: CodecInfo, h26xs: &Vec<Vec<u8>>) {
-        let mut decoder = HwDecoder::new(info.clone()).unwrap();
+    fn test_decoder(format: CodecFormat, h26xs: &Vec<Vec<u8>>) {
+        let mut decoder = HwDecoder::new(format).unwrap();
         let start = Instant::now();
         let mut cnt = 0;
         for h26x in h26xs {
             let _ = decoder.decode(h26x).unwrap();
             cnt += 1;
         }
-        let device = format!("{:?}", info.hwdevice).to_lowercase();
+        let device = format!("{:?}", decoder.info.hwdevice).to_lowercase();
         let device = device.split("_").last().unwrap();
-        println!("{} {}: {:?}", info.name, device, start.elapsed() / cnt);
+        println!(
+            "{} {}: {:?}",
+            decoder.info.name,
+            device,
+            start.elapsed() / cnt
+        );
     }
 }

libs/scrap/src/common/codec.rs

@@ -10,14 +10,12 @@ use crate::gpucodec::*;
 #[cfg(feature = "hwcodec")]
 use crate::hwcodec::*;
 #[cfg(feature = "mediacodec")]
-use crate::mediacodec::{
-    MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT,
-};
+use crate::mediacodec::{MediaCodecDecoder, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT};
 use crate::{
     aom::{self, AomDecoder, AomEncoder, AomEncoderConfig},
     common::GoogleImage,
     vpxcodec::{self, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig, VpxVideoCodecId},
-    CodecName, EncodeInput, EncodeYuvFormat, ImageRgb,
+    CodecFormat, CodecName, EncodeInput, EncodeYuvFormat, ImageRgb,
 };
 
 use hbb_common::{
@@ -96,13 +94,21 @@ pub struct Decoder {
     vp9: Option<VpxDecoder>,
     av1: Option<AomDecoder>,
     #[cfg(feature = "hwcodec")]
-    hw: HwDecoders,
+    h264_ram: Option<HwDecoder>,
+    #[cfg(feature = "hwcodec")]
+    h265_ram: Option<HwDecoder>,
     #[cfg(feature = "gpucodec")]
-    gpu: GpuDecoders,
+    h264_vram: Option<GpuDecoder>,
+    #[cfg(feature = "gpucodec")]
+    h265_vram: Option<GpuDecoder>,
+    #[cfg(feature = "mediacodec")]
+    h264_media_codec: MediaCodecDecoder,
+    #[cfg(feature = "mediacodec")]
+    h265_media_codec: MediaCodecDecoder,
+    format: CodecFormat,
+    valid: bool,
     #[cfg(feature = "hwcodec")]
     i420: Vec<u8>,
-    #[cfg(feature = "mediacodec")]
-    media_codec: MediaCodecDecoders,
 }
 
 #[derive(Debug, Clone)]
@@ -372,6 +378,7 @@ impl Decoder {
         id_for_perfer: Option<&str>,
         _flutter: bool,
         _luid: Option<i64>,
+        mark_unsupported: &Vec<CodecFormat>,
     ) -> SupportedDecoding {
         let (prefer, prefer_chroma) = Self::preference(id_for_perfer);
 
@@ -398,12 +405,12 @@ impl Decoder {
         }
         #[cfg(feature = "gpucodec")]
         if enable_gpucodec_option() && _flutter {
-            decoding.ability_h264 |= if GpuDecoder::available(CodecName::H264GPU, _luid).len() > 0 {
+            decoding.ability_h264 |= if GpuDecoder::available(CodecFormat::H264, _luid).len() > 0 {
                 1
             } else {
                 0
             };
-            decoding.ability_h265 |= if GpuDecoder::available(CodecName::H265GPU, _luid).len() > 0 {
+            decoding.ability_h265 |= if GpuDecoder::available(CodecFormat::H265, _luid).len() > 0 {
                 1
             } else {
                 0
@@ -424,72 +431,148 @@ impl Decoder {
                     0
                 };
         }
+        for unsupported in mark_unsupported {
+            match unsupported {
+                CodecFormat::VP8 => decoding.ability_vp8 = 0,
+                CodecFormat::VP9 => decoding.ability_vp9 = 0,
+                CodecFormat::AV1 => decoding.ability_av1 = 0,
+                CodecFormat::H264 => decoding.ability_h264 = 0,
+                CodecFormat::H265 => decoding.ability_h265 = 0,
+                _ => {}
+            }
+        }
         decoding
     }
 
-    pub fn exist_codecs(&self, _flutter: bool) -> CodecAbility {
-        #[allow(unused_mut)]
-        let mut ability = CodecAbility {
-            vp8: self.vp8.is_some(),
-            vp9: self.vp9.is_some(),
-            av1: self.av1.is_some(),
-            ..Default::default()
-        };
+    pub fn new(format: CodecFormat, _luid: Option<i64>) -> Decoder {
+        log::info!("try create new decoder, format: {format:?}, _luid: {_luid:?}");
+        let (mut vp8, mut vp9, mut av1) = (None, None, None);
         #[cfg(feature = "hwcodec")]
-        {
-            ability.h264 |= self.hw.h264.is_some();
-            ability.h265 |= self.hw.h265.is_some();
-        }
+        let (mut h264_ram, mut h265_ram) = (None, None);
         #[cfg(feature = "gpucodec")]
-        if _flutter {
-            ability.h264 |= self.gpu.h264.is_some();
-            ability.h265 |= self.gpu.h265.is_some();
-        }
+        let (mut h264_vram, mut h265_vram) = (None, None);
         #[cfg(feature = "mediacodec")]
-        {
-            ability.h264 = self.media_codec.h264.is_some();
-            ability.h265 = self.media_codec.h265.is_some();
-        }
-        ability
-    }
+        let (mut h264_media_codec, mut h265_media_codec) = (None, None);
+        let mut valid = false;
 
-    pub fn new(_luid: Option<i64>) -> Decoder {
-        let vp8 = VpxDecoder::new(VpxDecoderConfig {
-            codec: VpxVideoCodecId::VP8,
-        })
-        .ok();
-        let vp9 = VpxDecoder::new(VpxDecoderConfig {
-            codec: VpxVideoCodecId::VP9,
-        })
-        .ok();
-        let av1 = AomDecoder::new().ok();
+        match format {
+            CodecFormat::VP8 => {
+                match VpxDecoder::new(VpxDecoderConfig {
+                    codec: VpxVideoCodecId::VP8,
+                }) {
+                    Ok(v) => vp8 = Some(v),
+                    Err(e) => log::error!("create VP8 decoder failed: {}", e),
+                }
+                valid = vp8.is_some();
+            }
+            CodecFormat::VP9 => {
+                match VpxDecoder::new(VpxDecoderConfig {
+                    codec: VpxVideoCodecId::VP9,
+                }) {
+                    Ok(v) => vp9 = Some(v),
+                    Err(e) => log::error!("create VP9 decoder failed: {}", e),
+                }
+                valid = vp9.is_some();
+            }
+            CodecFormat::AV1 => {
+                match AomDecoder::new() {
+                    Ok(v) => av1 = Some(v),
+                    Err(e) => log::error!("create AV1 decoder failed: {}", e),
+                }
+                valid = av1.is_some();
+            }
+            CodecFormat::H264 => {
+                #[cfg(feature = "gpucodec")]
+                if !valid && enable_gpucodec_option() && _luid.clone().unwrap_or_default() != 0 {
+                    match GpuDecoder::new(format, _luid) {
+                        Ok(v) => h264_vram = Some(v),
+                        Err(e) => log::error!("create H264 vram decoder failed: {}", e),
+                    }
+                    valid = h264_vram.is_some();
+                }
+                #[cfg(feature = "hwcodec")]
+                if !valid && enable_hwcodec_option() {
+                    match HwDecoder::new(format) {
+                        Ok(v) => h264_ram = Some(v),
+                        Err(e) => log::error!("create H264 ram decoder failed: {}", e),
+                    }
+                    valid = h264_ram.is_some();
+                }
+                #[cfg(feature = "mediacodec")]
+                if !valid && enable_hwcodec_option() {
+                    h264_media_codec = MediaCodecDecoder::new(format);
+                    if h264_media_codec.is_none() {
+                        log::error!("create H264 media codec decoder failed");
+                    }
+                    valid = h264_media_codec.is_some();
+                }
+            }
+            CodecFormat::H265 => {
+                #[cfg(feature = "gpucodec")]
+                if !valid && enable_gpucodec_option() && _luid.clone().unwrap_or_default() != 0 {
+                    match GpuDecoder::new(format, _luid) {
+                        Ok(v) => h265_vram = Some(v),
+                        Err(e) => log::error!("create H265 vram decoder failed: {}", e),
+                    }
+                    valid = h265_vram.is_some();
+                }
+                #[cfg(feature = "hwcodec")]
+                if !valid && enable_hwcodec_option() {
+                    match HwDecoder::new(format) {
+                        Ok(v) => h265_ram = Some(v),
+                        Err(e) => log::error!("create H265 ram decoder failed: {}", e),
+                    }
+                    valid = h265_ram.is_some();
+                }
+                #[cfg(feature = "mediacodec")]
+                if !valid && enable_hwcodec_option() {
+                    h265_media_codec = MediaCodecDecoder::new(format);
+                    if h265_media_codec.is_none() {
+                        log::error!("create H265 media codec decoder failed");
+                    }
+                    valid = h265_media_codec.is_some();
+                }
+            }
+            CodecFormat::Unknown => {
+                log::error!("unknown codec format, cannot create decoder");
+            }
+        }
+        if !valid {
+            log::error!("failed to create {format:?} decoder");
+        } else {
+            log::info!("create {format:?} decoder success");
+        }
         Decoder {
             vp8,
             vp9,
             av1,
             #[cfg(feature = "hwcodec")]
-            hw: if enable_hwcodec_option() {
-                HwDecoder::new_decoders()
-            } else {
-                HwDecoders::default()
-            },
+            h264_ram,
+            #[cfg(feature = "hwcodec")]
+            h265_ram,
             #[cfg(feature = "gpucodec")]
-            gpu: if enable_gpucodec_option() && _luid.clone().unwrap_or_default() != 0 {
-                GpuDecoder::new_decoders(_luid)
-            } else {
-                GpuDecoders::default()
-            },
+            h264_vram,
+            #[cfg(feature = "gpucodec")]
+            h265_vram,
+            #[cfg(feature = "mediacodec")]
+            h264_media_codec,
+            #[cfg(feature = "mediacodec")]
+            h265_media_codec,
+            format,
+            valid,
             #[cfg(feature = "hwcodec")]
             i420: vec![],
-            #[cfg(feature = "mediacodec")]
-            media_codec: if enable_hwcodec_option() {
-                MediaCodecDecoder::new_decoders()
-            } else {
-                MediaCodecDecoders::default()
-            },
         }
     }
 
+    pub fn format(&self) -> CodecFormat {
+        self.format
+    }
+
+    pub fn valid(&self) -> bool {
+        self.valid
+    }
+
     // rgb [in/out] fmt and stride must be set in ImageRgb
     pub fn handle_video_frame(
         &mut self,
@@ -525,12 +608,12 @@ impl Decoder {
             video_frame::Union::H264s(h264s) => {
                 *chroma = Some(Chroma::I420);
                 #[cfg(feature = "gpucodec")]
-                if let Some(decoder) = &mut self.gpu.h264 {
+                if let Some(decoder) = &mut self.h264_vram {
                     *_pixelbuffer = false;
                     return Decoder::handle_gpu_video_frame(decoder, h264s, _texture);
                 }
                 #[cfg(feature = "hwcodec")]
-                if let Some(decoder) = &mut self.hw.h264 {
+                if let Some(decoder) = &mut self.h264_ram {
                     return Decoder::handle_hw_video_frame(decoder, h264s, rgb, &mut self.i420);
                 }
                 Err(anyhow!("don't support h264!"))
@@ -539,12 +622,12 @@ impl Decoder {
             video_frame::Union::H265s(h265s) => {
                 *chroma = Some(Chroma::I420);
                 #[cfg(feature = "gpucodec")]
-                if let Some(decoder) = &mut self.gpu.h265 {
+                if let Some(decoder) = &mut self.h265_vram {
                     *_pixelbuffer = false;
                     return Decoder::handle_gpu_video_frame(decoder, h265s, _texture);
                 }
                 #[cfg(feature = "hwcodec")]
-                if let Some(decoder) = &mut self.hw.h265 {
+                if let Some(decoder) = &mut self.h265_ram {
                     return Decoder::handle_hw_video_frame(decoder, h265s, rgb, &mut self.i420);
                 }
                 Err(anyhow!("don't support h265!"))
@@ -552,7 +635,7 @@ impl Decoder {
             #[cfg(feature = "mediacodec")]
             video_frame::Union::H264s(h264s) => {
                 *chroma = Some(Chroma::I420);
-                if let Some(decoder) = &mut self.media_codec.h264 {
+                if let Some(decoder) = &mut self.h264_media_codec {
                     Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb)
                 } else {
                     Err(anyhow!("don't support h264!"))
@@ -561,7 +644,7 @@ impl Decoder {
             #[cfg(feature = "mediacodec")]
             video_frame::Union::H265s(h265s) => {
                 *chroma = Some(Chroma::I420);
-                if let Some(decoder) = &mut self.media_codec.h265 {
+                if let Some(decoder) = &mut self.h265_media_codec {
                     Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb)
                 } else {
                     Err(anyhow!("don't support h265!"))

libs/scrap/src/common/gpucodec.rs

@@ -6,7 +6,7 @@ use std::{
 
 use crate::{
     codec::{base_bitrate, enable_gpucodec_option, EncoderApi, EncoderCfg, Quality},
-    AdapterDevice, CodecName, EncodeInput, EncodeYuvFormat, Pixfmt,
+    AdapterDevice, CodecFormat, CodecName, EncodeInput, EncodeYuvFormat, Pixfmt,
 };
 use gpucodec::gpu_common::{
     self, Available, DecodeContext, DynamicContext, EncodeContext, FeatureContext, MAX_GOP,
@@ -87,7 +87,10 @@ impl EncoderApi for GpuEncoder {
                         last_frame_len: 0,
                         same_bad_len_counter: 0,
                     }),
-                    Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
+                    Err(_) => {
+                        hbb_common::config::GpucodecConfig::clear();
+                        Err(anyhow!(format!("Failed to create encoder")))
+                    }
                 }
             }
             _ => Err(anyhow!("encoder type mismatch")),
@@ -300,8 +303,8 @@ pub struct GpuDecoders {
 }
 
 impl GpuDecoder {
-    pub fn try_get(name: CodecName, luid: Option<i64>) -> Option<DecodeContext> {
-        let v: Vec<_> = Self::available(name, luid);
+    pub fn try_get(format: CodecFormat, luid: Option<i64>) -> Option<DecodeContext> {
+        let v: Vec<_> = Self::available(format, luid);
         if v.len() > 0 {
             Some(v[0].clone())
         } else {
@@ -309,11 +312,11 @@ impl GpuDecoder {
         }
     }
 
-    pub fn available(name: CodecName, luid: Option<i64>) -> Vec<DecodeContext> {
+    pub fn available(format: CodecFormat, luid: Option<i64>) -> Vec<DecodeContext> {
         let luid = luid.unwrap_or_default();
-        let data_format = match name {
-            CodecName::H264GPU => gpu_common::DataFormat::H264,
-            CodecName::H265GPU => gpu_common::DataFormat::H265,
+        let data_format = match format {
+            CodecFormat::H264 => gpu_common::DataFormat::H264,
+            CodecFormat::H265 => gpu_common::DataFormat::H265,
             _ => return vec![],
         };
         get_available_config()
@@ -337,28 +340,18 @@ impl GpuDecoder {
         )
     }
 
-    pub fn new_decoders(luid: Option<i64>) -> GpuDecoders {
-        let mut h264: Option<GpuDecoder> = None;
-        let mut h265: Option<GpuDecoder> = None;
-        if let Ok(decoder) = GpuDecoder::new(CodecName::H264GPU, luid) {
-            h264 = Some(decoder);
-        }
-        if let Ok(decoder) = GpuDecoder::new(CodecName::H265GPU, luid) {
-            h265 = Some(decoder);
-        }
-        log::info!(
-            "new gpu decoders, support h264: {}, h265: {}",
-            h264.is_some(),
-            h265.is_some()
-        );
-        GpuDecoders { h264, h265 }
-    }
-
-    pub fn new(name: CodecName, luid: Option<i64>) -> ResultType<Self> {
-        let ctx = Self::try_get(name, luid).ok_or(anyhow!("Failed to get decode context"))?;
+    pub fn new(format: CodecFormat, luid: Option<i64>) -> ResultType<Self> {
+        log::info!("try create {format:?} vram decoder, luid: {luid:?}");
+        let ctx = Self::try_get(format, luid).ok_or(anyhow!("Failed to get decode context"))?;
         match Decoder::new(ctx) {
             Ok(decoder) => Ok(Self { decoder }),
-            Err(_) => Err(anyhow!(format!("Failed to create decoder"))),
+            Err(_) => {
+                hbb_common::config::GpucodecConfig::clear();
+                Err(anyhow!(format!(
+                    "Failed to create decoder, format: {:?}",
+                    format
+                )))
+            }
         }
     }
     pub fn decode(&mut self, data: &[u8]) -> ResultType<Vec<GpuDecoderImage>> {

libs/scrap/src/common/hwcodec.rs

@@ -1,10 +1,10 @@
 use crate::{
     codec::{base_bitrate, codec_thread_num, EncoderApi, EncoderCfg, Quality as Q},
-    hw, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
+    hw, CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
 };
 use hbb_common::{
     allow_err,
-    anyhow::{anyhow, Context},
+    anyhow::{anyhow, bail, Context},
     bytes::Bytes,
     config::HwCodecConfig,
     log,
@@ -94,7 +94,10 @@ impl EncoderApi for HwEncoder {
                         height: ctx.height as _,
                         bitrate,
                     }),
-                    Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
+                    Err(_) => {
+                        HwCodecConfig::clear();
+                        Err(anyhow!(format!("Failed to create encoder")))
+                    }
                 }
             }
             _ => Err(anyhow!("encoder type mismatch")),
@@ -230,31 +233,26 @@ impl HwDecoder {
         })
     }
 
-    pub fn new_decoders() -> HwDecoders {
+    pub fn new(format: CodecFormat) -> ResultType<Self> {
+        log::info!("try create {format:?} ram decoder");
         let best = HwDecoder::best();
-        let mut h264: Option<HwDecoder> = None;
-        let mut h265: Option<HwDecoder> = None;
-        let mut fail = false;
-
-        if let Some(info) = best.h264 {
-            h264 = HwDecoder::new(info).ok();
-            if h264.is_none() {
-                fail = true;
+        let info = match format {
+            CodecFormat::H264 => {
+                if let Some(info) = best.h264 {
+                    info
+                } else {
+                    bail!("no h264 decoder, should not be here");
+                }
             }
-        }
-        if let Some(info) = best.h265 {
-            h265 = HwDecoder::new(info).ok();
-            if h265.is_none() {
-                fail = true;
+            CodecFormat::H265 => {
+                if let Some(info) = best.h265 {
+                    info
+                } else {
+                    bail!("no h265 decoder, should not be here");
+                }
             }
-        }
-        if fail {
-            hwcodec_new_check_process();
-        }
-        HwDecoders { h264, h265 }
-    }
-
-    pub fn new(info: CodecInfo) -> ResultType<Self> {
+            _ => bail!("unsupported format: {:?}", format),
+        };
         let ctx = DecodeContext {
             name: info.name.clone(),
             device_type: info.hwdevice.clone(),
@@ -262,7 +260,10 @@ impl HwDecoder {
         };
         match Decoder::new(ctx) {
             Ok(decoder) => Ok(HwDecoder { decoder, info }),
-            Err(_) => Err(anyhow!(format!("Failed to create decoder"))),
+            Err(_) => {
+                HwCodecConfig::clear();
+                Err(anyhow!(format!("Failed to create decoder")))
+            }
         }
     }
     pub fn decode(&mut self, data: &[u8]) -> ResultType<Vec<HwDecoderImage>> {

libs/scrap/src/common/mediacodec.rs

@@ -10,7 +10,7 @@ use std::{
 use crate::ImageFormat;
 use crate::{
     codec::{EncoderApi, EncoderCfg},
-    I420ToABGR, I420ToARGB, ImageRgb,
+    CodecFormat, I420ToABGR, I420ToARGB, ImageRgb,
 };
 
 /// MediaCodec mime type name
@@ -37,17 +37,16 @@ impl Deref for MediaCodecDecoder {
     }
 }
 
-#[derive(Default)]
-pub struct MediaCodecDecoders {
-    pub h264: Option<MediaCodecDecoder>,
-    pub h265: Option<MediaCodecDecoder>,
-}
-
 impl MediaCodecDecoder {
-    pub fn new_decoders() -> MediaCodecDecoders {
-        let h264 = create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder);
-        let h265 = create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder);
-        MediaCodecDecoders { h264, h265 }
+    pub fn new(format: CodecFormat) -> Option<MediaCodecDecoder> {
+        match format {
+            CodecFormat::H264 => create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder),
+            CodecFormat::H265 => create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder),
+            _ => {
+                log::error!("Unsupported codec format: {}", format);
+                None
+            }
+        }
     }
 
     // rgb [in/out] fmt and stride must be set in ImageRgb

libs/scrap/src/common/mod.rs

@@ -251,7 +251,7 @@ pub enum CodecName {
     H265GPU,
 }
 
-#[derive(PartialEq, Debug, Clone)]
+#[derive(PartialEq, Debug, Clone, Copy)]
 pub enum CodecFormat {
     VP8,
     VP9,