diff --git a/Cargo.lock b/Cargo.lock index 7e679ef91..884df3c56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2231,6 +2231,19 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hwcodec" +version = "0.1.0" +source = "git+https://github.com/21pages/hwcodec#bfc558d2375928b0a59336cfc72336415db27066" +dependencies = [ + "bindgen", + "cc", + "log", + "serde 1.0.137", + "serde_derive", + "serde_json 1.0.81", +] + [[package]] name = "hyper" version = "0.14.19" @@ -4247,6 +4260,8 @@ dependencies = [ "gstreamer", "gstreamer-app", "gstreamer-video", + "hbb_common", + "hwcodec", "jni", "lazy_static", "libc", diff --git a/Cargo.toml b/Cargo.toml index 3cb3ffc91..f270f7b30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ appimage = [] use_samplerate = ["samplerate"] use_rubato = ["rubato"] use_dasp = ["dasp"] +hwcodec = ["scrap/hwcodec"] default = ["use_dasp"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/README-RU.md b/README-RU.md index 54c161cf0..755d91ca3 100644 --- a/README-RU.md +++ b/README-RU.md @@ -1,5 +1,5 @@

- RustDesk - Your remote desktop
+ RustDesk - Ваш удаленый рабочий стол
Servers • Build • Docker • @@ -15,10 +15,16 @@ Еще одно программное обеспечение для удаленного рабочего стола, написанное на Rust. Работает из коробки, не требует настройки. Вы полностью контролируете свои данные, не беспокоясь о безопасности. Вы можете использовать наш сервер ретрансляции, [настроить свой собственный](https://rustdesk.com/server), или [написать свой собственный сервер ретрансляции](https://github.com/rustdesk/rustdesk-server-demo). +![image](https://user-images.githubusercontent.com/71636191/171661982-430285f0-2e12-4b1d-9957-4a58e375304d.png) + RustDesk приветствует вклад каждого. Смотрите [`CONTRIBUTING.md`](CONTRIBUTING.md) для помощи в начале работы. +[**Как работает RustDesk?**](https://github.com/rustdesk/rustdesk/wiki/How-does-RustDesk-work%3F) + [**СКАЧАТЬ ПРИЛОЖЕНИЕ**](https://github.com/rustdesk/rustdesk/releases) +[Get it on F-Droid](https://f-droid.org/en/packages/com.carriez.flutter_hbb) + ## Бесплатные общедоступные серверы Ниже приведены серверы, для бесплатного использования, они могут меняться со временем. Если вы не находитесь рядом с одним из них, ваша сеть может работать медленно. @@ -81,7 +87,7 @@ export VCPKG_ROOT=$HOME/vcpkg vcpkg/vcpkg install libvpx libyuv opus ``` -### Исправление libvpx (Для Fedora) +### Исправление libvpx (для Fedora) ```sh cd vcpkg/buildtrees/libvpx/src diff --git a/build.py b/build.py index 45488ad63..2b7cd3f27 100644 --- a/build.py +++ b/build.py @@ -66,6 +66,11 @@ def make_parser(): default='', help='Integrate features, windows only.' 'Available: IddDriver, PrivacyMode. Special value is "ALL" and empty "". Default is empty.') + parser.add_argument( + '--hwcodec', + action='store_true', + help='Enable feature hwcodec, windows only.' + ) return parser @@ -89,11 +94,9 @@ def download_extract_features(features, res_dir): print(f'{feat} extract end') -def build_windows(feature): - features = get_features(feature) - if not features: - os.system('cargo build --release --features inline') - else: +def build_windows(args): + features = get_features(args.feature) + if features: print(f'Build with features {list(features.keys())}') res_dir = 'resources' if os.path.isdir(res_dir) and not os.path.islink(res_dir): @@ -102,8 +105,12 @@ def build_windows(feature): raise Exception(f'Find file {res_dir}, not a directory') os.makedirs(res_dir, exist_ok=True) download_extract_features(features, res_dir) - os.system('cargo build --release --features inline,with_rc') + with_rc = ',with_rc' if features else '' + hwcodec = ',hwcodec' if args.hwcodec else '' + cmd = 'cargo build --release --features inline' + with_rc + hwcodec + print(cmd) + os.system(cmd) def main(): parser = make_parser() @@ -123,7 +130,7 @@ def main(): os.system('git checkout src/ui/common.tis') version = get_version() if windows: - build_windows(args.feature) + build_windows(args) # os.system('upx.exe target/release/rustdesk.exe') os.system('mv target/release/rustdesk.exe target/release/RustDesk.exe') pa = os.environ.get('P') diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index 048301d7e..0538f1aef 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -1,13 +1,13 @@ syntax = "proto3"; package hbb; -message VP9 { +message EncodedVideoFrame { bytes data = 1; bool key = 2; int64 pts = 3; } -message VP9s { repeated VP9 frames = 1; } +message EncodedVideoFrames { repeated EncodedVideoFrame frames = 1; } message RGB { bool compress = 1; } @@ -19,9 +19,11 @@ message YUV { message VideoFrame { oneof union { - VP9s vp9s = 6; + EncodedVideoFrames vp9s = 6; RGB rgb = 7; YUV yuv = 8; + EncodedVideoFrames h264s = 10; + EncodedVideoFrames h265s = 11; } int64 timestamp = 9; } @@ -430,6 +432,12 @@ enum ImageQuality { Best = 4; } +message VideoCodecState { + int32 ScoreVpx = 1; + int32 ScoreH264 = 2; + int32 ScoreH265 = 3; +} + message OptionMessage { enum BoolOption { NotSet = 0; @@ -445,11 +453,14 @@ message OptionMessage { BoolOption disable_audio = 7; BoolOption disable_clipboard = 8; BoolOption enable_file_transfer = 9; + VideoCodecState video_codec_state = 10; } message TestDelay { int64 time = 1; bool from_client = 2; + uint32 last_delay = 3; + uint32 target_bitrate = 4; } message PublicKey { diff --git a/libs/hbb_common/src/config.rs b/libs/hbb_common/src/config.rs index cf3d669dc..b4d8eaff3 100644 --- a/libs/hbb_common/src/config.rs +++ b/libs/hbb_common/src/config.rs @@ -39,9 +39,16 @@ lazy_static::lazy_static! { pub static ref PROD_RENDEZVOUS_SERVER: Arc> = Default::default(); pub static ref APP_NAME: Arc> = Arc::new(RwLock::new("RustDesk".to_owned())); } -#[cfg(any(target_os = "android", target_os = "ios"))] +#[cfg(target_os = "android")] +lazy_static::lazy_static! { + pub static ref APP_DIR: Arc> = Arc::new(RwLock::new("/data/user/0/com.carriez.flutter_hbb/app_flutter".to_owned())); +} +#[cfg(target_os = "ios")] lazy_static::lazy_static! { pub static ref APP_DIR: Arc> = Default::default(); +} +#[cfg(any(target_os = "android", target_os = "ios"))] +lazy_static::lazy_static! { pub static ref APP_HOME_DIR: Arc> = Default::default(); } const CHARS: &'static [char] = &[ @@ -139,6 +146,8 @@ pub struct PeerConfig { pub disable_clipboard: bool, #[serde(default)] pub enable_file_transfer: bool, + #[serde(default)] + pub show_quality_monitor: bool, // the other scalar value must before this #[serde(default)] @@ -881,6 +890,22 @@ impl LanPeers { } } +#[derive(Debug, Default, Serialize, Deserialize, Clone)] +pub struct HwCodecConfig { + #[serde(default)] + pub options: HashMap, +} + +impl HwCodecConfig { + pub fn load() -> HwCodecConfig { + Config::load_::("_hwcodec") + } + + pub fn store(&self) { + Config::store_(self, "_hwcodec"); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/libs/scrap/Cargo.toml b/libs/scrap/Cargo.toml index c99ec45da..1b269d96e 100644 --- a/libs/scrap/Cargo.toml +++ b/libs/scrap/Cargo.toml @@ -18,6 +18,7 @@ cfg-if = "1.0" libc = "0.2" num_cpus = "1.13" lazy_static = "1.4" +hbb_common = { path = "../hbb_common" } [dependencies.winapi] version = "0.3" @@ -48,3 +49,6 @@ tracing = { version = "0.1", optional = true } gstreamer = { version = "0.16", optional = true } gstreamer-app = { version = "0.16", features = ["v1_10"], optional = true } gstreamer-video = { version = "0.16", optional = true } + +[target.'cfg(target_os = "windows")'.dependencies] +hwcodec = { git = "https://github.com/21pages/hwcodec", optional = true } diff --git a/libs/scrap/examples/capture_mag.rs b/libs/scrap/examples/capture_mag.rs index 3e15b4e69..c0505141e 100644 --- a/libs/scrap/examples/capture_mag.rs +++ b/libs/scrap/examples/capture_mag.rs @@ -21,6 +21,8 @@ fn get_display(i: usize) -> Display { #[cfg(windows)] fn record(i: usize) { + use std::time::Duration; + for d in Display::all().unwrap() { println!("{:?} {} {}", d.origin(), d.width(), d.height()); } @@ -40,7 +42,7 @@ fn record(i: usize) { println!("Filter window for cls {} name {}", wnd_cls, wnd_name); } - let frame = capture_mag.frame(0).unwrap(); + let frame = capture_mag.frame(Duration::from_millis(0)).unwrap(); println!("Capture data len: {}, Saving...", frame.len()); let mut bitflipped = Vec::with_capacity(w * h * 4); @@ -76,7 +78,7 @@ fn record(i: usize) { println!("Filter window for cls {} title {}", wnd_cls, wnd_title); } - let buffer = capture_mag.frame(0).unwrap(); + let buffer = capture_mag.frame(Duration::from_millis(0)).unwrap(); println!("Capture data len: {}, Saving...", buffer.len()); let mut frame = Default::default(); diff --git a/libs/scrap/examples/ffplay.rs b/libs/scrap/examples/ffplay.rs index a4ca1b35b..2c685b931 100644 --- a/libs/scrap/examples/ffplay.rs +++ b/libs/scrap/examples/ffplay.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + extern crate scrap; fn main() { @@ -29,7 +31,7 @@ fn main() { let mut out = child.stdin.unwrap(); loop { - match capturer.frame(0) { + match capturer.frame(Duration::from_millis(0)) { Ok(frame) => { // Write the frame, removing end-of-row padding. let stride = frame.len() / h; diff --git a/libs/scrap/examples/record-screen.rs b/libs/scrap/examples/record-screen.rs index 2a56c0dcd..e099a8d1e 100644 --- a/libs/scrap/examples/record-screen.rs +++ b/libs/scrap/examples/record-screen.rs @@ -13,10 +13,11 @@ use std::time::{Duration, Instant}; use std::{io, thread}; use docopt::Docopt; +use scrap::codec::{EncoderApi, EncoderCfg}; use webm::mux; use webm::mux::Track; -use scrap::codec as vpx_encode; +use scrap::vpxcodec as vpx_encode; use scrap::{Capturer, Display, STRIDE_ALIGN}; const USAGE: &'static str = " @@ -89,27 +90,22 @@ fn main() -> io::Result<()> { mux::Segment::new(mux::Writer::new(out)).expect("Could not initialize the multiplexer."); let (vpx_codec, mux_codec) = match args.flag_codec { - Codec::Vp8 => (vpx_encode::VideoCodecId::VP8, mux::VideoCodecId::VP8), - Codec::Vp9 => (vpx_encode::VideoCodecId::VP9, mux::VideoCodecId::VP9), + Codec::Vp8 => (vpx_encode::VpxVideoCodecId::VP8, mux::VideoCodecId::VP8), + Codec::Vp9 => (vpx_encode::VpxVideoCodecId::VP9, mux::VideoCodecId::VP9), }; let mut vt = webm.add_video_track(width, height, None, mux_codec); // Setup the encoder. - let mut vpx = vpx_encode::Encoder::new( - &vpx_encode::Config { - width, - height, - timebase: [1, 1000], - bitrate: args.flag_bv, - codec: vpx_codec, - rc_min_quantizer: 0, - rc_max_quantizer: 0, - speed: 6, - }, - 0, - ) + let mut vpx = vpx_encode::VpxEncoder::new(EncoderCfg::VPX(vpx_encode::VpxEncoderConfig { + width, + height, + timebase: [1, 1000], + bitrate: args.flag_bv, + codec: vpx_codec, + num_threads: 0, + })) .unwrap(); // Start recording. @@ -138,7 +134,7 @@ fn main() -> io::Result<()> { break; } - if let Ok(frame) = c.frame(0) { + if let Ok(frame) = c.frame(Duration::from_millis(0)) { let ms = time.as_secs() * 1000 + time.subsec_millis() as u64; for frame in vpx.encode(ms as i64, &frame, STRIDE_ALIGN).unwrap() { diff --git a/libs/scrap/examples/screenshot.rs b/libs/scrap/examples/screenshot.rs index b52ea11f7..b6bf8a1d9 100644 --- a/libs/scrap/examples/screenshot.rs +++ b/libs/scrap/examples/screenshot.rs @@ -34,7 +34,7 @@ fn record(i: usize) { loop { // Wait until there's a frame. - let buffer = match capturer.frame(0) { + let buffer = match capturer.frame(Duration::from_millis(0)) { Ok(buffer) => buffer, Err(error) => { if error.kind() == WouldBlock { @@ -83,7 +83,7 @@ fn record(i: usize) { loop { // Wait until there's a frame. - let buffer = match capturer.frame(0) { + let buffer = match capturer.frame(Duration::from_millis(0)) { Ok(buffer) => buffer, Err(error) => { if error.kind() == WouldBlock { diff --git a/libs/scrap/src/common/android.rs b/libs/scrap/src/common/android.rs index 1975a6505..8322da3cd 100644 --- a/libs/scrap/src/common/android.rs +++ b/libs/scrap/src/common/android.rs @@ -3,8 +3,8 @@ use crate::rgba_to_i420; use lazy_static::lazy_static; use serde_json::Value; use std::collections::HashMap; -use std::io; use std::sync::Mutex; +use std::{io, time::Duration}; lazy_static! { static ref SCREEN_SIZE: Mutex<(u16, u16, u16)> = Mutex::new((0, 0, 0)); // (width, height, scale) @@ -33,7 +33,7 @@ impl Capturer { self.display.height() as usize } - pub fn frame<'a>(&'a mut self, _timeout_ms: u32) -> io::Result> { + pub fn frame<'a>(&'a mut self, _timeout: Duration) -> io::Result> { if let Some(buf) = get_video_raw() { crate::would_block_if_equal(&mut self.saved_raw_data, buf)?; rgba_to_i420(self.width(), self.height(), buf, &mut self.bgra); diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index f1533d7cf..e06c0e29a 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -1,536 +1,327 @@ -// https://github.com/astraw/vpx-encode -// https://github.com/astraw/env-libvpx-sys -// https://github.com/rust-av/vpx-rs/blob/master/src/decoder.rs - -use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *}; -use std::os::raw::{c_int, c_uint}; -use std::{ptr, slice}; - -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -pub enum VideoCodecId { - VP8, - VP9, -} - -impl Default for VideoCodecId { - fn default() -> VideoCodecId { - VideoCodecId::VP9 - } -} - -pub struct Encoder { - ctx: vpx_codec_ctx_t, - width: usize, - height: usize, -} - -pub struct Decoder { - ctx: vpx_codec_ctx_t, -} - -#[derive(Debug)] -pub enum Error { - FailedCall(String), - BadPtr(String), -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::result::Result<(), std::fmt::Error> { - write!(f, "{:?}", self) - } -} - -impl std::error::Error for Error {} - -pub type Result = std::result::Result; - -macro_rules! call_vpx { - ($x:expr) => {{ - let result = unsafe { $x }; // original expression - let result_int = unsafe { std::mem::transmute::<_, i32>(result) }; - if result_int != 0 { - return Err(Error::FailedCall(format!( - "errcode={} {}:{}:{}:{}", - result_int, - module_path!(), - file!(), - line!(), - column!() - )) - .into()); - } - result - }}; -} - -macro_rules! call_vpx_ptr { - ($x:expr) => {{ - let result = unsafe { $x }; // original expression - let result_int = unsafe { std::mem::transmute::<_, isize>(result) }; - if result_int == 0 { - return Err(Error::BadPtr(format!( - "errcode={} {}:{}:{}:{}", - result_int, - module_path!(), - file!(), - line!(), - column!() - )) - .into()); - } - result - }}; -} - -impl Encoder { - pub fn new(config: &Config, num_threads: u32) -> Result { - let i; - if cfg!(feature = "VP8") { - i = match config.codec { - VideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), - VideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), - }; - } else { - i = call_vpx_ptr!(vpx_codec_vp9_cx()); - } - let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; - call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0)); - - // https://www.webmproject.org/docs/encoder-parameters/ - // default: c.rc_min_quantizer = 0, c.rc_max_quantizer = 63 - // try rc_resize_allowed later - - c.g_w = config.width; - c.g_h = config.height; - c.g_timebase.num = config.timebase[0]; - c.g_timebase.den = config.timebase[1]; - c.rc_target_bitrate = config.bitrate; - c.rc_undershoot_pct = 95; - c.rc_dropframe_thresh = 25; - if config.rc_min_quantizer > 0 { - c.rc_min_quantizer = config.rc_min_quantizer; - } - if config.rc_max_quantizer > 0 { - c.rc_max_quantizer = config.rc_max_quantizer; - } - let mut speed = config.speed; - if speed <= 0 { - speed = 6; - } - - c.g_threads = if num_threads == 0 { - num_cpus::get() as _ - } else { - num_threads - }; - c.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT; - // https://developers.google.com/media/vp9/bitrate-modes/ - // Constant Bitrate mode (CBR) is recommended for live streaming with VP9. - c.rc_end_usage = vpx_rc_mode::VPX_CBR; - // c.kf_min_dist = 0; - // c.kf_max_dist = 999999; - c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot - - /* - VPX encoder支持two-pass encode，这是为了rate control的。 - 对于两遍编码，就是需要整个编码过程做两次，第一次会得到一些新的控制参数来进行第二遍的编码， - 这样可以在相同的bitrate下得到最好的PSNR - */ - - let mut ctx = Default::default(); - call_vpx!(vpx_codec_enc_init_ver( - &mut ctx, - i, - &c, - 0, - VPX_ENCODER_ABI_VERSION as _ - )); - - if config.codec == VideoCodecId::VP9 { - // set encoder internal speed settings - // in ffmpeg, it is --speed option - /* - set to 0 or a positive value 1-16, the codec will try to adapt its - complexity depending on the time it spends encoding. Increasing this - number will make the speed go up and the quality go down. - Negative values mean strict enforcement of this - while positive values are adaptive - */ - /* https://developers.google.com/media/vp9/live-encoding - Speed 5 to 8 should be used for live / real-time encoding. - Lower numbers (5 or 6) are higher quality but require more CPU power. - Higher numbers (7 or 8) will be lower quality but more manageable for lower latency - use cases and also for lower CPU power devices such as mobile. - */ - call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, speed,)); - // set row level multi-threading - /* - as some people in comments and below have already commented, - more recent versions of libvpx support -row-mt 1 to enable tile row - multi-threading. This can increase the number of tiles by up to 4x in VP9 - (since the max number of tile rows is 4, regardless of video height). - To enable this, use -tile-rows N where N is the number of tile rows in - log2 units (so -tile-rows 1 means 2 tile rows and -tile-rows 2 means 4 tile - rows). The total number of active threads will then be equal to - $tile_rows * $tile_columns - */ - call_vpx!(vpx_codec_control_( - &mut ctx, - VP9E_SET_ROW_MT as _, - 1 as c_int - )); - - call_vpx!(vpx_codec_control_( - &mut ctx, - VP9E_SET_TILE_COLUMNS as _, - 4 as c_int - )); - } - - Ok(Self { - ctx, - width: config.width as _, - height: config.height as _, - }) - } - - pub fn encode(&mut self, pts: i64, data: &[u8], stride_align: usize) -> Result { - assert!(2 * data.len() >= 3 * self.width * self.height); - - let mut image = Default::default(); - call_vpx_ptr!(vpx_img_wrap( - &mut image, - vpx_img_fmt::VPX_IMG_FMT_I420, - self.width as _, - self.height as _, - stride_align as _, - data.as_ptr() as _, - )); - - call_vpx!(vpx_codec_encode( - &mut self.ctx, - &image, - pts as _, - 1, // Duration - 0, // Flags - VPX_DL_REALTIME as _, - )); - - Ok(EncodeFrames { - ctx: &mut self.ctx, - iter: ptr::null(), - }) - } - - /// Notify the encoder to return any pending packets - pub fn flush(&mut self) -> Result { - call_vpx!(vpx_codec_encode( - &mut self.ctx, - ptr::null(), - -1, // PTS - 1, // Duration - 0, // Flags - VPX_DL_REALTIME as _, - )); - - Ok(EncodeFrames { - ctx: &mut self.ctx, - iter: ptr::null(), - }) - } -} - -impl Drop for Encoder { - fn drop(&mut self) { - unsafe { - let result = vpx_codec_destroy(&mut self.ctx); - if result != VPX_CODEC_OK { - panic!("failed to destroy vpx codec"); - } - } - } -} - -#[derive(Clone, Copy, Debug)] -pub struct EncodeFrame<'a> { - /// Compressed data. - pub data: &'a [u8], - /// Whether the frame is a keyframe. - pub key: bool, - /// Presentation timestamp (in timebase units). - pub pts: i64, -} - -#[derive(Clone, Copy, Debug)] -pub struct Config { - /// The width (in pixels). - pub width: c_uint, - /// The height (in pixels). - pub height: c_uint, - /// The timebase numerator and denominator (in seconds). - pub timebase: [c_int; 2], - /// The target bitrate (in kilobits per second). - pub bitrate: c_uint, - /// The codec - pub codec: VideoCodecId, - pub rc_min_quantizer: u32, - pub rc_max_quantizer: u32, - pub speed: i32, -} - -pub struct EncodeFrames<'a> { - ctx: &'a mut vpx_codec_ctx_t, - iter: vpx_codec_iter_t, -} - -impl<'a> Iterator for EncodeFrames<'a> { - type Item = EncodeFrame<'a>; - fn next(&mut self) -> Option { - loop { - unsafe { - let pkt = vpx_codec_get_cx_data(self.ctx, &mut self.iter); - if pkt.is_null() { - return None; - } else if (*pkt).kind == vpx_codec_cx_pkt_kind::VPX_CODEC_CX_FRAME_PKT { - let f = &(*pkt).data.frame; - return Some(Self::Item { - data: slice::from_raw_parts(f.buf as _, f.sz as _), - key: (f.flags & VPX_FRAME_IS_KEY) != 0, - pts: f.pts, - }); - } else { - // Ignore the packet. - } - } - } - } -} - -impl Decoder { - /// Create a new decoder - /// - /// # Errors - /// - /// The function may fail if the underlying libvpx does not provide - /// the VP9 decoder. - pub fn new(codec: VideoCodecId, num_threads: u32) -> Result { - // This is sound because `vpx_codec_ctx` is a repr(C) struct without any field that can - // cause UB if uninitialized. - let i; - if cfg!(feature = "VP8") { - i = match codec { - VideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()), - VideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()), - }; - } else { - i = call_vpx_ptr!(vpx_codec_vp9_dx()); - } - let mut ctx = Default::default(); - let cfg = vpx_codec_dec_cfg_t { - threads: if num_threads == 0 { - num_cpus::get() as _ - } else { - num_threads - }, - w: 0, - h: 0, - }; - /* - unsafe { - println!("{}", vpx_codec_get_caps(i)); - } - */ - call_vpx!(vpx_codec_dec_init_ver( - &mut ctx, - i, - &cfg, - 0, - VPX_DECODER_ABI_VERSION as _, - )); - Ok(Self { ctx }) - } - - pub fn decode2rgb(&mut self, data: &[u8], rgba: bool) -> Result> { - let mut img = Image::new(); - for frame in self.decode(data)? { - drop(img); - img = frame; - } - for frame in self.flush()? { - drop(img); - img = frame; - } - if img.is_null() { - Ok(Vec::new()) - } else { - let mut out = Default::default(); - img.rgb(1, rgba, &mut out); - Ok(out) - } - } - - /// Feed some compressed data to the encoder - /// - /// The `data` slice is sent to the decoder - /// - /// It matches a call to `vpx_codec_decode`. - pub fn decode(&mut self, data: &[u8]) -> Result { - call_vpx!(vpx_codec_decode( - &mut self.ctx, - data.as_ptr(), - data.len() as _, - ptr::null_mut(), - 0, - )); - - Ok(DecodeFrames { - ctx: &mut self.ctx, - iter: ptr::null(), - }) - } - - /// Notify the decoder to return any pending frame - pub fn flush(&mut self) -> Result { - call_vpx!(vpx_codec_decode( - &mut self.ctx, - ptr::null(), - 0, - ptr::null_mut(), - 0 - )); - Ok(DecodeFrames { - ctx: &mut self.ctx, - iter: ptr::null(), - }) - } -} - -impl Drop for Decoder { - fn drop(&mut self) { - unsafe { - let result = vpx_codec_destroy(&mut self.ctx); - if result != VPX_CODEC_OK { - panic!("failed to destroy vpx codec"); - } - } - } -} - -pub struct DecodeFrames<'a> { - ctx: &'a mut vpx_codec_ctx_t, - iter: vpx_codec_iter_t, -} - -impl<'a> Iterator for DecodeFrames<'a> { - type Item = Image; - fn next(&mut self) -> Option { - let img = unsafe { vpx_codec_get_frame(self.ctx, &mut self.iter) }; - if img.is_null() { - return None; - } else { - return Some(Image(img)); - } - } -} - -// https://chromium.googlesource.com/webm/libvpx/+/bali/vpx/src/vpx_image.c -pub struct Image(*mut vpx_image_t); -impl Image { - #[inline] - pub fn new() -> Self { - Self(std::ptr::null_mut()) - } - - #[inline] - pub fn is_null(&self) -> bool { - self.0.is_null() - } - - #[inline] - pub fn width(&self) -> usize { - self.inner().d_w as _ - } - - #[inline] - pub fn height(&self) -> usize { - self.inner().d_h as _ - } - - #[inline] - pub fn format(&self) -> vpx_img_fmt_t { - // VPX_IMG_FMT_I420 - self.inner().fmt - } - - #[inline] - pub fn inner(&self) -> &vpx_image_t { - unsafe { &*self.0 } - } - - #[inline] - pub fn stride(&self, iplane: usize) -> i32 { - self.inner().stride[iplane] - } - - pub fn rgb(&self, stride_align: usize, rgba: bool, dst: &mut Vec) { - let h = self.height(); - let mut w = self.width(); - let bps = if rgba { 4 } else { 3 }; - w = (w + stride_align - 1) & !(stride_align - 1); - dst.resize(h * w * bps, 0); - let img = self.inner(); - unsafe { - if rgba { - super::I420ToARGB( - img.planes[0], - img.stride[0], - img.planes[1], - img.stride[1], - img.planes[2], - img.stride[2], - dst.as_mut_ptr(), - (w * bps) as _, - self.width() as _, - self.height() as _, - ); - } else { - super::I420ToRAW( - img.planes[0], - img.stride[0], - img.planes[1], - img.stride[1], - img.planes[2], - img.stride[2], - dst.as_mut_ptr(), - (w * bps) as _, - self.width() as _, - self.height() as _, - ); - } - } - } - - #[inline] - pub fn data(&self) -> (&[u8], &[u8], &[u8]) { - unsafe { - let img = self.inner(); - let h = (img.d_h as usize + 1) & !1; - let n = img.stride[0] as usize * h; - let y = slice::from_raw_parts(img.planes[0], n); - let n = img.stride[1] as usize * (h >> 1); - let u = slice::from_raw_parts(img.planes[1], n); - let v = slice::from_raw_parts(img.planes[2], n); - (y, u, v) - } - } -} - -impl Drop for Image { - fn drop(&mut self) { - if !self.0.is_null() { - unsafe { vpx_img_free(self.0) }; - } - } -} - -unsafe impl Send for vpx_codec_ctx_t {} +use std::ops::{Deref, DerefMut}; +#[cfg(feature = "hwcodec")] +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +#[cfg(feature = "hwcodec")] +use crate::hwcodec::*; +use crate::vpxcodec::*; + +use hbb_common::{ + anyhow::anyhow, + log, + message_proto::{video_frame, EncodedVideoFrames, Message, VideoCodecState}, + ResultType, +}; +#[cfg(feature = "hwcodec")] +use hbb_common::{config::Config2, lazy_static}; + +#[cfg(feature = "hwcodec")] +lazy_static::lazy_static! { + static ref PEER_DECODER_STATES: Arc>> = Default::default(); + static ref MY_DECODER_STATE: Arc> = Default::default(); +} +const SCORE_VPX: i32 = 90; + +#[derive(Debug, Clone)] +pub struct HwEncoderConfig { + pub codec_name: String, + pub width: usize, + pub height: usize, + pub bitrate: i32, +} + +#[derive(Debug, Clone)] +pub enum EncoderCfg { + VPX(VpxEncoderConfig), + HW(HwEncoderConfig), +} + +pub trait EncoderApi { + fn new(cfg: EncoderCfg) -> ResultType + where + Self: Sized; + + fn encode_to_message(&mut self, frame: &[u8], ms: i64) -> ResultType; + + fn use_yuv(&self) -> bool; + + fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>; +} + +pub struct DecoderCfg { + pub vpx: VpxDecoderConfig, +} + +pub struct Encoder { + pub codec: Box, +} + +impl Deref for Encoder { + type Target = Box; + + fn deref(&self) -> &Self::Target { + &self.codec + } +} + +impl DerefMut for Encoder { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.codec + } +} + +pub struct Decoder { + vpx: VpxDecoder, + #[cfg(feature = "hwcodec")] + hw: HwDecoders, + #[cfg(feature = "hwcodec")] + i420: Vec, +} + +#[derive(Debug, Clone)] +pub enum EncoderUpdate { + State(VideoCodecState), + Remove, + DisableHwIfNotExist, +} + +impl Encoder { + pub fn new(config: EncoderCfg) -> ResultType { + log::info!("new encoder:{:?}", config); + match config { + EncoderCfg::VPX(_) => Ok(Encoder { + codec: Box::new(VpxEncoder::new(config)?), + }), + + #[cfg(feature = "hwcodec")] + EncoderCfg::HW(_) => match HwEncoder::new(config) { + Ok(hw) => Ok(Encoder { + codec: Box::new(hw), + }), + Err(e) => { + HwEncoder::best(true, true); + Err(e) + } + }, + #[cfg(not(feature = "hwcodec"))] + _ => Err(anyhow!("unsupported encoder type")), + } + } + + // TODO + pub fn update_video_encoder(id: i32, update: EncoderUpdate) { + log::info!("encoder update: {:?}", update); + #[cfg(feature = "hwcodec")] + { + let mut states = PEER_DECODER_STATES.lock().unwrap(); + match update { + EncoderUpdate::State(state) => { + states.insert(id, state); + } + EncoderUpdate::Remove => { + states.remove(&id); + } + EncoderUpdate::DisableHwIfNotExist => { + if !states.contains_key(&id) { + states.insert(id, VideoCodecState::default()); + } + } + } + let current_encoder_name = HwEncoder::current_name(); + if states.len() > 0 { + let (best, _) = HwEncoder::best(false, true); + let enabled_h264 = best.h264.is_some() + && states.len() > 0 + && states.iter().all(|(_, s)| s.ScoreH264 > 0); + let enabled_h265 = best.h265.is_some() + && states.len() > 0 + && states.iter().all(|(_, s)| s.ScoreH265 > 0); + + // score encoder + let mut score_vpx = SCORE_VPX; + let mut score_h264 = best.h264.as_ref().map_or(0, |c| c.score); + let mut score_h265 = best.h265.as_ref().map_or(0, |c| c.score); + + // score decoder + score_vpx += states.iter().map(|s| s.1.ScoreVpx).sum::(); + if enabled_h264 { + score_h264 += states.iter().map(|s| s.1.ScoreH264).sum::(); + } + if enabled_h265 { + score_h265 += states.iter().map(|s| s.1.ScoreH265).sum::(); + } + + if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 { + *current_encoder_name.lock().unwrap() = Some(best.h265.unwrap().name); + } else if enabled_h264 && score_h264 >= score_vpx && score_h264 >= score_h265 { + *current_encoder_name.lock().unwrap() = Some(best.h264.unwrap().name); + } else { + *current_encoder_name.lock().unwrap() = None; + } + log::info!( + "connection count:{}, h264:{}, h265:{}, score: vpx({}), h264({}), h265({}), set current encoder name {:?}", + states.len(), + enabled_h264, + enabled_h265, + score_vpx, + score_h264, + score_h265, + current_encoder_name.lock().unwrap() + ) + } else { + *current_encoder_name.lock().unwrap() = None; + } + } + #[cfg(not(feature = "hwcodec"))] + { + let _ = id; + let _ = update; + } + } + #[inline] + pub fn current_hw_encoder_name() -> Option { + #[cfg(feature = "hwcodec")] + if check_hwcodec_config() { + return HwEncoder::current_name().lock().unwrap().clone(); + } else { + return None; + } + #[cfg(not(feature = "hwcodec"))] + return None; + } +} + +#[cfg(feature = "hwcodec")] +impl Drop for Decoder { + fn drop(&mut self) { + *MY_DECODER_STATE.lock().unwrap() = VideoCodecState { + ScoreVpx: SCORE_VPX, + ..Default::default() + }; + } +} + +impl Decoder { + pub fn video_codec_state() -> VideoCodecState { + // video_codec_state is mainted by creation and destruction of Decoder. + // It has been ensured to use after Decoder's creation. + #[cfg(feature = "hwcodec")] + if check_hwcodec_config() { + return MY_DECODER_STATE.lock().unwrap().clone(); + } else { + return VideoCodecState { + ScoreVpx: SCORE_VPX, + ..Default::default() + }; + } + #[cfg(not(feature = "hwcodec"))] + VideoCodecState { + ScoreVpx: SCORE_VPX, + ..Default::default() + } + } + + pub fn new(config: DecoderCfg) -> Decoder { + let vpx = VpxDecoder::new(config.vpx).unwrap(); + let decoder = Decoder { + vpx, + #[cfg(feature = "hwcodec")] + hw: HwDecoder::new_decoders(), + #[cfg(feature = "hwcodec")] + i420: vec![], + }; + + #[cfg(feature = "hwcodec")] + { + let mut state = MY_DECODER_STATE.lock().unwrap(); + state.ScoreVpx = SCORE_VPX; + state.ScoreH264 = decoder.hw.h264.as_ref().map_or(0, |d| d.info.score); + state.ScoreH265 = decoder.hw.h265.as_ref().map_or(0, |d| d.info.score); + } + + decoder + } + + pub fn handle_video_frame( + &mut self, + frame: &video_frame::Union, + rgb: &mut Vec, + ) -> ResultType { + match frame { + video_frame::Union::vp9s(vp9s) => { + Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, rgb) + } + #[cfg(feature = "hwcodec")] + video_frame::Union::h264s(h264s) => { + if let Some(decoder) = &mut self.hw.h264 { + Decoder::handle_hw_video_frame(decoder, h264s, rgb, &mut self.i420) + } else { + Err(anyhow!("don't support h264!")) + } + } + #[cfg(feature = "hwcodec")] + video_frame::Union::h265s(h265s) => { + if let Some(decoder) = &mut self.hw.h265 { + Decoder::handle_hw_video_frame(decoder, h265s, rgb, &mut self.i420) + } else { + Err(anyhow!("don't support h265!")) + } + } + _ => Err(anyhow!("unsupported video frame type!")), + } + } + + fn handle_vp9s_video_frame( + decoder: &mut VpxDecoder, + vp9s: &EncodedVideoFrames, + rgb: &mut Vec, + ) -> ResultType { + let mut last_frame = Image::new(); + for vp9 in vp9s.frames.iter() { + for frame in decoder.decode(&vp9.data)? { + drop(last_frame); + last_frame = frame; + } + } + for frame in decoder.flush()? { + drop(last_frame); + last_frame = frame; + } + if last_frame.is_null() { + Ok(false) + } else { + last_frame.rgb(1, true, rgb); + Ok(true) + } + } + + #[cfg(feature = "hwcodec")] + fn handle_hw_video_frame( + decoder: &mut HwDecoder, + frames: &EncodedVideoFrames, + rgb: &mut Vec, + i420: &mut Vec, + ) -> ResultType { + let mut ret = false; + for h264 in frames.frames.iter() { + for image in decoder.decode(&h264.data)? { + // TODO: just process the last frame + if image.bgra(rgb, i420).is_ok() { + ret = true; + } + } + } + return Ok(ret); + } +} + +#[cfg(feature = "hwcodec")] +fn check_hwcodec_config() -> bool { + if let Some(v) = Config2::get().options.get("enable-hwcodec") { + return v != "N"; + } + return true; // default is true +} diff --git a/libs/scrap/src/common/convert.rs b/libs/scrap/src/common/convert.rs index 1e5f2164d..306a217ea 100644 --- a/libs/scrap/src/common/convert.rs +++ b/libs/scrap/src/common/convert.rs @@ -49,6 +49,17 @@ extern "C" { height: c_int, ) -> c_int; + pub fn ARGBToNV12( + src_bgra: *const u8, + src_stride_bgra: c_int, + dst_y: *mut u8, + dst_stride_y: c_int, + dst_uv: *mut u8, + dst_stride_uv: c_int, + width: c_int, + height: c_int, + ) -> c_int; + pub fn NV12ToI420( src_y: *const u8, src_stride_y: c_int, @@ -91,6 +102,17 @@ extern "C" { width: c_int, height: c_int, ) -> c_int; + + pub fn NV12ToARGB( + src_y: *const u8, + src_stride_y: c_int, + src_uv: *const u8, + src_stride_uv: c_int, + dst_rgba: *mut u8, + dst_stride_rgba: c_int, + width: c_int, + height: c_int, + ) -> c_int; } // https://github.com/webmproject/libvpx/blob/master/vpx/src/vpx_image.c @@ -220,3 +242,192 @@ pub unsafe fn nv12_to_i420( height as _, ); } + +#[cfg(feature = "hwcodec")] +pub mod hw { + use hbb_common::{anyhow::anyhow, ResultType}; + use hwcodec::{ffmpeg::ffmpeg_linesize_offset_length, AVPixelFormat}; + + pub fn hw_bgra_to_i420( + width: usize, + height: usize, + stride: &[i32], + offset: &[i32], + length: i32, + src: &[u8], + dst: &mut Vec, + ) { + let stride_y = stride[0] as usize; + let stride_u = stride[1] as usize; + let stride_v = stride[2] as usize; + let offset_u = offset[0] as usize; + let offset_v = offset[1] as usize; + + dst.resize(length as _, 0); + let dst_y = dst.as_mut_ptr(); + let dst_u = dst[offset_u..].as_mut_ptr(); + let dst_v = dst[offset_v..].as_mut_ptr(); + unsafe { + super::ARGBToI420( + src.as_ptr(), + (src.len() / height) as _, + dst_y, + stride_y as _, + dst_u, + stride_u as _, + dst_v, + stride_v as _, + width as _, + height as _, + ); + } + } + + pub fn hw_bgra_to_nv12( + width: usize, + height: usize, + stride: &[i32], + offset: &[i32], + length: i32, + src: &[u8], + dst: &mut Vec, + ) { + let stride_y = stride[0] as usize; + let stride_uv = stride[1] as usize; + let offset_uv = offset[0] as usize; + + dst.resize(length as _, 0); + let dst_y = dst.as_mut_ptr(); + let dst_uv = dst[offset_uv..].as_mut_ptr(); + unsafe { + super::ARGBToNV12( + src.as_ptr(), + (src.len() / height) as _, + dst_y, + stride_y as _, + dst_uv, + stride_uv as _, + width as _, + height as _, + ); + } + } + + #[cfg(target_os = "windows")] + pub fn hw_nv12_to_bgra( + width: usize, + height: usize, + src_y: &[u8], + src_uv: &[u8], + src_stride_y: usize, + src_stride_uv: usize, + dst: &mut Vec, + i420: &mut Vec, + align: usize, + ) -> ResultType<()> { + let nv12_stride_y = src_stride_y; + let nv12_stride_uv = src_stride_uv; + if let Ok((linesize_i420, offset_i420, i420_len)) = + ffmpeg_linesize_offset_length(AVPixelFormat::AV_PIX_FMT_YUV420P, width, height, align) + { + dst.resize(width * height * 4, 0); + let i420_stride_y = linesize_i420[0]; + let i420_stride_u = linesize_i420[1]; + let i420_stride_v = linesize_i420[2]; + i420.resize(i420_len as _, 0); + + unsafe { + let i420_offset_y = i420.as_ptr().add(0) as _; + let i420_offset_u = i420.as_ptr().add(offset_i420[0] as _) as _; + let i420_offset_v = i420.as_ptr().add(offset_i420[1] as _) as _; + super::NV12ToI420( + src_y.as_ptr(), + nv12_stride_y as _, + src_uv.as_ptr(), + nv12_stride_uv as _, + i420_offset_y, + i420_stride_y, + i420_offset_u, + i420_stride_u, + i420_offset_v, + i420_stride_v, + width as _, + height as _, + ); + super::I420ToARGB( + i420_offset_y, + i420_stride_y, + i420_offset_u, + i420_stride_u, + i420_offset_v, + i420_stride_v, + dst.as_mut_ptr(), + (width * 4) as _, + width as _, + height as _, + ); + return Ok(()); + }; + } + return Err(anyhow!("get linesize offset failed")); + } + + #[cfg(not(target_os = "windows"))] + pub fn hw_nv12_to_bgra( + width: usize, + height: usize, + src_y: &[u8], + src_uv: &[u8], + src_stride_y: usize, + src_stride_uv: usize, + dst: &mut Vec, + ) -> ResultType<()> { + dst.resize(width * height * 4, 0); + unsafe { + match super::NV12ToARGB( + src_y.as_ptr(), + src_stride_y as _, + src_uv.as_ptr(), + src_stride_uv as _, + dst.as_mut_ptr(), + (width * 4) as _, + width as _, + height as _, + ) { + 0 => Ok(()), + _ => Err(anyhow!("NV12ToARGB failed")), + } + } + } + + pub fn hw_i420_to_bgra( + width: usize, + height: usize, + src_y: &[u8], + src_u: &[u8], + src_v: &[u8], + src_stride_y: usize, + src_stride_u: usize, + src_stride_v: usize, + dst: &mut Vec, + ) { + let src_y = src_y.as_ptr(); + let src_u = src_u.as_ptr(); + let src_v = src_v.as_ptr(); + dst.resize(width * height * 4, 0); + unsafe { + super::I420ToARGB( + src_y, + src_stride_y as _, + src_u, + src_stride_u as _, + src_v, + src_stride_v as _, + dst.as_mut_ptr(), + (width * 4) as _, + width as _, + height as _, + ); + }; + } +} diff --git a/libs/scrap/src/common/dxgi.rs b/libs/scrap/src/common/dxgi.rs index c0b4130bb..1a8c39885 100644 --- a/libs/scrap/src/common/dxgi.rs +++ b/libs/scrap/src/common/dxgi.rs @@ -1,5 +1,6 @@ use crate::dxgi; use std::io::ErrorKind::{NotFound, TimedOut, WouldBlock}; +use std::time::Duration; use std::{io, ops}; pub struct Capturer { @@ -40,8 +41,8 @@ impl Capturer { self.height } - pub fn frame<'a>(&'a mut self, timeout_ms: u32) -> io::Result> { - match self.inner.frame(timeout_ms) { + pub fn frame<'a>(&'a mut self, timeout_ms: Duration) -> io::Result> { + match self.inner.frame(timeout_ms.as_millis() as _) { Ok(frame) => Ok(Frame(frame)), Err(ref error) if error.kind() == TimedOut => Err(WouldBlock.into()), Err(error) => Err(error), @@ -135,7 +136,7 @@ impl CapturerMag { pub fn get_rect(&self) -> ((i32, i32), usize, usize) { self.inner.get_rect() } - pub fn frame<'a>(&'a mut self, _timeout_ms: u32) -> io::Result> { + pub fn frame<'a>(&'a mut self, _timeout_ms: Duration) -> io::Result> { self.inner.frame(&mut self.data)?; Ok(Frame(&self.data)) } diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs new file mode 100644 index 000000000..ff23978de --- /dev/null +++ b/libs/scrap/src/common/hwcodec.rs @@ -0,0 +1,344 @@ +use crate::{ + codec::{EncoderApi, EncoderCfg}, + hw, HW_STRIDE_ALIGN, +}; +use hbb_common::{ + anyhow::{anyhow, Context}, + config::HwCodecConfig, + lazy_static, log, + message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame}, + ResultType, +}; +use hwcodec::{ + decode::{DecodeContext, DecodeFrame, Decoder}, + encode::{EncodeContext, EncodeFrame, Encoder}, + ffmpeg::{CodecInfo, CodecInfos, DataFormat}, + AVPixelFormat, + Quality::{self, *}, + RateContorl::{self, *}, +}; +use std::sync::{Arc, Mutex}; + +lazy_static::lazy_static! { + static ref HW_ENCODER_NAME: Arc>> = Default::default(); +} + +const CFG_KEY_ENCODER: &str = "bestHwEncoders"; +const CFG_KEY_DECODER: &str = "bestHwDecoders"; + +const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_YUV420P; +const DEFAULT_TIME_BASE: [i32; 2] = [1, 30]; +const DEFAULT_GOP: i32 = 60; +const DEFAULT_HW_QUALITY: Quality = Quality_Default; +const DEFAULT_RC: RateContorl = RC_DEFAULT; + +pub struct HwEncoder { + encoder: Encoder, + yuv: Vec, + pub format: DataFormat, + pub pixfmt: AVPixelFormat, +} + +impl EncoderApi for HwEncoder { + fn new(cfg: EncoderCfg) -> ResultType + where + Self: Sized, + { + match cfg { + EncoderCfg::HW(config) => { + let ctx = EncodeContext { + name: config.codec_name.clone(), + width: config.width as _, + height: config.height as _, + pixfmt: DEFAULT_PIXFMT, + align: HW_STRIDE_ALIGN as _, + bitrate: config.bitrate * 1000, + timebase: DEFAULT_TIME_BASE, + gop: DEFAULT_GOP, + quality: DEFAULT_HW_QUALITY, + rc: DEFAULT_RC, + }; + let format = match Encoder::format_from_name(config.codec_name.clone()) { + Ok(format) => format, + Err(_) => { + return Err(anyhow!(format!( + "failed to get format from name:{}", + config.codec_name + ))) + } + }; + match Encoder::new(ctx.clone()) { + Ok(encoder) => Ok(HwEncoder { + encoder, + yuv: vec![], + format, + pixfmt: ctx.pixfmt, + }), + Err(_) => Err(anyhow!(format!("Failed to create encoder"))), + } + } + _ => Err(anyhow!("encoder type mismatch")), + } + } + + fn encode_to_message( + &mut self, + frame: &[u8], + _ms: i64, + ) -> ResultType { + let mut msg_out = Message::new(); + let mut vf = VideoFrame::new(); + let mut frames = Vec::new(); + for frame in self.encode(frame).with_context(|| "Failed to encode")? { + frames.push(EncodedVideoFrame { + data: frame.data, + pts: frame.pts as _, + ..Default::default() + }); + } + if frames.len() > 0 { + let frames = EncodedVideoFrames { + frames: frames.into(), + ..Default::default() + }; + match self.format { + DataFormat::H264 => vf.set_h264s(frames), + DataFormat::H265 => vf.set_h265s(frames), + } + msg_out.set_video_frame(vf); + Ok(msg_out) + } else { + Err(anyhow!("no valid frame")) + } + } + + fn use_yuv(&self) -> bool { + false + } + + fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> { + self.encoder.set_bitrate((bitrate * 1000) as _).ok(); + Ok(()) + } +} + +impl HwEncoder { + /// Get best encoders. + /// + /// # Parameter + /// `force_reset`: force to refresh config. + /// `write`: write to config file. + /// + /// # Return + /// `CodecInfos`: infos. + /// `bool`: whether the config is refreshed. + pub fn best(force_reset: bool, write: bool) -> (CodecInfos, bool) { + let config = get_config(CFG_KEY_ENCODER); + if !force_reset && config.is_ok() { + (config.unwrap(), false) + } else { + let ctx = EncodeContext { + name: String::from(""), + width: 1920, + height: 1080, + pixfmt: DEFAULT_PIXFMT, + align: HW_STRIDE_ALIGN as _, + bitrate: 0, + timebase: DEFAULT_TIME_BASE, + gop: DEFAULT_GOP, + quality: DEFAULT_HW_QUALITY, + rc: DEFAULT_RC, + }; + let encoders = CodecInfo::score(Encoder::avaliable_encoders(ctx)); + if write { + set_config(CFG_KEY_ENCODER, &encoders) + .map_err(|e| log::error!("{:?}", e)) + .ok(); + } + (encoders, true) + } + } + + pub fn current_name() -> Arc>> { + HW_ENCODER_NAME.clone() + } + + pub fn encode(&mut self, bgra: &[u8]) -> ResultType> { + match self.pixfmt { + AVPixelFormat::AV_PIX_FMT_YUV420P => hw::hw_bgra_to_i420( + self.encoder.ctx.width as _, + self.encoder.ctx.height as _, + &self.encoder.linesize, + &self.encoder.offset, + self.encoder.length, + bgra, + &mut self.yuv, + ), + AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_bgra_to_nv12( + self.encoder.ctx.width as _, + self.encoder.ctx.height as _, + &self.encoder.linesize, + &self.encoder.offset, + self.encoder.length, + bgra, + &mut self.yuv, + ), + } + + match self.encoder.encode(&self.yuv) { + Ok(v) => { + let mut data = Vec::::new(); + data.append(v); + Ok(data) + } + Err(_) => Ok(Vec::::new()), + } + } +} + +pub struct HwDecoder { + decoder: Decoder, + pub info: CodecInfo, +} + +pub struct HwDecoders { + pub h264: Option, + pub h265: Option, +} + +impl HwDecoder { + /// See HwEncoder::best + fn best(force_reset: bool, write: bool) -> (CodecInfos, bool) { + let config = get_config(CFG_KEY_DECODER); + if !force_reset && config.is_ok() { + (config.unwrap(), false) + } else { + let decoders = CodecInfo::score(Decoder::avaliable_decoders()); + if write { + set_config(CFG_KEY_DECODER, &decoders) + .map_err(|e| log::error!("{:?}", e)) + .ok(); + } + (decoders, true) + } + } + + pub fn new_decoders() -> HwDecoders { + let (best, _) = HwDecoder::best(false, true); + let mut h264: Option = None; + let mut h265: Option = None; + let mut fail = false; + + if let Some(info) = best.h264 { + h264 = HwDecoder::new(info).ok(); + if h264.is_none() { + fail = true; + } + } + if let Some(info) = best.h265 { + h265 = HwDecoder::new(info).ok(); + if h265.is_none() { + fail = true; + } + } + if fail { + HwDecoder::best(true, true); + } + HwDecoders { h264, h265 } + } + + pub fn new(info: CodecInfo) -> ResultType { + let ctx = DecodeContext { + name: info.name.clone(), + device_type: info.hwdevice.clone(), + }; + match Decoder::new(ctx) { + Ok(decoder) => Ok(HwDecoder { decoder, info }), + Err(_) => Err(anyhow!(format!("Failed to create decoder"))), + } + } + pub fn decode(&mut self, data: &[u8]) -> ResultType> { + match self.decoder.decode(data) { + Ok(v) => Ok(v.iter().map(|f| HwDecoderImage { frame: f }).collect()), + Err(_) => Ok(vec![]), + } + } +} + +pub struct HwDecoderImage<'a> { + frame: &'a DecodeFrame, +} + +impl HwDecoderImage<'_> { + pub fn bgra(&self, bgra: &mut Vec, i420: &mut Vec) -> ResultType<()> { + let frame = self.frame; + match frame.pixfmt { + AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to_bgra( + frame.width as _, + frame.height as _, + &frame.data[0], + &frame.data[1], + frame.linesize[0] as _, + frame.linesize[1] as _, + bgra, + i420, + HW_STRIDE_ALIGN, + ), + AVPixelFormat::AV_PIX_FMT_YUV420P => { + hw::hw_i420_to_bgra( + frame.width as _, + frame.height as _, + &frame.data[0], + &frame.data[1], + &frame.data[2], + frame.linesize[0] as _, + frame.linesize[1] as _, + frame.linesize[2] as _, + bgra, + ); + return Ok(()); + } + } + } +} + +fn get_config(k: &str) -> ResultType { + let v = HwCodecConfig::load() + .options + .get(k) + .unwrap_or(&"".to_owned()) + .to_owned(); + match CodecInfos::deserialize(&v) { + Ok(v) => Ok(v), + Err(_) => Err(anyhow!("Failed to get config:{}", k)), + } +} + +fn set_config(k: &str, v: &CodecInfos) -> ResultType<()> { + match v.serialize() { + Ok(v) => { + let mut config = HwCodecConfig::load(); + config.options.insert(k.to_owned(), v); + config.store(); + Ok(()) + } + Err(_) => Err(anyhow!("Failed to set config:{}", k)), + } +} + +pub fn check_config() { + let (encoders, update_encoders) = HwEncoder::best(false, false); + let (decoders, update_decoders) = HwDecoder::best(false, false); + if update_encoders || update_decoders { + if let Ok(encoders) = encoders.serialize() { + if let Ok(decoders) = decoders.serialize() { + let mut config = HwCodecConfig::load(); + config.options.insert(CFG_KEY_ENCODER.to_owned(), encoders); + config.options.insert(CFG_KEY_DECODER.to_owned(), decoders); + config.store(); + return; + } + } + log::error!("Failed to serialize codec info"); + } +} diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index 108a4ae31..792ea14e1 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -1,4 +1,4 @@ -pub use self::codec::*; +pub use self::vpxcodec::*; cfg_if! { if #[cfg(quartz)] { @@ -29,8 +29,12 @@ cfg_if! { pub mod codec; mod convert; +#[cfg(feature = "hwcodec")] +pub mod hwcodec; +pub mod vpxcodec; pub use self::convert::*; pub const STRIDE_ALIGN: usize = 64; // commonly used in libvpx vpx_img_alloc caller +pub const HW_STRIDE_ALIGN: usize = 0; // recommended by av_frame_get_buffer mod vpx; diff --git a/libs/scrap/src/common/quartz.rs b/libs/scrap/src/common/quartz.rs index b35b56b61..46ac1b56d 100644 --- a/libs/scrap/src/common/quartz.rs +++ b/libs/scrap/src/common/quartz.rs @@ -51,7 +51,7 @@ impl Capturer { self.inner.height() } - pub fn frame<'a>(&'a mut self, _timeout_ms: u32) -> io::Result> { + pub fn frame<'a>(&'a mut self, _timeout_ms: std::time::Duration) -> io::Result> { match self.frame.try_lock() { Ok(mut handle) => { let mut frame = None; diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs new file mode 100644 index 000000000..fc54b153c --- /dev/null +++ b/libs/scrap/src/common/vpxcodec.rs @@ -0,0 +1,599 @@ +// https://github.com/astraw/vpx-encode +// https://github.com/astraw/env-libvpx-sys +// https://github.com/rust-av/vpx-rs/blob/master/src/decoder.rs + +use hbb_common::anyhow::{anyhow, Context}; +use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame}; +use hbb_common::ResultType; + +use crate::codec::EncoderApi; +use crate::STRIDE_ALIGN; + +use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *}; +use std::os::raw::{c_int, c_uint}; +use std::{ptr, slice}; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum VpxVideoCodecId { + VP8, + VP9, +} + +impl Default for VpxVideoCodecId { + fn default() -> VpxVideoCodecId { + VpxVideoCodecId::VP9 + } +} + +pub struct VpxEncoder { + ctx: vpx_codec_ctx_t, + width: usize, + height: usize, +} + +pub struct VpxDecoder { + ctx: vpx_codec_ctx_t, +} + +#[derive(Debug)] +pub enum Error { + FailedCall(String), + BadPtr(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::result::Result<(), std::fmt::Error> { + write!(f, "{:?}", self) + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +macro_rules! call_vpx { + ($x:expr) => {{ + let result = unsafe { $x }; // original expression + let result_int = unsafe { std::mem::transmute::<_, i32>(result) }; + if result_int != 0 { + return Err(Error::FailedCall(format!( + "errcode={} {}:{}:{}:{}", + result_int, + module_path!(), + file!(), + line!(), + column!() + )) + .into()); + } + result + }}; +} + +macro_rules! call_vpx_ptr { + ($x:expr) => {{ + let result = unsafe { $x }; // original expression + let result_int = unsafe { std::mem::transmute::<_, isize>(result) }; + if result_int == 0 { + return Err(Error::BadPtr(format!( + "errcode={} {}:{}:{}:{}", + result_int, + module_path!(), + file!(), + line!(), + column!() + )) + .into()); + } + result + }}; +} + +impl EncoderApi for VpxEncoder { + fn new(cfg: crate::codec::EncoderCfg) -> ResultType + where + Self: Sized, + { + match cfg { + crate::codec::EncoderCfg::VPX(config) => { + let i; + if cfg!(feature = "VP8") { + i = match config.codec { + VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), + VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), + }; + } else { + i = call_vpx_ptr!(vpx_codec_vp9_cx()); + } + let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; + call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0)); + + // https://www.webmproject.org/docs/encoder-parameters/ + // default: c.rc_min_quantizer = 0, c.rc_max_quantizer = 63 + // try rc_resize_allowed later + + c.g_w = config.width; + c.g_h = config.height; + c.g_timebase.num = config.timebase[0]; + c.g_timebase.den = config.timebase[1]; + c.rc_target_bitrate = config.bitrate; + c.rc_undershoot_pct = 95; + c.rc_dropframe_thresh = 25; + c.g_threads = if config.num_threads == 0 { + num_cpus::get() as _ + } else { + config.num_threads + }; + c.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT; + // https://developers.google.com/media/vp9/bitrate-modes/ + // Constant Bitrate mode (CBR) is recommended for live streaming with VP9. + c.rc_end_usage = vpx_rc_mode::VPX_CBR; + // c.kf_min_dist = 0; + // c.kf_max_dist = 999999; + c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot + + /* + VPX encoder支持two-pass encode，这是为了rate control的。 + 对于两遍编码，就是需要整个编码过程做两次，第一次会得到一些新的控制参数来进行第二遍的编码， + 这样可以在相同的bitrate下得到最好的PSNR + */ + + let mut ctx = Default::default(); + call_vpx!(vpx_codec_enc_init_ver( + &mut ctx, + i, + &c, + 0, + VPX_ENCODER_ABI_VERSION as _ + )); + + if config.codec == VpxVideoCodecId::VP9 { + // set encoder internal speed settings + // in ffmpeg, it is --speed option + /* + set to 0 or a positive value 1-16, the codec will try to adapt its + complexity depending on the time it spends encoding. Increasing this + number will make the speed go up and the quality go down. + Negative values mean strict enforcement of this + while positive values are adaptive + */ + /* https://developers.google.com/media/vp9/live-encoding + Speed 5 to 8 should be used for live / real-time encoding. + Lower numbers (5 or 6) are higher quality but require more CPU power. + Higher numbers (7 or 8) will be lower quality but more manageable for lower latency + use cases and also for lower CPU power devices such as mobile. + */ + call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, 7,)); + // set row level multi-threading + /* + as some people in comments and below have already commented, + more recent versions of libvpx support -row-mt 1 to enable tile row + multi-threading. This can increase the number of tiles by up to 4x in VP9 + (since the max number of tile rows is 4, regardless of video height). + To enable this, use -tile-rows N where N is the number of tile rows in + log2 units (so -tile-rows 1 means 2 tile rows and -tile-rows 2 means 4 tile + rows). The total number of active threads will then be equal to + $tile_rows * $tile_columns + */ + call_vpx!(vpx_codec_control_( + &mut ctx, + VP9E_SET_ROW_MT as _, + 1 as c_int + )); + + call_vpx!(vpx_codec_control_( + &mut ctx, + VP9E_SET_TILE_COLUMNS as _, + 4 as c_int + )); + } + + Ok(Self { + ctx, + width: config.width as _, + height: config.height as _, + }) + } + _ => Err(anyhow!("encoder type mismatch")), + } + } + + fn encode_to_message(&mut self, frame: &[u8], ms: i64) -> ResultType { + let mut frames = Vec::new(); + for ref frame in self + .encode(ms, frame, STRIDE_ALIGN) + .with_context(|| "Failed to encode")? + { + frames.push(VpxEncoder::create_frame(frame)); + } + for ref frame in self.flush().with_context(|| "Failed to flush")? { + frames.push(VpxEncoder::create_frame(frame)); + } + + // to-do: flush periodically, e.g. 1 second + if frames.len() > 0 { + Ok(VpxEncoder::create_msg(frames)) + } else { + Err(anyhow!("no valid frame")) + } + } + + fn use_yuv(&self) -> bool { + true + } + + fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> { + let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() }; + new_enc_cfg.rc_target_bitrate = bitrate; + call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &new_enc_cfg)); + return Ok(()); + } +} + +impl VpxEncoder { + pub fn encode(&mut self, pts: i64, data: &[u8], stride_align: usize) -> Result { + assert!(2 * data.len() >= 3 * self.width * self.height); + + let mut image = Default::default(); + call_vpx_ptr!(vpx_img_wrap( + &mut image, + vpx_img_fmt::VPX_IMG_FMT_I420, + self.width as _, + self.height as _, + stride_align as _, + data.as_ptr() as _, + )); + + call_vpx!(vpx_codec_encode( + &mut self.ctx, + &image, + pts as _, + 1, // Duration + 0, // Flags + VPX_DL_REALTIME as _, + )); + + Ok(EncodeFrames { + ctx: &mut self.ctx, + iter: ptr::null(), + }) + } + + /// Notify the encoder to return any pending packets + pub fn flush(&mut self) -> Result { + call_vpx!(vpx_codec_encode( + &mut self.ctx, + ptr::null(), + -1, // PTS + 1, // Duration + 0, // Flags + VPX_DL_REALTIME as _, + )); + + Ok(EncodeFrames { + ctx: &mut self.ctx, + iter: ptr::null(), + }) + } + + #[inline] + fn create_msg(vp9s: Vec) -> Message { + let mut msg_out = Message::new(); + let mut vf = VideoFrame::new(); + vf.set_vp9s(EncodedVideoFrames { + frames: vp9s.into(), + ..Default::default() + }); + msg_out.set_video_frame(vf); + msg_out + } + + #[inline] + fn create_frame(frame: &EncodeFrame) -> EncodedVideoFrame { + EncodedVideoFrame { + data: frame.data.to_vec(), + key: frame.key, + pts: frame.pts, + ..Default::default() + } + } +} + +impl Drop for VpxEncoder { + fn drop(&mut self) { + unsafe { + let result = vpx_codec_destroy(&mut self.ctx); + if result != VPX_CODEC_OK { + panic!("failed to destroy vpx codec"); + } + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct EncodeFrame<'a> { + /// Compressed data. + pub data: &'a [u8], + /// Whether the frame is a keyframe. + pub key: bool, + /// Presentation timestamp (in timebase units). + pub pts: i64, +} + +#[derive(Clone, Copy, Debug)] +pub struct VpxEncoderConfig { + /// The width (in pixels). + pub width: c_uint, + /// The height (in pixels). + pub height: c_uint, + /// The timebase numerator and denominator (in seconds). + pub timebase: [c_int; 2], + /// The target bitrate (in kilobits per second). + pub bitrate: c_uint, + /// The codec + pub codec: VpxVideoCodecId, + pub num_threads: u32, +} + +#[derive(Clone, Copy, Debug)] +pub struct VpxDecoderConfig { + pub codec: VpxVideoCodecId, + pub num_threads: u32, +} + +pub struct EncodeFrames<'a> { + ctx: &'a mut vpx_codec_ctx_t, + iter: vpx_codec_iter_t, +} + +impl<'a> Iterator for EncodeFrames<'a> { + type Item = EncodeFrame<'a>; + fn next(&mut self) -> Option { + loop { + unsafe { + let pkt = vpx_codec_get_cx_data(self.ctx, &mut self.iter); + if pkt.is_null() { + return None; + } else if (*pkt).kind == vpx_codec_cx_pkt_kind::VPX_CODEC_CX_FRAME_PKT { + let f = &(*pkt).data.frame; + return Some(Self::Item { + data: slice::from_raw_parts(f.buf as _, f.sz as _), + key: (f.flags & VPX_FRAME_IS_KEY) != 0, + pts: f.pts, + }); + } else { + // Ignore the packet. + } + } + } + } +} + +impl VpxDecoder { + /// Create a new decoder + /// + /// # Errors + /// + /// The function may fail if the underlying libvpx does not provide + /// the VP9 decoder. + pub fn new(config: VpxDecoderConfig) -> Result { + // This is sound because `vpx_codec_ctx` is a repr(C) struct without any field that can + // cause UB if uninitialized. + let i; + if cfg!(feature = "VP8") { + i = match config.codec { + VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()), + VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()), + }; + } else { + i = call_vpx_ptr!(vpx_codec_vp9_dx()); + } + let mut ctx = Default::default(); + let cfg = vpx_codec_dec_cfg_t { + threads: if config.num_threads == 0 { + num_cpus::get() as _ + } else { + config.num_threads + }, + w: 0, + h: 0, + }; + /* + unsafe { + println!("{}", vpx_codec_get_caps(i)); + } + */ + call_vpx!(vpx_codec_dec_init_ver( + &mut ctx, + i, + &cfg, + 0, + VPX_DECODER_ABI_VERSION as _, + )); + Ok(Self { ctx }) + } + + pub fn decode2rgb(&mut self, data: &[u8], rgba: bool) -> Result> { + let mut img = Image::new(); + for frame in self.decode(data)? { + drop(img); + img = frame; + } + for frame in self.flush()? { + drop(img); + img = frame; + } + if img.is_null() { + Ok(Vec::new()) + } else { + let mut out = Default::default(); + img.rgb(1, rgba, &mut out); + Ok(out) + } + } + + /// Feed some compressed data to the encoder + /// + /// The `data` slice is sent to the decoder + /// + /// It matches a call to `vpx_codec_decode`. + pub fn decode(&mut self, data: &[u8]) -> Result { + call_vpx!(vpx_codec_decode( + &mut self.ctx, + data.as_ptr(), + data.len() as _, + ptr::null_mut(), + 0, + )); + + Ok(DecodeFrames { + ctx: &mut self.ctx, + iter: ptr::null(), + }) + } + + /// Notify the decoder to return any pending frame + pub fn flush(&mut self) -> Result { + call_vpx!(vpx_codec_decode( + &mut self.ctx, + ptr::null(), + 0, + ptr::null_mut(), + 0 + )); + Ok(DecodeFrames { + ctx: &mut self.ctx, + iter: ptr::null(), + }) + } +} + +impl Drop for VpxDecoder { + fn drop(&mut self) { + unsafe { + let result = vpx_codec_destroy(&mut self.ctx); + if result != VPX_CODEC_OK { + panic!("failed to destroy vpx codec"); + } + } + } +} + +pub struct DecodeFrames<'a> { + ctx: &'a mut vpx_codec_ctx_t, + iter: vpx_codec_iter_t, +} + +impl<'a> Iterator for DecodeFrames<'a> { + type Item = Image; + fn next(&mut self) -> Option { + let img = unsafe { vpx_codec_get_frame(self.ctx, &mut self.iter) }; + if img.is_null() { + return None; + } else { + return Some(Image(img)); + } + } +} + +// https://chromium.googlesource.com/webm/libvpx/+/bali/vpx/src/vpx_image.c +pub struct Image(*mut vpx_image_t); +impl Image { + #[inline] + pub fn new() -> Self { + Self(std::ptr::null_mut()) + } + + #[inline] + pub fn is_null(&self) -> bool { + self.0.is_null() + } + + #[inline] + pub fn width(&self) -> usize { + self.inner().d_w as _ + } + + #[inline] + pub fn height(&self) -> usize { + self.inner().d_h as _ + } + + #[inline] + pub fn format(&self) -> vpx_img_fmt_t { + // VPX_IMG_FMT_I420 + self.inner().fmt + } + + #[inline] + pub fn inner(&self) -> &vpx_image_t { + unsafe { &*self.0 } + } + + #[inline] + pub fn stride(&self, iplane: usize) -> i32 { + self.inner().stride[iplane] + } + + pub fn rgb(&self, stride_align: usize, rgba: bool, dst: &mut Vec) { + let h = self.height(); + let mut w = self.width(); + let bps = if rgba { 4 } else { 3 }; + w = (w + stride_align - 1) & !(stride_align - 1); + dst.resize(h * w * bps, 0); + let img = self.inner(); + unsafe { + if rgba { + super::I420ToARGB( + img.planes[0], + img.stride[0], + img.planes[1], + img.stride[1], + img.planes[2], + img.stride[2], + dst.as_mut_ptr(), + (w * bps) as _, + self.width() as _, + self.height() as _, + ); + } else { + super::I420ToRAW( + img.planes[0], + img.stride[0], + img.planes[1], + img.stride[1], + img.planes[2], + img.stride[2], + dst.as_mut_ptr(), + (w * bps) as _, + self.width() as _, + self.height() as _, + ); + } + } + } + + #[inline] + pub fn data(&self) -> (&[u8], &[u8], &[u8]) { + unsafe { + let img = self.inner(); + let h = (img.d_h as usize + 1) & !1; + let n = img.stride[0] as usize * h; + let y = slice::from_raw_parts(img.planes[0], n); + let n = img.stride[1] as usize * (h >> 1); + let u = slice::from_raw_parts(img.planes[1], n); + let v = slice::from_raw_parts(img.planes[2], n); + (y, u, v) + } + } +} + +impl Drop for Image { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { vpx_img_free(self.0) }; + } + } +} + +unsafe impl Send for vpx_codec_ctx_t {} diff --git a/libs/scrap/src/common/x11.rs b/libs/scrap/src/common/x11.rs index f8217e3b7..255819902 100644 --- a/libs/scrap/src/common/x11.rs +++ b/libs/scrap/src/common/x11.rs @@ -1,5 +1,5 @@ use crate::x11; -use std::{io, ops}; +use std::{io, ops, time::Duration}; pub struct Capturer(x11::Capturer); @@ -16,7 +16,7 @@ impl Capturer { self.0.display().rect().h as usize } - pub fn frame<'a>(&'a mut self, _timeout_ms: u32) -> io::Result> { + pub fn frame<'a>(&'a mut self, _timeout: Duration) -> io::Result> { Ok(Frame(self.0.frame()?)) } } diff --git a/libs/scrap/src/dxgi/mag.rs b/libs/scrap/src/dxgi/mag.rs index 0d63088b7..9adf26cdb 100644 --- a/libs/scrap/src/dxgi/mag.rs +++ b/libs/scrap/src/dxgi/mag.rs @@ -282,7 +282,11 @@ impl CapturerMag { let y = GetSystemMetrics(SM_YVIRTUALSCREEN); let w = GetSystemMetrics(SM_CXVIRTUALSCREEN); let h = GetSystemMetrics(SM_CYVIRTUALSCREEN); - if !(origin.0 == x as _ && origin.1 == y as _ && width == w as _ && height == h as _) { + if !(origin.0 == x as i32 + && origin.1 == y as i32 + && width == w as usize + && height == h as usize) + { return Err(Error::new( ErrorKind::Other, format!( @@ -510,10 +514,10 @@ impl CapturerMag { let y = GetSystemMetrics(SM_YVIRTUALSCREEN); let w = GetSystemMetrics(SM_CXVIRTUALSCREEN); let h = GetSystemMetrics(SM_CYVIRTUALSCREEN); - if !(self.rect.left == x as _ - && self.rect.top == y as _ - && self.rect.right == (x + w) as _ - && self.rect.bottom == (y + h) as _) + if !(self.rect.left == x as i32 + && self.rect.top == y as i32 + && self.rect.right == (x + w) as i32 + && self.rect.bottom == (y + h) as i32) { return Err(Error::new( ErrorKind::Other, diff --git a/src/client.rs b/src/client.rs index fed83cece..d63ce970c 100644 --- a/src/client.rs +++ b/src/client.rs @@ -12,6 +12,11 @@ use cpal::{ Device, Host, StreamConfig, }; use magnum_opus::{Channels::*, Decoder as AudioDecoder}; +use scrap::{ + codec::{Decoder, DecoderCfg}, + VpxDecoderConfig, VpxVideoCodecId, +}; + use sha2::{Digest, Sha256}; use uuid::Uuid; @@ -30,13 +35,12 @@ use hbb_common::{ tokio::time::Duration, AddrMangle, ResultType, Stream, }; -use scrap::{Decoder, Image, VideoCodecId}; pub use super::lang::*; pub mod file_trait; pub use file_trait::FileManager; pub mod helper; -pub use helper::LatencyController; +pub use helper::*; pub const SEC30: Duration = Duration::from_secs(30); pub struct Client; @@ -717,7 +721,12 @@ pub struct VideoHandler { impl VideoHandler { pub fn new(latency_controller: Arc>) -> Self { VideoHandler { - decoder: Decoder::new(VideoCodecId::VP9, (num_cpus::get() / 2) as _).unwrap(), + decoder: Decoder::new(DecoderCfg { + vpx: VpxDecoderConfig { + codec: VpxVideoCodecId::VP9, + num_threads: (num_cpus::get() / 2) as _, + }, + }), latency_controller, rgb: Default::default(), } @@ -731,33 +740,18 @@ impl VideoHandler { .update_video(vf.timestamp); } match &vf.union { - Some(video_frame::Union::vp9s(vp9s)) => self.handle_vp9s(vp9s), + Some(frame) => self.decoder.handle_video_frame(frame, &mut self.rgb), _ => Ok(false), } } - pub fn handle_vp9s(&mut self, vp9s: &VP9s) -> ResultType { - let mut last_frame = Image::new(); - for vp9 in vp9s.frames.iter() { - for frame in self.decoder.decode(&vp9.data)? { - drop(last_frame); - last_frame = frame; - } - } - for frame in self.decoder.flush()? { - drop(last_frame); - last_frame = frame; - } - if last_frame.is_null() { - Ok(false) - } else { - last_frame.rgb(1, true, &mut self.rgb); - Ok(true) - } - } - pub fn reset(&mut self) { - self.decoder = Decoder::new(VideoCodecId::VP9, 1).unwrap(); + self.decoder = Decoder::new(DecoderCfg { + vpx: VpxDecoderConfig { + codec: VpxVideoCodecId::VP9, + num_threads: 1, + }, + }); } } @@ -886,6 +880,8 @@ impl LoginConfigHandler { option.block_input = BoolOption::Yes.into(); } else if name == "unblock-input" { option.block_input = BoolOption::No.into(); + } else if name == "show-quality-monitor" { + config.show_quality_monitor = !config.show_quality_monitor; } else { let v = self.options.get(&name).is_some(); if v { @@ -918,15 +914,8 @@ impl LoginConfigHandler { n += 1; } else if q == "custom" { let config = PeerConfig::load(&self.id); - let mut it = config.custom_image_quality.iter(); - let bitrate = it.next(); - let quantizer = it.next(); - if let Some(bitrate) = bitrate { - if let Some(quantizer) = quantizer { - msg.custom_image_quality = bitrate << 8 | quantizer; - n += 1; - } - } + msg.custom_image_quality = config.custom_image_quality[0] << 8; + n += 1; } if self.get_toggle_option("show-remote-cursor") { msg.show_remote_cursor = BoolOption::Yes.into(); @@ -952,6 +941,11 @@ impl LoginConfigHandler { msg.disable_clipboard = BoolOption::Yes.into(); n += 1; } + // TODO: add option + let state = Decoder::video_codec_state(); + msg.video_codec_state = hbb_common::protobuf::MessageField::some(state); + n += 1; + if n > 0 { Some(msg) } else { @@ -988,6 +982,8 @@ impl LoginConfigHandler { self.config.disable_audio } else if name == "disable-clipboard" { self.config.disable_clipboard + } else if name == "show-quality-monitor" { + self.config.show_quality_monitor } else { !self.get_option(name).is_empty() } @@ -1009,17 +1005,17 @@ impl LoginConfigHandler { msg_out } - pub fn save_custom_image_quality(&mut self, bitrate: i32, quantizer: i32) -> Message { + pub fn save_custom_image_quality(&mut self, image_quality: i32) -> Message { let mut misc = Misc::new(); misc.set_option(OptionMessage { - custom_image_quality: bitrate << 8 | quantizer, + custom_image_quality: image_quality << 8, ..Default::default() }); let mut msg_out = Message::new(); msg_out.set_misc(misc); let mut config = self.load_config(); config.image_quality = "custom".to_owned(); - config.custom_image_quality = vec![bitrate, quantizer]; + config.custom_image_quality = vec![image_quality as _]; self.save_config(config); msg_out } @@ -1170,9 +1166,11 @@ where let latency_controller = LatencyController::new(); let latency_controller_cl = latency_controller.clone(); + // Create video_handler out of the thread below to ensure that the handler exists before client start. + // It will take a few tenths of a second for the first time, and then tens of milliseconds. + let mut video_handler = VideoHandler::new(latency_controller); std::thread::spawn(move || { - let mut video_handler = VideoHandler::new(latency_controller); loop { if let Ok(data) = video_receiver.recv() { match data { diff --git a/src/client/helper.rs b/src/client/helper.rs index abd20d312..ea12cb7ee 100644 --- a/src/client/helper.rs +++ b/src/client/helper.rs @@ -3,7 +3,7 @@ use std::{ time::Instant, }; -use hbb_common::log; +use hbb_common::{log, message_proto::{VideoFrame, video_frame}}; const MAX_LATENCY: i64 = 500; const MIN_LATENCY: i64 = 100; @@ -57,3 +57,33 @@ impl LatencyController { self.allow_audio } } + +#[derive(PartialEq, Debug, Clone)] +pub enum CodecFormat { + VP9, + H264, + H265, + Unknown, +} + +impl From<&VideoFrame> for CodecFormat { + fn from(it: &VideoFrame) -> Self { + match it.union { + Some(video_frame::Union::vp9s(_)) => CodecFormat::VP9, + Some(video_frame::Union::h264s(_)) => CodecFormat::H264, + Some(video_frame::Union::h265s(_)) => CodecFormat::H265, + _ => CodecFormat::Unknown, + } + } +} + +impl ToString for CodecFormat { + fn to_string(&self) -> String { + match self { + CodecFormat::VP9 => "VP9".into(), + CodecFormat::H264 => "H264".into(), + CodecFormat::H265 => "H265".into(), + CodecFormat::Unknown => "Unknow".into(), + } + } +} diff --git a/src/ipc.rs b/src/ipc.rs index 5eabbab66..24a156eba 100644 --- a/src/ipc.rs +++ b/src/ipc.rs @@ -73,7 +73,7 @@ pub enum FS { WriteOffset { id: i32, file_num: i32, - offset_blk: u32 + offset_blk: u32, }, CheckDigest { id: i32, @@ -336,6 +336,7 @@ async fn handle(data: Data, stream: &mut Connection) { .await ); } + _ => {} } } diff --git a/src/lang/cn.rs b/src/lang/cn.rs index 606f2a6e7..ba8e6d178 100644 --- a/src/lang/cn.rs +++ b/src/lang/cn.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "关于"), ("Mute", "静音"), ("Audio Input", "音频输入"), + ("Enhancements", "增强功能"), + ("Hardware Codec", "硬件编解码"), + ("Adaptive Bitrate", "自适应码率"), ("ID Server", "ID服务器"), ("Relay Server", "中继服务器"), ("API Server", "API服务器"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "优化反应时间"), ("Custom", "自定义画质"), ("Show remote cursor", "显示远程光标"), + ("Show quality monitor", "显示质量监测"), ("Disable clipboard", "禁止剪贴板"), ("Lock after session end", "断开后锁定远程电脑"), ("Insert", "插入"), diff --git a/src/lang/cs.rs b/src/lang/cs.rs index 696eabe2a..3c92ee71b 100644 --- a/src/lang/cs.rs +++ b/src/lang/cs.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "O aplikaci"), ("Mute", "Ztlumit"), ("Audio Input", "Vstup zvuku"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "Server pro identif."), ("Relay Server", "Předávací (relay) server"), ("API Server", "Server s API rozhraním"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Optimalizovat pro co nejnižší prodlevu odezvy"), ("Custom", "Uživatelsky určené"), ("Show remote cursor", "Zobrazovat ukazatel myši z protějšku"), + ("Show quality monitor", ""), ("Disable clipboard", "Vypnout schránku"), ("Lock after session end", "Po ukončení relace zamknout plochu"), ("Insert", "Vložit"), diff --git a/src/lang/da.rs b/src/lang/da.rs index da191e8c1..16b98359b 100644 --- a/src/lang/da.rs +++ b/src/lang/da.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "Omkring"), ("Mute", "Sluk for mikrofonen"), ("Audio Input", "Lydindgang"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "identifikations Server"), ("Relay Server", "Relæ Server"), ("API Server", "API Server"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Optimeret responstid"), ("Custom", "Brugerdefineret"), ("Show remote cursor", "Vis fjernbetjeningskontrolleret markør"), + ("Show quality monitor", ""), ("Disable clipboard", "Deaktiver udklipsholder"), ("Lock after session end", "Lås efter afslutningen af fjernstyring"), ("Insert", "Indsæt"), diff --git a/src/lang/de.rs b/src/lang/de.rs index 0307501b2..c2fd56002 100644 --- a/src/lang/de.rs +++ b/src/lang/de.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "Über"), ("Mute", "Stummschalten"), ("Audio Input", "Audio-Eingang"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "ID Server"), ("Relay Server", "Verbindungsserver Server"), ("API Server", "API Server"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Optimierte Reaktionszeit"), ("Custom", "Benutzerdefiniert"), ("Show remote cursor", "Ferngesteuerten Cursor anzeigen"), + ("Show quality monitor", ""), ("Disable clipboard", "Zwischenablage deaktivieren"), ("Lock after session end", "Sperren nach Sitzungsende"), ("Insert", "Einfügen"), diff --git a/src/lang/eo.rs b/src/lang/eo.rs index 75f3bc084..c9701d63a 100644 --- a/src/lang/eo.rs +++ b/src/lang/eo.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "Pri"), ("Mute", "Muta"), ("Audio Input", "Aŭdia enigo"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "Servilo de identigiloj"), ("Relay Server", "Relajsa servilo"), ("API Server", "Servilo de API"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Optimigi reakcia tempo"), ("Custom", "Personigi bilda kvalito"), ("Show remote cursor", "Montri foran kursoron"), + ("Show quality monitor", ""), ("Disable clipboard", "Malebligi poŝon"), ("Lock after session end", "Ŝlosi foran komputilon post malkonektado"), ("Insert", "Enmeti"), diff --git a/src/lang/es.rs b/src/lang/es.rs index f55071fda..60cc05341 100644 --- a/src/lang/es.rs +++ b/src/lang/es.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "Sobre"), ("Mute", "Silencio"), ("Audio Input", "Entrada de audio"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "ID server"), ("Relay Server", "Server relay"), ("API Server", "Server API"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Optimizar el tiempo de reacción"), ("Custom", "Personalizado"), ("Show remote cursor", "Mostrar cursor remoto"), + ("Show quality monitor", ""), ("Disable clipboard", "Deshabilitar portapapeles"), ("Lock after session end", "Bloquear después del final de la sesión"), ("Insert", "Insertar"), diff --git a/src/lang/fr.rs b/src/lang/fr.rs index 64500f9a6..f447c5acd 100644 --- a/src/lang/fr.rs +++ b/src/lang/fr.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "À propos de"), ("Mute", "Muet"), ("Audio Input", "Entrée audio"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "Serveur ID"), ("Relay Server", "Serveur relais"), ("API Server", "Serveur API"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Optimiser le temps de réaction"), ("Custom", "Qualité d'image personnalisée"), ("Show remote cursor", "Afficher le curseur distant"), + ("Show quality monitor", ""), ("Disable clipboard", "Désactiver le presse-papier"), ("Lock after session end", "Verrouiller l'ordinateur distant après la déconnexion"), ("Insert", "Insérer"), diff --git a/src/lang/id.rs b/src/lang/id.rs index d3da71833..b1c6aaa1e 100644 --- a/src/lang/id.rs +++ b/src/lang/id.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "Tentang"), ("Mute", "Bisukan"), ("Audio Input", "Masukkan Audio"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "Server ID"), ("Relay Server", "Server Relay"), ("API Server", "API Server"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Optimalkan waktu reaksi"), ("Custom", "Custom"), ("Show remote cursor", "Tampilkan remote kursor"), + ("Show quality monitor", ""), ("Disable clipboard", "Matikan papan klip"), ("Lock after session end", "Kunci setelah sesi berakhir"), ("Insert", "Menyisipkan"), diff --git a/src/lang/it.rs b/src/lang/it.rs index fa4c99e84..c756d2218 100644 --- a/src/lang/it.rs +++ b/src/lang/it.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "Informazioni"), ("Mute", "Silenzia"), ("Audio Input", "Input audio"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "ID server"), ("Relay Server", "Server relay"), ("API Server", "Server API"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Ottimizza il tempo di reazione"), ("Custom", "Personalizzato"), ("Show remote cursor", "Mostra il cursore remoto"), + ("Show quality monitor", ""), ("Disable clipboard", "Disabilita appunti"), ("Lock after session end", "Blocca al termine della sessione"), ("Insert", "Inserisci"), diff --git a/src/lang/ptbr.rs b/src/lang/ptbr.rs index 6ff30a3cd..946e47d4c 100644 --- a/src/lang/ptbr.rs +++ b/src/lang/ptbr.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "Sobre"), ("Mute", "Emudecer"), ("Audio Input", "Entrada de Áudio"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "Servidor de ID"), ("Relay Server", "Servidor de Relay"), ("API Server", "Servidor da API"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Otimizar tempo de reação"), ("Custom", "Personalizado"), ("Show remote cursor", "Mostrar cursor remoto"), + ("Show quality monitor", ""), ("Disable clipboard", "Desabilitar área de transferência"), ("Lock after session end", "Bloquear após o fim da sessão"), ("Insert", "Inserir"), diff --git a/src/lang/ru.rs b/src/lang/ru.rs index a50563674..d421bd85e 100644 --- a/src/lang/ru.rs +++ b/src/lang/ru.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "О RustDesk"), ("Mute", "Отключить звук"), ("Audio Input", "Аудиовход"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "ID-сервер"), ("Relay Server", "Сервер ретрансляции"), ("API Server", "API-сервер"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Оптимизировать время реакции"), ("Custom", "Пользовательский"), ("Show remote cursor", "Показать удаленный курсор"), + ("Show quality monitor", "Показать качество"), ("Disable clipboard", "Отключить буфер обмена"), ("Lock after session end", "Выход из учётной записи после завершения сеанса"), ("Insert", "Вставить"), @@ -121,9 +125,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Failed to connect via relay server", "Не удалось подключиться через сервер ретрансляции"), ("Failed to make direct connection to remote desktop", "Не удалось установить прямое подключение к удаленному рабочему столу"), ("Set Password", "Установить пароль"), - ("OS Password", "Пароль операционной системы"), + ("OS Password", "Пароль ОС"), ("install_tip", "В некоторых случаях из-за UAC RustDesk может работать некорректно на удаленном узле. Чтобы избежать UAC, нажмите кнопку ниже, чтобы установить RustDesk в системе"), - ("Click to upgrade", "Нажмите, чтобы проверить на наличие обновлений"), + ("Click to upgrade", "Нажмите, чтобы проверить наличие обновлений"), ("Click to download", "Нажмите, чтобы скачать"), ("Click to update", "Нажмите, чтобы обновить"), ("Configure", "Настроить"), @@ -132,14 +136,14 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Installing ...", "Устанавливается..."), ("Install", "Установить"), ("Installation", "Установка"), - ("Installation Path", "Папка установки"), + ("Installation Path", "Путь установки"), ("Create start menu shortcuts", "Создать ярлыки меню \"Пуск\""), ("Create desktop icon", "Создать значок на рабочем столе"), ("agreement_tip", "Если вы начнете установку, примите лицензионное соглашение"), ("Accept and Install", "Принять и установить"), ("End-user license agreement", "Лицензионное соглашение с конечным пользователем"), ("Generating ...", "Генерация..."), - ("Your installation is lower version.", "Ваша инсталяция является более ранней версией"), + ("Your installation is lower version.", "Ваша установка более ранней версии"), ("not_close_tcp_tip", "Не закрывать это окно при использовании туннеля"), ("Listening ...", "Ожидаем..."), ("Remote Host", "Удаленная машина"), @@ -159,11 +163,11 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Allow hearing sound", "Разрешить передачу звука"), ("Allow file copy and paste", "Разрешить копирование и вставку файлов"), ("Connected", "Подключено"), - ("Direct and encrypted connection", "Прямое и шифрованное соединение"), - ("Relayed and encrypted connection", "Коммутируемое и зашифрованное соединение"), + ("Direct and encrypted connection", "Прямое и зашифрованное соединение"), + ("Relayed and encrypted connection", "Ретранслируемое и зашифрованное соединение"), ("Direct and unencrypted connection", "Прямое и незашифрованное соединение"), - ("Relayed and unencrypted connection", "Коммутируемое и незашифрованное соединение"), - ("Enter Remote ID", "Введите удаленный идентификатор"), + ("Relayed and unencrypted connection", "Ретранслируемое и незашифрованное соединение"), + ("Enter Remote ID", "Введите удаленный ID"), ("Enter your password", "Введите пароль"), ("Logging in...", "Вход..."), ("Enable RDP session sharing", "Включить общий доступ к сеансу RDP"), @@ -219,12 +223,12 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Paste", "Вставить"), ("Paste here?", "Вставить сюда?"), ("Are you sure to close the connection?", "Вы уверены, что хотите закрыть соединение?"), - ("Download new version", "Загрузить новую версию"), + ("Download new version", "Скачать новую версию"), ("Touch mode", "Сенсорный режим"), ("Mouse mode", "Режим мыши"), ("One-Finger Tap", "Касание одним пальцем"), ("Left Mouse", "Левая кнопка мыши"), - ("One-Long Tap", "Одно долгое касание пальцем"), + ("One-Long Tap", "Одно долгое нажатие пальцем"), ("Two-Finger Tap", "Касание двумя пальцами"), ("Right Mouse", "Правая мышь"), ("One-Finger Move", "Движение одним пальцем"), @@ -255,7 +259,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Do you accept?", "Вы согласны?"), ("Open System Setting", "Открыть настройки системы"), ("How to get Android input permission?", "Как получить разрешение на ввод Android?"), - ("android_input_permission_tip1", "Чтобы удаленное устройство могло управлять вашим Android-устройством с помощью мыши или касания, вам необходимо разрешить RustDesk использовать службу «Специальные возможности»."), + ("android_input_permission_tip1", "Чтобы удаленное устройство могло управлять вашим Android-устройством с помощью мыши или касания, вам необходимо разрешить RustDesk использовать службу \"Специальные возможности\"."), ("android_input_permission_tip2", "Перейдите на следующую страницу системных настроек, найдите и войдите в [Установленные службы], включите службу [RustDesk Input]."), ("android_new_connection_tip", "Получен новый запрос на управление вашим текущим устройством."), ("android_service_will_start_tip", "Включение захвата экрана автоматически запускает службу, позволяя другим устройствам запрашивать соединение с этого устройства."), @@ -264,21 +268,21 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("android_start_service_tip", "Нажмите [Запуск промежуточного сервера] или ОТКРЫТЬ разрешение [Захват экрана], чтобы запустить службу демонстрации экрана."), ("Account", "Аккаунт"), ("Overwrite", "Перезаписать"), - ("This file exists, skip or overwrite this file?", "Этот файл существует, пропустить или перезаписать этот файл?"), + ("This file exists, skip or overwrite this file?", "Этот файл существует, пропустить или перезаписать файл?"), ("Quit", "Выйти"), ("doc_mac_permission", "https://rustdesk.com/docs/ru/manual/mac/#включение-разрешений"), ("Help", "Помощь"), - ("Failed", "Неуспешный"), + ("Failed", "Не удалось"), ("Succeeded", "Успешно"), ("Someone turns on privacy mode, exit", "Кто-то включает режим конфиденциальности, выйдите"), ("Unsupported", "Не поддерживается"), ("Peer denied", "Отказано в пире"), ("Please install plugins", "Пожалуйста, установите плагины"), ("Peer exit", "Одноранговый выход"), - ("Failed to turn off", "Не удалось отключить"), + ("Failed to turn off", "Не удалось выключить"), ("Turned off", "Выключен"), ("In privacy mode", "В режиме конфиденциальности"), ("Out privacy mode", "Выход из режима конфиденциальности"), - ("Language", ""), + ("Language", "Язык"), ].iter().cloned().collect(); } diff --git a/src/lang/sk.rs b/src/lang/sk.rs index 252faaf17..8c60abaa9 100644 --- a/src/lang/sk.rs +++ b/src/lang/sk.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "O RustDesk"), ("Mute", "Stíšiť"), ("Audio Input", "Zvukový vstup"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "ID server"), ("Relay Server", "Prepojovací server"), ("API Server", "API server"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Optimalizované pre čas odozvy"), ("Custom", "Vlastné"), ("Show remote cursor", "Zobrazovať vzdialený ukazovateľ myši"), + ("Show quality monitor", ""), ("Disable clipboard", "Vypnúť schránku"), ("Lock after session end", "Po skončení uzamknúť plochu"), ("Insert", "Vložiť"), diff --git a/src/lang/template.rs b/src/lang/template.rs index 6ad92db6e..ad963b323 100644 --- a/src/lang/template.rs +++ b/src/lang/template.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", ""), ("Mute", ""), ("Audio Input", ""), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", ""), ("Relay Server", ""), ("API Server", ""), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", ""), ("Custom", ""), ("Show remote cursor", ""), + ("Show quality monitor", ""), ("Disable clipboard", ""), ("Lock after session end", ""), ("Insert", ""), diff --git a/src/lang/tr.rs b/src/lang/tr.rs index 37502567c..d12b2881e 100644 --- a/src/lang/tr.rs +++ b/src/lang/tr.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "Hakkında"), ("Mute", "Sesi Kapat"), ("Audio Input", "Ses Girişi"), + ("Enhancements", ""), + ("Hardware Codec", ""), + ("Adaptive Bitrate", ""), ("ID Server", "ID Sunucu"), ("Relay Server", "Relay Sunucu"), ("API Server", "API Sunucu"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "Tepki süresini optimize et"), ("Custom", "Özel"), ("Show remote cursor", "Uzaktaki fare imlecini göster"), + ("Show quality monitor", ""), ("Disable clipboard", "Hafızadaki kopyalanmışları engelle"), ("Lock after session end", "Bağlantıdan sonra kilitle"), ("Insert", "Ekle"), diff --git a/src/lang/tw.rs b/src/lang/tw.rs index 0d81d7a8b..4f9813943 100644 --- a/src/lang/tw.rs +++ b/src/lang/tw.rs @@ -35,6 +35,9 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("About", "關於"), ("Mute", "靜音"), ("Audio Input", "音訊輸入"), + ("Enhancements", "增強功能"), + ("Hardware Codec", "硬件編解碼"), + ("Adaptive Bitrate", "自適應碼率"), ("ID Server", "ID 伺服器"), ("Relay Server", "轉送伺服器"), ("API Server", "API 伺服器"), @@ -105,6 +108,7 @@ pub static ref T: std::collections::HashMap<&'static str, &'static str> = ("Optimize reaction time", "回應速度最佳化"), ("Custom", "自訂"), ("Show remote cursor", "顯示遠端游標"), + ("Show quality monitor", "顯示質量監測"), ("Disable clipboard", "停用剪貼簿"), ("Lock after session end", "工作階段結束後鎖定電腦"), ("Insert", "插入"), diff --git a/src/main.rs b/src/main.rs index a5b1d7b04..2e2e4c8ac 100644 --- a/src/main.rs +++ b/src/main.rs @@ -151,6 +151,10 @@ fn main() { ipc::set_password(args[1].to_owned()).unwrap(); } return; + } else if args[0] == "--check-hwcodec-config" { + #[cfg(feature = "hwcodec")] + scrap::hwcodec::check_config(); + return; } } ui::start(&mut args[..]); diff --git a/src/platform/windows.rs b/src/platform/windows.rs index ab267440e..c9f83389a 100644 --- a/src/platform/windows.rs +++ b/src/platform/windows.rs @@ -1020,6 +1020,22 @@ copy /Y \"{tmp_path}\\Uninstall {app_name}.lnk\" \"{start_menu}\\\" // https://docs.microsoft.com/zh-cn/windows/win32/msi/uninstall-registry-key?redirectedfrom=MSDNa // https://www.windowscentral.com/how-edit-registry-using-command-prompt-windows-10 // https://www.tenforums.com/tutorials/70903-add-remove-allowed-apps-through-windows-firewall-windows-10-a.html + // Note: without if exist, the bat may exit in advance on some Windows7 https://github.com/rustdesk/rustdesk/issues/895 + let dels = format!( + " +if exist \"{mk_shortcut}\" del /f /q \"{mk_shortcut}\" +if exist \"{uninstall_shortcut}\" del /f /q \"{uninstall_shortcut}\" +if exist \"{tray_shortcut}\" del /f /q \"{tray_shortcut}\" +if exist \"{tmp_path}\\{app_name}.lnk\" del /f /q \"{tmp_path}\\{app_name}.lnk\" +if exist \"{tmp_path}\\Uninstall {app_name}.lnk\" del /f /q \"{tmp_path}\\Uninstall {app_name}.lnk\" +if exist \"{tmp_path}\\{app_name} Tray.lnk\" del /f /q \"{tmp_path}\\{app_name} Tray.lnk\" + ", + mk_shortcut = mk_shortcut, + uninstall_shortcut = uninstall_shortcut, + tray_shortcut = tray_shortcut, + tmp_path = tmp_path, + app_name = crate::get_app_name(), + ); let cmds = format!( " {uninstall_str} @@ -1048,12 +1064,7 @@ cscript \"{tray_shortcut}\" copy /Y \"{tmp_path}\\{app_name} Tray.lnk\" \"%PROGRAMDATA%\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\\" {shortcuts} copy /Y \"{tmp_path}\\Uninstall {app_name}.lnk\" \"{path}\\\" -del /f \"{mk_shortcut}\" -del /f \"{uninstall_shortcut}\" -del /f \"{tray_shortcut}\" -del /f \"{tmp_path}\\{app_name}.lnk\" -del /f \"{tmp_path}\\Uninstall {app_name}.lnk\" -del /f \"{tmp_path}\\{app_name} Tray.lnk\" +{dels} sc create {app_name} binpath= \"\\\"{exe}\\\" --import-config \\\"{config_path}\\\"\" start= auto DisplayName= \"{app_name} Service\" sc start {app_name} sc stop {app_name} @@ -1086,7 +1097,12 @@ sc delete {app_name} "timeout 300" } else { "" - } + }, + dels=if debug { + "" + } else { + &dels + }, ); run_cmds(cmds, debug, "install")?; std::thread::sleep(std::time::Duration::from_millis(2000)); @@ -1132,10 +1148,10 @@ fn get_uninstall() -> String { " {before_uninstall} reg delete {subkey} /f - rd /s /q \"{path}\" - rd /s /q \"{start_menu}\" - del /f /q \"%PUBLIC%\\Desktop\\{app_name}*\" - del /f /q \"%PROGRAMDATA%\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\{app_name} Tray.lnk\" + if exist \"{path}\" rd /s /q \"{path}\" + if exist \"{start_menu}\" rd /s /q \"{start_menu}\" + if exist \"%PUBLIC%\\Desktop\\{app_name}.lnk\" del /f /q \"%PUBLIC%\\Desktop\\{app_name}.lnk\" + if exist \"%PROGRAMDATA%\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\{app_name} Tray.lnk\" del /f /q \"%PROGRAMDATA%\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\{app_name} Tray.lnk\" ", before_uninstall=get_before_uninstall(), subkey=subkey, @@ -1182,11 +1198,8 @@ fn run_cmds(cmds: String, show: bool, tip: &str) -> ResultType<()> { .show(show) .force_prompt(true) .status(); - // leave the file for debug if execution failed - if let Ok(res) = res { - if res.success() { - allow_err!(std::fs::remove_file(tmp)); - } + if !show { + allow_err!(std::fs::remove_file(tmp)); } let _ = res?; Ok(()) diff --git a/src/server.rs b/src/server.rs index 9bafa09d1..3b68fc7dd 100644 --- a/src/server.rs +++ b/src/server.rs @@ -38,6 +38,7 @@ pub const NAME_POS: &'static str = ""; mod connection; mod service; +mod video_qos; pub mod video_service; use hbb_common::tcp::new_listener; @@ -320,6 +321,15 @@ pub async fn start_server(is_server: bool) { std::process::exit(-1); } }); + #[cfg(feature = "hwcodec")] + if let Ok(exe) = std::env::current_exe() { + std::thread::spawn(move || { + std::process::Command::new(exe) + .arg("--check-hwcodec-config") + .status() + .ok() + }); + } #[cfg(windows)] crate::platform::windows::bootstrap(); input_service::fix_key_down_timeout_loop(); diff --git a/src/server/connection.rs b/src/server/connection.rs index 304b20655..46c730092 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -3,6 +3,7 @@ use super::{input_service::*, *}; use crate::clipboard_file::*; #[cfg(not(any(target_os = "android", target_os = "ios")))] use crate::common::update_clipboard; +use crate::video_service; #[cfg(any(target_os = "android", target_os = "ios"))] use crate::{common::MOBILE_INFO2, mobile::connection_manager::start_channel}; use crate::{ipc, VERSION}; @@ -69,7 +70,6 @@ pub struct Connection { audio: bool, file: bool, last_test_delay: i64, - image_quality: i32, lock_after_session_end: bool, show_remote_cursor: bool, // by peer ip: String, @@ -105,7 +105,7 @@ impl Subscriber for ConnInner { } } -const TEST_DELAY_TIMEOUT: Duration = Duration::from_secs(3); +const TEST_DELAY_TIMEOUT: Duration = Duration::from_secs(1); const SEC30: Duration = Duration::from_secs(30); const H1: Duration = Duration::from_secs(3600); const MILLI1: Duration = Duration::from_millis(1); @@ -154,7 +154,6 @@ impl Connection { audio: Config::get_option("enable-audio").is_empty(), file: Config::get_option("enable-file-transfer").is_empty(), last_test_delay: 0, - image_quality: ImageQuality::Balanced.value(), lock_after_session_end: false, show_remote_cursor: false, ip: "".to_owned(), @@ -376,8 +375,11 @@ impl Connection { if time > 0 && conn.last_test_delay == 0 { conn.last_test_delay = time; let mut msg_out = Message::new(); + let qos = video_service::VIDEO_QOS.lock().unwrap(); msg_out.set_test_delay(TestDelay{ time, + last_delay:qos.current_delay, + target_bitrate:qos.target_bitrate, ..Default::default() }); conn.inner.send(msg_out.into()); @@ -394,8 +396,8 @@ impl Connection { let _ = privacy_mode::turn_off_privacy(0); } video_service::notify_video_frame_feched(id, None); - video_service::update_test_latency(id, 0); - video_service::update_image_quality(id, None); + scrap::codec::Encoder::update_video_encoder(id, scrap::codec::EncoderUpdate::Remove); + video_service::VIDEO_QOS.lock().unwrap().reset(); if let Err(err) = conn.try_port_forward_loop(&mut rx_from_cm).await { conn.on_close(&err.to_string(), false); } @@ -664,7 +666,7 @@ impl Connection { res.set_peer_info(pi); } else { try_activate_screen(); - match super::video_service::get_displays() { + match video_service::get_displays() { Err(err) => { res.set_error(format!("X11 error: {}", err)); } @@ -780,6 +782,22 @@ impl Connection { if let Some(message::Union::login_request(lr)) = msg.union { if let Some(o) = lr.option.as_ref() { self.update_option(o).await; + if let Some(q) = o.video_codec_state.clone().take() { + scrap::codec::Encoder::update_video_encoder( + self.inner.id(), + scrap::codec::EncoderUpdate::State(q), + ); + } else { + scrap::codec::Encoder::update_video_encoder( + self.inner.id(), + scrap::codec::EncoderUpdate::DisableHwIfNotExist, + ); + } + } else { + scrap::codec::Encoder::update_video_encoder( + self.inner.id(), + scrap::codec::EncoderUpdate::DisableHwIfNotExist, + ); } self.video_ack_required = lr.video_ack_required; if self.authorized { @@ -886,10 +904,11 @@ impl Connection { self.inner.send(msg_out.into()); } else { self.last_test_delay = 0; - let latency = crate::get_time() - t.time; - if latency > 0 { - super::video_service::update_test_latency(self.inner.id(), latency); - } + let new_delay = (crate::get_time() - t.time) as u32; + video_service::VIDEO_QOS + .lock() + .unwrap() + .update_network_delay(new_delay); } } else if self.authorized { match msg.union { @@ -1065,7 +1084,7 @@ impl Connection { }, Some(message::Union::misc(misc)) => match misc.union { Some(misc::Union::switch_display(s)) => { - super::video_service::switch_display(s.display); + video_service::switch_display(s.display); } Some(misc::Union::chat_message(c)) => { self.send_to_cm(ipc::Data::ChatMessage { text: c.text }); @@ -1075,7 +1094,7 @@ impl Connection { } Some(misc::Union::refresh_video(r)) => { if r { - super::video_service::refresh(); + video_service::refresh(); } } Some(misc::Union::video_received(_)) => { @@ -1095,13 +1114,20 @@ impl Connection { async fn update_option(&mut self, o: &OptionMessage) { log::info!("Option update: {:?}", o); if let Ok(q) = o.image_quality.enum_value() { - self.image_quality = q.value(); - super::video_service::update_image_quality(self.inner.id(), Some(q.value())); - } - let q = o.custom_image_quality; - if q > 0 { - self.image_quality = q; - super::video_service::update_image_quality(self.inner.id(), Some(q)); + let image_quality; + if let ImageQuality::NotSet = q { + if o.custom_image_quality > 0 { + image_quality = o.custom_image_quality; + } else { + image_quality = ImageQuality::Balanced.value(); + } + } else { + image_quality = q.value(); + } + video_service::VIDEO_QOS + .lock() + .unwrap() + .update_image_quality(image_quality); } if let Ok(q) = o.lock_after_session_end.enum_value() { if q != BoolOption::NotSet { diff --git a/src/server/video_qos.rs b/src/server/video_qos.rs new file mode 100644 index 000000000..efad1f9d9 --- /dev/null +++ b/src/server/video_qos.rs @@ -0,0 +1,219 @@ +use super::*; +use std::time::Duration; +const FPS: u8 = 30; +trait Percent { + fn as_percent(&self) -> u32; +} + +impl Percent for ImageQuality { + fn as_percent(&self) -> u32 { + match self { + ImageQuality::NotSet => 0, + ImageQuality::Low => 50, + ImageQuality::Balanced => 66, + ImageQuality::Best => 100, + } + } +} + +pub struct VideoQoS { + width: u32, + height: u32, + user_image_quality: u32, + current_image_quality: u32, + enable_abr: bool, + pub current_delay: u32, + pub fps: u8, // abr + pub target_bitrate: u32, // abr + updated: bool, + state: DelayState, + debounce_count: u32, +} + +#[derive(PartialEq, Debug)] +enum DelayState { + Normal = 0, + LowDelay = 200, + HighDelay = 500, + Broken = 1000, +} + +impl DelayState { + fn from_delay(delay: u32) -> Self { + if delay > DelayState::Broken as u32 { + DelayState::Broken + } else if delay > DelayState::HighDelay as u32 { + DelayState::HighDelay + } else if delay > DelayState::LowDelay as u32 { + DelayState::LowDelay + } else { + DelayState::Normal + } + } +} + +impl Default for VideoQoS { + fn default() -> Self { + VideoQoS { + fps: FPS, + user_image_quality: ImageQuality::Balanced.as_percent(), + current_image_quality: ImageQuality::Balanced.as_percent(), + enable_abr: false, + width: 0, + height: 0, + current_delay: 0, + target_bitrate: 0, + updated: false, + state: DelayState::Normal, + debounce_count: 0, + } + } +} + +impl VideoQoS { + pub fn set_size(&mut self, width: u32, height: u32) { + if width == 0 || height == 0 { + return; + } + self.width = width; + self.height = height; + } + + pub fn spf(&mut self) -> Duration { + if self.fps <= 0 { + self.fps = FPS; + } + Duration::from_secs_f32(1. / (self.fps as f32)) + } + + // update_network_delay periodically + // decrease the bitrate when the delay gets bigger + pub fn update_network_delay(&mut self, delay: u32) { + if self.current_delay.eq(&0) { + self.current_delay = delay; + return; + } + + self.current_delay = delay / 2 + self.current_delay / 2; + log::trace!( + "VideoQoS update_network_delay:{}, {}, state:{:?}", + self.current_delay, + delay, + self.state, + ); + + // ABR + if !self.enable_abr { + return; + } + let current_state = DelayState::from_delay(self.current_delay); + if current_state != self.state && self.debounce_count > 5 { + log::debug!( + "VideoQoS state changed:{:?} -> {:?}", + self.state, + current_state + ); + self.state = current_state; + self.debounce_count = 0; + self.refresh_quality(); + } else { + self.debounce_count += 1; + } + } + + fn refresh_quality(&mut self) { + match self.state { + DelayState::Normal => { + self.fps = FPS; + self.current_image_quality = self.user_image_quality; + } + DelayState::LowDelay => { + self.fps = FPS; + self.current_image_quality = std::cmp::min(self.user_image_quality, 50); + } + DelayState::HighDelay => { + self.fps = FPS / 2; + self.current_image_quality = std::cmp::min(self.user_image_quality, 25); + } + DelayState::Broken => { + self.fps = FPS / 4; + self.current_image_quality = 10; + } + } + let _ = self.generate_bitrate().ok(); + self.updated = true; + } + + // handle image_quality change from peer + pub fn update_image_quality(&mut self, image_quality: i32) { + let image_quality = Self::convert_quality(image_quality) as _; + log::debug!("VideoQoS update_image_quality: {}", image_quality); + if self.current_image_quality != image_quality { + self.current_image_quality = image_quality; + let _ = self.generate_bitrate().ok(); + self.updated = true; + } + + self.user_image_quality = self.current_image_quality; + } + + pub fn generate_bitrate(&mut self) -> ResultType { + // https://www.nvidia.com/en-us/geforce/guides/broadcasting-guide/ + if self.width == 0 || self.height == 0 { + bail!("Fail to generate_bitrate, width or height is not set"); + } + if self.current_image_quality == 0 { + self.current_image_quality = ImageQuality::Balanced.as_percent(); + } + + let base_bitrate = ((self.width * self.height) / 800) as u32; + + #[cfg(target_os = "android")] + { + // fix when andorid screen shrinks + let fix = Display::fix_quality() as u32; + log::debug!("Android screen, fix quality:{}", fix); + let base_bitrate = base_bitrate * fix; + self.target_bitrate = base_bitrate * self.current_image_quality / 100; + Ok(self.target_bitrate) + } + #[cfg(not(target_os = "android"))] + { + self.target_bitrate = base_bitrate * self.current_image_quality / 100; + Ok(self.target_bitrate) + } + } + + pub fn check_if_updated(&mut self) -> bool { + if self.updated { + self.updated = false; + return true; + } + return false; + } + + pub fn reset(&mut self) { + *self = Default::default(); + } + + pub fn check_abr_config(&mut self) -> bool { + self.enable_abr = if let Some(v) = Config2::get().options.get("enable-abr") { + v != "N" + } else { + true // default is true + }; + self.enable_abr + } + + pub fn convert_quality(q: i32) -> i32 { + if q == ImageQuality::Balanced.value() { + 100 * 2 / 3 + } else if q == ImageQuality::Low.value() { + 100 / 2 + } else if q == ImageQuality::Best.value() { + 100 + } else { + (q >> 8 & 0xFF) * 2 + } + } +} diff --git a/src/server/video_service.rs b/src/server/video_service.rs index b486cd311..e64ddd807 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -18,12 +18,16 @@ // to-do: // https://slhck.info/video/2017/03/01/rate-control.html -use super::*; +use super::{video_qos::VideoQoS, *}; use hbb_common::tokio::sync::{ mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, Mutex as TokioMutex, }; -use scrap::{Capturer, Config, Display, EncodeFrame, Encoder, Frame, VideoCodecId, STRIDE_ALIGN}; +use scrap::{ + codec::{Encoder, EncoderCfg, HwEncoderConfig}, + vpxcodec::{VpxEncoderConfig, VpxVideoCodecId}, + Capturer, Display, Frame, +}; use std::{ collections::HashSet, io::{ErrorKind::WouldBlock, Result}, @@ -38,14 +42,13 @@ lazy_static::lazy_static! { static ref CURRENT_DISPLAY: Arc> = Arc::new(Mutex::new(usize::MAX)); static ref LAST_ACTIVE: Arc> = Arc::new(Mutex::new(Instant::now())); static ref SWITCH: Arc> = Default::default(); - static ref TEST_LATENCIES: Arc>> = Default::default(); - static ref IMAGE_QUALITIES: Arc>> = Default::default(); static ref FRAME_FETCHED_NOTIFIER: (UnboundedSender<(i32, Option)>, Arc)>>>) = { let (tx, rx) = unbounded_channel(); (tx, Arc::new(TokioMutex::new(rx))) }; static ref PRIVACY_MODE_CONN_ID: Mutex = Mutex::new(0); static ref IS_CAPTURER_MAGNIFIER_SUPPORTED: bool = is_capturer_mag_supported(); + pub static ref VIDEO_QOS: Arc> = Default::default(); } fn is_capturer_mag_supported() -> bool { @@ -125,7 +128,7 @@ impl VideoFrameController { } trait TraitCapturer { - fn frame<'a>(&'a mut self, timeout_ms: u32) -> Result>; + fn frame<'a>(&'a mut self, timeout: Duration) -> Result>; #[cfg(windows)] fn is_gdi(&self) -> bool; @@ -134,8 +137,8 @@ trait TraitCapturer { } impl TraitCapturer for Capturer { - fn frame<'a>(&'a mut self, timeout_ms: u32) -> Result> { - self.frame(timeout_ms) + fn frame<'a>(&'a mut self, timeout: Duration) -> Result> { + self.frame(timeout) } #[cfg(windows)] @@ -151,7 +154,7 @@ impl TraitCapturer for Capturer { #[cfg(windows)] impl TraitCapturer for scrap::CapturerMag { - fn frame<'a>(&'a mut self, _timeout_ms: u32) -> Result> { + fn frame<'a>(&'a mut self, _timeout_ms: Duration) -> Result> { self.frame(_timeout_ms) } @@ -201,9 +204,11 @@ fn check_display_changed( } // Capturer object is expensive, avoiding to create it frequently. -fn create_capturer(privacy_mode_id: i32, display: Display) -> ResultType> { - let use_yuv = true; - +fn create_capturer( + privacy_mode_id: i32, + display: Display, + use_yuv: bool, +) -> ResultType> { #[cfg(not(windows))] let c: Option> = None; #[cfg(windows)] @@ -292,7 +297,7 @@ pub fn test_create_capturer(privacy_mode_id: i32, timeout_millis: u64) -> bool { let test_begin = Instant::now(); while test_begin.elapsed().as_millis() < timeout_millis as _ { if let Ok((_, _, display)) = get_current_display() { - if let Ok(_) = create_capturer(privacy_mode_id, display) { + if let Ok(_) = create_capturer(privacy_mode_id, display, true) { return true; } } @@ -320,9 +325,6 @@ fn run(sp: GenericService) -> ResultType<()> { #[cfg(windows)] ensure_close_virtual_device()?; - let fps = 30; - let wait = 1000 / fps; - let spf = time::Duration::from_secs_f32(1. / (fps as f32)); let (ndisplay, current, display) = get_current_display()?; let (origin, width, height) = (display.origin(), display.width(), display.height()); log::debug!( @@ -336,6 +338,38 @@ fn run(sp: GenericService) -> ResultType<()> { num_cpus::get(), ); + let mut video_qos = VIDEO_QOS.lock().unwrap(); + + video_qos.set_size(width as _, height as _); + let mut spf = video_qos.spf(); + let bitrate = video_qos.generate_bitrate()?; + let abr = video_qos.check_abr_config(); + drop(video_qos); + log::info!("init bitrate={}, abr enabled:{}", bitrate, abr); + + let encoder_cfg = match Encoder::current_hw_encoder_name() { + Some(codec_name) => EncoderCfg::HW(HwEncoderConfig { + codec_name, + width, + height, + bitrate: bitrate as _, + }), + None => EncoderCfg::VPX(VpxEncoderConfig { + width: width as _, + height: height as _, + timebase: [1, 1000], // Output timestamp precision + bitrate, + codec: VpxVideoCodecId::VP9, + num_threads: (num_cpus::get() / 2) as _, + }), + }; + + let mut encoder; + match Encoder::new(encoder_cfg) { + Ok(x) => encoder = x, + Err(err) => bail!("Failed to create encoder: {}", err), + } + let privacy_mode_id = *PRIVACY_MODE_CONN_ID.lock().unwrap(); #[cfg(not(windows))] let captuerer_privacy_mode_id = privacy_mode_id; @@ -355,26 +389,7 @@ fn run(sp: GenericService) -> ResultType<()> { } else { log::info!("In privacy mode, the peer side cannot watch the screen"); } - let mut c = create_capturer(captuerer_privacy_mode_id, display)?; - - let q = get_image_quality(); - let (bitrate, rc_min_quantizer, rc_max_quantizer, speed) = get_quality(width, height, q); - log::info!("bitrate={}, rc_min_quantizer={}", bitrate, rc_min_quantizer); - let cfg = Config { - width: width as _, - height: height as _, - timebase: [1, 1000], // Output timestamp precision - bitrate, - codec: VideoCodecId::VP9, - rc_min_quantizer, - rc_max_quantizer, - speed, - }; - let mut vpx; - match Encoder::new(&cfg, (num_cpus::get() / 2) as _) { - Ok(x) => vpx = x, - Err(err) => bail!("Failed to create encoder: {}", err), - } + let mut c = create_capturer(captuerer_privacy_mode_id, display, encoder.use_yuv())?; if *SWITCH.lock().unwrap() { log::debug!("Broadcasting display switch"); @@ -401,10 +416,24 @@ fn run(sp: GenericService) -> ResultType<()> { let mut try_gdi = 1; #[cfg(windows)] log::info!("gdi: {}", c.is_gdi()); + while sp.ok() { #[cfg(windows)] check_uac_switch(privacy_mode_id, captuerer_privacy_mode_id)?; + { + let mut video_qos = VIDEO_QOS.lock().unwrap(); + if video_qos.check_if_updated() { + log::debug!( + "qos is updated, target_bitrate:{}, fps:{}", + video_qos.target_bitrate, + video_qos.fps + ); + encoder.set_bitrate(video_qos.target_bitrate).unwrap(); + spf = video_qos.spf(); + } + } + if *SWITCH.lock().unwrap() { bail!("SWITCH"); } @@ -413,9 +442,6 @@ fn run(sp: GenericService) -> ResultType<()> { bail!("SWITCH"); } check_privacy_mode_changed(&sp, privacy_mode_id)?; - if get_image_quality() != q { - bail!("SWITCH"); - } #[cfg(windows)] { if crate::platform::windows::desktop_changed() { @@ -437,7 +463,7 @@ fn run(sp: GenericService) -> ResultType<()> { frame_controller.reset(); #[cfg(any(target_os = "android", target_os = "ios"))] - let res = match (*c).frame(wait as _) { + let res = match c.frame(spf) { Ok(frame) => { let time = now - start; let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64; @@ -448,7 +474,7 @@ fn run(sp: GenericService) -> ResultType<()> { } scrap::Frame::RAW(data) => { if (data.len() != 0) { - let send_conn_ids = handle_one_frame(&sp, data, ms, &mut vpx)?; + let send_conn_ids = handle_one_frame(&sp, data, ms, &mut encoder)?; frame_controller.set_send(now, send_conn_ids); } } @@ -460,11 +486,11 @@ fn run(sp: GenericService) -> ResultType<()> { }; #[cfg(not(any(target_os = "android", target_os = "ios")))] - let res = match (*c).frame(wait as _) { + let res = match c.frame(spf) { Ok(frame) => { let time = now - start; let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64; - let send_conn_ids = handle_one_frame(&sp, &frame, ms, &mut vpx)?; + let send_conn_ids = handle_one_frame(&sp, &frame, ms, &mut encoder)?; frame_controller.set_send(now, send_conn_ids); #[cfg(windows)] { @@ -531,7 +557,6 @@ fn run(sp: GenericService) -> ResultType<()> { Ok(()) } -#[inline] fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> ResultType<()> { let privacy_mode_id_2 = *PRIVACY_MODE_CONN_ID.lock().unwrap(); if privacy_mode_id != privacy_mode_id_2 { @@ -547,10 +572,11 @@ fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> Resu } #[inline] -fn create_msg(vp9s: Vec) -> Message { +#[cfg(any(target_os = "android", target_os = "ios"))] +fn create_msg(vp9s: Vec) -> Message { let mut msg_out = Message::new(); let mut vf = VideoFrame::new(); - vf.set_vp9s(VP9s { + vf.set_vp9s(EncodedVideoFrames { frames: vp9s.into(), ..Default::default() }); @@ -559,22 +585,12 @@ fn create_msg(vp9s: Vec) -> Message { msg_out } -#[inline] -fn create_frame(frame: &EncodeFrame) -> VP9 { - VP9 { - data: frame.data.to_vec(), - key: frame.key, - pts: frame.pts, - ..Default::default() - } -} - #[inline] fn handle_one_frame( sp: &GenericService, frame: &[u8], ms: i64, - vpx: &mut Encoder, + encoder: &mut Encoder, ) -> ResultType> { sp.snapshot(|sps| { // so that new sub and old sub share the same encoder after switch @@ -585,20 +601,8 @@ fn handle_one_frame( })?; let mut send_conn_ids: HashSet = Default::default(); - let mut frames = Vec::new(); - for ref frame in vpx - .encode(ms, frame, STRIDE_ALIGN) - .with_context(|| "Failed to encode")? - { - frames.push(create_frame(frame)); - } - for ref frame in vpx.flush().with_context(|| "Failed to flush")? { - frames.push(create_frame(frame)); - } - - // to-do: flush periodically, e.g. 1 second - if frames.len() > 0 { - send_conn_ids = sp.send_video_frame(create_msg(frames)); + if let Ok(msg) = encoder.encode_to_message(frame, ms) { + send_conn_ids = sp.send_video_frame(msg); } Ok(send_conn_ids) } @@ -618,7 +622,7 @@ pub fn handle_one_frame_encoded( Ok(()) })?; let mut send_conn_ids: HashSet = Default::default(); - let vp9_frame = VP9 { + let vp9_frame = EncodedVideoFrame { data: frame.to_vec(), key: true, pts: ms, @@ -748,82 +752,3 @@ fn get_current_display() -> ResultType<(usize, usize, Display)> { } return Ok((n, current, displays.remove(current))); } - -#[inline] -fn update_latency(id: i32, latency: i64, latencies: &mut HashMap) { - if latency <= 0 { - latencies.remove(&id); - } else { - latencies.insert(id, latency); - } -} - -pub fn update_test_latency(id: i32, latency: i64) { - update_latency(id, latency, &mut *TEST_LATENCIES.lock().unwrap()); -} - -fn convert_quality(q: i32) -> i32 { - let q = { - if q == ImageQuality::Balanced.value() { - (100 * 2 / 3, 12) - } else if q == ImageQuality::Low.value() { - (100 / 2, 18) - } else if q == ImageQuality::Best.value() { - (100, 12) - } else { - let bitrate = q >> 8 & 0xFF; - let quantizer = q & 0xFF; - (bitrate * 2, (100 - quantizer) * 36 / 100) - } - }; - if q.0 <= 0 { - 0 - } else { - q.0 << 8 | q.1 - } -} - -pub fn update_image_quality(id: i32, q: Option) { - match q { - Some(q) => { - let q = convert_quality(q); - if q > 0 { - IMAGE_QUALITIES.lock().unwrap().insert(id, q); - } else { - IMAGE_QUALITIES.lock().unwrap().remove(&id); - } - } - None => { - IMAGE_QUALITIES.lock().unwrap().remove(&id); - } - } -} - -fn get_image_quality() -> i32 { - IMAGE_QUALITIES - .lock() - .unwrap() - .values() - .min() - .unwrap_or(&convert_quality(ImageQuality::Balanced.value())) - .clone() -} - -#[inline] -fn get_quality(w: usize, h: usize, q: i32) -> (u32, u32, u32, i32) { - // https://www.nvidia.com/en-us/geforce/guides/broadcasting-guide/ - let bitrate = q >> 8 & 0xFF; - let quantizer = q & 0xFF; - let b = ((w * h) / 1000) as u32; - - #[cfg(target_os = "android")] - { - // fix when andorid screen shrinks - let fix = Display::fix_quality() as u32; - log::debug!("Android screen, fix quality:{}", fix); - let b = b * fix; - return (bitrate as u32 * b / 100, quantizer as _, 56, 7); - } - - (bitrate as u32 * b / 100, quantizer as _, 56, 7) -} diff --git a/src/ui.rs b/src/ui.rs index fc56d437c..f0134aceb 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -754,6 +754,13 @@ impl UI { ) } + fn has_hwcodec(&self) -> bool { + #[cfg(not(feature = "hwcodec"))] + return false; + #[cfg(feature = "hwcodec")] + return true; + } + fn get_langs(&self) -> String { crate::lang::LANGS.to_string() } @@ -833,6 +840,7 @@ impl sciter::EventHandler for UI { fn discover(); fn get_lan_peers(); fn get_uuid(); + fn has_hwcodec(); fn get_langs(); } } diff --git a/src/ui/header.tis b/src/ui/header.tis index 4b2615a45..35a132c90 100644 --- a/src/ui/header.tis +++ b/src/ui/header.tis @@ -159,6 +159,7 @@ class Header: Reactor.Component {

  • {svg_checkmark}{translate('Custom')}
  • {svg_checkmark}{translate('Show remote cursor')}
    • +
  • {svg_checkmark}{translate('Show quality monitor')}
    • {audio_enabled ?
  • {svg_checkmark}{translate('Mute')}
    • : ""} {is_win && pi.platform == 'Windows' && file_enabled ?
  • {svg_checkmark}{translate('Allow file copy and paste')}
    • : ""} {keyboard_enabled && clipboard_enabled ?
  • {svg_checkmark}{translate('Disable clipboard')}
    • : ""} @@ -315,7 +316,9 @@ class Header: Reactor.Component { handle_custom_image_quality(); } else if (me.id == "privacy-mode") { togglePrivacyMode(me.id); - } else if (me.attributes.hasClass("toggle-option")) { + } else if (me.id == "show-quality-monitor") { + toggleQualityMonitor(me.id); + }else if (me.attributes.hasClass("toggle-option")) { handler.toggle_option(me.id); toggleMenuState(); } else if (!me.attributes.hasClass("selected")) { @@ -333,15 +336,13 @@ class Header: Reactor.Component { function handle_custom_image_quality() { var tmp = handler.get_custom_image_quality(); - var bitrate0 = tmp[0] || 50; - var quantizer0 = tmp.length > 1 ? tmp[1] : 100; + var bitrate = (tmp[0] || 50); msgbox("custom", "Custom Image Quality", "
    \ -
    x% bitrate
    \ -
    x% quantizer
    \ +
    x% Bitrate
    \
    ", function(res=null) { if (!res) return; if (!res.bitrate) return; - handler.save_custom_image_quality(res.bitrate, res.quantizer); + handler.save_custom_image_quality(res.bitrate); toggleMenuState(); }); } @@ -357,7 +358,7 @@ function toggleMenuState() { for (var el in $$(menu#display-options>li)) { el.attributes.toggleClass("selected", values.indexOf(el.id) >= 0); } - for (var id in ["show-remote-cursor", "disable-audio", "enable-file-transfer", "disable-clipboard", "lock-after-session-end"]) { + for (var id in ["show-remote-cursor", "show-quality-monitor", "disable-audio", "enable-file-transfer", "disable-clipboard", "lock-after-session-end"]) { var el = self.select('#' + id); if (el) { var value = handler.get_toggle_option(id); @@ -425,6 +426,17 @@ function togglePrivacyMode(privacy_id) { } } +function toggleQualityMonitor(name) { + var show = handler.get_toggle_option(name); + if (show) { + $(#quality-monitor).style.set{ display: "none" }; + } else { + $(#quality-monitor).style.set{ display: "block" }; + } + handler.toggle_option(name); + toggleMenuState(); +} + handler.updateBlockInputState = function(input_blocked) { if (!input_blocked) { handler.toggle_option("block-input"); diff --git a/src/ui/index.tis b/src/ui/index.tis index 6a9573ca4..099ca2af8 100644 --- a/src/ui/index.tis +++ b/src/ui/index.tis @@ -199,6 +199,42 @@ class Languages: Reactor.Component { } } +var enhancementsMenu; +class Enhancements: Reactor.Component { + function this() { + enhancementsMenu = this; + } + + function render() { + var has_hwcodec = handler.has_hwcodec(); + var me = this; + self.timer(1ms, function() { me.toggleMenuState() }); + return
  • {translate('Enhancements')} + + {has_hwcodec ?
  • {svg_checkmark}{translate("Hardware Codec")}{"(beta)"}
    • : ""} +
  • {svg_checkmark}{translate("Adaptive Bitrate")}{"(beta)"}
    • +     +
    • ; + } + + function toggleMenuState() { + for (var el in $$(menu#enhancements-menu>li)) { + if (el.id && el.id.indexOf("enable-") == 0) { + var enabled = handler.get_option(el.id) != "N"; + el.attributes.toggleClass("selected", enabled); + } + } + + } + + event click $(menu#enhancements-menu>li) (_, me) { + var v = me.id; + if (v.indexOf("enable-") == 0) { + handler.set_option(v, handler.get_option(v) != 'N' ? 'N' : ''); + } + this.toggleMenuState(); + } +} function getUserName() { try { @@ -239,6 +275,7 @@ class MyIdMenu: Reactor.Component {
  • {svg_checkmark}{translate('Enable File Transfer')}
  • {svg_checkmark}{translate('Enable TCP Tunneling')}
    • +
  • {svg_checkmark}{translate('Enable remote configuration modification')}
  • {translate('ID/Relay Server')}
    • diff --git a/src/ui/remote.css b/src/ui/remote.css index 617285e9c..66c5ce80f 100644 --- a/src/ui/remote.css +++ b/src/ui/remote.css @@ -9,6 +9,16 @@ div#video-wrapper { background: #212121; } +div#quality-monitor { + top: 20px; + right: 20px; + background: #7571719c; + padding: 5px; + min-width: 150px; + color: azure; + border: solid azure; +} + video#handler { behavior: native-remote video; size: *; @@ -24,7 +34,7 @@ img#cursor { } .goup { - transform: rotate(90deg); + transform: rotate(90deg); } table#remote-folder-view { @@ -33,4 +43,4 @@ table#remote-folder-view { table#local-folder-view { context-menu: selector(menu#local-folder-view); -} +} \ No newline at end of file diff --git a/src/ui/remote.html b/src/ui/remote.html index 32c1409e2..d58c3449b 100644 --- a/src/ui/remote.html +++ b/src/ui/remote.html @@ -1,12 +1,13 @@ - - - - -
    -
    + +
    + + +
    - - -
    - -
    -
    -
    -
    - - + + + +
    +