Merge pull request #6084 from 21pages/av1_record

AV1 record

Cargo.lock

@@ -6712,18 +6712,16 @@ dependencies = [
 
 [[package]]
 name = "webm"
-version = "1.0.2"
+version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/21pages/rust-webm#d2c4d3ac133c7b0e4c0f656da710b48391981e64"
-checksum = "ecb047148a12ef1fd8ab26302bca7e82036f005c3073b48e17cc1b44ec577136"
 dependencies = [
  "webm-sys",
 ]
 
 [[package]]
 name = "webm-sys"
-version = "1.0.3"
+version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/21pages/rust-webm#d2c4d3ac133c7b0e4c0f656da710b48391981e64"
-checksum = "0ded6ec82ccf51fe265b0b2b1579cac839574ed910c17baac58e807f8a9de7f3"
 dependencies = [
  "cc",
 ]

flutter/lib/common/widgets/toolbar.dart

@@ -224,11 +224,8 @@ List<TTextMenu> toolbarControls(BuildContext context, String id, FFI ffi) {
     ));
   }
   // record
-  var codecFormat = ffi.qualityMonitorModel.data.codecFormat;
   if (!isDesktop &&
-      (ffi.recordingModel.start ||
+      (ffi.recordingModel.start || (perms["recording"] != false))) {
-          (perms["recording"] != false &&
-              (codecFormat == "VP8" || codecFormat == "VP9")))) {
     v.add(TTextMenu(
         child: Row(
           children: [

flutter/lib/desktop/pages/remote_page.dart

@@ -677,7 +677,8 @@ class _ImagePaintState extends State<ImagePaint> {
     } else {
       final key = cache.updateGetKey(scale);
       if (!cursor.cachedKeys.contains(key)) {
-        debugPrint("Register custom cursor with key $key (${cache.hotx},${cache.hoty})");
+        debugPrint(
+            "Register custom cursor with key $key (${cache.hotx},${cache.hoty})");
         // [Safety]
         // It's ok to call async registerCursor in current synchronous context,
         // because activating the cursor is also an async call and will always

flutter/lib/desktop/widgets/remote_toolbar.dart

@@ -478,7 +478,7 @@ class _RemoteToolbarState extends State<RemoteToolbar> {
       toolbarItems.add(_ChatMenu(id: widget.id, ffi: widget.ffi));
       toolbarItems.add(_VoiceCallMenu(id: widget.id, ffi: widget.ffi));
     }
-    toolbarItems.add(_RecordMenu(ffi: widget.ffi));
+    toolbarItems.add(_RecordMenu());
     toolbarItems.add(_CloseMenu(id: widget.id, ffi: widget.ffi));
     return Column(
       mainAxisSize: MainAxisSize.min,
@@ -1646,17 +1646,17 @@ class _VoiceCallMenu extends StatelessWidget {
 }
 
 class _RecordMenu extends StatelessWidget {
-  final FFI ffi;
+  const _RecordMenu({Key? key}) : super(key: key);
-  const _RecordMenu({Key? key, required this.ffi}) : super(key: key);
 
   @override
   Widget build(BuildContext context) {
-    var ffiModel = Provider.of<FfiModel>(context);
+    var ffi = Provider.of<FfiModel>(context);
     var recordingModel = Provider.of<RecordingModel>(context);
     final visible =
-        recordingModel.start || ffiModel.permissions['recording'] != false;
+        (recordingModel.start || ffi.permissions['recording'] != false) &&
+            ffi.pi.currentDisplay != kAllDisplayValue;
     if (!visible) return Offstage();
-    final menuButton = _IconMenuButton(
+    return _IconMenuButton(
       assetName: 'assets/rec.svg',
       tooltip: recordingModel.start
           ? 'Stop session recording'
@@ -1669,14 +1669,6 @@ class _RecordMenu extends StatelessWidget {
           ? _ToolbarTheme.hoverRedColor
           : _ToolbarTheme.hoverBlueColor,
     );
-    return ChangeNotifierProvider.value(
-        value: ffi.qualityMonitorModel,
-        child: Consumer<QualityMonitorModel>(
-            builder: (context, model, child) => Offstage(
-                  // If already started, AV1->Hidden/Stop, Other->Start, same as actual
-                  offstage: model.data.codecFormat == 'AV1',
-                  child: menuButton,
-                )));
   }
 }
 

flutter/lib/mobile/pages/remote_page.dart

@@ -763,7 +763,9 @@ void showOptions(
       children.add(InkWell(
           onTap: () {
             if (i == cur) return;
-            bind.sessionSwitchDisplay(sessionId: gFFI.sessionId, value: Int32List.fromList([i]));
+            gFFI.recordingModel.onClose();
+            bind.sessionSwitchDisplay(
+                sessionId: gFFI.sessionId, value: Int32List.fromList([i]));
             gFFI.dialogManager.dismissAll();
           },
           child: Ink(

flutter/lib/models/model.dart

@@ -860,6 +860,8 @@ class FfiModel with ChangeNotifier {
 
   // Directly switch to the new display without waiting for the response.
   switchToNewDisplay(int display, SessionID sessionId, String peerId) {
+    // VideoHandler creation is upon when video frames are received, so either caching commands(don't know next width/height) or stopping recording when switching displays.
+    parent.target?.recordingModel.onClose();
     // no need to wait for the response
     pi.currentDisplay = display;
     updateCurDisplay(sessionId);
@@ -868,7 +870,6 @@ class FfiModel with ChangeNotifier {
     } catch (e) {
       //
     }
-    parent.target?.recordingModel.onSwitchDisplay();
   }
 
   updateBlockInputState(Map<String, dynamic> evt, String peerId) {
@@ -1850,56 +1851,66 @@ class RecordingModel with ChangeNotifier {
     int? width = parent.target?.canvasModel.getDisplayWidth();
     int? height = parent.target?.canvasModel.getDisplayHeight();
     if (sessionId == null || width == null || height == null) return;
-    final currentDisplay = parent.target?.ffiModel.pi.currentDisplay;
+    final pi = parent.target?.ffiModel.pi;
-    if (currentDisplay != kAllDisplayValue) {
+    if (pi == null) return;
+    final currentDisplay = pi.currentDisplay;
+    if (currentDisplay == kAllDisplayValue) return;
     bind.sessionRecordScreen(
         sessionId: sessionId,
         start: true,
-          display: currentDisplay!,
+        display: currentDisplay,
         width: width,
         height: height);
   }
-  }
 
   toggle() async {
     if (isIOS) return;
     final sessionId = parent.target?.sessionId;
     if (sessionId == null) return;
+    final pi = parent.target?.ffiModel.pi;
+    if (pi == null) return;
+    final currentDisplay = pi.currentDisplay;
+    if (currentDisplay == kAllDisplayValue) return;
     _start = !_start;
     notifyListeners();
-    await bind.sessionRecordStatus(sessionId: sessionId, status: _start);
+    await _sendStatusMessage(sessionId, pi, _start);
     if (_start) {
-      final pi = parent.target?.ffiModel.pi;
-      if (pi != null) {
       sessionRefreshVideo(sessionId, pi);
+      if (versionCmp(pi.version, '1.2.4') >= 0) {
+        // will not receive SwitchDisplay since 1.2.4
+        onSwitchDisplay();
       }
     } else {
-      final currentDisplay = parent.target?.ffiModel.pi.currentDisplay;
-      if (currentDisplay != kAllDisplayValue) {
       bind.sessionRecordScreen(
           sessionId: sessionId,
           start: false,
-            display: currentDisplay!,
+          display: currentDisplay,
           width: 0,
           height: 0);
     }
   }
-  }
 
-  onClose() {
+  onClose() async {
     if (isIOS) return;
     final sessionId = parent.target?.sessionId;
     if (sessionId == null) return;
+    if (!_start) return;
     _start = false;
-    final currentDisplay = parent.target?.ffiModel.pi.currentDisplay;
+    final pi = parent.target?.ffiModel.pi;
-    if (currentDisplay != kAllDisplayValue) {
+    if (pi == null) return;
+    final currentDisplay = pi.currentDisplay;
+    if (currentDisplay == kAllDisplayValue) return;
+    await _sendStatusMessage(sessionId, pi, false);
     bind.sessionRecordScreen(
         sessionId: sessionId,
         start: false,
-          display: currentDisplay!,
+        display: currentDisplay,
         width: 0,
         height: 0);
   }
+
+  _sendStatusMessage(SessionID sessionId, PeerInfo pi, bool status) async {
+    await bind.sessionRecordStatus(sessionId: sessionId, status: status);
   }
 }
 

libs/scrap/Cargo.toml

@@ -19,7 +19,7 @@ cfg-if = "1.0"
 num_cpus = "1.15"
 lazy_static = "1.4"
 hbb_common = { path = "../hbb_common" }
-webm = "1.0"
+webm = { git = "https://github.com/21pages/rust-webm" }
 
 [dependencies.winapi]
 version = "0.3"

libs/scrap/src/common/record.rs

@@ -49,9 +49,12 @@ impl RecorderContext {
         }
         let file = if self.server { "s" } else { "c" }.to_string()
             + &self.id.clone()
-            + &chrono::Local::now().format("_%Y%m%d%H%M%S_").to_string()
+            + &chrono::Local::now().format("_%Y%m%d%H%M%S%3f_").to_string()
-            + &self.format.to_string()
+            + &self.format.to_string().to_lowercase()
-            + if self.format == CodecFormat::VP9 || self.format == CodecFormat::VP8 {
+            + if self.format == CodecFormat::VP9
+                || self.format == CodecFormat::VP8
+                || self.format == CodecFormat::AV1
+            {
                 ".webm"
             } else {
                 ".mp4"
@@ -83,6 +86,7 @@ pub enum RecordState {
 pub struct Recorder {
     pub inner: Box<dyn RecorderApi>,
     ctx: RecorderContext,
+    pts: Option<i64>,
 }
 
 impl Deref for Recorder {
@@ -101,19 +105,18 @@ impl DerefMut for Recorder {
 
 impl Recorder {
     pub fn new(mut ctx: RecorderContext) -> ResultType<Self> {
-        if ctx.format == CodecFormat::AV1 {
-            bail!("not support av1 recording");
-        }
         ctx.set_filename()?;
         let recorder = match ctx.format {
-            CodecFormat::VP8 | CodecFormat::VP9 => Recorder {
+            CodecFormat::VP8 | CodecFormat::VP9 | CodecFormat::AV1 => Recorder {
                 inner: Box::new(WebmRecorder::new(ctx.clone())?),
                 ctx,
+                pts: None,
             },
             #[cfg(feature = "hwcodec")]
             _ => Recorder {
                 inner: Box::new(HwRecorder::new(ctx.clone())?),
                 ctx,
+                pts: None,
             },
             #[cfg(not(feature = "hwcodec"))]
             _ => bail!("unsupported codec type"),
@@ -125,13 +128,16 @@ impl Recorder {
     fn change(&mut self, mut ctx: RecorderContext) -> ResultType<()> {
         ctx.set_filename()?;
         self.inner = match ctx.format {
-            CodecFormat::VP8 | CodecFormat::VP9 => Box::new(WebmRecorder::new(ctx.clone())?),
+            CodecFormat::VP8 | CodecFormat::VP9 | CodecFormat::AV1 => {
+                Box::new(WebmRecorder::new(ctx.clone())?)
+            }
             #[cfg(feature = "hwcodec")]
             _ => Box::new(HwRecorder::new(ctx.clone())?),
             #[cfg(not(feature = "hwcodec"))]
             _ => bail!("unsupported codec type"),
         };
         self.ctx = ctx;
+        self.pts = None;
         self.send_state(RecordState::NewFile(self.ctx.filename.clone()));
         Ok(())
     }
@@ -153,7 +159,10 @@ impl Recorder {
                         ..self.ctx.clone()
                     })?;
                 }
-                vp8s.frames.iter().map(|f| self.write_video(f)).count();
+                for f in vp8s.frames.iter() {
+                    self.check_pts(f.pts)?;
+                    self.write_video(f);
+                }
             }
             video_frame::Union::Vp9s(vp9s) => {
                 if self.ctx.format != CodecFormat::VP9 {
@@ -162,7 +171,22 @@ impl Recorder {
                         ..self.ctx.clone()
                     })?;
                 }
-                vp9s.frames.iter().map(|f| self.write_video(f)).count();
+                for f in vp9s.frames.iter() {
+                    self.check_pts(f.pts)?;
+                    self.write_video(f);
+                }
+            }
+            video_frame::Union::Av1s(av1s) => {
+                if self.ctx.format != CodecFormat::AV1 {
+                    self.change(RecorderContext {
+                        format: CodecFormat::AV1,
+                        ..self.ctx.clone()
+                    })?;
+                }
+                for f in av1s.frames.iter() {
+                    self.check_pts(f.pts)?;
+                    self.write_video(f);
+                }
             }
             #[cfg(feature = "hwcodec")]
             video_frame::Union::H264s(h264s) => {
@@ -172,8 +196,9 @@ impl Recorder {
                         ..self.ctx.clone()
                     })?;
                 }
-                if self.ctx.format == CodecFormat::H264 {
+                for f in h264s.frames.iter() {
-                    h264s.frames.iter().map(|f| self.write_video(f)).count();
+                    self.check_pts(f.pts)?;
+                    self.write_video(f);
                 }
             }
             #[cfg(feature = "hwcodec")]
@@ -184,8 +209,9 @@ impl Recorder {
                         ..self.ctx.clone()
                     })?;
                 }
-                if self.ctx.format == CodecFormat::H265 {
+                for f in h265s.frames.iter() {
-                    h265s.frames.iter().map(|f| self.write_video(f)).count();
+                    self.check_pts(f.pts)?;
+                    self.write_video(f);
                 }
             }
             _ => bail!("unsupported frame type"),
@@ -194,6 +220,17 @@ impl Recorder {
         Ok(())
     }
 
+    fn check_pts(&mut self, pts: i64) -> ResultType<()> {
+        // https://stackoverflow.com/questions/76379101/how-to-create-one-playable-webm-file-from-two-different-video-tracks-with-same-c
+        let old_pts = self.pts;
+        self.pts = Some(pts);
+        if old_pts.clone().unwrap_or_default() > pts {
+            log::info!("pts {:?}->{}, change record filename", old_pts, pts);
+            self.change(self.ctx.clone())?;
+        }
+        Ok(())
+    }
+
     fn send_state(&self, state: RecordState) {
         self.ctx.tx.as_ref().map(|tx| tx.send(state));
     }
@@ -230,10 +267,19 @@ impl RecorderApi for WebmRecorder {
             None,
             if ctx.format == CodecFormat::VP9 {
                 mux::VideoCodecId::VP9
-            } else {
+            } else if ctx.format == CodecFormat::VP8 {
                 mux::VideoCodecId::VP8
+            } else {
+                mux::VideoCodecId::AV1
             },
         );
+        if ctx.format == CodecFormat::AV1 {
+            // [129, 8, 12, 0] in 3.6.0, but zero works
+            let codec_private = vec![0, 0, 0, 0];
+            if !webm.set_codec_private(vt.track_number(), &codec_private) {
+                bail!("Failed to set codec private");
+            }
+        }
         Ok(WebmRecorder {
             vt,
             webm: Some(webm),

src/client.rs

@@ -999,16 +999,19 @@ pub struct VideoHandler {
     pub rgb: ImageRgb,
     recorder: Arc<Mutex<Option<Recorder>>>,
     record: bool,
+    _display: usize, // useful for debug
 }
 
 impl VideoHandler {
     /// Create a new video handler.
-    pub fn new() -> Self {
+    pub fn new(_display: usize) -> Self {
+        log::info!("new video handler for display #{_display}");
         VideoHandler {
             decoder: Decoder::new(),
             rgb: ImageRgb::new(ImageFormat::ARGB, crate::DST_STRIDE_RGBA),
             recorder: Default::default(),
             record: false,
+            _display,
         }
     }
 
@@ -1900,7 +1903,7 @@ where
                         if handler_controller_map.len() <= display {
                             for _i in handler_controller_map.len()..=display {
                                 handler_controller_map.push(VideoHandlerController {
-                                    handler: VideoHandler::new(),
+                                    handler: VideoHandler::new(_i),
                                     count: 0,
                                     duration: std::time::Duration::ZERO,
                                     skip_beginning: 0,
@@ -1960,6 +1963,7 @@ where
                         }
                     }
                     MediaData::RecordScreen(start, display, w, h, id) => {
+                        log::info!("record screen command: start:{start}, display:{display}");
                         if handler_controller_map.len() == 1 {
                             // Compatible with the sciter version(single ui session).
                             // For the sciter version, there're no multi-ui-sessions for one connection.

src/server/connection.rs

@@ -2104,7 +2104,7 @@ impl Connection {
                 display,
                 video_service::OPTION_REFRESH,
                 super::service::SERVICE_OPTION_VALUE_TRUE,
-            )
+            );
         });
     }
 
@@ -2145,6 +2145,7 @@ impl Connection {
 
             // Send display changed message.
             // For compatibility with old versions ( < 1.2.4 ).
+            // sciter need it in new version
             if let Some(msg_out) = video_service::make_display_changed_msg(self.display_idx, None) {
                 self.send(msg_out).await;
             }

src/server/video_service.rs

@@ -436,8 +436,7 @@ fn run(vs: VideoService) -> ResultType<()> {
     log::info!("init quality={:?}, abr enabled:{}", quality, abr);
     let codec_name = Encoder::negotiated_codec();
     let recorder = get_recorder(c.width, c.height, &codec_name);
-    let last_recording =
+    let last_recording = recorder.lock().unwrap().is_some() || video_qos.record();
-        (recorder.lock().unwrap().is_some() || video_qos.record()) && codec_name != CodecName::AV1;
     drop(video_qos);
     let encoder_cfg = get_encoder_config(&c, quality, last_recording);
 
@@ -479,8 +478,7 @@ fn run(vs: VideoService) -> ResultType<()> {
             allow_err!(encoder.set_quality(quality));
             video_qos.store_bitrate(encoder.bitrate());
         }
-        let recording = (recorder.lock().unwrap().is_some() || video_qos.record())
+        let recording = recorder.lock().unwrap().is_some() || video_qos.record();
-            && codec_name != CodecName::AV1;
         if recording != last_recording {
             bail!("SWITCH");
         }

src/ui/header.tis

@@ -144,7 +144,7 @@ class Header: Reactor.Component {
             <span #action>{svg_action}</span>
             <span #display>{svg_display}</span>
             <span #keyboard>{svg_keyboard}</span>
-            {recording_enabled && qualityMonitorData[4] != "AV1" ? <span #recording>{recording ? svg_recording_on : svg_recording_off}</span> : ""}
+            {recording_enabled ? <span #recording>{recording ? svg_recording_on : svg_recording_off}</span> : ""}
             {this.renderKeyboardPop()}
             {this.renderDisplayPop()}
             {this.renderActionPop()}
@@ -299,14 +299,22 @@ class Header: Reactor.Component {
         header.update();
         handler.record_status(recording);
         // 0 is just a dummy value. It will be ignored by the handler.
-        if (recording)
+        if (recording) {
             handler.refresh_video(0);
-        else
+            if (handler.version_cmp(pi.version, '1.2.4') >= 0) handler.record_screen(recording, pi.current_display, display_width, display_height);
-            handler.record_screen(false, 0, display_width, display_height);
+        }
+        else {
+            handler.record_screen(recording, pi.current_display, display_width, display_height);
+        }
     }
 
     event click $(#screen) (_, me) {
         if (pi.current_display == me.index) return;
+        if (recording) {
+            recording = false;
+            handler.record_screen(false, pi.current_display, display_width, display_height);
+            handler.record_status(false);
+        }
         handler.switch_display(me.index);
     }
     

src/ui/remote.rs

@@ -243,6 +243,7 @@ impl InvokeUiSession for SciterHandler {
         pi_sciter.set_item("sas_enabled", pi.sas_enabled);
         pi_sciter.set_item("displays", Self::make_displays_array(&pi.displays));
         pi_sciter.set_item("current_display", pi.current_display);
+        pi_sciter.set_item("version", pi.version.clone());
         self.call("updatePi", &make_args!(pi_sciter));
     }
 
@@ -469,6 +470,7 @@ impl sciter::EventHandler for SciterSession {
         fn restart_remote_device();
         fn request_voice_call();
         fn close_voice_call();
+        fn version_cmp(String, String);
     }
 }
 
@@ -757,6 +759,10 @@ impl SciterSession {
             log::error!("Failed to spawn IP tunneling: {}", err);
         }
     }
+
+    fn version_cmp(&self, v1: String, v2: String) -> i32 {
+        (hbb_common::get_version_number(&v1) - hbb_common::get_version_number(&v2)) as i32
+    }
 }
 
 pub fn make_fd(id: i32, entries: &Vec<FileEntry>, only_count: bool) -> Value {

src/ui/remote.tis

@@ -522,7 +522,6 @@ handler.updateQualityStatus = function(speed, fps, delay, bitrate, codec_format)
     bitrate ? qualityMonitorData[3] = bitrate:null;
     codec_format ? qualityMonitorData[4] = codec_format:null;
     qualityMonitor.update();
-    if (codec_format) header.update();
 }
 
 handler.setPermission = function(name, enabled) {