add zero copy mode hareware codec for windows (#6778)

Signed-off-by: 21pages <pages21@163.com>

flutter/lib/desktop/pages/desktop_setting_page.dart

@@ -388,10 +388,12 @@ class _GeneralState extends State<_General> {
   }
 
   Widget hwcodec() {
+    final hwcodec = bind.mainHasHwcodec();
+    final gpucodec = bind.mainHasGpucodec();
     return Offstage(
-      offstage: !bind.mainHasHwcodec(),
+      offstage: !(hwcodec || gpucodec),
       child: _Card(title: 'Hardware Codec', children: [
-        _OptionCheckBox(context, 'Enable hardware codec', 'enable-hwcodec'),
+        _OptionCheckBox(context, 'Enable hardware codec', 'enable-hwcodec')
       ]),
     );
   }

flutter/lib/desktop/pages/remote_page.dart

@@ -79,7 +79,6 @@ class _RemotePageState extends State<RemotePage>
   late RxBool _zoomCursor;
   late RxBool _remoteCursorMoved;
   late RxBool _keyboardEnabled;
-  final Map<int, RenderTexture> _renderTextures = {};
 
   var _blockableOverlayState = BlockableOverlayState();
 
@@ -212,9 +211,7 @@ class _RemotePageState extends State<RemotePage>
     // https://github.com/flutter/flutter/issues/64935
     super.dispose();
     debugPrint("REMOTE PAGE dispose session $sessionId ${widget.id}");
-    for (final texture in _renderTextures.values) {
-      await texture.destroy(closeSession);
-    }
+    _ffi.textureModel.onRemotePageDispose(closeSession);
     // ensure we leave this session, this is a double check
     _ffi.inputModel.enterOrLeave(false);
     DesktopMultiWindow.removeListener(this);
@@ -429,38 +426,6 @@ class _RemotePageState extends State<RemotePage>
     );
   }
 
-  Map<int, RenderTexture> _updateGetRenderTextures(int curDisplay) {
-    tryCreateTexture(int idx) {
-      if (!_renderTextures.containsKey(idx)) {
-        final renderTexture = RenderTexture();
-        _renderTextures[idx] = renderTexture;
-        renderTexture.create(idx, sessionId);
-      }
-    }
-
-    tryRemoveTexture(int idx) {
-      if (_renderTextures.containsKey(idx)) {
-        _renderTextures[idx]!.destroy(true);
-        _renderTextures.remove(idx);
-      }
-    }
-
-    if (curDisplay == kAllDisplayValue) {
-      final displays = _ffi.ffiModel.pi.getCurDisplays();
-      for (var i = 0; i < displays.length; i++) {
-        tryCreateTexture(i);
-      }
-    } else {
-      tryCreateTexture(curDisplay);
-      for (var i = 0; i < _ffi.ffiModel.pi.displays.length; i++) {
-        if (i != curDisplay) {
-          tryRemoveTexture(i);
-        }
-      }
-    }
-    return _renderTextures;
-  }
-
   Widget getBodyForDesktop(BuildContext context) {
     var paints = <Widget>[
       MouseRegion(onEnter: (evt) {
@@ -475,16 +440,19 @@ class _RemotePageState extends State<RemotePage>
         return Obx(
           () => _ffi.ffiModel.pi.isSet.isFalse
               ? Container(color: Colors.transparent)
-              : Obx(() => ImagePaint(
+              : Obx(() {
+                  _ffi.textureModel.updateCurrentDisplay(peerDisplay.value);
+                  return ImagePaint(
                     id: widget.id,
                     zoomCursor: _zoomCursor,
                     cursorOverImage: _cursorOverImage,
                     keyboardEnabled: _keyboardEnabled,
                     remoteCursorMoved: _remoteCursorMoved,
-                    renderTextures: _updateGetRenderTextures(peerDisplay.value),
                     listenerBuilder: (child) => _buildRawTouchAndPointerRegion(
                         child, enterView, leaveView),
-                  )),
+                    ffi: _ffi,
+                  );
+                }),
         );
       }))
     ];
@@ -515,22 +483,22 @@ class _RemotePageState extends State<RemotePage>
 }
 
 class ImagePaint extends StatefulWidget {
+  final FFI ffi;
   final String id;
   final RxBool zoomCursor;
   final RxBool cursorOverImage;
   final RxBool keyboardEnabled;
   final RxBool remoteCursorMoved;
-  final Map<int, RenderTexture> renderTextures;
   final Widget Function(Widget)? listenerBuilder;
 
   ImagePaint(
       {Key? key,
+      required this.ffi,
       required this.id,
       required this.zoomCursor,
       required this.cursorOverImage,
       required this.keyboardEnabled,
       required this.remoteCursorMoved,
-      required this.renderTextures,
       this.listenerBuilder})
       : super(key: key);
 
@@ -548,6 +516,11 @@ class _ImagePaintState extends State<ImagePaint> {
   RxBool get remoteCursorMoved => widget.remoteCursorMoved;
   Widget Function(Widget)? get listenerBuilder => widget.listenerBuilder;
 
+  @override
+  void initState() {
+    super.initState();
+  }
+
   @override
   Widget build(BuildContext context) {
     final m = Provider.of<ImageModel>(context);
@@ -668,10 +641,10 @@ class _ImagePaintState extends State<ImagePaint> {
     }
     final curDisplay = ffiModel.pi.currentDisplay;
     for (var i = 0; i < displays.length; i++) {
-      final textureId = widget
-          .renderTextures[curDisplay == kAllDisplayValue ? i : curDisplay]
-          ?.textureId;
-      if (textureId != null) {
+      final textureId = widget.ffi.textureModel
+          .getTextureId(curDisplay == kAllDisplayValue ? i : curDisplay);
+      if (true) {
+        // both "textureId.value != -1" and "true" seems ok
         children.add(Positioned(
           left: (displays[i].x - rect.left) * s + offset.dx,
           top: (displays[i].y - rect.top) * s + offset.dy,

flutter/lib/desktop/widgets/tabbar_widget.dart

@@ -151,18 +151,20 @@ class DesktopTabController {
       return false;
     }
     state.update((val) {
-      val!.selected = index;
-      Future.delayed(Duration(milliseconds: 100), (() {
-        if (val.pageController.hasClients) {
-          val.pageController.jumpToPage(index);
-        }
-        val.scrollController.itemCount = val.tabs.length;
-        if (val.scrollController.hasClients &&
-            val.scrollController.itemCount > index) {
-          val.scrollController
-              .scrollToItem(index, center: false, animate: true);
-        }
-      }));
+      if (val != null) {
+        val.selected = index;
+        Future.delayed(Duration(milliseconds: 100), (() {
+          if (val.pageController.hasClients) {
+            val.pageController.jumpToPage(index);
+          }
+          val.scrollController.itemCount = val.tabs.length;
+          if (val.scrollController.hasClients &&
+              val.scrollController.itemCount > index) {
+            val.scrollController
+                .scrollToItem(index, center: false, animate: true);
+          }
+        }));
+      }
     });
     if (callOnSelected) {
       if (state.value.tabs.length > index) {

flutter/lib/models/desktop_render_texture.dart

@@ -1,46 +1,241 @@
+import 'package:flutter/material.dart';
+import 'package:flutter_gpu_texture_renderer/flutter_gpu_texture_renderer.dart';
+import 'package:flutter_hbb/consts.dart';
+import 'package:flutter_hbb/models/model.dart';
 import 'package:get/get.dart';
 import 'package:texture_rgba_renderer/texture_rgba_renderer.dart';
 
 import '../../common.dart';
 import './platform_model.dart';
 
-final useTextureRender = bind.mainUseTextureRender();
+final useTextureRender =
+    bind.mainHasPixelbufferTextureRender() || bind.mainHasGpuTextureRender();
 
-class RenderTexture {
-  final RxInt textureId = RxInt(-1);
+class _PixelbufferTexture {
   int _textureKey = -1;
   int _display = 0;
   SessionID? _sessionId;
+  final support = bind.mainHasPixelbufferTextureRender();
+  bool _destroying = false;
+  int? _id;
 
   final textureRenderer = TextureRgbaRenderer();
 
-  RenderTexture();
-
   int get display => _display;
 
-  create(int d, SessionID sessionId) {
-    if (useTextureRender) {
+  create(int d, SessionID sessionId, FFI ffi) {
+    if (support) {
       _display = d;
       _textureKey = bind.getNextTextureKey();
       _sessionId = sessionId;
 
       textureRenderer.createTexture(_textureKey).then((id) async {
+        _id = id;
         if (id != -1) {
+          ffi.textureModel.setRgbaTextureId(display: d, id: id);
           final ptr = await textureRenderer.getTexturePtr(_textureKey);
-          platformFFI.registerTexture(sessionId, display, ptr);
-          textureId.value = id;
+          platformFFI.registerPixelbufferTexture(sessionId, display, ptr);
+          debugPrint(
+              "create pixelbuffer texture: peerId: ${ffi.id} display:$_display, textureId:$id");
         }
       });
     }
   }
 
-  destroy(bool unregisterTexture) async {
-    if (useTextureRender && _textureKey != -1 && _sessionId != null) {
+  destroy(bool unregisterTexture, FFI ffi) async {
+    if (!_destroying && support && _textureKey != -1 && _sessionId != null) {
+      _destroying = true;
       if (unregisterTexture) {
-        platformFFI.registerTexture(_sessionId!, display, 0);
+        platformFFI.registerPixelbufferTexture(_sessionId!, display, 0);
+        // sleep for a while to avoid the texture is used after it's unregistered.
+        await Future.delayed(Duration(milliseconds: 100));
       }
       await textureRenderer.closeTexture(_textureKey);
       _textureKey = -1;
+      _destroying = false;
+      debugPrint(
+          "destroy pixelbuffer texture: peerId: ${ffi.id} display:$_display, textureId:$_id");
+    }
+  }
+}
+
+class _GpuTexture {
+  int _textureId = -1;
+  SessionID? _sessionId;
+  final support = bind.mainHasGpuTextureRender();
+  bool _destroying = false;
+  int _display = 0;
+  int? _id;
+  int? _output;
+
+  int get display => _display;
+
+  final gpuTextureRenderer = FlutterGpuTextureRenderer();
+
+  _GpuTexture();
+
+  create(int d, SessionID sessionId, FFI ffi) {
+    if (support) {
+      _sessionId = sessionId;
+      _display = d;
+
+      gpuTextureRenderer.registerTexture().then((id) async {
+        _id = id;
+        if (id != null) {
+          _textureId = id;
+          ffi.textureModel.setGpuTextureId(display: d, id: id);
+          final output = await gpuTextureRenderer.output(id);
+          _output = output;
+          if (output != null) {
+            platformFFI.registerGpuTexture(sessionId, d, output);
+          }
+          debugPrint(
+              "create gpu texture: peerId: ${ffi.id} display:$_display, textureId:$id, output:$output");
+        }
+      }, onError: (err) {
+        debugPrint("Failed to register gpu texture:$err");
+      });
+    }
+  }
+
+  destroy(FFI ffi) async {
+    // must stop texture render, render unregistered texture cause crash
+    if (!_destroying && support && _sessionId != null && _textureId != -1) {
+      _destroying = true;
+      platformFFI.registerGpuTexture(_sessionId!, _display, 0);
+      // sleep for a while to avoid the texture is used after it's unregistered.
+      await Future.delayed(Duration(milliseconds: 100));
+      await gpuTextureRenderer.unregisterTexture(_textureId);
+      _textureId = -1;
+      _destroying = false;
+      debugPrint(
+          "destroy gpu texture: peerId: ${ffi.id} display:$_display, textureId:$_id, output:$_output");
+    }
+  }
+}
+
+class _Control {
+  RxInt textureID = (-1).obs;
+
+  int _rgbaTextureId = -1;
+  int get rgbaTextureId => _rgbaTextureId;
+  int _gpuTextureId = -1;
+  int get gpuTextureId => _gpuTextureId;
+  bool _isGpuTexture = false;
+  bool get isGpuTexture => _isGpuTexture;
+
+  setTextureType({bool gpuTexture = false}) {
+    _isGpuTexture = gpuTexture;
+    textureID.value = _isGpuTexture ? gpuTextureId : rgbaTextureId;
+  }
+
+  setRgbaTextureId(int id) {
+    _rgbaTextureId = id;
+    textureID.value = _isGpuTexture ? gpuTextureId : rgbaTextureId;
+  }
+
+  setGpuTextureId(int id) {
+    _gpuTextureId = id;
+    textureID.value = _isGpuTexture ? gpuTextureId : rgbaTextureId;
+  }
+}
+
+class TextureModel {
+  final WeakReference<FFI> parent;
+  final Map<int, _Control> _control = {};
+  final Map<int, _PixelbufferTexture> _pixelbufferRenderTextures = {};
+  final Map<int, _GpuTexture> _gpuRenderTextures = {};
+
+  TextureModel(this.parent);
+
+  setTextureType({required int display, required bool gpuTexture}) {
+    debugPrint("setTextureType: display:$display, isGpuTexture:$gpuTexture");
+    var texture = _control[display];
+    if (texture == null) {
+      texture = _Control();
+      _control[display] = texture;
+    }
+    texture.setTextureType(gpuTexture: gpuTexture);
+  }
+
+  setRgbaTextureId({required int display, required int id}) {
+    var ctl = _control[display];
+    if (ctl == null) {
+      ctl = _Control();
+      _control[display] = ctl;
+    }
+    ctl.setRgbaTextureId(id);
+  }
+
+  setGpuTextureId({required int display, required int id}) {
+    var ctl = _control[display];
+    if (ctl == null) {
+      ctl = _Control();
+      _control[display] = ctl;
+    }
+    ctl.setGpuTextureId(id);
+  }
+
+  RxInt getTextureId(int display) {
+    var ctl = _control[display];
+    if (ctl == null) {
+      ctl = _Control();
+      _control[display] = ctl;
+    }
+    return ctl.textureID;
+  }
+
+  updateCurrentDisplay(int curDisplay) {
+    final ffi = parent.target;
+    if (ffi == null) return;
+    tryCreateTexture(int idx) {
+      if (!_pixelbufferRenderTextures.containsKey(idx)) {
+        final renderTexture = _PixelbufferTexture();
+        _pixelbufferRenderTextures[idx] = renderTexture;
+        renderTexture.create(idx, ffi.sessionId, ffi);
+      }
+      if (!_gpuRenderTextures.containsKey(idx)) {
+        final renderTexture = _GpuTexture();
+        _gpuRenderTextures[idx] = renderTexture;
+        renderTexture.create(idx, ffi.sessionId, ffi);
+      }
+    }
+
+    tryRemoveTexture(int idx) {
+      _control.remove(idx);
+      if (_pixelbufferRenderTextures.containsKey(idx)) {
+        _pixelbufferRenderTextures[idx]!.destroy(true, ffi);
+        _pixelbufferRenderTextures.remove(idx);
+      }
+      if (_gpuRenderTextures.containsKey(idx)) {
+        _gpuRenderTextures[idx]!.destroy(ffi);
+        _gpuRenderTextures.remove(idx);
+      }
+    }
+
+    if (curDisplay == kAllDisplayValue) {
+      final displays = ffi.ffiModel.pi.getCurDisplays();
+      for (var i = 0; i < displays.length; i++) {
+        tryCreateTexture(i);
+      }
+    } else {
+      tryCreateTexture(curDisplay);
+      for (var i = 0; i < ffi.ffiModel.pi.displays.length; i++) {
+        if (i != curDisplay) {
+          tryRemoveTexture(i);
+        }
+      }
+    }
+  }
+
+  onRemotePageDispose(bool closeSession) async {
+    final ffi = parent.target;
+    if (ffi == null) return;
+    for (final texture in _pixelbufferRenderTextures.values) {
+      await texture.destroy(closeSession, ffi);
+    }
+    for (final texture in _gpuRenderTextures.values) {
+      await texture.destroy(ffi);
     }
   }
 }

flutter/lib/models/model.dart

@@ -2097,6 +2097,7 @@ class FFI {
   late final InputModel inputModel; // session
   late final ElevationModel elevationModel; // session
   late final CmFileModel cmFileModel; // cm
+  late final TextureModel textureModel; //session
 
   FFI(SessionID? sId) {
     sessionId = sId ?? (isDesktop ? Uuid().v4obj() : _constSessionId);
@@ -2116,6 +2117,7 @@ class FFI {
     inputModel = InputModel(WeakReference(this));
     elevationModel = ElevationModel(WeakReference(this));
     cmFileModel = CmFileModel(WeakReference(this));
+    textureModel = TextureModel(WeakReference(this));
   }
 
   /// Mobile reuse FFI
@@ -2195,6 +2197,9 @@ class FFI {
       }
     }
 
+    final hasPixelBufferTextureRender = bind.mainHasPixelbufferTextureRender();
+    final hasGpuTextureRender = bind.mainHasGpuTextureRender();
+
     final SimpleWrapper<bool> isToNewWindowNotified = SimpleWrapper(false);
     // Preserved for the rgba data.
     stream.listen((message) {
@@ -2240,7 +2245,9 @@ class FFI {
           }
         } else if (message is EventToUI_Rgba) {
           final display = message.field0;
-          if (useTextureRender) {
+          if (hasPixelBufferTextureRender) {
+            debugPrint("EventToUI_Rgba display:$display");
+            textureModel.setTextureType(display: display, gpuTexture: false);
             onEvent2UIRgba();
           } else {
             // Fetch the image buffer from rust codes.
@@ -2254,6 +2261,13 @@ class FFI {
               imageModel.onRgba(display, rgba);
             }
           }
+        } else if (message is EventToUI_Texture) {
+          final display = message.field0;
+          debugPrint("EventToUI_Texture display:$display");
+          if (hasGpuTextureRender) {
+            textureModel.setTextureType(display: display, gpuTexture: true);
+            onEvent2UIRgba();
+          }
         }
       }();
     });

flutter/lib/models/native_model.dart

@@ -99,9 +99,14 @@ class PlatformFFI {
 
   int getRgbaSize(SessionID sessionId, int display) =>
       _ffiBind.sessionGetRgbaSize(sessionId: sessionId, display: display);
-  void nextRgba(SessionID sessionId, int display) => _ffiBind.sessionNextRgba(sessionId: sessionId, display: display);
-  void registerTexture(SessionID sessionId, int display, int ptr) =>
-      _ffiBind.sessionRegisterTexture(sessionId: sessionId, display: display, ptr: ptr);
+  void nextRgba(SessionID sessionId, int display) =>
+      _ffiBind.sessionNextRgba(sessionId: sessionId, display: display);
+  void registerPixelbufferTexture(SessionID sessionId, int display, int ptr) =>
+      _ffiBind.sessionRegisterPixelbufferTexture(
+          sessionId: sessionId, display: display, ptr: ptr);
+  void registerGpuTexture(SessionID sessionId, int display, int ptr) =>
+      _ffiBind.sessionRegisterGpuTexture(
+          sessionId: sessionId, display: display, ptr: ptr);
 
   /// Init the FFI class, loads the native Rust core library.
   Future<void> init(String appType) async {

flutter/pubspec.lock

@@ -505,6 +505,15 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "0.0.4"
+  flutter_gpu_texture_renderer:
+    dependency: "direct main"
+    description:
+      path: "."
+      ref: "3865a99f60a92bea4d95bb5d55cf524b1bcbbf5a"
+      resolved-ref: "3865a99f60a92bea4d95bb5d55cf524b1bcbbf5a"
+      url: "https://github.com/21pages/flutter_gpu_texture_renderer"
+    source: git
+    version: "0.0.1"
   flutter_improved_scrolling:
     dependency: "direct main"
     description:

flutter/pubspec.yaml

@@ -92,6 +92,10 @@ dependencies:
   texture_rgba_renderer: ^0.0.16
   percent_indicator: ^4.2.2
   dropdown_button2: ^2.0.0
+  flutter_gpu_texture_renderer:
+    git:
+      url: https://github.com/21pages/flutter_gpu_texture_renderer
+      ref: 3865a99f60a92bea4d95bb5d55cf524b1bcbbf5a
   uuid: ^3.0.7
   auto_size_text_field: ^2.2.1
   flex_color_picker: ^3.3.0

flutter/windows/runner/flutter_window.cpp

@@ -4,6 +4,7 @@
 
 #include <desktop_multi_window/desktop_multi_window_plugin.h>
 #include <texture_rgba_renderer/texture_rgba_renderer_plugin_c_api.h>
+#include <flutter_gpu_texture_renderer/flutter_gpu_texture_renderer_plugin_c_api.h>
 
 #include "flutter/generated_plugin_registrant.h"
 
@@ -34,6 +35,8 @@ bool FlutterWindow::OnCreate() {
     auto *registry = flutter_view_controller->engine();
     TextureRgbaRendererPluginCApiRegisterWithRegistrar(
         registry->GetRegistrarForPlugin("TextureRgbaRendererPlugin"));
+    FlutterGpuTextureRendererPluginCApiRegisterWithRegistrar(
+        registry->GetRegistrarForPlugin("FlutterGpuTextureRendererPluginCApi"));
   });
   SetChildContent(flutter_controller_->view()->GetNativeWindow());
   return true;