Refact/multi window soft rendering (#8343)

* refact: multi_window_soft_rendering Signed-off-by: fufesou <linlong1266@gmail.com> * fix: window pos, potential wait for image Signed-off-by: fufesou <linlong1266@gmail.com> * comments Signed-off-by: fufesou <linlong1266@gmail.com> * remove debug print Signed-off-by: fufesou <linlong1266@gmail.com> * explicitly set rgba_data.size_got to false after init Signed-off-by: fufesou <linlong1266@gmail.com> * refact: multi window, merge images, render with texture Signed-off-by: fufesou <linlong1266@gmail.com> * revert, flutter.rs, rgba valid Signed-off-by: fufesou <linlong1266@gmail.com> * Add displays index before sending capture msg Signed-off-by: fufesou <linlong1266@gmail.com> * refact: multi window, soft rendering Signed-off-by: fufesou <linlong1266@gmail.com> * fix: build Signed-off-by: fufesou <linlong1266@gmail.com> --------- Signed-off-by: fufesou <linlong1266@gmail.com>
2025-10-29 17:00:05 +00:00 · 2024-06-13 18:03:41 +08:00
parent 8e12a34634
commit bc875a35b0
10 changed files with 160 additions and 82 deletions
--- a/flutter/lib/common.dart
+++ b/flutter/lib/common.dart
@@ -2924,10 +2924,10 @@ openMonitorInNewTabOrWindow(int i, String peerId, PeerInfo pi,
      kMainWindowId, kWindowEventOpenMonitorSession, jsonEncode(args));
 }

-setNewConnectWindowFrame(int windowId, String peerId, Rect? screenRect) async {
+setNewConnectWindowFrame(int windowId, String peerId, int? display, Rect? screenRect) async {
  if (screenRect == null) {
    await restoreWindowPosition(WindowType.RemoteDesktop,
-        windowId: windowId, peerId: peerId);
+        windowId: windowId, display: display, peerId: peerId);
  } else {
    await tryMoveToScreenAndSetFullscreen(screenRect);
  }
--- a/flutter/lib/common/widgets/setting_widgets.dart
+++ b/flutter/lib/common/widgets/setting_widgets.dart
@@ -234,12 +234,12 @@ List<(String, String)> otherDefaultSettings() {
    ('True color (4:4:4)', kOptionI444),
    ('Reverse mouse wheel', kKeyReverseMouseWheel),
    ('swap-left-right-mouse', kOptionSwapLeftRightMouse),
-    if (isDesktop && bind.mainGetUseTextureRender())
+    if (isDesktop)
      (
        'Show displays as individual windows',
        kKeyShowDisplaysAsIndividualWindows
      ),
-    if (isDesktop && bind.mainGetUseTextureRender())
+    if (isDesktop)
      (
        'Use all my displays for the remote session',
        kKeyUseAllMyDisplaysForTheRemoteSession
--- a/flutter/lib/common/widgets/toolbar.dart
+++ b/flutter/lib/common/widgets/toolbar.dart
@@ -586,7 +586,6 @@ Future<List<TToggleMenu>> toolbarDisplayToggle(
  if (pi.isSupportMultiDisplay &&
      PrivacyModeState.find(id).isEmpty &&
      pi.displaysCount.value > 1 &&
-      bind.mainGetUseTextureRender() &&
      bind.mainGetUserDefaultOption(key: kKeyShowMonitorsToolbar) == 'Y') {
    final value =
        bind.sessionGetDisplaysAsIndividualWindows(sessionId: ffi.sessionId) ==
@@ -602,9 +601,7 @@ Future<List<TToggleMenu>> toolbarDisplayToggle(
  }

  final isMultiScreens = !isWeb && (await getScreenRectList()).length > 1;
-  if (bind.mainGetUseTextureRender() &&
-      pi.isSupportMultiDisplay &&
-      isMultiScreens) {
+  if (pi.isSupportMultiDisplay && isMultiScreens) {
    final value = bind.sessionGetUseAllMyDisplaysForTheRemoteSession(
            sessionId: ffi.sessionId) ==
        'Y';
--- a/flutter/lib/desktop/pages/remote_page.dart
+++ b/flutter/lib/desktop/pages/remote_page.dart
@@ -617,10 +617,11 @@ class _ImagePaintState extends State<ImagePaint> {
      final paintWidth = c.getDisplayWidth() * s;
      final paintHeight = c.getDisplayHeight() * s;
      final paintSize = Size(paintWidth, paintHeight);
-      final paintWidget = m.useTextureRender
-          ? _BuildPaintTextureRender(
-              c, s, Offset.zero, paintSize, isViewOriginal())
-          : _buildScrollbarNonTextureRender(m, paintSize, s);
+      final paintWidget =
+          m.useTextureRender || widget.ffi.ffiModel.pi.forceTextureRender
+              ? _BuildPaintTextureRender(
+                  c, s, Offset.zero, paintSize, isViewOriginal())
+              : _buildScrollbarNonTextureRender(m, paintSize, s);
      return NotificationListener<ScrollNotification>(
          onNotification: (notification) {
            c.updateScrollPercent();
@@ -638,17 +639,18 @@ class _ImagePaintState extends State<ImagePaint> {
          ));
    } else {
      if (c.size.width > 0 && c.size.height > 0) {
-        final paintWidget = m.useTextureRender
-            ? _BuildPaintTextureRender(
-                c,
-                s,
-                Offset(
-                  isLinux ? c.x.toInt().toDouble() : c.x,
-                  isLinux ? c.y.toInt().toDouble() : c.y,
-                ),
-                c.size,
-                isViewOriginal())
-            : _buildScrollAuthNonTextureRender(m, c, s);
+        final paintWidget =
+            m.useTextureRender || widget.ffi.ffiModel.pi.forceTextureRender
+                ? _BuildPaintTextureRender(
+                    c,
+                    s,
+                    Offset(
+                      isLinux ? c.x.toInt().toDouble() : c.x,
+                      isLinux ? c.y.toInt().toDouble() : c.y,
+                    ),
+                    c.size,
+                    isViewOriginal())
+                : _buildScrollAutoNonTextureRender(m, c, s);
        return mouseRegion(child: _buildListener(paintWidget));
      } else {
        return Container();
@@ -664,7 +666,7 @@ class _ImagePaintState extends State<ImagePaint> {
    );
  }

-  Widget _buildScrollAuthNonTextureRender(
+  Widget _buildScrollAutoNonTextureRender(
      ImageModel m, CanvasModel c, double s) {
    return CustomPaint(
      size: Size(c.size.width, c.size.height),
--- a/flutter/lib/desktop/pages/remote_tab_page.dart
+++ b/flutter/lib/desktop/pages/remote_tab_page.dart
@@ -420,7 +420,7 @@ class _ConnectionTabPageState extends State<ConnectionTabPage> {
          await WindowController.fromWindowId(windowId()).setFullscreen(false);
          stateGlobal.setFullscreen(false, procWnd: false);
        }
-        await setNewConnectWindowFrame(windowId(), id!, screenRect);
+        await setNewConnectWindowFrame(windowId(), id!, display, screenRect);
        Future.delayed(Duration(milliseconds: isWindows ? 100 : 0), () async {
          await windowOnTop(windowId());
        });
--- a/flutter/lib/desktop/widgets/remote_toolbar.dart
+++ b/flutter/lib/desktop/widgets/remote_toolbar.dart
@@ -643,15 +643,12 @@ class _MonitorMenu extends StatelessWidget {
  }

  Widget buildMonitorSubmenuWidget(BuildContext context) {
-    final m = Provider.of<ImageModel>(context);
    return Column(
      mainAxisSize: MainAxisSize.min,
      children: [
        Row(children: buildMonitorList(context, false)),
-        supportIndividualWindows && m.useTextureRender ? Divider() : Offstage(),
-        supportIndividualWindows && m.useTextureRender
-            ? chooseDisplayBehavior()
-            : Offstage(),
+        supportIndividualWindows ? Divider() : Offstage(),
+        supportIndividualWindows ? chooseDisplayBehavior() : Offstage(),
      ],
    );
  }
@@ -737,10 +734,7 @@ class _MonitorMenu extends StatelessWidget {
    for (int i = 0; i < pi.displays.length; i++) {
      monitorList.add(buildMonitorButton(i));
    }
-    final m = Provider.of<ImageModel>(context);
-    if (supportIndividualWindows &&
-        m.useTextureRender &&
-        pi.displays.length > 1) {
+    if (supportIndividualWindows && pi.displays.length > 1) {
      monitorList.add(buildMonitorButton(kAllDisplayValue));
    }
    return monitorList;
@@ -824,7 +818,6 @@ class _MonitorMenu extends StatelessWidget {
    RxInt display = CurrentDisplayState.find(id);
    if (display.value != i) {
      final isChooseDisplayToOpenInNewWindow = pi.isSupportMultiDisplay &&
-          bind.mainGetUseTextureRender() &&
          bind.sessionGetDisplaysAsIndividualWindows(
                  sessionId: ffi.sessionId) ==
              'Y';
--- a/flutter/lib/models/model.dart
+++ b/flutter/lib/models/model.dart
@@ -1187,10 +1187,11 @@ class ImageModel with ChangeNotifier {

  onRgba(int display, Uint8List rgba) {
    final pid = parent.target?.id;
+    final rect = parent.target?.ffiModel.pi.getDisplayRect(display);
    img.decodeImageFromPixels(
        rgba,
-        parent.target?.ffiModel.rect?.width.toInt() ?? 0,
-        parent.target?.ffiModel.rect?.height.toInt() ?? 0,
+        rect?.width.toInt() ?? 0,
+        rect?.height.toInt() ?? 0,
        isWeb ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888,
        onPixelsCopied: () {
      // Unlock the rgba memory from rust codes.
@@ -2397,9 +2398,10 @@ class FFI {
      cursorModel.peerId = id;
    }

+    final isNewPeer = tabWindowId == null;
    // If tabWindowId != null, this session is a "tab -> window" one.
    // Else this session is a new one.
-    if (tabWindowId == null) {
+    if (isNewPeer) {
      // ignore: unused_local_variable
      final addRes = bind.sessionAddSync(
        sessionId: sessionId,
@@ -2424,14 +2426,25 @@ class FFI {
            'Unreachable, failed to add existed session to $id, $addRes');
        return;
      }
-      bind.sessionTryAddDisplay(
-          sessionId: sessionId, displays: Int32List.fromList(displays));
      ffiModel.pi.currentDisplay = display;
    }
    if (isDesktop && connType == ConnType.defaultConn) {
      textureModel.updateCurrentDisplay(display ?? 0);
    }
-    final stream = bind.sessionStart(sessionId: sessionId, id: id);
+
+    // CAUTION: `sessionStart()` and `sessionStartWithDisplays()` are an async functions.
+    // Though the stream is returned immediately, the stream may not be ready.
+    // Any operations that depend on the stream should be carefully handled.
+    late final Stream<EventToUI> stream;
+    if (isNewPeer || display == null || displays == null) {
+      stream = bind.sessionStart(sessionId: sessionId, id: id);
+    } else {
+      // We have to put displays in `sessionStart()` to make sure the stream is ready
+      // and then the displays' capturing requests can be sent.
+      stream = bind.sessionStartWithDisplays(
+          sessionId: sessionId, id: id, displays: Int32List.fromList(displays));
+    }
+
    if (isWeb) {
      platformFFI.setRgbaCallback((int display, Uint8List data) {
        onEvent2UIRgba();
@@ -2442,14 +2455,6 @@ class FFI {

    final cb = ffiModel.startEventListener(sessionId, id);

-    // Force refresh displays.
-    // The controlled side may not refresh the image when the (peer,display) is already subscribed.
-    if (displays != null) {
-      for (final display in displays) {
-        bind.sessionRefresh(sessionId: sessionId, display: display);
-      }
-    }
-
    imageModel.updateUserTextureRender();
    final hasGpuTextureRender = bind.mainHasGpuTextureRender();
    final SimpleWrapper<bool> isToNewWindowNotified = SimpleWrapper(false);
@@ -2500,8 +2505,8 @@ class FFI {
          }
        } else if (message is EventToUI_Rgba) {
          final display = message.field0;
-          if (imageModel.useTextureRender) {
-            debugPrint("EventToUI_Rgba display:$display");
+          if (imageModel.useTextureRender || ffiModel.pi.forceTextureRender) {
+            //debugPrint("EventToUI_Rgba display:$display");
            textureModel.setTextureType(display: display, gpuTexture: false);
            onEvent2UIRgba();
          } else {
@@ -2692,6 +2697,7 @@ class PeerInfo with ChangeNotifier {

  bool get isSupportMultiDisplay =>
      (isDesktop || isWebDesktop) && isSupportMultiUiSession;
+  bool get forceTextureRender => currentDisplay == kAllDisplayValue;

  bool get cursorEmbedded => tryGetDisplay()?.cursorEmbedded ?? false;

@@ -2700,30 +2706,32 @@ class PeerInfo with ChangeNotifier {
  bool get isAmyuniIdd =>
      platformAdditions[kPlatformAdditionsIddImpl] == 'amyuni_idd';

-  Display? tryGetDisplay() {
+  Display? tryGetDisplay({int? display}) {
    if (displays.isEmpty) {
      return null;
    }
-    if (currentDisplay == kAllDisplayValue) {
+    display ??= currentDisplay;
+    if (display == kAllDisplayValue) {
      return displays[0];
    } else {
-      if (currentDisplay > 0 && currentDisplay < displays.length) {
-        return displays[currentDisplay];
+      if (display > 0 && display < displays.length) {
+        return displays[display];
      } else {
        return displays[0];
      }
    }
  }

-  Display? tryGetDisplayIfNotAllDisplay() {
+  Display? tryGetDisplayIfNotAllDisplay({int? display}) {
    if (displays.isEmpty) {
      return null;
    }
-    if (currentDisplay == kAllDisplayValue) {
+    display ??= currentDisplay;
+    if (display == kAllDisplayValue) {
      return null;
    }
-    if (currentDisplay >= 0 && currentDisplay < displays.length) {
-      return displays[currentDisplay];
+    if (display >= 0 && display < displays.length) {
+      return displays[display];
    } else {
      return null;
    }
@@ -2747,6 +2755,12 @@ class PeerInfo with ChangeNotifier {
    }
    return 1.0;
  }
+
+  Rect? getDisplayRect(int display) {
+    final d = tryGetDisplayIfNotAllDisplay(display: display);
+    if (d == null) return null;
+    return Rect.fromLTWH(d.x, d.y, d.width.toDouble(), d.height.toDouble());
+  }
 }

 const canvasKey = 'canvas';
--- a/flutter/lib/web/bridge.dart
+++ b/flutter/lib/web/bridge.dart
@@ -63,11 +63,6 @@ class RustdeskImpl {
    return '';
  }

-  void sessionTryAddDisplay(
-      {required UuidValue sessionId,
-      required Int32List displays,
-      dynamic hint}) {}
-
  String sessionAddSync(
      {required UuidValue sessionId,
      required String id,
@@ -94,6 +89,14 @@ class RustdeskImpl {
    return Stream.empty();
  }

+  Stream<EventToUI> sessionStartWithDisplays(
+      {required UuidValue sessionId,
+      required String id,
+      required Int32List displays,
+      dynamic hint}) {
+    throw UnimplementedError();
+  }
+
  Future<bool?> sessionGetRemember(
      {required UuidValue sessionId, dynamic hint}) {
    return Future(