video_ack_required

2025-10-29 17:00:05 +00:00 · 2022-02-05 01:55:23 +08:00
parent 9ff3bd74c9
commit b348b3fdc8
65 changed files with 1307 additions and 58 deletions
--- a/src/codec.js
+++ b/src/codec.js
@@ -1,4 +1,5 @@
 // example: https://github.com/rgov/js-theora-decoder/blob/main/index.html
+// https://github.com/brion/ogv.js/releases, yarn add has no simd
 // dev: copy decoder files from node/ogv/dist/* to project dir
 // dist: .... to dist
 /*
@@ -20,8 +21,12 @@
 */

 export function loadVp9(callback) {
+  // Multithreading is used only if `options.threading` is true. 
+  // This requires browser support for the new `SharedArrayBuffer` and `Atomics` APIs, 
+  // currently available in Firefox and Chrome with experimental flags enabled.
+  // 所有主流浏览器均默认于2018年1月5日禁用SharedArrayBuffer
  window.OGVLoader.loadClass(
-    "OGVDecoderVideoVP9W",
+    "OGVDecoderVideoVP9SIMDW",
    (videoCodecClass) => {
      window.videoCodecClass = videoCodecClass;
      videoCodecClass({ videoFormat: {} }).then((decoder) => {
@@ -30,7 +35,7 @@ export function loadVp9(callback) {
        })
      })
    },
-    { worker: true }
+    { worker: true, threading: true }
  );
 }

--- a/src/common.ts
+++ b/src/common.ts
@@ -45,12 +45,11 @@ export function translate(locale: string, text: string): string {
 const zCode = "z".charCodeAt(0);
 const aCode = "a".charCodeAt(0);

-export function mapKey(name: string) {
+export function mapKey(name: string, isDesktop: Boolean) {
  const tmp = KEY_MAP[name];
-  if (!tmp) return undefined;
  if (tmp.length == 1) {
    const chr = tmp.charCodeAt(0);
-    if (chr > zCode || chr < aCode)
+    if (!isDesktop && (chr > zCode || chr < aCode))
      return KeyEvent.fromPartial({ unicode: chr });
    else return KeyEvent.fromPartial({ chr });
  }
--- a/src/connection.ts
+++ b/src/connection.ts
@@ -27,12 +27,14 @@ export default class Connection {
  _audioDecoder: any;
  _password: Uint8Array | undefined;
  _options: any;
+  _videoTestSpeed: number[];

  constructor() {
    this._msgbox = globals.msgbox;
    this._draw = globals.draw;
    this._msgs = [];
    this._id = "";
+    this._videoTestSpeed = [0, 0];
  }

  async start(id: string) {
@@ -58,7 +60,7 @@ export default class Connection {
    this.loadVideoDecoder();
    this.loadAudioDecoder();
    const uri = getDefaultUri();
-    const ws = new Websock(uri);
+    const ws = new Websock(uri, true);
    this._ws = ws;
    this._id = id;
    console.log(
@@ -75,7 +77,7 @@ export default class Connection {
      nat_type,
    });
    ws.sendRendezvous({ punch_hole_request });
-    const msg = ws.parseRendezvous(await ws.next());
+    const msg = (await ws.next()) as rendezvous.RendezvousMessage;
    ws.close();
    console.log(new Date() + ": Got relay response");
    const phr = msg.punch_hole_response;
@@ -113,7 +115,7 @@ export default class Connection {
    }
    const uuid = rr.uuid;
    console.log(new Date() + ": Connecting to relay server: " + uri);
-    const ws = new Websock(uri);
+    const ws = new Websock(uri, false, this.handleVideoFrame.bind(this));
    await ws.open();
    console.log(new Date() + ": Connected to relay server");
    this._ws = ws;
@@ -149,7 +151,7 @@ export default class Connection {
      this._ws?.sendMessage({});
      return;
    }
-    const msg = this._ws?.parseMessage(await this._ws?.next());
+    const msg = (await this._ws?.next()) as message.Message;
    let signedId: any = msg?.signed_id;
    if (!signedId) {
      console.error("Handshake failed: invalid message type");
@@ -198,7 +200,7 @@ export default class Connection {

  async msgLoop() {
    while (true) {
-      const msg = this._ws?.parseMessage(await this._ws?.next());
+      const msg = (await this._ws?.next()) as message.Message;
      if (msg?.hash) {
        this._hash = msg?.hash;
        if (!this._password)
@@ -251,6 +253,7 @@ export default class Connection {

  draw(frame: any) {
    this._draw?.(frame);
+    globals.draw(frame);
  }

  close() {
@@ -305,6 +308,7 @@ export default class Connection {
      my_name: "web", // to-do
      password,
      option: this.getOptionMessage(),
+      video_ack_required: true,
    });
    this._ws?.sendMessage({ login_request });
  }
@@ -341,6 +345,11 @@ export default class Connection {
    return n > 0 ? msg : undefined;
  }

+  sendVideoReceived() {
+    const misc = message.Misc.fromPartial({ video_received: true });
+    this._ws?.sendMessage({ misc });
+  }
+
  handleVideoFrame(vf: message.VideoFrame) {
    if (!this._firstFrame) {
      this.msgbox("", "", "");
@@ -348,11 +357,28 @@ export default class Connection {
    }
    if (vf.vp9s) {
      const dec = this._videoDecoder;
-      // dec.sync();
+      var tm = new Date().getTime();
+      var i = 0;
+      const n = vf.vp9s?.frames.length;
      vf.vp9s.frames.forEach((f) => {
        dec.processFrame(f.data.slice(0).buffer, (ok: any) => {
-          if (ok && dec.frameBuffer) {
+          i++;
+          if (i == n) this.sendVideoReceived();
+          if (ok && dec.frameBuffer && n == i) {
            this.draw(dec.frameBuffer);
+            const now = new Date().getTime();
+            var elapsed = now - tm;
+            this._videoTestSpeed[1] += elapsed;
+            this._videoTestSpeed[0] += 1;
+            if (this._videoTestSpeed[0] >= 30) {
+              console.log(
+                "video decoder: " +
+                  parseInt(
+                    "" + this._videoTestSpeed[1] / this._videoTestSpeed[0]
+                  )
+              );
+              this._videoTestSpeed = [0, 0];
+            }
          }
        });
      });
@@ -398,6 +424,8 @@ export default class Connection {
  handleMisc(misc: message.Misc) {
    if (misc.audio_format) {
      //
+    } else if (misc.chat_message) {
+      globals.pushEvent("chat", misc.chat_message.text);
    } else if (misc.permission_info) {
      const p = misc.permission_info;
      console.info("Change permission " + p.permission + " -> " + p.enabled);
@@ -457,18 +485,18 @@ export default class Connection {
    shift: Boolean,
    command: Boolean
  ) {
-    const key_event = mapKey(name);
+    const key_event = mapKey(name, globals.isDesktop());
    if (!key_event) return;
-    if (alt && name == 'VK_MENU') {
+    if (alt && name == "VK_MENU") {
      alt = false;
    }
-    if (ctrl && name == 'VK_CONTROL') {
+    if (ctrl && name == "VK_CONTROL") {
      ctrl = false;
    }
-    if (shift && name == 'VK_SHIFT') {
+    if (shift && name == "VK_SHIFT") {
      shift = false;
    }
-    if (command && name == 'Meta') {
+    if (command && name == "Meta") {
      command = false;
    }
    key_event.down = down;
@@ -581,9 +609,9 @@ export default class Connection {
    const misc = message.Misc.fromPartial({ option });
    this._ws?.sendMessage({ misc });
  }
-  
+
  getImageQuality() {
-    return this.getOption('image-quality');
+    return this.getOption("image-quality");
  }

  getImageQualityEnum(
@@ -631,7 +659,7 @@ export default class Connection {

 // @ts-ignore
 async function testDelay() {
-  const ws = new Websock(getDefaultUri(false));
+  const ws = new Websock(getDefaultUri(false), true);
  await ws.open();
  console.log(ws.latency());
 }
--- a/src/gen_js_from_hbb.ts
+++ b/src/gen_js_from_hbb.ts
@@ -684,12 +684,12 @@ export const KEY_MAP: any = {
  "VK_RBRACKET": "]",
  "VK_BACKSLASH": "\\",
  "VK_MINUS": "-",
-  "VK_PLUS": "= // it is =, but sciter return VK_PLUS",
-  "VK_DIVIDE": "Divide // numpad",
-  "VK_MULTIPLY": "Multiply // numpad",
-  "VK_SUBTRACT": "Subtract // numpad",
-  "VK_ADD": "Add // numpad",
-  "VK_DECIMAL": "Decimal // numpad",
+  "VK_PLUS": "=",
+  "VK_DIVIDE": "Divide",
+  "VK_MULTIPLY": "Multiply",
+  "VK_SUBTRACT": "Subtract",
+  "VK_ADD": "Add",
+  "VK_DECIMAL": "Decimal",
  "VK_F1": "F1",
  "VK_F2": "F2",
  "VK_F3": "F3",
--- a/src/globals.js
+++ b/src/globals.js
@@ -14,12 +14,15 @@ window.getRgba = () => {
  currentFrame = undefined;
  return tmp || null;
 }
-window.getLanguage = () => navigator.language;
 window.isMobile = () => {
  return /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|ipad|iris|kindle|Android|Silk|lge |maemo|midp|mmp|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows (ce|phone)|xda|xiino/i.test(navigator.userAgent)
    || /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test(navigator.userAgent.substr(0, 4));
 }

+export function isDesktop() {
+  return !isMobile;
+}
+
 export function msgbox(type, title, text) {
  if (!events) return;
  if (!type || (type == 'error' && !text)) return;
@@ -297,4 +300,4 @@ export function getPeers() {
  } catch (e) {
    return {};
  }
-}
+}
--- a/src/message.ts
+++ b/src/message.ts
@@ -642,6 +642,7 @@ export interface LoginRequest {
  option: OptionMessage | undefined;
  file_transfer: FileTransfer | undefined;
  port_forward: PortForward | undefined;
+  video_ack_required: boolean;
 }

 export interface ChatMessage {
@@ -944,6 +945,7 @@ export interface Misc {
  close_reason: string | undefined;
  refresh_video: boolean | undefined;
  option_response: OptionResponse | undefined;
+  video_received: boolean | undefined;
 }

 export interface Message {
@@ -1521,6 +1523,7 @@ function createBaseLoginRequest(): LoginRequest {
    option: undefined,
    file_transfer: undefined,
    port_forward: undefined,
+    video_ack_required: false,
  };
 }

@@ -1556,6 +1559,9 @@ export const LoginRequest = {
        writer.uint32(66).fork()
      ).ldelim();
    }
+    if (message.video_ack_required === true) {
+      writer.uint32(72).bool(message.video_ack_required);
+    }
    return writer;
  },

@@ -1587,6 +1593,9 @@ export const LoginRequest = {
        case 8:
          message.port_forward = PortForward.decode(reader, reader.uint32());
          break;
+        case 9:
+          message.video_ack_required = reader.bool();
+          break;
        default:
          reader.skipType(tag & 7);
          break;
@@ -1612,6 +1621,9 @@ export const LoginRequest = {
      port_forward: isSet(object.port_forward)
        ? PortForward.fromJSON(object.port_forward)
        : undefined,
+      video_ack_required: isSet(object.video_ack_required)
+        ? Boolean(object.video_ack_required)
+        : false,
    };
  },

@@ -1636,6 +1648,8 @@ export const LoginRequest = {
      (obj.port_forward = message.port_forward
        ? PortForward.toJSON(message.port_forward)
        : undefined);
+    message.video_ack_required !== undefined &&
+      (obj.video_ack_required = message.video_ack_required);
    return obj;
  },

@@ -1659,6 +1673,7 @@ export const LoginRequest = {
      object.port_forward !== undefined && object.port_forward !== null
        ? PortForward.fromPartial(object.port_forward)
        : undefined;
+    message.video_ack_required = object.video_ack_required ?? false;
    return message;
  },
 };
@@ -4466,6 +4481,7 @@ function createBaseMisc(): Misc {
    close_reason: undefined,
    refresh_video: undefined,
    option_response: undefined,
+    video_received: undefined,
  };
 }

@@ -4510,6 +4526,9 @@ export const Misc = {
        writer.uint32(90).fork()
      ).ldelim();
    }
+    if (message.video_received !== undefined) {
+      writer.uint32(96).bool(message.video_received);
+    }
    return writer;
  },

@@ -4553,6 +4572,9 @@ export const Misc = {
            reader.uint32()
          );
          break;
+        case 12:
+          message.video_received = reader.bool();
+          break;
        default:
          reader.skipType(tag & 7);
          break;
@@ -4587,6 +4609,9 @@ export const Misc = {
      option_response: isSet(object.option_response)
        ? OptionResponse.fromJSON(object.option_response)
        : undefined,
+      video_received: isSet(object.video_received)
+        ? Boolean(object.video_received)
+        : undefined,
    };
  },

@@ -4620,6 +4645,8 @@ export const Misc = {
      (obj.option_response = message.option_response
        ? OptionResponse.toJSON(message.option_response)
        : undefined);
+    message.video_received !== undefined &&
+      (obj.video_received = message.video_received);
    return obj;
  },

@@ -4651,6 +4678,7 @@ export const Misc = {
      object.option_response !== undefined && object.option_response !== null
        ? OptionResponse.fromPartial(object.option_response)
        : undefined;
+    message.video_received = object.video_received ?? undefined;
    return message;
  },
 };
--- a/src/ui.js
+++ b/src/ui.js
@@ -51,6 +51,7 @@ if (app) {
      const conn = globals.newConn();
      conn.setMsgbox(msgbox);
      conn.setDraw((f) => {
+        globals.draw(f);
        player.drawFrame(f);
      });
      document.querySelector('div#status').style.display = 'block';
--- a/src/websock.ts
+++ b/src/websock.ts
@@ -7,13 +7,19 @@ type Keys = "message" | "open" | "close" | "error";
 export default class Websock {
  _websocket: WebSocket;
  _eventHandlers: { [key in Keys]: Function };
-  _buf: Uint8Array[];
+  _buf: (rendezvous.RendezvousMessage | message.Message)[];
  _status: any;
  _latency: number;
  _secretKey: [Uint8Array, number, number] | undefined;
  _uri: string;
+  _isRendezvous: boolean;
+  _videoCallback: ((v: message.VideoFrame) => void) | undefined;

-  constructor(uri: string) {
+  constructor(
+    uri: string,
+    isRendezvous: boolean = true,
+    fn: ((v: message.VideoFrame) => void) | undefined = undefined
+  ) {
    this._eventHandlers = {
      message: (_: any) => {},
      open: () => {},
@@ -27,6 +33,8 @@ export default class Websock {
    this._websocket.onmessage = this._recv_message.bind(this);
    this._websocket.binaryType = "arraybuffer";
    this._latency = new Date().getTime();
+    this._isRendezvous = isRendezvous;
+    this._videoCallback = fn;
  }

  latency(): number {
@@ -98,7 +106,7 @@ export default class Websock {
      this._websocket.onclose = (e) => {
        if (this._status == "open") {
          // e.code 1000 means that the connection was closed normally.
-          reject('Reset by the peer');
+          reject("Reset by the peer");
        }
        this._status = e;
        console.error("WebSock.onclose: " + e);
@@ -112,14 +120,16 @@ export default class Websock {
        this._status = e;
        console.error("WebSock.onerror: " + e);
        this._eventHandlers.error(e);
-        reject(e['data']);
+        reject(e["data"]);
      };
    });
  }

-  async next(timeout = 12000): Promise<Uint8Array> {
+  async next(
+    timeout = 12000
+  ): Promise<rendezvous.RendezvousMessage | message.Message> {
    const func = (
-      resolve: (value: Uint8Array) => void,
+      resolve: (value: rendezvous.RendezvousMessage | message.Message) => void,
      reject: (reason: any) => void,
      tm0: number
    ) => {
@@ -166,7 +176,16 @@ export default class Websock {
        k[2] += 1;
        bytes = globals.decrypt(bytes, k[2], k[0]);
      }
-      this._buf.push(bytes);
+      if (this._isRendezvous) {
+        this._buf.push(this.parseRendezvous(bytes));
+      } else {
+        const m = this.parseMessage(bytes);
+        if (this._videoCallback && m.video_frame) {
+          this._videoCallback(m.video_frame);
+        } else {
+          this._buf.push(m);
+        }
+      }
    }
    this._eventHandlers.message(e.data);
  }