More work on SRS backend for radio playback and recording

frontend/react/src/audio/audiomanager.ts

@@ -2,16 +2,28 @@ import { AudioRadioSetting } from "../interfaces";
 import { getApp } from "../olympusapp";
 import { Buffer } from "buffer";
 import { MicrophoneHandler } from "./microphonehandler";
+import { PlaybackPipeline } from "./playbackpipeline";
 
 enum MessageType {
   audio,
   settings,
 }
 
+function fromBytes(array) {
+  let res = 0;
+  for (let i = 0; i < array.length; i++) {
+    res = res << 8;
+    res += array[array.length - i - 1];
+  }
+  return res;
+}
+
+var context = new AudioContext();
+
 export class AudioManager {
   #radioSettings: AudioRadioSetting[] = [
     {
-      frequency: 124000000,
+      frequency: 251000000,
       modulation: 0,
       ptt: false,
       tuned: false,
@@ -19,7 +31,7 @@ export class AudioManager {
     },
   ];
 
-  #microphoneHandlers: (MicrophoneHandler | null)[] =[];
+  #microphoneHandlers: (MicrophoneHandler | null)[] = [];
 
   #address: string = "localhost";
   #port: number = 4000;
@@ -38,6 +50,7 @@ export class AudioManager {
   }
 
   start() {
+    const pipeline = new PlaybackPipeline();
     let res = this.#address.match(/(?:http|https):\/\/(.+):/);
     let wsAddress = res ? res[1] : this.#address;
 
@@ -51,8 +64,13 @@ export class AudioManager {
       console.log(event);
     });
 
-    this.#socket.addEventListener("message", (event) => {
-      console.log("Message from server ", event.data);
+    this.#socket.addEventListener("message", async (event) => {
+      let bytes = event.data;
+      let packet = new Uint8Array(await bytes.arrayBuffer())
+      let audioLength = fromBytes(packet.slice(2, 4));
+      let audioData = packet.slice(6, 6 + audioLength);
+      let frequency = new DataView(packet.slice(6 + audioLength, 6 + audioLength  + 8).reverse().buffer).getFloat64(0);
+      pipeline.play(audioData.buffer);
     });
   }
 
@@ -80,9 +98,8 @@ export class AudioManager {
       if (setting.ptt && !this.#microphoneHandlers[idx]) {
         this.#microphoneHandlers[idx] = new MicrophoneHandler(this.#socket, setting);
       }
-    })
+    });
 
-    if (this.#socket?.readyState == 1)
-      this.#socket?.send(new Uint8Array([MessageType.settings, ...Buffer.from(JSON.stringify(message), "utf-8")]));
+    if (this.#socket?.readyState == 1) this.#socket?.send(new Uint8Array([MessageType.settings, ...Buffer.from(JSON.stringify(message), "utf-8")]));
   }
 }

frontend/react/src/audio/audiopacket.ts

@@ -60,7 +60,6 @@ export class AudioPacket {
     packet[4] = frequencyAudioLen[0];
     packet[5] = frequencyAudioLen[1];
 
-
     this.#packet = new Uint8Array([0].concat(packet));
   }
 

frontend/react/src/audio/capturepipeline.ts

@@ -0,0 +1,88 @@
+export class CapturePipeline {
+  sampleRate: any;
+  codec: any;
+  sourceId: any;
+  onrawdata: any;
+  onencoded: any;
+  deviceId: any;
+  audioContext: any;
+  mic: any;
+  source: any;
+  destination: any;
+  encoder: any;
+  audioTrackProcessor: any;
+  duration: any;
+
+  constructor(codec = "opus", sampleRate = 16000, duration = 40000) {
+    this.sampleRate = sampleRate;
+    this.codec = codec;
+    this.duration = duration;
+    this.onrawdata = null;
+    this.onencoded = null;
+  }
+  async connect() {
+    const mic = navigator.mediaDevices.getUserMedia({ audio: true });
+
+    this.audioContext = new AudioContext({
+      sampleRate: this.sampleRate,
+      latencyHint: "interactive",
+    });
+    this.mic = await mic;
+    this.source = this.audioContext.createMediaStreamSource(this.mic);
+    this.destination = this.audioContext.createMediaStreamDestination();
+    this.destination.channelCount = 1;
+    this.source.connect(this.destination);
+
+    this.encoder = new AudioEncoder({
+      output: this.handleEncodedData.bind(this),
+      error: this.handleEncodingError.bind(this),
+    });
+
+    this.encoder.configure({
+      codec: this.codec,
+      numberOfChannels: 1,
+      sampleRate: this.sampleRate,
+      opus: {
+        frameDuration: this.duration,
+      },
+      bitrateMode: "constant"
+    });
+
+    //@ts-ignore
+    this.audioTrackProcessor = new MediaStreamTrackProcessor({
+      track: this.destination.stream.getAudioTracks()[0],
+    });
+    this.audioTrackProcessor.readable.pipeTo(
+      new WritableStream({
+        write: this.handleRawData.bind(this),
+      })
+    );
+  }
+  disconnect() {
+    this.source.disconnect();
+    delete this.audioTrackProcessor;
+    delete this.encoder;
+    delete this.destination;
+    delete this.mic;
+    delete this.source;
+  }
+
+  handleEncodedData(chunk, metadata) {
+    if (this.onencoded) {
+      this.onencoded(chunk, metadata);
+    }
+    const data = new ArrayBuffer(chunk.byteLength);
+    chunk.copyTo(data);
+  }
+  handleEncodingError(e) {
+    console.log(e);
+  }
+
+  handleRawData(audioData) {
+    if (this.onrawdata) {
+      this.onrawdata(audioData);
+    }
+    this.encoder.encode(audioData);
+    audioData.close();
+  }
+}

frontend/react/src/audio/microphonehandler.ts

@@ -1,5 +1,6 @@
 import { AudioRadioSetting } from "../interfaces";
 import { AudioPacket } from "./audiopacket";
+import { CapturePipeline } from "./capturepipeline";
 
 export class MicrophoneHandler {
   #socket: WebSocket;
@@ -11,42 +12,23 @@ export class MicrophoneHandler {
 
     console.log("Starting microphone handler");
 
-    //@ts-ignore
-    let getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia || navigator.msGetUserMedia;
+    const pipeline = new CapturePipeline();
 
-    if (getUserMedia) {
-      //@ts-ignore
-      navigator.getUserMedia(
-        { audio: {
-          sampleRate: 16000,
-          channelCount: 1,
-          volume: 1.0
-        } },
-        (stream) => {
-          this.start_microphone(stream);
-        },
-         (e) => {
-          alert("Error capturing audio.");
-        }
-      );
-    } else {
-      alert("getUserMedia not supported in this browser.");
-    }
-  }
+    navigator.mediaDevices.enumerateDevices()
+    .then(function(devices) {
+      devices.forEach(function(device) {
+        console.log(device.kind + ": " + device.label +
+                " id = " + device.deviceId);
+      });
+    })
 
-  start_microphone(stream) {
-    const recorder = new MediaRecorder(stream);
-
-    // fires every one second and passes an BlobEvent
-    recorder.ondataavailable = async (event) => {
-      // get the Blob from the event
-      const blob = event.data;
-
-      let rawData = await blob.arrayBuffer();
-      let packet = new AudioPacket(new Uint8Array(rawData), this.#setting);
-      this.#socket.send(packet.getArray());
-    };
-
-    recorder.start(200);
+    pipeline.connect().then(() => {
+      pipeline.onencoded = (data) => {
+        let buffer = new ArrayBuffer(data.byteLength);
+        data.copyTo(buffer);
+        let packet = new AudioPacket(new Uint8Array(buffer), this.#setting);
+        this.#socket.send(packet.getArray());
+      }
+    })
   }
 }

frontend/react/src/audio/playbackpipeline.ts

@@ -0,0 +1,82 @@
+export class PlaybackPipeline {
+  sampleRate: any;
+  codec: any;
+  sourceId: any;
+  onrawdata: any;
+  ondecoded: any;
+  deviceId: any;
+  audioContext: any;
+  mic: any;
+  source: any;
+  destination: any;
+  decoder: any;
+  audioTrackProcessor: any;
+  duration: any;
+  trackGenerator: any;
+  writer: any;
+
+  constructor(codec = "opus", sampleRate = 16000, duration = 40000) {
+    this.sampleRate = sampleRate;
+    this.codec = codec;
+    this.duration = duration;
+    this.ondecoded = null;
+    this.audioContext = new AudioContext();
+
+    this.decoder = new AudioDecoder({
+      output: (chunk) => this.handleDecodedData(chunk),
+      error: this.handleDecodingError.bind(this),
+    });
+
+    this.decoder.configure({
+      codec: this.codec,
+      numberOfChannels: 1,
+      sampleRate: this.sampleRate,
+      opus: {
+        frameDuration: this.duration,
+      },
+      bitrateMode: "constant",
+    });
+
+    //@ts-ignore
+    this.trackGenerator = new MediaStreamTrackGenerator({ kind: "audio" });
+    this.writer = this.trackGenerator.writable.getWriter();
+
+    const stream = new MediaStream([this.trackGenerator]);
+
+    const mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
+    mediaStreamSource.connect(this.audioContext.destination)
+  }
+
+  play(buffer) {
+    const init = {
+      type: "key",
+      data: buffer,
+      timestamp: 23000000,
+      duration: 2000000,
+      transfer: [buffer],
+    };
+    //@ts-ignore
+    let chunk = new EncodedAudioChunk(init);
+
+    this.decoder.decode(chunk);
+  }
+
+  disconnect() {
+    this.source.disconnect();
+    delete this.audioTrackProcessor;
+    delete this.decoder;
+    delete this.destination;
+    delete this.mic;
+    delete this.source;
+  }
+
+  handleDecodedData(chunk) {
+    this.writer.ready.then(() => {
+        this.writer.write(chunk);
+    })
+  }
+  handleDecodingError(e) {
+    console.log(e);
+  }
+
+}