From b352bc824c445e58d0e067767342c5c7b5738b53 Mon Sep 17 00:00:00 2001 From: Pax1601 Date: Mon, 2 Sep 2024 07:51:51 +0200 Subject: [PATCH] More work on SRS backend for radio playback and recording --- frontend/react/package.json | 2 + frontend/react/src/audio/audiomanager.ts | 31 +++++-- frontend/react/src/audio/audiopacket.ts | 1 - frontend/react/src/audio/capturepipeline.ts | 88 +++++++++++++++++++ frontend/react/src/audio/microphonehandler.ts | 52 ++++------- frontend/react/src/audio/playbackpipeline.ts | 82 +++++++++++++++++ frontend/react/src/ui/panels/radiomenu.tsx | 2 +- .../react/src/ui/panels/unitcontrolmenu.tsx | 2 +- frontend/server/package.json | 2 +- frontend/server/src/audio/defaultdata.ts | 2 +- frontend/server/src/audio/srshandler.ts | 35 ++++++-- frontend/server/tsconfig.json | 4 +- 12 files changed, 247 insertions(+), 56 deletions(-) create mode 100644 frontend/react/src/audio/capturepipeline.ts create mode 100644 frontend/react/src/audio/playbackpipeline.ts diff --git a/frontend/react/package.json b/frontend/react/package.json index a77bbafd..0f73e905 100644 --- a/frontend/react/package.json +++ b/frontend/react/package.json @@ -16,6 +16,7 @@ "@fortawesome/react-fontawesome": "^0.2.0", "@tanem/svg-injector": "^10.1.68", "@turf/turf": "^6.5.0", + "@types/dom-webcodecs": "^0.1.11", "@types/leaflet": "^1.9.8", "@types/react-leaflet": "^3.0.0", "@types/turf": "^3.5.32", @@ -24,6 +25,7 @@ "leaflet": "^1.9.4", "leaflet-control-mini-map": "^0.4.0", "leaflet-path-drag": "^1.9.5", + "opus-decoder": "^0.7.6", "react": "^18.2.0", "react-dom": "^18.2.0", "react-icons": "^5.0.1", diff --git a/frontend/react/src/audio/audiomanager.ts b/frontend/react/src/audio/audiomanager.ts index dc177c6e..5510e229 100644 --- a/frontend/react/src/audio/audiomanager.ts +++ b/frontend/react/src/audio/audiomanager.ts @@ -2,16 +2,28 @@ import { AudioRadioSetting } from "../interfaces"; import { getApp } from "../olympusapp"; import { Buffer } from "buffer"; import { MicrophoneHandler } from "./microphonehandler"; +import { PlaybackPipeline } from "./playbackpipeline"; enum MessageType { audio, settings, } +function fromBytes(array) { + let res = 0; + for (let i = 0; i < array.length; i++) { + res = res << 8; + res += array[array.length - i - 1]; + } + return res; +} + +var context = new AudioContext(); + export class AudioManager { #radioSettings: AudioRadioSetting[] = [ { - frequency: 124000000, + frequency: 251000000, modulation: 0, ptt: false, tuned: false, @@ -19,7 +31,7 @@ export class AudioManager { }, ]; - #microphoneHandlers: (MicrophoneHandler | null)[] =[]; + #microphoneHandlers: (MicrophoneHandler | null)[] = []; #address: string = "localhost"; #port: number = 4000; @@ -38,6 +50,7 @@ export class AudioManager { } start() { + const pipeline = new PlaybackPipeline(); let res = this.#address.match(/(?:http|https):\/\/(.+):/); let wsAddress = res ? res[1] : this.#address; @@ -51,8 +64,13 @@ export class AudioManager { console.log(event); }); - this.#socket.addEventListener("message", (event) => { - console.log("Message from server ", event.data); + this.#socket.addEventListener("message", async (event) => { + let bytes = event.data; + let packet = new Uint8Array(await bytes.arrayBuffer()) + let audioLength = fromBytes(packet.slice(2, 4)); + let audioData = packet.slice(6, 6 + audioLength); + let frequency = new DataView(packet.slice(6 + audioLength, 6 + audioLength + 8).reverse().buffer).getFloat64(0); + pipeline.play(audioData.buffer); }); } @@ -80,9 +98,8 @@ export class AudioManager { if (setting.ptt && !this.#microphoneHandlers[idx]) { this.#microphoneHandlers[idx] = new MicrophoneHandler(this.#socket, setting); } - }) + }); - if (this.#socket?.readyState == 1) - this.#socket?.send(new Uint8Array([MessageType.settings, ...Buffer.from(JSON.stringify(message), "utf-8")])); + if (this.#socket?.readyState == 1) this.#socket?.send(new Uint8Array([MessageType.settings, ...Buffer.from(JSON.stringify(message), "utf-8")])); } } diff --git a/frontend/react/src/audio/audiopacket.ts b/frontend/react/src/audio/audiopacket.ts index f7700ee2..82afb48e 100644 --- a/frontend/react/src/audio/audiopacket.ts +++ b/frontend/react/src/audio/audiopacket.ts @@ -60,7 +60,6 @@ export class AudioPacket { packet[4] = frequencyAudioLen[0]; packet[5] = frequencyAudioLen[1]; - this.#packet = new Uint8Array([0].concat(packet)); } diff --git a/frontend/react/src/audio/capturepipeline.ts b/frontend/react/src/audio/capturepipeline.ts new file mode 100644 index 00000000..482f4894 --- /dev/null +++ b/frontend/react/src/audio/capturepipeline.ts @@ -0,0 +1,88 @@ +export class CapturePipeline { + sampleRate: any; + codec: any; + sourceId: any; + onrawdata: any; + onencoded: any; + deviceId: any; + audioContext: any; + mic: any; + source: any; + destination: any; + encoder: any; + audioTrackProcessor: any; + duration: any; + + constructor(codec = "opus", sampleRate = 16000, duration = 40000) { + this.sampleRate = sampleRate; + this.codec = codec; + this.duration = duration; + this.onrawdata = null; + this.onencoded = null; + } + async connect() { + const mic = navigator.mediaDevices.getUserMedia({ audio: true }); + + this.audioContext = new AudioContext({ + sampleRate: this.sampleRate, + latencyHint: "interactive", + }); + this.mic = await mic; + this.source = this.audioContext.createMediaStreamSource(this.mic); + this.destination = this.audioContext.createMediaStreamDestination(); + this.destination.channelCount = 1; + this.source.connect(this.destination); + + this.encoder = new AudioEncoder({ + output: this.handleEncodedData.bind(this), + error: this.handleEncodingError.bind(this), + }); + + this.encoder.configure({ + codec: this.codec, + numberOfChannels: 1, + sampleRate: this.sampleRate, + opus: { + frameDuration: this.duration, + }, + bitrateMode: "constant" + }); + + //@ts-ignore + this.audioTrackProcessor = new MediaStreamTrackProcessor({ + track: this.destination.stream.getAudioTracks()[0], + }); + this.audioTrackProcessor.readable.pipeTo( + new WritableStream({ + write: this.handleRawData.bind(this), + }) + ); + } + disconnect() { + this.source.disconnect(); + delete this.audioTrackProcessor; + delete this.encoder; + delete this.destination; + delete this.mic; + delete this.source; + } + + handleEncodedData(chunk, metadata) { + if (this.onencoded) { + this.onencoded(chunk, metadata); + } + const data = new ArrayBuffer(chunk.byteLength); + chunk.copyTo(data); + } + handleEncodingError(e) { + console.log(e); + } + + handleRawData(audioData) { + if (this.onrawdata) { + this.onrawdata(audioData); + } + this.encoder.encode(audioData); + audioData.close(); + } +} diff --git a/frontend/react/src/audio/microphonehandler.ts b/frontend/react/src/audio/microphonehandler.ts index c099e3ad..e421bca9 100644 --- a/frontend/react/src/audio/microphonehandler.ts +++ b/frontend/react/src/audio/microphonehandler.ts @@ -1,5 +1,6 @@ import { AudioRadioSetting } from "../interfaces"; import { AudioPacket } from "./audiopacket"; +import { CapturePipeline } from "./capturepipeline"; export class MicrophoneHandler { #socket: WebSocket; @@ -11,42 +12,23 @@ export class MicrophoneHandler { console.log("Starting microphone handler"); - //@ts-ignore - let getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia || navigator.msGetUserMedia; + const pipeline = new CapturePipeline(); - if (getUserMedia) { - //@ts-ignore - navigator.getUserMedia( - { audio: { - sampleRate: 16000, - channelCount: 1, - volume: 1.0 - } }, - (stream) => { - this.start_microphone(stream); - }, - (e) => { - alert("Error capturing audio."); - } - ); - } else { - alert("getUserMedia not supported in this browser."); - } - } + navigator.mediaDevices.enumerateDevices() + .then(function(devices) { + devices.forEach(function(device) { + console.log(device.kind + ": " + device.label + + " id = " + device.deviceId); + }); + }) - start_microphone(stream) { - const recorder = new MediaRecorder(stream); - - // fires every one second and passes an BlobEvent - recorder.ondataavailable = async (event) => { - // get the Blob from the event - const blob = event.data; - - let rawData = await blob.arrayBuffer(); - let packet = new AudioPacket(new Uint8Array(rawData), this.#setting); - this.#socket.send(packet.getArray()); - }; - - recorder.start(200); + pipeline.connect().then(() => { + pipeline.onencoded = (data) => { + let buffer = new ArrayBuffer(data.byteLength); + data.copyTo(buffer); + let packet = new AudioPacket(new Uint8Array(buffer), this.#setting); + this.#socket.send(packet.getArray()); + } + }) } } diff --git a/frontend/react/src/audio/playbackpipeline.ts b/frontend/react/src/audio/playbackpipeline.ts new file mode 100644 index 00000000..b8540f0c --- /dev/null +++ b/frontend/react/src/audio/playbackpipeline.ts @@ -0,0 +1,82 @@ +export class PlaybackPipeline { + sampleRate: any; + codec: any; + sourceId: any; + onrawdata: any; + ondecoded: any; + deviceId: any; + audioContext: any; + mic: any; + source: any; + destination: any; + decoder: any; + audioTrackProcessor: any; + duration: any; + trackGenerator: any; + writer: any; + + constructor(codec = "opus", sampleRate = 16000, duration = 40000) { + this.sampleRate = sampleRate; + this.codec = codec; + this.duration = duration; + this.ondecoded = null; + this.audioContext = new AudioContext(); + + this.decoder = new AudioDecoder({ + output: (chunk) => this.handleDecodedData(chunk), + error: this.handleDecodingError.bind(this), + }); + + this.decoder.configure({ + codec: this.codec, + numberOfChannels: 1, + sampleRate: this.sampleRate, + opus: { + frameDuration: this.duration, + }, + bitrateMode: "constant", + }); + + //@ts-ignore + this.trackGenerator = new MediaStreamTrackGenerator({ kind: "audio" }); + this.writer = this.trackGenerator.writable.getWriter(); + + const stream = new MediaStream([this.trackGenerator]); + + const mediaStreamSource = this.audioContext.createMediaStreamSource(stream); + mediaStreamSource.connect(this.audioContext.destination) + } + + play(buffer) { + const init = { + type: "key", + data: buffer, + timestamp: 23000000, + duration: 2000000, + transfer: [buffer], + }; + //@ts-ignore + let chunk = new EncodedAudioChunk(init); + + this.decoder.decode(chunk); + } + + disconnect() { + this.source.disconnect(); + delete this.audioTrackProcessor; + delete this.decoder; + delete this.destination; + delete this.mic; + delete this.source; + } + + handleDecodedData(chunk) { + this.writer.ready.then(() => { + this.writer.write(chunk); + }) + } + handleDecodingError(e) { + console.log(e); + } + +} diff --git a/frontend/react/src/ui/panels/radiomenu.tsx b/frontend/react/src/ui/panels/radiomenu.tsx index a4f0b0a0..d0b759f8 100644 --- a/frontend/react/src/ui/panels/radiomenu.tsx +++ b/frontend/react/src/ui/panels/radiomenu.tsx @@ -12,7 +12,7 @@ import { OlLabelToggle } from "../components/ollabeltoggle"; import { FaVolumeHigh } from "react-icons/fa6"; export function RadioMenu(props: { open: boolean; onClose: () => void; children?: JSX.Element | JSX.Element[] }) { - const [frequency1, setFrequency1] = useState(124000000); + const [frequency1, setFrequency1] = useState(251000000); const [ptt1, setPTT1] = useState(false); const [frequency2, setFrequency2] = useState(251000000); const [frequency3, setFrequency3] = useState(243000000); diff --git a/frontend/react/src/ui/panels/unitcontrolmenu.tsx b/frontend/react/src/ui/panels/unitcontrolmenu.tsx index 07922762..0b3bc280 100644 --- a/frontend/react/src/ui/panels/unitcontrolmenu.tsx +++ b/frontend/react/src/ui/panels/unitcontrolmenu.tsx @@ -1113,7 +1113,7 @@ export function UnitControlMenu(props: { open: boolean; onClose: () => void }) {
Radio frequency
- { + { if (activeAdvancedSettings) { activeAdvancedSettings.radio.frequency = value; setActiveAdvancedSettings(JSON.parse(JSON.stringify(activeAdvancedSettings))); diff --git a/frontend/server/package.json b/frontend/server/package.json index 5673d6cb..163fb81d 100644 --- a/frontend/server/package.json +++ b/frontend/server/package.json @@ -22,8 +22,8 @@ "express-basic-auth": "^1.2.1", "http-proxy-middleware": "^2.0.6", "morgan": "~1.9.1", - "node-opus": "^0.3.3", "open": "^10.0.0", + "opus-decoder": "^0.7.6", "regedit": "^5.1.2", "save": "^2.9.0", "sha256": "^0.2.0", diff --git a/frontend/server/src/audio/defaultdata.ts b/frontend/server/src/audio/defaultdata.ts index 962f6c49..d97cbaad 100644 --- a/frontend/server/src/audio/defaultdata.ts +++ b/frontend/server/src/audio/defaultdata.ts @@ -2,7 +2,7 @@ export var defaultSRSData = { ClientGuid: "", Name: "", Seat: 0, - Coalition: 0, + Coalition: 2, AllowRecord: false, RadioInfo: { radios: [ diff --git a/frontend/server/src/audio/srshandler.ts b/frontend/server/src/audio/srshandler.ts index a813408e..831b10d2 100644 --- a/frontend/server/src/audio/srshandler.ts +++ b/frontend/server/src/audio/srshandler.ts @@ -1,4 +1,7 @@ import { defaultSRSData } from "./defaultdata"; +const { OpusEncoder } = require("@discordjs/opus"); +const encoder = new OpusEncoder(16000, 1); + var net = require("net"); const SRS_VERSION = "2.1.0.10"; @@ -41,12 +44,10 @@ export class SRSHandler { this.ws.on("message", (data) => { switch (data[0]) { case MessageType.audio: - this.udp.send(data.slice(1), 5002, "localhost", function (error) { - if (error) { - console.log("Error!!!"); - } else { - console.log("Data sent"); - } + let audioBuffer = data.slice(1); + this.udp.send(audioBuffer, SRSPort, "localhost", (error) => { + if (error) + console.log(`Error sending data to SRS server: ${error}`); }); break; case MessageType.settings: @@ -54,7 +55,7 @@ export class SRSHandler { message.settings.forEach((setting, idx) => { this.data.RadioInfo.radios[idx].freq = setting.frequency; this.data.RadioInfo.radios[idx].modulation = setting.modulation; - }) + }); break; default: break; @@ -65,8 +66,12 @@ export class SRSHandler { }); /* TCP */ + this.tcp.on("error", (ex) => { + console.log("Could not connect to SRS Server"); + }); + this.tcp.connect(SRSPort, "localhost", () => { - console.log("Connected"); + console.log(`Connected to SRS Server on TCP Port ${SRSPort}`); this.syncInterval = setInterval(() => { let SYNC = { @@ -75,10 +80,24 @@ export class SRSHandler { Version: SRS_VERSION, }; + this.udp.send(this.data.ClientGuid, SRSPort, "localhost", (error) => { + if (error) console.log(`Error pinging SRS server on UDP: ${error}`); + }); + if (this.tcp.readyState == "open") this.tcp.write(`${JSON.stringify(SYNC)}\n`); else clearInterval(this.syncInterval); }, 1000); }); + + /* UDP */ + this.udp.on("listening", () => { + console.log(`Listening to SRS Server on UDP port ${SRSPort}`) + }); + + this.udp.on("message", (message, remote) => { + if (this.ws && message.length > 22) + this.ws.send(message); + }); } } diff --git a/frontend/server/tsconfig.json b/frontend/server/tsconfig.json index 6168f73c..24dfd8c7 100644 --- a/frontend/server/tsconfig.json +++ b/frontend/server/tsconfig.json @@ -2,7 +2,9 @@ "compilerOptions": { "outDir": "./build", "allowJs": true, - "target": "es5" + "target": "es5", + "module": "Node16", + "moduleResolution": "Node16" }, "include": [ "./src/**/*"