From 9bbcdac70415c5bc123214d53f2205e733bb7ab0 Mon Sep 17 00:00:00 2001 From: Pax1601 Date: Wed, 4 Sep 2024 19:41:05 +0200 Subject: [PATCH] More work on audio --- frontend/react/src/audio/audiomanager.ts | 95 ++++++++++------ frontend/react/src/audio/audiopacket.ts | 2 +- frontend/react/src/audio/audiosink.ts | 71 +++++++++++- frontend/react/src/audio/audiosource.ts | 33 ++++-- .../{audiofilesource.ts => filesource.ts} | 9 +- frontend/react/src/audio/microphonesource.ts | 6 +- frontend/react/src/audio/radiosink.ts | 77 +++++++++++++ frontend/react/src/audio/srsradio.ts | 83 -------------- frontend/react/src/audio/unitsink.ts | 34 ++++++ frontend/react/src/dom.d.ts | 2 +- frontend/react/src/eventscontext.tsx | 2 + frontend/react/src/interfaces.ts | 13 +-- frontend/react/src/statecontext.tsx | 1 + frontend/react/src/ui/panels/audiomenu.tsx | 75 ++++++++++++ .../ui/panels/components/audiosourcepanel.tsx | 79 ++++++++----- .../src/ui/panels/components/radiopanel.tsx | 33 ++---- frontend/react/src/ui/panels/header.tsx | 28 +++-- frontend/react/src/ui/panels/radiomenu.tsx | 63 ++--------- frontend/react/src/ui/panels/sidebar.tsx | 9 +- frontend/react/src/ui/ui.tsx | 10 ++ frontend/react/src/unit/unit.ts | 14 +++ frontend/server/src/audio/srshandler.ts | 107 ++++++++++++++---- 22 files changed, 555 insertions(+), 291 deletions(-) rename frontend/react/src/audio/{audiofilesource.ts => filesource.ts} (90%) create mode 100644 frontend/react/src/audio/radiosink.ts delete mode 100644 frontend/react/src/audio/srsradio.ts create mode 100644 frontend/react/src/audio/unitsink.ts create mode 100644 frontend/react/src/ui/panels/audiomenu.tsx diff --git a/frontend/react/src/audio/audiomanager.ts b/frontend/react/src/audio/audiomanager.ts index 18f151c8..95ef049e 100644 --- a/frontend/react/src/audio/audiomanager.ts +++ b/frontend/react/src/audio/audiomanager.ts @@ -1,12 +1,15 @@ import { AudioMessageType } from "../constants/constants"; import { MicrophoneSource } from "./microphonesource"; -import { SRSRadio } from "./srsradio"; +import { RadioSink } from "./radiosink"; import { getApp } from "../olympusapp"; import { fromBytes, makeID } from "../other/utils"; -import { AudioFileSource } from "./audiofilesource"; +import { FileSource } from "./filesource"; import { AudioSource } from "./audiosource"; import { Buffer } from "buffer"; import { PlaybackPipeline } from "./playbackpipeline"; +import { AudioSink } from "./audiosink"; +import { Unit } from "../unit/unit"; +import { UnitSink } from "./unitsink"; export class AudioManager { #audioContext: AudioContext; @@ -14,8 +17,8 @@ export class AudioManager { /* The playback pipeline enables audio playback on the speakers/headphones */ #playbackPipeline: PlaybackPipeline; - /* The SRS radio audio sinks used to transmit the audio stream to the SRS backend */ - #radios: SRSRadio[] = []; + /* The audio sinks used to transmit the audio stream to the SRS backend */ + #sinks: AudioSink[] = []; /* List of all possible audio sources (microphone, file stream etc...) */ #sources: AudioSource[] = []; @@ -59,44 +62,49 @@ export class AudioManager { /* Handle the reception of a new message */ this.#socket.addEventListener("message", (event) => { - this.#radios.forEach(async (radio) => { - /* Extract the audio data as array */ - let packetUint8Array = new Uint8Array(await event.data.arrayBuffer()); + this.#sinks.forEach(async (sink) => { + if (sink instanceof RadioSink) { + /* Extract the audio data as array */ + let packetUint8Array = new Uint8Array(await event.data.arrayBuffer()); - /* Extract the encoded audio data */ - let audioLength = fromBytes(packetUint8Array.slice(2, 4)); - let audioUint8Array = packetUint8Array.slice(6, 6 + audioLength); + /* Extract the encoded audio data */ + let audioLength = fromBytes(packetUint8Array.slice(2, 4)); + let audioUint8Array = packetUint8Array.slice(6, 6 + audioLength); - /* Extract the frequency value and play it on the speakers if we are listening to it*/ - let frequency = new DataView(packetUint8Array.slice(6 + audioLength, 6 + audioLength + 8).reverse().buffer).getFloat64(0); - if (radio.getSetting().frequency === frequency) { - this.#playbackPipeline.play(audioUint8Array.buffer); + /* Extract the frequency value and play it on the speakers if we are listening to it*/ + let frequency = new DataView(packetUint8Array.slice(6 + audioLength, 6 + audioLength + 8).reverse().buffer).getFloat64(0); + if (sink.getFrequency() === frequency) { + this.#playbackPipeline.play(audioUint8Array.buffer); + } } }); }); - /* Add two default radios */ - this.#radios = [new SRSRadio(), new SRSRadio()]; - document.dispatchEvent(new CustomEvent("radiosUpdated")); - /* Add the microphone source and connect it directly to the radio */ const microphoneSource = new MicrophoneSource(); microphoneSource.initialize().then(() => { - this.#radios.forEach((radio) => { - microphoneSource.getNode().connect(radio.getNode()); + this.#sinks.forEach((sink) => { + if (sink instanceof RadioSink) + microphoneSource.connect(sink); }); this.#sources.push(microphoneSource); document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + + /* Add two default radios */ + this.addRadio(); + this.addRadio(); }); } stop() { this.#sources.forEach((source) => { - source.getNode().disconnect(); + source.disconnect(); }); this.#sources = []; + this.#sinks = []; - this.#radios = []; + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); } setAddress(address) { @@ -108,27 +116,38 @@ export class AudioManager { } addFileSource(file) { - const newSource = new AudioFileSource(file); + const newSource = new FileSource(file); this.#sources.push(newSource); - newSource.getNode().connect(this.#radios[0].getNode()); + newSource.connect(this.#sinks[0]); document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); } - getRadios() { - return this.#radios; + addUnitSink(unit: Unit) { + this.#sinks.push(new UnitSink(unit)); + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getSinks() { + return this.#sinks; } addRadio() { - const newRadio = new SRSRadio(); - this.#sources[0].getNode().connect(newRadio.getNode()); - this.#radios.push(newRadio); - document.dispatchEvent(new CustomEvent("radiosUpdated")); + const newRadio = new RadioSink(); + this.#sinks.push(newRadio); + newRadio.setName(`Radio ${this.#sinks.length}`); + this.#sources[0].connect(newRadio); + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); } - removeRadio(idx) { - this.#radios[idx].getNode().disconnect(); - this.#radios.splice(idx, 1); - document.dispatchEvent(new CustomEvent("radiosUpdated")); + removeSink(sink) { + sink.disconnect(); + this.#sinks = this.#sinks.filter((v) => v != sink); + let idx = 1; + this.#sinks.forEach((sink) => { + if (sink instanceof RadioSink) + sink.setName(`Radio ${idx++}`); + }); + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); } getSources() { @@ -152,8 +171,12 @@ export class AudioManager { type: "Settings update", guid: this.#guid, coalition: 2, - settings: this.#radios.map((radio) => { - return radio.getSetting(); + settings: this.#sinks.filter((sink) => sink instanceof RadioSink).map((radio) => { + return { + frequency: radio.getFrequency(), + modulation: radio.getModulation(), + ptt: radio.getPtt(), + }; }), }; diff --git a/frontend/react/src/audio/audiopacket.ts b/frontend/react/src/audio/audiopacket.ts index 1be4d7cd..d4688c1d 100644 --- a/frontend/react/src/audio/audiopacket.ts +++ b/frontend/react/src/audio/audiopacket.ts @@ -30,7 +30,7 @@ export class AudioPacket { let encModulation: number[] = [settings.modulation]; let encEncryption: number[] = [0]; - let encUnitID: number[] = getBytes(100000001, 4); + let encUnitID: number[] = getBytes(0, 4); let encPacketID: number[] = getBytes(packetID, 8); packetID++; let encHops: number[] = [0]; diff --git a/frontend/react/src/audio/audiosink.ts b/frontend/react/src/audio/audiosink.ts index 0ac06f8d..bbf46cb8 100644 --- a/frontend/react/src/audio/audiosink.ts +++ b/frontend/react/src/audio/audiosink.ts @@ -1,3 +1,70 @@ +import { getApp } from "../olympusapp"; + export abstract class AudioSink { - abstract getNode(): AudioNode; -} \ No newline at end of file + #encoder: AudioEncoder; + #name: string; + #node: MediaStreamAudioDestinationNode; + #audioTrackProcessor: any; // TODO can we have typings? + #gainNode: GainNode; + + constructor() { + /* A gain node is used because it allows to connect multiple inputs */ + this.#gainNode = getApp().getAudioManager().getAudioContext().createGain(); + this.#node = getApp().getAudioManager().getAudioContext().createMediaStreamDestination(); + this.#node.channelCount = 1; + + this.#encoder = new AudioEncoder({ + output: (data) => this.handleEncodedData(data), + error: (e) => { + console.log(e); + }, + }); + + this.#encoder.configure({ + codec: "opus", + numberOfChannels: 1, + sampleRate: 16000, + //@ts-ignore // TODO why is it giving error? + opus: { + frameDuration: 40000, + }, + bitrateMode: "constant", + }); + + //@ts-ignore + this.#audioTrackProcessor = new MediaStreamTrackProcessor({ + track: this.#node.stream.getAudioTracks()[0], + }); + this.#audioTrackProcessor.readable.pipeTo( + new WritableStream({ + write: (arrayBuffer) => this.#handleRawData(arrayBuffer), + }) + ); + + this.#gainNode.connect(this.#node); + } + + setName(name) { + this.#name = name; + } + + getName() { + return this.#name; + } + + disconnect() { + this.getNode().disconnect(); + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getNode() { + return this.#gainNode; + } + + #handleRawData(audioData) { + this.#encoder.encode(audioData); + audioData.close(); + } + + abstract handleEncodedData(encodedAudioChunk: EncodedAudioChunk): void; +} diff --git a/frontend/react/src/audio/audiosource.ts b/frontend/react/src/audio/audiosource.ts index 2688f674..a257389e 100644 --- a/frontend/react/src/audio/audiosource.ts +++ b/frontend/react/src/audio/audiosource.ts @@ -1,22 +1,33 @@ -import { AudioSourceSetting } from "../interfaces"; import { AudioSink } from "./audiosink"; export abstract class AudioSource { - #setting: AudioSourceSetting = { - connectedTo: "", - filename: "", - playing: true, - }; + #connectedTo: AudioSink[] = []; + #name = ""; + #playing = false; - getSetting() { - return this.#setting; + connect(sink: AudioSink) { + this.getNode().connect(sink.getNode()); + this.#connectedTo.push(sink); + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); } - setSetting(setting: AudioSourceSetting) { - this.#setting = setting; + disconnect() { + this.getNode().disconnect(); + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + } + + setName(name) { + this.#name = name; + } + + getName() { + return this.#name; + } + + getConnectedTo() { + return this.#connectedTo; } abstract play(): void; abstract getNode(): AudioNode; - abstract getName(): string; } diff --git a/frontend/react/src/audio/audiofilesource.ts b/frontend/react/src/audio/filesource.ts similarity index 90% rename from frontend/react/src/audio/audiofilesource.ts rename to frontend/react/src/audio/filesource.ts index d3d47de7..38178798 100644 --- a/frontend/react/src/audio/audiofilesource.ts +++ b/frontend/react/src/audio/filesource.ts @@ -2,7 +2,7 @@ import { AudioSource } from "./audiosource"; import { bufferToF32Planar } from "../other/utils"; import { getApp } from "../olympusapp"; -export class AudioFileSource extends AudioSource { +export class FileSource extends AudioSource { #gainNode: GainNode; #file: File | null = null; #source: AudioBufferSourceNode; @@ -11,6 +11,8 @@ export class AudioFileSource extends AudioSource { super(); this.#file = file; + this.setName(this.#file?.name ?? "N/A"); + this.#gainNode = getApp().getAudioManager().getAudioContext().createGain(); } @@ -30,6 +32,7 @@ export class AudioFileSource extends AudioSource { this.#source = getApp().getAudioManager().getAudioContext().createBufferSource(); this.#source.buffer = arrayBuffer; this.#source.connect(this.#gainNode); + this.#source.loop = true; this.#source.start(); }); } @@ -44,8 +47,4 @@ export class AudioFileSource extends AudioSource { setGain(gain) { this.#gainNode.gain.setValueAtTime(gain, getApp().getAudioManager().getAudioContext().currentTime); } - - getName() { - return this.#file?.name ?? "N/A"; - } } diff --git a/frontend/react/src/audio/microphonesource.ts b/frontend/react/src/audio/microphonesource.ts index 92cec7b6..209ffcd2 100644 --- a/frontend/react/src/audio/microphonesource.ts +++ b/frontend/react/src/audio/microphonesource.ts @@ -6,6 +6,8 @@ export class MicrophoneSource extends AudioSource { constructor() { super(); + + this.setName("Microphone"); } async initialize() { @@ -22,8 +24,4 @@ export class MicrophoneSource extends AudioSource { play() { // TODO, now is always on } - - getName() { - return "Microphone" - } } diff --git a/frontend/react/src/audio/radiosink.ts b/frontend/react/src/audio/radiosink.ts new file mode 100644 index 00000000..aa8cb9e1 --- /dev/null +++ b/frontend/react/src/audio/radiosink.ts @@ -0,0 +1,77 @@ +import { AudioSink } from "./audiosink"; +import { AudioPacket } from "./audiopacket"; +import { getApp } from "../olympusapp"; + +export class RadioSink extends AudioSink { + #frequency = 251000000; + #modulation = 0; + #ptt = false; + #tuned = false; + #volume = 0.5; + + constructor() { + super(); + } + + setFrequency(frequency) { + this.#frequency = frequency; + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getFrequency() { + return this.#frequency; + } + + setModulation(modulation) { + this.#modulation = modulation; + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getModulation() { + return this.#modulation; + } + + setPtt(ptt) { + this.#ptt = ptt; + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getPtt() { + return this.#ptt; + } + + setTuned(tuned) { + this.#tuned = tuned; + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getTuned() { + return this.#tuned; + } + + setVolume(volume) { + this.#volume = volume; + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getVolume() { + return this.#volume; + } + + handleEncodedData(encodedAudioChunk: EncodedAudioChunk) { + let arrayBuffer = new ArrayBuffer(encodedAudioChunk.byteLength); + encodedAudioChunk.copyTo(arrayBuffer); + + if (this.#ptt) { + let packet = new AudioPacket( + new Uint8Array(arrayBuffer), + { + frequency: this.#frequency, + modulation: this.#modulation, + }, + getApp().getAudioManager().getGuid() + ); + getApp().getAudioManager().send(packet.getArray()); + } + } +} diff --git a/frontend/react/src/audio/srsradio.ts b/frontend/react/src/audio/srsradio.ts deleted file mode 100644 index 75196af9..00000000 --- a/frontend/react/src/audio/srsradio.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { AudioSink } from "./audiosink"; -import { AudioPacket } from "./audiopacket"; -import { getApp } from "../olympusapp"; - -export class SRSRadio extends AudioSink { - #encoder: AudioEncoder; - #node: MediaStreamAudioDestinationNode; - #audioTrackProcessor: any; // TODO can we have typings? - #gainNode: GainNode; - - #setting = { - frequency: 251000000, - modulation: 0, - ptt: false, - tuned: false, - volume: 0.5, - }; - - constructor() { - super(); - - /* A gain node is used because it allows to connect multiple inputs */ - this.#gainNode = getApp().getAudioManager().getAudioContext().createGain(); - this.#node = getApp().getAudioManager().getAudioContext().createMediaStreamDestination(); - this.#node.channelCount = 1; - - this.#encoder = new AudioEncoder({ - output: (data) => this.#handleEncodedData(data), - error: (e) => {console.log(e);}, - }); - - this.#encoder.configure({ - codec: 'opus', - numberOfChannels: 1, - sampleRate: 16000, - //@ts-ignore // TODO why is it giving error? - opus: { - frameDuration: 40000, - }, - bitrateMode: "constant" - }); - - //@ts-ignore - this.#audioTrackProcessor = new MediaStreamTrackProcessor({ - track: this.#node.stream.getAudioTracks()[0], - }); - this.#audioTrackProcessor.readable.pipeTo( - new WritableStream({ - write: (arrayBuffer) => this.#handleRawData(arrayBuffer), - }) - ); - - this.#gainNode.connect(this.#node); - } - - getSetting() { - return this.#setting; - } - - setSetting(setting) { - this.#setting = setting; - document.dispatchEvent(new CustomEvent("radiosUpdated")); - } - - getNode() { - return this.#gainNode; - } - - #handleEncodedData(audioBuffer) { - let arrayBuffer = new ArrayBuffer(audioBuffer.byteLength); - audioBuffer.copyTo(arrayBuffer); - - if (this.#setting.ptt) { - let packet = new AudioPacket(new Uint8Array(arrayBuffer), this.#setting, getApp().getAudioManager().getGuid()); - getApp().getAudioManager().send(packet.getArray()); - } - } - - #handleRawData(audioData) { - this.#encoder.encode(audioData); - audioData.close(); - } -} diff --git a/frontend/react/src/audio/unitsink.ts b/frontend/react/src/audio/unitsink.ts new file mode 100644 index 00000000..9390f9c0 --- /dev/null +++ b/frontend/react/src/audio/unitsink.ts @@ -0,0 +1,34 @@ +import { AudioSink } from "./audiosink"; +import { AudioPacket } from "./audiopacket"; +import { getApp } from "../olympusapp"; +import { Unit } from "../unit/unit"; + +export class UnitSink extends AudioSink { + #unit: Unit; + + constructor(unit: Unit) { + super(); + + this.#unit = unit; + this.setName(`${unit.getUnitName()} - ${unit.getName()}`); + } + + getUnit() { + return this.#unit; + } + + handleEncodedData(encodedAudioChunk: EncodedAudioChunk) { + let arrayBuffer = new ArrayBuffer(encodedAudioChunk.byteLength); + encodedAudioChunk.copyTo(arrayBuffer); + + let packet = new AudioPacket( + new Uint8Array(arrayBuffer), + { + frequency: 243000000, + modulation: 255, // HOPEFULLY this will never be used by SRS, indicates "loudspeaker" mode + }, + getApp().getAudioManager().getGuid() + ); + getApp().getAudioManager().send(packet.getArray()); + } +} diff --git a/frontend/react/src/dom.d.ts b/frontend/react/src/dom.d.ts index 19928962..4367da9e 100644 --- a/frontend/react/src/dom.d.ts +++ b/frontend/react/src/dom.d.ts @@ -27,7 +27,7 @@ interface CustomEventMap { showUnitContextMenu: CustomEvent; hideUnitContextMenu: CustomEvent; audioSourcesUpdated: CustomEvent; - radiosUpdated: CustomEvent; + audioSinksUpdated: CustomEvent; } declare global { diff --git a/frontend/react/src/eventscontext.tsx b/frontend/react/src/eventscontext.tsx index ac44f7fc..d57951db 100644 --- a/frontend/react/src/eventscontext.tsx +++ b/frontend/react/src/eventscontext.tsx @@ -9,6 +9,7 @@ export const EventsContext = createContext({ setOptionsMenuVisible: (e: boolean) => {}, setAirbaseMenuVisible: (e: boolean) => {}, setRadioMenuVisible: (e: boolean) => {}, + setAudioMenuVisible: (e: boolean) => {}, toggleMainMenuVisible: () => {}, toggleSpawnMenuVisible: () => {}, toggleUnitControlMenuVisible: () => {}, @@ -17,6 +18,7 @@ export const EventsContext = createContext({ toggleOptionsMenuVisible: () => {}, toggleAirbaseMenuVisible: () => {}, toggleRadioMenuVisible: () => {}, + toggleAudioMenuVisible: () => {}, }); export const EventsProvider = EventsContext.Provider; diff --git a/frontend/react/src/interfaces.ts b/frontend/react/src/interfaces.ts index 01c1661a..07f139f4 100644 --- a/frontend/react/src/interfaces.ts +++ b/frontend/react/src/interfaces.ts @@ -1,5 +1,6 @@ import { LatLng } from "leaflet"; import { Coalition, Context } from "./types/types"; +import { AudioSink } from "./audio/audiosink"; class Airbase {} @@ -292,17 +293,5 @@ export interface ServerStatus { paused: boolean; } -export interface SRSRadioSetting { - frequency: number; - modulation: number; - volume: number; - ptt: boolean; - tuned: boolean; -} -export interface AudioSourceSetting { - filename: string; - playing: boolean; - connectedTo: string; -} diff --git a/frontend/react/src/statecontext.tsx b/frontend/react/src/statecontext.tsx index 58cb62a3..9ba4b8ca 100644 --- a/frontend/react/src/statecontext.tsx +++ b/frontend/react/src/statecontext.tsx @@ -10,6 +10,7 @@ export const StateContext = createContext({ optionsMenuVisible: false, airbaseMenuVisible: false, radioMenuVisible: false, + audioMenuVisible: false, mapHiddenTypes: MAP_HIDDEN_TYPES_DEFAULTS, mapOptions: MAP_OPTIONS_DEFAULTS, mapSources: [] as string[], diff --git a/frontend/react/src/ui/panels/audiomenu.tsx b/frontend/react/src/ui/panels/audiomenu.tsx new file mode 100644 index 00000000..dccd3e09 --- /dev/null +++ b/frontend/react/src/ui/panels/audiomenu.tsx @@ -0,0 +1,75 @@ +import React, { useEffect, useState } from "react"; +import { Menu } from "./components/menu"; +import { getApp } from "../../olympusapp"; +import { FaQuestionCircle } from "react-icons/fa"; +import { AudioSourcePanel } from "./components/audiosourcepanel"; +import { AudioSource } from "../../audio/audiosource"; + +export function AudioMenu(props: { open: boolean; onClose: () => void; children?: JSX.Element | JSX.Element[] }) { + const [sources, setSources] = useState([] as AudioSource[]); + + useEffect(() => { + /* Force a rerender */ + document.addEventListener("audioSourcesUpdated", () => { + setSources( + getApp() + ?.getAudioManager() + .getSources() + .map((source) => source) + ); + }); + }, []); + + + return ( + +
The audio source panel allows you to add and manage audio sources.
+
+
+ +
+
+
Use the controls to apply effects and start/stop the playback of an audio source.
+
Sources can be connected to your radios, or attached to a unit to be played on loudspeakers.
+
+
+
+ <> + {sources + .map((source) => { + return ; + })} + + +
+ + ); +} diff --git a/frontend/react/src/ui/panels/components/audiosourcepanel.tsx b/frontend/react/src/ui/panels/components/audiosourcepanel.tsx index 1ba0f0df..8e45c77b 100644 --- a/frontend/react/src/ui/panels/components/audiosourcepanel.tsx +++ b/frontend/react/src/ui/panels/components/audiosourcepanel.tsx @@ -3,10 +3,12 @@ import { OlStateButton } from "../../components/olstatebutton"; import { faPlay, faRepeat } from "@fortawesome/free-solid-svg-icons"; import { getApp } from "../../../olympusapp"; import { AudioSource } from "../../../audio/audiosource"; -import { FaVolumeHigh } from "react-icons/fa6"; +import { FaTrash, FaVolumeHigh } from "react-icons/fa6"; import { OlRangeSlider } from "../../components/olrangeslider"; +import { FaUnlink } from "react-icons/fa"; +import { OlDropdown, OlDropdownItem } from "../../components/oldropdown"; -export function AudioSourcePanel(props: { index: number; source: AudioSource }) { +export function AudioSourcePanel(props: { source: AudioSource }) { return (
- Source: {props.source.getName()} -
- { - let sources = getApp().getAudioManager().getSources(); - sources[props.index].play(); - }} - tooltip="Play file" - > - { - //let setting = props.setting; - //setting.volume = parseFloat(ev.currentTarget.value) / 100; - //props.updateSetting(setting); - }} - className="my-auto" - /> - { - - }} - tooltip="Loop" - > + {props.source.getName()} + {props.source.getName() != "Microphone" && ( +
+ { + props.source.play(); + }} + tooltip="Play file" + > + { + //let setting = props.setting; + //setting.volume = parseFloat(ev.currentTarget.value) / 100; + //props.updateSetting(setting); + }} + className="my-auto" + /> + {}} tooltip="Loop"> +
+ )} + Connected to: +
+ {props.source.getConnectedTo().map((sink) => { + return ( +
+ {sink.getName()} + +
+ ); + })} + + {getApp() + .getAudioManager() + .getSinks() + .filter((sink) => !props.source.getConnectedTo().includes(sink)) + .map((sink) => { + return { + props.source.connect(sink); + }}>{sink.getName()}; + })} +
-
); } diff --git a/frontend/react/src/ui/panels/components/radiopanel.tsx b/frontend/react/src/ui/panels/components/radiopanel.tsx index f37278c8..9b471355 100644 --- a/frontend/react/src/ui/panels/components/radiopanel.tsx +++ b/frontend/react/src/ui/panels/components/radiopanel.tsx @@ -4,11 +4,10 @@ import { FaTrash } from "react-icons/fa6"; import { OlLabelToggle } from "../../components/ollabeltoggle"; import { OlStateButton } from "../../components/olstatebutton"; import { faEarListen, faMicrophoneLines } from "@fortawesome/free-solid-svg-icons"; -import { SRSRadio } from "../../../audio/srsradio"; -import { SRSRadioSetting } from "../../../interfaces"; +import { RadioSink } from "../../../audio/radiosink"; import { getApp } from "../../../olympusapp"; -export function RadioPanel(props: { index: number; setting: SRSRadioSetting, onSettingUpdate: (SRSRadioSetting) => void }) { +export function RadioPanel(props: { radio: RadioSink }) { return (
- Radio {props.index + 1} -
{getApp().getAudioManager().removeRadio(props.index);}}> + {props.radio.getName()} +
{getApp().getAudioManager().removeSink(props.radio);}}>
{ - let setting = props.setting; - setting.frequency = value; - props.onSettingUpdate(setting); + props.radio.setFrequency(value) }} />
{ - let setting = props.setting; - setting.modulation = setting.modulation === 1 ? 0 : 1; - props.onSettingUpdate(setting); + props.radio.setModulation(props.radio.getModulation() === 1 ? 0 : 1); }} > { - let setting = props.setting; - setting.ptt = !setting.ptt; - props.onSettingUpdate(setting); + props.radio.setPtt(!props.radio.getPtt()); }} tooltip="Talk on frequency" > { - let setting = props.setting; - setting.tuned = !setting.tuned; - props.onSettingUpdate(setting); + props.radio.setTuned(!props.radio.getTuned()); }} tooltip="Tune to radio" > diff --git a/frontend/react/src/ui/panels/header.tsx b/frontend/react/src/ui/panels/header.tsx index 4a10d6ad..561725da 100644 --- a/frontend/react/src/ui/panels/header.tsx +++ b/frontend/react/src/ui/panels/header.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useRef, useState } from "react"; import { OlRoundStateButton, OlStateButton, OlLockStateButton } from "../components/olstatebutton"; -import { faSkull, faCamera, faFlag, faLink, faUnlink, faBars } from "@fortawesome/free-solid-svg-icons"; +import { faSkull, faCamera, faFlag, faLink, faUnlink, faBars, faVolumeHigh } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { EventsConsumer } from "../../eventscontext"; import { StateConsumer } from "../../statecontext"; @@ -23,12 +23,12 @@ import { FaChevronLeft, FaChevronRight } from "react-icons/fa6"; export function Header() { const [scrolledLeft, setScrolledLeft] = useState(true); const [scrolledRight, setScrolledRight] = useState(false); + const [audioEnabled, setAudioEnabled] = useState(false); /* Initialize the "scroll" position of the element */ var scrollRef = useRef(null); useEffect(() => { - if (scrollRef.current) - onScroll(scrollRef.current); + if (scrollRef.current) onScroll(scrollRef.current); }); function onScroll(el) { @@ -54,10 +54,9 @@ export function Header() { dark:border-gray-800 dark:bg-olympus-900 `} > - + {!scrolledLeft && (
-
+
{}} tooltip="Lock/unlock protected units (from scripted mission)" /> + { + audioEnabled ? getApp().getAudioManager().stop() : getApp().getAudioManager().start(); + setAudioEnabled(!audioEnabled); + }} + tooltip="Enable/disable audio and radio backend" + icon={faVolumeHigh} + />
void; children?: JSX.Element | JSX.Element[] }) { - const [radioEnabled, setRadioEnabled] = useState(false); - const [radioSettings, setRadioSettings] = useState([] as SRSRadioSetting[]); + const [radios, setRadios] = useState([] as RadioSink[]); useEffect(() => { /* Force a rerender */ - document.addEventListener("radiosUpdated", () => { - setRadioSettings( + document.addEventListener("audioSinksUpdated", () => { + setRadios( getApp() ?.getAudioManager() - .getRadios() - .map((radio) => radio.getSetting()) + .getSinks() + .filter((sink) => sink instanceof RadioSink) + .map((radio) => radio) ); }); }, []); @@ -40,28 +40,10 @@ export function RadioMenu(props: { open: boolean; onClose: () => void; children? dark:text-white `} > -
- Enable radio: - { - radioEnabled ? getApp().getAudioManager().stop() : getApp().getAudioManager().start(); - setRadioEnabled(!radioEnabled); - }} - /> -
- {radioEnabled && radioSettings.map((setting, idx) => { - return ( - { - getApp().getAudioManager().getRadios()[idx].setSetting(setting); - }} - > - ); + {radios.map((radio) => { + return ; })} - {radioEnabled && radioSettings.length < 10 && ( + {radios.length < 10 && ( */ diff --git a/frontend/react/src/ui/panels/sidebar.tsx b/frontend/react/src/ui/panels/sidebar.tsx index 1294481d..b5edf8a7 100644 --- a/frontend/react/src/ui/panels/sidebar.tsx +++ b/frontend/react/src/ui/panels/sidebar.tsx @@ -1,9 +1,10 @@ import React, { useState } from "react"; import { OlStateButton } from "../components/olstatebutton"; -import { faGamepad, faRuler, faPencil, faEllipsisV, faCog, faQuestionCircle, faPlusSquare, faMagnifyingGlass, faPlaneDeparture, faRadio } from "@fortawesome/free-solid-svg-icons"; +import { faGamepad, faRuler, faPencil, faEllipsisV, faCog, faQuestionCircle, faPlusSquare, faMagnifyingGlass, faPlaneDeparture, faRadio, faVolumeHigh } from "@fortawesome/free-solid-svg-icons"; import { EventsConsumer } from "../../eventscontext"; import { StateConsumer } from "../../statecontext"; import { IDLE } from "../../constants/constants"; +import { faSpeakerDeck } from "@fortawesome/free-brands-svg-icons"; export function SideBar() { return ( @@ -64,6 +65,12 @@ export function SideBar() { icon={faRadio} tooltip="Hide/show radio menu" > +
diff --git a/frontend/react/src/ui/ui.tsx b/frontend/react/src/ui/ui.tsx index a4ee5af7..143b03de 100644 --- a/frontend/react/src/ui/ui.tsx +++ b/frontend/react/src/ui/ui.tsx @@ -23,6 +23,7 @@ import { MapContextMenu } from "./contextmenus/mapcontextmenu"; import { AirbaseMenu } from "./panels/airbasemenu"; import { Airbase } from "../mission/airbase"; import { RadioMenu } from "./panels/radiomenu"; +import { AudioMenu } from "./panels/audiomenu"; export type OlympusUIState = { mainMenuVisible: boolean; @@ -44,6 +45,7 @@ export function UI() { const [measureMenuVisible, setMeasureMenuVisible] = useState(false); const [drawingMenuVisible, setDrawingMenuVisible] = useState(false); const [radioMenuVisible, setRadioMenuVisible] = useState(false); + const [audioMenuVisible, setAudioMenuVisible] = useState(false); const [optionsMenuVisible, setOptionsMenuVisible] = useState(false); const [airbaseMenuVisible, setAirbaseMenuVisible] = useState(false); const [mapHiddenTypes, setMapHiddenTypes] = useState(MAP_HIDDEN_TYPES_DEFAULTS); @@ -100,6 +102,7 @@ export function UI() { setOptionsMenuVisible(false); setAirbaseMenuVisible(false); setRadioMenuVisible(false); + setAudioMenuVisible(false); } function checkPassword(password: string) { @@ -157,6 +160,7 @@ export function UI() { optionsMenuVisible: optionsMenuVisible, airbaseMenuVisible: airbaseMenuVisible, radioMenuVisible: radioMenuVisible, + audioMenuVisible: audioMenuVisible, mapOptions: mapOptions, mapHiddenTypes: mapHiddenTypes, mapSources: mapSources, @@ -174,6 +178,7 @@ export function UI() { setOptionsMenuVisible: setOptionsMenuVisible, setAirbaseMenuVisible: setAirbaseMenuVisible, setRadioMenuVisible: setRadioMenuVisible, + setAudioMenuVisible: setAudioMenuVisible, toggleMainMenuVisible: () => { hideAllMenus(); setMainMenuVisible(!mainMenuVisible); @@ -206,6 +211,10 @@ export function UI() { hideAllMenus(); setRadioMenuVisible(!radioMenuVisible); }, + toggleAudioMenuVisible: () => { + hideAllMenus(); + setAudioMenuVisible(!audioMenuVisible); + }, }} >
@@ -241,6 +250,7 @@ export function UI() { setDrawingMenuVisible(false)} /> setAirbaseMenuVisible(false)} airbase={airbase}/> setRadioMenuVisible(false)} /> + setAudioMenuVisible(false)} /> diff --git a/frontend/react/src/unit/unit.ts b/frontend/react/src/unit/unit.ts index cd9b7ee0..d4232ae2 100644 --- a/frontend/react/src/unit/unit.ts +++ b/frontend/react/src/unit/unit.ts @@ -74,6 +74,7 @@ import { faPeopleGroup, faQuestionCircle, faRoute, + faVolumeHigh, faXmarksLines, } from "@fortawesome/free-solid-svg-icons"; import { FaXmarksLines } from "react-icons/fa6"; @@ -850,6 +851,19 @@ export abstract class Unit extends CustomMarker { } ); + contextActionSet.addContextAction( + this, + "speaker", + "Make audio source", + "Make this unit an audio source (loudspeakers)", + faVolumeHigh, + null, + (units: Unit[], _1, _2) => { + units.forEach((unit) => getApp().getAudioManager().addUnitSink(unit)); + }, + { executeImmediately: true } + ); + contextActionSet.addDefaultContextAction( this, "default", diff --git a/frontend/server/src/audio/srshandler.ts b/frontend/server/src/audio/srshandler.ts index d402942e..577a6e29 100644 --- a/frontend/server/src/audio/srshandler.ts +++ b/frontend/server/src/audio/srshandler.ts @@ -3,6 +3,7 @@ const { OpusEncoder } = require("@discordjs/opus"); const encoder = new OpusEncoder(16000, 1); var net = require("net"); +var bufferString = ""; const SRS_VERSION = "2.1.0.10"; @@ -13,43 +14,43 @@ enum MessageType { settings, } +function fromBytes(array) { + let res = 0; + for (let i = 0; i < array.length; i++) { + res = res << 8; + res += array[array.length - i - 1]; + } + return res; +} + +function getBytes(value, length) { + let res: number[] = []; + for (let i = 0; i < length; i++) { + res.push(value & 255); + value = value >> 8; + } + return res; +} + export class SRSHandler { ws: any; tcp = new net.Socket(); udp = require("dgram").createSocket("udp4"); data = JSON.parse(JSON.stringify(defaultSRSData)); syncInterval: any; - packetQueue = []; + clients = []; + SRSPort = 0; constructor(ws, SRSPort) { this.data.Name = `Olympus${globalIndex}`; + this.SRSPort = SRSPort; globalIndex += 1; /* Websocket */ this.ws = ws; this.ws.on("error", console.error); this.ws.on("message", (data) => { - switch (data[0]) { - case MessageType.audio: - let audioBuffer = data.slice(1); - this.packetQueue.push(audioBuffer); - this.udp.send(audioBuffer, SRSPort, "localhost", (error) => { - if (error) - console.log(`Error sending data to SRS server: ${error}`); - }); - break; - case MessageType.settings: - let message = JSON.parse(data.slice(1)); - this.data.ClientGuid = message.guid; - this.data.Coalition = message.coalition; - message.settings.forEach((setting, idx) => { - this.data.RadioInfo.radios[idx].freq = setting.frequency; - this.data.RadioInfo.radios[idx].modulation = setting.modulation; - }); - break; - default: - break; - } + this.decodeData(data); }); this.ws.on("close", () => { this.tcp.end(); @@ -81,6 +82,20 @@ export class SRSHandler { }, 1000); }); + this.tcp.on("data", (data) => { + bufferString += data.toString(); + while (bufferString.includes("\n")) { + try { + let message = JSON.parse(bufferString.split("\n")[0]); + bufferString = bufferString.slice(bufferString.indexOf("\n") + 1); + if (message.Clients !== undefined) + this.clients = message.Clients; + } catch (e) { + console.log(e); + } + } + }); + /* UDP */ this.udp.on("listening", () => { console.log(`Listening to SRS Server on UDP port ${SRSPort}`); @@ -90,4 +105,52 @@ export class SRSHandler { if (this.ws && message.length > 22) this.ws.send(message); }); } + + decodeData(data){ + switch (data[0]) { + case MessageType.audio: + let packetUint8Array = new Uint8Array(data.slice(1)); + + let audioLength = fromBytes(packetUint8Array.slice(2, 4)); + let frequenciesLength = fromBytes(packetUint8Array.slice(4, 6)); + let modulation = fromBytes(packetUint8Array.slice(6 + audioLength + 8, 6 + audioLength + 8 + 1)); + let offset = 6 + audioLength + frequenciesLength; + + if (modulation == 255) { + packetUint8Array[6 + audioLength + 8] = 2; + this.clients.forEach((client) => { + getBytes(client.RadioInfo.unitId, 4).forEach((value, idx) => { + packetUint8Array[offset + idx] = value; + }); + + var dst = new ArrayBuffer(packetUint8Array.byteLength); + let newBuffer = new Uint8Array(dst); + newBuffer.set(new Uint8Array(packetUint8Array)); + this.udp.send(newBuffer, this.SRSPort, "localhost", (error) => { + if (error) + console.log(`Error sending data to SRS server: ${error}`); + }) + }) + } else { + this.udp.send(packetUint8Array, this.SRSPort, "localhost", (error) => { + if (error) + console.log(`Error sending data to SRS server: ${error}`); + }); + } + + + break; + case MessageType.settings: + let message = JSON.parse(data.slice(1)); + this.data.ClientGuid = message.guid; + this.data.Coalition = message.coalition; + message.settings.forEach((setting, idx) => { + this.data.RadioInfo.radios[idx].freq = setting.frequency; + this.data.RadioInfo.radios[idx].modulation = setting.modulation; + }); + break; + default: + break; + } + } }