diff --git a/frontend/react/package.json b/frontend/react/package.json index ca6df087..7b512e08 100644 --- a/frontend/react/package.json +++ b/frontend/react/package.json @@ -16,13 +16,16 @@ "@fortawesome/react-fontawesome": "^0.2.0", "@tanem/svg-injector": "^10.1.68", "@turf/turf": "^6.5.0", + "@types/dom-webcodecs": "^0.1.11", "@types/leaflet": "^1.9.8", "@types/react-leaflet": "^3.0.0", "@types/turf": "^3.5.32", + "buffer": "^6.0.3", "js-sha256": "^0.11.0", "leaflet": "^1.9.4", "leaflet-control-mini-map": "^0.4.0", "leaflet-path-drag": "^1.9.5", + "opus-decoder": "^0.7.6", "react": "^18.2.0", "react-dom": "^18.2.0", "react-icons": "^5.0.1", @@ -32,6 +35,7 @@ }, "devDependencies": { "@eslint/js": "^9.6.0", + "@types/node": "^22.5.1", "@types/react": "^18.2.66", "@types/react-dom": "^18.2.22", "@typescript-eslint/parser": "^7.14.1", @@ -49,6 +53,7 @@ "prettier": "^3.3.2", "tailwindcss": "^3.4.3", "typescript-eslint": "^7.14.1", - "vite": "^5.2.0" + "vite": "^5.2.0", + "web-audio-peak-meter": "^3.1.0" } } diff --git a/frontend/react/src/audio/audiolibrary.js b/frontend/react/src/audio/audiolibrary.js new file mode 100644 index 00000000..f5e31c89 --- /dev/null +++ b/frontend/react/src/audio/audiolibrary.js @@ -0,0 +1,289 @@ +// TODO Convert to typescript +// Audio library I shamelessly copied from the web + +// SAFARI Polyfills +if (!window.AudioBuffer.prototype.copyToChannel) { + window.AudioBuffer.prototype.copyToChannel = function copyToChannel(buffer, channel) { + this.getChannelData(channel).set(buffer); + }; +} +if (!window.AudioBuffer.prototype.copyFromChannel) { + window.AudioBuffer.prototype.copyFromChannel = function copyFromChannel(buffer, channel) { + buffer.set(this.getChannelData(channel)); + }; +} + +export class Effect { + constructor(context) { + this.name = "effect"; + this.context = context; + this.input = this.context.createGain(); + this.effect = null; + this.bypassed = false; + this.output = this.context.createGain(); + this.setup(); + this.wireUp(); + } + + setup() { + this.effect = this.context.createGain(); + } + + wireUp() { + this.input.connect(this.effect); + this.effect.connect(this.output); + } + + connect(destination) { + this.output.connect(destination); + } +} + +export class Sample { + constructor(context) { + this.context = context; + this.buffer = this.context.createBufferSource(); + this.buffer.start(); + this.sampleBuffer = null; + this.rawBuffer = null; + this.loaded = false; + this.output = this.context.createGain(); + this.output.gain.value = 0.1; + } + + play() { + if (this.loaded) { + this.buffer = this.context.createBufferSource(); + this.buffer.buffer = this.sampleBuffer; + this.buffer.connect(this.output); + this.buffer.start(this.context.currentTime); + } + } + + connect(input) { + this.output.connect(input); + } + + load(path) { + this.loaded = false; + return fetch(path) + .then((response) => response.arrayBuffer()) + .then((myBlob) => { + return new Promise((resolve, reject) => { + this.context.decodeAudioData(myBlob, resolve, reject); + }); + }) + .then((buffer) => { + this.sampleBuffer = buffer; + this.loaded = true; + return this; + }); + } +} + +export class AmpEnvelope { + constructor(context, gain = 1) { + this.context = context; + this.output = this.context.createGain(); + this.output.gain.value = gain; + this.partials = []; + this.velocity = 0; + this.gain = gain; + this._attack = 0; + this._decay = 0.001; + this._sustain = this.output.gain.value; + this._release = 0.001; + } + + on(velocity) { + this.velocity = velocity / 127; + this.start(this.context.currentTime); + } + + off(MidiEvent) { + return this.stop(this.context.currentTime); + } + + start(time) { + this.output.gain.value = 0; + this.output.gain.setValueAtTime(0, time); + this.output.gain.setTargetAtTime(1, time, this.attack + 0.00001); + this.output.gain.setTargetAtTime(this.sustain * this.velocity, time + this.attack, this.decay); + } + + stop(time) { + this.sustain = this.output.gain.value; + this.output.gain.cancelScheduledValues(time); + this.output.gain.setValueAtTime(this.sustain, time); + this.output.gain.setTargetAtTime(0, time, this.release + 0.00001); + } + + set attack(value) { + this._attack = value; + } + + get attack() { + return this._attack; + } + + set decay(value) { + this._decay = value; + } + + get decay() { + return this._decay; + } + + set sustain(value) { + this.gain = value; + this._sustain; + } + + get sustain() { + return this.gain; + } + + set release(value) { + this._release = value; + } + + get release() { + return this._release; + } + + connect(destination) { + this.output.connect(destination); + } +} + +export class Voice { + constructor(context, type = "sawtooth", gain = 0.1) { + this.context = context; + this.type = type; + this.value = -1; + this.gain = gain; + this.output = this.context.createGain(); + this.partials = []; + this.output.gain.value = this.gain; + this.ampEnvelope = new AmpEnvelope(this.context); + this.ampEnvelope.connect(this.output); + } + + init() { + let osc = this.context.createOscillator(); + osc.type = this.type; + osc.connect(this.ampEnvelope.output); + osc.start(this.context.currentTime); + this.partials.push(osc); + } + + on(MidiEvent) { + this.value = MidiEvent.value; + this.partials.forEach((osc) => { + osc.frequency.value = MidiEvent.frequency; + }); + this.ampEnvelope.on(MidiEvent.velocity || MidiEvent); + } + + off(MidiEvent) { + this.ampEnvelope.off(MidiEvent); + this.partials.forEach((osc) => { + osc.stop(this.context.currentTime + this.ampEnvelope.release * 4); + }); + } + + connect(destination) { + this.output.connect(destination); + } + + set detune(value) { + this.partials.forEach((p) => (p.detune.value = value)); + } + + set attack(value) { + this.ampEnvelope.attack = value; + } + + get attack() { + return this.ampEnvelope.attack; + } + + set decay(value) { + this.ampEnvelope.decay = value; + } + + get decay() { + return this.ampEnvelope.decay; + } + + set sustain(value) { + this.ampEnvelope.sustain = value; + } + + get sustain() { + return this.ampEnvelope.sustain; + } + + set release(value) { + this.ampEnvelope.release = value; + } + + get release() { + return this.ampEnvelope.release; + } +} +export class Noise extends Voice { + constructor(context, gain) { + super(context, gain); + this._length = 2; + } + + get length() { + return this._length || 2; + } + set length(value) { + this._length = value; + } + + init() { + var lBuffer = new Float32Array(this.length * this.context.sampleRate); + var rBuffer = new Float32Array(this.length * this.context.sampleRate); + for (let i = 0; i < this.length * this.context.sampleRate; i++) { + lBuffer[i] = 1 - 2 * Math.random(); + rBuffer[i] = 1 - 2 * Math.random(); + } + let buffer = this.context.createBuffer(2, this.length * this.context.sampleRate, this.context.sampleRate); + buffer.copyToChannel(lBuffer, 0); + buffer.copyToChannel(rBuffer, 1); + + let osc = this.context.createBufferSource(); + osc.buffer = buffer; + osc.loop = true; + osc.loopStart = 0; + osc.loopEnd = 2; + osc.start(this.context.currentTime); + osc.connect(this.ampEnvelope.output); + this.partials.push(osc); + } + + on(MidiEvent) { + this.value = MidiEvent.value; + this.ampEnvelope.on(MidiEvent.velocity || MidiEvent); + } +} + +export class Filter extends Effect { + constructor(context, type = "lowpass", cutoff = 1000, resonance = 0.9) { + super(context); + this.name = "filter"; + this.effect.frequency.value = cutoff; + this.effect.Q.value = resonance; + this.effect.type = type; + } + + setup() { + this.effect = this.context.createBiquadFilter(); + this.effect.connect(this.output); + this.wireUp(); + } +} diff --git a/frontend/react/src/audio/audiomanager.ts b/frontend/react/src/audio/audiomanager.ts new file mode 100644 index 00000000..66b59175 --- /dev/null +++ b/frontend/react/src/audio/audiomanager.ts @@ -0,0 +1,241 @@ +import { AudioMessageType } from "../constants/constants"; +import { MicrophoneSource } from "./microphonesource"; +import { RadioSink } from "./radiosink"; +import { getApp } from "../olympusapp"; +import { makeID } from "../other/utils"; +import { FileSource } from "./filesource"; +import { AudioSource } from "./audiosource"; +import { Buffer } from "buffer"; +import { PlaybackPipeline } from "./playbackpipeline"; +import { AudioSink } from "./audiosink"; +import { Unit } from "../unit/unit"; +import { UnitSink } from "./unitsink"; +import { AudioPacket, MessageType } from "./audiopacket"; + +export class AudioManager { + #audioContext: AudioContext; + + /* The playback pipeline enables audio playback on the speakers/headphones */ + #playbackPipeline: PlaybackPipeline; + + /* The audio sinks used to transmit the audio stream to the SRS backend */ + #sinks: AudioSink[] = []; + + /* List of all possible audio sources (microphone, file stream etc...) */ + #sources: AudioSource[] = []; + + /* The audio backend must be manually started so that the browser can detect the user is enabling audio. + Otherwise, no playback will be performed. */ + #running: boolean = false; + #address: string = "localhost"; + #port: number = 4000; + #socket: WebSocket | null = null; + #guid: string = makeID(22); + #SRSClientUnitIDs: number[] = []; + + constructor() { + document.addEventListener("configLoaded", () => { + let config = getApp().getConfig(); + if (config["WSPort"]) { + this.setPort(config["WSPort"]); + } + }); + + setInterval(() => { + this.#syncRadioSettings(); + }, 1000); + } + + start() { + this.#running = true; + this.#audioContext = new AudioContext({ sampleRate: 16000 }); + this.#playbackPipeline = new PlaybackPipeline(); + + /* Connect the audio websocket */ + let res = this.#address.match(/(?:http|https):\/\/(.+):/); + let wsAddress = res ? res[1] : this.#address; + this.#socket = new WebSocket(`ws://${wsAddress}:${this.#port}`); + + /* Log the opening of the connection */ + this.#socket.addEventListener("open", (event) => { + console.log("Connection to audio websocket successfull"); + }); + + /* Log any websocket errors */ + this.#socket.addEventListener("error", (event) => { + console.log(event); + }); + + /* Handle the reception of a new message */ + this.#socket.addEventListener("message", (event) => { + this.#sinks.forEach(async (sink) => { + if (sink instanceof RadioSink) { + /* Extract the audio data as array */ + let packetUint8Array = new Uint8Array(await event.data.arrayBuffer()); + + if (packetUint8Array[0] === MessageType.audio) { + /* Extract the encoded audio data */ + let audioPacket = new AudioPacket(); + audioPacket.fromByteArray(packetUint8Array.slice(1)); + + /* Extract the frequency value and play it on the speakers if we are listening to it*/ + audioPacket.getFrequencies().forEach((frequencyInfo) => { + if (sink.getFrequency() === frequencyInfo.frequency && sink.getModulation() === frequencyInfo.modulation) { + this.#playbackPipeline.playBuffer(audioPacket.getAudioData().buffer); + } + }); + } else { + this.#SRSClientUnitIDs = JSON.parse(new TextDecoder().decode(packetUint8Array.slice(1))).unitIDs; + } + } + }); + }); + + /* Add the microphone source and connect it directly to the radio */ + const microphoneSource = new MicrophoneSource(); + microphoneSource.initialize().then(() => { + this.#sinks.forEach((sink) => { + if (sink instanceof RadioSink) microphoneSource.connect(sink); + }); + this.#sources.push(microphoneSource); + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + + /* Add two default radios */ + this.addRadio(); + this.addRadio(); + + }); + document.dispatchEvent(new CustomEvent("audioManagerStateChanged")); + } + + stop() { + this.#running = false; + this.#sources.forEach((source) => { + source.disconnect(); + }); + this.#sinks.forEach((sink) => { + sink.disconnect(); + }); + this.#sources = []; + this.#sinks = []; + + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + document.dispatchEvent(new CustomEvent("audioManagerStateChanged")); + } + + setAddress(address) { + this.#address = address; + } + + setPort(port) { + this.#port = port; + } + + addFileSource(file) { + console.log(`Adding file source from ${file.name}`); + if (!this.#running) { + console.log("Audio manager not started, aborting..."); + return; + } + const newSource = new FileSource(file); + this.#sources.push(newSource); + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + } + + getSources() { + return this.#sources; + } + + removeSource(source: AudioSource) { + console.log(`Removing source ${source.getName()}`); + if (!this.#running) { + console.log("Audio manager not started, aborting..."); + return; + } + source.disconnect(); + this.#sources = this.#sources.filter((v) => v != source); + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + } + + addUnitSink(unit: Unit) { + console.log(`Adding unit sink for unit with ID ${unit.ID}`); + if (!this.#running) { + console.log("Audio manager not started, aborting..."); + return; + } + this.#sinks.push(new UnitSink(unit)); + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + addRadio() { + console.log("Adding new radio"); + if (!this.#running) { + console.log("Audio manager not started, aborting..."); + return; + } + const newRadio = new RadioSink(); + this.#sinks.push(newRadio); + newRadio.setName(`Radio ${this.#sinks.length}`); + this.#sources[0].connect(newRadio); + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getSinks() { + return this.#sinks; + } + + removeSink(sink) { + console.log(`Removing sink ${sink.getName()}`); + if (!this.#running) { + console.log("Audio manager not started, aborting..."); + return; + } + sink.disconnect(); + this.#sinks = this.#sinks.filter((v) => v != sink); + let idx = 1; + this.#sinks.forEach((sink) => { + if (sink instanceof RadioSink) sink.setName(`Radio ${idx++}`); + }); + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getGuid() { + return this.#guid; + } + + send(array) { + this.#socket?.send(array); + } + + getAudioContext() { + return this.#audioContext; + } + + getSRSClientsUnitIDs() { + return this.#SRSClientUnitIDs; + } + + isRunning() { + return this.#running; + } + + #syncRadioSettings() { + let message = { + type: "Settings update", + guid: this.#guid, + coalition: 2, + settings: this.#sinks + .filter((sink) => sink instanceof RadioSink) + .map((radio) => { + return { + frequency: radio.getFrequency(), + modulation: radio.getModulation(), + ptt: radio.getPtt(), + }; + }), + }; + + if (this.#socket?.readyState == 1) this.#socket?.send(new Uint8Array([AudioMessageType.settings, ...Buffer.from(JSON.stringify(message), "utf-8")])); + } +} diff --git a/frontend/react/src/audio/audiopacket.ts b/frontend/react/src/audio/audiopacket.ts new file mode 100644 index 00000000..a92042d9 --- /dev/null +++ b/frontend/react/src/audio/audiopacket.ts @@ -0,0 +1,210 @@ +// TODO This code is in common with the backend, would be nice to share it */ +import { byteArrayToDouble, byteArrayToInteger, doubleToByteArray, integerToByteArray } from "../other/utils"; +import { Buffer } from "buffer"; + +var packetID = 0; + +export enum MessageType { + audio, + settings, + unitIDs +} + +export class AudioPacket { + /* Mandatory data */ + #frequencies: { frequency: number; modulation: number; encryption: number }[] = []; + #audioData: Uint8Array; + #transmissionGUID: string; + #clientGUID: string; + + /* Default data */ + #unitID: number = 0; + #hops: number = 0; + + /* Usually internally set only */ + #packetID: number | null = null; + + fromByteArray(byteArray: Uint8Array) { + let totalLength = byteArrayToInteger(byteArray.slice(0, 2)); + let audioLength = byteArrayToInteger(byteArray.slice(2, 4)); + let frequenciesLength = byteArrayToInteger(byteArray.slice(4, 6)); + + /* Perform some sanity checks */ + if (totalLength !== byteArray.length) { + console.log( + `Warning, audio packet expected length is ${totalLength} but received length is ${byteArray.length}, aborting...` + ); + return; + } + + if (frequenciesLength % 10 !== 0) { + console.log( + `Warning, audio packet frequencies data length is ${frequenciesLength} which is not a multiple of 10, aborting...` + ); + return; + } + + /* Extract the audio data */ + this.#audioData = byteArray.slice(6, 6 + audioLength); + + /* Extract the frequencies */ + let offset = 6 + audioLength; + for (let idx = 0; idx < frequenciesLength / 10; idx++) { + this.#frequencies.push({ + frequency: byteArrayToDouble(byteArray.slice(offset, offset + 8)), + modulation: byteArray[offset + 8], + encryption: byteArray[offset + 9], + }); + offset += 10; + } + + /* If necessary increase the packetID */ + if (this.#packetID === null) this.#packetID = packetID++; + + /* Extract the remaining data */ + this.#unitID = byteArrayToInteger(byteArray.slice(offset, offset + 4)); + offset += 4; + this.#packetID = byteArrayToInteger(byteArray.slice(offset, offset + 8)); + offset += 8; + this.#hops = byteArrayToInteger(byteArray.slice(offset, offset + 1)); + offset += 1; + this.#transmissionGUID = new TextDecoder().decode(byteArray.slice(offset, offset + 22)); + offset += 22; + this.#clientGUID = new TextDecoder().decode(byteArray.slice(offset, offset + 22)); + offset += 22; + } + + toByteArray() { + /* Perform some sanity checks // TODO check correct values */ + if (this.#frequencies.length === 0) { + console.log( + "Warning, could not encode audio packet, no frequencies data provided, aborting..." + ); + return; + } + + if (this.#audioData === undefined) { + console.log( + "Warning, could not encode audio packet, no audio data provided, aborting..." + ); + return; + } + + if (this.#transmissionGUID === undefined) { + console.log( + "Warning, could not encode audio packet, no transmission GUID provided, aborting..." + ); + return; + } + + if (this.#clientGUID === undefined) { + console.log( + "Warning, could not encode audio packet, no client GUID provided, aborting..." + ); + return; + } + + // Prepare the array for the header + let header: number[] = [0, 0, 0, 0, 0, 0]; + + // Encode the frequencies data + let frequenciesData = [] as number[]; + this.#frequencies.forEach((data) => { + frequenciesData = frequenciesData.concat( + [...doubleToByteArray(data.frequency)], + [data.modulation], + [data.encryption] + ); + }); + + // Encode unitID, packetID, hops + let encUnitID: number[] = integerToByteArray(this.#unitID, 4); + let encPacketID: number[] = integerToByteArray(this.#packetID, 8); + let encHops: number[] = [this.#hops]; + + // Assemble packet + let encodedData: number[] = ([] as number[]).concat( + header, + [...this.#audioData], + frequenciesData, + encUnitID, + encPacketID, + encHops, + [...Buffer.from(this.#transmissionGUID, "utf-8")], + [...Buffer.from(this.#clientGUID, "utf-8")] + ); + + // Set the lengths of the parts + let encPacketLen = integerToByteArray(encodedData.length, 2); + encodedData[0] = encPacketLen[0]; + encodedData[1] = encPacketLen[1]; + + let encAudioLen = integerToByteArray(this.#audioData.length, 2); + encodedData[2] = encAudioLen[0]; + encodedData[3] = encAudioLen[1]; + + let frequencyAudioLen = integerToByteArray(frequenciesData.length, 2); + encodedData[4] = frequencyAudioLen[0]; + encodedData[5] = frequencyAudioLen[1]; + + return new Uint8Array([0].concat(encodedData)); + } + + setFrequencies( + frequencies: { frequency: number; modulation: number; encryption: number }[] + ) { + this.#frequencies = frequencies; + } + + getFrequencies() { + return this.#frequencies; + } + + setAudioData(audioData: Uint8Array) { + this.#audioData = audioData; + } + + getAudioData() { + return this.#audioData; + } + + setTransmissionGUID(transmissionGUID: string) { + this.#transmissionGUID = transmissionGUID; + } + + getTransmissionGUID() { + return this.#transmissionGUID; + } + + setClientGUID(clientGUID: string) { + this.#clientGUID = clientGUID; + } + + getClientGUID() { + return this.#clientGUID; + } + + setUnitID(unitID: number) { + this.#unitID = unitID; + } + + getUnitID() { + return this.#unitID; + } + + setPacketID(packetID: number) { + this.#packetID = packetID; + } + + getPacketID() { + return this.#packetID; + } + + setHops(hops: number) { + this.#hops = hops; + } + + getHops() { + return this.#hops; + } +} diff --git a/frontend/react/src/audio/audiosink.ts b/frontend/react/src/audio/audiosink.ts new file mode 100644 index 00000000..af90064a --- /dev/null +++ b/frontend/react/src/audio/audiosink.ts @@ -0,0 +1,28 @@ +import { getApp } from "../olympusapp"; + +/* Base audio sink class */ +export class AudioSink { + #name: string; + #gainNode: GainNode; + + constructor() { + this.#gainNode = getApp().getAudioManager().getAudioContext().createGain(); + } + + setName(name) { + this.#name = name; + } + + getName() { + return this.#name; + } + + disconnect() { + this.getInputNode().disconnect(); + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getInputNode() { + return this.#gainNode; + } +} diff --git a/frontend/react/src/audio/audiosource.ts b/frontend/react/src/audio/audiosource.ts new file mode 100644 index 00000000..e852f07a --- /dev/null +++ b/frontend/react/src/audio/audiosource.ts @@ -0,0 +1,69 @@ +import { getApp } from "../olympusapp"; +import { AudioSink } from "./audiosink"; +import { WebAudioPeakMeter } from "web-audio-peak-meter"; + +/* Base abstract audio source class */ +export abstract class AudioSource { + #connectedTo: AudioSink[] = []; + #name = ""; + #meter: WebAudioPeakMeter; + #volume: number = 1.0; + #gainNode: GainNode; + + constructor() { + this.#gainNode = getApp().getAudioManager().getAudioContext().createGain(); + + /* This library requires a div element to initialize the object. Create a fake element, we will read the data and render it ourselves. */ + this.#meter = new WebAudioPeakMeter(this.#gainNode, document.createElement("div")); + } + + connect(sink: AudioSink) { + this.getOutputNode().connect(sink.getInputNode()); + this.#connectedTo.push(sink); + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + } + + disconnect(sinkToDisconnect?: AudioSink) { + if (sinkToDisconnect !== undefined) { + this.getOutputNode().disconnect(sinkToDisconnect.getInputNode()); + this.#connectedTo = this.#connectedTo.filter((sink) => sink != sinkToDisconnect); + } else { + this.getOutputNode().disconnect(); + } + + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + } + + setName(name) { + this.#name = name; + } + + getName() { + return this.#name; + } + + getConnectedTo() { + return this.#connectedTo; + } + + setVolume(volume) { + this.#volume = volume; + this.#gainNode.gain.exponentialRampToValueAtTime(volume, getApp().getAudioManager().getAudioContext().currentTime + 0.02); + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + } + + getVolume() { + return this.#volume; + } + + getMeter() { + return this.#meter; + } + + getOutputNode() { + return this.#gainNode; + } + + /* Play method must be implemented by child classes */ + abstract play(): void; +} diff --git a/frontend/react/src/audio/audiounitpipeline.ts b/frontend/react/src/audio/audiounitpipeline.ts new file mode 100644 index 00000000..f10e0a21 --- /dev/null +++ b/frontend/react/src/audio/audiounitpipeline.ts @@ -0,0 +1,206 @@ +import { getApp } from "../olympusapp"; +import { Unit } from "../unit/unit"; +import { Filter, Noise } from "./audiolibrary"; +import { AudioPacket } from "./audiopacket"; + +const MAX_DISTANCE = 1852; // Ignore clients that are further away than 1NM, to save performance. + +export class AudioUnitPipeline { + #inputNode: GainNode; + #sourceUnit: Unit; + #unitID: number; + #gainNode: GainNode; + #destinationNode: MediaStreamAudioDestinationNode; + #audioTrackProcessor: any; + #encoder: AudioEncoder; + + #convolverNode: ConvolverNode; + #preDelayNode: DelayNode; + #multitapNodes: DelayNode[]; + #multitapGainNode: GainNode; + #wetGainNode: GainNode; + #tailOsc: Noise; + + #distance: number = 0; + + constructor(sourceUnit: Unit, unitID: number, inputNode: GainNode) { + this.#sourceUnit = sourceUnit; + this.#unitID = unitID; + + /* Initialize the Opus Encoder */ + this.#encoder = new AudioEncoder({ + output: (data) => this.handleEncodedData(data, unitID), + error: (e) => { + console.log(e); + }, + }); + + this.#encoder.configure({ + codec: "opus", + numberOfChannels: 1, + sampleRate: 16000, + //@ts-ignore // TODO why is it giving error? + opus: { + frameDuration: 40000, + }, + bitrateMode: "constant", + }); + + /* Create the destination node where the stream will be written to be encoded and sent to SRS */ + this.#destinationNode = getApp().getAudioManager().getAudioContext().createMediaStreamDestination(); + this.#destinationNode.channelCount = 1; + + /* Gain node to modulate the strength of the audio */ + this.#gainNode = getApp().getAudioManager().getAudioContext().createGain(); + + /* Create the track processor to encode and write the data to SRS */ + //@ts-ignore + this.#audioTrackProcessor = new MediaStreamTrackProcessor({ + track: this.#destinationNode.stream.getAudioTracks()[0], + }); + this.#audioTrackProcessor.readable.pipeTo( + new WritableStream({ + write: (audioData) => this.handleRawData(audioData), + }) + ); + + /* Create the pipeline */ + this.#inputNode = inputNode; + this.#setupEffects(); + + /* Create the interval task to update the data */ + setInterval(() => { + /* Get the destination unit and compute the distance to it */ + let destinationUnit = getApp().getUnitsManager().getUnitByID(this.#unitID); + if (destinationUnit) { + let distance = destinationUnit?.getPosition().distanceTo(this.#sourceUnit.getPosition()); + + /* The units positions are updated at a low frequency. Filter the distance to avoid sudden volume jumps */ + this.#distance = 0.9 * this.#distance + 0.1 * distance; + + /* Don't bother updating parameters if the client is too far away */ + if (this.#distance < MAX_DISTANCE) { + /* Compute a new gain decreasing with distance. */ + let newGain = 1.0 - Math.pow(this.#distance / 1000, 0.5); // Arbitrary + + /* Set the values of the main gain node and the multitap gain node, used for reverb effect */ + this.#gainNode.gain.setValueAtTime(newGain, getApp().getAudioManager().getAudioContext().currentTime); + this.#multitapGainNode.gain.setValueAtTime(newGain / 10, getApp().getAudioManager().getAudioContext().currentTime); + + /* Increase reverb and predelay with distance */ + let reverbTime = this.#distance / 1000 / 4; //Arbitrary + let preDelay = this.#distance / 1000 / 2; // Arbitrary + this.#preDelayNode.delayTime.setValueAtTime(preDelay, getApp().getAudioManager().getAudioContext().currentTime); + this.#multitapNodes.forEach((t, i) => { + t.delayTime.setValueAtTime(0.001 + i * (preDelay / 2), getApp().getAudioManager().getAudioContext().currentTime); + }); + this.#tailOsc.release = reverbTime / 3; + } + } + }, 100); + } + + handleEncodedData(encodedAudioChunk, unitID) { + /* Encode the data in SRS format and send it to the backend */ + let arrayBuffer = new ArrayBuffer(encodedAudioChunk.byteLength); + encodedAudioChunk.copyTo(arrayBuffer); + + let audioPacket = new AudioPacket(); + audioPacket.setAudioData(new Uint8Array(arrayBuffer)); + audioPacket.setFrequencies([ + { + frequency: 100, + modulation: 2, + encryption: 0, + }, + ]); + audioPacket.setClientGUID(getApp().getAudioManager().getGuid()); + audioPacket.setTransmissionGUID(getApp().getAudioManager().getGuid()); + + if (unitID !== 0) { + audioPacket.setUnitID(unitID); + getApp().getAudioManager().send(audioPacket.toByteArray()); + } + } + + handleRawData(audioData) { + /* Ignore players that are too far away */ + if (this.#distance < MAX_DISTANCE) { + this.#encoder.encode(audioData); + audioData.close(); + } + } + + #setupEffects() { + /* Create the nodes necessary for the pipeline */ + this.#convolverNode = getApp().getAudioManager().getAudioContext().createConvolver(); + this.#preDelayNode = getApp().getAudioManager().getAudioContext().createDelay(1); + this.#multitapGainNode = getApp().getAudioManager().getAudioContext().createGain(); + this.#wetGainNode = getApp().getAudioManager().getAudioContext().createGain(); + this.#multitapNodes = []; + for (let i = 2; i > 0; i--) { + this.#multitapNodes.push(getApp().getAudioManager().getAudioContext().createDelay(1)); + } + + /* Connect the nodes as follows + /------> pre delay -> convolver ------\ + input -> main gain -> wet gain -< >-> destination + \-> multitap[0] -> ... -> multitap[n]-/ + + The multitap nodes simulate distinct echoes coming from the original sound. Multitap[0] is the original sound. + The predelay and convolver nodes simulate reverb. + */ + + this.#inputNode.connect(this.#gainNode); + this.#gainNode.connect(this.#wetGainNode); + this.#wetGainNode.connect(this.#preDelayNode); + this.#wetGainNode.connect(this.#multitapNodes[0]); + this.#multitapNodes.map((t, i) => { + if (this.#multitapNodes[i + 1]) { + t.connect(this.#multitapNodes[i + 1]); + } + }); + this.#multitapNodes[this.#multitapNodes.length - 1].connect(this.#multitapGainNode); + this.#multitapGainNode.connect(this.#destinationNode); + this.#preDelayNode.connect(this.#convolverNode); + this.#convolverNode.connect(this.#destinationNode); + + /* Render the random noise needed for the convolver node to simulate reverb */ + this.#renderTail(0.1); //Arbitrary + } + + #renderTail(reverbTime) { + let attack = 0; + let decay = 0.0; + + /* Generate an offline audio context to render the reverb noise */ + const tailContext = new OfflineAudioContext( + 2, + getApp().getAudioManager().getAudioContext().sampleRate * reverbTime, + getApp().getAudioManager().getAudioContext().sampleRate + ); + + /* A noise oscillator and a two filters are added to smooth the reverb */ + this.#tailOsc = new Noise(tailContext, 1); + const tailLPFilter = new Filter(tailContext, "lowpass", 5000, 1); + const tailHPFilter = new Filter(tailContext, "highpass", 500, 1); + + /* Initialize and connect the oscillator with the filters */ + this.#tailOsc.init(); + this.#tailOsc.connect(tailHPFilter.input); + tailHPFilter.connect(tailLPFilter.input); + tailLPFilter.connect(tailContext.destination); + this.#tailOsc.attack = attack; + this.#tailOsc.decay = decay; + + setTimeout(() => { + /* Set the buffer of the convolver node */ + tailContext.startRendering().then((buffer) => { + this.#convolverNode.buffer = buffer; + }); + + this.#tailOsc.on({ frequency: 500, velocity: 127 }); + //tailOsc.off(); // TODO In the original example I copied, this was turned off. No idea why but it seems to work correctly if left on. To investigate. + }, 20); + } +} diff --git a/frontend/react/src/audio/filesource.ts b/frontend/react/src/audio/filesource.ts new file mode 100644 index 00000000..647c134a --- /dev/null +++ b/frontend/react/src/audio/filesource.ts @@ -0,0 +1,119 @@ +import { AudioSource } from "./audiosource"; +import { getApp } from "../olympusapp"; + +export class FileSource extends AudioSource { + #file: File; + #source: AudioBufferSourceNode; + #duration: number = 0; + #currentPosition: number = 0; + #updateInterval: any; + #lastUpdateTime: number = 0; + #playing = false; + #audioBuffer: AudioBuffer; + #restartTimeout: any; + #looping = false; + + constructor(file) { + super(); + this.#file = file; + + this.setName(this.#file?.name ?? "N/A"); + + /* Create the file reader and read the file from disk */ + var reader = new FileReader(); + reader.onload = (e) => { + var contents = e.target?.result; + if (contents) { + getApp() + .getAudioManager() + .getAudioContext() + /* Decode the audio file. This method takes care of codecs */ + .decodeAudioData(contents as ArrayBuffer, (audioBuffer) => { + this.#audioBuffer = audioBuffer; + this.#duration = audioBuffer.duration; + }); + } + }; + reader.readAsArrayBuffer(this.#file); + } + + play() { + /* A new buffer source must be created every time the file is played */ + this.#source = getApp().getAudioManager().getAudioContext().createBufferSource(); + this.#source.buffer = this.#audioBuffer; + this.#source.connect(this.getOutputNode()); + this.#source.loop = this.#looping; + + /* Start playing the file at the selected position */ + this.#source.start(0, this.#currentPosition); + this.#playing = true; + const now = Date.now() / 1000; + this.#lastUpdateTime = now; + + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + + this.#updateInterval = setInterval(() => { + /* Update the current position value every second */ + const now = Date.now() / 1000; + this.#currentPosition += now - this.#lastUpdateTime; + this.#lastUpdateTime = now; + + if (this.#currentPosition > this.#duration) { + this.#currentPosition = 0; + if (!this.#looping) this.pause(); + } + + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + }, 1000); + } + + pause() { + /* Disconnect the source and update the position to the current time (precisely)*/ + this.#source.stop(); + this.#source.disconnect(); + this.#playing = false; + + const now = Date.now() / 1000; + this.#currentPosition += now - this.#lastUpdateTime; + clearInterval(this.#updateInterval); + + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + } + + getPlaying() { + return this.#playing; + } + + getCurrentPosition() { + return this.#currentPosition; + } + + getDuration() { + return this.#duration; + } + + setCurrentPosition(percentPosition) { + /* To change the current play position we must: + 1) pause the current playback; + 2) update the current position value; + 3) after some time, restart playing. The delay is needed to avoid immediately restarting many times if the user drags the position slider; + */ + if (this.#playing) { + clearTimeout(this.#restartTimeout); + this.#restartTimeout = setTimeout(() => this.play(), 1000); + } + + this.pause(); + this.#currentPosition = (percentPosition / 100) * this.#duration; + } + + setLooping(looping) { + this.#looping = looping; + if (this.#source) this.#source.loop = looping; + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + } + + getLooping() { + return this.#looping; + } +} diff --git a/frontend/react/src/audio/microphonesource.ts b/frontend/react/src/audio/microphonesource.ts new file mode 100644 index 00000000..f744c804 --- /dev/null +++ b/frontend/react/src/audio/microphonesource.ts @@ -0,0 +1,25 @@ +import { getApp } from "../olympusapp"; +import { AudioSource } from "./audiosource"; + +export class MicrophoneSource extends AudioSource { + #sourceNode: MediaStreamAudioSourceNode; + + constructor() { + super(); + + this.setName("Microphone"); + } + + /* Asynchronously initialize the microphone and connect it to the output node */ + async initialize() { + const microphone = await navigator.mediaDevices.getUserMedia({ audio: true }); + if (getApp().getAudioManager().getAudioContext()) { + this.#sourceNode = getApp().getAudioManager().getAudioContext().createMediaStreamSource(microphone); + this.#sourceNode.connect(this.getOutputNode()); + } + } + + play() { + document.dispatchEvent(new CustomEvent("audioSourcesUpdated")); + } +} diff --git a/frontend/react/src/audio/playbackpipeline.ts b/frontend/react/src/audio/playbackpipeline.ts new file mode 100644 index 00000000..2d0895ea --- /dev/null +++ b/frontend/react/src/audio/playbackpipeline.ts @@ -0,0 +1,56 @@ +import { getApp } from "../olympusapp"; + +export class PlaybackPipeline { + #decoder = new AudioDecoder({ + output: (chunk) => this.#handleDecodedData(chunk), + error: (e) => console.log(e), + }); + #trackGenerator: any; // TODO can we have typings? + #writer: any; + #gainNode: GainNode; + + constructor() { + this.#decoder.configure({ + codec: 'opus', + numberOfChannels: 1, + sampleRate: 16000, + //@ts-ignore // TODO why is this giving an error? + opus: { + frameDuration: 40000, + }, + bitrateMode: "constant", + }); + + //@ts-ignore + this.#trackGenerator = new MediaStreamTrackGenerator({ kind: "audio" }); + this.#writer = this.#trackGenerator.writable.getWriter(); + + const stream = new MediaStream([this.#trackGenerator]); + const mediaStreamSource = getApp().getAudioManager().getAudioContext().createMediaStreamSource(stream); + + /* Connect to the device audio output */ + this.#gainNode = getApp().getAudioManager().getAudioContext().createGain(); + mediaStreamSource.connect(this.#gainNode); + this.#gainNode.connect(getApp().getAudioManager().getAudioContext().destination); + } + + playBuffer(arrayBuffer) { + const init = { + type: "key", + data: arrayBuffer, + timestamp: 0, + duration: 2000000, + transfer: [arrayBuffer], + }; + //@ts-ignore //TODO Typings? + let encodedAudioChunk = new EncodedAudioChunk(init); + + this.#decoder.decode(encodedAudioChunk); + } + + #handleDecodedData(audioData) { + this.#writer.ready.then(() => { + this.#writer.write(audioData); + }) + } +} diff --git a/frontend/react/src/audio/radiosink.ts b/frontend/react/src/audio/radiosink.ts new file mode 100644 index 00000000..d8f8cbff --- /dev/null +++ b/frontend/react/src/audio/radiosink.ts @@ -0,0 +1,120 @@ +import { AudioSink } from "./audiosink"; +import { AudioPacket } from "./audiopacket"; +import { getApp } from "../olympusapp"; + +/* Radio sink, basically implements a simple SRS Client in Olympus. Does not support encryption at this moment */ +export class RadioSink extends AudioSink { + #encoder: AudioEncoder; + #desinationNode: MediaStreamAudioDestinationNode; + #audioTrackProcessor: any; // TODO can we have typings? + #frequency = 251000000; + #modulation = 0; + #ptt = false; + #tuned = false; + #volume = 0.5; + + constructor() { + super(); + + this.#encoder = new AudioEncoder({ + output: (data) => this.handleEncodedData(data), + error: (e) => { + console.log(e); + }, + }); + + this.#encoder.configure({ + codec: "opus", + numberOfChannels: 1, + sampleRate: 16000, + //@ts-ignore // TODO why is it giving error? + opus: { + frameDuration: 40000, + }, + bitrateMode: "constant", + }); + + this.#desinationNode = getApp().getAudioManager().getAudioContext().createMediaStreamDestination(); + this.#desinationNode.channelCount = 1; + + //@ts-ignore + this.#audioTrackProcessor = new MediaStreamTrackProcessor({ + track: this.#desinationNode.stream.getAudioTracks()[0], + }); + this.#audioTrackProcessor.readable.pipeTo( + new WritableStream({ + write: (arrayBuffer) => this.handleRawData(arrayBuffer), + }) + ); + + this.getInputNode().connect(this.#desinationNode); + } + + setFrequency(frequency) { + this.#frequency = frequency; + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getFrequency() { + return this.#frequency; + } + + setModulation(modulation) { + this.#modulation = modulation; + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getModulation() { + return this.#modulation; + } + + setPtt(ptt) { + this.#ptt = ptt; + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getPtt() { + return this.#ptt; + } + + setTuned(tuned) { + this.#tuned = tuned; + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getTuned() { + return this.#tuned; + } + + setVolume(volume) { + this.#volume = volume; + document.dispatchEvent(new CustomEvent("audioSinksUpdated")); + } + + getVolume() { + return this.#volume; + } + + handleEncodedData(encodedAudioChunk: EncodedAudioChunk) { + let arrayBuffer = new ArrayBuffer(encodedAudioChunk.byteLength); + encodedAudioChunk.copyTo(arrayBuffer); + + if (this.#ptt) { + let audioPacket = new AudioPacket(); + audioPacket.setAudioData(new Uint8Array(arrayBuffer)); + audioPacket.setFrequencies([{ + frequency: this.#frequency, + modulation: this.#modulation, + encryption: 0 + }]) + audioPacket.setClientGUID(getApp().getAudioManager().getGuid()); + audioPacket.setTransmissionGUID(getApp().getAudioManager().getGuid()); + getApp().getAudioManager().send(audioPacket.toByteArray()); + } + } + + handleRawData(audioData) { + this.#encoder.encode(audioData); + audioData.close(); + } +} diff --git a/frontend/react/src/audio/unitsink.ts b/frontend/react/src/audio/unitsink.ts new file mode 100644 index 00000000..3799f4f4 --- /dev/null +++ b/frontend/react/src/audio/unitsink.ts @@ -0,0 +1,32 @@ +import { AudioSink } from "./audiosink"; +import { getApp } from "../olympusapp"; +import { Unit } from "../unit/unit"; +import { AudioUnitPipeline } from "./audiounitpipeline"; + +/* Unit sink to implement a "loudspeaker" external sound. Useful for stuff like 5MC calls, air sirens, +scramble calls and so on. Ideally, one may want to move this code to the backend*/ +export class UnitSink extends AudioSink { + #unit: Unit; + #unitPipelines: {[key: string]: AudioUnitPipeline} = {}; + + constructor(sourceUnit: Unit) { + super(); + + this.#unit = sourceUnit; + this.setName(`${sourceUnit.getUnitName()} - ${sourceUnit.getName()}`); + + /* TODO as of now, any client connecting after the sink was created will not receive the sound. Add ability to add new pipelines */ + getApp() + .getAudioManager() + .getSRSClientsUnitIDs() + .forEach((unitID) => { + if (unitID !== 0) { + this.#unitPipelines[unitID] = new AudioUnitPipeline(sourceUnit, unitID, this.getInputNode()); + } + }); + } + + getUnit() { + return this.#unit; + } +} diff --git a/frontend/react/src/constants/constants.ts b/frontend/react/src/constants/constants.ts index 111d961e..9948ea2d 100644 --- a/frontend/react/src/constants/constants.ts +++ b/frontend/react/src/constants/constants.ts @@ -356,3 +356,8 @@ export const GROUPING_ZOOM_TRANSITION = 13; export const MAX_SHOTS_SCATTER = 3; export const MAX_SHOTS_INTENSITY = 3; export const SHOTS_SCATTER_DEGREES = 10; + +export enum AudioMessageType { + audio, + settings, +} diff --git a/frontend/react/src/dom.d.ts b/frontend/react/src/dom.d.ts index 68f9e88e..a33116ed 100644 --- a/frontend/react/src/dom.d.ts +++ b/frontend/react/src/dom.d.ts @@ -26,6 +26,9 @@ interface CustomEventMap { hideMapContextMenu: CustomEvent; showUnitContextMenu: CustomEvent; hideUnitContextMenu: CustomEvent; + audioSourcesUpdated: CustomEvent; + audioSinksUpdated: CustomEvent; + audioManagerStateChanged: CustomEvent; } declare global { diff --git a/frontend/react/src/eventscontext.tsx b/frontend/react/src/eventscontext.tsx index fdf399d8..d57951db 100644 --- a/frontend/react/src/eventscontext.tsx +++ b/frontend/react/src/eventscontext.tsx @@ -8,6 +8,8 @@ export const EventsContext = createContext({ setDrawingMenuVisible: (e: boolean) => {}, setOptionsMenuVisible: (e: boolean) => {}, setAirbaseMenuVisible: (e: boolean) => {}, + setRadioMenuVisible: (e: boolean) => {}, + setAudioMenuVisible: (e: boolean) => {}, toggleMainMenuVisible: () => {}, toggleSpawnMenuVisible: () => {}, toggleUnitControlMenuVisible: () => {}, @@ -15,6 +17,8 @@ export const EventsContext = createContext({ toggleDrawingMenuVisible: () => {}, toggleOptionsMenuVisible: () => {}, toggleAirbaseMenuVisible: () => {}, + toggleRadioMenuVisible: () => {}, + toggleAudioMenuVisible: () => {}, }); export const EventsProvider = EventsContext.Provider; diff --git a/frontend/react/src/index.css b/frontend/react/src/index.css index 4f66ad7a..0367ad24 100644 --- a/frontend/react/src/index.css +++ b/frontend/react/src/index.css @@ -40,4 +40,8 @@ z-index: 2006; } +.vertical-slider { + writing-mode: vertical-lr !important; + direction: rtl !important; +} diff --git a/frontend/react/src/interfaces.ts b/frontend/react/src/interfaces.ts index 6c8c4904..07f139f4 100644 --- a/frontend/react/src/interfaces.ts +++ b/frontend/react/src/interfaces.ts @@ -1,5 +1,6 @@ import { LatLng } from "leaflet"; import { Coalition, Context } from "./types/types"; +import { AudioSink } from "./audio/audiosink"; class Airbase {} @@ -291,3 +292,6 @@ export interface ServerStatus { connected: boolean; paused: boolean; } + + + diff --git a/frontend/react/src/olympusapp.ts b/frontend/react/src/olympusapp.ts index 5e685cba..43e3b7f9 100644 --- a/frontend/react/src/olympusapp.ts +++ b/frontend/react/src/olympusapp.ts @@ -25,10 +25,11 @@ import { helicopterDatabase } from "./unit/databases/helicopterdatabase"; import { groundUnitDatabase } from "./unit/databases/groundunitdatabase"; import { navyUnitDatabase } from "./unit/databases/navyunitdatabase"; import { Coalition, Context } from "./types/types"; +import { AudioManager } from "./audio/audiomanager"; export var VERSION = "{{OLYMPUS_VERSION_NUMBER}}"; export var IP = window.location.toString(); -export var connectedToServer = true; // Temporary +export var connectedToServer = true; // TODO Temporary export class OlympusApp { /* Global data */ @@ -45,6 +46,7 @@ export class OlympusApp { #shortcutManager: ShortcutManager | null = null; #unitsManager: UnitsManager | null = null; #weaponsManager: WeaponsManager | null = null; + #audioManager: AudioManager | null = null; //#pluginsManager: // TODO /* Current context */ @@ -79,6 +81,10 @@ export class OlympusApp { getMissionManager() { return this.#missionManager as MissionManager; } + + getAudioManager() { + return this.#audioManager as AudioManager; + } /* TODO getPluginsManager() { @@ -151,9 +157,11 @@ export class OlympusApp { this.#shortcutManager = new ShortcutManager(); this.#unitsManager = new UnitsManager(); this.#weaponsManager = new WeaponsManager(); + this.#audioManager = new AudioManager(); /* Set the address of the server */ this.getServerManager().setAddress(window.location.href.split("?")[0].replace("vite/", "")); + this.getAudioManager().setAddress(window.location.href.split("?")[0].replace("vite/", "")); /* Setup all global events */ this.#setupEvents(); diff --git a/frontend/react/src/other/utils.ts b/frontend/react/src/other/utils.ts index 818d5c38..9359b6e2 100644 --- a/frontend/react/src/other/utils.ts +++ b/frontend/react/src/other/utils.ts @@ -532,3 +532,47 @@ export function getUnitsByLabel(filterString: string) { return [filteredAircraft, filteredHelicopters, filteredAirDefense, filteredGroundUnits, filteredNavyUnits]; } + +export function makeID(length) { + let result = ""; + const characters = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + const charactersLength = characters.length; + let counter = 0; + while (counter < length) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)); + counter += 1; + } + return result; +} + +export function byteArrayToInteger(array) { + let res = 0; + for (let i = 0; i < array.length; i++) { + res = res << 8; + res += array[array.length - i - 1]; + } + return res; +} + +export function integerToByteArray(value, length) { + let res: number[] = []; + for (let i = 0; i < length; i++) { + res.push(value & 255); + value = value >> 8; + } + return res; +} + +export function doubleToByteArray(number) { + var buffer = new ArrayBuffer(8); // JS numbers are 8 bytes long, or 64 bits + var longNum = new Float64Array(buffer); // so equivalent to Float64 + + longNum[0] = number; + + return Array.from(new Uint8Array(buffer)); +} + +export function byteArrayToDouble(array) { + return new DataView(array.reverse().buffer).getFloat64(0); +} diff --git a/frontend/react/src/radio/microphonehandler.ts b/frontend/react/src/radio/microphonehandler.ts new file mode 100644 index 00000000..90f6a68d --- /dev/null +++ b/frontend/react/src/radio/microphonehandler.ts @@ -0,0 +1,34 @@ +import { SRSRadioSetting } from "../interfaces"; +import { AudioPacket } from "./audiopacket"; +import { CapturePipeline } from "./capturepipeline"; + +export class MicrophoneHandler { + #socket: WebSocket; + #setting: SRSRadioSetting; + + constructor(socket, setting) { + this.#socket = socket; + this.#setting = setting; + + console.log("Starting microphone handler"); + + const pipeline = new CapturePipeline(); + + navigator.mediaDevices.enumerateDevices() + .then(function(devices) { + devices.forEach(function(device) { + console.log(device.kind + ": " + device.label + + " id = " + device.deviceId); + }); + }) + + pipeline.connect().then(() => { + pipeline.onencoded = (data) => { + let buffer = new ArrayBuffer(data.byteLength); + data.copyTo(buffer); + let packet = new AudioPacket(new Uint8Array(buffer), this.#setting); + this.#socket.send(packet.getArray()); + } + }) + } +} diff --git a/frontend/react/src/statecontext.tsx b/frontend/react/src/statecontext.tsx index 5c2d9577..9ba4b8ca 100644 --- a/frontend/react/src/statecontext.tsx +++ b/frontend/react/src/statecontext.tsx @@ -9,6 +9,8 @@ export const StateContext = createContext({ drawingMenuVisible: false, optionsMenuVisible: false, airbaseMenuVisible: false, + radioMenuVisible: false, + audioMenuVisible: false, mapHiddenTypes: MAP_HIDDEN_TYPES_DEFAULTS, mapOptions: MAP_OPTIONS_DEFAULTS, mapSources: [] as string[], diff --git a/frontend/react/src/ui/components/olfrequencyinput.tsx b/frontend/react/src/ui/components/olfrequencyinput.tsx new file mode 100644 index 00000000..eaf4e9a3 --- /dev/null +++ b/frontend/react/src/ui/components/olfrequencyinput.tsx @@ -0,0 +1,56 @@ +import React from "react"; +import { ChangeEvent } from "react"; +import { OlNumberInput } from "./olnumberinput"; + +export function OlFrequencyInput(props: { value: number; className?: string; onChange: (value: number) => void }) { + let frequency = props.value; + + return ( +
+ { + let newValue = Math.max(Math.min(Number(e.target.value), 400), 1) * 1000000; + let decimalPart = frequency - Math.floor(frequency / 1000000) * 1000000; + frequency = newValue + decimalPart; + props.onChange(frequency); + }} + onDecrease={() => { + frequency = Math.max(Math.min(Number(frequency - 1000000), 400000000), 1000000); + props.onChange(frequency); + }} + onIncrease={() => { + frequency = Math.max(Math.min(Number(frequency + 1000000), 400000000), 1000000); + props.onChange(frequency); + }} + value={Math.floor(frequency / 1000000)} + > +
.
+ { + let newValue = Math.max(Math.min(Number(e.target.value), 990), 0) * 1000; + let integerPart = Math.floor(frequency / 1000000) * 1000000; + frequency = newValue + integerPart; + props.onChange(frequency); + }} + onDecrease={() => { + frequency = Math.max(Math.min(Number(frequency - 25000), 400000000), 1000000); + props.onChange(frequency); + }} + onIncrease={() => { + frequency = Math.max(Math.min(Number(frequency + 25000), 400000000), 1000000); + props.onChange(frequency); + }} + value={(frequency - Math.floor(frequency / 1000000) * 1000000) / 1000} + > +
MHz
+
+ ); +} diff --git a/frontend/react/src/ui/components/olrangeslider.tsx b/frontend/react/src/ui/components/olrangeslider.tsx index 59f80b95..8412bc11 100644 --- a/frontend/react/src/ui/components/olrangeslider.tsx +++ b/frontend/react/src/ui/components/olrangeslider.tsx @@ -5,6 +5,8 @@ export function OlRangeSlider(props: { min?: number; max?: number; step?: number; + className?: string; + vertical?: boolean; onChange: (e: ChangeEvent) => void; }) { var elementRef = useRef(null); @@ -28,9 +30,11 @@ export function OlRangeSlider(props: { max={props.max ?? 100} step={props.step ?? 1} className={` + ${props.className} h-2 w-full cursor-pointer appearance-none rounded-lg bg-gray-200 dark:bg-gray-700 `} + /> ); } diff --git a/frontend/react/src/ui/modals/components/card.tsx b/frontend/react/src/ui/modals/components/card.tsx index 3e4f7623..e2d2ab57 100644 --- a/frontend/react/src/ui/modals/components/card.tsx +++ b/frontend/react/src/ui/modals/components/card.tsx @@ -1,6 +1,6 @@ import React from "react"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { faArrowRight, faCheckCircle, faExternalLink, faLink, faUnlink } from "@fortawesome/free-solid-svg-icons"; +import { faArrowRight } from "@fortawesome/free-solid-svg-icons"; export function Card(props: { children?: JSX.Element | JSX.Element[]; className?: string }) { return ( diff --git a/frontend/react/src/ui/panels/audiomenu.tsx b/frontend/react/src/ui/panels/audiomenu.tsx new file mode 100644 index 00000000..fff5a3ca --- /dev/null +++ b/frontend/react/src/ui/panels/audiomenu.tsx @@ -0,0 +1,107 @@ +import React, { useEffect, useState } from "react"; +import { Menu } from "./components/menu"; +import { getApp } from "../../olympusapp"; +import { FaQuestionCircle } from "react-icons/fa"; +import { AudioSourcePanel } from "./components/sourcepanel"; +import { AudioSource } from "../../audio/audiosource"; +import { FaVolumeHigh } from "react-icons/fa6"; + +export function AudioMenu(props: { open: boolean; onClose: () => void; children?: JSX.Element | JSX.Element[] }) { + const [sources, setSources] = useState([] as AudioSource[]); + const [audioManagerEnabled, setAudioManagerEnabled] = useState(false); + + useEffect(() => { + /* Force a rerender */ + document.addEventListener("audioSourcesUpdated", () => { + setSources( + getApp() + ?.getAudioManager() + .getSources() + .map((source) => source) + ); + }); + + document.addEventListener("audioManagerStateChanged", () => { + setAudioManagerEnabled(getApp().getAudioManager().isRunning()); + }); + }, []); + + return ( + +
The audio source panel allows you to add and manage audio sources.
+
+ {audioManagerEnabled && ( + <> +
+ +
+
+
Use the controls to apply effects and start/stop the playback of an audio source.
+
Sources can be connected to your radios, or attached to a unit to be played on loudspeakers.
+
+ + )} + {!audioManagerEnabled && ( + <> +
+ +
+
+
+ To enable the audio menu, first start the audio backend with the{" "} + + + {" "} + button on the navigation header. +
+
+ + )} +
+ +
+ <> + {sources.map((source) => { + return ; + })} + + {audioManagerEnabled && ( + + )} +
+
+ ); +} diff --git a/frontend/react/src/ui/panels/components/radiopanel.tsx b/frontend/react/src/ui/panels/components/radiopanel.tsx new file mode 100644 index 00000000..3209e686 --- /dev/null +++ b/frontend/react/src/ui/panels/components/radiopanel.tsx @@ -0,0 +1,61 @@ +import React, { useEffect, useState } from "react"; +import { OlFrequencyInput } from "../../components/olfrequencyinput"; +import { FaTrash } from "react-icons/fa6"; +import { OlLabelToggle } from "../../components/ollabeltoggle"; +import { OlStateButton } from "../../components/olstatebutton"; +import { faEarListen, faMicrophoneLines } from "@fortawesome/free-solid-svg-icons"; +import { RadioSink } from "../../../audio/radiosink"; +import { getApp } from "../../../olympusapp"; + +export function RadioPanel(props: { radio: RadioSink }) { + return ( +
+
+ {props.radio.getName()} +
{getApp().getAudioManager().removeSink(props.radio);}}> + +
+
+ { + props.radio.setFrequency(value) + }} + /> +
+ { + props.radio.setModulation(props.radio.getModulation() === 1 ? 0 : 1); + }} + > + + { + props.radio.setPtt(!props.radio.getPtt()); + }} + tooltip="Talk on frequency" + > + + { + props.radio.setTuned(!props.radio.getTuned()); + }} + tooltip="Tune to radio" + > +
+
+ ); +} diff --git a/frontend/react/src/ui/panels/components/sourcepanel.tsx b/frontend/react/src/ui/panels/components/sourcepanel.tsx new file mode 100644 index 00000000..de5e818e --- /dev/null +++ b/frontend/react/src/ui/panels/components/sourcepanel.tsx @@ -0,0 +1,141 @@ +import React, { useEffect, useState } from "react"; +import { OlStateButton } from "../../components/olstatebutton"; +import { faPause, faPlay, faRepeat, faStop } from "@fortawesome/free-solid-svg-icons"; +import { getApp } from "../../../olympusapp"; +import { AudioSource } from "../../../audio/audiosource"; +import { FaArrowRight, FaTrash, FaVolumeHigh } from "react-icons/fa6"; +import { OlRangeSlider } from "../../components/olrangeslider"; +import { FaUnlink } from "react-icons/fa"; +import { OlDropdown, OlDropdownItem } from "../../components/oldropdown"; +import { FileSource } from "../../../audio/filesource"; +import { MicrophoneSource } from "../../../audio/microphonesource"; + +export function AudioSourcePanel(props: { source: AudioSource }) { + const [meterLevel, setMeterLevel] = useState(0); + + useEffect(() => { + setInterval(() => { + setMeterLevel(props.source.getMeter().getPeaks().current[0]); + }, 50); + }, []); + + let availabileSinks = getApp() + .getAudioManager() + .getSinks() + .filter((sink) => !props.source.getConnectedTo().includes(sink)); + + return ( +
+
+ {props.source.getName()} + {!(props.source instanceof MicrophoneSource) && ( +
{ + getApp().getAudioManager().removeSource(props.source); + }} + > + +
+ )} +
+ +
+ {props.source instanceof FileSource && ( +
+ { + if (props.source instanceof FileSource) props.source.getPlaying() ? props.source.stop() : props.source.play(); + }} + tooltip="Play file" + > + 0 ? (props.source.getCurrentPosition() / props.source.getDuration()) * 100 : 0} + onChange={(ev) => { + if (props.source instanceof FileSource) props.source.setCurrentPosition(parseFloat(ev.currentTarget.value)); + }} + className="my-auto" + /> + { + if (props.source instanceof FileSource) props.source.setLooping(!props.source.getLooping()); + }} + tooltip="Loop" + > +
+ )} +
+
+ +
+
+
+
+
+ { + props.source.setVolume(parseFloat(ev.currentTarget.value) / 100); + }} + className="absolute top-[18px]" + /> +
+
+ {Math.round(props.source.getVolume() * 100)} +
+
+
+ + Connected to: +
+ {props.source.getConnectedTo().map((sink) => { + return ( +
+ + {sink.getName()} + props.source.disconnect(sink)}> +
+ ); + })} +
+ {availabileSinks.length > 0 && ( + + {availabileSinks.map((sink) => { + return ( + { + props.source.connect(sink); + }} + > + {sink.getName()} + + ); + })} + + )} +
+ ); +} diff --git a/frontend/react/src/ui/panels/header.tsx b/frontend/react/src/ui/panels/header.tsx index 4a10d6ad..561725da 100644 --- a/frontend/react/src/ui/panels/header.tsx +++ b/frontend/react/src/ui/panels/header.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useRef, useState } from "react"; import { OlRoundStateButton, OlStateButton, OlLockStateButton } from "../components/olstatebutton"; -import { faSkull, faCamera, faFlag, faLink, faUnlink, faBars } from "@fortawesome/free-solid-svg-icons"; +import { faSkull, faCamera, faFlag, faLink, faUnlink, faBars, faVolumeHigh } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { EventsConsumer } from "../../eventscontext"; import { StateConsumer } from "../../statecontext"; @@ -23,12 +23,12 @@ import { FaChevronLeft, FaChevronRight } from "react-icons/fa6"; export function Header() { const [scrolledLeft, setScrolledLeft] = useState(true); const [scrolledRight, setScrolledRight] = useState(false); + const [audioEnabled, setAudioEnabled] = useState(false); /* Initialize the "scroll" position of the element */ var scrollRef = useRef(null); useEffect(() => { - if (scrollRef.current) - onScroll(scrollRef.current); + if (scrollRef.current) onScroll(scrollRef.current); }); function onScroll(el) { @@ -54,10 +54,9 @@ export function Header() { dark:border-gray-800 dark:bg-olympus-900 `} > - + {!scrolledLeft && ( -
+
{}} tooltip="Lock/unlock protected units (from scripted mission)" /> + { + audioEnabled ? getApp().getAudioManager().stop() : getApp().getAudioManager().start(); + setAudioEnabled(!audioEnabled); + }} + tooltip="Enable/disable audio and radio backend" + icon={faVolumeHigh} + />
void; options: MapOptions; children?: JSX.Element | JSX.Element[] }) { +export function OptionsMenu(props: { open: boolean; onClose: () => void; options: MapOptions; children?: JSX.Element | JSX.Element[] }) { return (
void; children?: JSX.Element | JSX.Element[] }) { + const [radios, setRadios] = useState([] as RadioSink[]); + const [audioManagerEnabled, setAudioManagerEnabled] = useState(false); + + useEffect(() => { + /* Force a rerender */ + document.addEventListener("audioSinksUpdated", () => { + setRadios( + getApp() + ?.getAudioManager() + .getSinks() + .filter((sink) => sink instanceof RadioSink) + .map((radio) => radio) + ); + }); + + document.addEventListener("audioManagerStateChanged", () => { + setAudioManagerEnabled(getApp().getAudioManager().isRunning()); + }); + }, []); + + return ( + +
The radio menu allows you to talk on radio to the players online using SRS.
+
+ {audioManagerEnabled && ( + <> +
+ +
+
+
Use the radio controls to tune to a frequency, then click on the PTT button to talk.
+
You can add up to 10 radios. Use the audio effects menu to play audio tracks or to add background noises.
+
+ + )} + {!audioManagerEnabled && ( + <> +
+ +
+
+
+ To enable the radio menu, first start the audio backend with the{" "} + + + {" "} + button on the navigation header. +
+
+ + )} +
+ +
+ {radios.map((radio) => { + return ; + })} + {audioManagerEnabled && radios.length < 10 && ( + + )} +
+
+ ); +} diff --git a/frontend/react/src/ui/panels/sidebar.tsx b/frontend/react/src/ui/panels/sidebar.tsx index a4642cf7..b5edf8a7 100644 --- a/frontend/react/src/ui/panels/sidebar.tsx +++ b/frontend/react/src/ui/panels/sidebar.tsx @@ -1,9 +1,10 @@ import React, { useState } from "react"; import { OlStateButton } from "../components/olstatebutton"; -import { faGamepad, faRuler, faPencil, faEllipsisV, faCog, faQuestionCircle, faPlusSquare, faMagnifyingGlass, faPlaneDeparture } from "@fortawesome/free-solid-svg-icons"; +import { faGamepad, faRuler, faPencil, faEllipsisV, faCog, faQuestionCircle, faPlusSquare, faMagnifyingGlass, faPlaneDeparture, faRadio, faVolumeHigh } from "@fortawesome/free-solid-svg-icons"; import { EventsConsumer } from "../../eventscontext"; import { StateConsumer } from "../../statecontext"; import { IDLE } from "../../constants/constants"; +import { faSpeakerDeck } from "@fortawesome/free-brands-svg-icons"; export function SideBar() { return ( @@ -58,6 +59,18 @@ export function SideBar() { icon={faPlaneDeparture} tooltip="Hide/show airbase menu" > + +
diff --git a/frontend/react/src/ui/panels/unitcontrolmenu.tsx b/frontend/react/src/ui/panels/unitcontrolmenu.tsx index 395d06c6..0b3bc280 100644 --- a/frontend/react/src/ui/panels/unitcontrolmenu.tsx +++ b/frontend/react/src/ui/panels/unitcontrolmenu.tsx @@ -48,6 +48,7 @@ import { FaRadio } from "react-icons/fa6"; import { OlNumberInput } from "../components/olnumberinput"; import { Radio, TACAN } from "../../interfaces"; import { OlStringInput } from "../components/olstringinput"; +import { OlFrequencyInput } from "../components/olfrequencyinput"; export function UnitControlMenu(props: { open: boolean; onClose: () => void }) { const [selectedUnits, setSelectedUnits] = useState([] as Unit[]); @@ -1112,71 +1113,12 @@ export function UnitControlMenu(props: { open: boolean; onClose: () => void }) {
Radio frequency
- { - let newValue = Math.max(Math.min(Number(e.target.value), 400), 1) * 1000000; - if (activeAdvancedSettings) { - let decimalPart = activeAdvancedSettings.radio.frequency - Math.floor(activeAdvancedSettings.radio.frequency / 1000000) * 1000000; - activeAdvancedSettings.radio.frequency = newValue + decimalPart; - } + { + if (activeAdvancedSettings) { + activeAdvancedSettings.radio.frequency = value; setActiveAdvancedSettings(JSON.parse(JSON.stringify(activeAdvancedSettings))); - }} - onDecrease={() => { - if (activeAdvancedSettings) - activeAdvancedSettings.radio.frequency = Math.max( - Math.min(Number(activeAdvancedSettings.radio.frequency - 1000000), 400000000), - 1000000 - ); - setActiveAdvancedSettings(JSON.parse(JSON.stringify(activeAdvancedSettings))); - }} - onIncrease={() => { - if (activeAdvancedSettings) - activeAdvancedSettings.radio.frequency = Math.max( - Math.min(Number(activeAdvancedSettings.radio.frequency + 1000000), 400000000), - 1000000 - ); - setActiveAdvancedSettings(JSON.parse(JSON.stringify(activeAdvancedSettings))); - }} - value={activeAdvancedSettings ? Math.floor(activeAdvancedSettings.radio.frequency / 1000000) : 124} - > -
.
- { - let newValue = Math.max(Math.min(Number(e.target.value), 990), 0) * 1000; - if (activeAdvancedSettings) { - let integerPart = Math.floor(activeAdvancedSettings.radio.frequency / 1000000) * 1000000; - activeAdvancedSettings.radio.frequency = newValue + integerPart; - } - setActiveAdvancedSettings(JSON.parse(JSON.stringify(activeAdvancedSettings))); - }} - onDecrease={() => { - if (activeAdvancedSettings) - activeAdvancedSettings.radio.frequency = Math.max( - Math.min(Number(activeAdvancedSettings.radio.frequency - 25000), 400000000), - 1000000 - ); - setActiveAdvancedSettings(JSON.parse(JSON.stringify(activeAdvancedSettings))); - }} - onIncrease={() => { - if (activeAdvancedSettings) - activeAdvancedSettings.radio.frequency = Math.max( - Math.min(Number(activeAdvancedSettings.radio.frequency + 25000), 400000000), - 1000000 - ); - setActiveAdvancedSettings(JSON.parse(JSON.stringify(activeAdvancedSettings))); - }} - value={ - activeAdvancedSettings - ? (activeAdvancedSettings.radio.frequency - Math.floor(activeAdvancedSettings.radio.frequency / 1000000) * 1000000) / 1000 - : 0 } - > -
MHz
+ }}/>
diff --git a/frontend/react/src/ui/ui.tsx b/frontend/react/src/ui/ui.tsx index df013bc2..b3960178 100644 --- a/frontend/react/src/ui/ui.tsx +++ b/frontend/react/src/ui/ui.tsx @@ -9,7 +9,7 @@ import { SpawnMenu } from "./panels/spawnmenu"; import { UnitControlMenu } from "./panels/unitcontrolmenu"; import { MainMenu } from "./panels/mainmenu"; import { SideBar } from "./panels/sidebar"; -import { Options } from "./panels/options"; +import { OptionsMenu } from "./panels/optionsmenu"; import { MapHiddenTypes, MapOptions } from "../types/types"; import { BLUE_COMMANDER, CONTEXT_ACTION, GAME_MASTER, IDLE, MAP_HIDDEN_TYPES_DEFAULTS, MAP_OPTIONS_DEFAULTS, RED_COMMANDER } from "../constants/constants"; import { getApp, setupApp } from "../olympusapp"; @@ -22,6 +22,8 @@ import { ControlsPanel } from "./panels/controlspanel"; import { MapContextMenu } from "./contextmenus/mapcontextmenu"; import { AirbaseMenu } from "./panels/airbasemenu"; import { Airbase } from "../mission/airbase"; +import { RadioMenu } from "./panels/radiomenu"; +import { AudioMenu } from "./panels/audiomenu"; export type OlympusUIState = { mainMenuVisible: boolean; @@ -42,6 +44,8 @@ export function UI() { const [unitControlMenuVisible, setUnitControlMenuVisible] = useState(false); const [measureMenuVisible, setMeasureMenuVisible] = useState(false); const [drawingMenuVisible, setDrawingMenuVisible] = useState(false); + const [radioMenuVisible, setRadioMenuVisible] = useState(false); + const [audioMenuVisible, setAudioMenuVisible] = useState(false); const [optionsMenuVisible, setOptionsMenuVisible] = useState(false); const [airbaseMenuVisible, setAirbaseMenuVisible] = useState(false); const [mapHiddenTypes, setMapHiddenTypes] = useState(MAP_HIDDEN_TYPES_DEFAULTS); @@ -97,6 +101,8 @@ export function UI() { setDrawingMenuVisible(false); setOptionsMenuVisible(false); setAirbaseMenuVisible(false); + setRadioMenuVisible(false); + setAudioMenuVisible(false); } function checkPassword(password: string) { @@ -128,14 +134,6 @@ export function UI() { setLoginModalVisible(false); } - /* Temporary during devel */ - //useEffect(() => { - // window.setTimeout(() => { - // checkPassword("admin"); - // connect("devel"); - // }, 1000) - //}, []) - return (
{ hideAllMenus(); setMainMenuVisible(!mainMenuVisible); @@ -197,6 +199,14 @@ export function UI() { hideAllMenus(); setAirbaseMenuVisible(!airbaseMenuVisible); }, + toggleRadioMenuVisible: () => { + hideAllMenus(); + setRadioMenuVisible(!radioMenuVisible); + }, + toggleAudioMenuVisible: () => { + hideAllMenus(); + setAudioMenuVisible(!audioMenuVisible); + }, }} >
@@ -227,10 +237,12 @@ export function UI() {
setMainMenuVisible(false)} /> setSpawnMenuVisible(false)} /> - setOptionsMenuVisible(false)} options={mapOptions} /> + setOptionsMenuVisible(false)} options={mapOptions} /> setUnitControlMenuVisible(false)} /> setDrawingMenuVisible(false)} /> setAirbaseMenuVisible(false)} airbase={airbase}/> + setRadioMenuVisible(false)} /> + setAudioMenuVisible(false)} /> diff --git a/frontend/react/src/unit/unit.ts b/frontend/react/src/unit/unit.ts index cd9b7ee0..5e16d610 100644 --- a/frontend/react/src/unit/unit.ts +++ b/frontend/react/src/unit/unit.ts @@ -74,6 +74,7 @@ import { faPeopleGroup, faQuestionCircle, faRoute, + faVolumeHigh, faXmarksLines, } from "@fortawesome/free-solid-svg-icons"; import { FaXmarksLines } from "react-icons/fa6"; @@ -849,6 +850,19 @@ export abstract class Unit extends CustomMarker { if (targetPosition) getApp().getUnitsManager().addDestination(targetPosition, false, 0, units); } ); + + contextActionSet.addContextAction( + this, + "speaker", + "Make audio source", + "Make this unit an audio source (loudspeakers)", + faVolumeHigh, + null, + (units: Unit[], _1, _2) => { + units.forEach((unit) => getApp().getAudioManager().addUnitSink(unit)); + }, + { executeImmediately: true } + ); contextActionSet.addDefaultContextAction( this, diff --git a/frontend/server/Example.ogg b/frontend/server/Example.ogg new file mode 100644 index 00000000..39f957bf Binary files /dev/null and b/frontend/server/Example.ogg differ diff --git a/frontend/server/Richard_Wagner_-_The_Valkyrie_-_Ride_of_the_Valkyries.ogg b/frontend/server/Richard_Wagner_-_The_Valkyrie_-_Ride_of_the_Valkyries.ogg new file mode 100644 index 00000000..fb13b008 Binary files /dev/null and b/frontend/server/Richard_Wagner_-_The_Valkyrie_-_Ride_of_the_Valkyries.ogg differ diff --git a/frontend/server/package.json b/frontend/server/package.json index 2bda645b..0d981117 100644 --- a/frontend/server/package.json +++ b/frontend/server/package.json @@ -10,6 +10,7 @@ }, "private": true, "dependencies": { + "@discordjs/opus": "^0.9.0", "appjs": "^0.0.20", "appjs-win32": "^0.0.19", "body-parser": "^1.20.2", @@ -28,10 +29,13 @@ "srtm-elevation": "^2.1.2", "tcp-ping-port": "^1.0.1", "uuid": "^9.0.1", + "wavefile": "^11.0.0", + "web-audio-api": "^0.2.2", "ws": "^8.18.0", "yargs": "^17.7.2" }, "devDependencies": { + "opus-decoder": "^0.7.6", "ts-node": "^10.9.2", "typescript": "^5.5.3" } diff --git a/frontend/server/sample1.WAV b/frontend/server/sample1.WAV new file mode 100644 index 00000000..941c804f Binary files /dev/null and b/frontend/server/sample1.WAV differ diff --git a/frontend/server/sample3.WAV b/frontend/server/sample3.WAV new file mode 100644 index 00000000..084bad48 Binary files /dev/null and b/frontend/server/sample3.WAV differ diff --git a/frontend/server/src/app.ts b/frontend/server/src/app.ts index e2534d67..59384e48 100644 --- a/frontend/server/src/app.ts +++ b/frontend/server/src/app.ts @@ -5,6 +5,7 @@ import logger = require("morgan"); import fs = require("fs"); import bodyParser = require("body-parser"); import cors = require("cors"); +import { AudioBackend } from "./audio/audiobackend"; /* Load the proxy middleware plugin */ import httpProxyMiddleware = require("http-proxy-middleware"); @@ -83,5 +84,10 @@ module.exports = function (configLocation, viteProxy) { }); } + if (config["audio"]) { + let audioBackend = new AudioBackend(config["audio"]["SRSPort"], config["audio"]["WSPort"]); + audioBackend.start(); + } + return app; }; diff --git a/frontend/server/src/audio/audiobackend.ts b/frontend/server/src/audio/audiobackend.ts new file mode 100644 index 00000000..71331aa4 --- /dev/null +++ b/frontend/server/src/audio/audiobackend.ts @@ -0,0 +1,21 @@ +import { WebSocketServer } from "ws"; +import { SRSHandler } from "./srshandler"; + +export class AudioBackend { + SRSPort: number = 5002; + WSPort: number = 4000; + handlers: SRSHandler[] = []; + + constructor(SRSPort, WSPort) { + this.SRSPort = SRSPort ?? this.SRSPort; + this.WSPort = WSPort ?? this.WSPort; + } + + start() { + const wss = new WebSocketServer({ port: this.WSPort }); + + wss.on("connection", (ws) => { + this.handlers.push(new SRSHandler(ws, this.SRSPort)); + }); + } +} diff --git a/frontend/server/src/audio/audiopacket.ts b/frontend/server/src/audio/audiopacket.ts new file mode 100644 index 00000000..4c083b96 --- /dev/null +++ b/frontend/server/src/audio/audiopacket.ts @@ -0,0 +1,210 @@ +// TODO This code is in common with the frontend, would be nice to share it */ +import { byteArrayToDouble, byteArrayToInteger, doubleToByteArray, integerToByteArray } from "../utils"; +import { Buffer } from "buffer"; + +var packetID = 0; + +export enum MessageType { + audio, + settings, + unitIDs +} + +export class AudioPacket { + /* Mandatory data */ + #frequencies: { frequency: number; modulation: number; encryption: number }[] = []; + #audioData: Uint8Array; + #transmissionGUID: string; + #clientGUID: string; + + /* Default data */ + #unitID: number = 0; + #hops: number = 0; + + /* Usually internally set only */ + #packetID: number | null = null; + + fromByteArray(byteArray: Uint8Array) { + let totalLength = byteArrayToInteger(byteArray.slice(0, 2)); + let audioLength = byteArrayToInteger(byteArray.slice(2, 4)); + let frequenciesLength = byteArrayToInteger(byteArray.slice(4, 6)); + + /* Perform some sanity checks */ + if (totalLength !== byteArray.length) { + console.log( + `Warning, audio packet expected length is ${totalLength} but received length is ${byteArray.length}, aborting...` + ); + return; + } + + if (frequenciesLength % 10 !== 0) { + console.log( + `Warning, audio packet frequencies data length is ${frequenciesLength} which is not a multiple of 10, aborting...` + ); + return; + } + + /* Extract the audio data */ + this.#audioData = byteArray.slice(6, 6 + audioLength); + + /* Extract the frequencies */ + let offset = 6 + audioLength; + for (let idx = 0; idx < frequenciesLength / 10; idx++) { + this.#frequencies.push({ + frequency: byteArrayToDouble(byteArray.slice(offset, offset + 8)), + modulation: byteArray[offset + 8], + encryption: byteArray[offset + 9], + }); + offset += 10; + } + + /* If necessary increase the packetID */ + if (this.#packetID === null) this.#packetID = packetID++; + + /* Extract the remaining data */ + this.#unitID = byteArrayToInteger(byteArray.slice(offset, offset + 4)); + offset += 4; + this.#packetID = byteArrayToInteger(byteArray.slice(offset, offset + 8)); + offset += 8; + this.#hops = byteArrayToInteger(byteArray.slice(offset, offset + 1)); + offset += 1; + this.#transmissionGUID = new TextDecoder().decode(byteArray.slice(offset, offset + 22)); + offset += 22; + this.#clientGUID = new TextDecoder().decode(byteArray.slice(offset, offset + 22)); + offset += 22; + } + + toByteArray() { + /* Perform some sanity checks // TODO check correct values */ + if (this.#frequencies.length === 0) { + console.log( + "Warning, could not encode audio packet, no frequencies data provided, aborting..." + ); + return; + } + + if (this.#audioData === undefined) { + console.log( + "Warning, could not encode audio packet, no audio data provided, aborting..." + ); + return; + } + + if (this.#transmissionGUID === undefined) { + console.log( + "Warning, could not encode audio packet, no transmission GUID provided, aborting..." + ); + return; + } + + if (this.#clientGUID === undefined) { + console.log( + "Warning, could not encode audio packet, no client GUID provided, aborting..." + ); + return; + } + + // Prepare the array for the header + let header: number[] = [0, 0, 0, 0, 0, 0]; + + // Encode the frequencies data + let frequenciesData = [] as number[]; + this.#frequencies.forEach((data) => { + frequenciesData = frequenciesData.concat( + [...doubleToByteArray(data.frequency)], + [data.modulation], + [data.encryption] + ); + }); + + // Encode unitID, packetID, hops + let encUnitID: number[] = integerToByteArray(this.#unitID, 4); + let encPacketID: number[] = integerToByteArray(this.#packetID, 8); + let encHops: number[] = [this.#hops]; + + // Assemble packet + let encodedData: number[] = ([] as number[]).concat( + header, + [...this.#audioData], + frequenciesData, + encUnitID, + encPacketID, + encHops, + [...Buffer.from(this.#transmissionGUID, "utf-8")], + [...Buffer.from(this.#clientGUID, "utf-8")] + ); + + // Set the lengths of the parts + let encPacketLen = integerToByteArray(encodedData.length, 2); + encodedData[0] = encPacketLen[0]; + encodedData[1] = encPacketLen[1]; + + let encAudioLen = integerToByteArray(this.#audioData.length, 2); + encodedData[2] = encAudioLen[0]; + encodedData[3] = encAudioLen[1]; + + let frequencyAudioLen = integerToByteArray(frequenciesData.length, 2); + encodedData[4] = frequencyAudioLen[0]; + encodedData[5] = frequencyAudioLen[1]; + + return new Uint8Array([0].concat(encodedData)); + } + + setFrequencies( + frequencies: { frequency: number; modulation: number; encryption: number }[] + ) { + this.#frequencies = frequencies; + } + + getFrequencies() { + return this.#frequencies; + } + + setAudioData(audioData: Uint8Array) { + this.#audioData = audioData; + } + + getAudioData() { + return this.#audioData; + } + + setTransmissionGUID(transmissionGUID: string) { + this.#transmissionGUID = transmissionGUID; + } + + getTransmissionGUID() { + return this.#transmissionGUID; + } + + setClientGUID(clientGUID: string) { + this.#clientGUID = clientGUID; + } + + getClientGUID() { + return this.#clientGUID; + } + + setUnitID(unitID: number) { + this.#unitID = unitID; + } + + getUnitID() { + return this.#unitID; + } + + setPacketID(packetID: number) { + this.#packetID = packetID; + } + + getPacketID() { + return this.#packetID; + } + + setHops(hops: number) { + this.#hops = hops; + } + + getHops() { + return this.#hops; + } +} diff --git a/frontend/server/src/audio/defaultdata.ts b/frontend/server/src/audio/defaultdata.ts new file mode 100644 index 00000000..d97cbaad --- /dev/null +++ b/frontend/server/src/audio/defaultdata.ts @@ -0,0 +1,112 @@ +export var defaultSRSData = { + ClientGuid: "", + Name: "", + Seat: 0, + Coalition: 2, + AllowRecord: false, + RadioInfo: { + radios: [ + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + ], + unit: "", + unitId: 0, + iff: { + control: 2, + mode1: -1, + mode2: -1, + mode3: -1, + mode4: false, + mic: -1, + status: 0, + }, + ambient: { vol: 1.0, abType: "" }, + }, + LatLngPosition: { lat: 0.0, lng: 0.0, alt: 0.0 }, + }; \ No newline at end of file diff --git a/frontend/server/src/audio/srshandler.ts b/frontend/server/src/audio/srshandler.ts new file mode 100644 index 00000000..5d3eed32 --- /dev/null +++ b/frontend/server/src/audio/srshandler.ts @@ -0,0 +1,121 @@ +import { MessageType } from "./audiopacket"; +import { defaultSRSData } from "./defaultdata"; + +/* TCP/IP socket */ +var net = require("net"); +var bufferString = ""; + +const SRS_VERSION = "2.1.0.10"; +var globalIndex = 1; + +export class SRSHandler { + ws: any; + tcp = new net.Socket(); + udp = require("dgram").createSocket("udp4"); + data = JSON.parse(JSON.stringify(defaultSRSData)); + syncInterval: any; + clients = []; + SRSPort = 0; + + constructor(ws, SRSPort) { + this.data.Name = `Olympus${globalIndex}`; + this.SRSPort = SRSPort; + globalIndex += 1; + + /* Websocket */ + this.ws = ws; + this.ws.on("error", console.error); + this.ws.on("message", (data) => { + this.decodeData(data); + }); + this.ws.on("close", () => { + this.tcp.end(); + }); + + /* TCP */ + this.tcp.on("error", (ex) => { + console.log("Could not connect to SRS Server"); + }); + + this.tcp.connect(SRSPort, "localhost", () => { + console.log(`Connected to SRS Server on TCP Port ${SRSPort}`); + + this.syncInterval = setInterval(() => { + let SYNC = { + Client: this.data, + MsgType: 2, + Version: SRS_VERSION, + }; + + this.data.ClientGuid !== "" && + this.udp.send(this.data.ClientGuid, SRSPort, "localhost", (error) => { + if (error) console.log(`Error pinging SRS server on UDP: ${error}`); + }); + + if (this.tcp.readyState == "open") + this.tcp.write(`${JSON.stringify(SYNC)}\n`); + else clearInterval(this.syncInterval); + + let unitsBuffer = Buffer.from( + JSON.stringify({ + unitIDs: this.clients.map((client) => { + return client.RadioInfo.unitId; + }), + }), + "utf-8" + ); + + this.ws.send( + ([] as number[]).concat([MessageType.unitIDs], [...unitsBuffer]) + ); + }, 1000); + }); + + this.tcp.on("data", (data) => { + bufferString += data.toString(); + while (bufferString.includes("\n")) { + try { + let message = JSON.parse(bufferString.split("\n")[0]); + bufferString = bufferString.slice(bufferString.indexOf("\n") + 1); + if (message.Clients !== undefined) this.clients = message.Clients; + } catch (e) { + console.log(e); + } + } + }); + + /* UDP */ + this.udp.on("listening", () => { + console.log(`Listening to SRS Server on UDP port ${SRSPort}`); + }); + + this.udp.on("message", (message, remote) => { + if (this.ws && message.length > 22) + this.ws.send( + ([] as number[]).concat([MessageType.audio], [...message]) + ); + }); + } + + decodeData(data) { + switch (data[0]) { + case MessageType.audio: + const encodedData = new Uint8Array(data.slice(1)); + this.udp.send(encodedData, this.SRSPort, "localhost", (error) => { + if (error) console.log(`Error sending data to SRS server: ${error}`); + }); + break; + case MessageType.settings: + let message = JSON.parse(data.slice(1)); + this.data.ClientGuid = message.guid; + this.data.Coalition = message.coalition; + message.settings.forEach((setting, idx) => { + this.data.RadioInfo.radios[idx].freq = setting.frequency; + this.data.RadioInfo.radios[idx].modulation = setting.modulation; + }); + break; + default: + break; + } + } +} diff --git a/frontend/server/src/routes/resources.ts b/frontend/server/src/routes/resources.ts index d2d12506..728c2fc7 100644 --- a/frontend/server/src/routes/resources.ts +++ b/frontend/server/src/routes/resources.ts @@ -7,7 +7,7 @@ module.exports = function (configLocation) { if (fs.existsSync(configLocation)) { let rawdata = fs.readFileSync(configLocation, "utf-8"); const config = JSON.parse(rawdata); - res.send(JSON.stringify(config.frontend)); + res.send(JSON.stringify({...config.frontend, ...(config.audio ?? {}) })); res.end() } else { res.sendStatus(404); diff --git a/frontend/server/src/utils.ts b/frontend/server/src/utils.ts new file mode 100644 index 00000000..da095138 --- /dev/null +++ b/frontend/server/src/utils.ts @@ -0,0 +1,30 @@ +export function byteArrayToInteger(array) { + let res = 0; + for (let i = 0; i < array.length; i++) { + res = res << 8; + res += array[array.length - i - 1]; + } + return res; +} + +export function integerToByteArray(value, length) { + let res: number[] = []; + for (let i = 0; i < length; i++) { + res.push(value & 255); + value = value >> 8; + } + return res; +} + +export function doubleToByteArray(number) { + var buffer = new ArrayBuffer(8); // JS numbers are 8 bytes long, or 64 bits + var longNum = new Float64Array(buffer); // so equivalent to Float64 + + longNum[0] = number; + + return Array.from(new Uint8Array(buffer)); +} + +export function byteArrayToDouble(array) { + return new DataView(array.reverse().buffer).getFloat64(0); +} diff --git a/frontend/server/srs.js b/frontend/server/srs.js new file mode 100644 index 00000000..ef9e355e --- /dev/null +++ b/frontend/server/srs.js @@ -0,0 +1,219 @@ +const WaveFile = require('wavefile').WaveFile; + +var fs = require('fs'); +let source = fs.readFileSync('sample3.WAV'); +let wav = new WaveFile(source); +let wavBuffer = wav.toBuffer(); +const { OpusEncoder } = require('@discordjs/opus'); +const encoder = new OpusEncoder(16000, 1); + +let fileIndex = 0; +let packetID = 0; + +var udp = require("dgram"); +var udpClient = udp.createSocket("udp4"); + +let clientData = { + ClientGuid: "AZi9CkptY0yW_C-3YmI7oQ", + Name: "Olympus", + Seat: 0, + Coalition: 0, + AllowRecord: false, + RadioInfo: { + radios: [ + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + { + enc: false, + encKey: 0, + freq: 1.0, + modulation: 3, + secFreq: 1.0, + retransmit: false, + }, + ], + unit: "", + unitId: 0, + iff: { + control: 2, + mode1: -1, + mode2: -1, + mode3: -1, + mode4: false, + mic: -1, + status: 0, + }, + ambient: { vol: 1.0, abType: "" }, + }, + LatLngPosition: { lat: 0.0, lng: 0.0, alt: 0.0 }, +}; + +var net = require("net"); + +var tcpClient = new net.Socket(); + +tcpClient.on("data", function (data) { + console.log("Received: " + data); + +}); + +tcpClient.on("close", function () { + console.log("Connection closed"); +}); + +tcpClient.connect(5002, "localhost", function () { + console.log("Connected"); + + setTimeout(() => { + let SYNC = { + Client: clientData, + MsgType: 2, + Version: "2.1.0.10", + }; + let string = JSON.stringify(SYNC); + tcpClient.write(string + "\n"); + + setInterval(() => { + let slice = []; + for (let i = 0; i < 16000 * 0.04; i++) { + slice.push(wavBuffer[Math.round(fileIndex) * 2], wavBuffer[Math.round(fileIndex) * 2 + 1]); + fileIndex += 44100 / 16000; + } + const encoded = encoder.encode(new Uint8Array(slice)); + + let header = [ + 0, 0, + 0, 0, + 0, 0 + ] + + let encFrequency = [...doubleToByteArray(251000000)]; + let encModulation = [2]; + let encEncryption = [0]; + + let encUnitID = getBytes(100000001, 4); + let encPacketID = getBytes(packetID, 8); + packetID++; + let encHops = [0]; + + let packet = [].concat(header, [...encoded], encFrequency, encModulation, encEncryption, encUnitID, encPacketID, encHops, [...Buffer.from(clientData.ClientGuid, 'utf-8')], [...Buffer.from(clientData.ClientGuid, 'utf-8')]); + + let encPacketLen = getBytes(packet.length, 2); + packet[0] = encPacketLen[0]; + packet[1] = encPacketLen[1]; + + let encAudioLen = getBytes(encoded.length, 2); + packet[2] = encAudioLen[0]; + packet[3] = encAudioLen[1]; + + let frequencyAudioLen = getBytes(10, 2); + packet[4] = frequencyAudioLen[0]; + packet[5] = frequencyAudioLen[1]; + + let data = new Uint8Array(packet); + udpClient.send(data, 5002, "localhost", function (error) { + if (error) { + tcpClient.close(); + } else { + console.log("Data sent !!!"); + } + }); + }, 40); + }, 1000); +}); + +function getBytes(value, length) { + let res = []; + for (let i = 0; i < length; i++) { + res.push(value & 255); + value = value >> 8; + } + return res; +} + +function doubleToByteArray(number) { + var buffer = new ArrayBuffer(8); // JS numbers are 8 bytes long, or 64 bits + var longNum = new Float64Array(buffer); // so equivalent to Float64 + + longNum[0] = number; + + return Array.from(new Uint8Array(buffer)); +} \ No newline at end of file diff --git a/frontend/server/tsconfig.json b/frontend/server/tsconfig.json index 6168f73c..e5a37c12 100644 --- a/frontend/server/tsconfig.json +++ b/frontend/server/tsconfig.json @@ -2,7 +2,9 @@ "compilerOptions": { "outDir": "./build", "allowJs": true, - "target": "es5" + "target": "ES2023", + "module": "Node16", + "moduleResolution": "Node16" }, "include": [ "./src/**/*" diff --git a/olympus.json b/olympus.json index 383039c6..602660ac 100644 --- a/olympus.json +++ b/olympus.json @@ -1,7 +1,7 @@ { "backend": { "address": "localhost", - "port": 4512 + "port": 3001 }, "authentication": { "gameMasterPassword": "4b8823ed9e5c2392ab4a791913bb8ce41956ea32e308b760eefb97536746dd33", @@ -33,5 +33,9 @@ "DCS Map (Official)": "https://maps.dcsolympus.com/maps", "DCS Map (Alt.)": "https://refugees.dcsolympus.com/maps" } + }, + "audio": { + "SRSPort": 5002, + "WSPort": 4000 } }