From f17ee42d636ba9df2d122393366c86644ce7a37e Mon Sep 17 00:00:00 2001 From: Pax1601 Date: Tue, 3 Dec 2024 18:54:30 +0100 Subject: [PATCH] Implemented session data for audio --- frontend/react/src/audio/audiomanager.ts | 50 +++++-- frontend/react/src/audio/filesource.ts | 7 +- frontend/react/src/audio/speechcontroller.ts | 38 +++++- .../react/src/audio/texttospeechsource.ts | 11 +- frontend/react/src/constants/constants.ts | 3 +- frontend/react/src/controllers/awacs.ts | 7 +- frontend/react/src/interfaces.ts | 1 + frontend/react/src/sessiondata.ts | 30 +++- .../src/ui/components/olfrequencyinput.tsx | 4 +- .../react/src/ui/components/ollabeltoggle.tsx | 5 +- .../react/src/ui/components/olnumberinput.tsx | 11 +- .../react/src/ui/components/olstatebutton.tsx | 5 +- frontend/react/src/ui/components/oltoggle.tsx | 10 +- .../src/ui/modals/filesourceloadprompt.tsx | 129 ------------------ frontend/react/src/ui/panels/audiomenu.tsx | 13 +- .../ui/panels/components/radiosinkpanel.tsx | 7 +- .../src/ui/panels/components/sourcepanel.tsx | 85 +++++++++--- frontend/react/src/ui/ui.tsx | 4 +- 18 files changed, 212 insertions(+), 208 deletions(-) delete mode 100644 frontend/react/src/ui/modals/filesourceloadprompt.tsx diff --git a/frontend/react/src/audio/audiomanager.ts b/frontend/react/src/audio/audiomanager.ts index d0ecff6b..0fbcad0e 100644 --- a/frontend/react/src/audio/audiomanager.ts +++ b/frontend/react/src/audio/audiomanager.ts @@ -52,6 +52,7 @@ export class AudioManager { #SRSClientUnitIDs: number[] = []; #syncInterval: number; #speechRecognition: boolean = true; + #internalTextToSpeechSource: TextToSpeechSource; constructor() { ConfigLoadedEvent.on((config: OlympusConfig) => { @@ -161,23 +162,41 @@ export class AudioManager { newRadio?.setModulation(options.modulation); }); } else { - /* Add two default radios */ - this.addRadio(); - this.addRadio(); + /* Add two default radios and connect to the microphone*/ + let newRadio = this.addRadio(); + this.#sources.find((source) => source instanceof MicrophoneSource)?.connect(newRadio); + this.#sources.find((source) => source instanceof TextToSpeechSource)?.connect(newRadio); + + newRadio = this.addRadio(); + this.#sources.find((source) => source instanceof MicrophoneSource)?.connect(newRadio); + this.#sources.find((source) => source instanceof TextToSpeechSource)?.connect(newRadio); + } + + let sessionFileSources = getApp().getSessionDataManager().getSessionData().fileSources; + if (sessionFileSources) { + /* Load file sources */ + sessionFileSources.forEach((options) => { + this.addFileSource(); + }); } let sessionUnitSinks = getApp().getSessionDataManager().getSessionData().unitSinks; if (sessionUnitSinks) { - /* Load session radios */ + /* Load unit sinks */ sessionUnitSinks.forEach((options) => { let unit = getApp().getUnitsManager().getUnitByID(options.ID); if (unit) { - let newSink = this.addUnitSink(unit); + this.addUnitSink(unit); } }); } - let sessionFileSources = getApp().getSessionDataManager().getSessionData().fileSources; - if (sessionFileSources && sessionFileSources.length > 0) getApp().setState(OlympusState.LOAD_FILES); + + let sessionConnections = getApp().getSessionDataManager().getSessionData().connections; + if (sessionConnections) { + sessionConnections.forEach((connection) => { + this.#sources[connection[0]]?.connect(this.#sinks[connection[1]]); + }) + } this.#running = true; AudioManagerStateChangedEvent.dispatch(this.#running); @@ -190,6 +209,8 @@ export class AudioManager { this.#devices = devices; AudioManagerDevicesChangedEvent.dispatch(devices); }); + + this.#internalTextToSpeechSource = new TextToSpeechSource(); } stop() { @@ -220,9 +241,9 @@ export class AudioManager { this.#endpoint = endpoint; } - addFileSource(file) { - console.log(`Adding file source from ${file.name}`); - const newSource = new FileSource(file); + addFileSource() { + console.log(`Adding file source`); + const newSource = new FileSource(); this.#sources.push(newSource); AudioSourcesChangedEvent.dispatch(getApp().getAudioManager().getSources()); return newSource; @@ -250,11 +271,10 @@ export class AudioManager { addRadio() { console.log("Adding new radio"); const newRadio = new RadioSink(); - newRadio.speechDataAvailable = (blob) => this.#speechController.analyzeData(blob); + newRadio.speechDataAvailable = (blob) => this.#speechController.analyzeData(blob, newRadio); this.#sinks.push(newRadio); /* Set radio name by default to be incremental number */ newRadio.setName(`Radio ${this.#sinks.length}`); - this.#sources.find((source) => source instanceof MicrophoneSource)?.connect(newRadio); AudioSinksChangedEvent.dispatch(this.#sinks); return newRadio; } @@ -327,13 +347,17 @@ export class AudioManager { } setSpeechRecognition(speechRecognition: boolean) { - this.#speechRecognition = this.#speechRecognition; + this.#speechRecognition = speechRecognition; } getSpeechRecognition() { return this.#speechRecognition; } + getInternalTextToSpeechSource() { + return this.#internalTextToSpeechSource; + } + #syncRadioSettings() { /* Send the radio settings of each radio to the SRS backend */ let message = { diff --git a/frontend/react/src/audio/filesource.ts b/frontend/react/src/audio/filesource.ts index 229c699c..5e3b5859 100644 --- a/frontend/react/src/audio/filesource.ts +++ b/frontend/react/src/audio/filesource.ts @@ -15,13 +15,12 @@ export class FileSource extends AudioSource { #restartTimeout: any; #looping = false; - constructor(file) { - super(); + setFile(file: File) { this.#file = file; this.#filename = this.#file?.name; this.setName(this.#file?.name ?? "N/A"); - + /* Create the file reader and read the file from disk */ var reader = new FileReader(); reader.onload = (e) => { @@ -38,6 +37,8 @@ export class FileSource extends AudioSource { } }; reader.readAsArrayBuffer(this.#file); + + AudioSourcesChangedEvent.dispatch(getApp().getAudioManager().getSources()); } play() { diff --git a/frontend/react/src/audio/speechcontroller.ts b/frontend/react/src/audio/speechcontroller.ts index da2a9d84..1370bfdf 100644 --- a/frontend/react/src/audio/speechcontroller.ts +++ b/frontend/react/src/audio/speechcontroller.ts @@ -1,10 +1,12 @@ import { getApp } from "../olympusapp"; import { blobToBase64 } from "../other/utils"; +import { AudioSource } from "./audiosource"; +import { RadioSink } from "./radiosink"; export class SpeechController { constructor() {} - analyzeData(blob: Blob) { + analyzeData(blob: Blob, radio: RadioSink) { blobToBase64(blob) .then((base64) => { const requestOptions = { @@ -23,23 +25,45 @@ export class SpeechController { throw new Error("Error saving profile"); } }) - .then((text) => this.#executeCommand(text)) + .then((text) => this.#executeCommand(text.toLowerCase(), radio)) .catch((error) => console.error(error)); // Handle errors }) .catch((error) => console.error(error)); } - #executeCommand(text) { + playText(text, radio: RadioSink) { + const textToSpeechSource = getApp() + .getAudioManager() + .getInternalTextToSpeechSource(); + + textToSpeechSource.connect(radio); + textToSpeechSource.playText(text); + radio.setPtt(true); + textToSpeechSource.onMessageCompleted = () => { + radio.setPtt(false); + textToSpeechSource.disconnect(radio); + } + } + + #executeCommand(text, radio) { console.log(`Received speech command: ${text}`); if (text.indexOf("olympus") === 0 ) { - this.#olympusCommand(text); + this.#olympusCommand(text, radio); } else if (text.indexOf(getApp().getAWACSController()?.getCallsign().toLowerCase()) === 0) { - getApp().getAWACSController()?.executeCommand(text); + getApp().getAWACSController()?.executeCommand(text, radio); } } - #olympusCommand(text) { - + #olympusCommand(text, radio) { + if (text.indexOf("request straight") > 0 || text.indexOf("request straightin") > 0) { + this.playText("Confirm you are on step 13, being a pussy?", radio); + } + else if (text.indexOf("bolter") > 0) { + this.playText("What an idiot, I never boltered, 100% boarding rate", radio); + } + else if (text.indexOf("read back") > 0) { + this.playText(text.replace("olympus", ""), radio); + } } } diff --git a/frontend/react/src/audio/texttospeechsource.ts b/frontend/react/src/audio/texttospeechsource.ts index 85df8735..29c61c03 100644 --- a/frontend/react/src/audio/texttospeechsource.ts +++ b/frontend/react/src/audio/texttospeechsource.ts @@ -12,11 +12,12 @@ export class TextToSpeechSource extends AudioSource { #audioBuffer: AudioBuffer; #restartTimeout: any; #looping = false; + onMessageCompleted: () => void = () => {}; constructor() { super(); - this.setName("Text to speech") + this.setName("Text to speech"); } playText(text: string) { @@ -36,7 +37,7 @@ export class TextToSpeechSource extends AudioSource { } }) // Parse the response .then((blob) => { - return blob.arrayBuffer() + return blob.arrayBuffer(); }) .then((contents) => { getApp() @@ -76,7 +77,10 @@ export class TextToSpeechSource extends AudioSource { if (this.#currentPosition > this.#duration) { this.#currentPosition = 0; - if (!this.#looping) this.pause(); + if (!this.#looping) { + this.pause(); + this.onMessageCompleted(); + } } AudioSourcesChangedEvent.dispatch(getApp().getAudioManager().getSources()); @@ -133,4 +137,3 @@ export class TextToSpeechSource extends AudioSource { return this.#looping; } } - diff --git a/frontend/react/src/constants/constants.ts b/frontend/react/src/constants/constants.ts index 86de3a97..4fd77a29 100644 --- a/frontend/react/src/constants/constants.ts +++ b/frontend/react/src/constants/constants.ts @@ -286,8 +286,7 @@ export enum OlympusState { OPTIONS = "Options", AUDIO = "Audio", AIRBASE = "Airbase", - GAME_MASTER = "Game master", - LOAD_FILES = "Load files" + GAME_MASTER = "Game master" } export const NO_SUBSTATE = "No substate"; diff --git a/frontend/react/src/controllers/awacs.ts b/frontend/react/src/controllers/awacs.ts index da053f23..d8ea48c1 100644 --- a/frontend/react/src/controllers/awacs.ts +++ b/frontend/react/src/controllers/awacs.ts @@ -2,6 +2,7 @@ import { getApp } from "../olympusapp"; import { Coalition } from "../types/types"; import { Unit } from "../unit/unit"; import { bearing, coalitionToEnum, computeBearingRangeString, mToFt, rad2deg } from "../other/utils"; +import { TextToSpeechSource } from "../audio/texttospeechsource"; const trackStrings = ["North", "North-East", "East", "South-East", "South", "South-West", "West", "North-West", "North"]; const relTrackStrings = ["hot", "flank right", "beam right", "cold", "cold", "cold", "beam left", "flank left", "hot"]; @@ -11,9 +12,11 @@ export class AWACSController { #callsign: string = "Magic"; #referenceUnit: Unit; - constructor() {} + constructor() { - executeCommand(text) { + } + + executeCommand(text, radio) { if (text.indexOf("request picture") > 0) { console.log("Requested AWACS picture"); const readout = this.createPicture(true); diff --git a/frontend/react/src/interfaces.ts b/frontend/react/src/interfaces.ts index 90904d1f..32d9edd1 100644 --- a/frontend/react/src/interfaces.ts +++ b/frontend/react/src/interfaces.ts @@ -33,6 +33,7 @@ export interface SessionData { radios?: { frequency: number; modulation: number }[]; fileSources?: { filename: string; volume: number }[]; unitSinks?: {ID: number}[]; + connections?: any[]; } export interface ProfileOptions { diff --git a/frontend/react/src/sessiondata.ts b/frontend/react/src/sessiondata.ts index b257a12b..c01b3d15 100644 --- a/frontend/react/src/sessiondata.ts +++ b/frontend/react/src/sessiondata.ts @@ -1,3 +1,4 @@ +import { AudioSink } from "./audio/audiosink"; import { FileSource } from "./audio/filesource"; import { RadioSink } from "./audio/radiosink"; import { UnitSink } from "./audio/unitsink"; @@ -26,10 +27,23 @@ export class SessionDataManager { .filter((sink) => sink instanceof UnitSink) .map((unitSink) => { return { - ID: unitSink.getUnit().ID + ID: unitSink.getUnit().ID, }; }); + this.#sessionData.connections = []; + let counter = 0; + let sources = getApp().getAudioManager().getSources(); + let sinks = getApp().getAudioManager().getSinks(); + sources.forEach((source, idx) => { + counter++; + source.getConnectedTo().forEach((sink) => { + if (sinks.indexOf(sink as AudioSink) !== undefined) { + this.#sessionData.connections?.push([idx, sinks.indexOf(sink as AudioSink)]); + } + }); + }); + this.#saveSessionData(); } }); @@ -41,6 +55,20 @@ export class SessionDataManager { .map((fileSource) => { return { filename: fileSource.getFilename(), volume: fileSource.getVolume() }; }); + + this.#sessionData.connections = []; + let counter = 0; + let sources = getApp().getAudioManager().getSources(); + let sinks = getApp().getAudioManager().getSinks(); + sources.forEach((source, idx) => { + counter++; + source.getConnectedTo().forEach((sink) => { + if (sinks.indexOf(sink as AudioSink) !== undefined) { + this.#sessionData.connections?.push([idx, sinks.indexOf(sink as AudioSink)]); + } + }); + }); + this.#saveSessionData(); } }); diff --git a/frontend/react/src/ui/components/olfrequencyinput.tsx b/frontend/react/src/ui/components/olfrequencyinput.tsx index 138ce87c..82c5c07a 100644 --- a/frontend/react/src/ui/components/olfrequencyinput.tsx +++ b/frontend/react/src/ui/components/olfrequencyinput.tsx @@ -10,10 +10,10 @@ export function OlFrequencyInput(props: { value: number; className?: string; onC flex gap-2 `}> { - let newValue = Math.max(Math.min(Number(e.target.value), 400), 1) * 1000000; + let newValue = Math.max(Math.min(Number(e.target.value), 400), 0) * 1000000; let decimalPart = frequency - Math.floor(frequency / 1000000) * 1000000; frequency = newValue + decimalPart; props.onChange(frequency); diff --git a/frontend/react/src/ui/components/ollabeltoggle.tsx b/frontend/react/src/ui/components/ollabeltoggle.tsx index a31e8fa3..7b6c3e87 100644 --- a/frontend/react/src/ui/components/ollabeltoggle.tsx +++ b/frontend/react/src/ui/components/ollabeltoggle.tsx @@ -3,7 +3,10 @@ import React from "react"; export function OlLabelToggle(props: { toggled: boolean | undefined; leftLabel: string; rightLabel: string; onClick: () => void }) { return ( - - ); - })} - - -
- -
- - - ); -} diff --git a/frontend/react/src/ui/panels/audiomenu.tsx b/frontend/react/src/ui/panels/audiomenu.tsx index 60a15fa4..d3380fe4 100644 --- a/frontend/react/src/ui/panels/audiomenu.tsx +++ b/frontend/react/src/ui/panels/audiomenu.tsx @@ -202,18 +202,7 @@ export function AudioMenu(props: { open: boolean; onClose: () => void; children? focus:outline-none focus:ring-4 focus:ring-blue-300 hover:bg-blue-800 `} - onClick={() => { - var input = document.createElement("input"); - input.type = "file"; - input.click(); - input.onchange = (e: Event) => { - let target = e.target as HTMLInputElement; - if (target && target.files) { - var file = target.files[0]; - getApp().getAudioManager().addFileSource(file); - } - }; - }} + onClick={() => getApp().getAudioManager().addFileSource()} > Add audio source diff --git a/frontend/react/src/ui/panels/components/radiosinkpanel.tsx b/frontend/react/src/ui/panels/components/radiosinkpanel.tsx index 83bbe62f..94a10a56 100644 --- a/frontend/react/src/ui/panels/components/radiosinkpanel.tsx +++ b/frontend/react/src/ui/panels/components/radiosinkpanel.tsx @@ -18,11 +18,14 @@ export const RadioSinkPanel = forwardRef((props: { radio: RadioSink; shortcutKey
{ + setExpanded(!expanded); + }} >
void }, ref: ForwardedRef) => { const [meterLevel, setMeterLevel] = useState(0); @@ -48,7 +49,47 @@ export const AudioSourcePanel = forwardRef((props: { source: AudioSource; onExpa />
- {props.source.getName()} +
+ {props.source.getName() === "" ? ( + props.source instanceof FileSource ? ( +
+ No file selected + +
+ ) : ( + "No name" + ) + ) : ( + props.source.getName() + )} +
{!(props.source instanceof MicrophoneSource) && !(props.source instanceof TextToSpeechSource) && (
{(props.source instanceof FileSource || props.source instanceof TextToSpeechSource) && (
- {props.source instanceof TextToSpeechSource && - { - setText(ev.target.value); - }} - > - } + {props.source instanceof TextToSpeechSource && ( + { + setText(ev.target.value); + }} + > + )}
0 ? (props.source.getCurrentPosition() / props.source.getDuration()) * 100 : 0} onChange={(ev) => { - if (props.source instanceof FileSource || props.source instanceof TextToSpeechSource) props.source.setCurrentPosition(parseFloat(ev.currentTarget.value)); + if (props.source instanceof FileSource || props.source instanceof TextToSpeechSource) + props.source.setCurrentPosition(parseFloat(ev.currentTarget.value)); }} className="my-auto" /> @@ -124,9 +166,10 @@ export const AudioSourcePanel = forwardRef((props: { source: AudioSource; onExpa flex-row border-gray-500 `} > -
+
- - +
getApp().setState(OlympusState.IDLE)} />