fix: don't listen to the users input until agent didn't stop talking

Author: yaroslav8765

custom/composables/useAgentAudio.ts

@@ -49,7 +49,7 @@ function restartStandByAudio() {
 
 export const useAgentAudio = defineStore('agentAudio', () => {
   const agentStore = useAgentStore();
-  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | null>(null);
+  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | 'readyToRespond' >('readyToRespond');
   const isStreamingResponse = ref(false);
 
   let currentAbortController: AbortController | null = null;
@@ -59,15 +59,21 @@ export const useAgentAudio = defineStore('agentAudio', () => {
   let currentStreamingAudio: StreamingAudioState | null = null;
   let bufferedAudioChunks: ArrayBuffer[] = [];
   let bufferedAudioMimeType = 'audio/mpeg';
+  let wasAudioResponseReceived = false;
 
   function stopGenerationAndAudio() {
-    agentAudioMode.value = null;
+    setAudioModeReadyToRespond();
     stopCurrentAudioPlayback();
     currentAbortController?.abort();
   }
 
+  function setAudioModeReadyToRespond() {
+    agentAudioMode.value = 'readyToRespond';
+  }
+
   async function sendAudioToServerAndHandleResponse(blob: Blob) {
     currentAbortController = new AbortController();
+    wasAudioResponseReceived = false;
     const formData = new FormData();
     formData.append('file', blob, 'user_prompt.wav');
     formData.append('sessionId', agentStore.activeSessionId);
@@ -101,7 +107,9 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       }
     } finally {
       isStreamingResponse.value = false;
-      agentAudioMode.value = null;
+      if (!wasAudioResponseReceived) {
+        setAudioModeReadyToRespond();
+      }
     }
   }
 
@@ -177,14 +185,15 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       stopStandByAudio();
       agentStore.setCurrentChatStatus('ready');
       agentStore.addAgentMessage(event.data.response.text);
-      agentAudioMode.value = 'playingAgentResponse';
       return;
     }
 
     if (event.type === 'audio-start') {
+      wasAudioResponseReceived = true;
       isStreamingResponse.value = false;
       agentAudioMode.value = 'fetchingAudio';
       initializeAudioStream(event.data.mimeType);
+      agentAudioMode.value = 'playingAgentResponse';
       return;
     }
 
@@ -348,12 +357,12 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     detachStreamingAudio();
     destroyCurrentAudioElement();
     if (!dontResetMode) {
-      agentAudioMode.value = null;
+      setAudioModeReadyToRespond();
     }
   }
 
   function handleAudioEnded() {
-    agentAudioMode.value = null;
+    setAudioModeReadyToRespond();
     stopCurrentAudioPlayback();
   }
 

custom/speech_recognition_frontend/MicrophoneButon.vue

@@ -63,7 +63,7 @@ onMounted(() => {
   });
 });
 
-watch(agentAudioMode, (newVal) => {
+watch(agentAudioMode, async (newVal) => {
   if(newVal === 'streaming') {
     stopCurrentAudioPlayback(true);
     microphoneButtonMode.value = 'generating';
@@ -72,10 +72,11 @@ watch(agentAudioMode, (newVal) => {
   } else if (newVal === 'fetchingAudio') {
     //Generation is done, waiting for audio to be ready
   } else if (newVal === 'playingAgentResponse') {
-    // Audio is playing
-  } else {
+    // Audio is playing' 
+  } else if (newVal === 'readyToRespond') {
     if(isAudioChatMode.value) {
       microphoneButtonMode.value = 'listen';
+      await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
     } else {
       microphoneButtonMode.value = 'off';
     }
@@ -142,12 +143,8 @@ async function sendRecordForTranscription() {
   showAudioWavesAnimation.value = false;
   const recordBlob = await getRecord();
   if (recordBlob) {
-    console.log('Audio recorded, sending to server for transcription. Audio Blob size:', recordBlob.size, recordBlob.type);
     onStopRecording();
     await sendAudioToServerAndHandleResponse(recordBlob);
-    if (agentStore.isAudioChatMode) {
-      await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
-    }
   } else { 
     console.error('No audio recorded');
   }