fix(voice): reset VAD on premature STT EOT & guard empty recorder frames (#1181)

Co-authored-by: Claude <noreply@anthropic.com>

Author: toubatbrian

.changeset/tender-baboons-judge.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": patch
+---
+
+fix(voice): reset VAD on premature STT EOT & guard empty recorder frames

agents/src/voice/audio_recognition.ts

@@ -165,6 +165,7 @@ export class AudioRecognition {
   private isInterruptionEnabled: boolean;
   private isAgentSpeaking: boolean;
   private interruptionStreamChannel?: StreamChannel<InterruptionSentinel | AudioFrame>;
+  private closed = false;
 
   constructor(opts: AudioRecognitionOptions) {
     this.hooks = opts.recognitionHooks;
@@ -696,6 +697,15 @@ export class AudioRecognition {
             });
           });
         }
+        // STT EOT changes user state from speaking to listening without updating VAD internal states.
+        // VAD EOS will also skip updating user state from listening (STT enforced) to listening (VAD detected)
+        // and user state won't be updated until a new VAD SOS is received.
+        // Reset VAD so that incorrect end of turn from STT can be corrected by VAD interruption.
+        // If user is still speaking (an immediate VAD SOS will interrupt the agent).
+        if (this.vad && this.speaking) {
+          this.logger.warn('stt end of speech received while user is speaking, resetting vad');
+          this.resetVad();
+        }
         this.speaking = false;
         this.userTurnCommitted = true;
         this.lastSpeakingTime = Date.now();
@@ -1159,6 +1169,23 @@ export class AudioRecognition {
     });
   }
 
+  /**
+   * Reset the VAD by restarting its task. This is needed when STT sends a premature
+   * end-of-turn signal while the user is still speaking, so VAD can detect new speech
+   * and trigger interruptions correctly.
+   */
+  private resetVad() {
+    if (!this.vad) return;
+
+    this.vadTask?.cancelAndWait().finally(() => {
+      if (this.closed) return;
+      this.vadTask = Task.from(({ signal }) => this.createVadTask(this.vad, signal));
+      this.vadTask.result.catch((err) => {
+        this.logger.error(`Error running VAD task: ${err}`);
+      });
+    });
+  }
+
   commitUserTurn(audioDetached: boolean) {
     const commitUserTurnTask =
       (delayDuration: number = 500) =>
@@ -1206,6 +1233,7 @@ export class AudioRecognition {
   }
 
   async close() {
+    this.closed = true;
     this.detachInputAudioStream();
     this.silenceAudioWriter.releaseLock();
     await this.commitUserTurnTask?.cancelAndWait();

agents/src/voice/recorder_io/recorder_io.ts

@@ -679,11 +679,16 @@ class RecorderAudioOutput extends AudioOutput {
       pauseIdx++;
     }
 
-    if (buf.length > 0) {
+    // Filter out empty frames from split operations to avoid spurious buffer writes
+    const filteredBuf = buf.filter((f) => f.samplesPerChannel > 0);
+
+    if (filteredBuf.length > 0) {
       if (trailingSilenceDuration > 0) {
-        buf.push(createSilenceFrame(trailingSilenceDuration / 1000, sampleRate, numChannels));
+        filteredBuf.push(
+          createSilenceFrame(trailingSilenceDuration / 1000, sampleRate, numChannels),
+        );
       }
-      this.writeFn(buf);
+      this.writeFn(filteredBuf);
     }
 
     this.accFrames = [];