fix(voice): retain concrete language for turn detection (#2149)

Co-authored-by: rosetta-livekit-bot[bot] <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com>

Author: rosetta-livekit-bot[bot]

.changeset/green-mice-hear.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents': patch
+---
+
+Preserve concrete STT language hints when later transcripts report non-specific language codes.

agents/src/voice/audio_recognition.ts

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { Mutex } from '@livekit/mutex';
-import { AudioFrame, type ParticipantKind } from '@livekit/rtc-node';
+import type { AudioFrame, ParticipantKind } from '@livekit/rtc-node';
 import { ThrowsPromise } from '@livekit/throws-transformer/throws';
 import {
   type Context,
@@ -67,6 +67,8 @@ import {
 // Maximum number of chat items included in the `lk.chat_ctx` attribute of the
 // `eou_detection` span (mirrors Python's `_EOU_MAX_HISTORY_TURNS`).
 const EOU_MAX_HISTORY_TURNS = 6;
+const MIN_LANGUAGE_DETECTION_LENGTH = 5;
+const NON_SPECIFIC_LANGUAGE_CODES = new Set(['auto', 'multi']);
 
 export interface EndOfTurnInfo {
   /** The new transcript text from the user's speech. */
@@ -636,6 +638,16 @@ export class AudioRecognition {
     }
   }
 
+  private updateLastLanguage(language: LanguageCode | undefined, transcript: string): void {
+    if (!language || NON_SPECIFIC_LANGUAGE_CODES.has(language)) {
+      return;
+    }
+
+    if (!this.lastLanguage || transcript.length > MIN_LANGUAGE_DETECTION_LENGTH) {
+      this.lastLanguage = language;
+    }
+  }
+
   async start(options?: {
     sttPipeline?: STTPipeline;
     turnDetectorStream?: BaseStreamingTurnDetectorStream;
@@ -1092,9 +1104,10 @@ export class AudioRecognition {
 
     switch (ev.type) {
       case SpeechEventType.FINAL_TRANSCRIPT:
-        const transcript = ev.alternatives?.[0]?.text;
+        const transcript = ev.alternatives?.[0]?.text ?? '';
         const confidence = ev.alternatives?.[0]?.confidence ?? 0;
-        this.lastLanguage = ev.alternatives?.[0]?.language;
+        const language = ev.alternatives?.[0]?.language;
+        this.updateLastLanguage(language, transcript);
 
         if (!transcript) {
           // stt final transcript received but no transcript
@@ -1162,13 +1175,7 @@ export class AudioRecognition {
         const preflightConfidence = ev.alternatives?.[0]?.confidence ?? 0;
         const preflightLanguage = ev.alternatives?.[0]?.language;
 
-        const MIN_LANGUAGE_DETECTION_LENGTH = 5;
-        if (
-          !this.lastLanguage ||
-          (preflightLanguage && preflightTranscript.length > MIN_LANGUAGE_DETECTION_LENGTH)
-        ) {
-          this.lastLanguage = preflightLanguage;
-        }
+        this.updateLastLanguage(preflightLanguage, preflightTranscript);
 
         if (!preflightTranscript) {
           return;

agents/src/voice/audio_recognition_turn_detection.test.ts

@@ -30,6 +30,7 @@ import {
   MIN_SILENCE_DURATION_MS,
   type TurnDetectionEvent,
 } from '../inference/eot/base.js';
+import { type LanguageCode, asLanguageCode } from '../language.js';
 import { ChatContext } from '../llm/chat_context.js';
 import { initializeLogger } from '../log.js';
 import { Future } from '../utils.js';
@@ -55,6 +56,7 @@ interface RecognitionInternals {
   turnDetectorLatePredictionWarned: boolean;
   lastEmittedEotPrediction?: TurnDetectionEvent;
   lastSpeakingTime?: number;
+  lastLanguage?: LanguageCode;
   audioTranscript: string;
   audioInterimTranscript: string;
   audioPreflightTranscript: string;
@@ -68,6 +70,7 @@ interface RecognitionInternals {
   runEOUDetection: (chatCtx: ChatContext, trigger?: 'vad' | 'stt' | 'manual') => void;
   createVadTask: (vad: VAD | undefined, signal: AbortSignal) => Promise<void>;
   checkVadSilenceRequirement: (detector?: _TurnDetector | BaseStreamingTurnDetector) => void;
+  updateLastLanguage: (language: LanguageCode, transcript: string) => void;
   updateTurnDetector: (detector: _TurnDetector | BaseStreamingTurnDetector | undefined) => void;
   clearUserTurn: () => void;
 }
@@ -221,6 +224,35 @@ function flush(): Promise<void> {
   return new Promise((resolve) => setImmediate(resolve));
 }
 
+describe('TestLanguageTracking', () => {
+  it('does not replace a concrete language with a non-specific language', () => {
+    const { internals } = makeRecognition();
+    internals.lastLanguage = asLanguageCode('en');
+
+    internals.updateLastLanguage(asLanguageCode('multi'), 'ambiguous phrase');
+
+    expect(internals.lastLanguage).toBe(asLanguageCode('en'));
+  });
+
+  it('does not initialize language with a non-specific language', () => {
+    const { internals } = makeRecognition();
+    internals.lastLanguage = undefined;
+
+    internals.updateLastLanguage(asLanguageCode('multi'), 'we');
+
+    expect(internals.lastLanguage).toBeUndefined();
+  });
+
+  it('still updates concrete language after a long transcript', () => {
+    const { internals } = makeRecognition();
+    internals.lastLanguage = asLanguageCode('en');
+
+    internals.updateLastLanguage(asLanguageCode('fr'), 'bonjour');
+
+    expect(internals.lastLanguage).toBe(asLanguageCode('fr'));
+  });
+});
+
 /**
  * Drive `createVadTask` against a scripted VAD stream so VAD events flow
  * through the real handler. `feed()` resolves once the event has been processed