fix: account for STT fallback transcript alignment (livekit#6633)

Author: chenghao-mou

livekit-agents/livekit/agents/inference/stt.py

@@ -613,13 +613,23 @@ def __init__(
 
         vad = _resolve_vad_for_model(model, vad if is_given(vad) else None)
 
+        fallback_models: NotGivenOr[list[FallbackModel]] = NOT_GIVEN
+        if is_given(fallback):
+            fallback_models = _normalize_fallback(fallback)
+        models = [model]
+        if is_given(fallback_models):
+            models.extend(item["model"] for item in fallback_models)
+        aligned_transcript: Literal["word", False] = (
+            "word" if all(_aligned_transcript_for_model(item) for item in models) else False
+        )
+
         # chat_context follows the model's native support; the session decides whether to forward
         super().__init__(
             capabilities=stt.STTCapabilities(
                 streaming=True,
                 interim_results=True,
                 diarization=diarization_enabled,
-                aligned_transcript=_aligned_transcript_for_model(model),
+                aligned_transcript=aligned_transcript,
                 offline_recognize=False,
                 keyterms=_keyterms_extra_for_model(model) is not None,
                 chat_context=_supports_agent_context_carryover(model),
@@ -647,9 +657,6 @@ def __init__(
             raise ValueError(
                 "api_secret is required, either as argument or set LIVEKIT_API_SECRET environmental variable"
             )
-        fallback_models: NotGivenOr[list[FallbackModel]] = NOT_GIVEN
-        if is_given(fallback):
-            fallback_models = _normalize_fallback(fallback)
 
         self._opts = STTOptions(
             model=model,
@@ -741,11 +748,16 @@ def update_options(
 
             self._opts.model = model
             self._vad = _resolve_vad_for_model(model, self._vad)
+            models = [self._opts.model]
+            if is_given(self._opts.fallback):
+                models.extend(item["model"] for item in self._opts.fallback)
             self._capabilities = replace(
                 self._capabilities,
                 keyterms=_keyterms_extra_for_model(self._opts.model) is not None,
                 chat_context=_supports_agent_context_carryover(self._opts.model),
-                aligned_transcript=_aligned_transcript_for_model(self._opts.model),
+                aligned_transcript=(
+                    "word" if all(_aligned_transcript_for_model(item) for item in models) else False
+                ),
             )
         if is_given(language):
             self._opts.language = LanguageCode(language)

tests/test_inference_stt_aligned_transcript_claim.py

@@ -102,6 +102,29 @@ def test_unknown_models_do_not_claim_alignment(_fake_credentials: None) -> None:
     assert stt_impl.capabilities.aligned_transcript is False
 
 
+@pytest.mark.parametrize(
+    ("fallback", "expected"),
+    [
+        pytest.param("cartesia/ink-whisper", "word", id="aligned-fallback"),
+        pytest.param("cartesia/ink-2", False, id="unaligned-fallback"),
+        pytest.param("new-provider/new-turn-model", False, id="unknown-fallback"),
+    ],
+)
+def test_fallback_models_constrain_alignment_claim(
+    fallback: str, expected: object, _fake_credentials: None
+) -> None:
+    stt_impl = inference.STT(model="deepgram/nova-3", fallback=fallback)
+    assert stt_impl.capabilities.aligned_transcript == expected
+
+
+def test_alignment_update_still_accounts_for_fallback(_fake_credentials: None) -> None:
+    stt_impl = inference.STT(model="cartesia/ink-2", fallback="new-provider/new-turn-model")
+
+    stt_impl.update_options(model="deepgram/nova-3")
+
+    assert stt_impl.capabilities.aligned_transcript is False
+
+
 def test_gateway_ink2_payload_carries_no_word_alignment() -> None:
     """The advertised word alignment is not in the data the gateway sends."""
     stream = InferenceSpeechStream.__new__(InferenceSpeechStream)