feat: implement TTS aligned transcripts

agents/src/index.ts

@@ -34,6 +34,7 @@ export * from './types.js';
 export * from './utils.js';
 export * from './vad.js';
 export * from './version.js';
+export { createTimedString, isTimedString, type TimedString } from './voice/io.js';
 export * from './worker.js';
 
 export { cli, inference, ipc, llm, metrics, stream, stt, telemetry, tokenize, tts, voice };

agents/src/inference/stt.ts

@@ -16,7 +16,7 @@ import {
 } from '../stt/index.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
 import { type AudioBuffer, Event, Task, cancelAndWait, shortuuid, waitForAbort } from '../utils.js';
-import type { TimedString } from '../voice/io.js';
+import { type TimedString, createTimedString } from '../voice/io.js';
 import {
   type SttServerEvent,
   type SttTranscriptEvent,
@@ -489,13 +489,14 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
         confidence: data.confidence,
         text,
         words: data.words.map(
-          (word): TimedString => ({
-            text: word.word,
-            startTime: word.start + this.startTimeOffset,
-            endTime: word.end + this.startTimeOffset,
-            startTimeOffset: this.startTimeOffset,
-            confidence: word.confidence,
-          }),
+          (word): TimedString =>
+            createTimedString({
+              text: word.word,
+              startTime: word.start + this.startTimeOffset,
+              endTime: word.end + this.startTimeOffset,
+              startTimeOffset: this.startTimeOffset,
+              confidence: word.confidence,
+            }),
         ),
       };
 

agents/src/llm/realtime.ts

@@ -6,6 +6,7 @@ import { EventEmitter } from 'events';
 import type { ReadableStream } from 'node:stream/web';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { Task } from '../utils.js';
+import type { TimedString } from '../voice/io.js';
 import type { ChatContext, FunctionCall } from './chat_context.js';
 import type { ToolChoice, ToolContext } from './tool_context.js';
 
@@ -17,7 +18,10 @@ export interface InputSpeechStoppedEvent {
 
 export interface MessageGeneration {
   messageId: string;
-  textStream: ReadableStream<string>;
+  /**
+   * Text stream that may contain plain strings or TimedString objects with timestamps.
+   */
+  textStream: ReadableStream<string | TimedString>;
   audioStream: ReadableStream<AudioFrame>;
   modalities?: Promise<('text' | 'audio')[]>;
 }

agents/src/tts/stream_adapter.ts

@@ -3,7 +3,9 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';
 import type { APIConnectOptions } from '../types.js';
+import { USERDATA_TIMED_TRANSCRIPT } from '../types.js';
 import { Task } from '../utils.js';
+import { createTimedString } from '../voice/io.js';
 import type { ChunkedStream } from './tts.js';
 import { SynthesizeStream, TTS } from './tts.js';
 
@@ -13,7 +15,7 @@ export class StreamAdapter extends TTS {
   label: string;
 
   constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {
-    super(tts.sampleRate, tts.numChannels, { streaming: true });
+    super(tts.sampleRate, tts.numChannels, { streaming: true, alignedTranscript: true });
     this.#tts = tts;
     this.#sentenceTokenizer = sentenceTokenizer;
     this.label = this.#tts.label;
@@ -53,6 +55,8 @@ export class StreamAdapterWrapper extends SynthesizeStream {
   }
 
   protected async run() {
+    let cumulativeDuration = 0;
+
     const forwardInput = async () => {
       for await (const input of this.input) {
         if (this.abortController.signal.aborted) break;
@@ -99,8 +103,26 @@ export class StreamAdapterWrapper extends SynthesizeStream {
       await prevTask?.result;
       if (controller.signal.aborted) return;
 
+      // Create a TimedString with the sentence text and current cumulative duration
+      const timedString = createTimedString({
+        text: token,
+        startTime: cumulativeDuration,
+      });
+
+      let isFirstFrame = true;
       for await (const audio of audioStream) {
         if (controller.signal.aborted) break;
+
+        // Attach the TimedString to the first frame of this sentence
+        if (isFirstFrame) {
+          audio.frame.userdata[USERDATA_TIMED_TRANSCRIPT] = [timedString];
+          isFirstFrame = false;
+        }
+
+        // Track cumulative duration
+        const frameDuration = audio.frame.samplesPerChannel / audio.frame.sampleRate;
+        cumulativeDuration += frameDuration;
+
         this.queue.put(audio);
       }
     };

agents/src/tts/tts.ts

@@ -13,8 +13,11 @@ import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { recordException, traceTypes, tracer } from '../telemetry/index.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS, intervalForRetry } from '../types.js';
 import { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js';
+import type { TimedString } from '../voice/io.js';
 
-/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */
+/**
+ * SynthesizedAudio is a packet of speech synthesis as returned by the TTS.
+ */
 export interface SynthesizedAudio {
   /** Request ID (one segment could be made up of multiple requests) */
   requestId: string;
@@ -26,17 +29,27 @@ export interface SynthesizedAudio {
   deltaText?: string;
   /** Whether this is the last frame of the segment (streaming only) */
   final: boolean;
+  /**
+   * Timed transcripts associated with this audio packet (word-level timestamps).
+   */
+  timedTranscripts?: TimedString[];
 }
 
 /**
  * Describes the capabilities of the TTS provider.
+ *  tts/tts.py line 47-51
  *
  * @remarks
  * At present, only `streaming` is supplied to this interface, and the framework only supports
  * providers that do have a streaming endpoint.
  */
 export interface TTSCapabilities {
   streaming: boolean;
+  /**
+   * Whether this TTS supports aligned transcripts (word-level timestamps).
+   *  tts/tts.py line 50 - TTSCapabilities.aligned_transcript
+   */
+  alignedTranscript?: boolean;
 }
 
 export interface TTSError {

agents/src/types.ts

@@ -2,6 +2,11 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+/**
+ * Key used to store timed transcripts in AudioFrame.userdata.
+ */
+export const USERDATA_TIMED_TRANSCRIPT = 'lk.timed_transcripts';
+
 /**
  * Connection options for API calls, controlling retry and timeout behavior.
  */

agents/src/voice/agent.ts

@@ -26,9 +26,11 @@ import { StreamAdapter as STTStreamAdapter } from '../stt/index.js';
 import { SentenceTokenizer as BasicSentenceTokenizer } from '../tokenize/basic/index.js';
 import type { TTS } from '../tts/index.js';
 import { SynthesizeStream, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';
+import { USERDATA_TIMED_TRANSCRIPT } from '../types.js';
 import type { VAD } from '../vad.js';
 import type { AgentActivity } from './agent_activity.js';
 import type { AgentSession, TurnDetectionMode } from './agent_session.js';
+import type { TimedString } from './io.js';
 
 export const asyncLocalStorage = new AsyncLocalStorage<{ functionCall?: FunctionCall }>();
 export const STOP_RESPONSE_SYMBOL = Symbol('StopResponse');
@@ -70,6 +72,13 @@ export interface AgentOptions<UserData> {
   tts?: TTS | TTSModelString;
   allowInterruptions?: boolean;
   minConsecutiveSpeechDelay?: number;
+  /**
+   * Whether to use TTS-aligned transcripts for the transcription node input.
+   * When enabled and the TTS supports it, word-level timestamps from TTS
+   * will be forwarded to the transcription node instead of raw LLM text.
+   *  agent.py line 50, 80 - use_tts_aligned_transcript
+   */
+  useTtsAlignedTranscript?: boolean;
 }
 
 export class Agent<UserData = any> {
@@ -79,6 +88,10 @@ export class Agent<UserData = any> {
   private _vad?: VAD;
   private _llm?: LLM | RealtimeModel;
   private _tts?: TTS;
+  /**
+   * Whether to use TTS-aligned transcripts for the transcription node input.
+   */
+  private _useTtsAlignedTranscript?: boolean;
 
   /** @internal */
   _agentActivity?: AgentActivity;
@@ -102,6 +115,7 @@ export class Agent<UserData = any> {
     vad,
     llm,
     tts,
+    useTtsAlignedTranscript,
   }: AgentOptions<UserData>) {
     if (id) {
       this._id = id;
@@ -147,6 +161,8 @@ export class Agent<UserData = any> {
       this._tts = tts;
     }
 
+    this._useTtsAlignedTranscript = useTtsAlignedTranscript;
+
     this._agentActivity = undefined;
   }
 
@@ -166,6 +182,13 @@ export class Agent<UserData = any> {
     return this._tts;
   }
 
+  /**
+   * Whether to use TTS-aligned transcripts for the transcription node input.
+   */
+  get useTtsAlignedTranscript(): boolean | undefined {
+    return this._useTtsAlignedTranscript;
+  }
+
   get chatCtx(): ReadonlyChatContext {
     return new ReadonlyChatContext(this._chatCtx.items);
   }
@@ -190,10 +213,19 @@ export class Agent<UserData = any> {
 
   async onExit(): Promise<void> {}
 
+  /**
+   * Process transcription text (or TimedString) before outputting.
+   *
+   * @param text - The input text stream. When useTtsAlignedTranscript is enabled
+   *               and TTS supports aligned transcripts, this will be a stream of
+   *               TimedString objects with timing information.
+   * @param modelSettings - Model settings for the transcription node.
+   * @returns The processed text/TimedString stream, or null to disable transcription output.
+   */
   async transcriptionNode(
-    text: ReadableStream<string>,
+    text: ReadableStream<string | TimedString>,
     modelSettings: ModelSettings,
-  ): Promise<ReadableStream<string> | null> {
+  ): Promise<ReadableStream<string | TimedString> | null> {
     return Agent.default.transcriptionNode(this, text, modelSettings);
   }
 
@@ -395,6 +427,10 @@ export class Agent<UserData = any> {
               if (chunk === SynthesizeStream.END_OF_STREAM) {
                 break;
               }
+              // Attach timed transcripts to frame.userdata
+              if (chunk.timedTranscripts && chunk.timedTranscripts.length > 0) {
+                chunk.frame.userdata[USERDATA_TIMED_TRANSCRIPT] = chunk.timedTranscripts;
+              }
               controller.enqueue(chunk.frame);
             }
             controller.close();
@@ -410,9 +446,9 @@ export class Agent<UserData = any> {
 
     async transcriptionNode(
       agent: Agent,
-      text: ReadableStream<string>,
+      text: ReadableStream<string | TimedString>,
       _modelSettings: ModelSettings,
-    ): Promise<ReadableStream<string> | null> {
+    ): Promise<ReadableStream<string | TimedString> | null> {
       return text;
     },