Skip to content

Commit b1c8ba6

Browse files
committed
Merge branch 'main' into lukas/hanging-tee
2 parents 185e21e + a5bb4ff commit b1c8ba6

9 files changed

Lines changed: 74 additions & 16 deletions

File tree

.changeset/itchy-rocks-cheer.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@livekit/agents': patch
3+
---
4+
5+
Add AEC warmup functionality to AgentSession and AgentActivity

.changeset/silver-days-perform.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@livekit/agents-plugin-google": patch
3+
---
4+
5+
fix(google): move generateContentStream inside try/catch in Gemini TTS

.github/banner_dark.png

29 Bytes
Loading

.github/banner_light.png

412 Bytes
Loading

README.md

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,6 @@ understand.
2525
This is a Node.js distribution of the [LiveKit Agents framework](https://livekit.io/agents),
2626
originally written in Python.
2727

28-
Looking for the Python library? Check out [Agents](https://github.com/livekit/agents).
29-
3028
<!--END_DESCRIPTION-->
3129

3230
## ✨ 1.0 Release ✨
@@ -366,17 +364,17 @@ This project is licensed under `Apache-2.0`, and is [REUSE-3.2](https://reuse.so
366364
Refer to [the license](LICENSES/Apache-2.0.txt) for details.
367365

368366
<!--BEGIN_REPO_NAV-->
369-
370367
<br/><table>
371-
372368
<thead><tr><th colspan="2">LiveKit Ecosystem</th></tr></thead>
373369
<tbody>
374-
<tr><td>LiveKit SDKs</td><td><a href="https://github.com/livekit/client-sdk-js">Browser</a> · <a href="https://github.com/livekit/client-sdk-swift">iOS/macOS/visionOS</a> · <a href="https://github.com/livekit/client-sdk-android">Android</a> · <a href="https://github.com/livekit/client-sdk-flutter">Flutter</a> · <a href="https://github.com/livekit/client-sdk-react-native">React Native</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/client-sdk-unity">Unity</a> · <a href="https://github.com/livekit/client-sdk-unity-web">Unity (WebGL)</a></td></tr><tr></tr>
370+
<tr><td>Agents SDKs</td><td><a href="https://github.com/livekit/agents">Python</a> · <b>Node.js</b></td></tr><tr></tr>
371+
<tr><td>LiveKit SDKs</td><td><a href="https://github.com/livekit/client-sdk-js">Browser</a> · <a href="https://github.com/livekit/client-sdk-swift">Swift</a> · <a href="https://github.com/livekit/client-sdk-android">Android</a> · <a href="https://github.com/livekit/client-sdk-flutter">Flutter</a> · <a href="https://github.com/livekit/client-sdk-react-native">React Native</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/client-sdk-unity">Unity</a> · <a href="https://github.com/livekit/client-sdk-unity-web">Unity (WebGL)</a> · <a href="https://github.com/livekit/client-sdk-esp32">ESP32</a> · <a href="https://github.com/livekit/client-sdk-cpp">C++</a></td></tr><tr></tr>
372+
<tr><td>Starter Apps</td><td><a href="https://github.com/livekit-examples/agent-starter-python">Python Agent</a> · <a href="https://github.com/livekit-examples/agent-starter-node">TypeScript Agent</a> · <a href="https://github.com/livekit-examples/agent-starter-react">React App</a> · <a href="https://github.com/livekit-examples/agent-starter-swift">SwiftUI App</a> · <a href="https://github.com/livekit-examples/agent-starter-android">Android App</a> · <a href="https://github.com/livekit-examples/agent-starter-flutter">Flutter App</a> · <a href="https://github.com/livekit-examples/agent-starter-react-native">React Native App</a> · <a href="https://github.com/livekit-examples/agent-starter-embed">Web Embed</a></td></tr><tr></tr>
373+
<tr><td>UI Components</td><td><a href="https://github.com/livekit/components-js">React</a> · <a href="https://github.com/livekit/components-android">Android Compose</a> · <a href="https://github.com/livekit/components-swift">SwiftUI</a> · <a href="https://github.com/livekit/components-flutter">Flutter</a></td></tr><tr></tr>
375374
<tr><td>Server APIs</td><td><a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/server-sdk-go">Golang</a> · <a href="https://github.com/livekit/server-sdk-ruby">Ruby</a> · <a href="https://github.com/livekit/server-sdk-kotlin">Java/Kotlin</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/agence104/livekit-server-sdk-php">PHP (community)</a> · <a href="https://github.com/pabloFuente/livekit-server-sdk-dotnet">.NET (community)</a></td></tr><tr></tr>
376-
<tr><td>UI Components</td><td><a href="https://github.com/livekit/components-js">React</a> · <a href="https://github.com/livekit/components-android">Android Compose</a> · <a href="https://github.com/livekit/components-swift">SwiftUI</a></td></tr><tr></tr>
377-
<tr><td>Agents Frameworks</td><td><a href="https://github.com/livekit/agents">Python</a> · <b>Node.js</b> · <a href="https://github.com/livekit/agent-playground">Playground</a></td></tr><tr></tr>
378-
<tr><td>Services</td><td><a href="https://github.com/livekit/livekit">LiveKit server</a> · <a href="https://github.com/livekit/egress">Egress</a> · <a href="https://github.com/livekit/ingress">Ingress</a> · <a href="https://github.com/livekit/sip">SIP</a></td></tr><tr></tr>
379-
<tr><td>Resources</td><td><a href="https://docs.livekit.io">Docs</a> · <a href="https://github.com/livekit-examples">Example apps</a> · <a href="https://livekit.io/cloud">Cloud</a> · <a href="https://docs.livekit.io/home/self-hosting/deployment">Self-hosting</a> · <a href="https://github.com/livekit/livekit-cli">CLI</a></td></tr>
375+
<tr><td>Resources</td><td><a href="https://docs.livekit.io">Docs</a> · <a href="https://docs.livekit.io/mcp">Docs MCP Server</a> · <a href="https://github.com/livekit/livekit-cli">CLI</a> · <a href="https://cloud.livekit.io">LiveKit Cloud</a></td></tr><tr></tr>
376+
<tr><td>LiveKit Server OSS</td><td><a href="https://github.com/livekit/livekit">LiveKit server</a> · <a href="https://github.com/livekit/egress">Egress</a> · <a href="https://github.com/livekit/ingress">Ingress</a> · <a href="https://github.com/livekit/sip">SIP</a></td></tr><tr></tr>
377+
<tr><td>Community</td><td><a href="https://community.livekit.io">Developer Community</a> · <a href="https://livekit.io/join-slack">Slack</a> · <a href="https://x.com/livekit">X</a> · <a href="https://www.youtube.com/@livekit_io">YouTube</a></td></tr>
380378
</tbody>
381379
</table>
382380
<!--END_REPO_NAV-->

agents/src/voice/agent_activity.ts

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import type { Span } from '@opentelemetry/api';
77
import { ROOT_CONTEXT, context as otelContext, trace } from '@opentelemetry/api';
88
import { Heap } from 'heap-js';
99
import { AsyncLocalStorage } from 'node:async_hooks';
10-
import { ReadableStream } from 'node:stream/web';
10+
import { ReadableStream, TransformStream } from 'node:stream/web';
1111
import { type ChatContext, ChatMessage } from '../llm/chat_context.js';
1212
import {
1313
type ChatItem,
@@ -485,6 +485,20 @@ export class AgentActivity implements RecognitionHooks {
485485
void this.audioStream.close();
486486
this.audioStream = new MultiInputStream<AudioFrame>();
487487

488+
// Filter is applied on this.audioStream.stream (downstream of MultiInputStream) rather
489+
// than on the source audioStream via pipeThrough. pipeThrough locks its source stream, so
490+
// if it were applied directly on audioStream, that lock would survive MultiInputStream.close()
491+
// and make audioStream permanently locked for subsequent attachAudioInput calls (e.g. handoff).
492+
const aecWarmupAudioFilter = new TransformStream<AudioFrame, AudioFrame>({
493+
transform: (frame, controller) => {
494+
const shouldDiscardForAecWarmup =
495+
this.agentSession.agentState === 'speaking' && this.agentSession._aecWarmupRemaining > 0;
496+
if (!shouldDiscardForAecWarmup) {
497+
controller.enqueue(frame);
498+
}
499+
},
500+
});
501+
488502
this.audioStreamId = this.audioStream.addInputStream(audioStream);
489503

490504
if (this.realtimeSession && this.audioRecognition) {
@@ -756,6 +770,11 @@ export class AgentActivity implements RecognitionHooks {
756770
}
757771

758772
private interruptByAudioActivity(): void {
773+
if (this.agentSession._aecWarmupRemaining > 0) {
774+
// Disable interruption from audio activity while AEC warmup is active.
775+
return;
776+
}
777+
759778
if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
760779
// skip speech handle interruption if server side turn detection is enabled
761780
return;

agents/src/voice/agent_session.ts

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ export interface VoiceOptions {
7777
maxToolSteps: number;
7878
preemptiveGeneration: boolean;
7979
userAwayTimeout?: number | null;
80+
aecWarmupDuration: number | null;
8081
useTtsAlignedTranscript: boolean;
8182
}
8283

@@ -90,6 +91,7 @@ const defaultVoiceOptions: VoiceOptions = {
9091
maxToolSteps: 3,
9192
preemptiveGeneration: false,
9293
userAwayTimeout: 15.0,
94+
aecWarmupDuration: 3000,
9395
useTtsAlignedTranscript: true,
9496
} as const;
9597

@@ -158,6 +160,8 @@ export class AgentSession<
158160
private closingTask: Promise<void> | null = null;
159161
private userAwayTimer: NodeJS.Timeout | null = null;
160162

163+
private _aecWarmupTimer: NodeJS.Timeout | null = null;
164+
161165
// Connection options for STT, LLM, and TTS
162166
private _connOptions: ResolvedSessionConnectOptions;
163167

@@ -169,6 +173,9 @@ export class AgentSession<
169173
private userSpeakingSpan?: Span;
170174
private agentSpeakingSpan?: Span;
171175

176+
/** @internal */
177+
_aecWarmupRemaining = 0;
178+
172179
/** @internal */
173180
_recorderIO?: RecorderIO;
174181

@@ -241,6 +248,7 @@ export class AgentSession<
241248
// This is the "global" chat context, it holds the entire conversation history
242249
this._chatCtx = ChatContext.empty();
243250
this.options = { ...defaultVoiceOptions, ...voiceOptions };
251+
this._aecWarmupRemaining = this.options.aecWarmupDuration ?? 0;
244252

245253
this._onUserInputTranscribed = this._onUserInputTranscribed.bind(this);
246254
this.on(AgentSessionEventTypes.UserInputTranscribed, this._onUserInputTranscribed);
@@ -845,6 +853,14 @@ export class AgentSession<
845853
this.agentSpeakingSpan = undefined;
846854
}
847855

856+
if (state === 'speaking' && this._aecWarmupRemaining > 0 && this._aecWarmupTimer === null) {
857+
this._aecWarmupTimer = setTimeout(() => this._onAecWarmupExpired(), this._aecWarmupRemaining);
858+
this.logger.debug(
859+
{ warmupDurationMs: this._aecWarmupRemaining },
860+
'aec warmup active, disabling interruptions',
861+
);
862+
}
863+
848864
const oldState = this._agentState;
849865
this._agentState = state;
850866

@@ -938,6 +954,19 @@ export class AgentSession<
938954
}
939955
}
940956

957+
/** @internal */
958+
_onAecWarmupExpired(): void {
959+
if (this._aecWarmupRemaining > 0) {
960+
this.logger.debug('aec warmup expired, re-enabling interruptions');
961+
}
962+
963+
this._aecWarmupRemaining = 0;
964+
if (this._aecWarmupTimer !== null) {
965+
clearTimeout(this._aecWarmupTimer);
966+
this._aecWarmupTimer = null;
967+
}
968+
}
969+
941970
private _onUserInputTranscribed(ev: UserInputTranscribedEvent): void {
942971
if (this.userState === 'away' && ev.isFinal) {
943972
this.logger.debug('User returned from away state due to speech input');
@@ -969,6 +998,7 @@ export class AgentSession<
969998
}
970999

9711000
this._cancelUserAwayTimer();
1001+
this._onAecWarmupExpired();
9721002
this.off(AgentSessionEventTypes.UserInputTranscribed, this._onUserInputTranscribed);
9731003

9741004
if (this.activity) {

examples/src/basic_agent.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ export default defineAgent({
7171
// allow the LLM to generate a response while waiting for the end of turn
7272
preemptiveGeneration: true,
7373
useTtsAlignedTranscript: true,
74+
aecWarmupDuration: 3000,
7475
},
7576
connOptions: {
7677
// Example of overriding the default connection options for the LLM/TTS/STT

plugins/google/src/beta/gemini_tts.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -213,13 +213,13 @@ export class ChunkedStream extends tts.ChunkedStream {
213213
},
214214
];
215215

216-
const responseStream = await this.#tts.client.models.generateContentStream({
217-
model: this.#tts.opts.model,
218-
contents,
219-
config,
220-
});
221-
222216
try {
217+
const responseStream = await this.#tts.client.models.generateContentStream({
218+
model: this.#tts.opts.model,
219+
contents,
220+
config,
221+
});
222+
223223
for await (const response of responseStream) {
224224
await this.#processResponse(response, bstream, requestId);
225225
}

0 commit comments

Comments
 (0)