fix(google): align realtime behavior for Gemini 3.1 and harden session/tool handling (#1189)

Author: toubatbrian

.changeset/clever-feet-rush.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents-plugin-google": patch
+---
+
+fix(google): align realtime behavior for Gemini 3.1 and harden session/tool handling

plugins/google/src/beta/realtime/api_proto.ts

@@ -19,6 +19,7 @@ export type LiveAPIModels =
   | 'gemini-live-2.5-flash-preview-native-audio-09-2025' // Public preview https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash-live-api#live-2.5-flash-preview
   | 'gemini-live-2.5-flash-preview-native-audio' // still works, possibly an alias, but not mentioned in any docs or changelog
   // Gemini API models
+  | 'gemini-3.1-flash-live-preview' // https://ai.google.dev/gemini-api/docs/models/gemini-3.1-flash-live-preview
   | 'gemini-2.5-flash-native-audio-preview-12-2025' // https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-live
   | 'gemini-2.5-flash-native-audio-preview-09-2025' // https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-live
   | 'gemini-2.0-flash-exp'; // still works in Gemini API but not VertexAI

plugins/google/src/beta/realtime/realtime_api.ts

@@ -660,12 +660,10 @@ export class RealtimeSession extends llm.RealtimeSession {
     for (const f of this.resampleAudio(frame)) {
       for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {
         const realtimeInput: types.LiveClientRealtimeInput = {
-          mediaChunks: [
-            {
-              mimeType: 'audio/pcm',
-              data: Buffer.from(nf.data.buffer).toString('base64'),
-            },
-          ],
+          audio: {
+            mimeType: 'audio/pcm',
+            data: Buffer.from(nf.data.buffer).toString('base64'),
+          },
         };
         this.sendClientEvent({
           type: 'realtime_input',
@@ -684,6 +682,13 @@ export class RealtimeSession extends llm.RealtimeSession {
   }
 
   async generateReply(instructions?: string): Promise<llm.GenerationCreatedEvent> {
+    if (this.options.model === 'gemini-3.1-flash-live-preview') {
+      this.#logger.warn(
+        'generateReply is not compatible with gemini-3.1-flash-live-preview and will be ignored.',
+      );
+      throw new Error("generateReply is not compatible with 'gemini-3.1-flash-live-preview'");
+    }
+
     if (this.pendingGenerationFut && !this.pendingGenerationFut.done) {
       this.#logger.warn(
         'generateReply called while another generation is pending, cancelling previous.',
@@ -992,13 +997,16 @@ export class RealtimeSession extends llm.RealtimeSession {
             }
             break;
           case 'realtime_input':
-            const { mediaChunks, activityStart, activityEnd, text } = msg.value;
+            const { mediaChunks, audio, activityStart, activityEnd, text } = msg.value;
             if (this.pendingToolCallIds.size > 0) break;
             if (mediaChunks) {
               for (const mediaChunk of mediaChunks) {
                 await session.sendRealtimeInput({ media: mediaChunk });
               }
             }
+            if (audio) {
+              await session.sendRealtimeInput({ audio });
+            }
             if (text) {
               await session.sendRealtimeInput({ text });
             }
@@ -1152,6 +1160,16 @@ export class RealtimeSession extends llm.RealtimeSession {
         ),
       };
     }
+    if (obj.type === 'realtime_input' && obj.value?.audio) {
+      const ac = obj.value.audio as { mimeType?: string; data?: string };
+      obj.value = {
+        ...obj.value,
+        audio: {
+          ...ac,
+          data: typeof ac.data === 'string' ? this.truncateString(ac.data, maxLength) : ac.data,
+        },
+      };
+    }
     return obj;
   }
 
@@ -1265,17 +1283,15 @@ export class RealtimeSession extends llm.RealtimeSession {
         },
         languageCode: opts.language,
       },
-      tools: [
-        {
-          functionDeclarations: this.geminiDeclarations,
-          ...this.options.geminiTools,
-        },
-      ],
+      tools:
+        this.geminiDeclarations.length > 0 || this.options.geminiTools
+          ? [{ functionDeclarations: this.geminiDeclarations, ...this.options.geminiTools }]
+          : undefined,
       inputAudioTranscription: opts.inputAudioTranscription,
       outputAudioTranscription: opts.outputAudioTranscription,
-      sessionResumption: {
-        handle: this.sessionResumptionHandle,
-      },
+      sessionResumption: this.sessionResumptionHandle
+        ? { handle: this.sessionResumptionHandle }
+        : {},
     };
 
     // Add generation fields at TOP LEVEL (NO generationConfig!)
@@ -1527,6 +1543,8 @@ export class RealtimeSession extends llm.RealtimeSession {
         }),
       );
     }
+    // This closes message/text/audio/function streams so the consumer can continue.
+    this.markCurrentGenerationDone();
   }
 
   private handleToolCallCancellation(cancellation: types.LiveServerToolCallCancellation): void {