fix(google): add Gemini 3.1 Flash Live compatibility

Author: swayamg20

.changeset/fix-gemini-31-live-api.md

@@ -0,0 +1,13 @@
+---
+'@livekit/agents-plugin-google': patch
+---
+
+fix: compatibility with gemini-3.1-flash-live-preview (#1179)
+
+Two paths were broken with Gemini 3.1:
+
+1. **`pushAudio()`** — was sending `mediaChunks` (deprecated); now sends the `audio` field directly, which Gemini 3.1 requires.
+
+2. **`generateReply()`** — was using `sendClientContent` to trigger generation; Gemini 3.1 rejects this with "Request contains an invalid argument." Now uses `sendRealtimeInput({ text })` instead, which works across all Live API models.
+
+Also fixes spurious empty `tools` and `sessionResumption` fields being sent in the session setup config when they have no values, which could cause rejections on strict model versions.

plugins/google/src/beta/realtime/api_proto.ts

@@ -21,6 +21,7 @@ export type LiveAPIModels =
   // Gemini API models
   | 'gemini-2.5-flash-native-audio-preview-12-2025' // https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-live
   | 'gemini-2.5-flash-native-audio-preview-09-2025' // https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-live
+  | 'gemini-3.1-flash-live-preview' // https://ai.google.dev/gemini-api/docs/models
   | 'gemini-2.0-flash-exp'; // still works in Gemini API but not VertexAI
 
 /**

plugins/google/src/beta/realtime/realtime_api.ts

@@ -660,12 +660,10 @@ export class RealtimeSession extends llm.RealtimeSession {
     for (const f of this.resampleAudio(frame)) {
       for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {
         const realtimeInput: types.LiveClientRealtimeInput = {
-          mediaChunks: [
-            {
-              mimeType: 'audio/pcm',
-              data: Buffer.from(nf.data.buffer).toString('base64'),
-            },
-          ],
+          audio: {
+            mimeType: 'audio/pcm',
+            data: Buffer.from(nf.data.buffer).toString('base64'),
+          },
         };
         this.sendClientEvent({
           type: 'realtime_input',
@@ -704,26 +702,12 @@ export class RealtimeSession extends llm.RealtimeSession {
       this.inUserActivity = false;
     }
 
-    // Gemini requires the last message to end with user's turn
-    // so we need to add a placeholder user turn in order to trigger a new generation
-    const turns: types.Content[] = [];
-    if (instructions !== undefined) {
-      turns.push({
-        parts: [{ text: instructions }],
-        role: 'model',
-      });
-    }
-    turns.push({
-      parts: [{ text: '.' }],
-      role: 'user',
-    });
-
+    // Trigger generation via realtime text input.
+    // Gemini 3.1+ rejects sendClientContent — sendRealtimeInput({ text }) works on all Live models.
+    // Instructions are included in the text so the model uses them as context for the next reply.
     this.sendClientEvent({
-      type: 'content',
-      value: {
-        turns,
-        turnComplete: true,
-      },
+      type: 'realtime_input',
+      value: { text: instructions ?? '.' },
     });
 
     const timeoutHandle = setTimeout(() => {
@@ -992,12 +976,10 @@ export class RealtimeSession extends llm.RealtimeSession {
             }
             break;
           case 'realtime_input':
-            const { mediaChunks, activityStart, activityEnd, text } = msg.value;
+            const { audio, activityStart, activityEnd, text } = msg.value;
             if (this.pendingToolCallIds.size > 0) break;
-            if (mediaChunks) {
-              for (const mediaChunk of mediaChunks) {
-                await session.sendRealtimeInput({ media: mediaChunk });
-              }
+            if (audio) {
+              await session.sendRealtimeInput({ audio });
             }
             if (text) {
               await session.sendRealtimeInput({ text });
@@ -1141,15 +1123,14 @@ export class RealtimeSession extends llm.RealtimeSession {
     maxLength: number = 30,
   ): Record<string, unknown> {
     const obj: any = { ...event };
-    if (obj.type === 'realtime_input' && obj.value?.mediaChunks) {
+    if (obj.type === 'realtime_input' && obj.value?.audio) {
+      const mc = obj.value.audio as { mimeType?: string; data?: string };
       obj.value = {
         ...obj.value,
-        mediaChunks: (obj.value.mediaChunks as Array<{ mimeType?: string; data?: string }>).map(
-          (mc) => ({
-            ...mc,
-            data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,
-          }),
-        ),
+        audio: {
+          ...mc,
+          data: typeof mc.data === 'string' ? this.truncateString(mc.data, maxLength) : mc.data,
+        },
       };
     }
     return obj;
@@ -1265,17 +1246,15 @@ export class RealtimeSession extends llm.RealtimeSession {
         },
         languageCode: opts.language,
       },
-      tools: [
-        {
-          functionDeclarations: this.geminiDeclarations,
-          ...this.options.geminiTools,
-        },
-      ],
+      tools:
+        this.geminiDeclarations.length > 0 || this.options.geminiTools
+          ? [{ functionDeclarations: this.geminiDeclarations, ...this.options.geminiTools }]
+          : undefined,
       inputAudioTranscription: opts.inputAudioTranscription,
       outputAudioTranscription: opts.outputAudioTranscription,
-      sessionResumption: {
-        handle: this.sessionResumptionHandle,
-      },
+      sessionResumption: this.sessionResumptionHandle
+        ? { handle: this.sessionResumptionHandle }
+        : undefined,
     };
 
     // Add generation fields at TOP LEVEL (NO generationConfig!)