Add support for tools to Phonic Plugin

.changeset/pink-items-dig.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents-plugin-phonic': patch
+---
+
+Add support for LiveKit tools to Phonic plugin

examples/src/phonic_realtime_agent.ts

@@ -1,20 +1,37 @@
 // SPDX-FileCopyrightText: 2026 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { type JobContext, ServerOptions, cli, defineAgent, voice } from '@livekit/agents';
+import { type JobContext, ServerOptions, cli, defineAgent, llm, voice } from '@livekit/agents';
 import * as phonic from '@livekit/agents-plugin-phonic';
 import { fileURLToPath } from 'node:url';
+import { z } from 'zod';
+
+const toggleLight = llm.tool({
+  description: 'Toggle a light on or off. Available lights are A05, A06, A07, and A08.',
+  parameters: z.object({
+    light_id: z.string().describe('The ID of the light to toggle'),
+    state: z.enum(['on', 'off']).describe('Whether to turn the light on or off'),
+  }),
+  execute: async ({ light_id, state }) => {
+    console.log(`Turning ${state} light ${light_id}`);
+    await new Promise((resolve) => setTimeout(resolve, 1_000));
+    return `Light ${light_id} turned ${state}`;
+  },
+});
 
 export default defineAgent({
   entry: async (ctx: JobContext) => {
     const agent = new voice.Agent({
       instructions: 'You are a helpful voice AI assistant named Alex.',
+      tools: {
+        toggle_light: toggleLight,
+      },
     });
 
     const session = new voice.AgentSession({
       // Uses PHONIC_API_KEY environment variable when apiKey is not provided
       llm: new phonic.realtime.RealtimeModel({
-        voice: 'virginia',
+        voice: 'sabrina',
         welcomeMessage: 'Hey there, how can I help you today?',
         audioSpeed: 1.2,
       }),

plugins/phonic/README.md

@@ -5,20 +5,36 @@ Realtime voice AI integration for [Phonic](https://phonic.co/) with LiveKit Agen
 ## Usage
 
 ```typescript
-import { type JobContext, ServerOptions, cli, defineAgent, voice } from '@livekit/agents';
+import { type JobContext, ServerOptions, cli, defineAgent, llm, voice } from '@livekit/agents';
 import * as phonic from '@livekit/agents-plugin-phonic';
 import { fileURLToPath } from 'node:url';
+import { z } from 'zod';
+
+const toggleLight = llm.tool({
+  description: 'Toggle a light on or off. Available lights are A05, A06, A07, and A08.',
+  parameters: z.object({
+    light_id: z.string().describe('The ID of the light to toggle'),
+    state: z.enum(['on', 'off']).describe('Whether to turn the light on or off'),
+  }),
+  execute: async ({ light_id, state }) => {
+    console.log(`Turning ${state} light ${light_id}`);
+    return `Light ${light_id} turned ${state}`;
+  },
+});
 
 export default defineAgent({
   entry: async (ctx: JobContext) => {
     const agent = new voice.Agent({
       instructions: 'You are a helpful voice AI assistant named Alex.',
+      tools: {
+        toggle_light: toggleLight,
+      },
     });
 
     const session = new voice.AgentSession({
       // Uses PHONIC_API_KEY environment variable when apiKey is not provided
       llm: new phonic.realtime.RealtimeModel({
-        voice: 'virginia',
+        voice: 'sabrina',
         welcomeMessage: 'Hey there, how can I help you today?',
         audioSpeed: 1.2,
       }),

plugins/phonic/package.json

@@ -41,7 +41,7 @@
     "typescript": "^5.0.0"
   },
   "dependencies": {
-    "phonic": "^0.30.37"
+    "phonic": "^0.30.39"
   },
   "peerDependencies": {
     "@livekit/agents": "workspace:*",

plugins/phonic/src/realtime/realtime_model.ts

@@ -5,6 +5,7 @@ import type { APIConnectOptions } from '@livekit/agents';
 import {
   AudioByteStream,
   DEFAULT_API_CONNECT_OPTIONS,
+  Future,
   llm,
   log,
   shortuuid,
@@ -21,6 +22,7 @@ const PHONIC_NUM_CHANNELS = 1;
 const PHONIC_INPUT_FRAME_MS = 20;
 const DEFAULT_MODEL = 'merritt';
 const WS_CLOSE_NORMAL = 1000;
+const TOOL_CALL_OUTPUT_TIMEOUT_MS = 60_000;
 
 export interface RealtimeModelOptions {
   apiKey: string;
@@ -125,8 +127,6 @@ export class RealtimeModel extends llm.RealtimeModel {
       messageTruncation: false,
       turnDetection: true,
       userTranscription: true,
-      // TODO @Phonic-Co: Implement tool support
-      // Phonic has automatic tool reply generation, but tools are not supported with LiveKit Agents yet.
       autoToolReplyGeneration: true,
       manualFunctionCalls: false,
       audioOutput: true,
@@ -197,19 +197,17 @@ export class RealtimeSession extends llm.RealtimeSession {
   private logger = log();
   private closed = false;
   private configSent = false;
-  private instructionsReady: Promise<void>;
-  private resolveInstructionsReady: () => void;
+  private instructionsReady = new Future<void>();
+  private toolsReady = new Future<void>();
   private connectTask: Promise<void>;
+  private toolDefinitions: Record<string, unknown>[] = [];
+  private pendingToolCallIds = new Set<string>();
+  private readyToStart = false;
 
   constructor(realtimeModel: RealtimeModel) {
     super(realtimeModel);
     this.options = realtimeModel._options;
 
-    this.resolveInstructionsReady = () => {};
-    this.instructionsReady = new Promise<void>((resolve) => {
-      this.resolveInstructionsReady = resolve;
-    });
-
     this.client = new PhonicClient({
       apiKey: this.options.apiKey,
       baseUrl: this.options.baseUrl,
@@ -241,25 +239,70 @@ export class RealtimeSession extends llm.RealtimeSession {
       return;
     }
     this.options.instructions = instructions;
-    this.resolveInstructionsReady();
+    this.instructionsReady.resolve();
   }
 
-  async updateChatCtx(_chatCtx: llm.ChatContext): Promise<void> {
-    this.logger.warn('updateChatCtx is not supported by the Phonic realtime model.');
+  async updateChatCtx(chatCtx: llm.ChatContext): Promise<void> {
+    let sent = false;
+    for (const item of chatCtx.items) {
+      if (item.type === 'function_call_output' && this.pendingToolCallIds.has(item.callId)) {
+        this.pendingToolCallIds.delete(item.callId);
+        this.logger.info(`Sending tool call output for ${item.name} (call_id: ${item.callId})`);
+        this.socket?.sendToolCallOutput({
+          type: 'tool_call_output',
+          tool_call_id: item.callId,
+          output: item.output,
+        });
+        sent = true;
+      }
+    }
+    if (!sent) {
+      this.logger.warn(
+        'updateChatCtx called but no new tool call outputs to send. Phonic does not support general chat context updates.',
+      );
+    } else {
+      this.startNewAssistantTurn();
+    }
   }
 
   async updateTools(tools: llm.ToolContext): Promise<void> {
-    if (Object.keys(tools).length > 0) {
-      this.logger.warn('Tool use is not supported by the Phonic realtime model.');
+    if (this.configSent) {
+      this.logger.warn(
+        'updateTools called after config was already sent. Phonic does not support updating tools mid-session.',
+      );
+      return;
     }
+
+    this._tools = { ...tools };
+    this.toolDefinitions = Object.entries(tools)
+      .filter(([_, tool]) => llm.isFunctionTool(tool))
+      .map(([name, tool]) => ({
+        type: 'custom_websocket',
+        tool_schema: {
+          type: 'function',
+          function: {
+            name,
+            description: tool.description,
+            parameters: llm.toJsonSchema(tool.parameters),
+            strict: true,
+          },
+        },
+        tool_call_output_timeout_ms: TOOL_CALL_OUTPUT_TIMEOUT_MS,
+        // Tool chaining and tool calls during speech are not supported at this time
+        // for ease of implementation within the RealtimeSession generations framework
+        wait_for_speech_before_tool_call: true,
+        allow_tool_chaining: false,
+      }));
+
+    this.toolsReady.resolve();
   }
 
   updateOptions(_options: { toolChoice?: llm.ToolChoice | null }): void {
     this.logger.warn('updateOptions is not supported by the Phonic realtime model.');
   }
 
   pushAudio(frame: AudioFrame): void {
-    if (this.closed) {
+    if (this.closed || !this.readyToStart) {
       return;
     }
 
@@ -294,7 +337,9 @@ export class RealtimeSession extends llm.RealtimeSession {
   }
 
   async interrupt(): Promise<void> {
-    this.logger.warn('interrupt is not supported by the Phonic realtime model.');
+    this.logger.warn(
+      'interrupt() is not supported by Phonic realtime model. User interruptions are automatically handled by Phonic.',
+    );
   }
 
   async truncate(_options: { messageId: string; audioEndMs: number; audioTranscript?: string }) {
@@ -303,7 +348,8 @@ export class RealtimeSession extends llm.RealtimeSession {
 
   async close(): Promise<void> {
     this.closed = true;
-    this.resolveInstructionsReady();
+    this.instructionsReady.resolve();
+    this.toolsReady.resolve();
     this.closeCurrentGeneration({ interrupted: false });
     this.socket?.close();
     await this.connectTask;
@@ -332,8 +378,10 @@ export class RealtimeSession extends llm.RealtimeSession {
     });
 
     await this.socket.waitForOpen();
-    await this.instructionsReady;
+    await this.instructionsReady.await;
+    await this.toolsReady.await;
     if (this.closed) return;
+
     this.configSent = true;
     this.socket.sendConfig({
       type: 'config',
@@ -348,7 +396,7 @@ export class RealtimeSession extends llm.RealtimeSession {
       output_format: 'pcm_44100',
       recognized_languages: this.options.languages,
       audio_speed: this.options.audioSpeed,
-      tools: this.options.phonicTools,
+      tools: [...(this.options.phonicTools ?? []), ...this.toolDefinitions],
       boosted_keywords: this.options.boostedKeywords,
       generate_no_input_poke_text: this.options.generateNoInputPokeText,
       no_input_poke_sec: this.options.noInputPokeSec,
@@ -381,17 +429,12 @@ export class RealtimeSession extends llm.RealtimeSession {
       case 'user_finished_speaking':
         this.handleInputSpeechStopped();
         break;
+      case 'tool_call':
+        this.handleToolCall(message);
+        break;
       case 'error':
         this.emitError(new Error(message.error.message), false);
         break;
-      case 'tool_call':
-        this.emitError(
-          new Error(
-            `WebSocket tool calls are not yet supported by the Phonic realtime model with LiveKit Agents.`,
-          ),
-          false,
-        );
-        break;
       case 'assistant_ended_conversation':
         this.emitError(
           new Error(
@@ -404,11 +447,15 @@ export class RealtimeSession extends llm.RealtimeSession {
         this.conversationId = message.conversation_id;
         this.logger.info(`Phonic Conversation began with ID: ${this.conversationId}`);
         break;
-      case 'assistant_chose_not_to_respond':
+      case 'tool_call_interrupted':
+        this.handleToolCallInterrupted(message);
+        break;
       case 'ready_to_start_conversation':
+        this.readyToStart = true;
+        break;
+      case 'assistant_chose_not_to_respond':
       case 'input_cancelled':
       case 'tool_call_output_processed':
-      case 'tool_call_interrupted':
       case 'dtmf':
       default:
         break;
@@ -420,8 +467,13 @@ export class RealtimeSession extends llm.RealtimeSession {
      * Although Phonic sends audio chunks when the assistant is not speaking (i.e. containing silence or background noise),
      * we only process the chunks when the assistant is speaking to align with the generations model, whereby new streams are created for each turn.
      */
+    if (this.currentGeneration === undefined && message.text) {
+      this.logger.debug('Starting new generation due to text in audio chunk');
+      this.startNewAssistantTurn();
+    }
+
     const gen = this.currentGeneration;
-    if (!gen) return;
+    if (gen === undefined) return;
 
     if (message.text) {
       gen.outputText += message.text;
@@ -464,6 +516,32 @@ export class RealtimeSession extends llm.RealtimeSession {
     });
   }
 
+  private handleToolCall(message: Phonic.ToolCallPayload): void {
+    this.pendingToolCallIds.add(message.tool_call_id);
+
+    if (this.currentGeneration === undefined) {
+      this.logger.warn('Encountered tool call but no active generation. Starting new turn.');
+      this.startNewAssistantTurn();
+    }
+
+    this.currentGeneration!.functionChannel.write(
+      llm.FunctionCall.create({
+        callId: message.tool_call_id,
+        name: message.tool_name,
+        args: JSON.stringify(message.parameters),
+      }),
+    );
+    // At most 1 tool call is supported per turn due to `toolChaining: false`, allowing us to close the generation
+    this.closeCurrentGeneration({ interrupted: false });
+  }
+
+  private handleToolCallInterrupted(message: Phonic.ToolCallInterruptedPayload): void {
+    this.pendingToolCallIds.delete(message.tool_call_id);
+    this.logger.warn(
+      `Tool call for ${message.tool_name} (call_id: ${message.tool_call_id}) was cancelled due to user interruption.`,
+    );
+  }
+
   private handleInputSpeechStarted(): void {
     this.emit('input_speech_started', {});
     this.closeCurrentGeneration({ interrupted: true });