feat(phonic): handle user text input via generate_reply

examples/voice_agents/phonic_realtime_agent.py

@@ -46,7 +46,7 @@ async def toggle_light(self, light_id: str, state: str) -> str:
 server = AgentServer()
 
 
-@server.rtc_session()
+@server.rtc_session(agent_name="my-agent-june-18")
 async def entrypoint(ctx: JobContext):
     session = AgentSession()
     await session.start(agent=MyAgent(), room=ctx.room)

livekit-plugins/livekit-plugins-phonic/livekit/plugins/phonic/realtime/realtime_model.py

@@ -292,6 +292,7 @@ def __init__(self, realtime_model: RealtimeModel) -> None:
         self._tool_definitions: list[dict] = []
         self._forbid_speech_after_tool_call: set[str] = set()
         self._system_prompt_postfix: str = ""
+        self._pending_user_text: str | None = None
 
     async def _close_active_session(self) -> None:
         async with self._session_lock:
@@ -345,7 +346,9 @@ async def update_chat_ctx(self, chat_ctx: llm.ChatContext) -> None:
         diff_ops = llm.utils.compute_chat_ctx_diff(self._chat_ctx, chat_ctx)
         sent_tool_call_output = False
         sent_system_message = False
+        buffered_user_text = False
         forbid_speech = False
+        last_item_id = chat_ctx.items[-1].id if chat_ctx.items else None
 
         for _, item_id in diff_ops.to_create:
             item = chat_ctx.get_by_id(item_id)
@@ -379,9 +382,21 @@ async def update_chat_ctx(self, chat_ctx: llm.ChatContext) -> None:
                         )
                         sent_system_message = True
 
+            # Only treat a user message as text input when it's appended at the tail of the context.
+            if (
+                isinstance(item, llm.ChatMessage)
+                and item.role == "user"
+                and item_id == last_item_id
+            ):
+                text = item.text_content
+                if text:
+                    logger.info(f"Received user text input: {text}")
+                    self._pending_user_text = text
+                    buffered_user_text = True
+
         self._chat_ctx = chat_ctx.copy()
 
-        if not sent_tool_call_output and not sent_system_message:
+        if not sent_tool_call_output and not sent_system_message and not buffered_user_text:
             logger.warning(
                 "update_chat_ctx called but no new tool call outputs to send. "
                 "Phonic does not support general chat context updates."
@@ -545,6 +560,11 @@ def say(
             self._generate_reply_task.cancel()
         self._generate_reply_task = asyncio.create_task(self._send_say(text), name="phonic-say")
 
+        # say() drives speech directly and supersedes any buffered user text turn. Drop it
+        # here since _send_say won't consume it, otherwise it would leak into a later
+        # generate_reply.
+        self._pending_user_text = None
+
         self._close_current_generation(interrupted=False)
 
         if self._pending_generate_reply_fut and not self._pending_generate_reply_fut.done():
@@ -630,6 +650,7 @@ def _on_fut_done(f: asyncio.Future[llm.GenerationCreatedEvent]) -> None:
                 # external cancel: drop the queued send if it hasn't gone out yet
                 if not send_task.done():
                     send_task.cancel()
+                self._pending_user_text = None
 
         fut.add_done_callback(_on_fut_done)
         return fut
@@ -638,8 +659,24 @@ async def _send_generate_reply(self, payload: GenerateReplyPayload) -> None:
         await self._ready_to_start.wait()
         if self._session_should_close.is_set():
             return
+
+        system_message = payload.system_message
+        if self._pending_user_text:
+            user_text_instruction = (
+                f'The user sent the following text message: "{self._pending_user_text}". '
+                "Please respond to their message."
+            )
+            system_message = (
+                f"{system_message}\n\n{user_text_instruction}"
+                if system_message
+                else user_text_instruction
+            )
+            self._pending_user_text = None
+
         if self._socket:
-            await self._socket.send_generate_reply(payload)
+            await self._socket.send_generate_reply(
+                GenerateReplyPayload(system_message=system_message)
+            )
 
     def commit_audio(self) -> None:
         logger.warning("commit_audio is not supported by the Phonic realtime model.")