Skip to content

Commit a06046a

Browse files
toubatbriantoubatbrian
committed
update oai realtime
1 parent 3e0c6be commit a06046a

5 files changed

Lines changed: 124 additions & 35 deletions

File tree

‎agents/src/llm/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,10 @@ export {
6464
executeToolCall,
6565
oaiBuildFunctionInfo,
6666
oaiParams,
67+
serializeImage,
6768
toJsonSchema,
6869
type OpenAIFunctionParameters,
70+
type SerializedImage,
6971
} from './utils.js';
7072

7173
export {

‎plugins/openai/src/realtime/api_proto.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,11 @@ export interface InputAudioContent {
162162
audio: AudioBase64Bytes;
163163
}
164164

165+
export interface InputImageContent {
166+
type: 'input_image';
167+
image_url: string;
168+
}
169+
165170
export interface TextContent {
166171
type: 'text';
167172
text: string;
@@ -181,6 +186,7 @@ export interface AudioContent {
181186
export type Content =
182187
| InputTextContent
183188
| InputAudioContent
189+
| InputImageContent
184190
| TextContent
185191
| OutputTextContent
186192
| AudioContent;
@@ -206,7 +212,7 @@ export interface SystemItem extends BaseItem {
206212
export interface UserItem extends BaseItem {
207213
type: 'message';
208214
role: 'user';
209-
content: (InputTextContent | InputAudioContent)[];
215+
content: (InputTextContent | InputAudioContent | InputImageContent)[];
210216
}
211217

212218
export interface AssistantItem extends BaseItem {
@@ -361,7 +367,7 @@ export interface UserItemCreate {
361367
id: string;
362368
type: 'message';
363369
role: 'user';
364-
content: (InputTextContent | InputAudioContent)[];
370+
content: (InputTextContent | InputAudioContent | InputImageContent)[];
365371
}
366372

367373
export interface AssistantItemCreate {

‎plugins/openai/src/realtime/realtime_model.test.ts‎

Lines changed: 66 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@ import { livekitItemToOpenAIItem } from './realtime_model.js';
88

99
describe('livekitItemToOpenAIItem', () => {
1010
describe('message items', () => {
11-
it('should use output_text type for assistant messages', () => {
11+
it('should use output_text type for assistant messages', async () => {
1212
const assistantMessage = new llm.ChatMessage({
1313
role: 'assistant',
1414
content: 'Hello, how can I help you?',
1515
id: 'test-assistant-msg',
1616
});
1717

18-
const result = livekitItemToOpenAIItem(assistantMessage) as api_proto.AssistantItem;
18+
const result = (await livekitItemToOpenAIItem(assistantMessage)) as api_proto.AssistantItem;
1919

2020
expect(result.type).toBe('message');
2121
expect(result.role).toBe('assistant');
@@ -25,14 +25,14 @@ describe('livekitItemToOpenAIItem', () => {
2525
expect((content as api_proto.OutputTextContent).text).toBe('Hello, how can I help you?');
2626
});
2727

28-
it('should use input_text type for user messages', () => {
28+
it('should use input_text type for user messages', async () => {
2929
const userMessage = new llm.ChatMessage({
3030
role: 'user',
3131
content: 'What is the weather like?',
3232
id: 'test-user-msg',
3333
});
3434

35-
const result = livekitItemToOpenAIItem(userMessage) as api_proto.UserItem;
35+
const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;
3636

3737
expect(result.type).toBe('message');
3838
expect(result.role).toBe('user');
@@ -42,14 +42,14 @@ describe('livekitItemToOpenAIItem', () => {
4242
expect((content as api_proto.InputTextContent).text).toBe('What is the weather like?');
4343
});
4444

45-
it('should use input_text type for system messages', () => {
45+
it('should use input_text type for system messages', async () => {
4646
const systemMessage = new llm.ChatMessage({
4747
role: 'system',
4848
content: 'You are a helpful assistant.',
4949
id: 'test-system-msg',
5050
});
5151

52-
const result = livekitItemToOpenAIItem(systemMessage) as api_proto.UserItem;
52+
const result = (await livekitItemToOpenAIItem(systemMessage)) as api_proto.UserItem;
5353

5454
expect(result.type).toBe('message');
5555
expect(result.role).toBe('system');
@@ -58,48 +58,98 @@ describe('livekitItemToOpenAIItem', () => {
5858
expect(content.type).toBe('input_text');
5959
});
6060

61-
it('should convert developer role to system role', () => {
61+
it('should convert developer role to system role', async () => {
6262
const developerMessage = new llm.ChatMessage({
6363
role: 'developer',
6464
content: 'System instructions.',
6565
id: 'test-developer-msg',
6666
});
6767

68-
const result = livekitItemToOpenAIItem(developerMessage) as api_proto.UserItem;
68+
const result = (await livekitItemToOpenAIItem(developerMessage)) as api_proto.UserItem;
6969

7070
expect(result.type).toBe('message');
7171
expect(result.role).toBe('system');
7272
const content = result.content[0]!;
7373
expect(content.type).toBe('input_text');
7474
});
7575

76-
it('should handle multiple content items for assistant', () => {
76+
it('should handle multiple content items for assistant', async () => {
7777
const multiContentMessage = new llm.ChatMessage({
7878
role: 'assistant',
7979
content: ['First part.', 'Second part.'],
8080
id: 'test-multi-msg',
8181
});
8282

83-
const result = livekitItemToOpenAIItem(multiContentMessage) as api_proto.AssistantItem;
83+
const result = (await livekitItemToOpenAIItem(
84+
multiContentMessage,
85+
)) as api_proto.AssistantItem;
8486

8587
expect(result.content).toHaveLength(2);
8688
const content0 = result.content[0]!;
8789
const content1 = result.content[1]!;
8890
expect(content0.type).toBe('output_text');
8991
expect(content1.type).toBe('output_text');
9092
});
93+
94+
it('should convert image content to input_image for user messages', async () => {
95+
const base64Data =
96+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
97+
const imageContent = llm.createImageContent({
98+
image: `data:image/png;base64,${base64Data}`,
99+
mimeType: 'image/png',
100+
});
101+
102+
const userMessage = new llm.ChatMessage({
103+
role: 'user',
104+
content: [imageContent],
105+
id: 'test-image-msg',
106+
});
107+
108+
const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;
109+
110+
expect(result.type).toBe('message');
111+
expect(result.role).toBe('user');
112+
expect(result.content).toHaveLength(1);
113+
const content = result.content[0]!;
114+
expect(content.type).toBe('input_image');
115+
expect((content as api_proto.InputImageContent).image_url).toBe(
116+
`data:image/png;base64,${base64Data}`,
117+
);
118+
});
119+
120+
it('should handle mixed text and image content', async () => {
121+
const base64Data =
122+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
123+
const imageContent = llm.createImageContent({
124+
image: `data:image/png;base64,${base64Data}`,
125+
mimeType: 'image/png',
126+
});
127+
128+
const userMessage = new llm.ChatMessage({
129+
role: 'user',
130+
content: ['Describe this image:', imageContent],
131+
id: 'test-mixed-msg',
132+
});
133+
134+
const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;
135+
136+
expect(result.type).toBe('message');
137+
expect(result.content).toHaveLength(2);
138+
expect(result.content[0]!.type).toBe('input_text');
139+
expect(result.content[1]!.type).toBe('input_image');
140+
});
91141
});
92142

93143
describe('function_call items', () => {
94-
it('should convert function call items correctly', () => {
144+
it('should convert function call items correctly', async () => {
95145
const functionCall = new llm.FunctionCall({
96146
callId: 'call-123',
97147
name: 'get_weather',
98148
args: '{"location": "San Francisco"}',
99149
id: 'test-func-call',
100150
});
101151

102-
const result = livekitItemToOpenAIItem(functionCall) as api_proto.FunctionCallItem;
152+
const result = (await livekitItemToOpenAIItem(functionCall)) as api_proto.FunctionCallItem;
103153

104154
expect(result.type).toBe('function_call');
105155
expect(result.id).toBe('test-func-call');
@@ -110,15 +160,17 @@ describe('livekitItemToOpenAIItem', () => {
110160
});
111161

112162
describe('function_call_output items', () => {
113-
it('should convert function call output items correctly', () => {
163+
it('should convert function call output items correctly', async () => {
114164
const functionOutput = new llm.FunctionCallOutput({
115165
callId: 'call-123',
116166
output: 'The weather in San Francisco is sunny.',
117167
isError: false,
118168
id: 'test-func-output',
119169
});
120170

121-
const result = livekitItemToOpenAIItem(functionOutput) as api_proto.FunctionCallOutputItem;
171+
const result = (await livekitItemToOpenAIItem(
172+
functionOutput,
173+
)) as api_proto.FunctionCallOutputItem;
122174

123175
expect(result.type).toBe('function_call_output');
124176
expect(result.id).toBe('test-func-output');

‎plugins/openai/src/realtime/realtime_model.ts‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -461,7 +461,7 @@ export class RealtimeSession extends llm.RealtimeSession {
461461

462462
async updateChatCtx(_chatCtx: llm.ChatContext): Promise<void> {
463463
const unlock = await this.updateChatCtxLock.lock();
464-
const events = this.createChatCtxUpdateEvents(_chatCtx);
464+
const events = await this.createChatCtxUpdateEvents(_chatCtx);
465465
const futures: Future<void>[] = [];
466466

467467
for (const event of events) {
@@ -498,10 +498,10 @@ export class RealtimeSession extends llm.RealtimeSession {
498498
}
499499
}
500500

501-
private createChatCtxUpdateEvents(
501+
private async createChatCtxUpdateEvents(
502502
chatCtx: llm.ChatContext,
503503
addMockAudio: boolean = false,
504-
): (api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[] {
504+
): Promise<(api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[]> {
505505
const newChatCtx = chatCtx.copy();
506506
if (addMockAudio) {
507507
newChatCtx.items.push(createMockAudioItem());
@@ -533,7 +533,7 @@ export class RealtimeSession extends llm.RealtimeSession {
533533
}
534534
events.push({
535535
type: 'conversation.item.create',
536-
item: livekitItemToOpenAIItem(chatItem),
536+
item: await livekitItemToOpenAIItem(chatItem),
537537
previous_item_id: previousId ?? undefined,
538538
event_id: shortuuid('chat_ctx_create_'),
539539
} as api_proto.ConversationItemCreateEvent);
@@ -704,7 +704,7 @@ export class RealtimeSession extends llm.RealtimeSession {
704704
content: [_options.audioTranscript],
705705
});
706706
chatCtx.items[idx] = newItem;
707-
const events = this.createChatCtxUpdateEvents(chatCtx);
707+
const events = await this.createChatCtxUpdateEvents(chatCtx);
708708
for (const ev of events) {
709709
this.sendEvent(ev);
710710
}
@@ -845,7 +845,7 @@ export class RealtimeSession extends llm.RealtimeSession {
845845

846846
const oldChatCtx = this.remoteChatCtx;
847847
this.remoteChatCtx = new llm.RemoteChatContext();
848-
events.push(...this.createChatCtxUpdateEvents(chatCtx));
848+
events.push(...(await this.createChatCtxUpdateEvents(chatCtx)));
849849

850850
try {
851851
for (const ev of events) {
@@ -1560,7 +1560,6 @@ export class RealtimeSession extends llm.RealtimeSession {
15601560
if (event.error.message.startsWith('Cancellation failed')) {
15611561
return;
15621562
}
1563-
15641563
this.#logger.error({ error: event.error }, 'OpenAI Realtime API returned an error');
15651564
this.emitError({
15661565
error: new APIError(event.error.message, {
@@ -1622,7 +1621,7 @@ export class RealtimeSession extends llm.RealtimeSession {
16221621
}
16231622

16241623
/** @internal Exported for testing purposes */
1625-
export function livekitItemToOpenAIItem(item: llm.ChatItem): api_proto.ItemResource {
1624+
export async function livekitItemToOpenAIItem(item: llm.ChatItem): Promise<api_proto.ItemResource> {
16261625
switch (item.type) {
16271626
case 'function_call':
16281627
return {
@@ -1649,8 +1648,19 @@ export function livekitItemToOpenAIItem(item: llm.ChatItem): api_proto.ItemResou
16491648
text: c,
16501649
} as api_proto.InputTextContent | api_proto.OutputTextContent);
16511650
} else if (c.type === 'image_content') {
1652-
// not supported for now
1653-
continue;
1651+
const serialized = await llm.serializeImage(c);
1652+
if (serialized.externalUrl) {
1653+
log().warn('External URL is not supported for input_image in realtime API');
1654+
continue;
1655+
}
1656+
if (!serialized.base64Data) {
1657+
log().warn('Serialized image has no data bytes');
1658+
continue;
1659+
}
1660+
contentList.push({
1661+
type: 'input_image',
1662+
image_url: `data:${serialized.mimeType};base64,${serialized.base64Data}`,
1663+
} as api_proto.InputImageContent);
16541664
} else if (c.type === 'audio_content') {
16551665
if (role === 'user') {
16561666
const encodedAudio = Buffer.from(combineAudioFrames(c.frame).data).toString('base64');
@@ -1699,6 +1709,10 @@ function openAIItemToLivekitItem(item: api_proto.ItemResource): llm.ChatItem {
16991709
for (const c of contents) {
17001710
if (c.type === 'text' || c.type === 'input_text') {
17011711
content.push(c.text);
1712+
} else if (c.type === 'input_image' && (c as api_proto.InputImageContent).image_url) {
1713+
content.push(
1714+
llm.createImageContent({ image: (c as api_proto.InputImageContent).image_url }),
1715+
);
17021716
}
17031717
}
17041718
return llm.ChatMessage.create({

0 commit comments

Comments
 (0)