fix: preserve tool result images in chat completions (#626)

Author: 7Sageer

.changeset/fix-tool-result-images.md

@@ -0,0 +1,6 @@
+---
+"@moonshot-ai/kosong": patch
+"@moonshot-ai/kimi-code": patch
+---
+
+Preserve image outputs from tools when using OpenAI-compatible chat completions.

packages/kosong/src/providers/openai-legacy.ts

@@ -50,6 +50,8 @@ const OPENAI_CHAT_TOOL_CALL_ID_POLICY: ToolCallIdPolicy = {
   normalize: (id) => sanitizeToolCallId(id, 64),
   maxLength: 64,
 };
+const TOOL_RESULT_IMAGE_PROMPT = 'Attached image(s) from tool result:';
+const TOOL_RESULT_IMAGE_PLACEHOLDER = '(see attached image)';
 
 function extractReasoningContent(
   source: unknown,
@@ -165,7 +167,7 @@ function convertMessage(
       : toolMessageConversion;
 
     if (effectiveConversion !== null) {
-      result.content = convertToolMessageContent(message, effectiveConversion);
+      result.content = convertToolMessageContentForChat(message, effectiveConversion);
     } else {
       // Pure-text tool result with no conversion configured: serialize via the
       // generic content-part path so single-text messages become a plain string.
@@ -217,6 +219,70 @@ function convertMessage(
 
   return result;
 }
+
+function convertToolMessageContentForChat(
+  message: Message,
+  conversion: ToolMessageConversion,
+): string | OpenAIContentPart[] {
+  const content = convertToolMessageContent(message, conversion);
+  if (
+    typeof content === 'string' &&
+    content.length === 0 &&
+    message.content.some((part) => part.type === 'image_url')
+  ) {
+    return TOOL_RESULT_IMAGE_PLACEHOLDER;
+  }
+  return content;
+}
+
+function toolResultImageParts(message: Message): OpenAIContentPart[] {
+  const images: OpenAIContentPart[] = [];
+  for (const part of message.content) {
+    if (part.type !== 'image_url') continue;
+    images.push({
+      type: 'image_url',
+      image_url:
+        part.imageUrl.id === undefined
+          ? { url: part.imageUrl.url }
+          : { url: part.imageUrl.url, id: part.imageUrl.id },
+    });
+  }
+  return images;
+}
+
+function appendToolResultImagesMessage(
+  messages: OpenAIMessage[],
+  pendingToolResultImages: OpenAIContentPart[],
+): void {
+  if (pendingToolResultImages.length === 0) return;
+  messages.push({
+    role: 'user',
+    content: [{ type: 'text', text: TOOL_RESULT_IMAGE_PROMPT }, ...pendingToolResultImages],
+  });
+  pendingToolResultImages.length = 0;
+}
+
+function convertHistoryMessages(
+  history: readonly Message[],
+  reasoningKey: string | undefined,
+  toolMessageConversion: ToolMessageConversion,
+): OpenAIMessage[] {
+  const messages: OpenAIMessage[] = [];
+  const pendingToolResultImages: OpenAIContentPart[] = [];
+
+  for (const msg of history) {
+    if (msg.role !== 'tool') {
+      appendToolResultImagesMessage(messages, pendingToolResultImages);
+    }
+    messages.push(convertMessage(msg, reasoningKey, toolMessageConversion));
+    if (msg.role === 'tool') {
+      pendingToolResultImages.push(...toolResultImageParts(msg));
+    }
+  }
+
+  appendToolResultImagesMessage(messages, pendingToolResultImages);
+  return messages;
+}
 export class OpenAILegacyStreamedMessage implements StreamedMessage {
   private _id: string | null = null;
   private _usage: TokenUsage | null = null;
@@ -437,9 +503,9 @@ export class OpenAILegacyChatProvider implements ChatProvider {
       history,
       OPENAI_CHAT_TOOL_CALL_ID_POLICY,
     );
-    for (const msg of normalizedHistory) {
-      messages.push(convertMessage(msg, this._reasoningKey, this._toolMessageConversion));
-    }
+    messages.push(
+      ...convertHistoryMessages(normalizedHistory, this._reasoningKey, this._toolMessageConversion),
+    );
 
     const kwargs: Record<string, unknown> = normalizeGenerationKwargs(
       this._model,

packages/kosong/test/openai-legacy.test.ts

@@ -287,7 +287,7 @@ describe('OpenAILegacyChatProvider', () => {
       ]);
     });
 
-    it('tool call with image result flattens to text to satisfy API constraints', async () => {
+    it('tool call with image result keeps the tool result textual and reattaches images as user input', async () => {
       // OpenAI Chat Completions `tool` messages only accept text content.
       // Even when toolMessageConversion is unset, a tool result containing
       // image_url / audio_url / video_url parts must not be serialized as a
@@ -319,15 +319,86 @@ describe('OpenAILegacyChatProvider', () => {
       ];
       const body = await captureRequestBody(provider, '', [], history);
 
-      const toolMsg = (body['messages'] as Record<string, unknown>[])[2]!;
+      const messages = body['messages'] as Record<string, unknown>[];
+      const toolMsg = messages[2]!;
       expect(toolMsg['role']).toBe('tool');
       expect(toolMsg['tool_call_id']).toBe('call_abc123');
       // Content must be a plain string, not a content-part array.
       expect(typeof toolMsg['content']).toBe('string');
       // The text segment must survive; the image must not appear as a
-      // structured image_url part anywhere in the serialized content.
+      // structured image_url part inside the tool message.
       expect(toolMsg['content']).toContain('5');
       expect(Array.isArray(toolMsg['content'])).toBe(false);
+      expect(messages[3]).toEqual({
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Attached image(s) from tool result:' },
+          { type: 'image_url', image_url: { url: 'https://example.com/image.png' } },
+        ],
+      });
+    });
+
+    it('groups consecutive tool result images after all matching tool messages', async () => {
+      const provider = createProvider();
+      const history: Message[] = [
+        { role: 'user', content: [{ type: 'text', text: 'Fetch both images' }], toolCalls: [] },
+        {
+          role: 'assistant',
+          content: [{ type: 'text', text: 'ok' }],
+          toolCalls: [
+            { type: 'function', id: 'call_first', name: 'first_image', arguments: '{}' },
+            { type: 'function', id: 'call_second', name: 'second_image', arguments: '{}' },
+          ],
+        },
+        {
+          role: 'tool',
+          content: [
+            { type: 'image_url', imageUrl: { url: 'https://example.com/first.png' } },
+          ],
+          toolCallId: 'call_first',
+          toolCalls: [],
+        },
+        {
+          role: 'tool',
+          content: [
+            { type: 'text', text: 'second' },
+            { type: 'image_url', imageUrl: { url: 'https://example.com/second.png' } },
+          ],
+          toolCallId: 'call_second',
+          toolCalls: [],
+        },
+      ];
+      const body = await captureRequestBody(provider, '', [], history);
+
+      expect(body['messages']).toEqual([
+        { role: 'user', content: 'Fetch both images' },
+        {
+          role: 'assistant',
+          content: 'ok',
+          tool_calls: [
+            {
+              type: 'function',
+              id: 'call_first',
+              function: { name: 'first_image', arguments: '{}' },
+            },
+            {
+              type: 'function',
+              id: 'call_second',
+              function: { name: 'second_image', arguments: '{}' },
+            },
+          ],
+        },
+        { role: 'tool', content: '(see attached image)', tool_call_id: 'call_first' },
+        { role: 'tool', content: 'second', tool_call_id: 'call_second' },
+        {
+          role: 'user',
+          content: [
+            { type: 'text', text: 'Attached image(s) from tool result:' },
+            { type: 'image_url', image_url: { url: 'https://example.com/first.png' } },
+            { type: 'image_url', image_url: { url: 'https://example.com/second.png' } },
+          ],
+        },
+      ]);
     });
 
     it('parallel tool calls', async () => {