fix(translator): stop double-counting OpenAI cache tokens in Anthropic streaming usage

OpenAI's prompt_tokens_details.cached_tokens and cache_creation_input_tokens
are a breakdown *within* prompt_tokens (analogous to text_tokens/audio_tokens
in the same struct), not additive extras. Anthropic's native usage schema is
the opposite: input_tokens excludes cache tokens, which are reported as
separate, additive counters (cache_read_input_tokens,
cache_creation_input_tokens).

The previous commit forwarded OpenAI's cache token breakdown directly into
Anthropic's additive cache usage fields without accounting for the semantic
mismatch. Since Anthropic-consuming clients compute total context usage as
input_tokens + cache_read_input_tokens + cache_creation_input_tokens, this
caused the cached portion of every prompt to be counted twice: once inside
input_tokens (via OpenAI's prompt_tokens) and again via the cache fields.
Depending on the backend's exact reporting of cache_creation_input_tokens
(a field with no documented meaning in OpenAI's real Chat Completions API,
so its semantics are backend-defined), this could inflate the reported
input token count far beyond the real prompt size, up to the point of
clients rejecting requests as exceeding the model's context window.

Revert to not forwarding OpenAI's cache token breakdown into Anthropic's
cache usage fields, for both the client-facing message_delta.usage payload
and the gateway's internal TokenUsage metrics. input_tokens/output_tokens
continue to be backfilled correctly, which was the original motivating fix.
The pre-existing ptr.To(int64(0)) cache args to
ExtractTokenUsageFromExplicitCaching are left untouched to keep this change
minimal.

Signed-off-by: ajac-zero <ajcardoza2000@gmail.com>

Author: ajac-zero

internal/translator/anthropic_openai_test.go

@@ -385,7 +385,7 @@ func TestAnthropicToOpenAITranslator_ResponseBody_Streaming(t *testing.T) {
 
 	// Spot-check specific event data.
 	require.JSONEq(t, `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello!"}}`, events[2].data)
-	require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10,"output_tokens":5,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}}`, events[4].data)
+	require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10,"output_tokens":5}}`, events[4].data)
 	require.JSONEq(t, `{"type":"message_stop"}`, events[5].data)
 }
 

internal/translator/openai_helper.go

@@ -630,10 +630,8 @@ type sseMessageDeltaBody struct {
 }
 
 type sseOutputUsage struct {
-	InputTokens              int `json:"input_tokens"`
-	OutputTokens             int `json:"output_tokens"`
-	CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
-	CacheReadInputTokens     int `json:"cache_read_input_tokens"`
+	InputTokens  int `json:"input_tokens"`
+	OutputTokens int `json:"output_tokens"`
 }
 
 type sseMessageStop struct {
@@ -642,22 +640,20 @@ type sseMessageStop struct {
 
 // openAIStreamToAnthropicState tracks the state for converting OpenAI SSE chunks to Anthropic SSE events.
 type openAIStreamToAnthropicState struct {
-	buffer              bytes.Buffer
-	messageStarted      bool // flag indicating emitted message_start
-	hasOpenBlock        bool // flag indicating emitted content_block_start but not content_block_stop
-	hasThinkingBlock    bool // flag indicating the open block is a thinking block
-	closingEmitted      bool // flag indicating emitted content_block_stop + message_delta + message_stop
-	messageID           string
-	model               string
-	stopReason          string // Anthropic stop_reason, mapped from OpenAI finish_reason
-	inputTokens         int
-	outputTokens        int
-	cacheReadTokens     int // prompt tokens served from OpenAI's prompt cache (prompt_tokens_details.cached_tokens)
-	cacheCreationTokens int // tokens written to OpenAI's prompt cache (prompt_tokens_details.cache_creation_input_tokens)
-	tokenUsage          metrics.TokenUsage
-	blockIndex          int                       // current Anthropic content block index
-	activeTools         map[int64]*streamToolCall // keyed by OpenAI tool_call index
-	requestModel        string
+	buffer           bytes.Buffer
+	messageStarted   bool // flag indicating emitted message_start
+	hasOpenBlock     bool // flag indicating emitted content_block_start but not content_block_stop
+	hasThinkingBlock bool // flag indicating the open block is a thinking block
+	closingEmitted   bool // flag indicating emitted content_block_stop + message_delta + message_stop
+	messageID        string
+	model            string
+	stopReason       string // Anthropic stop_reason, mapped from OpenAI finish_reason
+	inputTokens      int
+	outputTokens     int
+	tokenUsage       metrics.TokenUsage
+	blockIndex       int                       // current Anthropic content block index
+	activeTools      map[int64]*streamToolCall // keyed by OpenAI tool_call index
+	requestModel     string
 }
 
 type streamToolCall struct {
@@ -743,15 +739,17 @@ func (s *openAIStreamToAnthropicState) handleChunk(chunk *openai.ChatCompletionR
 	if len(chunk.Choices) == 0 && chunk.Usage != nil {
 		s.inputTokens = chunk.Usage.PromptTokens
 		s.outputTokens = chunk.Usage.CompletionTokens
-		if details := chunk.Usage.PromptTokensDetails; details != nil {
-			s.cacheReadTokens = details.CachedTokens
-			s.cacheCreationTokens = details.CacheCreationTokens
-		}
+		// We don't forward OpenAI's prompt_tokens_details.cached_tokens/cache_creation_input_tokens
+		// here: they're a breakdown *within* prompt_tokens (like text_tokens/audio_tokens), not
+		// additive extras as in Anthropic's native usage schema, where input_tokens excludes
+		// cache tokens and they're reported as separate additive fields. Forwarding OpenAI's
+		// values as-is would cause Anthropic clients to double-count the cached portion on top
+		// of input_tokens.
 		s.tokenUsage = metrics.ExtractTokenUsageFromExplicitCaching(
 			int64(s.inputTokens),
 			int64(s.outputTokens),
-			ptr.To(int64(s.cacheReadTokens)),
-			ptr.To(int64(s.cacheCreationTokens)),
+			ptr.To(int64(0)),
+			ptr.To(int64(0)),
 		)
 		return s.emitClosingEvents(out)
 	}
@@ -1059,17 +1057,14 @@ func (s *openAIStreamToAnthropicState) emitClosingEvents(out *[]byte) error {
 		stopReason = string(anthropic.StopReasonEndTurn)
 	}
 
-	// message_delta.usage is cumulative per the Anthropic API spec, so we
-	// backfill input_tokens and cache counters here rather than in
-	// message_start, since OpenAI doesn't report prompt token usage until now.
+	// message_delta.usage is cumulative per the Anthropic API spec, so we backfill input_tokens
+	// here rather than in message_start, since OpenAI doesn't report prompt token usage until now.
 	msgDeltaPayload := sseMessageDelta{
 		Type:  "message_delta",
 		Delta: sseMessageDeltaBody{StopReason: stopReason, StopSequence: nil},
 		Usage: sseOutputUsage{
-			InputTokens:              s.inputTokens,
-			OutputTokens:             s.outputTokens,
-			CacheCreationInputTokens: s.cacheCreationTokens,
-			CacheReadInputTokens:     s.cacheReadTokens,
+			InputTokens:  s.inputTokens,
+			OutputTokens: s.outputTokens,
 		},
 	}
 	data, err := json.Marshal(msgDeltaPayload)

internal/translator/openai_helper_test.go

@@ -612,15 +612,16 @@ func TestOpenAIStreamToAnthropicState_ProcessBuffer_TextStreaming(t *testing.T)
 	require.JSONEq(t, `{"type":"content_block_stop","index":0}`, events[4].data)
 
 	assert.Equal(t, "message_delta", events[5].eventType)
-	require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10,"output_tokens":5,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}}`, events[5].data)
+	require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10,"output_tokens":5}}`, events[5].data)
 
 	assert.Equal(t, "message_stop", events[6].eventType)
 	require.JSONEq(t, `{"type":"message_stop"}`, events[6].data)
 }
 
+// TestOpenAIStreamToAnthropicState_ProcessBuffer_CachedTokens verifies that OpenAI's
+// prompt_tokens_details breakdown is not forwarded into Anthropic's additive cache usage
+// fields (see the comment in handleChunk for why).
 func TestOpenAIStreamToAnthropicState_ProcessBuffer_CachedTokens(t *testing.T) {
-	// Verify that OpenAI's prompt_tokens_details (cached_tokens / cache_creation_input_tokens)
-	// is forwarded into the Anthropic message_delta.usage cache fields, not hardcoded to zero.
 	state := &openAIStreamToAnthropicState{
 		activeTools:  make(map[int64]*streamToolCall),
 		requestModel: "claude-3",
@@ -647,15 +648,19 @@ func TestOpenAIStreamToAnthropicState_ProcessBuffer_CachedTokens(t *testing.T) {
 		}
 	}
 	require.NotEmpty(t, msgDeltaData)
-	require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":100,"output_tokens":5,"cache_creation_input_tokens":20,"cache_read_input_tokens":80}}`, msgDeltaData)
+	require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":100,"output_tokens":5}}`, msgDeltaData)
 
-	// Also verify the gateway's internal cost-tracking TokenUsage picks up the real cache counts.
+	// The gateway's internal cost-tracking TokenUsage must likewise report the plain
+	// prompt_tokens total without adding the cached breakdown on top of it.
+	total, ok := state.tokenUsage.InputTokens()
+	require.True(t, ok)
+	assert.Equal(t, uint32(100), total)
 	cached, ok := state.tokenUsage.CachedInputTokens()
 	require.True(t, ok)
-	assert.Equal(t, uint32(80), cached)
+	assert.Equal(t, uint32(0), cached)
 	cacheCreation, ok := state.tokenUsage.CacheCreationInputTokens()
 	require.True(t, ok)
-	assert.Equal(t, uint32(20), cacheCreation)
+	assert.Equal(t, uint32(0), cacheCreation)
 }
 
 func TestOpenAIStreamToAnthropicState_ProcessBuffer_ToolCallStreaming(t *testing.T) {
@@ -696,7 +701,7 @@ func TestOpenAIStreamToAnthropicState_ProcessBuffer_ToolCallStreaming(t *testing
 	require.JSONEq(t, `{"type":"content_block_stop","index":0}`, events[3].data)
 
 	assert.Equal(t, "message_delta", events[4].eventType)
-	require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":15,"output_tokens":10,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}}`, events[4].data)
+	require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":15,"output_tokens":10}}`, events[4].data)
 
 	assert.Equal(t, "message_stop", events[5].eventType)
 	require.JSONEq(t, `{"type":"message_stop"}`, events[5].data)
@@ -733,7 +738,7 @@ func TestOpenAIStreamToAnthropicState_ProcessBuffer_EndOfStreamClosing(t *testin
 		}
 	}
 	require.NotEmpty(t, msgDeltaData)
-	require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":0,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}}`, msgDeltaData)
+	require.JSONEq(t, `{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":0,"output_tokens":0}}`, msgDeltaData)
 }
 
 func TestOpenAIStreamToAnthropicState_ProcessBuffer_EmptyInput(t *testing.T) {

tests/data-plane/testupstream_test.go

@@ -1342,7 +1342,7 @@ event: content_block_stop
 data: {"type":"content_block_stop","index":0}
 
 event: message_delta
-data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10,"output_tokens":3,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}}
+data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":10,"output_tokens":3}}
 
 event: message_stop
 data: {"type":"message_stop"}`,
@@ -1379,7 +1379,7 @@ event: content_block_stop
 data: {"type":"content_block_stop","index":0}
 
 event: message_delta
-data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":50,"output_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}}
+data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":50,"output_tokens":15}}
 
 event: message_stop
 data: {"type":"message_stop"}`,