Merge branch 'main' into docs/quota-policy

nacx · web-flow · commit 4af8c72ee542 · 2026-03-16T19:47:45.000+01:00
diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go
@@ -27,6 +27,7 @@ type VersionedAPISchema struct {
 	// prefix field. This is to maintain backward compatibility. This will be removed in future releases.
 	//
 	// See https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers for details.
+	// +optional
 	Version *string `json:"version,omitempty"`
 
 	// Prefix is the prefix for the API.
@@ -41,6 +42,7 @@ type VersionedAPISchema struct {
 	// use prefix, so you can leave this field unset.
 	//
 	// See https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers for details.
+	// +optional
 	Prefix *string `json:"prefix,omitempty"`
 }
 
diff --git a/internal/extproc/processor_impl.go b/internal/extproc/processor_impl.go
@@ -590,13 +590,13 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) SetBackend(c
 	if u.modelNameOverride != "" {
 		u.requestHeaders[internalapi.ModelNameHeaderKeyDefault] = u.modelNameOverride
 	}
-	rp.upstreamFilter = u
-	u.parent = rp
+	u.parent = rp // Set parent before GetTranslator so it can access rp.eh
 
 	u.translator, err = u.parent.eh.GetTranslator(b.Schema, u.modelNameOverride)
 	if err != nil {
 		return fmt.Errorf("failed to create translator for backend %s: %w", b.Name, err)
 	}
+	rp.upstreamFilter = u // Only assign after translator is confirmed valid
 
 	switch redactor := u.translator.(type) {
 	case translator.ResponseRedactor:
diff --git a/internal/extproc/processor_impl_test.go b/internal/extproc/processor_impl_test.go
@@ -433,6 +433,43 @@ func Test_chatCompletionProcessorUpstreamFilter_SetBackend(t *testing.T) {
 	require.Zero(t, mm.inputTokenCount)
 	mm.RequireSelectedBackend(t, "some-backend")
 	require.Equal(t, r, p.parent)
+	// Verify upstreamFilter is NOT set when translator creation fails.
+	// This prevents a nil-translator panic when the router processes the response
+	// (the nil check on upstreamFilter at ProcessResponseHeaders/ProcessResponseBody
+	// must fall through to passThroughProcessor).
+	require.Nil(t, r.upstreamFilter, "upstreamFilter must remain nil when SetBackend fails")
+}
+
+// Test_chatCompletionProcessorUpstreamFilter_SetBackend_unsupportedSchema_noResponsePanic
+// verifies that when SetBackend fails due to an unsupported schema, subsequent
+// response processing does not panic. Before the fix for #1941, upstreamFilter
+// was assigned before the translator was created, so the router's nil check on
+// upstreamFilter would pass but the nil translator would cause a panic.
+func Test_chatCompletionProcessorUpstreamFilter_SetBackend_unsupportedSchema_noResponsePanic(t *testing.T) {
+	headers := map[string]string{":path": "/foo"}
+	mm := &mockMetrics{}
+	p := &chatCompletionProcessorUpstreamFilter{
+		requestHeaders: headers,
+		metrics:        mm,
+	}
+	r := &chatCompletionProcessorRouterFilter{}
+
+	err := p.SetBackend(t.Context(), &filterapi.Backend{
+		Name:   "bad-backend",
+		Schema: filterapi.VersionedAPISchema{Name: "unsupported-schema", Version: "v1"},
+	}, nil, r)
+	require.Error(t, err)
+	require.Nil(t, r.upstreamFilter, "upstreamFilter must remain nil on translator creation failure")
+
+	// Simulate response arriving after the failed SetBackend.
+	// This must NOT panic; it should fall through to passThroughProcessor.
+	resp, err := r.ProcessResponseHeaders(t.Context(), nil)
+	require.NoError(t, err)
+	require.NotNil(t, resp)
+
+	resp, err = r.ProcessResponseBody(t.Context(), &extprocv3.HttpBody{Body: []byte("error"), EndOfStream: true})
+	require.NoError(t, err)
+	require.NotNil(t, resp)
 }
 
 func Test_chatCompletionProcessorUpstreamFilter_ProcessRequestHeaders(t *testing.T) {
diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx
@@ -2524,12 +2524,12 @@ the ai-gateway, but by the vendor via proper versioning.
 /><ApiField
   name="version"
   type="string"
-  required="true"
+  required="false"
   description="Version is the version of the API schema.<br />When the name is set to AzureOpenAI, this version maps to `API Version` in the<br />Azure OpenAI API documentation (https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#rest-api-versioning).<br />**Deprecated Behavior**: When the name is set to `OpenAI`, this version field will behave as the<br />prefix field. This is to maintain backward compatibility. This will be removed in future releases.<br />See https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers for details."
 /><ApiField
   name="prefix"
   type="string"
-  required="true"
+  required="false"
   description="Prefix is the prefix for the API.<br />When the name is set to `OpenAI`, `chat completions` API endpoint will be `$\{this_field\}/chat/completions`.<br />It can be with or without a leading slash (`/`).<br />This is especially useful when routing to the backend that has an OpenAI compatible API but has a different<br />prefix. For example, Gemini OpenAI compatible API (https://ai.google.dev/gemini-api/docs/openai) uses<br />`/v1beta/openai` prefix. Another example is that Cohere AI (https://docs.cohere.com/v2/docs/compatibility-api)<br />uses `/compatibility/v1` prefix. On the other hand, DeepSeek (https://api-docs.deepseek.com/) doesn't<br />use prefix, so you can leave this field unset.<br />See https://aigateway.envoyproxy.io/docs/capabilities/llm-integrations/supported-providers for details."
 />
 
diff --git a/site/docs/capabilities/index.md b/site/docs/capabilities/index.md
@@ -16,6 +16,7 @@ Support for various Large Language Model providers:
 - **[Supported Providers](./llm-integrations/supported-providers.md)**: Compatible AI/LLM service providers
 - **[Supported Endpoints](./llm-integrations/supported-endpoints.md)**: Available API endpoints and operations
 - **[Vendor-Specific Fields](./llm-integrations/vendor-specific-fields.md)**: Use backend-specific parameters and access provider-unique capabilities in your OpenAI-compatible requests
+- **[Prompt Caching](./llm-integrations/prompt-caching.md)**: Provider-agnostic prompt caching using unified cache_control API
 
 ## Inference Optimization
 
diff --git a/site/docs/capabilities/llm-integrations/index.md b/site/docs/capabilities/llm-integrations/index.md
@@ -15,6 +15,7 @@ Envoy AI Gateway provides integration to multiple LLM Providers. This section pr
 ## Advanced Features
 
 - **[Vendor-Specific Fields](./vendor-specific-fields.md)** - Use backend-specific parameters and access provider-unique capabilities in your OpenAI-compatible requests
+- **[Prompt Caching](./prompt-caching.md)** - Provider-agnostic prompt caching using unified cache_control API
 
 ## Getting Started with Provider Connectivity
 
diff --git a/site/docs/capabilities/llm-integrations/prompt-caching.md b/site/docs/capabilities/llm-integrations/prompt-caching.md
@@ -0,0 +1,174 @@
+---
+id: prompt-caching
+title: Prompt Caching
+sidebar_position: 6
+---
+
+# Prompt Caching
+
+Envoy AI Gateway provides provider-agnostic prompt caching through a unified `cache_control` API. The same cache syntax works across multiple providers: Direct Anthropic, GCP Vertex AI (Claude models), and AWS Bedrock (Claude models). This reduces costs and improves response times by caching frequently-used content like system prompts, tool definitions, and reference documents.
+
+## Supported Providers
+
+| Provider               | API Schema            | Cache Support |
+| ---------------------- | --------------------- | ------------- |
+| Anthropic (Direct)     | `Anthropic`           | Native        |
+| GCP Vertex AI (Claude) | `GCPAnthropic`        | Translated    |
+| AWS Bedrock (Claude)   | `AWSBedrockAnthropic` | Translated    |
+
+## How It Works
+
+- Add `cache_control: {"type": "ephemeral"}` to content blocks in your request.
+- AI Gateway translates this to the provider-specific format automatically.
+- Cache is maintained per-provider; all providers require a minimum of 1,024 tokens for caching.
+- A maximum of 4 cache breakpoints are allowed per request across all providers.
+- On cache hit, the provider charges reduced input token costs.
+
+## Usage
+
+No gateway-side configuration is needed. Caching is controlled entirely at the request level by adding `cache_control` to the content blocks you want cached.
+
+### Basic Example: System Prompt Caching
+
+Cache a system prompt so that subsequent requests reuse the cached content:
+
+```json
+{
+  "model": "claude-sonnet-4-5",
+  "messages": [
+    {
+      "role": "system",
+      "content": [
+        {
+          "type": "text",
+          "text": "You are a helpful assistant with extensive knowledge...(long system prompt)...",
+          "cache_control": { "type": "ephemeral" }
+        }
+      ]
+    },
+    {
+      "role": "user",
+      "content": "What is the capital of France?"
+    }
+  ]
+}
+```
+
+### Multiple Cache Points
+
+You can place up to 4 cache breakpoints in a single request to cache different parts of the conversation:
+
+```json
+{
+  "model": "claude-sonnet-4-5",
+  "messages": [
+    {
+      "role": "system",
+      "content": [
+        {
+          "type": "text",
+          "text": "System instructions...",
+          "cache_control": { "type": "ephemeral" }
+        }
+      ]
+    },
+    {
+      "role": "user",
+      "content": [
+        {
+          "type": "text",
+          "text": "Reference document content...",
+          "cache_control": { "type": "ephemeral" }
+        },
+        {
+          "type": "text",
+          "text": "Question about the document"
+        }
+      ]
+    }
+  ]
+}
+```
+
+### Tool Definition Caching
+
+Cache complex tool schemas that remain the same across requests:
+
+```json
+{
+  "model": "claude-sonnet-4-5",
+  "messages": [
+    {
+      "role": "user",
+      "content": "Help me search for information about cloud computing trends."
+    }
+  ],
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "search_knowledge_base",
+        "description": "Search through a comprehensive knowledge base...",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "query": {
+              "type": "string",
+              "description": "Natural language search query"
+            }
+          },
+          "required": ["query"]
+        },
+        "cache_control": { "type": "ephemeral" }
+      }
+    }
+  ]
+}
+```
+
+## Response Format
+
+When caching is active, the response includes cache information in the `usage` field:
+
+```json
+{
+  "usage": {
+    "prompt_tokens": 2000,
+    "completion_tokens": 150,
+    "prompt_tokens_details": {
+      "cached_tokens": 1800
+    }
+  }
+}
+```
+
+- `cached_tokens` indicates the number of tokens served from cache at a reduced cost.
+- Cache write tokens are tracked internally for billing purposes.
+
+## Best Practices
+
+:::tip
+
+- Place `cache_control` on content that exceeds 1,024 tokens. Content below this threshold will not be cached.
+- Cache system prompts, tool definitions, and reference documents that do not change between requests.
+- Position cache breakpoints strategically -- cached content must appear at the beginning of the message.
+- Monitor `cached_tokens` in responses to verify caching effectiveness and measure cost savings.
+  :::
+
+## Provider-Specific Notes
+
+:::note
+
+- **All providers**: Minimum 1,024 tokens per cached block, maximum 4 cache breakpoints per request.
+- **Anthropic Direct**: Uses the native `cache_control` field directly with no translation.
+- **GCP Vertex AI**: AI Gateway translates `cache_control` to Vertex AI's caching format automatically.
+- **AWS Bedrock**: AI Gateway translates `cache_control` to Bedrock's cachePoint format automatically.
+- All providers support the `"ephemeral"` cache type.
+- Existing requests without `cache_control` continue to work with no changes.
+  :::
+
+## Further Reading
+
+- [Prompt Caching Examples](https://github.com/envoyproxy/ai-gateway/tree/main/examples/cache) -- Detailed examples with curl commands for each provider.
+- [Connecting to GCP Vertex AI](../../getting-started/connect-providers/gcp-vertexai.md) -- Set up GCP Vertex AI as a provider.
+- [Connecting to AWS Bedrock](../../getting-started/connect-providers/aws-bedrock.md) -- Set up AWS Bedrock as a provider.

Original file line number	Diff line number	Diff line change
`@@ -590,13 +590,13 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) SetBackend(c`
`590`	`590`	`if u.modelNameOverride != "" {`
`591`	`591`	`u.requestHeaders[internalapi.ModelNameHeaderKeyDefault] = u.modelNameOverride`
`592`	`592`	`}`
`593`		`- rp.upstreamFilter = u`
`594`		`- u.parent = rp`
	`593`	`+ u.parent = rp // Set parent before GetTranslator so it can access rp.eh`
`595`	`594`
`596`	`595`	`u.translator, err = u.parent.eh.GetTranslator(b.Schema, u.modelNameOverride)`
`597`	`596`	`if err != nil {`
`598`	`597`	`return fmt.Errorf("failed to create translator for backend %s: %w", b.Name, err)`
`599`	`598`	`}`
	`599`	`+ rp.upstreamFilter = u // Only assign after translator is confirmed valid`
`600`	`600`
`601`	`601`	`switch redactor := u.translator.(type) {`
`602`	`602`	`case translator.ResponseRedactor:`