pivot to using ratelimit overrides from metadata

Signed-off-by: Anurag Aggarwal <kanurag94@gmail.com>

Author: kanurag94

api/v1beta1/ai_gateway_route.go

@@ -206,19 +206,15 @@ type AIGatewayRouteSpec struct {
 	// +kubebuilder:validation:MaxItems=36
 	LLMRequestCosts []LLMRequestCost `json:"llmRequestCosts,omitempty"`
 
-	// RateLimitsFromHeaders emits per-request rate-limit override structs from request headers
-	// into io.envoy.ai_gateway dynamic metadata. Each header value must be formatted as
-	// "<count>/<unit>" (e.g. "100000/HOUR"). Overrides the global defaults from GatewayConfig
-	// for entries with the same MetadataKey.
-	//
-	// See RateLimitFromHeader for the security requirement to strip these headers from client
-	// requests using ClientTrafficPolicy.spec.headers.earlyRequestHeaders.remove.
+	// RateLimits defines per-route rate-limit override entries that emit per-request limit structs
+	// into io.envoy.ai_gateway dynamic metadata. Entries with the same MetadataKey override the
+	// gateway-level GlobalRateLimits defined in GatewayConfig.
 	//
 	// +optional
 	// +listType=map
 	// +listMapKey=metadataKey
 	// +kubebuilder:validation:MaxItems=36
-	RateLimitsFromHeaders []RateLimitFromHeader `json:"rateLimitsFromHeaders,omitempty"`
+	RateLimits []RateLimitOverride `json:"rateLimits,omitempty"`
 }
 
 // AIGatewayRouteRule is a rule that defines the routing behavior of the AIGatewayRoute.

api/v1beta1/gateway_config.go

@@ -81,18 +81,16 @@ type GatewayConfigSpec struct {
 	// +listMapKey=metadataKey
 	GlobalLLMRequestCosts []LLMRequestCost `json:"globalLLMRequestCosts,omitempty"`
 
-	// GlobalRateLimitsFromHeaders defines gateway-level defaults for emitting per-request
-	// rate-limit override structs from request headers into io.envoy.ai_gateway dynamic metadata.
-	// Route-scoped entries with the same MetadataKey take precedence.
-	//
-	// See RateLimitFromHeader for the security requirement to strip these headers from client
-	// requests using ClientTrafficPolicy.spec.headers.earlyRequestHeaders.remove.
+	// GlobalRateLimits defines gateway-level defaults for emitting per-request rate-limit
+	// override structs into io.envoy.ai_gateway dynamic metadata. Each entry maps a metadata
+	// key to a source that supplies the per-request limit value. Route-scoped entries with the
+	// same MetadataKey take precedence over these global defaults.
 	//
 	// +optional
 	// +listType=map
 	// +listMapKey=metadataKey
 	// +kubebuilder:validation:MaxItems=36
-	GlobalRateLimitsFromHeaders []RateLimitFromHeader `json:"globalRateLimitsFromHeaders,omitempty"`
+	GlobalRateLimits []RateLimitOverride `json:"globalRateLimits,omitempty"`
 }
 
 // GatewayConfigExtProc holds runtime-specific configuration for the external processor.

api/v1beta1/shared_types.go

@@ -165,43 +165,52 @@ const (
 	AIGatewayFilterMetadataNamespace = "io.envoy.ai_gateway"
 )
 
-// RateLimitFromHeader emits a per-request rate-limit override struct into io.envoy.ai_gateway
-// dynamic metadata from a trusted request header. The header value must be formatted as
-// "COUNT/UNIT" (e.g. "100000/HOUR"). The gateway parses it and writes a struct with
-// requests_per_unit and unit fields under MetadataKey — the struct
-// Envoy's RateLimit.Override.DynamicMetadata reads. If the header is absent or malformed the key is omitted.
-//
-// Security note: this feature reads the rate-limit value directly from an incoming request header.
-// A client that can set this header can inflate its own quota. To prevent spoofing, you MUST strip
-// the header from client requests before the HTTP filter chain (including ext-authz) processes it.
-// Use ClientTrafficPolicy.spec.headers.earlyRequestHeaders.remove on the associated Gateway:
-//
-//	apiVersion: gateway.envoyproxy.io/v1alpha1
-//	kind: ClientTrafficPolicy
-//	spec:
-//	  targetRefs:
-//	    - group: gateway.networking.k8s.io
-//	      kind: Gateway
-//	      name: <your-gateway>
-//	  headers:
-//	    earlyRequestHeaders:
-//	      remove: ["<header-name>"]
-//
-// After stripping, only the trusted ext-authz (or another filter running after the strip)
-// can set the header, ensuring the value reflects the actual tenant quota.
-type RateLimitFromHeader struct {
-	// MetadataKey is the key written under io.envoy.ai_gateway in the dynamic metadata.
+// RateLimitOverride emits a per-request rate-limit override struct into io.envoy.ai_gateway
+// dynamic metadata. The source value must be formatted as "COUNT/UNIT" (e.g. "100000/HOUR").
+// The gateway parses it and writes { requests_per_unit, unit } under MetadataKey — the struct
+// Envoy's RateLimit.Override.DynamicMetadata reads. If the source value is absent or malformed
+// the key is omitted and the BackendTrafficPolicy static default applies.
+type RateLimitOverride struct {
+	// MetadataKey is the key written under io.envoy.ai_gateway in the dynamic metadata,
+	// referenced by BackendTrafficPolicy.rateLimit.global.rules[].limit.fromMetadata.key.
 	//
 	// +kubebuilder:validation:Required
 	MetadataKey string `json:"metadataKey"`
-	// Header is the request header whose value encodes the rate limit as "<count>/<unit>",
-	// where unit is one of SECOND, MINUTE, HOUR, DAY.
+	// Source defines where the rate-limit value is read from.
+	// Exactly one field within Source must be set.
+	//
+	// +kubebuilder:validation:Required
+	Source RateLimitOverrideSource `json:"source"`
+}
+
+// RateLimitOverrideSource defines the origin of the rate-limit value.
+// Exactly one field must be set.
+//
+// +kubebuilder:validation:XValidation:rule="has(self.fromMetadata)",message="exactly one of fromMetadata must be set"
+type RateLimitOverrideSource struct {
+	// FromMetadata reads the rate-limit value from filter dynamic metadata set by a preceding
+	// Envoy filter (typically ext_authz). The value at the referenced namespace/key must be a
+	// string formatted as "<count>/<unit>" (e.g. "100000/HOUR"), where unit is one of
+	// SECOND, MINUTE, HOUR, DAY.
+	//
+	// Because dynamic metadata can only be set by Envoy filters — not by downstream clients —
+	// this source is safe against client spoofing with no additional configuration required.
 	//
-	// This header must be set exclusively by a trusted component (e.g. ext-authz).
-	// See the security note on RateLimitFromHeader for how to prevent client spoofing.
+	// +kubebuilder:validation:Required
+	FromMetadata RateLimitMetadataSource `json:"fromMetadata"`
+}
+
+// RateLimitMetadataSource identifies a string field in Envoy filter dynamic metadata.
+type RateLimitMetadataSource struct {
+	// Namespace is the filter metadata namespace to read from.
+	// For ext_authz this is typically "envoy.filters.http.ext_authz".
+	//
+	// +kubebuilder:validation:Required
+	Namespace string `json:"namespace"`
+	// Key is the field name within Namespace whose string value encodes the rate limit.
 	//
 	// +kubebuilder:validation:Required
-	Header string `json:"header"`
+	Key string `json:"key"`
 }
 
 // HTTPHeaderMutation defines the mutation of HTTP headers that will be applied to the request

api/v1beta1/zz_generated.deepcopy.go

@@ -138,16 +138,16 @@ func (c *GatewayController) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		return ctrl.Result{}, err
 	}
 	var defaultLLMCosts []aigv1b1.LLMRequestCost
-	var defaultRateLimitsFromHeaders []aigv1b1.RateLimitFromHeader
+	var defaultRateLimits []aigv1b1.RateLimitOverride
 	if gwConfig != nil {
 		defaultLLMCosts = gwConfig.Spec.GlobalLLMRequestCosts
-		defaultRateLimitsFromHeaders = gwConfig.Spec.GlobalRateLimitsFromHeaders
+		defaultRateLimits = gwConfig.Spec.GlobalRateLimits
 	}
 
 	// We need to create the filter config in Envoy Gateway system namespace because the sidecar extproc need
 	// to access it.
 	var hasEffectiveRoutes bool // indicates whether the filter config is effective (i.e., there is at least one active route).
-	hasEffectiveRoutes, err = c.reconcileFilterConfigSecret(ctx, FilterConfigSecretPerGatewayName(gw.Name, gw.Namespace), namespace, aiRoutes.Items, mcpRoutes.Items, uid, defaultLLMCosts, defaultRateLimitsFromHeaders)
+	hasEffectiveRoutes, err = c.reconcileFilterConfigSecret(ctx, FilterConfigSecretPerGatewayName(gw.Name, gw.Namespace), namespace, aiRoutes.Items, mcpRoutes.Items, uid, defaultLLMCosts, defaultRateLimits)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
@@ -250,12 +250,21 @@ func aigwLLMRequestCostToFilterAPI(cost aigv1b1.LLMRequestCost, routeName string
 	return out, nil
 }
 
-func aigwGlobalRateLimitFromHeaderToFilterAPI(h aigv1b1.RateLimitFromHeader) filterapi.GlobalRateLimitFromHeader {
-	return filterapi.GlobalRateLimitFromHeader{MetadataKey: h.MetadataKey, Header: strings.ToLower(h.Header)}
+func aigwGlobalRateLimitToFilterAPI(h aigv1b1.RateLimitOverride) filterapi.GlobalRateLimitOverride {
+	return filterapi.GlobalRateLimitOverride{
+		MetadataKey: h.MetadataKey,
+		Namespace:   h.Source.FromMetadata.Namespace,
+		Key:         h.Source.FromMetadata.Key,
+	}
 }
 
-func aigwRateLimitFromHeaderToFilterAPI(h aigv1b1.RateLimitFromHeader, routeName string) filterapi.RateLimitFromHeader {
-	return filterapi.RateLimitFromHeader{MetadataKey: h.MetadataKey, Header: strings.ToLower(h.Header), RouteName: routeName}
+func aigwRateLimitToFilterAPI(h aigv1b1.RateLimitOverride, routeName string) filterapi.RateLimitOverride {
+	return filterapi.RateLimitOverride{
+		MetadataKey: h.MetadataKey,
+		Namespace:   h.Source.FromMetadata.Namespace,
+		Key:         h.Source.FromMetadata.Key,
+		RouteName:   routeName,
+	}
 }
 
 // mergeBodyMutations merges route-level and backend-level BodyMutation with route-level taking precedence.
@@ -361,7 +370,7 @@ func (c *GatewayController) reconcileFilterConfigSecret(
 	mcpRoutes []aigv1b1.MCPRoute,
 	uuid string,
 	defaultLLMCosts []aigv1b1.LLMRequestCost,
-	defaultRateLimitsFromHeaders []aigv1b1.RateLimitFromHeader,
+	defaultRateLimits []aigv1b1.RateLimitOverride,
 ) (hasEffectiveRoute bool, _ error) {
 	// Precondition: aiGatewayRoutes is not empty as we early return if it is empty.
 	ec := &filterapi.Config{UUID: uuid, Version: version.Parse()}
@@ -378,10 +387,10 @@ func (c *GatewayController) reconcileFilterConfigSecret(
 		}
 		ec.GlobalLLMRequestCosts = append(ec.GlobalLLMRequestCosts, fc)
 	}
-	// Process global RateLimitsFromHeaders from GatewayConfig.
+	// Process global RateLimits from GatewayConfig.
 	// CRD enforces uniqueness via +listType=map, so no deduplication needed.
-	for _, h := range defaultRateLimitsFromHeaders {
-		ec.GlobalRateLimitsFromHeaders = append(ec.GlobalRateLimitsFromHeaders, aigwGlobalRateLimitFromHeaderToFilterAPI(h))
+	for _, h := range defaultRateLimits {
+		ec.GlobalRateLimits = append(ec.GlobalRateLimits, aigwGlobalRateLimitToFilterAPI(h))
 	}
 
 	// Models contributed by routes with no Spec.Hostnames. We only promote these to
@@ -523,12 +532,12 @@ func (c *GatewayController) reconcileFilterConfigSecret(
 				ec.LLMRequestCosts = append(ec.LLMRequestCosts, fc)
 			}
 
-			dedupRL := map[string]filterapi.RateLimitFromHeader{}
-			for _, h := range aiGatewayRoute.Spec.RateLimitsFromHeaders {
-				dedupRL[h.MetadataKey] = aigwRateLimitFromHeaderToFilterAPI(h, routeName)
+			dedupRL := map[string]filterapi.RateLimitOverride{}
+			for _, h := range aiGatewayRoute.Spec.RateLimits {
+				dedupRL[h.MetadataKey] = aigwRateLimitToFilterAPI(h, routeName)
 			}
 			for _, h := range dedupRL {
-				ec.RateLimitsFromHeaders = append(ec.RateLimitsFromHeaders, h)
+				ec.RateLimits = append(ec.RateLimits, h)
 			}
 		}
 	}

internal/controller/gateway.go

@@ -782,9 +782,9 @@ func TestGatewayController_reconcileFilterConfigSecret_SkipsDeletedRoutes(t *tes
 	require.Contains(t, fc.Backends[0].Name, "apple")
 }
 
-// TestGatewayController_reconcileFilterConfigSecret_RateLimitsFromHeaders verifies that
-// route-scoped RateLimitsFromHeaders and global RateLimitsFromHeaders are projected into the filter config.
-func TestGatewayController_reconcileFilterConfigSecret_RateLimitsFromHeaders(t *testing.T) {
+// TestGatewayController_reconcileFilterConfigSecret_RateLimits verifies that
+// route-scoped RateLimits and global RateLimits are projected into the filter config.
+func TestGatewayController_reconcileFilterConfigSecret_RateLimits(t *testing.T) {
 	fakeClient := requireNewFakeClientWithIndexes(t)
 	kube := fake2.NewClientset()
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel})))
@@ -799,9 +799,9 @@ func TestGatewayController_reconcileFilterConfigSecret_RateLimitsFromHeaders(t *
 				Rules: []aigv1b1.AIGatewayRouteRule{
 					{BackendRefs: []aigv1b1.AIGatewayRouteRuleBackendRef{{Name: "some-backend"}}},
 				},
-				RateLimitsFromHeaders: []aigv1b1.RateLimitFromHeader{
-					{MetadataKey: "llm_input_token_limit", Header: "x-aigw-limit-input"},
-					{MetadataKey: "llm_output_token_limit", Header: "x-aigw-limit-output"},
+				RateLimits: []aigv1b1.RateLimitOverride{
+					{MetadataKey: "llm_input_token_limit", Source: aigv1b1.RateLimitOverrideSource{FromMetadata: aigv1b1.RateLimitMetadataSource{Namespace: "my.ext_authz", Key: "input_limit"}}},
+					{MetadataKey: "llm_output_token_limit", Source: aigv1b1.RateLimitOverrideSource{FromMetadata: aigv1b1.RateLimitMetadataSource{Namespace: "my.ext_authz", Key: "output_limit"}}},
 				},
 			},
 		},
@@ -815,8 +815,8 @@ func TestGatewayController_reconcileFilterConfigSecret_RateLimitsFromHeaders(t *
 	})
 	require.NoError(t, err)
 
-	globalRateLimits := []aigv1b1.RateLimitFromHeader{
-		{MetadataKey: "llm_total_token_limit", Header: "x-aigw-limit-total"},
+	globalRateLimits := []aigv1b1.RateLimitOverride{
+		{MetadataKey: "llm_total_token_limit", Source: aigv1b1.RateLimitOverrideSource{FromMetadata: aigv1b1.RateLimitMetadataSource{Namespace: "my.ext_authz", Key: "total_limit"}}},
 	}
 
 	const someNamespace = "some-namespace"
@@ -833,15 +833,16 @@ func TestGatewayController_reconcileFilterConfigSecret_RateLimitsFromHeaders(t *
 	var fc filterapi.Config
 	require.NoError(t, yaml.Unmarshal([]byte(configStr), &fc))
 
-	require.Len(t, fc.GlobalRateLimitsFromHeaders, 1)
-	require.Equal(t, "llm_total_token_limit", fc.GlobalRateLimitsFromHeaders[0].MetadataKey)
-	require.Equal(t, "x-aigw-limit-total", fc.GlobalRateLimitsFromHeaders[0].Header)
+	require.Len(t, fc.GlobalRateLimits, 1)
+	require.Equal(t, "llm_total_token_limit", fc.GlobalRateLimits[0].MetadataKey)
+	require.Equal(t, "my.ext_authz", fc.GlobalRateLimits[0].Namespace)
+	require.Equal(t, "total_limit", fc.GlobalRateLimits[0].Key)
 
-	require.Len(t, fc.RateLimitsFromHeaders, 2)
-	for _, h := range fc.RateLimitsFromHeaders {
+	require.Len(t, fc.RateLimits, 2)
+	for _, h := range fc.RateLimits {
 		require.Equal(t, "ns/tenant-route", h.RouteName)
 	}
-	metadataKeys := []string{fc.RateLimitsFromHeaders[0].MetadataKey, fc.RateLimitsFromHeaders[1].MetadataKey}
+	metadataKeys := []string{fc.RateLimits[0].MetadataKey, fc.RateLimits[1].MetadataKey}
 	require.ElementsMatch(t, []string{"llm_input_token_limit", "llm_output_token_limit"}, metadataKeys)
 }
 

internal/controller/gateway_test.go

@@ -16,6 +16,7 @@ import (
 	extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
 	typev3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/envoyproxy/ai-gateway/internal/apischema/openai"
 	"github.com/envoyproxy/ai-gateway/internal/filterapi"
@@ -36,7 +37,7 @@ type modelsProcessor struct {
 var _ Processor = (*modelsProcessor)(nil)
 
 // NewModelsProcessor creates a new processor that returns the list of declared models.
-func NewModelsProcessor(config *filterapi.RuntimeConfig, requestHeaders map[string]string, logger *slog.Logger, isUpstreamFilter bool, _ bool) (Processor, error) {
+func NewModelsProcessor(config *filterapi.RuntimeConfig, requestHeaders map[string]string, _ map[string]*structpb.Struct, logger *slog.Logger, isUpstreamFilter bool, _ bool) (Processor, error) {
 	if isUpstreamFilter {
 		return passThroughProcessor{}, nil
 	}

internal/extproc/models_processor.go

@@ -34,7 +34,7 @@ func TestModels_ProcessRequestHeaders(t *testing.T) {
 			CreatedAt: now,
 		},
 	}}
-	p, err := NewModelsProcessor(cfg, nil, slog.Default(), false, false)
+	p, err := NewModelsProcessor(cfg, nil, nil, slog.Default(), false, false)
 	require.NoError(t, err)
 	res, err := p.ProcessRequestHeaders(t.Context(), &corev3.HeaderMap{
 		Headers: []*corev3.HeaderValue{{Key: "foo", Value: "bar"}},

internal/extproc/models_processor_test.go

@@ -11,12 +11,15 @@ import (
 
 	corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
+	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/envoyproxy/ai-gateway/internal/filterapi"
 )
 
 // ProcessorFactory is the factory function used to create new instances of a processor.
-type ProcessorFactory func(_ *filterapi.RuntimeConfig, _ map[string]string, _ *slog.Logger, isUpstreamFilter bool, enableRedaction bool) (Processor, error)
+// filterMetadata carries the Envoy filter dynamic metadata from ProcessingRequest.MetadataContext,
+// keyed by filter namespace. This is populated once per request on the RequestHeaders phase.
+type ProcessorFactory func(_ *filterapi.RuntimeConfig, _ map[string]string, filterMetadata map[string]*structpb.Struct, _ *slog.Logger, isUpstreamFilter bool, enableRedaction bool) (Processor, error)
 
 // Processor is the interface for the processor which corresponds to a single gRPC stream per the external processor filter.
 // This decouples the processor implementation detail from the server implementation.