feat(gateway): same-key retry for single-provider routes

Retry once (or up to the configured budget) against the same env-var
key when a model resolves to a single provider with a single env-var
key — i.e. no alternate key and no other provider to fall back to.
Covers both explicit provider/model requests and auto-routed requests
that resolve to a single provider. When another provider exists,
cross-provider fallback handles retries and same-key retry stays off.

Bounded by the resolved routing-config retry.maxRetries budget
(default 2 → up to 2 same-key retries; 0 disables). Auth failures
(401/403) and BYOK/custom providers are excluded. The abort listener
is re-armed before each same-key retry so a client disconnect during
the retried upstream call still cancels. Key count is read from the
resolved envVarName (which may be a regional override), not the
provider default.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: steebchen

apps/gateway/src/chat/chat.ts

@@ -196,6 +196,7 @@ import { extractTokenUsage } from "./tools/extract-token-usage.js";
 import { extractToolCalls } from "./tools/extract-tool-calls.js";
 import { getFinishReasonFromError } from "./tools/get-finish-reason-from-error.js";
 import {
+	getEnvKeyCount,
 	getProviderEnv,
 	getServiceTierIneligibleEnvIndices,
 	hasServiceTierEligibleEnvCredential,
@@ -235,6 +236,7 @@ import {
 	providerRetryKey,
 	selectNextProvider,
 	shouldRetryAlternateKey,
+	shouldRetrySameKey,
 	shouldRetryRequest,
 } from "./tools/retry-with-fallback.js";
 import {
@@ -6575,6 +6577,7 @@ chat.openapi(completions, async (c) => {
 				// --- Retry loop for provider fallback ---
 				const routingAttempts: RoutingAttempt[] = [];
 				const failedProviderIds = new Set<string>();
+				let sameKeyRetryCount = 0;
 				let res: Response | undefined;
 				for (
 					let retryAttempt = 0;
@@ -6775,8 +6778,23 @@ chat.openapi(completions, async (c) => {
 								maxRetries: routingCfg.retry.maxRetries,
 							});
 							const willRetrySameProvider = sameProviderRetryContext !== null;
+							const willRetrySameKey =
+								!willRetrySameProvider &&
+								!willRetryTimeout &&
+								shouldRetrySameKey({
+									usedProvider,
+									errorType: "upstream_timeout",
+									statusCode: 0,
+									envVarName,
+									envKeyCount: getEnvKeyCount(envVarName),
+									hasOtherProvider: (
+										routingMetadata?.providerScores ?? []
+									).some((s) => s.providerId !== usedProvider),
+									retryCount: sameKeyRetryCount,
+									maxRetries: routingCfg.retry.maxRetries,
+								});
 							const willRetryRequest =
-								willRetrySameProvider || willRetryTimeout;
+								willRetrySameProvider || willRetryTimeout || willRetrySameKey;
 
 							const baseLogEntry = createLogEntry(
 								requestId,
@@ -6874,6 +6892,30 @@ chat.openapi(completions, async (c) => {
 								continue;
 							}
 
+							if (willRetrySameKey) {
+								sameKeyRetryCount++;
+								// Re-add abort listener (removed by catch/finally on the
+								// failed attempt) so a client disconnect during the
+								// retried upstream call still cancels.
+								c.req.raw.signal.addEventListener("abort", onAbort);
+								routingAttempts.push(
+									buildRoutingAttempt(
+										usedProvider,
+										usedInternalModel,
+										0,
+										getErrorType(0),
+										false,
+										{
+											region: usedRegion,
+											apiKeyHash: usedApiKeyHash,
+											logId: attemptLogId,
+										},
+									),
+								);
+								retryAttempt--;
+								continue;
+							}
+
 							if (willRetryTimeout) {
 								routingAttempts.push(
 									buildRoutingAttempt(
@@ -7113,7 +7155,23 @@ chat.openapi(completions, async (c) => {
 								maxRetries: routingCfg.retry.maxRetries,
 							});
 							const willRetrySameProvider = sameProviderRetryContext !== null;
-							const willRetryRequest = willRetrySameProvider || willRetryFetch;
+							const willRetrySameKey =
+								!willRetrySameProvider &&
+								!willRetryFetch &&
+								shouldRetrySameKey({
+									usedProvider,
+									errorType: "network_error",
+									statusCode: 0,
+									envVarName,
+									envKeyCount: getEnvKeyCount(envVarName),
+									hasOtherProvider: (
+										routingMetadata?.providerScores ?? []
+									).some((s) => s.providerId !== usedProvider),
+									retryCount: sameKeyRetryCount,
+									maxRetries: routingCfg.retry.maxRetries,
+								});
+							const willRetryRequest =
+								willRetrySameProvider || willRetryFetch || willRetrySameKey;
 
 							const baseLogEntry = createLogEntry(
 								requestId,
@@ -7230,6 +7288,30 @@ chat.openapi(completions, async (c) => {
 								continue;
 							}
 
+							if (willRetrySameKey) {
+								sameKeyRetryCount++;
+								// Re-add abort listener (removed by catch/finally on the
+								// failed attempt) so a client disconnect during the
+								// retried upstream call still cancels.
+								c.req.raw.signal.addEventListener("abort", onAbort);
+								routingAttempts.push(
+									buildRoutingAttempt(
+										usedProvider,
+										usedInternalModel,
+										0,
+										getErrorType(0),
+										false,
+										{
+											region: usedRegion,
+											apiKeyHash: usedApiKeyHash,
+											logId: attemptLogId,
+										},
+									),
+								);
+								retryAttempt--;
+								continue;
+							}
+
 							if (willRetryFetch) {
 								routingAttempts.push(
 									buildRoutingAttempt(
@@ -7355,8 +7437,23 @@ chat.openapi(completions, async (c) => {
 							maxRetries: routingCfg.retry.maxRetries,
 						});
 						const willRetrySameProvider = sameProviderRetryContext !== null;
+						const willRetrySameKey =
+							!willRetrySameProvider &&
+							!willRetryHttpError &&
+							shouldRetrySameKey({
+								usedProvider,
+								errorType: finishReason,
+								statusCode: res.status,
+								envVarName,
+								envKeyCount: getEnvKeyCount(envVarName),
+								hasOtherProvider: (routingMetadata?.providerScores ?? []).some(
+									(s) => s.providerId !== usedProvider,
+								),
+								retryCount: sameKeyRetryCount,
+								maxRetries: routingCfg.retry.maxRetries,
+							});
 						const willRetryRequest =
-							willRetrySameProvider || willRetryHttpError;
+							willRetrySameProvider || willRetryHttpError || willRetrySameKey;
 
 						const baseLogEntry = createLogEntry(
 							requestId,
@@ -7514,6 +7611,30 @@ chat.openapi(completions, async (c) => {
 							continue;
 						}
 
+						if (willRetrySameKey) {
+							sameKeyRetryCount++;
+							// Re-add abort listener (removed by catch/finally on the
+							// failed attempt) so a client disconnect during the
+							// retried upstream call still cancels.
+							c.req.raw.signal.addEventListener("abort", onAbort);
+							routingAttempts.push(
+								buildRoutingAttempt(
+									usedProvider,
+									usedInternalModel,
+									res.status,
+									getErrorType(res.status),
+									false,
+									{
+										region: usedRegion,
+										apiKeyHash: usedApiKeyHash,
+										logId: attemptLogId,
+									},
+								),
+							);
+							retryAttempt--;
+							continue;
+						}
+
 						if (willRetryHttpError) {
 							routingAttempts.push(
 								buildRoutingAttempt(
@@ -7675,8 +7796,25 @@ chat.openapi(completions, async (c) => {
 							maxRetries: routingCfg.retry.maxRetries,
 						});
 						const willRetrySameProvider = sameProviderRetryContext !== null;
+						const willRetrySameKey =
+							!willRetrySameProvider &&
+							!willRetryStreamingError &&
+							shouldRetrySameKey({
+								usedProvider,
+								errorType,
+								statusCode: inferredStatusCode,
+								envVarName,
+								envKeyCount: getEnvKeyCount(envVarName),
+								hasOtherProvider: (routingMetadata?.providerScores ?? []).some(
+									(s) => s.providerId !== usedProvider,
+								),
+								retryCount: sameKeyRetryCount,
+								maxRetries: routingCfg.retry.maxRetries,
+							});
 						const willRetryRequest =
-							willRetrySameProvider || willRetryStreamingError;
+							willRetrySameProvider ||
+							willRetryStreamingError ||
+							willRetrySameKey;
 
 						const baseLogEntry = createLogEntry(
 							requestId,
@@ -7794,6 +7932,30 @@ chat.openapi(completions, async (c) => {
 							continue;
 						}
 
+						if (willRetrySameKey) {
+							sameKeyRetryCount++;
+							// Re-add abort listener (removed by catch/finally on the
+							// failed attempt) so a client disconnect during the
+							// retried upstream call still cancels.
+							c.req.raw.signal.addEventListener("abort", onAbort);
+							routingAttempts.push(
+								buildRoutingAttempt(
+									usedProvider,
+									usedInternalModel,
+									inferredStatusCode,
+									getErrorType(inferredStatusCode),
+									false,
+									{
+										region: usedRegion,
+										apiKeyHash: usedApiKeyHash,
+										logId: attemptLogId,
+									},
+								),
+							);
+							retryAttempt--;
+							continue;
+						}
+
 						if (willRetryStreamingError) {
 							routingAttempts.push(
 								buildRoutingAttempt(
@@ -10666,6 +10828,7 @@ chat.openapi(completions, async (c) => {
 	// --- Retry loop for provider fallback ---
 	const routingAttempts: RoutingAttempt[] = [];
 	const failedProviderIds = new Set<string>();
+	let sameKeyRetryCount = 0;
 	let canceled = false;
 	let fetchError: Error | null = null;
 	let isTimeoutFetchError = false;
@@ -10896,8 +11059,23 @@ chat.openapi(completions, async (c) => {
 				maxRetries: routingCfg.retry.maxRetries,
 			});
 			const willRetrySameProvider = sameProviderRetryContext !== null;
+			const willRetrySameKey =
+				!willRetrySameProvider &&
+				!willRetryFetchNonStreaming &&
+				shouldRetrySameKey({
+					usedProvider,
+					errorType: "network_error",
+					statusCode: 0,
+					envVarName,
+					envKeyCount: getEnvKeyCount(envVarName),
+					hasOtherProvider: (routingMetadata?.providerScores ?? []).some(
+						(s) => s.providerId !== usedProvider,
+					),
+					retryCount: sameKeyRetryCount,
+					maxRetries: routingCfg.retry.maxRetries,
+				});
 			const willRetryRequest =
-				willRetrySameProvider || willRetryFetchNonStreaming;
+				willRetrySameProvider || willRetryFetchNonStreaming || willRetrySameKey;
 
 			const baseLogEntry = createLogEntry(
 				requestId,
@@ -11015,6 +11193,30 @@ chat.openapi(completions, async (c) => {
 				continue;
 			}
 
+			if (willRetrySameKey) {
+				sameKeyRetryCount++;
+				// Re-add abort listener (removed by the per-attempt finally) so
+				// a client disconnect during the retried upstream call still
+				// cancels.
+				c.req.raw.signal.addEventListener("abort", onAbort);
+				routingAttempts.push(
+					buildRoutingAttempt(
+						usedProvider,
+						usedInternalModel,
+						0,
+						getErrorType(0),
+						false,
+						{
+							region: usedRegion,
+							apiKeyHash: usedApiKeyHash,
+							logId: attemptLogId,
+						},
+					),
+				);
+				retryAttempt--;
+				continue;
+			}
+
 			if (willRetryFetchNonStreaming) {
 				routingAttempts.push(
 					buildRoutingAttempt(
@@ -11278,8 +11480,23 @@ chat.openapi(completions, async (c) => {
 				maxRetries: routingCfg.retry.maxRetries,
 			});
 			const willRetrySameProvider = sameProviderRetryContext !== null;
+			const willRetrySameKey =
+				!willRetrySameProvider &&
+				!willRetryHttpNonStreaming &&
+				shouldRetrySameKey({
+					usedProvider,
+					errorType: finishReason,
+					statusCode: res.status,
+					envVarName,
+					envKeyCount: getEnvKeyCount(envVarName),
+					hasOtherProvider: (routingMetadata?.providerScores ?? []).some(
+						(s) => s.providerId !== usedProvider,
+					),
+					retryCount: sameKeyRetryCount,
+					maxRetries: routingCfg.retry.maxRetries,
+				});
 			const willRetryRequest =
-				willRetrySameProvider || willRetryHttpNonStreaming;
+				willRetrySameProvider || willRetryHttpNonStreaming || willRetrySameKey;
 
 			const baseLogEntry = createLogEntry(
 				requestId,
@@ -11456,6 +11673,30 @@ chat.openapi(completions, async (c) => {
 				continue;
 			}
 
+			if (willRetrySameKey) {
+				sameKeyRetryCount++;
+				// Re-add abort listener (removed by the per-attempt finally) so
+				// a client disconnect during the retried upstream call still
+				// cancels.
+				c.req.raw.signal.addEventListener("abort", onAbort);
+				routingAttempts.push(
+					buildRoutingAttempt(
+						usedProvider,
+						usedInternalModel,
+						res.status,
+						getErrorType(res.status),
+						false,
+						{
+							region: usedRegion,
+							apiKeyHash: usedApiKeyHash,
+							logId: attemptLogId,
+						},
+					),
+				);
+				retryAttempt--;
+				continue;
+			}
+
 			if (willRetryHttpNonStreaming) {
 				routingAttempts.push(
 					buildRoutingAttempt(

apps/gateway/src/chat/tools/get-provider-env.ts

@@ -2,6 +2,7 @@ import { HTTPException } from "hono/http-exception";
 
 import {
 	getRoundRobinValue,
+	parseCommaSeparatedEnv,
 	peekRoundRobinValue,
 } from "@/lib/round-robin-env.js";
 
@@ -147,3 +148,20 @@ export function getProviderEnv(
 
 	return { token: result.value, configIndex: result.index, envVarName: envVar };
 }
+
+/**
+ * Returns the number of comma-separated values configured in the named env
+ * var, or 0 if it's unset/empty. Pass the resolved `envVarName` from the
+ * provider context — it may be a regional override (e.g. `*__SINGAPORE`)
+ * rather than the provider's base var.
+ */
+export function getEnvKeyCount(envVarName: string | undefined): number {
+	if (!envVarName) {
+		return 0;
+	}
+	const value = process.env[envVarName];
+	if (!value) {
+		return 0;
+	}
+	return parseCommaSeparatedEnv(value).length;
+}