fix(tw): retry Stripe 429s in the test harness (TW_WORKER_MODE-gated)

stripe-node's maxNetworkRetries never retries 429s (_shouldRetry covers
connection errors, 409, 5xx, and stripe-should-retry only), so rate-limit
collateral killed tests outright — the largest single infra-flake cluster
(~30 tests across the measured runs).

withTwStripeRateLimitRetry wraps the three factory-constructed clients
with a lazy deep Proxy: thenable-returning calls get up to 3 retries on
429/rate_limit with 1s/2s/4s jittered backoff (8s cap), honoring
Retry-After when larger. Wrapping sits OUTSIDE instrumentStripe so each
retry gets its own OTel span. Identity function unless TW_WORKER_MODE=1
and NODE_ENV != production — the same double-gate as stripeRetryOptions,
so prod clients are provably untouched. Retrying 429s specifically is
idempotency-safe: a rate-limited request was rejected before processing.

Known trade-off: rewrapped calls return plain promises, dropping Stripe's
autoPaging* extras — verified unused repo-wide.

Author: SirTenzin

server/src/external/connect/createStripeCli.ts

@@ -16,6 +16,7 @@ import {
 	initPlatformStripe,
 	stripeRetryOptions,
 } from "./initStripeCli.js";
+import { withTwStripeRateLimitRetry } from "./twStripeRateLimitRetry.js";
 
 export const createStripeCli = ({
 	org,
@@ -64,7 +65,9 @@ export const createStripeCli = ({
 						: undefined,
 					...stripeRetryOptions(),
 				});
-				return skipInstrumentation ? client : instrumentStripe({ client });
+				return withTwStripeRateLimitRetry(
+					skipInstrumentation ? client : instrumentStripe({ client }),
+				);
 			},
 		});
 	}

server/src/external/connect/initStripeCli.ts

@@ -16,6 +16,7 @@ import {
 } from "./clientCache/cacheKeyUtils.js";
 import { getOrCreateStripeClient } from "./clientCache/stripeClientCache.js";
 import { getConnectWebhookSecret } from "./connectUtils.js";
+import { withTwStripeRateLimitRetry } from "./twStripeRateLimitRetry.js";
 
 // tw-swarm seam: a forked worker can't change its snapshotted process env, so
 // its per-worker pool key is read from a file written at fork-bind time.
@@ -98,7 +99,9 @@ export const initMasterStripe = (params?: {
 					: undefined,
 				...stripeRetryOptions(),
 			});
-			return params?.skipInstrumentation ? client : instrumentStripe({ client });
+			return withTwStripeRateLimitRetry(
+				params?.skipInstrumentation ? client : instrumentStripe({ client }),
+			);
 		},
 	});
 };
@@ -158,7 +161,9 @@ export const initPlatformStripe = ({
 				apiVersion: legacyVersion ? ("2025-02-24.acacia" as any) : undefined,
 				...stripeRetryOptions(),
 			});
-			return skipInstrumentation ? client : instrumentStripe({ client });
+			return withTwStripeRateLimitRetry(
+				skipInstrumentation ? client : instrumentStripe({ client }),
+			);
 		},
 	});
 };

server/src/external/connect/twStripeRateLimitRetry.ts

@@ -0,0 +1,89 @@
+import type Stripe from "stripe";
+
+/**
+ * tw-swarm seam: retry-on-429 for the test harness only, same gate as
+ * stripeRetryOptions/twWorkerStripeKey. The SDK's maxNetworkRetries never
+ * retries 429s (RequestSender._shouldRetry: connection errors, 409, 5xx,
+ * stripe-should-retry only), so rate-limit collateral kills tests outright.
+ * Deliberately NOT enabled in prod: blind retries on non-idempotent billing
+ * calls risk double-charging when a request succeeded but the response died.
+ */
+const isTwWorkerHarness = (): boolean =>
+	process.env.TW_WORKER_MODE === "1" &&
+	process.env.NODE_ENV !== "production";
+
+const MAX_RETRIES = 3;
+const BASE_DELAY_MS = 1000;
+const MAX_DELAY_MS = 8000;
+
+const isRateLimitError = (error: unknown): boolean => {
+	const e = error as { statusCode?: number; code?: string; type?: string };
+	return (
+		e?.statusCode === 429 ||
+		e?.code === "rate_limit" ||
+		e?.type === "StripeRateLimitError"
+	);
+};
+
+/** Exponential backoff with jitter; honors Stripe's Retry-After when larger. */
+const retryDelayMs = (attempt: number, error: unknown): number => {
+	const headers = (error as { headers?: Record<string, string> })?.headers;
+	const retryAfterSec = Number(headers?.["retry-after"]);
+	const backoff = Math.min(BASE_DELAY_MS * 2 ** attempt, MAX_DELAY_MS);
+	const jittered = backoff * (0.5 + Math.random() * 0.5);
+	return Number.isFinite(retryAfterSec) && retryAfterSec > 0
+		? Math.max(retryAfterSec * 1000, jittered)
+		: jittered;
+};
+
+const wrapFn = (
+	fn: (...args: unknown[]) => unknown,
+	self: unknown,
+): ((...args: unknown[]) => unknown) =>
+	function wrapped(...args: unknown[]) {
+		const invoke = () => Reflect.apply(fn, self, args);
+		const first = invoke();
+		// Sync return (config getters etc.) — pass through untouched.
+		if (!(first && typeof (first as Promise<unknown>).then === "function")) {
+			return first;
+		}
+		return (async () => {
+			let attempt = 0;
+			let pending = first as Promise<unknown>;
+			for (;;) {
+				try {
+					return await pending;
+				} catch (error) {
+					if (!isRateLimitError(error) || attempt >= MAX_RETRIES) {
+						throw error;
+					}
+					await new Promise((resolve) =>
+						setTimeout(resolve, retryDelayMs(attempt, error)),
+					);
+					attempt += 1;
+					pending = invoke() as Promise<unknown>;
+				}
+			}
+		})();
+	};
+
+const wrapNamespace = <T extends object>(target: T): T =>
+	new Proxy(target, {
+		get(t, prop, _receiver) {
+			const value = Reflect.get(t, prop, t);
+			if (typeof prop !== "string" || prop.startsWith("_")) return value;
+			if (typeof value === "function") {
+				return wrapFn(value as (...args: unknown[]) => unknown, t);
+			}
+			if (value && typeof value === "object" && !Array.isArray(value)) {
+				return wrapNamespace(value);
+			}
+			return value;
+		},
+	});
+
+/** Identity outside the tw worker harness — prod clients are untouched. */
+export const withTwStripeRateLimitRetry = (client: Stripe): Stripe => {
+	if (!isTwWorkerHarness()) return client;
+	return wrapNamespace(client);
+};