fix(web): back off billing retries instead of hammering a failing transport (#6446)

The packaged client's first open of an unmaterialized team project fires a
burst of od:// requests; when that burst hits transient transport failures
the proxy answers with synthetic 502s, and the billing read's retry loop
made it worse: every mounted consumer armed its own fixed 5s timer, each
timer's retry event fanned out to every consumer, and each listener
force-refetched past the coalescing cache — one failure became a standing
5s-cadence request storm that fed the very burst it was waiting out
(observed by QA as hundreds of repeated billing requests while the left
panel span).

- Replace the per-hook fixed 5s retry timer with one module-level schedule
  per requestKey whose delay grows 5s → 10s → 20s → 40s → 60s cap while
  failures are consecutive; a success resets the count but deliberately
  leaves a pending timer armed so stale consumers still re-sync.
- Stop forcing the retry read. Failures are never cached, so the plain
  coalesced path is a genuine refetch, and a retry that lands just after a
  concurrent consumer's success now joins that fresh result instead of
  evicting it and issuing another request.
- Hard-expiry revalidation keeps its forced read via an explicit
  force flag on the retry event — its cached answer is void by definition,
  which the plain path cannot know.

Red tests first: both new specs fail on the previous implementation
(fixed-cadence retry fires at t=10s; retry stampedes a third request past a
0.5s-old success) and pass with the fix. 403 stays fail-closed with no
retry; the 83 web test files touching workspace context/billing all pass.

Author: lefarcen

apps/web/src/collab/useWorkspaceContext.ts

@@ -1009,6 +1009,7 @@ class WorkspaceBillingHttpError extends Error {
 export function resetWorkspaceBillingCache(): void {
   cachedWorkspaceBillingResponses.clear();
   resetWorkspaceBillingInterestRegistry();
+  resetWorkspaceBillingRetrySchedules();
 }
 
 type BillingInvalidation = Extract<
@@ -1101,7 +1102,6 @@ export function useWorkspaceBillingResponse(
   const activeScopeKeyRef = useRef<string | null>(billingScopeKey);
   const activeRequestKeyRef = useRef<string | null>(billingRequestKey);
   const requestEpochRef = useRef(0);
-  const retryTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
   const runtimeManagedRef = useRef(false);
   const interestOwnerIdRef = useRef('');
   if (!interestOwnerIdRef.current) {
@@ -1115,7 +1115,9 @@ export function useWorkspaceBillingResponse(
     return () => {
       mountedRef.current = false;
       requestEpochRef.current += 1;
-      if (retryTimerRef.current) clearTimeout(retryTimerRef.current);
+      // The retry schedule is module-level and deliberately survives this
+      // unmount: another consumer of the same requestKey may still be
+      // mounted, and a timer that fires with no listeners is a no-op.
     };
   }, []);
 
@@ -1195,10 +1197,7 @@ export function useWorkspaceBillingResponse(
         activeRequestKeyRef.current === requestKey
       ) {
         runtimeManagedRef.current = Boolean(response.workspaceRuntime);
-        if (retryTimerRef.current) {
-          clearTimeout(retryTimerRef.current);
-          retryTimerRef.current = null;
-        }
+        clearWorkspaceBillingRetryFailures(requestKey);
         cachedWorkspaceBillingResponses.set(scopeKey, response);
         setState({ scopeKey, response });
       }
@@ -1231,20 +1230,10 @@ export function useWorkspaceBillingResponse(
             },
           });
         }
-        if (!revoked && !retryTimerRef.current) {
-          retryTimerRef.current = setTimeout(() => {
-            retryTimerRef.current = null;
-            if (
-              mountedRef.current &&
-              activeScopeKeyRef.current === scopeKey &&
-              activeRequestKeyRef.current === requestKey
-            ) {
-              window.dispatchEvent(new CustomEvent(WORKSPACE_BILLING_RETRY_EVENT, {
-                detail: { requestKey },
-              }));
-            }
-          }, WORKSPACE_BILLING_RETRY_MS);
-        }
+        // A revoked read (403) fails closed and must not retry. Everything
+        // else — including the packaged client's synthetic proxy 502s —
+        // retries on the shared, exponentially backed-off schedule.
+        if (!revoked) scheduleWorkspaceBillingRetry(requestKey);
       }
     }
   }, [
@@ -1306,8 +1295,13 @@ export function useWorkspaceBillingResponse(
       const expired = enforceWorkspaceBillingHardExpiry(current);
       cachedWorkspaceBillingResponses.set(scopeKey, expired);
       setState({ scopeKey, response: expired });
+      // `force: true` — hard expiry KNOWS the cached answer is void (that is
+      // the whole point of the timer), so the revalidation must bypass any
+      // settled coalescing entry, exactly like an identity change. Failure
+      // retries deliberately dispatch WITHOUT force so they can share a
+      // concurrent consumer's fresh success instead.
       window.dispatchEvent(new CustomEvent(WORKSPACE_BILLING_RETRY_EVENT, {
-        detail: { requestKey },
+        detail: { requestKey, force: true },
       }));
     };
     timer = setTimeout(
@@ -1381,8 +1375,15 @@ export function useWorkspaceBillingResponse(
       if (event.key === WORKSPACE_BILLING_REFRESH_STORAGE_KEY) refreshAfterIdentityChange();
     };
     const onRetry = (event: Event) => {
-      const requestKey = (event as CustomEvent<{ requestKey?: string }>).detail?.requestKey;
-      if (requestKey === activeRequestKeyRef.current) void loadBilling(false, true);
+      const detail = (event as CustomEvent<{ requestKey?: string; force?: boolean }>).detail;
+      if (detail?.requestKey !== activeRequestKeyRef.current) return;
+      // Failure retries are deliberately NOT forced: a failed read is never
+      // cached, so the plain coalesced path is a genuine refetch — and when a
+      // concurrent consumer just succeeded, joining that fresh result re-syncs
+      // this one without adding another request to an already-struggling
+      // transport. Hard-expiry revalidation dispatches with `force: true`
+      // because its cached answer is void by definition.
+      void loadBilling(false, detail?.force === true);
     };
     window.addEventListener('focus', refresh);
     window.addEventListener('pageshow', refresh);
@@ -1577,8 +1578,74 @@ export function workspaceBillingSnapshotForContext(
 }
 
 const WORKSPACE_BILLING_POLL_MS = 30_000;
-const WORKSPACE_BILLING_RETRY_MS = 5_000;
+const WORKSPACE_BILLING_RETRY_BASE_MS = 5_000;
+const WORKSPACE_BILLING_RETRY_MAX_MS = 60_000;
 const WORKSPACE_BILLING_RETRY_EVENT = 'od:workspace-billing-retry';
+
+/**
+ * One retry schedule per billing `requestKey`, shared by every mounted
+ * consumer — the module-level counterpart of the per-hook timer it replaced.
+ *
+ * Two properties are load-bearing for the packaged (od://) client, whose
+ * proxy answers with synthetic 502s (`OD_PROTOCOL_PROXY_FAILED`) when the
+ * bursty first-open request load hits a transient transport failure:
+ *
+ *  1. The delay grows exponentially (5s → 10s → 20s → 40s → 60s cap) while
+ *     failures are consecutive. A fixed 5s cadence against a struggling
+ *     transport is self-defeating — each retry adds to the very burst that
+ *     is producing the 502s it is retrying.
+ *  2. The schedule is keyed once per requestKey, not once per mounted hook.
+ *     N consumers failing on the same shared read used to arm N timers whose
+ *     N events each fanned out to N listeners; one schedule dispatches one
+ *     retry event per cycle and the listeners' coalesced reads share one
+ *     network request.
+ *
+ * A success only RESETS the consecutive-failure count — it deliberately does
+ * not cancel a pending timer. Consumers hold their own state, so a success
+ * observed by one consumer has not reached the others; letting the pending
+ * retry fire re-syncs them through the coalescing cache (a fresh success
+ * within the share window costs zero network requests).
+ */
+type WorkspaceBillingRetrySchedule = {
+  consecutiveFailures: number;
+  timer: ReturnType<typeof setTimeout> | null;
+};
+const workspaceBillingRetrySchedules = new Map<string, WorkspaceBillingRetrySchedule>();
+
+function scheduleWorkspaceBillingRetry(requestKey: string): void {
+  if (typeof window === 'undefined') return;
+  let schedule = workspaceBillingRetrySchedules.get(requestKey);
+  if (!schedule) {
+    schedule = { consecutiveFailures: 0, timer: null };
+    workspaceBillingRetrySchedules.set(requestKey, schedule);
+  }
+  if (schedule.timer != null) return;
+  const delay = Math.min(
+    WORKSPACE_BILLING_RETRY_BASE_MS * 2 ** schedule.consecutiveFailures,
+    WORKSPACE_BILLING_RETRY_MAX_MS,
+  );
+  schedule.consecutiveFailures += 1;
+  schedule.timer = setTimeout(() => {
+    schedule.timer = null;
+    // Dispatch unconditionally: listeners filter on their own active
+    // requestKey, and an event nobody is mounted for is a no-op.
+    window.dispatchEvent(
+      new CustomEvent(WORKSPACE_BILLING_RETRY_EVENT, { detail: { requestKey } }),
+    );
+  }, delay);
+}
+
+function clearWorkspaceBillingRetryFailures(requestKey: string): void {
+  const schedule = workspaceBillingRetrySchedules.get(requestKey);
+  if (schedule) schedule.consecutiveFailures = 0;
+}
+
+function resetWorkspaceBillingRetrySchedules(): void {
+  for (const schedule of workspaceBillingRetrySchedules.values()) {
+    if (schedule.timer != null) clearTimeout(schedule.timer);
+  }
+  workspaceBillingRetrySchedules.clear();
+}
 export const WORKSPACE_BILLING_REFRESH_EVENT = 'od:workspace-billing-refresh';
 const WORKSPACE_BILLING_REFRESH_STORAGE_KEY = 'od.workspaceBilling.refreshAt';
 

apps/web/tests/useWorkspaceBilling.scope.test.tsx

@@ -9,6 +9,7 @@ import type {
 import { resetCoalescedGet } from '../src/lib/coalesced-get';
 import {
   lastResolvedWorkspaceContext,
+  notifyWorkspaceBillingRefresh,
   notifyWorkspaceContextRefresh,
   resetWorkspaceBillingCache,
   resetWorkspaceContextCache,
@@ -938,6 +939,162 @@ describe('useWorkspaceBilling explicit scope', () => {
     );
   }, 7_000);
 
+  it('backs off exponentially while billing keeps failing and re-arms at the base delay after a success', async () => {
+    // Packaged-client regression (first-open loading spin): the od:// proxy
+    // answers billing with synthetic 502s under bursty first-open load, and a
+    // FIXED 5s retry cadence kept feeding the burst it was waiting out. The
+    // schedule must grow 5s → 10s → 20s → 40s … and reset once a read succeeds.
+    vi.useFakeTimers();
+    Object.defineProperty(document, 'visibilityState', {
+      configurable: true,
+      get: () => 'hidden', // silence the 30s compatibility poll — this test meters retries only
+    });
+    let billingCalls = 0;
+    let failBilling = true;
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+        const url = String(input);
+        if (url === '/api/workspace/directory') {
+          return workspaceDirectoryResponse(teamContext('workspace-a'));
+        }
+        if (url === '/api/workspace/context') {
+          return new Response(JSON.stringify({ context: teamContext('workspace-a') }), {
+            status: 200,
+            headers: { 'content-type': 'application/json' },
+          });
+        }
+        const interest = billingInterestResponse(input, init);
+        if (interest) return interest;
+        if (url.startsWith('/api/workspace/billing?')) {
+          billingCalls += 1;
+          if (failBilling) {
+            return new Response(JSON.stringify({ error: 'od_protocol_proxy_failed' }), {
+              status: 502,
+              headers: { 'content-type': 'application/json' },
+            });
+          }
+          return new Response(JSON.stringify(billingResponse('workspace-a', '2.50')), {
+            status: 200,
+            headers: { 'content-type': 'application/json' },
+          });
+        }
+        throw new Error(`unexpected fetch ${url}`);
+      }),
+    );
+    const flush = async (ms: number) => {
+      await act(async () => {
+        await vi.advanceTimersByTimeAsync(ms);
+      });
+    };
+
+    try {
+      const hook = renderHook(() => useWorkspaceBillingResponse());
+      await flush(0);
+      await flush(0);
+      expect(billingCalls).toBe(1); // mount read failed (502)
+
+      await flush(5_000); // 1st retry at base 5s
+      expect(billingCalls).toBe(2);
+
+      await flush(5_000); // t=10s — a fixed 5s cadence would fire here
+      expect(billingCalls).toBe(2); // backed off: next retry is 10s out
+
+      await flush(5_000); // t=15s — the 10s retry lands
+      expect(billingCalls).toBe(3);
+
+      await flush(20_000); // t=35s — the 20s retry lands
+      expect(billingCalls).toBe(4);
+
+      failBilling = false;
+      await flush(40_000); // t=75s — the 40s retry lands and SUCCEEDS
+      expect(billingCalls).toBe(5);
+      expect(hook.result.current?.workspaceBalance?.balanceUsd).toBe('2.50');
+
+      // A later failure starts over at the base delay — the success reset
+      // the consecutive-failure count.
+      failBilling = true;
+      act(() => {
+        notifyWorkspaceBillingRefresh();
+      });
+      await flush(0);
+      expect(billingCalls).toBe(6); // pushed refresh failed (502)
+      await flush(5_000); // retry at base 5s again, not 60s
+      expect(billingCalls).toBe(7);
+    } finally {
+      delete (document as unknown as Record<string, unknown>).visibilityState;
+    }
+  }, 15_000);
+
+  it('a retry joins a concurrent consumer\'s fresh success instead of stampeding another request', async () => {
+    // De-forcing the retry path: failures are never cached, so the plain
+    // coalesced read is a genuine refetch — and when another mounted consumer
+    // just fetched a fresh success, the retry must share it (zero new network
+    // requests against a struggling transport) rather than evict it.
+    vi.useFakeTimers();
+    Object.defineProperty(document, 'visibilityState', {
+      configurable: true,
+      get: () => 'hidden',
+    });
+    let billingCalls = 0;
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+        const url = String(input);
+        if (url === '/api/workspace/directory') {
+          return workspaceDirectoryResponse(teamContext('workspace-a'));
+        }
+        if (url === '/api/workspace/context') {
+          return new Response(JSON.stringify({ context: teamContext('workspace-a') }), {
+            status: 200,
+            headers: { 'content-type': 'application/json' },
+          });
+        }
+        const interest = billingInterestResponse(input, init);
+        if (interest) return interest;
+        if (url.startsWith('/api/workspace/billing?')) {
+          billingCalls += 1;
+          if (billingCalls === 1) {
+            return new Response(JSON.stringify({ error: 'od_protocol_proxy_failed' }), {
+              status: 502,
+              headers: { 'content-type': 'application/json' },
+            });
+          }
+          return new Response(JSON.stringify(billingResponse('workspace-a', '2.50')), {
+            status: 200,
+            headers: { 'content-type': 'application/json' },
+          });
+        }
+        throw new Error(`unexpected fetch ${url}`);
+      }),
+    );
+    const flush = async (ms: number) => {
+      await act(async () => {
+        await vi.advanceTimersByTimeAsync(ms);
+      });
+    };
+
+    try {
+      const first = renderHook(() => useWorkspaceBillingResponse());
+      await flush(0);
+      await flush(0);
+      expect(billingCalls).toBe(1); // first consumer's mount read failed → retry armed at +5s
+
+      await flush(4_500);
+      const second = renderHook(() => useWorkspaceBillingResponse());
+      await flush(0);
+      await flush(0);
+      expect(billingCalls).toBe(2); // second consumer's mount read succeeded
+      expect(second.result.current?.workspaceBalance?.balanceUsd).toBe('2.50');
+
+      await flush(500); // t=5s — the first consumer's retry fires
+      expect(billingCalls).toBe(2); // joined the 0.5s-old success; no third request
+      expect(first.result.current?.workspaceBalance?.balanceUsd).toBe('2.50');
+    } finally {
+      delete (document as unknown as Record<string, unknown>).visibilityState;
+    }
+  }, 15_000);
+
   it('keeps daemon-authored last-good state on a retryable directory outage', async () => {
     let billingCalls = 0;
     vi.stubGlobal(