Skip to content

Commit 64c9862

Browse files
perf/robustness(api-runner): hard per-turn timeout (anti-wedge)
A heavy MoA run hung ~1h45m on a single non-returning /chat/completions call — the httpx read timeout didn't fire (kept-alive stream). Wrap each turn's request in asyncio.wait_for(timeout=SPECA_API_TURN_TIMEOUT, default 150s); on timeout log an api_error and continue to the next turn so one stuck model can't wedge the whole run. 119 runner tests green.
1 parent cee8d03 commit 64c9862

1 file changed

Lines changed: 12 additions & 3 deletions

File tree

scripts/orchestrator/api_runner.py

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,8 @@ def __init__(
392392
_tok = _os.environ.get("SPECA_API_MAX_TOKENS", "").strip()
393393
self.max_tokens = int(_tok) if _tok.isdigit() and int(_tok) > 0 else 16384
394394
self.reasoning_effort = _os.environ.get("SPECA_API_REASONING_EFFORT", "").strip().lower() or None
395+
_tt = _os.environ.get("SPECA_API_TURN_TIMEOUT", "").strip()
396+
self.turn_timeout = float(_tt) if _tt.replace(".", "", 1).isdigit() and float(_tt) > 0 else 150.0
395397

396398
async def run_batch(
397399
self,
@@ -542,11 +544,18 @@ async def _execute_batch(
542544
})
543545

544546
try:
545-
response = await client.post(
546-
"/chat/completions",
547-
json=request_body,
547+
response = await asyncio.wait_for(
548+
client.post("/chat/completions", json=request_body),
549+
timeout=self.turn_timeout,
548550
)
549551
response.raise_for_status()
552+
except (asyncio.TimeoutError, httpx.TimeoutException) as e:
553+
log_entries.append({
554+
"type": "api_error", "turn": turn, "status": 0,
555+
"error": f"turn timeout after {self.turn_timeout}s (anti-wedge)",
556+
})
557+
await asyncio.sleep(2)
558+
continue
550559
except httpx.HTTPStatusError as e:
551560
error_text = e.response.text[:500] if e.response else str(e)
552561
log_entries.append({

0 commit comments

Comments
 (0)