Skip to content

Commit edcd151

Browse files
Merge branch 'Th0rgal:master' into master
2 parents 9d50ff4 + 058c6e0 commit edcd151

1 file changed

Lines changed: 40 additions & 0 deletions

File tree

src/api/supervision.rs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,13 @@ pub(crate) async fn stuck_mission_watchdog_loop(
318318
// previous tick. Entries for dead scopes are pruned each pass.
319319
let mut oom_seen: std::collections::HashMap<String, u64> = std::collections::HashMap::new();
320320

321+
// Worker missions we have already auto-resumed once this process lifetime.
322+
// Auto-resume is a single supervised retry for workers whose runner died
323+
// (orphan / deploy SIGTERM) — an environmental interruption, not the
324+
// worker's own fault. If the resumed worker dies again it stays
325+
// interrupted and the boss handles it, so this can never resume-storm.
326+
let mut auto_resumed_workers: HashSet<Uuid> = HashSet::new();
327+
321328
loop {
322329
tokio::time::sleep(CHECK_INTERVAL).await;
323330

@@ -465,6 +472,39 @@ pub(crate) async fn stuck_mission_watchdog_loop(
465472
.to_string(),
466473
),
467474
});
475+
476+
// One supervised auto-resume for orphaned WORKER missions. The
477+
// boss used to babysit this by hand (10 manual resume_worker
478+
// calls in one campaign); a runner death is environmental, so a
479+
// single retry is safe. Once-only per process: a worker that dies
480+
// again stays interrupted for the boss to triage.
481+
if mission.parent_mission_id.is_some() && auto_resumed_workers.insert(mission.id) {
482+
tracing::info!(
483+
mission_id = %mission.id,
484+
parent = ?mission.parent_mission_id,
485+
"Stuck-mission watchdog: auto-resuming orphaned worker once"
486+
);
487+
let (resume_tx, resume_rx) = oneshot::channel();
488+
if cmd_tx
489+
.send(ControlCommand::ResumeMission {
490+
mission_id: mission.id,
491+
clean_workspace: false,
492+
skip_message: false,
493+
respond: resume_tx,
494+
})
495+
.await
496+
.is_ok()
497+
{
498+
match resume_rx.await {
499+
Ok(Ok(_)) => {}
500+
Ok(Err(e)) => tracing::warn!(
501+
mission_id = %mission.id,
502+
"Auto-resume failed: {}; leaving interrupted for the boss", e
503+
),
504+
Err(_) => {}
505+
}
506+
}
507+
}
468508
}
469509
}
470510
}

0 commit comments

Comments
 (0)