fix: isolate job heartbeat and contain payload panics

Author: St4NNi

core/src/structs/job.rs

@@ -111,6 +111,7 @@ pub enum JobPayload {
         steps: u32,
         step_sleep_ms: u64,
         fail_at: Option<u32>,
+        panic_at: Option<u32>,
         cleanup_marker: Option<String>,
     },
 }
@@ -414,6 +415,7 @@ mod tests {
                 steps: 3,
                 step_sleep_ms: 0,
                 fail_at: None,
+                panic_at: None,
                 cleanup_marker: Some("/tmp/probe-marker".to_string()),
             },
             user(1, 2),

operations/src/jobs/executor.rs

@@ -66,13 +66,15 @@ pub async fn dispatch_payload(ctx: &JobContext, payload: &JobPayload) -> JobRunO
             steps,
             step_sleep_ms,
             fail_at,
+            panic_at,
             cleanup_marker,
         } => {
             run_probe(
                 ctx,
                 *steps,
                 *step_sleep_ms,
                 *fail_at,
+                *panic_at,
                 cleanup_marker.as_deref(),
             )
             .await
@@ -96,6 +98,7 @@ async fn run_probe(
     steps: u32,
     step_sleep_ms: u64,
     fail_at: Option<u32>,
+    panic_at: Option<u32>,
     cleanup_marker: Option<&str>,
 ) -> JobRunOutcome {
     ctx.progress.set_total(steps as u64);
@@ -106,6 +109,9 @@ async fn run_probe(
         if ctx.cancel.is_cancelled() {
             return JobRunOutcome::Cancelled;
         }
+        if panic_at == Some(step) {
+            panic!("probe panic at step {step}");
+        }
         if fail_at == Some(step) {
             return JobRunOutcome::Failed(JobError::retryable(format!(
                 "probe failed at step {step}"

operations/src/jobs/prune.rs

@@ -153,6 +153,7 @@ mod tests {
                 steps: 1,
                 step_sleep_ms: 0,
                 fail_at: None,
+                panic_at: None,
                 cleanup_marker: None,
             },
             UserId::new(Ulid::from_bytes([2u8; 16]), RealmId([1u8; 32])),