feat: add health check for RUNNING jobs

- New ABNORMAL state in enum Jobstate — set when health check detects
  a RUNNING job is stuck (output empty AND no child processes)
- s_check_running_health() — periodic check, runs every 10s in server loop
- count_child_pids() — reads /proc/PID/children to detect zombie jobs
- health_state field in struct Job — 0=UNCHECKED, 1=NORMAL, skip on rechecks
- On ABNORMAL: free slots via free_cores(), release CPU binding,
  state changes to ABNORMAL so it's excluded from slot accounting
- s_remove_job() rejects removing ABNORMAL jobs (use ts -k instead)
- s_send_output() allows ABNORMAL state so ts -k can get the PID
- Only adds new code — zero changes to existing logic paths

Author: kylincaster

job_ops.c

@@ -505,7 +505,7 @@ void s_send_output(int s, int jobid) {
   } else {
     p = get_job(jobid);
     if (p != 0 && p->state != RUNNING && p->state != FINISHED && p->state != SKIPPED
-        && p->state != PAUSE)
+        && p->state != PAUSE && p->state != ABNORMAL)
       p = 0;
     /* Thaw PAUSEd jobs so signals can be delivered */
     if (p != 0 && p->state == PAUSE) {

jobs.c

@@ -252,6 +252,79 @@ static int s_check_timeout() {
     return had_timeout;
 }
 
+/* Forward declaration */
+static int count_child_pids(pid_t parent_pid);
+
+/* Health check for RUNNING jobs.
+   Checks jobs that have been RUNNING for ≥300 seconds and are HEALTH_UNCHECKED.
+   Conditions indicating an unhealthy job:
+     1) output log file exists but is empty (stat size == 0)
+     2) no child processes (only the shell waiting)
+   Fulfilling any condition → HEALTH_ABNORMAL, free cores & CPU binding.
+   Otherwise → HEALTH_NORMAL (skip on future checks).
+   Does NOT freeze cgroup — ts -k must keep working as usual. */
+void s_check_running_health(void) {
+    size_t n = vec_size(&active_jobs);
+    for (size_t i = 0; i < n; i++) {
+        struct Job *p = (struct Job *)vec_get(&active_jobs, i);
+
+        /* Only check RUNNING, UNCHECKED jobs with a real PID */
+        if (p->state != RUNNING) continue;
+        if (p->health_state != 0) continue;  /* 0 = HEALTH_UNCHECKED */
+        if (p->pid <= 0) continue;
+
+        /* Must have been running for at least 5 minutes (300 seconds) */
+        time_t elapsed = get_monotonic_sec() - p->info.start_time;
+        if (elapsed < 300) continue;
+
+        int abnormal = 0;
+
+        /* Conditions: output file empty AND no child processes — both required */
+        if (!abnormal) {
+            int output_empty = 0;
+            int no_children = 0;
+
+            if (p->output_filename != NULL && p->output_filename[0] != '\0') {
+                struct stat st;
+                if (stat(p->output_filename, &st) == 0 && S_ISREG(st.st_mode) && st.st_size == 0) {
+                    output_empty = 1;
+                }
+            }
+
+            if (count_child_pids(p->pid) == 0) {
+                no_children = 1;
+            }
+
+            if (output_empty && no_children) {
+                printf("Job[%d] health ABNORMAL: output empty + no child processes (pid=%d)\n",
+                       p->jobid, p->pid);
+                abnormal = 1;
+            }
+        }
+
+        if (abnormal) {
+            p->state = ABNORMAL;
+            /* Free slots so new jobs can dispatch */
+            if (p->num_allocated != 0) {
+                free_cores(p);
+            }
+#ifdef TS_CPU_BIND
+            /* Free CPU binding if allocated */
+            if (p->cpu_alloc) {
+                cpu_bind_free((struct CpuAlloc *)p->cpu_alloc);
+                p->cpu_alloc = NULL;
+            }
+            if (cpu_bind_defrag_enabled())
+                cpu_bind_defrag_start();
+#endif
+            printf("Job[%d] -> ABNORMAL (slots freed)\n", p->jobid);
+        } else {
+            p->health_state = 1;  /* HEALTH_NORMAL */
+            printf("Job[%d] -> HEALTH_NORMAL\n", p->jobid);
+        }
+    }
+}
+
 int s_update_slots_usage() {
     int timeout_flag = s_check_timeout();
 
@@ -318,6 +391,37 @@ static int is_descendant_pid(pid_t parent_pid, pid_t target_pid) {
     return 0;
 }
 
+/* Count direct child PIDs of a process by reading /proc/<pid>/children.
+   Returns the number of children, 0 if none or error. */
+static int count_child_pids(pid_t parent_pid) {
+    char path[256];
+    char buf[4096];
+    int fd;
+
+    snprintf(path, sizeof(path), "/proc/%d/task/%d/children", parent_pid, parent_pid);
+    fd = open(path, O_RDONLY);
+    if (fd == -1) {
+        return 0;
+    }
+
+    ssize_t n = read(fd, buf, sizeof(buf) - 1);
+    close(fd);
+
+    if (n <= 0) {
+        return 0;
+    }
+    buf[n] = '\0';
+
+    int count = 0;
+    char *saveptr;
+    char *token = strtok_r(buf, " ", &saveptr);
+    while (token != NULL) {
+        count++;
+        token = strtok_r(NULL, " ", &saveptr);
+    }
+    return count;
+}
+
 /* Find which running job a PID belongs to.
    If deep_search != 0, also checks child/grandchild processes.
    Returns jobid if found, -1 if not. */
@@ -524,6 +628,7 @@ char const *jstate2string(enum Jobstate s) {
     case WAIT:           jobstate = "wait    "; break;
     case DELINK:         jobstate = "delink  "; break;
     case LOCKED:         jobstate = "locked  "; break;
+    case ABNORMAL:       jobstate = "abnormal"; break;
     case PAUSE:          jobstate = "pause   "; break;
     default:             jobstate = "UNKNOWN ";
     }
@@ -1284,6 +1389,16 @@ int s_remove_job(int s, int *jobid, struct User *client) {
         return 0;
     }
 
+    if (p->state == ABNORMAL) {
+        if (*jobid == -1)
+            snprintf(buff, 255, "Last job is abnormal.\n");
+        else
+            snprintf(buff, 255, "Abnormal job [%i] PID: %d by `%s` (use ts -k to kill).\n",
+                     *jobid, p->pid, p->user->name);
+        s_send_removejob_nok(s, buff);
+        return 0;
+    }
+
     *jobid = p->jobid;
     delete_DB(p->jobid, "Jobs");
     p->state = FINISHED;

jobs.h

@@ -18,6 +18,7 @@ struct Msg;
 enum Jobstate {
   QUEUED,
   RUNNING,
+  ABNORMAL,
   PAUSE,
   FINISHED,
   SKIPPED,
@@ -74,6 +75,8 @@ struct Job {
   int num_allocated;
   int no_cpu_binding;
   int client_socket;
+  /* 0=HEALTH_UNCHECKED, 1=HEALTH_NORMAL, 2=HEALTH_ABNORMAL */
+  int health_state;
 #ifdef TS_CPU_BIND
   void *cpu_alloc;              /* struct CpuAlloc * 由 cpu_bind 管理 */
 #endif
@@ -127,6 +130,7 @@ void s_kill_all_jobs(int s, struct User *user);
 void s_count_running_jobs(int s, struct User *user);
 void s_check_holdon(void);
 
+void s_check_running_health(void);
 int s_check_running_pid(pid_t pid);
 void s_read_sqlite(void);
 int s_find_pid(pid_t target_pid, int deep_search);

server.c

@@ -413,6 +413,15 @@ static void server_loop(int ls) {
       }
     } // nconnections
 
+    { /* Periodic health check for RUNNING jobs — every ~10 seconds */
+        static int health_ticks = 0;
+        health_ticks++;
+        if (health_ticks >= 10) {
+            health_ticks = 0;
+            s_check_running_health();
+        }
+    }
+
     next_run_job();
     s_check_holdon();
   } // end of while (keep_loop)