fix: timeout/PAUSE job lifecycle — CPU binding, cgroup cleanup, and block -r on PAUSE

Four interrelated fixes for PAUSE (timeout) job handling:

1. Block 'ts -r' on PAUSE jobs (like RUNNING)
   - s_remove_job() now rejects PAUSE state the same as RUNNING
   - Avoids cgroups_clean_job() on live processes

2. Free CPU binding on PAUSE (safe_pause_job)
   - cpu_bind_free() when pausing, so allocator can reassign CPUs
   - Re-allocate on PAUSE→RUNNING resume in config_running()

3. Fix double CPU allocation for QUEUED→RUNNING
   - Guard re-allocation in config_running() with p->pid != 0
   - QUEUED jobs get allocated later in s_process_runjob_ok()

4. Fix cgroup cleanup order for v1
   - Swap cleanup_freeze after cleanup_cpu in cgroups_clean_job()
   - CPU cleanup kills processes first, freeze rmdir succeeds immediately

Also: cgroups v1/v2 init validation, reconnect improvements,
start-service flag, non-fatal cpu.cfs_quota_us warnings.

Author: kylincaster

.claude/remote-trigger-audit.jsonl

@@ -0,0 +1,8 @@
+{"auditId":"a8e5dadf-b5d2-4cc2-90a2-08692b83bf6e","ok":false,"error":"Not authenticated with a claude.ai account. Run /login and try again.","createdAt":1780452897448}
+{"auditId":"13526caa-c23f-4d07-95d4-daebf154d6bc","ok":false,"error":"Not authenticated with a claude.ai account. Run /login and try again.","createdAt":1780452903533}
+{"auditId":"cae3aff9-3566-42ab-be48-5d4c4306e040","ok":false,"error":"Not authenticated with a claude.ai account. Run /login and try again.","createdAt":1780452925545}
+{"auditId":"77586bc1-5c0e-484d-87f5-1182ba975879","ok":false,"error":"Not authenticated with a claude.ai account. Run /login and try again.","createdAt":1780452943937}
+{"auditId":"cddb29a3-41e0-440d-aedf-26f9a4e9f407","ok":false,"error":"Not authenticated with a claude.ai account. Run /login and try again.","createdAt":1780452954295}
+{"auditId":"331a804d-cb4c-4304-b637-79a09ce19494","ok":false,"error":"Not authenticated with a claude.ai account. Run /login and try again.","createdAt":1780452971227}
+{"auditId":"deb5a7b1-db62-4185-a5c5-00f7a895fbed","ok":false,"error":"Not authenticated with a claude.ai account. Run /login and try again.","createdAt":1780452976216}
+{"auditId":"f21b7b91-e3f7-4813-9f73-a63aa070dddd","ok":false,"error":"Not authenticated with a claude.ai account. Run /login and try again.","createdAt":1780453012872}

.codegraph/.gitignore

@@ -0,0 +1,16 @@
+# CodeGraph data files
+# These are local to each machine and should not be committed
+
+# Database
+*.db
+*.db-wal
+*.db-shm
+
+# Cache
+cache/
+
+# Logs
+*.log
+
+# Hook markers
+.dirty

cgroups.c

@@ -93,13 +93,14 @@ static inline void cg_group_name(int jobid, pid_t pid, char *buf, size_t size) {
 void cgroups_v2_init(void) {
     FILE *fp = fopen(CGROUP_CONTROLLERS, "r");
     if (!fp) {
-        fprintf(stderr, "Warning: cannot open %s\n", CGROUP_CONTROLLERS);
-        return;
+        error("Cannot open %s — cgroup v2 not available", CGROUP_CONTROLLERS);
     }
 
     char line[256];
     int has_cpu = 0;
+#ifdef TS_CPU_BIND
     int has_cpuset = 0;
+#endif
     if (fgets(line, sizeof(line), fp)) {
         if (strstr(line, "cpu")) {
             has_cpu = 1;
@@ -112,19 +113,65 @@ void cgroups_v2_init(void) {
     }
     fclose(fp);
 
-    if (has_cpu) {
-        cg_write(CGROUP_SUBTREE_CONTROL, "+cpu");
+    /* cpu controller is required */
+    if (!has_cpu) {
+        error("cgroup v2: 'cpu' controller not found in %s. "
+              "The kernel or container does not support it.",
+              CGROUP_CONTROLLERS);
+    }
+
+    /* Write +cpu and verify */
+    if (cg_write(CGROUP_SUBTREE_CONTROL, "+cpu") != 0) {
+        error("cgroup v2: failed to enable 'cpu' controller — "
+              "check %s permissions", CGROUP_SUBTREE_CONTROL);
     }
+    {
+        char verify[256] = "";
+        FILE *vfp = fopen(CGROUP_SUBTREE_CONTROL, "r");
+        if (vfp) {
+            if (fgets(verify, sizeof(verify), vfp))
+                verify[strcspn(verify, "\n")] = '\0';
+            fclose(vfp);
+        }
+        if (!strstr(verify, "cpu")) {
+            error("cgroup v2: 'cpu' controller not propagated to %s "
+                  "(got \"%s\"). Check parent cgroup subtree_control.",
+                  CGROUP_SUBTREE_CONTROL, verify);
+        }
+    }
+
 #ifdef TS_CPU_BIND
-    if (has_cpuset) {
-        cg_write(CGROUP_SUBTREE_CONTROL, "+cpuset");
+    if (!has_cpuset) {
+        error("cgroup v2: 'cpuset' controller not found in %s. "
+              "CPU binding requires it.",
+              CGROUP_CONTROLLERS);
+    }
+
+    if (cg_write(CGROUP_SUBTREE_CONTROL, "+cpuset") != 0) {
+        error("cgroup v2: failed to enable 'cpuset' controller — "
+              "check %s permissions", CGROUP_SUBTREE_CONTROL);
+    }
+    {
+        char verify[256] = "";
+        FILE *vfp = fopen(CGROUP_SUBTREE_CONTROL, "r");
+        if (vfp) {
+            if (fgets(verify, sizeof(verify), vfp))
+                verify[strcspn(verify, "\n")] = '\0';
+            fclose(vfp);
+        }
+        if (!strstr(verify, "cpuset")) {
+            error("cgroup v2: 'cpuset' controller not propagated to %s "
+                  "(got \"%s\"). Check parent cgroup subtree_control.",
+                  CGROUP_SUBTREE_CONTROL, verify);
+        }
     }
 #endif
 }
 
 static int cgroups_v2_cpu(int jobid, pid_t pid, int cpus) {
     char path[512];
     char group[64];
+    int err = 0;
 
     cg_group_name(jobid, pid, group, sizeof(group));
     snprintf(path, sizeof(path), CGROUP_DIR "/%s", group);
@@ -139,15 +186,17 @@ static int cgroups_v2_cpu(int jobid, pid_t pid, int cpus) {
 
     snprintf(path, sizeof(path), CGROUP_DIR "/%s/" CGROUP_CPU_MAX, group);
     if (cg_write(path, "%ld %ld", quota, period) != 0) {
-        return -1;
+        fprintf(stderr, "Warning: cannot set cpu.max for job %d — "
+                        "running without CPU limit\n", jobid);
+        err = 1;
     }
 
     snprintf(path, sizeof(path), CGROUP_DIR "/%s/" CGROUP_PROCS, group);
     if (cg_write(path, "%d", pid) != 0) {
         return -1;
     }
 
-    return 0;
+    return err ? -1 : 0;
 }
 
 static int cgroups_v2_wait_frozen(int jobid, pid_t pid) {
@@ -366,9 +415,47 @@ static int cgroups_v2_freeze_ok(int jobid, pid_t pid) {
 
 /* ---- v1 internal ---- */
 
+void cgroups_v1_init(void) {
+    struct stat st;
+
+    /* cpu controller: must exist and be writable */
+    if (stat(CGROUP_V1_CPU_DIR, &st) != 0 || !S_ISDIR(st.st_mode)) {
+        error("cgroup v1: '%s' not found — kernel or container "
+              "does not support cpu controller", CGROUP_V1_CPU_DIR);
+    }
+    if (access(CGROUP_V1_CPU_DIR, W_OK) != 0) {
+        error("cgroup v1: '%s' not writable — run as root",
+              CGROUP_V1_CPU_DIR);
+    }
+
+    /* freezer controller: must exist and be writable */
+    if (stat(CGROUP_V1_FREEZE_DIR, &st) != 0 || !S_ISDIR(st.st_mode)) {
+        error("cgroup v1: '%s' not found — kernel or container "
+              "does not support freezer controller", CGROUP_V1_FREEZE_DIR);
+    }
+    if (access(CGROUP_V1_FREEZE_DIR, W_OK) != 0) {
+        error("cgroup v1: '%s' not writable — run as root",
+              CGROUP_V1_FREEZE_DIR);
+    }
+
+#ifdef TS_CPU_BIND
+    /* cpuset controller: required for CPU binding */
+    if (stat("/sys/fs/cgroup/cpuset", &st) != 0 || !S_ISDIR(st.st_mode)) {
+        error("cgroup v1: '/sys/fs/cgroup/cpuset' not found — "
+              "CPU binding requires cpuset controller");
+    }
+    if (access("/sys/fs/cgroup/cpuset", W_OK) != 0) {
+        error("cgroup v1: '/sys/fs/cgroup/cpuset' not writable — "
+              "run as root");
+    }
+#endif
+}
+
 static int cgroups_v1_cpu(int jobid, pid_t pid, int cpus) {
     char buf[256];
     char group[64];
+    int err = 0;
+
     cg_group_name(jobid, pid, group, sizeof(group));
     snprintf(buf, sizeof(buf), CGROUP_V1_CPU_DIR "/%s", group);
     printf("Create cgroups folder at %s\n", buf);
@@ -383,19 +470,23 @@ static int cgroups_v1_cpu(int jobid, pid_t pid, int cpus) {
 
     snprintf(path, sizeof(path), "%s/cpu.cfs_period_us", buf);
     if (cg_write(path, "%ld", period) != 0) {
-        return -1;
+        fprintf(stderr, "Warning: cannot set cpu.cfs_period_us for job %d — "
+                        "running without CPU limit\n", jobid);
+        err = 1;
     }
 
     snprintf(path, sizeof(path), "%s/cpu.cfs_quota_us", buf);
     if (cg_write(path, "%ld", quota) != 0) {
-        return -1;
+        fprintf(stderr, "Warning: cannot set cpu.cfs_quota_us for job %d — "
+                        "running without CPU limit\n", jobid);
+        err = 1;
     }
 
     snprintf(path, sizeof(path), "%s/cgroup.procs", buf);
     if (cg_write(path, "%d", pid) != 0) {
         return -1;
     }
-    return 0;
+    return err ? -1 : 0;
 }
 
 static int cgroups_v1_freeze(int jobid, pid_t pid) {
@@ -721,8 +812,8 @@ void cgroups_clean_job(const struct Job *p) {
     cgroups_v2_cleanup(group);
 #else
     printf("clear cgroups: %s\n", group);
-    cgroups_v1_cleanup_freeze(group);
     cgroups_v1_cleanup_cpu(group);
+    cgroups_v1_cleanup_freeze(group);
 #ifdef TS_CPU_BIND
     cgroups_v1_cleanup_cpuset(group);
 #endif
@@ -757,6 +848,11 @@ void cgroups_set_cpuset(int jobid, pid_t pid, const void *valloc)
     cgroup_io_wait_if_busy();
 
     const struct CpuAlloc *alloc = (const struct CpuAlloc *)valloc;
+
+    /* 防御：分配失败时不写 cpuset（避免写空 cpuset.cpus → cgroup.procs ENOSPC） */
+    if (!alloc || alloc->error || alloc->count <= 0)
+        return;
+
     char *cpus_str = cpu_bind_format_cpus(alloc);
     char *mems_str = cpu_bind_format_mems(alloc);
     char path[512];

cgroups.h

@@ -22,6 +22,7 @@ int cgroups_freeze_ok(int jobid, pid_t pid);
 #ifdef CGROUP_V2
 void cgroups_v2_init(void);
 #endif
+void cgroups_v1_init(void);
 
 #ifdef TS_CPU_BIND
 void cgroups_set_cpuset(int jobid, pid_t pid, const void *alloc);

cpu_bind/HANDOVER.md

@@ -0,0 +1,101 @@
+# CPU 绑定分配器 — 交接文档
+
+## 文件结构
+
+```
+cpu_bind/
+├── topology.h          # 硬编码系统拓扑（2节点 x 4组 x 4核 = 32核）
+├── cpu_bind.h          # 公共 API 声明
+├── cpu_bind.c          # 完整实现
+├── test_cpu_bind.c     # 16 个测试用例
+└── Makefile            # 独立编译（依赖 ../vec.h ../vec.o）
+```
+
+## 公共 API
+
+```c
+void              cpu_bind_init(void);                      // 初始化
+int               cpu_bind_enabled(void);                   // 是否启用
+struct CpuAlloc  *cpu_bind_alloc_init(int jobid, int max_cpus); // 创建空 alloc
+void              cpu_bind_alloc(struct CpuAlloc *alloc, int N); // 分配 N 个 CPU
+void              cpu_bind_free(struct CpuAlloc *alloc);    // 释放
+int               cpu_bind_defrag(void);                    // 碎片整理
+char             *cpu_bind_format_cpus(const struct CpuAlloc *alloc); // "0,1,2,3"
+char             *cpu_bind_format_mems(const struct CpuAlloc *alloc); // "0" 或 "0,1"
+```
+
+## struct CpuAlloc
+
+```c
+struct CpuAlloc {
+    int primary_node;           // 主节点（-1=不限制）
+    int mem_nodes;              // 涉及节点位图
+    int count;                  // 已分配的 CPU 数
+    int slots;                  // 容量（os_cpus[] 最大元素数）
+    int jobid;
+    int error;                  // 0=成功 1=异常
+    int quality;                // 碎片 group 数（0=完美 1=1个碎片 2=2个碎片）
+    int os_cpus[];              // 柔性数组
+};
+```
+
+## 全局变量
+
+- `cpu_owner[MAX_OS_CPU]` — CPU→jobid 映射（0=空闲）
+- `cpu_to_group[MAX_OS_CPU]` — CPU→group 索引表（`cpu_bind_init` 中构建）
+- `cpu_bind_disabled` — `TS_NO_CPU_BIND` 环境变量禁用
+- `cpu_allocs` — `vec_t` 全局活跃 alloc 列表（供 defrag 使用）
+
+## 内部函数调用链
+
+```
+cpu_bind_alloc()
+  ├─ single_ok? → alloc_from_groups(N, node->group_ids, ...)
+  └─ 否则 → cross_node_merge()
+              └─ alloc_from_groups(N, collected_ids, ...)
+
+alloc_from_groups(N, group_ids, ...)
+  while remaining > 0:
+    ① best-fit 单组（free_count ≥ remaining，excess 最小）
+    ② 无 → 最大空闲组
+    └─ take_from_group(group, take, alloc)
+
+take_from_group()
+  → 在 group→os_cpus[] 中取 N 个空闲 CPU（不要求连续）
+  → 写 cpu_owner，更新 free_count
+  → free_count > 0 则 alloc->quality++
+  → 返回实际取的个数
+
+cpu_bind_post_alloc()
+  → qsort(os_cpus)
+  → 校验①：每个 CPU owner == jobid
+  → 校验②：owner 总数 == count
+  → 可 #define CPU_BIND_NO_POST_CHECK 关闭
+```
+
+## defrag 流程
+
+```
+cpu_bind_defrag()
+  ① 按 (quality ASC, slots DESC) 排序 cpu_allocs
+  ② 找 quality=0 的边界
+  ③ 清空 cpu_owner，重置 free_count（用 num_cores 直接赋值）
+  ④ 恢复 quality=0 allocs 的 cpu_owner
+  ⑤ 重分 quality>0 allocs（手动重置字段，re-alloc）
+```
+
+## 测试
+
+```bash
+cd cpu_bind && make test
+# 16/16 passed, 零警告
+```
+
+`test_cpu_bind.c` — 16 个测试覆盖：初始化、禁用、空/超容量分配、单核→整组→跨组→跨节点、耗尽+重分配、双释放安全、格式化、并发不重叠、defrag。
+
+## 待办 / 后续
+
+- [ ] defrag 的质量排序和重分逻辑需要实际验证（目前只有空 vec 测试）
+- [ ] `cpu_bind_free` 中的 vec 扫描删除是 O(n)，数据量大时可优化
+- [ ] `cpu_bind_post_alloc` 的校验目前硬编码，可通过编译开关关闭
+- [ ] 与 ts 主项目的集成（`cpu_bind.c` 需要接入 server 的 job 生命周期）

cpu_bind/Makefile

@@ -0,0 +1,20 @@
+# cpu_bind/Makefile — 独立 CPU 绑定模块编译 & 测试
+
+CC      = gcc
+CFLAGS  = -std=c11 -Wall -Wextra -g -D_DEFAULT_SOURCE -I..
+
+.PHONY: all test clean
+
+all: test_cpu_bind
+
+cpu_bind.o: cpu_bind.c cpu_bind.h topology.h ../vec.h
+	$(CC) $(CFLAGS) -c cpu_bind.c -o cpu_bind.o
+
+test_cpu_bind: test_cpu_bind.c cpu_bind.o cpu_bind.h topology.h ../vec.h
+	$(CC) $(CFLAGS) -o test_cpu_bind test_cpu_bind.c cpu_bind.o ../vec.o
+
+test: test_cpu_bind
+	./test_cpu_bind
+
+clean:
+	rm -f *.o test_cpu_bind