fix(scheduler): break non-converging SSP catch-up loop + row-level diagnostic

apps/benchmark/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/benchmark",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "description": "Performance benchmark CLI for sp00ky scheduler + SSP",
   "private": true,
   "type": "module",

apps/cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sp00ky-cli"
-version = "0.0.1-canary.113"
+version = "0.0.1-canary.114"
 edition = "2021"
 
 [[bin]]

apps/cli/npm/cli-darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/cli-darwin-arm64",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "description": "macOS Apple Silicon binary for @spooky-sync/cli",
   "license": "MIT",
   "os": [

apps/cli/npm/cli-darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/cli-darwin-x64",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "description": "macOS Intel binary for @spooky-sync/cli",
   "license": "MIT",
   "os": [

apps/cli/npm/cli-linux-arm64/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/cli-linux-arm64",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "description": "Linux ARM64 binary for @spooky-sync/cli",
   "license": "MIT",
   "os": [

apps/cli/npm/cli-linux-x64/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/cli-linux-x64",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "description": "Linux x64 binary for @spooky-sync/cli",
   "license": "MIT",
   "os": [

apps/cli/npm/cli-win32-x64/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/cli-win32-x64",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "description": "Windows x64 binary for @spooky-sync/cli",
   "license": "MIT",
   "os": [

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/cli",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "description": "Generate TypeScript/Dart types from SurrealDB schema files",
   "type": "module",
   "main": "./dist/syncgen.cjs",

apps/devtools-mcp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/devtools-mcp",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "description": "MCP server for Sp00ky Sync devtools",
   "license": "MIT",
   "type": "module",

apps/devtools/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/devtools",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "description": "Chrome DevTools extension for debugging and inspecting Sp00ky state",
   "private": true,
   "type": "module",

apps/landing-page/package.json

@@ -1,7 +1,7 @@
 {
   "name": "landing-page",
   "type": "module",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "scripts": {
     "dev": "astro dev",
     "dev:landingpage": "astro dev",

apps/scheduler/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scheduler"
-version = "0.0.1-canary.113"
+version = "0.0.1-canary.114"
 edition = "2021"
 
 [dependencies]

apps/scheduler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scheduler",
-  "version": "0.0.1-canary.113",
+  "version": "0.0.1-canary.114",
   "private": true,
   "scripts": {
     "build": "docker buildx build --target scheduler -t mono424/spooky-scheduler:dev -f ../../Dockerfile ../..",

apps/scheduler/src/main.rs

@@ -55,6 +55,8 @@ async fn main() -> Result<()> {
         config: std::sync::Arc::new(config.clone()),
         status: scheduler.status.clone(),
         event_buffer: scheduler.event_buffer.clone(),
+        seq_counter: std::sync::Arc::clone(&scheduler.seq_counter),
+        reclone_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
     };
     let ssp_router = scheduler::ssp_management::create_ssp_router(ssp_mgmt_state);
 

apps/scheduler/src/router.rs

@@ -27,6 +27,12 @@ pub struct SspPool {
     /// to re-bootstrap. The next heartbeat from these SSPs returns 409 so
     /// they tear down and re-register against the current frozen snapshot.
     forced_resync: HashSet<String>,
+    /// Consecutive catch-up verification failures per SSP, reset on any pass.
+    /// A plain re-bootstrap can't fix a *deterministic* scheduler-vs-circuit
+    /// hash gap (the SSP refetches the same diverging state every cycle), so
+    /// this counter lets the catch-up path escalate — re-clone the replica,
+    /// then admit anyway — instead of looping forever. See `poll_and_replay_ssp`.
+    catchup_failures: HashMap<String, u32>,
     strategy: LoadBalanceStrategy,
     round_robin_index: usize,
     max_buffer_size: usize,
@@ -41,12 +47,28 @@ impl SspPool {
             message_buffers: HashMap::new(),
             ssp_snapshot_seqs: HashMap::new(),
             forced_resync: HashSet::new(),
+            catchup_failures: HashMap::new(),
             strategy,
             round_robin_index: 0,
             max_buffer_size,
         }
     }
 
+    /// Record one more consecutive catch-up verification failure for this SSP
+    /// and return the new running count. Cleared by `reset_catchup_failures`
+    /// on any successful verification (or admit).
+    pub fn record_catchup_failure(&mut self, ssp_id: &str) -> u32 {
+        let entry = self.catchup_failures.entry(ssp_id.to_string()).or_insert(0);
+        *entry += 1;
+        *entry
+    }
+
+    /// Reset the consecutive catch-up failure count for this SSP (on a pass,
+    /// or once we admit it to broadcast to break the loop).
+    pub fn reset_catchup_failures(&mut self, ssp_id: &str) {
+        self.catchup_failures.remove(ssp_id);
+    }
+
     /// Flag an SSP for forced re-bootstrap on its next heartbeat. Used by
     /// the integrity-check path when the SSP's circuit hashes disagree with
     /// the scheduler's frozen snapshot — the SSP is told (via 409) to wipe
@@ -219,6 +241,7 @@ impl SspPool {
         self.message_buffers.remove(ssp_id);
         self.ssp_snapshot_seqs.remove(ssp_id);
         self.forced_resync.remove(ssp_id);
+        self.catchup_failures.remove(ssp_id);
         self.ssps.remove(ssp_id)
     }
 
@@ -232,6 +255,7 @@ impl SspPool {
         self.message_buffers.clear();
         self.ssp_snapshot_seqs.clear();
         self.forced_resync.clear();
+        self.catchup_failures.clear();
         self.round_robin_index = 0;
         count
     }
@@ -340,3 +364,36 @@ impl SspPool {
             .any(|s| matches!(s, SspState::Bootstrapping | SspState::Replaying))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn pool() -> SspPool {
+        SspPool::new(LoadBalanceStrategy::RoundRobin, 100)
+    }
+
+    #[test]
+    fn catchup_failures_count_up_then_reset() {
+        let mut p = pool();
+        assert_eq!(p.record_catchup_failure("ssp-0"), 1);
+        assert_eq!(p.record_catchup_failure("ssp-0"), 2);
+        assert_eq!(p.record_catchup_failure("ssp-0"), 3);
+        // Independent per SSP.
+        assert_eq!(p.record_catchup_failure("ssp-1"), 1);
+        // Reset clears only the named SSP and restarts its streak.
+        p.reset_catchup_failures("ssp-0");
+        assert_eq!(p.record_catchup_failure("ssp-0"), 1);
+        assert_eq!(p.record_catchup_failure("ssp-1"), 2);
+    }
+
+    #[test]
+    fn remove_clears_catchup_failures() {
+        let mut p = pool();
+        p.record_catchup_failure("ssp-0");
+        p.record_catchup_failure("ssp-0");
+        p.remove("ssp-0");
+        // A re-registered SSP of the same id starts a fresh streak.
+        assert_eq!(p.record_catchup_failure("ssp-0"), 1);
+    }
+}