feat(harness/soak-ab): restart-count signal + crash-loop alert

PR #8 makes unknown-head / reorg-gap FATAL in stream mode (exit 75, EX_TEMPFAIL) with restart as
the designed recovery. soak-b.service now: Restart=on-failure (covers 75) + an ExecStartPre that
appends a UTC timestamp to $SOAK_B_RESTART_LOG on every (re)start (ReadWritePaths widened to cover
it). ab-diff.mjs reads that log (restartStats, unit-tested) and surfaces restartCount / lastRestartAt
/ restartsLastHour in soak-ab-status.json, raising a crash-loop alert at >3 restarts/hour.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: dzhelezov

harness/soak-ab/ab-diff.mjs

@@ -71,6 +71,34 @@ export function compareBucketHashes(aBuckets, bBuckets) {
   return { ok: mismatches.length === 0, mismatches, onlyA, onlyB };
 }
 
+// Restart accounting from the systemd ExecStartPre log (one UTC-timestamp line per (re)start). In
+// stream mode a fatal unknown-head / reorg-gap exits 75 and systemd restarts — designed recovery, so
+// only the RATE matters: >3 restarts in the trailing hour is a crash-loop alert.
+export function restartStats(lines, nowMs = Date.now()) {
+  const stamps = [];
+  for (const line of lines) {
+    const m = line.trim().match(/^(\d{4}-\d\d-\d\dT[\d:.]+Z)/);
+    if (m) {
+      stamps.push(Date.parse(m[1]));
+    }
+  }
+  stamps.sort((x, y) => x - y);
+
+  const hourAgo = nowMs - 3_600_000;
+  const restartsLastHour = stamps.filter((t) => t >= hourAgo).length;
+  const lastRestartAt =
+    stamps.length > 0
+      ? new Date(stamps[stamps.length - 1]).toISOString()
+      : null;
+
+  return {
+    restartCount: stamps.length,
+    lastRestartAt,
+    restartsLastHour,
+    crashLoop: restartsLastHour > 3,
+  };
+}
+
 // ── psql adapters ──────────────────────────────────────────────────────────────────────────────
 
 const SEP = '\x1f'; // unit separator — never appears in the hex/numeric fields we select
@@ -316,10 +344,41 @@ async function main() {
       ? 'PASS'
       : 'PENDING';
 
+  // Soak B restart signal (from the systemd ExecStartPre log).
+  const restartLog =
+    process.env.RESTART_LOG ?? `${process.env.HOME ?? '.'}/soak-b-restarts.log`;
+  let restarts = {
+    restartCount: 0,
+    lastRestartAt: null,
+    restartsLastHour: 0,
+    crashLoop: false,
+  };
+  try {
+    restarts = restartStats(readFileSync(restartLog, 'utf8').split('\n'));
+  } catch {
+    // no restart log yet — Soak B not started, or first boot
+  }
+
+  const alerts = [];
+  if (restarts.crashLoop) {
+    alerts.push(
+      `crash-loop: ${restarts.restartsLastHour} restarts in the last hour (>3)`,
+    );
+  }
+  if (verdict === 'FAIL') {
+    alerts.push(
+      'finalized-diff: an unexpected finalized-overlap divergence (see diffClasses)',
+    );
+  }
+
   const status = {
     ts: new Date().toISOString(),
     chains: results.map((r) => r.chain),
     verdict,
+    restartCount: restarts.restartCount,
+    lastRestartAt: restarts.lastRestartAt,
+    restartsLastHour: restarts.restartsLastHour,
+    alerts,
     diffClasses: Object.fromEntries(results.map((r) => [r.chain, r.classes])),
     lagA: Object.fromEntries(results.map((r) => [r.chain, r.lagA ?? null])),
     lagB: Object.fromEntries(results.map((r) => [r.chain, r.lagB ?? null])),

harness/soak-ab/ab-diff.test.mjs

@@ -4,6 +4,7 @@ import {
   checkpointMonotonic,
   classifyTxDiff,
   compareBucketHashes,
+  restartStats,
 } from './ab-diff.mjs';
 
 test('classifyTxDiff: B missing A parent txs, all log-referenced → expected class (PASS)', () => {
@@ -67,3 +68,43 @@ test('compareBucketHashes: shared buckets must match; one-sided buckets are repo
   assert.equal(badRes.mismatches.length, 1);
   assert.equal(badRes.mismatches[0].bucket, '1');
 });
+
+test('restartStats: counts restarts, reports last, flags crash-loop only above 3/hour', () => {
+  const now = Date.parse('2026-07-03T12:00:00Z');
+  // 2 restarts within the hour, 1 old → not a crash-loop
+  const calm = restartStats(
+    [
+      '2026-07-03T09:00:00Z restart',
+      '2026-07-03T11:30:00Z restart',
+      '2026-07-03T11:55:00Z restart',
+      '', // trailing blank line from the log split
+    ],
+    now,
+  );
+  assert.equal(calm.restartCount, 3);
+  assert.equal(calm.lastRestartAt, '2026-07-03T11:55:00.000Z');
+  assert.equal(calm.restartsLastHour, 2);
+  assert.equal(calm.crashLoop, false);
+
+  // 4 restarts within the hour → crash-loop
+  const loop = restartStats(
+    [
+      '2026-07-03T11:10:00Z restart',
+      '2026-07-03T11:25:00Z restart',
+      '2026-07-03T11:40:00Z restart',
+      '2026-07-03T11:59:00Z restart',
+    ],
+    now,
+  );
+  assert.equal(loop.restartsLastHour, 4);
+  assert.equal(loop.crashLoop, true);
+
+  // empty / never-started
+  const none = restartStats([], now);
+  assert.deepEqual(none, {
+    restartCount: 0,
+    lastRestartAt: null,
+    restartsLastHour: 0,
+    crashLoop: false,
+  });
+});

harness/soak-ab/deploy-soak-b.sh

@@ -25,6 +25,8 @@ PGADMIN_URL="${PGADMIN_URL:-postgres:///postgres}"
 MEM_HIGH="${SOAK_B_MEM_HIGH:-6G}"
 MEM_MAX="${SOAK_B_MEM_MAX:-8G}"
 UNIT_DIR="${SYSTEMD_DIR:-/etc/systemd/system}"
+RESTART_LOG="${SOAK_B_RESTART_LOG:-$HOME/soak-b-restarts.log}"
+RESTART_LOG_DIR="$(dirname "$RESTART_LOG")"
 
 HERE="$(cd "$(dirname "$0")" && pwd)"
 
@@ -89,12 +91,15 @@ chmod 600 "$ENVFILE"
 echo "  wrote $ENVFILE (chmod 600)"
 
 # ── 4. render + install the unit; enable but DO NOT start ──
+touch "$RESTART_LOG" 2>/dev/null || true
 RENDER="$(mktemp)"
 sed -e "s#@@WORKDIR@@#${WORKDIR}#g" \
     -e "s#@@ENVFILE@@#${ENVFILE}#g" \
     -e "s#@@PORT@@#${PORT}#g" \
     -e "s#@@MEM_HIGH@@#${MEM_HIGH}#g" \
     -e "s#@@MEM_MAX@@#${MEM_MAX}#g" \
+    -e "s#@@RESTART_LOG@@#${RESTART_LOG}#g" \
+    -e "s#@@RESTART_LOG_DIR@@#${RESTART_LOG_DIR}#g" \
     "$HERE/soak-b.service" > "$RENDER"
 
 if [ -w "$UNIT_DIR" ] || [ "$(id -u)" = 0 ]; then
@@ -109,3 +114,4 @@ else
 fi
 rm -f "$RENDER"
 echo "▶ Soak B provisioned. Start manually; then run harness/soak-ab/ab-diff.mjs hourly."
+echo "   restart log: $RESTART_LOG (ab-diff.mjs reads restartCount/lastRestartAt; >3/h = crash-loop alert)"

harness/soak-ab/soak-b.service

@@ -26,12 +26,21 @@ Environment=CI=true
 # The soak indexes the SAME 3 chains as Soak A; the app's ponder.config.ts reads SOAK_CHAINS.
 Environment=SOAK_CHAINS=eth,base,arbitrum
 
+# Restart counting is a first-class soak signal. Each (re)start appends a UTC timestamp to the
+# restart log; ab-diff.mjs reads it into restartCount / lastRestartAt and alerts on >3 restarts/hour
+# (crash-loop). In PORTAL_REALTIME=stream mode unknown-head and reorg-gap conditions are FATAL by
+# design and exit 75 (EX_TEMPFAIL) — Restart=on-failure makes that the intended restart-recovery,
+# NOT a soak failure (only the crash-loop RATE is an alert).
+Environment=SOAK_B_RESTART_LOG=@@RESTART_LOG@@
+ExecStartPre=/bin/sh -c 'echo "$(date -u +%%Y-%%m-%%dT%%H:%%M:%%SZ) restart" >> "$SOAK_B_RESTART_LOG"'
+
 ExecStart=@@WORKDIR@@/node_modules/.bin/ponder start --schema soak_b --port @@PORT@@
 
 # RSS cap — MemoryHigh throttles, MemoryMax is the hard kill ceiling (the soak asserts it holds).
 MemoryHigh=@@MEM_HIGH@@
 MemoryMax=@@MEM_MAX@@
 
+# on-failure covers exit 75 (the designed stream-mode restart) and any crash/OOM/signal.
 Restart=on-failure
 RestartSec=10
 # 2 planned SIGKILL+restart drills mid-soak are exercised by the operator via `systemctl kill -s KILL`.
@@ -40,7 +49,7 @@ TimeoutStopSec=30
 
 NoNewPrivileges=true
 ProtectSystem=strict
-ReadWritePaths=@@WORKDIR@@
+ReadWritePaths=@@WORKDIR@@ @@RESTART_LOG_DIR@@
 
 [Install]
 WantedBy=multi-user.target

harness/validate/README.md

@@ -125,7 +125,14 @@ memory. It asserts: **logs** strict row-set + field identity (PRIMARY, must be 0
 identity (total_difficulty excluded); **transactions** are EXACTLY the expected class — B may be
 missing parent txs for realtime-ingested spans, each referenced by an A-side log; anything else FAILS;
 per-1000-block checkpoint hashes match; `_ponder_checkpoint` is monotonic. It writes
-`soak-ab-status.json` for the hourly monitor.
+`soak-ab-status.json` (`verdict`, `restartCount`, `lastRestartAt`, `restartsLastHour`, `alerts`,
+`diffClasses`, `lagA`/`lagB`, `counters`) for the hourly monitor.
+
+**Restart signal** — in `PORTAL_REALTIME=stream` mode, unknown-head / reorg-gap are FATAL by design
+and exit **75 (EX_TEMPFAIL)**; `Restart=on-failure` makes that a designed restart-recovery, not a
+soak failure. The unit's `ExecStartPre` appends a UTC timestamp to `$SOAK_B_RESTART_LOG` on every
+(re)start; `ab-diff.mjs` reads it into `restartCount`/`lastRestartAt` and raises a **crash-loop**
+alert when restarts exceed 3/hour. Point `ab-diff.mjs` at the same log via `RESTART_LOG=…`.
 
 **Guardrails** (`deploy-soak-b.sh`): DB is `euler_rt_b` and nothing else; port is never `:9547`; the
 `euler` prod DB is never touched; secrets are copied into a `chmod 600` env file, never printed/committed.