test(chaos): fault-proxy campaign — byte-identical backfill under injected Portal faults (#149)

* harness: add fault proxy chaos campaign

* harness(fault-proxy): committee candor fixes — honest head-scenario + AIMD labels, biome gate

Addresses the blind 3-model REQUEST-CHANGES review of the P1 fault-proxy chaos
campaign. The core correctness property (digest-identity + interval-tiling, fails
closed) is preserved and unweakened; the fixes stop the head-* and AIMD *labels*
from overclaiming.

1. biome gate: `biome format --write` the 11 scenario JSONs (semantically identical)
   and rename the unused `init` param at chaos-aimd-trace.cjs:66 → `_init`, so
   `biome@2.5.2 check chaos-aimd-trace.cjs scenarios/faultproxy/` exits 0 (error-level).

2. AIMD sparse candor (finding #3): a single gate sample can no longer assert
   `recovered`. `sparse` (samples < 3) is reported honestly ("sparse — insufficient
   samples"), never as recovery. AIMD stays SECONDARY — sparse does not fail a
   scenario whose digest+intervals pass; only a genuine bounds violation (concurrency
   outside [min,max]) still fails the AIMD gate.

3. head recovery elif false-pass (finding #4): recovery now runs ONLY for head-freeze
   and ONLY when phase 1 did not complete (done != 1). A failed head_recovery_check
   propagates instead of retaining a stale pass. `byte-identical-recovery` is reserved
   for a genuine stall+resume; an unexpected completion labels `byte-identical-completion`.

4. head-freeze label candor (finding #5): honest note — "stalled without corruption;
   clean restart on the same DB (fault removed) completed byte-identical" — no
   partial-progress-resume implication.

5. head-regression-100k & head-flap vacuity (findings #1/#2, unanimous): these
   manipulate the head ABOVE the pinned endBlock so it never intersects the fixed
   range [20529207,20579207]; a fixed-range historical backfill never rolls back an
   already-finalized+indexed block, so they prove nothing the clean baseline did.
   Marked NOT-APPLICABLE (distinct verdict, not a head-tolerance PASS, not a fail) with
   a note pointing head-reorg/rollback coverage to the stream-mode soak. head-freeze
   (frozen head BELOW endBlock) remains the one meaningful in-range head scenario.
   Chosen the ORCHESTRATOR-approved N/A fallback over an in-range redesign because a
   genuine in-range regression/flap is not achievable non-contrived: regression is
   anchored to the drifting live head, and an in-range flap only duplicates head-freeze
   without exercising rollback (a stream-mode property).

Verified: biome check EXIT 0 on changed files; bash -n clean; head-freeze re-run on
the throwaway PG (54329) PASS byte-identical-recovery, digest == baseline, intervals
tile exactly, honest note emitted; N/A routing confirmed by code-trace; proxy.test.mjs
6/6.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* harness(fault-proxy): repo-relative RESULT_FILE/CX_SUMMARY paths (drop hardcoded absolute paths)

Env-overridable defaults under $ART (the gitignored artifacts dir) so the
harness is portable and carries no machine-specific absolute paths.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: dzhelezov

.gitignore

@@ -1,6 +1,7 @@
 node_modules/
 harness/results.*.json
 harness/bench/results.md
+harness/chaos/.faultproxy/
 harness/euler/vaults.*.json
 *.log
 .DS_Store

harness/chaos/chaos-aimd-trace.cjs

@@ -0,0 +1,134 @@
+const fs = require('node:fs');
+
+const traceFile =
+  process.env.CHAOS_AIMD_TRACE_FILE || process.env.AIMD_TRACE_FILE || '';
+let seq = 0;
+let inFlight = 0;
+let maxInFlight = 0;
+
+function append(event) {
+  if (!traceFile) return;
+  const record = {
+    ts: new Date().toISOString(),
+    pid: process.pid,
+    run: Number(process.env.CHAOS_RUN || 0),
+    attempt: Number(process.env.CHAOS_ATTEMPT || 0),
+    seq: ++seq,
+    ...event,
+  };
+  try {
+    fs.appendFileSync(traceFile, `${JSON.stringify(record)}\n`);
+  } catch {
+    // Evidence-only hook. Never perturb the app under test.
+  }
+}
+
+function parseGateLine(args) {
+  const line = args.map((a) => (typeof a === 'string' ? a : '')).join(' ');
+  const m = line.match(
+    /\[portalGate\]\s+concurrency_limit=(\d+)\s+active=(\d+)\s+buffered_rows=(\d+)/,
+  );
+  if (!m) return;
+  append({
+    event: 'aimd-gate-log',
+    source: 'portalGate-log',
+    concurrency: Number(m[1]),
+    limit: Number(m[1]),
+    active: Number(m[2]),
+    rows: Number(m[3]),
+  });
+}
+
+const originalLog = console.log;
+console.log = function aimdConsoleLog(...args) {
+  try {
+    parseGateLine(args);
+  } catch {
+    // Ignore trace parse errors.
+  }
+  return originalLog.apply(this, args);
+};
+
+function isFinalizedStream(input) {
+  const url =
+    typeof input === 'string'
+      ? input
+      : input instanceof URL
+        ? input.href
+        : input?.url;
+  return typeof url === 'string' && url.includes('/finalized-stream');
+}
+
+const originalFetch = globalThis.fetch;
+if (typeof originalFetch !== 'function') {
+  append({ event: 'aimd-env', error: 'global fetch is unavailable' });
+} else {
+  globalThis.fetch = async function aimdFetch(input, _init) {
+    const traced = isFinalizedStream(input);
+    if (!traced) {
+      return originalFetch.apply(this, arguments);
+    }
+
+    inFlight += 1;
+    maxInFlight = Math.max(maxInFlight, inFlight);
+    append({
+      event: 'aimd-fetch-start',
+      source: 'fetch-derived',
+      inFlight,
+      concurrency: inFlight,
+      maxInFlight,
+    });
+
+    try {
+      const response = await originalFetch.apply(this, arguments);
+      append({
+        event: 'aimd-fetch-end',
+        source: 'fetch-derived',
+        status: response?.status ?? null,
+        ok: response?.ok ?? null,
+        inFlight,
+        concurrency: inFlight,
+        maxInFlight,
+      });
+      return response;
+    } catch (error) {
+      append({
+        event: 'aimd-fetch-error',
+        source: 'fetch-derived',
+        error: error instanceof Error ? error.name : 'Error',
+        message: error instanceof Error ? error.message : String(error),
+        inFlight,
+        concurrency: inFlight,
+        maxInFlight,
+      });
+      throw error;
+    } finally {
+      inFlight = Math.max(0, inFlight - 1);
+      append({
+        event: 'aimd-fetch-release',
+        source: 'fetch-derived',
+        inFlight,
+        concurrency: inFlight,
+        maxInFlight,
+      });
+    }
+  };
+
+  append({
+    event: 'aimd-env',
+    source: 'hook',
+    min: Number(process.env.PORTAL_MIN_CONCURRENCY || 1),
+    max: Number(process.env.PORTAL_MAX_CONCURRENCY || 32),
+    start: Number(process.env.PORTAL_START_CONCURRENCY || 4),
+  });
+}
+
+process.on('exit', () => {
+  append({
+    event: 'aimd-exit',
+    source: 'hook',
+    inFlight,
+    concurrency: inFlight,
+    maxInFlight,
+  });
+});

harness/chaos/chaos-pg-driver.sh

@@ -1813,9 +1813,11 @@ backend_label_cmd () {
   echo "$label"
 }
 
-case "${1:-campaign}" in
-  selftest) selftest ;;
-  campaign) campaign ;;
-  backend-label) backend_label_cmd ;;
-  *) echo "usage: chaos-pg-driver.sh [campaign|selftest|backend-label]"; exit 2 ;;
-esac
+if [ -z "${CHAOS_LIB_ONLY:-}" ]; then
+  case "${1:-campaign}" in
+    selftest) selftest ;;
+    campaign) campaign ;;
+    backend-label) backend_label_cmd ;;
+    *) echo "usage: chaos-pg-driver.sh [campaign|selftest|backend-label]"; exit 2 ;;
+  esac
+fi