Fix test flakiness

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
             launch.sh dashboard.sh harvest.sh \
             costs.sh progress.sh \
             lib/*.sh lib/drivers/*.sh \
-            tests/test_*.sh tests/test.sh
+            tests/*.sh
 
   unit-tests:
     strategy:

tests/_test_env.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Keep test behavior independent of host Git settings.
+export GIT_CONFIG_GLOBAL=/dev/null
+export GIT_CONFIG_SYSTEM=/dev/null
+export GIT_CONFIG_COUNT=0
+unset GIT_CONFIG GIT_CONFIG_PARAMETERS

tests/manual_interactive_e2e.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -euo pipefail
 
+# shellcheck source=_test_env.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"
+
 # Manual interactive E2E fixture generator.
 #
 # This is intentionally not named test_*.sh, so tests/test.sh --unit

tests/runtime_signal_trap.sh

@@ -2,6 +2,9 @@
 # shellcheck disable=SC2034
 set -uo pipefail
 
+# shellcheck source=_test_env.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"
+
 # tests/runtime_signal_trap.sh
 #
 # Drive the harness's emergency-push paths under real Docker.
@@ -180,6 +183,22 @@ wait_for_session() {
     return 1
 }
 
+wait_for_test_commit() {
+    local name="$1" tag="$2" timeout="${3:-30}"
+    local i subject
+    for i in $(seq 1 "$timeout"); do
+        subject=$(docker exec "$name" \
+            git -C /workspace log -1 --format='%s' 2>/dev/null || true)
+        if [ "$subject" = "test commit [${tag}]" ]; then
+            return 0
+        fi
+        sleep 1
+    done
+    echo "    timed out waiting for test commit [${tag}] in ${name}"
+    docker logs "$name" 2>&1 | tail -15 | sed 's/^/      /'
+    return 1
+}
+
 await_exit() {
     local name="$1" timeout="${2:-90}"
     local i status
@@ -205,8 +224,7 @@ count_landed() {
 echo
 echo "=== Item 3: SIGTERM mid-session ==="
 start_container "${NAMES[0]}" -e TC_TAG=sigterm-tag
-wait_for_session "${NAMES[0]}" || exit 1
-sleep 2
+wait_for_test_commit "${NAMES[0]}" sigterm-tag || exit 1
 echo "  sending docker stop -t 60..."
 docker stop -t 60 "${NAMES[0]}" >/dev/null
 EXIT_3=$(await_exit "${NAMES[0]}")
@@ -221,8 +239,7 @@ echo
 
 echo "=== Item 4: SIGINT mid-session ==="
 start_container "${NAMES[1]}" -e TC_TAG=sigint-tag
-wait_for_session "${NAMES[1]}" || exit 1
-sleep 2
+wait_for_test_commit "${NAMES[1]}" sigint-tag || exit 1
 echo "  sending docker kill --signal=SIGINT..."
 docker kill --signal=SIGINT "${NAMES[1]}" >/dev/null
 EXIT_4=$(await_exit "${NAMES[1]}")

tests/test.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -euo pipefail
 
+# shellcheck source=_test_env.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"
+
 # Smoke test: launch agents with a counting prompt, verify results.
 # Each agent N writes test-results/agent-N.txt with N*100..N*100+99.
 #

tests/test_activity_filter.sh

@@ -2,6 +2,9 @@
 # shellcheck disable=SC2034
 set -euo pipefail
 
+# shellcheck source=_test_env.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"
+
 # Unit tests for lib/activity-filter.sh.
 # No Docker or API key required.
 

tests/test_ci_workflows.sh

@@ -1,11 +1,16 @@
 #!/bin/bash
 set -euo pipefail
 
+# shellcheck source=_test_env.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"
+
 # Unit tests for GitHub Actions workflow shape.
 
 PASS=0
 FAIL=0
-REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+TESTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$TESTS_DIR/.." && pwd)"
+CI_YML="$REPO_ROOT/.github/workflows/ci.yml"
 INTEGRATION_YML="$REPO_ROOT/.github/workflows/integration.yml"
 
 assert_eq() {
@@ -32,6 +37,23 @@ assert_eq "workflow has no full-matrix job" "0" \
 assert_eq "workflow does not run --all" "0" \
     "$(grep -cF './tests/test.sh --all' "$INTEGRATION_YML" || true)"
 
+echo ""
+echo "=== 2. Shared test environment ==="
+
+missing_env=""
+for test_file in "$TESTS_DIR"/*.sh; do
+    [ "$test_file" = "$TESTS_DIR/_test_env.sh" ] && continue
+    if ! grep -qF \
+            'source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"' \
+            "$test_file"; then
+        missing_env="${missing_env} $(basename "$test_file")"
+    fi
+done
+
+assert_eq "all test scripts source the shared environment" "" "$missing_env"
+assert_eq "CI checks every test script" "1" \
+    "$(grep -cF 'tests/*.sh' "$CI_YML")"
+
 echo ""
 echo "==============================="
 echo "  ${PASS} passed, ${FAIL} failed"

tests/test_config.sh

@@ -2,6 +2,9 @@
 # shellcheck disable=SC2034
 set -euo pipefail
 
+# shellcheck source=_test_env.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"
+
 # Unit tests for swarm.json config parsing.
 # No Docker or API key required -- validates jq expressions only.
 

tests/test_costs.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -euo pipefail
 
+# shellcheck source=_test_env.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"
+
 # Unit tests for costs.sh aggregation and formatting logic.
 # No Docker or API key required.
 

tests/test_dashboard.sh

@@ -2,6 +2,9 @@
 # shellcheck disable=SC2034,SC2016
 set -euo pipefail
 
+# shellcheck source=_test_env.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"
+
 # Unit tests for dashboard.sh helper functions:
 #   format_model, truncate_str, and column layout (with optional Tag).
 # No Docker or API key required.

tests/test_drivers.sh

@@ -2,6 +2,9 @@
 # shellcheck disable=SC2034
 set -euo pipefail
 
+# shellcheck source=_test_env.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"
+
 # Unit tests for the driver role interface.
 # No Docker or API key required.
 
@@ -1323,14 +1326,15 @@ for _drv in claude-code codex-cli gemini-cli; do
         "$(grep -cE 'stdbuf -oL tee "\$logfile"' "$DRIVERS_DIR/$_drv.sh" || true)"
 done
 
-# --- Behavioral tests below require `setsid` and `stdbuf` on the
-# host (both GNU utilities).  Stock macOS ships neither, so skip
-# these assertions when they're missing -- the structural pins
-# above are what prevent the bug from silently regressing; the
-# behavioural tests just provide end-to-end confidence on Linux CI.
+# --- Behavioral tests below require `setsid`, `stdbuf`, and `timeout`
+# on the host (all GNU utilities). Stock macOS ships none of them, so
+# skip these assertions when they're missing. The structural pins above
+# prevent silent regressions; the behavioural tests provide end-to-end
+# confidence on Linux CI.
 if ! command -v setsid >/dev/null 2>&1 \
-        || ! command -v stdbuf >/dev/null 2>&1; then
-    echo "  SKIP: reaper behavioural tests (setsid/stdbuf unavailable)"
+        || ! command -v stdbuf >/dev/null 2>&1 \
+        || ! command -v timeout >/dev/null 2>&1; then
+    echo "  SKIP: reaper behavioural tests (setsid/stdbuf/timeout unavailable)"
 else
     # --- Behavioral: fake CLI that forks a surviving child and exits.
     # Without reaping, the child's stdout FD would keep the pipeline
@@ -1346,23 +1350,25 @@ CLI
     chmod +x "$TMPDIR/reap_cli.sh"
 
     # Exit-0 path: pipeline drains, stdout tee'd, exit code preserved.
-    _reap_start=$(date +%s)
-    _run_reaped "$TMPDIR/reap_zero.log" "$TMPDIR/reap_cli.sh" 0 "hello from CLI"
-    _reap_ec=$?
-    _reap_elapsed=$(($(date +%s) - _reap_start))
+    # The timeout bounds a process-group regression without making
+    # normal elapsed time part of the assertion.
+    _reap_ec=0
+    timeout --kill-after=5 30 bash -c \
+        'source "$1"; shift; _run_reaped "$@"' \
+        _ "$DRIVERS_DIR/_common.sh" "$TMPDIR/reap_zero.log" \
+        "$TMPDIR/reap_cli.sh" 0 "hello from CLI" || _reap_ec=$?
 
     assert_eq "reaped run: exit code preserved (0)" "0" "$_reap_ec"
     assert_eq "reaped run: stdout captured in logfile" "hello from CLI" \
         "$(cat "$TMPDIR/reap_zero.log")"
-    # Generous bound for slow CI; actual drain is ~10-50ms.
-    assert_eq "reaped run: drains despite surviving child" "true" \
-        "$([ "$_reap_elapsed" -lt 5 ] && echo true || echo false)"
 
     # Non-zero exit path: exit code is likewise propagated.
     # `|| _reap_ec=$?` keeps set -e from aborting the test script.
     _reap_ec=0
-    _run_reaped "$TMPDIR/reap_nz.log" "$TMPDIR/reap_cli.sh" 42 "boom" \
-        || _reap_ec=$?
+    timeout --kill-after=5 30 bash -c \
+        'source "$1"; shift; _run_reaped "$@"' \
+        _ "$DRIVERS_DIR/_common.sh" "$TMPDIR/reap_nz.log" \
+        "$TMPDIR/reap_cli.sh" 42 "boom" || _reap_ec=$?
     assert_eq "reaped run: non-zero exit preserved (42)" "42" "$_reap_ec"
 
     # --- Defensive cleanup: if the kill-group regressed, sweep any
@@ -1460,13 +1466,12 @@ assert_eq "_run_reaped reaps watchdog after kill" "1" \
     "$(grep -cE 'wait "\$_wd_pid" 2>/dev/null' "$COMMON_FILE")"
 
 # --- §40b. Behavioral: a deadlocked CLI is force-killed by the
-# watchdog within the configured timeout window.
+# watchdog.
 #
-# Skips on hosts without setsid/stdbuf -- the structural pins
-# above are the main regression guard; the behavioural test just
-# provides live coverage on Linux CI.  We drive the watchdog at
-# SWARM_ACTIVITY_POLL=1, SWARM_WATCHDOG_GRACE=1 so the full
-# escalation completes in ~3s and the test adds ~5s to the suite.
+# Skips on hosts without setsid/stdbuf/timeout -- the structural
+# pins above are the main regression guard; the behavioural test
+# provides live coverage on Linux CI. We use the shortest supported
+# polling and grace intervals to keep the suite quick.
 
 # Small helper to count matches without tripping assert_eq with
 # grep's noisy `0 + exit 1 + || echo 0` behaviour (which was
@@ -1478,8 +1483,9 @@ _wd_count() {
 }
 
 if ! command -v setsid >/dev/null 2>&1 \
-        || ! command -v stdbuf >/dev/null 2>&1; then
-    echo "  SKIP: watchdog behavioural test (setsid/stdbuf unavailable)"
+        || ! command -v stdbuf >/dev/null 2>&1 \
+        || ! command -v timeout >/dev/null 2>&1; then
+    echo "  SKIP: watchdog behavioural test (setsid/stdbuf/timeout unavailable)"
 else
     # Synthetic CLI: emits one line, then sleeps indefinitely.
     # The log mtime will freeze after the initial write and the
@@ -1497,17 +1503,18 @@ CLI
     #   "alive" write advances _last_mtime), second poll at t=2s
     #   (mtime stale for ≥2s → SIGTERM → the bash default handler
     #   exits → the grace loop returns early).  End-to-end ~2-3s.
-    _wd_start=$(date +%s)
+    # The outer timeout bounds a regression without making normal
+    # elapsed time an assertion.
+    _wd_ec=0
     SWARM_ACTIVITY_TIMEOUT=2 \
     SWARM_ACTIVITY_POLL=1 \
     SWARM_WATCHDOG_GRACE=1 \
-        _run_reaped "$TMPDIR/wd_run.log" "$TMPDIR/wd_cli.sh" \
-        >/dev/null 2>&1 || true
-    _wd_elapsed=$(($(date +%s) - _wd_start))
+        timeout --kill-after=5 30 bash -c \
+        'source "$1"; shift; _run_reaped "$@"' \
+        _ "$COMMON_FILE" "$TMPDIR/wd_run.log" "$TMPDIR/wd_cli.sh" \
+        >/dev/null 2>&1 || _wd_ec=$?
 
-    assert_eq "watchdog terminates deadlocked CLI within budget" \
-        "true" \
-        "$([ "$_wd_elapsed" -lt 10 ] && echo true || echo false)"
+    assert_eq "watchdog propagates SIGTERM exit (143)" "143" "$_wd_ec"
 
     # The watchdog's diagnostic line lands on stderr, which
     # _run_reaped redirects to <logfile>.err.  Its presence is
@@ -1523,9 +1530,8 @@ CLI
         "$(head -1 "$TMPDIR/wd_run.log" 2>/dev/null)"
 
     # --- Watchdog must stay dormant when the knob is unset or
-    # non-numeric.  Run a short well-behaved CLI and verify no
-    # watchdog diagnostics leak and elapsed stays near the CLI's
-    # own runtime rather than inflating to the grace window.
+    # non-numeric. Run a short well-behaved CLI and verify the
+    # watchdog does not alter its observable behavior.
     cat > "$TMPDIR/wd_quick.sh" <<'CLI'
 #!/bin/bash
 echo "done"
@@ -1534,19 +1540,21 @@ CLI
     chmod +x "$TMPDIR/wd_quick.sh"
 
     unset SWARM_ACTIVITY_TIMEOUT SWARM_ACTIVITY_POLL SWARM_WATCHDOG_GRACE
-    _wd_start=$(date +%s)
+    _wd_ec=0
     _run_reaped "$TMPDIR/wd_quick.log" "$TMPDIR/wd_quick.sh" \
-        >/dev/null 2>&1 || true
-    _wd_elapsed=$(($(date +%s) - _wd_start))
-    assert_eq "unset timeout: watchdog stays dormant" "true" \
-        "$([ "$_wd_elapsed" -lt 3 ] && echo true || echo false)"
+        >/dev/null 2>&1 || _wd_ec=$?
+    assert_eq "unset timeout: exit code preserved (0)" "0" "$_wd_ec"
+    assert_eq "unset timeout: stdout captured" "done" \
+        "$(cat "$TMPDIR/wd_quick.log")"
     assert_eq "unset timeout: no watchdog diagnostics" "0" \
         "$(_wd_count 'swarm watchdog' "$TMPDIR/wd_quick.log.err")"
 
     # Non-integer value must degrade to disabled, not crash.
+    _wd_ec=0
     SWARM_ACTIVITY_TIMEOUT="nonsense" \
         _run_reaped "$TMPDIR/wd_bad.log" "$TMPDIR/wd_quick.sh" \
-        >/dev/null 2>&1 || true
+        >/dev/null 2>&1 || _wd_ec=$?
+    assert_eq "non-integer timeout: run succeeds" "0" "$_wd_ec"
     assert_eq "non-integer timeout: no watchdog diagnostics" "0" \
         "$(_wd_count 'swarm watchdog' "$TMPDIR/wd_bad.log.err")"
 

tests/test_format.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -euo pipefail
 
+# shellcheck source=_test_env.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh"
+
 # Unit tests for formatting functions used by dashboard.sh and costs.sh.
 # No Docker or API key required.