run-migrate.sh

#!/usr/bin/env bash
set -euo pipefail

#
# Agent-assisted version-migration benchmark / release gate.
#
# Takes the checked-in migrate-fixture/ app (a Conference Manager pinned to a RELEASED
# Brace version), hands a headless Claude Code agent the migration guide(s) for the next
# version, and asserts the app still passes its full test suite after the upgrade.
#
# A migration is a behaviour-preserving change, so the existing test suite is the grading
# oracle: every test green on the FROM version must stay green on the TO version. The agent
# fixes application code only; it never edits the tests.
#
# Usage:
#   ./run-migrate.sh                                  # from = fixture pin, to = brace pom version
#   ./run-migrate.sh --from 0.1.6 --to 0.1.7
#   ./run-migrate.sh --to 0.1.7 --guide-mode withheld # measure the guide's marginal value
#   ./run-migrate.sh --fix-model sonnet
#
# Env:
#   BRACE_REPO   path to the brace framework checkout (default: ~/code/brace)
#

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
RESULTS_DIR="$SCRIPT_DIR/results"
TESTS="$SCRIPT_DIR/tests"
FIXTURE_DIR="$SCRIPT_DIR/migrate-fixture"
BRACE_REPO="${BRACE_REPO:-$HOME/code/brace}"
MIGRATE_DIR="$BRACE_REPO/docs/migrations"

MAX_COMPILE_RETRIES=5
MAX_BOOT_TEST_RETRIES=4
PORT=8080

# The frozen oracle: the FULL suite must stay green across the upgrade.
ORACLE_TESTS="test_conference.py test_feature_availability.py test_feature_waitlist.py test_feature_ratings.py test_feature_multiday.py test_feature_notifications.py test_migrate_ops.py"

# ---------- parse arguments ----------

FROM=""
TO=""
GUIDE_MODE="provided"   # provided | withheld
FIX_MODEL="opus"

while [[ $# -gt 0 ]]; do
    case $1 in
        --from)       FROM="$2"; shift 2 ;;
        --to)         TO="$2"; shift 2 ;;
        --guide-mode) GUIDE_MODE="$2"; shift 2 ;;
        --fix-model)  FIX_MODEL="$2"; shift 2 ;;
        --port)       PORT="$2"; shift 2 ;;
        *)            echo "Unknown arg: $1"; exit 1 ;;
    esac
done

mkdir -p "$RESULTS_DIR"

# Default FROM = the version the fixture is pinned to.
if [ -z "$FROM" ]; then
    FROM=$(sed -n 's/.*<brace\.version>\(.*\)<\/brace\.version>.*/\1/p' "$FIXTURE_DIR/pom.xml" | head -1)
fi
# Default TO = the brace repo's current pom version.
if [ -z "$TO" ]; then
    TO=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' "$BRACE_REPO/pom.xml" | head -1)
fi

if [ -z "$FROM" ] || [ -z "$TO" ]; then
    echo "Could not determine FROM ($FROM) / TO ($TO) versions."; exit 1
fi

echo "=== Migration: Brace $FROM -> $TO  (guide-mode: $GUIDE_MODE, fix-model: $FIX_MODEL) ==="

# ---------- helpers (shared shape with run-chain.sh) ----------

kill_port() {
    local port=$1
    local pids
    pids=$(lsof -ti:"$port" 2>/dev/null || true)
    [ -z "$pids" ] && return 0
    echo "$pids" | xargs kill 2>/dev/null || true
    sleep 2
    pids=$(lsof -ti:"$port" 2>/dev/null || true)
    if [ -n "$pids" ]; then echo "$pids" | xargs kill -9 2>/dev/null || true; sleep 1; fi
    return 0
}

start_app() {
    local work_dir=$1 port=$2
    kill_port "$port"
    cd "$work_dir"
    SERVER_PORT="$port" java -jar target/conference-manager-1.0-SNAPSHOT.jar > "$work_dir/app.log" 2>&1 &
    APP_PID=$!
    for i in $(seq 1 30); do
        if curl -sf "http://localhost:$port/events" > /dev/null 2>&1; then return 0; fi
        if ! kill -0 "$APP_PID" 2>/dev/null; then return 1; fi
        sleep 1
    done
    return 1
}

stop_app() {
    if [ -n "${APP_PID:-}" ]; then
        kill "$APP_PID" 2>/dev/null || true
        for i in $(seq 1 5); do kill -0 "$APP_PID" 2>/dev/null || break; sleep 1; done
        kill -9 "$APP_PID" 2>/dev/null || true
        wait "$APP_PID" 2>/dev/null || true
        APP_PID=""
    fi
}

parse_test_results() {
    local xml_file=$1
    [ -f "$xml_file" ] || { echo "0 0 0"; return; }
    python3 -c "
import xml.etree.ElementTree as ET
root = ET.parse('$xml_file').getroot()
ts = root.find('testsuite') if root.tag != 'testsuite' else root
tests = int(ts.get('tests', 0)); failures = int(ts.get('failures', 0)); errors = int(ts.get('errors', 0))
print(f'{tests - failures - errors} {failures + errors} {tests}')
"
}

parse_failure_details() {
    local xml_file=$1
    python3 -c "
import xml.etree.ElementTree as ET
root = ET.parse('$xml_file').getroot()
ts = root.find('testsuite') if root.tag != 'testsuite' else root
for tc in ts.findall('.//testcase'):
    for tag in ('failure','error'):
        f = tc.find(tag)
        if f is not None:
            print(f'FAILED: {tc.get(\"classname\")}.{tc.get(\"name\")}')
            print('  ' + (f.get('message','')[:300]))
            print()
" 2>/dev/null || echo "Could not parse test results"
}

accumulate_stats() {
    local output=$1
    local cost
    cost=$(echo "$output" | jq -r '.total_cost_usd // 0')
    TOTAL_COST=$(echo "$TOTAL_COST + $cost" | bc)
    INVOCATIONS+=("$output")
}

# Run an agent turn (resume if we have a session), accumulate cost, return result text.
agent_turn() {
    local prompt=$1 budget=$2
    local resume_flag=""
    [ -n "${SESSION_ID:-}" ] && resume_flag="--resume $SESSION_ID"
    local out
    out=$(cd "$WORK_DIR" && claude -p "$prompt" \
        --output-format json --permission-mode bypassPermissions \
        --model "$FIX_MODEL" --max-budget-usd "$budget" $resume_flag 2>/dev/null)
    SESSION_ID=$(echo "$out" | jq -r '.session_id // empty')
    accumulate_stats "$out"
    echo "$out" | jq -r '.result // empty'
}

# ---------- ensure the TO toolchain exists (jar in ~/.m2 + launcher cache for `brace agents-md`) ----------

ensure_to_toolchain() {
    local v=$1
    local cache="$HOME/.brace/toolchains/$v"
    local need_build=false
    [ -d "$cache/lib" ] || need_build=true
    # Is the jar resolvable from ~/.m2?
    if [ ! -f "$HOME/.m2/repository/com/larvalabs/brace/$v/brace-$v.jar" ]; then need_build=true; fi

    if [ "$need_build" = true ]; then
        echo "  Building + installing brace $v from $BRACE_REPO ..."
        ( cd "$BRACE_REPO" && mvn -q -DskipTests install ) || { echo "  ERROR: brace $v build failed"; return 1; }
    fi

    # Stage the launcher toolchain cache so `brace agents-md` resolves an unreleased TO.
    if [ ! -d "$cache/lib" ]; then
        local zip="$BRACE_REPO/target/brace-$v.zip"
        if [ ! -f "$zip" ]; then echo "  ERROR: dist zip not found: $zip"; return 1; fi
        local tmp; tmp=$(mktemp -d)
        unzip -q "$zip" -d "$tmp"
        mkdir -p "$HOME/.brace/toolchains"
        rm -rf "$cache"
        mv "$tmp/brace-$v" "$cache"
        rm -rf "$tmp"
        echo "  Staged brace $v toolchain at $cache"
    fi
    return 0
}

# ---------- collect migration guides in the (FROM, TO] range ----------
# Guides are named brace-<lo>-to-<hi>.md. Include those whose boundary lies within the
# upgrade range. Concatenated in ascending order of <hi>.

collect_guides() {
    local from=$1 to=$2
    python3 - "$MIGRATE_DIR" "$from" "$to" <<'PY'
import sys, re, os
mig_dir, frm, to = sys.argv[1], sys.argv[2], sys.argv[3]
def parse(v):
    v = v.replace('-SNAPSHOT','')
    return tuple(int(x) for x in re.findall(r'\d+', v))
fv, tv = parse(frm), parse(to)
guides = []
for name in os.listdir(mig_dir):
    m = re.match(r'brace-(\d[\w.]*?)-to-(\d[\w.]*?)\.md$', name)
    if not m: continue
    lo, hi = parse(m.group(1)), parse(m.group(2))
    # guide covers a step inside our range: lo >= from and hi <= to
    if lo >= fv and hi <= tv:
        guides.append((hi, name))
guides.sort()
for _, name in guides:
    p = os.path.join(mig_dir, name)
    sys.stdout.write(f"\n\n===== Migration guide: {name} =====\n\n")
    with open(p) as f:
        sys.stdout.write(f.read())
PY
}

# ---------- main ----------

WORK_DIR="$SCRIPT_DIR/work/brace-migrate-${FROM}-to-${TO}"
RESULT_FILE="$RESULTS_DIR/brace-migrate-${FROM}-to-${TO}-${GUIDE_MODE}.json"
XML_FILE="$WORK_DIR/test-results.xml"

ensure_to_toolchain "$TO" || exit 1

GUIDES=$(collect_guides "$FROM" "$TO")
if [ "$GUIDE_MODE" = "provided" ] && [ -z "$GUIDES" ]; then
    echo "  WARNING: no migration guides found for $FROM -> $TO in $MIGRATE_DIR"
fi

# Fresh copy of the fixture.
rm -rf "$WORK_DIR"
mkdir -p "$(dirname "$WORK_DIR")"
cp -r "$FIXTURE_DIR" "$WORK_DIR"
rm -rf "$WORK_DIR/target"
cd "$WORK_DIR"
git init -q; git add -A; git commit -q -m "fixture: Conference Manager on brace $FROM"

# ---------- baseline gate: fixture must be green on FROM before we touch anything ----------

echo "  Baseline: building + testing fixture on brace $FROM ..."
mvn -q package -DskipTests > /dev/null 2>&1 || { echo "  ERROR: fixture failed to build on $FROM"; exit 1; }
BASELINE_PASSED=0; BASELINE_TOTAL=0; BASELINE_FAILED=0
if start_app "$WORK_DIR" "$PORT"; then
    cd "$TESTS"
    TEST_BASE_URL="http://localhost:$PORT" python3 -m pytest $ORACLE_TESTS --tb=no -q --junitxml="$XML_FILE" > /dev/null 2>&1 || true
    read BASELINE_PASSED BASELINE_FAILED BASELINE_TOTAL <<< "$(parse_test_results "$XML_FILE")"
    stop_app
fi
echo "  Baseline: $BASELINE_PASSED/$BASELINE_TOTAL passed"
if [ "$BASELINE_FAILED" != "0" ] || [ "$BASELINE_TOTAL" -eq 0 ]; then
    echo "  ERROR: fixture is not green on $FROM — fix the fixture before migrating. Aborting."
    exit 1
fi

# ---------- agent migration ----------

SESSION_ID=""
TOTAL_COST="0"
INVOCATIONS=()
START_TIME=$(date +%s)

GUIDE_BLOCK=""
if [ "$GUIDE_MODE" = "provided" ]; then
    GUIDE_BLOCK="Follow these migration guides in order. Apply every change they require.

$GUIDES"
else
    GUIDE_BLOCK="No migration guide is provided. Determine the necessary changes from the new framework version: run \`brace agents-md\` to regenerate BRACE-AGENTS.md, read it, and use the compile/boot/test errors you will be shown to drive the fixes."
fi

echo "  Migrating with the agent ($GUIDE_MODE) ..."
agent_turn "You are upgrading an existing Brace application from Brace $FROM to Brace $TO.

The project is a Maven project pinned to Brace $FROM via the <brace.version> property in pom.xml. Perform the upgrade:

1. Bump <brace.version> in pom.xml from $FROM to $TO.
2. Apply all required code changes for this version step.
3. Run \`brace agents-md\` to refresh BRACE-AGENTS.md from the new framework version.

Constraints:
- Make minimal, targeted edits. Do not refactor working code or rewrite features.
- Do NOT modify any test files — the test suite is the acceptance oracle and must keep passing unchanged.
- Do NOT delete functionality to avoid a migration step. The admin surface in app/controller/AdminController.java must be migrated properly, not removed.
- The app must still listen on port $PORT.

$GUIDE_BLOCK" 12 > "$WORK_DIR/migrate.txt"

echo "  Migration turn complete. Cost so far: \$$TOTAL_COST"
kill_port "$PORT"

# ---------- compile loop ----------

COMPILE_ATTEMPTS=0
COMPILED=false
for attempt in $(seq 1 $MAX_COMPILE_RETRIES); do
    COMPILE_ATTEMPTS=$attempt
    if (cd "$WORK_DIR" && mvn -q compile > /dev/null 2>&1); then COMPILED=true; break; fi
    [ "$attempt" -eq "$MAX_COMPILE_RETRIES" ] && break
    ERRORS=$(cd "$WORK_DIR" && mvn compile 2>&1 | grep -E "ERROR|error:" | head -40)
    echo "  Compile failed (attempt $attempt) — sending errors to agent"
    agent_turn "The project failed to compile after the migration. Fix the compilation errors with minimal changes. Errors:

$ERRORS" 5 > /dev/null
done

# ---------- boot + test loop (startup failures, e.g. rejected route patterns, feed back too) ----------

FIX_ATTEMPTS=0
TESTS_PASSED=0; TESTS_FAILED=0; TOTAL_TESTS=0
if [ "$COMPILED" = true ]; then
    (cd "$WORK_DIR" && mvn -q package -DskipTests > /dev/null 2>&1) || true

    for attempt in $(seq 0 $MAX_BOOT_TEST_RETRIES); do
        if ! start_app "$WORK_DIR" "$PORT"; then
            echo "  App failed to start (attempt $((attempt+1)))"
            if [ "$attempt" -lt "$MAX_BOOT_TEST_RETRIES" ]; then
                FIX_ATTEMPTS=$((FIX_ATTEMPTS+1))
                BOOTLOG=$(tail -30 "$WORK_DIR/app.log" 2>/dev/null)
                agent_turn "The application compiled but fails to start after the migration. Fix the startup error with minimal changes (do not modify tests). Startup log:

$BOOTLOG" 5 > /dev/null
                (cd "$WORK_DIR" && mvn -q package -DskipTests > /dev/null 2>&1) || true
                continue
            fi
            break
        fi

        cd "$TESTS"
        TEST_BASE_URL="http://localhost:$PORT" python3 -m pytest $ORACLE_TESTS --tb=short -q --junitxml="$XML_FILE" > /dev/null 2>&1 || true
        read TESTS_PASSED TESTS_FAILED TOTAL_TESTS <<< "$(parse_test_results "$XML_FILE")"

        if [ "$TESTS_FAILED" = "0" ] && [ "$TESTS_PASSED" -gt "0" ]; then
            echo "  All $TESTS_PASSED/$TOTAL_TESTS tests passed"
            stop_app; break
        fi
        echo "  $TESTS_PASSED/$TOTAL_TESTS passed ($TESTS_FAILED failed)"
        if [ "$attempt" -lt "$MAX_BOOT_TEST_RETRIES" ]; then
            FIX_ATTEMPTS=$((FIX_ATTEMPTS+1))
            DETAIL=$(parse_failure_details "$XML_FILE")
            stop_app
            agent_turn "Some tests fail after the migration. Fix the application code only (never the tests) with minimal, targeted changes. All other tests must keep passing. Failures:

$DETAIL

Test files live in $TESTS/ — running: $ORACLE_TESTS" 5 > /dev/null
            (cd "$WORK_DIR" && mvn -q package -DskipTests > /dev/null 2>&1) || true
        else
            stop_app
        fi
    done
fi

# ---------- metrics ----------

END_TIME=$(date +%s)
WALL=$((END_TIME - START_TIME))

# Did the agent bump the dependency version?
WORK_PIN=$(sed -n 's/.*<brace\.version>\(.*\)<\/brace\.version>.*/\1/p' "$WORK_DIR/pom.xml" | head -1)
VERSION_BUMPED=false; [ "$WORK_PIN" = "$TO" ] && VERSION_BUMPED=true

# Did `brace agents-md` run? (BRACE-AGENTS.md is new in 0.1.7; otherwise check it changed.)
AGENTS_MD_REFRESHED=false; [ -f "$WORK_DIR/BRACE-AGENTS.md" ] && AGENTS_MD_REFRESHED=true

# How much application code did the migration touch?
LINES_CHANGED=$(cd "$WORK_DIR" && git diff --numstat HEAD 2>/dev/null | awk '{a+=$1; d+=$2} END {print a+d+0}')

BEHAVIOR_PRESERVED=false
if [ "$TESTS_FAILED" = "0" ] && [ "$TESTS_PASSED" = "$BASELINE_TOTAL" ]; then BEHAVIOR_PRESERVED=true; fi

MODEL_USAGE=$(printf '%s\n' "${INVOCATIONS[@]}" | jq -s '
    [.[].modelUsage // {} | to_entries[]] | group_by(.key) |
    map({key: .[0].key, value: {
        inputTokens: (map(.value.inputTokens // 0) | add),
        outputTokens: (map(.value.outputTokens // 0) | add),
        cacheReadInputTokens: (map(.value.cacheReadInputTokens // 0) | add),
        cacheCreationInputTokens: (map(.value.cacheCreationInputTokens // 0) | add),
        costUSD: (map(.value.costUSD // 0) | add)
    }}) | from_entries' 2>/dev/null || echo '{}')

cat > "$RESULT_FILE" <<JSON
{
    "benchmark": "migrate",
    "framework": "brace",
    "migration_from": "$FROM",
    "migration_to": "$TO",
    "guide_mode": "$GUIDE_MODE",
    "fix_model": "$FIX_MODEL",
    "baseline_passed": $BASELINE_PASSED,
    "baseline_total": $BASELINE_TOTAL,
    "total_cost_usd": $TOTAL_COST,
    "model_usage": $MODEL_USAGE,
    "compile_attempts": $COMPILE_ATTEMPTS,
    "compiled": $COMPILED,
    "fix_attempts": $FIX_ATTEMPTS,
    "tests_passed": $TESTS_PASSED,
    "tests_failed": $TESTS_FAILED,
    "total_tests": $TOTAL_TESTS,
    "behavior_preserved": $BEHAVIOR_PRESERVED,
    "version_bumped_correctly": $VERSION_BUMPED,
    "agents_md_refreshed": $AGENTS_MD_REFRESHED,
    "lines_changed": ${LINES_CHANGED:-0},
    "wall_clock_seconds": $WALL
}
JSON

echo ""
echo "=== Result: brace $FROM -> $TO ($GUIDE_MODE) ==="
jq '.' "$RESULT_FILE"
echo ""
if [ "$BEHAVIOR_PRESERVED" = true ] && [ "$FIX_ATTEMPTS" -eq 0 ]; then
    echo "GATE PASS: clean migration, zero fix loops."
elif [ "$BEHAVIOR_PRESERVED" = true ]; then
    echo "GATE WARN: migration succeeded but took $FIX_ATTEMPTS fix loop(s) — the guide left work the agent had to discover. Close the gap before tagging."
else
    echo "GATE FAIL: behaviour not preserved ($TESTS_PASSED/$BASELINE_TOTAL). The migration guide is incomplete or wrong."
fi
cd "$SCRIPT_DIR"