run-chain.sh

#!/usr/bin/env bash
set -euo pipefail

#
# Chained Feature Addition Benchmark
#
# Runs feature additions sequentially, each building on the output of the previous.
# Each feature round measures the cost of adding one feature to a growing codebase.
#
# Usage:
#   ./run-chain.sh 2 3          # run features 2 then 3
#   ./run-chain.sh 2            # run feature 2 only
#   ./run-chain.sh 2 3 --brace  # brace only
#   ./run-chain.sh 2 3 4 5      # run features 2 through 5
#
# Feature map:
#   1 = Speaker Availability (existing — run via run-feature.sh)
#   2 = Waitlist with Auto-Promotion
#   3 = Ratings & Speaker Stats
#   4 = Multi-Day Events & Tracks (refactor) [future]
#   5 = Notifications & Activity Feed [future]
#
# Requires: completed previous feature in work/<framework>-feature<N-1>-run1/
#   Feature 2 requires: work/<fw>-feature-run1/  (output of feature 1)
#   Feature 3 requires: work/<fw>-feature2-run1/ (output of feature 2)
#

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
RESULTS_DIR="$SCRIPT_DIR/results"
TESTS="$SCRIPT_DIR/tests"
MAX_COMPILE_RETRIES=5
MAX_TEST_RETRIES=3

# ---------- feature configuration ----------

declare -A FEATURE_SPECS
FEATURE_SPECS[2]="feature-spec-2-waitlist.md"
FEATURE_SPECS[3]="feature-spec-3-ratings.md"
FEATURE_SPECS[4]="feature-spec-4-multiday.md"
FEATURE_SPECS[5]="feature-spec-5-notifications.md"

# Test files are cumulative — each feature adds its test file
declare -A FEATURE_TESTS
FEATURE_TESTS[2]="test_conference.py test_feature_availability.py test_feature_waitlist.py"
FEATURE_TESTS[3]="test_conference.py test_feature_availability.py test_feature_waitlist.py test_feature_ratings.py"
FEATURE_TESTS[4]="test_conference.py test_feature_availability.py test_feature_waitlist.py test_feature_ratings.py test_feature_multiday.py"
FEATURE_TESTS[5]="test_conference.py test_feature_availability.py test_feature_waitlist.py test_feature_ratings.py test_feature_multiday.py test_feature_notifications.py"

# Source directory pattern: where to copy from for each feature
source_dir_for_feature() {
    local framework=$1
    local feature=$2
    local run=$3
    if [ "$feature" -eq 2 ]; then
        echo "$SCRIPT_DIR/work/${framework}-feature-run${run}"
    else
        local prev=$((feature - 1))
        echo "$SCRIPT_DIR/work/${framework}-feature${prev}-run${run}"
    fi
}

# Work directory: where this feature's output goes
work_dir_for_feature() {
    local framework=$1
    local feature=$2
    local run=$3
    echo "$SCRIPT_DIR/work/${framework}-feature${feature}-run${run}"
}

# Result file
result_file_for_feature() {
    local framework=$1
    local feature=$2
    local run=$3
    echo "$RESULTS_DIR/${framework}-feature${feature}-run${run}.json"
}

# ---------- port assignment ----------

declare -A FRAMEWORK_PORTS
FRAMEWORK_PORTS[brace]=8080
FRAMEWORK_PORTS[spring]=8081
FRAMEWORK_PORTS[hono]=8082

# ---------- parse arguments ----------

FEATURES=()
FRAMEWORKS="brace spring hono"
FIX_MODEL="opus"
RUN_NUMBER=1

while [[ $# -gt 0 ]]; do
    case $1 in
        --brace)     FRAMEWORKS="brace"; shift ;;
        --spring)    FRAMEWORKS="spring"; shift ;;
        --hono)      FRAMEWORKS="hono"; shift ;;
        --fix-model) FIX_MODEL="$2"; shift 2 ;;
        --run)       RUN_NUMBER="$2"; shift 2 ;;
        *)           FEATURES+=("$1"); shift ;;
    esac
done

if [ ${#FEATURES[@]} -eq 0 ]; then
    echo "Usage: $0 <feature-numbers...> [--brace|--spring|--hono] [--fix-model MODEL] [--run N]"
    echo "Example: $0 2 3 --brace --fix-model opus"
    exit 1
fi

mkdir -p "$RESULTS_DIR"

# ---------- helpers ----------

is_node_project() {
    [ -f "$1/package.json" ]
}

kill_port() {
    local port=$1
    local pids=$(lsof -ti:$port 2>/dev/null || true)
    if [ -z "$pids" ]; then return 0; fi

    # SIGTERM first
    echo "$pids" | xargs kill 2>/dev/null || true
    sleep 2

    # Check if still alive, SIGKILL if needed
    pids=$(lsof -ti:$port 2>/dev/null || true)
    if [ -n "$pids" ]; then
        echo "    Force-killing stubborn process on port $port"
        echo "$pids" | xargs kill -9 2>/dev/null || true
        sleep 1
    fi

    # Verify port is free
    for i in $(seq 1 10); do
        if ! lsof -ti:$port > /dev/null 2>&1; then
            return 0
        fi
        sleep 1
    done
    echo "    WARNING: port $port still in use after cleanup"
    return 1
}

start_app() {
    local work_dir=$1
    local port=$2

    # Ensure port is free before starting
    kill_port "$port"

    cd "$work_dir"
    if is_node_project "$work_dir"; then
        node dist/index.js > /dev/null 2>&1 &
    else
        SERVER_PORT=$port java -jar target/conference-manager-1.0-SNAPSHOT.jar > /dev/null 2>&1 &
    fi
    APP_PID=$!
    for i in $(seq 1 30); do
        if curl -sf "http://localhost:$port/events" > /dev/null 2>&1; then
            return 0
        fi
        if ! kill -0 "$APP_PID" 2>/dev/null; then
            echo "    App process died during startup"
            return 1
        fi
        sleep 1
    done
    echo "    App failed to start within 30s"
    return 1
}

stop_app() {
    if [ -n "${APP_PID:-}" ]; then
        kill "$APP_PID" 2>/dev/null || true
        # Wait up to 5 seconds for graceful shutdown
        for i in $(seq 1 5); do
            kill -0 "$APP_PID" 2>/dev/null || break
            sleep 1
        done
        # Force kill if still alive
        kill -9 "$APP_PID" 2>/dev/null || true
        wait "$APP_PID" 2>/dev/null || true
        APP_PID=""
    fi
}

parse_test_results() {
    local xml_file=$1
    if [ ! -f "$xml_file" ]; then
        echo "0 0 0"
        return
    fi
    python3 -c "
import xml.etree.ElementTree as ET
root = ET.parse('$xml_file').getroot()
ts = root.find('testsuite') if root.tag != 'testsuite' else root
tests = int(ts.get('tests', 0))
failures = int(ts.get('failures', 0))
errors = int(ts.get('errors', 0))
passed = tests - failures - errors
print(f'{passed} {failures + errors} {tests}')
"
}

parse_failure_details() {
    local xml_file=$1
    python3 -c "
import xml.etree.ElementTree as ET
root = ET.parse('$xml_file').getroot()
ts = root.find('testsuite') if root.tag != 'testsuite' else root
for tc in ts.findall('.//testcase'):
    fail = tc.find('failure')
    if fail is not None:
        print(f'FAILED: {tc.get(\"classname\")}.{tc.get(\"name\")}')
        msg = fail.get('message', '')[:200]
        print(f'  {msg}')
        print()
" 2>/dev/null || echo "Could not parse test results"
}

accumulate_stats() {
    local output=$1
    local cost=$(echo "$output" | jq -r '.total_cost_usd // 0')
    TOTAL_COST=$(echo "$TOTAL_COST + $cost" | bc)
    INVOCATIONS+=("$output")
}

generate_project_claude_md() {
    local work_dir=$1
    local framework=$2

    echo "    Generating project-specific CLAUDE.md..."

    local overview=""
    overview+="## Project-Specific Context\n\n"

    if is_node_project "$work_dir"; then
        local src_files=$(find "$work_dir/src" -name "*.ts" -type f | sort)
        overview+="### Source Files\n\n\`\`\`\n"
        for f in $src_files; do
            local rel=${f#$work_dir/}
            overview+="$rel\n"
        done
        overview+="\`\`\`\n\n"
        overview+="### Routes\n\n\`\`\`\n"
        local routes=$(grep -rhE "app\.(get|post|put|delete)\(" "$work_dir/src/" 2>/dev/null | sed 's/^[[:space:]]*//' || true)
        overview+="$routes\n\`\`\`\n"
    else
        local src_files=$(find "$work_dir/src" -name "*.java" -type f | sort)
        local migration_files=$(find "$work_dir/src" -name "*.sql" -type f 2>/dev/null | sort)
        overview+="### Source Files\n\n\`\`\`\n"
        for f in $src_files; do
            local rel=${f#$work_dir/}
            overview+="$rel\n"
        done
        overview+="\`\`\`\n\n"
        overview+="### Entities\n\n"
        for f in $(find "$work_dir/src" -path "*/model/*.java" -type f | sort); do
            local class=$(basename "$f" .java)
            local fields=$(grep "public " "$f" | grep -v "class " | sed 's/.*public /- /' | sed 's/;$//')
            overview+="**$class**: $fields\n"
        done
        overview+="\n"
        if [ "$framework" = "brace" ]; then
            overview+="### Routes (from App.java)\n\n\`\`\`\n"
            local routes=$(grep -E "app\.(get|post|put|delete)" "$work_dir/src/main/java/app/App.java" 2>/dev/null | sed 's/^[[:space:]]*//' || true)
            overview+="$routes\n\`\`\`\n"
        fi
    fi

    printf "\n$overview" >> "$work_dir/CLAUDE.md"
}

# ---------- run one feature for one framework ----------

run_feature() {
    local framework=$1
    local feature=$2
    local run=$3
    local port=${FRAMEWORK_PORTS[$framework]}
    local spec_file="$SCRIPT_DIR/${FEATURE_SPECS[$feature]}"
    local test_files="${FEATURE_TESTS[$feature]}"
    local src_dir=$(source_dir_for_feature "$framework" "$feature" "$run")
    local work_dir=$(work_dir_for_feature "$framework" "$feature" "$run")
    local result_file=$(result_file_for_feature "$framework" "$feature" "$run")
    local xml_file="$work_dir/test-results.xml"

    echo "=== $framework feature $feature run $run (port $port) ==="

    # Verify spec exists
    if [ ! -f "$spec_file" ]; then
        echo "  ERROR: Spec file not found: $spec_file"
        return 1
    fi

    # Verify source exists
    if [ ! -d "$src_dir" ]; then
        echo "  ERROR: Source directory not found: $src_dir"
        echo "  Run the previous feature first."
        return 1
    fi

    # Copy from previous feature's output
    rm -rf "$work_dir"
    mkdir -p "$(dirname "$work_dir")"
    cp -r "$src_dir" "$work_dir"
    cd "$work_dir"

    # Update port references
    if [ "$port" != "8080" ]; then
        sed -i '' "s/8080/$port/g" CLAUDE.md
        if is_node_project "$work_dir"; then
            find src -type f -name "*.ts" -exec sed -i '' "s/8080/$port/g" {} + 2>/dev/null || true
        else
            find src -type f \( -name "*.java" -o -name "*.properties" \) -exec sed -i '' "s/8080/$port/g" {} + 2>/dev/null || true
        fi
    fi

    # Clean up artifacts from previous run
    rm -f test-results.xml plan.txt execution.txt plan-raw.json execution-raw.json
    rm -f compile-fix-*.txt test-fix-*.txt

    # Reinstall Node.js dependencies
    if is_node_project "$work_dir"; then
        echo "  Reinstalling npm dependencies..."
        rm -rf node_modules
        npm install --silent 2>&1 || { echo "  ERROR: npm install failed"; return 1; }
    fi

    # Update CLAUDE.md with current project state
    generate_project_claude_md "$work_dir" "$framework"

    # Fresh git state
    rm -rf .git
    git init -q
    git add -A
    git commit -q -m "initial - before feature $feature"

    SESSION_ID=""
    TOTAL_COST="0"
    INVOCATIONS=()
    local plan_cost="0"
    local exec_cost="0"
    local compile_attempts=0
    local test_fix_attempts=0
    local start_time=$(date +%s)
    local test_progression="[]"

    # Phase 1: Plan
    echo "  Phase 1: Planning feature $feature..."
    local plan_output
    plan_output=$(claude -p "$(cat "$spec_file")

Read the existing codebase to understand the current implementation before planning. DO NOT IMPLEMENT ANYTHING YET. Create a detailed task breakdown for adding this feature. For each task, indicate which model (haiku, sonnet, or opus) you would delegate it to using subagents. Use the agent you think is best for each task." \
        --output-format json \
        --permission-mode bypassPermissions \
        --model opus \
        --max-budget-usd 2 \
        2>/dev/null)

    SESSION_ID=$(echo "$plan_output" | jq -r '.session_id // empty')
    plan_cost=$(echo "$plan_output" | jq -r '.total_cost_usd // 0')
    accumulate_stats "$plan_output"
    echo "$plan_output" | jq -r '.result // empty' > "$work_dir/plan.txt"
    echo "$plan_output" > "$work_dir/plan-raw.json"
    echo "  Plan complete. Cost: \$$plan_cost"

    # Phase 2: Execute
    echo "  Phase 2: Executing feature $feature..."
    local resume_flag=""
    if [ -n "${SESSION_ID:-}" ]; then
        resume_flag="--resume $SESSION_ID"
    fi
    local exec_output
    exec_output=$(claude -p "Now execute the plan you just created. Delegate each task to subagents using the model you specified. Run independent tasks in parallel where possible. IMPORTANT: All existing tests must continue to pass — do not break existing functionality. The app must listen on port $port." \
        --output-format json \
        --permission-mode bypassPermissions \
        --model opus \
        --max-budget-usd 10 \
        $resume_flag \
        2>/dev/null)

    SESSION_ID=$(echo "$exec_output" | jq -r '.session_id // empty')
    exec_cost=$(echo "$exec_output" | jq -r '.total_cost_usd // 0')
    accumulate_stats "$exec_output"
    echo "$exec_output" | jq -r '.result // empty' > "$work_dir/execution.txt"
    echo "$exec_output" > "$work_dir/execution-raw.json"
    echo "  Execution complete. Cost: \$$exec_cost (total: \$$TOTAL_COST)"

    # Kill any process left on port by the execution phase
    echo "  Cleaning up port $port..."
    kill_port "$port"

    # Compile loop
    local compiled=false
    local compile_cmd package_cmd
    if is_node_project "$work_dir"; then
        compile_cmd="npx tsc --noEmit"
        package_cmd="npx tsc"
    else
        compile_cmd="mvn compile -q"
        package_cmd="mvn package -q -DskipTests"
    fi

    for attempt in $(seq 1 $MAX_COMPILE_RETRIES); do
        compile_attempts=$attempt
        echo "  Compile attempt $attempt..."
        if cd "$work_dir" && $compile_cmd 2>/dev/null; then
            compiled=true
            echo "  Compile OK"
            break
        fi
        if [ "$attempt" -eq "$MAX_COMPILE_RETRIES" ]; then
            echo "  Compile failed after $MAX_COMPILE_RETRIES attempts"
            break
        fi
        local errors
        errors=$(cd "$work_dir" && $compile_cmd 2>&1 | tail -50)
        echo "  Sending compile errors to AI..."
        local resume_flag=""
        if [ -n "${SESSION_ID:-}" ]; then
            resume_flag="--resume $SESSION_ID"
        fi
        local fix_output
        fix_output=$(claude -p "The project failed to compile. Fix the compilation errors. Make minimal changes — do not refactor or change working code. Here are the errors:

$errors" \
            --output-format json \
            --permission-mode bypassPermissions \
            --model $FIX_MODEL \
            --max-budget-usd 5 \
            $resume_flag \
            2>/dev/null)
        SESSION_ID=$(echo "$fix_output" | jq -r '.session_id // empty')
        accumulate_stats "$fix_output"
    done

    # Test loop
    local tests_passed=0
    local tests_failed=0
    local total_tests=0

    if [ "$compiled" = true ]; then
        echo "  Packaging..."
        cd "$work_dir" && $package_cmd 2>/dev/null || true

        echo "  Starting app..."
        if start_app "$work_dir" "$port"; then

            for test_attempt in $(seq 0 $MAX_TEST_RETRIES); do
                echo "  Running all tests (attempt $((test_attempt + 1)))..."
                cd "$TESTS"
                TEST_BASE_URL="http://localhost:$port" python3 -m pytest $test_files -v --tb=short --junitxml="$xml_file" 2>&1 || true

                read tests_passed tests_failed total_tests <<< $(parse_test_results "$xml_file")

                test_progression=$(echo "$test_progression" | jq \
                    --argjson attempt "$test_attempt" \
                    --argjson passed "$tests_passed" \
                    --argjson failed "$tests_failed" \
                    --argjson total "$total_tests" \
                    '. + [{"attempt": $attempt, "passed": $passed, "failed": $failed, "total": $total}]')

                if [ "$tests_failed" = "0" ] && [ "$tests_passed" -gt "0" ]; then
                    echo "  All $tests_passed/$total_tests tests passed!"
                    break
                fi

                echo "  $tests_passed/$total_tests tests passed ($tests_failed failed)"

                if [ "$test_attempt" -lt "$MAX_TEST_RETRIES" ]; then
                    test_fix_attempts=$((test_fix_attempts + 1))
                    echo "  Sending failures to AI..."
                    stop_app

                    local failed_detail
                    failed_detail=$(parse_failure_details "$xml_file")

                    local resume_flag=""
                    if [ -n "${SESSION_ID:-}" ]; then
                        resume_flag="--resume $SESSION_ID"
                    fi
                    local fix_output
                    fix_output=$(cd "$work_dir" && claude -p "Some integration tests are failing. Fix the issues directly — make minimal, targeted changes. Do not refactor or change any code that isn't related to the failures. All other tests must continue to pass. Do not modify the tests, fix the application code only. Here are the failures:

$failed_detail

The test files are at $TESTS/ — the tests being run are: $test_files" \
                        --output-format json \
                        --permission-mode bypassPermissions \
                        --model $FIX_MODEL \
                        --max-budget-usd 5 \
                        $resume_flag \
                        2>/dev/null)
                    SESSION_ID=$(echo "$fix_output" | jq -r '.session_id // empty')
                    accumulate_stats "$fix_output"

                    cd "$work_dir"
                    $package_cmd 2>/dev/null || true
                    start_app "$work_dir" "$port" || break
                fi
            done

            stop_app
        fi
    fi

    local end_time=$(date +%s)
    local wall_clock=$((end_time - start_time))
    local fix_cost=$(echo "$TOTAL_COST - $plan_cost - $exec_cost" | bc)

    # Merge modelUsage
    local model_usage
    model_usage=$(printf '%s\n' "${INVOCATIONS[@]}" | jq -s '
        [.[].modelUsage // {} | to_entries[]] |
        group_by(.key) |
        map({
            key: .[0].key,
            value: {
                inputTokens: (map(.value.inputTokens // 0) | add),
                outputTokens: (map(.value.outputTokens // 0) | add),
                cacheReadInputTokens: (map(.value.cacheReadInputTokens // 0) | add),
                cacheCreationInputTokens: (map(.value.cacheCreationInputTokens // 0) | add),
                costUSD: (map(.value.costUSD // 0) | add)
            }
        }) | from_entries
    ' 2>/dev/null || echo '{}')

    cat > "$result_file" <<RESULTS_EOF
{
    "benchmark": "feature-${feature}",
    "framework": "$framework",
    "feature": $feature,
    "run": $run,
    "fix_model": "$FIX_MODEL",
    "total_cost_usd": $TOTAL_COST,
    "phase_costs": {
        "plan": $plan_cost,
        "execution": $exec_cost,
        "fix": $fix_cost
    },
    "model_usage": $model_usage,
    "compile_attempts": $compile_attempts,
    "compiled": $compiled,
    "test_fix_attempts": $test_fix_attempts,
    "test_progression": $test_progression,
    "tests_passed": $tests_passed,
    "tests_failed": $tests_failed,
    "total_tests": $total_tests,
    "wall_clock_seconds": $wall_clock
}
RESULTS_EOF

    echo "  Results:"
    cat "$result_file" | jq '.'
    echo ""
    cd "$SCRIPT_DIR"
}

# ---------- main loop ----------

for feature in "${FEATURES[@]}"; do
    echo ""
    echo "========================================="
    echo "  FEATURE $feature"
    echo "========================================="
    echo ""

    # Run frameworks in parallel for this feature
    pids=()
    for fw in $FRAMEWORKS; do
        run_feature "$fw" "$feature" "$RUN_NUMBER" &
        pids+=($!)
    done
    for pid in "${pids[@]}"; do
        wait "$pid" || true
    done
done

# ---------- summary ----------

echo ""
echo "=== Chain Summary ==="
for feature in "${FEATURES[@]}"; do
    echo ""
    echo "--- Feature $feature ---"
    for fw in $FRAMEWORKS; do
        local_result=$(result_file_for_feature "$fw" "$feature" "$RUN_NUMBER")
        if [ -f "$local_result" ]; then
            cost=$(jq -r '.total_cost_usd' "$local_result")
            passed=$(jq -r '.tests_passed' "$local_result")
            total=$(jq -r '.total_tests' "$local_result")
            fixes=$(jq -r '.test_fix_attempts' "$local_result")
            wall=$(jq -r '.wall_clock_seconds' "$local_result")
            printf "  %-8s  \$%.2f  %d/%d tests  %d fixes  %ds\n" "$fw" "$cost" "$passed" "$total" "$fixes" "$wall"
        else
            printf "  %-8s  (no results)\n" "$fw"
        fi
    done
done