run-feature.sh

#!/usr/bin/env bash
set -euo pipefail

#
# Benchmark 2: Cold-Start Feature Addition
#
# Takes an existing greenfield build, updates CLAUDE.md with project-specific info,
# starts a NEW session (no history), and measures the cost of adding a feature.
#
# Usage:
#   ./run-feature.sh 1               # run both frameworks
#   ./run-feature.sh 1 --brace       # brace only
#   ./run-feature.sh 1 --spring      # spring only
#
# Requires: a completed greenfield run in work/<framework>-run1/
#

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
RESULTS_DIR="$SCRIPT_DIR/results"
FEATURE_SPEC="$SCRIPT_DIR/feature-spec.md"
TESTS="$SCRIPT_DIR/tests"
MAX_COMPILE_RETRIES=5
MAX_TEST_RETRIES=3

# ---------- port assignment ----------

declare -A FRAMEWORK_PORTS
FRAMEWORK_PORTS[brace]=8080
FRAMEWORK_PORTS[spring]=8081
FRAMEWORK_PORTS[hono]=8082

# Parse arguments
RUNS=1
FRAMEWORKS="brace spring hono"
FIX_MODEL="sonnet"

while [[ $# -gt 0 ]]; do
    case $1 in
        --brace)    FRAMEWORKS="brace"; shift ;;
        --spring)   FRAMEWORKS="spring"; shift ;;
        --hono)     FRAMEWORKS="hono"; shift ;;
        --fix-model) FIX_MODEL="$2"; shift 2 ;;
        *)          RUNS=$1; shift ;;
    esac
done

mkdir -p "$RESULTS_DIR"

# ---------- helpers ----------

is_node_project() {
    [ -f "$1/package.json" ]
}

kill_port() {
    local port=$1
    local pids=$(lsof -ti:$port 2>/dev/null || true)
    if [ -z "$pids" ]; then return 0; fi
    echo "$pids" | xargs kill 2>/dev/null || true
    sleep 2
    pids=$(lsof -ti:$port 2>/dev/null || true)
    if [ -n "$pids" ]; then
        echo "  Force-killing stubborn process on port $port"
        echo "$pids" | xargs kill -9 2>/dev/null || true
        sleep 1
    fi
    for i in $(seq 1 10); do
        if ! lsof -ti:$port > /dev/null 2>&1; then return 0; fi
        sleep 1
    done
    echo "  WARNING: port $port still in use after cleanup"
    return 1
}

start_app() {
    local work_dir=$1
    local port=$2
    kill_port "$port"
    cd "$work_dir"
    if is_node_project "$work_dir"; then
        node dist/index.js > /dev/null 2>&1 &
    else
        SERVER_PORT=$port java -jar target/conference-manager-1.0-SNAPSHOT.jar > /dev/null 2>&1 &
    fi
    APP_PID=$!
    for i in $(seq 1 30); do
        if curl -sf "http://localhost:$port/events" > /dev/null 2>&1; then
            return 0
        fi
        if ! kill -0 "$APP_PID" 2>/dev/null; then
            echo "  App process died during startup"
            return 1
        fi
        sleep 1
    done
    echo "  App failed to start within 30s"
    return 1
}

stop_app() {
    if [ -n "${APP_PID:-}" ]; then
        kill "$APP_PID" 2>/dev/null || true
        for i in $(seq 1 5); do
            kill -0 "$APP_PID" 2>/dev/null || break
            sleep 1
        done
        kill -9 "$APP_PID" 2>/dev/null || true
        wait "$APP_PID" 2>/dev/null || true
        APP_PID=""
    fi
}

parse_test_results() {
    local xml_file=$1
    if [ ! -f "$xml_file" ]; then
        echo "0 0 0"
        return
    fi
    python3 -c "
import xml.etree.ElementTree as ET
root = ET.parse('$xml_file').getroot()
ts = root.find('testsuite') if root.tag != 'testsuite' else root
tests = int(ts.get('tests', 0))
failures = int(ts.get('failures', 0))
errors = int(ts.get('errors', 0))
passed = tests - failures - errors
print(f'{passed} {failures + errors} {tests}')
"
}

parse_failure_details() {
    local xml_file=$1
    python3 -c "
import xml.etree.ElementTree as ET
root = ET.parse('$xml_file').getroot()
ts = root.find('testsuite') if root.tag != 'testsuite' else root
for tc in ts.findall('.//testcase'):
    fail = tc.find('failure')
    if fail is not None:
        print(f'FAILED: {tc.get(\"classname\")}.{tc.get(\"name\")}')
        msg = fail.get('message', '')[:200]
        print(f'  {msg}')
        print()
" 2>/dev/null || echo "Could not parse test results"
}

accumulate_stats() {
    local output=$1
    local cost=$(echo "$output" | jq -r '.total_cost_usd // 0')
    TOTAL_COST=$(echo "$TOTAL_COST + $cost" | bc)
    INVOCATIONS+=("$output")
}

# Generate project-specific CLAUDE.md content by reading the existing codebase
generate_project_claude_md() {
    local work_dir=$1
    local framework=$2

    echo "  Generating project-specific CLAUDE.md..."

    local overview=""
    overview+="## Project-Specific Context\n\n"

    if is_node_project "$work_dir"; then
        # TypeScript project
        local src_files=$(find "$work_dir/src" -name "*.ts" -type f | sort)

        overview+="### Source Files\n\n"
        overview+="\`\`\`\n"
        for f in $src_files; do
            local rel=${f#$work_dir/}
            overview+="$rel\n"
        done
        overview+="\`\`\`\n\n"

        # Add route summary from TypeScript files
        overview+="### Routes\n\n"
        overview+="\`\`\`\n"
        local routes=$(grep -rhE "app\.(get|post|put|delete)\(" "$work_dir/src/" 2>/dev/null | sed 's/^[[:space:]]*//' || true)
        overview+="$routes\n"
        overview+="\`\`\`\n"
    else
        # Java project
        local src_files=$(find "$work_dir/src" -name "*.java" -type f | sort)
        local migration_files=$(find "$work_dir/src" -name "*.sql" -type f 2>/dev/null | sort)

        overview+="### Source Files\n\n"
        overview+="\`\`\`\n"
        for f in $src_files; do
            local rel=${f#$work_dir/}
            overview+="$rel\n"
        done
        overview+="\`\`\`\n\n"

        # Add entity summary
        overview+="### Entities\n\n"
        for f in $(find "$work_dir/src" -path "*/model/*.java" -type f | sort); do
            local class=$(basename "$f" .java)
            local fields=$(grep "public " "$f" | grep -v "class " | sed 's/.*public /- /' | sed 's/;$//')
            overview+="**$class**: $fields\n"
        done
        overview+="\n"

        # Add route summary for Brace (from App.java)
        if [ "$framework" = "brace" ]; then
            overview+="### Routes (from App.java)\n\n"
            overview+="\`\`\`\n"
            local routes=$(grep -E "app\.(get|post|put|delete)" "$work_dir/src/main/java/app/App.java" 2>/dev/null | sed 's/^[[:space:]]*//' || true)
            overview+="$routes\n"
            overview+="\`\`\`\n"
        fi
    fi

    # Append to existing CLAUDE.md
    printf "\n$overview" >> "$work_dir/CLAUDE.md"
}

# ---------- main benchmark function ----------

run_feature_benchmark() {
    local framework=$1
    local run_number=$2
    local port=${FRAMEWORK_PORTS[$framework]}
    local source_dir="$SCRIPT_DIR/work/${framework}-run1"
    local work_dir="$SCRIPT_DIR/work/${framework}-feature-run${run_number}"
    local result_file="$RESULTS_DIR/${framework}-feature-run${run_number}.json"
    local xml_file="$work_dir/test-results.xml"

    echo "=== $framework feature run $run_number (port $port) ==="

    # Verify source exists
    if [ ! -d "$source_dir" ]; then
        echo "  ERROR: No greenfield build found at $source_dir. Run ./run.sh first."
        return 1
    fi

    # Copy from the greenfield build
    rm -rf "$work_dir"
    mkdir -p "$(dirname "$work_dir")"
    cp -r "$source_dir" "$work_dir"

    cd "$work_dir"

    # Update port references in source files and CLAUDE.md
    if [ "$port" != "8080" ]; then
        sed -i '' "s/8080/$port/g" CLAUDE.md
        if is_node_project "$work_dir"; then
            find src -type f -name "*.ts" -exec sed -i '' "s/8080/$port/g" {} + 2>/dev/null || true
        else
            find src -type f \( -name "*.java" -o -name "*.properties" \) -exec sed -i '' "s/8080/$port/g" {} + 2>/dev/null || true
        fi
    fi

    # Clean up any previous test artifacts
    rm -f test-results.xml
    rm -f plan.txt execution.txt plan-raw.json execution-raw.json
    rm -f compile-fix-*.txt test-fix-*.txt

    # Reinstall Node.js dependencies (cp -r breaks symlinks in node_modules/.bin)
    if is_node_project "$work_dir"; then
        echo "  Reinstalling npm dependencies..."
        cd "$work_dir"
        rm -rf node_modules
        if ! npm install --silent 2>&1; then
            echo "  ERROR: npm install failed"
            return 1
        fi
    fi

    # Update CLAUDE.md with project-specific info
    generate_project_claude_md "$work_dir" "$framework"

    # Fresh git state
    rm -rf .git
    git init -q
    git add -A
    git commit -q -m "initial - existing conference manager"

    SESSION_ID=""
    TOTAL_COST="0"
    INVOCATIONS=()
    local plan_cost="0"
    local exec_cost="0"
    local compile_attempts=0
    local test_fix_attempts=0
    local start_time=$(date +%s)
    local test_progression="[]"

    # Phase 1: Plan the feature
    echo "  Phase 1: Planning feature addition..."
    local plan_output
    plan_output=$(claude -p "$(cat "$FEATURE_SPEC")

Read the existing codebase to understand the current implementation before planning. DO NOT IMPLEMENT ANYTHING YET. Create a detailed task breakdown for adding this feature. For each task, indicate which model (haiku, sonnet, or opus) you would delegate it to using subagents. Use the agent you think is best for each task." \
        --output-format json \
        --permission-mode bypassPermissions \
        --model opus \
        --max-budget-usd 2 \
        2>/dev/null)

    SESSION_ID=$(echo "$plan_output" | jq -r '.session_id // empty')
    plan_cost=$(echo "$plan_output" | jq -r '.total_cost_usd // 0')
    accumulate_stats "$plan_output"
    echo "$plan_output" | jq -r '.result // empty' > "$work_dir/plan.txt"
    echo "$plan_output" > "$work_dir/plan-raw.json"
    echo "  Plan complete. Cost: \$$plan_cost"

    # Phase 2: Execute
    echo "  Phase 2: Executing feature addition..."
    local exec_output
    local resume_flag=""
    if [ -n "${SESSION_ID:-}" ]; then
        resume_flag="--resume $SESSION_ID"
    fi
    exec_output=$(claude -p "Now execute the plan you just created. Delegate each task to subagents using the model you specified. Run independent tasks in parallel where possible. IMPORTANT: All existing tests must continue to pass — do not break existing functionality. The app must listen on port $port." \
        --output-format json \
        --permission-mode bypassPermissions \
        --model opus \
        --max-budget-usd 10 \
        $resume_flag \
        2>/dev/null)

    SESSION_ID=$(echo "$exec_output" | jq -r '.session_id // empty')
    exec_cost=$(echo "$exec_output" | jq -r '.total_cost_usd // 0')
    accumulate_stats "$exec_output"
    echo "$exec_output" | jq -r '.result // empty' > "$work_dir/execution.txt"
    echo "$exec_output" > "$work_dir/execution-raw.json"
    echo "  Execution complete. Cost: \$$exec_cost (total: \$$TOTAL_COST)"

    # Kill any process left on port by the execution phase
    echo "  Cleaning up port $port..."
    kill_port "$port"

    # Compile loop
    local compiled=false
    local compile_cmd package_cmd
    if is_node_project "$work_dir"; then
        compile_cmd="npx tsc --noEmit"
        package_cmd="npx tsc"
    else
        compile_cmd="mvn compile -q"
        package_cmd="mvn package -q -DskipTests"
    fi

    for attempt in $(seq 1 $MAX_COMPILE_RETRIES); do
        compile_attempts=$attempt
        echo "  Compile attempt $attempt..."
        if cd "$work_dir" && $compile_cmd 2>/dev/null; then
            compiled=true
            echo "  Compile OK"
            break
        fi

        if [ "$attempt" -eq "$MAX_COMPILE_RETRIES" ]; then
            echo "  Compile failed after $MAX_COMPILE_RETRIES attempts"
            break
        fi

        local errors
        errors=$(cd "$work_dir" && $compile_cmd 2>&1 | tail -50)
        echo "  Sending compile errors to AI..."
        local fix_output
        local resume_flag=""
        if [ -n "${SESSION_ID:-}" ]; then
            resume_flag="--resume $SESSION_ID"
        fi
        fix_output=$(claude -p "The project failed to compile. Fix the compilation errors. Make minimal changes — do not refactor or change working code. Here are the errors:

$errors" \
            --output-format json \
            --permission-mode bypassPermissions \
            --model $FIX_MODEL \
            --max-budget-usd 5 \
            $resume_flag \
            2>/dev/null)
        SESSION_ID=$(echo "$fix_output" | jq -r '.session_id // empty')
        accumulate_stats "$fix_output"
        echo "$fix_output" | jq -r '.result // empty' > "$work_dir/compile-fix-${attempt}.txt"
    done

    # Test loop — run BOTH original and feature tests
    local tests_passed=0
    local tests_failed=0
    local total_tests=0

    if [ "$compiled" = true ]; then
        echo "  Packaging..."
        cd "$work_dir" && $package_cmd 2>/dev/null || true

        echo "  Starting app..."
        if start_app "$work_dir" "$port"; then

            for test_attempt in $(seq 0 $MAX_TEST_RETRIES); do
                echo "  Running all tests (attempt $((test_attempt + 1)))..."
                cd "$TESTS"
                # Run both original and feature tests
                TEST_BASE_URL="http://localhost:$port" python3 -m pytest test_conference.py test_feature_availability.py -v --tb=short --junitxml="$xml_file" 2>&1 || true

                read tests_passed tests_failed total_tests <<< $(parse_test_results "$xml_file")

                test_progression=$(echo "$test_progression" | jq \
                    --argjson attempt "$test_attempt" \
                    --argjson passed "$tests_passed" \
                    --argjson failed "$tests_failed" \
                    --argjson total "$total_tests" \
                    '. + [{"attempt": $attempt, "passed": $passed, "failed": $failed, "total": $total}]')

                if [ "$tests_failed" = "0" ] && [ "$tests_passed" -gt "0" ]; then
                    echo "  All $tests_passed/$total_tests tests passed!"
                    break
                fi

                echo "  $tests_passed/$total_tests tests passed ($tests_failed failed)"

                if [ "$test_attempt" -lt "$MAX_TEST_RETRIES" ]; then
                    test_fix_attempts=$((test_fix_attempts + 1))
                    echo "  Sending failures to AI..."

                    stop_app

                    local failed_detail
                    failed_detail=$(parse_failure_details "$xml_file")

                    local fix_output
                    local resume_flag=""
                    if [ -n "${SESSION_ID:-}" ]; then
                        resume_flag="--resume $SESSION_ID"
                    fi
                    fix_output=$(cd "$work_dir" && claude -p "Some integration tests are failing. Fix the issues directly — make minimal, targeted changes. Do not refactor or change any code that isn't related to the failures. All other tests must continue to pass. Do not modify the tests, fix the application code only. Here are the failures:

$failed_detail

The test files are at $TESTS/test_conference.py and $TESTS/test_feature_availability.py" \
                        --output-format json \
                        --permission-mode bypassPermissions \
                        --model $FIX_MODEL \
                        --max-budget-usd 5 \
                        $resume_flag \
                        2>/dev/null)
                    SESSION_ID=$(echo "$fix_output" | jq -r '.session_id // empty')
                    accumulate_stats "$fix_output"
                    echo "$fix_output" | jq -r '.result // empty' > "$work_dir/test-fix-${test_attempt}.txt"

                    cd "$work_dir"
                    $package_cmd 2>/dev/null || true
                    start_app "$work_dir" "$port" || break
                fi
            done

            stop_app
        fi
    fi

    local end_time=$(date +%s)
    local wall_clock=$((end_time - start_time))

    local fix_cost=$(echo "$TOTAL_COST - $plan_cost - $exec_cost" | bc)

    # Merge modelUsage
    local model_usage
    model_usage=$(printf '%s\n' "${INVOCATIONS[@]}" | jq -s '
        [.[].modelUsage // {} | to_entries[]] |
        group_by(.key) |
        map({
            key: .[0].key,
            value: {
                inputTokens: (map(.value.inputTokens // 0) | add),
                outputTokens: (map(.value.outputTokens // 0) | add),
                cacheReadInputTokens: (map(.value.cacheReadInputTokens // 0) | add),
                cacheCreationInputTokens: (map(.value.cacheCreationInputTokens // 0) | add),
                costUSD: (map(.value.costUSD // 0) | add)
            }
        }) | from_entries
    ' 2>/dev/null || echo '{}')

    cat > "$result_file" <<RESULTS_EOF
{
    "benchmark": "feature-addition",
    "framework": "$framework",
    "run": $run_number,
    "fix_model": "$FIX_MODEL",
    "total_cost_usd": $TOTAL_COST,
    "phase_costs": {
        "plan": $plan_cost,
        "execution": $exec_cost,
        "fix": $fix_cost
    },
    "model_usage": $model_usage,
    "compile_attempts": $compile_attempts,
    "compiled": $compiled,
    "test_fix_attempts": $test_fix_attempts,
    "test_progression": $test_progression,
    "tests_passed": $tests_passed,
    "tests_failed": $tests_failed,
    "total_tests": $total_tests,
    "wall_clock_seconds": $wall_clock
}
RESULTS_EOF

    echo "  Results:"
    cat "$result_file" | jq '.'
    echo ""
    cd "$SCRIPT_DIR"
}

# Run benchmarks — frameworks in parallel, runs sequential
for run in $(seq 1 $RUNS); do
    pids=()
    for fw in $FRAMEWORKS; do
        run_feature_benchmark "$fw" "$run" &
        pids+=($!)
    done
    # Wait for all frameworks to complete this run
    for pid in "${pids[@]}"; do
        wait "$pid" || true
    done
done

# Summary
echo "=== Feature Addition Summary ==="
for fw in $FRAMEWORKS; do
    echo ""
    echo "$fw:"
    jq -s '{
        avg_cost_usd: (map(.total_cost_usd) | add / length),
        avg_compile_attempts: (map(.compile_attempts) | add / length),
        avg_test_fix_attempts: (map(.test_fix_attempts) | add / length),
        avg_tests_passed: (map(.tests_passed) | add / length),
        total_tests: (map(.total_tests) | max),
        avg_wall_clock: (map(.wall_clock_seconds) | add / length),
        runs: length
    }' "$RESULTS_DIR"/${fw}-feature-*.json 2>/dev/null || echo "  No results"
done