test_routing_framework.sh

#!/bin/bash

# ===============================================================
# AgentFlow Routing Framework Comprehensive Test Suite
# ===============================================================
# Tests the sophisticated multi-model routing system with:
# - Intelligent task-based routing
# - Performance optimization
# - Fallback mechanisms
# - A/B testing framework
# - Load balancing
# - Error handling and recovery
# ===============================================================

set -e

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
PURPLE='\033[0;35m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color

# Configuration
API_URL="http://localhost:3001/api/v1"
TEST_API_KEY="test-routing-key-2024"
WEBSOCKET_URL="ws://localhost:3001"
CONCURRENT_USERS=10
TEST_DURATION=30
LOG_FILE="routing_test_results_$(date +%Y%m%d_%H%M%S).log"

# Test counters
TOTAL_TESTS=0
PASSED_TESTS=0
FAILED_TESTS=0

# Performance metrics
declare -A ROUTING_TIMES=()
declare -A MODEL_SELECTIONS=()
declare -A FALLBACK_TRIGGERS=()
declare -A AB_TEST_RESULTS=()

echo "==============================================================="
echo "🚀 AgentFlow Routing Framework Test Suite"
echo "==============================================================="
echo "Testing Date: $(date)"
echo "API Endpoint: $API_URL"
echo "Log File: $LOG_FILE"
echo "==============================================================="

# Logging function
log_test() {
    local test_name="$1"
    local passed="$2"
    local details="$3"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    
    TOTAL_TESTS=$((TOTAL_TESTS + 1))
    
    if [ "$passed" = true ]; then
        PASSED_TESTS=$((PASSED_TESTS + 1))
        echo -e "${GREEN}✅ PASS${NC} | $test_name | $details"
        echo "[$timestamp] PASS | $test_name | $details" >> "$LOG_FILE"
    else
        FAILED_TESTS=$((FAILED_TESTS + 1))
        echo -e "${RED}❌ FAIL${NC} | $test_name | $details"
        echo "[$timestamp] FAIL | $test_name | $details" >> "$LOG_FILE"
    fi
}

# Helper function to make API calls with retries
api_call() {
    local method="$1"
    local endpoint="$2"
    local data="$3"
    local max_retries=3
    local retry=0
    
    while [ $retry -lt $max_retries ]; do
        if [ "$method" = "POST" ]; then
            response=$(curl -s -X POST "$API_URL$endpoint" \
                -H "Content-Type: application/json" \
                -H "x-api-key: $TEST_API_KEY" \
                -d "$data" \
                -w "\n%{http_code}" 2>/dev/null)
        else
            response=$(curl -s -X GET "$API_URL$endpoint" \
                -H "x-api-key: $TEST_API_KEY" \
                -w "\n%{http_code}" 2>/dev/null)
        fi
        
        if [ $? -eq 0 ]; then
            echo "$response"
            return 0
        fi
        
        retry=$((retry + 1))
        sleep 1
    done
    
    echo "API_CALL_FAILED"
    return 1
}

# Check prerequisites
check_prerequisites() {
    echo -e "${BLUE}🔍 Checking Prerequisites...${NC}"
    
    # Check if backend is running
    if ! curl -s "$API_URL/health" > /dev/null 2>&1; then
        echo -e "${RED}❌ Backend server not running at $API_URL${NC}"
        echo "Please start the backend server first:"
        echo "cd backend && npm start"
        exit 1
    fi
    
    # Check if Ollama is running
    if ! curl -s "http://localhost:11434/api/tags" > /dev/null 2>&1; then
        echo -e "${RED}❌ Ollama not running at localhost:11434${NC}"
        echo "Please start Ollama first: ollama serve"
        exit 1
    fi
    
    # Check required tools
    for tool in curl jq bc; do
        if ! command -v $tool > /dev/null 2>&1; then
            echo -e "${RED}❌ Required tool not found: $tool${NC}"
            exit 1
        fi
    done
    
    log_test "Prerequisites Check" true "All requirements met"
}

# Test 1: Basic Routing Functionality
test_basic_routing() {
    echo -e "\n${BLUE}🧭 Test 1: Basic Routing Functionality${NC}"
    
    # Test data for different task types
    declare -A test_prompts=(
        ["conversation"]="Hello, how are you today?"
        ["analysis"]="Analyze the pros and cons of renewable energy sources"
        ["technical"]="Write a Python function to sort a list using quicksort algorithm"
        ["quick_response"]="What is the capital of France?"
        ["multimodal"]="Analyze this image and describe what you see"
        ["creative"]="Write a short story about a robot discovering emotions"
        ["planning"]="Create a project plan for launching a mobile app"
        ["research"]="Research the latest developments in quantum computing"
    )
    
    for task_type in "${!test_prompts[@]}"; do
        prompt="${test_prompts[$task_type]}"
        
        response=$(api_call "POST" "/models/route" "{
            \"prompt\": \"$prompt\",
            \"context\": {\"task_type\": \"$task_type\"},
            \"options\": {\"strategy\": \"intelligent\"}
        }")
        
        if [ "$response" != "API_CALL_FAILED" ]; then
            http_code=$(echo "$response" | tail -n1)
            body=$(echo "$response" | head -n -1)
            
            if [ "$http_code" = "200" ] && echo "$body" | jq -e '.success == true' > /dev/null 2>&1; then
                recommended_model=$(echo "$body" | jq -r '.recommended_model')
                confidence=$(echo "$body" | jq -r '.confidence')
                detected_task=$(echo "$body" | jq -r '.task_analysis.task_type')
                routing_time=$(echo "$body" | jq -r '.routing_decision.routing_time // 0')
                
                # Track routing performance
                ROUTING_TIMES["$task_type"]=$routing_time
                MODEL_SELECTIONS["$task_type"]=$recommended_model
                
                # Validate routing logic
                routing_valid=true
                case $task_type in
                    "multimodal"|"image")
                        if [[ ! "$recommended_model" =~ "moondream" ]]; then
                            routing_valid=false
                        fi
                        ;;
                    "technical"|"analysis"|"research")
                        if [[ ! "$recommended_model" =~ ("phi3.5"|"llama3.2") ]]; then
                            routing_valid=false
                        fi
                        ;;
                    "quick_response")
                        if [[ ! "$recommended_model" =~ ("gemma2"|"llama3.2") ]]; then
                            routing_valid=false
                        fi
                        ;;
                esac
                
                if [ "$routing_valid" = true ]; then
                    log_test "Basic Routing - $task_type" true "Model: $recommended_model, Confidence: $confidence, Task: $detected_task, Time: ${routing_time}ms"
                else
                    log_test "Basic Routing - $task_type" false "Suboptimal routing: $recommended_model for $task_type"
                fi
            else
                log_test "Basic Routing - $task_type" false "API Error: $http_code"
            fi
        else
            log_test "Basic Routing - $task_type" false "API call failed"
        fi
    done
}

# Test 2: Performance-Based Routing
test_performance_routing() {
    echo -e "\n${BLUE}⚡ Test 2: Performance-Based Routing${NC}"
    
    # Test performance-optimized routing
    response=$(api_call "POST" "/models/route" "{
        \"prompt\": \"Tell me a quick fact about space\",
        \"context\": {},
        \"options\": {\"strategy\": \"performance\"}
    }")
    
    if [ "$response" != "API_CALL_FAILED" ]; then
        http_code=$(echo "$response" | tail -n1)
        body=$(echo "$response" | head -n -1)
        
        if [ "$http_code" = "200" ]; then
            recommended_model=$(echo "$body" | jq -r '.recommended_model')
            routing_time=$(echo "$body" | jq -r '.routing_decision.routing_time // 0')
            strategy=$(echo "$body" | jq -r '.routing_decision.strategy')
            
            # Performance routing should prioritize fast models
            if [[ "$recommended_model" =~ ("gemma2"|"llama3.2") ]] && [ "$strategy" = "performance" ]; then
                log_test "Performance Routing" true "Model: $recommended_model, Strategy: $strategy, Time: ${routing_time}ms"
            else
                log_test "Performance Routing" false "Expected performance-optimized model, got: $recommended_model"
            fi
        else
            log_test "Performance Routing" false "HTTP $http_code"
        fi
    else
        log_test "Performance Routing" false "API call failed"
    fi
}

# Test 3: Fallback Mechanism
test_fallback_mechanism() {
    echo -e "\n${BLUE}🔄 Test 3: Fallback Mechanism${NC}"
    
    # Test with non-existent preferred model to trigger fallback
    response=$(api_call "POST" "/models/route" "{
        \"prompt\": \"Hello world\",
        \"context\": {},
        \"options\": {
            \"preferredModel\": \"non-existent-model\",
            \"strategy\": \"intelligent\"
        }
    }")
    
    if [ "$response" != "API_CALL_FAILED" ]; then
        http_code=$(echo "$response" | tail -n1)
        body=$(echo "$response" | head -n -1)
        
        if [ "$http_code" = "200" ]; then
            recommended_model=$(echo "$body" | jq -r '.recommended_model')
            reason=$(echo "$body" | jq -r '.routing_decision.reason // ""')
            
            # Should fallback to a valid model
            valid_models=("llama3.2:1b" "gemma2:2b" "phi3.5:latest" "moondream:latest")
            is_valid=false
            for model in "${valid_models[@]}"; do
                if [[ "$recommended_model" == "$model" ]]; then
                    is_valid=true
                    break
                fi
            done
            
            if [ "$is_valid" = true ]; then
                FALLBACK_TRIGGERS["fallback_test"]="$recommended_model"
                log_test "Fallback Mechanism" true "Fallback to: $recommended_model, Reason: $reason"
            else
                log_test "Fallback Mechanism" false "Invalid fallback model: $recommended_model"
            fi
        else
            log_test "Fallback Mechanism" false "HTTP $http_code"
        fi
    else
        log_test "Fallback Mechanism" false "API call failed"
    fi
}

# Test 4: Model Health Monitoring
test_model_health() {
    echo -e "\n${BLUE}🏥 Test 4: Model Health Monitoring${NC}"
    
    response=$(api_call "GET" "/models")
    
    if [ "$response" != "API_CALL_FAILED" ]; then
        http_code=$(echo "$response" | tail -n1)
        body=$(echo "$response" | head -n -1)
        
        if [ "$http_code" = "200" ]; then
            # Check if all expected models are reported
            models=$(echo "$body" | jq -r '.models[].name' 2>/dev/null)
            
            if echo "$models" | grep -q "llama3.2" && 
               echo "$models" | grep -q "gemma2" && 
               echo "$models" | grep -q "phi3.5"; then
                
                # Check health status
                healthy_count=$(echo "$body" | jq '[.models[] | select(.status == "healthy")] | length' 2>/dev/null)
                total_count=$(echo "$body" | jq '.models | length' 2>/dev/null)
                
                if [ "$healthy_count" -gt 0 ]; then
                    log_test "Model Health Monitoring" true "Healthy models: $healthy_count/$total_count"
                else
                    log_test "Model Health Monitoring" false "No healthy models available"
                fi
            else
                log_test "Model Health Monitoring" false "Missing expected models"
            fi
        else
            log_test "Model Health Monitoring" false "HTTP $http_code"
        fi
    else
        log_test "Model Health Monitoring" false "API call failed"
    fi
}

# Test 5: A/B Testing Framework
test_ab_testing() {
    echo -e "\n${BLUE}🧪 Test 5: A/B Testing Framework${NC}"
    
    # Configure A/B test
    ab_config="{
        \"testName\": \"routing_performance_test\",
        \"taskType\": \"conversation\",
        \"testModel\": \"phi3.5:latest\",
        \"controlModel\": \"llama3.2:1b\",
        \"testPercentage\": 0.5
    }"
    
    # Run multiple requests to test A/B distribution
    declare -A ab_results=()
    ab_results["phi3.5:latest"]=0
    ab_results["llama3.2:1b"]=0
    ab_results["other"]=0
    
    for i in {1..20}; do
        response=$(api_call "POST" "/models/route" "{
            \"prompt\": \"Hello, tell me a joke\",
            \"context\": {},
            \"options\": {\"enableABTesting\": true}
        }")
        
        if [ "$response" != "API_CALL_FAILED" ]; then
            http_code=$(echo "$response" | tail -n1)
            body=$(echo "$response" | head -n -1)
            
            if [ "$http_code" = "200" ]; then
                recommended_model=$(echo "$body" | jq -r '.recommended_model')
                
                if [[ "$recommended_model" == "phi3.5:latest" ]]; then
                    ab_results["phi3.5:latest"]=$((ab_results["phi3.5:latest"] + 1))
                elif [[ "$recommended_model" == "llama3.2:1b" ]]; then
                    ab_results["llama3.2:1b"]=$((ab_results["llama3.2:1b"] + 1))
                else
                    ab_results["other"]=$((ab_results["other"] + 1))
                fi
            fi
        fi
        
        sleep 0.1  # Small delay between requests
    done
    
    # Analyze distribution
    phi_count=${ab_results["phi3.5:latest"]}
    llama_count=${ab_results["llama3.2:1b"]}
    other_count=${ab_results["other"]}
    
    total_valid=$((phi_count + llama_count))
    
    if [ $total_valid -gt 10 ]; then
        # Check if distribution is roughly 50/50 (allowing for randomness)
        phi_percentage=$(echo "scale=2; $phi_count * 100 / $total_valid" | bc)
        
        AB_TEST_RESULTS["phi3.5_percentage"]=$phi_percentage
        AB_TEST_RESULTS["llama_count"]=$llama_count
        AB_TEST_RESULTS["phi_count"]=$phi_count
        
        log_test "A/B Testing Framework" true "Distribution - Phi3.5: $phi_count, Llama: $llama_count, Percentage: $phi_percentage%"
    else
        log_test "A/B Testing Framework" false "Insufficient valid responses for A/B testing"
    fi
}

# Test 6: Concurrent Load Testing
test_concurrent_routing() {
    echo -e "\n${BLUE}🚀 Test 6: Concurrent Load Testing${NC}"
    
    # Create temporary directory for concurrent test results
    temp_dir="/tmp/agentflow_concurrent_$$"
    mkdir -p "$temp_dir"
    
    # Launch concurrent requests
    for i in $(seq 1 $CONCURRENT_USERS); do
        (
            start_time=$(date +%s%3N)
            
            response=$(api_call "POST" "/models/route" "{
                \"prompt\": \"Concurrent test request $i\",
                \"context\": {\"user_id\": \"test_user_$i\"},
                \"options\": {\"strategy\": \"balanced\"}
            }")
            
            end_time=$(date +%s%3N)
            response_time=$((end_time - start_time))
            
            if [ "$response" != "API_CALL_FAILED" ]; then
                http_code=$(echo "$response" | tail -n1)
                if [ "$http_code" = "200" ]; then
                    echo "SUCCESS,$i,$response_time" > "$temp_dir/result_$i.txt"
                else
                    echo "FAIL,$i,$response_time" > "$temp_dir/result_$i.txt"
                fi
            else
                echo "FAIL,$i,0" > "$temp_dir/result_$i.txt"
            fi
        ) &
    done
    
    # Wait for all background jobs to complete
    wait
    
    # Analyze results
    success_count=0
    total_response_time=0
    max_response_time=0
    
    for result_file in "$temp_dir"/result_*.txt; do
        if [ -f "$result_file" ]; then
            result=$(cat "$result_file")
            IFS=',' read -r status user_id response_time <<< "$result"
            
            if [ "$status" = "SUCCESS" ]; then
                success_count=$((success_count + 1))
                total_response_time=$((total_response_time + response_time))
                
                if [ $response_time -gt $max_response_time ]; then
                    max_response_time=$response_time
                fi
            fi
        fi
    done
    
    # Calculate metrics
    if [ $success_count -gt 0 ]; then
        avg_response_time=$((total_response_time / success_count))
        success_rate=$(echo "scale=2; $success_count * 100 / $CONCURRENT_USERS" | bc)
        
        # Cleanup
        rm -rf "$temp_dir"
        
        # Pass if success rate > 90% and avg response time < 5000ms
        if (( $(echo "$success_rate >= 90" | bc -l) )) && [ $avg_response_time -lt 5000 ]; then
            log_test "Concurrent Load Testing" true "Success Rate: $success_rate%, Avg Response: ${avg_response_time}ms, Max: ${max_response_time}ms"
        else
            log_test "Concurrent Load Testing" false "Success Rate: $success_rate%, Avg Response: ${avg_response_time}ms (threshold: 90%, 5000ms)"
        fi
    else
        rm -rf "$temp_dir"
        log_test "Concurrent Load Testing" false "No successful responses"
    fi
}

# Test 7: Routing Performance Benchmarks
test_routing_performance() {
    echo -e "\n${BLUE}📊 Test 7: Routing Performance Benchmarks${NC}"
    
    declare -A benchmark_results=()
    total_routing_time=0
    benchmark_count=0
    
    # Test routing speed for different complexity levels
    declare -A benchmark_prompts=(
        ["simple"]="Hi"
        ["medium"]="Explain the concept of machine learning in simple terms"
        ["complex"]="Analyze the economic impact of artificial intelligence on global employment markets, considering both positive and negative effects, historical precedents, and potential mitigation strategies"
    )
    
    for complexity in "${!benchmark_prompts[@]}"; do
        prompt="${benchmark_prompts[$complexity]}"
        
        # Run 5 iterations for each complexity level
        complexity_times=()
        for i in {1..5}; do
            start_time=$(date +%s%3N)
            
            response=$(api_call "POST" "/models/route" "{
                \"prompt\": \"$prompt\",
                \"context\": {},
                \"options\": {\"bypassCache\": true}
            }")
            
            end_time=$(date +%s%3N)
            routing_time=$((end_time - start_time))
            
            if [ "$response" != "API_CALL_FAILED" ]; then
                http_code=$(echo "$response" | tail -n1)
                if [ "$http_code" = "200" ]; then
                    complexity_times+=($routing_time)
                    total_routing_time=$((total_routing_time + routing_time))
                    benchmark_count=$((benchmark_count + 1))
                fi
            fi
        done
        
        # Calculate average for this complexity
        if [ ${#complexity_times[@]} -gt 0 ]; then
            sum=0
            for time in "${complexity_times[@]}"; do
                sum=$((sum + time))
            done
            avg_time=$((sum / ${#complexity_times[@]}))
            benchmark_results["$complexity"]=$avg_time
        fi
    done
    
    # Overall performance analysis
    if [ $benchmark_count -gt 0 ]; then
        avg_routing_time=$((total_routing_time / benchmark_count))
        
        # Pass if average routing time < 200ms
        if [ $avg_routing_time -lt 200 ]; then
            log_test "Routing Performance" true "Avg routing time: ${avg_routing_time}ms (Simple: ${benchmark_results[simple]}ms, Medium: ${benchmark_results[medium]}ms, Complex: ${benchmark_results[complex]}ms)"
        else
            log_test "Routing Performance" false "Avg routing time: ${avg_routing_time}ms (threshold: 200ms)"
        fi
    else
        log_test "Routing Performance" false "No successful benchmark responses"
    fi
}

# Test 8: Error Handling and Recovery
test_error_handling() {
    echo -e "\n${BLUE}🛡️ Test 8: Error Handling and Recovery${NC}"
    
    # Test with malformed request
    response=$(api_call "POST" "/models/route" "{
        \"invalid_field\": \"test\"
    }")
    
    if [ "$response" != "API_CALL_FAILED" ]; then
        http_code=$(echo "$response" | tail -n1)
        if [ "$http_code" = "400" ]; then
            log_test "Error Handling - Malformed Request" true "Properly returned HTTP 400"
        else
            log_test "Error Handling - Malformed Request" false "Expected HTTP 400, got $http_code"
        fi
    else
        log_test "Error Handling - Malformed Request" false "API call failed"
    fi
    
    # Test with empty prompt
    response=$(api_call "POST" "/models/route" "{
        \"prompt\": \"\",
        \"context\": {},
        \"options\": {}
    }")
    
    if [ "$response" != "API_CALL_FAILED" ]; then
        http_code=$(echo "$response" | tail -n1)
        if [ "$http_code" = "400" ]; then
            log_test "Error Handling - Empty Prompt" true "Properly rejected empty prompt"
        else
            log_test "Error Handling - Empty Prompt" false "Expected HTTP 400, got $http_code"
        fi
    else
        log_test "Error Handling - Empty Prompt" false "API call failed"
    fi
}

# Test 9: Model Selection Logic Validation
test_model_selection_logic() {
    echo -e "\n${BLUE}🧠 Test 9: Model Selection Logic Validation${NC}"
    
    # Test specific model selection scenarios
    declare -A selection_tests=(
        ["vision_task"]='{"prompt": "Describe this image in detail", "context": {"hasImage": true}}'
        ["math_problem"]='{"prompt": "Solve this differential equation: dy/dx = x^2 + 3x + 2", "context": {}}'
        ["casual_chat"]='{"prompt": "How was your day?", "context": {}}'
        ["urgent_query"]='{"prompt": "What time is it?", "context": {"urgency": "high"}}'
    )
    
    declare -A expected_models=(
        ["vision_task"]="moondream"
        ["math_problem"]="phi3.5"
        ["casual_chat"]="llama3.2"
        ["urgent_query"]="gemma2"
    )
    
    for test_case in "${!selection_tests[@]}"; do
        response=$(api_call "POST" "/models/route" "${selection_tests[$test_case]}")
        
        if [ "$response" != "API_CALL_FAILED" ]; then
            http_code=$(echo "$response" | tail -n1)
            body=$(echo "$response" | head -n -1)
            
            if [ "$http_code" = "200" ]; then
                recommended_model=$(echo "$body" | jq -r '.recommended_model')
                expected="${expected_models[$test_case]}"
                
                if [[ "$recommended_model" =~ "$expected" ]]; then
                    log_test "Model Selection - $test_case" true "Correctly selected $recommended_model (expected: $expected)"
                else
                    log_test "Model Selection - $test_case" false "Selected $recommended_model, expected: $expected"
                fi
            else
                log_test "Model Selection - $test_case" false "HTTP $http_code"
            fi
        else
            log_test "Model Selection - $test_case" false "API call failed"
        fi
    done
}

# Generate comprehensive report
generate_report() {
    echo -e "\n${PURPLE}📋 Routing Framework Test Report${NC}"
    echo "==============================================================="
    echo "Test Execution Time: $(date)"
    echo "Total Tests: $TOTAL_TESTS"
    echo "Passed: $PASSED_TESTS"
    echo "Failed: $FAILED_TESTS"
    
    if [ $TOTAL_TESTS -gt 0 ]; then
        success_rate=$(echo "scale=2; $PASSED_TESTS * 100 / $TOTAL_TESTS" | bc)
        echo "Success Rate: $success_rate%"
    fi
    
    echo ""
    echo "📊 Performance Metrics:"
    echo "----------------------"
    
    # Routing times by task type
    if [ ${#ROUTING_TIMES[@]} -gt 0 ]; then
        echo "Routing Times by Task Type:"
        for task_type in "${!ROUTING_TIMES[@]}"; do
            echo "  - $task_type: ${ROUTING_TIMES[$task_type]}ms"
        done
    fi
    
    # Model selection distribution
    if [ ${#MODEL_SELECTIONS[@]} -gt 0 ]; then
        echo ""
        echo "Model Selection Distribution:"
        declare -A model_counts=()
        for task_type in "${!MODEL_SELECTIONS[@]}"; do
            model="${MODEL_SELECTIONS[$task_type]}"
            model_counts["$model"]=$((model_counts["$model"] + 1))
        done
        
        for model in "${!model_counts[@]}"; do
            echo "  - $model: ${model_counts[$model]} selections"
        done
    fi
    
    # A/B Test Results
    if [ ${#AB_TEST_RESULTS[@]} -gt 0 ]; then
        echo ""
        echo "A/B Testing Results:"
        for metric in "${!AB_TEST_RESULTS[@]}"; do
            echo "  - $metric: ${AB_TEST_RESULTS[$metric]}"
        done
    fi
    
    echo ""
    echo "🎯 Routing Framework Assessment:"
    echo "------------------------------"
    
    if (( $(echo "$success_rate >= 90" | bc -l) )); then
        echo -e "${GREEN}✅ EXCELLENT${NC} - Routing framework is production-ready"
    elif (( $(echo "$success_rate >= 75" | bc -l) )); then
        echo -e "${YELLOW}⚠️  GOOD${NC} - Minor optimizations needed"
    else
        echo -e "${RED}❌ NEEDS WORK${NC} - Significant issues detected"
    fi
    
    echo ""
    echo "💡 Recommendations:"
    echo "------------------"
    
    # Performance recommendations
    avg_routing_time=0
    if [ ${#ROUTING_TIMES[@]} -gt 0 ]; then
        total_time=0
        for time in "${ROUTING_TIMES[@]}"; do
            total_time=$((total_time + time))
        done
        avg_routing_time=$((total_time / ${#ROUTING_TIMES[@]}))
    fi
    
    if [ $avg_routing_time -gt 200 ]; then
        echo "- Optimize routing algorithm for faster decision making (current avg: ${avg_routing_time}ms)"
    fi
    
    if [ $FAILED_TESTS -gt 0 ]; then
        echo "- Review failed test cases and improve error handling"
    fi
    
    if [ ${#AB_TEST_RESULTS[@]} -eq 0 ]; then
        echo "- Implement A/B testing framework for continuous optimization"
    fi
    
    echo ""
    echo "📁 Detailed logs saved to: $LOG_FILE"
    echo "==============================================================="
}

# Main execution
main() {
    echo "Starting Routing Framework Test Suite..."
    echo ""
    
    # Initialize log file
    echo "AgentFlow Routing Framework Test Results - $(date)" > "$LOG_FILE"
    echo "===============================================================" >> "$LOG_FILE"
    
    # Run test suite
    check_prerequisites
    test_basic_routing
    test_performance_routing
    test_fallback_mechanism
    test_model_health
    test_ab_testing
    test_concurrent_routing
    test_routing_performance
    test_error_handling
    test_model_selection_logic
    
    # Generate final report
    generate_report
    
    # Exit with appropriate code
    if [ $FAILED_TESTS -eq 0 ]; then
        echo -e "\n${GREEN}🎉 All routing tests passed successfully!${NC}"
        exit 0
    else
        echo -e "\n${RED}⚠️  Some routing tests failed. Check the report above.${NC}"
        exit 1
    fi
}

# Run the test suite
main "$@"