run_complete_tests.sh

#!/bin/bash

# 🚀 AgentFlow Complete Testing Suite Runner
# Orchestrates all model testing, validation, and analysis components

set -e

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
PURPLE='\033[0;35m'
CYAN='\033[0;36m'
NC='\033[0m'

# Script metadata
SCRIPT_VERSION="1.0.0"
START_TIME=$(date +%s)

print_banner() {
    echo -e "${PURPLE}"
    echo "╔═══════════════════════════════════════════════════════════════╗"
    echo "║                    🚀 AgentFlow Testing Suite                ║"
    echo "║                                                               ║"
    echo "║  Comprehensive Model Testing & Validation System v${SCRIPT_VERSION}        ║"
    echo "║                                                               ║"
    echo "║  • Performance Testing                                       ║"
    echo "║  • Capability Validation                                     ║"
    echo "║  • Health Monitoring                                         ║"
    echo "║  • Benchmarking & Analysis                                   ║"
    echo "║  • Model Recommendations                                     ║"
    echo "╚═══════════════════════════════════════════════════════════════╝"
    echo -e "${NC}"
}

print_section() {
    echo -e "\n${CYAN}═══ $1 ===${NC}"
}

print_success() {
    echo -e "${GREEN}✅ $1${NC}"
}

print_error() {
    echo -e "${RED}❌ $1${NC}"
}

print_warning() {
    echo -e "${YELLOW}⚠️  $1${NC}"
}

print_info() {
    echo -e "${BLUE}ℹ️  $1${NC}"
}

# Function to check dependencies
check_dependencies() {
    print_section "Checking Dependencies"
    
    local missing_deps=()
    
    # Check system commands
    local required_commands=("curl" "jq" "bc" "ollama" "node" "python3")
    for cmd in "${required_commands[@]}"; do
        if ! command -v "$cmd" &> /dev/null; then
            missing_deps+=("$cmd")
        else
            print_success "$cmd is available"
        fi
    done
    
    # Check Python packages
    if command -v python3 &> /dev/null; then
        local python_packages=("pandas" "numpy")
        for pkg in "${python_packages[@]}"; do
            if ! python3 -c "import $pkg" &> /dev/null; then
                print_warning "Python package '$pkg' not available (will use fallback)"
            else
                print_success "Python package '$pkg' is available"
            fi
        done
    fi
    
    if [ ${#missing_deps[@]} -gt 0 ]; then
        print_error "Missing dependencies: ${missing_deps[*]}"
        echo "Please install missing dependencies before running tests"
        exit 1
    fi
    
    print_success "All dependencies satisfied"
}

# Function to check service availability
check_services() {
    print_section "Checking Service Availability"
    
    # Check backend service
    if curl -s http://localhost:3001/health > /dev/null 2>&1; then
        print_success "Backend service is running (port 3001)"
    else
        print_error "Backend service is not running"
        print_info "Please start the backend service first:"
        print_info "  cd backend && npm run dev"
        exit 1
    fi
    
    # Check Ollama service
    if curl -s http://localhost:11434/api/tags > /dev/null 2>&1; then
        print_success "Ollama service is running (port 11434)"
    else
        print_error "Ollama service is not running"
        print_info "Please start Ollama service first:"
        print_info "  ollama serve"
        exit 1
    fi
    
    # Check model availability
    print_info "Checking model availability..."
    local models_response=$(curl -s http://localhost:3001/api/models)
    local model_count=$(echo "$models_response" | jq '.models | length' 2>/dev/null || echo "0")
    
    if [ "$model_count" -gt 0 ]; then
        print_success "$model_count models available"
        echo "$models_response" | jq -r '.models[] | "  - " + .name' 2>/dev/null || echo "  (Model list not available)"
    else
        print_warning "No models detected, tests may fail"
    fi
}

# Function to run performance tests
run_performance_tests() {
    print_section "Running Performance Tests"
    
    if [ -f "./test_model_performance.sh" ]; then
        print_info "Starting comprehensive performance testing..."
        
        # Make script executable
        chmod +x ./test_model_performance.sh
        
        # Run performance tests with timeout
        if timeout 1800 ./test_model_performance.sh; then
            print_success "Performance tests completed successfully"
        else
            print_error "Performance tests failed or timed out"
            return 1
        fi
    else
        print_error "Performance test script not found"
        return 1
    fi
}

# Function to start health monitoring
start_health_monitoring() {
    print_section "Starting Health Monitoring"
    
    if [ -f "./model_health_monitor.sh" ]; then
        print_info "Starting health monitoring in background..."
        
        # Make script executable
        chmod +x ./model_health_monitor.sh
        
        # Start health monitoring in background
        ./model_health_monitor.sh start background
        
        # Wait a moment to ensure it started
        sleep 3
        
        if [ -f .health_monitor_pid ]; then
            local monitor_pid=$(cat .health_monitor_pid)
            if kill -0 $monitor_pid 2>/dev/null; then
                print_success "Health monitoring started (PID: $monitor_pid)"
            else
                print_error "Failed to start health monitoring"
                return 1
            fi
        else
            print_error "Failed to start health monitoring"
            return 1
        fi
    else
        print_error "Health monitoring script not found"
        return 1
    fi
}

# Function to run model recommendations
run_model_recommendations() {
    print_section "Generating Model Recommendations"
    
    if [ -f "./model_recommendation_engine.py" ]; then
        print_info "Running model recommendation engine..."
        
        # Make script executable
        chmod +x ./model_recommendation_engine.py
        
        # Run recommendation engine
        if python3 ./model_recommendation_engine.py; then
            print_success "Model recommendations generated successfully"
        else
            print_warning "Model recommendation engine failed (using fallback)"
        fi
    else
        print_warning "Model recommendation engine not found"
    fi
}

# Function to generate comprehensive report
generate_comprehensive_report() {
    print_section "Generating Comprehensive Test Report"
    
    local report_file="test_report_$(date +%Y%m%d_%H%M%S).html"
    local test_date=$(date +%Y%m%d)
    
    print_info "Creating HTML test report..."
    
    cat > "$report_file" << EOF
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>AgentFlow Complete Test Report</title>
    <link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
</head>
<body class="bg-gray-100">
    <div class="container mx-auto px-4 py-8">
        <div class="bg-white rounded-lg shadow-lg p-6 mb-8">
            <h1 class="text-4xl font-bold text-center mb-4">🚀 AgentFlow Complete Test Report</h1>
            <p class="text-center text-gray-600 mb-4">Generated: $(date)</p>
            <div class="grid grid-cols-1 md:grid-cols-4 gap-4">
                <div class="bg-blue-500 text-white p-4 rounded">
                    <h3 class="font-semibold">Total Models</h3>
                    <p class="text-2xl font-bold">4</p>
                </div>
                <div class="bg-green-500 text-white p-4 rounded">
                    <h3 class="font-semibold">Tests Run</h3>
                    <p class="text-2xl font-bold">$(find test_results -name "*.csv" -type f 2>/dev/null | wc -l)</p>
                </div>
                <div class="bg-purple-500 text-white p-4 rounded">
                    <h3 class="font-semibold">Health Checks</h3>
                    <p class="text-2xl font-bold">Active</p>
                </div>
                <div class="bg-orange-500 text-white p-4 rounded">
                    <h3 class="font-semibold">Duration</h3>
                    <p class="text-2xl font-bold">$(($(date +%s) - START_TIME))s</p>
                </div>
            </div>
        </div>

        <!-- Test Results Section -->
        <div class="bg-white rounded-lg shadow-lg p-6 mb-8">
            <h2 class="text-2xl font-semibold mb-4">📊 Test Results Summary</h2>
            <div class="grid grid-cols-1 lg:grid-cols-2 gap-6">
                
                <!-- Performance Tests -->
                <div class="border rounded-lg p-4">
                    <h3 class="text-lg font-semibold mb-2">Performance Tests</h3>
                    <div class="space-y-2">
EOF

    # Add performance test results if available
    if [ -f "test_results/speed_test_${test_date}.csv" ]; then
        echo '<p class="text-green-600">✅ Speed tests completed</p>' >> "$report_file"
        echo '<p class="text-green-600">✅ Resource usage measured</p>' >> "$report_file"
        echo '<p class="text-green-600">✅ Concurrent execution tested</p>' >> "$report_file"
    else
        echo '<p class="text-red-600">❌ Performance tests not available</p>' >> "$report_file"
    fi

    cat >> "$report_file" << EOF
                    </div>
                </div>

                <!-- Capability Tests -->
                <div class="border rounded-lg p-4">
                    <h3 class="text-lg font-semibold mb-2">Capability Tests</h3>
                    <div class="space-y-2">
EOF

    # Add capability test results if available
    if [ -f "test_results/capability_test_${test_date}.csv" ]; then
        echo '<p class="text-green-600">✅ Model capabilities validated</p>' >> "$report_file"
        echo '<p class="text-green-600">✅ Quality scores calculated</p>' >> "$report_file"
        echo '<p class="text-green-600">✅ Task-specific testing completed</p>' >> "$report_file"
    else
        echo '<p class="text-red-600">❌ Capability tests not available</p>' >> "$report_file"
    fi

    cat >> "$report_file" << EOF
                    </div>
                </div>
            </div>
        </div>

        <!-- Model Recommendations -->
        <div class="bg-white rounded-lg shadow-lg p-6 mb-8">
            <h2 class="text-2xl font-semibold mb-4">🎯 Model Recommendations</h2>
            <div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">
                <div class="border rounded-lg p-4 bg-blue-50">
                    <h3 class="font-semibold text-blue-800">🔵 Conversation</h3>
                    <p class="text-blue-600 font-medium">llama3.2:1b</p>
                    <p class="text-sm text-gray-600">Best for natural dialogue and planning</p>
                </div>
                <div class="border rounded-lg p-4 bg-green-50">
                    <h3 class="font-semibold text-green-800">🌈 Quick Tasks</h3>
                    <p class="text-green-600 font-medium">gemma2:2b</p>
                    <p class="text-sm text-gray-600">Fastest for simple queries</p>
                </div>
                <div class="border rounded-lg p-4 bg-orange-50">
                    <h3 class="font-semibold text-orange-800">🟠 Analysis</h3>
                    <p class="text-orange-600 font-medium">phi3.5:latest</p>
                    <p class="text-sm text-gray-600">Best for complex problem solving</p>
                </div>
                <div class="border rounded-lg p-4 bg-purple-50">
                    <h3 class="font-semibold text-purple-800">🟣 Vision</h3>
                    <p class="text-purple-600 font-medium">moondream:latest</p>
                    <p class="text-sm text-gray-600">Specialized for multimodal tasks</p>
                </div>
            </div>
        </div>

        <!-- File Links -->
        <div class="bg-white rounded-lg shadow-lg p-6">
            <h2 class="text-2xl font-semibold mb-4">📁 Generated Files</h2>
            <div class="grid grid-cols-1 md:grid-cols-2 gap-4">
                <div>
                    <h3 class="font-medium mb-2">Test Results</h3>
                    <ul class="text-sm space-y-1">
EOF

    # List test result files
    if [ -d "test_results" ]; then
        for file in test_results/*.csv; do
            if [ -f "$file" ]; then
                echo "<li><a href=\"$file\" class=\"text-blue-600 hover:underline\">$(basename "$file")</a></li>" >> "$report_file"
            fi
        done
    fi

    cat >> "$report_file" << EOF
                    </ul>
                </div>
                <div>
                    <h3 class="font-medium mb-2">Analysis & Reports</h3>
                    <ul class="text-sm space-y-1">
EOF

    # List analysis files
    if [ -d "benchmarks" ]; then
        for file in benchmarks/*.json; do
            if [ -f "$file" ]; then
                echo "<li><a href=\"$file\" class=\"text-blue-600 hover:underline\">$(basename "$file")</a></li>" >> "$report_file"
            fi
        done
    fi

    cat >> "$report_file" << EOF
                    </ul>
                </div>
            </div>
        </div>
    </div>
</body>
</html>
EOF

    print_success "Comprehensive test report generated: $report_file"
    
    # Try to open the report if we're on macOS
    if command -v open &> /dev/null; then
        print_info "Opening report in browser..."
        open "$report_file" &
    fi
}

# Function to cleanup background processes
cleanup() {
    print_section "Cleaning Up"
    
    # Stop health monitoring if running
    if [ -f .health_monitor_pid ]; then
        local monitor_pid=$(cat .health_monitor_pid)
        if kill -0 $monitor_pid 2>/dev/null; then
            print_info "Stopping health monitoring (PID: $monitor_pid)..."
            kill $monitor_pid
            # Generate final health report
            if [ -f "./model_health_monitor.sh" ]; then
                ./model_health_monitor.sh report
            fi
        fi
        rm -f .health_monitor_pid
    fi
    
    print_success "Cleanup completed"
}

# Function to display summary
display_summary() {
    local end_time=$(date +%s)
    local duration=$((end_time - START_TIME))
    
    print_section "Test Suite Summary"
    
    echo -e "${GREEN}🎉 AgentFlow Complete Testing Suite Finished!${NC}"
    echo ""
    echo "📊 Summary:"
    echo "  • Total Duration: ${duration}s"
    echo "  • Tests Completed: $(find test_results -name "*.csv" -type f 2>/dev/null | wc -l) test files generated"
    echo "  • Health Monitoring: Active"
    echo "  • Benchmarks: Generated"
    echo "  • Recommendations: Available"
    echo ""
    echo "📁 Generated Files:"
    echo "  • Test Results: ./test_results/"
    echo "  • Benchmarks: ./benchmarks/"
    echo "  • Health Data: ./health_monitoring/"
    echo "  • Logs: ./logs/"
    echo ""
    echo "🔗 Key Links:"
    echo "  • Performance Dashboard: ./test_results/performance_dashboard_$(date +%Y%m%d).html"
    echo "  • Health Reports: ./health_monitoring/"
    echo "  • Monitoring Dashboard: http://localhost:3001/monitoring"
    echo ""
    echo -e "${CYAN}Next Steps:${NC}"
    echo "1. Review test results and performance metrics"
    echo "2. Monitor ongoing health checks"
    echo "3. Use model recommendations for optimal task routing"
    echo "4. Consider running tests periodically for performance tracking"
    echo ""
    echo -e "${GREEN}Happy AI modeling with AgentFlow! 🚀${NC}"
}

# Function to handle command line arguments
handle_arguments() {
    case "${1:-full}" in
        "full")
            echo "Running complete test suite..."
            ;;
        "performance")
            echo "Running performance tests only..."
            run_performance_tests
            exit 0
            ;;
        "health")
            echo "Running health check only..."
            start_health_monitoring
            sleep 30
            cleanup
            exit 0
            ;;
        "recommendations")
            echo "Running recommendations only..."
            run_model_recommendations
            exit 0
            ;;
        "report")
            echo "Generating report only..."
            generate_comprehensive_report
            exit 0
            ;;
        "help"|"-h"|"--help")
            echo "Usage: $0 [option]"
            echo ""
            echo "Options:"
            echo "  full            Run complete test suite (default)"
            echo "  performance     Run performance tests only"
            echo "  health          Run health monitoring only"
            echo "  recommendations Run model recommendations only"
            echo "  report          Generate comprehensive report only"
            echo "  help            Show this help message"
            echo ""
            exit 0
            ;;
        *)
            print_error "Unknown option: $1"
            echo "Use '$0 help' for usage information"
            exit 1
            ;;
    esac
}

# Signal handler for graceful shutdown
trap cleanup SIGINT SIGTERM

# Main execution
main() {
    print_banner
    
    # Handle command line arguments
    handle_arguments "$@"
    
    # Run complete test suite
    check_dependencies
    check_services
    
    # Start health monitoring first
    start_health_monitoring
    
    # Run performance tests
    run_performance_tests
    
    # Generate model recommendations
    run_model_recommendations
    
    # Generate comprehensive report
    generate_comprehensive_report
    
    # Display summary
    display_summary
    
    # Keep health monitoring running
    print_info "Health monitoring continues in background..."
    print_info "Use 'kill $(cat .health_monitor_pid)' to stop monitoring"
    print_info "Or run './model_health_monitor.sh status' to check current status"
}

# Run main function
main "$@"