model_recommendation_engine.py

#!/usr/bin/env python3

"""
🤖 AgentFlow Model Recommendation Engine
Analyzes performance data and provides intelligent model recommendations
"""

import json
import csv
import os
import datetime
import numpy as np
import pandas as pd
from typing import Dict, List, Tuple, Optional
from dataclasses import dataclass
from pathlib import Path
import logging

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

@dataclass
class ModelPerformance:
    """Data class for model performance metrics"""
    model_id: str
    avg_response_time: float
    quality_score: float
    success_rate: float
    resource_usage: float
    specialty: str
    failure_count: int
    last_updated: datetime.datetime

@dataclass
class TaskRequirement:
    """Data class for task requirements"""
    task_type: str
    priority_speed: int  # 1-10 scale
    priority_quality: int  # 1-10 scale
    priority_reliability: int  # 1-10 scale
    max_response_time: float
    min_quality_score: float

class ModelRecommendationEngine:
    """
    Intelligent model recommendation engine for AgentFlow
    """
    
    def __init__(self, data_dir: str = "."):
        self.data_dir = Path(data_dir)
        self.test_results_dir = self.data_dir / "test_results"
        self.benchmarks_dir = self.data_dir / "benchmarks"
        self.health_dir = self.data_dir / "health_monitoring"
        
        # Model configurations
        self.models = {
            "llama3.2:1b": {
                "company": "Meta",
                "specialty": "Conversation & Planning",
                "size": "1.3GB",
                "optimal_for": ["conversations", "planning", "creative_tasks"],
                "strengths": ["Natural dialogue", "Context understanding", "Creative writing"],
                "weaknesses": ["Complex analysis", "Mathematical computation"]
            },
            "gemma2:2b": {
                "company": "Google", 
                "specialty": "Speed & Efficiency",
                "size": "1.6GB",
                "optimal_for": ["quick_responses", "simple_queries", "real_time"],
                "strengths": ["Fast responses", "Efficient processing", "Simple tasks"],
                "weaknesses": ["Complex reasoning", "Long-form content"]
            },
            "phi3.5:latest": {
                "company": "Microsoft",
                "specialty": "Analysis & Problem Solving",
                "size": "2.2GB", 
                "optimal_for": ["analysis", "reasoning", "technical_tasks", "research"],
                "strengths": ["Complex analysis", "Mathematical reasoning", "Technical accuracy"],
                "weaknesses": ["Response speed", "Simple queries"]
            },
            "moondream:latest": {
                "company": "Independent",
                "specialty": "Vision & Multimodal",
                "size": "829MB",
                "optimal_for": ["image_analysis", "ocr", "visual_qa", "multimodal"],
                "strengths": ["Image understanding", "Visual processing", "Multimodal tasks"],
                "weaknesses": ["Text-only tasks", "Speed for complex reasoning"]
            }
        }
        
        # Task type mappings
        self.task_requirements = {
            "conversation": TaskRequirement("conversation", 8, 7, 9, 3.0, 7.0),
            "quick_answer": TaskRequirement("quick_answer", 10, 6, 8, 2.0, 6.0),
            "analysis": TaskRequirement("analysis", 4, 10, 7, 15.0, 8.5),
            "planning": TaskRequirement("planning", 6, 8, 8, 5.0, 7.5),
            "creative": TaskRequirement("creative", 5, 9, 7, 8.0, 8.0),
            "math": TaskRequirement("math", 6, 10, 9, 10.0, 9.0),
            "vision": TaskRequirement("vision", 5, 8, 8, 10.0, 7.5),
            "research": TaskRequirement("research", 3, 10, 8, 20.0, 9.0),
            "simple_task": TaskRequirement("simple_task", 9, 6, 9, 2.5, 6.5)
        }
        
        self.performance_data = {}
        self.load_performance_data()
    
    def load_performance_data(self) -> None:
        """Load performance data from test results and benchmarks"""
        logger.info("Loading performance data...")
        
        try:
            # Load from latest benchmark file
            latest_benchmark = self._get_latest_benchmark()
            if latest_benchmark:
                self._load_benchmark_data(latest_benchmark)
            
            # Load from test result files
            self._load_test_results()
            
            # Load health monitoring data
            self._load_health_data()
            
            logger.info(f"Loaded performance data for {len(self.performance_data)} models")
            
        except Exception as e:
            logger.error(f"Error loading performance data: {e}")
            self._create_fallback_data()
    
    def _get_latest_benchmark(self) -> Optional[Path]:
        """Get the most recent benchmark file"""
        if not self.benchmarks_dir.exists():
            return None
        
        benchmark_files = list(self.benchmarks_dir.glob("benchmark_*.json"))
        if not benchmark_files:
            return None
        
        return max(benchmark_files, key=lambda x: x.stat().st_mtime)
    
    def _load_benchmark_data(self, benchmark_file: Path) -> None:
        """Load data from benchmark JSON file"""
        try:
            with open(benchmark_file, 'r') as f:
                data = json.load(f)
            
            for model_id, model_data in data.get("model_performance", {}).items():
                self.performance_data[model_id] = ModelPerformance(
                    model_id=model_id,
                    avg_response_time=float(model_data.get("average_response_time", 5.0)),
                    quality_score=float(model_data.get("average_quality_score", 7.0)),
                    success_rate=0.9,  # Default
                    resource_usage=50.0,  # Default
                    specialty=model_data.get("specialty", "General"),
                    failure_count=0,
                    last_updated=datetime.datetime.now()
                )
                
        except Exception as e:
            logger.warning(f"Error loading benchmark data: {e}")
    
    def _load_test_results(self) -> None:
        """Load data from CSV test result files"""
        try:
            # Load speed test results
            speed_file = self._get_latest_csv_file("speed_test_*.csv")
            if speed_file:
                self._process_speed_data(speed_file)
            
            # Load capability test results
            capability_file = self._get_latest_csv_file("capability_test_*.csv")
            if capability_file:
                self._process_capability_data(capability_file)
                
        except Exception as e:
            logger.warning(f"Error loading test results: {e}")
    
    def _get_latest_csv_file(self, pattern: str) -> Optional[Path]:
        """Get the most recent CSV file matching pattern"""
        if not self.test_results_dir.exists():
            return None
        
        csv_files = list(self.test_results_dir.glob(pattern))
        if not csv_files:
            return None
            
        return max(csv_files, key=lambda x: x.stat().st_mtime)
    
    def _process_speed_data(self, speed_file: Path) -> None:
        """Process speed test CSV data"""
        try:
            df = pd.read_csv(speed_file)
            
            for model in df['Model'].unique():
                model_data = df[df['Model'] == model]
                successful_tests = model_data[model_data['Success'] == 'success']
                
                if len(successful_tests) > 0:
                    avg_time = successful_tests['Response_Time'].mean()
                    success_rate = len(successful_tests) / len(model_data)
                    
                    if model in self.performance_data:
                        self.performance_data[model].avg_response_time = avg_time
                        self.performance_data[model].success_rate = success_rate
                    
        except Exception as e:
            logger.warning(f"Error processing speed data: {e}")
    
    def _process_capability_data(self, capability_file: Path) -> None:
        """Process capability test CSV data"""
        try:
            df = pd.read_csv(capability_file)
            
            for model in df['Model'].unique():
                model_data = df[df['Model'] == model]
                successful_tests = model_data[model_data['Success'] == 'success']
                
                if len(successful_tests) > 0:
                    avg_quality = successful_tests['Quality_Score'].mean()
                    
                    if model in self.performance_data:
                        self.performance_data[model].quality_score = avg_quality
                    
        except Exception as e:
            logger.warning(f"Error processing capability data: {e}")
    
    def _load_health_data(self) -> None:
        """Load health monitoring data"""
        try:
            health_file = self._get_latest_csv_file("health_log_*.csv", self.health_dir)
            if health_file:
                df = pd.read_csv(health_file)
                
                for model in df['Model'].unique():
                    model_data = df[df['Model'] == model]
                    latest_entry = model_data.iloc[-1]  # Get most recent
                    
                    if model in self.performance_data:
                        self.performance_data[model].failure_count = latest_entry.get('FailureCount', 0)
                        
        except Exception as e:
            logger.warning(f"Error loading health data: {e}")
    
    def _create_fallback_data(self) -> None:
        """Create fallback performance data if no files are available"""
        logger.info("Creating fallback performance data")
        
        fallback_data = {
            "llama3.2:1b": (2.5, 8.0, 0.85),
            "gemma2:2b": (1.8, 7.0, 0.90),
            "phi3.5:latest": (4.2, 9.0, 0.80),
            "moondream:latest": (3.1, 7.5, 0.82)
        }
        
        for model_id, (response_time, quality, success_rate) in fallback_data.items():
            self.performance_data[model_id] = ModelPerformance(
                model_id=model_id,
                avg_response_time=response_time,
                quality_score=quality,
                success_rate=success_rate,
                resource_usage=45.0,
                specialty=self.models.get(model_id, {}).get("specialty", "General"),
                failure_count=0,
                last_updated=datetime.datetime.now()
            )
    
    def calculate_model_score(self, model_id: str, task_type: str) -> float:
        """Calculate a composite score for a model on a specific task"""
        if model_id not in self.performance_data:
            return 0.0
        
        perf = self.performance_data[model_id]
        req = self.task_requirements.get(task_type, self.task_requirements["conversation"])
        
        # Speed score (inverted - lower time is better)
        speed_score = min(10, max(1, 10 - (perf.avg_response_time / req.max_response_time) * 5))
        
        # Quality score (direct)
        quality_score = min(10, max(1, perf.quality_score))
        
        # Reliability score (based on success rate and failure count)
        reliability_score = min(10, max(1, perf.success_rate * 10 - perf.failure_count))
        
        # Specialty bonus (if model specialty matches task)
        specialty_bonus = 1.0
        model_config = self.models.get(model_id, {})
        if task_type in model_config.get("optimal_for", []):
            specialty_bonus = 1.2
        
        # Weighted composite score
        composite = (
            speed_score * (req.priority_speed / 10) +
            quality_score * (req.priority_quality / 10) +
            reliability_score * (req.priority_reliability / 10)
        ) * specialty_bonus
        
        return round(composite, 2)
    
    def recommend_model(self, task_type: str, context: Dict = None) -> Dict:
        """Get the best model recommendation for a specific task"""
        if not self.performance_data:
            return self._get_fallback_recommendation(task_type)
        
        recommendations = []
        
        for model_id in self.performance_data.keys():
            score = self.calculate_model_score(model_id, task_type)
            perf = self.performance_data[model_id]
            model_config = self.models.get(model_id, {})
            
            recommendations.append({
                "model_id": model_id,
                "score": score,
                "company": model_config.get("company", "Unknown"),
                "specialty": model_config.get("specialty", "General"),
                "avg_response_time": perf.avg_response_time,
                "quality_score": perf.quality_score,
                "success_rate": perf.success_rate,
                "strengths": model_config.get("strengths", []),
                "weaknesses": model_config.get("weaknesses", []),
                "reasoning": self._generate_reasoning(model_id, task_type, score)
            })
        
        # Sort by score (descending)
        recommendations.sort(key=lambda x: x["score"], reverse=True)
        
        return {
            "task_type": task_type,
            "timestamp": datetime.datetime.now().isoformat(),
            "primary_recommendation": recommendations[0] if recommendations else None,
            "alternatives": recommendations[1:3] if len(recommendations) > 1 else [],
            "all_options": recommendations,
            "task_requirements": {
                "max_response_time": self.task_requirements[task_type].max_response_time,
                "min_quality_score": self.task_requirements[task_type].min_quality_score,
                "priorities": {
                    "speed": self.task_requirements[task_type].priority_speed,
                    "quality": self.task_requirements[task_type].priority_quality,
                    "reliability": self.task_requirements[task_type].priority_reliability
                }
            }
        }
    
    def _generate_reasoning(self, model_id: str, task_type: str, score: float) -> str:
        """Generate human-readable reasoning for the recommendation"""
        perf = self.performance_data[model_id]
        model_config = self.models.get(model_id, {})
        
        reasons = []
        
        # Performance-based reasoning
        if perf.avg_response_time < 2.0:
            reasons.append("Very fast response time")
        elif perf.avg_response_time > 5.0:
            reasons.append("Slower but thorough processing")
        
        if perf.quality_score >= 8.5:
            reasons.append("High quality responses")
        elif perf.quality_score < 7.0:
            reasons.append("Quick but basic responses")
        
        # Specialty-based reasoning
        if task_type in model_config.get("optimal_for", []):
            reasons.append(f"Specialized for {task_type} tasks")
        
        # Score-based reasoning
        if score >= 8.0:
            reasons.append("Excellent match for requirements")
        elif score >= 6.0:
            reasons.append("Good fit with some trade-offs")
        else:
            reasons.append("May not be optimal for this task")
        
        return "; ".join(reasons)
    
    def _get_fallback_recommendation(self, task_type: str) -> Dict:
        """Provide fallback recommendation when no performance data is available"""
        fallback_mapping = {
            "conversation": "llama3.2:1b",
            "quick_answer": "gemma2:2b",
            "analysis": "phi3.5:latest",
            "math": "phi3.5:latest",
            "vision": "moondream:latest",
            "planning": "llama3.2:1b",
            "creative": "llama3.2:1b",
            "research": "phi3.5:latest",
            "simple_task": "gemma2:2b"
        }
        
        recommended_model = fallback_mapping.get(task_type, "llama3.2:1b")
        
        return {
            "task_type": task_type,
            "timestamp": datetime.datetime.now().isoformat(),
            "primary_recommendation": {
                "model_id": recommended_model,
                "score": 7.0,
                "reasoning": "Fallback recommendation based on model specialty"
            },
            "note": "Based on default model specialties (no performance data available)"
        }
    
    def get_performance_summary(self) -> Dict:
        """Get a comprehensive performance summary of all models"""
        summary = {
            "timestamp": datetime.datetime.now().isoformat(),
            "total_models": len(self.performance_data),
            "models": {}
        }
        
        for model_id, perf in self.performance_data.items():
            model_config = self.models.get(model_id, {})
            
            summary["models"][model_id] = {
                "company": model_config.get("company", "Unknown"),
                "specialty": model_config.get("specialty", "General"),
                "size": model_config.get("size", "Unknown"),
                "performance": {
                    "avg_response_time": perf.avg_response_time,
                    "quality_score": perf.quality_score,
                    "success_rate": perf.success_rate,
                    "failure_count": perf.failure_count
                },
                "strengths": model_config.get("strengths", []),
                "weaknesses": model_config.get("weaknesses", []),
                "optimal_for": model_config.get("optimal_for", []),
                "last_updated": perf.last_updated.isoformat()
            }
        
        return summary
    
    def generate_recommendation_matrix(self) -> Dict:
        """Generate a comprehensive recommendation matrix for all task types"""
        matrix = {
            "timestamp": datetime.datetime.now().isoformat(),
            "task_recommendations": {}
        }
        
        for task_type in self.task_requirements.keys():
            recommendation = self.recommend_model(task_type)
            matrix["task_recommendations"][task_type] = {
                "primary": recommendation["primary_recommendation"]["model_id"] if recommendation["primary_recommendation"] else None,
                "score": recommendation["primary_recommendation"]["score"] if recommendation["primary_recommendation"] else 0,
                "alternatives": [alt["model_id"] for alt in recommendation["alternatives"]]
            }
        
        return matrix
    
    def export_recommendations(self, output_file: str = None) -> str:
        """Export comprehensive recommendations to JSON file"""
        if output_file is None:
            output_file = f"model_recommendations_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
        
        recommendations = {
            "performance_summary": self.get_performance_summary(),
            "recommendation_matrix": self.generate_recommendation_matrix(),
            "task_specific_recommendations": {}
        }
        
        # Generate detailed recommendations for each task type
        for task_type in self.task_requirements.keys():
            recommendations["task_specific_recommendations"][task_type] = self.recommend_model(task_type)
        
        output_path = Path(output_file)
        with open(output_path, 'w') as f:
            json.dump(recommendations, f, indent=2, default=str)
        
        logger.info(f"Recommendations exported to {output_path}")
        return str(output_path)

def main():
    """Main function for command-line usage"""
    engine = ModelRecommendationEngine()
    
    # Generate and display recommendations
    print("🤖 AgentFlow Model Recommendation Engine")
    print("=" * 50)
    
    # Show performance summary
    summary = engine.get_performance_summary()
    print(f"\n📊 Performance Summary ({summary['total_models']} models loaded)")
    
    for model_id, data in summary["models"].items():
        print(f"\n{model_id} ({data['company']})")
        print(f"  Specialty: {data['specialty']}")
        print(f"  Response Time: {data['performance']['avg_response_time']:.2f}s")
        print(f"  Quality Score: {data['performance']['quality_score']:.1f}/10")
        print(f"  Success Rate: {data['performance']['success_rate']:.1%}")
    
    # Show recommendation matrix
    print("\n🎯 Recommendation Matrix")
    matrix = engine.generate_recommendation_matrix()
    
    for task_type, rec in matrix["task_recommendations"].items():
        print(f"\n{task_type.upper()}:")
        print(f"  Primary: {rec['primary']} (score: {rec['score']:.1f})")
        if rec["alternatives"]:
            print(f"  Alternatives: {', '.join(rec['alternatives'])}")
    
    # Export detailed recommendations
    output_file = engine.export_recommendations()
    print(f"\n📄 Detailed recommendations exported to: {output_file}")

if __name__ == "__main__":
    main()