evaluate.py

import os
from langsmith import Client
from langsmith.evaluation import evaluate  # <--- New standard evaluation import
from implementation.answer_advanced import answer_question

# Initialize the LangSmith Client
client = Client()

def main():
    dataset_name = "Insurellm Test Dataset"
    
    # 1. Create a Test Dataset in LangSmith if it doesn't exist yet
    if not client.has_dataset(dataset_name=dataset_name):
        dataset = client.create_dataset(
            dataset_name=dataset_name, 
            description="Evaluation dataset for Insurellm QA verification."
        )
        
        # Add basic benchmark question-and-answer examples
        client.create_examples(
            dataset_id=dataset.id,
            examples=[
                {
                    "inputs": {"question": "What is Insurellm?"},
                    "outputs": {"reference": "Insurellm is an expert insurance assistant platform."}
                },
                {
                    "inputs": {"question": "How do I file a policy claim?"},
                    "outputs": {"reference": "Claims must be submitted through the secure company portal."}
                }
            ]
        )
        print(f"🎉 Created dataset: '{dataset_name}' with test examples.")
    else:
        print(f"📚 Dataset '{dataset_name}' already exists. Using it.")

    # 2. Define our target runner wrapper function
    def evaluate_target(inputs: dict) -> dict:
        user_question = inputs["question"]
        
        # Run your exact local pipeline (Rewriting + Retrieval + Rerank + Ollama)
        answer, retrieved_docs = answer_question(question=user_question, history=[])
        
        # Package output for LangSmith evaluation formats
        return {
            "output": answer,
            "context": [doc.page_content for doc in retrieved_docs]
        }

    print("🚀 Running evaluation pipeline locally against LangSmith bench...")
    
    # 3. Use the new standard evaluate function
    experiment_results = evaluate(
        evaluate_target,
        data=dataset_name,
        experiment_prefix="Llama3.2-Advanced-RAG-Eval",
    )
    
    print("✅ Evaluation complete! Open your LangSmith Dashboard web UI to view detailed trace metrics.")

if __name__ == "__main__":
    main()