coverage_analysis.sh

#!/bin/bash

# Code Coverage Analysis for Property-Based Tests
# This script analyzes the code coverage achieved by property-based tests

set -e

echo "📊 Code Coverage Analysis for Property-Based Tests"
echo "================================================="

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Function to analyze test coverage for a specific file
analyze_test_coverage() {
    local test_file="$1"
    local target_file="$2"
    local description="$3"
    
    echo ""
    echo "🔍 Analyzing $description"
    echo "------------------------"
    
    if [ ! -f "$test_file" ]; then
        echo -e "${RED}❌ Test file not found: $test_file${NC}"
        return 1
    fi
    
    if [ ! -f "$target_file" ]; then
        echo -e "${RED}❌ Target file not found: $target_file${NC}"
        return 1
    fi
    
    # Count test functions
    local test_count=$(grep -c "fn test_" "$test_file" 2>/dev/null || echo "0")
    echo -e "${BLUE}📈 Test Functions: $test_count${NC}"
    
    # Count properties
    local prop_count=$(grep -c "Property [0-9]:" "$test_file" 2>/dev/null || echo "0")
    echo -e "${BLUE}🎯 Properties: $prop_count${NC}"
    
    # Count lines in target file
    local target_lines=$(wc -l < "$target_file")
    echo -e "${BLUE}📄 Target File Lines: $target_lines${NC}"
    
    # Count lines in test file
    local test_lines=$(wc -l < "$test_file")
    echo -e "${BLUE}🧪 Test File Lines: $test_lines${NC}"
    
    # Calculate test-to-code ratio
    local ratio=$(echo "scale=2; $test_lines * 100 / $target_lines" | bc 2>/dev/null || echo "0")
    echo -e "${BLUE}📊 Test-to-Code Ratio: ${ratio}%${NC}"
    
    # Check for comprehensive coverage areas
    local coverage_areas=0
    
    if grep -q "Core Properties" "$test_file"; then ((coverage_areas++)); fi
    if grep -q "Input Validation" "$test_file"; then ((coverage_areas++)); fi
    if grep -q "Error Handling" "$test_file"; then ((coverage_areas++)); fi
    if grep -q "Performance" "$test_file"; then ((coverage_areas++)); fi
    if grep -q "Consistency" "$test_file"; then ((coverage_areas++)); fi
    if grep -q "Edge Cases" "$test_file"; then ((coverage_areas++)); fi
    
    echo -e "${BLUE}🎯 Coverage Areas: $coverage_areas/6${NC}"
    
    # Check for mock models
    if grep -q "struct Mock" "$test_file"; then
        echo -e "${GREEN}✅ Mock Models: Present${NC}"
    else
        echo -e "${YELLOW}⚠️  Mock Models: Not found${NC}"
    fi
    
    # Check for proptest usage
    if grep -q "proptest!" "$test_file"; then
        echo -e "${GREEN}✅ Property-Based Testing: Present${NC}"
    else
        echo -e "${YELLOW}⚠️  Property-Based Testing: Not found${NC}"
    fi
    
    # Check for custom strategies
    if grep -q "mod strategies" "$test_file"; then
        echo -e "${GREEN}✅ Custom Strategies: Present${NC}"
    else
        echo -e "${YELLOW}⚠️  Custom Strategies: Not found${NC}"
    fi
    
    return 0
}

echo ""
echo "📋 Property-Based Test Coverage Analysis"
echo "========================================"

# Analyze each test file and its corresponding target
test_files=(
    "tests/clip_text_encode_property_tests.rs:comfy-nodes/src/conditioning.rs:CLIPTextEncode"
    "tests/vae_encode_property_tests.rs:comfy-nodes/src/vae.rs:VAEEncode"
    "tests/vae_decode_property_tests.rs:comfy-nodes/src/vae.rs:VAEDecode"
    "tests/tensor_compatibility_validation_tests.rs:comfy-nodes/src:vae.rs:Tensor Compatibility"
    "tests/vae_tiling_functionality_tests.rs:comfy-nodes/src/vae.rs:VAE Tiling"
    "tests/vae_half_precision_tests.rs:comfy-nodes/src/vae.rs:VAE Half-Precision"
    "tests/vae_comprehensive_error_handling_tests.rs:comfy-nodes/src/vae.rs:VAE Error Handling"
)

total_tests=0
total_properties=0
total_coverage_areas=0
files_analyzed=0

for test_info in "${test_files[@]}"; do
    IFS=':' read -r test_file target_file description <<< "$test_info"
    
    if analyze_test_coverage "$test_file" "$target_file" "$description"; then
        ((files_analyzed++))
        
        # Extract counts for summary
        test_count=$(grep -c "fn test_" "$test_file" 2>/dev/null || echo "0")
        prop_count=$(grep -c "Property [0-9]:" "$test_file" 2>/dev/null || echo "0")
        
        total_tests=$((total_tests + test_count))
        total_properties=$((total_properties + prop_count))
        
        # Count coverage areas
        coverage_areas=0
        if grep -q "Core Properties" "$test_file"; then ((coverage_areas++)); fi
        if grep -q "Input Validation" "$test_file"; then ((coverage_areas++)); fi
        if grep -q "Error Handling" "$test_file"; then ((coverage_areas++)); fi
        if grep -q "Performance" "$test_file"; then ((coverage_areas++)); fi
        if grep -q "Consistency" "$test_file"; then ((coverage_areas++)); fi
        if grep -q "Edge Cases" "$test_file"; then ((coverage_areas++)); fi
        
        total_coverage_areas=$((total_coverage_areas + coverage_areas))
    fi
done

echo ""
echo "📊 Overall Coverage Summary"
echo "=========================="
echo -e "${BLUE}Files Analyzed: $files_analyzed${NC}"
echo -e "${BLUE}Total Test Functions: $total_tests${NC}"
echo -e "${BLUE}Total Properties: $total_properties${NC}"
echo -e "${BLUE}Total Coverage Areas: $total_coverage_areas${NC}"

# Calculate coverage percentage
if [ $files_analyzed -gt 0 ]; then
    avg_coverage_areas=$(echo "scale=1; $total_coverage_areas / $files_analyzed" | bc 2>/dev/null || echo "0")
    echo -e "${BLUE}Average Coverage Areas per File: ${avg_coverage_areas}/6${NC}"
fi

echo ""
echo "🎯 Coverage Quality Assessment"
echo "============================="

# Assess coverage quality
if [ $total_tests -ge 70 ]; then
    echo -e "${GREEN}✅ Test Function Coverage: EXCELLENT ($total_tests functions)${NC}"
elif [ $total_tests -ge 50 ]; then
    echo -e "${YELLOW}⚠️  Test Function Coverage: GOOD ($total_tests functions)${NC}"
else
    echo -e "${RED}❌ Test Function Coverage: NEEDS IMPROVEMENT ($total_tests functions)${NC}"
fi

if [ $total_properties -ge 80 ]; then
    echo -e "${GREEN}✅ Property Coverage: EXCELLENT ($total_properties properties)${NC}"
elif [ $total_properties -ge 60 ]; then
    echo -e "${YELLOW}⚠️  Property Coverage: GOOD ($total_properties properties)${NC}"
else
    echo -e "${RED}❌ Property Coverage: NEEDS IMPROVEMENT ($total_properties properties)${NC}"
fi

if [ $total_coverage_areas -ge 35 ]; then
    echo -e "${GREEN}✅ Coverage Area Coverage: EXCELLENT ($total_coverage_areas areas)${NC}"
elif [ $total_coverage_areas -ge 25 ]; then
    echo -e "${YELLOW}⚠️  Coverage Area Coverage: GOOD ($total_coverage_areas areas)${NC}"
else
    echo -e "${RED}❌ Coverage Area Coverage: NEEDS IMPROVEMENT ($total_coverage_areas areas)${NC}"
fi

echo ""
echo "🔍 Detailed Coverage Analysis"
echo "============================="

# Check for specific coverage patterns
echo "📋 Coverage Pattern Analysis:"

# Check for property-based testing usage
prop_test_files=$(grep -l "proptest!" tests/*property* tests/*validation* tests/*tiling* tests/*half* tests/*error* 2>/dev/null | wc -l)
echo -e "${BLUE}Property-Based Test Files: $prop_test_files${NC}"

# Check for mock model usage
mock_files=$(grep -l "struct Mock" tests/*property* tests/*validation* tests/*tiling* tests/*half* tests/*error* 2>/dev/null | wc -l)
echo -e "${BLUE}Files with Mock Models: $mock_files${NC}"

# Check for custom strategies
strategy_files=$(grep -l "mod strategies" tests/*property* tests/*validation* tests/*tiling* tests/*half* tests/*error* 2>/dev/null | wc -l)
echo -e "${BLUE}Files with Custom Strategies: $strategy_files${NC}"

# Check for edge case testing
edge_case_files=$(grep -l "edge_case" tests/*property* tests/*validation* tests/*tiling* tests/*half* tests/*error* 2>/dev/null | wc -l)
echo -e "${BLUE}Files with Edge Case Testing: $edge_case_files${NC}"

echo ""
echo "📈 Coverage Recommendations"
echo "=========================="

if [ $total_tests -lt 70 ]; then
    echo -e "${YELLOW}⚠️  Consider adding more test functions${NC}"
fi

if [ $total_properties -lt 80 ]; then
    echo -e "${YELLOW}⚠️  Consider adding more property validations${NC}"
fi

if [ $total_coverage_areas -lt 35 ]; then
    echo -e "${YELLOW}⚠️  Consider adding more coverage areas${NC}"
fi

if [ $prop_test_files -lt 7 ]; then
    echo -e "${YELLOW}⚠️  Consider adding more property-based test files${NC}"
fi

if [ $mock_files -lt 7 ]; then
    echo -e "${YELLOW}⚠️  Consider adding more mock models${NC}"
fi

echo ""
echo "🏁 Coverage Analysis Complete"
echo "============================"

# Final assessment
if [ $total_tests -ge 70 ] && [ $total_properties -ge 80 ] && [ $total_coverage_areas -ge 35 ]; then
    echo -e "${GREEN}✅ Overall Coverage: EXCELLENT${NC}"
    echo -e "${GREEN}✅ Property-based tests provide comprehensive coverage${NC}"
    echo -e "${GREEN}✅ Ready for production use${NC}"
elif [ $total_tests -ge 50 ] && [ $total_properties -ge 60 ] && [ $total_coverage_areas -ge 25 ]; then
    echo -e "${YELLOW}⚠️  Overall Coverage: GOOD${NC}"
    echo -e "${YELLOW}⚠️  Consider adding more tests for better coverage${NC}"
else
    echo -e "${RED}❌ Overall Coverage: NEEDS IMPROVEMENT${NC}"
    echo -e "${RED}❌ Consider significantly expanding test coverage${NC}"
fi