analyze_coverage_alerts.py

#!/usr/bin/env python3

# DISCLAIMER
# This scipt was largely written by Claude Code and only checked and adapted by the authors.

import sys
import json
import re
from typing import List, Dict, Tuple
from enum import Enum
from collections import defaultdict, Counter


class CommentType(Enum):
    FUZZER_UNAVAILABLE = "fuzzer_unavailable"
    TIMEOUT_ERROR = "timeout_error"
    MEMORY_ERROR = "memory_error"
    COVERAGE_DEGRADED = "coverage_degraded"
    COVERAGE_DEGRADED_DETAIL = "coverage_degraded_detail"
    DEADLY_SIGNAL = "deadly_signal"
    TARGET_EXITED = "target_exited"
    BINARY_FILE_MATCH = "binary_file_match"
    UNKNOWN = "unknown"


# Define regex patterns for each comment type
patterns = {
    CommentType.FUZZER_UNAVAILABLE: re.compile(r"Fuzzer no longer available!"),
    CommentType.TIMEOUT_ERROR: re.compile(r"Coverage error: timeout after \d+ seconds"),
    CommentType.MEMORY_ERROR: re.compile(
        r"Coverage error: out-of-memory \(used: \d+Mb; limit: \d+Mb\)"
    ),
    CommentType.COVERAGE_DEGRADED: re.compile(r"Coverage has degraded!"),
    CommentType.COVERAGE_DEGRADED_DETAIL: re.compile(
        r"Degraded from \d{4}-\d{2}-\d{2} \(\d+\.\d+%\)\s+to \d{4}-\d{2}-\d{2} \(\d+\.\d+%\)!",
        re.MULTILINE,
    ),
    CommentType.DEADLY_SIGNAL: re.compile(r"Coverage error: deadly signal"),
    CommentType.TARGET_EXITED: re.compile(r"Coverage error: fuzz target exited"),
    CommentType.BINARY_FILE_MATCH: re.compile(
        r"Coverage error: Binary file .+ matches"
    ),
}

# Define which comment types should be combined for reporting
COVERAGE_ERROR_TYPES = {
    CommentType.TIMEOUT_ERROR,
    CommentType.MEMORY_ERROR,
    CommentType.DEADLY_SIGNAL,
    CommentType.TARGET_EXITED,
    CommentType.BINARY_FILE_MATCH,
}


def classify_comment(comment: str) -> set[CommentType]:
    """
    Classify a single comment based on regex patterns.
    Args:
        comment (str): The comment text to classify
    Returns:
        set[CommentType]: Set of all unique comment types that match, or {UNKNOWN} if any part is unmatched
    """
    comment = comment.strip()
    if not comment:
        return set()

    matches = set()
    # Normalize whitespace for processing - collapse multiple spaces/newlines to single spaces
    normalized_comment = re.sub(r"\s+", " ", comment)
    remaining_text = normalized_comment

    # Process each pattern and remove matched portions
    for comment_type, pattern in patterns.items():
        # Also normalize the pattern matching to handle whitespace consistently
        pattern_for_normalized = pattern
        if comment_type == CommentType.COVERAGE_DEGRADED_DETAIL:
            # Special handling for degraded detail pattern with flexible whitespace
            pattern_for_normalized = re.compile(
                r"Degraded from \d{4}-\d{2}-\d{2} \(\d+\.\d+%\) to \d{4}-\d{2}-\d{2} \(\d+\.\d+%\)!",
                re.IGNORECASE,
            )

        # Find all matches for this pattern
        pattern_matches = list(pattern_for_normalized.finditer(remaining_text))
        if pattern_matches:
            matches.add(comment_type)
            # Remove all matched text (in reverse order to maintain positions)
            for match in reversed(pattern_matches):
                remaining_text = (
                    remaining_text[: match.start()] + remaining_text[match.end() :]
                )

    # Clean up remaining text (remove extra whitespace)
    remaining_text = remaining_text.strip()

    # If there's unmatched content, classify as unknown
    if remaining_text:
        return {CommentType.UNKNOWN}

    return matches


def combine_comment_counts(comment_counts: Counter) -> Counter:
    """
    Combine coverage error types for reporting while preserving individual counts.
    Args:
        comment_counts (Counter): Original comment type counts
    Returns:
        Counter: Modified counts with coverage errors combined
    """
    combined_counts = Counter(comment_counts)

    # Calculate total coverage errors
    coverage_error_total = sum(
        combined_counts[error_type]
        for error_type in COVERAGE_ERROR_TYPES
        if error_type in combined_counts
    )

    # Remove individual coverage error types
    for error_type in COVERAGE_ERROR_TYPES:
        if error_type in combined_counts:
            del combined_counts[error_type]

    # Add combined coverage error count if there were any
    if coverage_error_total > 0:
        combined_counts["coverage_errors_combined"] = coverage_error_total

    return combined_counts


def analyze_fuzzer_data(filename):
    """
    Analyze fuzzer data and provide comprehensive statistics.
    Returns:
        dict: Dictionary containing all statistics
    """
    with open(filename, "r") as f:
        data = json.load(f)

    # Initialize counters
    stats = {
        "total_projects": len(data),
        "total_fuzzers": 0,
        "fuzzers_over_30_pct": 0,
        "fuzzers_over_30_pct_with_alert": 0,
        "fuzzers_over_30_pct_without_alert": 0,
        "fuzzers_under_30_pct": 0,
        "fuzzers_under_30_pct_with_alert": 0,
        "fuzzers_under_30_pct_without_alert": 0,
        "total_with_alerts": 0,
        "total_without_alerts": 0,
        "comment_type_counts": Counter(),
        "unknown_comments": [],
        # Project-level aggregate stats
        "projects_with_alerts": 0,
        "projects_without_alerts": 0,
        "projects_with_over_30_pct_coverage": 0,
        "projects_all_fuzzers_over_30_pct": 0,
        "projects_all_fuzzers_under_30_pct": 0,
        "projects_with_comments": 0,
    }

    for project in data:
        project_name = project.get("project_name", "Unknown")
        project_fuzzers = project.get("fuzzers", [])

        if not project_fuzzers:
            continue

        # Track project-level characteristics
        project_has_alerts = False
        project_has_over_30_pct = False
        project_all_over_30_pct = True
        project_has_comments = False

        for fuzzer in project_fuzzers:
            stats["total_fuzzers"] += 1

            coverage_pct = fuzzer.get("coverage_percentage", 0)
            has_alert = fuzzer.get("has_alert_danger", False)
            comments = fuzzer.get("comments", "")

            # Track project-level flags
            if has_alert:
                project_has_alerts = True
            if coverage_pct > 30:
                project_has_over_30_pct = True
            else:
                project_all_over_30_pct = False
            if comments:
                project_has_comments = True

            # Count alerts
            if has_alert:
                stats["total_with_alerts"] += 1
            else:
                stats["total_without_alerts"] += 1

            # Categorize by coverage and alert status
            if coverage_pct > 30:
                stats["fuzzers_over_30_pct"] += 1
                if has_alert:
                    stats["fuzzers_over_30_pct_with_alert"] += 1
                else:
                    stats["fuzzers_over_30_pct_without_alert"] += 1
            else:
                stats["fuzzers_under_30_pct"] += 1
                if has_alert:
                    stats["fuzzers_under_30_pct_with_alert"] += 1
                else:
                    stats["fuzzers_under_30_pct_without_alert"] += 1

            # Classify comments if they exist
            if comments:
                classified = classify_comment(comments)
                for comment_type in classified:
                    stats["comment_type_counts"][comment_type] += 1

                # Store unknown comments for debugging
                if CommentType.UNKNOWN in classified:
                    stats["unknown_comments"].append(
                        {
                            "project": project_name,
                            "fuzzer": fuzzer.get("name", "Unknown"),
                            "comment": comments,
                        }
                    )

        # Update project-level aggregate stats
        if project_has_alerts:
            stats["projects_with_alerts"] += 1
        else:
            stats["projects_without_alerts"] += 1

        if project_has_over_30_pct:
            stats["projects_with_over_30_pct_coverage"] += 1

        if project_all_over_30_pct:
            stats["projects_all_fuzzers_over_30_pct"] += 1
        elif not project_has_over_30_pct:  # All fuzzers under 30%
            stats["projects_all_fuzzers_under_30_pct"] += 1

        if project_has_comments:
            stats["projects_with_comments"] += 1

    return stats


def print_latex_table(stats):
    """Generate and print a LaTeX table."""
    print("\n" + "=" * 60)
    print("LATEX TABLE")
    print("=" * 60)
    print()

    # Calculate key values
    total_fuzzers = stats["total_fuzzers"]
    total_projects = stats["total_projects"]

    # For projects: calculate projects with at least one fuzzer >30% coverage AND alert
    projects_over_30_and_alert = 0

    # We need to recalculate this from the data since we don't track it directly
    # For now, we'll use a conservative estimate based on existing stats
    # This would ideally be calculated during the data analysis phase

    print("\\begin{table}[htbp]")
    print("\\centering")
    print("\\caption{Fuzz Introspector --- Degradation Alerts}")
    print("\\label{tab:degradation-alerts}")
    print("\\begin{tabular}{lcccc}")
    print("\\toprule")
    print(
        " & \\textbf{Total} & \\textbf{With Alert} & \\textbf{Coverage: <30\\% / >30\\%} & \\textbf{Over 30\\% and Alert} \\\\"
    )
    print("\\midrule")

    # Projects row
    print(
        f"Projects & {total_projects} & {stats['projects_with_alerts']} & -- & -- \\\\"
    )

    # Fuzzers row
    fuzzers_coverage_split = (
        f"{stats['fuzzers_under_30_pct']} / {stats['fuzzers_over_30_pct']}"
    )
    print(
        f"Fuzz Targets & {total_fuzzers} & {stats['total_with_alerts']} & {fuzzers_coverage_split} & {stats['fuzzers_over_30_pct_with_alert']} \\\\"
    )

    print("\\bottomrule")
    print("\\end{tabular}")
    print("\\end{table}")
    print()
    print(
        "% Note: This table requires the booktabs package for \\toprule, \\midrule, \\bottomrule"
    )
    print("% Add \\usepackage{booktabs} to your LaTeX preamble")
    print("% Coverage split format: <30% / >30%")
    print("% Projects 'Over 30% and Alert' column shows '--' as this metric would need")
    print(
        "% additional analysis to determine projects with at least one fuzzer >30% AND alert"
    )


def print_comment_type_latex_table(stats):
    """Generate and print a LaTeX table for comment type analysis."""
    print("\n" + "=" * 60)
    print("COMMENT TYPE ANALYSIS - LATEX TABLE")
    print("=" * 60)
    print()

    # Get combined comment counts
    combined_counts = combine_comment_counts(stats["comment_type_counts"])

    if not combined_counts:
        print("% No comment types found in the data")
        return

    print("\\begin{table}[htbp]")
    print("\\centering")
    print("\\caption{Issue Type Counts}")
    print("\\label{tab:issue-types}")
    print("\\begin{tabular}{lr}")
    print("\\toprule")
    print("\\textbf{Issue Type} & \\textbf{Count} \\\\")
    print("\\midrule")

    # Sort by count (descending) and format for LaTeX
    sorted_types = combined_counts.most_common()
    for comment_type, count in sorted_types:
        if comment_type == "coverage_errors_combined":
            type_name = "Coverage Errors (Combined)"
        elif hasattr(comment_type, "value"):
            # Convert enum values to readable format
            type_name = comment_type.value.replace("_", " ").title()
            # Handle specific cases for better readability
            if type_name == "Fuzzer Unavailable":
                type_name = "Fuzzer Unavailable"
            elif type_name == "Coverage Degraded":
                type_name = "Coverage Degraded"
            elif type_name == "Coverage Degraded Detail":
                continue
            elif type_name == "Unknown":
                type_name = "Unknown Issues"
        else:
            type_name = str(comment_type).replace("_", " ").title()

        # Escape special LaTeX characters
        type_name = type_name.replace("&", "\\&").replace("%", "\\%")
        print(f"{type_name} & {count} \\\\")

    print("\\bottomrule")
    print("\\end{tabular}")
    print("\\end{table}")
    print()
    print(
        "% Note: This table shows the breakdown of different issue types found in fuzzer comments"
    )
    print(
        "% Coverage Errors (Combined) includes: timeout, memory, deadly signal, target exited, and binary file match errors"
    )


def print_detailed_stats(stats):
    """Print comprehensive statistics in a readable format."""
    print("=" * 60)
    print("FUZZER ANALYSIS REPORT")
    print("=" * 60)

    print(f"\n📊 OVERALL STATISTICS")
    print(f"   Total Projects: {stats['total_projects']}")
    print(f"   Total Fuzzers: {stats['total_fuzzers']}")
    print(f"   Fuzzers with Alerts: {stats['total_with_alerts']}")
    print(f"   Fuzzers without Alerts: {stats['total_without_alerts']}")

    print(f"\n🎯 COVERAGE BREAKDOWN (>30% threshold)")
    print(f"   Fuzzers with >30% coverage: {stats['fuzzers_over_30_pct']}")
    print(f"     • With alerts: {stats['fuzzers_over_30_pct_with_alert']}")
    print(f"     • Without alerts: {stats['fuzzers_over_30_pct_without_alert']}")
    print(f"   Fuzzers with ≤30% coverage: {stats['fuzzers_under_30_pct']}")
    print(f"     • With alerts: {stats['fuzzers_under_30_pct_with_alert']}")
    print(f"     • Without alerts: {stats['fuzzers_under_30_pct_without_alert']}")

    # Calculate percentages
    if stats["total_fuzzers"] > 0:
        over_30_pct = (stats["fuzzers_over_30_pct"] / stats["total_fuzzers"]) * 100
        alert_pct = (stats["total_with_alerts"] / stats["total_fuzzers"]) * 100
        print(f"\n📈 PERCENTAGES")
        print(f"   Fuzzers with >30% coverage: {over_30_pct:.1f}%")
        print(f"   Fuzzers with alerts: {alert_pct:.1f}%")

    print(f"\n🏗️ PROJECT-LEVEL STATISTICS")
    print(f"   Projects with at least one alert: {stats['projects_with_alerts']}")
    print(f"   Projects with no alerts: {stats['projects_without_alerts']}")
    print(
        f"   Projects with at least one fuzzer >30% coverage: {stats['projects_with_over_30_pct_coverage']}"
    )
    print(
        f"   Projects where ALL fuzzers have >30% coverage: {stats['projects_all_fuzzers_over_30_pct']}"
    )
    print(
        f"   Projects where ALL fuzzers have ≤30% coverage: {stats['projects_all_fuzzers_under_30_pct']}"
    )
    print(f"   Projects with comments/issues: {stats['projects_with_comments']}")

    # Calculate project-level percentages
    if stats["total_projects"] > 0:
        projects_with_alerts_pct = (
            stats["projects_with_alerts"] / stats["total_projects"]
        ) * 100
        projects_good_coverage_pct = (
            stats["projects_all_fuzzers_over_30_pct"] / stats["total_projects"]
        ) * 100
        projects_with_issues_pct = (
            stats["projects_with_comments"] / stats["total_projects"]
        ) * 100
        print(f"\n📈 PROJECT PERCENTAGES")
        print(f"   Projects with alerts: {projects_with_alerts_pct:.1f}%")
        print(
            f"   Projects with all fuzzers >30% coverage: {projects_good_coverage_pct:.1f}%"
        )
        print(f"   Projects with issues/comments: {projects_with_issues_pct:.1f}%")

    print(f"\n🚨 COMMENT TYPE ANALYSIS")
    if stats["comment_type_counts"]:
        # Combine coverage error types for display
        combined_counts = combine_comment_counts(stats["comment_type_counts"])

        # Sort by count (descending)
        sorted_types = combined_counts.most_common()
        for comment_type, count in sorted_types:
            if comment_type == "coverage_errors_combined":
                type_name = "Coverage Errors (Combined)"
            elif hasattr(comment_type, "value"):
                type_name = comment_type.value.replace("_", " ").title()
            else:
                type_name = str(comment_type).replace("_", " ").title()
            print(f"   {type_name}: {count}")
    else:
        print("   No comments found")

    print(f"\n🎯 KEY METRICS")
    print(
        f"   Fuzzers with >30% coverage AND alerts: {stats['fuzzers_over_30_pct_with_alert']}"
    )
    print(
        f"   Projects with at least one alert: {stats['projects_with_alerts']} / {stats['total_projects']}"
    )

    # Show unknown comments if any
    if stats["unknown_comments"]:
        print(f"\n❓ UNKNOWN COMMENT PATTERNS ({len(stats['unknown_comments'])} found)")
        print("   These comments contain unrecognized patterns:")
        for i, item in enumerate(stats["unknown_comments"][:5], 1):  # Show first 5
            print(f"   {i}. {item['project']} / {item['fuzzer']}")
            print(
                f'      "{item["comment"][:100]}{"..." if len(item["comment"]) > 100 else ""}"'
            )
        if len(stats["unknown_comments"]) > 5:
            print(f"   ... and {len(stats['unknown_comments']) - 5} more")


if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: python script.py <json_file>")
        sys.exit(1)

    json_file = sys.argv[1]
    stats = analyze_fuzzer_data(json_file)
    print_detailed_stats(stats)
    print_latex_table(stats)
    print_comment_type_latex_table(stats)