fix-encoding.py

#!/usr/bin/env python3
# /// script
# requires-python = ">=3.10"
# dependencies = ["ftfy"]
# ///
"""
fix-encoding.py - Fix UTF-8 encoding issues in markdown documentation files.

Uses ftfy for automatic mojibake detection/repair, with manual fallbacks
for edge cases like emoji-to-ASCII conversion.

Usage:
    uv run fix-encoding.py [directory]
    # or: python fix-encoding.py [directory]  (if ftfy is installed)

Options:
    --dry-run    Show what would be fixed without making changes
    --verbose    Show detailed output
"""

import argparse
import re
import sys
from pathlib import Path

try:
    import ftfy
except ImportError:
    print("Error: ftfy is required. Install with: pip install ftfy")
    sys.exit(1)


# Manual replacements for symbols that should become ASCII
# (ftfy fixes mojibake but preserves valid Unicode - we want ASCII output)
SYMBOL_REPLACEMENTS = {
    # Checkmarks and crosses
    "✅": "Yes",
    "❌": "No",
    "✓": "[done]",
    "☑": "[x]",
    "☐": "[ ]",

    # Arrows
    "→": "->",
    "←": "<-",
    "↑": "^",
    "↓": "v",
    "⇄": "<->",
    "⇒": "=>",
    "⇐": "<=",

    # Bullets and daggers
    "•": "*",
    "·": "*",
    "†": "*",
    "‡": "**",
    "§": "S",  # Section symbol

    # Dashes
    "—": "--",  # em dash
    "–": "-",   # en dash

    # Quotes (smart quotes to straight)
    "'": "'",
    "'": "'",
    """: '"',
    """: '"',

    # Math symbols
    "×": "x",
    "÷": "/",
    "≥": ">=",
    "≤": "<=",
    "≠": "!=",
    "±": "+/-",
    "¼": "1/4",
    "½": "1/2",
    "¾": "3/4",

    # Emoji
    "🛠️": "[dev]",
    "🛠": "[dev]",
    "⏳": "[pending]",
    "⚠️": "[!]",
    "⚠": "[!]",
    "ℹ️": "[i]",
    "ℹ": "[i]",

    # Misc
    "…": "...",
    "©": "(c)",
    "®": "(R)",
    "™": "(TM)",
    "°": " deg",
}

# Regex patterns for stubborn mojibake that ftfy might miss
REGEX_REPLACEMENTS = [
    # Corrupted arrow patterns (various encodings)
    (re.compile(r"â\*['\"]"), "->"),
    (re.compile(r"’"), "'"),
    (re.compile(r"â€"), "-"),
    (re.compile(r"Ã--"), "x"),
    (re.compile(r"×"), "x"),
    (re.compile(r"§"), "S"),
    (re.compile(r"¼"), "1/4"),
    (re.compile(r"ÂÂ"), ""),
    (re.compile(r'"Â\d+/\d+"?'), ""),  # Corrupted fractions
    # Triple-encoded garbage (Ã followed by control chars)
    (re.compile(r'"Ã.{1,3}\d+/\d+'), ""),  # e.g., "Â1/4
    (re.compile(r"Ã[\x80-\xbf]"), ""),  # Ã followed by continuation byte
    # Cleanup remaining  (C2 byte from broken UTF-8)
    (re.compile(r'"Â\d+/\d+'), ""),  # e.g., "Â1/4
    (re.compile(r"Â(?=\d|[^\w])"), ""),  # Â before digit or non-word char
]


def fix_encoding(text: str) -> str:
    """Fix encoding issues in text using ftfy and manual replacements."""

    # Step 1: Use ftfy to fix mojibake automatically
    # ftfy.fix_text handles most UTF-8 encoding issues
    fixed = ftfy.fix_text(
        text,
        normalization="NFC",  # Normalize to composed form
        explain=False,
    )

    # Step 2: Apply regex patterns for stubborn mojibake
    for pattern, replacement in REGEX_REPLACEMENTS:
        fixed = pattern.sub(replacement, fixed)

    # Step 3: Replace Unicode symbols with ASCII equivalents
    for symbol, replacement in SYMBOL_REPLACEMENTS.items():
        fixed = fixed.replace(symbol, replacement)

    return fixed


def has_non_ascii(text: str) -> bool:
    """Check if text contains non-ASCII characters."""
    return any(ord(char) > 127 for char in text)


def process_file(filepath: Path, dry_run: bool = False) -> tuple[bool, bool]:
    """
    Process a single file.

    Returns:
        (had_issues, was_fixed): Whether file had issues and whether it was fixed
    """
    try:
        content = filepath.read_text(encoding="utf-8", errors="replace")
    except Exception as e:
        print(f"  Error reading {filepath}: {e}")
        return False, False

    if not has_non_ascii(content):
        return False, False

    fixed_content = fix_encoding(content)

    if fixed_content == content:
        # No changes made despite having non-ASCII
        return True, False

    if not dry_run:
        try:
            filepath.write_text(fixed_content, encoding="utf-8")
        except Exception as e:
            print(f"  Error writing {filepath}: {e}")
            return True, False

    return True, True


def find_markdown_files(directory: Path) -> list[Path]:
    """Find all markdown files in directory recursively."""
    return sorted(directory.rglob("*.md"))


def main():
    parser = argparse.ArgumentParser(
        description="Fix UTF-8 encoding issues in markdown files"
    )
    parser.add_argument(
        "directory",
        nargs="?",
        default=".",
        help="Directory to process (default: current directory)",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Show what would be fixed without making changes",
    )
    parser.add_argument(
        "--verbose", "-v",
        action="store_true",
        help="Show detailed output",
    )

    args = parser.parse_args()

    target_dir = Path(args.directory)
    if not target_dir.is_dir():
        print(f"Error: Directory '{target_dir}' not found")
        sys.exit(1)

    print(f"Fixing encoding in: {target_dir}")
    if args.dry_run:
        print("(dry run - no changes will be made)")
    print()

    files = find_markdown_files(target_dir)
    if not files:
        print(f"No markdown files found in {target_dir}")
        sys.exit(0)

    print(f"Found {len(files)} markdown file(s)")
    print()

    fixed_count = 0
    skipped_count = 0
    problem_files = []

    for filepath in files:
        had_issues, was_fixed = process_file(filepath, dry_run=args.dry_run)

        if had_issues:
            if was_fixed:
                print(f"Fixed: {filepath}")
                fixed_count += 1
            else:
                if args.verbose:
                    print(f"Issues remain: {filepath}")
                problem_files.append(filepath)
        else:
            skipped_count += 1
            if args.verbose:
                print(f"Clean: {filepath}")

    print()
    print(f"Done. Fixed {fixed_count} file(s), skipped {skipped_count} clean file(s).")

    # Verify results
    print()
    print("Verifying...")

    remaining_issues = []
    for filepath in files:
        try:
            content = filepath.read_text(encoding="utf-8")
            if has_non_ascii(content):
                remaining_issues.append(filepath)
                # Show sample of remaining issues
                lines = content.split("\n")
                for i, line in enumerate(lines, 1):
                    if has_non_ascii(line):
                        # Truncate long lines
                        display_line = line[:80] + "..." if len(line) > 80 else line
                        print(f"  {filepath}:{i}: {display_line}")
                        break  # Just show first issue per file
        except Exception:
            pass

    if not remaining_issues:
        print("All files clean!")
    else:
        print()
        print(f"Warning: {len(remaining_issues)} file(s) still have non-ASCII characters.")
        print("These may be intentional (e.g., non-English content) or need manual review.")


if __name__ == "__main__":
    main()