shortcuts.py

#!/usr/bin/env python3
"""Steam shortcut ID generation and shortcuts.vdf read/write.

Implements the same CRC32-based ID generation algorithm used by
Steam ROM Manager, so our shortcuts are fully compatible.

The shortcuts.vdf format is a binary key-value format used by Steam
to store non-Steam game shortcuts.
"""

import binascii
import json
import logging
import struct
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, List, Optional, Tuple

logger = logging.getLogger(__name__)

# ═══════════════════════════════════════════════════════════════════════
# App ID Generation (matches SRM exactly)
# ═══════════════════════════════════════════════════════════════════════

def generate_preliminary_id(exe: str, appname: str) -> int:
    """Generate the preliminary ID used by Steam for non-Steam shortcuts.

    This matches SRM's generatePreliminaryId() exactly.

    Args:
        exe: The executable path string.
        appname: The app/game display name.

    Returns:
        64-bit preliminary ID.
    """
    key = exe + appname
    crc = binascii.crc32(key.encode("utf-8")) & 0xFFFFFFFF
    top = crc | 0x80000000
    return (top << 32) | 0x02000000


def generate_app_id(exe: str, appname: str) -> str:
    """Generate the full app ID (used for Big Picture grids).

    Matches SRM's generateAppId().

    Args:
        exe: The executable path string.
        appname: The app/game display name.

    Returns:
        String representation of the app ID.
    """
    return str(generate_preliminary_id(exe, appname))


def generate_short_app_id(exe: str, appname: str) -> str:
    """Generate the short app ID (used for grid image filenames).

    Matches SRM's generateShortAppId().
    This is the ID used in grid image filenames: {shortAppId}.png, etc.

    Args:
        exe: The executable path string.
        appname: The app/game display name.

    Returns:
        String representation of the short app ID.
    """
    return str(int(generate_app_id(exe, appname)) >> 32)


def generate_shortcut_id(exe: str, appname: str) -> int:
    """Generate the shortcut ID (stored in shortcuts.vdf as appid).

    Matches SRM's generateShortcutId().

    Args:
        exe: The executable path string.
        appname: The app/game display name.

    Returns:
        Signed 32-bit shortcut ID.
    """
    preliminary = generate_preliminary_id(exe, appname)
    unsigned = preliminary >> 32
    # Convert to signed 32-bit
    if unsigned >= 0x80000000:
        return unsigned - 0x100000000
    return unsigned


def shorten_app_id(long_id: str) -> str:
    """Convert a full app ID to short app ID.

    Args:
        long_id: Full app ID string.

    Returns:
        Short app ID string.
    """
    return str(int(long_id) >> 32)


def lengthen_app_id(short_id: str) -> str:
    """Convert a short app ID to full app ID.

    Args:
        short_id: Short app ID string.

    Returns:
        Full app ID string.
    """
    return str((int(short_id) << 32) | 0x02000000)


# ═══════════════════════════════════════════════════════════════════════
# Shortcut Data Structure
# ═══════════════════════════════════════════════════════════════════════

@dataclass
class SteamShortcut:
    """Represents a single non-Steam game shortcut."""
    appid: int                          # Signed 32-bit shortcut ID
    appname: str                        # Display name
    exe: str                            # Executable path (quoted)
    start_dir: str                      # Start-in directory (quoted)
    icon: str = ""                      # Icon path
    shortcut_path: str = ""             # Shortcut path
    launch_options: str = ""            # Additional launch arguments
    is_hidden: int = 0                  # Hidden from library
    allow_desktop_config: int = 1       # Allow desktop config
    allow_overlay: int = 1              # Allow Steam overlay
    openvr: int = 0                     # VR game
    devkit: int = 0                     # Dev kit
    devkit_game_id: str = ""            # Dev kit game ID
    devkit_override_app_id: int = 0     # Dev kit override
    last_play_time: int = 0             # Unix timestamp
    flatpak_appid: str = ""             # Flatpak app ID (custom field)
    tags: Dict[str, str] = field(default_factory=dict)  # Tags/categories

    @property
    def short_app_id(self) -> str:
        """Get the short app ID used for grid image filenames."""
        # Convert signed shortcut ID back to unsigned
        unsigned = self.appid if self.appid >= 0 else self.appid + 0x100000000
        return str(unsigned)

    @property
    def full_app_id(self) -> str:
        """Get the full app ID used for Big Picture grids."""
        return lengthen_app_id(self.short_app_id)


# ═══════════════════════════════════════════════════════════════════════
# shortcuts.vdf Binary Parser
# ═══════════════════════════════════════════════════════════════════════

# VDF binary type markers
VDF_TYPE_OBJECT = 0x00
VDF_TYPE_STRING = 0x01
VDF_TYPE_INT32 = 0x02
VDF_TYPE_END = 0x08

# Known integer fields in shortcuts.vdf
INT32_FIELDS = {
    "appid", "ishidden", "allowdesktopconfig", "allowoverlay",
    "openvr", "devkit", "devkitoverrideappid", "lastplaytime",
}


def read_shortcuts_vdf(path: Path) -> List[SteamShortcut]:
    """Read and parse a shortcuts.vdf file.

    Args:
        path: Path to shortcuts.vdf file.

    Returns:
        List of SteamShortcut objects.

    Raises:
        FileNotFoundError: If the file doesn't exist.
        ValueError: If the file format is invalid.
    """
    if not path.exists():
        raise FileNotFoundError(f"shortcuts.vdf not found: {path}")

    data = path.read_bytes()
    shortcuts = []
    pos = 0

    # Skip top-level object marker and "shortcuts" key
    if data[pos:pos + 1] == b'\x00':
        pos += 1
        # Read "shortcuts" key
        end = data.index(b'\x00', pos)
        pos = end + 1

    while pos < len(data):
        # Each shortcut starts with 0x00 + index string
        if data[pos:pos + 1] == b'\x08':
            break  # End of shortcuts object
        if data[pos:pos + 1] == b'\x00':
            pos += 1
            # Read index
            end = data.index(b'\x00', pos)
            pos = end + 1

            # Parse shortcut fields
            shortcut_data, pos = _parse_vdf_object(data, pos)
            shortcut = _dict_to_shortcut(shortcut_data)
            if shortcut:
                shortcuts.append(shortcut)
        else:
            pos += 1

    logger.info(f"Read {len(shortcuts)} shortcuts from {path}")
    return shortcuts


def _parse_vdf_object(data: bytes, pos: int) -> Tuple[dict, int]:
    """Parse a VDF binary object starting at pos.

    Returns:
        Tuple of (parsed dict, new position).
    """
    result = {}

    while pos < len(data):
        type_byte = data[pos]
        pos += 1

        if type_byte == VDF_TYPE_END:
            break

        # Read key name
        end = data.index(b'\x00', pos)
        key = data[pos:end].decode('utf-8', errors='replace').lower()
        pos = end + 1

        if type_byte == VDF_TYPE_STRING:
            # String value
            end = data.index(b'\x00', pos)
            value = data[pos:end].decode('utf-8', errors='replace')
            pos = end + 1
            result[key] = value

        elif type_byte == VDF_TYPE_INT32:
            # 32-bit integer
            value = struct.unpack('<i', data[pos:pos + 4])[0]
            pos += 4
            result[key] = value

        elif type_byte == VDF_TYPE_OBJECT:
            # Nested object (e.g., tags)
            sub_obj, pos = _parse_vdf_object(data, pos)
            result[key] = sub_obj

    return result, pos


def _dict_to_shortcut(data: dict) -> Optional[SteamShortcut]:
    """Convert a parsed VDF dict to a SteamShortcut object."""
    try:
        return SteamShortcut(
            appid=data.get("appid", 0),
            appname=data.get("appname", data.get("AppName", "")),
            exe=data.get("exe", data.get("Exe", "")),
            start_dir=data.get("startdir", data.get("StartDir", "")),
            icon=data.get("icon", data.get("icon", "")),
            shortcut_path=data.get("shortcutpath", ""),
            launch_options=data.get("launchoptions", data.get("LaunchOptions", "")),
            is_hidden=data.get("ishidden", data.get("IsHidden", 0)),
            allow_desktop_config=data.get("allowdesktopconfig", data.get("AllowDesktopConfig", 1)),
            allow_overlay=data.get("allowoverlay", data.get("AllowOverlay", 1)),
            openvr=data.get("openvr", data.get("OpenVR", 0)),
            devkit=data.get("devkit", data.get("Devkit", 0)),
            devkit_game_id=data.get("devkitgameid", data.get("DevkitGameID", "")),
            devkit_override_app_id=data.get("devkitoverrideappid", data.get("DevkitOverrideAppID", 0)),
            last_play_time=data.get("lastplaytime", data.get("LastPlayTime", 0)),
            flatpak_appid=data.get("flatpakappid", data.get("FlatpakAppID", "")),
            tags=data.get("tags", {}),
        )
    except Exception as e:
        logger.warning(f"Failed to parse shortcut: {e}")
        return None


def write_shortcuts_vdf(path: Path, shortcuts: List[SteamShortcut]) -> None:
    """Write shortcuts to a shortcuts.vdf file.

    Args:
        path: Output path for shortcuts.vdf.
        shortcuts: List of SteamShortcut objects to write.
    """
    buf = bytearray()

    # Top-level object: \x00 "shortcuts" \x00
    buf.append(VDF_TYPE_OBJECT)
    buf.extend(b'shortcuts\x00')

    for i, sc in enumerate(shortcuts):
        # Object start: \x00 "index" \x00
        buf.append(VDF_TYPE_OBJECT)
        buf.extend(str(i).encode('utf-8'))
        buf.append(0x00)

        # Write fields
        _write_int32(buf, "appid", sc.appid)
        _write_string(buf, "AppName", sc.appname)
        _write_string(buf, "Exe", sc.exe)
        _write_string(buf, "StartDir", sc.start_dir)
        _write_string(buf, "icon", sc.icon)
        _write_string(buf, "ShortcutPath", sc.shortcut_path)
        _write_string(buf, "LaunchOptions", sc.launch_options)
        _write_int32(buf, "IsHidden", sc.is_hidden)
        _write_int32(buf, "AllowDesktopConfig", sc.allow_desktop_config)
        _write_int32(buf, "AllowOverlay", sc.allow_overlay)
        _write_int32(buf, "OpenVR", sc.openvr)
        _write_int32(buf, "Devkit", sc.devkit)
        _write_string(buf, "DevkitGameID", sc.devkit_game_id)
        _write_int32(buf, "DevkitOverrideAppID", sc.devkit_override_app_id)
        _write_int32(buf, "LastPlayTime", sc.last_play_time)
        _write_string(buf, "FlatpakAppID", sc.flatpak_appid)

        # Tags sub-object
        buf.append(VDF_TYPE_OBJECT)
        buf.extend(b'tags\x00')
        for tag_idx, tag_val in sc.tags.items():
            _write_string(buf, str(tag_idx), str(tag_val))
        buf.append(VDF_TYPE_END)

        # End of shortcut object
        buf.append(VDF_TYPE_END)

    # End of shortcuts object
    buf.append(VDF_TYPE_END)

    # End of root object (top-level container wrapping "shortcuts")
    buf.append(VDF_TYPE_END)

    # Backup existing file
    if path.exists():
        backup = path.with_suffix('.vdf.bak')
        import shutil
        shutil.copy2(path, backup)
        logger.info(f"Backed up existing shortcuts.vdf to {backup}")

    # Write atomically via temp file (prevents truncation on failure)
    import os as _os
    tmp = path.with_suffix('.vdf.tmp')
    tmp.write_bytes(bytes(buf))
    _os.replace(tmp, path)
    logger.info(f"Wrote {len(shortcuts)} shortcuts to {path}")


def _write_string(buf: bytearray, key: str, value: str) -> None:
    """Write a string field to the VDF buffer."""
    buf.append(VDF_TYPE_STRING)
    buf.extend(key.encode('utf-8'))
    buf.append(0x00)
    buf.extend(value.encode('utf-8'))
    buf.append(0x00)


def _write_int32(buf: bytearray, key: str, value: int) -> None:
    """Write a 32-bit integer field to the VDF buffer."""
    buf.append(VDF_TYPE_INT32)
    buf.extend(key.encode('utf-8'))
    buf.append(0x00)
    buf.extend(struct.pack('<i', value))


# ═══════════════════════════════════════════════════════════════════════
# High-Level Operations
# ═══════════════════════════════════════════════════════════════════════

def find_shortcuts_vdf(steam_path: Path, user_id: Optional[str] = None) -> Path:
    """Find the shortcuts.vdf file for a Steam user.

    Args:
        steam_path: Steam installation path.
        user_id: Specific user ID, or None to auto-detect.

    Returns:
        Path to shortcuts.vdf.

    Raises:
        FileNotFoundError: If shortcuts.vdf cannot be found.
    """
    userdata = steam_path / "userdata"

    if user_id:
        vdf = userdata / user_id / "config" / "shortcuts.vdf"
        if vdf.exists():
            return vdf
        raise FileNotFoundError(f"shortcuts.vdf not found for user {user_id}")

    # Auto-detect: find first user with a shortcuts.vdf
    for user_dir in sorted(userdata.iterdir()):
        if not user_dir.is_dir():
            continue
        vdf = user_dir / "config" / "shortcuts.vdf"
        if vdf.exists():
            logger.info(f"Found shortcuts.vdf for user {user_dir.name}")
            return vdf

    raise FileNotFoundError("No shortcuts.vdf found in any user profile")


def get_existing_shortcuts(steam_path: Path,
                           user_id: Optional[str] = None) -> List[SteamShortcut]:
    """Get all existing non-Steam shortcuts.

    Args:
        steam_path: Steam installation path.
        user_id: Specific user ID, or None to auto-detect.

    Returns:
        List of existing shortcuts, or empty list if none found.
    """
    try:
        vdf_path = find_shortcuts_vdf(steam_path, user_id)
        return read_shortcuts_vdf(vdf_path)
    except FileNotFoundError:
        return []


def add_shortcuts(steam_path: Path, new_shortcuts: List[SteamShortcut],
                  user_id: Optional[str] = None,
                  replace_existing: bool = False,
                  remove_by_tags: Optional[set] = None) -> Tuple[int, int]:
    """Add new shortcuts to shortcuts.vdf, preserving existing ones.

    Args:
        steam_path: Steam installation path.
        new_shortcuts: Shortcuts to add.
        user_id: Specific user ID, or None to auto-detect.
        replace_existing: If True, replace shortcuts with matching appid.
        remove_by_tags: If provided, remove all existing shortcuts whose first
            tag value matches any value in this set before adding new ones.
            Use this to replace all SRM-managed shortcuts for a system.

    Returns:
        Tuple of (added_count, skipped_count).
    """
    existing = get_existing_shortcuts(steam_path, user_id)

    # Remove all existing entries for managed systems (tag-based purge).
    # This handles format migration (SRM format -> SGM format) where app IDs differ.
    if remove_by_tags:
        before = len(existing)
        existing = [
            sc for sc in existing
            if not any(str(v) in remove_by_tags for v in sc.tags.values())
        ]
        removed = before - len(existing)
        if removed:
            logger.info(f"Removed {removed} existing shortcuts by tag: {remove_by_tags}")

    existing_ids = {sc.appid for sc in existing}

    added = 0
    skipped = 0

    for new_sc in new_shortcuts:
        if new_sc.appid in existing_ids:
            if replace_existing:
                # Remove old entry
                existing = [sc for sc in existing if sc.appid != new_sc.appid]
                existing.append(new_sc)
                added += 1
            else:
                skipped += 1
        else:
            existing.append(new_sc)
            added += 1

    # Find or create the VDF path
    try:
        vdf_path = find_shortcuts_vdf(steam_path, user_id)
    except FileNotFoundError:
        # Create the config directory if needed
        userdata = steam_path / "userdata"
        if user_id:
            config_dir = userdata / user_id / "config"
        else:
            # Use first user directory
            for user_dir in sorted(userdata.iterdir()):
                if user_dir.is_dir():
                    config_dir = user_dir / "config"
                    break
            else:
                raise FileNotFoundError("No user profile found in Steam userdata")
        config_dir.mkdir(parents=True, exist_ok=True)
        vdf_path = config_dir / "shortcuts.vdf"

    # Skip writing if nothing changed — avoids truncation if killed mid-write
    if added == 0:
        if not remove_by_tags:
            logger.info(f"No new shortcuts to add, skipping VDF write")
            return added, skipped
        # Even if nothing added, write if we removed entries by tag

    write_shortcuts_vdf(vdf_path, existing)
    logger.info(f"Added {added} shortcuts, skipped {skipped} existing")
    return added, skipped


def find_localconfig_vdf(steam_path: Path, user_id: Optional[str] = None) -> Path:
    """Find the localconfig.vdf for a Steam user.

    Args:
        steam_path: Steam installation path.
        user_id: Specific Steam user ID, or None to auto-detect.

    Returns:
        Path to localconfig.vdf.

    Raises:
        FileNotFoundError: If not found.
    """
    userdata = steam_path / "userdata"
    if user_id:
        candidates = [userdata / user_id]
    else:
        candidates = sorted(userdata.iterdir()) if userdata.exists() else []

    for user_dir in candidates:
        vdf = user_dir / "config" / "localconfig.vdf"
        if vdf.exists():
            return vdf

    raise FileNotFoundError("localconfig.vdf not found in Steam userdata")


def update_steam_collections(steam_path: Path,
                              shortcuts_by_category: Dict[str, List[int]],
                              user_id: Optional[str] = None) -> bool:
    """Add shortcuts to Steam library collections in cloud storage.

    Creates native ``uc-*`` user collections (same format Steam uses natively),
    replacing any old ``srm-*`` collections from Steam ROM Manager.

    Also merges legacy-named collections (e.g. "Commodore 64" -> "C64") into
    the canonical collection and marks the legacy ones as deleted.

    ``cloud-storage-namespace-1.json`` is the authoritative collection source.
    On import we:
      1. Remove (mark deleted) all old ``srm-*`` entries.
      2. Merge legacy-named collections into their canonical equivalents.
      3. Write new ``uc-*`` entries with the full game list.
      4. Update ``localconfig.vdf`` user-collections cache for parity.

    Args:
        steam_path: Steam installation path.
        shortcuts_by_category: Mapping of category name -> list of unsigned int32
            short app IDs (generate_short_app_id(), NOT generate_shortcut_id()).
            Steam's collection system stores unsigned 32-bit values.
        user_id: Specific Steam user ID, or None to auto-detect.

    Returns:
        True on success, False on failure.
    """
    import base64
    import hashlib
    import json
    import re
    import time as _time

    try:
        vdf_path = find_localconfig_vdf(steam_path, user_id)
    except FileNotFoundError as e:
        logger.warning(f"Could not find localconfig.vdf: {e}")
        return False

    cloud_path = vdf_path.parent / "cloudstorage" / "cloud-storage-namespace-1.json"
    cloud_modified_path = (
        vdf_path.parent / "cloudstorage" / "cloud-storage-namespace-1.modified.json"
    )

    if not cloud_path.exists():
        logger.debug(f"Cloud storage not found at {cloud_path}, skipping")
        return False

    try:
        cloud_data: list = json.load(cloud_path.open(encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as e:
        logger.warning(f"Failed to read cloud storage: {e}")
        return False

    # --- helpers --------------------------------------------------------
    def _make_uc_id(category: str) -> str:
        """Derive a stable uc- ID from the category name."""
        digest = hashlib.sha1(f"sgm-{category}".encode()).digest()
        # Use url-safe base64, strip padding, take first 16 chars
        b64 = base64.urlsafe_b64encode(digest).rstrip(b"=").decode()
        return f"uc-{b64[:16]}"

    def _get_max_version(data: list) -> int:
        v = 1
        for item in data:
            if isinstance(item, list) and len(item) >= 2:
                try:
                    v = max(v, int(str(item[1].get("version", "0"))))
                except (ValueError, TypeError):
                    pass
        return v

    def _parse_collection_value(entry: dict) -> dict:
        """Parse the 'value' JSON string from a cloud collection entry."""
        value_str = entry.get("value", "")
        if value_str:
            try:
                return json.loads(value_str)
            except json.JSONDecodeError:
                pass
        return {}

    # Build lookup: key -> list index
    cloud_index: dict[str, int] = {}
    for i, item in enumerate(cloud_data):
        if isinstance(item, list) and len(item) >= 2 and isinstance(item[1], dict):
            cloud_index[item[1].get("key", "")] = i

    next_version = _get_max_version(cloud_data) + 1
    timestamp_ms = int(_time.time() * 1000)

    # 1. Mark all old srm-* collection entries as deleted
    for key, idx in list(cloud_index.items()):
        if key.startswith("user-collections.srm-"):
            deleted_entry = {
                "key": key,
                "timestamp": timestamp_ms,
                "is_deleted": True,
                "version": str(next_version),
            }
            cloud_data[idx] = [key, deleted_entry]
            next_version += 1
            logger.debug(f"Marked old SRM collection as deleted: {key}")

    # 2. Merge legacy-named collections into canonical ones
    # Build a mapping: legacy_name -> canonical_category
    # e.g. "Commodore 64" -> "C64", "Nintendo Entertainment System" -> "NES"
    legacy_to_canonical: dict[str, str] = {}
    try:
        from emulators import get_registry
        registry = get_registry()
        for sys_id, sys_def in registry.list_systems().items():
            canonical = sys_def.get_steam_category()
            legacy_to_canonical[canonical] = canonical  # self-map for safety
            for tag in sys_def.all_category_tags():
                if tag != canonical:
                    legacy_to_canonical[tag] = canonical
    except Exception as e:
        logger.debug(f"Could not load system definitions for legacy mapping: {e}")

    # Find existing collections whose names are legacy aliases
    legacy_coll_ids: dict[str, str] = {}  # coll_id -> canonical category name
    for key, idx in list(cloud_index.items()):
        if not key.startswith("user-collections.uc-"):
            continue
        entry = cloud_data[idx][1] if isinstance(cloud_data[idx], list) else cloud_data[idx]
        if entry.get("is_deleted", False):
            continue
        value = _parse_collection_value(entry)
        coll_name = value.get("name", "")
        if not coll_name:
            continue
        canonical = legacy_to_canonical.get(coll_name)
        if canonical and canonical != coll_name:
            legacy_coll_ids[key] = canonical
            # Merge the legacy collection's games into shortcuts_by_category
            legacy_added = [int(x) for x in value.get("added", [])
                           if isinstance(x, int) or (isinstance(x, str) and x.isdigit())]
            existing = shortcuts_by_category.setdefault(canonical, [])
            # Add games that aren't already in the canonical list
            existing_set = set(existing)
            for app_id in legacy_added:
                if app_id not in existing_set:
                    existing.append(app_id)
                    existing_set.add(app_id)
            logger.info(f"Merged legacy collection {coll_name!r} into {canonical!r} ({len(legacy_added)} games)")

    # Mark legacy collections as deleted
    for key, canonical in legacy_coll_ids.items():
        idx = cloud_index[key]
        coll_id = key.replace("user-collections.", "")
        deleted_entry = {
            "key": key,
            "timestamp": timestamp_ms,
            "is_deleted": True,
            "version": str(next_version),
        }
        cloud_data[idx] = [key, deleted_entry]
        next_version += 1
        logger.info(f"Marked legacy collection as deleted: {key} (merged into {canonical!r})")

    # 2b. Merge same-name duplicates (e.g. SRM-created "Heroic" vs SGM-created "Heroic")
    # When multiple uc-* collections share the same name, keep only the SGM one
    # and merge the others' games into it.
    # Also delete any uc-* collection that duplicates a built-in Steam collection
    # (e.g. "Favorites" or "Hidden").
    BUILTIN_COLLECTIONS = {"favorite", "favorites", "hidden"}

    name_to_keys: dict[str, list[str]] = {}  # collection name -> list of cloud keys
    for key, idx in list(cloud_index.items()):
        if not key.startswith("user-collections.uc-"):
            continue
        entry = cloud_data[idx][1] if isinstance(cloud_data[idx], list) else cloud_data[idx]
        if entry.get("is_deleted", False):
            continue
        value = _parse_collection_value(entry)
        coll_name = value.get("name", "")
        if coll_name:
            name_to_keys.setdefault(coll_name, []).append(key)

    for coll_name, keys in name_to_keys.items():
        # Delete uc-* collections that duplicate built-in Steam collections
        if coll_name.lower() in BUILTIN_COLLECTIONS:
            for key in keys:
                idx = cloud_index[key]
                deleted_entry = {
                    "key": key,
                    "timestamp": timestamp_ms,
                    "is_deleted": True,
                    "version": str(next_version),
                }
                cloud_data[idx] = [key, deleted_entry]
                next_version += 1
                logger.info(f"Marked built-in duplicate collection as deleted: {key} ({coll_name!r})")
            continue

        if len(keys) <= 1:
            continue
        # Determine which key is the SGM canonical one
        sgm_key = f"user-collections.{_make_uc_id(coll_name)}"
        # Merge all non-SGM collections into the SGM one
        for key in keys:
            if key == sgm_key:
                continue
            idx = cloud_index[key]
            entry = cloud_data[idx][1] if isinstance(cloud_data[idx], list) else cloud_data[idx]
            value = _parse_collection_value(entry)
            dup_added = [int(x) for x in value.get("added", [])
                        if isinstance(x, int) or (isinstance(x, str) and x.isdigit())]
            # Merge games into the canonical category
            existing = shortcuts_by_category.setdefault(coll_name, [])
            existing_set = set(existing)
            for app_id in dup_added:
                if app_id not in existing_set:
                    existing.append(app_id)
                    existing_set.add(app_id)
            # Mark the duplicate as deleted
            deleted_entry = {
                "key": key,
                "timestamp": timestamp_ms,
                "is_deleted": True,
                "version": str(next_version),
            }
            cloud_data[idx] = [key, deleted_entry]
            next_version += 1
            logger.info(f"Marked duplicate collection as deleted: {key} (merged into {coll_name!r})")

    # 3. Write new uc-* entries
    # Skip built-in Steam collection names that should not be duplicated
    BUILTIN_COLLECTIONS = {"favorite", "favorites", "hidden"}

    new_collections: dict[str, dict] = {}  # coll_id -> value dict (for localconfig)
    for category, app_ids in shortcuts_by_category.items():
        if not app_ids:
            continue

        # Skip built-in Steam collections (Favorites, Hidden)
        if category.lower() in BUILTIN_COLLECTIONS:
            logger.debug(f"Skipping built-in collection: {category!r}")
            continue

        coll_id = _make_uc_id(category)
        cloud_key = f"user-collections.{coll_id}"

        value_obj = {
            "id": coll_id,
            "name": category,
            "added": list(app_ids),
            "removed": [],
        }
        new_entry = {
            "key": cloud_key,
            "timestamp": timestamp_ms,
            "value": json.dumps(value_obj, separators=(",", ":")),
            "version": str(next_version),
            "conflictResolutionMethod": "custom",
            "strMethodId": "union-collections",
        }

        if cloud_key in cloud_index:
            cloud_data[cloud_index[cloud_key]] = [cloud_key, new_entry]
        else:
            cloud_data.append([cloud_key, new_entry])
            cloud_index[cloud_key] = len(cloud_data) - 1

        # For localconfig cache, Steam expects NO "name" field — just id/added/removed.
        # Including "name" prevents Steam from showing games in the collection UI.
        new_collections[coll_id] = {
            "id": coll_id,
            "added": list(app_ids),
            "removed": [],
        }
        next_version += 1
        logger.info(f"Wrote collection {category!r} ({coll_id}) with {len(app_ids)} games")

    # 4. Persist cloud storage
    try:
        import os as _os
        tmp_cloud = cloud_path.with_suffix(".json.sgm_tmp")
        with tmp_cloud.open("w", encoding="utf-8") as f:
            json.dump(cloud_data, f, separators=(",", ":"))
        _os.replace(tmp_cloud, cloud_path)
        cloud_modified_path.write_text("[]", encoding="utf-8")
        logger.info("Updated cloud-storage-namespace-1.json with Steam collections")
    except (OSError, json.JSONDecodeError) as e:
        logger.warning(f"Failed to write cloud storage collections: {e}")
        return False

    # 5. Update localconfig.vdf user-collections cache (best-effort)
    try:
        content = vdf_path.read_text(encoding="utf-8")
    except OSError as e:
        logger.error(f"Failed to read localconfig.vdf: {e}")
        return True  # cloud write succeeded; localconfig is optional cache

    pattern = re.compile(r'("user-collections"\t+)"(.*?)"(\s*\n)', re.DOTALL)
    match = pattern.search(content)

    if match:
        raw_json = match.group(2)
        # VDF escaping: inner quotes are stored as \" in the file, which Python
        # reads as the two chars \  ". We need to unescape them to parse JSON.
        raw_json = raw_json.replace('\\\\"', '\x01').replace('\\"', '"').replace('\x01', '\\"')
        try:
            collections: dict = json.loads(raw_json)
        except json.JSONDecodeError:
            collections = {}
    else:
        collections = {}

    # Remove old srm-* keys, legacy collection keys, and strip "name" from entries.
    # Also remove any legacy collection IDs that were merged.
    legacy_coll_id_set = {key.replace("user-collections.", "") for key in legacy_coll_ids}
    cleaned: dict = {}
    for k, v in collections.items():
        if k.startswith("srm-"):
            continue  # drop old SRM keys
        if k in legacy_coll_id_set:
            continue  # drop merged legacy collection keys
        if isinstance(v, dict) and "name" in v:
            v = {kk: vv for kk, vv in v.items() if kk != "name"}
        cleaned[k] = v
    cleaned.update(new_collections)
    collections = cleaned

    new_json = json.dumps(collections, separators=(",", ":")).replace('"', '\\"')

    if match:
        new_content = (
            content[: match.start()]
            + match.group(1)
            + f'"{new_json}"'
            + match.group(3)
            + content[match.end():]
        )
    else:
        insert_line = f'\t\t"user-collections"\t\t"{new_json}"\n'
        insert_match = re.search(r'("user-roaming-config-store".*?\n)', content, re.DOTALL)
        if insert_match:
            pos = insert_match.end()
            new_content = content[:pos] + insert_line + content[pos:]
        else:
            new_content = content.rstrip() + "\n" + insert_line

    try:
        import os
        tmp = vdf_path.with_suffix(".vdf.sgm_tmp")
        tmp.write_text(new_content, encoding="utf-8")
        os.replace(tmp, vdf_path)
        logger.info("Updated localconfig.vdf with Steam collections")
    except OSError as e:
        logger.error(f"Failed to write localconfig.vdf: {e}")

    return True



def delete_steam_collections(steam_path: Path,
                             category_names: set,
                             user_id: Optional[str] = None) -> bool:
    """Mark Steam collections as deleted in cloud storage.

    This is the removal counterpart to update_steam_collections().  It marks
    the ``uc-*`` entries for the given category names as ``is_deleted`` so
    Steam stops showing them.

    Args:
        steam_path: Steam installation path.
        category_names: Set of collection names to delete (e.g. {"Atari 2600"}).
        user_id: Specific Steam user ID, or None to auto-detect.

    Returns:
        True on success, False on failure.
    """
    import base64
    import hashlib
    import json
    import re
    import time as _time

    try:
        vdf_path = find_localconfig_vdf(steam_path, user_id)
    except FileNotFoundError as e:
        logger.warning(f"Could not find localconfig.vdf: {e}")
        return False

    cloud_path = vdf_path.parent / "cloudstorage" / "cloud-storage-namespace-1.json"
    cloud_modified_path = (
        vdf_path.parent / "cloudstorage" / "cloud-storage-namespace-1.modified.json"
    )

    if not cloud_path.exists():
        logger.debug(f"Cloud storage not found at {cloud_path}, skipping")
        return False

    try:
        cloud_data: list = json.load(cloud_path.open(encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as e:
        logger.warning(f"Failed to read cloud storage: {e}")
        return False

    def _make_uc_id(category: str) -> str:
        digest = hashlib.sha1(f"sgm-{category}".encode()).digest()
        b64 = base64.urlsafe_b64encode(digest).rstrip(b"=").decode()
        return f"uc-{b64[:16]}"

    def _get_max_version(data: list) -> int:
        v = 1
        for item in data:
            if isinstance(item, list) and len(item) >= 2:
                try:
                    v = max(v, int(str(item[1].get("version", "0"))))
                except (ValueError, TypeError):
                    pass
        return v

    cloud_index: dict[str, int] = {}
    for i, item in enumerate(cloud_data):
        if isinstance(item, list) and len(item) >= 2 and isinstance(item[1], dict):
            cloud_index[item[1].get("key", "")] = i

    next_version = _get_max_version(cloud_data) + 1
    timestamp_ms = int(_time.time() * 1000)

    deleted_count = 0
    for category in category_names:
        coll_id = _make_uc_id(category)
        cloud_key = f"user-collections.{coll_id}"

        deleted_entry = {
            "key": cloud_key,
            "timestamp": timestamp_ms,
            "is_deleted": True,
            "version": str(next_version),
        }

        if cloud_key in cloud_index:
            cloud_data[cloud_index[cloud_key]] = [cloud_key, deleted_entry]
        else:
            # Add a new deleted entry so Steam syncs the deletion
            cloud_data.append([cloud_key, deleted_entry])

        next_version += 1
        deleted_count += 1
        logger.info(f"Marked collection {category!r} as deleted")

    if deleted_count == 0:
        logger.debug("No collections found to delete")
        return True

    try:
        import os as _os
        tmp_cloud = cloud_path.with_suffix(".json.sgm_tmp")
        with tmp_cloud.open("w", encoding="utf-8") as f:
            json.dump(cloud_data, f, separators=(",", ":"))
        _os.replace(tmp_cloud, cloud_path)
        cloud_modified_path.write_text("[]", encoding="utf-8")
        logger.info(f"Deleted {deleted_count} collection(s) from cloud storage")
        return True
    except (OSError, json.JSONDecodeError) as e:
        logger.warning(f"Failed to write cloud storage: {e}")
        return False




@dataclass
class CloudCollection:
    """Represents a Steam collection read from cloud storage."""
    coll_id: str
    name: str
    added: list[int]
    removed: list[int]
    is_deleted: bool = False


def read_cloud_collections(steam_path: Path,
                            user_id: Optional[str] = None
                            ) -> list[CloudCollection]:
    """Read all Steam collections from cloud-storage-namespace-1.json.

    Args:
        steam_path: Steam installation path.
        user_id: Specific Steam user ID, or None to auto-detect.

    Returns:
        List of CloudCollection objects. Deleted collections have is_deleted=True.
    """
    import json
    import re

    try:
        vdf_path = find_localconfig_vdf(steam_path, user_id)
    except FileNotFoundError as e:
        logger.warning(f"Could not find localconfig.vdf: {e}")
        return []

    cloud_path = vdf_path.parent / "cloudstorage" / "cloud-storage-namespace-1.json"
    if not cloud_path.exists():
        return []

    try:
        cloud_data: list = json.load(cloud_path.open(encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as e:
        logger.warning(f"Failed to read cloud storage: {e}")
        return []

    collections: list[CloudCollection] = []
    for item in cloud_data:
        if not isinstance(item, list) or len(item) < 2:
            continue
        key = item[0]
        entry = item[1] if isinstance(item[1], dict) else {}

        if not isinstance(key, str) or not key.startswith("user-collections."):
            continue
        if key.startswith("user-collections.srm-"):
            continue  # skip legacy SRM collections

        coll_id = key.replace("user-collections.", "")
        is_deleted = entry.get("is_deleted", False)

        if is_deleted:
            collections.append(CloudCollection(
                coll_id=coll_id,
                name="",
                added=[],
                removed=[],
                is_deleted=True,
            ))
            continue

        value_str = entry.get("value", "")
        if value_str:
            try:
                value: dict = json.loads(value_str)
            except json.JSONDecodeError:
                continue
        else:
            value = {}

        # Try to get name from value, or fall back to localconfig
        name = value.get("name", "")
        if not name:
            name = _get_collection_name_from_localconfig(vdf_path, coll_id)

        added_raw = value.get("added", [])
        removed_raw = value.get("removed", [])
        added = [int(x) for x in added_raw if isinstance(x, int) or (isinstance(x, str) and x.isdigit())]
        removed = [int(x) for x in removed_raw if isinstance(x, int) or (isinstance(x, str) and x.isdigit())]

        collections.append(CloudCollection(
            coll_id=coll_id,
            name=name,
            added=added,
            removed=removed,
            is_deleted=False,
        ))

    return collections


def _get_collection_name_from_localconfig(vdf_path: Path, coll_id: str) -> str:
    """Look up a collection name from localconfig.vdf user-collections cache."""
    import json
    import re
    try:
        content = vdf_path.read_text(encoding="utf-8")
    except OSError:
        return ""

    pattern = re.compile(r'"user-collections"	+"(.*?)"(\s*\n)', re.DOTALL)
    match = pattern.search(content)
    if not match:
        return ""

    raw_json = match.group(2)
    raw_json = raw_json.replace('\"', '\x01').replace('\"', '"').replace('\x01', '\"')
    try:
        collections: dict = json.loads(raw_json)
    except json.JSONDecodeError:
        return ""

    entry = collections.get(coll_id, {})
    return entry.get("name", "")




def modify_cloud_collection(steam_path: Path,
                              coll_id: str,
                              add_app_ids: Optional[list[int]] = None,
                              remove_app_ids: Optional[list[int]] = None,
                              user_id: Optional[str] = None) -> bool:
    """Add or remove app IDs from a Steam collection in cloud storage.

    Args:
        steam_path: Steam installation path.
        coll_id: Collection ID (e.g. "uc-+6pgB5fyxXlz").
        add_app_ids: List of unsigned int32 app IDs to add.
        remove_app_ids: List of unsigned int32 app IDs to remove.
        user_id: Specific Steam user ID, or None to auto-detect.

    Returns:
        True on success, False on failure.
    """
    import time as _time

    if not add_app_ids and not remove_app_ids:
        logger.debug("No changes specified for collection modification")
        return True

    try:
        vdf_path = find_localconfig_vdf(steam_path, user_id)
    except FileNotFoundError as e:
        logger.warning(f"Could not find localconfig.vdf: {e}")
        return False

    cloud_path = vdf_path.parent / "cloudstorage" / "cloud-storage-namespace-1.json"
    if not cloud_path.exists():
        logger.warning(f"Cloud storage not found at {cloud_path}")
        return False

    try:
        cloud_data: list = json.load(cloud_path.open(encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as e:
        logger.warning(f"Failed to read cloud storage: {e}")
        return False

    # Build index
    cloud_index: dict[str, int] = {}
    for i, item in enumerate(cloud_data):
        if isinstance(item, list) and len(item) >= 2 and isinstance(item[1], dict):
            cloud_index[item[1].get("key", "")] = i

    cloud_key = f"user-collections.{coll_id}"
    next_version = 1
    for item in cloud_data:
        if isinstance(item, list) and len(item) >= 2:
            try:
                v = int(str(item[1].get("version", "0")))
                next_version = max(next_version, v + 1)
            except (ValueError, TypeError):
                pass

    timestamp_ms = int(_time.time() * 1000)

    if cloud_key in cloud_index:
        entry = cloud_data[cloud_index[cloud_key]][1]
        if entry.get("is_deleted"):
            logger.warning(f"Collection {coll_id} is marked as deleted")
            return False

        value_str = entry.get("value", "{}")
        try:
            value: dict = json.loads(value_str)
        except json.JSONDecodeError:
            value = {"id": coll_id, "added": [], "removed": []}

        # Parse existing added/removed
        current_added = [int(x) for x in value.get("added", [])
                         if isinstance(x, int) or (isinstance(x, str) and x.isdigit())]
        current_removed = [int(x) for x in value.get("removed", [])
                           if isinstance(x, int) or (isinstance(x, str) and x.isdigit())]

        # Apply removals first (from both added and removed)
        if remove_app_ids:
            for aid in remove_app_ids:
                if aid in current_added:
                    current_added.remove(aid)
                if aid not in current_removed:
                    current_removed.append(aid)

        # Apply additions (from added, move from removed)
        if add_app_ids:
            for aid in add_app_ids:
                if aid in current_removed:
                    current_removed.remove(aid)
                if aid not in current_added:
                    current_added.append(aid)

        value["added"] = current_added
        value["removed"] = current_removed
    else:
        # Collection doesn't exist, create it
        value = {
            "id": coll_id,
            "name": "",
            "added": add_app_ids or [],
            "removed": remove_app_ids or [],
        }

    new_entry = {
        "key": cloud_key,
        "timestamp": timestamp_ms,
        "value": json.dumps(value, separators=(",", ":")),
        "version": str(next_version),
        "conflictResolutionMethod": "custom",
        "strMethodId": "union-collections",
    }

    if cloud_key in cloud_index:
        cloud_data[cloud_index[cloud_key]] = [cloud_key, new_entry]
    else:
        cloud_data.append([cloud_key, new_entry])
        cloud_index[cloud_key] = len(cloud_data) - 1

    # Update localconfig cache too
    _update_localconfig_collection(vdf_path, coll_id, value)

    try:
        import os as _os
        tmp_cloud = cloud_path.with_suffix(".json.sgm_tmp")
        with tmp_cloud.open("w", encoding="utf-8") as f:
            json.dump(cloud_data, f, separators=(",", ":"))
        _os.replace(tmp_cloud, cloud_path)
        cloud_modified_path = vdf_path.parent / "cloudstorage" / "cloud-storage-namespace-1.modified.json"
        cloud_modified_path.write_text("[]", encoding="utf-8")
        logger.info(f"Modified collection {coll_id}")
        return True
    except (OSError, json.JSONDecodeError) as e:
        logger.warning(f"Failed to write cloud storage: {e}")
        return False


def _update_localconfig_collection(vdf_path: Path, coll_id: str, value: dict) -> None:
    """Update localconfig.vdf user-collections cache for a specific collection."""
    import re
    try:
        content = vdf_path.read_text(encoding="utf-8")
    except OSError:
        return

    pattern = re.compile(r'"user-collections"	+"(.*?)"(\s*)', re.DOTALL)
    match = pattern.search(content)

    if match:
        raw_json = match.group(2)
        raw_json = raw_json.replace('\"', '
').replace('\"', '"').replace('
', '\"')
        try:
            collections: dict = json.loads(raw_json)
        except json.JSONDecodeError:
            collections = {}
    else:
        collections = {}

    collections[coll_id] = {
        "id": value.get("id", coll_id),
        "added": value.get("added", []),
        "removed": value.get("removed", []),
    }

    new_json = json.dumps(collections, separators=(",", ":")).replace('"', '\"')

    if match:
        new_content = (
            content[: match.start()]
            + match.group(1)
            + f'"{new_json}"'
            + match.group(3)
            + content[match.end():]
        )
    else:
        insert_line = f'\t\t"user-collections"\t\t"{new_json}"\n'
        insert_match = re.search(r'("user-roaming-config-store".*?\n)', content, re.DOTALL)
        if insert_match:
            pos = insert_match.end()
            new_content = content[:pos] + insert_line + content[pos:]
        else:
            new_content = content.rstrip() + "\n" + insert_line

    try:
        import os
        tmp = vdf_path.with_suffix(".vdf.sgm_tmp")
        tmp.write_text(new_content, encoding="utf-8")
        os.replace(tmp, vdf_path)
        logger.debug(f"Updated localconfig.vdf for collection {coll_id}")
    except OSError as e:
        logger.warning(f"Failed to update localconfig.vdf: {e}")


def delete_cloud_collection(steam_path: Path,
                              coll_id: str,
                              user_id: Optional[str] = None) -> bool:
    """Mark a Steam collection as deleted in cloud storage.

    Args:
        steam_path: Steam installation path.
        coll_id: Collection ID to delete.
        user_id: Specific Steam user ID, or None to auto-detect.

    Returns:
        True on success, False on failure.
    """
    import time as _time

    try:
        vdf_path = find_localconfig_vdf(steam_path, user_id)
    except FileNotFoundError as e:
        logger.warning(f"Could not find localconfig.vdf: {e}")
        return False

    cloud_path = vdf_path.parent / "cloudstorage" / "cloud-storage-namespace-1.json"
    if not cloud_path.exists():
        return False

    try:
        cloud_data: list = json.load(cloud_path.open(encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as e:
        logger.warning(f"Failed to read cloud storage: {e}")
        return False

    cloud_key = f"user-collections.{coll_id}"
    cloud_index: dict[str, int] = {}
    for i, item in enumerate(cloud_data):
        if isinstance(item, list) and len(item) >= 2 and isinstance(item[1], dict):
            cloud_index[item[1].get("key", "")] = i

    next_version = 1
    for item in cloud_data:
        if isinstance(item, list) and len(item) >= 2:
            try:
                v = int(str(item[1].get("version", "0")))
                next_version = max(next_version, v + 1)
            except (ValueError, TypeError):
                pass

    timestamp_ms = int(_time.time() * 1000)

    deleted_entry = {
        "key": cloud_key,
        "timestamp": timestamp_ms,
        "is_deleted": True,
        "version": str(next_version),
    }

    if cloud_key in cloud_index:
        cloud_data[cloud_index[cloud_key]] = [cloud_key, deleted_entry]
    else:
        cloud_data.append([cloud_key, deleted_entry])

    try:
        import os as _os
        tmp_cloud = cloud_path.with_suffix(".json.sgm_tmp")
        with tmp_cloud.open("w", encoding="utf-8") as f:
            json.dump(cloud_data, f, separators=(",", ":"))
        _os.replace(tmp_cloud, cloud_path)
        cloud_modified_path = vdf_path.parent / "cloudstorage" / "cloud-storage-namespace-1.modified.json"
        cloud_modified_path.write_text("[]", encoding="utf-8")
        logger.info(f"Deleted collection {coll_id}")
        return True
    except (OSError, json.JSONDecodeError) as e:
        logger.warning(f"Failed to write cloud storage: {e}")
        return False


# Emulator Plugin Integration
def create_shortcut_with_plugin(rom_path: str, rom_name: str, system_def,
                                emulator_plugin=None,
                                launch_options: str = "",
                                tags: Optional[Dict[str, str]] = None) -> SteamShortcut:
    """Create a SteamShortcut using an EmulatorPlugin.

    This function provides a clean interface between ROM scanning and
    shortcut creation, supporting both the legacy EmulatorConfig system
    and the new EmulatorPlugin system.

    Args:
        rom_path: Path to the ROM file or title ID (for title-ID mode).
        rom_name: Steam display name for the game.
        system_def: SystemDef/SystemPlugin for the system.
        emulator_plugin: Optional EmulatorPlugin instance.
            If None, falls back to system_def.emulator (legacy).
        launch_options: Additional launch arguments.
        tags: Dictionary of tag indices to values.

    Returns:
        SteamShortcut object ready to be added to shortcuts.vdf.
    """
    from pathlib import Path as _Path

    # Handle both SystemPlugin (new) and SystemDef wrapper (backward compat)
    launch_mode = "rom"
    if hasattr(system_def, 'launch_mode'):
        launch_mode = system_def.launch_mode
    elif hasattr(system_def, 'emulator') and system_def.emulator:
        launch_mode = getattr(system_def.emulator, 'launch_mode', 'rom')

    # Determine the executable and launch mode
    use_title_id = (rom_path and len(rom_path) > 10 and launch_mode == "title_id")

    if emulator_plugin:
        # Use the new plugin system
        exe = emulator_plugin.get_steam_exe(rom_path)
    elif use_title_id:
        # Legacy Vita3K title-ID mode
        if hasattr(system_def, 'emulator') and system_def.emulator:
            exe = system_def.emulator.get_steam_exe(rom_path)
        else:
            exe = rom_path
    else:
        # Legacy ROM file mode
        if hasattr(system_def, 'emulator') and system_def.emulator:
            exe = system_def.emulator.get_steam_exe(str(_Path(rom_path)))
        else:
            exe = str(_Path(rom_path))

    # Generate IDs (must use the same exe format as SRM for compatibility)
    appid = generate_shortcut_id(exe, rom_name)

    # Determine start directory
    if use_title_id:
        # For Vita3K: start_dir is the emulator binary directory
        if emulator_plugin:
            exe_path = emulator_plugin.find_executable()
        elif hasattr(system_def, 'emulator') and system_def.emulator:
            exe_path = getattr(system_def.emulator, 'emulator', 'Vita3K')
        else:
            exe_path = 'Vita3K'
        start_dir = f'"{_Path(exe_path).parent}"'
    else:
        start_dir = f'"{str(_Path(rom_path).parent)}"'

    # Build tags - get steam_category
    if tags is None:
        if hasattr(system_def, 'steam_category'):
            category = system_def.steam_category
        elif hasattr(system_def, 'get_steam_category'):
            category = system_def.get_steam_category()
        else:
            category = "ROM"
        tags = {"0": category}

    sc = SteamShortcut(
        appid=appid,
        appname=rom_name,
        exe=exe,
        start_dir=start_dir,
        launch_options=launch_options,
        tags=tags,
    )
    return sc