perf(backend): mirror ROM filter values into a narrow trigger-maintained table

backend/alembic/env.py

@@ -11,7 +11,7 @@
 from models.collection import VirtualCollection
 from models.firmware import Firmware  # noqa
 from models.platform import Platform  # noqa
-from models.rom import Rom, RomMetadata, SiblingRom  # noqa
+from models.rom import Rom, RomFacets, RomMetadata, SiblingRom  # noqa
 from models.user import User  # noqa
 
 # this is the Alembic Config object, which provides

backend/alembic/versions/0100_roms_facets_table.py

@@ -0,0 +1,146 @@
+"""Mirror the ROM filter values into a narrow, trigger-maintained table
+
+Building the filter dropdowns aggregated the facet columns straight off
+``roms``. Those columns are cheap to compute since 0098, but they live inline
+with the raw provider-metadata blobs, so the aggregation still had to read
+every row of a multi-gigabyte table: on a ~90k-game library that is seconds of
+pure disk reading on any database whose buffer pool is smaller than the table
+(the stock 128-256 MB defaults always are).
+
+``roms_facets`` holds only the filter values (a few MB for the same library) and
+is kept in sync by AFTER INSERT/UPDATE triggers on ``roms``, so every write path
+(ORM, bulk update, scan) stays consistent without an application hook. Deletes
+ride the ``ON DELETE CASCADE`` foreign key.
+
+Revision ID: 0100_roms_facets_table
+Revises: 0099_platform_description
+Create Date: 2026-07-22 00:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op  # type: ignore[attr-defined]
+
+from utils.database import CustomJSON, is_postgresql
+
+# revision identifiers, used by Alembic.
+revision = "0100_roms_facets_table"
+down_revision = "0099_platform_description"
+branch_labels = None
+depends_on = None
+
+
+# (roms_facets column, roms source column)
+_MIRRORED_COLUMNS = [
+    ("platform_id", "platform_id"),
+    ("genres", "generated_genres"),
+    ("franchises", "generated_franchises"),
+    ("collections", "generated_collections"),
+    ("companies", "generated_companies"),
+    ("game_modes", "generated_game_modes"),
+    ("age_ratings", "generated_age_ratings"),
+    ("player_count", "generated_player_count"),
+    ("regions", "regions"),
+    ("languages", "languages"),
+    ("tags", "tags"),
+]
+
+_TARGET_COLUMNS = ", ".join(target for target, _ in _MIRRORED_COLUMNS)
+_SOURCE_COLUMNS = ", ".join(source for _, source in _MIRRORED_COLUMNS)
+
+_BACKFILL_SQL = (
+    f"INSERT INTO roms_facets (rom_id, {_TARGET_COLUMNS})\n"  # nosec B608
+    f"SELECT id, {_SOURCE_COLUMNS} FROM roms"
+)
+
+_MYSQL_UPSERT_BODY = (
+    f"INSERT INTO roms_facets (rom_id, {_TARGET_COLUMNS})\n"  # nosec B608
+    + "VALUES (NEW.id, "
+    + ", ".join(f"NEW.{source}" for _, source in _MIRRORED_COLUMNS)
+    + ")\nON DUPLICATE KEY UPDATE\n"
+    + ",\n".join(f"{target} = VALUES({target})" for target, _ in _MIRRORED_COLUMNS)
+    + ",\nupdated_at = CURRENT_TIMESTAMP"
+)
+
+_MYSQL_TRIGGERS = {
+    "roms_facets_after_insert": "AFTER INSERT",
+    "roms_facets_after_update": "AFTER UPDATE",
+}
+
+_POSTGRES_SYNC_FN = f"""
+CREATE OR REPLACE FUNCTION romm_sync_rom_facets() RETURNS trigger
+    LANGUAGE plpgsql AS $$
+BEGIN
+    INSERT INTO roms_facets (rom_id, {_TARGET_COLUMNS})
+    VALUES (NEW.id, {", ".join(f"NEW.{source}" for _, source in _MIRRORED_COLUMNS)})
+    ON CONFLICT (rom_id) DO UPDATE SET
+        {", ".join(f"{target} = EXCLUDED.{target}" for target, _ in _MIRRORED_COLUMNS)},
+        updated_at = NOW();
+    RETURN NULL;
+END $$
+"""  # nosec B608
+
+_POSTGRES_TRIGGER = """
+CREATE TRIGGER roms_facets_sync AFTER INSERT OR UPDATE ON roms
+    FOR EACH ROW EXECUTE FUNCTION romm_sync_rom_facets()
+"""
+
+
+def upgrade() -> None:
+    op.create_table(
+        "roms_facets",
+        sa.Column(
+            "rom_id",
+            sa.Integer(),
+            sa.ForeignKey("roms.id", ondelete="CASCADE"),
+            primary_key=True,
+        ),
+        sa.Column("platform_id", sa.Integer(), nullable=False),
+        sa.Column("genres", CustomJSON(), nullable=True),
+        sa.Column("franchises", CustomJSON(), nullable=True),
+        sa.Column("collections", CustomJSON(), nullable=True),
+        sa.Column("companies", CustomJSON(), nullable=True),
+        sa.Column("game_modes", CustomJSON(), nullable=True),
+        sa.Column("age_ratings", CustomJSON(), nullable=True),
+        sa.Column("player_count", sa.String(length=100), nullable=True),
+        sa.Column("regions", CustomJSON(), nullable=True),
+        sa.Column("languages", CustomJSON(), nullable=True),
+        sa.Column("tags", CustomJSON(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+    )
+    op.create_index("idx_roms_facets_platform_id", "roms_facets", ["platform_id"])
+
+    op.execute(_BACKFILL_SQL)
+
+    if is_postgresql(op.get_bind()):
+        op.execute(_POSTGRES_SYNC_FN)
+        op.execute(_POSTGRES_TRIGGER)
+    else:
+        for name, timing in _MYSQL_TRIGGERS.items():
+            op.execute(
+                f"CREATE TRIGGER {name} {timing} ON roms\n"
+                f"FOR EACH ROW\n{_MYSQL_UPSERT_BODY}"
+            )
+
+
+def downgrade() -> None:
+    if is_postgresql(op.get_bind()):
+        op.execute("DROP TRIGGER IF EXISTS roms_facets_sync ON roms")
+        op.execute("DROP FUNCTION IF EXISTS romm_sync_rom_facets()")
+    else:
+        for name in _MYSQL_TRIGGERS:
+            op.execute(f"DROP TRIGGER IF EXISTS {name}")
+
+    op.drop_index("idx_roms_facets_platform_id", table_name="roms_facets")
+    op.drop_table("roms_facets")

backend/handler/database/roms_handler.py

@@ -46,6 +46,7 @@
 from models.rom import (
     METADATA_SOURCE_COLUMNS,
     Rom,
+    RomFacets,
     RomFile,
     RomFileCategory,
     RomMetadata,
@@ -126,6 +127,23 @@
 # 3 is the default minimum size in InnoDB
 FULLTEXT_MIN_TOKEN_SIZE = 3
 
+# Filter dropdowns read the narrow `roms_facets` mirror instead of `roms`,
+# whose rows carry the raw metadata blobs. Column order matches the unpacking
+# in `_collect_filter_values`.
+_FILTER_VALUES_SELECT = select(
+    RomFacets.genres,
+    RomFacets.franchises,
+    RomFacets.collections,
+    RomFacets.companies,
+    RomFacets.game_modes,
+    RomFacets.age_ratings,
+    RomFacets.player_count,
+    RomFacets.regions,
+    RomFacets.languages,
+    RomFacets.tags,
+    RomFacets.platform_id,
+)
+
 # Cached ROM filter values (genres/franchises/etc.) so it doesn't get
 # recomputed on every call to /api/roms
 ROM_FILTERS_CACHE_VERSION_KEY = "filter_values:ver"
@@ -2347,24 +2365,7 @@ def with_filter_values(
 
         ids_subq = query.order_by(None).with_only_columns(Rom.id).scalar_subquery()  # type: ignore
 
-        statement = (
-            select(
-                RomMetadata.genres,
-                RomMetadata.franchises,
-                RomMetadata.collections,
-                RomMetadata.companies,
-                RomMetadata.game_modes,
-                RomMetadata.age_ratings,
-                RomMetadata.player_count,
-                Rom.regions,
-                Rom.languages,
-                Rom.tags,
-                Rom.platform_id,
-            )
-            .select_from(Rom)
-            .join(RomMetadata, Rom.id == RomMetadata.rom_id)
-            .where(Rom.id.in_(ids_subq))
-        )
+        statement = _FILTER_VALUES_SELECT.where(RomFacets.rom_id.in_(ids_subq))
 
         result = self._collect_filter_values(session, statement)
         if redis_key is not None and version is not None:
@@ -2379,18 +2380,4 @@ def get_rom_filters(
         """
         Returns all filter values across all ROM metadata
         """
-        statement = select(
-            RomMetadata.genres,
-            RomMetadata.franchises,
-            RomMetadata.collections,
-            RomMetadata.companies,
-            RomMetadata.game_modes,
-            RomMetadata.age_ratings,
-            RomMetadata.player_count,
-            Rom.regions,
-            Rom.languages,
-            Rom.tags,
-            Rom.platform_id,
-        )
-
-        return self._collect_filter_values(session, statement)
+        return self._collect_filter_values(session, _FILTER_VALUES_SELECT)

backend/models/rom.py

@@ -234,6 +234,44 @@ class RomMetadata(BaseModel):
     rom: Mapped[Rom] = relationship(lazy="joined", back_populates="metadatum")
 
 
+class RomFacets(BaseModel):
+    """Narrow mirror of the per-ROM values that back the filter dropdowns.
+
+    The same values live on `roms` (as STORED generated columns, plus the
+    region/language/tag columns), but those rows also carry the raw provider
+    metadata blobs, so aggregating them reads the whole multi-gigabyte table.
+    This table holds a few MB of the same data and is kept in sync by triggers
+    on `roms`, so no write path has to remember to update it.
+    """
+
+    __tablename__ = "roms_facets"
+
+    __table_args__ = (Index("idx_roms_facets_platform_id", "platform_id"),)
+
+    rom_id: Mapped[int] = mapped_column(
+        ForeignKey("roms.id", ondelete="CASCADE"), primary_key=True
+    )
+    platform_id: Mapped[int] = mapped_column(Integer(), nullable=False)
+
+    genres: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[])
+    franchises: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[])
+    collections: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[])
+    companies: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[])
+    game_modes: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[])
+    age_ratings: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[])
+    player_count: Mapped[str | None] = mapped_column(String(length=100), default="1")
+    regions: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[])
+    languages: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[])
+    tags: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[])
+
+    created_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=True), server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=True), server_default=func.now()
+    )
+
+
 class Rom(BaseModel):
     __tablename__ = "roms"
 

backend/tests/handler/database/test_rom_facets.py

@@ -0,0 +1,56 @@
+"""Checks for the `roms_facets` mirror that backs the filter dropdowns.
+
+The table holds a copy of each ROM's filter values (migration 0100) and is
+maintained by database triggers on `roms`, not by application code, so these
+tests write through the normal handlers and assert the mirror follows.
+"""
+
+from sqlalchemy import select
+
+from handler.database import db_rom_handler
+from handler.database.base_handler import sync_session
+from models.rom import Rom, RomFacets
+
+
+def _facets(rom_id: int) -> RomFacets | None:
+    with sync_session.begin() as session:
+        return session.scalar(select(RomFacets).where(RomFacets.rom_id == rom_id))
+
+
+class TestRomFacets:
+    def test_insert_mirrors_the_rom(self, rom: Rom):
+        facets = _facets(rom.id)
+        assert facets is not None
+        assert facets.platform_id == rom.platform_id
+
+    def test_update_mirrors_derived_and_raw_values(self, rom: Rom):
+        db_rom_handler.update_rom(
+            rom.id,
+            {
+                "igdb_metadata": {"genres": ["Action", "RPG"], "franchises": ["Zelda"]},
+                "regions": ["USA"],
+                "tags": ["Proto"],
+            },
+        )
+
+        facets = _facets(rom.id)
+        assert facets is not None
+        assert facets.genres == ["Action", "RPG"]
+        assert facets.franchises == ["Zelda"]
+        assert facets.regions == ["USA"]
+        assert facets.tags == ["Proto"]
+
+    def test_delete_cascades(self, rom: Rom):
+        rom_id = rom.id
+        db_rom_handler.delete_rom(rom_id)
+
+        assert _facets(rom_id) is None
+
+    def test_filter_values_follow_a_metadata_edit(self, rom: Rom):
+        db_rom_handler.update_rom(
+            rom.id, {"igdb_metadata": {"genres": ["Puzzle"]}, "languages": ["En"]}
+        )
+
+        filters = db_rom_handler.get_rom_filters()
+        assert "Puzzle" in filters["genres"]
+        assert "En" in filters["languages"]

backend/tools/generate_test_data.py

@@ -60,6 +60,7 @@
 # NOTE: `roms_metadata` and `sibling_roms` are DB VIEWS, not tables: the
 # aggregated metadata and sibling links are derived from each rom's provider
 # columns, so we populate those columns instead of inserting into the views.
+# `roms_facets` is a real table, but triggers on `roms` fill it in.
 from models.rom import (  # noqa: E402
     Rom,
     RomFile,