Merge branch 'master' into feature/device-flow-authorization

Author: gantoine

.github/workflows/pytest.yml

@@ -71,7 +71,11 @@ jobs:
       - name: Initiate MariaDB database
         if: matrix.db == 'mariadb'
         run: |
-          mysql --host 127.0.0.1 --port ${{ job.services.mariadb.ports['3306'] }} -uroot -ppasswd -e "GRANT ALL PRIVILEGES ON romm_test.* TO 'romm_test'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;"
+          # Grant on the `romm_test%` namespace so the test user can create the
+          # per-worker databases (romm_test_gw0, ...) used under pytest-xdist,
+          # without granting it global privileges. The backticks are escaped so
+          # the shell doesn't treat them as command substitution.
+          mysql --host 127.0.0.1 --port ${{ job.services.mariadb.ports['3306'] }} -uroot -ppasswd -e "GRANT ALL PRIVILEGES ON \`romm\_test%\`.* TO 'romm_test'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;"
 
       - name: Run python tests
         env:
@@ -80,9 +84,11 @@ jobs:
           ROMM_DB_DRIVER: ${{ matrix.db }}
           REDIS_HOST: 127.0.0.1
           REDIS_PORT: ${{ job.services.valkey.ports['6379'] }}
+          HYPOTHESIS_PROFILE: ci
         run: |
           cd backend
-          uv run pytest -vv --maxfail=10 --junitxml=pytest-report.xml --cov --cov-report xml:coverage.xml --cov-config=.coveragerc .
+          # GitHub-hosted Linux runners have 4 vCPUs; run one worker per core.
+          uv run pytest -n 4 -vv --maxfail=10 --junitxml=pytest-report.xml --cov --cov-report xml:coverage.xml --cov-config=.coveragerc .
 
       - name: Publish test results
         uses: EnricoMi/publish-unit-test-result-action/linux@v2.20.0

.gitignore

@@ -54,6 +54,7 @@ romm_mock
 backend/romm_test/resources
 backend/romm_test/logs
 .pytest_cache
+.hypothesis
 /romm_test
 .coverage
 pytest-report.xml

backend/adapters/services/igdb.py

@@ -495,7 +495,7 @@ class SlugToIGDB(TypedDict):
         "category": "Computer",
         "family_name": "Atari",
         "family_slug": "atari",
-        "generation": -1,
+        "generation": 4,
         "id": 63,
         "name": "Atari ST/STE",
         "slug": "atari-st",

backend/alembic/env.py

@@ -41,6 +41,15 @@ def include_object(object, name, type_, reflected, compare_to):
     ]:  # Virtual table
         return False
 
+    # Skip DB-specific search indexes in autogenerate
+    # to avoid false drop/create operations
+    if type_ == "index" and name in (
+        "idx_roms_name_fs_name_fulltext",
+        "idx_roms_name_trgm",
+        "idx_roms_fs_name_trgm",
+    ):
+        return False
+
     return True
 
 

backend/alembic/versions/0084_add_roms_search_index.py

@@ -0,0 +1,137 @@
+"""Speed up gallery search and name ordering on the roms table
+
+This migration adds the database structures the gallery relies on to search
+and sort large libraries without full table scans:
+
+- Search indexes on roms.name and roms.fs_name, tailored per backend so the
+  search query stays index-backed: a FULLTEXT index on MySQL/MariaDB, and
+  pg_trgm GIN indexes on PostgreSQL (installing the pg_trgm extension first).
+- A plain idx_roms_name index on roms.name to accelerate name lookups and
+  range scans.
+- A precomputed, indexed name_sort_key column for natural-sort ordering.
+  Sorting by name previously ran a per-row regexp (strip leading articles,
+  zero-pad numbers) that no index could cover, forcing a full sort on every
+  page. The key is now stored on write and backfilled here, so ordering by
+  name — including deep-offset pages — uses idx_roms_name_sort_key.
+
+downgrade() drops every object created here in reverse order, leaving the
+pg_trgm extension in place since other objects may depend on it.
+
+Revision ID: 0084_add_roms_search_index
+Revises: 0083_rom_category_soundtrack
+Create Date: 2026-06-16 00:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+from models.rom import NAME_SORT_KEY_MAX_LENGTH, compute_name_sort_key
+from utils.database import is_mariadb, is_mysql, is_postgresql
+
+# revision identifiers, used by Alembic.
+revision = "0084_add_roms_search_index"
+down_revision = "0083_rom_category_soundtrack"
+branch_labels = None
+depends_on = None
+
+FULLTEXT_INDEX_NAME = "idx_roms_name_fs_name_fulltext"
+PG_NAME_INDEX = "idx_roms_name_trgm"
+PG_FS_NAME_INDEX = "idx_roms_fs_name_trgm"
+
+_BACKFILL_BATCH = 1000
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+
+    # 1. DB-specific search indexes on roms.name and roms.fs_name.
+    if is_mysql(bind) or is_mariadb(bind):
+        op.execute(
+            sa.text(
+                f"CREATE FULLTEXT INDEX {FULLTEXT_INDEX_NAME} "
+                "ON roms (name, fs_name)"
+            )
+        )
+    elif is_postgresql(bind):
+        # pg_trgm is a trusted extension since PostgreSQL 13, so a non-superuser
+        # with CREATE on the database can install it.
+        op.execute(sa.text("CREATE EXTENSION IF NOT EXISTS pg_trgm"))
+        op.execute(
+            sa.text(
+                f"CREATE INDEX IF NOT EXISTS {PG_NAME_INDEX} "
+                "ON roms USING gin (name gin_trgm_ops)"
+            )
+        )
+        op.execute(
+            sa.text(
+                f"CREATE INDEX IF NOT EXISTS {PG_FS_NAME_INDEX} "
+                "ON roms USING gin (fs_name gin_trgm_ops)"
+            )
+        )
+
+    # 2. Plain index on roms.name.
+    with op.batch_alter_table("roms", schema=None) as batch_op:
+        batch_op.create_index(
+            "idx_roms_name",
+            ["name"],
+            unique=False,
+            if_not_exists=True,
+        )
+
+    # 3. Precomputed name_sort_key column for natural-sort ordering.
+    op.add_column(
+        "roms",
+        sa.Column(
+            "name_sort_key",
+            sa.String(length=NAME_SORT_KEY_MAX_LENGTH),
+            nullable=True,
+        ),
+        if_not_exists=True,
+    )
+
+    roms = sa.table(
+        "roms",
+        sa.column("id", sa.Integer),
+        sa.column("name", sa.String),
+        sa.column("name_sort_key", sa.String),
+    )
+
+    result = bind.execute(sa.select(roms.c.id, roms.c.name))
+    update_stmt = (
+        roms.update()
+        .where(roms.c.id == sa.bindparam("_id"))
+        .values(name_sort_key=sa.bindparam("_key"))
+    )
+    while True:
+        batch = result.fetchmany(_BACKFILL_BATCH)
+        if not batch:
+            break
+        bind.execute(
+            update_stmt,
+            [{"_id": row.id, "_key": compute_name_sort_key(row.name)} for row in batch],
+        )
+
+    op.create_index(
+        "idx_roms_name_sort_key", "roms", ["name_sort_key"], if_not_exists=True
+    )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+
+    # 3. name_sort_key column and its index.
+    op.drop_index("idx_roms_name_sort_key", table_name="roms", if_exists=True)
+    op.drop_column("roms", "name_sort_key", if_exists=True)
+
+    # 2. Plain index on roms.name.
+    with op.batch_alter_table("roms", schema=None) as batch_op:
+        batch_op.drop_index("idx_roms_name", if_exists=True)
+
+    # 1. DB-specific search indexes.
+    if is_mysql(bind) or is_mariadb(bind):
+        op.execute(sa.text(f"DROP INDEX {FULLTEXT_INDEX_NAME} ON roms"))
+    elif is_postgresql(bind):
+        # Leave the pg_trgm extension in place; other objects may depend on it.
+        op.execute(sa.text(f"DROP INDEX IF EXISTS {PG_FS_NAME_INDEX}"))
+        op.execute(sa.text(f"DROP INDEX IF EXISTS {PG_NAME_INDEX}"))

backend/alembic/versions/0085_rom_category_screenshot.py

@@ -0,0 +1,81 @@
+"""Update rom_file.category column enum to include screenshots.
+
+Revision ID: 0085_rom_category_screenshot
+Revises: 0084_add_roms_search_index
+Create Date: 2026-06-17 00:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+from utils.database import is_postgresql
+
+# revision identifiers, used by Alembic.
+revision = "0085_rom_category_screenshot"
+down_revision = "0084_add_roms_search_index"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    connection = op.get_bind()
+
+    if is_postgresql(connection):
+        # `ALTER TYPE ... ADD VALUE` must run outside a transaction in PostgreSQL.
+        # autocommit_block breaks out of alembic's wrapping transaction for
+        # exactly this operation.
+        with op.get_context().autocommit_block():
+            op.execute(
+                "ALTER TYPE romfilecategory ADD VALUE IF NOT EXISTS 'SCREENSHOT'"
+            )
+    else:
+        rom_file_category_enum = sa.Enum(
+            "GAME",
+            "DLC",
+            "HACK",
+            "MANUAL",
+            "PATCH",
+            "UPDATE",
+            "MOD",
+            "DEMO",
+            "TRANSLATION",
+            "PROTOTYPE",
+            "CHEAT",
+            "SOUNDTRACK",
+            "SCREENSHOT",
+            name="romfilecategory",
+        )
+        with op.batch_alter_table("rom_files", schema=None) as batch_op:
+            batch_op.alter_column(
+                "category", type_=rom_file_category_enum, nullable=True
+            )
+
+
+def downgrade() -> None:
+    # PostgreSQL cannot remove enum values, so the downgrade is a no-op there.
+    # On other backends, narrow the enum back to its pre-screenshot set. Any
+    # rows still holding the new value would block this, but the upload flow
+    # is additive and a re-upgrade is a no-op.
+    connection = op.get_bind()
+
+    if is_postgresql(connection):
+        return
+
+    rom_file_category_enum = sa.Enum(
+        "GAME",
+        "DLC",
+        "HACK",
+        "MANUAL",
+        "PATCH",
+        "UPDATE",
+        "MOD",
+        "DEMO",
+        "TRANSLATION",
+        "PROTOTYPE",
+        "CHEAT",
+        "SOUNDTRACK",
+        name="romfilecategory",
+    )
+    with op.batch_alter_table("rom_files", schema=None) as batch_op:
+        batch_op.alter_column("category", type_=rom_file_category_enum, nullable=True)

backend/alembic/versions/0086_screenshot_visibility.py

@@ -0,0 +1,49 @@
+"""Add is_gallery + is_public flags to screenshots for per-user community
+gallery screenshots (distinct from auto-captured save/state thumbnails).
+
+Revision ID: 0086_screenshot_visibility
+Revises: 0085_rom_category_screenshot
+Create Date: 2026-06-18 00:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0086_screenshot_visibility"
+down_revision = "0085_rom_category_screenshot"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "screenshots",
+        sa.Column(
+            "is_gallery",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+        if_not_exists=True,
+    )
+    op.add_column(
+        "screenshots",
+        sa.Column(
+            "is_public",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+        if_not_exists=True,
+    )
+    op.create_index(
+        "idx_screenshots_public", "screenshots", ["is_public"], if_not_exists=True
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("idx_screenshots_public", table_name="screenshots", if_exists=True)
+    op.drop_column("screenshots", "is_public", if_exists=True)
+    op.drop_column("screenshots", "is_gallery", if_exists=True)

backend/alembic/versions/0087_save_state_visibility.py

@@ -0,0 +1,38 @@
+"""Add is_public flag to saves and states for per-user community sharing
+(mirrors screenshots/notes visibility).
+
+Revision ID: 0087_save_state_visibility
+Revises: 0086_screenshot_visibility
+Create Date: 2026-06-20 00:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0087_save_state_visibility"
+down_revision = "0086_screenshot_visibility"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    for table in ("saves", "states"):
+        op.add_column(
+            table,
+            sa.Column(
+                "is_public",
+                sa.Boolean(),
+                nullable=False,
+                server_default=sa.text("false"),
+            ),
+            if_not_exists=True,
+        )
+        op.create_index(f"idx_{table}_public", table, ["is_public"], if_not_exists=True)
+
+
+def downgrade() -> None:
+    for table in ("saves", "states"):
+        op.drop_index(f"idx_{table}_public", table_name=table, if_exists=True)
+        op.drop_column(table, "is_public", if_exists=True)

backend/config/__init__.py

@@ -239,6 +239,7 @@ def _get_env(var: str, fallback: str | None = None) -> str | None:
 
 # FRONTEND
 KIOSK_MODE: Final[bool] = safe_str_to_bool(_get_env("KIOSK_MODE"))
+DISABLE_LOGS_VIEWER: Final[bool] = safe_str_to_bool(_get_env("DISABLE_LOGS_VIEWER"))
 
 # LOGGING
 LOGLEVEL: Final[str] = _get_env("LOGLEVEL", "INFO").upper()

backend/conftest.py

@@ -0,0 +1,15 @@
+import os
+
+# When running under pytest-xdist, give each worker its own database so the
+# autouse `clear_database` fixture in one worker can't wipe rows another worker
+# is mid-test with. This must run before any application module (config /
+# database handlers) is imported, so the engine built at import time binds to
+# the per-worker name. As the rootdir conftest, this file is imported before
+# `tests/conftest.py` (which imports those modules).
+#
+# The Redis cache needs no equivalent handling: under pytest it is an in-process
+# FakeRedis, so each worker process is already isolated.
+_xdist_worker = os.environ.get("PYTEST_XDIST_WORKER")
+if _xdist_worker:
+    _base_db_name = os.environ.get("DB_NAME", "romm_test")
+    os.environ["DB_NAME"] = f"{_base_db_name}_{_xdist_worker}"