Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion backend/endpoints/platform.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from logger.logger import log
from models.permission import PermAction, PermEntity
from models.platform import Platform
from utils.platforms import get_supported_platforms
from utils.platforms import get_filesystem_platforms, get_supported_platforms
from utils.router import APIRouter

router = APIRouter(
Expand Down Expand Up @@ -98,6 +98,18 @@ def get_supported_platforms_endpoint(request: Request) -> list[PlatformSchema]:
return get_supported_platforms()


@protected_route(router.get, "/filesystem", [Scope.PLATFORMS_READ])
async def get_filesystem_platforms_endpoint(request: Request) -> list[PlatformSchema]:
"""Retrieve platform folders on disk that have no database row yet.

These folders (created for a platform but not scanned yet) are invisible
to the scan picker because they have no ROMs imported, so they carry no
database row. Returning them lets the UI offer a first scan for them.
"""

return await get_filesystem_platforms()


@protected_route(
router.get,
"/{id}",
Expand Down
27 changes: 23 additions & 4 deletions backend/endpoints/sockets/scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -700,6 +700,7 @@ async def scan_platforms(
roms_ids: list[int] | None = None,
launchbox_remote_enabled: bool = True,
playmatch_enabled: bool = True,
platform_fs_slugs: list[str] | None = None,
) -> ScanStats:
"""Scan all the listed platforms and fetch metadata from different sources

Expand All @@ -708,10 +709,15 @@ async def scan_platforms(
metadata_sources (list[str]): List of metadata sources to be used
scan_type (ScanType): Type of scan to be performed.
roms_ids (list[int], optional): List of selected roms to be scanned.
platform_fs_slugs (list[str], optional): Filesystem slugs of folders to
scan that have no database row yet (never-scanned platforms).
"""
if not roms_ids:
roms_ids = []

if not platform_fs_slugs:
platform_fs_slugs = []

socket_manager = _get_socket_manager()
scan_stats = ScanStats()

Expand All @@ -738,10 +744,20 @@ async def scan_platforms(
db_platforms = db_platform_handler.get_platforms()
db_platforms_by_slug = {p.fs_slug: p for p in db_platforms}

platform_list = [
p.fs_slug for p in db_platforms if p.id in platform_ids
] or fs_platforms
platform_list = sorted(platform_list)
# Selected platforms arrive as database ids (existing platforms) and/or
# filesystem slugs (a folder can be an existing platform or one on disk
# without a database row yet). Both resolve to a folder slug the scanner
# walks. Known database platforms are always accepted; a bare slug is only
# accepted when it maps to a folder on disk. When nothing is selected,
# every filesystem platform is scanned.
selected_slugs = [p.fs_slug for p in db_platforms if p.id in platform_ids]
for fs_slug in platform_fs_slugs:
if fs_slug in selected_slugs:
continue
if fs_slug in db_platforms_by_slug or fs_slug in fs_platforms:
selected_slugs.append(fs_slug)

platform_list = sorted(selected_slugs or fs_platforms)
Comment thread
gantoine marked this conversation as resolved.
Outdated

# A "new platforms" scan skips platforms that already exist in the database,
# so they must be excluded from the totals to keep the tracker accurate. This
Expand Down Expand Up @@ -904,6 +920,7 @@ async def scan_handler(sid: str, options: dict[str, Any]):
log.info(f"{emoji.EMOJI_MAGNIFYING_GLASS_TILTED_RIGHT} Scanning")

platform_ids = options.get("platforms", [])
platform_fs_slugs = options.get("platform_fs_slugs", [])
scan_type = ScanType[options.get("type", "quick").upper()]
roms_ids = options.get("roms_ids", [])
metadata_sources = options.get("apis", [])
Expand All @@ -918,6 +935,7 @@ async def scan_handler(sid: str, options: dict[str, Any]):
roms_ids=roms_ids,
launchbox_remote_enabled=launchbox_remote_enabled,
playmatch_enabled=playmatch_enabled,
platform_fs_slugs=platform_fs_slugs,
)

return high_prio_queue.enqueue(
Expand All @@ -928,6 +946,7 @@ async def scan_handler(sid: str, options: dict[str, Any]):
roms_ids=roms_ids,
launchbox_remote_enabled=launchbox_remote_enabled,
playmatch_enabled=playmatch_enabled,
platform_fs_slugs=platform_fs_slugs,
job_timeout=SCAN_TIMEOUT, # Timeout (default of 4 hours)
result_ttl=TASK_RESULT_TTL,
meta={
Expand Down
13 changes: 13 additions & 0 deletions backend/tests/endpoints/sockets/test_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,19 @@ async def test_complete_scan_counts_all_selected(self, patched, mocker):
assert result.total_platforms == 3
assert result.total_roms == 300

async def test_scan_selected_filesystem_slug(self, patched, mocker):
"""A never-scanned folder can be targeted by its filesystem slug."""
result = await scan_platforms(
platform_ids=[],
metadata_sources=[],
scan_type=ScanType.QUICK,
platform_fs_slugs=["new1"],
)

# Only the selected folder is scanned, not every filesystem platform.
assert result.total_platforms == 1
assert result.total_roms == 100


class TestShouldScanRom:
def test_new_platforms_scan_with_no_rom(self):
Expand Down
29 changes: 29 additions & 0 deletions backend/tests/endpoints/test_platform.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from unittest.mock import patch

from fastapi import status


Expand All @@ -14,6 +16,33 @@ def test_get_platforms(client, access_token, platform):
assert len(platforms) == 1


def test_get_filesystem_platforms(client, access_token, platform):
with patch(
"utils.platforms.fs_platform_handler.get_platforms"
) as mock_get_platforms:
# A folder already backed by a database row is excluded; a brand-new
# folder with no database row is returned as an id -1 / 0-rom entry.
mock_get_platforms.return_value = [platform.fs_slug, "segacd"]

response = client.get("/api/platforms/filesystem")
assert response.status_code == status.HTTP_403_FORBIDDEN

response = client.get(
"/api/platforms/filesystem",
headers={"Authorization": f"Bearer {access_token}"},
)
assert response.status_code == status.HTTP_200_OK

platforms = response.json()
fs_slugs = [p["fs_slug"] for p in platforms]
assert platform.fs_slug not in fs_slugs
assert "segacd" in fs_slugs

segacd = next(p for p in platforms if p["fs_slug"] == "segacd")
assert segacd["id"] == -1
assert segacd["rom_count"] == 0


def test_update_platform_custom_name(client, access_token, platform):
# The body is an embedded key, not a bare scalar; sending {"custom_name": ...}
# must be accepted (regression against the single-body-param 422).
Expand Down
161 changes: 100 additions & 61 deletions backend/utils/platforms.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from datetime import datetime, timezone

from config.config_manager import config_manager as cm
from endpoints.responses.platform import PlatformSchema
from handler.database import db_platform_handler
from handler.filesystem import fs_platform_handler
from handler.metadata import (
meta_flashpoint_handler,
meta_hasheous_handler,
Expand All @@ -17,6 +19,74 @@
from models.platform import Platform


def _build_unmatched_platform(slug: str, fs_slug: str, now: datetime) -> PlatformSchema:
"""Build a PlatformSchema for a platform that has no database row yet.

Metadata is resolved from the local provider catalogs only (no network
calls). The entry carries id -1 and rom_count 0, matching the shape used
for unmatched supported platforms.
"""
igdb_platform = meta_igdb_handler.get_platform(slug)
moby_platform = meta_moby_handler.get_platform(slug)
ss_platform = meta_ss_handler.get_platform(slug)
ra_platform = meta_ra_handler.get_platform(slug)
launchbox_platform = meta_launchbox_handler.get_platform(slug)
hasheous_platform = meta_hasheous_handler.get_platform(slug)
tgdb_platform = meta_tgdb_handler.get_platform(slug)
flashpoint_platform = meta_flashpoint_handler.get_platform(slug)
hltb_platform = meta_hltb_handler.get_platform(slug)

platform_attrs = {
"id": -1,
"name": slug.replace("-", " ").title(),
"fs_slug": fs_slug,
"slug": slug,
"roms": [],
"rom_count": 0,
"created_at": now,
"updated_at": now,
"fs_size_bytes": 0,
"missing_from_fs": False,
}

platform_attrs.update(
{
**hltb_platform,
**flashpoint_platform,
**hasheous_platform,
**tgdb_platform,
**launchbox_platform,
**ra_platform,
**moby_platform,
**ss_platform,
**igdb_platform,
"igdb_id": igdb_platform.get("igdb_id")
or hasheous_platform.get("igdb_id")
or None,
"ra_id": ra_platform.get("ra_id") or hasheous_platform.get("ra_id") or None,
"tgdb_id": moby_platform.get("tgdb_id")
or hasheous_platform.get("tgdb_id")
or tgdb_platform.get("tgdb_id")
or None,
"name": igdb_platform.get("name")
or ss_platform.get("name")
or moby_platform.get("name")
or ra_platform.get("name")
or launchbox_platform.get("name")
or hasheous_platform.get("name")
or tgdb_platform.get("name")
or flashpoint_platform.get("name")
or hltb_platform.get("name")
or slug.replace("-", " ").title(),
"url_logo": igdb_platform.get("url_logo")
or tgdb_platform.get("url_logo")
or "",
}
)

return PlatformSchema.model_validate(Platform(**platform_attrs))


def get_supported_platforms() -> list[PlatformSchema]:
"""Get all supported platforms with metadata from various sources.

Expand Down Expand Up @@ -48,67 +118,36 @@ def get_supported_platforms() -> list[PlatformSchema]:
supported_platforms.append(PlatformSchema.model_validate(db_platform))
continue

igdb_platform = meta_igdb_handler.get_platform(slug)
moby_platform = meta_moby_handler.get_platform(slug)
ss_platform = meta_ss_handler.get_platform(slug)
ra_platform = meta_ra_handler.get_platform(slug)
launchbox_platform = meta_launchbox_handler.get_platform(slug)
hasheous_platform = meta_hasheous_handler.get_platform(slug)
tgdb_platform = meta_tgdb_handler.get_platform(slug)
flashpoint_platform = meta_flashpoint_handler.get_platform(slug)
hltb_platform = meta_hltb_handler.get_platform(slug)

platform_attrs = {
"id": -1,
"name": slug.replace("-", " ").title(),
"fs_slug": slug,
"slug": slug,
"roms": [],
"rom_count": 0,
"created_at": now,
"updated_at": now,
"fs_size_bytes": 0,
"missing_from_fs": False,
}
supported_platforms.append(_build_unmatched_platform(slug, slug, now))

platform_attrs.update(
{
**hltb_platform,
**flashpoint_platform,
**hasheous_platform,
**tgdb_platform,
**launchbox_platform,
**ra_platform,
**moby_platform,
**ss_platform,
**igdb_platform,
"igdb_id": igdb_platform.get("igdb_id")
or hasheous_platform.get("igdb_id")
or None,
"ra_id": ra_platform.get("ra_id")
or hasheous_platform.get("ra_id")
or None,
"tgdb_id": moby_platform.get("tgdb_id")
or hasheous_platform.get("tgdb_id")
or tgdb_platform.get("tgdb_id")
or None,
"name": igdb_platform.get("name")
or ss_platform.get("name")
or moby_platform.get("name")
or ra_platform.get("name")
or launchbox_platform.get("name")
or hasheous_platform.get("name")
or tgdb_platform.get("name")
or flashpoint_platform.get("name")
or hltb_platform.get("name")
or slug.replace("-", " ").title(),
"url_logo": igdb_platform.get("url_logo")
or tgdb_platform.get("url_logo")
or "",
}
)
return supported_platforms

platform = Platform(**platform_attrs)
supported_platforms.append(PlatformSchema.model_validate(platform))

return supported_platforms
async def get_filesystem_platforms() -> list[PlatformSchema]:
"""Get platform folders that exist on disk but have no database row yet.

A folder created for a platform (optionally bound in Library Management)
has no database row until its first scan imports a ROM, so it is invisible
to the scan platform picker. Surfacing these folders lets a first scan
target a brand-new platform. Slugs are resolved through the platform
binding/version config; metadata is resolved locally (no network calls).
"""
cnfg = cm.get_config()
fs_slugs = await fs_platform_handler.get_platforms()
existing_fs_slugs = {p.fs_slug for p in db_platform_handler.get_platforms()}

now = datetime.now(timezone.utc)
filesystem_platforms = []

for fs_slug in fs_slugs:
if fs_slug in existing_fs_slugs:
continue

slug = (
cnfg.PLATFORMS_BINDING.get(fs_slug)
or cnfg.PLATFORMS_VERSIONS.get(fs_slug)
or fs_slug
)
filesystem_platforms.append(_build_unmatched_platform(slug, fs_slug, now))

return filesystem_platforms
1 change: 1 addition & 0 deletions frontend/src/locales/bg_BG/scan.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"firmware-found": "Намерен фърмуер: {n} файл | Намерен фърмуер: {n} файла",
"firmware-scanned-n": "Фърмуер: {n} сканирани",
"firmware-scanned-with-details": "Фърмуер: {n_scanned_firmware} сканирани, от които {n_new_firmware} нови",
"folder-not-scanned": "Не сканирано",
"hash-calculation-disabled": "Изчисляването на хешове е деактивирано",
"hash-matchers": "Хеш разпознавачи",
"hashes": "Преизчисли хешове",
Expand Down
1 change: 1 addition & 0 deletions frontend/src/locales/cs_CZ/scan.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"firmware-found": "Firmware nalezen: {n} soubor | Firmware nalezen: {n} soubory",
"firmware-scanned-n": "Firmware: {n} naskenováno",
"firmware-scanned-with-details": "Firmware: naskenováno {n_scanned_firmware}, z toho {n_new_firmware} nových",
"folder-not-scanned": "Neskenováno",
"hash-calculation-disabled": "Výpočet hash je zakázán",
"hash-matchers": "Párovače hashů",
"hashes": "Přepočítat hashe",
Expand Down
1 change: 1 addition & 0 deletions frontend/src/locales/de_DE/scan.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"firmware-found": "Firmware gefunden: {n} Datei | Firmware gefunden: {n} Dateien",
"firmware-scanned-n": "Firmware: {n} gescannt",
"firmware-scanned-with-details": "Firmware: {n_scanned_firmware} gescannt, davon {n_new_firmware} neu",
"folder-not-scanned": "Nicht gescannt",
"hash-calculation-disabled": "Hash-Berechnung ist deaktiviert",
"hash-matchers": "Hash-Matcher",
"hashes": "Hashes neu berechnen",
Expand Down
1 change: 1 addition & 0 deletions frontend/src/locales/en_GB/scan.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"firmware-found": "Firmware found: {n} file | Firmware found: {n} files",
"firmware-scanned-n": "Firmware: {n} scanned",
"firmware-scanned-with-details": "Firmware: {n_scanned_firmware} scanned, with {n_new_firmware} new",
"folder-not-scanned": "Not scanned",
"hash-calculation-disabled": "Hash calculation is disabled",
"hash-matchers": "Hash matchers",
"hashes": "Recalculate hashes",
Expand Down
1 change: 1 addition & 0 deletions frontend/src/locales/en_US/scan.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"firmware-found": "Firmware found: {n} file | Firmware found: {n} files",
"firmware-scanned-n": "Firmware: {n} scanned",
"firmware-scanned-with-details": "Firmware: {n_scanned_firmware} scanned, with {n_new_firmware} new",
"folder-not-scanned": "Not scanned",
"hash-calculation-disabled": "Hash calculation is disabled",
"hash-matchers": "Hash matchers",
"hashes": "Recalculate hashes",
Expand Down
1 change: 1 addition & 0 deletions frontend/src/locales/es_ES/scan.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"firmware-found": "Firmware encontrado: {n} archivo | Firmware encontrado: {n} archivos",
"firmware-scanned-n": "Firmware: {n} analizado",
"firmware-scanned-with-details": "Firmware: {n_scanned_firmware} analizados, con {n_new_firmware} nuevos",
"folder-not-scanned": "Sin escanear",
"hash-calculation-disabled": "El cálculo de hash está deshabilitado",
"hash-matchers": "Emparejadores por hash",
"hashes": "Recalcular hashes",
Expand Down
1 change: 1 addition & 0 deletions frontend/src/locales/fr_FR/scan.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"firmware-found": "Firmware trouvé : {n} fichier | Firmware trouvé : {n} fichiers",
"firmware-scanned-n": "Firmware : {n} analysé",
"firmware-scanned-with-details": "Firmware : {n_scanned_firmware} analysés, dont {n_new_firmware} nouveaux",
"folder-not-scanned": "Non scanné",
"hash-calculation-disabled": "Le calcul de hachage est désactivé",
"hash-matchers": "Comparateurs par hachage",
"hashes": "Recalculer les hachages",
Expand Down
Loading
Loading