refactor(transfer): make queue admission durable

Author: jxxghp

app/application/chain/data.py

@@ -10,8 +10,11 @@
 from dataclasses import dataclass
 from typing import Any, Optional
 
+from app.application.transfer import TransferAdmissionRepository
+
 
 OperFactory = Callable[[], Any]
+TransferAdmissionRepositoryFactory = Callable[[], TransferAdmissionRepository]
 
 
 @dataclass(frozen=True, slots=True)
@@ -23,7 +26,7 @@ class ChainDataPorts:
     workflow: OperFactory
     download_history: OperFactory
     transfer_history: OperFactory
-    transfer_pending: OperFactory
+    transfer_pending: TransferAdmissionRepositoryFactory
     media_server: OperFactory
     download_failure: OperFactory
     user: OperFactory
@@ -77,12 +80,6 @@ class TransferHistoryPortProxy(_ChainDataPortProxy):
     port_name = "transfer_history"
 
 
-class TransferPendingPortProxy(_ChainDataPortProxy):
-    """待整理数据端口代理。"""
-
-    port_name = "transfer_pending"
-
-
 class MediaServerPortProxy(_ChainDataPortProxy):
     """媒体服务器数据端口代理。"""
 
@@ -156,8 +153,8 @@ def get_chain_transfer_history_port() -> Any:
     return get_chain_data_ports().transfer_history()
 
 
-def get_chain_transfer_pending_port() -> Any:
-    """创建待整理数据端口实例。"""
+def get_chain_transfer_pending_port() -> TransferAdmissionRepository:
+    """创建类型化的整理任务 durable admission 仓储。"""
     return get_chain_data_ports().transfer_pending()
 
 

app/application/transfer.py

@@ -19,14 +19,10 @@
 from dataclasses import dataclass
 from pathlib import Path
 from time import monotonic
-from typing import Callable, Dict, List, Optional, Tuple, Union
+from typing import Callable, Dict, List, Optional, Protocol, Tuple, Union
 
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, PrivateAttr
 
-from app.schemas.transfer import MetaInfo as _SchemaMetaInfo
-from app.schemas.transfer import MusicInfo as _SchemaMusicInfo
-from app.schemas.transfer import MusicMeta as _SchemaMusicMeta
-from app.schemas.workflow import MediaInfo as _SchemaMediaInfo
 from app.adapters.system.host import SystemUtils
 from app.application.agent import get_prompt_manager, get_running_agent_manager
 from app.domain.context import MediaInfo, MusicInfo
@@ -40,6 +36,9 @@
 from app.schemas.media import OptionalMediaIdentityMixin, resolve_media_identity
 from app.schemas.system import TransferDirectoryConf
 from app.schemas.tmdb import TmdbEpisode
+from app.schemas.transfer import MetaInfo as _SchemaMetaInfo
+from app.schemas.transfer import MusicInfo as _SchemaMusicInfo
+from app.schemas.transfer import MusicMeta as _SchemaMusicMeta
 from app.schemas.transfer import TransferInfo, TransferJob, TransferJobTask
 from app.schemas.types import (
     MUSIC_ENTITY_ALBUM,
@@ -48,7 +47,7 @@
     MediaType,
     ReplyMode,
 )
-
+from app.schemas.workflow import MediaInfo as _SchemaMediaInfo
 
 
 class TransferTask(OptionalMediaIdentityMixin, BaseModel):
@@ -81,6 +80,16 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel):
     manual: Optional[bool] = False
     background: Optional[bool] = True
     preview: Optional[bool] = False
+    _admission_task_id: Optional[str] = PrivateAttr(default=None)
+
+    @property
+    def admission_task_id(self) -> Optional[str]:
+        """返回仅供宿主持久准入和终态结算使用的内部任务标识。"""
+        return self._admission_task_id
+
+    def bind_admission_task_id(self, task_id: str) -> None:
+        """绑定持久准入生成的稳定身份，不改变插件可见序列化字段。"""
+        self._admission_task_id = task_id
 
     def to_dict(self):
         """
@@ -113,36 +122,86 @@ class TransferQueue(BaseModel):
     result: Optional[TransferInfo] = None
 
 
+TRANSFER_ADMISSION_ACCEPTED = "accepted"
+
+
+@dataclass(frozen=True, slots=True)
+class TransferAdmission:
+    """描述已经持久化、可在进程退出后恢复的整理任务准入事实。"""
+
+    task_id: str
+    storage: str
+    src_path: str
+    state: str
+    created_at: str
+    updated_at: str
+    last_error: Optional[str] = None
+
+
+class TransferAdmissionRepository(Protocol):
+    """整理任务 durable admission 所需的类型化持久化端口。"""
+
+    def admit(self, *, storage: str, src_path: str) -> TransferAdmission:
+        """幂等登记源文件并返回稳定任务身份。"""
+        ...
+
+    def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]:
+        """按登记顺序返回等待恢复或执行的任务。"""
+        ...
+
+    def record_enqueue_failure(self, *, task_id: str, error: str) -> None:
+        """记录内存队列接收失败，保留任务供后续恢复。"""
+        ...
+
+    def discard_task(self, *, task_id: str) -> int:
+        """按稳定任务身份删除已经到达终态的登记。"""
+        ...
+
+
 class TransferQueueService:
     """协调整理任务登记、入队、移除和队列视图查询。"""
 
     def __init__(
             self,
             *,
             register_task: Callable[[TransferTask], bool],
+            admit_task: Callable[[TransferTask], TransferAdmission],
             enqueue: Callable[[TransferQueue], None],
             before_enqueue: Callable[[TransferTask], None],
-            after_enqueue: Callable[[TransferTask], None],
+            enqueue_failed: Callable[[TransferTask, Exception], None],
             remove_task: Callable[[FileItem], None],
             list_tasks: Callable[[], List[TransferJob]],
             expire_tasks: Callable[[], None],
     ) -> None:
         """保存队列用例依赖，避免 Application 服务绑定具体线程队列实现。"""
         self._register_task = register_task
+        self._admit_task = admit_task
         self._enqueue = enqueue
         self._before_enqueue = before_enqueue
-        self._after_enqueue = after_enqueue
+        self._enqueue_failed = enqueue_failed
         self._remove_task = remove_task
         self._list_tasks = list_tasks
         self._expire_tasks = expire_tasks
 
     def put(self, task: TransferTask, callback: Callable) -> bool:
-        """登记并入队一个整理任务，保持原有副作用顺序。"""
+        """先持久化准入事实再入队；任何前置失败都撤销内存作业视图。"""
         if not task or not self._register_task(task):
             return False
-        self._before_enqueue(task)
-        self._enqueue(TransferQueue(task=task, callback=callback))
-        self._after_enqueue(task)
+        try:
+            admission = self._admit_task(task)
+            task.bind_admission_task_id(admission.task_id)
+        except Exception:
+            self._remove_task(task.fileitem)
+            raise
+        try:
+            self._before_enqueue(task)
+            self._enqueue(TransferQueue(task=task, callback=callback))
+        except Exception as err:
+            try:
+                self._enqueue_failed(task, err)
+            finally:
+                self._remove_task(task.fileitem)
+            raise
         return True
 
     def remove(self, fileitem: FileItem) -> None:

app/chain/transfer.py

@@ -52,6 +52,7 @@
 from app.application.transfer import (
     FailedRetryScheduler,
     JobManager,
+    TransferAdmission,
     TransferFailureNotification,
     TransferFailureNotificationAggregator,
     TransferQueue,
@@ -225,8 +226,8 @@ def __init__(self) -> None:
         self.retry_scheduler = FailedRetryScheduler()
         # 整理失败通知聚合器
         self.failure_notification_aggregator = TransferFailureNotificationAggregator()
-        # 待整理文件落盘登记，用于进程重启后回放内存队列里未完成的任务
-        self._pendingoper = get_chain_transfer_pending_port()
+        # durable admission 仓储先于内存入队保存任务，进程退出后仍可恢复。
+        self._transfer_admissions = get_chain_transfer_pending_port()
         # 转移成功的文件清单
         self._success_target_files: Dict[Tuple, List[str]] = {}
         # 批次级刮削缓冲，避免同一批多文件入库重复触发目录刮削
@@ -867,7 +868,8 @@ def put_to_queue(self, task: TransferTask) -> bool:
         """
         添加到待整理队列
         :param task: 任务信息
-        :return: True表示任务已添加到队列，False表示任务无效或已存在（重复）
+        :return: True表示任务已添加，False表示链已关闭或任务无效/重复
+        :raises Exception: 持久准入、批次登记或内存入队失败
         """
         with self._worker_state_lock:
             if self._closing:
@@ -879,9 +881,10 @@ def _transfer_queue_service(self) -> TransferQueueService:
         """构建保持旧队列对象和私有兼容接缝的应用服务。"""
         return TransferQueueService(
             register_task=self.__put_to_jobview,
+            admit_task=self.__admit_transfer,
             enqueue=self._queue.put,
             before_enqueue=self._register_scrape_batch_task,
-            after_enqueue=self.__register_pending,
+            enqueue_failed=self.__record_enqueue_failure,
             remove_task=self.jobview.remove_task,
             list_tasks=self.jobview.list_jobs,
             expire_tasks=self.__expire_stale_transfer_tasks,
@@ -936,17 +939,19 @@ def __replay_pending(
         if stop_event.is_set():
             return
         try:
-            pendings = self._pendingoper.list_all()
+            pendings = self._transfer_admissions.list_accepted()
         except Exception as err:
             logger.error(f"读取待整理文件登记失败：{err}")
             return
         if not pendings:
             return
         logger.info(f"发现 {len(pendings)} 个上次未整理完的文件，正在重新送入整理链 ...")
         replayed = 0
-        for storage, src_path in pendings:
+        for admission in pendings:
             if stop_event.is_set():
                 break
+            storage = admission.storage
+            src_path = admission.src_path
             try:
                 fileitem, should_discard = self.__build_replay_fileitem(storage, src_path)
                 # stat 等同步 I/O 返回后重新检查，关闭期间不得注销尚未完成的登记。
@@ -955,7 +960,9 @@ def __replay_pending(
                 if not fileitem:
                     if should_discard:
                         # 源文件确认已消失，注销登记避免每次启动重复回放
-                        self._pendingoper.discard(storage=storage, src_path=src_path)
+                        self._transfer_admissions.discard_task(
+                            task_id=admission.task_id
+                        )
                     continue
                 self.do_transfer(fileitem=fileitem)
                 replayed += 1
@@ -1009,19 +1016,35 @@ def __build_replay_fileitem(storage: str, src_path: str) -> Tuple[Optional[FileI
             modify_time=modify_time,
         ), False
 
-    def __register_pending(self, task: TransferTask):
-        """
-        落盘登记一个待整理文件，登记失败不影响正常入队。
-        :param task: 任务信息
-        """
+    def __admit_transfer(self, task: TransferTask) -> TransferAdmission:
+        """在内存入队前持久化源文件并返回稳定任务身份。"""
         fileitem = task.fileitem if task else None
-        if not fileitem or not fileitem.path:
-            return
+        if not fileitem or not fileitem.storage or not fileitem.path:
+            raise ValueError("整理任务缺少源文件身份")
+        return self._transfer_admissions.admit(
+            storage=fileitem.storage,
+            src_path=fileitem.path,
+        )
+
+    def __record_enqueue_failure(
+            self,
+            task: TransferTask,
+            error: Exception,
+    ) -> None:
+        """记录内存入队失败并撤销该任务的批次占位。"""
         try:
-            self._pendingoper.register(storage=fileitem.storage, src_path=fileitem.path)
-        except Exception as err:
-            # 登记只是重启后的补救手段，失败不能阻断正常整理
-            logger.debug(f"登记待整理文件失败: {fileitem.path} - {err}")
+            if task.admission_task_id:
+                self._transfer_admissions.record_enqueue_failure(
+                    task_id=task.admission_task_id,
+                    error=str(error),
+                )
+        except Exception as record_error:
+            logger.error(
+                "记录整理任务入队失败原因异常："
+                f"{task.admission_task_id} - {record_error}"
+            )
+        finally:
+            self._finish_scrape_batch_task(task)
 
     def __discard_pending(self, task: TransferTask):
         """
@@ -1031,13 +1054,17 @@ def __discard_pending(self, task: TransferTask):
         重试预算重新送入整理链；留在本表反而会每次重启都重复回放。
         :param task: 任务信息
         """
-        fileitem = task.fileitem if task else None
-        if not fileitem or not fileitem.path:
+        if not task or not task.admission_task_id:
             return
         try:
-            self._pendingoper.discard(storage=fileitem.storage, src_path=fileitem.path)
+            self._transfer_admissions.discard_task(
+                task_id=task.admission_task_id
+            )
         except Exception as err:
-            logger.debug(f"注销待整理文件登记失败: {fileitem.path} - {err}")
+            logger.error(
+                "注销整理任务 durable admission 失败: "
+                f"{task.admission_task_id} - {err}"
+            )
 
     def __put_to_jobview(self, task: TransferTask) -> bool:
         """
@@ -2751,7 +2778,15 @@ def _get_cached_extra_meta(
                     preview=preview,
                 )
                 if background:
-                    if self.put_to_queue(task=transfer_task):
+                    try:
+                        queued = self.put_to_queue(task=transfer_task)
+                    except Exception as err:
+                        all_success = False
+                        message = f"{file_path.name} 加入整理队列失败：{err}"
+                        err_msgs.append(message)
+                        logger.error(message)
+                        continue
+                    if queued:
                         logger.info(f"{file_path.name} 已添加到整理队列")
                     else:
                         logger.debug(f"{file_path.name} 已在整理队列中，跳过")