fix(face-clusters): run global reclustering as async job; remove 10s axios cap (AOSSIE-Org#1345)

The shared axios client applied a hard 10s timeout to every backend request
with no per-call override, so synchronous endpoints that scale with library
size (notably global face reclustering) aborted in the UI after ~10s while the
backend kept running to completion, leaving the UI and DB inconsistent.

Backend
- POST /face-clusters/global-recluster now starts the full DBSCAN pass as a
  background task and returns a task_id immediately (202 Accepted), running the
  blocking work via asyncio.to_thread so the event loop is not blocked.
- New GET /face-clusters/global-recluster/{task_id} to poll the job status
  (running | complete | error).
- Concurrency guard: a second trigger while one is running rejoins the in-flight
  task instead of starting a second pass that would race on the cluster tables.
- Cleanup loop reaps finished task results after a TTL (registered in the app
  lifespan); running tasks are never reaped. Terminal results are not deleted on
  first poll, so repeated polls (multiple tabs/retries) stay consistent.

Frontend
- Raise the default axios timeout 10s -> 30s and add LONG_REQUEST_TIMEOUT_MS
  (120s) applied per-call to the face-search / multi-person-search endpoints.
- Replace the one-shot reclustering mutation with a useGlobalRecluster polling
  hook wired into the Settings recluster button (same loader/toast UX).

Scoped to the face-clustering side of the issue: memories endpoints are left
untouched (handled separately), and a live progress bar / SSE is deferred to a
follow-up.

Author: VanshajPoonia

backend/app/routes/face_clusters.py

@@ -1,7 +1,10 @@
+import asyncio
 import logging
 from binascii import Error as Base64Error
 import base64
-from typing import Annotated
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Annotated, Dict, Optional
 import uuid
 import os
 from fastapi import APIRouter, HTTPException, Query, status
@@ -20,8 +23,10 @@
     ErrorResponse,
     GetClustersResponse,
     GetClustersData,
-    GlobalReclusterResponse,
-    GlobalReclusterData,
+    GlobalReclusterStartData,
+    GlobalReclusterStartResponse,
+    GlobalReclusterStatusData,
+    GlobalReclusterStatusResponse,
     ClusterMetadata,
     GetClusterImagesResponse,
     GetClusterImagesData,
@@ -38,6 +43,83 @@
 router = APIRouter()
 
 
+# Global reclustering runs synchronously over every face embedding in the
+# library and can take well past any reasonable HTTP timeout on large
+# libraries, so it runs as a background task that the client polls instead
+# of blocking the request.
+@dataclass
+class ReclusterTask:
+    status: str = "running"  # running | complete | error
+    clusters_created: Optional[int] = None
+    faces_skipped: Optional[int] = None
+    message: Optional[str] = None
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    task: Optional[asyncio.Task] = None
+
+
+recluster_tasks: Dict[str, ReclusterTask] = {}
+
+# Only one global reclustering job may run at a time: a full pass deletes and
+# rebuilds every cluster, so two concurrent runs would race on the same tables.
+# Holds the task_id of the in-flight job, or None when idle. Safe without a
+# lock because it is only read/written from the (single-threaded) event loop
+# with no await between check-and-set.
+_active_recluster_task_id: Optional[str] = None
+
+# How long a finished task's result is retained for polling before the cleanup
+# loop reaps it. Running tasks are never reaped (they are bounded to one by the
+# concurrency guard above).
+RECLUSTER_TASK_TTL_MINUTES = 15
+
+
+async def _run_global_recluster(task_id: str):
+    global _active_recluster_task_id
+    entry = recluster_tasks[task_id]
+    try:
+        result, total_faces_skipped = await asyncio.to_thread(
+            cluster_util_face_clusters_sync, force_full_reclustering=True
+        )
+
+        entry.status = "complete"
+        entry.clusters_created = result or 0
+        entry.faces_skipped = total_faces_skipped
+        entry.message = (
+            "No faces found to cluster"
+            if not result
+            else "Global reclustering completed successfully."
+        )
+        logger.info("Global reclustering completed successfully (task_id=%s)", task_id)
+    except Exception as e:
+        logger.error(f"Global reclustering failed: {str(e)}")
+        entry.status = "error"
+        entry.message = f"Global reclustering failed: {str(e)}"
+    finally:
+        # Release the concurrency guard so a new job can be started, while the
+        # finished result stays in recluster_tasks for the client to poll.
+        if _active_recluster_task_id == task_id:
+            _active_recluster_task_id = None
+
+
+async def _cleanup_stale_recluster_tasks():
+    """Periodically drop finished reclustering results once they age out.
+
+    Running tasks are left untouched (a legitimate recluster can run for a
+    long time, and the concurrency guard already bounds them to one).
+    """
+    while True:
+        await asyncio.sleep(300)  # run every 5 minutes
+        now = datetime.now(timezone.utc)
+        stale = [
+            tid
+            for tid, entry in recluster_tasks.items()
+            if entry.status != "running"
+            and (now - entry.created_at).total_seconds()
+            > RECLUSTER_TASK_TTL_MINUTES * 60
+        ]
+        for tid in stale:
+            recluster_tasks.pop(tid, None)
+
+
 @router.put(
     "/{cluster_id}",
     response_model=RenameClusterResponse,
@@ -308,51 +390,95 @@ def face_tagging(
 
 @router.post(
     "/global-recluster",
-    response_model=GlobalReclusterResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=GlobalReclusterStartResponse,
     responses={code: {"model": ErrorResponse} for code in [500]},
 )
-def trigger_global_reclustering():
+async def trigger_global_reclustering():
     """
-    Manually trigger global face reclustering.
+    Start a global face reclustering job in the background.
     This forces full reclustering regardless of the 24-hour rule.
+
+    Returns immediately with a task_id; poll
+    GET /face-clusters/global-recluster/{task_id} for the result, since
+    reclustering runs over every face embedding and can take a long time
+    on large libraries.
+
+    If a reclustering job is already running, its task_id is returned instead
+    of starting a second (concurrent runs would race on the cluster tables).
     """
-    try:
-        logger.info("Starting manual global face reclustering...")
+    global _active_recluster_task_id
 
-        result, total_faces_skipped = cluster_util_face_clusters_sync(
-            force_full_reclustering=True
+    if _active_recluster_task_id is not None:
+        logger.info(
+            "Global reclustering already in progress (task_id=%s); reusing it",
+            _active_recluster_task_id,
+        )
+        return GlobalReclusterStartResponse(
+            success=True,
+            message="Global reclustering already in progress.",
+            data=GlobalReclusterStartData(task_id=_active_recluster_task_id),
         )
 
-        if result == 0:
-            return GlobalReclusterResponse(
-                success=True,
-                message="No faces found to cluster",
-                data=GlobalReclusterData(
-                    clusters_created=0, faces_skipped=total_faces_skipped
-                ),
-            )
+    task_id = str(uuid.uuid4())
+    entry = ReclusterTask()
+    recluster_tasks[task_id] = entry
+    _active_recluster_task_id = task_id
+    entry.task = asyncio.create_task(_run_global_recluster(task_id))
 
-        logger.info("Global reclustering completed successfully")
+    logger.info("Started manual global face reclustering (task_id=%s)", task_id)
 
-        return GlobalReclusterResponse(
-            success=True,
-            message="Global reclustering completed successfully.",
-            data=GlobalReclusterData(
-                clusters_created=result, faces_skipped=total_faces_skipped
-            ),
-        )
+    return GlobalReclusterStartResponse(
+        success=True,
+        message="Global reclustering started.",
+        data=GlobalReclusterStartData(task_id=task_id),
+    )
 
-    except Exception as e:
-        logger.error(f"Global reclustering failed: {str(e)}")
+
+@router.get(
+    "/global-recluster/{task_id}",
+    response_model=GlobalReclusterStatusResponse,
+    responses={code: {"model": ErrorResponse} for code in [404]},
+)
+async def get_global_recluster_status(task_id: str):
+    """Poll the status of a previously started global reclustering job."""
+    entry = recluster_tasks.get(task_id)
+    if entry is None:
         raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            status_code=status.HTTP_404_NOT_FOUND,
             detail=ErrorResponse(
                 success=False,
-                error="Internal server error",
-                message=f"Global reclustering failed: {str(e)}",
+                error="Task Not Found",
+                message=f"Recluster task '{task_id}' not found or already consumed.",
             ).model_dump(),
         )
 
+    if entry.status == "running":
+        return GlobalReclusterStatusResponse(
+            success=True,
+            data=GlobalReclusterStatusData(status="running"),
+        )
+
+    # Terminal state: leave the entry in place so repeated polls (multiple
+    # tabs, retries) return the same result; _cleanup_stale_recluster_tasks
+    # reaps it once it ages out.
+    if entry.status == "error":
+        return GlobalReclusterStatusResponse(
+            success=False,
+            message=entry.message,
+            data=GlobalReclusterStatusData(status="error"),
+        )
+
+    return GlobalReclusterStatusResponse(
+        success=True,
+        message=entry.message,
+        data=GlobalReclusterStatusData(
+            status="complete",
+            clusters_created=entry.clusters_created,
+            faces_skipped=entry.faces_skipped,
+        ),
+    )
+
 
 @router.post(
     "/multi-search",

backend/app/schemas/face_clusters.py

@@ -1,5 +1,5 @@
 from pydantic import BaseModel
-from typing import List, Optional, Dict, Union, Any
+from typing import List, Literal, Optional, Dict, Union, Any
 
 
 # Request Models
@@ -74,16 +74,32 @@ class GetClusterImagesResponse(BaseModel):
     data: Optional[GetClusterImagesData] = None
 
 
-class GlobalReclusterData(BaseModel):
+class GlobalReclusterStartData(BaseModel):
+    task_id: str
+
+
+class GlobalReclusterStartResponse(BaseModel):
+    """Returned immediately when a global reclustering job is started."""
+
+    success: bool
+    message: Optional[str] = None
+    error: Optional[str] = None
+    data: Optional[GlobalReclusterStartData] = None
+
+
+class GlobalReclusterStatusData(BaseModel):
+    status: Literal["running", "complete", "error"]
     clusters_created: Optional[int] = None
     faces_skipped: Optional[int] = None
 
 
-class GlobalReclusterResponse(BaseModel):
+class GlobalReclusterStatusResponse(BaseModel):
+    """Polled to check on the progress of a global reclustering job."""
+
     success: bool
     message: Optional[str] = None
     error: Optional[str] = None
-    data: Optional[GlobalReclusterData] = None
+    data: Optional[GlobalReclusterStatusData] = None
 
 
 class MultiPersonSearchRequest(BaseModel):

backend/main.py

@@ -25,7 +25,10 @@
 from app.routes.folders import router as folders_router
 from app.routes.albums import router as albums_router
 from app.routes.images import router as images_router
-from app.routes.face_clusters import router as face_clusters_router
+from app.routes.face_clusters import (
+    router as face_clusters_router,
+    _cleanup_stale_recluster_tasks,
+)
 from app.routes.user_preferences import router as user_preferences_router
 from app.routes.memories import router as memories_router
 from app.routes.shutdown import router as shutdown_router
@@ -66,12 +69,17 @@ async def lifespan(app: FastAPI):
 
     # Start the SSE model download cleanup task
     cleanup_task = asyncio.create_task(_cleanup_stale_tasks())
+    # Start the global-reclustering finished-task cleanup loop
+    recluster_cleanup_task = asyncio.create_task(_cleanup_stale_recluster_tasks())
 
     try:
         yield
     finally:
         cleanup_task.cancel()
-        await asyncio.gather(cleanup_task, return_exceptions=True)
+        recluster_cleanup_task.cancel()
+        await asyncio.gather(
+            cleanup_task, recluster_cleanup_task, return_exceptions=True
+        )
         app.state.executor.shutdown(wait=True)
 
 

frontend/src/api/api-functions/face_clusters.ts

@@ -1,5 +1,5 @@
 import { faceClustersEndpoints } from '../apiEndpoints';
-import { apiClient } from '../axiosConfig';
+import { apiClient, LONG_REQUEST_TIMEOUT_MS } from '../axiosConfig';
 import { APIResponse } from '@/types/API';
 import { BackendRes } from '@/hooks/useQueryExtension';
 import type { Image } from '@/types/Media';
@@ -53,6 +53,7 @@ export const fetchSearchedFaces = async (
   const response = await apiClient.post<APIResponse>(
     faceClustersEndpoints.searchForFaces,
     request,
+    { timeout: LONG_REQUEST_TIMEOUT_MS },
   );
   return response.data;
 };
@@ -63,6 +64,7 @@ export const fetchSearchedFacesBase64 = async (
   const response = await apiClient.post<BackendRes<Image[]>>(
     faceClustersEndpoints.searchForFacesBase64,
     request,
+    { timeout: LONG_REQUEST_TIMEOUT_MS },
   );
   return response.data;
 };
@@ -72,15 +74,39 @@ export interface GlobalReclusterData {
   faces_skipped: number | null;
 }
 
-export const triggerGlobalReclustering = async (): Promise<
-  BackendRes<GlobalReclusterData>
+export interface GlobalReclusterStartData {
+  task_id: string;
+}
+
+export interface GlobalReclusterStatusData {
+  status: 'running' | 'complete' | 'error';
+  clusters_created: number | null;
+  faces_skipped: number | null;
+}
+
+// Global reclustering runs over every face embedding in the library and can
+// take well past any reasonable HTTP timeout, so the backend runs it as a
+// background job: this kicks it off and returns a task_id immediately.
+export const startGlobalReclustering = async (): Promise<
+  BackendRes<GlobalReclusterStartData>
 > => {
-  const response = await apiClient.post<BackendRes<GlobalReclusterData>>(
+  const response = await apiClient.post<BackendRes<GlobalReclusterStartData>>(
     faceClustersEndpoints.globalRecluster,
   );
   return response.data;
 };
 
+// Poll this with the task_id returned by startGlobalReclustering until
+// status is 'complete' or 'error'.
+export const getGlobalReclusterStatus = async (
+  taskId: string,
+): Promise<BackendRes<GlobalReclusterStatusData>> => {
+  const response = await apiClient.get<BackendRes<GlobalReclusterStatusData>>(
+    faceClustersEndpoints.globalReclusterStatus(taskId),
+  );
+  return response.data;
+};
+
 export interface MultiPersonSearchRequest {
   cluster_ids: string[];
   match_mode: 'match_any' | 'match_all';
@@ -92,6 +118,7 @@ export const fetchMultiPersonSearch = async (
   const response = await apiClient.post<APIResponse>(
     faceClustersEndpoints.multiPersonSearch,
     request,
+    { timeout: LONG_REQUEST_TIMEOUT_MS },
   );
   return response.data;
 };

frontend/src/api/apiEndpoints.ts

@@ -10,6 +10,8 @@ export const faceClustersEndpoints = {
   renameCluster: (clusterId: string) => `/face-clusters/${clusterId}`,
   getClusterImages: (clusterId: string) => `/face-clusters/${clusterId}/images`,
   globalRecluster: '/face-clusters/global-recluster',
+  globalReclusterStatus: (taskId: string) =>
+    `/face-clusters/global-recluster/${taskId}`,
   multiPersonSearch: '/face-clusters/multi-search',
 };