Face Quality and Adaptive Density-Aware Face Clustering

backend/app/config/settings.py

@@ -35,3 +35,22 @@
     DATABASE_PATH = os.path.join(user_data_dir("PictoPy"), "database", "PictoPy.db")
 THUMBNAIL_IMAGES_PATH = os.path.join(user_data_dir("PictoPy"), "thumbnails")
 IMAGES_PATH = "./images"
+
+# Clustering Configuration
+PICTO_CLUSTERING_EPS = float(os.getenv("PICTO_CLUSTERING_EPS", "0.75"))
+PICTO_CLUSTERING_MIN_SAMPLES = int(os.getenv("PICTO_CLUSTERING_MIN_SAMPLES", "2"))
+PICTO_CLUSTERING_SIMILARITY_THRESHOLD = float(
+    os.getenv("PICTO_CLUSTERING_SIMILARITY_THRESHOLD", "0.85")
+)
+PICTO_CLUSTERING_MERGE_THRESHOLD = float(
+    os.getenv("PICTO_CLUSTERING_MERGE_THRESHOLD", "0.7")
+)
+PICTO_CLUSTERING_CONF_THRESHOLD = float(
+    os.getenv("PICTO_CLUSTERING_CONF_THRESHOLD", "0.45")
+)
+PICTO_CLUSTERING_BLUR_THRESHOLD = float(
+    os.getenv("PICTO_CLUSTERING_BLUR_THRESHOLD", "80.0")
+)
+PICTO_CLUSTERING_MIN_FACE_SIZE = int(
+    os.getenv("PICTO_CLUSTERING_MIN_FACE_SIZE", "1600")
+)

backend/app/models/FaceDetector.py

@@ -7,6 +7,12 @@
 from app.models.YOLO import YOLO
 from app.database.faces import db_insert_face_embeddings_by_image_id
 from app.logging.setup_logging import get_logger
+from app.config.settings import (
+    PICTO_CLUSTERING_CONF_THRESHOLD,
+    PICTO_CLUSTERING_BLUR_THRESHOLD,
+    PICTO_CLUSTERING_MIN_FACE_SIZE,
+)
+from app.utils.face_quality import face_passes_quality_gate
 
 # Initialize logger
 logger = get_logger(__name__)
@@ -16,7 +22,7 @@ class FaceDetector:
     def __init__(self):
         self.yolo_detector = YOLO(
             YOLO_util_get_model_path("face"),
-            conf_threshold=0.45,
+            conf_threshold=PICTO_CLUSTERING_CONF_THRESHOLD,
             iou_threshold=0.45,
         )
         self.facenet = FaceNet(FaceNet_util_get_model_path())
@@ -34,26 +40,38 @@ def detect_faces(self, image_id: str, image_path: str, forSearch: bool = False):
         logger.info(f"Detected {len(boxes)} faces in image {image_id}.")
 
         processed_faces, embeddings, bboxes, confidences = [], [], [], []
+        faces_skipped = 0
 
         for box, score in zip(boxes, scores):
-            if score > self.yolo_detector.conf_threshold:
-                x1, y1, x2, y2 = map(int, box)
+            x1, y1, x2, y2 = map(int, box)
 
-                # Create bounding box dictionary in JSON format
-                bbox = {"x": x1, "y": y1, "width": x2 - x1, "height": y2 - y1}
-                bboxes.append(bbox)
-                confidences.append(float(score))
+            padding = 20
+            face_img = img[
+                max(0, y1 - padding) : min(img.shape[0], y2 + padding),
+                max(0, x1 - padding) : min(img.shape[1], x2 + padding),
+            ]
 
-                padding = 20
-                face_img = img[
-                    max(0, y1 - padding) : min(img.shape[0], y2 + padding),
-                    max(0, x1 - padding) : min(img.shape[1], x2 + padding),
-                ]
-                processed_face = FaceNet_util_preprocess_image(face_img)
-                processed_faces.append(processed_face)
+            if not face_passes_quality_gate(
+                face_crop=face_img,
+                bbox=(x1, y1, x2, y2),
+                conf_score=float(score),
+                conf_threshold=self.yolo_detector.conf_threshold,
+                blur_threshold=PICTO_CLUSTERING_BLUR_THRESHOLD,
+                min_face_size=PICTO_CLUSTERING_MIN_FACE_SIZE,
+            ):
+                faces_skipped += 1
+                continue
 
-                embedding = self.facenet.get_embedding(processed_face)
-                embeddings.append(embedding)
+            # Create bounding box dictionary in JSON format
+            bbox = {"x": x1, "y": y1, "width": x2 - x1, "height": y2 - y1}
+            bboxes.append(bbox)
+            confidences.append(float(score))
+
+            processed_face = FaceNet_util_preprocess_image(face_img)
+            processed_faces.append(processed_face)
+
+            embedding = self.facenet.get_embedding(processed_face)
+            embeddings.append(embedding)
 
         if not forSearch and embeddings:
             db_insert_face_embeddings_by_image_id(
@@ -64,6 +82,7 @@ def detect_faces(self, image_id: str, image_path: str, forSearch: bool = False):
             "ids": f"{class_ids}",
             "processed_faces": processed_faces,
             "num_faces": len(embeddings),
+            "faces_skipped": faces_skipped,
         }
 
     def close(self):

backend/app/routes/face_clusters.py

@@ -9,8 +9,9 @@
     db_get_cluster_by_id,
     db_update_cluster,
     db_get_all_clusters_with_face_counts,
-    db_get_images_by_cluster_id,  # Add this import
+    db_get_images_by_cluster_id,
 )
+from app.utils.face_clusters import cluster_util_face_clusters_sync
 from app.schemas.face_clusters import (
     RenameClusterRequest,
     RenameClusterResponse,
@@ -313,24 +314,27 @@ def trigger_global_reclustering():
     try:
         logger.info("Starting manual global face reclustering...")
 
-        # Use the smart clustering function with force flag set to True
-        from app.utils.face_clusters import cluster_util_face_clusters_sync
-
-        result = cluster_util_face_clusters_sync(force_full_reclustering=True)
+        result, total_faces_skipped = cluster_util_face_clusters_sync(
+            force_full_reclustering=True
+        )
 
         if result == 0:
             return GlobalReclusterResponse(
                 success=True,
                 message="No faces found to cluster",
-                data=GlobalReclusterData(clusters_created=0),
+                data=GlobalReclusterData(
+                    clusters_created=0, faces_skipped=total_faces_skipped
+                ),
             )
 
         logger.info("Global reclustering completed successfully")
 
         return GlobalReclusterResponse(
             success=True,
             message="Global reclustering completed successfully.",
-            data=GlobalReclusterData(clusters_created=result),
+            data=GlobalReclusterData(
+                clusters_created=result, faces_skipped=total_faces_skipped
+            ),
         )
 
     except Exception as e:

backend/app/schemas/face_clusters.py

@@ -76,6 +76,7 @@ class GetClusterImagesResponse(BaseModel):
 
 class GlobalReclusterData(BaseModel):
     clusters_created: Optional[int] = None
+    faces_skipped: Optional[int] = None
 
 
 class GlobalReclusterResponse(BaseModel):

backend/app/utils/face_clusters.py

@@ -6,12 +6,13 @@
 import sqlite3
 from datetime import datetime
 from sklearn.cluster import DBSCAN
+from sklearn.neighbors import NearestNeighbors
 from sklearn.metrics.pairwise import cosine_distances
 from sklearn.metrics.pairwise import cosine_similarity
 
 
 from collections import defaultdict, Counter
-from typing import List, Dict, Optional, Union
+from typing import List, Dict, Optional, Union, Tuple
 from numpy.typing import NDArray
 from app.database.connection import get_db_connection
 
@@ -26,7 +27,13 @@
     db_get_metadata,
     db_update_metadata,
 )
-from app.config.settings import DATABASE_PATH
+from app.config.settings import (
+    DATABASE_PATH,
+    PICTO_CLUSTERING_EPS,
+    PICTO_CLUSTERING_MIN_SAMPLES,
+    PICTO_CLUSTERING_SIMILARITY_THRESHOLD,
+    PICTO_CLUSTERING_MERGE_THRESHOLD,
+)
 from app.logging.setup_logging import get_logger
 
 # Initialize logger
@@ -102,10 +109,10 @@ def cluster_util_face_clusters_sync(force_full_reclustering: bool = False):
     metadata = db_get_metadata()
     if force_full_reclustering or cluster_util_is_reclustering_needed(metadata):
         # Perform clustering operation
-        results = cluster_util_cluster_all_face_embeddings()
+        results, total_faces_skipped = cluster_util_cluster_all_face_embeddings()
 
         if not results:
-            return 0
+            return 0, total_faces_skipped
 
         results = [result.to_dict() for result in results]
 
@@ -153,13 +160,15 @@ def cluster_util_face_clusters_sync(force_full_reclustering: bool = False):
             current_metadata = metadata or {}
             current_metadata["reclustering_time"] = datetime.now().timestamp()
             db_update_metadata(current_metadata, cursor)
-        return len(cluster_list)
+        return len(cluster_list), total_faces_skipped
     else:
-        face_cluster_mappings = cluster_util_assign_cluster_to_faces_without_clusterId()
+        face_cluster_mappings, total_faces_skipped = (
+            cluster_util_assign_cluster_to_faces_without_clusterId()
+        )
         with get_db_connection() as conn:
             cursor = conn.cursor()
             db_update_face_cluster_ids_batch(face_cluster_mappings, cursor)
-        return len(face_cluster_mappings)
+        return len(face_cluster_mappings), total_faces_skipped
 
 
 def _validate_embedding(embedding: NDArray, min_norm: float = 1e-6) -> bool:
@@ -185,12 +194,26 @@ def _validate_embedding(embedding: NDArray, min_norm: float = 1e-6) -> bool:
     return True
 
 
+def estimate_eps(embeddings: np.ndarray, k: int) -> Optional[float]:
+    if len(embeddings) <= k:
+        return None
+
+    nn = NearestNeighbors(n_neighbors=k + 1, metric="cosine")
+    nn.fit(embeddings)
+    distances, _ = nn.kneighbors(embeddings)
+
+    kth_distances = distances[:, -1]
+    kth_distances.sort()
+    estimated_eps = np.percentile(kth_distances, 90)
+    return float(estimated_eps)
+
+
 def cluster_util_cluster_all_face_embeddings(
-    eps: float = 0.75,
-    min_samples: int = 2,
-    similarity_threshold: float = 0.85,
+    eps: float = PICTO_CLUSTERING_EPS,
+    min_samples: int = PICTO_CLUSTERING_MIN_SAMPLES,
+    similarity_threshold: float = PICTO_CLUSTERING_SIMILARITY_THRESHOLD,
     merge_threshold: float = None,
-) -> List[ClusterResult]:
+) -> Tuple[List[ClusterResult], int]:
     """
     Cluster face embeddings using DBSCAN with similarity validation.
 
@@ -232,9 +255,11 @@ def cluster_util_cluster_all_face_embeddings(
     if invalid_count > 0:
         logger.warning(f"Filtered out {invalid_count} invalid embeddings")
 
+    total_faces_skipped = invalid_count
+
     if not embeddings:
         logger.error("No valid embeddings found after validation")
-        return []
+        return [], total_faces_skipped
 
     logger.info(f"Total valid faces to cluster: {len(face_ids)}")
 
@@ -259,6 +284,15 @@ def cluster_util_cluster_all_face_embeddings(
         f"Applied similarity threshold: {similarity_threshold} (max_distance: {max_distance:.3f})"
     )
 
+    estimated_eps = estimate_eps(embeddings_array, k=min_samples)
+    if estimated_eps is not None:
+        logger.info(f"Adaptive eps estimated: {estimated_eps:.4f}")
+        eps = estimated_eps
+    else:
+        logger.warning(
+            f"Too few embeddings for eps estimation, using config default: {eps}"
+        )
+
     # Perform DBSCAN clustering with precomputed distances
     dbscan = DBSCAN(
         eps=eps,
@@ -307,17 +341,21 @@ def cluster_util_cluster_all_face_embeddings(
 
     # Post-clustering merge: merge similar clusters based on representative faces
     # Use similarity_threshold if merge_threshold not explicitly provided
-    effective_merge_threshold = merge_threshold if merge_threshold is not None else 0.7
+    effective_merge_threshold = (
+        merge_threshold
+        if merge_threshold is not None
+        else PICTO_CLUSTERING_MERGE_THRESHOLD
+    )
     results = _merge_similar_clusters(
         results, merge_threshold=effective_merge_threshold
     )
 
-    return results
+    return results, total_faces_skipped
 
 
 def cluster_util_assign_cluster_to_faces_without_clusterId(
     similarity_threshold: float = 0.8,
-) -> List[Dict]:
+) -> Tuple[List[Dict], int]:
     """
     Assign cluster IDs to faces that don't have clusters using nearest mean method with similarity threshold.
 
@@ -339,13 +377,13 @@ def cluster_util_assign_cluster_to_faces_without_clusterId(
     # Get faces without cluster assignments
     unassigned_faces = db_get_faces_unassigned_clusters()
     if not unassigned_faces:
-        return []
+        return [], 0
 
     # Get cluster mean embeddings
     cluster_means = db_get_cluster_mean_embeddings()
 
     if not cluster_means:
-        return []
+        return [], 0
 
     # Prepare data for nearest neighbor assignment with validation
     cluster_ids = []
@@ -370,7 +408,7 @@ def cluster_util_assign_cluster_to_faces_without_clusterId(
 
     if not mean_embeddings:
         logger.error("No valid cluster means found after validation")
-        return []
+        return [], 0
 
     mean_embeddings_array = np.array(mean_embeddings)
 
@@ -415,7 +453,9 @@ def cluster_util_assign_cluster_to_faces_without_clusterId(
             f"Skipped {skipped_invalid} faces with invalid embeddings during assignment"
         )
 
-    return face_cluster_mappings
+    total_faces_skipped = skipped_invalid
+
+    return face_cluster_mappings, total_faces_skipped
 
 
 def _merge_similar_clusters(

backend/app/utils/face_quality.py

@@ -0,0 +1,37 @@
+import cv2
+import numpy as np
+
+
+def face_passes_quality_gate(
+    face_crop: np.ndarray,
+    bbox: tuple,  # (x1, y1, x2, y2)
+    conf_score: float,
+    conf_threshold: float,
+    blur_threshold: float,
+    min_face_size: int,
+) -> bool:
+    """
+    Evaluates a detected face against quality thresholds before it proceeds
+    to embedding. All checks must pass for the face to be considered valid.
+    """
+    # 1. Completeness check (Confidence)
+    if conf_score < conf_threshold:
+        return False
+
+    # 2. Size check
+    x1, y1, x2, y2 = bbox
+    area = (x2 - x1) * (y2 - y1)
+    if area < min_face_size:
+        return False
+
+    # 3. Blur check
+    if len(face_crop.shape) == 3:
+        gray = cv2.cvtColor(face_crop, cv2.COLOR_BGR2GRAY)
+    else:
+        gray = face_crop
+
+    variance = cv2.Laplacian(gray, cv2.CV_64F).var()
+    if variance < blur_threshold:
+        return False
+
+    return True