perf: Update breakpoint_threshold_type and breakpoint_threshold_amount para… …meters to ProcessingConfig for better code scalability and maintainability

Author: ldw129

embedder_service/models/chunker_with_embedder.py

@@ -28,5 +28,10 @@ def embed_query(self, text: str) -> List[float]:
 def chunker(config: ProcessingConfig):
     """Helper function to get tools based on current request's config"""
     emb_model = SentenceTransformerEmbeddings(config.embedding_model)
-    sem_chunker = SemanticChunker(emb_model, config.breakpoint_threshold_type, config.breakpoint_threshold_amount)
+    sem_chunker = SemanticChunker(
+        emb_model, 
+        breakpoint_threshold_type=config.breakpoint_threshold_type, 
+        breakpoint_threshold_amount=config.breakpoint_threshold_amount
+    )
+    
     return sem_chunker

embedder_service/models/embed.py

@@ -1,11 +1,13 @@
 from pydantic import BaseModel, Field
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, get_args
 import os
 from langchain_experimental.text_splitter import BreakpointThresholdType
 
 _EMBEDDING_MODEL_DEFAULT = "all-MiniLM-L6-v2"
 EMBEDDING_MODEL_NAME = os.getenv("EMBEDDING_MODEL", _EMBEDDING_MODEL_DEFAULT)
 
+breakpoints = get_args(BreakpointThresholdType)
+
 
 class ProcessingConfig(BaseModel):
     """Config model for embed API endpoint."""
@@ -18,9 +20,9 @@ class ProcessingConfig(BaseModel):
     embedding_model: str = Field(
         default=EMBEDDING_MODEL_NAME, description="Sentence Transformer model")
     breakpoint_threshold_type: BreakpointThresholdType = Field(
-        default="percentile", description="Breakpoint threshold type")
+        default=breakpoints[0], description="Breakpoint threshold type")
     breakpoint_threshold_amount: Optional[float] = Field(
-        default=90, description="Breakpoint threshold amount")
+        default=90.0, description="Breakpoint threshold amount")
     min_chunk_size: int = Field(default=100, description="Minimum chunk size")
     max_chunk_size: int = Field(default=1000, description="Maximum chunk size")
     store_in_chroma: bool = Field(