merge origin/main

Author: MoSchmidt

.gitignore

@@ -19,7 +19,7 @@ backend/ingestion/data
 backend/inquiro-env/
 
 # Claude Code
-.claude
+.claude/
 
 # Evaluation data and results
 backend/evaluation_data/

CLAUDE.md

@@ -81,4 +81,4 @@ Frontend API types are generated from backend OpenAPI schema. When backend endpo
 
 ### Frontend
 - ESLint + Prettier
-- Husky + lint-staged for pre-commit hooks
+- Husky + lint-staged for pre-commit hooks

backend/.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
--   repo: https://github.com/astral-sh/ruff-pre-commit
+  - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: 'v0.5.0' # Use a recent ruff version
     hooks:
-    -   id: ruff
-        args: [--fix, --exit-non-zero-on-fix]
-    -   id: ruff-format
+      - id: ruff
+        args: [ --fix, --exit-non-zero-on-fix ]
+      - id: ruff-format

backend/app/constants/database_constants.py

@@ -18,3 +18,11 @@ class PaperType(Enum):
     WORKSHOP = "WORKSHOP"
     THESIS = "THESIS"
     OTHER = "OTHER"
+
+
+class PaperContentStatus(Enum):
+    """Lifecycle status of paper content parsing and storage."""
+    PENDING = "PENDING"
+    PROCESSING = "PROCESSING"
+    SUCCEEDED = "SUCCEEDED"
+    FAILED = "FAILED"

backend/app/core/config.py

@@ -29,6 +29,12 @@ class Settings(BaseSettings):
     # --- OpenAI ---
     OPENAI_API_KEY: Optional[str] = None
 
+    # --- Docling PDF Conversion ---
+    DOCLING_WORKERS: int = 2
+    DOCLING_TIMEOUT_SECONDS: int = 300
+    DOCLING_MAX_RETRIES: int = 3
+    DOCLING_RETRY_BASE_DELAY: int = 30  # seconds, doubles each attempt
+
     model_config = SettingsConfigDict(
         env_file=os.getenv("ENV_FILE", "dev.env"),
         extra="ignore",

backend/app/core/database.py

@@ -36,6 +36,7 @@ async def init_db() -> None:
             "app.models.project",
             "app.models.paper",
             "app.models.project_paper",
+            "app.models.paper_content",
     ):
         import_module(module)
 

backend/app/core/deps.py

@@ -15,7 +15,7 @@ def get_specter2_proximity_embedder() -> Specter2Embedder:
 @lru_cache(maxsize=1)
 def get_specter2_query_embedder() -> Specter2Embedder:
     """Return a shared Specter2Embedder instance, initialized once."""
-    return Specter2Embedder(model="allenai/specter2_adhoc_query")
+    return Specter2Embedder(adapter="allenai/specter2_adhoc_query")
 
 
 @lru_cache(maxsize=1)

backend/app/core/security.py

@@ -19,7 +19,7 @@ def create_access_token(data: Dict[str, Any], expires_delta: Optional[timedelta]
 
     to_encode = data.copy()
     expire = datetime.now(timezone.utc) + (
-        expires_delta or timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
+            expires_delta or timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
     )
     to_encode.update({"exp": expire})
     return jwt.encode(to_encode, settings.JWT_SECRET_KEY, algorithm=ALGORITHM)
@@ -30,7 +30,7 @@ def create_refresh_token(data: Dict[str, Any], expires_delta: Optional[timedelta
 
     to_encode = data.copy()
     expire = datetime.now(timezone.utc) + (
-        expires_delta or timedelta(days=settings.REFRESH_TOKEN_EXPIRE_DAYS)
+            expires_delta or timedelta(days=settings.REFRESH_TOKEN_EXPIRE_DAYS)
     )
     to_encode.update({"exp": expire, "type": "refresh"})
     return jwt.encode(to_encode, settings.JWT_SECRET_KEY, algorithm=ALGORITHM)

backend/app/llm/embeddings/specter2.py

@@ -2,7 +2,8 @@
 from typing import List, Optional
 
 import torch
-from transformers import AutoAdapterModel, AutoTokenizer
+from adapters import AutoAdapterModel
+from transformers import AutoTokenizer
 
 logger = logging.getLogger(__name__)
 
@@ -20,31 +21,46 @@ def build_specter2_text(title: str, abstract: str, tokenizer: AutoTokenizer) ->
 class Specter2Embedder:
     """Wrapper around the SPECTER2 retrieval (proximity) model."""
 
-    def __init__(self, model: str = "allenai/specter2", device: Optional[str] = None) -> None:
+    def __init__(
+        self, adapter: str = "allenai/specter2_base", device: Optional[str] = None
+    ) -> None:
         """Load SPECTER2 tokenizer, base model, and proximity adapter."""
+
         if device is None:
             device = "cuda" if torch.cuda.is_available() else "cpu"
 
         self.device = torch.device(device)
-        logger.info("Loading SPECTER2 base + proximity adapter on %s...", self.device)
+        self.adapter = adapter
 
-        # Load tokenizer + base model
-        self.tokenizer = AutoTokenizer.from_pretrained("allenai/specter2_base")
+        # Tokenizer (shared across all variants)
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            "allenai/specter2_base",
+            local_files_only=False,
+            trust_remote_code=False,
+        )
 
-        self.model = AutoAdapterModel.from_pretrained("allenai/specter2_base")
+        # Adapter-aware model
+        self.model = AutoAdapterModel.from_pretrained(
+            "allenai/specter2_base",
+            local_files_only=False,
+            trust_remote_code=False,
+        )
 
-        # Load retrieval adapter (proximity)
+        # Load and activate adapter
         self.model.load_adapter(
-            model,
-            load_as="specter2",
-            set_active=True,
+            adapter,
             source="hf",
+            set_active=True,
         )
 
         self.model.to(self.device)
         self.model.eval()
 
-        logger.info("SPECTER2 model + adapter loaded successfully.")
+        logger.info(
+            "Loaded SPECTER2 model with adapter '%s' on device '%s'",
+            adapter,
+            self.device,
+        )
 
     def embed_batch(self, texts: List[str]) -> List[Optional[List[float]]]:
         """

backend/app/llm/evaluation/README.md

@@ -34,6 +34,7 @@ python -m app.llm.evaluation.dataset_generator \
 Note: The default output is `evaluation_data/dataset.json` (directories are created automatically).
 
 This will create the dataset file with test cases containing:
+
 - `ground_truth_keywords`: The original keywords
 - `user_input`: Generated natural language query
 - `metadata`: Generation info
@@ -58,6 +59,7 @@ python -m app.llm.evaluation.prompt_evaluator \
 Note: The default output is `evaluation_results/results.json` (directories are created automatically).
 
 The results will include:
+
 - `mean_jaccard`: Average Jaccard similarity score
 - `std_jaccard`: Standard deviation
 - `min_jaccard` / `max_jaccard`: Score range
@@ -66,6 +68,7 @@ The results will include:
 ## Metrics
 
 **Jaccard Similarity**: Measures overlap between extracted and ground truth keywords
+
 - Formula: `|A ∩ B| / |A ∪ B|`
 - Range: 0.0 (no overlap) to 1.0 (perfect match)
 - Keywords are normalized (lowercase, stripped) before comparison
@@ -127,6 +130,7 @@ python -m app.llm.evaluation.evaluate_all_prompts \
 ```
 
 This will:
+
 - Evaluate all prompts matching the pattern (default: `prompt_*.txt`)
 - Save individual results for each prompt
 - Generate a comparison report ranking prompts by mean Jaccard score