Skip to content

Commit b736d3f

Browse files
committed
Merge remote-tracking branch 'origin/main' into backend/safety_features
2 parents 6a7aeac + 4fddc99 commit b736d3f

23 files changed

Lines changed: 523 additions & 91 deletions

backend/app/repositories/search_repository.py

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,38 @@
1-
from typing import List, Tuple
1+
from datetime import date
2+
from typing import List, Optional, Tuple, Union
23

3-
from sqlalchemy import select
4+
from sqlalchemy import ColumnElement, and_, or_, select
45
from sqlalchemy.ext.asyncio import AsyncSession
56

67
from app.models.paper import Paper as PaperModel
8+
from app.schemas.search_dto import (
9+
AdvancedSearchFilter,
10+
ConditionGroup,
11+
TextCondition,
12+
)
13+
14+
# Mapping from DTO field names to SQLAlchemy model columns
15+
_FIELD_COLUMN = {
16+
"title": PaperModel.title,
17+
"abstract": PaperModel.abstract,
18+
}
719

820

921
class SearchRepository:
1022
"""Repository for search-related database operations."""
1123

1224
@staticmethod
1325
async def search_papers_by_embeddings(
14-
db: AsyncSession, embeddings: List[List[float]], limit: int = 5, threshold: float = 0.4
26+
db: AsyncSession,
27+
embeddings: List[List[float]],
28+
limit: int = 5,
29+
threshold: float = 0.4,
30+
search_filter: Optional[AdvancedSearchFilter] = None,
1531
) -> List[Tuple[PaperModel, float]]:
1632
"""
1733
Perform a vector search for papers based on a list of embeddings.
1834
Returns a list of (PaperModel, avg_distance) tuples ordered by ascending distance.
35+
Optionally applies advanced search filters (year range, text conditions).
1936
"""
2037

2138
# Build distance expressions
@@ -30,6 +47,62 @@ async def search_papers_by_embeddings(
3047
.limit(limit)
3148
)
3249

50+
if search_filter:
51+
clauses = SearchRepository._build_filter_clauses(search_filter)
52+
if clauses:
53+
stmt = stmt.where(and_(*clauses))
54+
3355
result = await db.execute(stmt)
3456
rows = result.fetchall()
3557
return [(paper, float(dist)) for paper, dist in rows]
58+
59+
@staticmethod
60+
def _build_filter_clauses(search_filter: AdvancedSearchFilter) -> list:
61+
"""Build a list of top-level SQLAlchemy filter clauses from the advanced filter."""
62+
clauses = []
63+
64+
if search_filter.year_from is not None:
65+
clauses.append(PaperModel.published_at >= date(search_filter.year_from, 1, 1))
66+
67+
if search_filter.year_to is not None:
68+
clauses.append(PaperModel.published_at <= date(search_filter.year_to, 12, 31))
69+
70+
condition_clause = SearchRepository._build_group_clause(search_filter.root)
71+
if condition_clause is not None:
72+
clauses.append(condition_clause)
73+
74+
return clauses
75+
76+
@staticmethod
77+
def _build_group_clause(group: ConditionGroup) -> Optional[ColumnElement[bool]]:
78+
"""Recursively build an AND/OR clause from a ConditionGroup."""
79+
if not group.children:
80+
return None
81+
82+
child_clauses = []
83+
for child in group.children:
84+
clause = SearchRepository._build_node_clause(child)
85+
if clause is not None:
86+
child_clauses.append(clause)
87+
88+
if not child_clauses:
89+
return None
90+
91+
if group.operator == "AND":
92+
return and_(*child_clauses)
93+
return or_(*child_clauses)
94+
95+
@staticmethod
96+
def _build_node_clause(
97+
node: Union[TextCondition, ConditionGroup],
98+
) -> Optional[ColumnElement[bool]]:
99+
"""Build a clause for a single node (condition or nested group)."""
100+
if node.type == "group":
101+
return SearchRepository._build_group_clause(node)
102+
103+
column = _FIELD_COLUMN[node.field]
104+
pattern = f"%{node.value}%"
105+
106+
if node.operator == "contains":
107+
return column.ilike(pattern)
108+
return or_(column.is_(None), ~column.ilike(pattern))

backend/app/routes/search_routes.py

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1+
import json
2+
13
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile, status
24
from sqlalchemy.ext.asyncio import AsyncSession
35

46
from app.core.database import get_db
57
from app.core.limiter import limiter
6-
from app.schemas.search_dto import SearchRequest, SearchResponse
8+
from app.schemas.search_dto import AdvancedSearchFilter, SearchRequest, SearchResponse
79
from app.services.search_service import SearchService
810

911
router = APIRouter(prefix="/search", tags=["Search"])
@@ -23,12 +25,14 @@ async def search(
2325
) -> SearchResponse:
2426
"""
2527
Returns a list of papers that match the search query.
26-
27-
Currently, this returns the 5 most recently fetched papers, regardless
28-
of the query string.
28+
Optionally accepts an advanced filter with year range and text conditions.
2929
"""
3030

31-
papers = await SearchService.search_papers(payload.query, db)
31+
papers = await SearchService.search_papers(
32+
query=payload.query,
33+
db=db,
34+
search_filter=payload.filter,
35+
)
3236
return SearchResponse.model_validate(papers)
3337

3438

@@ -48,21 +52,42 @@ async def search_by_pdf(
4852
max_length=5000,
4953
description="Optional: query specifying what you want to find in relation to the paper",
5054
),
55+
advanced_filter: str | None = Form(
56+
default=None,
57+
description="Optional: JSON-encoded advanced search filter",
58+
),
5159
db: AsyncSession = Depends(get_db),
5260
) -> SearchResponse:
5361
"""
54-
Returns a list of papers that are relevant to the uploaded PDF
62+
Returns a list of papers that are relevant to the uploaded PDF.
5563
The PDF is analyzed, turned into semantic search queries, and used for vector search on our DB.
64+
Optionally accepts a JSON-encoded advanced filter.
5665
"""
5766
if pdf.content_type != "application/pdf":
5867
raise HTTPException(
5968
status_code=400,
6069
detail="Invalid file type: Only PDF files are supported.",
6170
)
6271

72+
search_filter = None
73+
if advanced_filter:
74+
try:
75+
search_filter = AdvancedSearchFilter.model_validate(json.loads(advanced_filter))
76+
except json.JSONDecodeError as exc:
77+
raise HTTPException(
78+
status_code=400,
79+
detail="Invalid filter JSON.",
80+
) from exc
81+
except ValueError as exc:
82+
raise HTTPException(
83+
status_code=400,
84+
detail="Invalid filter.",
85+
) from exc
86+
6387
papers = await SearchService.search_papers_from_pdf(
6488
pdf_file=pdf,
6589
db=db,
6690
query=query,
91+
search_filter=search_filter,
6792
)
6893
return SearchResponse.model_validate(papers)

backend/app/schemas/search_dto.py

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,52 @@
1-
from typing import List
1+
from __future__ import annotations
22

3-
from pydantic import BaseModel, Field
3+
from typing import Annotated, List, Literal, Optional, Union
4+
5+
from pydantic import BaseModel, Field, model_validator
46

57
from app.schemas.paper_dto import PaperDto
68

79

10+
class TextCondition(BaseModel):
11+
"""Single text-based filter condition on a paper field."""
12+
13+
type: Literal["condition"]
14+
field: Literal["title", "abstract"]
15+
operator: Literal["contains", "not_contains"]
16+
value: str
17+
18+
19+
class ConditionGroup(BaseModel):
20+
"""Logical group combining multiple conditions with AND / OR."""
21+
22+
type: Literal["group"]
23+
operator: Literal["AND", "OR"]
24+
children: List[Annotated[Union[TextCondition, ConditionGroup], Field(discriminator="type")]]
25+
26+
27+
class AdvancedSearchFilter(BaseModel):
28+
"""Structured filter with optional year range and boolean condition tree."""
29+
30+
year_from: Optional[int] = Field(default=None, ge=1, le=9999)
31+
year_to: Optional[int] = Field(default=None, ge=1, le=9999)
32+
root: ConditionGroup
33+
34+
@model_validator(mode="after")
35+
def check_year_range(self) -> AdvancedSearchFilter:
36+
"""Validate year_from is larger or equal to year_to."""
37+
if self.year_from is not None and self.year_to is not None:
38+
if self.year_from > self.year_to:
39+
raise ValueError("year_from must be <= year_to")
40+
return self
41+
42+
843
class SearchRequest(BaseModel):
944
"""
1045
Request to search for specified query
1146
"""
1247

1348
query: str = Field(..., max_length=5000)
49+
filter: Optional[AdvancedSearchFilter] = None
1450

1551

1652
class SearchResponse(BaseModel):

backend/app/services/search_service.py

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from app.core.deps import get_openai_provider, get_specter2_query_embedder
99
from app.core.safety import SafetyService
1010
from app.repositories.search_repository import SearchRepository
11-
from app.schemas.search_dto import PaperDto, SearchResponse
11+
from app.schemas.search_dto import AdvancedSearchFilter, PaperDto, SearchResponse
1212
from app.utils.author_utils import normalize_authors
1313
from app.utils.pdf_utils import pdf_bytes_to_text
1414
from app.utils.token_utils import ensure_fits_token_limit
@@ -25,7 +25,11 @@ class SearchService:
2525
MAX_PDF_KEYWORD_INPUT_TOKENS = 280_000
2626

2727
@staticmethod
28-
async def search_papers(query: str, db: AsyncSession) -> SearchResponse:
28+
async def search_papers(
29+
query: str,
30+
db: AsyncSession,
31+
search_filter: Optional[AdvancedSearchFilter] = None,
32+
) -> SearchResponse:
2933
"""
3034
Search using a free-text query
3135
"""
@@ -47,13 +51,15 @@ async def search_papers(query: str, db: AsyncSession) -> SearchResponse:
4751
keywords=keywords,
4852
db=db,
4953
user_query=query,
54+
search_filter=search_filter,
5055
)
5156

5257
@staticmethod
5358
async def search_papers_from_pdf(
54-
pdf_file: UploadFile,
55-
db: AsyncSession,
56-
query: Optional[str] = None,
59+
pdf_file: UploadFile,
60+
db: AsyncSession,
61+
query: Optional[str] = None,
62+
search_filter: Optional[AdvancedSearchFilter] = None,
5763
) -> SearchResponse:
5864
"""
5965
Search using a PDF as the primary signal.
@@ -109,15 +115,18 @@ async def search_papers_from_pdf(
109115
logger.info("PDF search keywords: %s", keywords)
110116

111117
label = query or pdf_file.filename or "pdf-search"
112-
return await SearchService._search_with_keywords(keywords=keywords, db=db, user_query=label)
118+
return await SearchService._search_with_keywords(
119+
keywords=keywords, db=db, user_query=label, search_filter=search_filter
120+
)
113121

114122
# ---------- Shared search pipeline ----------
115123

116124
@staticmethod
117125
async def _search_with_keywords(
118-
keywords: List[str],
119-
db: AsyncSession,
120-
user_query: str,
126+
keywords: List[str],
127+
db: AsyncSession,
128+
user_query: str,
129+
search_filter: Optional[AdvancedSearchFilter] = None,
121130
) -> SearchResponse:
122131
"""
123132
Core embedding + vector-search + DTO mapping pipeline.
@@ -134,6 +143,7 @@ async def _search_with_keywords(
134143
db=db,
135144
embeddings=embeddings,
136145
limit=10,
146+
search_filter=search_filter,
137147
)
138148

139149
results: List[PaperDto] = []
@@ -165,9 +175,9 @@ async def _search_with_keywords(
165175

166176
@staticmethod
167177
async def _extract_keywords_with_retry(
168-
openai_provider: Any,
169-
query: str,
170-
max_retries: int = 2,
178+
openai_provider: Any,
179+
query: str,
180+
max_retries: int = 2,
171181
) -> List[str]:
172182
"""
173183
Extract keywords from the provider with retries and format normalization
@@ -211,10 +221,10 @@ async def _extract_keywords_with_retry(
211221

212222
@staticmethod
213223
async def _extract_pdf_keywords_with_retry(
214-
openai_provider: Any,
215-
pdf_text: str,
216-
query: Optional[str],
217-
max_retries: int = 2,
224+
openai_provider: Any,
225+
pdf_text: str,
226+
query: Optional[str],
227+
max_retries: int = 2,
218228
) -> List[str]:
219229
"""
220230
Extract keywords from PDF text and optional user query with retries.

frontend/src/api/.openapi-generator/FILES

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@ base.ts
88
common.ts
99
configuration.ts
1010
index.ts
11+
models/advanced-search-filter.ts
1112
models/chat-message-dto.ts
13+
models/condition-group-children-inner.ts
14+
models/condition-group.ts
1215
models/httpvalidation-error.ts
1316
models/index.ts
14-
models/location-inner.ts
1517
models/login-request.ts
1618
models/login-response.ts
1719
models/paper-chat-request.ts
@@ -27,6 +29,8 @@ models/refresh-request.ts
2729
models/refresh-response.ts
2830
models/search-request.ts
2931
models/search-response.ts
32+
models/text-condition.ts
3033
models/user-create.ts
3134
models/user-response.ts
35+
models/validation-error-loc-inner.ts
3236
models/validation-error.ts
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
7.19.0
1+
7.18.0-SNAPSHOT

frontend/src/api/apis/authentication-api.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import type { AxiosPromise, AxiosInstance, RawAxiosRequestConfig } from 'axios';
1818
import globalAxios from 'axios';
1919
// Some imports not used depending on template conditions
2020
// @ts-ignore
21-
import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction, replaceWithSerializableTypeIfNeeded } from '../common';
21+
import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction } from '../common';
2222
// @ts-ignore
2323
import { BASE_PATH, COLLECTION_FORMATS, type RequestArgs, BaseAPI, RequiredError, operationServerMap } from '../base';
2424
// @ts-ignore

frontend/src/api/apis/paper-api.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import type { AxiosPromise, AxiosInstance, RawAxiosRequestConfig } from 'axios';
1818
import globalAxios from 'axios';
1919
// Some imports not used depending on template conditions
2020
// @ts-ignore
21-
import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction, replaceWithSerializableTypeIfNeeded } from '../common';
21+
import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction } from '../common';
2222
// @ts-ignore
2323
import { BASE_PATH, COLLECTION_FORMATS, type RequestArgs, BaseAPI, RequiredError, operationServerMap } from '../base';
2424
// @ts-ignore

frontend/src/api/apis/projects-api.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import type { AxiosPromise, AxiosInstance, RawAxiosRequestConfig } from 'axios';
1818
import globalAxios from 'axios';
1919
// Some imports not used depending on template conditions
2020
// @ts-ignore
21-
import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction, replaceWithSerializableTypeIfNeeded } from '../common';
21+
import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction } from '../common';
2222
// @ts-ignore
2323
import { BASE_PATH, COLLECTION_FORMATS, type RequestArgs, BaseAPI, RequiredError, operationServerMap } from '../base';
2424
// @ts-ignore

0 commit comments

Comments
 (0)