feat(vector_io): add classifier reranker type for post-retrieval chunk filtering

Add "classifier" as a new reranker type that scores chunks by quality or
answerability (rather than relevance) and filters below a confidence
threshold. Uses the same inference rerank API as neural reranking but
with a different objective model.

Refs: #5728

Signed-off-by: Varsha Prasad Narsing <vnarsing@redhat.com>
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: Varsha Prasad Narsing <varshaprasad96@gmail.com>

Author: varshaprasad96

src/ogx/providers/utils/memory/openai_vector_store_mixin.py

@@ -1159,6 +1159,9 @@ def _build_reranker_params(
                     reranker_params["weights"] = ranking_options.weights
             elif ranking_options.ranker == "neural":
                 reranker_params["model"] = ranking_options.model
+            elif ranking_options.ranker == "classifier":
+                reranker_params["model"] = ranking_options.model
+                reranker_params["confidence_threshold"] = ranking_options.score_threshold or 0.0
             else:
                 logger.debug("Unknown ranker value, passing through", ranker=ranking_options.ranker)
 

src/ogx/providers/utils/memory/vector_store.py

@@ -94,6 +94,7 @@ def validate_tiktoken_encoding(name: str = "cl100k_base") -> None:
 RERANKER_TYPE_RRF = "rrf"
 RERANKER_TYPE_WEIGHTED = "weighted"
 RERANKER_TYPE_NORMALIZED = "normalized"
+RERANKER_TYPE_CLASSIFIER = "classifier"
 
 
 def parse_pdf(data: bytes) -> str:
@@ -345,6 +346,8 @@ async def query_chunks(
         elif reranker_type == "neural":
             # Neural reranking is being applied after initial retrieval
             neural_reranking_enabled = True
+        elif reranker_type == "classifier":
+            reranker_type = RERANKER_TYPE_CLASSIFIER
         elif reranker_type == "normalized":
             reranker_type = RERANKER_TYPE_NORMALIZED
         else:
@@ -397,8 +400,27 @@ async def query_chunks(
         if neural_reranking_enabled and response.chunks:
             response = await self.apply_neural_rerank(query_string, response, desired_max_num_results, reranker_params)
 
+        # Apply classifier reranking if enabled
+        if reranker_type == RERANKER_TYPE_CLASSIFIER and response.chunks:
+            response = await self.apply_classifier_rerank(
+                query_string, response, desired_max_num_results, reranker_params
+            )
+
         return response
 
+    @staticmethod
+    def _extract_chunk_texts(
+        chunks: list[EmbeddedChunk],
+    ) -> list[str | OpenAIChatCompletionContentPartTextParam | OpenAIChatCompletionContentPartImageParam]:
+        """Extract text content from chunks for reranking."""
+        texts: list[str | OpenAIChatCompletionContentPartTextParam | OpenAIChatCompletionContentPartImageParam] = []
+        for chunk in chunks:
+            if isinstance(chunk.content, str):
+                texts.append(chunk.content)
+            else:
+                texts.append(interleaved_content_as_str(chunk.content))
+        return texts
+
     async def apply_neural_rerank(
         self,
         query_string: str,
@@ -421,15 +443,7 @@ async def apply_neural_rerank(
             )
             return response
 
-        # Extract text contents from chunks for reranking
-        text_from_chunks: list[
-            str | OpenAIChatCompletionContentPartTextParam | OpenAIChatCompletionContentPartImageParam
-        ] = []
-        for chunk in response.chunks:
-            if isinstance(chunk.content, str):
-                text_from_chunks.append(chunk.content)
-            else:
-                text_from_chunks.append(interleaved_content_as_str(chunk.content))
+        text_from_chunks = self._extract_chunk_texts(response.chunks)
 
         try:
             rerank_response = await self.inference_api.rerank(
@@ -464,6 +478,77 @@ async def apply_neural_rerank(
 
         return QueryChunksResponse(chunks=reranked_chunks, scores=reranked_scores)
 
+    async def apply_classifier_rerank(
+        self,
+        query_string: str,
+        response: QueryChunksResponse,
+        desired_max_num_results: int,
+        reranker_params: dict[str, Any],
+    ) -> QueryChunksResponse:
+        """Rerank and filter chunks using a classifier model via the inference API.
+
+        Like neural reranking, this uses an inference model to score chunks.
+        The difference is the objective: classifier models score for quality or
+        answerability rather than relevance. Chunks below the confidence threshold
+        are filtered out.
+        """
+        classifier_model = reranker_params.get("model")
+
+        if not classifier_model and self.vector_stores_config and self.vector_stores_config.default_reranker_model:
+            config = self.vector_stores_config.default_reranker_model
+            classifier_model = f"{config.provider_id}/{config.model_id}"
+
+        if not classifier_model:
+            log.warning(
+                "Classifier reranking requested but no model configured. Returning results without classification."
+            )
+            return response
+
+        text_from_chunks = self._extract_chunk_texts(response.chunks)
+
+        try:
+            rerank_response = await self.inference_api.rerank(
+                RerankRequest(
+                    model=classifier_model,
+                    query=query_string,
+                    items=text_from_chunks,
+                    max_num_results=desired_max_num_results,
+                )
+            )
+        except Exception as e:
+            log.error("Classifier reranking failed, returning original results", error=str(e))
+            return response
+
+        confidence_threshold = reranker_params.get("confidence_threshold", 0.0)
+
+        classified_chunks = []
+        classified_scores = []
+        for classified_chunk in rerank_response.data:
+            if (
+                classified_chunk.index < len(response.chunks)
+                and classified_chunk.relevance_score >= confidence_threshold
+            ):
+                classified_chunks.append(response.chunks[classified_chunk.index])
+                classified_scores.append(classified_chunk.relevance_score)
+
+        if not classified_chunks:
+            log.warning(
+                "Classifier rerank filtered all chunks",
+                model=classifier_model,
+                threshold=confidence_threshold,
+                original_count=len(response.chunks),
+            )
+        else:
+            log.info(
+                "Classifier rerank complete",
+                model=classifier_model,
+                threshold=confidence_threshold,
+                before=len(response.chunks),
+                after=len(classified_chunks),
+            )
+
+        return QueryChunksResponse(chunks=classified_chunks, scores=classified_scores)
+
     # Note: File processing for vector stores now happens at the
     # openai_attach_file_to_vector_store level using file_id.
     # This VectorStoreWithIndex class focuses on chunk operations.

src/ogx_api/vector_io/models.py

@@ -482,6 +482,9 @@ class SearchRankingOptions(BaseModel):
         - "weighted": Weighted combination of vector and keyword scores
         - "rrf": Reciprocal Rank Fusion algorithm
         - "neural": Neural reranking model (requires model parameter)
+        - "classifier": Classification model that scores chunks by quality/answerability and
+          filters below a confidence threshold (requires model parameter, optional confidence_threshold
+          via score_threshold)
         Note: For OpenAI API compatibility, any string value is accepted, but only the above values are supported.
     :param score_threshold: (Optional) Minimum relevance score threshold for results. Default: 0.0
     :param alpha: (Optional) Weight factor for weighted ranker (0-1).

tests/unit/providers/vector_io/test_classifier_reranker.py

@@ -0,0 +1,165 @@
+# Copyright (c) The OGX Contributors.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""Tests for the classifier reranker type in the search pipeline."""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from ogx.providers.utils.memory.vector_store import (
+    RERANKER_TYPE_CLASSIFIER,
+    VectorStoreWithIndex,
+)
+from ogx_api import ChunkMetadata, EmbeddedChunk, QueryChunksResponse
+
+
+def _make_chunk(content: str, chunk_id: str = "c1") -> EmbeddedChunk:
+    return EmbeddedChunk(
+        content=content,
+        chunk_id=chunk_id,
+        metadata={"document_id": chunk_id},
+        chunk_metadata=ChunkMetadata(document_id=chunk_id, chunk_id=chunk_id),
+        embedding=[0.1, 0.2, 0.3],
+        embedding_model="test",
+        embedding_dimension=3,
+    )
+
+
+class TestClassifierRerankerConstant:
+    def test_classifier_constant_defined(self):
+        assert RERANKER_TYPE_CLASSIFIER == "classifier"
+
+
+class TestApplyClassifierRerank:
+    @pytest.fixture
+    def mock_store(self):
+        store = MagicMock(spec=VectorStoreWithIndex)
+        store.inference_api = AsyncMock()
+        store.vector_stores_config = None
+        store.apply_classifier_rerank = VectorStoreWithIndex.apply_classifier_rerank.__get__(store)
+        store._extract_chunk_texts = VectorStoreWithIndex._extract_chunk_texts
+        return store
+
+    async def test_filters_below_confidence_threshold(self, mock_store):
+        chunks = [_make_chunk("high quality", "c1"), _make_chunk("low quality", "c2")]
+        response = QueryChunksResponse(chunks=chunks, scores=[0.9, 0.3])
+
+        rerank_data = MagicMock()
+        rerank_data.data = [
+            MagicMock(index=0, relevance_score=0.85),
+            MagicMock(index=1, relevance_score=0.2),
+        ]
+        mock_store.inference_api.rerank.return_value = rerank_data
+
+        result = await mock_store.apply_classifier_rerank(
+            "test query", response, 5, {"model": "test-classifier", "confidence_threshold": 0.5}
+        )
+
+        assert len(result.chunks) == 1
+        assert result.chunks[0].content == "high quality"
+        assert result.scores[0] == 0.85
+
+    async def test_keeps_all_above_threshold(self, mock_store):
+        chunks = [_make_chunk("a", "c1"), _make_chunk("b", "c2")]
+        response = QueryChunksResponse(chunks=chunks, scores=[0.9, 0.8])
+
+        rerank_data = MagicMock()
+        rerank_data.data = [
+            MagicMock(index=0, relevance_score=0.9),
+            MagicMock(index=1, relevance_score=0.8),
+        ]
+        mock_store.inference_api.rerank.return_value = rerank_data
+
+        result = await mock_store.apply_classifier_rerank(
+            "test query", response, 5, {"model": "test-classifier", "confidence_threshold": 0.1}
+        )
+
+        assert len(result.chunks) == 2
+
+    async def test_zero_threshold_keeps_all(self, mock_store):
+        chunks = [_make_chunk("a", "c1"), _make_chunk("b", "c2")]
+        response = QueryChunksResponse(chunks=chunks, scores=[0.5, 0.1])
+
+        rerank_data = MagicMock()
+        rerank_data.data = [
+            MagicMock(index=0, relevance_score=0.5),
+            MagicMock(index=1, relevance_score=0.1),
+        ]
+        mock_store.inference_api.rerank.return_value = rerank_data
+
+        result = await mock_store.apply_classifier_rerank(
+            "test query", response, 5, {"model": "test-classifier", "confidence_threshold": 0.0}
+        )
+
+        assert len(result.chunks) == 2
+
+    async def test_no_model_returns_original(self, mock_store):
+        chunks = [_make_chunk("a", "c1")]
+        response = QueryChunksResponse(chunks=chunks, scores=[0.5])
+
+        result = await mock_store.apply_classifier_rerank("test query", response, 5, {})
+
+        assert len(result.chunks) == 1
+        mock_store.inference_api.rerank.assert_not_called()
+
+    async def test_inference_error_returns_original(self, mock_store):
+        chunks = [_make_chunk("a", "c1")]
+        response = QueryChunksResponse(chunks=chunks, scores=[0.5])
+        mock_store.inference_api.rerank.side_effect = RuntimeError("model unavailable")
+
+        result = await mock_store.apply_classifier_rerank("test query", response, 5, {"model": "bad-model"})
+
+        assert len(result.chunks) == 1
+        assert result.scores[0] == 0.5
+
+    async def test_calls_rerank_with_correct_model(self, mock_store):
+        chunks = [_make_chunk("content", "c1")]
+        response = QueryChunksResponse(chunks=chunks, scores=[0.5])
+
+        rerank_data = MagicMock()
+        rerank_data.data = [MagicMock(index=0, relevance_score=0.9)]
+        mock_store.inference_api.rerank.return_value = rerank_data
+
+        await mock_store.apply_classifier_rerank("my query", response, 5, {"model": "my-org/quality-classifier"})
+
+        call_args = mock_store.inference_api.rerank.call_args[0][0]
+        assert call_args.model == "my-org/quality-classifier"
+        assert call_args.query == "my query"
+
+    async def test_out_of_bounds_index_ignored(self, mock_store):
+        chunks = [_make_chunk("only one", "c1")]
+        response = QueryChunksResponse(chunks=chunks, scores=[0.5])
+
+        rerank_data = MagicMock()
+        rerank_data.data = [
+            MagicMock(index=0, relevance_score=0.9),
+            MagicMock(index=99, relevance_score=0.8),
+        ]
+        mock_store.inference_api.rerank.return_value = rerank_data
+
+        result = await mock_store.apply_classifier_rerank("query", response, 5, {"model": "test-model"})
+
+        assert len(result.chunks) == 1
+        assert result.scores[0] == 0.9
+
+    async def test_all_filtered_returns_empty(self, mock_store):
+        chunks = [_make_chunk("low", "c1"), _make_chunk("also low", "c2")]
+        response = QueryChunksResponse(chunks=chunks, scores=[0.3, 0.2])
+
+        rerank_data = MagicMock()
+        rerank_data.data = [
+            MagicMock(index=0, relevance_score=0.1),
+            MagicMock(index=1, relevance_score=0.05),
+        ]
+        mock_store.inference_api.rerank.return_value = rerank_data
+
+        result = await mock_store.apply_classifier_rerank(
+            "query", response, 5, {"model": "test-model", "confidence_threshold": 0.5}
+        )
+
+        assert len(result.chunks) == 0
+        assert len(result.scores) == 0