refactor(api): single context-building implementation (#1079)

Consolidate the three copies of context assembly into
open_notebook/utils/context_builder.py:

- POST /api/chat/context now delegates to build_notebook_context()
  (same request/response shapes, same string-matching config semantics)
- The source-chat graph now calls build_source_context() instead of the
  495-line generalized ContextBuilder class, which had exactly one
  caller and whose notebook/notes/priority-config flexibility was dead
- POST /api/notebooks/{notebook_id}/context removed: it duplicated
  /api/chat/context with a slightly different envelope and had zero
  callers (frontend, docs, tests)

Behavior is pinned by new characterization tests written before the
refactor (tests/test_context_endpoint_characterization.py) plus unit
tests for build_source_context.

Author: lfnovo

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Changed
+- Context building consolidated into a single implementation (`open_notebook/utils/context_builder.py`): the copy-pasted source/note assembly loops behind `POST /api/chat/context` and the removed notebook-context endpoint, plus the 495-line generalized `ContextBuilder` class (whose only caller was the source-chat graph), are now two focused functions — `build_notebook_context()` (backs `POST /api/chat/context`, unchanged request/response shapes and config semantics) and `build_source_context()` (backs the source-chat graph, same context shape and 50k-token budget). Pinned by new characterization tests — no behavior change for the surviving paths
+- **Removed** `POST /api/notebooks/{notebook_id}/context`: it duplicated `POST /api/chat/context` (same assembly logic, slightly different response envelope) and had zero callers — frontend, docs and tests only use `/api/chat/context`. If you called it programmatically, switch to `POST /api/chat/context` (body: `{notebook_id, context_config}`; response fields: `context.sources`/`context.notes`, `token_count`, `char_count`)
 - Backend provider metadata now lives in a single registry (`open_notebook/ai/provider_registry.py`): env var config, modalities, connection-test models, OpenAI-compatible discovery URLs and docs links are defined once per provider, and `PROVIDER_ENV_CONFIG`, `PROVIDER_MODALITIES`, `TEST_MODELS` and `OPENAI_COMPAT_PROVIDERS` are derived from it. Adding a provider drops from ~6 hand-synced dicts to the registry plus two manual copies (the `SupportedProvider` Literal and the frontend provider table), both enforced by tests
 - Frontend convention cleanup (no user-facing change): hook files unified to kebab-case (`useNotebookChat.ts`/`useSourceChat.ts` → `use-notebook-chat.ts`/`use-source-chat.ts`), `src/components/source/` merged into `src/components/sources/`, the localStorage auth-token parsing ritual extracted into a single `getAuthToken()` helper (`src/lib/auth-token.ts`), and non-streaming raw `fetch` calls routed through `apiClient` (podcast audio download, auth-status check). SSE/streaming paths and the login/checkAuth credential probes deliberately keep raw `fetch`
 - Pruned unused langchain packages: removed `langchain-community` and `langchain-deepseek` from the dependencies (nothing imports them — DeepSeek and xAI route through esperanto's OpenAI-compatible path, which uses `langchain-openai`). The remaining `langchain-*` provider packages are documented as runtime requirements of esperanto's dynamic `to_langchain()` and the whole langchain/langgraph family now carries explicit upper bounds; `langchain-core` and `langchain-text-splitters` (both directly imported but previously only transitive) are now declared explicitly

api/main.py

@@ -19,7 +19,6 @@
     auth,
     chat,
     config,
-    context,
     credentials,
     embedding,
     embedding_rebuild,
@@ -382,7 +381,6 @@ async def open_notebook_error_handler(request: Request, exc: OpenNotebookError):
     embedding_rebuild.router, prefix="/api/embeddings", tags=["embeddings"]
 )
 app.include_router(settings.router, prefix="/api", tags=["settings"])
-app.include_router(context.router, prefix="/api", tags=["context"])
 app.include_router(sources.router, prefix="/api", tags=["sources"])
 app.include_router(insights.router, prefix="/api", tags=["insights"])
 app.include_router(commands_router.router, prefix="/api", tags=["commands"])

api/models.py

@@ -386,30 +386,6 @@ class SourceListResponse(BaseModel):
     processing_info: Optional[Dict[str, Any]] = None
 
 
-# Context API models
-class ContextConfig(BaseModel):
-    sources: Dict[str, str] = Field(
-        default_factory=dict, description="Source inclusion config {source_id: level}"
-    )
-    notes: Dict[str, str] = Field(
-        default_factory=dict, description="Note inclusion config {note_id: level}"
-    )
-
-
-class ContextRequest(BaseModel):
-    notebook_id: str = Field(..., description="Notebook ID to get context for")
-    context_config: Optional[ContextConfig] = Field(
-        None, description="Context configuration"
-    )
-
-
-class ContextResponse(BaseModel):
-    notebook_id: str
-    sources: List[Dict[str, Any]] = Field(..., description="Source context data")
-    notes: List[Dict[str, Any]] = Field(..., description="Note context data")
-    total_tokens: Optional[int] = Field(None, description="Estimated token count")
-
-
 # Insights API models
 class SourceInsightResponse(BaseModel):
     id: str

api/routers/chat.py

@@ -12,21 +12,16 @@
     SuccessResponse,
     extract_chat_messages,
     get_session_or_404,
-    normalize_record_id,
 )
 from open_notebook.database.repository import ensure_record_id, repo_query
-from open_notebook.domain.notebook import (
-    ChatSession,
-    Note,
-    Notebook,
-    Source,
-    SourceInsight,
-)
+from open_notebook.domain.notebook import ChatSession, Notebook
 from open_notebook.exceptions import (
     NotFoundError,
     OpenNotebookError,
 )
 from open_notebook.graphs.chat import graph as chat_graph
+from open_notebook.utils import token_count
+from open_notebook.utils.context_builder import build_notebook_context
 from open_notebook.utils.graph_utils import get_session_message_count
 
 router = APIRouter()
@@ -402,100 +397,12 @@ async def build_context(request: BuildContextRequest):
         if not notebook:
             raise HTTPException(status_code=404, detail="Notebook not found")
 
-        context_data: dict[str, list[dict[str, str]]] = {"sources": [], "notes": []}
-        total_content = ""
-
-        # Process context configuration if provided
-        if request.context_config:
-            # Process sources
-            for source_id, status in request.context_config.get("sources", {}).items():
-                if "not in" in status:
-                    continue
-
-                try:
-                    # Add table prefix if not present
-                    full_source_id = normalize_record_id("source", source_id)
-
-                    try:
-                        source = await Source.get(full_source_id)
-                    except Exception:
-                        continue
-
-                    if "insights" in status:
-                        source_context = await source.get_context(context_size="short")
-                        context_data["sources"].append(source_context)
-                        total_content += str(source_context)
-                    elif "full content" in status:
-                        source_context = await source.get_context(context_size="long")
-                        context_data["sources"].append(source_context)
-                        total_content += str(source_context)
-                except Exception as e:
-                    logger.warning(f"Error processing source {source_id}: {str(e)}")
-                    continue
-
-            # Process notes
-            for note_id, status in request.context_config.get("notes", {}).items():
-                if "not in" in status:
-                    continue
-
-                try:
-                    # Add table prefix if not present
-                    full_note_id = normalize_record_id("note", note_id)
-                    note = await Note.get(full_note_id)
-                    if not note:
-                        continue
-
-                    if "full content" in status:
-                        note_context = note.get_context(context_size="long")
-                        context_data["notes"].append(note_context)
-                        total_content += str(note_context)
-                except Exception as e:
-                    logger.warning(f"Error processing note {note_id}: {str(e)}")
-                    continue
-        else:
-            # Default behavior - include all sources and notes with short context
-            sources = await notebook.get_sources()
-            try:
-                insights_by_source = await SourceInsight.get_for_sources(
-                    [source.id for source in sources if source.id]
-                )
-            except Exception as e:
-                # Match the per-source fallback below: a hiccup fetching
-                # insights shouldn't fail the whole context request.
-                logger.warning(f"Error batch-fetching source insights: {str(e)}")
-                insights_by_source = {}
-            for source in sources:
-                try:
-                    source_context = await source.get_context(
-                        context_size="short",
-                        insights=insights_by_source.get(source.id or "", []),
-                    )
-                    context_data["sources"].append(source_context)
-                    total_content += str(source_context)
-                except Exception as e:
-                    logger.warning(f"Error processing source {source.id}: {str(e)}")
-                    continue
-
-            notes = await notebook.get_notes()
-            for note in notes:
-                try:
-                    note_context = note.get_context(context_size="short")
-                    context_data["notes"].append(note_context)
-                    total_content += str(note_context)
-                except Exception as e:
-                    logger.warning(f"Error processing note {note.id}: {str(e)}")
-                    continue
-
-        # Calculate character and token counts
+        context_data, total_content = await build_notebook_context(
+            notebook, request.context_config
+        )
+
         char_count = len(total_content)
-        # Use token count utility if available
-        try:
-            from open_notebook.utils import token_count
-
-            estimated_tokens = token_count(total_content) if total_content else 0
-        except ImportError:
-            # Fallback to simple estimation
-            estimated_tokens = char_count // 4
+        estimated_tokens = token_count(total_content) if total_content else 0
 
         return BuildContextResponse(
             context=context_data, token_count=estimated_tokens, char_count=char_count

api/routers/context.py

@@ -60,7 +60,7 @@ Instead of memorizing endpoints, use the interactive API docs:
 **Chat** - Conversational AI interface
 - `GET/POST /chat/sessions` - Manage chat sessions
 - `POST /chat/execute` - Send message and get response
-- `POST /chat/context/build` - Prepare context for chat
+- `POST /chat/context` - Prepare context for chat
 
 **Search** - Find content by text or semantic similarity
 - `POST /search` - Full-text or vector search

docs/7-DEVELOPMENT/api-reference.md

@@ -35,10 +35,11 @@ All embedding is fire-and-forget through the surreal-commands worker — nothing
 
 ## Context building (`utils/context_builder.py`)
 
-`ContextBuilder` assembles LLM context from selected sources/notes/insights under a token budget:
+The single implementation behind both context consumers:
 
-- Each `ContextItem` counts its own tokens on construction; `build()` adds items in priority order (default weights: source 100 > insight 75 > note 50, see `ContextConfig`) and stops once `max_tokens` is exceeded (no prorating).
-- Fetching is lazy (nothing is loaded until `build()`), and every call re-fetches — there is no cache layer.
+- `build_notebook_context()` backs `POST /api/chat/context` (chat panel + podcast generation): it assembles source/note contexts from the inclusion config, whose status strings are matched textually ("not in" skips, "insights" → short context, "full content" → long context). Without a config, every source and note is included with its short context. Per-item failures are logged and skipped.
+- `build_source_context()` backs the source-chat graph: one source's short context plus its insights, truncated to a token budget by dropping insights (last-fetched first).
+- Every call re-fetches — there is no cache layer.
 - Token counting uses `o200k_base` via tiktoken and is an estimate (±5-10% vs. the actual model); `token_count()` falls back to a coarse estimate if tiktoken is unavailable.
 
 ## Encryption (`utils/encryption.py`) {#encryption}

docs/7-DEVELOPMENT/content-processing.md

@@ -15,7 +15,7 @@
 from open_notebook.domain.notebook import Source, SourceInsight
 from open_notebook.exceptions import OpenNotebookError
 from open_notebook.utils import clean_thinking_content
-from open_notebook.utils.context_builder import ContextBuilder
+from open_notebook.utils.context_builder import build_source_context
 from open_notebook.utils.error_classifier import classify_error
 from open_notebook.utils.text_utils import extract_text_content
 
@@ -37,7 +37,7 @@ def call_model_with_source_context(
     Main function that builds source context and calls the model.
 
     This function:
-    1. Uses ContextBuilder to build source-specific context
+    1. Uses build_source_context to build source-specific context
     2. Applies the source_chat Jinja2 prompt template
     3. Handles model provisioning with override support
     4. Tracks context indicators for referenced insights/content
@@ -58,19 +58,18 @@ def _call_model_with_source_context_inner(
     if not source_id:
         raise ValueError("source_id is required in state")
 
-    # Build source context using ContextBuilder (run async code in new loop)
+    # Build source context using build_source_context (run async code in new loop)
     def build_context():
         """Build context in a new event loop"""
         new_loop = asyncio.new_event_loop()
         try:
             asyncio.set_event_loop(new_loop)
-            context_builder = ContextBuilder(
-                source_id=source_id,
-                include_insights=True,
-                include_notes=False,  # Focus on source-specific content
-                max_tokens=50000,  # Reasonable limit for source context
+            return new_loop.run_until_complete(
+                build_source_context(
+                    source_id=source_id,
+                    max_tokens=50000,  # Reasonable limit for source context
+                )
             )
-            return new_loop.run_until_complete(context_builder.build())
         finally:
             new_loop.close()
             asyncio.set_event_loop(None)
@@ -192,7 +191,7 @@ def _format_source_context(context_data: Dict) -> str:
     Format the context data into a readable string for the prompt.
 
     Args:
-        context_data: Context data from ContextBuilder
+        context_data: Context data from build_source_context
 
     Returns:
         Formatted context string