refactor(ai): single provider registry as the backend source of truth

Provider metadata (env vars, modalities, connection-test models,
OpenAI-compatible discovery URLs, display names, docs links) is now
defined once in open_notebook/ai/provider_registry.py. The existing
surfaces are derived from it, keeping every import and call-site shape
unchanged:

- api/credentials_service.py: PROVIDER_ENV_CONFIG, PROVIDER_MODALITIES
  and the discovery url_map are built from the registry
- open_notebook/ai/connection_tester.py: TEST_MODELS derived
- open_notebook/ai/model_discovery.py: OPENAI_COMPAT_PROVIDERS built
  from registry entries with a discovery URL (quirk hooks stay local)

The SupportedProvider Literal (typing, can't be built at runtime) and
the frontend provider tables remain manual copies; the cross-check
tests now assert registry keys == Literal == frontend list, plus
registry internal consistency and discovery-table coverage.

New GET /api/providers endpoint exposes the registry (name, display
name, modalities, docs_url, env-configured status) so clients can stop
hardcoding provider lists (frontend adoption is a follow-up).

Docs updated: open_notebook/AGENTS.md and docs/7-DEVELOPMENT/credentials.md
now describe the registry instead of the four-place sync rule.

Author: lfnovo

CHANGELOG.md

@@ -8,10 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Changed
+- Backend provider metadata now lives in a single registry (`open_notebook/ai/provider_registry.py`): env var config, modalities, connection-test models, OpenAI-compatible discovery URLs and docs links are defined once per provider, and `PROVIDER_ENV_CONFIG`, `PROVIDER_MODALITIES`, `TEST_MODELS` and `OPENAI_COMPAT_PROVIDERS` are derived from it. Adding a provider drops from ~6 hand-synced dicts to the registry plus two manual copies (the `SupportedProvider` Literal and the frontend provider table), both enforced by tests
 - Pruned unused langchain packages: removed `langchain-community` and `langchain-deepseek` from the dependencies (nothing imports them — DeepSeek and xAI route through esperanto's OpenAI-compatible path, which uses `langchain-openai`). The remaining `langchain-*` provider packages are documented as runtime requirements of esperanto's dynamic `to_langchain()` and the whole langchain/langgraph family now carries explicit upper bounds; `langchain-core` and `langchain-text-splitters` (both directly imported but previously only transitive) are now declared explicitly
 - The two Docker images (regular and single-container) are now built from a single multi-stage `Dockerfile` with shared stages — regular is the default (`runtime`) target, single-container is `--target single` — so deploy fixes (tiktoken pre-cache, env defaults, npm retry logic) no longer have to be applied twice. `Dockerfile.single` and `supervisord.single.conf` were removed; the single image appends a small `supervisord.surrealdb.conf` to the shared `supervisord.conf` at build time. Published image names and tags are unchanged
 
 ### Added
+- New `GET /api/providers` endpoint returning provider metadata from the registry (name, display name, modalities, docs URL, whether it is configured via environment variables), so clients can enumerate supported providers instead of hardcoding them
 - Release confidence process, documented and executable: `.github/RELEASE_PROCESS.md` now covers the risk-based test matrix (buckets A/B/C), the Docker image gate, the fix-loop re-test policy and the communication/credits/retro structure, backed by a new decision record (ADR-005) and versioned tooling under `scripts/release-test/` — `make release-test TAG= OLD_TAG=` runs fresh-install + upgrade scenarios against real images, and `make release-stack TAG= [DUMP=]` boots a browsable, isolated release-candidate stack (optionally with a copy of dev data) for manual verification
 - CI now gates every PR on `ruff check` (backend lint), `npm run lint` (frontend ESLint) and `npm run build` (frontend production build), in addition to the existing test suites
 

api/credentials_service.py

@@ -20,6 +20,7 @@
     classify_model_type,
     fetch_anthropic_model_ids,
 )
+from open_notebook.ai.provider_registry import PROVIDERS
 from open_notebook.domain.credential import Credential
 from open_notebook.utils.encryption import get_secret_from_env
 from open_notebook.utils.url_validation import validate_url
@@ -28,61 +29,17 @@
 # Constants
 # =============================================================================
 
-# Provider environment variable configuration.
+# Provider environment variable configuration, derived from the provider
+# registry (open_notebook/ai/provider_registry.py — the source of truth).
 # - "required": ALL listed env vars must be set for the provider to be considered configured.
 # - "required_any": at least ONE of the listed env vars must be set.
 # - "optional": additional env vars used during migration but not required.
 PROVIDER_ENV_CONFIG: Dict[str, dict] = {
-    "openai": {"required": ["OPENAI_API_KEY"]},
-    "anthropic": {"required": ["ANTHROPIC_API_KEY"]},
-    "google": {"required_any": ["GOOGLE_API_KEY", "GEMINI_API_KEY"]},
-    "groq": {"required": ["GROQ_API_KEY"]},
-    "mistral": {"required": ["MISTRAL_API_KEY"]},
-    "deepseek": {"required": ["DEEPSEEK_API_KEY"]},
-    "xai": {"required": ["XAI_API_KEY"]},
-    "openrouter": {"required": ["OPENROUTER_API_KEY"]},
-    "voyage": {"required": ["VOYAGE_API_KEY"]},
-    "elevenlabs": {"required": ["ELEVENLABS_API_KEY"]},
-    "deepgram": {"required": ["DEEPGRAM_API_KEY"]},
-    "ollama": {"required": ["OLLAMA_API_BASE"]},
-    "vertex": {
-        "required": ["VERTEX_PROJECT", "VERTEX_LOCATION"],
-        "optional": ["GOOGLE_APPLICATION_CREDENTIALS"],
-    },
-    "azure": {
-        "required": ["AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_API_VERSION"],
-        "optional": [
-            "AZURE_OPENAI_ENDPOINT_LLM",
-            "AZURE_OPENAI_ENDPOINT_EMBEDDING",
-            "AZURE_OPENAI_ENDPOINT_STT",
-            "AZURE_OPENAI_ENDPOINT_TTS",
-        ],
-    },
-    "openai_compatible": {
-        "required_any": ["OPENAI_COMPATIBLE_BASE_URL", "OPENAI_COMPATIBLE_API_KEY"],
-    },
-    "dashscope": {"required": ["DASHSCOPE_API_KEY"]},
-    "minimax": {"required": ["MINIMAX_API_KEY"]},
+    name: spec.env_config() for name, spec in PROVIDERS.items()
 }
 
 PROVIDER_MODALITIES: Dict[str, List[str]] = {
-    "openai": ["language", "embedding", "speech_to_text", "text_to_speech"],
-    "anthropic": ["language"],
-    "google": ["language", "embedding", "speech_to_text", "text_to_speech"],
-    "groq": ["language", "speech_to_text"],
-    "mistral": ["language", "embedding", "speech_to_text", "text_to_speech"],
-    "deepseek": ["language"],
-    "xai": ["language", "text_to_speech"],
-    "openrouter": ["language", "embedding"],
-    "voyage": ["embedding"],
-    "elevenlabs": ["text_to_speech", "speech_to_text"],
-    "deepgram": ["text_to_speech"],
-    "ollama": ["language", "embedding"],
-    "vertex": ["language", "embedding", "text_to_speech"],
-    "azure": ["language", "embedding", "speech_to_text", "text_to_speech"],
-    "openai_compatible": ["language", "embedding", "speech_to_text", "text_to_speech"],
-    "dashscope": ["language"],
-    "minimax": ["language"],
+    name: list(spec.modalities) for name, spec in PROVIDERS.items()
 }
 
 
@@ -426,16 +383,11 @@ def models_endpoint(url: str) -> str:
             model_names = list(ANTHROPIC_FALLBACK_MODELS)
         return [{"name": m, "provider": "anthropic"} for m in model_names]
 
-    # API-based discovery URLs (OpenAI-style /models endpoints)
+    # API-based discovery URLs (OpenAI-style /models endpoints), from the registry
     url_map = {
-        "openai": "https://api.openai.com/v1/models",
-        "groq": "https://api.groq.com/openai/v1/models",
-        "mistral": "https://api.mistral.ai/v1/models",
-        "deepseek": "https://api.deepseek.com/models",
-        "xai": "https://api.x.ai/v1/models",
-        "openrouter": "https://openrouter.ai/api/v1/models",
-        "dashscope": "https://dashscope.aliyuncs.com/compatible-mode/v1/models",
-        "minimax": "https://api.minimax.io/v1/models",
+        name: spec.openai_compat_discovery_url
+        for name, spec in PROVIDERS.items()
+        if spec.openai_compat_discovery_url
     }
 
     if provider == "ollama":

api/main.py

@@ -30,6 +30,7 @@
     notebooks,
     notes,
     podcasts,
+    providers,
     search,
     settings,
     source_chat,
@@ -391,6 +392,7 @@ async def open_notebook_error_handler(request: Request, exc: OpenNotebookError):
 app.include_router(chat.router, prefix="/api", tags=["chat"])
 app.include_router(source_chat.router, prefix="/api", tags=["source-chat"])
 app.include_router(credentials.router, prefix="/api", tags=["credentials"])
+app.include_router(providers.router, prefix="/api", tags=["providers"])
 app.include_router(languages.router, prefix="/api", tags=["languages"])
 
 

api/models.py

@@ -571,10 +571,11 @@ class MigrationResult(BaseModel):
 # Notebook delete cascade models
 # Credential models
 
-# Kept in sync with the frontend's ALL_PROVIDERS
-# (frontend/src/app/(dashboard)/settings/api-keys/page.tsx), TEST_MODELS
-# (open_notebook/ai/connection_tester.py), and PROVIDER_ENV_CONFIG
-# (api/credentials_service.py) - all three independently agree on this set.
+# Kept in sync with the provider registry
+# (open_notebook/ai/provider_registry.py PROVIDERS — the backend source of
+# truth) and the frontend's ALL_PROVIDERS (frontend/src/lib/providers.tsx).
+# A Literal can't be built at runtime, so this is one of the two remaining
+# manual copies; tests/test_credential_provider_validation.py enforces the sync.
 SupportedProvider = Literal[
     "openai",
     "anthropic",
@@ -596,6 +597,22 @@ class MigrationResult(BaseModel):
 ]
 
 
+class ProviderInfoResponse(BaseModel):
+    """Provider metadata from the provider registry."""
+
+    name: str = Field(..., description="Provider identifier (e.g. openai)")
+    display_name: str = Field(..., description="Human-friendly provider name")
+    modalities: List[str] = Field(
+        ..., description="Default modalities supported by the provider"
+    )
+    docs_url: Optional[str] = Field(
+        None, description="Where to get an API key / set the provider up"
+    )
+    env_configured: bool = Field(
+        ..., description="Whether the provider is configured via environment variables"
+    )
+
+
 class CreateCredentialRequest(BaseModel):
     """Request to create a new credential."""
 

api/routers/providers.py

@@ -0,0 +1,35 @@
+"""
+Providers Router
+
+Exposes the provider registry (open_notebook/ai/provider_registry.py) so
+clients can enumerate supported providers and their metadata instead of
+keeping their own copies.
+
+Endpoints:
+- GET /providers - List all supported providers with metadata
+"""
+
+from typing import List
+
+from fastapi import APIRouter
+
+from api.credentials_service import check_env_configured
+from api.models import ProviderInfoResponse
+from open_notebook.ai.provider_registry import PROVIDERS
+
+router = APIRouter(prefix="/providers", tags=["providers"])
+
+
+@router.get("", response_model=List[ProviderInfoResponse])
+async def list_providers():
+    """List all supported AI providers with their registry metadata."""
+    return [
+        ProviderInfoResponse(
+            name=spec.name,
+            display_name=spec.display_name,
+            modalities=list(spec.modalities),
+            docs_url=spec.docs_url,
+            env_configured=check_env_configured(spec.name),
+        )
+        for spec in PROVIDERS.values()
+    ]

docs/7-DEVELOPMENT/credentials.md

@@ -36,7 +36,7 @@ Settings UI ──► /credentials API ──► Credential record (encrypted, S
 
 CRUD plus lifecycle operations: `POST /credentials/{id}/test` (connection check), `/discover` (list available models), `/register-models` (create Model records from discovery), and two migration endpoints (`/migrate-from-env`, `/migrate-from-provider-config`). Swagger at `/docs` documents the shapes.
 
-**Supported providers (17)** are enforced by the `SupportedProvider` Literal in `api/models.py`, kept in sync with three other locations (frontend `ALL_PROVIDERS`, `connection_tester.TEST_MODELS`, `credentials_service.PROVIDER_ENV_CONFIG`):
+**Supported providers (17)** are defined once in the provider registry (`open_notebook/ai/provider_registry.py` `PROVIDERS`) — env vars, modalities, test models, discovery URLs and docs links all live there, and `connection_tester.TEST_MODELS`, `credentials_service.PROVIDER_ENV_CONFIG`/`PROVIDER_MODALITIES` and `model_discovery.OPENAI_COMPAT_PROVIDERS` are derived from it. `GET /api/providers` exposes the registry to clients. Two manual copies remain, enforced by `tests/test_credential_provider_validation.py`: the `SupportedProvider` Literal in `api/models.py` (typing can't be derived at runtime) and the frontend `ALL_PROVIDERS` tables (`frontend/src/lib/providers.tsx`):
 
 - Simple API key: openai, anthropic, google, groq, mistral, deepseek, xai, openrouter, voyage, elevenlabs, deepgram, dashscope, minimax
 - URL-based: ollama

open_notebook/AGENTS.md

@@ -12,7 +12,7 @@ Normative rules for working on the Python backend. Architecture and design ratio
 ## API layer (`api/`)
 
 - Structure is routes → services → models. Routers stay thin; business logic goes in `*_service.py`.
-- `SupportedProvider` must stay in sync across **four** locations: `api/models.py` (Literal), frontend `ALL_PROVIDERS`, `connection_tester.py` `TEST_MODELS`, and `credentials_service.py` `PROVIDER_ENV_CONFIG`. Adding a provider means touching all four.
+- Provider metadata (env vars, modalities, test models, discovery URLs, docs links) lives in the registry: `open_notebook/ai/provider_registry.py` `PROVIDERS`. `TEST_MODELS`, `PROVIDER_ENV_CONFIG`, `PROVIDER_MODALITIES` and `OPENAI_COMPAT_PROVIDERS` are derived from it, and `GET /api/providers` exposes it. Adding a provider = add it to the registry, plus **two** manual copies: the `SupportedProvider` Literal in `api/models.py` (typing can't be derived at runtime) and the frontend tables in `frontend/src/lib/providers.tsx` — both enforced by `tests/test_credential_provider_validation.py`.
 - NEVER return API key values from any endpoint — metadata only.
 - Every user-supplied URL field must go through `validate_url()` (`open_notebook/utils/url_validation.py`, async) for SSRF protection. Private IPs/localhost are intentionally allowed (self-hosted Ollama, LM Studio).
 - Errors: raise typed exceptions from `open_notebook.exceptions` — global handlers map them to HTTP status codes (`NotFoundError`→404, `InvalidInputError`→400, `AuthenticationError`→401, `RateLimitError`→429, `ConfigurationError`→422, `NetworkError`/`ExternalServiceError`→502, `OpenNotebookError`→500). Don't raise bare `HTTPException` for domain errors.

open_notebook/ai/connection_tester.py

@@ -14,6 +14,7 @@
 import httpx
 from loguru import logger
 
+from open_notebook.ai.provider_registry import PROVIDERS
 from open_notebook.utils.url_validation import validate_url
 
 
@@ -47,8 +48,9 @@ def _is_vertex_credentials_file_error(exc: Exception) -> bool:
     return isinstance(exc, (OSError, json.JSONDecodeError, GoogleAuthError))
 
 
-# Test models for each provider - uses minimal/cheapest models for testing
-# Format: (model_name, model_type)
+# Test models for each provider - uses minimal/cheapest models for testing.
+# Derived from the provider registry (the source of truth for test models).
+# Format: (model_name, model_type); None model = dynamic (first available).
 #
 # Prefer a provider-maintained floating alias where one exists, so a model
 # retirement doesn't silently break the connection test (see #970: Google
@@ -57,25 +59,8 @@ def _is_vertex_credentials_file_error(exc: Exception) -> bool:
 # its own. The provider test also no longer treats a model-level failure as
 # a connection failure (see `_connection_failure_reason`), so even if an
 # alias ever breaks, the test still reports the credentials correctly.
-TEST_MODELS = {
-    "openai": ("gpt-3.5-turbo", "language"),
-    "anthropic": ("claude-3-haiku-20240307", "language"),
-    "google": ("gemini-flash-latest", "language"),
-    "groq": ("llama-3.1-8b-instant", "language"),
-    "mistral": ("mistral-small-latest", "language"),
-    "deepseek": ("deepseek-chat", "language"),
-    "xai": ("grok-beta", "language"),
-    "openrouter": ("openai/gpt-3.5-turbo", "language"),
-    "voyage": ("voyage-3-lite", "embedding"),
-    "elevenlabs": ("eleven_multilingual_v2", "text_to_speech"),
-    "deepgram": ("aura-2-thalia-en", "text_to_speech"),
-    "ollama": (None, "language"),  # Dynamic - will use first available model
-    # Complex providers with additional configuration
-    "vertex": ("gemini-flash-latest", "language"),  # Uses Google Vertex AI
-    "azure": ("gpt-35-turbo", "language"),  # Azure OpenAI deployment name
-    "openai_compatible": (None, "language"),  # Dynamic - will use first available model
-    "dashscope": ("qwen-plus", "language"),
-    "minimax": ("MiniMax-M2.5", "language"),
+TEST_MODELS: dict = {
+    name: (spec.test_model, spec.test_model_type) for name, spec in PROVIDERS.items()
 }
 
 

open_notebook/ai/model_discovery.py

@@ -14,6 +14,7 @@
 from loguru import logger
 
 from open_notebook.ai.models import Model
+from open_notebook.ai.provider_registry import PROVIDERS
 from open_notebook.database.repository import repo_query
 from open_notebook.domain.credential import Credential
 
@@ -218,43 +219,28 @@ class ProviderDiscoverySpec:
     description: Optional[Callable[[dict], Optional[str]]] = None
 
 
+# Per-provider quirk hooks that can't live in the (pure data) registry.
+_COMPAT_CLASSIFY: Dict[str, Callable[[dict], str]] = {
+    "mistral": _classify_mistral,
+    # OpenRouter models are typically language models
+    "openrouter": lambda model: "language",
+}
+_COMPAT_DESCRIPTION: Dict[str, Callable[[dict], Optional[str]]] = {
+    "openrouter": lambda model: model.get("name"),
+}
+
+# Built from the provider registry: every provider with an
+# `openai_compat_discovery_url` gets a discovery spec. The API key env var
+# is the provider's (single) required env var from the registry.
 OPENAI_COMPAT_PROVIDERS: Dict[str, ProviderDiscoverySpec] = {
-    "openai": ProviderDiscoverySpec(
-        url="https://api.openai.com/v1/models",
-        env_var="OPENAI_API_KEY",
-    ),
-    "groq": ProviderDiscoverySpec(
-        url="https://api.groq.com/openai/v1/models",
-        env_var="GROQ_API_KEY",
-    ),
-    "mistral": ProviderDiscoverySpec(
-        url="https://api.mistral.ai/v1/models",
-        env_var="MISTRAL_API_KEY",
-        classify=_classify_mistral,
-    ),
-    "deepseek": ProviderDiscoverySpec(
-        url="https://api.deepseek.com/models",
-        env_var="DEEPSEEK_API_KEY",
-    ),
-    "xai": ProviderDiscoverySpec(
-        url="https://api.x.ai/v1/models",
-        env_var="XAI_API_KEY",
-    ),
-    "openrouter": ProviderDiscoverySpec(
-        url="https://openrouter.ai/api/v1/models",
-        env_var="OPENROUTER_API_KEY",
-        # OpenRouter models are typically language models
-        classify=lambda model: "language",
-        description=lambda model: model.get("name"),
-    ),
-    "dashscope": ProviderDiscoverySpec(
-        url="https://dashscope.aliyuncs.com/compatible-mode/v1/models",
-        env_var="DASHSCOPE_API_KEY",
-    ),
-    "minimax": ProviderDiscoverySpec(
-        url="https://api.minimax.io/v1/models",
-        env_var="MINIMAX_API_KEY",
-    ),
+    name: ProviderDiscoverySpec(
+        url=spec.openai_compat_discovery_url,
+        env_var=spec.required_env[0],
+        classify=_COMPAT_CLASSIFY.get(name),
+        description=_COMPAT_DESCRIPTION.get(name),
+    )
+    for name, spec in PROVIDERS.items()
+    if spec.openai_compat_discovery_url
 }