Skip to content

Commit a8a2755

Browse files
philipph-askuiphilipph-askui
committed
feat: add support for openai-compatible APIs as vlm_provider and image_qa_provider
1 parent e9575c1 commit a8a2755

16 files changed

Lines changed: 1581 additions & 0 deletions

‎src/askui/model_providers/__init__.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@
1212
- `AnthropicVlmProvider` — VLM via direct Anthropic API
1313
- `AnthropicImageQAProvider` — image Q&A via direct Anthropic API
1414
- `GoogleImageQAProvider` — image Q&A via Google Gemini API (direct, no proxy)
15+
- `OpenAIVlmProvider` — VLM via any OpenAI-compatible API
16+
- `OpenAIImageQAProvider` — image Q&A via any OpenAI-compatible API
17+
- `OllamaVlmProvider` — VLM via local Ollama instance (OpenAI-compatible)
18+
- `OllamaImageQAProvider` — image Q&A via local Ollama instance (OpenAI-compatible)
1519
"""
1620

1721
from askui.model_providers.anthropic_image_qa_provider import AnthropicImageQAProvider
@@ -22,6 +26,10 @@
2226
from askui.model_providers.detection_provider import DetectionProvider
2327
from askui.model_providers.google_image_qa_provider import GoogleImageQAProvider
2428
from askui.model_providers.image_qa_provider import ImageQAProvider
29+
from askui.model_providers.ollama_image_qa_provider import OllamaImageQAProvider
30+
from askui.model_providers.ollama_vlm_provider import OllamaVlmProvider
31+
from askui.model_providers.openai_image_qa_provider import OpenAIImageQAProvider
32+
from askui.model_providers.openai_vlm_provider import OpenAIVlmProvider
2533
from askui.model_providers.vlm_provider import VlmProvider
2634
from askui.utils.model_pricing import ModelPricing
2735

@@ -35,5 +43,9 @@
3543
"GoogleImageQAProvider",
3644
"ImageQAProvider",
3745
"ModelPricing",
46+
"OllamaImageQAProvider",
47+
"OllamaVlmProvider",
48+
"OpenAIImageQAProvider",
49+
"OpenAIVlmProvider",
3850
"VlmProvider",
3951
]

‎src/askui/model_providers/ollama_image_qa_provider.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
"""OllamaImageQAProvider — image Q&A via a local Ollama instance."""
2+
3+
from openai import OpenAI
4+
5+
from askui.model_providers.openai_image_qa_provider import OpenAIImageQAProvider
6+
7+
_DEFAULT_BASE_URL = "http://localhost:11434/v1"
8+
_DEFAULT_MODEL_ID = "qwen3.5"
9+
10+
11+
class OllamaImageQAProvider(OpenAIImageQAProvider):
12+
"""Image Q&A provider that routes requests to a local Ollama instance.
13+
14+
Thin convenience wrapper around `OpenAIImageQAProvider` with Ollama
15+
defaults (``base_url``, ``api_key``, ``model_id``).
16+
17+
Args:
18+
model_id (str, optional): Ollama model to use. Defaults to
19+
``"qwen3.5"``.
20+
base_url (str, optional): Base URL for the Ollama OpenAI-compatible
21+
API. Defaults to ``"http://localhost:11434/v1"``.
22+
client (`OpenAI` | None, optional): Pre-configured OpenAI client.
23+
If provided, ``base_url`` is ignored.
24+
25+
Example:
26+
```python
27+
from askui import AgentSettings, ComputerAgent
28+
from askui.model_providers import OllamaImageQAProvider
29+
30+
agent = ComputerAgent(settings=AgentSettings(
31+
image_qa_provider=OllamaImageQAProvider(
32+
model_id="llava",
33+
)
34+
))
35+
```
36+
"""
37+
38+
def __init__(
39+
self,
40+
model_id: str = _DEFAULT_MODEL_ID,
41+
base_url: str = _DEFAULT_BASE_URL,
42+
client: OpenAI | None = None,
43+
) -> None:
44+
super().__init__(
45+
model_id=model_id,
46+
api_key="ollama", # Ollama requires no auth; OpenAI SDK needs a value
47+
base_url=base_url,
48+
client=client,
49+
)

‎src/askui/model_providers/ollama_vlm_provider.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
"""OllamaVlmProvider — VLM access via a local Ollama instance."""
2+
3+
from openai import OpenAI
4+
5+
from askui.model_providers.openai_vlm_provider import OpenAIVlmProvider
6+
7+
_DEFAULT_BASE_URL = "http://localhost:11434/v1"
8+
_DEFAULT_MODEL_ID = "qwen3.5"
9+
10+
11+
class OllamaVlmProvider(OpenAIVlmProvider):
12+
"""VLM provider that routes requests to a local Ollama instance.
13+
14+
Thin convenience wrapper around `OpenAIVlmProvider` with Ollama
15+
defaults (``base_url``, ``api_key``, ``model_id``).
16+
17+
Args:
18+
model_id (str, optional): Ollama model to use. Defaults to
19+
``"qwen3.5"``.
20+
base_url (str, optional): Base URL for the Ollama OpenAI-compatible
21+
API. Defaults to ``"http://localhost:11434/v1"``.
22+
client (`OpenAI` | None, optional): Pre-configured OpenAI client.
23+
If provided, ``base_url`` is ignored.
24+
25+
Example:
26+
```python
27+
from askui import AgentSettings, ComputerAgent
28+
from askui.model_providers import OllamaVlmProvider
29+
30+
agent = ComputerAgent(settings=AgentSettings(
31+
vlm_provider=OllamaVlmProvider(
32+
model_id="qwen3.5",
33+
)
34+
))
35+
```
36+
"""
37+
38+
def __init__(
39+
self,
40+
model_id: str = _DEFAULT_MODEL_ID,
41+
base_url: str = _DEFAULT_BASE_URL,
42+
client: OpenAI | None = None,
43+
) -> None:
44+
super().__init__(
45+
model_id=model_id,
46+
api_key="ollama", # Ollama requires no auth; OpenAI SDK needs a value
47+
base_url=base_url,
48+
client=client,
49+
)

‎src/askui/model_providers/openai_image_qa_provider.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
"""OpenAIImageQAProvider — image Q&A via any OpenAI-compatible API."""
2+
3+
from functools import cached_property
4+
from typing import Type
5+
6+
from openai import OpenAI
7+
from typing_extensions import override
8+
9+
from askui.model_providers.image_qa_provider import ImageQAProvider
10+
from askui.models.openai.get_model import OpenAIGetModel
11+
from askui.models.shared.settings import GetSettings
12+
from askui.models.types.response_schemas import ResponseSchema
13+
from askui.utils.source_utils import Source
14+
15+
16+
class OpenAIImageQAProvider(ImageQAProvider):
17+
"""Image Q&A provider for any OpenAI-compatible API.
18+
19+
Works with OpenAI, Ollama, vLLM, LM Studio, Together AI, and any
20+
other service that exposes an OpenAI-compatible ``/v1/chat/completions``
21+
endpoint.
22+
23+
Args:
24+
model_id (str): Model name to use.
25+
api_key (str | None, optional): API key. Reads ``OPENAI_API_KEY``
26+
from the environment if not provided.
27+
base_url (str | None, optional): Base URL for the API. Defaults
28+
to the OpenAI API (``https://api.openai.com/v1``).
29+
client (`OpenAI` | None, optional): Pre-configured OpenAI client.
30+
If provided, ``api_key`` and ``base_url`` are ignored.
31+
32+
Example:
33+
```python
34+
from askui import AgentSettings, ComputerAgent
35+
from askui.model_providers import OpenAIImageQAProvider
36+
37+
agent = ComputerAgent(settings=AgentSettings(
38+
image_qa_provider=OpenAIImageQAProvider(
39+
model_id="gpt-4o",
40+
api_key="sk-...",
41+
)
42+
))
43+
```
44+
"""
45+
46+
def __init__(
47+
self,
48+
model_id: str,
49+
api_key: str | None = None,
50+
base_url: str | None = None,
51+
client: OpenAI | None = None,
52+
) -> None:
53+
self._model_id = model_id
54+
self._client = client or OpenAI(
55+
api_key=api_key,
56+
base_url=base_url,
57+
)
58+
59+
@cached_property
60+
def _get_model(self) -> OpenAIGetModel:
61+
"""Lazily initialise the `OpenAIGetModel` on first use."""
62+
return OpenAIGetModel(model_id=self._model_id, client=self._client)
63+
64+
@override
65+
def query(
66+
self,
67+
query: str,
68+
source: Source,
69+
response_schema: Type[ResponseSchema] | None,
70+
get_settings: GetSettings,
71+
) -> ResponseSchema | str:
72+
result: ResponseSchema | str = self._get_model.get(
73+
query=query,
74+
source=source,
75+
response_schema=response_schema,
76+
get_settings=get_settings,
77+
)
78+
return result

‎src/askui/model_providers/openai_vlm_provider.py‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
"""OpenAIVlmProvider — VLM access via any OpenAI-compatible API."""
2+
3+
import os
4+
from functools import cached_property
5+
from typing import Any
6+
7+
from openai import OpenAI
8+
from typing_extensions import override
9+
10+
from askui.model_providers.vlm_provider import VlmProvider
11+
from askui.models.openai.messages_api import OpenAIMessagesApi
12+
from askui.models.shared.agent_message_param import (
13+
MessageParam,
14+
ThinkingConfigParam,
15+
ToolChoiceParam,
16+
)
17+
from askui.models.shared.prompts import SystemPrompt
18+
from askui.models.shared.tools import ToolCollection
19+
from askui.utils.model_pricing import ModelPricing
20+
21+
_DEFAULT_MODEL_ID = "gpt-5.4"
22+
23+
24+
class OpenAIVlmProvider(VlmProvider):
25+
"""VLM provider for any OpenAI-compatible API.
26+
27+
Works with OpenAI, Ollama, vLLM, LM Studio, Together AI, and any
28+
other service that exposes an OpenAI-compatible ``/v1/chat/completions``
29+
endpoint.
30+
31+
Args:
32+
model_id (str): Model name to use.
33+
api_key (str | None, optional): API key. Reads ``OPENAI_API_KEY``
34+
from the environment if not provided.
35+
base_url (str | None, optional): Base URL for the API. Defaults
36+
to the OpenAI API (``https://api.openai.com/v1``).
37+
client (`OpenAI` | None, optional): Pre-configured OpenAI client.
38+
If provided, ``api_key`` and ``base_url`` are ignored.
39+
40+
Example:
41+
```python
42+
from askui import AgentSettings, ComputerAgent
43+
from askui.model_providers import OpenAIVlmProvider
44+
45+
agent = ComputerAgent(settings=AgentSettings(
46+
vlm_provider=OpenAIVlmProvider(
47+
model_id="gpt-4o",
48+
api_key="sk-...",
49+
)
50+
))
51+
```
52+
"""
53+
54+
def __init__(
55+
self,
56+
model_id: str | None = None,
57+
api_key: str | None = None,
58+
base_url: str | None = None,
59+
client: OpenAI | None = None,
60+
input_cost_per_million_tokens: float | None = None,
61+
output_cost_per_million_tokens: float | None = None,
62+
cache_write_cost_per_million_tokens: float | None = None,
63+
cache_read_cost_per_million_tokens: float | None = None,
64+
) -> None:
65+
self._model_id_value = (
66+
model_id or os.environ.get("VLM_PROVIDER_MODEL_ID") or _DEFAULT_MODEL_ID
67+
)
68+
if client is not None:
69+
self._client = client
70+
else:
71+
self._client = OpenAI(
72+
api_key=api_key,
73+
base_url=base_url,
74+
)
75+
76+
self._pricing = ModelPricing.for_model(
77+
self._model_id_value,
78+
input_cost_per_million_tokens=input_cost_per_million_tokens,
79+
output_cost_per_million_tokens=output_cost_per_million_tokens,
80+
cache_write_cost_per_million_tokens=cache_write_cost_per_million_tokens,
81+
cache_read_cost_per_million_tokens=cache_read_cost_per_million_tokens,
82+
)
83+
84+
@property
85+
@override
86+
def model_id(self) -> str:
87+
return self._model_id_value
88+
89+
@property
90+
@override
91+
def pricing(self) -> ModelPricing | None:
92+
return self._pricing
93+
94+
@cached_property
95+
def _messages_api(self) -> OpenAIMessagesApi:
96+
"""Lazily initialise the `OpenAIMessagesApi` on first use."""
97+
return OpenAIMessagesApi(client=self._client)
98+
99+
@override
100+
def create_message(
101+
self,
102+
messages: list[MessageParam],
103+
tools: ToolCollection | None = None,
104+
max_tokens: int | None = None,
105+
system: SystemPrompt | None = None,
106+
thinking: ThinkingConfigParam | None = None,
107+
tool_choice: ToolChoiceParam | None = None,
108+
temperature: float | None = None,
109+
provider_options: dict[str, Any] | None = None,
110+
) -> MessageParam:
111+
return self._messages_api.create_message(
112+
messages=messages,
113+
model_id=self._model_id_value,
114+
tools=tools,
115+
max_tokens=max_tokens,
116+
system=system,
117+
thinking=thinking,
118+
tool_choice=tool_choice,
119+
temperature=temperature,
120+
provider_options=provider_options,
121+
)

‎src/askui/models/openai/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Model integration via OpenAI-compatible APIs."""

0 commit comments

Comments
 (0)