Skip to content

Commit 1c22655

Browse files
authored
fix(test): repair 6 failing tests on develop CI (#19)
Two unrelated groups, both pre-existing on develop: 1. tests/unit/core/test_inject_llm_provider.py (3 failures) Tests pass llm_provider="openai"/"anthropic" (string) which eagerly builds a real provider requiring OPENAI_API_KEY/ANTHROPIC_API_KEY. They only check identity wiring and never call the LLM. Add an autouse fixture that supplies dummy env keys so construction succeeds offline. 2. tests/unit/test_llm_providers.py::TestOpenAIReasoningTokens (3 failures) Thinking models (gpt-5*) route through the Responses API since #9/#15, but these tests still mocked client.chat.completions.create — so the awaited call hit an un-mocked MagicMock ("can't be awaited"). Rewrite to mock client.responses.create with a Responses-API-shaped response (output items + usage.output_tokens_details.reasoning_tokens) via a shared helper.
1 parent 7a479fc commit 1c22655

2 files changed

Lines changed: 68 additions & 54 deletions

File tree

tests/unit/core/test_inject_llm_provider.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,21 @@
1+
import pytest
2+
13
from dana.common.llm import LLM
24
from dana.common.llm.providers import OpenAIProvider
35
from dana.common.resource.rlm_resource import RLMResource
46
from dana.core.agent.star_agent import STARAgent
57
from dana.core.memory import LTMemory
68

79

10+
@pytest.fixture(autouse=True)
11+
def _dummy_provider_env_keys(monkeypatch):
12+
"""Tests here verify provider wiring/identity, not real API calls.
13+
String-path construction (llm_provider='openai'/'anthropic') reads env
14+
keys at provider build time; supply dummies so it succeeds offline."""
15+
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
16+
monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key")
17+
18+
819
def test_rlm_resource_uses_injected_llm(tmp_path):
920
prov = OpenAIProvider(api_key="test-key", model="gpt-4")
1021
llm = LLM(provider=prov)

tests/unit/test_llm_providers.py

Lines changed: 57 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,44 @@ async def test_chat_api_error(self, provider):
6666
await provider.chat(messages)
6767

6868

69+
def _responses_api_response(
70+
*,
71+
text: str,
72+
model: str,
73+
input_tokens: int = 10,
74+
output_tokens: int = 5,
75+
reasoning_tokens: int | None = None,
76+
):
77+
"""Build a Mock shaped like an OpenAI Responses API response.
78+
79+
``reasoning_tokens`` maps to ``usage.output_tokens_details.reasoning_tokens``
80+
— the field gpt-5/o3/o4 populate on the Responses path. Pass a value (incl. 0)
81+
to exercise the provider's falsy→None coercion; pass None to omit details.
82+
"""
83+
msg_item = Mock(type="message")
84+
msg_item.content = [Mock(type="output_text", text=text)]
85+
output_details = Mock(reasoning_tokens=reasoning_tokens) if reasoning_tokens is not None else None
86+
resp = Mock()
87+
resp.output = [msg_item]
88+
resp.status = "completed"
89+
resp.model = model
90+
resp.usage = Mock(
91+
input_tokens=input_tokens,
92+
output_tokens=output_tokens,
93+
total_tokens=input_tokens + output_tokens,
94+
input_tokens_details=None,
95+
output_tokens_details=output_details,
96+
)
97+
return resp
98+
99+
69100
class TestOpenAIReasoningTokens:
70-
"""Unit tests for OpenAI reasoning tokens parsing (thinking models)"""
101+
"""Unit tests for OpenAI reasoning tokens parsing (thinking models).
102+
103+
Thinking models (gpt-5*) route through the Responses API, so these tests
104+
mock ``client.responses.create`` with a Responses-shaped response rather
105+
than the Chat Completions ``choices`` shape the legacy path used.
106+
"""
71107

72108
@pytest.fixture
73109
def provider(self):
@@ -79,30 +115,19 @@ def provider(self):
79115

80116
@pytest.mark.asyncio
81117
async def test_chat_with_reasoning_tokens(self, provider):
82-
"""Test that reasoning_tokens are parsed from thinking model response"""
83-
mock_response = Mock()
84-
mock_response.choices = [Mock()]
85-
mock_response.choices[0].message.content = "The answer is 42"
86-
mock_response.choices[0].message.tool_calls = None
87-
mock_response.choices[0].finish_reason = "stop"
88-
mock_response.model = "gpt-5-thinking-mini"
89-
90-
# Mock usage with completion_tokens_details containing reasoning_tokens
91-
mock_response.usage = Mock()
92-
mock_response.usage.prompt_tokens = 50
93-
mock_response.usage.completion_tokens = 200
94-
mock_response.usage.total_tokens = 250
95-
mock_response.usage.prompt_tokens_details = None
96-
97-
# This is the key part - completion_tokens_details with reasoning_tokens
98-
mock_completion_details = Mock()
99-
mock_completion_details.reasoning_tokens = 150
100-
mock_response.usage.completion_tokens_details = mock_completion_details
118+
"""reasoning_tokens parsed from Responses usage.output_tokens_details."""
119+
mock_response = _responses_api_response(
120+
text="The answer is 42",
121+
model="gpt-5-thinking-mini",
122+
input_tokens=50,
123+
output_tokens=200,
124+
reasoning_tokens=150,
125+
)
101126

102127
async def mock_create(*args, **kwargs):
103128
return mock_response
104129

105-
with patch.object(provider.client.chat.completions, "create", side_effect=mock_create):
130+
with patch.object(provider.client.responses, "create", side_effect=mock_create):
106131
messages = [LLMMessage(role="user", content="What is the meaning of life?")]
107132
response = await provider.chat(messages)
108133

@@ -113,57 +138,35 @@ async def mock_create(*args, **kwargs):
113138

114139
@pytest.mark.asyncio
115140
async def test_chat_without_reasoning_tokens(self, provider):
116-
"""Test that reasoning_tokens is None for non-thinking models"""
117-
mock_response = Mock()
118-
mock_response.choices = [Mock()]
119-
mock_response.choices[0].message.content = "Hello!"
120-
mock_response.choices[0].message.tool_calls = None
121-
mock_response.choices[0].finish_reason = "stop"
122-
mock_response.model = "gpt-4"
123-
mock_response.usage = Mock()
124-
mock_response.usage.prompt_tokens = 10
125-
mock_response.usage.completion_tokens = 5
126-
mock_response.usage.total_tokens = 15
127-
mock_response.usage.prompt_tokens_details = None
128-
mock_response.usage.completion_tokens_details = None # No reasoning details
141+
"""reasoning_tokens is None when output_tokens_details is absent."""
142+
mock_response = _responses_api_response(text="Hello!", model="gpt-4")
129143

130144
async def mock_create(*args, **kwargs):
131145
return mock_response
132146

133-
with patch.object(provider.client.chat.completions, "create", side_effect=mock_create):
147+
with patch.object(provider.client.responses, "create", side_effect=mock_create):
134148
messages = [LLMMessage(role="user", content="Hello")]
135149
response = await provider.chat(messages)
136150

137151
assert response.reasoning_tokens is None
138152

139153
@pytest.mark.asyncio
140154
async def test_chat_with_zero_reasoning_tokens(self, provider):
141-
"""Test that zero reasoning_tokens is treated as None (falsy)"""
142-
mock_response = Mock()
143-
mock_response.choices = [Mock()]
144-
mock_response.choices[0].message.content = "Quick response"
145-
mock_response.choices[0].message.tool_calls = None
146-
mock_response.choices[0].finish_reason = "stop"
147-
mock_response.model = "gpt-5-thinking-mini"
148-
mock_response.usage = Mock()
149-
mock_response.usage.prompt_tokens = 10
150-
mock_response.usage.completion_tokens = 5
151-
mock_response.usage.total_tokens = 15
152-
mock_response.usage.prompt_tokens_details = None
153-
154-
# Zero reasoning tokens (model didn't use thinking)
155-
mock_completion_details = Mock()
156-
mock_completion_details.reasoning_tokens = 0
157-
mock_response.usage.completion_tokens_details = mock_completion_details
155+
"""Zero reasoning_tokens is coerced to None (falsy)."""
156+
mock_response = _responses_api_response(
157+
text="Quick response",
158+
model="gpt-5-thinking-mini",
159+
reasoning_tokens=0,
160+
)
158161

159162
async def mock_create(*args, **kwargs):
160163
return mock_response
161164

162-
with patch.object(provider.client.chat.completions, "create", side_effect=mock_create):
165+
with patch.object(provider.client.responses, "create", side_effect=mock_create):
163166
messages = [LLMMessage(role="user", content="Hi")]
164167
response = await provider.chat(messages)
165168

166-
# Zero is falsy, so reasoning_tokens should be None
169+
# Zero is falsy → provider coerces to None
167170
assert response.reasoning_tokens is None
168171

169172

0 commit comments

Comments
 (0)