fix(evaluator): give the structured-output probe room for reasoning models

The probe capped output at 128 tokens. A reasoning model spends that
budget thinking and returns truncated or empty content, which is
indistinguishable from an endpoint that ignored the encoding, so
detection concluded UNSUPPORTED and fell back to the unenforced prompt
instruction.

Live against integrate.api.nvidia.com, nvidia/nemotron-3-nano-30b-a3b
and nvidia/llama-3.3-nemotron-super-49b-v1.5 both returned empty or
truncated content at 128 tokens and a valid probe response at 2048.

Raise the budget and warn when a probe is truncated, so an inconclusive
detection is visible rather than silently downgrading enforcement.

Also hoist the trials/target validation out of the evaluation session.

Signed-off-by: Sandy Chapman <schapman@nvidia.com>

Author: SandyChapman

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py

@@ -166,17 +166,17 @@ async def run(
         runtime_config = resolved_config.model_copy(update={"run_id": run_id})
         started_at = datetime.now(UTC)
 
+        if (trials is None) == (target is None):
+            raise ValueError("provide exactly one of trials or target")
+
         async with begin_evaluation_session():
             # Branch on which seam was supplied so the type checker can narrow ``target`` to a
             # concrete ``AgentEvalTarget`` without a cast.
             if trials is not None:
-                if target is not None:
-                    raise ValueError("provide exactly one of trials or target")
                 trial_list = list(trials)
-            elif target is not None:
-                trial_list = await self._generate_trials(tasks=task_list, target=target, config=runtime_config)
             else:
-                raise ValueError("provide exactly one of trials or target")
+                assert target is not None
+                trial_list = await self._generate_trials(tasks=task_list, target=target, config=runtime_config)
             scores = await self._score_trials(
                 tasks=task_list,
                 trials=trial_list,

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/structured_output.py

@@ -16,6 +16,10 @@
 
 _logger = logging.getLogger(__name__)
 
+#: Probes must survive a reasoning model's thinking budget; too small and truncated output looks
+#: identical to an endpoint that ignored the encoding.
+_PROBE_MAX_TOKENS = 4096
+
 #: The marker name must stay unguessable: the probe never shows this schema to the model, so a
 #: mode only passes when the server actually injected the grammar.
 _DEFAULT_PROBE_SCHEMA: dict = {
@@ -165,6 +169,14 @@ def _looks_like_unsupported_guided_json_error(message: str) -> bool:
     return False
 
 
+def _probe_was_truncated(response: dict) -> bool:
+    """Return whether the probe ran out of output budget before producing content."""
+    choices = response.get("choices")
+    if not isinstance(choices, list) or not choices:
+        return False
+    return choices[0].get("finish_reason") == "length"
+
+
 def _extract_chat_content(response: dict) -> str | None:
     choices = response.get("choices")
     if not isinstance(choices, list) or not choices:
@@ -249,7 +261,9 @@ async def _probe_structured_output_mode(
     base_request = {
         "messages": [{"role": "user", "content": probe_message}],
         "temperature": 0,
-        "max_tokens": 128,
+        # Generous because reasoning models spend most of a small budget thinking and return
+        # truncated or empty content, which is indistinguishable from an unhonoured encoding.
+        "max_tokens": _PROBE_MAX_TOKENS,
     }
     # Deep-copied so `probe_schema` stays unreachable from the request: Pydantic copies only a
     # dict's top level, and `probe_schema` is what the response is validated against.
@@ -270,6 +284,14 @@ async def _probe_structured_output_mode(
             content = _extract_chat_content(response)
             if content and _is_probe_valid_json(content, probe_schema):
                 return mode
+            if _probe_was_truncated(response):
+                _logger.warning(
+                    "Structured output probe for %s hit the %d-token budget, so support for %s "
+                    "could not be determined; enforcement may be dropped for this run.",
+                    model.name,
+                    _PROBE_MAX_TOKENS,
+                    mode.value,
+                )
         except Exception as e:
             if _looks_like_unsupported_guided_json_error(str(e)):
                 continue

packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py

@@ -921,6 +921,8 @@ def test_metric_row_error_is_none_for_a_trial_that_did_not_fail() -> None:
     row = _metric_row(AgentEvalTask(id="task-1", intent="Fix it.", inputs={}), _candidate_trial())
 
     assert row["trial"]["error"] is None
+
+
 class _PreflightCountingMetric(_ConstantMetric):
     """Metric that records how many times its preflight ran."""
 

packages/nemo_evaluator_sdk/tests/test_structured_output.py

@@ -519,3 +519,37 @@ async def inference_fn(model, request, max_retries, **kwargs):
 
     assert set(modes) == {StructuredOutputMode.OPENAI_RESPONSE_FORMAT}
     assert sorted(calls) == ["a", "b"], f"each concurrent run must probe in its own cache, got {calls}"
+
+
+@pytest.mark.asyncio
+async def test_truncated_probe_is_reported_rather_than_silently_unsupported(caplog):
+    """A reasoning model can spend the whole probe budget thinking and return empty content.
+
+    Truncation is indistinguishable from an endpoint ignoring the encoding, so it must at least be
+    surfaced instead of quietly resolving to the unenforced prompt fallback.
+    """
+
+    async def truncating_inference_fn(model, request, max_retries, **kwargs):
+        return {"choices": [{"finish_reason": "length", "message": {"content": ""}}]}
+
+    with caplog.at_level(logging.WARNING):
+        mode = await detect_structured_output_mode(
+            model=_test_model(), inference_fn=truncating_inference_fn, api_key=None
+        )
+
+    assert mode == StructuredOutputMode.UNSUPPORTED
+    assert [r for r in caplog.records if "could not be determined" in r.message]
+
+
+@pytest.mark.asyncio
+async def test_probe_budget_accommodates_a_reasoning_model():
+    """The probe must not use a budget a reasoning model burns before emitting content."""
+    seen: list[int] = []
+
+    async def inference_fn(model, request, max_retries, **kwargs):
+        seen.append(request["max_tokens"])
+        return {"choices": [{"message": {"content": '{"__nmp_probe_score": 1}'}}]}
+
+    await detect_structured_output_mode(model=_test_model(), inference_fn=inference_fn, api_key=None)
+
+    assert seen[0] >= 2048, f"probe budget {seen[0]} is too small for a reasoning model"