Skip to content

Commit 2a236a1

Browse files
committed
fix(evaluator): detect structured output support from the endpoint
Structured output mode was chosen from `Model.format`, but support is a property of the endpoint, not of the label attached to the model. integrate.api.nvidia.com is served under the `nim` label yet rejects `nvext.guided_json` with a 400 and silently ignores root `guided_json`, while honouring OpenAI `response_format`. Preflight probed only the two guided_json placements for `nim`, so it resolved to UNSUPPORTED and fell back to an unenforced "return JSON" prompt instruction. Judges kept scoring and nothing surfaced the downgrade. Probe every endpoint with the same ordered candidate list, trying `response_format` first. `Model.format` is deprecated, ignored, and now defaults to `openai`. Detection is cached per run behind a re-entrant ContextVar session. Run scoping is what makes caching a negative result safe: a probe can fail for reasons unrelated to capability, and a process-global negative would disable enforcement until restart. Five generation paths built a hook and issued requests without probing: target generation, agent evaluation, the multi-metric benchmark path, ProfBench, and direct `compute_scores()`. All now resolve first, and a hook used unprobed logs a warning once so a future path is visible rather than silent. The default `max_tokens` cap was NIM-only, so flipping the default format would have silently removed it. It is now unconditional, which changes behaviour for openai-format models that never set it. Signed-off-by: Sandy Chapman <schapman@nvidia.com>
1 parent a2e5eae commit 2a236a1

37 files changed

Lines changed: 1700 additions & 224 deletions

File tree

docs/evaluator/agent-eval/targets-and-runners.mdx

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,14 @@ bare model do before you wrap it in an agent?). The evaluator prompts it with ea
3131
|---|---|---|
3232
| `url` | yes | endpoint URL (e.g. `.../v1/chat/completions` or `.../v1/completions`) |
3333
| `name` | yes | model identifier, stamped on trials |
34-
| `format` | no | `ModelFormat.NVIDIA_NIM` (default), `ModelFormat.OPEN_AI`, or `ModelFormat.LLAMA_STACK` — serialized as `nim` / `openai` / `llama_stack` |
34+
| `format` | no | **deprecated and ignored** — structured output support is probed from the endpoint during preflight |
3535
| `api_key_secret` | no | credential reference — `workspace/secret_name` or `secret_name` |
3636

3737
```python
38-
from nemo_evaluator_sdk.enums import ModelFormat
3938
from nemo_evaluator_sdk.values import Model
4039

4140
target = Model(url="https://integrate.api.nvidia.com/v1/chat/completions", name="meta/llama-3.1-8b-instruct",
42-
format=ModelFormat.OPEN_AI, api_key_secret="NVIDIA_API_KEY")
41+
api_key_secret="NVIDIA_API_KEY")
4342
```
4443

4544
For a local `run()`, `api_key_secret` names an **environment variable** in your process; for a submitted

docs/evaluator/manage-tasks-tasksets.mdx

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -315,13 +315,12 @@ each run.
315315
from nemo_evaluator.api.schemas import TasksetRef
316316
from nemo_evaluator.jobs.agent_spec import AgentEvalInputSpec, ModelTarget
317317
from nemo_evaluator_sdk.values import Model
318-
from nemo_evaluator_sdk.enums import ModelFormat
319318

320319
# Instead of inlining AgentEvalTaskInput objects, point `tasks` at a stored taskset.
321320
input_spec = AgentEvalInputSpec(
322321
tasks=TasksetRef("default/geography-suite"),
323322
target=ModelTarget(
324-
model=Model(url="https://integrate.api.nvidia.com/v1", name="meta/llama-3.3-70b-instruct", format=ModelFormat.OPEN_AI),
323+
model=Model(url="https://integrate.api.nvidia.com/v1", name="meta/llama-3.3-70b-instruct"),
325324
),
326325
)
327326
```

docs/evaluator/metrics/llm-as-a-judge.mdx

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,12 @@ from nemo_evaluator_sdk import (
6969
RunConfig,
7070
LLMJudgeMetric
7171
)
72-
from nemo_evaluator_sdk.enums import ModelFormat
7372

7473

7574
metric = LLMJudgeMetric(
7675
model=Model(
7776
url="<judge-nim-url>/v1",
7877
name="meta/llama-3.1-70b-instruct",
79-
format=ModelFormat.NVIDIA_NIM,
8078
),
8179
scores=[
8280
RangeScore(
@@ -171,14 +169,12 @@ Use rubric scores when you want categorical labels with explicit descriptions:
171169

172170
```python
173171
from nemo_evaluator_sdk import JSONScoreParser, Model, RubricScore, LLMJudgeMetric
174-
from nemo_evaluator_sdk.enums import ModelFormat
175172
from nemo_evaluator_sdk.values import Rubric
176173

177174
metric = LLMJudgeMetric(
178175
model=Model(
179176
url="<judge-nim-url>/v1",
180177
name="meta/llama-3.1-70b-instruct",
181-
format=ModelFormat.NVIDIA_NIM,
182178
),
183179
scores=[
184180
RubricScore(
@@ -282,14 +278,12 @@ For production workloads, submit the same metric and dataset as a durable platfo
282278

283279
```python
284280
from nemo_evaluator_sdk import RunConfig, JSONScoreParser, Model, RubricScore, LLMJudgeMetric
285-
from nemo_evaluator_sdk.enums import ModelFormat
286281
from nemo_evaluator_sdk.values import Rubric
287282

288283
metric = LLMJudgeMetric(
289284
model=Model(
290285
url="<judge-nim-url>/v1",
291286
name="meta/llama-3.1-70b-instruct",
292-
format=ModelFormat.NVIDIA_NIM,
293287
),
294288
scores=[
295289
RubricScore(
@@ -561,7 +555,6 @@ metric = {
561555
"model": {
562556
"url": "<judge-url>/v1",
563557
"name": "meta/llama-3.1-70b-instruct",
564-
"format": "nim"
565558
},
566559
"scores": [
567560
{
@@ -608,7 +601,6 @@ metric = {
608601
"model": {
609602
"url": "https://api.example.com/v1",
610603
"name": "gpt-4",
611-
"format": "openai",
612604
"api_key_secret": "judge-api-key",
613605
},
614606
# ... scores and prompt_template
@@ -647,7 +639,6 @@ metric = {
647639
"model": {
648640
"url": "<nim-url>/v1",
649641
"name": "nvidia/llama-3.3-nemotron-super-49b-v1",
650-
"format": "nim",
651642
},
652643
# ... scores ...
653644
"system_prompt": "'detailed thinking on'",

docs/evaluator/metrics/model-configuration.mdx

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ from nemo_evaluator_sdk import Model
3535
model = Model(
3636
url="https://integrate.api.nvidia.com/v1",
3737
name="meta/llama-3.1-70b-instruct",
38-
format="nim",
3938
api_key_secret="NVIDIA_API_KEY",
4039
)
4140
```
@@ -44,7 +43,7 @@ model = Model(
4443
|-------|----------|-------------|
4544
| `url` | Yes | Base URL of the inference endpoint. |
4645
| `name` | Yes | Model name to send in inference requests. |
47-
| `format` | No | API format: `"nim"`, `"openai"`, or `"llama_stack"`. Defaults to `"nim"`. |
46+
| `format` | No | **Deprecated and ignored.** Structured output support is detected from the endpoint during preflight rather than inferred from this label. |
4847
| `api_key_secret` | No | Model API key reference. See [Model API Authentication](#model-api-authentication). |
4948

5049
<a id="model-api-authentication"></a>
@@ -82,7 +81,6 @@ from nemo_evaluator_sdk import (
8281
model = Model(
8382
url="https://integrate.api.nvidia.com/v1",
8483
name="meta/llama-3.1-70b-instruct",
85-
format="nim",
8684
api_key_secret="NVIDIA_API_KEY",
8785
)
8886

@@ -116,7 +114,6 @@ from nemo_evaluator_sdk import Model, RangeScore, LLMJudgeMetric
116114
judge_model = Model(
117115
url="https://integrate.api.nvidia.com/v1",
118116
name="meta/llama-3.1-70b-instruct",
119-
format="nim",
120117
api_key_secret="NVIDIA_API_KEY",
121118
)
122119
metric = LLMJudgeMetric(

docs/notebooks/ndd_evaluator.mdx

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -575,7 +575,6 @@ def run_evaluation(
575575
model_kwargs = {
576576
"url": model_spec["url"],
577577
"name": model_spec["model_id"],
578-
"format": "openai",
579578
}
580579
if secret_key_name:
581580
model_kwargs["api_key_secret"] = secret_key_name

packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -411,7 +411,6 @@
411411
" model=Model(\n",
412412
" url=\"https://integrate.api.nvidia.com/v1/chat/completions\",\n",
413413
" name=os.environ.get(\"JUDGE_MODEL\", \"nvidia/nvidia-nemotron-nano-9b-v2\"),\n",
414-
" format=\"openai\",\n",
415414
" # api_key_secret names the env var to read; model.api_key resolves os.environ[JUDGE_API_KEY_ENV].\n",
416415
" api_key_secret=SecretRef(JUDGE_API_KEY_ENV),\n",
417416
" ),\n",

packages/nemo_evaluator_sdk/examples/profbench/profbench.py

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@
2020
from nemo_evaluator_sdk.agent_eval.scores import AgentEvalTaskScore
2121
from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask
2222
from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput
23-
from nemo_evaluator_sdk.execution.metric_execution import generate_online_sample
23+
from nemo_evaluator_sdk.execution.metric_execution import (
24+
generate_online_sample,
25+
resolve_target_structured_output_mode,
26+
)
2427
from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
2528
from nemo_evaluator_sdk.values import InferenceParams, Model, RunConfigOnlineModel
2629
from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor
@@ -246,7 +249,14 @@ def __init__(
246249
self.default_headers = default_headers
247250

248251
async def judge(self, request: ProfBenchJudgeRequest) -> ProfBenchJudgeDecision:
249-
preprocess_hooks, postprocess_hooks = inference.new_hooks(self.params, model_format=self.model.format)
252+
preprocess_hooks, postprocess_hooks = inference.new_hooks(self.params)
253+
# Hooks are rebuilt per judge call, so this relies on detection being cached per endpoint.
254+
await resolve_target_structured_output_mode(
255+
preprocess_hooks=preprocess_hooks,
256+
model=self.model,
257+
inference_fn=self.inference_fn,
258+
params=self.params,
259+
)
250260
sample = await generate_online_sample(
251261
target=self.model,
252262
row={"prompt": _render_judge_prompt(request)},

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py

Lines changed: 50 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,15 @@
4646
make_agent_inference_fn,
4747
new_agent_inference_client,
4848
)
49-
from nemo_evaluator_sdk.execution.metric_execution import generate_online_sample, run_sync
49+
from nemo_evaluator_sdk.execution.metric_execution import (
50+
generate_online_sample,
51+
resolve_target_structured_output_mode,
52+
run_sync,
53+
)
54+
from nemo_evaluator_sdk.structured_output import structured_output_mode_session
5055
from nemo_evaluator_sdk.execution.samples import build_metric_input
5156
from nemo_evaluator_sdk.inference import InferenceFn
52-
from nemo_evaluator_sdk.metrics.protocol import Metric, validate_metric_result
57+
from nemo_evaluator_sdk.metrics.protocol import Metric, MetricWithPreflight, validate_metric_result
5358
from nemo_evaluator_sdk.metrics.utils import metric_type_name
5459
from nemo_evaluator_sdk.values import (
5560
Agent,
@@ -161,22 +166,26 @@ async def run(
161166
runtime_config = resolved_config.model_copy(update={"run_id": run_id})
162167
started_at = datetime.now(UTC)
163168

164-
# Branch on which seam was supplied so the type checker can narrow ``target`` to a
165-
# concrete ``AgentEvalTarget`` without a cast.
166-
if trials is not None:
167-
if target is not None:
169+
# One detection session for the whole run: generation probes the target and scoring probes
170+
# any judge model, and imported-trial runs still score, so scoping this to generation alone
171+
# would leave judges probing per call.
172+
async with structured_output_mode_session():
173+
# Branch on which seam was supplied so the type checker can narrow ``target`` to a
174+
# concrete ``AgentEvalTarget`` without a cast.
175+
if trials is not None:
176+
if target is not None:
177+
raise ValueError("provide exactly one of trials or target")
178+
trial_list = list(trials)
179+
elif target is not None:
180+
trial_list = await self._generate_trials(tasks=task_list, target=target, config=runtime_config)
181+
else:
168182
raise ValueError("provide exactly one of trials or target")
169-
trial_list = list(trials)
170-
elif target is not None:
171-
trial_list = await self._generate_trials(tasks=task_list, target=target, config=runtime_config)
172-
else:
173-
raise ValueError("provide exactly one of trials or target")
174-
scores = await self._score_trials(
175-
tasks=task_list,
176-
trials=trial_list,
177-
config=runtime_config,
178-
run_id=run_id,
179-
)
183+
scores = await self._score_trials(
184+
tasks=task_list,
185+
trials=trial_list,
186+
config=runtime_config,
187+
run_id=run_id,
188+
)
180189
runner_scores = _collect_runner_aggregate_scores(target) if target is not None else []
181190
finished_at = datetime.now(UTC)
182191
metadata = RunMetadata(
@@ -237,6 +246,19 @@ async def _score_trials(
237246
if not task.metrics:
238247
raise ValueError(f"task {task.id!r} does not declare any metrics")
239248

249+
# Agent-eval scores metrics directly rather than through prepare_metric_for_execution, so
250+
# nothing else runs their preflight. An LLM judge detects its endpoint's structured-output
251+
# encoding there; without this it would score using the provisional guess from new_hooks.
252+
# Deduplicated by identity because the same metric object is scored once per trial. The run
253+
# session would collapse repeat probes to one request anyway; this just avoids the repeated
254+
# awaits. Identity is stable here: `tasks` holds every metric for the duration of the loop.
255+
preflighted: set[int] = set()
256+
for task in tasks:
257+
for metric in task.metrics:
258+
if isinstance(metric, MetricWithPreflight) and id(metric) not in preflighted:
259+
preflighted.add(id(metric))
260+
await metric.preflight()
261+
240262
semaphore = asyncio.Semaphore(config.parallelism)
241263

242264
async def guarded_score(task: AgentEvalTask, trial: AgentEvalTrial, metric: Metric) -> AgentEvalTaskScore:
@@ -309,6 +331,8 @@ async def _generate_trials(
309331
params = _resolve_live_params(config, target)
310332
prompt_template = config.prompt_template or _default_prompt_template(target)
311333
semaphore = asyncio.Semaphore(params.parallelism)
334+
# Hooks are built per row below; the run-level session opened by run() is what keeps the
335+
# endpoint probe to one round trip for the whole pass instead of one per row.
312336

313337
# Use the injected transport client when provided; otherwise build a default for the
314338
# resolved target type and close it when generation finishes.
@@ -393,10 +417,18 @@ async def _generate_sample(
393417
# The transport client is a real class union, so isinstance narrowing is enough there.
394418
if isinstance(target, Model):
395419
model_params = cast(RunConfigOnlineModel, params)
396-
preprocess_hooks, postprocess_hooks = inference.new_hooks(model_params, model_format=target.format)
420+
preprocess_hooks, postprocess_hooks = inference.new_hooks(model_params)
397421
model_inference_fn = (
398422
cast(InferenceFn, inference_fn) if inference_fn is not None else inference.make_inference_request
399423
)
424+
# Hooks are built per row here, so this relies on detection being cached per endpoint:
425+
# without the probe the request would carry whichever encoding new_hooks guessed.
426+
await resolve_target_structured_output_mode(
427+
preprocess_hooks=preprocess_hooks,
428+
model=target,
429+
inference_fn=model_inference_fn,
430+
params=model_params,
431+
)
400432
return await generate_online_sample(
401433
target=target,
402434
row=row,

0 commit comments

Comments
 (0)