|
46 | 46 | make_agent_inference_fn, |
47 | 47 | new_agent_inference_client, |
48 | 48 | ) |
49 | | -from nemo_evaluator_sdk.execution.metric_execution import generate_online_sample, run_sync |
| 49 | +from nemo_evaluator_sdk.execution.metric_execution import ( |
| 50 | + generate_online_sample, |
| 51 | + resolve_target_structured_output_mode, |
| 52 | + run_sync, |
| 53 | +) |
| 54 | +from nemo_evaluator_sdk.structured_output import structured_output_mode_session |
50 | 55 | from nemo_evaluator_sdk.execution.samples import build_metric_input |
51 | 56 | from nemo_evaluator_sdk.inference import InferenceFn |
52 | | -from nemo_evaluator_sdk.metrics.protocol import Metric, validate_metric_result |
| 57 | +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricWithPreflight, validate_metric_result |
53 | 58 | from nemo_evaluator_sdk.metrics.utils import metric_type_name |
54 | 59 | from nemo_evaluator_sdk.values import ( |
55 | 60 | Agent, |
@@ -161,22 +166,26 @@ async def run( |
161 | 166 | runtime_config = resolved_config.model_copy(update={"run_id": run_id}) |
162 | 167 | started_at = datetime.now(UTC) |
163 | 168 |
|
164 | | - # Branch on which seam was supplied so the type checker can narrow ``target`` to a |
165 | | - # concrete ``AgentEvalTarget`` without a cast. |
166 | | - if trials is not None: |
167 | | - if target is not None: |
| 169 | + # One detection session for the whole run: generation probes the target and scoring probes |
| 170 | + # any judge model, and imported-trial runs still score, so scoping this to generation alone |
| 171 | + # would leave judges probing per call. |
| 172 | + async with structured_output_mode_session(): |
| 173 | + # Branch on which seam was supplied so the type checker can narrow ``target`` to a |
| 174 | + # concrete ``AgentEvalTarget`` without a cast. |
| 175 | + if trials is not None: |
| 176 | + if target is not None: |
| 177 | + raise ValueError("provide exactly one of trials or target") |
| 178 | + trial_list = list(trials) |
| 179 | + elif target is not None: |
| 180 | + trial_list = await self._generate_trials(tasks=task_list, target=target, config=runtime_config) |
| 181 | + else: |
168 | 182 | raise ValueError("provide exactly one of trials or target") |
169 | | - trial_list = list(trials) |
170 | | - elif target is not None: |
171 | | - trial_list = await self._generate_trials(tasks=task_list, target=target, config=runtime_config) |
172 | | - else: |
173 | | - raise ValueError("provide exactly one of trials or target") |
174 | | - scores = await self._score_trials( |
175 | | - tasks=task_list, |
176 | | - trials=trial_list, |
177 | | - config=runtime_config, |
178 | | - run_id=run_id, |
179 | | - ) |
| 183 | + scores = await self._score_trials( |
| 184 | + tasks=task_list, |
| 185 | + trials=trial_list, |
| 186 | + config=runtime_config, |
| 187 | + run_id=run_id, |
| 188 | + ) |
180 | 189 | runner_scores = _collect_runner_aggregate_scores(target) if target is not None else [] |
181 | 190 | finished_at = datetime.now(UTC) |
182 | 191 | metadata = RunMetadata( |
@@ -239,6 +248,19 @@ async def _score_trials( |
239 | 248 | if not task.metrics: |
240 | 249 | raise ValueError(f"task {task.id!r} does not declare any metrics") |
241 | 250 |
|
| 251 | + # Agent-eval scores metrics directly rather than through prepare_metric_for_execution, so |
| 252 | + # nothing else runs their preflight. An LLM judge detects its endpoint's structured-output |
| 253 | + # encoding there; without this it would score using the provisional guess from new_hooks. |
| 254 | + # Deduplicated by identity because the same metric object is scored once per trial. The run |
| 255 | + # session would collapse repeat probes to one request anyway; this just avoids the repeated |
| 256 | + # awaits. Identity is stable here: `tasks` holds every metric for the duration of the loop. |
| 257 | + preflighted: set[int] = set() |
| 258 | + for task in tasks: |
| 259 | + for metric in task.metrics: |
| 260 | + if isinstance(metric, MetricWithPreflight) and id(metric) not in preflighted: |
| 261 | + preflighted.add(id(metric)) |
| 262 | + await metric.preflight() |
| 263 | + |
242 | 264 | semaphore = asyncio.Semaphore(config.parallelism) |
243 | 265 |
|
244 | 266 | async def guarded_score(task: AgentEvalTask, trial: AgentEvalTrial, metric: Metric) -> AgentEvalTaskScore: |
@@ -311,6 +333,8 @@ async def _generate_trials( |
311 | 333 | params = _resolve_live_params(config, target) |
312 | 334 | prompt_template = config.prompt_template or _default_prompt_template(target) |
313 | 335 | semaphore = asyncio.Semaphore(params.parallelism) |
| 336 | + # Hooks are built per row below; the run-level session opened by run() is what keeps the |
| 337 | + # endpoint probe to one round trip for the whole pass instead of one per row. |
314 | 338 |
|
315 | 339 | # Use the injected transport client when provided; otherwise build a default for the |
316 | 340 | # resolved target type and close it when generation finishes. |
@@ -395,10 +419,18 @@ async def _generate_sample( |
395 | 419 | # The transport client is a real class union, so isinstance narrowing is enough there. |
396 | 420 | if isinstance(target, Model): |
397 | 421 | model_params = cast(RunConfigOnlineModel, params) |
398 | | - preprocess_hooks, postprocess_hooks = inference.new_hooks(model_params, model_format=target.format) |
| 422 | + preprocess_hooks, postprocess_hooks = inference.new_hooks(model_params) |
399 | 423 | model_inference_fn = ( |
400 | 424 | cast(InferenceFn, inference_fn) if inference_fn is not None else inference.make_inference_request |
401 | 425 | ) |
| 426 | + # Hooks are built per row here, so this relies on detection being cached per endpoint: |
| 427 | + # without the probe the request would carry whichever encoding new_hooks guessed. |
| 428 | + await resolve_target_structured_output_mode( |
| 429 | + preprocess_hooks=preprocess_hooks, |
| 430 | + model=target, |
| 431 | + inference_fn=model_inference_fn, |
| 432 | + params=model_params, |
| 433 | + ) |
402 | 434 | return await generate_online_sample( |
403 | 435 | target=target, |
404 | 436 | row=row, |
|
0 commit comments