fix(evaluator): skip agent-eval preflight for metrics with no scoreable trial

Failed trials short-circuit to a failed score without invoking the metric,
but preflight ran for every metric regardless. A run whose trials all failed
therefore probed the judge endpoint for nothing, and could abort outright:
preflight resolves the judge model, which raises for an unresolved reference.
Importing a batch of failed trials returned an exception instead of failed
scores.

Preflight a metric only when one of its tasks has a trial that will be scored.

Signed-off-by: Sandy Chapman <schapman@nvidia.com>

Author: SandyChapman

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py

@@ -10,7 +10,7 @@
 import asyncio
 import uuid
 from collections import defaultdict
-from collections.abc import Awaitable, Callable, Sequence
+from collections.abc import Awaitable, Callable, Mapping, Sequence
 from datetime import UTC, datetime
 from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version as package_version
@@ -245,7 +245,7 @@ async def _score_trials(
             if not task.metrics:
                 raise ValueError(f"task {task.id!r} does not declare any metrics")
 
-        await _preflight_task_metrics(tasks)
+        await _preflight_task_metrics(tasks, trials_by_task)
 
         semaphore = asyncio.Semaphore(config.parallelism)
 
@@ -386,14 +386,22 @@ async def generate_one(index: int, task: AgentEvalTask) -> AgentEvalTrial:
                 await close_client()
 
 
-async def _preflight_task_metrics(tasks: Sequence[AgentEvalTask]) -> None:
-    """Run each distinct metric's preflight once.
+async def _preflight_task_metrics(
+    tasks: Sequence[AgentEvalTask],
+    trials_by_task: Mapping[str, Sequence[AgentEvalTrial]],
+) -> None:
+    """Run each distinct metric's preflight once, skipping metrics with nothing to score.
 
     Agent-eval scores metrics directly rather than through ``prepare_metric_for_execution``, so
-    nothing else runs their preflight.
+    nothing else runs their preflight. Failed trials short-circuit to a failed score without
+    invoking the metric, so a task whose trials all failed must not trigger one: preflight resolves
+    the judge endpoint, making it both a wasted request and a way for the run to abort.
     """
     preflighted: set[int] = set()
     for task in tasks:
+        scoreable = any(trial.status != AgentEvalTrialStatus.FAILED for trial in trials_by_task.get(task.id, ()))
+        if not scoreable:
+            continue
         for metric in task.metrics:
             if isinstance(metric, MetricWithPreflight) and id(metric) not in preflighted:
                 preflighted.add(id(metric))

packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py

@@ -972,3 +972,65 @@ async def test_run_preflights_each_metric_once_across_tasks() -> None:
     )
 
     assert metric.preflight_calls == 1
+
+
+class _ProbingMetric(_ConstantMetric):
+    """Metric whose preflight probes a remote endpoint, as an LLM judge's does."""
+
+    preflight_calls: int = 0
+
+    async def preflight(self) -> None:
+        self.preflight_calls += 1
+        raise RuntimeError("endpoint probe failed")
+
+
+def _failed_trial(task_id: str = "task-1", trial_id: str = "trial-1") -> AgentEvalTrial:
+    return AgentEvalTrial(
+        id=trial_id,
+        task_id=task_id,
+        status=AgentEvalTrialStatus.FAILED,
+        output=AgentOutput(output_text=""),
+    )
+
+
+@pytest.mark.asyncio
+async def test_failed_trials_are_scored_without_preflighting_their_metric() -> None:
+    """A run whose trials all failed must not probe the judge endpoint.
+
+    Failed trials short-circuit to a failed score without invoking the metric, so preflighting is
+    both a wasted remote call and a new way for the run to abort: preflight resolves the judge
+    model, and that raises for an unresolved reference. Importing a batch of failed trials must
+    still yield failed scores rather than an exception.
+    """
+    metric = _ProbingMetric()
+
+    result = await AgentEvaluator().run(
+        tasks=[_task(metric)],
+        trials=[_failed_trial()],
+        config=AgentEvalRunConfig(parallelism=1),
+    )
+
+    assert metric.preflight_calls == 0
+    assert [score.status for score in result.scores] == [AgentEvalScoreStatus.FAILED]
+
+
+@pytest.mark.asyncio
+async def test_metric_is_preflighted_when_any_trial_is_scoreable() -> None:
+    """One completed trial is enough to need the endpoint resolved."""
+    metric = _PreflightCountingMetric()
+
+    await AgentEvaluator().run(
+        tasks=[_task(metric, task_id="task-1"), _task(metric, task_id="task-2")],
+        trials=[
+            _failed_trial(task_id="task-1", trial_id="trial-1"),
+            AgentEvalTrial(
+                id="trial-2",
+                task_id="task-2",
+                status=AgentEvalTrialStatus.COMPLETED,
+                output=AgentOutput(output_text="Candidate answer"),
+            ),
+        ],
+        config=AgentEvalRunConfig(parallelism=1),
+    )
+
+    assert metric.preflight_calls == 1