fix: handle unknown latency in optimization reports

Author: 16yunH

examples/optimization/eval_optimize_loop/loop/analysis.py

@@ -490,6 +490,11 @@ def unique_proposals(
             raise RuntimeError("AgentOptimizer produced no auditable prompt candidate")
         return unique
 
+    @staticmethod
+    def _pareto_latency_ms(resources: ResourceUsage) -> float:
+        """Return a dominance-safe latency value for Pareto comparisons."""
+        return math.inf if resources.p95_latency_ms is None else resources.p95_latency_ms
+
     @staticmethod
     def mark_pareto(candidates: list[CandidateEvaluation]) -> None:
         """Mark gate-eligible candidates not dominated on quality, tokens, and latency."""
@@ -498,20 +503,20 @@ def mark_pareto(candidates: list[CandidateEvaluation]) -> None:
             if not candidate.gate.accepted:
                 candidate.pareto_optimal = False
                 continue
+            candidate_latency = RegressionAnalyzer._pareto_latency_ms(candidate.audit.resources)
             dominated = False
             for other in eligible:
                 if other is candidate:
                     continue
+                other_latency = RegressionAnalyzer._pareto_latency_ms(other.audit.resources)
                 no_worse = (other.validation.pass_rate >= candidate.validation.pass_rate
                             and other.validation.average_score >= candidate.validation.average_score
                             and other.audit.resources.total_tokens <= candidate.audit.resources.total_tokens
-                            and (other.audit.resources.p95_latency_ms
-                                 or 0.0) <= (candidate.audit.resources.p95_latency_ms or 0.0))
+                            and other_latency <= candidate_latency)
                 strictly_better = (other.validation.pass_rate > candidate.validation.pass_rate
                                    or other.validation.average_score > candidate.validation.average_score
                                    or other.audit.resources.total_tokens < candidate.audit.resources.total_tokens
-                                   or (other.audit.resources.p95_latency_ms
-                                       or 0.0) < (candidate.audit.resources.p95_latency_ms or 0.0))
+                                   or other_latency < candidate_latency)
                 if no_worse and strictly_better:
                     dominated = True
                     break

examples/optimization/eval_optimize_loop/loop/reporting.py

@@ -145,8 +145,9 @@ def render_markdown(report: OptimizationReport) -> str:
     for item in report.candidates:
         usage = item.audit.resources
         cost = "unavailable" if usage.cost_usd is None else f"${usage.cost_usd:.4f}"
+        latency = "unavailable" if usage.p95_latency_ms is None else f"{usage.p95_latency_ms:.1f} ms"
         lines.append(f"| `{item.candidate_id}` | {usage.metric_calls} | {usage.judge_calls} | "
-                     f"{usage.total_tokens} | {usage.p95_latency_ms or 0.0:.1f} ms | "
+                     f"{usage.total_tokens} | {latency} | "
                      f"{usage.duration_seconds:.3f} s | {cost} | `{usage.cost_measurement}` |")
     lines.append("")
     return "\n".join(lines)

tests/evaluation/test_eval_optimize_loop_example.py

@@ -486,6 +486,41 @@ async def test_report_includes_paired_bootstrap_uncertainty_and_pareto_selection
     assert robust_checks["validation_gain_ci_lower_bound"].passed is True
 
 
+@pytest.mark.asyncio
+async def test_unknown_latency_is_not_treated_as_zero_for_pareto_or_markdown(tmp_path: Path, ) -> None:
+    EvalOptimizePipeline, PipelineSpec = _load_public_interface()
+    report = await EvalOptimizePipeline(
+        PipelineSpec.from_file(
+            _EXAMPLE_DIR / "pipeline.json",
+            output_dir=tmp_path / "unknown-latency",
+        )).run()
+
+    from eval_optimize_loop.loop.analysis import RegressionAnalyzer
+    from eval_optimize_loop.loop.reporting import render_markdown
+
+    robust = next(item for item in report.candidates if item.candidate_id == "robust")
+    unknown_latency = robust.model_copy(deep=True)
+    measured_latency = robust.model_copy(deep=True)
+    unknown_latency.candidate_id = "unknown-latency"
+    measured_latency.candidate_id = "measured-latency"
+    unknown_latency.audit.resources.p95_latency_ms = None
+    measured_latency.audit.resources.p95_latency_ms = 8.0
+
+    RegressionAnalyzer(seed=91, bootstrap_samples=100,
+                       confidence_level=0.95).mark_pareto([unknown_latency, measured_latency])
+    assert unknown_latency.pareto_optimal is False
+    assert measured_latency.pareto_optimal is True
+
+    report_for_rendering = report.model_copy(deep=True)
+    rendered_robust = next(item for item in report_for_rendering.candidates if item.candidate_id == "robust")
+    rendered_robust.audit.resources.p95_latency_ms = None
+    markdown = render_markdown(report_for_rendering)
+    resource_section = markdown.split("### Candidate evaluation resources", maxsplit=1)[1]
+    resource_row = next(line for line in resource_section.splitlines() if line.startswith("| `robust` |"))
+    assert "| unavailable |" in resource_row
+    assert "0.0 ms" not in resource_row
+
+
 @pytest.mark.asyncio
 async def test_data_quality_fails_closed_on_cross_split_near_duplicate(tmp_path: Path, ) -> None:
     EvalOptimizePipeline, PipelineSpec = _load_public_interface()