|
| 1 | +"""Built-in evaluators: accuracy (LLM judge), reliability (tools), performance (SLOs).""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +import re |
| 6 | +from typing import TYPE_CHECKING |
| 7 | + |
| 8 | +from hive.evals.types import CaseResult, EvalCase, EvalRun |
| 9 | +from hive.runtime.types import Message |
| 10 | + |
| 11 | +if TYPE_CHECKING: |
| 12 | + from hive.models.base import BaseProvider |
| 13 | + |
| 14 | +_JUDGE_RUBRIC = ( |
| 15 | + "Rate how well the actual answer satisfies the expected answer on a scale of 0 to " |
| 16 | + "10 (10 = fully correct and complete, 0 = wrong or missing). Reply with ONLY the " |
| 17 | + "number on the first line, then a one-sentence justification." |
| 18 | +) |
| 19 | + |
| 20 | + |
| 21 | +def _build_judge_prompt(instruction: str, expected: str, actual: str) -> str: |
| 22 | + """Build the judge prompt by concatenation. |
| 23 | +
|
| 24 | + Avoids ``str.format``/f-string substitution so brace characters in the instruction, |
| 25 | + expected answer, or actual output (e.g. JSON like ``{"k": 1}``) can't raise. |
| 26 | + """ |
| 27 | + return ( |
| 28 | + "You are grading an AI assistant's answer.\n\n" |
| 29 | + f"Task given to the assistant:\n{instruction}\n\n" |
| 30 | + f"Expected answer (reference):\n{expected}\n\n" |
| 31 | + f"Assistant's actual answer:\n{actual}\n\n" |
| 32 | + f"{_JUDGE_RUBRIC}" |
| 33 | + ) |
| 34 | + |
| 35 | + |
| 36 | +class AccuracyEval: |
| 37 | + """LLM-as-judge: score the agent's output against an expected answer (0..1).""" |
| 38 | + |
| 39 | + name = "accuracy" |
| 40 | + |
| 41 | + def __init__(self, judge: BaseProvider, threshold: float = 0.6): |
| 42 | + self._judge = judge |
| 43 | + self._threshold = threshold |
| 44 | + |
| 45 | + async def evaluate(self, run: EvalRun, case: EvalCase) -> CaseResult: |
| 46 | + if case.expected_answer is None: |
| 47 | + return CaseResult( |
| 48 | + case, self.name, True, 1.0, "no expected_answer (skipped)", run, skipped=True |
| 49 | + ) |
| 50 | + prompt = _build_judge_prompt(case.instruction, case.expected_answer, run.output) |
| 51 | + result = await self._judge.generate_with_metadata(messages=[Message.user(prompt)]) |
| 52 | + score, detail = self._parse(result.message.content) |
| 53 | + return CaseResult(case, self.name, score >= self._threshold, score, detail, run) |
| 54 | + |
| 55 | + @staticmethod |
| 56 | + def _parse(text: str) -> tuple[float, str]: |
| 57 | + match = re.search(r"\b(10|\d)(?:\.\d+)?\b", text) |
| 58 | + if not match: |
| 59 | + return 0.0, f"unparseable judge reply: {text[:80]}" |
| 60 | + score = min(1.0, float(match.group(0)) / 10.0) |
| 61 | + return score, text.strip()[:200] |
| 62 | + |
| 63 | + |
| 64 | +class ReliabilityEval: |
| 65 | + """Assert the agent actually called the expected tools. |
| 66 | +
|
| 67 | + ``mode="all"`` (default): every expected tool must appear. ``mode="exact"``: the |
| 68 | + set of called tools must equal the expected set. |
| 69 | + """ |
| 70 | + |
| 71 | + name = "reliability" |
| 72 | + |
| 73 | + def __init__(self, mode: str = "all"): |
| 74 | + if mode not in ("all", "exact"): |
| 75 | + raise ValueError("mode must be 'all' or 'exact'") |
| 76 | + self._mode = mode |
| 77 | + |
| 78 | + async def evaluate(self, run: EvalRun, case: EvalCase) -> CaseResult: |
| 79 | + expected = set(case.expected_tools or []) |
| 80 | + actual = set(run.tool_calls) |
| 81 | + if not expected: |
| 82 | + return CaseResult( |
| 83 | + case, self.name, True, 1.0, "no expected_tools (skipped)", run, skipped=True |
| 84 | + ) |
| 85 | + hit = expected & actual |
| 86 | + score = len(hit) / len(expected) |
| 87 | + if self._mode == "exact": |
| 88 | + passed = actual == expected |
| 89 | + else: |
| 90 | + passed = expected <= actual |
| 91 | + missing = sorted(expected - actual) |
| 92 | + extra = sorted(actual - expected) |
| 93 | + detail = f"called={sorted(actual)} missing={missing} extra={extra}" |
| 94 | + return CaseResult(case, self.name, passed, score, detail, run) |
| 95 | + |
| 96 | + |
| 97 | +class PerformanceEval: |
| 98 | + """Check a run against latency and/or cost SLOs.""" |
| 99 | + |
| 100 | + name = "performance" |
| 101 | + |
| 102 | + def __init__(self, max_seconds: float | None = None, max_cost_usd: float | None = None): |
| 103 | + self._max_seconds = max_seconds |
| 104 | + self._max_cost_usd = max_cost_usd |
| 105 | + |
| 106 | + async def evaluate(self, run: EvalRun, case: EvalCase) -> CaseResult: |
| 107 | + ok_time = self._max_seconds is None or run.duration_seconds <= self._max_seconds |
| 108 | + ok_cost = self._max_cost_usd is None or run.cost_usd <= self._max_cost_usd |
| 109 | + passed = ok_time and ok_cost |
| 110 | + # Score is the worse of the two budget ratios (1.0 = well within budget). |
| 111 | + # Guard with `is not None` (not truthiness) so a 0.0 budget is still scored -- |
| 112 | + # otherwise passed=False could pair with an empty-ratios score of 1.0. |
| 113 | + ratios = [] |
| 114 | + if self._max_seconds is not None: |
| 115 | + ratios.append( |
| 116 | + max(0.0, 1.0 - run.duration_seconds / self._max_seconds) |
| 117 | + if self._max_seconds > 0 |
| 118 | + else (0.0 if run.duration_seconds > 0 else 1.0) |
| 119 | + ) |
| 120 | + if self._max_cost_usd is not None: |
| 121 | + ratios.append( |
| 122 | + max(0.0, 1.0 - run.cost_usd / self._max_cost_usd) |
| 123 | + if self._max_cost_usd > 0 |
| 124 | + else (0.0 if run.cost_usd > 0 else 1.0) |
| 125 | + ) |
| 126 | + score = min(ratios) if ratios else 1.0 |
| 127 | + detail = f"{run.duration_seconds:.2f}s, ${run.cost_usd:.4f}" |
| 128 | + return CaseResult(case, self.name, passed, score, detail, run) |
0 commit comments