Merge branch 'feat/agentos-control-plane' into feat/agentos-deploy

Author: chiruu12

src/hive/evals/evaluators.py

@@ -11,20 +11,26 @@
 if TYPE_CHECKING:
     from hive.models.base import BaseProvider
 
-_JUDGE_PROMPT = """You are grading an AI assistant's answer.
+_JUDGE_RUBRIC = (
+    "Rate how well the actual answer satisfies the expected answer on a scale of 0 to "
+    "10 (10 = fully correct and complete, 0 = wrong or missing). Reply with ONLY the "
+    "number on the first line, then a one-sentence justification."
+)
 
-Task given to the assistant:
-{instruction}
 
-Expected answer (reference):
-{expected}
+def _build_judge_prompt(instruction: str, expected: str, actual: str) -> str:
+    """Build the judge prompt by concatenation.
 
-Assistant's actual answer:
-{actual}
-
-Rate how well the actual answer satisfies the expected answer on a scale of 0 to 10
-(10 = fully correct and complete, 0 = wrong or missing). Reply with ONLY the number
-on the first line, then a one-sentence justification."""
+    Avoids ``str.format``/f-string substitution so brace characters in the instruction,
+    expected answer, or actual output (e.g. JSON like ``{"k": 1}``) can't raise.
+    """
+    return (
+        "You are grading an AI assistant's answer.\n\n"
+        f"Task given to the assistant:\n{instruction}\n\n"
+        f"Expected answer (reference):\n{expected}\n\n"
+        f"Assistant's actual answer:\n{actual}\n\n"
+        f"{_JUDGE_RUBRIC}"
+    )
 
 
 class AccuracyEval:
@@ -41,9 +47,7 @@ async def evaluate(self, run: EvalRun, case: EvalCase) -> CaseResult:
             return CaseResult(
                 case, self.name, True, 1.0, "no expected_answer (skipped)", run, skipped=True
             )
-        prompt = _JUDGE_PROMPT.format(
-            instruction=case.instruction, expected=case.expected_answer, actual=run.output
-        )
+        prompt = _build_judge_prompt(case.instruction, case.expected_answer, run.output)
         result = await self._judge.generate_with_metadata(messages=[Message.user(prompt)])
         score, detail = self._parse(result.message.content)
         return CaseResult(case, self.name, score >= self._threshold, score, detail, run)

src/hive/evals/runner.py

@@ -29,13 +29,16 @@ def __init__(self, agent: Agent):
     async def run(self, case: EvalCase) -> EvalRun:
         async with self._lock:
             calls: list[str] = []
+            # Snapshot and restore any pre-existing observer instead of clearing it,
+            # so an agent built with Agent(on_tool=...) keeps its callback afterward.
+            previous_on_tool = self._agent._on_tool
             self._agent.observe_tools(lambda name, args, ok: calls.append(name))
             try:
                 result = await self._agent.run(
                     Task(instruction=case.instruction, context=case.context)
                 )
             finally:
-                self._agent.observe_tools(None)
+                self._agent.observe_tools(previous_on_tool)
         return EvalRun(
             instruction=case.instruction,
             output=result.output,

src/hive/runtime/agent.py

@@ -598,7 +598,13 @@ async def _run_loop(self, task: Task) -> TaskResult:
                             output = "[output withheld by guardrail]"
                         elif finding.action is GuardrailAction.REDACT:
                             output = finding.text
-                self._write_conversation_log(task.id, conversation.get_messages(), "completed")
+                # The raw assistant message is already in the conversation; replace it
+                # with the sanitized output for the on-disk log too, so a redacting
+                # guardrail doesn't leak the unredacted content into the JSON log file.
+                log_messages = conversation.get_messages()
+                if output != response.content:
+                    log_messages = [*log_messages[:-1], Message.assistant(output)]
+                self._write_conversation_log(task.id, log_messages, "completed")
                 return TaskResult(
                     task_id=task.id,
                     status=TaskStatus.COMPLETED,

tests/evals/test_evals.py

@@ -128,6 +128,14 @@ async def test_no_expected_answer_skips(self) -> None:
         res = await AccuracyEval(judge=judge).evaluate(_run("x"), EvalCase("q"))
         assert res.passed and res.skipped and "skipped" in res.detail
 
+    @pytest.mark.asyncio
+    async def test_braces_in_content_do_not_crash_judge(self) -> None:
+        judge = ScriptedProvider([Message.assistant("8\nok")])
+        run = _run('{"key": "value", "n": {}}')  # braces in the actual output
+        case = EvalCase("Parse JSON: {}", expected_answer="{'k': 1}")
+        res = await AccuracyEval(judge=judge).evaluate(run, case)
+        assert res.score == pytest.approx(0.8)
+
 
 class TestReportMetrics:
     def test_skipped_cases_excluded_from_rates(self) -> None:
@@ -153,6 +161,19 @@ def test_duplicate_evaluator_names_rejected(self) -> None:
         with pytest.raises(ValueError, match="duplicate evaluator"):
             EvalSuite(AgentEvalRunner(agent), [ReliabilityEval(), ReliabilityEval()])
 
+    @pytest.mark.asyncio
+    async def test_runner_restores_pre_existing_observer(self) -> None:
+        seen: list[str] = []
+        agent = Agent(
+            name="solver",
+            model=ScriptedProvider([Message.assistant("done")]),  # type: ignore[arg-type]
+            on_tool=lambda n, a, ok: seen.append(n),
+        )
+        await AgentEvalRunner(agent).run(EvalCase("hi"))
+        # The caller's original observer is restored, not dropped.
+        assert agent._on_tool is not None
+        assert agent._on_tool(("x"), {}, True) is None and seen == ["x"]
+
 
 class TestSuiteIntegration:
     @pytest.mark.asyncio

tests/runtime/test_guardrails.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from pathlib import Path
 from typing import Any
 
 import pytest
@@ -126,6 +127,23 @@ async def test_output_pii_is_redacted(self) -> None:
         assert "leak@corp.com" not in result.output
         assert "REDACTED" in result.output
 
+    @pytest.mark.asyncio
+    async def test_redacted_output_not_leaked_to_conversation_log(self, tmp_path: Path) -> None:
+        from hive.runtime.types import Task
+
+        agent = Agent(
+            name="a",
+            model=MockProvider("the email is leak@corp.com"),  # type: ignore[arg-type]
+            guardrails=build_guardrail_pipeline(GuardrailConfig(enabled=True)),
+            conversation_log_dir=tmp_path,
+        )
+        await agent.run(Task(instruction="contact?"))
+        logged = "\n".join(p.read_text() for p in tmp_path.rglob("*.json"))
+        # The on-disk conversation log must not contain the unredacted PII.
+        assert logged  # a log was written
+        assert "leak@corp.com" not in logged
+        assert "REDACTED" in logged
+
     @pytest.mark.asyncio
     async def test_disabled_guardrails_passthrough(self) -> None:
         from hive.runtime.types import Task