evals: judge function calls, and show the judge the calls the bot made

changelog/5713.added.md

@@ -0,0 +1,3 @@
+- Eval scenarios can put a function call to the judge: `eval:` is accepted on `function_call` and `function_call_stopped` expectations alongside `calls:` (or the `name:`/`args:` shorthand), and each call matched by name, and by any verbatim `args:`, is judged by name and arguments over the conversation so far. A scenario can therefore check what `args:` cannot match word for word, such as "the suggestion is about OpenTelemetry tracing, submitted for Jennifer Smith". A rejected call fails the expectation with kind `judge_no` and the judge's reason; a `continue` counts as a `no`, since a call is not a partial reply. The parser no longer warns about `eval:` on these two events, and a judged scenario that asserts on calls asks the bot for the `full` report level, so the judge sees the arguments.
+
+    The scripted judge now sees the bot's function calls too: every `function_call` the harness matches goes into the judge's conversation as an assistant message `[tool call] name(args)`, in arrival order with the reply's segments, and the judge is told such a line is a call the bot made and part of its reply. A `response` criterion can therefore check that a confirmation matches what was actually submitted. `EvalJudge` gains `add_tool_call()` and `evaluate_call()`, and `pipecat.evals.judge.format_tool_call()` formats a call as one line.

src/pipecat/evals/judge.py

@@ -48,6 +48,10 @@
     "arrived as several consecutive 'assistant' messages — against the given "
     "criterion, using the earlier turns only as context. The reply may still be "
     "streaming in. "
+    "An 'assistant' message of the form '[tool call] name(arguments)' is a function the "
+    "bot called at that point, with the arguments it passed; it is part of the bot's "
+    "reply, and a completed call is stronger evidence of an action (a booking, a "
+    "lookup) than the bot saying it did it. "
     "When the bot spoke its reply, the 'assistant' text is an automatic speech-to-text "
     "transcription, so it may contain homophones, misspellings, split or merged words, and "
     "missing punctuation. Always judge it by the intended spoken meaning, never by its exact "
@@ -78,6 +82,17 @@
     "Answer yes, no, or continue."
 )
 
+# The ask for an ``eval:`` on a function call. It names the call and gives its
+# arguments as JSON, so the verdict is about that call rather than about what
+# the bot said around it; the call is also in the conversation as a
+# '[tool call]' line (see :meth:`EvalJudge.add_tool_call`).
+JUDGE_CALL_ASK_TEMPLATE = (
+    "The bot called the function `{name}` with arguments `{args}`. "
+    "Does this call satisfy this criterion?\n\n"
+    "Criterion: {criterion}\n\n"
+    "Answer yes or no."
+)
+
 
 RUN_JUDGE_SYSTEM_INSTRUCTION = (
     "You are a strict but fair judge evaluating a complete conversation between a user "
@@ -225,6 +240,23 @@ def add_assistant_message(self, text: str | None) -> None:
         if text and text.strip():
             self._context.add_message({"role": "assistant", "content": text})
 
+    def add_tool_call(self, name: str, args: dict | None) -> None:
+        """Record a function call the bot made, as part of its current reply.
+
+        The call goes into the conversation as an assistant message
+        ``[tool call] name(args)``, the line the simulation judge's transcript
+        uses, so a later ``eval:`` can check the bot's words against what it
+        actually did.
+
+        Args:
+            name: The function's name.
+            args: The call's arguments, written as compact JSON; ``None`` or
+                empty gives ``name()``.
+        """
+        self._context.add_message(
+            {"role": "assistant", "content": f"[tool call] {format_tool_call(name, args)}"}
+        )
+
     async def evaluate(self, criterion: str) -> JudgeVerdict:
         """Judge whether the bot's latest reply satisfies ``criterion``, in the conversation so far.
 
@@ -239,6 +271,27 @@ async def evaluate(self, criterion: str) -> JudgeVerdict:
         ask = JUDGE_ASK_TEMPLATE.format(criterion=criterion)
         return await self._evaluate(criterion, JUDGE_SYSTEM_INSTRUCTION, ask)
 
+    async def evaluate_call(self, name: str, args: dict | None, criterion: str) -> JudgeVerdict:
+        """Judge whether a function call the bot made satisfies ``criterion``, in the conversation so far.
+
+        The ask names the call and its arguments, so the verdict is about that
+        call. A ``continue`` makes no sense for a call — it is not a partial
+        reply — and callers treat it as a ``no``.
+
+        Args:
+            name: The function's name.
+            args: The call's arguments, shown to the judge as compact JSON.
+            criterion: Natural-language description of what the call should be.
+
+        Returns:
+            A :class:`JudgeVerdict`, cached by ``(call, criterion, conversation)``
+            like :meth:`evaluate`.
+        """
+        ask = JUDGE_CALL_ASK_TEMPLATE.format(
+            name=name, args=_compact_json(args), criterion=criterion
+        )
+        return await self._evaluate(criterion, JUDGE_SYSTEM_INSTRUCTION, ask)
+
     async def evaluate_run(
         self, transcript: Sequence[dict], criteria: dict[str, str], success: str
     ) -> "RunVerdicts":
@@ -351,6 +404,24 @@ async def _call_judge_text(
         return response
 
 
+def format_tool_call(name: str, args: dict | None) -> str:
+    """A function call as one line: ``name({"k":"v"})``, or ``name()`` without arguments.
+
+    Args:
+        name: The function's name.
+        args: The call's arguments, or ``None``.
+
+    Returns:
+        The call with its arguments as compact JSON.
+    """
+    return f"{name}({_compact_json(args) if args else ''})"
+
+
+def _compact_json(args: dict | None) -> str:
+    """``args`` as JSON without the whitespace, non-ASCII text left readable."""
+    return json.dumps(args or {}, separators=(",", ":"), ensure_ascii=False)
+
+
 # The reason a verdict carries when the judge gave none.
 _NO_VERDICT = "(judge gave no verdict)"
 

src/pipecat/evals/matcher.py

@@ -34,7 +34,9 @@ class ExpectationMatcher:
     between. A reply with a content check aggregates its segments and
     re-checks on each, so an interim "Let me check" is rolled past rather
     than taken for the answer. A turn's function calls match by name in any
-    order.
+    order, and each one popped while matching is added to the judge's
+    conversation, so a reply's ``eval:`` can be checked against what the bot
+    actually did.
     """
 
     def __init__(self, *, stream: EvalEventStream, judge: EvalJudge | None, trace: EvalTrace):
@@ -258,7 +260,20 @@ async def _match_function_calls(
         turn_idx: int,
         exp_idx: int,
     ) -> EvalAssertionFailure | None:
-        """Match every call in the expectation, in any order, within the budget; else a failure naming the call that was missing or whose args did not match."""
+        """Match every call in the expectation, in any order, within the budget; else a failure naming the call that was missing or whose args did not match.
+
+        With ``eval:``, each matched call is also put to the judge, and the
+        first one it rejects fails the expectation.
+        """
+        if expectation.eval is not None and self._judge is None:
+            return self._failure(
+                expectation,
+                turn_idx,
+                exp_idx,
+                "scenario uses 'eval:' but no judge could be built",
+                "no_judge",
+            )
+
         matched: list[str] = []
         for spec in expectation.calls or []:
             want = spec.args or None
@@ -292,11 +307,47 @@ async def _match_function_calls(
                     f"function call {missing!r} not seen (matched: {seen})",
                     "missing_function_call",
                 )
+            judge_failure = await self._check_call_judge(event, expectation, turn_idx, exp_idx)
+            if judge_failure:
+                return judge_failure
             matched.append(str(event.get("name")))
 
         self.last_match_text = ", ".join(matched) or "function call"
         return None
 
+    async def _check_call_judge(
+        self,
+        event: dict,
+        expectation: EvalExpectation,
+        turn_idx: int,
+        exp_idx: int,
+    ) -> EvalAssertionFailure | None:
+        """Put a matched call to the judge if ``eval:`` was set on the expectation.
+
+        The judge is asked about the call by name and arguments, over the
+        conversation so far. A call is not a partial reply, so a ``continue``
+        fails it like a ``no``.
+        """
+        if expectation.eval is None:
+            return None
+        # _match_function_calls fails before matching when there is no judge.
+        assert self._judge is not None
+        name = str(event.get("name") or "?")
+        args = event.get("args") or {}
+        with logger.contextualize(eval_pipeline="judge"):
+            verdict = await self._judge.evaluate_call(name, args, expectation.eval)
+        self._trace.log(f"eval: {verdict.verdict} ({self._match_summary(event)}) {verdict.reason}")
+        if verdict.passed:
+            return None
+        return self._failure(
+            expectation,
+            turn_idx,
+            exp_idx,
+            f"eval {expectation.eval!r} on {self._match_summary(event)}: "
+            f"judge said {verdict.verdict} — {verdict.reason}",
+            "judge_no",
+        )
+
     async def _next_function_call(
         self,
         name: str | None,
@@ -308,7 +359,10 @@ async def _next_function_call(
 
         Calls seen but not yet claimed are buffered, so a turn's calls can arrive
         in any order and a call the LLM corrects and repeats still satisfies it.
-        Raises TimeoutError at ``deadline``.
+        Every ``function_call`` popped from the stream, claimed or buffered, goes
+        into the judge's conversation once, as it arrives, so a later ``eval:``
+        sees what the bot did before it spoke. Raises TimeoutError at
+        ``deadline``.
         """
 
         def matches(ev: dict) -> bool:
@@ -329,6 +383,8 @@ def matches(ev: dict) -> bool:
             event = await self._stream.next_any(deadline)
             if event.get("type") not in FUNCTION_CALL_EVENTS:
                 continue
+            if event.get("type") == "function_call" and self._judge is not None:
+                self._judge.add_tool_call(str(event.get("name") or "?"), event.get("args"))
             if matches(event):
                 return event
             self._pending_function_calls.append(event)

src/pipecat/evals/results.py

@@ -21,7 +21,7 @@
 # report "10x timeout on turn 3" without parsing free-text reasons.
 FAILURE_KINDS = (
     "timeout",  # no event of the expected type arrived within the budget
-    "judge_no",  # the judge rejected the reply
+    "judge_no",  # the judge rejected the reply, or a function call's `eval:`
     "judge_continue",  # the judge never accepted the reply before the budget ran out
     "no_judge",  # the scenario uses `eval:` but no judge could be built
     "no_content",  # the matched event carried no text to judge

src/pipecat/evals/script.py

@@ -79,6 +79,20 @@
     natural-language criterion the event's text content must satisfy, evaluated
     by a judge LLM (see :mod:`pipecat.evals.judge`).
 
+    On ``function_call`` and ``function_call_stopped`` the criterion is about
+    the call instead: each call ``calls:`` (or the ``name:``/``args:``
+    shorthand) matches is put to the judge by name and arguments, over the
+    conversation so far, which is how a scenario checks what ``args:`` cannot
+    match verbatim::
+
+        - event: function_call
+          calls: [{name: submit_session_suggestion}]
+          eval: "the suggestion is for a session about OpenTelemetry tracing, submitted for Jennifer Smith"
+
+    The judge also sees every call the harness matched as a ``[tool call]``
+    line in the bot's reply, so a later ``response`` criterion can check the
+    bot's words against what it actually submitted.
+
 ``absent: true``
     invert the expectation: assert that NO event of this type arrives before the
     ``within_ms`` budget expires (default 60s — set ``within_ms`` explicitly to
@@ -212,11 +226,11 @@
 from pipecat.utils.deprecation import deprecated
 
 # Events whose payloads carry bot-generated text the judge can sensibly
-# evaluate. Asserting ``eval:`` on anything else (user transcripts, tool
-# calls, interruption signals) produces a parser warning — the test controls
-# user input deterministically, so judging it adds cost without signal.
-# ``response`` is the modality-agnostic alias, resolved to one of the others
-# after parsing (see _resolve_response_events).
+# evaluate. Asserting ``eval:`` on anything else but a function call (user
+# transcripts, interruption signals) produces a parser warning — the test
+# controls user input deterministically, so judging it adds cost without
+# signal. ``response`` is the modality-agnostic alias, resolved to one of the
+# others after parsing (see _resolve_response_events).
 JUDGEABLE_EVENTS = frozenset({"response", "llm_response", "tts_response"})
 
 # Events carrying a function call, matched by name and arguments rather than by
@@ -268,9 +282,10 @@ class EvalExpectation:
             only when all of them are found. Built from ``calls:`` in the YAML, or
             from the single ``name:``/``args:`` shorthand.
         eval: Optional natural-language criterion the event's text content
-            must satisfy. Evaluated by a judge LLM. Only meaningful on the
-            bot-generated text events: ``response``, ``llm_response``, and
-            ``tts_response``.
+            must satisfy. Evaluated by a judge LLM. Meaningful on the
+            bot-generated text events (``response``, ``llm_response``, and
+            ``tts_response``) and on the function-call events, where it is
+            about each matched call's name and arguments rather than text.
         absent: When True, the expectation is inverted: it passes only when NO
             event of this type arrives before the ``within_ms`` budget expires,
             and fails as soon as one does. Matches on event type only;
@@ -531,12 +546,20 @@ def wants_response(self) -> bool:
         return any(exp.event == "response" for turn in self.turns for exp in turn.expect)
 
     def required_report_level(self) -> str | None:
-        """The function-call report level the scenario's assertions need: ``full`` for args, ``name`` for names, else ``None``."""
+        """The function-call report level the scenario's assertions need: ``full`` for args, ``name`` for names, else ``None``.
+
+        A judged scenario that asserts on calls needs ``full`` too: the judge
+        reads the calls the harness matches, with their arguments, whether the
+        ``eval:`` is on the call itself or on the reply after it.
+        """
         needs_name = False
+        judged = any(exp.eval is not None for turn in self.turns for exp in turn.expect)
         for turn in self.turns:
             for exp in turn.expect:
                 if exp.event not in FUNCTION_CALL_EVENTS:
                     continue
+                if judged:
+                    return "full"
                 # name/args live in exp.calls (the parser normalizes the single
                 # name:/args: shorthand into it too).
                 for call in exp.calls or []:
@@ -729,12 +752,17 @@ def _parse_expectation(e: Any, path: Path, turn_idx: int, exp_idx: int) -> EvalE
         )
 
     criterion = e.get("eval")
-    if criterion is not None and event not in JUDGEABLE_EVENTS:
+    if (
+        criterion is not None
+        and event not in JUDGEABLE_EVENTS
+        and event not in FUNCTION_CALL_EVENTS
+    ):
         logger.warning(
             f"{path}: turn #{turn_idx} expectation #{exp_idx}: 'eval:' on "
             f"event {event!r} — judge only makes sense on bot-generated text "
-            f"events ({', '.join(sorted(JUDGEABLE_EVENTS))}). Will run but is "
-            "unlikely to be meaningful."
+            f"events ({', '.join(sorted(JUDGEABLE_EVENTS))}) and function calls "
+            f"({', '.join(FUNCTION_CALL_EVENTS)}). Will run but is unlikely to be "
+            "meaningful."
         )
 
     absent = e.get("absent", False)

tests/test_evals_judge.py

@@ -6,7 +6,13 @@
 
 import unittest
 
-from pipecat.evals.judge import EvalJudge, JudgeVerdict, _parse_run_verdicts, _parse_verdict
+from pipecat.evals.judge import (
+    EvalJudge,
+    JudgeVerdict,
+    _parse_run_verdicts,
+    _parse_verdict,
+    format_tool_call,
+)
 
 
 class TestParseRunVerdicts(unittest.TestCase):
@@ -222,6 +228,61 @@ async def test_empty_response_fails(self):
         self.assertFalse(v.passed)
 
 
+class TestJudgeToolCalls(unittest.IsolatedAsyncioTestCase):
+    """Function calls in the judge's conversation, and a verdict on one call."""
+
+    def test_format_tool_call_is_compact_json(self):
+        self.assertEqual(
+            format_tool_call("book", {"city": "São Paulo", "guests": 2}),
+            'book({"city":"São Paulo","guests":2})',
+        )
+        self.assertEqual(format_tool_call("refresh", None), "refresh()")
+        self.assertEqual(format_tool_call("refresh", {}), "refresh()")
+
+    async def test_add_tool_call_is_an_assistant_line(self):
+        svc = _FakeLLMService(['{"verdict": "yes", "reason": "ok"}'])
+        judge = EvalJudge(svc)
+        judge.add_user_message("Book a table for two.")
+        judge.add_tool_call("book_table", {"guests": 2})
+        judge.add_assistant_message("Done, a table for two.")
+        await judge.evaluate("confirms the booking it made")
+        messages = svc.calls[0]["messages"][:-1]  # the verdict ask is last
+        self.assertEqual(
+            [(m["role"], m["content"]) for m in messages],
+            [
+                ("user", "Book a table for two."),
+                ("assistant", '[tool call] book_table({"guests":2})'),
+                ("assistant", "Done, a table for two."),
+            ],
+        )
+        self.assertIn("[tool call] name(arguments)", svc.calls[0]["system_instruction"])
+
+    async def test_evaluate_call_asks_about_the_named_call(self):
+        svc = _FakeLLMService(['{"verdict": "no", "reason": "wrong speaker"}'])
+        judge = EvalJudge(svc)
+        judge.add_tool_call("submit", {"speaker": "Ann"})
+        v = await judge.evaluate_call("submit", {"speaker": "Ann"}, "submitted for Bob")
+        self.assertFalse(v.passed)
+        self.assertEqual(v.reason, "wrong speaker")
+        ask = svc.calls[0]["messages"][-1]["content"]
+        self.assertIn('called the function `submit` with arguments `{"speaker":"Ann"}`', ask)
+        self.assertIn("Criterion: submitted for Bob", ask)
+
+    async def test_evaluate_call_caches_per_call(self):
+        """The same criterion on two different calls is two questions."""
+        svc = _FakeLLMService(
+            ['{"verdict": "yes", "reason": "a"}', '{"verdict": "no", "reason": "b"}']
+        )
+        judge = EvalJudge(svc)
+        first = await judge.evaluate_call("submit", {"n": 1}, "n is one")
+        again = await judge.evaluate_call("submit", {"n": 1}, "n is one")
+        second = await judge.evaluate_call("submit", {"n": 2}, "n is one")
+        self.assertTrue(first.passed)
+        self.assertTrue(again.passed)
+        self.assertFalse(second.passed)
+        self.assertEqual(len(svc.calls), 2)
+
+
 class TestJudgeVerdictDataclass(unittest.TestCase):
     def test_construction(self):
         v = JudgeVerdict(verdict="yes", reason="ok", raw_response="raw")