Add LLM routing metadata to generation observability 🤖🤖🤖

Signed-off-by: Clement Pakkam Isaac <232634418+cpakkamisaac-sae@users.noreply.github.com>

Author: cpakkamisaac-sae

examples/README.md

@@ -547,7 +547,7 @@ class DataAgent(Agent, llm=llm):
 
 ### LLM cascading resolution
 
-Configure LLMs at any granularity — class default, method override, instance override — for cost/latency optimization, A/B testing, or gradual rollouts.
+Configure LLMs at any granularity — class default, method override, instance override, or per-call override — for cost/latency optimization, A/B testing, or gradual rollouts.
 
 ```python
 class MyAgent(Agent, llm=default_llm):              # 1. class default
@@ -557,6 +557,7 @@ class MyAgent(Agent, llm=default_llm):              # 1. class default
         ...
 
 agent = MyAgent(llm=different_llm)                  # 4. instance override
+result = await agent.complex_task(llm=temporary_llm)  # 5. per-call override
 ```
 
 ### Event-driven history

src/nooa/runtime/actor.py

@@ -62,6 +62,8 @@
 
 logger = logging.getLogger(__name__)
 
+_MISSING = object()
+
 
 @contextmanager
 def _harness_metrics_lifecycle(should_trace: bool):
@@ -2530,13 +2532,18 @@ async def _execute_with_generation(
         method_name: str,
     ) -> Any:
         """Execute a method that needs LLM generation."""
+        base_method = getattr(method, "__func__", method)
+        try:
+            has_user_llm_param = "llm" in inspect.signature(method).parameters
+        except (TypeError, ValueError):
+            has_user_llm_param = False
+
         # Extract framework parameters (don't pass to generated method)
         call_strategy = kwargs.pop("_strategy", None)
-        call_llm = kwargs.pop("llm", None)
+        call_llm = kwargs.pop("llm", _MISSING) if not has_user_llm_param else _MISSING
         call_session_locals = kwargs.pop("_session_locals", None)
 
         # Get strategy with priority: call-level > decorator > default
-        base_method = getattr(method, "__func__", method)
         decorator_strategy = getattr(base_method, "_plan_strategy", None)
 
         # DEBUG: Log strategy retrieval for nested method debugging
@@ -2559,9 +2566,18 @@ async def _execute_with_generation(
 
         # Resolve LLM client with priority: call-level > @strategy decorator > agent's default
         plan_llm = getattr(base_method, "_plan_llm", None)
-        llm_client = call_llm or plan_llm or getattr(self.agent, "_llm", None)
+        if call_llm is not _MISSING and call_llm is not None:
+            llm_client = call_llm
+            llm_selection_source = "call_site"
+        elif plan_llm is not None:
+            llm_client = plan_llm
+            llm_selection_source = "decorator"
+        else:
+            llm_client = getattr(self.agent, "_llm", None)
+            llm_selection_source = "agent_default"
         if llm_client is None:
             raise RuntimeError(f"No LLM client available for {method_name}")
+        llm_model_name = getattr(llm_client, "model", "") or ""
 
         # Resolve truncation config: method-level @strategy(truncation=...) > agent-level
         method_truncation = getattr(base_method, "_strategy_truncation", None)
@@ -2605,6 +2621,10 @@ async def _execute_with_generation(
                 generation_id=generation_id,
                 parent_generation_id=parent_generation_id,
                 agent_call_id=self._agent_call_id,
+                **{
+                    "llm.model_name": llm_model_name,
+                    "llm.selection_source": llm_selection_source,
+                },
                 **strategy_kwargs,  # Add strategy config parameters
             )
 

src/nooa/runtime/method_wrapper.py

@@ -11,6 +11,7 @@
 """
 
 import asyncio
+import inspect
 import logging
 from collections.abc import Callable
 from functools import wraps
@@ -34,6 +35,8 @@
 
 logger = logging.getLogger(__name__)
 
+_MISSING = object()
+
 
 async def _flush_litellm_journal() -> None:
     # Three yields drain litellm's GLOBAL_LOGGING_WORKER chain before joining the
@@ -117,12 +120,21 @@ async def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
             # Strip framework kwargs before validation — they're consumed by
             # _execute_with_generation, not the user's method signature.
             _fw_session_locals = kwargs.pop("_session_locals", None)
+            _fw_llm = _MISSING
+            try:
+                _has_user_llm_param = "llm" in inspect.signature(original_func).parameters
+            except (TypeError, ValueError):
+                _has_user_llm_param = False
+            if not _has_user_llm_param:
+                _fw_llm = kwargs.pop("llm", _MISSING)
             try:
                 ArgumentValidator().validate(original_func, args, kwargs, _tc)
             finally:
                 # Restore so _execute_with_generation can pop them
                 if _fw_session_locals is not None:
                     kwargs["_session_locals"] = _fw_session_locals
+                if _fw_llm is not _MISSING:
+                    kwargs["llm"] = _fw_llm
 
             # Strategy resolution if not provided
             if resolved_strategy is None:

tests/runtime/test_llm_routing_observability.py

@@ -0,0 +1,264 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""LLM routing controls and generation observability."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any, cast
+
+import pytest
+
+from nooa import Agent, strategy
+from nooa.runtime.hooks import set_hooks
+from nooa.strategies import PredictStrategy
+from nooa.unifiedllm import FakeLLMClient, LLMResponse
+
+
+def _resp(value: str) -> LLMResponse:
+    return LLMResponse(
+        raw_response=None,
+        content=f'{{"value": "{value}"}}',
+        tool_calls=[],
+        finish_reason="stop",
+        assistant_message={"role": "assistant", "content": f'{{"value": "{value}"}}'},
+    )
+
+
+def _llm(model: str, value: str) -> FakeLLMClient:
+    client = FakeLLMClient(scripted_responses=[_resp(value)])
+    client.model = model
+    return client
+
+
+class RoutingHooks:
+    """Capture generation hook metadata for routing assertions."""
+
+    def __init__(self) -> None:
+        self.generations: list[dict[str, Any]] = []
+
+    def before_generation(
+        self,
+        agent: Any,
+        method_name: str,
+        strategy: str,
+        generation_id: str,
+        parent_generation_id: str | None,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        self.generations.append(
+            {
+                "method_name": method_name,
+                "strategy": strategy,
+                "generation_id": generation_id,
+                **kwargs,
+            }
+        )
+        return {"generation_id": generation_id}
+
+    def after_generation(
+        self,
+        agent: Any,
+        method_name: str,
+        result: Any,
+        exception: Exception | None,
+        context: Any,
+        generation_id: str,
+    ) -> None:
+        pass
+
+    def on_messages_built(
+        self,
+        agent: Any,
+        method_name: str,
+        messages: list[dict[str, Any]],
+        generation_id: str,
+        **kwargs: Any,
+    ) -> None:
+        pass
+
+    def before_agent_call(
+        self,
+        agent: Any,
+        method_name: str,
+        args: tuple[Any, ...],
+        kwargs: dict[str, Any],
+        call_id: str,
+        parent_call_id: str | None,
+        **extra: Any,
+    ) -> dict[str, Any]:
+        return {"call_id": call_id}
+
+    def after_agent_call(
+        self,
+        agent: Any,
+        method_name: str,
+        result: Any,
+        exception: Exception | None,
+        context: Any,
+        **kwargs: Any,
+    ) -> None:
+        pass
+
+    def before_code_execution(
+        self,
+        agent: Any,
+        code: str,
+        execution_id: str,
+        generation_id: str | None = None,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        return {"execution_id": execution_id}
+
+    def after_code_execution(
+        self,
+        agent: Any,
+        code: str,
+        result: Any,
+        exception: Exception | None,
+        context: Any,
+        execution_id: str,
+        **kwargs: Any,
+    ) -> None:
+        pass
+
+    def before_method_invocation(
+        self,
+        agent: Any,
+        method_name: str,
+        args: tuple[Any, ...],
+        kwargs: dict[str, Any],
+        invocation_id: str,
+        **extra: Any,
+    ) -> dict[str, Any]:
+        return {"invocation_id": invocation_id}
+
+    def after_method_invocation(
+        self,
+        agent: Any,
+        method_name: str,
+        result: Any,
+        exception: Exception | None,
+        context: Any,
+        invocation_id: str,
+        **kwargs: Any,
+    ) -> None:
+        pass
+
+    def before_tool_execution(
+        self,
+        agent: Any,
+        tool_name: str,
+        arguments: dict[str, Any],
+        execution_id: str,
+        generation_id: str | None = None,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        return {"execution_id": execution_id}
+
+    def after_tool_execution(
+        self,
+        agent: Any,
+        tool_name: str,
+        arguments: dict[str, Any],
+        result: Any,
+        exception: Exception | None,
+        context: Any,
+        execution_id: str,
+        **kwargs: Any,
+    ) -> None:
+        pass
+
+
+@pytest.fixture
+def routing_hooks() -> Iterator[RoutingHooks]:
+    hooks = RoutingHooks()
+    set_hooks(cast(Any, hooks))
+    try:
+        yield hooks
+    finally:
+        set_hooks(None)
+
+
+@pytest.mark.asyncio
+async def test_call_site_llm_override_wins_and_is_observable(routing_hooks: RoutingHooks) -> None:
+    default_llm = _llm("default-model", "default")
+    decorator_llm = _llm("decorator-model", "decorator")
+    override_llm = _llm("override-model", "override")
+
+    class RoutingAgent(Agent, llm=default_llm):
+        @strategy(PredictStrategy(), llm=decorator_llm)
+        async def summarize(self, text: str) -> str:
+            """Summarize text."""
+            ...
+
+    agent = RoutingAgent()
+
+    assert await agent.summarize("hello", llm=override_llm) == "override"  # pyright: ignore[reportCallIssue]
+
+    assert default_llm.call_count == 0
+    assert decorator_llm.call_count == 0
+    assert override_llm.call_count == 1
+    assert routing_hooks.generations[-1]["llm.model_name"] == "override-model"
+    assert routing_hooks.generations[-1]["llm.selection_source"] == "call_site"
+
+
+@pytest.mark.asyncio
+async def test_decorator_llm_selection_is_observable(routing_hooks: RoutingHooks) -> None:
+    default_llm = _llm("default-model", "default")
+    decorator_llm = _llm("decorator-model", "decorator")
+
+    class RoutingAgent(Agent, llm=default_llm):
+        @strategy(PredictStrategy(), llm=decorator_llm)
+        async def summarize(self, text: str) -> str:
+            """Summarize text."""
+            ...
+
+    agent = RoutingAgent()
+
+    assert await agent.summarize("hello") == "decorator"
+
+    assert default_llm.call_count == 0
+    assert decorator_llm.call_count == 1
+    assert routing_hooks.generations[-1]["llm.model_name"] == "decorator-model"
+    assert routing_hooks.generations[-1]["llm.selection_source"] == "decorator"
+
+
+@pytest.mark.asyncio
+async def test_agent_default_llm_selection_is_observable(routing_hooks: RoutingHooks) -> None:
+    default_llm = _llm("default-model", "default")
+
+    class RoutingAgent(Agent, llm=default_llm):
+        @strategy(PredictStrategy())
+        async def summarize(self, text: str) -> str:
+            """Summarize text."""
+            ...
+
+    agent = RoutingAgent()
+
+    assert await agent.summarize("hello") == "default"
+
+    assert default_llm.call_count == 1
+    assert routing_hooks.generations[-1]["llm.model_name"] == "default-model"
+    assert routing_hooks.generations[-1]["llm.selection_source"] == "agent_default"
+
+
+@pytest.mark.asyncio
+async def test_user_parameter_named_llm_is_not_consumed_as_framework_override(
+    routing_hooks: RoutingHooks,
+) -> None:
+    default_llm = _llm("default-model", "ok")
+
+    class RoutingAgent(Agent, llm=default_llm):
+        @strategy(PredictStrategy())
+        async def summarize(self, llm: str) -> str:
+            """Summarize using the user-supplied label."""
+            ...
+
+    agent = RoutingAgent()
+
+    assert await agent.summarize(llm="customer-visible-label") == "ok"
+
+    assert default_llm.call_count == 1
+    assert routing_hooks.generations[-1]["llm.model_name"] == "default-model"
+    assert routing_hooks.generations[-1]["llm.selection_source"] == "agent_default"