feat(observability): log classifier reasoning

Signed-off-by: Todd Fisher <todd.fisher@gmail.com>

Author: taf2

crates/libsy-llm-client/src/lib.rs

@@ -30,7 +30,10 @@ pub mod run;
 pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig};
 pub use client::{ModelConfig, TranslatingLlmClient};
 pub use error::{LlmClientError, Result};
-pub use observation::{LlmCallObservation, LlmCallStartObservation, RunObservation, RunObserver};
+pub use observation::{
+    ClassifierContentObservation, LlmCallObservation, LlmCallStartObservation, RunObservation,
+    RunObserver,
+};
 pub use raw::RawResponse;
-pub use run::{ClientRouter, run};
+pub use run::{ClientRouter, ObservationConfig, run, run_with_observation_config};
 pub use switchyard_translation::RawEventStream;

crates/libsy-llm-client/src/observation.rs

@@ -6,7 +6,24 @@
 use std::sync::Arc;
 use std::time::Duration;
 
-use switchyard_protocol::{Decision, ModelId, Usage};
+use switchyard_protocol::{Decision, LlmRequest, ModelId, Usage};
+
+/// Prompt and model-produced content from one non-answer classifier or judge call.
+#[derive(Clone, Debug)]
+pub struct ClassifierContentObservation {
+    /// Model that produced the routing verdict.
+    pub selected_model: ModelId,
+    /// Exact normalized request sent to the classifier target, excluding transport headers.
+    pub request: LlmRequest,
+    /// Model-produced reasoning content, when the provider returned it separately.
+    pub reasoning: Option<String>,
+    /// Text verdict consumed by the routing policy, including invalid replies.
+    pub verdict: Option<String>,
+    /// Whether the provider call itself completed successfully.
+    pub is_success: bool,
+    /// Time spent waiting for the classifier call to resolve.
+    pub duration: Duration,
+}
 
 /// One model call observed immediately before it is sent to its routed client.
 #[derive(Clone, Debug)]
@@ -39,6 +56,8 @@ pub enum RunObservation {
     RoutingDecision(Decision),
     /// A model call about to start.
     LlmCallStarted(LlmCallStartObservation),
+    /// Prompt, reasoning, and verdict from a classifier or judge call.
+    ClassifierContent(ClassifierContentObservation),
     /// A completed model call.
     LlmCall(LlmCallObservation),
     /// Routing time recorded by the `switchyard.routing_overhead_ms` metric.

crates/libsy-llm-client/src/run.rs

@@ -20,11 +20,13 @@ use std::time::{Duration, Instant};
 use parking_lot::Mutex;
 use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, drive_with_decision_observer};
 use switchyard_protocol::{
-    Decision, LlmClientError, ModelId, Request, Response, RoutedLlmClient, RoutingFallbackReason,
+    ContentBlock, Decision, LlmClientError, ModelId, Request, Response, RoutedLlmClient,
+    RoutingFallbackReason, completion_text,
 };
 
 use crate::observation::{
-    LlmCallObservation, LlmCallStartObservation, RunObservation, RunObserver,
+    ClassifierContentObservation, LlmCallObservation, LlmCallStartObservation, RunObservation,
+    RunObserver,
 };
 use crate::{metrics, observability};
 
@@ -46,6 +48,31 @@ pub async fn run(
     clients: ClientRouter,
     request: Request,
     observer: Option<RunObserver>,
+) -> Result<(Vec<Decision>, Response)> {
+    run_with_observation_config(
+        algorithm,
+        clients,
+        request,
+        observer,
+        ObservationConfig::default(),
+    )
+    .await
+}
+
+/// Controls optional content captured for a [`RunObserver`].
+#[derive(Clone, Copy, Debug, Default)]
+pub struct ObservationConfig {
+    /// Capture normalized prompts, reasoning, and verdicts for non-answer calls.
+    pub classifier_content: bool,
+}
+
+/// Runs one request like [`run`] with explicit observation-content controls.
+pub async fn run_with_observation_config(
+    algorithm: Arc<dyn Algorithm>,
+    clients: ClientRouter,
+    request: Request,
+    observer: Option<RunObserver>,
+    observation_config: ObservationConfig,
 ) -> Result<(Vec<Decision>, Response)> {
     let algorithm_name = algorithm.name().to_string();
     // The output from `serve` goes in here: when each successful routed call was in
@@ -64,6 +91,7 @@ pub async fn run(
                     clients.clone(),
                     call,
                     observer.clone(),
+                    observation_config,
                     Arc::clone(&routed_calls),
                 )
             }
@@ -122,10 +150,18 @@ async fn serve(
     clients: ClientRouter,
     call: CallModel,
     observer: Option<RunObserver>,
+    observation_config: ObservationConfig,
     // Output parameter because `drive` takes a function that returns a plain `Result<()>`.
     routed_calls: Arc<Mutex<RoutedCallWindows>>,
 ) -> Result<()> {
-    let result = call_first_available(&clients, &call, &observer, &routed_calls).await;
+    let result = call_first_available(
+        &clients,
+        &call,
+        &observer,
+        observation_config,
+        &routed_calls,
+    )
+    .await;
     call.respond(result)
 }
 
@@ -134,6 +170,7 @@ async fn call_first_available(
     clients: &ClientRouter,
     call: &CallModel,
     observer: &Option<RunObserver>,
+    observation_config: ObservationConfig,
     routed_calls: &Arc<Mutex<RoutedCallWindows>>,
 ) -> Result<Response> {
     for (index, target) in call.models.iter().enumerate() {
@@ -144,6 +181,7 @@ async fn call_first_available(
             request,
             call,
             observer,
+            observation_config,
             routed_calls,
             index,
             call.models.len(),
@@ -212,6 +250,7 @@ async fn call_one(
     request: Request,
     call: &CallModel,
     observer: &Option<RunObserver>,
+    observation_config: ObservationConfig,
     routed_calls: &Arc<Mutex<RoutedCallWindows>>,
     // index is for span log
     index: usize,
@@ -229,6 +268,9 @@ async fn call_one(
         span.record("gen_ai.conversation.id", session_id);
     }
     let is_answer_call = call.is_answer_call;
+    let classifier_request =
+        (observer.is_some() && observation_config.classifier_content && !is_answer_call)
+            .then(|| request.llm_request.clone());
     // Resolved before the clock starts: picking the client is Switchyard's work, not
     // the provider's, so it belongs in the routing overhead.
     let client = clients.route(model_id);
@@ -253,6 +295,22 @@ async fn call_one(
     });
     let result = observability::observe_client_call(result);
     if let Some(observer) = observer {
+        if let Some(request) = classifier_request {
+            let aggregate = result
+                .as_ref()
+                .ok()
+                .and_then(|response| response.llm_response.as_agg());
+            observer(RunObservation::ClassifierContent(
+                ClassifierContentObservation {
+                    selected_model: model_id.clone(),
+                    request,
+                    reasoning: aggregate.and_then(classifier_reasoning),
+                    verdict: aggregate.and_then(classifier_verdict),
+                    is_success: result.is_ok(),
+                    duration,
+                },
+            ));
+        }
         observer(RunObservation::LlmCall(LlmCallObservation {
             selected_model: model_id.clone(),
             is_answer_call,
@@ -272,6 +330,28 @@ async fn call_one(
     result
 }
 
+fn classifier_reasoning(response: &switchyard_protocol::AggLlmResponse) -> Option<String> {
+    nonempty_join(response.outputs.iter().flat_map(|output| {
+        output.content.iter().filter_map(|block| match block {
+            ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
+            _ => None,
+        })
+    }))
+}
+
+fn classifier_verdict(response: &switchyard_protocol::AggLlmResponse) -> Option<String> {
+    let verdict = completion_text(response);
+    (!verdict.is_empty()).then_some(verdict)
+}
+
+fn nonempty_join<'a>(parts: impl Iterator<Item = &'a str>) -> Option<String> {
+    let joined = parts
+        .filter(|part| !part.is_empty())
+        .collect::<Vec<_>>()
+        .join("\n");
+    (!joined.is_empty()).then_some(joined)
+}
+
 /// Whether a failed candidate is worth routing around.
 fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {
     let LibsyError::ClientCall { source, .. } = error else {

crates/libsy-llm-client/tests/observability.rs

@@ -343,8 +343,18 @@ impl RoutedLlmClient for ClassifierClient {
             tokio::time::sleep(self.classifier_delay).await;
             r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#
         };
+        let mut response = text_response(Some(model_id.to_string()), completion);
+        if model_id == self.classifier_model_id {
+            response.outputs[0].content.insert(
+                0,
+                ContentBlock::Reasoning {
+                    text: "The task is bounded and supported.".to_string(),
+                    signature: None,
+                },
+            );
+        }
         Ok(Response {
-            llm_response: LlmResponse::Agg(text_response(Some(model_id.to_string()), completion)),
+            llm_response: LlmResponse::Agg(response),
             metadata: None,
         })
     }
@@ -1292,6 +1302,57 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib
     Ok(())
 }
 
+#[tokio::test]
+async fn observed_classifier_call_reports_prompt_reasoning_and_verdict()
+-> switchyard_libsy::Result<()> {
+    let _guard = serialize_test().lock().await;
+    let observations = Arc::new(Mutex::new(Vec::new()));
+    let observed = Arc::clone(&observations);
+    let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));
+    let client = Arc::new(ClassifierClient {
+        classifier_model_id: "classifier".into(),
+        classifier_delay: Duration::ZERO,
+        routed_delay: Duration::ZERO,
+    }) as Arc<dyn RoutedLlmClient>;
+
+    switchyard_llm_client::run_with_observation_config(
+        classifier_router("classifier", "weak", "strong")?,
+        ClientRouter::single(client),
+        classifier_request(),
+        Some(observer),
+        switchyard_llm_client::ObservationConfig {
+            classifier_content: true,
+        },
+    )
+    .await?;
+
+    let observations = observations.lock();
+    let content = observations
+        .iter()
+        .find_map(|observation| match observation {
+            RunObservation::ClassifierContent(content) => Some(content),
+            _ => None,
+        })
+        .ok_or_else(|| test_error("expected classifier content observation"))?;
+    assert_eq!(content.selected_model, "classifier");
+    assert_eq!(
+        content.reasoning.as_deref(),
+        Some("The task is bounded and supported.")
+    );
+    assert_eq!(
+        content.verdict.as_deref(),
+        Some(
+            r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#
+        )
+    );
+    assert_eq!(
+        content.request.messages,
+        classifier_request().llm_request.messages
+    );
+    assert!(content.is_success);
+    Ok(())
+}
+
 #[tokio::test]
 async fn classifier_fail_open_records_each_failure_stage() -> switchyard_libsy::Result<()> {
     let _guard = serialize_test().lock().await;

crates/switchyard-server/README.md

@@ -102,6 +102,12 @@ served model. The legacy `proxy_x_session_id` remains a fallback when no normali
 present. The endpoint returns `404` when the session has no records and is not registered when
 routing logging is disabled.
 
+Add `--routing-log-classifier-content` to append a `classifier_content` event containing the
+normalized classifier request, provider-returned reasoning, and raw verdict. This is opt-in
+because the classifier request and reasoning can repeat user-provided secrets. Transport headers
+and provider credentials are never included. Store this log with restricted permissions and
+rotation appropriate for sensitive request content.
+
 An `llm_classifier` route sends each task to `classifier_target` for a capability verdict, then
 routes to `weak_target` or `strong_target`. Beyond the three targets it accepts these keys; only
 `base_threshold` is required, and anything the judge cannot decide routes to `strong_target`:

crates/switchyard-server/src/cli.rs

@@ -52,6 +52,10 @@ pub(crate) struct ServerArgs {
     #[arg(long, value_name = "PATH")]
     routing_log_file: Option<PathBuf>,
 
+    /// Include classifier prompts, model reasoning, and verdicts in the routing log.
+    #[arg(long, requires = "routing_log_file")]
+    routing_log_classifier_content: bool,
+
     /// TLS certificate path in PEM format.
     #[arg(long, requires = "tls_key")]
     tls_cert: Option<PathBuf>,
@@ -72,6 +76,7 @@ impl ServerArgs {
         if let Some(path) = self.routing_log_file {
             state = state.with_routing_log(path)?;
         }
+        state = state.with_routing_log_classifier_content(self.routing_log_classifier_content);
         let tls = match (self.tls_cert, self.tls_key) {
             (Some(cert), Some(key)) => {
                 if !cert.exists() || !key.exists() {