feat(reborn): Context management — progressive tool disclosure (flag-gated, default off)

crates/ironclaw_llm/src/nearai_chat.rs

@@ -581,14 +581,16 @@ impl LlmProvider for NearAiChatProvider {
         };
 
         let (input_tokens, output_tokens) = parse_usage(response.usage.as_ref());
+        let cached_tokens = parse_cached_tokens(response.usage.as_ref());
+        emit_context_shadow_usage(input_tokens, output_tokens, cached_tokens);
 
         Ok(CompletionResponse {
             content,
             finish_reason,
             input_tokens,
             output_tokens,
             reasoning: provider_reasoning,
-            cache_read_input_tokens: 0,
+            cache_read_input_tokens: cached_tokens.unwrap_or(0).min(input_tokens),
             cache_creation_input_tokens: 0,
         })
     }
@@ -692,14 +694,16 @@ impl LlmProvider for NearAiChatProvider {
         };
 
         let (input_tokens, output_tokens) = parse_usage(response.usage.as_ref());
+        let cached_tokens = parse_cached_tokens(response.usage.as_ref());
+        emit_context_shadow_usage(input_tokens, output_tokens, cached_tokens);
 
         Ok(ToolCompletionResponse {
             content,
             tool_calls,
             finish_reason,
             input_tokens,
             output_tokens,
-            cache_read_input_tokens: 0,
+            cache_read_input_tokens: cached_tokens.unwrap_or(0).min(input_tokens),
             cache_creation_input_tokens: 0,
             reasoning: provider_reasoning,
         })
@@ -1139,6 +1143,16 @@ struct ChatCompletionUsage {
     completion_tokens: Option<u64>,
     #[serde(default)]
     total_tokens: Option<u64>,
+    #[serde(default)]
+    prompt_tokens_details: Option<PromptTokensDetails>,
+    #[serde(default)]
+    cached_tokens: Option<u64>,
+}
+
+#[derive(Debug, Deserialize, Default)]
+struct PromptTokensDetails {
+    #[serde(default)]
+    cached_tokens: Option<u64>,
 }
 
 fn saturate_u32(val: u64) -> u32 {
@@ -1161,6 +1175,33 @@ fn emit_reasoning_trace(reasoning: Option<&str>) {
     }
 }
 
+fn emit_context_shadow_usage(
+    prompt_tokens: u32,
+    completion_tokens: u32,
+    cached_tokens: Option<u32>,
+) {
+    const CONTEXT_SHADOW_TARGET: &str = "ironclaw::reborn::context_shadow";
+    let cached_tokens_field = cached_tokens.map(i64::from).unwrap_or(-1);
+    if let Some(cached_tokens) = cached_tokens.filter(|_| prompt_tokens > 0) {
+        tracing::debug!(
+            target: CONTEXT_SHADOW_TARGET,
+            prompt_tokens,
+            completion_tokens,
+            cached_tokens = cached_tokens_field,
+            cache_hit_ratio = cached_tokens as f64 / prompt_tokens as f64,
+            "nearai chat usage shadow measurement"
+        );
+    } else {
+        tracing::debug!(
+            target: CONTEXT_SHADOW_TARGET,
+            prompt_tokens,
+            completion_tokens,
+            cached_tokens = cached_tokens_field,
+            "nearai chat usage shadow measurement"
+        );
+    }
+}
+
 fn parse_usage(usage: Option<&ChatCompletionUsage>) -> (u32, u32) {
     let Some(u) = usage else {
         return (0, 0);
@@ -1177,6 +1218,16 @@ fn parse_usage(usage: Option<&ChatCompletionUsage>) -> (u32, u32) {
     (input, output)
 }
 
+fn parse_cached_tokens(usage: Option<&ChatCompletionUsage>) -> Option<u32> {
+    let usage = usage?;
+    usage
+        .prompt_tokens_details
+        .as_ref()
+        .and_then(|details| details.cached_tokens)
+        .or(usage.cached_tokens)
+        .map(saturate_u32)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -2740,6 +2791,8 @@ mod tests {
             prompt_tokens: Some(100),
             completion_tokens: Some(50),
             total_tokens: Some(150),
+            prompt_tokens_details: None,
+            cached_tokens: None,
         };
         assert_eq!(parse_usage(Some(&usage)), (100, 50));
     }
@@ -2755,6 +2808,8 @@ mod tests {
             prompt_tokens: Some(100),
             completion_tokens: None,
             total_tokens: Some(180),
+            prompt_tokens_details: None,
+            cached_tokens: None,
         };
         // output = total - prompt = 80
         assert_eq!(parse_usage(Some(&usage)), (100, 80));
@@ -2766,6 +2821,8 @@ mod tests {
             prompt_tokens: None,
             completion_tokens: None,
             total_tokens: Some(200),
+            prompt_tokens_details: None,
+            cached_tokens: None,
         };
         // input = 0 (no prompt), output = total = 200
         assert_eq!(parse_usage(Some(&usage)), (0, 200));
@@ -2777,6 +2834,8 @@ mod tests {
             prompt_tokens: None,
             completion_tokens: None,
             total_tokens: None,
+            prompt_tokens_details: None,
+            cached_tokens: None,
         };
         assert_eq!(parse_usage(Some(&usage)), (0, 0));
     }
@@ -2931,13 +2990,80 @@ mod tests {
         assert_eq!(usage.total_tokens, Some(500));
     }
 
+    #[test]
+    fn test_usage_deserialize_nested_cached_tokens() {
+        let json = r#"{
+            "prompt_tokens": 100,
+            "completion_tokens": 25,
+            "total_tokens": 125,
+            "prompt_tokens_details": {
+                "cached_tokens": 80
+            }
+        }"#;
+        let usage: ChatCompletionUsage = serde_json::from_str(json).unwrap();
+        assert_eq!(parse_usage(Some(&usage)), (100, 25));
+        assert_eq!(parse_cached_tokens(Some(&usage)), Some(80));
+    }
+
+    #[test]
+    fn test_usage_deserialize_top_level_cached_tokens() {
+        let json = r#"{
+            "prompt_tokens": 100,
+            "completion_tokens": 25,
+            "total_tokens": 125,
+            "cached_tokens": 40
+        }"#;
+        let usage: ChatCompletionUsage = serde_json::from_str(json).unwrap();
+        assert_eq!(parse_cached_tokens(Some(&usage)), Some(40));
+    }
+
+    #[test]
+    fn test_usage_deserialize_prefers_nested_cached_tokens() {
+        let json = r#"{
+            "prompt_tokens": 100,
+            "completion_tokens": 25,
+            "total_tokens": 125,
+            "prompt_tokens_details": {
+                "cached_tokens": 80
+            },
+            "cached_tokens": 40
+        }"#;
+        let usage: ChatCompletionUsage = serde_json::from_str(json).unwrap();
+        assert_eq!(parse_cached_tokens(Some(&usage)), Some(80));
+    }
+
+    #[test]
+    fn test_usage_deserialize_cached_tokens_absent() {
+        let json = r#"{
+            "prompt_tokens": 100,
+            "completion_tokens": 25,
+            "total_tokens": 125
+        }"#;
+        let usage: ChatCompletionUsage = serde_json::from_str(json).unwrap();
+        assert_eq!(parse_cached_tokens(Some(&usage)), None);
+    }
+
+    #[test]
+    fn test_usage_without_details_still_parses_token_counts() {
+        let json = r#"{
+            "prompt_tokens": 10,
+            "completion_tokens": 5,
+            "total_tokens": 15
+        }"#;
+        let usage: ChatCompletionUsage = serde_json::from_str(json).unwrap();
+        assert_eq!(parse_usage(Some(&usage)), (10, 5));
+        assert_eq!(parse_cached_tokens(Some(&usage)), None);
+    }
+
     #[test]
     fn test_usage_deserialize_empty_object() {
         let json = "{}";
         let usage: ChatCompletionUsage = serde_json::from_str(json).unwrap();
         assert!(usage.prompt_tokens.is_none());
         assert!(usage.completion_tokens.is_none());
         assert!(usage.total_tokens.is_none());
+        assert!(usage.prompt_tokens_details.is_none());
+        assert!(usage.cached_tokens.is_none());
     }
 
     // -- ChatCompletionToolCall serde roundtrip --------------------------------

crates/ironclaw_loop_support/src/capability_surface_filter.rs

@@ -74,6 +74,25 @@ impl LoopCapabilityPort for CapabilitySurfaceVisibleFilter {
         Ok(definitions)
     }
 
+    fn provider_tool_call_capability_ids(
+        &self,
+        tool_call: &ProviderToolCall,
+    ) -> Result<ProviderToolCallCapabilityIds, AgentLoopHostError> {
+        // Delegate resolution to the inner port (e.g. the tool-disclosure
+        // forgiving path) rather than the LoopCapabilityPort default, which only
+        // searches THIS filter's already-filtered `tool_definitions` and so
+        // rejects every deferred/disclosed tool before the inner port can resolve
+        // it. Apply the same visible-scope check as `validate`/`register` so a
+        // genuinely non-visible call is still rejected.
+        let capability_ids = self.inner.provider_tool_call_capability_ids(tool_call)?;
+        validate_provider_tool_call_capability_scope(
+            capability_ids.clone(),
+            |capability_id| self.permits(capability_id),
+            "provider tool call is outside the model-visible capability view",
+        )?;
+        Ok(capability_ids)
+    }
+
     fn validate_provider_tool_call(
         &self,
         tool_call: &ProviderToolCall,
@@ -161,6 +180,24 @@ impl LoopCapabilityPort for CapabilitySurfaceProfileFilter {
         Ok(definitions)
     }
 
+    fn provider_tool_call_capability_ids(
+        &self,
+        tool_call: &ProviderToolCall,
+    ) -> Result<ProviderToolCallCapabilityIds, AgentLoopHostError> {
+        // Delegate to the inner port instead of the LoopCapabilityPort default
+        // (which only searches this filter's filtered `tool_definitions`), then
+        // apply the run-profile scope — mirroring `validate`/`register`.
+        let capability_ids = self.inner.provider_tool_call_capability_ids(tool_call)?;
+        if !matches!(self.allow_set.as_ref(), CapabilityAllowSet::All) {
+            validate_provider_tool_call_capability_scope(
+                capability_ids.clone(),
+                |capability_id| self.allow_set.permits(capability_id),
+                "provider tool call is outside the run-profile surface",
+            )?;
+        }
+        Ok(capability_ids)
+    }
+
     fn validate_provider_tool_call(
         &self,
         tool_call: &ProviderToolCall,
@@ -1384,4 +1421,66 @@ mod tests {
 
         assert_eq!(surface.version.as_str(), "surface-v1");
     }
+
+    #[test]
+    fn visible_filter_delegates_provider_tool_call_resolution_to_inner() {
+        // Regression: the model gateway pre-check calls
+        // `provider_tool_call_capability_ids` on whatever capability port it holds
+        // — here a CapabilitySurfaceVisibleFilter. If the filter fell back to the
+        // LoopCapabilityPort default (which only searches its OWN already-filtered
+        // `tool_definitions`), every deferred/disclosed tool would be rejected
+        // before the inner forgiving port could resolve it — which is exactly the
+        // "outside the visible capability surface" failure that killed every
+        // deferred extension-tool call. The filter must delegate to inner, then
+        // apply the visible-scope check.
+        let inner = Arc::new(SpyPort::default());
+        // Only the advertised tool is in tool_definitions (the filtered surface)...
+        *inner
+            .tool_definitions
+            .lock()
+            .expect("tool definitions lock") =
+            vec![provider_definition("demo.advertised", "demo__advertised")];
+        // ...but the inner port can still RESOLVE a deferred tool (mimicking the
+        // tool-disclosure forgiving path) that is NOT in tool_definitions.
+        inner
+            .provider_call_capability_ids
+            .lock()
+            .expect("provider call capability ids lock")
+            .insert(
+                "demo__deferred".to_string(),
+                provider_call_capability_ids(&["demo.deferred"]),
+            );
+        // The deferred tool's capability id IS in the model-visible view (it was
+        // disclosed via tool_search), so the call must resolve.
+        let filter = CapabilitySurfaceVisibleFilter::new(
+            Arc::clone(&inner) as Arc<dyn LoopCapabilityPort>,
+            [
+                capability_id("demo.advertised"),
+                capability_id("demo.deferred"),
+            ],
+        );
+
+        let resolved = filter
+            .provider_tool_call_capability_ids(&provider_call("demo__deferred"))
+            .expect("deferred-but-visible tool resolves through the inner forgiving path");
+        assert_eq!(
+            resolved.provider_capability_id,
+            capability_id("demo.deferred")
+        );
+
+        // A resolvable tool whose capability id is NOT in the visible view is still
+        // rejected by the scope check.
+        inner
+            .provider_call_capability_ids
+            .lock()
+            .expect("provider call capability ids lock")
+            .insert(
+                "demo__hidden".to_string(),
+                provider_call_capability_ids(&["demo.hidden"]),
+            );
+        let error = filter
+            .provider_tool_call_capability_ids(&provider_call("demo__hidden"))
+            .expect_err("non-visible tool is rejected");
+        assert_eq!(error.kind, AgentLoopHostErrorKind::InvalidInvocation);
+    }
 }

crates/ironclaw_loop_support/src/lib.rs

@@ -5,6 +5,14 @@
 //! not own provider clients, tool dispatchers, secrets, or runtime handles.
 #![warn(unreachable_pub)]
 
+/// Build tag for the capability-surface-filter provider-tool-call resolution path.
+/// Bumped when that resolution logic changes so a startup marker can prove THIS
+/// crate (not just a stale incremental codegen unit) is actually linked in.
+/// `delegate-to-inner` => the filters delegate `provider_tool_call_capability_ids`
+/// to the inner port (reaching the tool-disclosure forgiving path) instead of the
+/// LoopCapabilityPort default.
+pub const CAPABILITY_FILTER_RESOLUTION_BUILD: &str = "delegate-to-inner";
+
 use std::{
     collections::{HashMap, HashSet},
     sync::{

crates/ironclaw_reborn/src/context_shadow.rs

@@ -0,0 +1,24 @@
+/// Deterministic shadow token estimator for prompt-surface diagnostics.
+#[allow(clippy::manual_div_ceil)]
+pub(crate) fn estimate_tokens(s: &str) -> u32 {
+    let char_count = s.chars().count();
+    ((char_count + 3) / 4).min(u32::MAX as usize) as u32
+}
+
+#[cfg(test)]
+mod tests {
+    use super::estimate_tokens;
+
+    #[test]
+    fn estimate_tokens_uses_ceiling_char_count_over_four() {
+        assert_eq!(estimate_tokens(""), 0);
+        assert_eq!(estimate_tokens("a"), 1);
+        assert_eq!(estimate_tokens("abcd"), 1);
+        assert_eq!(estimate_tokens("abcde"), 2);
+    }
+
+    #[test]
+    fn estimate_tokens_counts_chars_not_bytes() {
+        assert_eq!(estimate_tokens("ééééé"), 2);
+    }
+}

crates/ironclaw_reborn/src/lib.rs

@@ -17,6 +17,7 @@
 //! the boundary tests are designed to prevent.
 
 pub mod app_loop_family;
+mod context_shadow;
 pub mod driver_registry;
 pub mod failure_categories;
 pub mod hook_gate_refs;
@@ -34,6 +35,8 @@ pub mod runtime;
 pub mod subagent;
 pub mod text_loop_driver;
 pub mod thread_scope;
+mod tool_disclosure;
+mod tool_disclosure_port;
 pub mod turn_run_executor;
 pub mod turn_runner;