perf: add savings audit output

headroom/cli/perf.py

@@ -79,8 +79,11 @@ def perf(hours: float, raw: bool, output_format: str) -> None:
                 "tokens_before",
                 "tokens_after",
                 "tokens_saved",
+                "prompt_reduction_tokens",
+                "accounting_delta_tokens",
                 "savings_pct",
                 "list_price_per_mtok",
+                "estimated_list_price_savings_usd",
             ]
             writer = csv.DictWriter(buf, fieldnames=fieldnames)
             writer.writeheader()

headroom/perf/analyzer.py

@@ -544,6 +544,28 @@ def format_report(report: PerfReport) -> str:
         lines.append(f"Total saved:  {total_saved:,} tokens")
         lines.append("")
 
+        audit = build_savings_audit(report)
+        lines.append("Savings Audit")
+        lines.append("-" * 40)
+        lines.append(
+            "  Raw prompt reduction: "
+            f"{audit['prompt_reduction_tokens']:,} tokens "
+            f"({audit['prompt_reduction_pct']:.1f}%)"
+        )
+        lines.append(
+            "  Logged savings:       "
+            f"{audit['logged_tokens_saved']:,} tokens "
+            f"({audit['logged_savings_pct']:.1f}%)"
+        )
+        lines.append(f"  Accounting delta:     {audit['accounting_delta_tokens']:+,} tokens")
+        delta_count = audit["record_counts"]["with_accounting_delta"]
+        if delta_count:
+            lines.append(f"  Records with delta:   {delta_count}/{len(records)}")
+        impossible_count = audit["record_counts"]["logged_saved_gt_tokens_before"]
+        if impossible_count:
+            lines.append(f"  Impossible records:   {impossible_count} saved more than before")
+        lines.append("")
+
         # Per-model breakdown with list prices
         by_model: dict[str, list[PerfRecord]] = {}
         for r in records:
@@ -803,6 +825,106 @@ def _pct(saved: int, before: int) -> float:
     return round(saved / before * 100, 1) if before > 0 else 0.0
 
 
+def _prompt_reduction_tokens(record: PerfRecord) -> int:
+    """Raw prompt-token delta for a record, excluding cache/accounting effects."""
+    return max(record.tokens_before - record.tokens_after, 0)
+
+
+def _audit_record(record: PerfRecord) -> dict:
+    prompt_reduction = _prompt_reduction_tokens(record)
+    return {
+        "timestamp": record.timestamp,
+        "request_id": record.request_id,
+        "model": record.model,
+        "client": record.client,
+        "tokens_before": record.tokens_before,
+        "tokens_after": record.tokens_after,
+        "logged_tokens_saved": record.tokens_saved,
+        "prompt_reduction_tokens": prompt_reduction,
+        "accounting_delta_tokens": record.tokens_saved - prompt_reduction,
+        "cache_read_tokens": record.cache_read,
+        "cache_write_tokens": record.cache_write,
+        "optimization_ms": record.optimization_ms,
+        "transforms": record.transforms,
+    }
+
+
+def _record_accounting_reasons(record: PerfRecord) -> list[str]:
+    reasons: list[str] = []
+    if min(record.tokens_before, record.tokens_after, record.tokens_saved) < 0:
+        reasons.append("negative_token_count")
+    if record.tokens_before > 0 and record.tokens_saved > record.tokens_before:
+        reasons.append("logged_saved_gt_tokens_before")
+    if record.tokens_after > record.tokens_before and record.tokens_saved > 0:
+        reasons.append("prompt_grew_but_logged_savings_positive")
+    if record.tokens_saved != _prompt_reduction_tokens(record):
+        reasons.append("logged_saved_ne_prompt_delta")
+    return reasons
+
+
+def build_savings_audit(report: PerfReport, *, limit: int = 10) -> dict:
+    """Explain how headline savings relate to raw before/after token deltas."""
+    records = report.perf_records
+    total_before = sum(r.tokens_before for r in records)
+    total_after = sum(r.tokens_after for r in records)
+    logged_saved = sum(r.tokens_saved for r in records)
+    prompt_reduction = sum(_prompt_reduction_tokens(r) for r in records)
+    prompt_growth = sum(max(r.tokens_after - r.tokens_before, 0) for r in records)
+    accounting_delta = logged_saved - prompt_reduction
+
+    records_with_delta = [r for r in records if r.tokens_saved != _prompt_reduction_tokens(r)]
+    impossible_saved_gt_before = [
+        r for r in records if r.tokens_before > 0 and r.tokens_saved > r.tokens_before
+    ]
+    prompt_growth_with_savings = [
+        r for r in records if r.tokens_after > r.tokens_before and r.tokens_saved > 0
+    ]
+    negative_token_counts = [
+        r for r in records if min(r.tokens_before, r.tokens_after, r.tokens_saved) < 0
+    ]
+
+    suspicious_records = []
+    for record in sorted(
+        records_with_delta,
+        key=lambda r: abs(r.tokens_saved - _prompt_reduction_tokens(r)),
+        reverse=True,
+    )[:limit]:
+        suspicious_records.append(
+            {
+                **_audit_record(record),
+                "reasons": _record_accounting_reasons(record),
+            }
+        )
+
+    return {
+        "formula": {
+            "raw_prompt_reduction": "sum(max(tokens_before - tokens_after, 0))",
+            "logged_savings": "sum(tokens_saved)",
+            "accounting_delta": "logged_savings - raw_prompt_reduction",
+        },
+        "total_tokens_before": total_before,
+        "total_tokens_after": total_after,
+        "logged_tokens_saved": logged_saved,
+        "logged_savings_pct": _pct(logged_saved, total_before),
+        "prompt_reduction_tokens": prompt_reduction,
+        "prompt_reduction_pct": _pct(prompt_reduction, total_before),
+        "prompt_growth_tokens": prompt_growth,
+        "accounting_delta_tokens": accounting_delta,
+        "record_counts": {
+            "total": len(records),
+            "with_accounting_delta": len(records_with_delta),
+            "logged_saved_gt_tokens_before": len(impossible_saved_gt_before),
+            "prompt_grew_but_logged_savings_positive": len(prompt_growth_with_savings),
+            "negative_token_count": len(negative_token_counts),
+        },
+        "top_saving_requests": [
+            _audit_record(r)
+            for r in sorted(records, key=lambda rec: rec.tokens_saved, reverse=True)[:limit]
+        ],
+        "suspicious_records": suspicious_records,
+    }
+
+
 def _percentile(data: list[float], pct: float) -> float:
     if not data:
         return 0.0
@@ -1029,6 +1151,7 @@ def build_perf_summary(report: PerfReport) -> dict:
     total_before = sum(r.tokens_before for r in records)
     total_after = sum(r.tokens_after for r in records)
     total_saved = sum(r.tokens_saved for r in records)
+    savings_audit = build_savings_audit(report)
 
     total_cr = sum(r.cache_read for r in records)
     total_cw = sum(r.cache_write for r in records)
@@ -1039,19 +1162,29 @@ def build_perf_summary(report: PerfReport) -> dict:
     for r in records:
         by_model_groups.setdefault(r.model, []).append(r)
     by_model = []
+    estimated_list_price_savings_usd = 0.0
     for model, recs in sorted(by_model_groups.items()):
         m_before = sum(r.tokens_before for r in recs)
         m_after = sum(r.tokens_after for r in recs)
         m_saved = sum(r.tokens_saved for r in recs)
+        m_prompt_reduction = sum(_prompt_reduction_tokens(r) for r in recs)
+        list_price = _get_list_price(model)
+        if list_price:
+            estimated_list_price_savings_usd += m_saved * list_price / 1_000_000
         by_model.append(
             {
                 "model": model,
                 "requests": len(recs),
                 "tokens_before": m_before,
                 "tokens_after": m_after,
                 "tokens_saved": m_saved,
+                "prompt_reduction_tokens": m_prompt_reduction,
+                "accounting_delta_tokens": m_saved - m_prompt_reduction,
                 "savings_pct": _pct(m_saved, m_before),
-                "list_price_per_mtok": _get_list_price(model),
+                "list_price_per_mtok": list_price,
+                "estimated_list_price_savings_usd": round(m_saved * list_price / 1_000_000, 4)
+                if list_price
+                else None,
             }
         )
 
@@ -1086,6 +1219,8 @@ def build_perf_summary(report: PerfReport) -> dict:
         "total_tokens_after": total_after,
         "tokens_saved": total_saved,
         "savings_pct": _pct(total_saved, total_before),
+        "estimated_list_price_savings_usd": round(estimated_list_price_savings_usd, 4),
+        "savings_audit": savings_audit,
         "cache_read_tokens": total_cr,
         "cache_write_tokens": total_cw,
         "cache_hit_pct": cache_hit_pct,

tests/test_cli_perf_format.py

@@ -17,6 +17,7 @@
     TransformRecord,
     build_overhead_summary,
     build_perf_summary,
+    build_savings_audit,
     perf_records_as_dicts,
 )
 
@@ -95,6 +96,8 @@ def test_build_perf_summary_totals_and_pct():
     assert summary["cache_write_tokens"] == 200
     assert summary["cache_hit_pct"] == 83.3
     assert summary["window_hours"] == 24.0
+    assert summary["savings_audit"]["prompt_reduction_tokens"] == 1000
+    assert summary["savings_audit"]["accounting_delta_tokens"] == 0
 
 
 def test_build_perf_summary_by_model_and_transform():
@@ -162,6 +165,40 @@ def test_build_overhead_summary_attributes_slow_stages():
     assert overhead["top_slow_requests"][0]["slowest_stage"] == "kompress"
 
 
+def test_build_savings_audit_surfaces_accounting_deltas():
+    report = PerfReport(
+        perf_records=[
+            PerfRecord(
+                timestamp="2026-06-05 10:00:00,000",
+                request_id="hr_bad",
+                model="gpt-5",
+                tokens_before=100,
+                tokens_after=90,
+                tokens_saved=250,
+            ),
+            PerfRecord(
+                timestamp="2026-06-05 10:01:00,000",
+                request_id="hr_growth",
+                model="gpt-5",
+                tokens_before=100,
+                tokens_after=120,
+                tokens_saved=1,
+            ),
+        ]
+    )
+
+    audit = build_savings_audit(report)
+
+    assert audit["logged_tokens_saved"] == 251
+    assert audit["prompt_reduction_tokens"] == 10
+    assert audit["accounting_delta_tokens"] == 241
+    assert audit["record_counts"]["with_accounting_delta"] == 2
+    assert audit["record_counts"]["logged_saved_gt_tokens_before"] == 1
+    assert audit["record_counts"]["prompt_grew_but_logged_savings_positive"] == 1
+    assert audit["suspicious_records"][0]["request_id"] == "hr_bad"
+    assert "logged_saved_gt_tokens_before" in audit["suspicious_records"][0]["reasons"]
+
+
 def test_perf_records_as_dicts_roundtrips_fields():
     dicts = perf_records_as_dicts(_sample_report())
     assert len(dicts) == 2
@@ -238,6 +275,8 @@ def test_perf_csv_by_model(runner, monkeypatch):
     assert {r["model"] for r in rows} == {"claude-sonnet-4.5", "claude-opus-4-8"}
     sonnet = next(r for r in rows if r["model"] == "claude-sonnet-4.5")
     assert sonnet["tokens_saved"] == "600"
+    assert sonnet["prompt_reduction_tokens"] == "600"
+    assert sonnet["accounting_delta_tokens"] == "0"
 
 
 def test_perf_csv_raw_per_record(runner, monkeypatch):