Add experiment-level cost tracking with token-based estimation (#23)

Harbor's AgentContext.cost_usd is almost always null, so the dashboard
and experiment views have been showing no cost information at all. This
adds a static LiteLLM-sourced pricing table (oddish/model_pricing.py)
that estimates trial cost from the captured input / cache / output token
counts when no native cost is reported.

The trial response now carries a cost_is_estimated flag and the
experiment summary bar aggregates cost across all trials in an
experiment, marking mixed native + estimated totals with a trailing '*'
and pure-estimate totals with a leading '~'.

Co-authored-by: Cursor Agent <cursoragent@cursor.com>

Author: RishiDesai

frontend/src/components/experiment-detail-view.tsx

@@ -80,6 +80,10 @@ type ExperimentSummary = {
   failCount: number;
   harnessErrorCount: number;
   pendingCount: number;
+  costUsd: number;
+  costTrialCount: number;
+  costHasEstimated: boolean;
+  costHasNative: boolean;
 };
 
 function buildExperimentSummary(tasksForExperiment: Task[]): ExperimentSummary {
@@ -96,11 +100,25 @@ function buildExperimentSummary(tasksForExperiment: Task[]): ExperimentSummary {
   let harnessErrorCount = 0;
   let pendingCount = 0;
 
+  let costUsd = 0;
+  let costTrialCount = 0;
+  let costHasEstimated = false;
+  let costHasNative = false;
+
   for (const task of tasksForExperiment) {
     const trials = task.trials ?? [];
     if (trials.length > 0) {
       // Compute from the (already version-filtered) trials array
       for (const trial of trials) {
+        if (trial.cost_usd != null) {
+          costUsd += trial.cost_usd;
+          costTrialCount += 1;
+          if (trial.cost_is_estimated === true) {
+            costHasEstimated = true;
+          } else {
+            costHasNative = true;
+          }
+        }
         if (trial.status === "success" && trial.reward != null) {
           rewardSum += trial.reward;
           rewardTotal++;
@@ -146,9 +164,21 @@ function buildExperimentSummary(tasksForExperiment: Task[]): ExperimentSummary {
     failCount,
     harnessErrorCount,
     pendingCount,
+    costUsd,
+    costTrialCount,
+    costHasEstimated,
+    costHasNative,
   };
 }
 
+function formatCostUsd(value: number): string {
+  if (!Number.isFinite(value) || value <= 0) return "$0.00";
+  if (value < 0.01) return `$${value.toFixed(4)}`;
+  if (value < 1) return `$${value.toFixed(3)}`;
+  if (value < 100) return `$${value.toFixed(2)}`;
+  return `$${value.toLocaleString(undefined, { maximumFractionDigits: 0 })}`;
+}
+
 function ExperimentHeaderMeta({
   isLoading,
   isInitialLoading,
@@ -229,6 +259,34 @@ function ExperimentSummaryBar({
             : "—"}
         </div>
         <div className="text-muted-foreground">•</div>
+        <div
+          className="font-mono text-muted-foreground"
+          title={
+            summary.costTrialCount > 0
+              ? `Summed across ${summary.costTrialCount} trial${
+                  summary.costTrialCount === 1 ? "" : "s"
+                }${
+                  summary.costHasEstimated && summary.costHasNative
+                    ? ". Mixed native + estimated values; ~ marks estimates."
+                    : summary.costHasEstimated
+                      ? ". Estimated from token counts × static model pricing."
+                      : ". Reported by the agent runtime."
+                }`
+              : "No cost data reported yet"
+          }
+        >
+          Cost{" "}
+          {summary.costTrialCount > 0 ? (
+            <>
+              {summary.costHasEstimated && !summary.costHasNative ? "~" : ""}
+              {formatCostUsd(summary.costUsd)}
+              {summary.costHasEstimated && summary.costHasNative ? "*" : ""}
+            </>
+          ) : (
+            "—"
+          )}
+        </div>
+        <div className="text-muted-foreground">•</div>
         <div className="flex items-center gap-2 font-mono text-muted-foreground">
           <span className="inline-flex items-center gap-0.5 text-emerald-400">
             {summary.passCount}

frontend/src/lib/types.ts

@@ -71,6 +71,11 @@ export interface Trial {
   queue_info?: TrialQueueInfo | null;
   task_version?: number | null;
   task_version_id?: string | null;
+  input_tokens?: number | null;
+  cache_tokens?: number | null;
+  output_tokens?: number | null;
+  cost_usd?: number | null;
+  cost_is_estimated?: boolean | null;
   created_at: string;
   started_at?: string | null;
   finished_at?: string | null;

oddish/src/oddish/core/endpoints.py

@@ -124,6 +124,10 @@ async def list_tasks_core(
                 TrialModel.has_trajectory,
                 TrialModel.phase_timing,
                 TrialModel.analysis_status,
+                TrialModel.input_tokens,
+                TrialModel.cache_tokens,
+                TrialModel.output_tokens,
+                TrialModel.cost_usd,
                 TrialModel.created_at,
                 TrialModel.started_at,
                 TrialModel.finished_at,

oddish/src/oddish/core/helpers.py

@@ -19,8 +19,33 @@
     TrialModel,
     TrialStatus,
 )
+from oddish.model_pricing import estimate_cost_usd
 from oddish.schemas import TaskStatusResponse, TrialQueueInfo, TrialResponse
 
+
+def _resolve_trial_cost(
+    trial: TrialModel, model_name: str | None
+) -> tuple[float | None, bool | None]:
+    """Return ``(cost_usd, cost_is_estimated)`` for a trial.
+
+    Prefers the native cost reported by the agent runtime. Falls back to
+    estimating from the pricing table when native cost is missing but we
+    have token counts and a known model.
+    """
+    if trial.cost_usd is not None:
+        return float(trial.cost_usd), False
+    if trial.input_tokens is None and trial.output_tokens is None:
+        return None, None
+    estimated = estimate_cost_usd(
+        model_name or trial.model,
+        trial.input_tokens,
+        trial.output_tokens,
+        trial.cache_tokens,
+    )
+    if estimated is None:
+        return None, None
+    return estimated, True
+
 _ANALYSIS_SUMMARY_UNSET = object()
 _VERSION_ID_UNSET: object = object()
 _QUEUE_PENDING_STATUSES = {TrialStatus.QUEUED, TrialStatus.RETRYING}
@@ -204,6 +229,7 @@ def build_trial_response(
     """Build a TrialResponse from a TrialModel."""
     normalized_model = settings.normalize_trial_model(trial.agent, trial.model)
     task_version, task_version_id = _resolve_trial_version_fields(trial)
+    cost_usd, cost_is_estimated = _resolve_trial_cost(trial, normalized_model)
     return TrialResponse(
         id=trial.id,
         name=trial.name,
@@ -227,7 +253,8 @@ def build_trial_response(
         input_tokens=trial.input_tokens,
         cache_tokens=trial.cache_tokens,
         output_tokens=trial.output_tokens,
-        cost_usd=trial.cost_usd,
+        cost_usd=cost_usd,
+        cost_is_estimated=cost_is_estimated,
         phase_timing=trial.phase_timing,
         has_trajectory=trial.has_trajectory,
         analysis_status=trial.analysis_status,
@@ -265,6 +292,7 @@ def build_compact_trial_response(
         )
     normalized_model = settings.normalize_trial_model(trial.agent, trial.model)
     task_version, task_version_id = _resolve_trial_version_fields(trial)
+    cost_usd, cost_is_estimated = _resolve_trial_cost(trial, normalized_model)
 
     return TrialResponse(
         id=trial.id,
@@ -286,10 +314,11 @@ def build_compact_trial_response(
         reward=trial.reward,
         error_message=trial.error_message,
         result=None,
-        input_tokens=None,
-        cache_tokens=None,
-        output_tokens=None,
-        cost_usd=None,
+        input_tokens=trial.input_tokens,
+        cache_tokens=trial.cache_tokens,
+        output_tokens=trial.output_tokens,
+        cost_usd=cost_usd,
+        cost_is_estimated=cost_is_estimated,
         phase_timing=trial.phase_timing,
         has_trajectory=trial.has_trajectory,
         analysis_status=trial.analysis_status,

oddish/src/oddish/model_pricing.py

@@ -0,0 +1,123 @@
+"""Model pricing table for estimating trial cost from token counts.
+
+Harbor does not populate ``cost_usd`` on AgentContext for every provider, so
+we fall back to a static pricing table to estimate cost from the captured
+input / cache / output token counts. Prices are per-token (not per-million)
+and sourced from LiteLLM's ``model_prices_and_context_window.json`` reference.
+
+Model-name matching uses substring patterns because trajectory / agent data
+uses varying formats (e.g. ``claude-sonnet-4-5-20250929``,
+``anthropic/claude-opus-4-5``, ``bedrock/global.anthropic.claude-sonnet-4-5``,
+``gpt-5.1-codex``). More-specific patterns MUST appear before less-specific
+ones to avoid false substring matches (e.g. ``gpt-5-mini`` before ``gpt-5``).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ModelPricing:
+    """Per-token pricing for a single model family."""
+
+    input: float
+    output: float
+    cache_read: float | None = None
+
+
+PRICING_TABLE: list[tuple[str, ModelPricing]] = [
+    # Anthropic — Claude 4.x family. Opus 4.5 dropped in price vs 4 / 4.1.
+    ("claude-opus-4-5", ModelPricing(input=5e-6, output=25e-6, cache_read=5e-7)),
+    ("claude-opus-4-1", ModelPricing(input=15e-6, output=75e-6, cache_read=1.5e-6)),
+    ("claude-opus-4", ModelPricing(input=15e-6, output=75e-6, cache_read=1.5e-6)),
+    ("claude-sonnet-4-5", ModelPricing(input=3e-6, output=15e-6, cache_read=3e-7)),
+    ("claude-sonnet-4", ModelPricing(input=3e-6, output=15e-6, cache_read=3e-7)),
+    ("claude-haiku-4-5", ModelPricing(input=1e-6, output=5e-6, cache_read=1e-7)),
+    ("claude-haiku-4", ModelPricing(input=1e-6, output=5e-6, cache_read=1e-7)),
+    # Anthropic — older Claude 3.x still showing up in legacy runs.
+    ("claude-3-7-sonnet", ModelPricing(input=3e-6, output=15e-6, cache_read=3e-7)),
+    ("claude-3-5-sonnet", ModelPricing(input=3e-6, output=15e-6, cache_read=3e-7)),
+    ("claude-3-5-haiku", ModelPricing(input=8e-7, output=4e-6, cache_read=8e-8)),
+    ("claude-3.5-sonnet", ModelPricing(input=3e-6, output=15e-6, cache_read=3e-7)),
+    ("claude-3.5-haiku", ModelPricing(input=8e-7, output=4e-6, cache_read=8e-8)),
+    ("claude-3-opus", ModelPricing(input=15e-6, output=75e-6, cache_read=1.5e-6)),
+    # Google — Gemini 3.x
+    ("gemini-3-pro", ModelPricing(input=2e-6, output=12e-6, cache_read=2e-7)),
+    ("gemini-3-flash", ModelPricing(input=5e-7, output=3e-6, cache_read=5e-8)),
+    # Google — Gemini 2.5
+    ("gemini-2.5-flash-lite", ModelPricing(input=1e-7, output=4e-7, cache_read=1e-8)),
+    ("gemini-2.5-flash", ModelPricing(input=3e-7, output=2.5e-6, cache_read=3e-8)),
+    ("gemini-2.5-pro", ModelPricing(input=1.25e-6, output=10e-6, cache_read=1.25e-7)),
+    # OpenAI — GPT-5.x
+    ("gpt-5.1-codex-mini", ModelPricing(input=2.5e-7, output=2e-6, cache_read=2.5e-8)),
+    ("gpt-5.1-codex", ModelPricing(input=1.25e-6, output=10e-6, cache_read=1.25e-7)),
+    ("gpt-5.1", ModelPricing(input=1.25e-6, output=10e-6, cache_read=1.25e-7)),
+    ("gpt-5-codex", ModelPricing(input=1.25e-6, output=10e-6, cache_read=1.25e-7)),
+    ("gpt-5-mini", ModelPricing(input=2.5e-7, output=2e-6, cache_read=2.5e-8)),
+    ("gpt-5-nano", ModelPricing(input=5e-8, output=4e-7, cache_read=5e-9)),
+    ("gpt-5-pro", ModelPricing(input=15e-6, output=120e-6)),
+    ("gpt-5", ModelPricing(input=1.25e-6, output=10e-6, cache_read=1.25e-7)),
+    # OpenAI — GPT-4.1 / 4o.
+    ("gpt-4.1-mini", ModelPricing(input=4e-7, output=1.6e-6, cache_read=1e-7)),
+    ("gpt-4.1-nano", ModelPricing(input=1e-7, output=4e-7, cache_read=2.5e-8)),
+    ("gpt-4.1", ModelPricing(input=2e-6, output=8e-6, cache_read=5e-7)),
+    ("gpt-4o-mini", ModelPricing(input=1.5e-7, output=6e-7, cache_read=7.5e-8)),
+    ("gpt-4o", ModelPricing(input=2.5e-6, output=10e-6, cache_read=1.25e-6)),
+    # OpenAI — reasoning.
+    ("o4-mini", ModelPricing(input=1.1e-6, output=4.4e-6, cache_read=2.75e-7)),
+    ("o3-pro", ModelPricing(input=20e-6, output=80e-6)),
+    ("o3-mini", ModelPricing(input=1.1e-6, output=4.4e-6, cache_read=5.5e-7)),
+    ("o3", ModelPricing(input=2e-6, output=8e-6, cache_read=5e-7)),
+    # OpenAI — standalone codex endpoint.
+    ("codex-mini", ModelPricing(input=1.5e-6, output=6e-6, cache_read=3.75e-7)),
+]
+
+
+def _find_pricing(model_name: str) -> ModelPricing | None:
+    lower = model_name.lower()
+    for pattern, pricing in PRICING_TABLE:
+        if pattern in lower:
+            return pricing
+    return None
+
+
+def has_pricing(model_name: str | None) -> bool:
+    """Return True iff the pricing table has an entry for this model."""
+    if not model_name:
+        return False
+    return _find_pricing(model_name) is not None
+
+
+def estimate_cost_usd(
+    model_name: str | None,
+    input_tokens: int | None,
+    output_tokens: int | None,
+    cached_tokens: int | None = None,
+) -> float | None:
+    """Estimate USD cost from token counts and a model name.
+
+    ``cached_tokens`` is the subset of ``input_tokens`` that were served from
+    prompt cache; those tokens are billed at ``cache_read`` instead of the
+    full input rate. Returns ``None`` when the model is not in the pricing
+    table or there are no tokens to price.
+    """
+    if not model_name:
+        return None
+    input_total = int(input_tokens or 0)
+    output_total = int(output_tokens or 0)
+    if input_total == 0 and output_total == 0:
+        return None
+    pricing = _find_pricing(model_name)
+    if pricing is None:
+        return None
+
+    cached = int(cached_tokens or 0)
+    uncached_input = max(0, input_total - cached)
+    cache_rate = pricing.cache_read if pricing.cache_read is not None else pricing.input
+
+    return (
+        uncached_input * pricing.input
+        + cached * cache_rate
+        + output_total * pricing.output
+    )

oddish/src/oddish/schemas.py

@@ -420,7 +420,23 @@ class TrialResponse(BaseModel):
     )
     cache_tokens: int | None = Field(None, description="Cache tokens used")
     output_tokens: int | None = Field(None, description="Output tokens generated")
-    cost_usd: float | None = Field(None, description="Estimated cost in USD")
+    cost_usd: float | None = Field(
+        None,
+        description=(
+            "Trial cost in USD. Native value from the agent runtime when "
+            "available; otherwise estimated from token counts and a static "
+            "model pricing table (see ``cost_is_estimated``)."
+        ),
+    )
+    cost_is_estimated: bool | None = Field(
+        None,
+        description=(
+            "True when ``cost_usd`` was derived from the static model "
+            "pricing table because the agent runtime did not report a "
+            "native cost. False when the cost came directly from the "
+            "runtime. Null when no cost is available."
+        ),
+    )
 
     # Per-phase timing breakdown
     phase_timing: dict | None = Field(