|
| 1 | +"""Public (unauthenticated) analysis reads for shared experiments. |
| 2 | +
|
| 3 | +These live here rather than beside their siblings in |
| 4 | +``oddish/core/sharing/public.py`` because they read the hosted analysis |
| 5 | +services (``api.services.*``), and the ``oddish`` package may not import the |
| 6 | +backend. The share-token resolvers go the other way, which is allowed. |
| 7 | +
|
| 8 | +Both routes are cache reads. Their authenticated counterparts generate on a |
| 9 | +miss -- a Claude call per trajectory summary, a claude-code run per comparison |
| 10 | +-- and neither may be reachable without a login: the spend is unbounded by |
| 11 | +anything the caller has to hold, and the comparison additionally parks one of |
| 12 | +an API container's three connections for minutes. A miss here is a 404. |
| 13 | +""" |
| 14 | + |
| 15 | +from __future__ import annotations |
| 16 | + |
| 17 | +import logging |
| 18 | + |
| 19 | +from fastapi import APIRouter, HTTPException, Query |
| 20 | + |
| 21 | +from api.services.blocks.analyzer.cohort.cohort_prompts import short_model_name |
| 22 | +from oddish.core.model_display_names import ( |
| 23 | + display_model_name, |
| 24 | + load_model_display_names, |
| 25 | +) |
| 26 | +from oddish.core.sharing.helpers import ( |
| 27 | + get_public_task_for_experiment, |
| 28 | + get_public_trial_for_experiment, |
| 29 | +) |
| 30 | +from oddish.db import get_session |
| 31 | + |
| 32 | +logger = logging.getLogger(__name__) |
| 33 | + |
| 34 | +router = APIRouter(tags=["Public"]) |
| 35 | + |
| 36 | + |
| 37 | +@router.get("/public/experiments/{public_token}/trials/{trial_id}/trajectory/summary") |
| 38 | +async def get_public_trial_trajectory_summary( |
| 39 | + public_token: str, trial_id: str |
| 40 | +) -> dict: |
| 41 | + """The stored trajectory summary for a public trial.""" |
| 42 | + from api.services.summarize_trajectory import load_stored_summary |
| 43 | + |
| 44 | + async with get_session() as session: |
| 45 | + trial = await get_public_trial_for_experiment(session, public_token, trial_id) |
| 46 | + if trial is None: |
| 47 | + raise HTTPException(status_code=404, detail="Trial not found") |
| 48 | + summary = await load_stored_summary(session, trial) |
| 49 | + if summary is None: |
| 50 | + raise HTTPException( |
| 51 | + status_code=404, detail="No trajectory summary for this trial" |
| 52 | + ) |
| 53 | + return summary |
| 54 | + |
| 55 | + |
| 56 | +def _short_name_aliases(names: dict[str, str]) -> dict[str, str]: |
| 57 | + """Index the alias table by the spelling the comparison actually stores. |
| 58 | +
|
| 59 | + ``load_model_display_names`` keys on the full id (``anthropic/claude-opus-4-8``) |
| 60 | + because that is what ``trials.model`` holds. The comparison does not: both |
| 61 | + ``models[].model`` and ``trial_models`` are written through |
| 62 | + ``short_model_name``, which strips the provider and region |
| 63 | + (``global.anthropic.claude-opus-4-8`` -> ``claude-opus-4-8``). Masking on the |
| 64 | + full id alone therefore matches nothing and publishes the real names. |
| 65 | +
|
| 66 | + A short name can collide -- ``global.anthropic.…`` and ``us.anthropic.…`` |
| 67 | + reduce to one string. When two aliases disagree on a collision the key is |
| 68 | + dropped rather than guessed: naming the wrong model misattributes the |
| 69 | + behaviour the analysis describes, which is worse than leaving the short |
| 70 | + name showing. |
| 71 | + """ |
| 72 | + short: dict[str, str] = {} |
| 73 | + dropped: set[str] = set() |
| 74 | + for key in sorted(names): |
| 75 | + alias = names[key] |
| 76 | + name = short_model_name(key) |
| 77 | + if not name or name == key: |
| 78 | + continue |
| 79 | + if short.setdefault(name, alias) != alias: |
| 80 | + dropped.add(name) |
| 81 | + for name in dropped: |
| 82 | + short.pop(name, None) |
| 83 | + logger.warning( |
| 84 | + "model display names disagree for short name %r; leaving it unmasked", |
| 85 | + name, |
| 86 | + ) |
| 87 | + return {**short, **names} |
| 88 | + |
| 89 | + |
| 90 | +def _mask_models(comparison: dict, names: dict[str, str]) -> dict: |
| 91 | + """Rewrite the comparison's model ids through the operator alias table. |
| 92 | +
|
| 93 | + Mutating a copy, not the argument: the dict is an ``AnalyzerBlock`` row's |
| 94 | + ``output``, and the session it came from is still open. |
| 95 | + """ |
| 96 | + if not names: |
| 97 | + return comparison |
| 98 | + names = _short_name_aliases(names) |
| 99 | + masked = dict(comparison) |
| 100 | + models = masked.get("models") |
| 101 | + if isinstance(models, dict): |
| 102 | + masked["models"] = { |
| 103 | + side: [ |
| 104 | + {**entry, "model": display_model_name(entry.get("model"), names)} |
| 105 | + if isinstance(entry, dict) |
| 106 | + else entry |
| 107 | + for entry in entries |
| 108 | + ] |
| 109 | + if isinstance(entries, list) |
| 110 | + else entries |
| 111 | + for side, entries in models.items() |
| 112 | + } |
| 113 | + trial_models = masked.get("trial_models") |
| 114 | + if isinstance(trial_models, dict): |
| 115 | + masked["trial_models"] = { |
| 116 | + trial_id: display_model_name(model, names) |
| 117 | + for trial_id, model in trial_models.items() |
| 118 | + } |
| 119 | + return masked |
| 120 | + |
| 121 | + |
| 122 | +async def _version_is_in_experiment( |
| 123 | + session, experiment_id: str, task_version_id: str |
| 124 | +) -> bool: |
| 125 | + """Whether the shared experiment has a trial on this task version. |
| 126 | +
|
| 127 | + Membership goes through ``trial_in_experiment`` -- a collection gathers |
| 128 | + trials that keep the ``experiment_id`` of wherever they ran, so an FK-only |
| 129 | + filter misses them. |
| 130 | + """ |
| 131 | + from sqlalchemy import select |
| 132 | + |
| 133 | + from oddish.core.experiment_membership import trial_in_experiment |
| 134 | + from oddish.db.models import TrialModel |
| 135 | + |
| 136 | + return ( |
| 137 | + await session.execute( |
| 138 | + select(TrialModel.id) |
| 139 | + .where( |
| 140 | + TrialModel.task_version_id == task_version_id, |
| 141 | + TrialModel.is_probe.is_(False), |
| 142 | + trial_in_experiment(experiment_id), |
| 143 | + ) |
| 144 | + .limit(1) |
| 145 | + ) |
| 146 | + ).scalar_one_or_none() is not None |
| 147 | + |
| 148 | + |
| 149 | +@router.get("/public/experiments/{public_token}/tasks/{task_id}/agent-capabilities") |
| 150 | +async def get_public_task_agent_capabilities( |
| 151 | + public_token: str, |
| 152 | + task_id: str, |
| 153 | + version: int | None = Query( |
| 154 | + None, |
| 155 | + description=( |
| 156 | + "Compare this task version instead of the current one. A share " |
| 157 | + "page pins the version its trials ran on, so without this an " |
| 158 | + "older version would show the current version's comparison." |
| 159 | + ), |
| 160 | + ), |
| 161 | +) -> dict: |
| 162 | + """The stored successful-vs-failing comparison for a public task version.""" |
| 163 | + from api.services.agent_capabilities import load_stored_analysis |
| 164 | + |
| 165 | + async with get_session() as session: |
| 166 | + resolved = await get_public_task_for_experiment(session, public_token, task_id) |
| 167 | + if resolved is None: |
| 168 | + raise HTTPException(status_code=404, detail="Task not found") |
| 169 | + experiment, task, _ = resolved |
| 170 | + if not task.current_version_id: |
| 171 | + raise HTTPException(status_code=404, detail="Task not found") |
| 172 | + version_id = task.current_version_id |
| 173 | + if version is not None: |
| 174 | + from oddish.db.models import TaskVersionModel |
| 175 | + from sqlalchemy import select |
| 176 | + |
| 177 | + version_id = ( |
| 178 | + await session.execute( |
| 179 | + select(TaskVersionModel.id).where( |
| 180 | + TaskVersionModel.task_id == task.id, |
| 181 | + TaskVersionModel.version == version, |
| 182 | + ) |
| 183 | + ) |
| 184 | + ).scalar_one_or_none() |
| 185 | + if version_id is None: |
| 186 | + raise HTTPException(status_code=404, detail="Task version not found") |
| 187 | + # The token publishes an experiment, not a task's whole history. Without |
| 188 | + # this, `?version=` walks every version the task ever had -- including |
| 189 | + # ones this experiment never ran, whose trial ids, models and trajectory |
| 190 | + # quotes the share was never meant to carry. Bound it to the versions |
| 191 | + # the share actually displays. |
| 192 | + if not await _version_is_in_experiment(session, experiment.id, version_id): |
| 193 | + raise HTTPException(status_code=404, detail="Task version not found") |
| 194 | + comparison = await load_stored_analysis( |
| 195 | + session, version_id, task_id=task.id |
| 196 | + ) |
| 197 | + if comparison is None: |
| 198 | + raise HTTPException( |
| 199 | + status_code=404, detail="No comparison stored for this version" |
| 200 | + ) |
| 201 | + names = await load_model_display_names(session) |
| 202 | + # Stamped at serve time, matching the authenticated route: the id is what |
| 203 | + # the UI addresses a version by, while this route takes the number. |
| 204 | + return {**_mask_models(comparison, names), "task_version_id": version_id} |
0 commit comments