promote: staging -> main (7c8b8c10)

backend/api/app.py

@@ -258,6 +258,7 @@ def create_app() -> FastAPI:
         reports,
         skills,
         public,
+        public_analysis,
         slack,
         tags,
         tasks,
@@ -283,6 +284,7 @@ def create_app() -> FastAPI:
     api.include_router(skills.router)
     api.include_router(documents.router)
     api.include_router(public.router)
+    api.include_router(public_analysis.router)
     api.include_router(slack.router)
     api.include_router(admin.router)
     api.include_router(cost_excluded_keys.router)

backend/api/routers/__init__.py

@@ -7,6 +7,7 @@
     dashboard,
     github_webhooks,
     orgs,
+    public_analysis,
     slack,
     tasks,
     trials,
@@ -20,6 +21,7 @@
     "github_webhooks",
     "orgs",
     "public",
+    "public_analysis",
     "slack",
     "tasks",
     "trials",

backend/api/routers/public_analysis.py

@@ -0,0 +1,204 @@
+"""Public (unauthenticated) analysis reads for shared experiments.
+
+These live here rather than beside their siblings in
+``oddish/core/sharing/public.py`` because they read the hosted analysis
+services (``api.services.*``), and the ``oddish`` package may not import the
+backend. The share-token resolvers go the other way, which is allowed.
+
+Both routes are cache reads. Their authenticated counterparts generate on a
+miss -- a Claude call per trajectory summary, a claude-code run per comparison
+-- and neither may be reachable without a login: the spend is unbounded by
+anything the caller has to hold, and the comparison additionally parks one of
+an API container's three connections for minutes. A miss here is a 404.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, HTTPException, Query
+
+from api.services.blocks.analyzer.cohort.cohort_prompts import short_model_name
+from oddish.core.model_display_names import (
+    display_model_name,
+    load_model_display_names,
+)
+from oddish.core.sharing.helpers import (
+    get_public_task_for_experiment,
+    get_public_trial_for_experiment,
+)
+from oddish.db import get_session
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Public"])
+
+
+@router.get("/public/experiments/{public_token}/trials/{trial_id}/trajectory/summary")
+async def get_public_trial_trajectory_summary(
+    public_token: str, trial_id: str
+) -> dict:
+    """The stored trajectory summary for a public trial."""
+    from api.services.summarize_trajectory import load_stored_summary
+
+    async with get_session() as session:
+        trial = await get_public_trial_for_experiment(session, public_token, trial_id)
+        if trial is None:
+            raise HTTPException(status_code=404, detail="Trial not found")
+        summary = await load_stored_summary(session, trial)
+    if summary is None:
+        raise HTTPException(
+            status_code=404, detail="No trajectory summary for this trial"
+        )
+    return summary
+
+
+def _short_name_aliases(names: dict[str, str]) -> dict[str, str]:
+    """Index the alias table by the spelling the comparison actually stores.
+
+    ``load_model_display_names`` keys on the full id (``anthropic/claude-opus-4-8``)
+    because that is what ``trials.model`` holds. The comparison does not: both
+    ``models[].model`` and ``trial_models`` are written through
+    ``short_model_name``, which strips the provider and region
+    (``global.anthropic.claude-opus-4-8`` -> ``claude-opus-4-8``). Masking on the
+    full id alone therefore matches nothing and publishes the real names.
+
+    A short name can collide -- ``global.anthropic.…`` and ``us.anthropic.…``
+    reduce to one string. When two aliases disagree on a collision the key is
+    dropped rather than guessed: naming the wrong model misattributes the
+    behaviour the analysis describes, which is worse than leaving the short
+    name showing.
+    """
+    short: dict[str, str] = {}
+    dropped: set[str] = set()
+    for key in sorted(names):
+        alias = names[key]
+        name = short_model_name(key)
+        if not name or name == key:
+            continue
+        if short.setdefault(name, alias) != alias:
+            dropped.add(name)
+    for name in dropped:
+        short.pop(name, None)
+        logger.warning(
+            "model display names disagree for short name %r; leaving it unmasked",
+            name,
+        )
+    return {**short, **names}
+
+
+def _mask_models(comparison: dict, names: dict[str, str]) -> dict:
+    """Rewrite the comparison's model ids through the operator alias table.
+
+    Mutating a copy, not the argument: the dict is an ``AnalyzerBlock`` row's
+    ``output``, and the session it came from is still open.
+    """
+    if not names:
+        return comparison
+    names = _short_name_aliases(names)
+    masked = dict(comparison)
+    models = masked.get("models")
+    if isinstance(models, dict):
+        masked["models"] = {
+            side: [
+                {**entry, "model": display_model_name(entry.get("model"), names)}
+                if isinstance(entry, dict)
+                else entry
+                for entry in entries
+            ]
+            if isinstance(entries, list)
+            else entries
+            for side, entries in models.items()
+        }
+    trial_models = masked.get("trial_models")
+    if isinstance(trial_models, dict):
+        masked["trial_models"] = {
+            trial_id: display_model_name(model, names)
+            for trial_id, model in trial_models.items()
+        }
+    return masked
+
+
+async def _version_is_in_experiment(
+    session, experiment_id: str, task_version_id: str
+) -> bool:
+    """Whether the shared experiment has a trial on this task version.
+
+    Membership goes through ``trial_in_experiment`` -- a collection gathers
+    trials that keep the ``experiment_id`` of wherever they ran, so an FK-only
+    filter misses them.
+    """
+    from sqlalchemy import select
+
+    from oddish.core.experiment_membership import trial_in_experiment
+    from oddish.db.models import TrialModel
+
+    return (
+        await session.execute(
+            select(TrialModel.id)
+            .where(
+                TrialModel.task_version_id == task_version_id,
+                TrialModel.is_probe.is_(False),
+                trial_in_experiment(experiment_id),
+            )
+            .limit(1)
+        )
+    ).scalar_one_or_none() is not None
+
+
+@router.get("/public/experiments/{public_token}/tasks/{task_id}/agent-capabilities")
+async def get_public_task_agent_capabilities(
+    public_token: str,
+    task_id: str,
+    version: int | None = Query(
+        None,
+        description=(
+            "Compare this task version instead of the current one. A share "
+            "page pins the version its trials ran on, so without this an "
+            "older version would show the current version's comparison."
+        ),
+    ),
+) -> dict:
+    """The stored successful-vs-failing comparison for a public task version."""
+    from api.services.agent_capabilities import load_stored_analysis
+
+    async with get_session() as session:
+        resolved = await get_public_task_for_experiment(session, public_token, task_id)
+        if resolved is None:
+            raise HTTPException(status_code=404, detail="Task not found")
+        experiment, task, _ = resolved
+        if not task.current_version_id:
+            raise HTTPException(status_code=404, detail="Task not found")
+        version_id = task.current_version_id
+        if version is not None:
+            from oddish.db.models import TaskVersionModel
+            from sqlalchemy import select
+
+            version_id = (
+                await session.execute(
+                    select(TaskVersionModel.id).where(
+                        TaskVersionModel.task_id == task.id,
+                        TaskVersionModel.version == version,
+                    )
+                )
+            ).scalar_one_or_none()
+            if version_id is None:
+                raise HTTPException(status_code=404, detail="Task version not found")
+        # The token publishes an experiment, not a task's whole history. Without
+        # this, `?version=` walks every version the task ever had -- including
+        # ones this experiment never ran, whose trial ids, models and trajectory
+        # quotes the share was never meant to carry. Bound it to the versions
+        # the share actually displays.
+        if not await _version_is_in_experiment(session, experiment.id, version_id):
+            raise HTTPException(status_code=404, detail="Task version not found")
+        comparison = await load_stored_analysis(
+            session, version_id, task_id=task.id
+        )
+        if comparison is None:
+            raise HTTPException(
+                status_code=404, detail="No comparison stored for this version"
+            )
+        names = await load_model_display_names(session)
+    # Stamped at serve time, matching the authenticated route: the id is what
+    # the UI addresses a version by, while this route takes the number.
+    return {**_mask_models(comparison, names), "task_version_id": version_id}

backend/api/routers/tasks.py

@@ -98,7 +98,7 @@
     stamp_experiment_owner,
 )
 from dashboard_attribution import resolve_search_authors
-from api.services.cohort_comparison import get_or_generate_comparison
+from api.services.agent_capabilities import get_or_generate_analysis
 from oddish.core.tasks import (
     complete_task_upload,
     initialize_task_upload,
@@ -1605,14 +1605,18 @@ async def get_task_detail(
         return await get_task_detail_core(session, task_id=task_id, org_id=auth.org_id)
 
 
-@router.get("/tasks/{task_id}/cohort-comparison")
-async def get_task_cohort_comparison(
+@router.get("/tasks/{task_id}/agent-capabilities")
+# Pre-rename path. Kept so a frontend deploy that lags this one -- or a
+# rollback to it -- keeps working; undocumented so only the new path is
+# published. Remove once no released frontend calls it.
+@router.get("/tasks/{task_id}/cohort-comparison", include_in_schema=False)
+async def get_task_agent_capabilities(
     task_id: str,
     auth: Annotated[AuthContext, Depends(require_auth)],
     refresh: bool = Query(
         False,
         description=(
-            "Discard the stored comparison and generate a new one. Costs an "
+            "Discard the stored analysis and generate a new one. Costs an "
             "LLM call, so it needs the same scope as an analysis rerun."
         ),
     ),
@@ -1661,7 +1665,7 @@ async def get_task_cohort_comparison(
             ).scalar_one_or_none()
             if version_id is None:
                 raise HTTPException(status_code=404, detail="Task version not found")
-        result = await get_or_generate_comparison(
+        result = await get_or_generate_analysis(
             session,
             version_id,
             task_id=task.id,