promote: staging -> main (0012be9d)

backend/api/services/blocks/analyzer/cohort/cohort_comparison_block.py

@@ -9,13 +9,28 @@
 from api.services.blocks.analyzer.cohort import cohort_prompts as cp
 from api.services.blocks.analyzer.cohort.cohort_taxonomy import BehaviorCategory
 
-SCHEMA_VERSION = 1
+# 2: added `summary`. 3: added `mode` and `models`, and relaxed the gate to one
+# populated cohort -- stored rows carry neither field and were generated under
+# the two-cohort framing, so they have to regenerate to gain either.
+SCHEMA_VERSION = 3
 
 # A trial whose summary covers less than this share of its own step span is
 # reported to the reader rather than averaged over silently.
 MIN_COVERAGE = 0.5
 
 
+def _model_counts(trials: list[dict]) -> list[dict]:
+    """[{model, trials}], most frequent first, then alphabetical for stability."""
+    counts: dict[str, int] = {}
+    for t in trials:
+        name = cp.short_model_name(t.get("model") or "unknown")
+        counts[name] = counts.get(name, 0) + 1
+    return [
+        {"model": m, "trials": n}
+        for m, n in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
+    ]
+
+
 def _non_empty_text(value: str) -> str:
     value = value.strip()
     if not value:
@@ -100,6 +115,13 @@ class CohortComparisonOutput(BaseModel):
     cohort_success: list[str]
     cohort_failure: list[str]
     categories: list[CategoryComparison]
+    # LAST, and the order is load-bearing. This schema is handed to the model
+    # as `response_format` / `output_schema`, and constrained decoding emits
+    # fields in schema order -- so a `summary` declared above `categories`
+    # would be generated before the rows it is supposed to be bound by,
+    # exactly inverting the prompt's "write summary last" rule and inviting a
+    # headline the categories do not support.
+    summary: NonEmptyText
 
 
 class CohortInput(BaseModel):
@@ -138,7 +160,9 @@ def sections(self) -> list[dict]:
                 "name": "preamble",
                 "raw_input": {},
                 "schema": _Empty,
-                "formatter": lambda _d: cp.PREAMBLE,
+                "formatter": lambda _d: cp.preamble(
+                    successful=ci.successful, failing=ci.failing
+                ),
             },
             {
                 "name": "task",
@@ -190,13 +214,45 @@ def to_output(self, raw: str) -> dict:
             parsed.model_dump(mode="json"), ci.successful, ci.failing
         )
         out["dropped"] = dropped
+        # The headline was written against the categories the model produced,
+        # and validation runs after it. If a whole category failed citation
+        # checks and was removed, the summary can name a split the panel no
+        # longer shows -- an unsourced claim sitting above sourced rows, which
+        # is the one thing this feature is built not to do. Drop it rather
+        # than let it describe a comparison that is no longer on screen.
+        # Observation-level drops leave the category standing, so the theme
+        # still holds and the summary survives them.
+        if dropped.get("categories"):
+            out.pop("summary", None)
         # Cohort membership is a fact we already hold, not something to take
         # from the model. The UI renders these lengths as "N successful, M
         # failing"; leaving the model's lists in place would let a fabricated
         # or truncated array misreport how much evidence the comparison rests
         # on -- the same class of problem the citation check exists to stop.
         out["cohort_success"] = [t["trial_id"] for t in ci.successful]
         out["cohort_failure"] = [t["trial_id"] for t in ci.failing]
+        # Which models landed on which side, counted here rather than asked of
+        # the model. "opus-4-8 succeeded 6 times" is arithmetic over rows we
+        # already hold; routing it through an LLM would make a checkable fact
+        # into an unverifiable claim, and validate_evidence has no way to
+        # police a count.
+        out["models"] = {
+            "successful": _model_counts(ci.successful),
+            "failing": _model_counts(ci.failing),
+        }
+        # trial -> model, so a citation can name the model it came from. The
+        # chips say which models ran on a side; without this the reader cannot
+        # tell which of them the cited behaviour belongs to, and a side listing
+        # fourteen models next to one citation reads as if all fourteen did it.
+        out["trial_models"] = {
+            t["trial_id"]: cp.short_model_name(t.get("model") or "unknown")
+            for t in (*ci.successful, *ci.failing)
+        }
+        # Single-cohort runs describe rather than compare, and the UI needs to
+        # know which without re-deriving it from two list lengths.
+        out["mode"] = (
+            "comparison" if ci.successful and ci.failing else "single"
+        )
         # Surfaced in the UI so a reader can see when the comparison rests on
         # thin evidence, rather than the feature averaging over it silently.
         out["thin_coverage"] = [

backend/api/services/blocks/analyzer/cohort/cohort_prompts.py

@@ -14,13 +14,74 @@
     BehaviorCategory,
 )
 
+# Vendor tokens that appear as a routing prefix on a stored model id.
+# Stripping is a whitelist, not a split on ".": `gpt-5.4` and
+# `claude-opus-4-8` carry dots of their own, and a generic split would render
+# them as "4" and "8".
+_VENDOR_PREFIXES = (
+    "anthropic",
+    "openai",
+    "google",
+    "meta",
+    "mistral",
+    "amazon",
+    "cohere",
+)
+
+
+# Cross-region inference profile prefixes on stored Bedrock ids. Not just
+# "global.": Opus 4.1 and Opus 4 have no global profile and are stored as
+# "us.anthropic...". Mirrors oddish.config._BEDROCK_REGION_PREFIXES.
+_REGION_PREFIXES = ("global.", "us.", "eu.", "apac.", "apn.")
+
+
+def short_model_name(raw: str) -> str:
+    """``global.anthropic.claude-opus-4-8`` -> ``claude-opus-4-8``.
+
+    Lives here rather than in the block because BOTH readers need it: the
+    prompt, which shows the model on each trial, and the chips built from
+    ``_model_counts``. Two spellings of one id read as two models.
+    """
+    name = raw.split("/")[-1]
+    for prefix in _REGION_PREFIXES:
+        if name.startswith(prefix):
+            name = name[len(prefix) :]
+            break
+    head, _, rest = name.partition(".")
+    if rest and head in _VENDOR_PREFIXES:
+        name = rest
+    return name
+
+
 PREAMBLE = (
     "You are comparing two cohorts of recorded agent runs on the same task: "
     "runs that succeeded for good reasons, and runs that failed for good "
     "reasons. A developer wants to know what the successful runs did "
     "differently."
 )
 
+# A task whose runs all failed (or all succeeded) has one cohort, and it is
+# often the most interesting case there is. Say so plainly rather than leaving
+# the model to infer it from an empty list -- an empty <cohort> block with a
+# "compare the two" preamble above it invites inventing the missing side.
+SINGLE_PREAMBLE = (
+    "You are describing ONE cohort of recorded agent runs on the same task: "
+    "{label} runs. There is no second cohort -- every classified run on this "
+    "task version landed on this side. A developer wants to know what these "
+    "runs did. Do NOT speculate about how a run on the other side would have "
+    "behaved, and do not describe the absent side at all: put every "
+    "observation in `{field}` and leave the other list empty."
+)
+
+
+def preamble(*, successful: list[dict], failing: list[dict]) -> str:
+    """Which framing the run gets, decided by which cohorts actually exist."""
+    if successful and failing:
+        return PREAMBLE
+    if successful:
+        return SINGLE_PREAMBLE.format(label="successful", field="successful")
+    return SINGLE_PREAMBLE.format(label="failing", field="failing")
+
 
 def taxonomy_section() -> str:
     """The categories WITH definitions.
@@ -43,7 +104,12 @@ def cohort_section(label: str, trials: list[dict]) -> str:
     """One cohort's trials, as component streams the model can cite."""
     lines = [f"<cohort name=\"{label}\">"]
     for t in trials:
-        lines.append(f'  <trial id="{t["trial_id"]}">')
+        # Same shortener the chips use. Two spellings of one model id --
+        # `global.anthropic.claude-opus-4-8` in the prose the model writes,
+        # `claude-opus-4-8` on the chip beside it -- read as two models.
+        model = short_model_name(t.get("model") or "") if t.get("model") else ""
+        attrs = f' model="{model}"' if model else ""
+        lines.append(f'  <trial id="{t["trial_id"]}"{attrs}>')
         for c in t.get("components") or []:
             ids = c.get("step_ids") or []
             if not ids:

backend/api/services/cohort_comparison.py

@@ -67,6 +67,11 @@ async def resolve_cohorts(
                     TrialModel.id,
                     TrialModel.total_steps,
                     TrialModel.trajectory_summary,
+                    # Which model produced the run. Attribution is computed
+                    # from these in code, never asked of the LLM: "which model
+                    # did better" is a counting question, and a fabricated
+                    # answer to it would be indistinguishable from a real one.
+                    TrialModel.model,
                 ).where(
                     TrialModel.task_version_id == task_version_id,
                     TrialModel.is_probe.is_(False),
@@ -81,6 +86,7 @@ async def resolve_cohorts(
         ).all()
         ids = [r[0] for r in rows]
         total_steps = {r[0]: r[1] for r in rows}
+        models = {r[0]: r[3] for r in rows}
         # Prefer the mirror on the trial row. summarize_trajectory writes every
         # summary to trials.trajectory_summary as well as analyzer_blocks, and
         # the mirror is what the sibling QA surfaces read -- post-trial reads
@@ -116,6 +122,7 @@ async def resolve_cohorts(
             out[cls].append(
                 {
                     "trial_id": tid,
+                    "model": models.get(tid),
                     "components": comps,
                     "covered_steps": len(all_ids),
                     "span": span,
@@ -340,7 +347,11 @@ async def get_or_generate_comparison(
     )
 
     successful, failing = await resolve_cohorts(session, task_version_id)
-    if len(successful) < MIN_COHORT or len(failing) < MIN_COHORT:
+    # One populated side is enough. Requiring both meant a task whose runs all
+    # failed -- the case a reader most wants explained -- got silence, even
+    # with ten classified failures on the table. What a cohort did is worth
+    # reporting on its own; what it did *differently* just needs two.
+    if max(len(successful), len(failing)) < MIN_COHORT:
         return None
 
     current = cohort_hash(

backend/tests/test_cohort_comparison_block.py

@@ -5,6 +5,7 @@
 from oddish.blocks.block import BlockParseError
 
 from api.services.blocks.analyzer.cohort.cohort_comparison_block import (
+    SCHEMA_VERSION,
     CohortComparisonBlock,
     CohortInput,
 )
@@ -42,6 +43,7 @@ def _raw(evidence, schema_version=99):
             "schema_version": schema_version,
             "cohort_success": ["t1"],
             "cohort_failure": ["t2"],
+            "summary": "Agents took a test baseline before editing.",
             "categories": [
                 {
                     "category": "testing_verification",
@@ -77,10 +79,82 @@ def test_prompt_contains_both_cohorts_and_definitions():
 def test_to_output_parses_and_stamps_schema_version():
     out = _block().to_output(_raw([GOOD_EVIDENCE]))
     # The block owns schema_version; a model-supplied value is overwritten.
-    assert out["schema_version"] == 1
+    # Asserted against the constant: pinning the literal made a deliberate
+    # version bump look like a regression.
+    assert out["schema_version"] == SCHEMA_VERSION
+    assert SCHEMA_VERSION != 99
     assert out["categories"][0]["category"] == "testing_verification"
 
 
+def test_short_model_name_keeps_dots_that_belong_to_the_name():
+    """A generic split on "." would turn gpt-5.4 into "4"."""
+    from api.services.blocks.analyzer.cohort.cohort_prompts import (
+        short_model_name,
+    )
+
+    assert short_model_name("global.anthropic.claude-opus-4-8") == "claude-opus-4-8"
+    assert short_model_name("anthropic/claude-fable-5") == "claude-fable-5"
+    assert short_model_name("gpt-5.4") == "gpt-5.4"
+    assert short_model_name("gemini-3.5-flash") == "gemini-3.5-flash"
+
+
+def test_short_model_name_strips_every_region_prefix():
+    """Opus 4.1 / Opus 4 have no "global." inference profile: they are stored
+    as "us.anthropic...", and a global-only strip left them long."""
+    from api.services.blocks.analyzer.cohort.cohort_prompts import (
+        short_model_name,
+    )
+
+    assert (
+        short_model_name("us.anthropic.claude-opus-4-1-20250805-v1:0")
+        == "claude-opus-4-1-20250805-v1:0"
+    )
+    assert short_model_name("eu.anthropic.claude-sonnet-4-5") == "claude-sonnet-4-5"
+    assert short_model_name("apac.anthropic.claude-haiku-4-5") == "claude-haiku-4-5"
+    assert short_model_name("bedrock/apn.amazon.nova-pro-v1:0") == "nova-pro-v1:0"
+
+
+def test_model_counts_are_ordered_and_stripped():
+    out = _block(
+        successful=[
+            {**TRIAL, "trial_id": "t1", "model": "global.anthropic.claude-opus-4-8"},
+            {**TRIAL, "trial_id": "t3", "model": "global.anthropic.claude-opus-4-8"},
+            {**TRIAL, "trial_id": "t4", "model": "gemini-3.5-flash"},
+        ],
+    ).to_output(_raw([GOOD_EVIDENCE]))
+    assert out["models"]["successful"] == [
+        {"model": "claude-opus-4-8", "trials": 2},
+        {"model": "gemini-3.5-flash", "trials": 1},
+    ]
+
+
+def test_mode_is_single_when_one_cohort_is_empty():
+    """All-failed and all-succeeded tasks are the cases a reader most wants
+    explained; the payload has to say which so the UI drops a column rather
+    than drawing an empty one."""
+    out = _block(failing=[]).to_output(_raw([GOOD_EVIDENCE]))
+    assert out["mode"] == "single"
+    assert out["models"]["failing"] == []
+    assert _block().to_output(_raw([GOOD_EVIDENCE]))["mode"] == "comparison"
+
+
+def test_summary_survives_a_clean_comparison():
+    out = _block().to_output(_raw([GOOD_EVIDENCE]))
+    assert out["dropped"]["categories"] == 0
+    assert out["summary"]
+
+
+def test_summary_is_dropped_when_a_category_is():
+    """A headline written against categories that validation then removed is
+    an unsourced claim above sourced rows -- exactly what the citation check
+    exists to prevent, so it must not outlive them."""
+    fabricated = {**GOOD_EVIDENCE, "trial_id": "does-not-exist"}
+    out = _block().to_output(_raw([fabricated]))
+    assert out["dropped"]["categories"] == 1
+    assert out["categories"] == []
+    assert "summary" not in out
+
+
 def test_to_output_validates_citations_before_the_block_persists():
     """Validation must happen in the transform, not after block.run().
 

backend/tests/test_cohort_comparison_schema.py

@@ -78,6 +78,7 @@ def test_full_output_parses():
         schema_version=1,
         cohort_success=["t1"],
         cohort_failure=["t2"],
+        summary="Agents took a test baseline before editing.",
         categories=[
             CategoryComparison(
                 category=BehaviorCategory.TESTING_VERIFICATION,