test(verdict): golden-set regression harness for verdict prompt changes

oddish/tests/test_verdict_golden_set.py

@@ -0,0 +1,49 @@
+"""Offline golden-set layer: every scenario in ``verdict_golden_set`` must
+render a prompt and round-trip a synthetic judge reply through VerdictBlock's
+parse path. This guards the two things a prompt/schema rewrite (like #1120's
+``is_good`` -> ``verdict``) can silently break in CI -- rendering and parsing.
+Whether the real model still returns the expected verdicts is the live
+layer's job (``test_verdict_golden_set_live.py``)."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from oddish.analyze.models import TaskVerdictModel
+from verdict_golden_set import GOLDEN_CASES, build_block
+
+_CASE_IDS = [c.name for c in GOLDEN_CASES]
+
+
+@pytest.mark.parametrize("case", GOLDEN_CASES, ids=_CASE_IDS)
+def test_prompt_renders_the_scenario(case):
+    prompt = build_block(case).build_prompt()
+
+    # build_prompt() raising (not the sentinel) is the degraded path; getting
+    # here means it rendered. Now pin that the scenario's substance made it in.
+    assert f"One benchmark task ran {len(case.classifications)} times" in prompt
+    for trial in case.classifications:
+        assert trial.trial_name in prompt
+        assert trial.evidence in prompt
+    for marker in case.expects_in_prompt:
+        assert marker in prompt
+
+
+@pytest.mark.parametrize("case", GOLDEN_CASES, ids=_CASE_IDS)
+def test_synthetic_reply_roundtrips_the_parse_path(case):
+    out = build_block(case).to_verdict(json.dumps(case.response()))
+
+    assert out["verdict"] == case.expected_verdict
+    assert out["confidence"] == case.expected_confidence
+    model = TaskVerdictModel(**out)
+    assert model.is_good is (case.expected_verdict == "accept")
+
+
+def test_golden_set_stays_representative():
+    """The set must keep exercising both verdicts and all three confidence
+    levels, or the live backtest degrades into a one-sided check."""
+    assert {c.expected_verdict for c in GOLDEN_CASES} == {"accept", "reject"}
+    assert {c.expected_confidence for c in GOLDEN_CASES} == {"high", "medium", "low"}
+    assert len({c.name for c in GOLDEN_CASES}) == len(GOLDEN_CASES)

oddish/tests/test_verdict_golden_set_live.py

@@ -0,0 +1,94 @@
+"""Opt-in live backtest: the REAL verdict model judges the golden set.
+
+Skipped unless the credentials for ``settings.verdict_model``'s configured
+OpenAI provider resolve (the same check ``_build_openai_client`` performs),
+so normal CI never touches the network. Run it with output visible:
+
+    uv run pytest tests/test_verdict_golden_set_live.py -s
+
+To compare two prompt versions, point ``ODDISH_VERDICT_PROMPT_FILE`` at an
+alternative prompt file and re-run; the run patches the loaded prompt text
+(``classifier._VERDICT_PROMPT``), which is what ``build_verdict_prompt``
+formats, so no production code changes are needed.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from unittest.mock import AsyncMock
+
+import pytest
+
+from oddish.analyze import classifier
+from oddish.analyze.classifier import VERDICT_TIMEOUT
+from oddish.config import OPENAI_PROVIDER_OPENAI, settings
+from verdict_golden_set import GOLDEN_CASES
+
+
+def _verdict_credentials_resolve() -> bool:
+    try:
+        if settings.get_openai_provider() == OPENAI_PROVIDER_OPENAI:
+            settings.require_public_openai_config()
+        else:
+            settings.require_azure_openai_config()
+    except Exception:
+        return False
+    return True
+
+
+requires_verdict_model = pytest.mark.skipif(
+    not _verdict_credentials_resolve(),
+    reason="no credentials for the configured OpenAI provider; "
+    "the live verdict backtest is opt-in",
+)
+
+
+@requires_verdict_model
+@pytest.mark.asyncio
+async def test_live_backtest_over_golden_set(monkeypatch):
+    from oddish.blocks.analyzer.analyzer_block import AnalyzerBlock
+    from oddish.workers.queue.qa_handler import synthesize_task_verdict
+
+    # The backtest measures the model, not persistence: no S3/Postgres needed.
+    monkeypatch.setattr(AnalyzerBlock, "save_to_s3", AsyncMock())
+    monkeypatch.setattr(AnalyzerBlock, "save_to_db", AsyncMock())
+    monkeypatch.setattr(AnalyzerBlock, "record_cost", AsyncMock())
+
+    override = os.environ.get("ODDISH_VERDICT_PROMPT_FILE")
+    if override:
+        monkeypatch.setattr(classifier, "_VERDICT_PROMPT", Path(override).read_text())
+
+    rows = []
+    for case in GOLDEN_CASES:
+        try:
+            verdict = await synthesize_task_verdict(
+                case.classifications,
+                case.baseline,
+                case.quality_check_passed,
+                VERDICT_TIMEOUT,
+                pre_trial_items=case.pre_trial_items,
+            )
+            got, confidence = verdict.verdict, verdict.confidence
+        except Exception as exc:  # noqa: BLE001 - one dead case must not hide the table
+            got, confidence = f"error({type(exc).__name__})", "-"
+        rows.append((case.name, case.expected_verdict, got, confidence))
+
+    width = max(len(r[0]) for r in rows)
+    header = f"{'case':<{width}}  {'expected':<8}  {'got':<8}  confidence"
+    print(
+        f"\nverdict model: {settings.verdict_model}"
+        + (f"  prompt: {override}" if override else "")
+    )
+    print(header)
+    print("-" * len(header))
+    for name, expected, got, confidence in rows:
+        flag = "" if got == expected else "  <-- MISMATCH"
+        print(f"{name:<{width}}  {expected:<8}  {got:<8}  {confidence}{flag}")
+    accepts = sum(1 for r in rows if r[2] == "accept")
+    print(f"accept rate: {accepts}/{len(rows)}")
+
+    mismatches = [r for r in rows if r[2] != r[1]]
+    assert not mismatches, f"{len(mismatches)} golden case(s) drifted: " + ", ".join(
+        f"{n} expected {e} got {g}" for n, e, g, _ in mismatches
+    )