fix(judge): collapse judge_llm.py onto judge.py's transport

src/clawbench/runner/judge_llm.py

@@ -17,20 +17,28 @@
 constrained by the instruction. This pulls reported scores up by ~10-30pp
 and matches the public Sonnet 4.6 reference (Intercepted 58% / Reward 44%).
 
+The rubric prompt below is the only thing this module owns. Transport,
+api_type dispatch, retries, and verdict parsing are judge.py's, so the two
+rubrics cannot drift in the ways they had: this module used to lack
+judge_context, route openai-responses at /chat/completions, reject
+google-generative-ai outright, and cap max_tokens at 800.
+
 Usage (same interface as judge.py for drop-in replacement):
     from judge_llm import judge_request
     verdict = judge_request(model_cfg, judge_model, instruction, intercept)
 """
 
 from __future__ import annotations
 
-import json
-import time
-import urllib.request
-import urllib.error
 from typing import Any
 
-from clawbench.runner.judge import _coerce_match
+from clawbench.runner.judge import (
+    _build_user_msg,
+    _parse_verdict,  # noqa: F401  — re-export: keeps the lenient path under test
+    _run_judge,
+)
+
+__all__ = ["JUDGE_SYSTEM", "judge_request"]
 
 
 JUDGE_SYSTEM = """You are a lenient evaluator for a web-agent benchmark.
@@ -77,148 +85,20 @@
 """
 
 
-def _build_user_msg(instruction: str, intercept: dict[str, Any]) -> str:
-    req = intercept.get("request") or {}
-    body = req.get("body")
-    if isinstance(body, (dict, list)):
-        body_str = json.dumps(body, ensure_ascii=False, indent=2)[:6000]
-    else:
-        body_str = str(body)[:6000] if body is not None else "(empty)"
-    return (
-        f"INSTRUCTION:\n{instruction}\n\n"
-        f"INTERCEPTED REQUEST:\n"
-        f"  url: {req.get('url')}\n"
-        f"  method: {req.get('method')}\n"
-        f"  body:\n{body_str}\n"
-    )
-
-
-def _post_json(
-    url: str, headers: dict[str, str], payload: dict[str, Any], timeout: int = 60
-) -> dict[str, Any]:
-    data = json.dumps(payload).encode("utf-8")
-    req = urllib.request.Request(
-        url, data=data, headers={**headers, "Content-Type": "application/json"}
-    )
-    with urllib.request.urlopen(req, timeout=timeout) as resp:
-        return json.loads(resp.read())
-
-
-def _call_openai_chat(model_cfg: dict, model_name: str, system: str, user: str) -> str:
-    url = f"{model_cfg['base_url'].rstrip('/')}/chat/completions"
-    headers = {"Authorization": f"Bearer {model_cfg['api_key']}"}
-    payload = {
-        "model": model_name,
-        "messages": [
-            {"role": "system", "content": system},
-            {"role": "user", "content": user},
-        ],
-        "max_tokens": 800,
-        "temperature": 0.0,
-    }
-    resp = _post_json(url, headers, payload)
-    return resp["choices"][0]["message"]["content"]
-
-
-def _call_anthropic_messages(
-    model_cfg: dict, model_name: str, system: str, user: str
-) -> str:
-    base = model_cfg.get("base_url", "https://api.anthropic.com").rstrip("/")
-    url = f"{base}/v1/messages"
-    headers = {
-        "x-api-key": model_cfg["api_key"],
-        "anthropic-version": model_cfg.get("anthropic_version", "2023-06-01"),
-    }
-    payload = {
-        "model": model_name,
-        "system": system,
-        "messages": [{"role": "user", "content": user}],
-        "max_tokens": 800,
-        "temperature": 0.0,
-    }
-    resp = _post_json(url, headers, payload)
-    content = resp.get("content", [])
-    return "".join(
-        b.get("text", "")
-        for b in content
-        if isinstance(b, dict) and b.get("type") == "text"
-    )
-
-
-def _parse_verdict(raw: str) -> tuple[bool | None, str]:
-    """Best-effort parse of the judge's reply into (match, reason).
-
-    Returns None when the reply carries no usable verdict, so an unparseable
-    judge response is reported as inconclusive instead of silently passing.
-    """
-    try:
-        # Strip markdown fences if any
-        s = raw.strip()
-        if s.startswith("```"):
-            s = s.split("\n", 1)[1] if "\n" in s else s
-            if s.endswith("```"):
-                s = s.rsplit("\n", 1)[0] if "\n" in s else s.rstrip("`")
-        obj = json.loads(s)
-        return _coerce_match(obj.get("match")), str(obj.get("reason", ""))
-    except Exception:
-        # Keyword fallback for replies that are not valid JSON. An unparseable
-        # reply is inconclusive (None), never an implicit pass.
-        low = raw.lower()
-        if "match" not in low:
-            return None, raw[:200] or "unparseable"
-        after = low.split("match", 1)[1][:80]
-        if "false" in after:
-            return False, raw[:200]
-        if "true" in after:
-            return True, raw[:200]
-        return None, raw[:200] or "unparseable"
-
-
 def judge_request(
-    model_cfg: dict, judge_model_name: str, instruction: str, intercept: dict[str, Any]
+    model_cfg: dict,
+    judge_model_name: str,
+    instruction: str,
+    intercept: dict[str, Any],
+    *,
+    judge_context: dict[str, Any] | None = None,
+    retries: int = 2,
 ) -> dict[str, Any]:
-    """Run a single lenient judge call. Returns dict with keys match/reason/judge_model/raw/error."""
-    system = JUDGE_SYSTEM
-    user = _build_user_msg(instruction, intercept)
-    api_type = model_cfg.get("api_type", "openai-completions")
-    raw = ""
-    err = None
-    for attempt in range(3):
-        try:
-            if api_type in ("openai-completions", "openai-responses"):
-                raw = _call_openai_chat(model_cfg, judge_model_name, system, user)
-            elif api_type == "anthropic-messages":
-                raw = _call_anthropic_messages(
-                    model_cfg, judge_model_name, system, user
-                )
-            else:
-                raise NotImplementedError(
-                    f"judge_llm: unsupported api_type {api_type!r}"
-                )
-            break
-        except urllib.error.HTTPError as e:
-            err = f"http_{e.code}"
-            if e.code in (429, 500, 502, 503):
-                time.sleep(2**attempt)
-                continue
-            break
-        except Exception as e:
-            err = f"err_{type(e).__name__}: {e}"
-            break
-    if not raw:
-        return {
-            "match": None,
-            "reason": "",
-            "judge_model": judge_model_name,
-            "raw": "",
-            "error": err,
-            "rubric": "lenient",
-        }
-    m, reason = _parse_verdict(raw)
-    return {
-        "match": m,
-        "reason": reason,
-        "judge_model": judge_model_name,
-        "raw": raw[:500],
-        "rubric": "lenient",
-    }
+    """Judge an intercepted HTTP request under the lenient rubric.
+
+    Same call signature and return shape as judge.judge_request, plus a
+    ``rubric`` key naming which rubric produced the verdict.
+    """
+    user = _build_user_msg(instruction, intercept, judge_context)
+    verdict = _run_judge(model_cfg, judge_model_name, JUDGE_SYSTEM, user, retries)
+    return {**verdict, "rubric": "lenient"}

tests/test_lenient_judge_parity.py

@@ -0,0 +1,153 @@
+"""The lenient rubric must reach the judge transport identically to the strict one.
+
+judge_llm.py used to re-implement _post_json / _build_user_msg / the api_type
+dispatch. That copy drifted: no judge_context, openai-responses routed at
+/chat/completions, google-generative-ai unsupported, and max_tokens capped at
+800 (a reasoning judge truncates and lands on the inconclusive path). These
+tests pin the collapsed behaviour so the two rubrics cannot diverge again.
+"""
+
+from __future__ import annotations
+
+import inspect
+from typing import Any
+
+import pytest
+
+from clawbench.runner import judge, judge_llm
+
+
+@pytest.fixture
+def captured(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]:
+    """Record every HTTP payload the judge would send, without a network."""
+    calls: list[dict[str, Any]] = []
+
+    def fake_post(
+        url: str, headers: dict[str, str], payload: dict[str, Any], timeout: int = 60
+    ) -> dict[str, Any]:
+        calls.append({"url": url, "headers": headers, "payload": payload})
+        return {
+            "choices": [{"message": {"content": '{"match": true, "reason": "ok"}'}}],
+            "output_text": '{"match": true, "reason": "ok"}',
+            "content": [{"type": "text", "text": '{"match": true, "reason": "ok"}'}],
+        }
+
+    monkeypatch.setattr(judge, "_post_json", fake_post)
+    return calls
+
+
+INTERCEPT = {"request": {"url": "https://shop.test/cart", "method": "POST", "body": {}}}
+
+
+def _cfg(api_type: str, base_url: str = "https://api.test/v1") -> dict[str, Any]:
+    return {"api_type": api_type, "base_url": base_url, "api_key": "k"}
+
+
+# --- the four documented divergences ----------------------------------------
+
+
+def test_lenient_judge_accepts_judge_context(captured: list[dict[str, Any]]) -> None:
+    """run.py passes judge_context to the strict judge; the lenient signature
+    used to reject it outright with a TypeError."""
+    judge_llm.judge_request(
+        _cfg("openai-completions"),
+        "judge-model",
+        "buy a red shirt",
+        INTERCEPT,
+        judge_context={"rubric": "must be red"},
+    )
+
+    user_msg = captured[0]["payload"]["messages"][1]["content"]
+    assert "HIDDEN JUDGE CONTEXT" in user_msg
+    assert "must be red" in user_msg
+
+
+def test_lenient_judge_routes_openai_responses_to_the_responses_endpoint(
+    captured: list[dict[str, Any]],
+) -> None:
+    judge_llm.judge_request(_cfg("openai-responses"), "m", "do a thing", INTERCEPT)
+
+    assert captured[0]["url"].endswith("/responses")
+
+
+def test_lenient_judge_supports_gemini(captured: list[dict[str, Any]]) -> None:
+    """google-generative-ai used to raise NotImplementedError on this path."""
+    verdict = judge_llm.judge_request(
+        _cfg("google-generative-ai", "https://generativelanguage.googleapis.com"),
+        "gemini-3-pro",
+        "do a thing",
+        INTERCEPT,
+    )
+
+    assert verdict["match"] is True
+    assert "/v1beta/openai/chat/completions" in captured[0]["url"]
+
+
+@pytest.mark.parametrize(
+    "api_type", ["openai-completions", "openai-responses", "anthropic-messages"]
+)
+def test_lenient_judge_gives_a_reasoning_judge_room_to_answer(
+    api_type: str, captured: list[dict[str, Any]]
+) -> None:
+    """The default judge is a reasoning model that burns hidden tokens before
+    emitting its JSON line; 800 truncated it into the inconclusive path."""
+    judge_llm.judge_request(_cfg(api_type), "m", "do a thing", INTERCEPT)
+
+    payload = captured[0]["payload"]
+    budget = payload.get("max_tokens") or payload.get("max_output_tokens")
+    assert budget == 4096
+
+
+# --- the rubrics stay distinct in the one way they should --------------------
+
+
+def test_the_two_rubrics_differ_only_in_the_system_prompt(
+    captured: list[dict[str, Any]],
+) -> None:
+    args = (_cfg("openai-completions"), "m", "buy a red shirt", INTERCEPT)
+
+    judge.judge_request(*args)
+    judge_llm.judge_request(*args)
+
+    strict, lenient = captured
+    assert strict["payload"]["messages"][0]["content"] == judge.JUDGE_SYSTEM
+    assert lenient["payload"]["messages"][0]["content"] == judge_llm.JUDGE_SYSTEM
+    assert "strict evaluator" in judge.JUDGE_SYSTEM
+    assert "lenient evaluator" in judge_llm.JUDGE_SYSTEM
+
+    # everything else on the wire is identical
+    assert strict["url"] == lenient["url"]
+    assert strict["headers"] == lenient["headers"]
+    assert strict["payload"]["messages"][1] == lenient["payload"]["messages"][1]
+    assert strict["payload"]["max_tokens"] == lenient["payload"]["max_tokens"]
+
+
+def test_lenient_verdict_is_tagged_with_its_rubric(
+    captured: list[dict[str, Any]],
+) -> None:
+    verdict = judge_llm.judge_request(_cfg("openai-completions"), "m", "x", INTERCEPT)
+
+    assert verdict["rubric"] == "lenient"
+    assert set(
+        judge.judge_request(_cfg("openai-completions"), "m", "x", INTERCEPT)
+    ) <= set(verdict)
+
+
+def test_lenient_signature_matches_the_strict_one() -> None:
+    """rescore.py calls both through one dict of judge functions."""
+    strict = inspect.signature(judge.judge_request)
+    lenient = inspect.signature(judge_llm.judge_request)
+
+    assert list(strict.parameters) == list(lenient.parameters)
+    for name, param in strict.parameters.items():
+        assert lenient.parameters[name].kind == param.kind
+        assert lenient.parameters[name].default == param.default
+
+
+def test_lenient_module_no_longer_reimplements_the_transport() -> None:
+    """Regression guard for the duplication itself: the ~100 copied lines of
+    _post_json / _call_* are what drifted in the first place."""
+    src = inspect.getsource(judge_llm)
+
+    for copied in ("def _post_json", "def _call_openai_chat", "def _build_user_msg"):
+        assert copied not in src