fix(judge): a bad --judge model no longer discards a completed run

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 
 ### Fixed
 - Fixed an issue where malformed per-run metadata could prevent `batch-summary.json` from being written and, when configured, uploaded.
+- Fixed the issue that an invalid judge model would lose the `run-meta.json` file.
 
 ## [0.9.2] - 2026-08-18
 ### Added

src/clawbench/runner/run.py

@@ -21,6 +21,7 @@
     HARNESSES,
     IMAGE,
     WORKSPACE_ROOT,
+    ModelConfigError,
     harness_image,
     load_model_config,
     load_runtime_env,
@@ -213,10 +214,28 @@ def main():
         safe_model = "human"
         harness_tag = "human"
     else:
-        model_cfg = load_model_config(args.model)
+        try:
+            model_cfg = load_model_config(args.model)
+        except ModelConfigError as e:
+            # No output_dir exists yet at this point, so there is nothing
+            # for a run-meta.json to record — exit immediately like other
+            # pre-flight validation above (e.g. missing PURELY_MAIL_* env).
+            print(f"ERROR: {e}")
+            sys.exit(1)
         safe_model = re.sub(r"[/:]+", "--", args.model)
         harness_tag = args.harness
 
+    # Resolve the judge model now rather than after the agent run. A bad
+    # --judge is a typo in the command line, and finding out about it 30
+    # minutes later — once the agent has already finished — helps nobody.
+    startup_judge_cfg: dict | None = None
+    if not args.human and args.judge and not args.no_judge:
+        try:
+            startup_judge_cfg = load_model_config(args.judge)
+        except ModelConfigError as e:
+            print(f"ERROR: --judge {args.judge!r}: {e}")
+            sys.exit(1)
+
     container = f"clawbench-{harness_tag}-{case_name}-{safe_model}-{int(time.time())}"
     run_dir_name = f"{harness_tag}-{case_name}-{safe_model}-{ts}"
 
@@ -232,7 +251,7 @@ def main():
     extra_info_warnings: list[str] = []
     intercepted = False
     host_port: int | None = None
-    judge_cfg: dict | None = None
+    judge_cfg: dict | None = startup_judge_cfg
     personal_info_metadata: dict[str, Any] | None = None
     browser_session: BrowserSession | None = None
     browser_runtime_finalized = False
@@ -601,7 +620,9 @@ def handle_sigint(sig, frame):
             try:
                 from clawbench.runner.judge import judge_request
 
-                judge_cfg = load_model_config(args.judge)
+                # Validated at startup; reload only if that was skipped.
+                if judge_cfg is None:
+                    judge_cfg = load_model_config(args.judge)
                 instruction_text = (
                     task.get("instruction") if isinstance(task, dict) else ""
                 ) or ""

src/clawbench/runner/run_support/config.py

@@ -32,6 +32,7 @@
     "MODELS_YAML",
     "WORKSPACE_ROOT",
     "HarnessRegistry",
+    "ModelConfigError",
     "harness_image",
     "load_dotenv",
     "load_harness_registry",
@@ -44,6 +45,16 @@
 ]
 
 
+class ModelConfigError(Exception):
+    """Raised when a model config in models/models.yaml is missing or invalid.
+
+    A plain Exception (not SystemExit) so callers that load a model mid-run
+    (e.g. the judge stage, after the agent has already produced results) can
+    catch it and continue instead of the process dying before run-meta.json
+    is written.
+    """
+
+
 HARNESSES = HARNESS_REGISTRY.harnesses
 DEFAULT_HARNESS = HARNESS_REGISTRY.default
 BASE_IMAGE = HARNESS_REGISTRY.base_image
@@ -107,12 +118,16 @@ def load_dotenv(path: Path) -> dict[str, str]:
 
 
 def load_models_yaml() -> dict:
-    """Load all model definitions from models/models.yaml."""
+    """Load all model definitions from models/models.yaml.
+
+    Raises ModelConfigError rather than exiting, for the same reason
+    load_model_config does: this runs inside the judge stage too, where a
+    SystemExit would escape the handlers and lose the run's metadata.
+    """
     if not MODELS_YAML.exists():
-        print(
-            f"ERROR: {MODELS_YAML} not found (copy models.example.yaml and fill in your keys)"
+        raise ModelConfigError(
+            f"{MODELS_YAML} not found (copy models.example.yaml and fill in your keys)"
         )
-        sys.exit(1)
     return yaml.safe_load(MODELS_YAML.read_text()) or {}
 
 
@@ -161,9 +176,10 @@ def load_model_config(model: str) -> dict:
     """
     all_models = load_models_yaml()
     if model not in all_models:
-        print(f"ERROR: model '{model}' not found in {MODELS_YAML}")
-        print(f"Available models: {', '.join(sorted(all_models))}")
-        sys.exit(1)
+        raise ModelConfigError(
+            f"model '{model}' not found in {MODELS_YAML}. "
+            f"Available models: {', '.join(sorted(all_models))}"
+        )
 
     # Validate model name characters. Note: '/' and ':' are valid in
     # vendor-prefixed ids like 'anthropic/claude-sonnet-4-6' or
@@ -173,21 +189,20 @@ def load_model_config(model: str) -> dict:
     # that sanitization.
     bad = [c for c in ' \\*?"<>|' if c in model]
     if bad:
-        print(
-            f"ERROR: model name '{model}' contains illegal character(s): "
+        raise ModelConfigError(
+            f"model name '{model}' contains illegal character(s): "
             f"{' '.join(repr(c) for c in bad)}"
         )
-        sys.exit(1)
 
     config = dict(all_models[model])
     config["model"] = model  # the YAML key IS the model name
 
     required = ["base_url", "api_type"]
     missing = [k for k in required if not config.get(k)]
     if missing:
-        for k in missing:
-            print(f"ERROR: Required field '{k}' missing for model '{model}'")
-        sys.exit(1)
+        raise ModelConfigError(
+            f"required field(s) missing for model '{model}': {', '.join(missing)}"
+        )
 
     # Normalize API keys: api_keys list wins, else wrap api_key into list.
     if config.get("api_keys"):
@@ -196,7 +211,6 @@ def load_model_config(model: str) -> dict:
         config["api_keys"] = [config["api_key"]]
 
     if not config.get("api_keys"):
-        print(f"ERROR: no api_key or api_keys for model '{model}'")
-        sys.exit(1)
+        raise ModelConfigError(f"no api_key or api_keys for model '{model}'")
 
     return config