Fail closed on PurelyMail API errors

Author: rgarcia

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 - Updated the harbor adaptor to support the full V2 lenient & strict and reports numeric results.
 
 ### Fixed
+- Fail task setup when PurelyMail returns an API error instead of emitting credentials for an account that was not created.
 - Fixed an issue where malformed per-run metadata could prevent `batch-summary.json` from being written and, when configured, uploaded.
 - Fixed the issue that an invalid judge model would lose the `run-meta.json` file.
 

src/clawbench/runner/run_support/email.py

@@ -18,7 +18,14 @@ def purelymail_request(endpoint: str, body: dict, api_key: str) -> dict:
         method="POST",
     )
     with urlopen(req, timeout=15) as resp:
-        return json.loads(resp.read())
+        result = json.loads(resp.read())
+    if not isinstance(result, dict):
+        raise RuntimeError(f"PurelyMail {endpoint} returned an invalid response")
+    if result.get("type") == "error":
+        code = f" ({result['code']})" if result.get("code") else ""
+        message = f": {result['message']}" if result.get("message") else ""
+        raise RuntimeError(f"PurelyMail {endpoint} failed{code}{message}")
+    return result
 
 
 def create_email(api_key: str, domain: str) -> tuple[str, str]:

src/clawbench/runtime/harbor/cleanup-email.py

@@ -28,7 +28,14 @@ def purelymail_request(endpoint: str, body: dict, api_key: str) -> dict:
         method="POST",
     )
     with urlopen(req, timeout=15) as resp:
-        return json.loads(resp.read())
+        result = json.loads(resp.read())
+    if not isinstance(result, dict):
+        raise RuntimeError(f"PurelyMail {endpoint} returned an invalid response")
+    if result.get("type") == "error":
+        code = f" ({result['code']})" if result.get("code") else ""
+        message = f": {result['message']}" if result.get("message") else ""
+        raise RuntimeError(f"PurelyMail {endpoint} failed{code}{message}")
+    return result
 
 
 def main() -> int:

src/clawbench/runtime/harbor/prepare-task.py

@@ -30,7 +30,14 @@ def purelymail_request(endpoint: str, body: dict, api_key: str) -> dict:
         method="POST",
     )
     with urlopen(req, timeout=15) as resp:
-        return json.loads(resp.read())
+        result = json.loads(resp.read())
+    if not isinstance(result, dict):
+        raise RuntimeError(f"PurelyMail {endpoint} returned an invalid response")
+    if result.get("type") == "error":
+        code = f" ({result['code']})" if result.get("code") else ""
+        message = f": {result['message']}" if result.get("message") else ""
+        raise RuntimeError(f"PurelyMail {endpoint} failed{code}{message}")
+    return result
 
 
 def create_email(api_key: str, domain: str) -> tuple[str, str]:

tests/test_purelymail_errors.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+from clawbench.runner.run_support import email as native_email
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+
+class FakeResponse:
+    def __init__(self, payload: object) -> None:
+        self.payload = payload
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args) -> None:
+        return None
+
+    def read(self) -> bytes:
+        return json.dumps(self.payload).encode()
+
+
+def load_harbor_script(name: str):
+    path = REPO_ROOT / "src" / "clawbench" / "runtime" / "harbor" / name
+    spec = importlib.util.spec_from_file_location(name.replace("-", "_"), path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.mark.parametrize(
+    "module",
+    [
+        native_email,
+        pytest.param(load_harbor_script("prepare-task.py"), id="harbor-prepare"),
+        pytest.param(load_harbor_script("cleanup-email.py"), id="harbor-cleanup"),
+    ],
+)
+def test_purelymail_api_errors_fail_closed(monkeypatch, module) -> None:
+    monkeypatch.setattr(
+        module,
+        "urlopen",
+        lambda *_args, **_kwargs: FakeResponse(
+            {
+                "type": "error",
+                "code": "invalidToken",
+                "message": "Token not valid.",
+            }
+        ),
+    )
+
+    with pytest.raises(
+        RuntimeError,
+        match=r"PurelyMail createUser failed \(invalidToken\): Token not valid\.",
+    ):
+        module.purelymail_request("createUser", {}, "secret-token")
+
+
+def test_purelymail_rejects_non_object_responses(monkeypatch) -> None:
+    monkeypatch.setattr(
+        native_email,
+        "urlopen",
+        lambda *_args, **_kwargs: FakeResponse(["unexpected"]),
+    )
+
+    with pytest.raises(RuntimeError, match="returned an invalid response"):
+        native_email.purelymail_request("createUser", {}, "secret-token")