Enforce Daytona computer-use evidence

Author: homanp

pithos/runtime_plugins.py

@@ -23,6 +23,7 @@
 from .computer_use import (
     COMPUTER_USE_DAYTONA,
     COMPUTER_USE_NONE,
+    DAYTONA_COMMANDS,
     computer_use_agent_env,
     install_computer_use_commands,
     normalize_computer_use_mode,
@@ -1048,6 +1049,25 @@ def _verdict_from_agent_result(
             )
         verdict = _verdict_from_payload(finding, plan, verdict_data, plugin=self.name)
         verdict.artifacts["transcript"] = str(finding_dir / "live-agent-transcript.jsonl")
+        if _profile_computer_use_mode(self.profile) == COMPUTER_USE_DAYTONA and not (
+            _agent_result_has_daytona_command(result)
+        ):
+            return RuntimeVerdict(
+                finding_id=finding.id,
+                title=finding.title,
+                plugin=self.name,
+                status=RUNTIME_BLOCKED,
+                confidence="low",
+                evidence=[
+                    "Daytona computer use was requested, but the live-agent transcript did not "
+                    "include a bash tool call invoking any cu-* command.",
+                    "The verifier requires computer-use evidence before accepting a Daytona "
+                    "live-agent verdict.",
+                ],
+                error="missing Daytona computer-use evidence",
+                artifacts={"transcript": str(finding_dir / "live-agent-transcript.jsonl")},
+                plan=plan.to_dict(),
+            )
         return verdict
 
     def _blocked(
@@ -1280,6 +1300,25 @@ def _localize_live_agent_paths(text: str, *, repo_path: Path, artifacts_path: Pa
     )
 
 
+def _agent_result_has_daytona_command(result) -> bool:
+    for msg in getattr(result, "messages", []) or []:
+        if msg.get("role") != "assistant":
+            continue
+        content = msg.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if not isinstance(block, dict) or block.get("type") != "toolCall":
+                continue
+            if block.get("name") != "bash":
+                continue
+            args = block.get("arguments")
+            command = args.get("command") if isinstance(args, dict) else ""
+            if any(cmd in str(command) for cmd in DAYTONA_COMMANDS):
+                return True
+    return False
+
+
 def _extract_verdict_payload(text: str) -> dict[str, object] | None:
     raw = parse_xml_tag(text, "runtime_verdict_json") or text.strip()
     if raw.startswith("```"):
@@ -1359,8 +1398,11 @@ def _live_agent_system_prompt(computer_use: str = COMPUTER_USE_NONE) -> str:
 - Do not start Xvfb, XFCE, x11vnc, noVNC, or a nested Daytona sandbox yourself.
 - Use bash-accessible local commands to control the existing desktop:
   `cu-info`, `cu-screenshot`, `cu-click`, `cu-type`, `cu-key`, `cu-scroll`, and `cu-drag`.
-- Take a screenshot before and after desktop actions, and use the image evidence to decide
-  whether the action affected the expected screen.
+- Your first bash actions must run `cu-info` and `cu-screenshot` before reading source, installing
+  dependencies, running tests, or writing reproduction scripts.
+- Take another screenshot after desktop/UI actions, and use the image evidence to decide whether the
+  action affected the expected screen.
+- Include the `cu-*` commands you ran and their artifact paths in the final runtime verdict evidence.
 - Treat coordinates as screenshot/API-space coordinates when display scaling is reported.
 - If UI actions do not affect the expected screen, check `DISPLAY`, run `cu-info`, and use
   diagnostics such as `xdpyinfo` before trying more actions.

pithos/runtime_verifier.py

@@ -554,7 +554,7 @@ def _environment_preflight(
         )
     if execute_app and computer_use_mode == COMPUTER_USE_DAYTONA:
         if backend != "local":
-            warnings.append(
+            issues.append(
                 "Daytona computer use controls the outer Superagent desktop; use "
                 "--sandbox-mode local so live agents can run the generated cu-* commands "
                 "in that environment"

tests/test_runtime_verifier.py

@@ -273,15 +273,35 @@ def test_runtime_preflight_records_daytona_computer_use(tmp_path):
         allow_inferred_runtime=True,
         provider="test-provider",
         model="test-model",
-        sandbox_mode="docker",
+        sandbox_mode="local",
         computer_use="daytona",
     )
 
     assert result.preflight["computer_use"] == "daytona"
+    assert result.preflight["ready"] is True
     assert result.preflight["computer_use_daytona"]["local_api_url"] == DAYTONA_LOCAL_API_URL
     assert "cu-screenshot" in result.preflight["computer_use_daytona"]["commands"]
     assert result.profile.verification["computer_use"] == "daytona"
-    assert any("Daytona computer use controls the outer" in w for w in result.preflight["warnings"])
+
+
+def test_runtime_preflight_rejects_daytona_without_local_runtime(tmp_path):
+    repo = tmp_path / "app"
+    repo.mkdir()
+    out_dir = tmp_path / "results" / "app" / "run" / "verify"
+
+    result = run_runtime_preflight(
+        repo_path=repo,
+        results_dir=out_dir,
+        execute_app=True,
+        allow_inferred_runtime=True,
+        provider="test-provider",
+        model="test-model",
+        sandbox_mode="docker",
+        computer_use="daytona",
+    )
+
+    assert result.preflight["ready"] is False
+    assert any("Daytona computer use controls the outer" in i for i in result.preflight["issues"])
 
 
 def test_runtime_preflight_reports_discovered_env_sources(tmp_path, monkeypatch):
@@ -709,11 +729,23 @@ async def fake_run_agent_process(prompt, **kwargs):
             {
                 "status": "not_reproduced",
                 "confidence": "high",
-                "evidence": ["ran Daytona computer-use verification"],
+                "evidence": ["ran cu-info and cu-screenshot before verification"],
             }
         )
         return SimpleNamespace(
             error=None,
+            messages=[
+                {
+                    "role": "assistant",
+                    "content": [
+                        {
+                            "type": "toolCall",
+                            "name": "bash",
+                            "arguments": {"command": "cu-info && cu-screenshot before.png"},
+                        }
+                    ],
+                }
+            ],
             last_assistant_message=payload,
             find_tagged_message=lambda _tag: payload,
         )
@@ -740,6 +772,85 @@ async def fake_run_agent_process(prompt, **kwargs):
     assert summary["environment"]["computer_use"] == "daytona"
 
 
+def test_daytona_computer_use_blocks_live_agent_without_cu_evidence(tmp_path, monkeypatch):
+    repo = tmp_path / "app"
+    repo.mkdir()
+    profile = repo / ".pithos" / "runtime.yaml"
+    profile.parent.mkdir()
+    profile.write_text(
+        """
+verification:
+  execute_app: true
+environment:
+  sandbox: local
+""",
+        encoding="utf-8",
+    )
+    results = tmp_path / "results" / "app" / "run"
+    results.mkdir(parents=True)
+    (results / "run-summary.json").write_text(json.dumps({"repo": {"path": str(repo)}}))
+    (results / "VULN-FINDINGS.json").write_text(
+        json.dumps(
+            [
+                {
+                    "id": "F001",
+                    "title": "Generic live authorization bypass",
+                    "category": "authorization bypass",
+                    "files": ["src/server.ts"],
+                }
+            ]
+        )
+    )
+    (results / "TRIAGE.json").write_text(
+        json.dumps({"findings": [{"id": "F001", "title": "Generic live authorization bypass"}]})
+    )
+
+    async def fake_run_agent_process(prompt, **kwargs):
+        payload = json.dumps(
+            {
+                "status": "not_reproduced",
+                "confidence": "high",
+                "evidence": ["ran source-only verification"],
+            }
+        )
+        return SimpleNamespace(
+            error=None,
+            messages=[
+                {
+                    "role": "assistant",
+                    "content": [
+                        {
+                            "type": "toolCall",
+                            "name": "bash",
+                            "arguments": {"command": "npm test"},
+                        }
+                    ],
+                }
+            ],
+            last_assistant_message=payload,
+            find_tagged_message=lambda _tag: payload,
+        )
+
+    monkeypatch.setattr(runtime_plugins, "run_agent_process", fake_run_agent_process)
+    monkeypatch.setattr(
+        runtime_plugins.docker_ops,
+        "run",
+        lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("docker was called")),
+    )
+
+    result = run_verify_repo(
+        triage_path=results,
+        execute_app=True,
+        provider="anthropic",
+        model="claude-sonnet-4-5",
+        sandbox_mode="local",
+        computer_use="daytona",
+    )
+
+    assert result.verdicts[0].status == "blocked"
+    assert result.verdicts[0].error == "missing Daytona computer-use evidence"
+
+
 def test_verify_repo_confirms_polar_style_source_oracles(tmp_path):
     repo = tmp_path / "polar"
     (repo / "server/polar/oauth2/grants").mkdir(parents=True)