Merge pull request #7 from superagent-ai/homanp/remove-unsafe-to-test

feat: Remove unsafe-to-test runtime verdict

Author: homanp

pithos/runtime_artifacts.py

@@ -13,14 +13,12 @@
 RUNTIME_NOT_REPRODUCED = "not_reproduced"
 RUNTIME_BLOCKED = "blocked"
 RUNTIME_INCONCLUSIVE = "inconclusive_runtime"
-RUNTIME_UNSAFE = "unsafe_to_test"
 
 RUNTIME_STATUSES = {
     RUNTIME_CONFIRMED,
     RUNTIME_NOT_REPRODUCED,
     RUNTIME_BLOCKED,
     RUNTIME_INCONCLUSIVE,
-    RUNTIME_UNSAFE,
 }
 
 
@@ -66,7 +64,6 @@ class VerificationPlan:
     preconditions: list[str] = field(default_factory=list)
     probe: dict[str, Any] = field(default_factory=dict)
     oracle: dict[str, Any] = field(default_factory=dict)
-    unsafe: bool = False
 
     def to_dict(self) -> dict[str, Any]:
         return asdict(self)

pithos/runtime_plugins.py

@@ -26,7 +26,6 @@
     RUNTIME_CONFIRMED,
     RUNTIME_INCONCLUSIVE,
     RUNTIME_NOT_REPRODUCED,
-    RUNTIME_UNSAFE,
     ProbeArtifact,
     RuntimeFinding,
     RuntimeVerdict,
@@ -779,7 +778,7 @@ def make_plan(self, finding: RuntimeFinding) -> VerificationPlan:
                 "Live verification was explicitly enabled.",
                 "Runtime profile preflight completed successfully.",
                 "The agent may run the project only in the copied sandbox workspace.",
-                "External side effects require declared mocks or staging targets.",
+                "Live verification uses configured staging, sandbox, mock, local, or loopback resources.",
             ],
             probe={
                 "kind": "agent_led_live_verification",
@@ -791,7 +790,6 @@ def make_plan(self, finding: RuntimeFinding) -> VerificationPlan:
                 "confirmed_runtime": "The agent reproduced the unsafe behavior with live evidence.",
                 "not_reproduced": "The live system rejected or prevented the unsafe behavior.",
                 "inconclusive_runtime": "The agent could not reach a reliable runtime conclusion.",
-                "unsafe_to_test": "The required reproduction would touch undeclared external side effects.",
             },
         )
 
@@ -819,19 +817,6 @@ def verify(self, finding: RuntimeFinding, out_dir: Path) -> RuntimeVerdict:
                 plan,
                 "runtime environment preflight failed; see verify/RUNTIME-SETUP.md",
             )
-        elif _unsafe_without_mock(self.profile, finding):
-            verdict = RuntimeVerdict(
-                finding_id=finding.id,
-                title=finding.title,
-                plugin=self.name,
-                status=RUNTIME_UNSAFE,
-                confidence="medium",
-                evidence=[
-                    "The finding appears to require an external or destructive side effect.",
-                    "The runtime profile does not declare a safe mock or staging target for that effect.",
-                ],
-                plan=plan.to_dict(),
-            )
         else:
             try:
                 verdict = self._run_live_agent(finding, finding_dir, plan)
@@ -1237,29 +1222,6 @@ def _route_from_files(files: list[str]) -> str | None:
     return None
 
 
-def _unsafe_without_mock(profile: AppRuntimeProfile, finding: RuntimeFinding) -> bool:
-    text = " ".join(
-        [
-            finding.title,
-            finding.category,
-            finding.exploit_scenario,
-            *finding.evidence,
-            *finding.files,
-        ]
-    ).lower()
-    external_keys = {
-        "stripe": ("stripe", "billing", "charge", "payment", "payout"),
-        "email": ("email", "smtp", "sendgrid", "mailgun"),
-        "aws": ("aws", "s3", "ses", "sns", "sqs"),
-        "webhook": ("webhook", "ssrf", "callback", "outbound"),
-        "production": ("production", "prod data", "real customer"),
-    }
-    for mock_name, needles in external_keys.items():
-        if any(needle in text for needle in needles) and not profile.mocks.get(mock_name):
-            return True
-    return False
-
-
 def _local_pi_env(app: EphemeralAppSandbox, agent_cfg: dict[str, object]) -> dict[str, str]:
     env = app.env()
     env["PI_OFFLINE"] = "1"
@@ -1347,11 +1309,10 @@ def _live_agent_system_prompt() -> str:
 Hard requirements:
 - Do not modify the original source checkout. Work only inside /work/repo and /work/artifacts.
 - Do not fix code, commit code, open PRs, or make unrelated changes.
-- Do not contact real third-party services, production systems, billing providers, email providers,
-  cloud storage, webhook targets, or customer data unless the runtime profile explicitly declares a
-  safe mock or staging target.
-- Prefer local tests, scripts, seeded data, local services, and loopback targets.
-- If reproduction would be destructive or external, return unsafe_to_test.
+- Use only the configured staging, sandbox, mock, local, or loopback resources. Do not contact
+  production systems or real customer data.
+- Prefer local tests, scripts, seeded data, local services, and loopback targets when they provide
+  reliable evidence.
 - Output exactly one <runtime_verdict_json>...</runtime_verdict_json> object.
 """
 
@@ -1384,7 +1345,7 @@ def _live_agent_prompt(profile: AppRuntimeProfile, finding: RuntimeFinding) -> s
 Write any supporting evidence under /work/artifacts. Return only:
 <runtime_verdict_json>
 {{
-  "status": "confirmed_runtime|not_reproduced|inconclusive_runtime|blocked|unsafe_to_test",
+  "status": "confirmed_runtime|not_reproduced|inconclusive_runtime|blocked",
   "confidence": "high|medium|low",
   "evidence": [
     "what you ran",

pithos/runtime_verifier.py

@@ -40,9 +40,7 @@ def status(self) -> str:
             return "runtime_findings_confirmed"
         if not self.profile.verification.get("execute_app"):
             return "completed_static_only"
-        if any(
-            v.status in {"blocked", "inconclusive_runtime", "unsafe_to_test"} for v in self.verdicts
-        ):
+        if any(v.status in {"blocked", "inconclusive_runtime"} for v in self.verdicts):
             return "completed_with_unverified"
         return "completed"
 
@@ -313,7 +311,6 @@ def _counts(verdicts: list[RuntimeVerdict]) -> dict[str, int]:
         "not_reproduced": 0,
         "blocked": 0,
         "inconclusive_runtime": 0,
-        "unsafe_to_test": 0,
     }
     for v in verdicts:
         out[v.status] = out.get(v.status, 0) + 1
@@ -325,7 +322,7 @@ def _summary_status(verdicts: list[RuntimeVerdict], *, execute_app: bool) -> str
         return "runtime_findings_confirmed"
     if not execute_app:
         return "completed_static_only"
-    if any(v.status in {"blocked", "inconclusive_runtime", "unsafe_to_test"} for v in verdicts):
+    if any(v.status in {"blocked", "inconclusive_runtime"} for v in verdicts):
         return "completed_with_unverified"
     return "completed"
 

tests/test_runtime_verifier.py

@@ -360,6 +360,77 @@ def test_live_verify_runs_profile_agent_command(tmp_path):
     assert (results / "runtime" / "F001" / "agent-command.json").exists()
 
 
+def test_live_verify_runs_external_side_effect_findings_in_staging(tmp_path):
+    repo = tmp_path / "app"
+    repo.mkdir()
+    (repo / "verify_agent.py").write_text(
+        """
+import json
+import os
+
+with open(os.environ["PITHOS_FINDING_JSON"], encoding="utf-8") as f:
+    finding = json.load(f)
+verdict = {
+    "status": "not_reproduced",
+    "confidence": "high",
+    "evidence": [f"exercised staged payment flow for {finding['id']}"],
+}
+with open(os.environ["PITHOS_VERDICT_JSON"], "w", encoding="utf-8") as f:
+    json.dump(verdict, f)
+""",
+        encoding="utf-8",
+    )
+    profile = repo / ".pithos" / "runtime.yaml"
+    profile.parent.mkdir()
+    profile.write_text(
+        f"""
+verification:
+  execute_app: true
+  agent_command: "{sys.executable} verify_agent.py"
+environment:
+  sandbox: local
+""",
+        encoding="utf-8",
+    )
+    results = tmp_path / "results" / "app" / "run"
+    results.mkdir(parents=True)
+    (results / "run-summary.json").write_text(json.dumps({"repo": {"path": str(repo)}}))
+    (results / "VULN-FINDINGS.json").write_text(
+        json.dumps(
+            [
+                {
+                    "id": "F001",
+                    "title": "Stripe charge endpoint can capture arbitrary staged payments",
+                    "category": "payment provider side effect",
+                    "files": ["src/payments.ts"],
+                }
+            ]
+        )
+    )
+    (results / "TRIAGE.json").write_text(
+        json.dumps(
+            {
+                "findings": [
+                    {
+                        "id": "F001",
+                        "title": "Stripe charge endpoint can capture arbitrary staged payments",
+                    }
+                ]
+            }
+        )
+    )
+
+    result = run_verify_repo(triage_path=results, execute_app=True)
+    summary = json.loads((results / "runtime" / "runtime-summary.json").read_text())
+
+    assert result.status == "completed"
+    assert result.verdicts[0].plugin == "live_agent"
+    assert result.verdicts[0].status == "not_reproduced"
+    assert summary["counts"]["not_reproduced"] == 1
+    assert len(summary["counts"]) == 5
+    assert (results / "runtime" / "F001" / "agent-command.json").exists()
+
+
 def test_live_verify_runs_pi_agent_locally_without_docker(tmp_path, monkeypatch):
     repo = tmp_path / "app"
     repo.mkdir()