Merge pull request #6 from superagent-ai/homanp/unify-sandbox-modes

feat: Unify runtime sandbox modes

Author: homanp

AGENTS.md

@@ -160,8 +160,8 @@ command, optional Dockerfile build metadata, start and healthcheck commands, env
 personas, services, and mocks. Secret values must come from the shell, CI secret store, or Cursor-style
 environment secrets; do not commit them. If `--execute-app` is passed and required setup is missing,
 PITHOS writes `verify/RUNTIME-SETUP.md` instead of starting the target app.
-Local live runtime setup defaults to `environment.sandbox: docker`; use `subprocess` only for tests or
-explicit local smoke checks.
+Live runtime setup defaults to `environment.sandbox: docker`; use `environment.sandbox: local`
+only in a disposable outer sandbox, ephemeral CI runner, or explicit local smoke check.
 Live verification does not load `.env.local` or other env files; users must explicitly export every
 required variable before running PITHOS.
 Use `pithos runtime init` to create a starter `.pithos/runtime.yaml`; it inspects `.env.example` for

docs/output.md

@@ -76,6 +76,10 @@ verification:
   execute_app: true
 ```
 
+`environment.sandbox` accepts `docker` or `local`. Docker is isolated and remains
+the default; local runs setup commands and live verification agents directly in
+the current environment, so use it only inside a disposable outer sandbox.
+
 Secret values should come from the shell, CI secret store, or Cursor-style
 environment secrets. The profile should name variables, not contain secret
 values.

pithos/cli.py

@@ -152,8 +152,9 @@ def main(argv: list[str] | None = None) -> int:
         choices=SANDBOX_MODES,
         default=os.environ.get("PITHOS_SANDBOX_MODE", DEFAULT_SANDBOX_MODE),
         help=(
-            "Static agent isolation mode: docker runs agents in containers; "
-            "local runs pi directly and trusts the outer environment"
+            "Sandbox mode for static scan and live verification: docker runs agents "
+            "and app commands in containers; local runs them directly and trusts the "
+            "outer environment"
         ),
     )
     p_run.add_argument("--votes", type=int, default=DEFAULT_REPO_VOTES)
@@ -358,6 +359,7 @@ def _cmd_run(args: argparse.Namespace) -> int:
             model=args.model,
             agent_env=agent_env,
             pi_config_dir=pi_config_dir,
+            sandbox_mode=args.sandbox_mode,
             event_sink=event_sink,
         )
     except KeyboardInterrupt:

pithos/runtime_plugins.py

@@ -12,13 +12,14 @@
 import asyncio
 import json
 import re
+import shutil
 import time
 import urllib.request
 from pathlib import Path
 from typing import Protocol
 
 from . import agent_image, docker_ops, sandbox as agent_sandbox
-from .agent import parse_xml_tag, run_agent
+from .agent import parse_xml_tag, run_agent, run_agent_process
 from .events import NULL_EVENT_SINK
 from .runtime_artifacts import (
     RUNTIME_BLOCKED,
@@ -946,6 +947,34 @@ def _run_pi_agent(
             return self._blocked(finding, plan, "verification agent provider/model is missing")
         with EphemeralAppSandbox(self.profile, finding_dir) as app:
             assert app.repo_copy is not None
+            assert app.root is not None
+            transcript = finding_dir / "live-agent-transcript.jsonl"
+            if app.backend == "local":
+                result = asyncio.run(
+                    run_agent_process(
+                        _localize_live_agent_paths(
+                            _live_agent_prompt(self.profile, finding),
+                            repo_path=app.repo_copy,
+                            artifacts_path=finding_dir,
+                        ),
+                        command_prefix=["pi"],
+                        provider=provider,
+                        model=model,
+                        session_dir=str(app.root / ".pi-sessions"),
+                        cwd=str(app.root),
+                        env=_local_pi_env(app, agent_cfg),
+                        transcript_path=str(transcript),
+                        progress_prefix=f"[verify {finding.id}]",
+                        tools=["read", "bash"],
+                        system_prompt=_localize_live_agent_paths(
+                            _live_agent_system_prompt(),
+                            repo_path=app.repo_copy,
+                            artifacts_path=finding_dir,
+                        ),
+                        event_sink=getattr(self, "event_sink", NULL_EVENT_SINK),
+                    )
+                )
+                return self._verdict_from_agent_result(finding, finding_dir, plan, result)
             mounts = agent_sandbox.provider_mounts(
                 Path(str(agent_cfg["pi_config_dir"])) if agent_cfg.get("pi_config_dir") else None
             )
@@ -968,7 +997,6 @@ def _run_pi_agent(
                 mounts=mounts,
             )
             try:
-                transcript = finding_dir / "live-agent-transcript.jsonl"
                 result = asyncio.run(
                     run_agent(
                         _live_agent_prompt(self.profile, finding),
@@ -984,6 +1012,15 @@ def _run_pi_agent(
                 )
             finally:
                 docker_ops.rm(container)
+        return self._verdict_from_agent_result(finding, finding_dir, plan, result)
+
+    def _verdict_from_agent_result(
+        self,
+        finding: RuntimeFinding,
+        finding_dir: Path,
+        plan: VerificationPlan,
+        result,
+    ) -> RuntimeVerdict:
         output = result.find_tagged_message("runtime_verdict_json")
         (finding_dir / "live-agent-output.txt").write_text(
             result.last_assistant_message or output,
@@ -1223,6 +1260,30 @@ def _unsafe_without_mock(profile: AppRuntimeProfile, finding: RuntimeFinding) ->
     return False
 
 
+def _local_pi_env(app: EphemeralAppSandbox, agent_cfg: dict[str, object]) -> dict[str, str]:
+    env = app.env()
+    env["PI_OFFLINE"] = "1"
+    env["PI_SKIP_VERSION_CHECK"] = "1"
+    env["PI_TELEMETRY"] = "0"
+    if agent_cfg.get("pi_config_dir"):
+        assert app.root is not None
+        pi_config_dir = Path(str(agent_cfg["pi_config_dir"]))
+        target = app.root / "home" / ".pi" / "agent"
+        target.mkdir(parents=True, exist_ok=True)
+        for name in ("auth.json", "models.json"):
+            source = pi_config_dir / name
+            if source.is_file():
+                shutil.copy2(source, target / name)
+        env["HOME"] = str(app.root / "home")
+    return env
+
+
+def _localize_live_agent_paths(text: str, *, repo_path: Path, artifacts_path: Path) -> str:
+    return text.replace("/work/repo", str(repo_path)).replace(
+        "/work/artifacts", str(artifacts_path)
+    )
+
+
 def _extract_verdict_payload(text: str) -> dict[str, object] | None:
     raw = parse_xml_tag(text, "runtime_verdict_json") or text.strip()
     if raw.startswith("```"):

pithos/runtime_sandbox.py

@@ -19,6 +19,10 @@
 from . import agent_image, docker_ops, sandbox as agent_sandbox
 from .runtime_profile import AppRuntimeProfile, required_env_names
 
+RUNTIME_SANDBOX_BACKENDS = ("docker", "local")
+LEGACY_RUNTIME_SANDBOX_BACKENDS = {"subprocess": "local"}
+RUNTIME_SANDBOX_ENV = "PITHOS_RUNTIME_SANDBOX_BACKEND"
+
 
 @dataclass(frozen=True)
 class CommandResult:
@@ -276,16 +280,7 @@ def _write_command_log(self, result: CommandResult) -> None:
 
     @property
     def backend(self) -> str:
-        raw = (
-            self.profile.environment.get("sandbox")
-            or self.profile.environment.get("backend")
-            or os.environ.get("PITHOS_RUNTIME_SANDBOX_BACKEND")
-            or "docker"
-        )
-        backend = str(raw)
-        if backend not in {"docker", "subprocess"}:
-            raise RuntimeError(f"unsupported runtime sandbox backend: {backend}")
-        return backend
+        return normalize_runtime_sandbox_backend(runtime_sandbox_backend_value(self.profile))
 
     def _ensure_container(self) -> str:
         if self._container:
@@ -375,6 +370,39 @@ def _resolve_env_value(value: str) -> str:
     return value
 
 
+def runtime_sandbox_backend_value(
+    profile: AppRuntimeProfile, *, default: str | None = None
+) -> object:
+    return (
+        profile.environment.get("sandbox")
+        or profile.environment.get("backend")
+        or os.environ.get(RUNTIME_SANDBOX_ENV)
+        or default
+        or "docker"
+    )
+
+
+def normalize_runtime_sandbox_backend(raw: object) -> str:
+    backend = str(raw).strip().lower()
+    backend = LEGACY_RUNTIME_SANDBOX_BACKENDS.get(backend, backend)
+    if backend not in RUNTIME_SANDBOX_BACKENDS:
+        allowed = ", ".join(RUNTIME_SANDBOX_BACKENDS)
+        raise RuntimeError(f"unsupported runtime sandbox backend: {raw}; choose {allowed}")
+    return backend
+
+
+def legacy_runtime_sandbox_warning(raw: object) -> str | None:
+    backend = str(raw).strip().lower()
+    if backend in LEGACY_RUNTIME_SANDBOX_BACKENDS:
+        replacement = LEGACY_RUNTIME_SANDBOX_BACKENDS[backend]
+        return (
+            f"{backend} runtime sandbox backend is deprecated; use {replacement}. "
+            f"{replacement} runs app commands and live agents in the current environment "
+            "and is not isolated."
+        )
+    return None
+
+
 def _service_commands(services: dict[str, Any]) -> list[tuple[str, str]]:
     commands: list[tuple[str, str]] = []
     for name, value in services.items():

pithos/runtime_verifier.py

@@ -18,6 +18,11 @@
 from .runtime_artifacts import RuntimeFinding, RuntimeVerdict, write_runtime_json
 from .runtime_plugins import select_plugin
 from .runtime_profile import AppRuntimeProfile, load_runtime_profile, required_env_names
+from .runtime_sandbox import (
+    legacy_runtime_sandbox_warning,
+    normalize_runtime_sandbox_backend,
+    runtime_sandbox_backend_value,
+)
 
 
 @dataclass(frozen=True)
@@ -55,6 +60,7 @@ def run_verify_repo(
     model: str | None = None,
     agent_env: dict[str, str] | None = None,
     pi_config_dir: Path | None = None,
+    sandbox_mode: str | None = None,
     event_sink: EventSinkLike = NULL_EVENT_SINK,
 ) -> RuntimeVerifyResult:
     start = time.time()
@@ -79,6 +85,7 @@ def run_verify_repo(
         allow_inferred_runtime=allow_inferred_runtime,
         provider=provider,
         model=model,
+        sandbox_mode=sandbox_mode,
     )
     if execute_app and not preflight["ready"]:
         _write_setup_guide(out_dir / "RUNTIME-SETUP.md", profile, preflight)
@@ -104,8 +111,12 @@ def run_verify_repo(
             "pi_config_dir": str(pi_config_dir) if pi_config_dir else None,
             "auth_env_keys": sorted((agent_env or {}).keys()),
         }
+    environment = dict(profile.environment)
+    if preflight["sandbox_backend"] in {"docker", "local"}:
+        environment["sandbox"] = preflight["sandbox_backend"]
     profile = replace(
         profile,
+        environment=environment,
         verification={
             **profile.verification,
             "execute_app": execute_app,
@@ -326,23 +337,32 @@ def _environment_preflight(
     allow_inferred_runtime: bool,
     provider: str | None,
     model: str | None,
+    sandbox_mode: str | None,
 ) -> dict[str, Any]:
     required = required_env_names(profile)
     missing = [name for name in required if not os.environ.get(name)]
     issues: list[str] = []
     warnings: list[str] = []
-    backend = str(
-        profile.environment.get("sandbox")
-        or profile.environment.get("backend")
-        or os.environ.get("PITHOS_RUNTIME_SANDBOX_BACKEND")
-        or "docker"
-    )
-    if execute_app and backend not in {"docker", "subprocess"}:
-        issues.append(f"unsupported runtime sandbox backend: {backend}")
+    raw_backend = runtime_sandbox_backend_value(profile, default=sandbox_mode)
+    try:
+        backend = normalize_runtime_sandbox_backend(raw_backend)
+    except RuntimeError:
+        backend = str(raw_backend)
+        if execute_app:
+            issues.append(
+                f"unsupported runtime sandbox backend: {raw_backend}; choose docker or local"
+            )
     if execute_app and backend == "docker" and not shutil.which("docker"):
         issues.append("Docker runtime sandbox requested but docker is not installed")
-    if execute_app and backend == "subprocess":
-        warnings.append("subprocess runtime backend is for local tests only and is not isolated")
+    if execute_app and backend == "local":
+        legacy_warning = legacy_runtime_sandbox_warning(raw_backend)
+        warnings.append(
+            legacy_warning
+            or (
+                "local runtime sandbox runs app commands and live agents in the current "
+                "environment and is not isolated"
+            )
+        )
     if execute_app and profile.inferred and not allow_inferred_runtime:
         issues.append(
             "live verification requires an explicit .pithos/runtime.yaml profile "

tests/test_cli.py

@@ -63,6 +63,7 @@ def fake_verify(**kwargs):
         assert kwargs["triage_path"] == triage_dir / "TRIAGE.json"
         assert kwargs["results_dir"] == triage_dir / "verify"
         assert kwargs["execute_app"] is False
+        assert kwargs["sandbox_mode"] == "docker"
         return VerifyResult()
 
     monkeypatch.setattr(cli, "_resolve_agent_env", lambda provider, pi_config_dir=None: ({}, None))
@@ -120,7 +121,12 @@ async def fake_scan(**kwargs):
     monkeypatch.setattr(cli, "_provider_auth_status", lambda provider, pi_config_dir=None: True)
     monkeypatch.setattr(cli, "resolve_repo_source", lambda *a, **k: Source())
     monkeypatch.setattr(cli, "run_repo_static", fake_scan)
-    monkeypatch.setattr(cli, "run_verify_repo", lambda **kwargs: VerifyResult())
+
+    def fake_verify(**kwargs):
+        assert kwargs["sandbox_mode"] == "local"
+        return VerifyResult()
+
+    monkeypatch.setattr(cli, "run_verify_repo", fake_verify)
 
     rc = cli.main(
         [
@@ -285,6 +291,7 @@ def fake_verify(**kwargs):
         assert kwargs["allow_inferred_runtime"] is True
         assert kwargs["provider"] == "anthropic"
         assert kwargs["model"] == "claude-sonnet-4-5"
+        assert kwargs["sandbox_mode"] == "docker"
         return VerifyResult()
 
     monkeypatch.setattr(cli, "_resolve_agent_env", lambda provider, pi_config_dir=None: ({}, None))

tests/test_runtime_sandbox.py

@@ -15,7 +15,7 @@ def test_sandbox_uses_explicit_env_and_runs_install(tmp_path, monkeypatch):
     profile = AppRuntimeProfile(
         repo_path=repo,
         install_command=f"python -c \"import os; open('{marker}', 'w').write(os.environ['FROM_FILE'])\"",
-        environment={"sandbox": "subprocess"},
+        environment={"sandbox": "local"},
         env_files=[".env.test"],
         env={"INLINE_VALUE": "inline", "FROM_FILE": "${FROM_FILE}"},
         required_env=["REQUIRED_TOKEN"],