Merge pull request #8 from superagent-ai/homanp/early-runtime-preflight

feat: Run live runtime preflight before static scan

Author: homanp

pithos/cli.py

@@ -16,10 +16,10 @@
 
 from .events import event_mode_from_env, make_event_sink
 from .exec_backend import SANDBOX_MODES
-from .repo_runner import DEFAULT_REPO_VOTES, run_repo_static
+from .repo_runner import DEFAULT_REPO_VOTES, allocate_repo_run_dir, run_repo_static
 from .repo_source import DEFAULT_REPO_CACHE, redact_secrets, resolve_repo_source
 from .runtime_template import generate_runtime_profile_template, write_runtime_profile_template
-from .runtime_verifier import run_verify_repo
+from .runtime_verifier import run_runtime_preflight, run_verify_repo
 from . import sandbox
 
 
@@ -331,6 +331,24 @@ def _cmd_run(args: argparse.Namespace) -> int:
     print(file=out)
 
     try:
+        run_out_dir = (
+            allocate_repo_run_dir(args.results_dir, repo_path) if args.execute_app else None
+        )
+        runtime_preflight = None
+        if args.execute_app:
+            runtime_preflight = run_runtime_preflight(
+                repo_path=repo_path,
+                profile_path=args.runtime_profile,
+                results_dir=run_out_dir / "verify" if run_out_dir else None,
+                execute_app=args.execute_app,
+                allow_inferred_runtime=args.allow_inferred_runtime,
+                provider=args.provider,
+                model=args.model,
+                agent_env=agent_env,
+                pi_config_dir=pi_config_dir,
+                sandbox_mode=args.sandbox_mode,
+                event_sink=event_sink,
+            )
         scan = asyncio.run(
             run_repo_static(
                 repo_path=repo_path,
@@ -344,6 +362,7 @@ def _cmd_run(args: argparse.Namespace) -> int:
                 allow_web=allow_web,
                 collect_advisories=args.advisories,
                 sandbox_mode=args.sandbox_mode,
+                out_dir=run_out_dir,
                 event_sink=event_sink,
             )
         )
@@ -360,6 +379,7 @@ def _cmd_run(args: argparse.Namespace) -> int:
             agent_env=agent_env,
             pi_config_dir=pi_config_dir,
             sandbox_mode=args.sandbox_mode,
+            preflight_result=runtime_preflight,
             event_sink=event_sink,
         )
     except KeyboardInterrupt:

pithos/repo_runner.py

@@ -60,6 +60,11 @@ def _safe_repo_name(path: Path) -> str:
     return re.sub(r"[^A-Za-z0-9_.-]+", "-", name).strip("-") or "repo"
 
 
+def allocate_repo_run_dir(results_dir: Path, repo_path: Path) -> Path:
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    return results_dir / _safe_repo_name(repo_path.resolve()) / timestamp
+
+
 def _git_metadata(repo: Path) -> dict[str, Any]:
     def run(*args: str) -> str | None:
         try:
@@ -485,13 +490,13 @@ async def run_repo_static(
     allow_web: bool = False,
     collect_advisories: bool = True,
     sandbox_mode: str = "docker",
+    out_dir: Path | None = None,
     event_sink: EventSinkLike = NULL_EVENT_SINK,
 ) -> RepoRunResult:
     start = time.time()
     repo_path = repo_path.resolve()
     backend = create_execution_backend(sandbox_mode)
-    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
-    out_dir = results_dir / _safe_repo_name(repo_path) / timestamp
+    out_dir = out_dir or allocate_repo_run_dir(results_dir, repo_path)
     out_dir.mkdir(parents=True, exist_ok=True)
 
     errors: list[str] = []

pithos/runtime_verifier.py

@@ -45,6 +45,46 @@ def status(self) -> str:
         return "completed"
 
 
+@dataclass(frozen=True)
+class RuntimePreflightResult:
+    repo_path: Path
+    out_dir: Path
+    profile: AppRuntimeProfile
+    preflight: dict[str, Any]
+    elapsed_s: float
+
+
+def run_runtime_preflight(
+    *,
+    repo_path: Path,
+    profile_path: Path | None = None,
+    results_dir: Path | None = None,
+    execute_app: bool = False,
+    allow_inferred_runtime: bool = False,
+    provider: str | None = None,
+    model: str | None = None,
+    agent_env: dict[str, str] | None = None,
+    pi_config_dir: Path | None = None,
+    sandbox_mode: str | None = None,
+    event_sink: EventSinkLike = NULL_EVENT_SINK,
+) -> RuntimePreflightResult:
+    """Run runtime setup checks without requiring static triage artifacts."""
+    return _run_runtime_preflight(
+        repo_path=repo_path,
+        profile_path=profile_path,
+        results_dir=results_dir,
+        execute_app=execute_app,
+        allow_inferred_runtime=allow_inferred_runtime,
+        provider=provider,
+        model=model,
+        agent_env=agent_env,
+        pi_config_dir=pi_config_dir,
+        sandbox_mode=sandbox_mode,
+        event_sink=event_sink,
+        emit_events=True,
+    )
+
+
 def run_verify_repo(
     *,
     triage_path: Path,
@@ -59,14 +99,17 @@ def run_verify_repo(
     agent_env: dict[str, str] | None = None,
     pi_config_dir: Path | None = None,
     sandbox_mode: str | None = None,
+    preflight_result: RuntimePreflightResult | None = None,
     event_sink: EventSinkLike = NULL_EVENT_SINK,
 ) -> RuntimeVerifyResult:
     start = time.time()
     triage_file = _resolve_triage_file(triage_path)
     repo = (repo_path or _repo_from_summary(triage_file) or Path.cwd()).expanduser().resolve()
-    profile = load_runtime_profile(repo, profile_path)
-    out_dir = results_dir or triage_file.parent / "runtime"
+    out_dir = results_dir or (preflight_result.out_dir if preflight_result else None)
+    out_dir = out_dir or triage_file.parent / "runtime"
     out_dir.mkdir(parents=True, exist_ok=True)
+    if preflight_result is not None:
+        execute_app = bool(preflight_result.preflight.get("execute_app", execute_app))
     event_sink.emit(
         "runtime_verification_started",
         {
@@ -77,52 +120,24 @@ def run_verify_repo(
         },
         stage="runtime",
     )
-    preflight = _environment_preflight(
-        profile,
-        execute_app=execute_app,
-        allow_inferred_runtime=allow_inferred_runtime,
-        provider=provider,
-        model=model,
-        sandbox_mode=sandbox_mode,
-    )
-    if execute_app and not preflight["ready"]:
-        _write_setup_guide(out_dir / "RUNTIME-SETUP.md", profile, preflight)
-        event_sink.emit(
-            "artifact_written",
-            artifact_data(out_dir / "RUNTIME-SETUP.md", artifact_type="runtime_setup"),
-            stage="runtime",
+    if preflight_result is None:
+        preflight_result = _run_runtime_preflight(
+            repo_path=repo,
+            profile_path=profile_path,
+            results_dir=out_dir,
+            execute_app=execute_app,
+            allow_inferred_runtime=allow_inferred_runtime,
+            provider=provider,
+            model=model,
+            agent_env=agent_env,
+            pi_config_dir=pi_config_dir,
+            sandbox_mode=sandbox_mode,
+            event_sink=event_sink,
+            emit_events=False,
         )
-    _write_runtime_artifact(
-        out_dir / "environment-summary.json",
-        preflight,
-        event_sink=event_sink,
-        stage="runtime",
-        artifact_type="runtime_environment_summary",
-    )
-
-    agent_config = {}
-    if execute_app and provider and model:
-        agent_config = {
-            "runner": "pi",
-            "provider": provider,
-            "model": model,
-            "pi_config_dir": str(pi_config_dir) if pi_config_dir else None,
-            "auth_env_keys": sorted((agent_env or {}).keys()),
-        }
-    environment = dict(profile.environment)
-    if preflight["sandbox_backend"] in {"docker", "local"}:
-        environment["sandbox"] = preflight["sandbox_backend"]
-    profile = replace(
-        profile,
-        environment=environment,
-        verification={
-            **profile.verification,
-            "execute_app": execute_app,
-            "allow_inferred_runtime": allow_inferred_runtime,
-            "preflight": preflight,
-            "agent": {**dict(profile.verification.get("agent") or {}), **agent_config},
-        },
-    )
+    profile = preflight_result.profile
+    preflight = preflight_result.preflight
+    repo = preflight_result.repo_path
     findings = _load_merged_findings(triage_file)
     if finding_ids:
         wanted = set(finding_ids)
@@ -211,6 +226,141 @@ def run_verify_repo(
     )
 
 
+def _run_runtime_preflight(
+    *,
+    repo_path: Path,
+    profile_path: Path | None,
+    results_dir: Path | None,
+    execute_app: bool,
+    allow_inferred_runtime: bool,
+    provider: str | None,
+    model: str | None,
+    agent_env: dict[str, str] | None,
+    pi_config_dir: Path | None,
+    sandbox_mode: str | None,
+    event_sink: EventSinkLike,
+    emit_events: bool,
+) -> RuntimePreflightResult:
+    start = time.time()
+    repo = Path(repo_path).expanduser().resolve()
+    profile = load_runtime_profile(repo, profile_path)
+    out_dir = results_dir or default_results_dir(repo)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    if emit_events:
+        event_sink.emit(
+            "runtime_preflight_started",
+            {
+                "repo": str(repo),
+                "results_dir": str(out_dir),
+                "execute_app": execute_app,
+            },
+            stage="runtime",
+        )
+    preflight = _environment_preflight(
+        profile,
+        execute_app=execute_app,
+        allow_inferred_runtime=allow_inferred_runtime,
+        provider=provider,
+        model=model,
+        sandbox_mode=sandbox_mode,
+    )
+    profile = _profile_with_runtime_context(
+        profile,
+        preflight,
+        execute_app=execute_app,
+        allow_inferred_runtime=allow_inferred_runtime,
+        provider=provider,
+        model=model,
+        agent_env=agent_env,
+        pi_config_dir=pi_config_dir,
+    )
+    _write_runtime_preflight_artifacts(
+        out_dir,
+        profile,
+        preflight,
+        execute_app=execute_app,
+        event_sink=event_sink,
+    )
+    elapsed = time.time() - start
+    if emit_events:
+        event_sink.emit(
+            "runtime_preflight_finished",
+            {
+                "ready": preflight["ready"],
+                "issues": len(preflight["issues"]),
+                "elapsed_s": elapsed,
+            },
+            stage="runtime",
+        )
+    return RuntimePreflightResult(
+        repo_path=repo,
+        out_dir=out_dir,
+        profile=profile,
+        preflight=preflight,
+        elapsed_s=elapsed,
+    )
+
+
+def _profile_with_runtime_context(
+    profile: AppRuntimeProfile,
+    preflight: dict[str, Any],
+    *,
+    execute_app: bool,
+    allow_inferred_runtime: bool,
+    provider: str | None,
+    model: str | None,
+    agent_env: dict[str, str] | None,
+    pi_config_dir: Path | None,
+) -> AppRuntimeProfile:
+    agent_config = {}
+    if execute_app and provider and model:
+        agent_config = {
+            "runner": "pi",
+            "provider": provider,
+            "model": model,
+            "pi_config_dir": str(pi_config_dir) if pi_config_dir else None,
+            "auth_env_keys": sorted((agent_env or {}).keys()),
+        }
+    environment = dict(profile.environment)
+    if preflight["sandbox_backend"] in {"docker", "local"}:
+        environment["sandbox"] = preflight["sandbox_backend"]
+    return replace(
+        profile,
+        environment=environment,
+        verification={
+            **profile.verification,
+            "execute_app": execute_app,
+            "allow_inferred_runtime": allow_inferred_runtime,
+            "preflight": preflight,
+            "agent": {**dict(profile.verification.get("agent") or {}), **agent_config},
+        },
+    )
+
+
+def _write_runtime_preflight_artifacts(
+    out_dir: Path,
+    profile: AppRuntimeProfile,
+    preflight: dict[str, Any],
+    *,
+    execute_app: bool,
+    event_sink: EventSinkLike,
+) -> None:
+    if execute_app and not preflight["ready"]:
+        _write_setup_guide(out_dir / "RUNTIME-SETUP.md", profile, preflight)
+        event_sink.emit(
+            "artifact_written",
+            artifact_data(out_dir / "RUNTIME-SETUP.md", artifact_type="runtime_setup"),
+            stage="runtime",
+        )
+    _write_runtime_artifact(
+        out_dir / "environment-summary.json",
+        preflight,
+        event_sink=event_sink,
+        stage="runtime",
+        artifact_type="runtime_environment_summary",
+    )
+
+
 def _write_runtime_artifact(
     path: Path,
     data: Any,