fix(p0-6): redact scenario error + stderr in e2b reports (#432)

* test(e2b-report): add failing scaffold for error redaction in summary + results.json (F008 S1)

* fix(e2b-report): redact scenario error in summary.md (F008 S2)

* fix(e2b-report): redact scenario error in results.json (F008 S3)

* fix(e2b-comprehensive): redact scenario error + stderr in transcripts (F008 S4)

Author: 0xmariowu

scripts/e2b/report.py

@@ -2,9 +2,12 @@
 
 from __future__ import annotations
 
+import json
 from datetime import datetime
 from pathlib import Path
 
+from autosearch.core.redact import redact
+from scripts.e2b.lib.reporter import redacted_json_ready
 from scripts.e2b.sandbox_runner import ScenarioResult
 
 
@@ -85,7 +88,8 @@ def render(results: list[ScenarioResult], summary: dict, output_dir: Path) -> st
             if r.report_length:
                 lines.append(f"  - 报告长度：{r.report_length} 字符")
             if r.error:
-                lines.append(f"  - ⚠️ 错误：`{r.error[:120]}`")
+                redacted_error = redact(str(r.error))
+                lines.append(f"  - ⚠️ 错误：`{redacted_error[:120]}`")
             # Notable details
             details = r.details
             if "pubmed_ok" in details:
@@ -105,7 +109,8 @@ def render(results: list[ScenarioResult], summary: dict, output_dir: Path) -> st
     if summary["failures"]:
         lines += ["## 失败场景", ""]
         for f in summary["failures"]:
-            lines.append(f"- **{f['id']} {f['name']}** (score={f['score']}): {f['error']}")
+            redacted_error = redact(str(f.get("error", "")))
+            lines.append(f"- **{f['id']} {f['name']}** (score={f['score']}): {redacted_error}")
         lines.append("")
 
     if summary.get("bonus_total", 0) > 0:
@@ -139,6 +144,16 @@ def render(results: list[ScenarioResult], summary: dict, output_dir: Path) -> st
     return report
 
 
+def render_results_json(results: list[ScenarioResult], summary: dict, output_dir: Path) -> str:
+    payload = {
+        "summary": summary,
+        "results": [r.to_dict() for r in results],
+    }
+    content = json.dumps(redacted_json_ready(payload), indent=2, ensure_ascii=False) + "\n"
+    (output_dir / "results.json").write_text(content, encoding="utf-8")
+    return content
+
+
 def _conclusion(summary: dict) -> str:
     r = summary["readiness"]
     score = summary["overall_score"]

scripts/e2b/run_comprehensive_tests.py

@@ -16,7 +16,6 @@
 
 import argparse
 import asyncio
-import json
 import os
 import sys
 import time
@@ -29,7 +28,7 @@
 sys.path.insert(0, str(ROOT))
 
 from scripts.e2b.evaluate import compute_summary  # noqa: E402
-from scripts.e2b.report import render  # noqa: E402
+from scripts.e2b.report import render, render_results_json  # noqa: E402
 from scripts.e2b.sandbox_runner import (  # noqa: E402
     ScenarioResult,
     _collect_keys,
@@ -406,6 +405,14 @@ async def run_scenario_in_sandbox(
                     await kill_sandbox(client, sandbox_id)
 
 
+def write_outputs(results: list[ScenarioResult], output_dir: Path) -> dict:
+    """Write comprehensive results without leaking scenario text secrets."""
+    summary = compute_summary(results)
+    render_results_json(results, summary, output_dir)
+    render(results, summary, output_dir)
+    return summary
+
+
 # ── Main ──────────────────────────────────────────────────────────────────────
 
 
@@ -476,16 +483,7 @@ async def main(argv: list[str] | None = None) -> int:
     output_dir.mkdir(parents=True, exist_ok=True)
 
     # Compute summary & write outputs
-    summary = compute_summary(list(results))
-    (output_dir / "results.json").write_text(
-        json.dumps(
-            {"summary": summary, "results": [r.to_dict() for r in results]},
-            indent=2,
-            ensure_ascii=False,
-        ),
-        encoding="utf-8",
-    )
-    render(list(results), summary, output_dir)
+    summary = write_outputs(list(results), output_dir)
 
     # Print final summary
     emoji = {"READY": "🟢", "BETA": "🟡", "NOT_READY": "🔴"}.get(summary["readiness"], "⚪")

tests/scripts/test_e2b_report_redact.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from scripts.e2b import report, run_comprehensive_tests
+from scripts.e2b.evaluate import compute_summary
+from scripts.e2b.sandbox_runner import ScenarioResult
+
+
+FAKE_API_KEY = "sk-" + "FAKEKEY1234567890abcdef"
+
+
+def _scenario_with_secret_error() -> ScenarioResult:
+    return ScenarioResult(
+        scenario_id="A1",
+        category="A",
+        name="fake_secret_error",
+        score=0,
+        passed=False,
+        details={},
+        error=f"request failed with {FAKE_API_KEY}",
+        duration_s=1.2,
+    )
+
+
+def _scenario_with_secret_transcript() -> ScenarioResult:
+    return ScenarioResult(
+        scenario_id="A2",
+        category="A",
+        name="fake_secret_transcript",
+        score=0,
+        passed=False,
+        details={
+            "stderr": f"stderr leaked {FAKE_API_KEY}",
+            "transcript": [
+                {"role": "assistant", "content": f"traceback leaked {FAKE_API_KEY}"},
+            ],
+        },
+        error=f"scenario failed with {FAKE_API_KEY}",
+        duration_s=2.3,
+    )
+
+
+def _summary_for(result: ScenarioResult) -> dict:
+    return compute_summary([result])
+
+
+def test_scenario_error_redacted_in_summary(tmp_path: Path) -> None:
+    scenario = _scenario_with_secret_error()
+
+    report.render([scenario], _summary_for(scenario), tmp_path)
+
+    content = (tmp_path / "summary.md").read_text(encoding="utf-8")
+    assert FAKE_API_KEY not in content
+    assert "[REDACTED]" in content
+
+
+def test_scenario_error_redacted_in_results_json(tmp_path: Path) -> None:
+    scenario = _scenario_with_secret_error()
+
+    report.render_results_json([scenario], _summary_for(scenario), tmp_path)
+
+    content = (tmp_path / "results.json").read_text(encoding="utf-8")
+    assert FAKE_API_KEY not in content
+    assert "[REDACTED]" in content
+
+
+def test_comprehensive_transcripts_redacted(tmp_path: Path) -> None:
+    scenario = _scenario_with_secret_transcript()
+
+    run_comprehensive_tests.write_outputs([scenario], tmp_path)
+
+    for path in (tmp_path / "summary.md", tmp_path / "results.json"):
+        content = path.read_text(encoding="utf-8")
+        assert FAKE_API_KEY not in content
+        assert "[REDACTED]" in content