|
| 1 | +#!/bin/bash |
| 2 | +# Run VMVM through all sandbox-bench suites with the correct tenant per suite. |
| 3 | +# |
| 4 | +# Tenant IDs are read from .env.meta (see .env.meta.example). |
| 5 | + |
| 6 | +set -euo pipefail |
| 7 | + |
| 8 | +# Load Meta-specific env vars |
| 9 | +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" |
| 10 | +if [[ -f "$SCRIPT_DIR/.env.meta" ]]; then |
| 11 | + set -a; source "$SCRIPT_DIR/.env.meta"; set +a |
| 12 | +else |
| 13 | + echo "ERROR: .env.meta not found. Copy .env.meta.example → .env.meta and fill in your values." >&2 |
| 14 | + exit 1 |
| 15 | +fi |
| 16 | + |
| 17 | +BENCH="$SCRIPT_DIR/venv/bin/sandbox-bench" |
| 18 | +OUTDIR="$SCRIPT_DIR/results" |
| 19 | +RUNS="${1:-3}" |
| 20 | + |
| 21 | +echo "=== VMVM sandbox-bench (${RUNS} runs per suite) ===" |
| 22 | +echo "" |
| 23 | + |
| 24 | +# 1) basic + performance |
| 25 | +echo "--- basic + performance ---" |
| 26 | +VMVM_TENANT_ID="${VMVM_TENANT_ID_BASIC:?Set VMVM_TENANT_ID_BASIC in .env.meta}" "$BENCH" run -p vmvm \ |
| 27 | + -s basic -s performance \ |
| 28 | + -n "$RUNS" \ |
| 29 | + -o "$OUTDIR/vmvm-basic-perf.json" |
| 30 | +echo "" |
| 31 | + |
| 32 | +# 2) competitive |
| 33 | +echo "--- competitive ---" |
| 34 | +VMVM_TENANT_ID="${VMVM_TENANT_ID_COMPETITIVE:?Set VMVM_TENANT_ID_COMPETITIVE in .env.meta}" "$BENCH" run -p vmvm \ |
| 35 | + -s competitive \ |
| 36 | + -n "$RUNS" \ |
| 37 | + -o "$OUTDIR/vmvm-competitive.json" |
| 38 | +echo "" |
| 39 | + |
| 40 | +# 3) swe |
| 41 | +echo "--- swe ---" |
| 42 | +VMVM_TENANT_ID="${VMVM_TENANT_ID_SWE:?Set VMVM_TENANT_ID_SWE in .env.meta}" "$BENCH" run -p vmvm \ |
| 43 | + -s swe \ |
| 44 | + -n "$RUNS" \ |
| 45 | + -o "$OUTDIR/vmvm-swe.json" |
| 46 | +echo "" |
| 47 | + |
| 48 | +# 4) environment |
| 49 | +echo "--- environment ---" |
| 50 | +VMVM_TENANT_ID="${VMVM_TENANT_ID_ENVIRONMENT:?Set VMVM_TENANT_ID_ENVIRONMENT in .env.meta}" "$BENCH" run -p vmvm \ |
| 51 | + -s environment \ |
| 52 | + -n "$RUNS" \ |
| 53 | + -o "$OUTDIR/vmvm-environment.json" |
| 54 | +echo "" |
| 55 | + |
| 56 | +# 5) Merge all suite results into one combined file |
| 57 | +echo "--- Merging results ---" |
| 58 | +"$SCRIPT_DIR/venv/bin/python3" - "$OUTDIR" <<'PYEOF' |
| 59 | +import json, sys, os |
| 60 | +
|
| 61 | +outdir = sys.argv[1] |
| 62 | +
|
| 63 | +files = [ |
| 64 | + ("vmvm-basic-perf.json", ["basic", "performance"]), |
| 65 | + ("vmvm-competitive.json", ["competitive"]), |
| 66 | + ("vmvm-swe.json", ["swe"]), |
| 67 | + ("vmvm-environment.json", ["environment"]), |
| 68 | +] |
| 69 | +
|
| 70 | +# Collect per-suite data from each run |
| 71 | +all_suite_results = {} |
| 72 | +all_capabilities = {} |
| 73 | +all_trace = [] |
| 74 | +total_time = 0.0 |
| 75 | +total_tool_calls = 0 |
| 76 | +total_friction = 0 |
| 77 | +total_errors = 0 |
| 78 | +error_messages = [] |
| 79 | +cold_start = 0.0 |
| 80 | +warm_start = None |
| 81 | +agent_spawn = None |
| 82 | +file_io_tp = None |
| 83 | +discoverability = 3.5 |
| 84 | +
|
| 85 | +for fname, expected_suites in files: |
| 86 | + path = os.path.join(outdir, fname) |
| 87 | + if not os.path.exists(path): |
| 88 | + print(f" SKIP {fname} (not found)") |
| 89 | + continue |
| 90 | +
|
| 91 | + with open(path) as f: |
| 92 | + data = json.load(f) |
| 93 | +
|
| 94 | + r = data["results"][0] |
| 95 | + print(f" {fname}: {r['total_time_seconds']:.1f}s, grade {r['grade']}, suites {r['suites_run']}") |
| 96 | +
|
| 97 | + total_time += r["total_time_seconds"] |
| 98 | + total_tool_calls += r["tool_calls"] |
| 99 | + total_friction += r["friction_points"] |
| 100 | + total_errors += r["errors"] |
| 101 | + error_messages.extend(r.get("error_messages", [])) |
| 102 | + all_trace.extend(r.get("trace", [])) |
| 103 | +
|
| 104 | + for suite_name, phases in r.get("suite_results", {}).items(): |
| 105 | + all_suite_results[suite_name] = phases |
| 106 | +
|
| 107 | + for cap, supported in r.get("capabilities", {}).items(): |
| 108 | + all_capabilities[cap] = supported |
| 109 | +
|
| 110 | + if r.get("cold_start_seconds", 0) > cold_start: |
| 111 | + cold_start = r["cold_start_seconds"] |
| 112 | + if r.get("warm_start_seconds") is not None: |
| 113 | + warm_start = r["warm_start_seconds"] |
| 114 | + if r.get("agent_spawn_seconds") is not None: |
| 115 | + agent_spawn = r["agent_spawn_seconds"] |
| 116 | + if r.get("file_io_throughput_mbps") is not None: |
| 117 | + file_io_tp = r["file_io_throughput_mbps"] |
| 118 | +
|
| 119 | +# Compute capability score |
| 120 | +cap_tested = len(all_capabilities) |
| 121 | +cap_supported = sum(1 for v in all_capabilities.values() if v) |
| 122 | +cap_score = cap_supported / cap_tested if cap_tested > 0 else 0.0 |
| 123 | +
|
| 124 | +# Build merged result using the scoring module |
| 125 | +suites_run = list(all_suite_results.keys()) |
| 126 | +
|
| 127 | +merged_result = { |
| 128 | + "provider": "vmvm", |
| 129 | + "success": True, |
| 130 | + "total_time_seconds": total_time, |
| 131 | + "tool_calls": total_tool_calls, |
| 132 | + "friction_points": total_friction, |
| 133 | + "errors": total_errors, |
| 134 | + "error_messages": error_messages, |
| 135 | + "estimated_cost_usd": 0.0, |
| 136 | + "sandbox_cost_usd": 0.0, |
| 137 | + "discoverability_score": discoverability, |
| 138 | + "score": 0.0, # will be recalculated |
| 139 | + "grade": "", |
| 140 | + "trace": all_trace, |
| 141 | + "suites_run": suites_run, |
| 142 | + "suite_results": all_suite_results, |
| 143 | + "capabilities": all_capabilities, |
| 144 | + "capability_score": cap_score, |
| 145 | + "cold_start_seconds": cold_start, |
| 146 | + "warm_start_seconds": warm_start, |
| 147 | + "agent_spawn_seconds": agent_spawn, |
| 148 | + "file_io_throughput_mbps": file_io_tp, |
| 149 | +} |
| 150 | +
|
| 151 | +# Recalculate score using the same algorithm as sandbox-bench |
| 152 | +MAX_TIME = 300; MAX_CALLS = 50; MAX_FRICTION = 5; MAX_ERRORS = 10; MAX_COST = 5.0 |
| 153 | +
|
| 154 | +def norm(v, mx): return min(1.0, v / mx) |
| 155 | +
|
| 156 | +has_caps = bool(all_capabilities) |
| 157 | +if has_caps: |
| 158 | + w = {"time": 0.25, "tool_calls": 0.10, "friction": 0.15, "errors": 0.20, "cost": 0.10, "discoverability": 0.10, "capabilities": 0.10} |
| 159 | +else: |
| 160 | + w = {"time": 0.30, "tool_calls": 0.15, "friction": 0.15, "errors": 0.20, "cost": 0.10, "discoverability": 0.10} |
| 161 | +
|
| 162 | +score = ( |
| 163 | + (1 - norm(total_time, MAX_TIME)) * w["time"] |
| 164 | + + (1 - norm(total_tool_calls, MAX_CALLS)) * w["tool_calls"] |
| 165 | + + (1 - norm(total_friction, MAX_FRICTION)) * w["friction"] |
| 166 | + + (1 - norm(total_errors, MAX_ERRORS)) * w["errors"] |
| 167 | + + (1 - norm(0.0, MAX_COST)) * w["cost"] |
| 168 | + + (discoverability / 5.0) * w["discoverability"] |
| 169 | +) |
| 170 | +if has_caps: |
| 171 | + score += cap_score * w["capabilities"] |
| 172 | +
|
| 173 | +score = round(score * 100, 1) |
| 174 | +if score >= 85: grade = "A" |
| 175 | +elif score >= 70: grade = "B" |
| 176 | +elif score >= 55: grade = "C" |
| 177 | +elif score >= 40: grade = "D" |
| 178 | +else: grade = "F" |
| 179 | +
|
| 180 | +merged_result["score"] = score |
| 181 | +merged_result["grade"] = grade |
| 182 | +
|
| 183 | +combined = { |
| 184 | + "config": { |
| 185 | + "providers": ["vmvm"], |
| 186 | + "suites": ["full"], |
| 187 | + "agent_mode": False, |
| 188 | + "model": "claude-opus-4", |
| 189 | + "runs": 3, |
| 190 | + }, |
| 191 | + "results": [merged_result], |
| 192 | +} |
| 193 | +
|
| 194 | +out_path = os.path.join(outdir, "vmvm-full.json") |
| 195 | +with open(out_path, "w") as f: |
| 196 | + json.dump(combined, f, indent=2) |
| 197 | +
|
| 198 | +print(f"\n Merged: {len(suites_run)} suites, {cap_supported}/{cap_tested} capabilities") |
| 199 | +print(f" Score: {score} / Grade: {grade}") |
| 200 | +print(f" Written to {out_path}") |
| 201 | +PYEOF |
| 202 | + |
| 203 | +echo "" |
| 204 | +echo "=== Done ===" |
0 commit comments