|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Babysit cycle for the three gpt-5.6-terra reasoning-effort experiments. |
| 3 | +
|
| 4 | +One cycle = poll -> classify -> report -> (delete infra) -> (re-top-up). |
| 5 | +
|
| 6 | +Targets live in config.json so they survive a container restart (see PLAN.md). |
| 7 | +Non-CUA cells are topped up directly; CUA cells are left to cua_fill.py, which |
| 8 | +respects the runbook 3.6 concurrency cap. |
| 9 | +
|
| 10 | +Degenerate trials are REPORTED but never auto-deleted: in the LOW arm short |
| 11 | +trajectories are the treatment effect being measured, and systematically |
| 12 | +pruning them biases pass@k (runbook 4b). |
| 13 | +
|
| 14 | +Usage: babysit.py [--delete] [--topup] |
| 15 | +""" |
| 16 | +import json, subprocess, os, sys, statistics, collections |
| 17 | + |
| 18 | +HERE = "/home/user/terra-run" |
| 19 | +ODDISH = "/home/user/oddish/oddish/.venv/bin/oddish" |
| 20 | +ARMS = ["LOW", "MEDIUM", "HIGH"] |
| 21 | +EXP_OF = {"LOW": "17b6f7d9", "MEDIUM": "c071229f", "HIGH": "a706e700"} |
| 22 | +CUA = {"excel-clone", "mastodon-clone", "s3-clone", "slack-clone"} |
| 23 | + |
| 24 | +CFG = json.load(open(f"{HERE}/config.json")) |
| 25 | +N_NONCUA, N_CUA, CUA_CAP = CFG["noncua_target"], CFG["cua_target"], CFG["cua_cap"] |
| 26 | + |
| 27 | + |
| 28 | +def load_env(): |
| 29 | + env = dict(os.environ) |
| 30 | + for line in open("/home/user/oddish/.env"): |
| 31 | + line = line.strip() |
| 32 | + if line and not line.startswith("#") and "=" in line: |
| 33 | + k, v = line.split("=", 1) |
| 34 | + env[k.strip()] = v.strip().strip('"').strip("'") |
| 35 | + return env |
| 36 | + |
| 37 | + |
| 38 | +def main(): |
| 39 | + do_delete, do_topup = "--delete" in sys.argv, "--topup" in sys.argv |
| 40 | + env = load_env() |
| 41 | + r = subprocess.run(["python3", f"{HERE}/poll_all.py"], capture_output=True, |
| 42 | + text=True, env=env, cwd=HERE, timeout=2400) |
| 43 | + print(r.stdout.strip() or r.stderr.strip()[:500]) |
| 44 | + st = json.load(open(f"{HERE}/state.json")) |
| 45 | + |
| 46 | + cells, now = st["cells"], st["polled_at"] |
| 47 | + tasks = sorted({k.split("|")[1] for k in cells}) |
| 48 | + noncua = [t for t in tasks if t not in CUA] |
| 49 | + cua = [t for t in tasks if t in CUA] |
| 50 | + cua_live = sum(c["PENDING"] for k, c in cells.items() |
| 51 | + if k.split("|")[1] in CUA and k.split("|")[2] != "nop") |
| 52 | + |
| 53 | + out = [f"# Babysit report — {now}", "", |
| 54 | + f"Phase **{CFG['phase']}** — targets: non-CUA **{N_NONCUA}**/cell, " |
| 55 | + f"CUA **{N_CUA}**/cell.", "", |
| 56 | + f"CUA in-flight (judge-consuming): **{cua_live}** / cap {CUA_CAP} " |
| 57 | + f"— {'OK' if cua_live <= CUA_CAP else 'OVER CAP'}", ""] |
| 58 | + |
| 59 | + grand = collections.Counter() |
| 60 | + for label, tlist, target in (("non-CUA", noncua, N_NONCUA), ("CUA", cua, N_CUA)): |
| 61 | + for arm in ARMS: |
| 62 | + rows = [(t, cells.get(f"{arm}|{t}|terra", |
| 63 | + {"VALID": 0, "PENDING": 0, "INFRA": 0, |
| 64 | + "OTHER": 0, "SKIPPED": 0, "pass": 0})) |
| 65 | + for t in tlist] |
| 66 | + v = sum(c["VALID"] for _, c in rows) |
| 67 | + p = sum(c["PENDING"] for _, c in rows) |
| 68 | + i = sum(c["INFRA"] + c["OTHER"] + c["SKIPPED"] for _, c in rows) |
| 69 | + ps = sum(c["pass"] for _, c in rows) |
| 70 | + grand.update({"valid": v, "pending": p, "infra": i, "pass": ps, |
| 71 | + f"{label}_valid": v, f"{label}_pass": ps}) |
| 72 | + done = sum(1 for _, c in rows if c["VALID"] >= target) |
| 73 | + out += [f"## {label} — {arm} ({EXP_OF[arm]})", "", |
| 74 | + f"valid={v}/{len(tlist)*target} pending={p} infra={i} " |
| 75 | + f"pass={ps} — {done}/{len(tlist)} tasks at target", "", |
| 76 | + "| task | valid | pending | infra | pass |", "|---|---|---|---|---|"] |
| 77 | + for t, c in rows: |
| 78 | + flag = "" if c["VALID"] >= target else " ⏳" |
| 79 | + out.append(f"| `{t}`{flag} | {c['VALID']} | {c['PENDING']} | " |
| 80 | + f"{c['INFRA']+c['OTHER']+c['SKIPPED']} | {c['pass']} |") |
| 81 | + out.append("") |
| 82 | + |
| 83 | + out += [f"**Fleet:** valid={grand['valid']} pending={grand['pending']} " |
| 84 | + f"infra={grand['infra']} pass={grand['pass']}", ""] |
| 85 | + |
| 86 | + bad = (st["trials"].get("INFRA", []) + st["trials"].get("OTHER", []) |
| 87 | + + st["trials"].get("SKIPPED", [])) |
| 88 | + if bad: |
| 89 | + out += ["## Infra / skipped — delete + rerun", ""] |
| 90 | + for tr in bad: |
| 91 | + out.append(f"- `{tr['id']}` {tr['arm']}/{tr['task']}/{tr['agent']} " |
| 92 | + f"[{tr['status']}] {tr['err']}") |
| 93 | + out.append("") |
| 94 | + |
| 95 | + short = [tr for tr in st["trials"].get("VALID", []) |
| 96 | + if tr["kind"] == "terra" and (tr["steps"] or 0) < 15] |
| 97 | + if short: |
| 98 | + out += ["## Degenerate candidates (<15 steps) — reported, not deleted", ""] |
| 99 | + for tr in sorted(short, key=lambda x: x["steps"] or 0): |
| 100 | + out.append(f"- `{tr['id']}` {tr['arm']}/{tr['task']} " |
| 101 | + f"steps={tr['steps']} reward={tr['reward']}") |
| 102 | + out.append("") |
| 103 | + |
| 104 | + allsteps = [tr["steps"] for tr in st["trials"].get("VALID", []) |
| 105 | + if tr["kind"] == "terra" and tr["steps"]] |
| 106 | + if allsteps: |
| 107 | + out.append(f"Step distribution (valid terra, n={len(allsteps)}): " |
| 108 | + f"min={min(allsteps)} p50={int(statistics.median(allsteps))} " |
| 109 | + f"max={max(allsteps)}") |
| 110 | + cost = sum(tr["cost"] or 0 for recs in st["trials"].values() for tr in recs |
| 111 | + if tr["kind"] == "terra") |
| 112 | + out.append(f"Terra spend so far: ${cost:,.2f}") |
| 113 | + |
| 114 | + report = "\n".join(out) + "\n" |
| 115 | + open(f"{HERE}/babysit-latest.md", "w").write(report) |
| 116 | + print(report) |
| 117 | + with open(f"{HERE}/hb-fleet.txt", "w") as f: |
| 118 | + f.write(f"{now} | FLEET phase={CFG['phase']} | valid={grand['valid']} " |
| 119 | + f"pending={grand['pending']} infra={grand['infra']} | " |
| 120 | + f"cua_inflight={cua_live}/{CUA_CAP} | ${cost:,.0f}\n") |
| 121 | + |
| 122 | + if do_delete and bad: |
| 123 | + denv = dict(env) |
| 124 | + denv["ODDISH_API_KEY"] = denv["ODDISH_ADMIN_API_KEY"] |
| 125 | + ids = [tr["id"] for tr in bad] |
| 126 | + for i in range(0, len(ids), 20): |
| 127 | + chunk = ids[i:i + 20] |
| 128 | + args = [ODDISH, "delete"] + sum([["-t", x] for x in chunk], []) + ["--json"] |
| 129 | + rr = subprocess.run(args, capture_output=True, text=True, env=denv, |
| 130 | + timeout=600) |
| 131 | + print(f"deleted {len(chunk)}: rc={rr.returncode} {rr.stdout[:200]}") |
| 132 | + |
| 133 | + if do_topup: |
| 134 | + # Only non-CUA here; CUA refills go through cua_fill.py's throttle. |
| 135 | + shortc = [] |
| 136 | + for arm in ARMS: |
| 137 | + for t in noncua: |
| 138 | + c = cells.get(f"{arm}|{t}|terra") |
| 139 | + held = sum(c[k] for k in ("VALID", "PENDING", "INFRA", "OTHER", |
| 140 | + "SKIPPED")) if c else 0 |
| 141 | + if held < N_NONCUA: |
| 142 | + shortc.append((arm, t, N_NONCUA - held)) |
| 143 | + if not shortc: |
| 144 | + print(f"topup: nothing short — all non-CUA cells hold {N_NONCUA}") |
| 145 | + for arm, t, gap in shortc: |
| 146 | + print(f"topup: {arm}/{t} short by {gap}") |
| 147 | + for attempt in range(1, 6): |
| 148 | + rr = subprocess.run( |
| 149 | + [ODDISH, "run", "-p", f"{HERE}/ds-{arm.lower()}", "-t", t, |
| 150 | + "-a", "codex", "-m", "openai/gpt-5.6-terra", |
| 151 | + "--n-trials", str(N_NONCUA), "-e", "modal", |
| 152 | + "-E", EXP_OF[arm], "--override-memory-mb", "65536", |
| 153 | + "--no-baseline-gate", |
| 154 | + "--ak", f"reasoning_effort={arm.lower()}", |
| 155 | + "--ae", f"ODDISH_EVAL_NONCE={arm}-{t}-{now}-{attempt}", |
| 156 | + "--background", "--force"], |
| 157 | + capture_output=True, text=True, env=env, timeout=400) |
| 158 | + if "new version" in rr.stdout.lower(): |
| 159 | + print(f" ABORT {arm}/{t}: submit wants a NEW TASK VERSION") |
| 160 | + break |
| 161 | + if "Task submitted!" in rr.stdout: |
| 162 | + line = [l.strip() for l in rr.stdout.splitlines() if "Trials:" in l] |
| 163 | + print(f" ok {line[0] if line else ''}") |
| 164 | + break |
| 165 | + print(f" attempt {attempt} failed") |
| 166 | + if attempt < 5: |
| 167 | + subprocess.run(["sleep", str(5 * attempt * attempt)]) |
| 168 | + |
| 169 | + |
| 170 | +if __name__ == "__main__": |
| 171 | + main() |
0 commit comments