swe: re-apply spec pre_install after env reset; keep CI shards on disk (#128)

The attributed nightly (28560892001, 87/500 unresolved with per-stage
log tails) decomposed into three classes, two of them ours:

1. Missing pre_install (48 instances). Task images bake the specs'
   pre_install tracked-file edits into /testbed at build time and
   prepare_env's `git reset --hard` reverts them. Without sphinx's
   `sed 's/pytest/pytest -rA/' tox.ini` the log parser sees zero
   per-test lines, so every sphinx run scored unresolved with exit 0
   ("33 passed ... congratulations" in the tail); without astropy's
   setuptools pin the editable reinstall resolves a modern build chain
   on Python 3.9 and dies. _prepare_tests now re-applies pre_install
   ahead of eval_commands/install.

2. Runner disk exhaustion (24 instances): per-instance task images
   (~1-2 GB each, never reused) filled the runner mid-shard —
   "failed to register layer: ... no space left on device". The
   example gains --rmi-after (drop an instance's image once its
   rollout persists) and the nightly frees ~20 GB of preinstalled
   toolchains up front.

3. The remaining 11 are genuine gold-patch test failures
   (astropy-7606/8707/8872, psf-requests-1766/1921/2317, five sphinx)
   matching the environment-drift sets reported upstream
   (SWE-bench/SWE-bench#294, #484); they stay visible pending an
   explicit exclusion decision after the next full run.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: Meirtz

.github/workflows/nightly-ci.yml

@@ -52,6 +52,14 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      # ~25 task images per shard exhaust the runner's default free
+      # space even with per-instance cleanup; drop the preinstalled
+      # toolchains we never use (~20 GB).
+      - name: free runner disk
+        run: |
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL
+          df -h /
+
       - uses: astral-sh/setup-uv@v5
         with:
           enable-cache: true
@@ -90,6 +98,7 @@ jobs:
           uv run python main.py \
             --ground-truth \
             --fail-on-unresolved \
+            --rmi-after \
             --bundle "$BUNDLE" \
             --docker-platform linux/amd64 \
             --dataset "$DATASET" \

examples/run-swe-rollouts/main.py

@@ -33,6 +33,7 @@
 import json
 import logging
 import os
+import subprocess
 import sys
 from pathlib import Path
 from typing import Any
@@ -215,6 +216,7 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
     parser.add_argument("--shard-index", type=int, default=0)
     parser.add_argument("--ground-truth", action="store_true")
     parser.add_argument("--fail-on-unresolved", action="store_true")
+    parser.add_argument("--rmi-after", action="store_true")
     parser.add_argument("--concurrency", type=int, default=1)
     parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1"))
     parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY", ""))
@@ -279,10 +281,19 @@ async def main(argv: list[str] | None = None) -> int:
     out_dir = Path(args.out)
     out_dir.mkdir(parents=True, exist_ok=True)
 
+    # Instance images are per-instance and never reused within a run;
+    # on CI runners, keeping ~25 of them exhausts the disk mid-shard.
+    images = {str(row["instance_id"]): dataset.image(row) for row in rows}
+
     def _persist(rollout: Any) -> None:
         (out_dir / f"{rollout.instance_id}.json").write_text(json.dumps(rollout.to_dict(), indent=2))
         verdict = "PASS" if rollout.resolved else (rollout.skipped or rollout.error or "FAIL")
         print(f"[{rollout.instance_id}] {verdict} ({rollout.duration_s:.1f}s)")
+        if args.rmi_after and rollout.instance_id in images:
+            subprocess.run(
+                ["docker", "rmi", "-f", images[rollout.instance_id]],
+                capture_output=True,
+            )
 
     print(f"selected {len(rows)} instance(s) (concurrency={args.concurrency}, ground_truth={args.ground_truth})")
     rollouts = await run_rollouts(

plugins/datasets/swebench/src/score.py

@@ -135,6 +135,12 @@ async def _apply_model_patch(patch: str, workdir: str, env: dict[str, str], time
 async def _prepare_tests(instance: dict, workdir: str, env: dict[str, str], timeout: float) -> tuple[bool, str]:
     specs = MAP_REPO_VERSION_TO_SPECS[instance["repo"]][instance["version"]]
     commands = [f"git config --global --add safe.directory {shlex.quote(workdir)}"]
+    # Task images bake `pre_install` edits into /testbed's tracked files
+    # at build time (sphinx: `sed 's/pytest/pytest -rA/' tox.ini` so the
+    # log parser sees per-test lines; astropy: a setuptools pin that the
+    # editable reinstall needs). `prepare_env`'s `git reset --hard`
+    # reverts those edits, so re-apply them before install/eval.
+    commands.extend(str(command) for command in specs.get("pre_install", []))
     commands.extend(str(command) for command in specs.get("eval_commands", []))
 
     install = str(specs.get("install") or "").strip()

tests/test_swe_env.py

@@ -183,3 +183,55 @@ async def test_score_apply_failure_is_attributed(monkeypatch) -> None:
     assert result["patch_applied"] is False
     assert result["failure_stage"] == "apply_patch"
     assert result["log_tail"] == "$ git apply\nerror: corrupt"
+
+
+async def test_prepare_tests_reapplies_pre_install_before_install(monkeypatch, tmp_path) -> None:
+    # Task images bake pre_install's tracked-file edits in at build time
+    # and prepare_env's `git reset --hard` reverts them — _prepare_tests
+    # must re-apply them ahead of the (re)install.
+    monkeypatch.setattr(
+        swe_score,
+        "MAP_REPO_VERSION_TO_SPECS",
+        {
+            "demo/demo": {
+                "1.0": {
+                    "pre_install": ["sed -i 's/pytest/pytest -rA/' tox.ini"],
+                    "eval_commands": ["export LANG=C"],
+                    "install": "python -m pip install -e .",
+                    "test_cmd": "pytest -rA",
+                }
+            }
+        },
+    )
+    ran: list[str] = []
+
+    async def record(command, workdir, env, timeout, *, conda=False):
+        ran.append(command)
+        return 0, "", False
+
+    monkeypatch.setattr(swe_score, "_run", record)
+    monkeypatch.setattr(swe_score, "_remove_untracked_paths", _async(None))
+
+    ok, log = await swe_score._prepare_tests(
+        {
+            "repo": "demo/demo",
+            "version": "1.0",
+            "base_commit": "abc",
+            "test_patch": (
+                "diff --git a/tests/t.py b/tests/t.py\n"
+                "--- a/tests/t.py\n"
+                "+++ b/tests/t.py\n"
+                "@@ -1 +1 @@\n"
+                "-a\n"
+                "+b\n"
+            ),
+        },
+        str(tmp_path),
+        {},
+        60.0,
+    )
+
+    assert ok and log == ""
+    sed_at = next(i for i, c in enumerate(ran) if c.startswith("sed "))
+    install_at = next(i for i, c in enumerate(ran) if "pip install" in c)
+    assert sed_at < install_at