Merge pull request #580 from morluto/agent/optimize-test-template-copy

perf(tests): hardlink immutable blobs in template copy

Author: morluto

tests/support/state.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import os
 import shutil
 import tempfile
 from collections.abc import Callable
@@ -97,14 +98,46 @@ def publish_template(
 
 
 def copy_template(template: Path, destination: Path) -> Path:
-    """Copy an immutable template into a new mutable per-test directory."""
+    """Copy an immutable template into a new mutable per-test directory.
+
+    Blobs under ``blobs/`` are content-addressed and immutable, so they are
+    hardlinked instead of copied.  This avoids copying ~7 MB of blob data per
+    test across 80+ composition tests — a significant I/O reduction.
+    """
 
     template = Path(template)
     destination = Path(destination)
     if not template.is_dir():
         raise FileNotFoundError(f"template directory does not exist: {template}")
     if destination.exists():
         raise FileExistsError(f"mutable test state already exists: {destination}")
-    destination.parent.mkdir(parents=True, exist_ok=True)
-    shutil.copytree(template, destination)
+    destination.mkdir(parents=True, exist_ok=True)
+
+    # Copy everything except the blobs directory with a normal recursive copy,
+    # then hardlink the content-addressed blobs.
+    for entry in os.scandir(template):
+        src = Path(entry.path)
+        dst = destination / entry.name
+        if entry.name == "blobs":
+            _hardlink_tree(src, dst)
+        elif entry.is_dir():
+            shutil.copytree(src, dst)
+        else:
+            shutil.copy2(src, dst)
     return destination
+
+
+def _hardlink_tree(src: Path, dst: Path) -> None:
+    """Recursively hardlink all files from *src* into *dst*."""
+
+    dst.mkdir(parents=True, exist_ok=True)
+    for entry in os.scandir(src):
+        s = Path(entry.path)
+        d = dst / entry.name
+        if entry.is_dir(follow_symlinks=False):
+            _hardlink_tree(s, d)
+        elif entry.is_file(follow_symlinks=False):
+            os.link(s, d)
+        else:
+            # Fallback for any unusual file types.
+            shutil.copy2(s, d)

tests/unit/support/test_copy_template.py

@@ -0,0 +1,62 @@
+"""Tests for copy_template hardlink optimization."""
+import os
+from pathlib import Path
+
+from tests.support.state import copy_template
+
+
+def test_copy_template_hardlinks_blobs(tmp_path: Path) -> None:
+    """Blobs should be hardlinked (not copied) for I/O efficiency."""
+    template = tmp_path / "template"
+    template.mkdir()
+    blob = template / "blobs" / "sha256" / "00" / "abc123"
+    blob.parent.mkdir(parents=True)
+    blob.write_bytes(b"blob content")
+    (template / "metadata.sqlite3").write_bytes(b"database")
+
+    dest = tmp_path / "destination"
+    copy_template(template, dest)
+
+    assert (dest / "blobs" / "sha256" / "00" / "abc123").read_bytes() == b"blob content"
+    template_inode = os.stat(blob).st_ino
+    dest_inode = os.stat(dest / "blobs" / "sha256" / "00" / "abc123").st_ino
+    assert template_inode == dest_inode, "blob should be hardlinked"
+
+    template_meta = os.stat(template / "metadata.sqlite3").st_ino
+    dest_meta = os.stat(dest / "metadata.sqlite3").st_ino
+    assert template_meta != dest_meta, "metadata should be copied, not hardlinked"
+
+
+def test_copy_template_preserves_all_files(tmp_path: Path) -> None:
+    """All non-blob files should be present in the destination."""
+    template = tmp_path / "template"
+    template.mkdir()
+    (template / "blobs").mkdir()
+    blob = template / "blobs" / "sha256" / "00" / "def456"
+    blob.parent.mkdir(parents=True)
+    blob.write_bytes(b"blob")
+    (template / "metadata.sqlite3").write_bytes(b"database")
+    (template / "metadata.sqlite3-shm").write_bytes(b"shm")
+    (template / "metadata.sqlite3-wal").write_bytes(b"wal")
+
+    dest = tmp_path / "destination"
+    copy_template(template, dest)
+
+    for name in ("metadata.sqlite3", "metadata.sqlite3-shm", "metadata.sqlite3-wal"):
+        assert (dest / name).exists(), f"{name} should be copied"
+
+
+def test_copy_template_raises_on_existing_destination(tmp_path: Path) -> None:
+    """copy_template should refuse to overwrite an existing destination."""
+    template = tmp_path / "template"
+    template.mkdir()
+    (template / "blobs").mkdir()
+    (template / "blobs" / "sha256" / "00" / "abc").parent.mkdir(parents=True)
+    (template / "blobs" / "sha256" / "00" / "abc").write_bytes(b"blob")
+
+    dest = tmp_path / "destination"
+    dest.mkdir()
+    import pytest
+
+    with pytest.raises(FileExistsError):
+        copy_template(template, dest)