tests i guess

Author: sneakers-the-rat

improvement/tests/test_improve.py

@@ -1,30 +1,25 @@
 """Unit tests for the Oracle's deterministic machinery.
 
-No real model: the generator's two model calls (choose a path, grow the code)
-are driven by small scripted fakes so the loop's logic — path coercion,
-truncation-vs-error classification, the write→continue→fix progression — is
-exercised directly and fast. Real generation is covered in test_integration.py.
+No real model: the model calls (name a file, grow a bite of code, summarise) are
+driven by small scripted fakes so the orchestration — issue shuffling,
+issue/source interleaving, the write→continue→fix progression, and accumulating
+MULTIPLE files across inspirations — is exercised directly and fast. Real
+generation is covered in test_integration.py.
 """
 from __future__ import annotations
 
-import ast
+import re
 
 from oracle import improve
 
 
-# A fake model keyed on which step is asking (we control the prompt text, so
-# this is test-side dispatch, not production parsing).
-def oracle_fake(*, path="src/mechanism.py", write="", cont="", fix="", reason="A bold vision"):
+def model(route):
+    """Wrap route(messages)->str into a fake generate(), recording calls."""
     calls = []
 
     def generate(messages, *, num_predict=None, temperature=None):
         calls.append(messages)
-        u = messages[-1]["content"]
-        if "Existing Python modules" in u:             return path
-        if "electrifying sentence" in u:               return reason
-        if "continues from exactly where" in u:        return cont
-        if "does not parse" in u:                      return fix
-        return write
+        return route(messages)
 
     generate.calls = calls
     return generate
@@ -33,6 +28,33 @@ def generate(messages, *, num_predict=None, temperature=None):
 BIG = 10 ** 9  # effectively no deadline for the fast unit loops
 
 
+# --- issue shuffling: ALL issues surface, in a wandering order ---------------
+def test_collect_issue_list_includes_every_issue(monkeypatch):
+    fake = [{"number": i, "title": f"t{i}", "body": "", "labels": []} for i in range(6)]
+    monkeypatch.setattr(improve, "_fetch_issues", lambda: fake)
+    items = improve.collect_issue_list()
+    assert len(items) == 6
+    for i in range(6):
+        assert any(f"#{i} t{i}" in it for it in items)   # not just one issue — all of them
+
+
+def test_collect_issue_list_actually_shuffles(monkeypatch):
+    fake = [{"number": i, "title": f"t{i}", "body": "", "labels": []} for i in range(6)]
+    monkeypatch.setattr(improve, "_fetch_issues", lambda: fake)
+    orders = set()
+    for seed in range(6):
+        improve._RNG.seed(seed)
+        items = improve.collect_issue_list()
+        orders.add(tuple(int(re.search(r"#(\d+)", it).group(1)) for it in items))
+    assert len(orders) > 1   # different seeds → different orders → it really shuffles
+
+
+def test_interleave_alternates_then_trails():
+    assert improve._interleave([1, 3], [2, 4]) == [1, 2, 3, 4]
+    assert improve._interleave([1], [2, 4, 6]) == [1, 2, 4, 6]
+    assert improve._interleave([], [2]) == [2]
+
+
 # --- path coercion -----------------------------------------------------------
 def test_bare_directory_becomes_a_file(scratch):
     scratch()
@@ -45,145 +67,133 @@ def test_traversal_is_contained(scratch):
     assert improve.coerce_to_src("../../etc/passwd", "data").is_relative_to(improve.SRC_DIR)
 
 
-def test_absolute_path_is_contained(scratch):
-    scratch()
-    assert improve.coerce_to_src("/etc/shadow", "data").is_relative_to(improve.SRC_DIR)
-
-
-# --- content validation ------------------------------------------------------
-def test_valid_python_passes(scratch):
-    scratch()
-    assert improve.content_problems(improve.SRC_DIR / "m.py", "def f():\n    return 1\n") == []
-
-
+# --- content validation + code extraction ------------------------------------
 def test_broken_python_is_flagged(scratch):
     scratch()
     probs = improve.content_problems(improve.SRC_DIR / "m.py", "def f():\n    return +\n")
     assert len(probs) == 1 and "syntax error" in probs[0].lower()
 
 
-def test_markdown_is_not_syntax_checked(scratch):
-    scratch()
-    assert improve.content_problems(improve.SRC_DIR / "notes.md", "def not python (((") == []
-
-
-# --- code extraction (model wraps in fences despite instructions) ------------
-def test_strip_code_fenced():
+def test_strip_code_fenced_and_dangling():
     assert improve._strip_code("```python\nX = 1\n```") == "X = 1"
-
-
-def test_strip_code_dangling_open_fence():
     assert improve._strip_code("```python\nX = 1") == "X = 1"
-
-
-def test_strip_code_raw():
     assert improve._strip_code("X = 1\n") == "X = 1"
 
 
-# --- truncation vs. logic error ---------------------------------------------
-def test_truncation_detected_for_unclosed_bracket():
-    code = "x = [1, 2,"
-    assert improve._is_truncation(code, improve._syntax_error(code))
-
-
-def test_truncation_detected_for_unterminated_paren_midfile():
-    code = "def f(start):\n    total = (start +"
-    assert improve._is_truncation(code, improve._syntax_error(code))
-
+def test_truncation_vs_logic_error():
+    assert improve._is_truncation("x = [1, 2,", improve._syntax_error("x = [1, 2,"))
+    broke = "def f(:\n    return 1"
+    assert not improve._is_truncation(broke, improve._syntax_error(broke))
 
-def test_logic_error_is_not_truncation():
-    code = "def f(:\n    return 1"   # malformed signature, complete file
-    e = improve._syntax_error(code)
-    assert e is not None and not improve._is_truncation(code, e)
 
-
-# --- choose the target file (a separate, constrained call) -------------------
+# --- choosing the target (informed by one inspiration) -----------------------
 def test_choose_target_extracts_path(scratch):
     scratch({"src/mechanism.py": ""})
-    t = improve.choose_target(lambda m, **k: "I shall forge src/mechanism.py today", "t", "s", "i")
+    t = improve.choose_target(lambda m, **k: "src/mechanism.py", "tree", ("issue", "#1 do x"), [])
     assert t.name == "mechanism.py" and t.is_relative_to(improve.SRC_DIR)
 
 
-def test_choose_target_can_create_a_new_file(scratch):
+def test_choose_target_can_create_new_file(scratch):
     scratch({"src/mechanism.py": ""})
-    t = improve.choose_target(lambda m, **k: "src/orrery.py", "t", "s", "i")
-    assert t.name == "orrery.py" and not t.exists()   # a NEW module, created later by write
-    assert t.is_relative_to(improve.SRC_DIR)
-
-
-def test_choose_target_defaults_on_junk(scratch):
-    scratch()
-    t = improve.choose_target(lambda m, **k: "no path here!!", "t", "s", "i")
-    assert t.suffix == ".py" and t.is_relative_to(improve.SRC_DIR)
+    t = improve.choose_target(lambda m, **k: "src/orrery.py", "tree", ("issue", "#1"), [])
+    assert t.name == "orrery.py" and not t.exists() and t.is_relative_to(improve.SRC_DIR)
 
 
-def test_choose_target_forces_py_extension(scratch):
+def test_choose_target_forces_py_and_defaults_on_junk(scratch):
     scratch()
-    t = improve.choose_target(lambda m, **k: "src/notes", "t", "s", "i")
-    assert t.suffix == ".py"
+    assert improve.choose_target(lambda m, **k: "src/notes", "t", ("issue", "x"), []).suffix == ".py"
+    assert improve.choose_target(lambda m, **k: "garbage", "t", ("issue", "x"), []).suffix == ".py"
 
 
-# --- grow the code: write → continue (truncated) / fix (broken) --------------
+# --- growing a file: write → continue (truncated) / fix (broken) -------------
 def test_grow_file_valid_on_first_write(scratch):
     scratch()
-    target = improve.SRC_DIR / "m.py"
-    g = oracle_fake(write="VALUE = 42\n")
-    code, last = improve.grow_file(g, target, "t", "s", "i", deadline=BIG, max_rounds=4)
-    assert improve.content_problems(target, code) == []
-    assert len(g.calls) == 1  # no repair needed
+    g = model(lambda m: "VALUE = 42\n")
+    code, last = improve.grow_file(g, improve.SRC_DIR / "m.py", "t", ("issue", "x"), "",
+                                   deadline=BIG, max_rounds=4)
+    assert improve.content_problems(improve.SRC_DIR / "m.py", code) == [] and len(g.calls) == 1
 
 
 def test_grow_file_continues_a_truncated_draft(scratch):
     scratch()
-    target = improve.SRC_DIR / "m.py"
-    g = oracle_fake(write="def tooth_count(start=13):\n    total = (start +",
-                    cont=" 1)\n    return total\n")
-    code, last = improve.grow_file(g, target, "t", "s", "i", deadline=BIG, max_rounds=4)
-    assert improve.content_problems(target, code) == []
+
+    def route(m):
+        if "continues from exactly where" in m[-1]["content"]:
+            return " 1)\n    return total\n"
+        return "def tooth_count(start=13):\n    total = (start +"
+
+    code, _ = improve.grow_file(model(route), improve.SRC_DIR / "m.py", "t", ("issue", "x"), "",
+                                deadline=BIG, max_rounds=4)
+    assert improve.content_problems(improve.SRC_DIR / "m.py", code) == []
     assert "return total" in code
 
 
 def test_grow_file_fixes_a_broken_draft(scratch):
     scratch()
-    target = improve.SRC_DIR / "m.py"
-    g = oracle_fake(write="def f(:\n    return 1", fix="def f():\n    return 1\n")
-    code, last = improve.grow_file(g, target, "t", "s", "i", deadline=BIG, max_rounds=4)
-    assert improve.content_problems(target, code) == []
-    # it took the FIX path (not continue), proving the classification routed correctly
-    assert any("does not parse" in m[-1]["content"] for m in g.calls)
 
+    def route(m):
+        return "def f():\n    return 1\n" if "does not parse" in m[-1]["content"] else "def f(:\n    return 1"
 
-# --- orchestration -----------------------------------------------------------
-def test_generate_improvement_end_to_end(scratch):
-    scratch({"src/mechanism.py": ""})
-    g = oracle_fake(path="src/mechanism.py", write="VALUE = 42\n", reason="Forge the back dial")
-    reason, files, last, valid = improve.generate_improvement(g, "t", "s", "i", deadline_seconds=999)
-    assert valid and len(files) == 1
-    target, code = files[0]
-    assert target.name == "mechanism.py" and code.strip() == "VALUE = 42"
-    assert reason == "Forge the back dial"
-    written = improve.write_blocks(files)
-    ast.parse((improve.SRC_DIR / "mechanism.py").read_text())
-    improve.write_pr_outputs(reason, written, valid=valid)
-    assert improve.PR_TITLE_PATH.read_text().strip()
-    assert "Vision" in improve.PR_BODY_PATH.read_text()
-
-
-def test_generate_improvement_reports_invalid_when_unrepairable(scratch):
-    scratch()
-    # always-broken write, and continue/fix also broken → never parses
-    g = oracle_fake(write="def f(:", fix="def f(:", cont="def f(:")
+    g = model(route)
+    code, _ = improve.grow_file(g, improve.SRC_DIR / "m.py", "t", ("issue", "x"), "",
+                                deadline=BIG, max_rounds=4)
+    assert improve.content_problems(improve.SRC_DIR / "m.py", code) == []
+    assert any("does not parse" in m[-1]["content"] for m in g.calls)   # took the FIX path
+
+
+# --- orchestration: MANY files across MANY inspirations ----------------------
+def test_generates_multiple_files_across_inspirations(scratch):
+    scratch({"src/mech.py": "X = 1\n"})
+
+    def route(m):
+        u = m[-1]["content"]
+        if "electrifying sentence" in u:
+            return "A vision spanning several files"
+        if "Pick the ONE file" in u:
+            return "src/from_issue.py" if "An open issue" in u else "src/from_source.py"
+        return "VALUE = 1\n"   # every write/fix produces valid code
+
+    g = model(route)
     reason, files, last, valid = improve.generate_improvement(
-        g, "t", "s", "i", deadline_seconds=999, max_rounds=3)
-    assert files and valid is False  # ships best effort, flagged not-valid
+        g, "tree", [("src/mech.py", "X = 1\n")], ["#7 forge a gear", "#9 wind the dial"],
+        deadline_seconds=999, max_files=5)
+    rels = {t.name for t, _ in files}
+    assert {"from_issue.py", "from_source.py"} <= rels   # both issue- and source-driven files
+    assert len(files) >= 2 and valid
+    assert reason == "A vision spanning several files"
+
+
+def test_respects_max_files(scratch):
+    scratch()
+    route = lambda m: ("one" if "electrifying" in m[-1]["content"]
+                       else f"src/f{len(m)}.py" if "Pick the ONE file" in m[-1]["content"]
+                       else "V = 1\n")
+    # many inspirations, but cap at 1 file
+    _, files, _, _ = improve.generate_improvement(
+        model(route), "t", [], ["#1", "#2", "#3", "#4"], deadline_seconds=999, max_files=1)
+    assert len(files) == 1
+
+
+def test_reports_invalid_when_unrepairable(scratch):
+    scratch()
+
+    def route(m):
+        u = m[-1]["content"]
+        if "Pick the ONE file" in u: return "src/broken.py"
+        if "electrifying sentence" in u: return "a flawed vision"
+        return "def f(:"   # never parses, in any mode
+
+    _, files, _, valid = improve.generate_improvement(
+        model(route), "t", [], ["#1"], deadline_seconds=999, max_files=1, max_rounds=3)
+    assert files and valid is False   # ships best effort, flagged invalid
 
 
 def test_deadline_guard_makes_no_model_calls():
     def boom(*a, **k):
         raise AssertionError("model called past deadline")
-    clock = iter([0, 100])  # now()=0 sets deadline=50; next now()=100 is past it
-    reason, files, last, valid = improve.generate_improvement(
-        boom, "t", "s", "i", deadline_seconds=50, now=lambda: next(clock))
+    clock = iter([0, 100])   # now()=0 sets deadline=50; next now()=100 is past it
+    _, files, _, valid = improve.generate_improvement(
+        boom, "t", [("src/a.py", "x")], ["#1"], deadline_seconds=50, now=lambda: next(clock))
     assert files == [] and valid is False
 
 
@@ -195,15 +205,10 @@ def test_ollama_generate_builds_request_and_parses(monkeypatch):
     class FakeResp:
         def __enter__(self): return self
         def __exit__(self, *a): return False
-        def read(self): return json.dumps({"message": {"content": "the gear turns"}}).encode()
-
-    def fake_urlopen(req, timeout=None):
-        captured["url"] = req.full_url
-        captured["body"] = json.loads(req.data)
-        return FakeResp()
-
-    monkeypatch.setattr(improve.urllib.request, "urlopen", fake_urlopen)
-    out = improve.ollama_generate([{"role": "user", "content": "hi"}], temperature=0.4, num_predict=7)
-    assert out == "the gear turns"
-    assert captured["url"].endswith("/api/chat")
-    assert captured["body"]["options"]["num_predict"] == 7
+        def read(self): return json.dumps({"message": {"content": "ok"}}).encode()
+
+    monkeypatch.setattr(improve.urllib.request, "urlopen",
+                        lambda req, timeout=None: captured.update(body=json.loads(req.data),
+                                                                  url=req.full_url) or FakeResp())
+    assert improve.ollama_generate([{"role": "user", "content": "hi"}], num_predict=7) == "ok"
+    assert captured["url"].endswith("/api/chat") and captured["body"]["options"]["num_predict"] == 7

improvement/tests/test_integration.py

@@ -1,9 +1,10 @@
 """Integration tests that drive the ACTUAL local Ollama model.
 
-Kept fast and bounded: tiny ``num_predict`` (the file is grown iteratively, so
-small steps are fine), a short internal deadline, and a hard 60s per-test
-timeout. The ``model`` fixture pulls the model if it isn't present; if no server
-is reachable these skip.
+Kept fast and bounded: tiny ``num_predict`` (the file is grown iteratively),
+a short internal deadline, ``max_files=1`` so a test touches one file, and a
+hard 60s per-test timeout. The multi-file / interleave / shuffle orchestration
+is covered deterministically in test_improve.py. The ``model`` fixture pulls the
+model if absent; if no server is reachable these skip.
 
     pdm run pytest -m integration -v
 """
@@ -20,39 +21,31 @@
 DEADLINE = 45   # internal soft budget — stays under the 60s hard timeout
 
 
-def _ctx():
-    return improve.repo_tree(), improve.collect_source(), "(no open issues)"
-
-
 def test_real_choose_target_returns_a_py_path(model, quick_budget, scratch):
     scratch({"src/mechanism.py": "", "src/back_dial.py": ""})
-    target = improve.choose_target(improve.ollama_generate, *_ctx())
+    target = improve.choose_target(
+        improve.ollama_generate, improve.repo_tree(), ("issue", "#1 add a gear counter"), [])
     assert target.suffix == ".py" and target.is_relative_to(improve.SRC_DIR)
 
 
 def test_real_generates_valid_python(model, quick_budget, scratch):
     scratch({"src/mechanism.py": "", "src/main.py": ""})
-    tree, source, issues = _ctx()
     reason, files, last, valid = improve.generate_improvement(
-        improve.ollama_generate, tree, source, issues, deadline_seconds=DEADLINE)
+        improve.ollama_generate, improve.repo_tree(), improve.source_files(), [],
+        deadline_seconds=DEADLINE, max_files=1)
     assert files, f"no file produced; last response:\n{last[:400]}"
     target, code = files[0]
     assert target.suffix == ".py" and target.is_relative_to(improve.SRC_DIR)
     assert valid, f"{target.name} did not parse:\n{code[:400]}"
     ast.parse(code)
 
 
-def test_real_writes_code_despite_a_distracting_recipe(model, quick_budget, scratch):
-    """The regression empty-repo tests missed: an existing recipe used to derail
-    the model into copying it. It must still produce valid Python."""
-    scratch({
-        "src/recipes/banana_pudding.md": "# Banana Pudding\n\nEggs, butter, bananas, milk...\n",
-        "src/mechanism.py": "",
-    })
-    tree, source, issues = _ctx()
+def test_real_answers_an_issue_with_valid_python(model, quick_budget, scratch):
+    scratch({"src/mechanism.py": ""})
     reason, files, last, valid = improve.generate_improvement(
-        improve.ollama_generate, tree, source, issues, deadline_seconds=DEADLINE)
+        improve.ollama_generate, improve.repo_tree(), improve.source_files(),
+        ["#1 Add a function that counts the teeth on a gear"],
+        deadline_seconds=DEADLINE, max_files=1)
     assert files, f"no file produced; last:\n{last[:400]}"
     target, code = files[0]
-    assert target.suffix == ".py", f"wrote {target.name}, not Python"
-    assert valid, f"{target.name} did not parse:\n{code[:400]}"
+    assert target.suffix == ".py" and valid, f"{target.name} invalid:\n{code[:400]}"