fix: make the GEPA skill-evolution path work end-to-end

evolution/core/fitness.py

@@ -104,18 +104,33 @@ def score(
         )
 
 
-def skill_fitness_metric(example: dspy.Example, prediction: dspy.Prediction, trace=None) -> float:
+def skill_fitness_metric(
+    example: dspy.Example,
+    prediction: dspy.Prediction,
+    trace=None,
+    pred_name=None,
+    pred_trace=None,
+):
     """DSPy-compatible metric function for skill optimization.
 
-    This is what gets passed to dspy.GEPA(metric=...).
-    Returns a float 0-1 score.
+    This is what gets passed to dspy.GEPA(metric=...). GEPA's
+    GEPAFeedbackMetric protocol calls it with (gold, pred, trace, pred_name,
+    pred_trace); MIPROv2 and direct holdout scoring call it with the first
+    two or three arguments only, so the extra parameters default to None.
+
+    Returns a float 0-1 score — except when GEPA requests predictor-level
+    feedback (pred_name is not None), where it returns
+    dspy.Prediction(score=..., feedback=...) with a deterministic hint about
+    which expected-behavior terms are missing, giving the reflection LM
+    something concrete to act on.
     """
     # The prediction should have an 'output' field with the agent's response
     agent_output = getattr(prediction, "output", "") or ""
     expected = getattr(example, "expected_behavior", "") or ""
-    task = getattr(example, "task_input", "") or ""
 
     if not agent_output.strip():
+        if pred_name is not None:
+            return dspy.Prediction(score=0.0, feedback="The response was empty.")
         return 0.0
 
     # Quick heuristic scoring (for speed during optimization)
@@ -129,11 +144,27 @@ def skill_fitness_metric(example: dspy.Example, prediction: dspy.Prediction, tra
     # Simple keyword overlap as a fast proxy
     expected_words = set(expected_lower.split())
     output_words = set(output_lower.split())
+    missing: list[str] = []
     if expected_words:
         overlap = len(expected_words & output_words) / len(expected_words)
         score = 0.3 + (0.7 * overlap)
+        missing = sorted(
+            w for w in (expected_words - output_words) if len(w) > 4
+        )[:8]
+
+    score = min(1.0, max(0.0, score))
+
+    if pred_name is not None:
+        if missing:
+            feedback = (
+                f"Score {score:.2f}. The response does not address these "
+                f"expected-behavior elements: {', '.join(missing)}."
+            )
+        else:
+            feedback = f"Score {score:.2f}. The response covers the expected behavior."
+        return dspy.Prediction(score=score, feedback=feedback)
 
-    return min(1.0, max(0.0, score))
+    return score
 
 
 def _parse_score(value) -> float:

evolution/skills/evolve_skill.py

@@ -43,6 +43,11 @@ def evolve(
     hermes_repo: Optional[str] = None,
     run_tests: bool = False,
     dry_run: bool = False,
+    max_tokens: Optional[int] = None,
+    temperature: Optional[float] = None,
+    num_threads: Optional[int] = None,
+    lm_timeout: Optional[float] = None,
+    lm_retries: Optional[int] = None,
 ):
     """Main evolution function — orchestrates the full optimization loop."""
 
@@ -118,7 +123,9 @@ def evolve(
     # ── 3. Validate constraints on baseline ─────────────────────────────
     console.print(f"\n[bold]Validating baseline constraints[/bold]")
     validator = ConstraintValidator(config)
-    baseline_constraints = validator.validate_all(skill["body"], "skill")
+    # Validate the full file (frontmatter + body): skill_structure checks
+    # frontmatter, which load_skill strips from `body`.
+    baseline_constraints = validator.validate_all(skill["raw"], "skill")
     all_pass = True
     for c in baseline_constraints:
         icon = "✓" if c.passed else "✗"
@@ -136,8 +143,21 @@ def evolve(
     console.print(f"  Optimizer model: {optimizer_model}")
     console.print(f"  Eval model: {eval_model}")
 
-    # Configure DSPy
-    lm = dspy.LM(eval_model)
+    # Configure DSPy. Generation kwargs are only passed when explicitly set,
+    # so the default behavior is unchanged. Reasoning models served by local
+    # OpenAI-compatible endpoints (low server-side max_tokens defaults) need
+    # an explicit budget or the thinking phase consumes it and content comes
+    # back empty.
+    lm_kwargs = {}
+    if max_tokens is not None:
+        lm_kwargs["max_tokens"] = max_tokens
+    if temperature is not None:
+        lm_kwargs["temperature"] = temperature
+    if lm_timeout is not None:
+        lm_kwargs["timeout"] = lm_timeout
+    if lm_retries is not None:
+        lm_kwargs["num_retries"] = lm_retries
+    lm = dspy.LM(eval_model, **lm_kwargs)
     dspy.configure(lm=lm)
 
     # Create the baseline skill module
@@ -153,9 +173,20 @@ def evolve(
     start_time = time.time()
 
     try:
+        # dspy.GEPA requires exactly one budget parameter (auto /
+        # max_full_evals / max_metric_calls) — there is no `max_steps` —
+        # and a reflection LM for proposing mutations.
+        # num_threads matters on serial local endpoints: parallel rollouts
+        # queue behind each other and time out in cascade once queue depth
+        # times per-request latency exceeds the client timeout.
+        gepa_kwargs = {}
+        if num_threads is not None:
+            gepa_kwargs["num_threads"] = num_threads
         optimizer = dspy.GEPA(
             metric=skill_fitness_metric,
-            max_steps=iterations,
+            max_full_evals=iterations,
+            reflection_lm=dspy.LM(optimizer_model, **lm_kwargs),
+            **gepa_kwargs,
         )
 
         optimized_module = optimizer.compile(
@@ -185,7 +216,9 @@ def evolve(
 
     # ── 7. Validate evolved skill ───────────────────────────────────────
     console.print(f"\n[bold]Validating evolved skill[/bold]")
-    evolved_constraints = validator.validate_all(evolved_body, "skill", baseline_text=skill["body"])
+    # Same rule as the baseline: validate the reassembled file, not the bare
+    # body — otherwise skill_structure always fails and nothing ever deploys.
+    evolved_constraints = validator.validate_all(evolved_full, "skill", baseline_text=skill["raw"])
     all_pass = True
     for c in evolved_constraints:
         icon = "✓" if c.passed else "✗"
@@ -303,7 +336,14 @@ def evolve(
 @click.option("--hermes-repo", default=None, help="Path to hermes-agent repo")
 @click.option("--run-tests", is_flag=True, help="Run full pytest suite as constraint gate")
 @click.option("--dry-run", is_flag=True, help="Validate setup without running optimization")
-def main(skill, iterations, eval_source, dataset_path, optimizer_model, eval_model, hermes_repo, run_tests, dry_run):
+@click.option("--max-tokens", default=None, type=int,
+              help="Generation budget per LM call (needed for reasoning models on local endpoints)")
+@click.option("--temperature", default=None, type=float, help="Sampling temperature for LM calls")
+@click.option("--num-threads", default=None, type=int,
+              help="Parallel rollouts for GEPA evaluation (use 1 for serial local endpoints)")
+@click.option("--lm-timeout", default=None, type=float, help="Per-request LM timeout in seconds")
+@click.option("--lm-retries", default=None, type=int, help="LM retry count on failures")
+def main(skill, iterations, eval_source, dataset_path, optimizer_model, eval_model, hermes_repo, run_tests, dry_run, max_tokens, temperature, num_threads, lm_timeout, lm_retries):
     """Evolve a Hermes Agent skill using DSPy + GEPA optimization."""
     evolve(
         skill_name=skill,
@@ -315,6 +355,11 @@ def main(skill, iterations, eval_source, dataset_path, optimizer_model, eval_mod
         hermes_repo=hermes_repo,
         run_tests=run_tests,
         dry_run=dry_run,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        num_threads=num_threads,
+        lm_timeout=lm_timeout,
+        lm_retries=lm_retries,
     )
 
 

evolution/skills/skill_module.py

@@ -84,33 +84,40 @@ def find_skill(skill_name: str, hermes_agent_path: Path) -> Optional[Path]:
 class SkillModule(dspy.Module):
     """A DSPy module that wraps a skill file for optimization.
 
-    The skill text (body) is the parameter that GEPA optimizes.
-    On each forward pass, the module:
-    1. Uses the skill text as instructions
+    The skill text (body) is the parameter that GEPA optimizes. GEPA's
+    candidate space is exactly the `signature.instructions` of the module's
+    named predictors (see dspy.teleprompt.gepa), so the skill text must live
+    in the signature instructions — not in an input field, which GEPA never
+    mutates. On each forward pass, the module:
+    1. Uses the skill text as the predictor's signature instructions
     2. Processes the task input
     3. Returns the agent's response
+
+    `skill_text` is a read-through property over the predictor's current
+    instructions, so after `optimizer.compile()` it reflects the evolved
+    text rather than the original.
     """
 
     class TaskWithSkill(dspy.Signature):
-        """Complete a task following the provided skill instructions.
+        """Complete a task following the provided skill instructions."""
 
-        You are an AI agent following specific skill instructions to complete a task.
-        Read the skill instructions carefully and follow the procedure described.
-        """
-        skill_instructions: str = dspy.InputField(desc="The skill instructions to follow")
         task_input: str = dspy.InputField(desc="The task to complete")
         output: str = dspy.OutputField(desc="Your response following the skill instructions")
 
     def __init__(self, skill_text: str):
         super().__init__()
-        self.skill_text = skill_text
-        self.predictor = dspy.ChainOfThought(self.TaskWithSkill)
+        signature = self.TaskWithSkill.with_instructions(skill_text)
+        self.predictor = dspy.ChainOfThought(signature)
+
+    @property
+    def skill_text(self) -> str:
+        """The current skill text — evolved instructions after optimization."""
+        for _, predictor in self.named_predictors():
+            return predictor.signature.instructions
+        raise AttributeError("SkillModule has no predictors")
 
     def forward(self, task_input: str) -> dspy.Prediction:
-        result = self.predictor(
-            skill_instructions=self.skill_text,
-            task_input=task_input,
-        )
+        result = self.predictor(task_input=task_input)
         return dspy.Prediction(output=result.output)
 
 

tests/core/test_constraints.py

@@ -96,3 +96,25 @@ def test_empty_skill_fails(self, validator):
         results = validator.validate_all("", "skill")
         failed = [r for r in results if not r.passed]
         assert len(failed) > 0
+
+
+class TestValidateFullFileNotBody:
+    """Regression: skill_structure must be checked on the full file.
+
+    load_skill() strips frontmatter into a separate field; validating the
+    bare body always fails skill_structure and blocks every deploy.
+    """
+
+    def test_realistic_full_file_passes(self, validator):
+        raw = (
+            "---\nname: real-skill\ndescription: Does something real\n---\n\n"
+            "# Procedure\n1. Step"
+        )
+        results = validator.validate_all(raw, "skill")
+        assert all(r.passed for r in results)
+
+    def test_bare_body_fails_structure(self, validator):
+        body = "# Procedure\n1. Step"
+        results = validator.validate_all(body, "skill")
+        structure = [r for r in results if r.constraint_name == "skill_structure"][0]
+        assert not structure.passed

tests/core/test_fitness.py

@@ -0,0 +1,49 @@
+"""Tests for the GEPA-compatible fitness metric."""
+
+import dspy
+
+from evolution.core.fitness import skill_fitness_metric
+
+
+def _example_and_pred():
+    example = dspy.Example(
+        task_input="task",
+        expected_behavior="verify evidence before concluding done",
+    )
+    prediction = dspy.Prediction(output="I will verify the evidence first")
+    return example, prediction
+
+
+class TestMetricContract:
+    def test_direct_call_returns_float(self):
+        example, prediction = _example_and_pred()
+        score = skill_fitness_metric(example, prediction)
+        assert isinstance(score, float)
+        assert 0.0 <= score <= 1.0
+
+    def test_miprov2_style_call_returns_float(self):
+        example, prediction = _example_and_pred()
+        score = skill_fitness_metric(example, prediction, None)
+        assert isinstance(score, float)
+
+    def test_gepa_reflection_call_returns_feedback(self):
+        # GEPA's GEPAFeedbackMetric protocol: (gold, pred, trace, pred_name,
+        # pred_trace); predictor-level calls expect Prediction(score, feedback).
+        example, prediction = _example_and_pred()
+        result = skill_fitness_metric(example, prediction, None, "predictor", None)
+        assert isinstance(result, dspy.Prediction)
+        assert 0.0 <= result.score <= 1.0
+        assert result.feedback
+
+    def test_empty_output_scores_zero(self):
+        example, _ = _example_and_pred()
+        assert skill_fitness_metric(example, dspy.Prediction(output="")) == 0.0
+
+    def test_gepa_accepts_metric_with_valid_budget(self):
+        # Regression: GEPA has no `max_steps` — the old call always raised
+        # TypeError and silently fell back to MIPROv2.
+        lm = dspy.LM("openai/test", api_base="http://127.0.0.1:1/v1", api_key="x")
+        optimizer = dspy.GEPA(
+            metric=skill_fitness_metric, max_full_evals=5, reflection_lm=lm,
+        )
+        assert optimizer is not None

tests/skills/test_skill_module.py

@@ -2,7 +2,7 @@
 
 import pytest
 from pathlib import Path
-from evolution.skills.skill_module import load_skill, reassemble_skill
+from evolution.skills.skill_module import SkillModule, load_skill, reassemble_skill
 
 
 SAMPLE_SKILL = """---
@@ -90,3 +90,30 @@ def test_evolved_body_replaces_original(self):
 
         assert "EVOLVED" in result
         assert "New and improved" in result
+
+
+class TestSkillModuleOptimizable:
+    """The skill text must live where GEPA mutates: signature.instructions.
+
+    GEPA's candidate space is {name: pred.signature.instructions} over
+    named_predictors() (dspy.teleprompt.gepa). A plain module attribute is
+    invisible to it, so the evolved text could never reach the saved artifact.
+    """
+
+    def test_skill_text_lives_in_signature_instructions(self):
+        text = "# My Skill\nAlways verify before concluding."
+        module = SkillModule(text)
+
+        predictors = list(module.named_predictors())
+        assert len(predictors) == 1
+        _, predictor = predictors[0]
+        assert predictor.signature.instructions == text
+
+    def test_skill_text_property_reflects_mutation(self):
+        module = SkillModule("# Original")
+
+        # Mutate the way GEPA applies candidates: rewrite the instructions.
+        for _, predictor in module.named_predictors():
+            predictor.signature = predictor.signature.with_instructions("# Evolved")
+
+        assert module.skill_text == "# Evolved"