Optimize-the-optimizer: evolve search controllers

benchmarks/README.md

@@ -54,6 +54,7 @@ uv run skydiscover-run benchmarks/math/circle_packing/initial_program.py \
 | [`ale_bench/`](ale_bench/) | Algorithms | 10 | Algorithmic contest problems (C++, ALE-Bench) |
 | [`image_gen/`](image_gen/) | Creative | 1 | AI image generation evolution |
 | [`prompt_optimization/`](prompt_optimization/) | Prompts | 1 | Evolve natural-language prompts, not code (HotPotQA) |
+| [`optimize_the_optimizer/`](optimize_the_optimizer/) | Meta-search | 1 | Evolve a search controller on a black-box portfolio |
 
 Each benchmark directory has its own README with setup and run instructions.
 

benchmarks/optimize_the_optimizer/README.md

@@ -0,0 +1,21 @@
+# Optimize the optimizer
+
+EvoX²-style benchmark: the discovery object is a **search controller**
+(selection + mutation / `ask` policy), scored by how well it optimizes a
+portfolio of black-box functions under a fixed eval budget.
+
+Held-out problems (Ackley, Rosenbrock) dominate `combined_score` so the
+controller must transfer, not memorize the train suite.
+
+## Run
+
+```bash
+uv run skydiscover-run benchmarks/optimize_the_optimizer/initial_program.py \
+  benchmarks/optimize_the_optimizer/evaluator.py \
+  -c benchmarks/optimize_the_optimizer/config.yaml -s best_of_n -i 50
+```
+
+```bash
+python3 benchmarks/optimize_the_optimizer/evaluator.py \
+  benchmarks/optimize_the_optimizer/initial_program.py
+```

benchmarks/optimize_the_optimizer/config.yaml

@@ -0,0 +1,46 @@
+# Optimize-the-optimizer (EvoX² wedge): evolve a search controller.
+# Usage:
+#   uv run skydiscover-run benchmarks/optimize_the_optimizer/initial_program.py \
+#     benchmarks/optimize_the_optimizer/evaluator.py \
+#     -c benchmarks/optimize_the_optimizer/config.yaml -s best_of_n -i 50
+
+language: python
+diff_based_generation: true
+max_iterations: 100
+checkpoint_interval: 10
+max_solution_length: 50000
+
+llm:
+  api_base: https://api.openai.com/v1
+  models:
+    - name: "gpt-5"
+      weight: 1.0
+  max_tokens: 12288
+  timeout: 600
+
+prompt:
+  system_message: |
+    You are evolving a *search controller* (parent selection + mutation / ask
+    policy), not a solution to a single math problem.
+
+    Implement `SearchController` with:
+      - __init__(dim, bounds, rng)
+      - initial_population() -> list[np.ndarray]
+      - ask(population, n) -> list[np.ndarray]
+
+    The evaluator runs your controller on black-box minimization problems
+    (sphere, rastrigin, …) under a fixed evaluation budget and scores
+    held-out problems most heavily. Improve exploration/exploitation,
+    adaptive step sizes, elitism, etc. Stay within the provided API.
+
+evaluator:
+  timeout: 120
+  cascade_evaluation: true
+  cascade_thresholds:
+    - 0.15
+    - 0.4
+
+monitor:
+  enabled: true
+  port: 8765
+  host: "127.0.0.1"

benchmarks/optimize_the_optimizer/evaluator.py

@@ -0,0 +1,113 @@
+"""Score a search controller on train + held-out black-box problems."""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+import time
+import traceback
+from typing import Any
+
+import numpy as np
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+if _HERE not in sys.path:
+    sys.path.insert(0, _HERE)
+
+from initial_program import run_controller  # noqa: E402
+from problems import TEST_PROBLEMS, TRAIN_PROBLEMS  # noqa: E402
+
+BUDGET = 64
+SEEDS = (0, 1, 2)
+
+
+def _load_controller(program_path: str):
+    spec = importlib.util.spec_from_file_location("candidate_controller", program_path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"cannot load {program_path}")
+    module = importlib.util.module_from_spec(spec)
+    # dataclasses (3.14+) look up cls.__module__ in sys.modules during decoration.
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    if not hasattr(module, "SearchController"):
+        raise AttributeError("program must define SearchController")
+    # Prefer the candidate's run_controller if present; else shared harness.
+    run_fn = getattr(module, "run_controller", run_controller)
+    return module.SearchController, run_fn
+
+
+def _normalize_score(raw: float, problem_name: str) -> float:
+    """Map raw maximize-scores into roughly [0, 1] per problem family."""
+    # Heuristic ceilings from random search baselines on these bounds/budgets.
+    floors = {
+        "sphere_2d": -20.0,
+        "sphere_4d": -20.0,
+        "rastrigin_2d": -80.0,
+        "ackley_2d": -15.0,
+        "rosenbrock_2d": -200.0,
+    }
+    ceilings = {
+        "sphere_2d": 0.0,
+        "sphere_4d": 0.0,
+        "rastrigin_2d": 0.0,
+        "ackley_2d": 0.0,
+        "rosenbrock_2d": 0.0,
+    }
+    lo = floors.get(problem_name, -50.0)
+    hi = ceilings.get(problem_name, 0.0)
+    if hi <= lo:
+        return 0.0
+    return float(np.clip((raw - lo) / (hi - lo), 0.0, 1.0))
+
+
+def _eval_suite(controller_cls, run_fn, problems) -> dict[str, float]:
+    scores = []
+    per: dict[str, float] = {}
+    for problem in problems:
+        seed_scores = []
+        for seed in SEEDS:
+            raw = run_fn(
+                controller_cls,
+                problem.fn,
+                problem.dim,
+                problem.bounds,
+                BUDGET,
+                seed,
+            )
+            seed_scores.append(_normalize_score(raw, problem.name))
+        mean = float(np.mean(seed_scores))
+        per[problem.name] = mean
+        scores.append(mean)
+    return {"mean": float(np.mean(scores)) if scores else 0.0, "per_problem": per}
+
+
+def evaluate(program_path: str) -> dict[str, Any]:
+    t0 = time.time()
+    try:
+        controller_cls, run_fn = _load_controller(program_path)
+        train = _eval_suite(controller_cls, run_fn, TRAIN_PROBLEMS)
+        test = _eval_suite(controller_cls, run_fn, TEST_PROBLEMS)
+        combined = 0.65 * test["mean"] + 0.35 * train["mean"]
+        return {
+            "combined_score": float(combined),
+            "test_score": float(test["mean"]),
+            "train_score": float(train["mean"]),
+            "latency_s": float(time.time() - t0),
+            "artifacts": {
+                "train": train["per_problem"],
+                "test": test["per_problem"],
+            },
+        }
+    except Exception as exc:  # noqa: BLE001
+        return {
+            "combined_score": 0.0,
+            "error": f"{type(exc).__name__}: {exc}",
+            "traceback": traceback.format_exc(),
+            "latency_s": float(time.time() - t0),
+        }
+
+
+if __name__ == "__main__":
+    path = sys.argv[1] if len(sys.argv) > 1 else os.path.join(_HERE, "initial_program.py")
+    print(evaluate(path))

benchmarks/optimize_the_optimizer/initial_program.py

@@ -0,0 +1,108 @@
+"""Baseline search controller for the optimize-the-optimizer benchmark.
+
+Evolve ``SearchController`` — parent selection + mutation under a fixed eval
+budget on held-out black-box problems. This is the EvoX² wedge: the candidate
+*is* a search strategy, scored by how well it optimizes other tasks.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable
+
+import numpy as np
+
+
+@dataclass
+class Candidate:
+    x: np.ndarray
+    score: float  # higher is better (negated objective)
+
+
+# EVOLVE-BLOCK-START
+class SearchController:
+    """Population search strategy evaluated under a fixed black-box budget."""
+
+    def __init__(self, dim: int, bounds: tuple[float, float], rng: np.random.Generator):
+        self.dim = dim
+        self.lo, self.hi = bounds
+        self.rng = rng
+        self.population_size = 8
+        self.mutation_scale = 0.25
+
+    def initial_population(self) -> list[np.ndarray]:
+        return [
+            self.rng.uniform(self.lo, self.hi, size=self.dim)
+            for _ in range(self.population_size)
+        ]
+
+    def select_parents(self, population: list[Candidate], k: int = 2) -> list[Candidate]:
+        """Tournament selection."""
+        parents = []
+        for _ in range(k):
+            a, b = self.rng.choice(len(population), size=2, replace=False)
+            parents.append(population[a] if population[a].score >= population[b].score else population[b])
+        return parents
+
+    def mutate(self, parent: Candidate) -> np.ndarray:
+        noise = self.rng.normal(scale=self.mutation_scale, size=self.dim)
+        child = parent.x + noise * (self.hi - self.lo)
+        return np.clip(child, self.lo, self.hi)
+
+    def ask(self, population: list[Candidate], n: int) -> list[np.ndarray]:
+        """Propose ``n`` new points given the current population."""
+        if not population:
+            return self.initial_population()[:n]
+        proposals = []
+        for _ in range(n):
+            parents = self.select_parents(population, k=2)
+            # Blend + mutate.
+            alpha = float(self.rng.random())
+            blend = alpha * parents[0].x + (1 - alpha) * parents[1].x
+            child = self.mutate(Candidate(blend, 0.0))
+            proposals.append(child)
+        return proposals
+
+
+# EVOLVE-BLOCK-END
+
+
+def run_controller(
+    controller_cls: type,
+    objective: Callable[[np.ndarray], float],
+    dim: int,
+    bounds: tuple[float, float],
+    budget: int,
+    seed: int,
+) -> float:
+    """Run ``controller_cls`` for ``budget`` evaluations; return best maximize-score."""
+    rng = np.random.default_rng(seed)
+    ctrl = controller_cls(dim, bounds, rng)
+    population: list[Candidate] = []
+    best = -float("inf")
+    remaining = budget
+
+    init = ctrl.initial_population()
+    for x in init:
+        if remaining <= 0:
+            break
+        score = -float(objective(x))
+        population.append(Candidate(x=np.asarray(x, dtype=np.float64), score=score))
+        best = max(best, score)
+        remaining -= 1
+
+    while remaining > 0:
+        batch = min(len(population) or 1, remaining)
+        proposals = ctrl.ask(population, batch)
+        for x in proposals:
+            if remaining <= 0:
+                break
+            score = -float(objective(np.asarray(x, dtype=np.float64)))
+            population.append(Candidate(x=np.asarray(x, dtype=np.float64), score=score))
+            best = max(best, score)
+            remaining -= 1
+            # Keep population bounded.
+            if len(population) > 40:
+                population.sort(key=lambda c: c.score, reverse=True)
+                population = population[:24]
+    return best

benchmarks/optimize_the_optimizer/problems.py

@@ -0,0 +1,50 @@
+"""Black-box toy suite used to score search controllers (optimize-the-optimizer)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class Problem:
+    name: str
+    dim: int
+    bounds: tuple[float, float]
+    # Minimize; evaluator will negate into a maximize score.
+    fn: Callable[[np.ndarray], float]
+
+
+def _sphere(x: np.ndarray) -> float:
+    return float(np.sum(x * x))
+
+
+def _rastrigin(x: np.ndarray) -> float:
+    n = x.size
+    return float(10 * n + np.sum(x * x - 10 * np.cos(2 * np.pi * x)))
+
+
+def _ackley(x: np.ndarray) -> float:
+    n = x.size
+    a = -20 * np.exp(-0.2 * np.sqrt(np.sum(x * x) / n))
+    b = -np.exp(np.sum(np.cos(2 * np.pi * x)) / n)
+    return float(a + b + 20 + np.e)
+
+
+def _rosenbrock(x: np.ndarray) -> float:
+    return float(np.sum(100.0 * (x[1:] - x[:-1] ** 2) ** 2 + (1 - x[:-1]) ** 2))
+
+
+# Train problems (used during search). Hold out ackley + rosenbrock.
+TRAIN_PROBLEMS: list[Problem] = [
+    Problem("sphere_2d", 2, (-3.0, 3.0), _sphere),
+    Problem("sphere_4d", 4, (-2.0, 2.0), _sphere),
+    Problem("rastrigin_2d", 2, (-3.0, 3.0), _rastrigin),
+]
+
+TEST_PROBLEMS: list[Problem] = [
+    Problem("ackley_2d", 2, (-3.0, 3.0), _ackley),
+    Problem("rosenbrock_2d", 2, (-2.0, 2.0), _rosenbrock),
+]