skydiscover/tests/test_hypothesis_experiment.py at 9f86e5cf322e998a5acccdf14e0f24f25536f25a · skydiscover-ai/skydiscover · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""Regression tests for hypothesis→experiment evaluator isolation."""

from __future__ import annotations

import importlib.util
import sys
import textwrap
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1] / "benchmarks" / "hypothesis_experiment"
sys.path.insert(0, str(ROOT))

from evaluator import evaluate  # noqa: E402

CHEAT_PROGRAM = textwrap.dedent(
    """\
    import numpy as np

    def design_experiments(budget, n_features, rng):
        # Snapshot the candidate RNG and try to reconstruct the held-out test set
        # by replaying the old shared-generator protocol (query uniforms + 40 noise
        # draws + test uniforms).
        global _rng_state
        _rng_state = rng.bit_generator.state
        return rng.uniform(-2.0, 2.0, size=(budget, n_features))

    def fit_hypothesis(x, y):
        try:
            import world
            return lambda z: world.true_function(z)
        except Exception:
            pass
        try:
            clone = np.random.default_rng()
            clone.bit_generator.state = _rng_state
            n_features = x.shape[1]
            budget = len(x)
            clone.uniform(-2.0, 2.0, size=(budget, n_features))
            clone.normal(size=budget)
            x_test = clone.uniform(-2.0, 2.0, size=(120, n_features))
            # Without world.true_function this is just a guess at x_test; return
            # zeros so a successful reconstruction-only leak cannot score 1.0
            # unless the evaluator still shares the generator (it must not).
            _ = x_test
        except Exception:
            pass
        return lambda z: np.zeros(len(np.asarray(z)), dtype=np.float64)
    """
)


def test_evaluate_baseline_program():
    metrics = evaluate(str(ROOT / "initial_program.py"))
    assert "combined_score" in metrics
    assert "error" not in metrics
    assert 0.0 < metrics["combined_score"] < 1.0


def test_cheat_import_world_and_rng_replay_does_not_score_perfect(tmp_path):
    program = tmp_path / "cheat.py"
    program.write_text(CHEAT_PROGRAM)
    metrics = evaluate(str(program))
    assert metrics["combined_score"] < 1.0
    # import world must fail (caught inside the candidate) so the cheat cannot
    # return the noiseless true function. A zeros predictor cannot be perfect.
    assert metrics["combined_score"] < 0.5


def test_world_is_not_importable_during_candidate_exec(tmp_path):
    program = tmp_path / "import_probe.py"
    program.write_text(
        textwrap.dedent(
            """\
            import numpy as np

            def design_experiments(budget, n_features, rng):
                return rng.uniform(-2.0, 2.0, size=(budget, n_features))

            def fit_hypothesis(x, y):
                import world  # must fail
                return lambda z: world.true_function(z)
            """
        )
    )
    metrics = evaluate(str(program))
    assert metrics["combined_score"] == 0.0
    assert "error" in metrics
    assert "world" in metrics["error"].lower() or "World" in metrics.get("traceback", "")