test(benchmarks): isolate synthetic module loading (#746)

Author: morluto

benchmarks/validation/test_benchmark_planner.py

@@ -9,7 +9,7 @@
 import sys
 from importlib.machinery import SourceFileLoader
 from pathlib import Path
-from types import SimpleNamespace
+from types import ModuleType, SimpleNamespace
 
 import pytest
 
@@ -18,13 +18,21 @@
 PATH_POLICY_PATH = ROOT / ".github" / "scripts" / "_ci_paths.py"
 VALIDATION_PLAN_PATH = ROOT / "benchmarks" / "tooling" / "validation_plan.py"
 VALIDATOR_PATH = ROOT / ".github" / "scripts" / "validate-benchmark-plan"
-_SPEC = importlib.util.spec_from_loader(
-    "benchmark_planner", SourceFileLoader("benchmark_planner", str(PLANNER_PATH))
-)
-assert _SPEC is not None and _SPEC.loader is not None
-planner = importlib.util.module_from_spec(_SPEC)
-sys.modules["benchmark_planner"] = planner
-_SPEC.loader.exec_module(planner)
+
+
+def _load_script(module_name: str, path: Path) -> ModuleType:
+    spec = importlib.util.spec_from_loader(
+        module_name, SourceFileLoader(module_name, str(path))
+    )
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    with pytest.MonkeyPatch.context() as module_state:
+        module_state.setitem(sys.modules, module_name, module)
+        spec.loader.exec_module(module)
+    return module
+
+
+planner = _load_script("benchmark_planner", PLANNER_PATH)
 
 
 @pytest.fixture(autouse=True)
@@ -38,6 +46,34 @@ def stable_digests(monkeypatch: pytest.MonkeyPatch) -> None:
     )
 
 
+@pytest.mark.parametrize("preserve_existing", [False, True])
+def test_load_script_scopes_sys_modules_registration(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    *,
+    preserve_existing: bool,
+) -> None:
+    module_name = "_benchmark_planner_module_probe"
+    source = tmp_path / "module_probe.py"
+    source.write_text(
+        "import sys\nregistered_while_loading = sys.modules[__name__]\n",
+        encoding="utf-8",
+    )
+    sentinel = ModuleType("sentinel")
+    if preserve_existing:
+        monkeypatch.setitem(sys.modules, module_name, sentinel)
+    else:
+        monkeypatch.delitem(sys.modules, module_name, raising=False)
+
+    loaded = _load_script(module_name, source)
+
+    assert vars(loaded)["registered_while_loading"] is loaded
+    if preserve_existing:
+        assert sys.modules[module_name] is sentinel
+    else:
+        assert module_name not in sys.modules
+
+
 def _matrix(result: dict[str, str]) -> list[dict[str, object]]:
     return json.loads(result["benchmark-oracle-matrix"])
 

benchmarks/validation/test_harbor_task_workflow.py

@@ -7,19 +7,56 @@
 import sys
 from importlib.machinery import SourceFileLoader
 from pathlib import Path
+from types import ModuleType
 from typing import Any
 
 import pytest
 
 ROOT = Path(__file__).resolve().parents[2]
 TOOL_PATH = ROOT / "tools" / "harbor_task_workflow.py"
-_SPEC = importlib.util.spec_from_loader(
-    "harbor_task_workflow", SourceFileLoader("harbor_task_workflow", str(TOOL_PATH))
-)
-assert _SPEC is not None and _SPEC.loader is not None
-workflow = importlib.util.module_from_spec(_SPEC)
-sys.modules[_SPEC.name] = workflow
-_SPEC.loader.exec_module(workflow)
+
+
+def _load_script(module_name: str, path: Path) -> ModuleType:
+    spec = importlib.util.spec_from_loader(
+        module_name, SourceFileLoader(module_name, str(path))
+    )
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    with pytest.MonkeyPatch.context() as module_state:
+        module_state.setitem(sys.modules, module_name, module)
+        spec.loader.exec_module(module)
+    return module
+
+
+workflow = _load_script("harbor_task_workflow", TOOL_PATH)
+
+
+@pytest.mark.parametrize("preserve_existing", [False, True])
+def test_load_script_scopes_sys_modules_registration(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    *,
+    preserve_existing: bool,
+) -> None:
+    module_name = "_harbor_task_workflow_module_probe"
+    source = tmp_path / "module_probe.py"
+    source.write_text(
+        "import sys\nregistered_while_loading = sys.modules[__name__]\n",
+        encoding="utf-8",
+    )
+    sentinel = ModuleType("sentinel")
+    if preserve_existing:
+        monkeypatch.setitem(sys.modules, module_name, sentinel)
+    else:
+        monkeypatch.delitem(sys.modules, module_name, raising=False)
+
+    loaded = _load_script(module_name, source)
+
+    assert vars(loaded)["registered_while_loading"] is loaded
+    if preserve_existing:
+        assert sys.modules[module_name] is sentinel
+    else:
+        assert module_name not in sys.modules
 
 
 def test_resolve_selection_uses_planner_owned_host_matrix() -> None:

benchmarks/validation/test_verifier_support_schema.py

@@ -6,6 +6,7 @@
 import json
 import sys
 from pathlib import Path
+from types import ModuleType
 
 import pytest
 
@@ -18,16 +19,45 @@
 )
 
 
-def _load_module():
-    spec = importlib.util.spec_from_file_location("_vs_under_test", _TEMPLATE)
+def _load_module(module_name: str, path: Path) -> ModuleType:
+    spec = importlib.util.spec_from_file_location(module_name, path)
     assert spec is not None and spec.loader is not None
     module = importlib.util.module_from_spec(spec)
-    sys.modules["_vs_under_test"] = module
-    spec.loader.exec_module(module)
+    with pytest.MonkeyPatch.context() as module_state:
+        module_state.setitem(sys.modules, module_name, module)
+        spec.loader.exec_module(module)
     return module
 
 
-_VS = _load_module()
+_VS = _load_module("_vs_under_test", _TEMPLATE)
+
+
+@pytest.mark.parametrize("preserve_existing", [False, True])
+def test_load_module_scopes_sys_modules_registration(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    *,
+    preserve_existing: bool,
+) -> None:
+    module_name = "_verifier_support_schema_module_probe"
+    source = tmp_path / "module_probe.py"
+    source.write_text(
+        "import sys\nregistered_while_loading = sys.modules[__name__]\n",
+        encoding="utf-8",
+    )
+    sentinel = ModuleType("sentinel")
+    if preserve_existing:
+        monkeypatch.setitem(sys.modules, module_name, sentinel)
+    else:
+        monkeypatch.delitem(sys.modules, module_name, raising=False)
+
+    loaded = _load_module(module_name, source)
+
+    assert vars(loaded)["registered_while_loading"] is loaded
+    if preserve_existing:
+        assert sys.modules[module_name] is sentinel
+    else:
+        assert module_name not in sys.modules
 
 
 @pytest.fixture