fix(evals): guard registry membership checks against malformed expected_tools and one-shot iterables

Author: subaoyan16

evals/agent_trajectory/metrics.py

@@ -273,7 +273,6 @@ def load_golden_samples(
     if not isinstance(data, list):
         raise ValueError(f"golden samples file must contain a JSON list, got {type(data).__name__}")
 
-    known = set(known_tool_names) if known_tool_names is not None else None
     golden_fields = {f.name for f in fields(GoldenSample)}
     samples: List[GoldenSample] = []
     seen_ids: set = set()
@@ -287,7 +286,7 @@ def load_golden_samples(
         if sample.id in seen_ids:
             raise ValueError(f"duplicate sample id: {sample.id}")
         seen_ids.add(sample.id)
-        issues = validate_golden_sample(sample, known)
+        issues = validate_golden_sample(sample, known_tool_names)
         if issues:
             raise ValueError(f"sample '{sample.id}': " + "; ".join(issues))
         samples.append(sample)
@@ -302,7 +301,9 @@ def validate_golden_sample(
 
     When ``known_tool_names`` is provided, ``expected_tools`` must be a subset
     of it; the caller supplies the authoritative registry names (this module
-    deliberately does not import ``src/``).
+    deliberately does not import ``src/``).  Any ``Iterable[str]`` is accepted
+    — including one-shot generators — and materialized once internally, so
+    membership checks never consume the caller's iterable.
 
     Field *types* are part of the structural contract — hand-edited golden
     JSON must fail with a clear message instead of crashing or silently
@@ -311,6 +312,7 @@ def validate_golden_sample(
     boolean.
     """
     issues: List[str] = []
+    known = set(known_tool_names) if known_tool_names is not None else None
     if not isinstance(sample.id, str) or not sample.id.strip():
         issues.append("id must be a non-empty string")
     if not isinstance(sample.task_description, str) or not sample.task_description.strip():
@@ -323,8 +325,10 @@ def validate_golden_sample(
         issues.append("expected_tools must be a non-empty list")
     elif any(not isinstance(t, str) or not t.strip() for t in sample.expected_tools):
         issues.append("expected_tools must contain only non-empty strings")
-    if known_tool_names is not None:
-        unknown = [t for t in sample.expected_tools if isinstance(t, str) and t.strip() and t not in known_tool_names]
+    elif known is not None:
+        # Only reachable when expected_tools is a non-empty list of non-empty
+        # strings, so malformed values can never crash the membership check.
+        unknown = [t for t in sample.expected_tools if t not in known]
         if unknown:
             issues.append(f"unknown expected_tools: {', '.join(unknown)}")
     if not isinstance(sample.skills, list):

tests/test_agent_trajectory_metrics.py

@@ -428,6 +428,28 @@ def test_mistyped_fields_fail_validation_not_crash(self):
         assert any("allowed_max_steps must be an integer" in i for i in issues)
         assert any("allow_optional_tools must be a boolean" in i for i in issues)
 
+    def test_non_iterable_expected_tools_with_known_names_does_not_crash(self):
+        sample = GoldenSample(
+            id="x",
+            task_description="t",
+            stock_code="600519",
+            expected_tools=1,
+        )
+        issues = validate_golden_sample(sample, {"get_realtime_quote"})
+        assert any("expected_tools must be a list" in i for i in issues)
+
+    def test_registry_membership_with_one_shot_generator(self):
+        # The helper accepts any Iterable[str]; a one-shot generator must be
+        # materialized internally so membership checks never consume it.
+        sample = GoldenSample(
+            id="x",
+            task_description="t",
+            stock_code="600519",
+            expected_tools=["b", "a"],
+        )
+        known = (name for name in ["a", "b"])
+        assert validate_golden_sample(sample, known) == []
+
 
 class TestLoadGoldenSamplesErrors:
     @staticmethod
@@ -517,6 +539,19 @@ def test_string_expected_tools_raises(self, tmp_path):
         with pytest.raises(ValueError, match="expected_tools must be a list"):
             load_golden_samples(path=path)
 
+    def test_non_list_expected_tools_with_known_names_raises_valueerror(self, tmp_path):
+        # Regression for OR-COR-4e0e3cf1: the registry membership check must
+        # not iterate a rejected non-list value and leak a TypeError.
+        sample = {
+            "id": "x",
+            "task_description": "t",
+            "stock_code": "600519",
+            "expected_tools": 1,
+        }
+        path = self._write_sample(tmp_path, [sample])
+        with pytest.raises(ValueError, match="expected_tools must be a list"):
+            load_golden_samples(path=path, known_tool_names={"get_realtime_quote"})
+
     def test_non_bool_allow_optional_tools_raises(self, tmp_path):
         sample = {
             "id": "x",