fix(evals): reject duplicate expected tools, add machine-readable expected_outcomes contract

- validate_golden_sample rejects duplicate expected_tools names so a
  hand-edited golden file cannot inflate hit rates (["quote", "quote",
  "history"] with one quote call must read 1/2, not 2/3); scoring
  normalizes duplicates to first occurrences as a defense
- GoldenSample gains expected_outcomes (guarded / cached / retry): a
  required outcome not observed in the log is reported as a violation, so
  the 600036 guard/retry sample now actually detects a trajectory that
  skips the out-of-scope call it describes
- 13 regression tests (71 total), flake8/black clean

Author: subaoyan16

docs/CHANGELOG.md

@@ -45,7 +45,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 - [修复] 阻止任意更新的非 bundled 指数候选（含 legacy `static` 子集）在 remote 缺失/损坏时以 active-index 子集覆盖 bundled baseline：所有非 bundled 候选必须为 bundled active-index canonical 集合的合法超集，否则回退 bundled 并记录 WARNING。
 - [新功能] 桌面端全局右上角增加更新入口，与设置页共用更新状态；普通浏览器 WebUI 不展示，且不会在挂载时重复触发后台检查。
 - [修复] 桌面端右上角更新入口与设置页共用检查中状态，避免一侧检查时另一侧仍可重复触发 GitHub Releases 检查；主进程手动检查路径同步增加 in-flight 防重。
-- [新功能] 新增 agent trajectory 评估管线（`evals/agent_trajectory`）：纯函数指标模块消费 `tool_calls_log` 计算工具命中率、冗余与缓存调用、失败重试与步数效率（含 `golden_samples.json` 3 个样例与单元测试）；不修改 `src/` 执行语义、不接入 CI 阻断门、不新增运行时配置。
+- [新功能] 新增 agent trajectory 评估管线（`evals/agent_trajectory`）：纯函数指标模块消费 `tool_calls_log` 计算工具命中率、冗余与缓存调用、失败重试与步数效率（含 `golden_samples.json` 3 个样例、`expected_outcomes` 轨迹特征断言与单元测试）；不修改 `src/` 执行语义、不接入 CI 阻断门、不新增运行时配置。
 
 ## [3.31.0] - 2026-08-23
 

evals/agent_trajectory/golden_samples.json

@@ -24,6 +24,7 @@
     "skills": [],
     "expected_tools": ["get_stock_info", "get_daily_history"],
     "allowed_max_steps": 10,
-    "allow_optional_tools": true
+    "allow_optional_tools": true,
+    "expected_outcomes": ["guarded", "retry"]
   }
 ]

evals/agent_trajectory/metrics.py

@@ -46,6 +46,14 @@
   success`` counts exactly one retry, not two.
 * ``cached_calls``: entries with ``cached=True`` (runner semantics: reuse of a
   non-retriable failure result).
+* ``expected_outcomes``: machine-readable trajectory features a golden sample
+  may require, from the fixed vocabulary ``guarded`` / ``cached`` / ``retry``.
+  ``guarded`` is observed when any entry carries ``guarded=True`` (the stock-
+  scope guard interception), ``cached`` when any entry carries ``cached=True``
+  and ``retry`` when at least one retry is counted per the contract above.  A
+  required outcome that is not observed is reported as a violation, so a
+  golden sample that declares guard / cache / retry expectations cannot be
+  passed by a trajectory that skips the behaviour it describes.
 * ``max_steps_touched``: the log does not carry ``max_steps`` itself, so this
   is the conservative heuristic ``max(step) >= golden.allowed_max_steps`` —
   a proxy for "the run reached the step budget", not proof of the loop
@@ -62,13 +70,21 @@
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional
 
+#: Machine-readable trajectory features a golden sample may require of a log
+#: (``guarded`` = a guarded call, ``cached`` = a cached call, ``retry`` = at
+#: least one retry).  See the "Metric semantics" section of the module docstring.
+EXPECTED_OUTCOME_TAGS = ("guarded", "cached", "retry")
+
 
 @dataclass
 class GoldenSample:
     """Expected trajectory for one evaluation task.
 
     ``expected_tools`` are the tool names the agent should call; tools outside
     this set are tolerated only when ``allow_optional_tools`` is true.
+    ``expected_outcomes`` are required trajectory features from the vocabulary
+    ``guarded`` / ``cached`` / ``retry`` (see the module docstring); every
+    declared outcome must be observable in the log.
     """
 
     id: str
@@ -78,6 +94,7 @@ class GoldenSample:
     skills: List[str] = field(default_factory=list)
     allowed_max_steps: int = 10
     allow_optional_tools: bool = True
+    expected_outcomes: List[str] = field(default_factory=list)
 
 
 @dataclass
@@ -150,6 +167,7 @@ def compute_trajectory_metrics(
     key_retries: Dict[tuple, int] = {}
     failed_calls = 0
     cached_calls = 0
+    guarded_calls = 0
     redundant_calls = 0
     distinct_steps = 0
     max_step = 0
@@ -171,6 +189,8 @@ def compute_trajectory_metrics(
             failed_calls += 1
         if entry.get("cached"):
             cached_calls += 1
+        if entry.get("guarded"):
+            guarded_calls += 1
 
         key = (tool, _args_key(_entry_arguments(entry)))
         if key_counts.get(key, 0):
@@ -196,18 +216,26 @@ def compute_trajectory_metrics(
     distinct_steps = max(distinct_steps, total)
     max_step = max(max_step, total)
 
+    retries = sum(key_retries.values())
+    violations: List[str] = []
+
     if isinstance(golden.expected_tools, list):
         expected = [t for t in golden.expected_tools if isinstance(t, str) and t]
     else:
         # Defend against hand-edited samples passing a bare string:
         # validation rejects it at load time, but scoring must not misparse
         # it into per-character tool names either.
         expected = []
+    # A hand-edited sample may repeat a tool name; normalize to first
+    # occurrences before scoring so the hit rate cannot be inflated
+    # (["quote", "quote"] with one quote call must read 1/2, not 2/3).
+    if len(set(expected)) != len(expected):
+        violations.append("expected_tools contains duplicate names")
+        expected = list(dict.fromkeys(expected))
     missing_expected = [t for t in expected if t not in used_tools]
     expected_hit_rate = (len(expected) - len(missing_expected)) / len(expected) if expected else 0.0
     optional_tools_used = [t for t in used_tools if t not in expected]
 
-    violations: List[str] = []
     if not expected:
         violations.append("golden sample has no expected_tools")
 
@@ -221,6 +249,31 @@ def compute_trajectory_metrics(
     if optional_tools_used and not optional_allowed:
         violations.append(f"optional tools used but not allowed: {', '.join(optional_tools_used)}")
 
+    # Machine-readable outcome expectations: each declared tag must be
+    # observable in the log, otherwise the sample's guard / cache / retry
+    # contract is violated (see the module docstring).
+    if not isinstance(golden.expected_outcomes, list):
+        violations.append("expected_outcomes must be a list of outcome tags")
+        outcomes: List[str] = []
+    else:
+        outcomes = [t for t in golden.expected_outcomes if isinstance(t, str) and t]
+        if len(set(outcomes)) != len(outcomes):
+            violations.append("expected_outcomes contains duplicate tags")
+            outcomes = list(dict.fromkeys(outcomes))
+        unknown = [t for t in outcomes if t not in EXPECTED_OUTCOME_TAGS]
+        if unknown:
+            violations.append(f"unknown expected outcome tags: {', '.join(unknown)}")
+    observed = []
+    if guarded_calls:
+        observed.append("guarded")
+    if cached_calls:
+        observed.append("cached")
+    if retries:
+        observed.append("retry")
+    missing_outcomes = [t for t in outcomes if t in EXPECTED_OUTCOME_TAGS and t not in observed]
+    if missing_outcomes:
+        violations.append(f"expected outcomes not observed: {', '.join(missing_outcomes)}")
+
     limit = golden.allowed_max_steps
     if isinstance(limit, bool) or not isinstance(limit, int):
         violations.append("allowed_max_steps is not an integer")
@@ -237,7 +290,7 @@ def compute_trajectory_metrics(
         redundant_calls=redundant_calls,
         cached_calls=cached_calls,
         failed_calls=failed_calls,
-        retries=sum(key_retries.values()),
+        retries=retries,
         distinct_steps=distinct_steps,
         max_steps_touched=max_steps_touched,
         violations=violations,
@@ -323,9 +376,11 @@ def validate_golden_sample(
 
     Field *types* are part of the structural contract — hand-edited golden
     JSON must fail with a clear message instead of crashing or silently
-    passing: text fields must be strings, ``expected_tools``/``skills`` must
-    be lists, ``allowed_max_steps`` an integer and ``allow_optional_tools`` a
-    boolean.
+    passing: text fields must be strings, ``expected_tools``/``skills`` and
+    ``expected_outcomes`` must be lists, ``allowed_max_steps`` an integer and
+    ``allow_optional_tools`` a boolean.  ``expected_tools`` and
+    ``expected_outcomes`` must be duplicate-free, and outcome tags must come
+    from the fixed vocabulary ``guarded`` / ``cached`` / ``retry``.
     """
     issues: List[str] = []
     known = set(known_tool_names) if known_tool_names is not None else None
@@ -341,6 +396,8 @@ def validate_golden_sample(
         issues.append("expected_tools must be a non-empty list")
     elif any(not isinstance(t, str) or not t.strip() for t in sample.expected_tools):
         issues.append("expected_tools must contain only non-empty strings")
+    elif len(set(sample.expected_tools)) != len(sample.expected_tools):
+        issues.append("expected_tools must not contain duplicate names")
     elif known is not None:
         # Only reachable when expected_tools is a non-empty list of non-empty
         # strings, so malformed values can never crash the membership check.
@@ -357,4 +414,14 @@ def validate_golden_sample(
         issues.append("allowed_max_steps must be >= 1")
     if not isinstance(sample.allow_optional_tools, bool):
         issues.append("allow_optional_tools must be a boolean")
+    if not isinstance(sample.expected_outcomes, list):
+        issues.append("expected_outcomes must be a list of outcome tags")
+    elif any(not isinstance(t, str) or not t.strip() for t in sample.expected_outcomes):
+        issues.append("expected_outcomes must contain only non-empty strings")
+    elif len(set(sample.expected_outcomes)) != len(sample.expected_outcomes):
+        issues.append("expected_outcomes must not contain duplicate tags")
+    else:
+        unknown = [t for t in sample.expected_outcomes if t not in EXPECTED_OUTCOME_TAGS]
+        if unknown:
+            issues.append(f"unknown expected_outcomes: {', '.join(unknown)}")
     return issues