Refactor MCP SDK 2.0 boundaries and remove reasoning workflows (#956)

* refactor(sdk): preserve concrete worker contracts

* refactor(mcp): stabilize tools and remove reasoning mode

* refactor(eval): remove reasoning-coupled analyzers

* fix(lean): parse pickle acknowledgements separately

Author: morluto

benchmarks/datasets/mathematical-benchmarks-v1/README.md

@@ -54,27 +54,6 @@ both jobs; only the treatment adds the Jacobian sidecar and MCP config. This
 paired setup is for workflow comparison; the public dataset is not held-out
 evidence.
 
-To isolate the external reasoning-log protocol, keep Jacobian enabled in both
-arms and vary only its mode:
-
-```sh
-# Control: identical Jacobian portfolio without the log protocol.
-JACOBIAN_REASONING_LOG_MODE=off make agent-eval \
-  DATASET=mathematical-benchmarks-v1 JACOBIAN_ENABLED=1 \
-  TASKS=graph-counterexample EVAL_EXECUTE=1
-
-# Treatment: identical Jacobian portfolio with required bracketing.
-JACOBIAN_REASONING_LOG_MODE=required make agent-eval \
-  DATASET=mathematical-benchmarks-v1 JACOBIAN_ENABLED=1 \
-  TASKS=graph-counterexample EVAL_EXECUTE=1
-```
-
-Bind `reasoning_log_mode` in each runtime snapshot's condition. The normalizer
-requires a structurally complete `PLAN`/call-cycle/`FINAL` trace for every
-`REQUIRED` trial and excludes summary text from normalized evidence. Public
-workflow observations remain non-causal; protected runs must also freeze the
-model, prompt, agent version, task digests, sampling settings, and budgets.
-
 To evaluate the canonical `math.find` and `math.run` surface, keep each model,
 task set, and prompt condition in a separate result root:
 
@@ -93,8 +72,8 @@ tool adoption. Public-suite observations remain directional workflow evidence,
 not a causal performance claim.
 
 Jacobian-enabled jobs collect both Codex ATIF and the Jacobian sidecar's MCP
-runtime log. The runtime log is authoritative for `math.find`, `math.run`, and
-`reasoning.write` counts and failed capability attempts; the normalizer does
+runtime log. The runtime log is authoritative for `math.find` and `math.run`
+counts and failed capability attempts; the normalizer does
 not infer executions from JavaScript source text. A missing configured trace or
 sidecar log makes the observation incomplete. Control jobs collect ATIF only
 because they do not start the Jacobian sidecar.

benchmarks/datasets/mathematical-benchmarks-v1/jacobian-observation.compose.yaml

@@ -20,8 +20,6 @@ services:
       - --port
       - "8000"
       - --allow-anonymous
-      - --reasoning-log-mode
-      - ${JACOBIAN_REASONING_LOG_MODE:-off}
       - --state-dir
       - /state
     volumes:

benchmarks/datasets/symbolic-coordination-v1/README.md

@@ -54,6 +54,6 @@ make harbor-oracle-task DATASET=symbolic-coordination-v1 TASKS="<task ids>"
 make harbor-check
 ```
 
-PR1 contains no comparison job, model run, post-solution audit, reasoning-log
-analysis, or training contract. Product-surface observations that did not block
+PR1 contains no comparison job, model run, post-solution audit, or training
+contract. Product-surface observations that did not block
 the pilot are recorded in [deferred capability gaps](CAPABILITY_GAPS.md).

benchmarks/schemas/held-out-manifest.schema.json

@@ -118,7 +118,6 @@
         "id": {"const": "C2"},
         "role": {"const": "PRIMARY_TREATMENT"},
         "jacobian_enabled": {"const": true},
-        "reasoning_log_mode": {"enum": ["REQUIRED", "AUDIT", "OFF"]},
         "image": {"type": "string", "pattern": "^[^@]+@sha256:[0-9a-f]{64}$"},
         "source_sha": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
         "platform": {"type": "string", "pattern": "^[a-z0-9_]+/[a-z0-9_]+$"},

benchmarks/schemas/observation-evidence.schema.json

@@ -2,7 +2,7 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://jacobian.invalid/benchmarks/schemas/observation-evidence.schema.json",
   "title": "Jacobian Harbor workflow observation evidence",
-  "description": "Strict normalized observation evidence v3. Binds the immutable benchmark snapshot ID and Harbor version (read from explicit job/runtime/held-out manifest bindings, never invented), normalized evaluation arguments, runtime/model/agent/repetition/budgets/results/rewards/verifier states/failures, and manifest-driven per-artifact identity. Fails closed: VALID only when execution is complete and every binding matches. The reasoning_protocol trial member is required. No v2 compatibility.",
+  "description": "Strict normalized observation evidence v4. Binds the immutable benchmark snapshot ID and Harbor version (read from explicit job/runtime/held-out manifest bindings, never invented), normalized evaluation arguments, runtime/model/agent/repetition/budgets/results/rewards/verifier states/failures, and manifest-driven per-artifact identity. Fails closed: VALID only when execution is complete and every binding matches. No compatibility path for earlier evidence versions.",
   "type": "object",
   "additionalProperties": false,
   "required": [
@@ -24,7 +24,7 @@
     "validation_failures"
   ],
   "properties": {
-    "schema_version": {"const": "3"},
+    "schema_version": {"const": "4"},
     "evidence_class": {"type": "string", "minLength": 1},
     "causal_claim_authorized": {"const": false},
     "status": {"enum": ["VALID", "INCOMPLETE"]},
@@ -244,7 +244,6 @@
         "artifacts",
         "tool_calls",
         "tool_errors",
-        "reasoning_protocol",
         "raw_result_digest"
       ],
       "properties": {
@@ -310,7 +309,6 @@
           "additionalProperties": {"type": "integer"}
         },
         "tool_errors": {"type": "integer", "minimum": 0},
-        "reasoning_protocol": {"$ref": "#/$defs/reasoning_protocol"},
         "raw_result_digest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}
       }
     },
@@ -369,7 +367,6 @@
         "id": {"type": "string", "minLength": 1},
         "role": {"type": "string", "minLength": 1},
         "jacobian_enabled": {"type": "boolean"},
-        "reasoning_log_mode": {"enum": ["REQUIRED", "AUDIT", "OFF"]},
         "image": {"type": "string", "minLength": 1},
         "source_sha": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
         "platform": {"type": "string", "pattern": "^[a-z0-9_]+/[a-z0-9_]+$"},
@@ -379,42 +376,6 @@
         "policy_digest": {"$ref": "#/$defs/digest"}
       }
     },
-    "reasoning_protocol": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "status",
-        "mode",
-        "requirement_status",
-        "plan_count",
-        "before_tool_count",
-        "after_tool_count",
-        "final_count",
-        "run_count",
-        "bound_invoke_count",
-        "missing_after_tool_count",
-        "pending_call_count",
-        "unavailable_after_tool_count",
-        "reported_actual_mismatch_count",
-        "summary_characters"
-      ],
-      "properties": {
-        "status": {"enum": ["COMPLETE", "INCOMPLETE"]},
-        "mode": {"enum": ["REQUIRED", "AUDIT", "OFF", "UNKNOWN"]},
-        "requirement_status": {"enum": ["COMPLETE", "INCOMPLETE", "NOT_REQUIRED"]},
-        "plan_count": {"type": "integer", "minimum": 0},
-        "before_tool_count": {"type": "integer", "minimum": 0},
-        "after_tool_count": {"type": "integer", "minimum": 0},
-        "final_count": {"type": "integer", "minimum": 0},
-        "run_count": {"type": "integer", "minimum": 0},
-        "bound_invoke_count": {"type": "integer", "minimum": 0},
-        "missing_after_tool_count": {"type": "integer", "minimum": 0},
-        "pending_call_count": {"type": "integer", "minimum": 0},
-        "unavailable_after_tool_count": {"type": "integer", "minimum": 0},
-        "reported_actual_mismatch_count": {"type": "integer", "minimum": 0},
-        "summary_characters": {"type": "integer", "minimum": 0}
-      }
-    },
     "runtime_invariants": {
       "type": "object",
       "additionalProperties": false,

benchmarks/tooling/observation_artifacts.py

@@ -17,10 +17,9 @@
 from typing import Any
 
 from benchmarks.tooling.errors import HarborSuiteError
-from jacobian.eval.telemetry import parse_reasoning_protocol_trace
 
 _MCP_TOOL_CALL = re.compile(
-    r"\bMCP tool call tool=(math\.(?:find|run)|reasoning\.write)\b"
+    r"\bMCP tool call tool=(math\.(?:find|run))\b"
     r".{0,512}?\bstatus=(success|error)\b"
     r".{0,512}?\brequest_digest=([0-9a-f]{16}|none)\b",
     re.DOTALL,
@@ -36,7 +35,6 @@ def _canonical_tool_name(value: str) -> str:
     aliases = {
         "mcp__jacobian__math_find": "math.find",
         "mcp__jacobian__math_run": "math.run",
-        "mcp__jacobian__reasoning_write": "reasoning.write",
     }
     return aliases.get(value, value)
 
@@ -611,9 +609,7 @@ def trial_artifacts(
         )
     ]
     ignored_tools = (
-        frozenset({"math.find", "math.run", "reasoning.write"})
-        if runtime_logs
-        else frozenset()
+        frozenset({"math.find", "math.run"}) if runtime_logs else frozenset()
     )
     errors = sum(
         _read_trace(path, calls, ignored_tools=ignored_tools) for path in agent_traces
@@ -624,67 +620,6 @@ def trial_artifacts(
     return artifacts, dict(sorted(calls.items())), errors, failures
 
 
-def trial_reasoning_protocol(
-    trial_path: Path | None,
-    artifacts: list[dict[str, Any]],
-) -> dict[str, int | str]:
-    """Extract bounded reasoning-protocol facts without retaining summary text."""
-
-    empty: dict[str, int | str] = {
-        "status": "INCOMPLETE",
-        "plan_count": 0,
-        "before_tool_count": 0,
-        "after_tool_count": 0,
-        "final_count": 0,
-        "run_count": 0,
-        "bound_invoke_count": 0,
-        "missing_after_tool_count": 0,
-        "pending_call_count": 0,
-        "unavailable_after_tool_count": 0,
-        "reported_actual_mismatch_count": 0,
-        "summary_characters": 0,
-    }
-    if trial_path is None:
-        return empty
-    root = trial_path.parent
-    candidates = [
-        _artifact_host_path(root, artifact)
-        for artifact in artifacts
-        if Path(artifact["artifact_path"]).suffix in {".json", ".jsonl"}
-        and any(
-            marker in Path(artifact["artifact_path"]).name.lower()
-            for marker in ("trajectory", "atif", "telemetry")
-        )
-    ]
-    observed: list[dict[str, int | str]] = []
-    for path in candidates:
-        try:
-            protocol = parse_reasoning_protocol_trace(path)
-        except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError):
-            continue
-        if isinstance(protocol, dict) and any(
-            int(protocol.get(field, 0)) > 0
-            for field in (
-                "plan_count",
-                "before_tool_count",
-                "after_tool_count",
-                "final_count",
-                "bound_invoke_count",
-            )
-        ):
-            observed.append(protocol)
-    if len(observed) == 1:
-        return observed[0]
-    if not observed:
-        return empty
-    combined = dict(empty)
-    for field in empty:
-        if field == "status":
-            continue
-        combined[field] = sum(int(item.get(field, 0)) for item in observed)
-    return combined
-
-
 def artifact_source_reuse(trials: list[dict[str, Any]]) -> list[str]:
     """Reject reuse of one canonical host source path across trials."""
 
@@ -707,5 +642,4 @@ def artifact_source_reuse(trials: list[dict[str, Any]]) -> list[str]:
 __all__ = [
     "artifact_source_reuse",
     "trial_artifacts",
-    "trial_reasoning_protocol",
 ]

benchmarks/tooling/observation_comparison.py

@@ -89,60 +89,6 @@ def _mcnemar_exact(control: list[float], treatment: list[float]) -> float | None
     return float(min(1.0, 2 * tail))
 
 
-_VARYING_CONDITION_KEYS = frozenset({"id", "role", "reasoning_log_mode"})
-
-
-def _reasoning_condition_failures(
-    control_condition: Any, treatment_condition: Any
-) -> list[str]:
-    """Validate the reasoning-log condition pair and its non-varying invariants.
-
-    Only applies when both arms have Jacobian enabled — a baseline
-    comparison (control has Jacobian disabled) is not a reasoning-log
-    isolation experiment and must not be rejected here.
-    """
-
-    failures: list[str] = []
-    if not (
-        isinstance(control_condition, dict)
-        and isinstance(treatment_condition, dict)
-        and control_condition.get("jacobian_enabled") is True
-        and treatment_condition.get("jacobian_enabled") is True
-    ):
-        return failures
-    control_mode = control_condition.get("reasoning_log_mode")
-    treatment_mode = treatment_condition.get("reasoning_log_mode")
-    if control_mode is None and treatment_mode is None:
-        return failures
-    if (control_mode, treatment_mode) != ("OFF", "REQUIRED"):
-        failures.append(
-            "reasoning-log comparison must pair OFF control with REQUIRED treatment"
-        )
-    # Compare every condition field except the explicitly varying
-    # identity, role, and reasoning_log_mode fields.  Divergent
-    # image, server_version, policy_profile, catalog_digest, or
-    # policy_digest means the two arms used different Jacobian code
-    # or capability portfolios and cannot support a valid
-    # reasoning-log comparison.
-    if isinstance(control_condition, dict) and isinstance(treatment_condition, dict):
-        control_invariant = {
-            k: v
-            for k, v in control_condition.items()
-            if k not in _VARYING_CONDITION_KEYS
-        }
-        treatment_invariant = {
-            k: v
-            for k, v in treatment_condition.items()
-            if k not in _VARYING_CONDITION_KEYS
-        }
-        if control_invariant != treatment_invariant:
-            failures.append(
-                "reasoning-log condition invariants differ outside "
-                "the allowed id/role/mode fields"
-            )
-    return failures
-
-
 def _comparison_failures(
     control: dict[str, Any], treatment: dict[str, Any]
 ) -> list[str]:
@@ -177,12 +123,6 @@ def _comparison_failures(
     )
     if control.get("fixed_invariants") != treatment.get("fixed_invariants"):
         failures.append("fixed invariants differ")
-    control_condition = control.get("runtime_snapshot", {}).get("condition")
-    treatment_condition = treatment.get("runtime_snapshot", {}).get("condition")
-    if isinstance(control_condition, dict) or isinstance(treatment_condition, dict):
-        failures.extend(
-            _reasoning_condition_failures(control_condition, treatment_condition)
-        )
     if control.get("job", {}).get("comparison_signature") != treatment.get(
         "job", {}
     ).get("comparison_signature"):