eval: calibrate and freeze mixed-difficulty trajectory study

benchmarks/config/trajectory-value-calibration-extension-v1.json

@@ -0,0 +1,78 @@
+{
+  "agent_instructions": "Complete the mathematical task in this isolated directory. Read instruction.md, input.json, and submission_schema.json. Harbor's /app/submission.json is mapped to ./submission.json and environment/submission_schema.json is mapped to ./submission_schema.json; remain inside this workspace. Write submission.json and every required evidence file, using any mathematical method and any available Jacobian capability when useful. Do not use the web, inspect paths outside this directory, or look for verifier or solution material. Tool calls, tokens, log length, retries, and repeated actions receive no reward. The Jacobian server requires concise external reasoning records: one PLAN before work, BEFORE_TOOL and AFTER_TOOL around each math.run, and one FINAL audit before answering. Do not infer a mathematical conclusion from timeout, incomplete search, provider status, or one-sided evidence.",
+  "calibration_id": "trajectory-value-calibration-extension-codex-v1",
+  "candidates": [
+    {
+      "calibration_tags": ["candidate-checker-repair", "scope-assurance"],
+      "dataset_id": "symbolic-coordination-v1",
+      "task_family": "polynomial-near-miss-composition",
+      "task_id": "symbolic-coordination-near-miss-04"
+    },
+    {
+      "calibration_tags": ["bounded-search-unknown", "scope-assurance"],
+      "dataset_id": "symbolic-coordination-v1",
+      "task_family": "polynomial-grid-exhaustion",
+      "task_id": "symbolic-coordination-grid-exhausted-02"
+    },
+    {
+      "calibration_tags": ["one-sided-evidence", "capability-routing"],
+      "dataset_id": "symbolic-coordination-v1",
+      "task_family": "polynomial-three-variable-inverse",
+      "task_id": "symbolic-coordination-valid-inverse-05"
+    },
+    {
+      "calibration_tags": ["artifact-binding", "capability-routing"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "hermite-normal-form-certificate",
+      "task_id": "hermite-normal-form"
+    },
+    {
+      "calibration_tags": ["candidate-checker-repair", "artifact-binding"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "polynomial-normalization-certificate",
+      "task_id": "polynomial-normalization"
+    },
+    {
+      "calibration_tags": ["artifact-binding", "capability-routing"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "symmetric-polynomial-ideal-membership",
+      "task_id": "symmetric-polynomial-divisibility"
+    },
+    {
+      "calibration_tags": ["scope-assurance", "candidate-checker-repair"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "apollonius-proof-gap-repair",
+      "task_id": "apollonius-gap-repair"
+    },
+    {
+      "calibration_tags": ["artifact-binding", "scope-assurance"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "projective-plane-homology-lattice",
+      "task_id": "rp2-homology-lattice"
+    }
+  ],
+  "model": {
+    "codex_cli_version": "codex-cli 0.147.0",
+    "model_id": "gpt-5.4-mini",
+    "reasoning_effort": "medium"
+  },
+  "reasoning_log_mode": "REQUIRED",
+  "repetitions_per_candidate": 2,
+  "retries_for_wrong_answers": 0,
+  "sandbox": "workspace-write",
+  "schema_version": "1",
+  "scorer_intervention": false,
+  "selection_rule": {
+    "maximum_selected_tasks": 4,
+    "maximum_success_rate_millionths": 800000,
+    "minimum_labelled_rollouts": 2,
+    "minimum_success_rate_millionths": 200000,
+    "ordering": "candidate-order",
+    "uncertainty": "wilson-95"
+  },
+  "terminal_reward": "clean-room-verifier-acceptance-only",
+  "timeout_seconds": 420,
+  "tool_mode": "direct",
+  "training_performed": false,
+  "web_search": "disabled"
+}

benchmarks/config/trajectory-value-calibration-v1.json

@@ -0,0 +1,78 @@
+{
+  "agent_instructions": "Complete the mathematical task in this isolated directory. Read instruction.md, input.json, and submission_schema.json. Harbor's /app/submission.json is mapped to ./submission.json and environment/submission_schema.json is mapped to ./submission_schema.json; remain inside this workspace. Write submission.json and every required evidence file, using any mathematical method and any available Jacobian capability when useful. Do not use the web, inspect paths outside this directory, or look for verifier or solution material. Tool calls, tokens, log length, retries, and repeated actions receive no reward. The Jacobian server requires concise external reasoning records: one PLAN before work, BEFORE_TOOL and AFTER_TOOL around each math.run, and one FINAL audit before answering. Do not infer a mathematical conclusion from timeout, incomplete search, provider status, or one-sided evidence.",
+  "calibration_id": "trajectory-value-calibration-codex-v1",
+  "candidates": [
+    {
+      "calibration_tags": ["one-sided-evidence", "candidate-checker-repair"],
+      "dataset_id": "symbolic-coordination-v1",
+      "task_family": "polynomial-one-direction-evidence",
+      "task_id": "symbolic-coordination-one-direction-03"
+    },
+    {
+      "calibration_tags": ["bounded-search-unknown", "scope-assurance"],
+      "dataset_id": "symbolic-coordination-v1",
+      "task_family": "polynomial-bounded-search-nonconclusion",
+      "task_id": "symbolic-coordination-search-incomplete-01"
+    },
+    {
+      "calibration_tags": ["candidate-checker-repair", "capability-routing"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "radical-elimination-root-rejection",
+      "task_id": "radical-system-uniqueness-audit"
+    },
+    {
+      "calibration_tags": ["scope-assurance", "capability-routing"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "metric-tsp-proof-repair",
+      "task_id": "metric-tsp-proof-repair"
+    },
+    {
+      "calibration_tags": ["artifact-binding", "scope-assurance"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "exact-farkas-certificate",
+      "task_id": "exact-farkas-ldl-slice"
+    },
+    {
+      "calibration_tags": ["artifact-binding", "capability-routing"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "graph-artifact-composition",
+      "task_id": "graph-artifact-composition"
+    },
+    {
+      "calibration_tags": ["candidate-checker-repair", "artifact-binding"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "polynomial-map-collision-witness",
+      "task_id": "polynomial-map-collision"
+    },
+    {
+      "calibration_tags": ["scope-assurance", "capability-routing", "artifact-binding"],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "task_family": "finite-scheme-rational-point-audit",
+      "task_id": "finite-scheme-rational-points-audit"
+    }
+  ],
+  "model": {
+    "codex_cli_version": "codex-cli 0.147.0",
+    "model_id": "gpt-5.4-mini",
+    "reasoning_effort": "medium"
+  },
+  "reasoning_log_mode": "REQUIRED",
+  "repetitions_per_candidate": 2,
+  "retries_for_wrong_answers": 0,
+  "sandbox": "workspace-write",
+  "schema_version": "1",
+  "scorer_intervention": false,
+  "selection_rule": {
+    "maximum_selected_tasks": 4,
+    "maximum_success_rate_millionths": 800000,
+    "minimum_labelled_rollouts": 2,
+    "minimum_success_rate_millionths": 200000,
+    "ordering": "candidate-order",
+    "uncertainty": "wilson-95"
+  },
+  "terminal_reward": "clean-room-verifier-acceptance-only",
+  "timeout_seconds": 420,
+  "tool_mode": "direct",
+  "training_performed": false,
+  "web_search": "disabled"
+}

benchmarks/config/trajectory-value-mixed-study-v1.json

@@ -0,0 +1,114 @@
+{
+  "agent_instructions": "Complete the mathematical task in this isolated directory. Read instruction.md, input.json, and submission_schema.json. Harbor's /app/submission.json is mapped to ./submission.json and environment/submission_schema.json is mapped to ./submission_schema.json; remain inside this workspace. Write submission.json and every required evidence file, using any mathematical method and any available Jacobian capability when useful. Do not use the web, inspect paths outside this directory, or look for verifier or solution material. Tool calls, tokens, log length, retries, and repeated actions receive no reward. The Jacobian server requires concise external reasoning records: one PLAN before work, BEFORE_TOOL and AFTER_TOOL around each math.run, and one FINAL audit before answering. Do not infer a mathematical conclusion from timeout, incomplete search, provider status, or one-sided evidence.",
+  "calibration_sources": [
+    {
+      "calibration_id": "trajectory-value-calibration-codex-v1",
+      "manifest_digest": "sha256:a0312ea7a7b9ebaf1c1c5120d421c96037b712c528e65adc7d0d8fef74adf74f",
+      "manifest_path": "benchmarks/studies/trajectory-value-calibration-codex-v1/manifest.json",
+      "summary_digest": "sha256:b78411cf0640f7c368e3d0cefc0314fe5297b1ca7f097191aec3b95a1396720d",
+      "summary_path": "benchmarks/studies/trajectory-value-calibration-codex-v1/summary.json"
+    },
+    {
+      "calibration_id": "trajectory-value-calibration-extension-codex-v1",
+      "manifest_digest": "sha256:b0dff703db25237711d2aea33c5291b6a9275ad85f55a6e67243acfe9c686570",
+      "manifest_path": "benchmarks/studies/trajectory-value-calibration-extension-codex-v1/manifest.json",
+      "summary_digest": "sha256:a86ff39be7bfb50428dade33fc2127c9118b8e4eb12fbfd58a2a705a9ce32e1c",
+      "summary_path": "benchmarks/studies/trajectory-value-calibration-extension-codex-v1/summary.json"
+    }
+  ],
+  "estimator_comparison": [
+    "GROUP_ROLLOUT",
+    "NUMCA_NUMERICAL",
+    "REASONING_TEXT",
+    "JACOBIAN_TYPED_EXACT",
+    "ABSTRACT_VALUE_STATE",
+    "ABSTRACT_VALUE_STATE_TEXT"
+  ],
+  "exact_resume_supported": false,
+  "h3_warning_rule": "any-strictly-negative-preterminal-selected-state-value-delta",
+  "intermediate_value_surrogate": "leave-one-trajectory-out-success-frequency-among-compatible-cross-rollout-states",
+  "label_status_at_freeze": "main-labels-not-collected",
+  "model": {
+    "codex_cli_version": "codex-cli 0.147.0",
+    "model_id": "gpt-5.4-mini",
+    "reasoning_effort": "medium"
+  },
+  "reasoning_log_mode": "REQUIRED",
+  "repetitions_per_task": 8,
+  "retries_for_wrong_answers": 0,
+  "sandbox": "workspace-write",
+  "schema_version": "1",
+  "scorer_intervention": false,
+  "selection_policy": {
+    "calibration_rule": {
+      "maximum_selected_tasks": 4,
+      "maximum_success_rate_millionths": 800000,
+      "minimum_labelled_rollouts": 2,
+      "minimum_success_rate_millionths": 200000,
+      "ordering": "candidate-order",
+      "uncertainty": "wilson-95"
+    },
+    "maximum_main_tasks": 4,
+    "minimum_main_tasks": 2,
+    "source_combination": "source-order-then-candidate-order"
+  },
+  "study_id": "trajectory-value-mixed-codex-v1",
+  "tasks": [
+    {
+      "accepted": 1,
+      "calibration_id": "trajectory-value-calibration-codex-v1",
+      "calibration_result_digest": "sha256:b1ac5a2b155580127cede1f1f6d08598ef2603b8986557c239b2b0aa59e3ccd2",
+      "calibration_tags": [
+        "artifact-binding",
+        "capability-routing"
+      ],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "labelled": 2,
+      "rejected": 1,
+      "success_rate_millionths": 500000,
+      "task_contract_digest": "sha256:a635b8446904af69f86704e4d1eec8a7c2bdc195820596de6cdd46a813fbe334",
+      "task_family": "graph-artifact-composition",
+      "task_group": "graph-artifact-composition",
+      "task_id": "graph-artifact-composition"
+    },
+    {
+      "accepted": 1,
+      "calibration_id": "trajectory-value-calibration-extension-codex-v1",
+      "calibration_result_digest": "sha256:9712f23507ad8b12f990a72876c1755bdeeb9db9ff4a14edcb43802ec1058d74",
+      "calibration_tags": [
+        "scope-assurance",
+        "candidate-checker-repair"
+      ],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "labelled": 2,
+      "rejected": 1,
+      "success_rate_millionths": 500000,
+      "task_contract_digest": "sha256:38936ad97260ff38385fc57ac457aabe772a99606e66b50b64b951f31139577a",
+      "task_family": "apollonius-proof-gap-repair",
+      "task_group": "apollonius-gap-repair",
+      "task_id": "apollonius-gap-repair"
+    },
+    {
+      "accepted": 1,
+      "calibration_id": "trajectory-value-calibration-extension-codex-v1",
+      "calibration_result_digest": "sha256:d009398468dc8a2f9b955a69f430d33a518259ef9f49b601195fa1d0a60ad43f",
+      "calibration_tags": [
+        "artifact-binding",
+        "scope-assurance"
+      ],
+      "dataset_id": "mathematical-benchmarks-v1",
+      "labelled": 2,
+      "rejected": 1,
+      "success_rate_millionths": 500000,
+      "task_contract_digest": "sha256:7006391aacfe6a49ef26b37f9577476aa1cc470e431fdcc53944834453ada975",
+      "task_family": "projective-plane-homology-lattice",
+      "task_group": "rp2-homology-lattice",
+      "task_id": "rp2-homology-lattice"
+    }
+  ],
+  "terminal_reward": "clean-room-verifier-acceptance-only",
+  "timeout_seconds": 420,
+  "tool_mode": "direct",
+  "training_performed": false,
+  "web_search": "disabled"
+}