Skip to content

Commit 64118c5

Browse files
cursoragentmorluto
andcommitted
fix(eval): sync remaining trajectory-value review hardenings
Co-authored-by: morluto <morluto@users.noreply.github.com>
1 parent 32c6bd7 commit 64118c5

784 files changed

Lines changed: 103549 additions & 7 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
{
2+
"agent_instructions": "Complete the mathematical task in this isolated directory. Read instruction.md, input.json, and submission_schema.json. Harbor's /app/submission.json is mapped to ./submission.json and environment/submission_schema.json is mapped to ./submission_schema.json; remain inside this workspace. Write submission.json and every required evidence file, using any mathematical method and any available Jacobian capability when useful. Do not use the web, inspect paths outside this directory, or look for verifier or solution material. Tool calls, tokens, log length, retries, and repeated actions receive no reward. The Jacobian server requires concise external reasoning records: one PLAN before work, BEFORE_TOOL and AFTER_TOOL around each math.run, and one FINAL audit before answering. Do not infer a mathematical conclusion from timeout, incomplete search, provider status, or one-sided evidence.",
3+
"calibration_sources": [
4+
{
5+
"calibration_id": "trajectory-value-calibration-codex-v1",
6+
"manifest_digest": "sha256:d087a1fff3632ba6f1a34fdfb983cbaae0812faa85dea4d040fdcf3ec311188f",
7+
"manifest_path": "benchmarks/studies/trajectory-value-calibration-codex-v1/manifest.json",
8+
"summary_digest": "sha256:1de7997f93a1eeeb09508600f8b40ddf12b36a5036e7030cc15f7248589b8df2",
9+
"summary_path": "benchmarks/studies/trajectory-value-calibration-codex-v1/summary.json"
10+
},
11+
{
12+
"calibration_id": "trajectory-value-calibration-extension-codex-v1",
13+
"manifest_digest": "sha256:3df9d76e6d79682ef381b60da4d47ab89577d358b3f14eb98c15ede5b07927a0",
14+
"manifest_path": "benchmarks/studies/trajectory-value-calibration-extension-codex-v1/manifest.json",
15+
"summary_digest": "sha256:8f2224b72ca9f468284e6f01fc07486785cc0bd4b30b17de4ecb4240c49d48dd",
16+
"summary_path": "benchmarks/studies/trajectory-value-calibration-extension-codex-v1/summary.json"
17+
}
18+
],
19+
"estimator_comparison": [
20+
"GROUP_ROLLOUT",
21+
"NUMCA_NUMERICAL",
22+
"REASONING_TEXT",
23+
"JACOBIAN_TYPED_EXACT",
24+
"ABSTRACT_VALUE_STATE",
25+
"ABSTRACT_VALUE_STATE_TEXT"
26+
],
27+
"exact_resume_supported": false,
28+
"h3_warning_rule": "any-strictly-negative-preterminal-selected-state-value-delta",
29+
"intermediate_value_surrogate": "leave-one-trajectory-out-success-frequency-among-compatible-cross-rollout-states",
30+
"label_status_at_freeze": "main-labels-not-collected",
31+
"model": {
32+
"codex_cli_version": "codex-cli 0.147.0",
33+
"model_id": "gpt-5.4-mini",
34+
"reasoning_effort": "medium"
35+
},
36+
"reasoning_log_mode": "REQUIRED",
37+
"repetitions_per_task": 8,
38+
"retries_for_wrong_answers": 0,
39+
"sandbox": "workspace-write",
40+
"schema_version": "1",
41+
"scorer_intervention": false,
42+
"selection_policy": {
43+
"calibration_rule": {
44+
"maximum_selected_tasks": 4,
45+
"maximum_success_rate_millionths": 800000,
46+
"minimum_labelled_rollouts": 2,
47+
"minimum_success_rate_millionths": 200000,
48+
"ordering": "candidate-order",
49+
"uncertainty": "wilson-95"
50+
},
51+
"maximum_main_tasks": 4,
52+
"minimum_main_tasks": 1,
53+
"source_combination": "source-order-then-candidate-order"
54+
},
55+
"study_id": "trajectory-value-mixed-codex-v1",
56+
"tasks": [
57+
{
58+
"accepted": 1,
59+
"calibration_id": "trajectory-value-calibration-extension-codex-v1",
60+
"calibration_result_digest": "sha256:9712f23507ad8b12f990a72876c1755bdeeb9db9ff4a14edcb43802ec1058d74",
61+
"calibration_tags": [
62+
"scope-assurance",
63+
"candidate-checker-repair"
64+
],
65+
"dataset_id": "mathematical-benchmarks-v1",
66+
"labelled": 2,
67+
"rejected": 1,
68+
"success_rate_millionths": 500000,
69+
"task_contract_digest": "sha256:38936ad97260ff38385fc57ac457aabe772a99606e66b50b64b951f31139577a",
70+
"task_family": "apollonius-proof-gap-repair",
71+
"task_group": "apollonius-gap-repair",
72+
"task_id": "apollonius-gap-repair"
73+
}
74+
],
75+
"terminal_reward": "clean-room-verifier-acceptance-only",
76+
"timeout_seconds": 420,
77+
"tool_mode": "direct",
78+
"training_performed": false,
79+
"web_search": "disabled"
80+
}

0 commit comments

Comments
 (0)