Skip to content

Commit f9e197e

Browse files
kaoru0822-kitaujicursoragent
kaoru0822-kitauji
authored and
cursoragent
committed
eval: record initial difficulty calibration
1 parent 81e981b commit f9e197e

247 files changed

Lines changed: 30020 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎benchmarks/config/trajectory-value-calibration-extension-v1.json‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
{
2+
"agent_instructions": "Complete the mathematical task in this isolated directory. Read instruction.md, input.json, and submission_schema.json. Harbor's /app/submission.json is mapped to ./submission.json and environment/submission_schema.json is mapped to ./submission_schema.json; remain inside this workspace. Write submission.json and every required evidence file, using any mathematical method and any available Jacobian capability when useful. Do not use the web, inspect paths outside this directory, or look for verifier or solution material. Tool calls, tokens, log length, retries, and repeated actions receive no reward. The Jacobian server requires concise external reasoning records: one PLAN before work, BEFORE_TOOL and AFTER_TOOL around each math.run, and one FINAL audit before answering. Do not infer a mathematical conclusion from timeout, incomplete search, provider status, or one-sided evidence.",
3+
"calibration_id": "trajectory-value-calibration-extension-codex-v1",
4+
"candidates": [
5+
{
6+
"calibration_tags": ["candidate-checker-repair", "scope-assurance"],
7+
"dataset_id": "symbolic-coordination-v1",
8+
"task_family": "polynomial-near-miss-composition",
9+
"task_id": "symbolic-coordination-near-miss-04"
10+
},
11+
{
12+
"calibration_tags": ["bounded-search-unknown", "scope-assurance"],
13+
"dataset_id": "symbolic-coordination-v1",
14+
"task_family": "polynomial-grid-exhaustion",
15+
"task_id": "symbolic-coordination-grid-exhausted-02"
16+
},
17+
{
18+
"calibration_tags": ["one-sided-evidence", "capability-routing"],
19+
"dataset_id": "symbolic-coordination-v1",
20+
"task_family": "polynomial-three-variable-inverse",
21+
"task_id": "symbolic-coordination-valid-inverse-05"
22+
},
23+
{
24+
"calibration_tags": ["artifact-binding", "capability-routing"],
25+
"dataset_id": "mathematical-benchmarks-v1",
26+
"task_family": "hermite-normal-form-certificate",
27+
"task_id": "hermite-normal-form"
28+
},
29+
{
30+
"calibration_tags": ["candidate-checker-repair", "artifact-binding"],
31+
"dataset_id": "mathematical-benchmarks-v1",
32+
"task_family": "polynomial-normalization-certificate",
33+
"task_id": "polynomial-normalization"
34+
},
35+
{
36+
"calibration_tags": ["artifact-binding", "capability-routing"],
37+
"dataset_id": "mathematical-benchmarks-v1",
38+
"task_family": "symmetric-polynomial-ideal-membership",
39+
"task_id": "symmetric-polynomial-divisibility"
40+
},
41+
{
42+
"calibration_tags": ["scope-assurance", "candidate-checker-repair"],
43+
"dataset_id": "mathematical-benchmarks-v1",
44+
"task_family": "apollonius-proof-gap-repair",
45+
"task_id": "apollonius-gap-repair"
46+
},
47+
{
48+
"calibration_tags": ["artifact-binding", "scope-assurance"],
49+
"dataset_id": "mathematical-benchmarks-v1",
50+
"task_family": "projective-plane-homology-lattice",
51+
"task_id": "rp2-homology-lattice"
52+
}
53+
],
54+
"model": {
55+
"codex_cli_version": "codex-cli 0.147.0",
56+
"model_id": "gpt-5.4-mini",
57+
"reasoning_effort": "medium"
58+
},
59+
"reasoning_log_mode": "REQUIRED",
60+
"repetitions_per_candidate": 2,
61+
"retries_for_wrong_answers": 0,
62+
"sandbox": "workspace-write",
63+
"schema_version": "1",
64+
"scorer_intervention": false,
65+
"selection_rule": {
66+
"maximum_selected_tasks": 4,
67+
"maximum_success_rate_millionths": 800000,
68+
"minimum_labelled_rollouts": 2,
69+
"minimum_success_rate_millionths": 200000,
70+
"ordering": "candidate-order",
71+
"uncertainty": "wilson-95"
72+
},
73+
"terminal_reward": "clean-room-verifier-acceptance-only",
74+
"timeout_seconds": 420,
75+
"tool_mode": "direct",
76+
"training_performed": false,
77+
"web_search": "disabled"
78+
}

‎benchmarks/studies/trajectory-value-calibration-codex-v1/manifest.json‎

Lines changed: 512 additions & 0 deletions
Large diffs are not rendered by default.

‎benchmarks/studies/trajectory-value-calibration-codex-v1/runs/exact-farkas-ldl-slice-cal-r01/codex.jsonl‎

Lines changed: 47 additions & 0 deletions
Large diffs are not rendered by default.

‎benchmarks/studies/trajectory-value-calibration-codex-v1/runs/exact-farkas-ldl-slice-cal-r01/codex.stderr‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Reading additional input from stdin...

‎benchmarks/studies/trajectory-value-calibration-codex-v1/runs/exact-farkas-ldl-slice-cal-r01/jacobian-mcp.stderr‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
Uninstalled 1 package in 2ms
2+
Installed 1 package in 2ms
3+
INFO: Started server process [26587]
4+
INFO: Waiting for application startup.
5+
[08/07/26 01:28:14] INFO StreamableHTTP streamable_http_manager.py:151
6+
session manager
7+
started
8+
INFO: Application startup complete.
9+
INFO: Uvicorn running on http://127.0.0.1:59868 (Press CTRL+C to quit)
10+
[08/07/26 01:28:16] WARNING lean.check is not reference_installation.py:114
11+
installed: The pinned
12+
Lean 4.31.0 runtime
13+
is unavailable.
14+
INFO Created new streamable_http_manager.py:303
15+
transport with
16+
session ID:
17+
e0ad8727c664467ba61f
18+
af6816cb723f
19+
[08/07/26 01:28:43] INFO MCP tool call tool=reasoning.write server.py:335
20+
status=success
21+
request_digest=d4735e3a265e16ee
22+
trace_digest=d4735e3a
23+
trace_source=request_id
24+
duration_ms=5.545 response_bytes=860
25+
argument_digest=sha256:1cac1dff1d1a45
26+
86f948b411691db39bc5728bf235e6385d6bf
27+
cc872b6f67ba8
28+
[08/07/26 01:28:46] INFO MCP tool call tool=math.find server.py:335
29+
status=success
30+
request_digest=4e07408562bedb8b
31+
trace_digest=4e074085
32+
trace_source=request_id
33+
duration_ms=137.048
34+
response_bytes=14271
35+
argument_digest=sha256:0fd734fb632adc
36+
12c9c7e332023c28f870339e55ac9b11ce1fe
37+
a7a7bbe07eba3
38+
[08/07/26 01:29:19] INFO MCP tool call tool=math.find server.py:335
39+
status=success
40+
request_digest=4b227777d4dd1fc6
41+
trace_digest=4b227777
42+
trace_source=request_id
43+
duration_ms=57.828
44+
response_bytes=2871
45+
argument_digest=sha256:b9876307991891
46+
45607955b017009788445f4c9e71a52198916
47+
a48aa09b4b848
48+
[08/07/26 01:30:18] WARNING MCP tool reasoning.write failed server.py:318
49+
╭─ Traceback (most recent call last─╮
50+
│ /Users/saionjitoko0822/Documents/ │
51+
│ GitHub/jacobian-trajectory-value- │
52+
│ pr5/src/jacobian/adapters/mcp/ser │
53+
│ ver.py:314 in wrapped │
54+
│ │
55+
│ 311 │ @wraps(handler) │
56+
│ 312 │ async def wrapped(*args │
57+
│ 313 │ │ try: │
58+
│ ❱ 314 │ │ │ return await ha │
59+
│ 315 │ │ except MCPError: │
60+
│ 316 │ │ │ raise │
61+
│ 317 │ │ except Exception as │
62+
│ │
63+
│ /Users/saionjitoko0822/Documents/ │
64+
│ GitHub/jacobian-trajectory-value- │
65+
│ pr5/src/jacobian/adapters/mcp/too │
66+
│ ls.py:443 in reasoning_write │
67+
│ │
68+
│ 440 │ ctx: Context[AppState, │
69+
│ 441 ) -> ReasoningWriteResult: │
70+
│ 442 │ active_runtime = _runti │
71+
│ ❱ 443 │ request = ReasoningWrit │
72+
│ 444 │ │ phase=phase, │
73+
│ 445 │ │ summary=summary, │
74+
│ 446 │ │ run_id=run_id, │
75+
│ │
76+
│ /Users/saionjitoko0822/Documents/ │
77+
│ GitHub/jacobian-trajectory-value- │
78+
│ pr5/.venv/lib/python3.12/site-pac │
79+
│ kages/pydantic/main.py:263 in │
80+
│ __init__ │
81+
│ │
82+
│ 260 │ │ """ │
83+
│ 261 │ │ # `__tracebackhide │
84+
│ from tracebacks │
85+
│ 262 │ │ __tracebackhide__ │
86+
│ ❱ 263 │ │ validated_self = s │
87+
│ self_instance=self) │
88+
│ 264 │ │ if self is not val │
89+
│ 265 │ │ │ warnings.warn( │
90+
│ 266 │ │ │ │ 'A custom │
91+
╰───────────────────────────────────╯
92+
ValidationError: 1 validation error
93+
for ReasoningWriteRequest
94+
Value error, fields do not match
95+
FINAL phase contract
96+
[type=value_error,
97+
input_value={'phase':
98+
<ReasoningPhase...pleteness_status':
99+
None}, input_type=dict]
100+
For further information visit
101+
https://errors.pydantic.dev/2.13/v/va
102+
lue_error
103+
INFO MCP tool call tool=reasoning.write server.py:335
104+
status=success
105+
request_digest=ef2d127de37b942b
106+
trace_digest=ef2d127d
107+
trace_source=request_id
108+
duration_ms=70.279 response_bytes=401
109+
argument_digest=sha256:75bb607fae1857
110+
f4833ddf722fae90c92b95dffb455e15371b4
111+
d335f486d86ae
112+
[08/07/26 01:30:26] WARNING MCP tool reasoning.write failed server.py:318
113+
╭─ Traceback (most recent call last─╮
114+
│ /Users/saionjitoko0822/Documents/ │
115+
│ GitHub/jacobian-trajectory-value- │
116+
│ pr5/src/jacobian/adapters/mcp/ser │
117+
│ ver.py:314 in wrapped │
118+
│ │
119+
│ 311 │ @wraps(handler) │
120+
│ 312 │ async def wrapped(*args │
121+
│ 313 │ │ try: │
122+
│ ❱ 314 │ │ │ return await ha │
123+
│ 315 │ │ except MCPError: │
124+
│ 316 │ │ │ raise │
125+
│ 317 │ │ except Exception as │
126+
│ │
127+
│ /Users/saionjitoko0822/Documents/ │
128+
│ GitHub/jacobian-trajectory-value- │
129+
│ pr5/src/jacobian/adapters/mcp/too │
130+
│ ls.py:443 in reasoning_write │
131+
│ │
132+
│ 440 │ ctx: Context[AppState, │
133+
│ 441 ) -> ReasoningWriteResult: │
134+
│ 442 │ active_runtime = _runti │
135+
│ ❱ 443 │ request = ReasoningWrit │
136+
│ 444 │ │ phase=phase, │
137+
│ 445 │ │ summary=summary, │
138+
│ 446 │ │ run_id=run_id, │
139+
│ │
140+
│ /Users/saionjitoko0822/Documents/ │
141+
│ GitHub/jacobian-trajectory-value- │
142+
│ pr5/.venv/lib/python3.12/site-pac │
143+
│ kages/pydantic/main.py:263 in │
144+
│ __init__ │
145+
│ │
146+
│ 260 │ │ """ │
147+
│ 261 │ │ # `__tracebackhide │
148+
│ from tracebacks │
149+
│ 262 │ │ __tracebackhide__ │
150+
│ ❱ 263 │ │ validated_self = s │
151+
│ self_instance=self) │
152+
│ 264 │ │ if self is not val │
153+
│ 265 │ │ │ warnings.warn( │
154+
│ 266 │ │ │ │ 'A custom │
155+
╰───────────────────────────────────╯
156+
ValidationError: 1 validation error
157+
for ReasoningWriteRequest
158+
Value error, reported result fields
159+
require INTERPRETED
160+
[type=value_error,
161+
input_value={'phase':
162+
<ReasoningPhase...s.COMPLETE:
163+
'COMPLETE'>}, input_type=dict]
164+
For further information visit
165+
https://errors.pydantic.dev/2.13/v/va
166+
lue_error
167+
INFO MCP tool call tool=reasoning.write server.py:335
168+
status=success
169+
request_digest=e7f6c011776e8db7
170+
trace_digest=e7f6c011
171+
trace_source=request_id
172+
duration_ms=43.675 response_bytes=401
173+
argument_digest=sha256:6e2c8717bb9a3e
174+
618dca4ce930aee626c3513c3333f25648854
175+
436308a88e7d4
176+
[08/07/26 01:30:39] INFO Terminating session: streamable_http.py:814
177+
e0ad8727c664467ba61faf6816cb
178+
723f

‎benchmarks/studies/trajectory-value-calibration-codex-v1/runs/exact-farkas-ldl-slice-cal-r01/jacobian-mcp.stdout‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
INFO: 127.0.0.1:59876 - "POST /mcp HTTP/1.1" 200 OK
2+
INFO: 127.0.0.1:59876 - "POST /mcp HTTP/1.1" 200 OK
3+
INFO: 127.0.0.1:59876 - "POST /mcp HTTP/1.1" 200 OK
4+
INFO: 127.0.0.1:59902 - "POST /mcp HTTP/1.1" 200 OK
5+
INFO: 127.0.0.1:59904 - "POST /mcp HTTP/1.1" 202 Accepted
6+
INFO: 127.0.0.1:59904 - "GET /mcp HTTP/1.1" 200 OK
7+
INFO: 127.0.0.1:59905 - "POST /mcp HTTP/1.1" 200 OK
8+
INFO: 127.0.0.1:59938 - "POST /mcp HTTP/1.1" 200 OK
9+
INFO: 127.0.0.1:59938 - "POST /mcp HTTP/1.1" 200 OK
10+
INFO: 127.0.0.1:59959 - "POST /mcp HTTP/1.1" 200 OK
11+
INFO: 127.0.0.1:59996 - "POST /mcp HTTP/1.1" 200 OK
12+
INFO: 127.0.0.1:60003 - "POST /mcp HTTP/1.1" 200 OK
13+
INFO: 127.0.0.1:60008 - "DELETE /mcp HTTP/1.1" 200 OK
14+
INFO: 127.0.0.1:60013 - "POST /mcp HTTP/1.1" 200 OK
15+
INFO: 127.0.0.1:60013 - "POST /mcp HTTP/1.1" 200 OK

‎benchmarks/studies/trajectory-value-calibration-codex-v1/runs/exact-farkas-ldl-slice-cal-r01/reasoning-log.jsonl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"schema_version":"1","run_id":"2cab6408-e6d0-4b92-b1ec-a114edc78b7f","sequence":0,"kind":"PLAN","occurred_at":"2026-08-07T08:28:43.532125+00:00","payload":{"summary":"Compute exact rational sign checks and certify positive definiteness of the 4x4 slice using an exact method, then write schema-valid submission and digest-bound evidence."}}

0 commit comments

Comments
 (0)