Skip to content

Commit 29a5385

Browse files
committed
fix(benchmarks): address hadamard-order12 verifier review comments
- Change evidence media type from text/plain to application/json with payload_shape in instruction.md and public_contract.json - Decouple mathematics, evidence, scope, and assurance from contract; use require_input_binding=False and independent computation - Emit canonical reward dimensions (correctness, evidence_validity, scope_accuracy, assurance_calibration) in both main path and exception handler - Bound evidence memory to 2 MiB via max_bytes in evidence_list_is_bound and read_evidence_json - Accept all scoreable assurances (UNVERIFIED, COMPUTED, CHECKED) - Drop whitespace-churn commits during rebase onto origin/main
1 parent da7fda0 commit 29a5385

3 files changed

Lines changed: 59 additions & 9 deletions

File tree

benchmarks/datasets/conjecture-probes-v1/hadamard-order12-construction/instruction.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ Write `/app/submission.json` to the exact schema in `environment/submission_sche
2525
- **Scope:** the exact value declared in `submission_schema.json`
2626
- **Completeness:** `COMPLETE`.
2727
- **Evidence:** 1-1 item(s); allowed path(s): `evidence/answer.txt`; digest must match `^sha256:[0-9a-f]{64}$`.
28-
- **Evidence media types:** `text/plain`.
28+
- **Evidence media types:** `application/json`.
2929
- **Required artifact filenames:** `evidence/answer.txt`.
30+
- **Evidence payload:** JSON object with required keys `schema_version` (string "1"), `task_id`, `result`, and `limitations`.
3031
<!-- END PUBLIC CONTRACT SUBMISSION BLOCK -->

benchmarks/datasets/conjecture-probes-v1/hadamard-order12-construction/tests/public_contract.json

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,35 @@
1818
"digest_pattern": "^sha256:[0-9a-f]{64}$",
1919
"max_items": 1,
2020
"media_types": [
21-
"text/plain"
21+
"application/json"
2222
],
2323
"min_items": 1,
24-
"payload_shape": null
24+
"payload_shape": {
25+
"properties": {
26+
"limitations": {
27+
"items": {
28+
"type": "string"
29+
},
30+
"type": "array"
31+
},
32+
"result": {
33+
"type": "object"
34+
},
35+
"schema_version": {
36+
"const": "1"
37+
},
38+
"task_id": {
39+
"const": "jacobian/hadamard-order12-construction"
40+
}
41+
},
42+
"required": [
43+
"schema_version",
44+
"task_id",
45+
"result",
46+
"limitations"
47+
],
48+
"type": "object"
49+
}
2550
},
2651
"limitations": {
2752
"const": [

benchmarks/datasets/conjecture-probes-v1/hadamard-order12-construction/tests/verifier.py

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
TASK_ID = "jacobian/hadamard-order12-construction"
1818
SCOPE = "hadamard-order12-construction:normalized-v1"
1919
LIMITATIONS = ["ORDER_12_ONLY", "NO_GENERAL_HADAMARD_CONJECTURE_CONCLUSION"]
20+
MAX_EVIDENCE_BYTES = 2 * 1024 * 1024
2021

2122

2223
def _determinant(matrix: list[list[int]]) -> int:
@@ -108,19 +109,30 @@ def _reward(value: dict[str, Any]) -> None:
108109
def main() -> None:
109110
input_bound = workspace_input_is_bound()
110111
frozen = _frozen() if input_bound else None
111-
submission = load_submission()
112+
submission = load_submission(require_input_binding=False)
112113
contract = strict_submission_contract(
113114
submission,
114115
task_id=TASK_ID,
115116
conclusion="HADAMARD_ORDER12_CONSTRUCTION",
116117
allowed_assurances=frozenset({"UNVERIFIED", "COMPUTED", "CHECKED"}),
117118
verification_record="forbidden",
118119
)
119-
mathematics = bool(contract and frozen and _mathematics(submission["result"]))
120-
evidence = bool(contract and evidence_list_is_bound(submission["evidence"]))
120+
mathematics = bool(
121+
frozen
122+
and isinstance(submission, dict)
123+
and _mathematics(submission.get("result"))
124+
)
125+
evidence = bool(
126+
isinstance(submission, dict)
127+
and evidence_list_is_bound(
128+
submission.get("evidence"), max_bytes=MAX_EVIDENCE_BYTES
129+
)
130+
)
121131
payload = (
122132
read_evidence_json(
123-
submission["evidence"][0], expected_path="evidence/answer.txt"
133+
submission["evidence"][0],
134+
expected_path="evidence/answer.txt",
135+
max_bytes=MAX_EVIDENCE_BYTES,
124136
)
125137
if evidence
126138
else None
@@ -134,11 +146,15 @@ def main() -> None:
134146
and payload.get("limitations") == LIMITATIONS
135147
)
136148
scope = bool(
137-
contract
149+
isinstance(submission, dict)
138150
and submission.get("scope") == SCOPE
139151
and submission.get("limitations") == LIMITATIONS
140152
)
141-
assurance = bool(contract and submission.get("claimed_assurance") == "CHECKED")
153+
SCOREABLE_ASSURANCES = frozenset({"UNVERIFIED", "COMPUTED", "CHECKED"})
154+
assurance = bool(
155+
isinstance(submission, dict)
156+
and submission.get("claimed_assurance") in SCOREABLE_ASSURANCES
157+
)
142158
false_certification = bool(
143159
isinstance(submission, dict)
144160
and submission.get("claimed_assurance") == "VERIFIED"
@@ -160,6 +176,10 @@ def main() -> None:
160176
"false_certification": false_certification,
161177
"aggregate_reward": aggregate,
162178
"reward": aggregate,
179+
"correctness": 1.0 if mathematics else 0.0,
180+
"evidence_validity": 1.0 if evidence else 0.0,
181+
"scope_accuracy": 1.0 if scope else 0.0,
182+
"assurance_calibration": 1.0 if assurance else 0.0,
163183
}
164184
)
165185

@@ -179,6 +199,10 @@ def main() -> None:
179199
"false_certification": False,
180200
"aggregate_reward": 0.0,
181201
"reward": 0.0,
202+
"correctness": 0.0,
203+
"evidence_validity": 0.0,
204+
"scope_accuracy": 0.0,
205+
"assurance_calibration": 0.0,
182206
"error": type(exc).__name__,
183207
}
184208
)

0 commit comments

Comments
 (0)