Skip to content

Commit a877969

Browse files
author
Grace Lee Rui Yue
committed
benchmarks: emit canonical Oracle reward
1 parent 7fd5c87 commit a877969

2 files changed

Lines changed: 22 additions & 2 deletions

File tree

  • benchmarks/datasets/conjecture-probes-v1/zarankiewicz-projective-plane-certificate/tests

benchmarks/datasets/conjecture-probes-v1/zarankiewicz-projective-plane-certificate/tests/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
22
LABEL jacobian.task="jacobian/zarankiewicz-projective-plane-certificate" \
3-
jacobian.checksum="1dc69575adcc69cf13cd9dfe86a42ee3e1b6bfad7ffa86d64eee7f54134fa4b4"
3+
jacobian.checksum="5229efdc8b61cfa1f124c6d3f364d2148adcaf986282e50e9381fd54c88f7f39"
44
RUN python -m pip install --no-cache-dir attrs==26.1.0 jsonschema==4.26.0 jsonschema-specifications==2025.9.1 referencing==0.37.0 rpds-py==2026.6.3 typing-extensions==4.16.0
55
COPY input.json public_contract.json test.sh verifier.py verifier_support.py /tests/
66
COPY input.json /app/input.json

benchmarks/datasets/conjecture-probes-v1/zarankiewicz-projective-plane-certificate/tests/verifier.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,9 +282,29 @@ def main() -> None:
282282
"scope": float(scope_ok),
283283
"assurance": float(assurance_ok),
284284
"aggregate_reward": aggregate,
285+
"reward": aggregate,
286+
"false_certification": bool(
287+
isinstance(raw, dict) and raw.get("claimed_assurance") == "VERIFIED"
288+
),
285289
}
286290
)
287291

288292

289293
if __name__ == "__main__":
290-
main()
294+
try:
295+
main()
296+
except BaseException as exc:
297+
reward(
298+
{
299+
"protocol": 0.0,
300+
"input_binding": 0.0,
301+
"mathematics": 0.0,
302+
"evidence": 0.0,
303+
"scope": 0.0,
304+
"assurance": 0.0,
305+
"false_certification": False,
306+
"aggregate_reward": 0.0,
307+
"reward": 0.0,
308+
"error": type(exc).__name__,
309+
}
310+
)

0 commit comments

Comments
 (0)