Skip to content

Commit a3cbb91

Browse files
morlutomorluto
authored
fix(verifier): retain false-certification diagnostics (#773)
1 parent 3097458 commit a3cbb91

13 files changed

Lines changed: 289 additions & 36 deletions

File tree

‎benchmarks/datasets/provider-feasibility-v1/cddlib/tests/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
22
RUN python -m pip install --no-cache-dir attrs==26.1.0 jsonschema==4.26.0 jsonschema-specifications==2025.9.1 referencing==0.37.0 rpds-py==2026.6.3 typing-extensions==4.16.0
3-
LABEL jacobian.checksum="1585fef67c7beb04d830498e9c9db28f62eb4564446806875a426bcd0d30154e"
3+
LABEL jacobian.checksum="106e2699dc32961b5e82101ef8e0a18db002ad6734c5db4a48510b75a41cc908"
44
COPY expected.json input.json test.sh verifier.py verifier_support.py public_contract.json /tests/
55
RUN chmod +x /tests/test.sh

‎benchmarks/datasets/provider-feasibility-v1/cddlib/tests/verifier.py‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,15 @@
22

33
import hashlib
44
import json
5+
import math
56
from pathlib import Path
67

7-
from verifier_support import load_submission, read_evidence_json
8+
from verifier_support import (
9+
MAX_SUBMISSION_BYTES,
10+
is_regular_bounded_file,
11+
load_submission,
12+
read_evidence_json,
13+
)
814

915
expected = json.loads(Path("/tests/expected.json").read_text())
1016
submission = load_submission()
@@ -116,6 +122,36 @@ def _execution_bound(report: object) -> bool:
116122
}
117123
and _execution_bound(report)
118124
)
125+
126+
127+
def _reject_nonfinite_json(value: str) -> None:
128+
raise ValueError(f"non-finite JSON number: {value}")
129+
130+
131+
def _finite_json_float(value: str) -> float:
132+
parsed = float(value)
133+
if not math.isfinite(parsed):
134+
raise ValueError(f"out-of-range JSON number: {value}")
135+
return parsed
136+
137+
138+
def _raw_claimed_verified() -> bool:
139+
"""Detect a VERIFIED claim in the raw submission without accepting it."""
140+
141+
try:
142+
path = Path("/app/submission.json")
143+
if not is_regular_bounded_file(path, max_bytes=MAX_SUBMISSION_BYTES):
144+
return False
145+
raw = json.loads(
146+
path.read_text(),
147+
parse_constant=_reject_nonfinite_json,
148+
parse_float=_finite_json_float,
149+
)
150+
except (OSError, ValueError, RecursionError, MemoryError):
151+
return False
152+
return isinstance(raw, dict) and raw.get("claimed_assurance") == "VERIFIED"
153+
154+
119155
target = Path("/logs/verifier/reward.json")
120156
target.parent.mkdir(parents=True, exist_ok=True)
121157
target.write_text(
@@ -124,10 +160,7 @@ def _execution_bound(report: object) -> bool:
124160
"reward": 1.0 if valid else 0.0,
125161
"pin_fidelity": float(valid),
126162
"provider_outcome": float(valid),
127-
"false_certification": bool(
128-
isinstance(submission, dict)
129-
and submission.get("claimed_assurance") == "VERIFIED"
130-
),
163+
"false_certification": _raw_claimed_verified(),
131164
},
132165
sort_keys=True,
133166
)

‎benchmarks/datasets/provider-feasibility-v1/cgal/tests/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
22
RUN python -m pip install --no-cache-dir attrs==26.1.0 jsonschema==4.26.0 jsonschema-specifications==2025.9.1 referencing==0.37.0 rpds-py==2026.6.3 typing-extensions==4.16.0
3-
LABEL jacobian.checksum="5ad96c74b9ba96d5891d4e1d5de6bc95b50fb7d7d7b7cfb829bcd1d54c009244"
3+
LABEL jacobian.checksum="97ca3943e3286259cb064b56672cafe549e5880dbd9d4b5cb618919f595bed9f"
44
COPY expected.json input.json test.sh verifier.py verifier_support.py public_contract.json /tests/
55
RUN chmod +x /tests/test.sh

‎benchmarks/datasets/provider-feasibility-v1/cgal/tests/verifier.py‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,15 @@
22

33
import hashlib
44
import json
5+
import math
56
from pathlib import Path
67

7-
from verifier_support import load_submission, read_evidence_json
8+
from verifier_support import (
9+
MAX_SUBMISSION_BYTES,
10+
is_regular_bounded_file,
11+
load_submission,
12+
read_evidence_json,
13+
)
814

915
expected = json.loads(Path("/tests/expected.json").read_text())
1016
submission = load_submission()
@@ -116,6 +122,36 @@ def _execution_bound(report: object) -> bool:
116122
}
117123
and _execution_bound(report)
118124
)
125+
126+
127+
def _reject_nonfinite_json(value: str) -> None:
128+
raise ValueError(f"non-finite JSON number: {value}")
129+
130+
131+
def _finite_json_float(value: str) -> float:
132+
parsed = float(value)
133+
if not math.isfinite(parsed):
134+
raise ValueError(f"out-of-range JSON number: {value}")
135+
return parsed
136+
137+
138+
def _raw_claimed_verified() -> bool:
139+
"""Detect a VERIFIED claim in the raw submission without accepting it."""
140+
141+
try:
142+
path = Path("/app/submission.json")
143+
if not is_regular_bounded_file(path, max_bytes=MAX_SUBMISSION_BYTES):
144+
return False
145+
raw = json.loads(
146+
path.read_text(),
147+
parse_constant=_reject_nonfinite_json,
148+
parse_float=_finite_json_float,
149+
)
150+
except (OSError, ValueError, RecursionError, MemoryError):
151+
return False
152+
return isinstance(raw, dict) and raw.get("claimed_assurance") == "VERIFIED"
153+
154+
119155
target = Path("/logs/verifier/reward.json")
120156
target.parent.mkdir(parents=True, exist_ok=True)
121157
target.write_text(
@@ -124,10 +160,7 @@ def _execution_bound(report: object) -> bool:
124160
"reward": 1.0 if valid else 0.0,
125161
"pin_fidelity": float(valid),
126162
"provider_outcome": float(valid),
127-
"false_certification": bool(
128-
isinstance(submission, dict)
129-
and submission.get("claimed_assurance") == "VERIFIED"
130-
),
163+
"false_certification": _raw_claimed_verified(),
131164
},
132165
sort_keys=True,
133166
)

‎benchmarks/datasets/provider-feasibility-v1/gudhi/tests/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
22
RUN python -m pip install --no-cache-dir attrs==26.1.0 jsonschema==4.26.0 jsonschema-specifications==2025.9.1 referencing==0.37.0 rpds-py==2026.6.3 typing-extensions==4.16.0
3-
LABEL jacobian.checksum="fdcee93619002c2cf6afbb8a404473174db3c5b4612250abf7562b0448e5ba8f"
3+
LABEL jacobian.checksum="98e516278161bd26f2f0fd1ed5d4acea371dae10cd44ce66d165a3cf5b1afd8a"
44
COPY expected.json input.json test.sh verifier.py verifier_support.py public_contract.json /tests/
55
RUN chmod +x /tests/test.sh

‎benchmarks/datasets/provider-feasibility-v1/gudhi/tests/verifier.py‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,15 @@
22

33
import hashlib
44
import json
5+
import math
56
from pathlib import Path
67

7-
from verifier_support import load_submission, read_evidence_json
8+
from verifier_support import (
9+
MAX_SUBMISSION_BYTES,
10+
is_regular_bounded_file,
11+
load_submission,
12+
read_evidence_json,
13+
)
814

915
expected = json.loads(Path("/tests/expected.json").read_text())
1016
submission = load_submission()
@@ -110,6 +116,36 @@ def _execution_bound(report: object) -> bool:
110116
}
111117
and _execution_bound(report)
112118
)
119+
120+
121+
def _reject_nonfinite_json(value: str) -> None:
122+
raise ValueError(f"non-finite JSON number: {value}")
123+
124+
125+
def _finite_json_float(value: str) -> float:
126+
parsed = float(value)
127+
if not math.isfinite(parsed):
128+
raise ValueError(f"out-of-range JSON number: {value}")
129+
return parsed
130+
131+
132+
def _raw_claimed_verified() -> bool:
133+
"""Detect a VERIFIED claim in the raw submission without accepting it."""
134+
135+
try:
136+
path = Path("/app/submission.json")
137+
if not is_regular_bounded_file(path, max_bytes=MAX_SUBMISSION_BYTES):
138+
return False
139+
raw = json.loads(
140+
path.read_text(),
141+
parse_constant=_reject_nonfinite_json,
142+
parse_float=_finite_json_float,
143+
)
144+
except (OSError, ValueError, RecursionError, MemoryError):
145+
return False
146+
return isinstance(raw, dict) and raw.get("claimed_assurance") == "VERIFIED"
147+
148+
113149
target = Path("/logs/verifier/reward.json")
114150
target.parent.mkdir(parents=True, exist_ok=True)
115151
target.write_text(
@@ -118,10 +154,7 @@ def _execution_bound(report: object) -> bool:
118154
"reward": 1.0 if valid else 0.0,
119155
"pin_fidelity": float(valid),
120156
"provider_outcome": float(valid),
121-
"false_certification": bool(
122-
isinstance(submission, dict)
123-
and submission.get("claimed_assurance") == "VERIFIED"
124-
),
157+
"false_certification": _raw_claimed_verified(),
125158
},
126159
sort_keys=True,
127160
)

‎benchmarks/datasets/provider-feasibility-v1/lean-repl/tests/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
22
RUN python -m pip install --no-cache-dir attrs==26.1.0 jsonschema==4.26.0 jsonschema-specifications==2025.9.1 referencing==0.37.0 rpds-py==2026.6.3 typing-extensions==4.16.0
3-
LABEL jacobian.checksum="3d50f293af713b8e09b3fda0c9331ea5487dd73096ab20372d7ff9c8d58c50fe"
3+
LABEL jacobian.checksum="4a43327abe126dd2c968b3d8eb800350ee9c9bbe826833026e2a090248d170f7"
44
COPY expected.json input.json test.sh verifier.py verifier_support.py public_contract.json /tests/
55
RUN chmod +x /tests/test.sh

‎benchmarks/datasets/provider-feasibility-v1/lean-repl/tests/verifier.py‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
from __future__ import annotations
22

33
import json
4+
import math
45
from pathlib import Path
56

6-
from verifier_support import load_submission, read_evidence_json
7+
from verifier_support import (
8+
MAX_SUBMISSION_BYTES,
9+
is_regular_bounded_file,
10+
load_submission,
11+
read_evidence_json,
12+
)
713

814
expected = json.loads(Path("/tests/expected.json").read_text())
915
submission = load_submission()
@@ -111,6 +117,36 @@ def _task_trace_matches(task, expected):
111117
"limitations",
112118
}
113119
)
120+
121+
122+
def _reject_nonfinite_json(value: str) -> None:
123+
raise ValueError(f"non-finite JSON number: {value}")
124+
125+
126+
def _finite_json_float(value: str) -> float:
127+
parsed = float(value)
128+
if not math.isfinite(parsed):
129+
raise ValueError(f"out-of-range JSON number: {value}")
130+
return parsed
131+
132+
133+
def _raw_claimed_verified() -> bool:
134+
"""Detect a VERIFIED claim in the raw submission without accepting it."""
135+
136+
try:
137+
path = Path("/app/submission.json")
138+
if not is_regular_bounded_file(path, max_bytes=MAX_SUBMISSION_BYTES):
139+
return False
140+
raw = json.loads(
141+
path.read_text(),
142+
parse_constant=_reject_nonfinite_json,
143+
parse_float=_finite_json_float,
144+
)
145+
except (OSError, ValueError, RecursionError, MemoryError):
146+
return False
147+
return isinstance(raw, dict) and raw.get("claimed_assurance") == "VERIFIED"
148+
149+
114150
target = Path("/logs/verifier/reward.json")
115151
target.parent.mkdir(parents=True, exist_ok=True)
116152
target.write_text(
@@ -119,10 +155,7 @@ def _task_trace_matches(task, expected):
119155
"reward": 1.0 if valid else 0.0,
120156
"pin_fidelity": float(valid),
121157
"provider_outcome": float(valid),
122-
"false_certification": bool(
123-
isinstance(submission, dict)
124-
and submission.get("claimed_assurance") == "VERIFIED"
125-
),
158+
"false_certification": _raw_claimed_verified(),
126159
},
127160
sort_keys=True,
128161
)

‎benchmarks/datasets/provider-feasibility-v1/nauty/tests/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
FROM python:3.12-slim@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de
22
RUN python -m pip install --no-cache-dir attrs==26.1.0 jsonschema==4.26.0 jsonschema-specifications==2025.9.1 referencing==0.37.0 rpds-py==2026.6.3 typing-extensions==4.16.0
3-
LABEL jacobian.checksum="a30b75b039561a6ed0c6d6bb5de70583a5f05b88b4d7fbcb3836d5197ced44ce"
3+
LABEL jacobian.checksum="92d2a5e79adc9ce5d951efbbf73e2a3fce41e9337238da057827540556c9f093"
44
COPY expected.json input.json test.sh verifier.py verifier_support.py public_contract.json /tests/
55
RUN chmod +x /tests/test.sh

‎benchmarks/datasets/provider-feasibility-v1/nauty/tests/verifier.py‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
from __future__ import annotations
22

33
import json
4+
import math
45
from pathlib import Path
56

6-
from verifier_support import load_submission, read_evidence_json
7+
from verifier_support import (
8+
MAX_SUBMISSION_BYTES,
9+
is_regular_bounded_file,
10+
load_submission,
11+
read_evidence_json,
12+
)
713

814
expected = json.loads(Path("/tests/expected.json").read_text())
915
submission = load_submission()
@@ -118,6 +124,36 @@ def _execution_bound(report: object) -> bool:
118124
}
119125
and _execution_bound(report)
120126
)
127+
128+
129+
def _reject_nonfinite_json(value: str) -> None:
130+
raise ValueError(f"non-finite JSON number: {value}")
131+
132+
133+
def _finite_json_float(value: str) -> float:
134+
parsed = float(value)
135+
if not math.isfinite(parsed):
136+
raise ValueError(f"out-of-range JSON number: {value}")
137+
return parsed
138+
139+
140+
def _raw_claimed_verified() -> bool:
141+
"""Detect a VERIFIED claim in the raw submission without accepting it."""
142+
143+
try:
144+
path = Path("/app/submission.json")
145+
if not is_regular_bounded_file(path, max_bytes=MAX_SUBMISSION_BYTES):
146+
return False
147+
raw = json.loads(
148+
path.read_text(),
149+
parse_constant=_reject_nonfinite_json,
150+
parse_float=_finite_json_float,
151+
)
152+
except (OSError, ValueError, RecursionError, MemoryError):
153+
return False
154+
return isinstance(raw, dict) and raw.get("claimed_assurance") == "VERIFIED"
155+
156+
121157
target = Path("/logs/verifier/reward.json")
122158
target.parent.mkdir(parents=True, exist_ok=True)
123159
target.write_text(
@@ -126,10 +162,7 @@ def _execution_bound(report: object) -> bool:
126162
"reward": 1.0 if valid else 0.0,
127163
"pin_fidelity": float(valid),
128164
"provider_outcome": float(valid),
129-
"false_certification": bool(
130-
isinstance(submission, dict)
131-
and submission.get("claimed_assurance") == "VERIFIED"
132-
),
165+
"false_certification": _raw_claimed_verified(),
133166
},
134167
sort_keys=True,
135168
)

0 commit comments

Comments
 (0)