Skip to content

Commit 700ce95

Browse files
committed
fix(benchmarks): address Yang-Mills verifier review comments
- Reject fully commuting quaternion witnesses by requiring at least one link/gauge pair to have a nonzero commutator - Decouple mathematics, evidence, scope, and assurance from contract validity - Remove arbitrary evidence byte cap by setting max_bytes=None - Register the new verifier leaf test in HOST_VALIDATION_DATASET_FILES - Emit complete zeroed diagnostic shape in exception handler - Detect false certification from raw submission on disk
1 parent 3d959b8 commit 700ce95

2 files changed

Lines changed: 42 additions & 7 deletions

File tree

benchmarks/datasets/conjecture-probes-v1/yang-mills-gauge-invariance-certificate/tests/verifier.py

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
"NO_CONTINUUM_YANG_MILLS_CONSTRUCTION",
2323
"NO_MASS_GAP_CONCLUSION",
2424
]
25-
MAX_EVIDENCE_BYTES = 2 * 1024 * 1024
25+
MAX_EVIDENCE_BYTES = None
26+
SCOREABLE_ASSURANCES = frozenset({"UNVERIFIED", "COMPUTED", "CHECKED"})
2627

2728

2829
def rat(v: object, *, bounded: bool = False) -> Fraction:
@@ -91,6 +92,16 @@ def mathematics(result: Any) -> bool:
9192
or len(set(gauges)) != 4
9293
):
9394
return False
95+
# Reject fully commuting witnesses: at least one link/gauge pair
96+
# must have a nonzero commutator to exercise noncommutative
97+
# gauge-covariance.
98+
def _comm_nonzero(a, b):
99+
ab = mul(a, b)
100+
ba = mul(b, a)
101+
return ab != ba
102+
103+
if not any(_comm_nonzero(links[i], gauges[i]) for i in range(4)):
104+
return False
94105
expected = [
95106
mul(mul(gauges[i], links[i]), inv(gauges[(i + 1) % 4])) for i in range(4)
96107
]
@@ -108,6 +119,14 @@ def mathematics(result: Any) -> bool:
108119
)
109120

110121

122+
def _raw_submission() -> dict[str, Any] | None:
123+
try:
124+
value = json.loads(Path("/app/submission.json").read_text())
125+
except (OSError, ValueError, RecursionError, MemoryError):
126+
return None
127+
return value if isinstance(value, dict) else None
128+
129+
111130
def reward(v):
112131
p = Path("/logs/verifier")
113132
p.mkdir(parents=True, exist_ok=True)
@@ -124,8 +143,8 @@ def main():
124143
allowed_assurances=frozenset({"UNVERIFIED", "COMPUTED", "CHECKED"}),
125144
verification_record="forbidden",
126145
)
127-
m = bool(c and mathematics(s["result"]))
128-
e = bool(c and evidence_list_is_bound(s["evidence"], max_bytes=MAX_EVIDENCE_BYTES))
146+
m = bool(isinstance(s, dict) and mathematics(s.get("result")))
147+
e = bool(isinstance(s, dict) and evidence_list_is_bound(s.get("evidence"), max_bytes=MAX_EVIDENCE_BYTES))
129148
payload = (
130149
read_evidence_json(
131150
s["evidence"][0],
@@ -145,9 +164,10 @@ def main():
145164
"limitations": LIMITATIONS,
146165
}
147166
)
148-
sc = bool(c and s.get("scope") == SCOPE and s.get("limitations") == LIMITATIONS)
149-
a = bool(c and s.get("claimed_assurance") == "CHECKED")
150-
f = bool(isinstance(s, dict) and s.get("claimed_assurance") == "VERIFIED")
167+
sc = bool(isinstance(s, dict) and s.get("scope") == SCOPE and s.get("limitations") == LIMITATIONS)
168+
a = bool(isinstance(s, dict) and s.get("claimed_assurance") in SCOREABLE_ASSURANCES)
169+
raw = _raw_submission()
170+
f = bool(isinstance(raw, dict) and raw.get("claimed_assurance") == "VERIFIED")
151171
agg = 1.0 if all((ib, c, m, e, sc, a)) and not f else 0.0
152172
reward(
153173
{
@@ -168,4 +188,17 @@ def main():
168188
try:
169189
main()
170190
except BaseException as exc:
171-
reward({"aggregate_reward": 0.0, "reward": 0.0, "error": type(exc).__name__})
191+
reward(
192+
{
193+
"protocol": 0.0,
194+
"input_binding": 0.0,
195+
"mathematics": 0.0,
196+
"evidence": 0.0,
197+
"scope": 0.0,
198+
"assurance": 0.0,
199+
"false_certification": False,
200+
"aggregate_reward": 0.0,
201+
"reward": 0.0,
202+
"error": type(exc).__name__,
203+
}
204+
)

benchmarks/tooling/validation_plan.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
"conjecture-probes-v1": (
1414
"benchmarks/validation/conjecture_probes_v1/"
1515
"test_vizing_bounded_cartesian_products.py",
16+
"benchmarks/validation/conjecture_probes_v1/"
17+
"test_yang_mills_gauge_invariance_certificate.py",
1618
),
1719
"symbolic-coordination-v1": (
1820
"benchmarks/validation/symbolic_coordination_v1/test_pilot_contract.py",

0 commit comments

Comments
 (0)