Skip to content

Commit 77bd6b3

Browse files
cursoragentmorluto
andcommitted
Merge main into #622; drop superseded schema-load weakening
#615 landed load_submission_raw and reverted weakening load_submission, which had collapsed host-verifier expectations (correctness staying 1.0 on envelope-only failures). Resolve conflicts by adopting main's tree so this branch matches the supported diagnostic split. Co-authored-by: morluto <morluto@users.noreply.github.com>
2 parents 3de6c02 + 6536175 commit 77bd6b3

373 files changed

Lines changed: 5766 additions & 1655 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/scripts/manage-test-timings

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -300,10 +300,16 @@ def merge(
300300
payload = load_json_bytes(path.read_bytes(), str(path))
301301
durations = validate_durations(payload, str(path), suite)
302302
duplicates = merged.keys() & durations.keys()
303-
if duplicates:
304-
duplicate = min(duplicates)
305-
raise ValueError(f"duplicate timing entry across shards: {duplicate}")
306-
merged.update(durations)
303+
for nodeid in sorted(duplicates):
304+
previous = merged[nodeid]
305+
incoming = durations[nodeid]
306+
if incoming > previous:
307+
merged[nodeid] = incoming
308+
warning(
309+
f"duplicate timing entry across shards for {nodeid}; "
310+
f"keeping max({previous}, {incoming})"
311+
)
312+
merged.update({k: v for k, v in durations.items() if k not in duplicates})
307313

308314
envelope = {
309315
"version": 1,

benchmarks/datasets/conjecture-probes-v1/vizing-bounded-cartesian-products/tests/verifier_support.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,11 @@ def load_submission(
115115
)
116116
except (OSError, ValueError, RecursionError, MemoryError, TypeError):
117117
return None
118-
return value if isinstance(value, dict) else None
118+
return (
119+
value
120+
if isinstance(value, dict) and _public_submission_is_valid(value)
121+
else None
122+
)
119123

120124

121125
def _public_submission_is_valid(submission: object) -> bool:

benchmarks/datasets/mathematical-benchmarks-v1/algebraic-independence-transfer-audit/tests/verifier_support.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,11 @@ def load_submission(
115115
)
116116
except (OSError, ValueError, RecursionError, MemoryError, TypeError):
117117
return None
118-
return value if isinstance(value, dict) else None
118+
return (
119+
value
120+
if isinstance(value, dict) and _public_submission_is_valid(value)
121+
else None
122+
)
119123

120124

121125
def _public_submission_is_valid(submission: object) -> bool:

benchmarks/datasets/mathematical-benchmarks-v1/alternating-recurrence-stability-certificate/tests/verifier_support.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,11 @@ def load_submission(
115115
)
116116
except (OSError, ValueError, RecursionError, MemoryError, TypeError):
117117
return None
118-
return value if isinstance(value, dict) else None
118+
return (
119+
value
120+
if isinstance(value, dict) and _public_submission_is_valid(value)
121+
else None
122+
)
119123

120124

121125
def _public_submission_is_valid(submission: object) -> bool:

benchmarks/datasets/mathematical-benchmarks-v1/apollonius-gap-repair/tests/verifier_support.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,11 @@ def load_submission(
115115
)
116116
except (OSError, ValueError, RecursionError, MemoryError, TypeError):
117117
return None
118-
return value if isinstance(value, dict) else None
118+
return (
119+
value
120+
if isinstance(value, dict) and _public_submission_is_valid(value)
121+
else None
122+
)
119123

120124

121125
def _public_submission_is_valid(submission: object) -> bool:

benchmarks/datasets/mathematical-benchmarks-v1/autoformalization-semantic-audit/tests/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ FROM ghcr.io/astral-sh/uv:0.8.4-python3.12-bookworm-slim@sha256:dc7e1d08f8ca9798
22
RUN python -m pip install --no-cache-dir attrs==26.1.0 jsonschema==4.26.0 jsonschema-specifications==2025.9.1 referencing==0.37.0 rpds-py==2026.6.3 typing-extensions==4.16.0
33
# Exact semantic-alignment verifier for an autoformalized orthogonality claim.
44
LABEL jacobian.task="jacobian/autoformalization-semantic-audit" \
5-
jacobian.checksum="a60077fa4e38dcb24aea4d660782a88e85c9815828a69266e08875ad18b72355"
5+
jacobian.checksum="6de7e3bf86ef8439758ccfefce419656c95112e1d1dedc0dd727046991182687"
66
COPY expected.json input.json test.sh verifier.py verifier_support.py public_contract.json /tests/
77
COPY input.json /app/input.json
88
RUN chmod +x /tests/test.sh && \

benchmarks/datasets/mathematical-benchmarks-v1/autoformalization-semantic-audit/tests/verifier.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -218,9 +218,9 @@ def main():
218218
contract and math_correct and limitations_correct and not false_certification
219219
)
220220
reward = (
221-
0
222-
if not correct
223-
else 0.7 + 0.1 * evidence_valid + 0.1 * scope_correct + 0.1 * assurance_correct
221+
0.0
222+
if not correct or not evidence_valid
223+
else 0.8 + 0.1 * scope_correct + 0.1 * assurance_correct
224224
)
225225

226226
Path("/logs/verifier").mkdir(parents=True, exist_ok=True)

benchmarks/datasets/mathematical-benchmarks-v1/autoformalization-semantic-audit/tests/verifier_support.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,11 @@ def load_submission(
115115
)
116116
except (OSError, ValueError, RecursionError, MemoryError, TypeError):
117117
return None
118-
return value if isinstance(value, dict) else None
118+
return (
119+
value
120+
if isinstance(value, dict) and _public_submission_is_valid(value)
121+
else None
122+
)
119123

120124

121125
def _public_submission_is_valid(submission: object) -> bool:

benchmarks/datasets/mathematical-benchmarks-v1/bounded-variation-uniform-limit/tests/verifier_support.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,11 @@ def load_submission(
115115
)
116116
except (OSError, ValueError, RecursionError, MemoryError, TypeError):
117117
return None
118-
return value if isinstance(value, dict) else None
118+
return (
119+
value
120+
if isinstance(value, dict) and _public_submission_is_valid(value)
121+
else None
122+
)
119123

120124

121125
def _public_submission_is_valid(submission: object) -> bool:

benchmarks/datasets/mathematical-benchmarks-v1/c4-characteristic-invariant-audit/tests/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ FROM ghcr.io/astral-sh/uv:0.8.4-python3.12-bookworm-slim@sha256:dc7e1d08f8ca9798
22
RUN python -m pip install --no-cache-dir attrs==26.1.0 jsonschema==4.26.0 jsonschema-specifications==2025.9.1 referencing==0.37.0 rpds-py==2026.6.3 typing-extensions==4.16.0
33
# Exact graph-semantic verifier for two incompatible C4 invariants.
44
LABEL jacobian.task="jacobian/c4-characteristic-invariant-audit" \
5-
jacobian.checksum="7d3a08303c325fd059b67f660e20182ec8a60ee1949fb6fbfedfb1275ee19a0a"
5+
jacobian.checksum="5664082658eb19ea223758ec9bb0491037c53cc6e0a6ff7bf792db2798999a2f"
66
COPY expected.json input.json test.sh verifier.py verifier_support.py public_contract.json /tests/
77
COPY input.json /app/input.json
88
RUN chmod +x /tests/test.sh

0 commit comments

Comments
 (0)