Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions benchmarks/schemas/observation-evidence.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@
"dataset": {"type": "string", "minLength": 1},
"condition": {"type": "string", "minLength": 1},
"snapshot_id": {
"type": "string",
"type": ["string", "null"],
"pattern": "^sha256:[0-9a-f]{64}$"
},
"harbor_version": {"type": "string", "minLength": 1},
"harbor_version": {"type": ["string", "null"], "minLength": 1},
"eval_args": {"$ref": "#/$defs/eval_args"},
"job": {"$ref": "#/$defs/job"},
"runtime_snapshot": {"$ref": "#/$defs/runtime_snapshot"},
Expand All @@ -55,6 +55,11 @@
},
"then": {
"properties": {
"snapshot_id": {
"type": "string",
"pattern": "^sha256:[0-9a-f]{64}$"
},
"harbor_version": {"type": "string", "minLength": 1},
"trials": {
"type": "array",
"items": {
Expand Down
18 changes: 17 additions & 1 deletion benchmarks/validation/test_observation_artifact_binding.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ def test_observation_rejects_missing_snapshot_id(
],
}
job_path = _write_observation_job(
tmp_path, job, snapshot_id=None, harbor_version=_HARBOR_VERSION
tmp_path, job, snapshot_id=None, harbor_version=None
)
result_path = _write_result(tmp_path)

Expand All @@ -245,6 +245,22 @@ def test_observation_rejects_missing_snapshot_id(

assert evidence["status"] == "INCOMPLETE"
assert any("snapshot_id" in f and "missing" in f for f in failures)
assert any("harbor_version" in f and "missing" in f for f in failures)

schema = json.loads(
(
Path(__file__).resolve().parents[1]
/ "schemas"
/ "observation-evidence.schema.json"
).read_text(encoding="utf-8")
)
from jsonschema import Draft202012Validator

assert list(Draft202012Validator(schema).iter_errors(evidence)) == []

evidence["status"] = "VALID"
errors = list(Draft202012Validator(schema).iter_errors(evidence))
assert errors, "VALID evidence must retain reproducibility bindings"


def test_observation_rejects_mismatched_harbor_version(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,7 @@ def test_ring_mismatches_fail_closed(polynomial_services) -> None:
assert result.artifact_uris == (), mutation


@pytest.mark.timeout(60)
def test_corrupted_found_candidate_does_not_verify(
authorized_polynomial_services,
) -> None:
Expand Down