fix(benchmarks): identify provider feasibility tasks accurately

Author: morluto

benchmarks/datasets/provider-feasibility-v1/cgal/task.toml

@@ -4,8 +4,8 @@ artifacts = ["/app/input.json", "/app/submission.json", "/app/evidence"]
 [task]
 name = "jacobian/cgal"
 version = "1.0.0"
-description = "Reproduce the pinned cddlib H/V feasibility contract."
-keywords = ["provider", "cddlib", "polyhedra", "feasibility"]
+description = "Reproduce the pinned CGAL 6.2 exact-Delaunay triangulation spike."
+keywords = ["provider", "cgal", "computational-geometry", "delaunay-triangulation"]
 
 [metadata]
 author_name = "Jacobian contributors"

benchmarks/datasets/provider-feasibility-v1/gudhi/task.toml

@@ -4,8 +4,8 @@ artifacts = ["/app/input.json", "/app/submission.json", "/app/evidence"]
 [task]
 name = "jacobian/gudhi"
 version = "1.0.0"
-description = "Reproduce the pinned cddlib H/V feasibility contract."
-keywords = ["provider", "cddlib", "polyhedra", "feasibility"]
+description = "Reproduce the pinned GUDHI 3.13.0 persistent homology spike."
+keywords = ["provider", "gudhi", "persistent-homology", "topological-data-analysis"]
 
 [metadata]
 author_name = "Jacobian contributors"

benchmarks/datasets/provider-feasibility-v1/lean-repl/task.toml

@@ -4,8 +4,8 @@ artifacts = ["/app/input.json", "/app/submission.json", "/app/evidence"]
 [task]
 name = "jacobian/lean-repl"
 version = "1.0.0"
-description = "Reproduce the pinned cddlib H/V feasibility contract."
-keywords = ["provider", "cddlib", "polyhedra", "feasibility"]
+description = "Reproduce the pinned Lean 4.31.0 REPL proof-state transport spike."
+keywords = ["provider", "lean-repl", "interactive-theorem-proving", "proof-state"]
 
 [metadata]
 author_name = "Jacobian contributors"

benchmarks/datasets/provider-feasibility-v1/nauty/task.toml

@@ -4,8 +4,8 @@ artifacts = ["/app/input.json", "/app/submission.json", "/app/evidence"]
 [task]
 name = "jacobian/nauty"
 version = "1.0.0"
-description = "Reproduce the pinned cddlib H/V feasibility contract."
-keywords = ["provider", "cddlib", "polyhedra", "feasibility"]
+description = "Reproduce the pinned nauty 2.9.3 graph generation and canonical labeling spike."
+keywords = ["provider", "nauty", "graph-isomorphism", "canonical-labeling"]
 
 [metadata]
 author_name = "Jacobian contributors"

benchmarks/datasets/provider-feasibility-v1/regina/task.toml

@@ -4,8 +4,8 @@ artifacts = ["/app/input.json", "/app/submission.json", "/app/evidence"]
 [task]
 name = "jacobian/regina"
 version = "1.0.0"
-description = "Reproduce the pinned cddlib H/V feasibility contract."
-keywords = ["provider", "cddlib", "polyhedra", "feasibility"]
+description = "Reproduce the pinned Regina 7.4.1 3-manifold triangulation and normal surface spike."
+keywords = ["provider", "regina", "low-dimensional-topology", "normal-surfaces"]
 
 [metadata]
 author_name = "Jacobian contributors"

benchmarks/validation/test_verifier_security_regressions.py

@@ -63,6 +63,68 @@ def test_provider_separate_verifier_publishes_bound_input_artifact(
     assert text.index('"/app/input.json"') < text.index('"/app/submission.json"')
 
 
+def test_provider_feasibility_task_descriptions_are_provider_specific() -> None:
+    """Provider-feasibility task descriptions must identify their own provider.
+
+    A copy/paste from cddlib left five tasks advertising a cddlib H/V
+    polyhedra contract they do not exercise.  Each task's ``description`` and
+    ``keywords`` must reference its own ``required_provider`` so catalog
+    consumers can distinguish the tasks without reading their implementation.
+    """
+
+    import re
+    import tomllib
+
+    def _normalize_tokens(text: str) -> set[str]:
+        return set(re.findall(r"[a-z0-9]+", text.lower().replace("-", " ")))
+
+    for task_name in PROVIDER_TASKS:
+        task = DATASETS / "provider-feasibility-v1" / task_name
+        cfg = tomllib.loads((task / "task.toml").read_text(encoding="utf-8"))
+        provider = cfg["metadata"]["required_provider"]
+        description = cfg["task"]["description"]
+        keywords = cfg["task"]["keywords"]
+        description_tokens = _normalize_tokens(description)
+        keyword_tokens = set().union(*(_normalize_tokens(k) for k in keywords))
+        provider_tokens = _normalize_tokens(provider)
+        # Every provider's description must mention every token of its own
+        # provider name (e.g. "lean" and "repl" for "lean-repl").
+        missing_in_description = provider_tokens - description_tokens
+        assert not missing_in_description, (
+            f"{task_name}: description does not mention required_provider "
+            f"{provider!r} tokens {sorted(missing_in_description)}: "
+            f"{description!r}"
+        )
+        missing_in_keywords = provider_tokens - keyword_tokens
+        assert not missing_in_keywords, (
+            f"{task_name}: keywords do not include required_provider "
+            f"{provider!r} tokens {sorted(missing_in_keywords)}: {keywords!r}"
+        )
+        # No task may advertise another provider's full name in its
+        # description or keywords.  Check the contiguous normalized form so
+        # "cgal" is not flagged by the word "c" in an unrelated description.
+        description_norm = " ".join(
+            description.lower().replace("-", " ").replace("_", " ").split()
+        )
+        keywords_norm = " ".join(
+            " ".join(k.lower().replace("-", " ").split()) for k in keywords
+        )
+        for other in PROVIDER_TASKS:
+            other_norm = " ".join(other.lower().replace("-", " ").split())
+            if other_norm == " ".join(provider.lower().replace("-", " ").split()):
+                continue
+            if other_norm in description_norm:
+                raise AssertionError(
+                    f"{task_name}: description references unrelated provider "
+                    f"{other!r}: {description!r}"
+                )
+            if other_norm in keywords_norm:
+                raise AssertionError(
+                    f"{task_name}: keywords reference unrelated provider "
+                    f"{other!r}: {keywords!r}"
+                )
+
+
 def test_reliability_recomputes_input_and_rejects_coerced_state_count(
     tmp_path: Path,
 ) -> None: