bugfix: remove _comment field from benchmark.json and enforce schema exclusivity

Remove the undocumented _comment field from SkillTestRunner output and add test assertion to enforce field exclusivity, preventing similar schema violations in the future.

Author: cowwoc

.cat/issues/v2/v2.1/remove-comment-from-benchmark-json-enforce-schema-exclusivity/index.json

@@ -1 +1,5 @@
-{"status": "open"}
+{
+  "status" : "closed",
+  "resolution" : "implemented",
+  "target_branch" : "v2.1"
+}

client/src/main/java/io/github/cowwoc/cat/hooks/skills/SkillTestRunner.java

@@ -528,7 +528,6 @@ public void persistArtifacts(String[] args, PrintStream out) throws IOException
     Path benchmarkJsonPath = benchmarkDir.resolve("benchmark.json");
     JsonMapper mapper = scope.getJsonMapper();
     ObjectNode root = mapper.createObjectNode();
-    root.put("_comment", "Auto-generated snapshot of the last benchmark run. Not a config file.");
     root.put("session_id", sessionId);
     root.put("phase", phase);
     root.put("timestamp", timestamp);

client/src/test/java/io/github/cowwoc/cat/hooks/test/SkillTestRunnerTest.java

@@ -17,6 +17,9 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 
 import static io.github.cowwoc.requirements13.java.DefaultJavaValidators.requireThat;
 
@@ -951,6 +954,12 @@ public void persistArtifactsWritesBenchmarkJson() throws IOException
       requireThat(root.path("skill").path("path").asString(), "skill.path").isEqualTo("skill.md");
       requireThat(root.path("skill").path("sha256").asString(""), "skill.sha256").isNotBlank();
       requireThat(root.path("test_cases").path("sha256").asString(""), "test_cases.sha256").isNotBlank();
+
+      // Assert exclusivity: no undocumented fields
+      List<String> fieldNames = new ArrayList<>(root.propertyNames());
+      Collections.sort(fieldNames);
+      List<String> expectedFieldNames = List.of("phase", "session_id", "skill", "test_cases", "timestamp");
+      requireThat(fieldNames, "fieldNames").isEqualTo(expectedFieldNames);
     }
     finally
     {

plugin/concepts/skill-benchmarking.md

@@ -14,7 +14,8 @@ Skill benchmarking produces quantitative evidence about whether a skill improves
 benchmark run executes the same test cases in two configurations — `with-skill` (skill active) and
 `without-skill` (skill inactive) — grades the outputs against defined assertions, and aggregates
 pass rates, timing, and token counts. The result is a benchmark JSON that supports A/B comparison
-and pattern analysis.
+and pattern analysis. The `benchmark.json` snapshot stored under each skill's `benchmark/` directory
+is auto-generated by `SkillTestRunner`; do not edit it manually.
 
 The full loop: write test cases -> spawn parallel runs -> grade outputs -> aggregate -> analyze ->
 review with user -> improve skill -> repeat.

plugin/skills/get-output-agent/benchmark/benchmark.json

@@ -1,5 +1,4 @@
 {
-  "_comment" : "Auto-generated snapshot of the last benchmark run. Not a config file.",
   "session_id" : "9ae46230-f04d-4c75-9a98-ae196923503e",
   "phase" : "sprt",
   "timestamp" : "2026-03-21T16:38:00Z",

plugin/skills/status-agent/benchmark/benchmark.json

@@ -1,5 +1,4 @@
 {
-  "_comment": "Auto-generated snapshot of the last benchmark run. Not a config file.",
   "session_id": "754d0e28-baa5-48ce-92c8-d96f578ffb93",
   "phase": "post-hardening",
   "timestamp": "2026-03-16T21:16:53Z",