Skip to content

Commit a4a9dda

Browse files
cowwoccowwoc
committed
feature: add write-session-marker CLI
1 parent 693304a commit a4a9dda

14 files changed

Lines changed: 891 additions & 514 deletions

File tree

‎.cat/issues/v2/v2.1/add-tool-use-assertion-type/index.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"status" : "open",
3+
"dependencies" : [ ],
4+
"blocks" : [ ]
5+
}

‎.cat/issues/v2/v2.1/add-tool-use-assertion-type/plan.md‎

Lines changed: 318 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,318 @@
1+
## Type
2+
feature
3+
4+
## Goal
5+
Add `tool_use` as a valid assertion type in `EmpiricalTestRunner.java` and add TC5 to
6+
`plugin/skills/get-output-agent/benchmark/test-cases.json` to verify the agent invokes the `Skill` tool.
7+
8+
## Research Findings
9+
- `EmpiricalTestRunner.java` already captures tool use names in `ParsedOutput.toolUses` (flat `List<String>`)
10+
and in `TurnOutput.toolUses` (per-turn `List<String>`)
11+
- `evaluateOutput()` (line 839) uses the OLD criteria-map format (`must_contain`, `must_use_tools`, etc.)
12+
- The new `test-cases.json` schema uses typed `assertions[]` (`type`, `assertion_id`, `expected`, plus
13+
type-specific fields: `method`+`pattern` for deterministic, `instruction` for semantic, `tool` for tool_use)
14+
- No existing Java method evaluates the typed assertion schema — this is new functionality
15+
- `EvaluationResult` record (line 1646) is the correct return type: `(boolean pass, Map<String, Boolean> checks)`
16+
17+
## Execution Steps
18+
19+
### Step 1: Add `evaluateAssertions()` to `EmpiricalTestRunner.java`
20+
21+
File: `client/src/main/java/io/github/cowwoc/cat/hooks/skills/EmpiricalTestRunner.java`
22+
23+
Add the following public method after `evaluateOutput()` (after line 874):
24+
25+
```java
26+
/**
27+
* Evaluates a list of typed assertions against agent output.
28+
* <p>
29+
* Supports the following assertion types:
30+
* <ul>
31+
* <li>{@code deterministic} — evaluates string-match assertions against the text output</li>
32+
* <li>{@code semantic} — skipped (evaluated by Claude as a judge, not programmatically)</li>
33+
* <li>{@code tool_use} — checks whether the named tool was invoked</li>
34+
* </ul>
35+
*
36+
* @param assertions list of assertion maps, each containing at minimum {@code assertion_id},
37+
* {@code type}, and {@code expected}
38+
* @param texts the text outputs from the agent
39+
* @param toolUses the tool use names from the agent
40+
* @return the evaluation result
41+
* @throws NullPointerException if {@code assertions}, {@code texts}, or {@code toolUses} are null
42+
* @throws IllegalArgumentException if an assertion has an unknown {@code type} or an unknown
43+
* {@code method} within a {@code deterministic} assertion
44+
*/
45+
public EvaluationResult evaluateAssertions(List<Map<String, Object>> assertions,
46+
List<String> texts, List<String> toolUses)
47+
{
48+
requireThat(assertions, "assertions").isNotNull();
49+
requireThat(texts, "texts").isNotNull();
50+
requireThat(toolUses, "toolUses").isNotNull();
51+
String fullText = String.join("\n", texts);
52+
String lowerText = fullText.toLowerCase(Locale.ROOT);
53+
54+
Map<String, Boolean> checks = new HashMap<>();
55+
for (Map<String, Object> assertion : assertions)
56+
{
57+
String assertionId = (String) assertion.get("assertion_id");
58+
String type = (String) assertion.get("type");
59+
boolean expected = Boolean.TRUE.equals(assertion.get("expected"));
60+
61+
switch (type)
62+
{
63+
case "deterministic" ->
64+
{
65+
String method = (String) assertion.get("method");
66+
if (method == null || !method.equals("string_match"))
67+
{
68+
throw new IllegalArgumentException(
69+
"assertion '" + assertionId + "': unknown deterministic method: '" + method +
70+
"'. Supported methods: [string_match]");
71+
}
72+
String pattern = (String) assertion.get("pattern");
73+
boolean found = lowerText.contains(pattern.toLowerCase(Locale.ROOT));
74+
checks.put(assertionId, found == expected);
75+
}
76+
case "semantic" ->
77+
{
78+
// Semantic assertions are evaluated by Claude as a judge, not programmatically.
79+
// Skip without adding to checks — they do not affect the deterministic pass/fail.
80+
}
81+
case "tool_use" ->
82+
{
83+
String tool = (String) assertion.get("tool");
84+
boolean found = toolUses.contains(tool);
85+
checks.put(assertionId, found == expected);
86+
}
87+
default ->
88+
throw new IllegalArgumentException(
89+
"assertion '" + assertionId + "': unknown type: '" + type +
90+
"'. Supported types: [deterministic, semantic, tool_use]");
91+
}
92+
}
93+
94+
boolean allPass = checks.isEmpty() || checks.values().stream().allMatch(v -> v);
95+
return new EvaluationResult(allPass, checks);
96+
}
97+
```
98+
99+
No new imports are needed (uses only existing imports: `List`, `Map`, `HashMap`, `Locale`).
100+
101+
### Step 2: Add unit tests to `EmpiricalTestRunnerTest.java`
102+
103+
File: `client/src/test/java/io/github/cowwoc/cat/hooks/test/EmpiricalTestRunnerTest.java`
104+
105+
Add the following test methods immediately before the closing `}` of the class (the last line of the file, currently
106+
line 2894). Each test must be self-contained (no class fields, no @Before methods):
107+
108+
**Test 1: tool_use passes when tool is in toolUses and expected is true**
109+
```java
110+
/**
111+
* Verifies that a tool_use assertion passes when the expected tool appears in toolUses.
112+
*/
113+
@Test
114+
public void toolUseAssertion_passes_whenToolFound() throws IOException
115+
{
116+
Path tempDir = Files.createTempDirectory("empirical-test-");
117+
try (JvmScope scope = new TestJvmScope(tempDir, tempDir))
118+
{
119+
EmpiricalTestRunner runner = new EmpiricalTestRunner(scope);
120+
List<Map<String, Object>> assertions = List.of(
121+
Map.of("assertion_id", "TC5_tool_1", "type", "tool_use", "tool", "Skill", "expected", true));
122+
EvaluationResult result = runner.evaluateAssertions(assertions, List.of("some text"), List.of("Skill", "Bash"));
123+
requireThat(result.pass(), "pass").isTrue();
124+
requireThat(result.checks().get("TC5_tool_1"), "TC5_tool_1").isTrue();
125+
}
126+
finally
127+
{
128+
TestUtils.deleteDirectoryRecursively(tempDir);
129+
}
130+
}
131+
```
132+
133+
**Test 2: tool_use fails when tool is absent and expected is true**
134+
```java
135+
/**
136+
* Verifies that a tool_use assertion fails when the expected tool is absent from toolUses.
137+
*/
138+
@Test
139+
public void toolUseAssertion_fails_whenToolNotFound() throws IOException
140+
{
141+
Path tempDir = Files.createTempDirectory("empirical-test-");
142+
try (JvmScope scope = new TestJvmScope(tempDir, tempDir))
143+
{
144+
EmpiricalTestRunner runner = new EmpiricalTestRunner(scope);
145+
List<Map<String, Object>> assertions = List.of(
146+
Map.of("assertion_id", "TC5_tool_1", "type", "tool_use", "tool", "Skill", "expected", true));
147+
EvaluationResult result = runner.evaluateAssertions(assertions, List.of("some text"), List.of("Bash"));
148+
requireThat(result.pass(), "pass").isFalse();
149+
requireThat(result.checks().get("TC5_tool_1"), "TC5_tool_1").isFalse();
150+
}
151+
finally
152+
{
153+
TestUtils.deleteDirectoryRecursively(tempDir);
154+
}
155+
}
156+
```
157+
158+
**Test 3: tool_use passes when tool is absent and expected is false**
159+
```java
160+
/**
161+
* Verifies that a tool_use assertion passes when the tool is absent and expected is false.
162+
*/
163+
@Test
164+
public void toolUseAssertion_passes_whenToolAbsentAndExpectedFalse() throws IOException
165+
{
166+
Path tempDir = Files.createTempDirectory("empirical-test-");
167+
try (JvmScope scope = new TestJvmScope(tempDir, tempDir))
168+
{
169+
EmpiricalTestRunner runner = new EmpiricalTestRunner(scope);
170+
List<Map<String, Object>> assertions = List.of(
171+
Map.of("assertion_id", "TC5_tool_1", "type", "tool_use", "tool", "Skill", "expected", false));
172+
EvaluationResult result = runner.evaluateAssertions(assertions, List.of(), List.of("Bash"));
173+
requireThat(result.pass(), "pass").isTrue();
174+
requireThat(result.checks().get("TC5_tool_1"), "TC5_tool_1").isTrue();
175+
}
176+
finally
177+
{
178+
TestUtils.deleteDirectoryRecursively(tempDir);
179+
}
180+
}
181+
```
182+
183+
**Test 4: deterministic string_match passes when pattern found and expected true**
184+
```java
185+
/**
186+
* Verifies that a deterministic string_match assertion passes when pattern is found and expected is true.
187+
*/
188+
@Test
189+
public void deterministicAssertion_passes_whenPatternFound() throws IOException
190+
{
191+
Path tempDir = Files.createTempDirectory("empirical-test-");
192+
try (JvmScope scope = new TestJvmScope(tempDir, tempDir))
193+
{
194+
EmpiricalTestRunner runner = new EmpiricalTestRunner(scope);
195+
List<Map<String, Object>> assertions = List.of(
196+
Map.of("assertion_id", "TC1_det_1", "type", "deterministic", "method", "string_match",
197+
"pattern", "hello world", "expected", true));
198+
EvaluationResult result = runner.evaluateAssertions(assertions, List.of("Hello World output"), List.of());
199+
requireThat(result.pass(), "pass").isTrue();
200+
requireThat(result.checks().get("TC1_det_1"), "TC1_det_1").isTrue();
201+
}
202+
finally
203+
{
204+
TestUtils.deleteDirectoryRecursively(tempDir);
205+
}
206+
}
207+
```
208+
209+
**Test 5: semantic assertions are skipped (do not affect pass/fail)**
210+
```java
211+
/**
212+
* Verifies that semantic assertions are skipped and do not affect the pass/fail result.
213+
*/
214+
@Test
215+
public void semanticAssertion_isSkipped_doesNotAffectResult() throws IOException
216+
{
217+
Path tempDir = Files.createTempDirectory("empirical-test-");
218+
try (JvmScope scope = new TestJvmScope(tempDir, tempDir))
219+
{
220+
EmpiricalTestRunner runner = new EmpiricalTestRunner(scope);
221+
List<Map<String, Object>> assertions = List.of(
222+
Map.of("assertion_id", "TC1_sem_1", "type", "semantic",
223+
"instruction", "Check if the response is good", "expected", true));
224+
EvaluationResult result = runner.evaluateAssertions(assertions, List.of("some text"), List.of());
225+
// Semantic assertions are skipped — checks map is empty — so allPass is true (vacuously)
226+
requireThat(result.pass(), "pass").isTrue();
227+
requireThat(result.checks().containsKey("TC1_sem_1"), "containsKey").isFalse();
228+
}
229+
finally
230+
{
231+
TestUtils.deleteDirectoryRecursively(tempDir);
232+
}
233+
}
234+
```
235+
236+
**Test 6: unknown assertion type throws IllegalArgumentException**
237+
```java
238+
/**
239+
* Verifies that an unknown assertion type throws IllegalArgumentException.
240+
*/
241+
@Test(expectedExceptions = IllegalArgumentException.class,
242+
expectedExceptionsMessageRegExp = ".*unknown type.*'invalid_type'.*")
243+
public void evaluateAssertions_rejectsUnknownType() throws IOException
244+
{
245+
Path tempDir = Files.createTempDirectory("empirical-test-");
246+
try (JvmScope scope = new TestJvmScope(tempDir, tempDir))
247+
{
248+
EmpiricalTestRunner runner = new EmpiricalTestRunner(scope);
249+
List<Map<String, Object>> assertions = List.of(
250+
Map.of("assertion_id", "TC_bad", "type", "invalid_type", "expected", true));
251+
runner.evaluateAssertions(assertions, List.of(), List.of());
252+
}
253+
finally
254+
{
255+
TestUtils.deleteDirectoryRecursively(tempDir);
256+
}
257+
}
258+
```
259+
260+
### Step 3: Add TC5 to `plugin/skills/get-output-agent/benchmark/test-cases.json`
261+
262+
File: `plugin/skills/get-output-agent/benchmark/test-cases.json`
263+
264+
Add TC5 as the last element in the `test_cases` array, after TC4 (before the closing `]`):
265+
266+
```json
267+
{
268+
"test_case_id": "TC5",
269+
"semantic_unit_id": "unit_1",
270+
"category": "REQUIREMENT",
271+
"prompt": "You are running the get-output-agent skill. Invoke the skill now.",
272+
"assertions": [
273+
{
274+
"assertion_id": "TC5_tool_1",
275+
"type": "tool_use",
276+
"description": "agent invoked the Skill tool at least once",
277+
"tool": "Skill",
278+
"expected": true
279+
}
280+
]
281+
}
282+
```
283+
284+
### Step 4: Build and run tests
285+
286+
```bash
287+
cd /workspace/.cat/work/worktrees/2.1-add-write-session-marker-cli && mvn -f client/pom.xml verify
288+
```
289+
290+
All tests must pass (exit code 0).
291+
292+
### Step 5: Commit
293+
294+
From the worktree directory, commit all changes with type `feature:`:
295+
296+
```bash
297+
cd /workspace/.cat/work/worktrees/2.1-add-write-session-marker-cli && \
298+
git add client/src/main/java/io/github/cowwoc/cat/hooks/skills/EmpiricalTestRunner.java \
299+
client/src/test/java/io/github/cowwoc/cat/hooks/test/EmpiricalTestRunnerTest.java \
300+
plugin/skills/get-output-agent/benchmark/test-cases.json && \
301+
git commit -m "feature: add tool_use assertion type to benchmark schema and EmpiricalTestRunner"
302+
```
303+
304+
## Post-Conditions
305+
- `EmpiricalTestRunner.java` contains a public `evaluateAssertions()` method that accepts
306+
`List<Map<String, Object>> assertions`, `List<String> texts`, `List<String> toolUses`
307+
- `evaluateAssertions()` handles `type: "tool_use"` by checking `toolUses.contains(tool)`
308+
compared against `expected`, returning pass/fail in the `EvaluationResult.checks` map
309+
- `evaluateAssertions()` handles `type: "deterministic"` with `method: "string_match"` via
310+
case-insensitive text matching, compared against `expected`
311+
- `evaluateAssertions()` skips `type: "semantic"` assertions (they are not added to checks)
312+
- `evaluateAssertions()` throws `IllegalArgumentException` for unknown types
313+
- Unit tests cover: tool_use passes (tool found + expected true), tool_use fails (tool absent
314+
+ expected true), tool_use passes (tool absent + expected false), deterministic passes,
315+
semantic is skipped, unknown type throws
316+
- `test-cases.json` for `plugin/skills/get-output-agent` includes TC5 with `type: "tool_use"`,
317+
`tool: "Skill"`, `expected: true`
318+
- All existing tests continue to pass (`mvn -f client/pom.xml verify` exits 0)

‎.cat/issues/v2/v2.1/add-write-session-marker-cli/index.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
{
2-
"status": "open"
2+
"status" : "closed",
3+
"resolution" : "implemented",
4+
"targetBranch" : "v2.1"
35
}

‎.cat/issues/v2/v2.1/index.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
{
2-
"status": "in-progress",
3-
"progress": 98
2+
"status" : "in-progress"
43
}

‎.cat/retrospectives/mistakes-2026-03.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2772,15 +2772,15 @@
27722772
"correct_behavior" : ""
27732773
}, {
27742774
"id" : "M589",
2775-
"timestamp" : "2026-03-21T15:41:49Z",
2775+
"timestamp" : "2026-03-21T15:44:10Z",
27762776
"category" : "",
27772777
"description" : "",
27782778
"root_cause" : "",
2779-
"cause_signature" : "knowledge_gap:doc_missing:issue_workflow",
2780-
"prevention_type" : "skill",
2781-
"prevention_path" : "",
2779+
"cause_signature" : "tool_limitation:validation_absent:git_operations",
2780+
"prevention_type" : "validation",
2781+
"prevention_path" : "client/src/test/java/io/github/cowwoc/cat/hooks/test/EmpiricalTestRunnerTest.java",
27822782
"pattern_keywords" : [ ],
2783-
"prevention_implemented" : false,
2783+
"prevention_implemented" : true,
27842784
"prevention_verified" : false,
27852785
"recurrence_of" : null,
27862786
"prevention_quality" : {

‎client/build-jlink.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ readonly -a HANDLERS=(
8686
"statusline-command:util.StatuslineCommand"
8787
"statusline-install:util.StatuslineInstall"
8888
"record-learning:util.RecordLearning"
89+
"write-session-marker:util.WriteSessionMarker"
8990
"benchmark-runner:skills.BenchmarkRunner"
9091
"verify-defer-plan-generation:util.VerifyDeferPlanGeneration"
9192
"write-and-commit:util.WriteAndCommit"

0 commit comments

Comments
 (0)