Skip to content

Commit 9326737

Browse files
feat(bench): add consolidation version and retrieved facts in results
- Introduce ConsolidationVersion in bench.Config and report - Append options for consolidation version in bench.Run - Include retrieved facts for diagnostics in QAResult - Update .gitignore to exclude Taskwarrior DB - Add Unit tests for consolidation prompt versions and behavior
1 parent 0148c51 commit 9326737

6 files changed

Lines changed: 159 additions & 10 deletions

File tree

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,6 @@ __debug_bin*
1515

1616
# benchmark datasets (downloaded, not committed)
1717
/bench/data/
18+
19+
# Per-project Taskwarrior DB (tk)
20+
.task/

bench/run.go

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ type Config struct {
4545
AnswerVersion string // answer prompt version; defaults to DefaultAnswerVersion
4646
Strategy string // write strategy; defaults to StrategyAdditive
4747

48+
// ConsolidationVersion selects the consolidation prompt version, used only
49+
// when Strategy is consolidate. Empty means the library default. Lets the
50+
// harness A/B prompt versions (e.g. v1 vs the conservative v2).
51+
ConsolidationVersion string
52+
4853
// Progress, if set, is called after each sample is fully scored. Lets the
4954
// runner print live progress without the library writing to stdout.
5055
Progress func(doneSamples, totalSamples int)
@@ -79,17 +84,21 @@ func Run(ctx context.Context, samples []Sample, cfg Config) (Report, error) {
7984
judge = eval.Judge{LLM: cfg.LLM}
8085
}
8186

82-
mem, err := mneme.New(
87+
opts := []mneme.Option{
8388
mneme.WithStore(cfg.Store),
8489
mneme.WithLLM(cfg.LLM),
8590
mneme.WithEmbedder(cfg.Embedder),
8691
mneme.WithStrategy(strategy),
87-
)
92+
}
93+
if cfg.ConsolidationVersion != "" {
94+
opts = append(opts, mneme.WithConsolidationVersion(cfg.ConsolidationVersion))
95+
}
96+
mem, err := mneme.New(opts...)
8897
if err != nil {
8998
return Report{}, fmt.Errorf("build memory: %w", err)
9099
}
91100

92-
report := Report{K: cfg.K, Strategy: cfg.Strategy}
101+
report := Report{K: cfg.K, Strategy: cfg.Strategy, ConsolidationVersion: cfg.ConsolidationVersion}
93102
for i, s := range samples {
94103
results, err := runSample(ctx, mem, answerLLM, judge, cfg, s)
95104
if err != nil {
@@ -129,6 +138,7 @@ func runSample(ctx context.Context, mem mneme.Memory, answerLLM provider.LLM, ju
129138
Gold: q.Answer,
130139
Predicted: pred,
131140
Category: q.Category,
141+
Retrieved: factTexts(facts),
132142
EM: exactMatch(pred, q.Answer),
133143
F1: f1(pred, q.Answer),
134144
Judge: judge.Same(ctx, pred, q.Answer),
@@ -137,6 +147,16 @@ func runSample(ctx context.Context, mem mneme.Memory, answerLLM provider.LLM, ju
137147
return results, nil
138148
}
139149

150+
// factTexts pulls the fact statements from search hits, for the -v diagnostic
151+
// dump (so a wrong answer can be traced to what the model actually saw).
152+
func factTexts(facts []mneme.Fact) []string {
153+
out := make([]string, len(facts))
154+
for i, f := range facts {
155+
out[i] = f.Text
156+
}
157+
return out
158+
}
159+
140160
// ingestMessages prepends the session date (when present) as a single dated
141161
// note so the extractor has an anchor for relative dates in the turns. The note
142162
// uses role "user" because the pipeline ignores system messages. Structured

bench/score.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@ type QAResult struct {
1717
Predicted string
1818
Category string
1919

20+
// Retrieved is the text of the top-k facts fed to the answer model for this
21+
// question. Carried for the -v diagnostic dump so a wrong answer can be
22+
// traced to a missing/merged/distorted fact rather than just observed.
23+
Retrieved []string
24+
2025
EM bool // normalized exact match
2126
F1 float64 // SQuAD-style token-overlap F1
2227
Judge bool // eval.Judge semantic match
@@ -29,6 +34,9 @@ type Report struct {
2934
Results []QAResult
3035
K int
3136
Strategy string
37+
// ConsolidationVersion records which consolidation prompt version was used
38+
// (empty for additive runs or the library default), for run provenance.
39+
ConsolidationVersion string
3240
}
3341

3442
// CategoryStat aggregates one category (or the whole run) into mean metrics.

cmd/bench/main.go

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ func run() error {
4848
embedKind = flag.String("embedder", "openai", "embedder: openai | fake")
4949
k = flag.Int("k", 5, "search top-k facts fed to the answer model")
5050
strategy = flag.String("strategy", bench.StrategyAdditive, "write strategy: additive | consolidate")
51+
cprompt = flag.String("cprompt", "", "consolidation prompt version (e.g. v1 | v2); empty = library default. Only used with -strategy consolidate")
5152
limit = flag.Int("limit", 0, "if >0, run only the first N samples (smoke test)")
5253
maxQ = flag.Int("maxq", 0, "if >0, cap questions per sample (smoke/cost control)")
5354
out = flag.String("out", "bench/RESULTS.md", "results file to write (markdown); empty to skip")
@@ -113,11 +114,12 @@ func run() error {
113114
defer cancel()
114115

115116
report, err := bench.Run(ctx, samples, bench.Config{
116-
LLM: llm,
117-
Embedder: embedder,
118-
Store: st,
119-
K: *k,
120-
Strategy: *strategy,
117+
LLM: llm,
118+
Embedder: embedder,
119+
Store: st,
120+
K: *k,
121+
Strategy: *strategy,
122+
ConsolidationVersion: *cprompt,
121123
Progress: func(done, total int) {
122124
fmt.Fprintf(os.Stderr, "\r scored %d/%d samples", done, total)
123125
if done == total {
@@ -141,6 +143,7 @@ func run() error {
141143
limit: *limit,
142144
maxQ: *maxQ,
143145
samples: len(samples),
146+
cprompt: *cprompt,
144147
})
145148
fmt.Print(md)
146149

@@ -207,6 +210,7 @@ func loadDataset(dataset, path string) ([]bench.Sample, error) {
207210
type runMeta struct {
208211
dataset, path, model, embedder string
209212
limit, maxQ, samples int
213+
cprompt string // consolidation prompt version, if set
210214
}
211215

212216
// render builds the human + markdown report: a per-category table (the lever
@@ -216,8 +220,12 @@ type runMeta struct {
216220
func render(report bench.Report, m runMeta) string {
217221
var b strings.Builder
218222
fmt.Fprintf(&b, "# mneme bench results — %s\n\n", m.dataset)
219-
fmt.Fprintf(&b, "dataset: `%s` · model: `%s` · embedder: `%s` · k: %d · strategy: `%s`\n\n",
223+
fmt.Fprintf(&b, "dataset: `%s` · model: `%s` · embedder: `%s` · k: %d · strategy: `%s`",
220224
m.path, m.model, m.embedder, report.K, report.Strategy)
225+
if m.cprompt != "" {
226+
fmt.Fprintf(&b, " · cprompt: `%s`", m.cprompt)
227+
}
228+
b.WriteString("\n\n")
221229
fmt.Fprintf(&b, "Metrics: **EM** = normalized exact match, **F1** = token-overlap F1, **Judge** = LLM semantic match. n = question count.\n\n")
222230

223231
bounded := m.limit > 0 || m.maxQ > 0
@@ -244,6 +252,9 @@ func render(report bench.Report, m runMeta) string {
244252

245253
fmt.Fprintf(&b, "\n## Reproduce\n\n```sh\ngo run ./cmd/bench -dataset %s -path %s -k %d -strategy %s",
246254
m.dataset, m.path, report.K, report.Strategy)
255+
if m.cprompt != "" {
256+
fmt.Fprintf(&b, " -cprompt %s", m.cprompt)
257+
}
247258
if m.limit > 0 {
248259
fmt.Fprintf(&b, " -limit %d", m.limit)
249260
}
@@ -271,6 +282,9 @@ func printAnswers(report bench.Report) {
271282
mark = "✓"
272283
}
273284
fmt.Fprintf(os.Stderr, "[%s] %s\n Q: %s\n gold: %s\n pred: %s\n", mark, r.Category, r.Question, r.Gold, r.Predicted)
285+
for _, f := range r.Retrieved {
286+
fmt.Fprintf(os.Stderr, " · %s\n", f)
287+
}
274288
}
275289
fmt.Fprintln(os.Stderr)
276290
}

prompt.go

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,14 +129,81 @@ Return ONLY a JSON object, no prose and no code fences, in exactly this shape:
129129
{"memory":[{"id":"<int|new>","text":"<the resulting fact>","event":"ADD|UPDATE|DELETE|NONE"}]}
130130
If nothing should change, return {"memory":[]}.`
131131

132+
// consolidationPromptV2 is a more conservative consolidation prompt. v1 lifted
133+
// the multi-hop and temporal bench categories (it reconciles changed facts in
134+
// place) but regressed single-hop by -0.10 Judge / -0.15 F1: the model would
135+
// UPDATE or DELETE facts that were already correct, or merge two distinct facts
136+
// into one less-specific statement, losing the precise token a single-hop
137+
// answer needed (bench/RESULTS.md "next levers" #1).
138+
//
139+
// v2 keeps v1's UPDATE-on-genuine-change behavior — the driver of the multi-hop
140+
// and temporal gains — but biases hard toward preserving correct facts and
141+
// forbids merging or generalizing. It must retain the opening "maintain a
142+
// person's long-term memory" phrase: the offline bench and unit-test fakes route
143+
// the consolidation call by that substring (TestConsolidationPromptsKeepRoutingPhrase).
144+
const consolidationPromptV2 = `You maintain a person's long-term memory.
145+
146+
You are given the CURRENT MEMORIES (each with a numeric id) and a set of NEW
147+
FACTS just extracted from a conversation. Decide how the new facts change the
148+
stored memory, and output one operation per change.
149+
150+
Your default stance is to PRESERVE what is already stored. Most new facts are
151+
either brand-new information (ADD) or restatements of something already known
152+
(NONE). UPDATE and DELETE are the rare, deliberate operations — use them only
153+
when a new fact unmistakably changes or contradicts one specific existing
154+
memory. When in doubt, ADD or do nothing; never overwrite or remove a correct
155+
memory on a guess.
156+
157+
EVENTS (choose exactly one per operation):
158+
- ADD: the new fact is genuinely new information not covered by any current
159+
memory. Use id "new" and put the new fact in "text". When a new fact is merely
160+
adjacent to an existing one (same topic, different detail), ADD it as its own
161+
fact rather than folding it into the existing memory.
162+
- UPDATE: the SAME attribute of the SAME entity took a new value — a move, a new
163+
job title, a changed preference, an explicitly corrected detail. Reference the
164+
existing memory's id and write the full corrected fact in "text". An UPDATE
165+
must keep every proper noun, quantity and date the original had except the one
166+
value that genuinely changed, and must stay at least as specific as the memory
167+
it replaces. Prefer UPDATE over deleting and re-adding when the same underlying
168+
fact changed.
169+
- DELETE: an existing memory is explicitly contradicted or made obsolete by the
170+
new facts and must be removed. Reference its id.
171+
- NONE: an existing memory is unaffected, or a new fact merely restates
172+
something already known. This makes no change, and is the right choice
173+
whenever you are unsure.
174+
175+
RULES
176+
- Only emit operations that change something. A current memory the new facts do
177+
not touch should simply be left alone (you may omit it, or mark it NONE).
178+
- Never invent ids. UPDATE, DELETE and NONE must reference an id that appears in
179+
CURRENT MEMORIES exactly. ADD must use the literal id "new".
180+
- Never merge two distinct memories into one, and never make a memory vaguer or
181+
drop a specific detail it already had. Two facts that merely share a topic are
182+
different memories — keep them separate.
183+
- Do not rewrite a memory that is still accurate. If it is correct, leave it
184+
(NONE) and ADD any genuinely new detail as a separate fact.
185+
- DELETE only on real contradiction or obsolescence — never just because two
186+
facts are about a similar topic.
187+
- Each resulting fact must stay self-contained and specific: resolve pronouns to
188+
names, keep proper nouns, quantities and dates — the same standard as
189+
extraction.
190+
191+
OUTPUT
192+
Return ONLY a JSON object, no prose and no code fences, in exactly this shape:
193+
{"memory":[{"id":"<int|new>","text":"<the resulting fact>","event":"ADD|UPDATE|DELETE|NONE"}]}
194+
If nothing should change, return {"memory":[]}.`
195+
132196
// DefaultConsolidationVersion is the consolidation prompt version used unless
133-
// WithConsolidationVersion overrides it.
197+
// WithConsolidationVersion overrides it. Held at v1 until a bench re-run shows
198+
// v2 recovers single-hop without giving back the multi-hop/temporal gains; the
199+
// flip to v2 is the decision that "gates consolidation-as-default".
134200
const DefaultConsolidationVersion = "v1"
135201

136202
// consolidationPrompts maps a version name to its consolidation system prompt.
137203
// Separate from promptVersions on purpose (see consolidationPromptV1).
138204
var consolidationPrompts = map[string]string{
139205
"v1": consolidationPromptV1,
206+
"v2": consolidationPromptV2,
140207
}
141208

142209
// ConsolidationPromptVersions returns the registered consolidation prompt

prompt_test.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,3 +78,40 @@ func TestExtractionPromptCoversPrinciples(t *testing.T) {
7878
}
7979
}
8080
}
81+
82+
func TestConsolidationPromptVersioning(t *testing.T) {
83+
if consolidationSystemPrompt("v1") != consolidationPromptV1 {
84+
t.Error("v1 should map to consolidationPromptV1")
85+
}
86+
if consolidationSystemPrompt("v2") != consolidationPromptV2 {
87+
t.Error("v2 should map to consolidationPromptV2")
88+
}
89+
if consolidationSystemPrompt("does-not-exist") != consolidationPrompts[DefaultConsolidationVersion] {
90+
t.Error("unknown version should fall back to the default")
91+
}
92+
if len(ConsolidationPromptVersions()) < 2 {
93+
t.Errorf("expected at least two consolidation prompt versions, got %d", len(ConsolidationPromptVersions()))
94+
}
95+
}
96+
97+
// The consolidation call is routed by this exact phrase in the offline bench and
98+
// unit-test fakes (consolidate_test.go, bench/bench_test.go); every registered
99+
// consolidation prompt must keep it or those fakes misroute the call.
100+
func TestConsolidationPromptsKeepRoutingPhrase(t *testing.T) {
101+
const routing = "maintain a person's long-term memory"
102+
for v, p := range consolidationPrompts {
103+
if !strings.Contains(p, routing) {
104+
t.Errorf("consolidation prompt %q is missing routing phrase %q", v, routing)
105+
}
106+
}
107+
}
108+
109+
// v2 exists to stop the single-hop regression measured in bench/RESULTS.md: it
110+
// must bias toward preserving correct facts and forbid merging/generalizing.
111+
func TestConsolidationPromptV2IsConservative(t *testing.T) {
112+
for _, want := range []string{"PRESERVE", "When in doubt", "Never merge", "still accurate"} {
113+
if !strings.Contains(consolidationPromptV2, want) {
114+
t.Errorf("consolidation prompt v2 missing conservative guardrail %q", want)
115+
}
116+
}
117+
}

0 commit comments

Comments
 (0)