|
| 1 | +/** |
| 2 | + * Same-session read dedup store (ADR-0003). |
| 3 | + * |
| 4 | + * Records which files an agent has already read in a session so a repeat read |
| 5 | + * of an UNCHANGED, non-trivial file (within the same compaction epoch) can be |
| 6 | + * answered with a tiny pointer instead of re-serving the packet / re-reading |
| 7 | + * the raw file. State lives at `<projectRoot>/.engram/served-reads-<session>.json` |
| 8 | + * (gitignored; relative paths only, no absolute/home paths). The set is cleared |
| 9 | + * on PreCompact (`clearServedReads`) so dedup never points at content that |
| 10 | + * context compaction evicted. |
| 11 | + * |
| 12 | + * Every function is best-effort and NEVER throws — a dedup-store failure must |
| 13 | + * never break a Read. On any error we simply don't dedup. |
| 14 | + */ |
| 15 | +import { |
| 16 | + readFileSync, |
| 17 | + writeFileSync, |
| 18 | + existsSync, |
| 19 | + statSync, |
| 20 | + mkdirSync, |
| 21 | + rmSync, |
| 22 | + readdirSync, |
| 23 | +} from "node:fs"; |
| 24 | +import { join, relative } from "node:path"; |
| 25 | + |
| 26 | +/** Below this, re-reading the raw file is cheaper than the dedup pointer + the |
| 27 | + * "scroll up" friction — so we don't dedup tiny files. ~100 tokens. */ |
| 28 | +const DEDUP_MIN_BYTES = 400; |
| 29 | + |
| 30 | +/** Served-read sets from sessions older than this are pruned (best-effort). */ |
| 31 | +const STALE_MS = 24 * 60 * 60 * 1000; |
| 32 | + |
| 33 | +/** |
| 34 | + * A recorded read is only dedup-eligible for this long. Recall-safety backstop |
| 35 | + * for the case where context content is evicted WITHOUT a PreCompact/SessionStart |
| 36 | + * reset firing (silent overflow, micro-compaction): an old recorded read is more |
| 37 | + * likely to have left the window. Expiry causes a re-serve (a false negative — |
| 38 | + * the safe direction), never a false "you already have this". |
| 39 | + */ |
| 40 | +const DEDUP_TTL_MS = 30 * 60 * 1000; |
| 41 | + |
| 42 | +/** |
| 43 | + * Cap on entries per session set. Bounds the blast radius of volume-based |
| 44 | + * eviction (a very long, read-heavy session that overflows the window before any |
| 45 | + * compaction event) to the most-recently-read files — the ones genuinely still |
| 46 | + * in context — and keeps the on-disk set small. Oldest entries are evicted. |
| 47 | + */ |
| 48 | +const MAX_ENTRIES = 256; |
| 49 | + |
| 50 | +/** Sanitise the session id into a safe filename fragment (no path traversal). */ |
| 51 | +function safeSession(sessionId: string): string | null { |
| 52 | + const s = sessionId.replace(/[^a-zA-Z0-9_-]/g, "").slice(0, 64); |
| 53 | + return s.length > 0 ? s : null; |
| 54 | +} |
| 55 | + |
| 56 | +function storePath(projectRoot: string, session: string): string { |
| 57 | + return join(projectRoot, ".engram", `served-reads-${session}.json`); |
| 58 | +} |
| 59 | + |
| 60 | +type ServedMap = Record<string, { mtimeMs: number; size: number; at: number }>; |
| 61 | + |
| 62 | +function load(path: string): ServedMap { |
| 63 | + try { |
| 64 | + const parsed = JSON.parse(readFileSync(path, "utf-8")) as unknown; |
| 65 | + return parsed && typeof parsed === "object" ? (parsed as ServedMap) : {}; |
| 66 | + } catch { |
| 67 | + return {}; |
| 68 | + } |
| 69 | +} |
| 70 | + |
| 71 | +/** Evict the oldest entries (by `at`) so a session set never exceeds the cap. */ |
| 72 | +function capEntries(map: ServedMap): void { |
| 73 | + const keys = Object.keys(map); |
| 74 | + if (keys.length <= MAX_ENTRIES) return; |
| 75 | + keys |
| 76 | + .sort((a, b) => map[a].at - map[b].at) |
| 77 | + .slice(0, keys.length - MAX_ENTRIES) |
| 78 | + .forEach((k) => delete map[k]); |
| 79 | +} |
| 80 | + |
| 81 | +/** Best-effort prune of served-read sets left behind by old sessions. */ |
| 82 | +function pruneStale(dir: string): void { |
| 83 | + try { |
| 84 | + const now = Date.now(); |
| 85 | + for (const name of readdirSync(dir)) { |
| 86 | + if (!name.startsWith("served-reads-") || !name.endsWith(".json")) continue; |
| 87 | + const p = join(dir, name); |
| 88 | + try { |
| 89 | + if (now - statSync(p).mtimeMs > STALE_MS) rmSync(p, { force: true }); |
| 90 | + } catch { |
| 91 | + /* ignore one bad file */ |
| 92 | + } |
| 93 | + } |
| 94 | + } catch { |
| 95 | + /* ignore */ |
| 96 | + } |
| 97 | +} |
| 98 | + |
| 99 | +/** |
| 100 | + * Record a full read; return `true` iff this is a repeat read of an unchanged, |
| 101 | + * non-trivial file already seen this session (the caller should dedup). On a |
| 102 | + * first read (or a changed/tiny file) it records and returns `false`. |
| 103 | + */ |
| 104 | +export function dedupOrRecord( |
| 105 | + projectRoot: string, |
| 106 | + sessionId: string, |
| 107 | + absPath: string |
| 108 | +): boolean { |
| 109 | + try { |
| 110 | + const session = safeSession(sessionId); |
| 111 | + if (!session) return false; |
| 112 | + |
| 113 | + let st: { mtimeMs: number; size: number }; |
| 114 | + try { |
| 115 | + const s = statSync(absPath); |
| 116 | + st = { mtimeMs: s.mtimeMs, size: s.size }; |
| 117 | + } catch { |
| 118 | + return false; // can't verify the file → never dedup |
| 119 | + } |
| 120 | + |
| 121 | + const now = Date.now(); |
| 122 | + const key = relative(projectRoot, absPath).split(/[\\/]/).join("/"); |
| 123 | + const path = storePath(projectRoot, session); |
| 124 | + const firstOfSession = !existsSync(path); |
| 125 | + const map = load(path); |
| 126 | + const prev = map[key]; |
| 127 | + |
| 128 | + if ( |
| 129 | + prev && |
| 130 | + prev.mtimeMs === st.mtimeMs && |
| 131 | + prev.size === st.size && |
| 132 | + st.size >= DEDUP_MIN_BYTES && |
| 133 | + now - prev.at < DEDUP_TTL_MS // recall-safety: stale records re-serve |
| 134 | + ) { |
| 135 | + return true; // unchanged, recent repeat of a non-trivial file → dedup |
| 136 | + } |
| 137 | + |
| 138 | + // First read (or changed, or tiny, or stale): record and continue. `at` is |
| 139 | + // refreshed only on a real (re-)serve, so dedup-eligibility ages from when |
| 140 | + // the content last entered context, not from the last pointer. |
| 141 | + map[key] = { mtimeMs: st.mtimeMs, size: st.size, at: now }; |
| 142 | + capEntries(map); |
| 143 | + try { |
| 144 | + const engramDir = join(projectRoot, ".engram"); |
| 145 | + if (!existsSync(engramDir)) mkdirSync(engramDir, { recursive: true }); |
| 146 | + writeFileSync(path, JSON.stringify(map)); |
| 147 | + if (firstOfSession) pruneStale(engramDir); |
| 148 | + } catch { |
| 149 | + /* recording is best-effort; failure just means no dedup next time */ |
| 150 | + } |
| 151 | + return false; |
| 152 | + } catch { |
| 153 | + return false; |
| 154 | + } |
| 155 | +} |
| 156 | + |
| 157 | +/** |
| 158 | + * Clear a session's served-read set — called on PreCompact, the eviction |
| 159 | + * boundary, so a post-compaction re-read always re-serves the content. |
| 160 | + */ |
| 161 | +export function clearServedReads(projectRoot: string, sessionId: string): void { |
| 162 | + try { |
| 163 | + const session = safeSession(sessionId); |
| 164 | + if (!session) return; |
| 165 | + rmSync(storePath(projectRoot, session), { force: true }); |
| 166 | + } catch { |
| 167 | + /* ignore */ |
| 168 | + } |
| 169 | +} |
0 commit comments