|
| 1 | +"""Structural, model-free context compression for token-budgeted assembly. |
| 2 | +
|
| 3 | +Renovates entropy/relevance-guided context selection into jCodeMunch's read-only, |
| 4 | +local idiom. Given an oversized symbol body and a per-item token cap, this ranks |
| 5 | +lines by a structural information signal (normalized Shannon entropy of the |
| 6 | +line's tokens, weighted by length), keeps the highest-signal lines, and ALWAYS |
| 7 | +keeps "keystone" lines — control-flow, returns/raises, signatures, and |
| 8 | +decision/constraint/negation cues whose removal could silently flip meaning. |
| 9 | +Everything else is elided, replaced by an honest ``… N low-signal line(s) |
| 10 | +elided …`` marker so the caller can never mistake a pruned view for the full body. |
| 11 | +
|
| 12 | +No model, no network, no state: the signal is pure structure (entropy of the |
| 13 | +token distribution), so this runs headless and deterministic. The result is a |
| 14 | +labeled VIEW, never an edit — consistent with the read-only charter. Callers opt |
| 15 | +in explicitly (``get_ranked_context(compress=True)``); default paths never invoke |
| 16 | +it, so their output is byte-identical. |
| 17 | +""" |
| 18 | + |
| 19 | +import math |
| 20 | +import re |
| 21 | +from collections import Counter |
| 22 | +from dataclasses import dataclass |
| 23 | + |
| 24 | +# Lines whose removal could silently change behavior — never elided. Covers |
| 25 | +# control flow, exits, boolean/comparison operators, and natural-language |
| 26 | +# constraint cues that carry the "what must hold" of a block. |
| 27 | +_KEYSTONE_RE = re.compile( |
| 28 | + r"\b(return|raise|yield|assert|if|elif|else|for|while|try|except|finally|" |
| 29 | + r"with|def|class|async|await|break|continue|throw|match|case|goto|guard)\b" |
| 30 | + r"|=>|->|:=|==|!=|<=|>=|&&|\|\|" |
| 31 | + r"|\b(?:must|only if|unless|never|always|require|ensure|shall)\b", |
| 32 | + re.IGNORECASE, |
| 33 | +) |
| 34 | +# Definition/signature lines — the contract of the symbol, always kept. |
| 35 | +_SIGNATURE_RE = re.compile( |
| 36 | + r"^\s*(?:@|def |class |func |function |fn |public |private |protected |" |
| 37 | + r"static |export |const |var |let |type |interface |struct |impl )" |
| 38 | +) |
| 39 | +# A decorator/annotation line is a keystone too (already covered by ``@`` above). |
| 40 | + |
| 41 | +_TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|\S") |
| 42 | + |
| 43 | + |
| 44 | +@dataclass |
| 45 | +class PruneResult: |
| 46 | + """Outcome of a keystone-protected prune. ``text`` is a labeled view.""" |
| 47 | + |
| 48 | + text: str |
| 49 | + total_lines: int |
| 50 | + kept_lines: int |
| 51 | + elided_lines: int |
| 52 | + is_pruned: bool |
| 53 | + |
| 54 | + |
| 55 | +def _token_entropy(text: str) -> float: |
| 56 | + """Normalized Shannon entropy (0..1) of the line's token distribution.""" |
| 57 | + toks = _TOKEN_RE.findall(text) |
| 58 | + n = len(toks) |
| 59 | + if n <= 1: |
| 60 | + return 0.0 |
| 61 | + counts = Counter(toks) |
| 62 | + ent = -sum((c / n) * math.log2(c / n) for c in counts.values()) |
| 63 | + return ent / math.log2(n) |
| 64 | + |
| 65 | + |
| 66 | +def line_signal(line: str) -> float: |
| 67 | + """Per-line information signal: token entropy weighted by content length. |
| 68 | +
|
| 69 | + Dense, varied lines (real logic) outrank repetitive or near-empty ones. A |
| 70 | + long line of distinct tokens scores highest; boilerplate and padding lowest. |
| 71 | + """ |
| 72 | + stripped = line.strip() |
| 73 | + if not stripped: |
| 74 | + return 0.0 |
| 75 | + return _token_entropy(line) * math.log2(len(stripped) + 2) |
| 76 | + |
| 77 | + |
| 78 | +def is_keystone(line: str) -> bool: |
| 79 | + """True if the line must be preserved regardless of its information signal.""" |
| 80 | + stripped = line.strip() |
| 81 | + if not stripped: |
| 82 | + return False |
| 83 | + if _SIGNATURE_RE.match(line): |
| 84 | + return True |
| 85 | + return bool(_KEYSTONE_RE.search(stripped)) |
| 86 | + |
| 87 | + |
| 88 | +def _emit(lines: list[str], keep: list[bool]) -> str: |
| 89 | + """Reconstruct the kept lines, collapsing dropped runs into elision markers.""" |
| 90 | + out: list[str] = [] |
| 91 | + run = 0 |
| 92 | + for i, ln in enumerate(lines): |
| 93 | + if keep[i]: |
| 94 | + if run: |
| 95 | + out.append(f" # … {run} low-signal line(s) elided …") |
| 96 | + run = 0 |
| 97 | + out.append(ln) |
| 98 | + else: |
| 99 | + run += 1 |
| 100 | + if run: |
| 101 | + out.append(f" # … {run} low-signal line(s) elided …") |
| 102 | + return "\n".join(out) |
| 103 | + |
| 104 | + |
| 105 | +def prune_source(source: str, max_tokens: int, count_tokens) -> PruneResult: |
| 106 | + """Keystone-protected prune of ``source`` toward ``max_tokens``. |
| 107 | +
|
| 108 | + Keeps every keystone line plus the highest-signal remaining lines that fit |
| 109 | + the budget. ``max_tokens`` is a SOFT per-item cap: keystone protection wins |
| 110 | + over the budget (correctness over compression), and elision markers add a |
| 111 | + few tokens, so the returned view may modestly exceed the cap — the caller's |
| 112 | + outer greedy packer still enforces the hard total. Returns the full source |
| 113 | + unchanged (``is_pruned=False``) when it already fits. |
| 114 | +
|
| 115 | + ``count_tokens`` is the same token estimator the caller packs with, so the |
| 116 | + prune and the pack agree on cost. Cost is measured once per line (O(L)). |
| 117 | + """ |
| 118 | + lines = source.splitlines() |
| 119 | + if not lines: |
| 120 | + return PruneResult(source, 0, 0, 0, False) |
| 121 | + if count_tokens(source) <= max_tokens: |
| 122 | + return PruneResult(source, len(lines), len(lines), 0, False) |
| 123 | + |
| 124 | + line_cost = [max(1, count_tokens(ln)) for ln in lines] |
| 125 | + keep = [False] * len(lines) |
| 126 | + used = 0 |
| 127 | + optional: list[tuple[float, int, int]] = [] # (-signal, index, index) for stable sort |
| 128 | + for i, ln in enumerate(lines): |
| 129 | + if is_keystone(ln): |
| 130 | + keep[i] = True |
| 131 | + used += line_cost[i] |
| 132 | + else: |
| 133 | + # Blank lines are near-free structure; rank them lowest so real |
| 134 | + # content is admitted first, but let them fill leftover budget. |
| 135 | + optional.append((-line_signal(ln), i, i)) |
| 136 | + |
| 137 | + optional.sort() # highest signal first (negated), ties by original order |
| 138 | + for _, i, _ in optional: |
| 139 | + if used + line_cost[i] > max_tokens: |
| 140 | + continue |
| 141 | + keep[i] = True |
| 142 | + used += line_cost[i] |
| 143 | + |
| 144 | + kept = sum(keep) |
| 145 | + elided = len(lines) - kept |
| 146 | + if elided == 0: |
| 147 | + return PruneResult(source, len(lines), kept, 0, False) |
| 148 | + return PruneResult(_emit(lines, keep), len(lines), kept, elided, True) |
0 commit comments