release: v1.108.161 — BM25 tokenizer: Unicode word splitting + CJK character bigrams

_TOKEN_RE was [a-zA-Z0-9]{2,}, treating every non-ASCII character as a
separator: CJK symbol names/summaries/docstrings produced zero BM25
tokens and accented Latin was mangled. Now splits on Unicode word
boundaries and expands CJK runs into overlapping character bigrams,
identical at index and query time. camelCase/snake_case splitting,
stemming, abbreviation expansion, and pure-ASCII tokenization are
unchanged. Suite parity with jdocmunch-mcp v1.114.1 (#91) and
jdatamunch-mcp v1.23.1. No reindex needed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: jgravelle

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 All notable changes to jcodemunch-mcp are documented here.
 
+## [1.108.161] - 2026-07-23 - BM25 tokenizer: Unicode word splitting + CJK character bigrams
+
+### Fixed
+- **The BM25 tokenizer no longer discards non-ASCII text.** `_TOKEN_RE` was
+  `[a-zA-Z0-9]{2,}`, so every non-ASCII character acted as a separator: CJK
+  symbol names, summaries, and docstrings produced zero BM25 tokens (the
+  lexical channel contributed nothing for those corpora) and accented Latin
+  identifiers were mangled (`café` → `caf`). The tokenizer now splits on
+  Unicode word boundaries and expands CJK runs (Hangul, Hiragana/Katakana,
+  Han) into overlapping character bigrams — applied identically at index and
+  query time, so bigram overlap is the match signal. Mixed-script tokens
+  split cleanly; camelCase/snake_case splitting, stemming, abbreviation
+  expansion, and pure-ASCII tokenization are unchanged. Suite parity with
+  jdocmunch-mcp v1.114.1 (#91 there) and jdatamunch-mcp v1.23.1. No reindex
+  needed — BM25 tokenizes stored index fields at scoring time.
+
 ## [1.108.160] - 2026-07-23 - Multi-checkout "different working tree" responses
 
 ### Changed

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "jcodemunch-mcp"
-version = "1.108.160"
+version = "1.108.161"
 description = "Token-efficient MCP server for source code exploration via tree-sitter AST parsing"
 readme = "README.md"
 requires-python = ">=3.10"

src/jcodemunch_mcp/tools/search_symbols.py

@@ -35,7 +35,29 @@
 
 # Pre-compiled regexes for _tokenize (called ~9000× on cold BM25 build)
 _CAMEL_RE = re.compile(r"([a-z])([A-Z])")
-_TOKEN_RE = re.compile(r"[a-zA-Z0-9]{2,}")
+# Unicode word runs (was [a-zA-Z0-9]{2,}, which silently dropped ALL
+# non-ASCII — CJK names/docstrings produced zero BM25 tokens; jdoc #91 class).
+_TOKEN_RE = re.compile(r"[^\W_]+")
+# CJK scripts carry no whitespace word boundaries; runs in these ranges are
+# expanded to overlapping character bigrams (same expansion at index and
+# query time, so bigram overlap is the match signal).
+_CJK_RE = re.compile(
+    "[ᄀ-ᇿ"  # Hangul Jamo
+    "぀-ヿ"  # Hiragana + Katakana
+    "㄰-㆏"  # Hangul Compatibility Jamo
+    "ㇰ-ㇿ"  # Katakana Phonetic Extensions
+    "㐀-䶿"  # CJK Unified Ideographs Extension A
+    "一-鿿"  # CJK Unified Ideographs
+    "가-힯"  # Hangul Syllables
+    "豈-﫿]+"  # CJK Compatibility Ideographs
+)
+
+
+def _cjk_bigrams(run: str) -> list[str]:
+    """Overlapping character bigrams for a CJK run; a lone char passes through."""
+    if len(run) == 1:
+        return [run]
+    return [run[i : i + 2] for i in range(len(run) - 1)]
 
 # Search result cache (Feature 5 — session-aware routing)
 import threading
@@ -181,11 +203,21 @@ def _tokenize(text: str) -> list[str]:
     if not text:
         return []
     text = _CAMEL_RE.sub(r"\1_\2", text)
+    # Pad CJK runs with spaces so mixed-script tokens split cleanly.
+    text = _CJK_RE.sub(lambda m: " " + m.group(0) + " ", text)
     raw_tokens = [t.lower() for t in _TOKEN_RE.findall(text)]
 
     result = []
     seen: set[str] = set()
     for tok in raw_tokens:
+        if _CJK_RE.fullmatch(tok):
+            # Bigram expansion; stemming/abbreviations are English-only.
+            for bg in _cjk_bigrams(tok):
+                result.append(bg)
+                seen.add(bg)
+            continue
+        if len(tok) < 2:
+            continue
         result.append(tok)
         seen.add(tok)
         # Stemmed form

tests/test_v1_108_161.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""v1.108.161 — BM25 tokenizer no longer discards non-ASCII text (jdoc #91 class).
+
+_TOKEN_RE was [a-zA-Z0-9]{2,}, so every non-ASCII character acted as a
+separator: CJK symbol names/summaries/docstrings produced zero BM25 tokens
+and accented Latin was mangled. Now splits on Unicode word boundaries and
+expands CJK runs into overlapping character bigrams — identical at index and
+query time, so bigram overlap is the match signal. Suite parity with
+jdocmunch-mcp v1.114.1 and jdatamunch-mcp v1.23.1.
+"""
+
+from jcodemunch_mcp.tools.search_symbols import _cjk_bigrams, _tokenize
+
+
+class TestCJKTokenization:
+    def test_korean_produces_bigrams(self):
+        toks = _tokenize("초과근무 승인 규칙")
+        assert toks == ["초과", "과근", "근무", "승인", "규칙"]
+
+    def test_mixed_camelcase_and_korean(self):
+        toks = _tokenize("OvertimeService 초과근무 계산")
+        assert "overtime" in toks and "service" in toks
+        assert "초과" in toks and "근무" in toks and "계산" in toks
+
+    def test_accented_latin_survives_intact(self):
+        # Was ['caf', 've'] (single-char fragments dropped by the old {2,}).
+        toks = _tokenize("café naïve")
+        assert "café" in toks and "naïve" in toks
+
+    def test_japanese_bigrams(self):
+        toks = _tokenize("日本語のドキュメント")
+        assert "日本" in toks and "本語" in toks
+
+    def test_single_cjk_char_kept(self):
+        assert _tokenize("車") == ["車"]
+
+    def test_mixed_script_run_splits(self):
+        toks = _tokenize("초과근무OvertimeService")
+        assert "overtime" in toks and "초과" in toks
+
+    def test_bigram_helper(self):
+        assert _cjk_bigrams("가") == ["가"]
+        assert _cjk_bigrams("가나") == ["가나"]
+        assert _cjk_bigrams("가나다") == ["가나", "나다"]
+
+    def test_ascii_behavior_unchanged(self):
+        # camelCase split + stemming + abbreviation expansion all intact.
+        assert _tokenize("getUserData") == ["get", "user", "data"]
+        toks = _tokenize("config")
+        assert "config" in toks and "configuration" in toks
+        # Single-char ASCII tokens still dropped (old {2,} behavior).
+        assert _tokenize("a b") == []