Skip to content

Commit 81fea62

Browse files
tribixbitetribixbite
committed
fix: language isolation for Dict Manager and beam search
Dictionary Manager: - MainDictionarySource was hardcoded to en_enhanced.json - Added languageCode parameter to load correct dictionary - Binary dictionary loading for non-English languages - WordListFragment passes language code to MainDictionarySource Beam Search Trie: - Defensive check in getVocabularyTrie() - Returns null if primary is non-English but trie is English - Prevents English vocabulary contamination - Logs error to diagnose initialization issues — claude-opus-4-5-20251101
1 parent f599f2a commit 81fea62

6 files changed

Lines changed: 112 additions & 20 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1919

2020
---
2121

22+
## [1.1.89] - 2025-01-05
23+
24+
### Fixed - Language Isolation
25+
- **Dictionary Manager**: Now loads correct language dictionary (was always English)
26+
- **Beam Search Trie**: Defensive check prevents English trie contamination
27+
- Returns null if primary is non-English but trie wasn't replaced
28+
- Logs error to help diagnose initialization issues
29+
- **MainDictionarySource**: Added language parameter, loads binary dictionaries for non-English
30+
31+
---
32+
2233
## [1.1.88] - 2025-01-05
2334

2435
### Added - Multilanguage Support 🌍

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ dependencies {
5050
// UPDATE ONLY THESE THREE VALUES FOR A NEW RELEASE:
5151
ext.VERSION_MAJOR = 1
5252
ext.VERSION_MINOR = 1
53-
ext.VERSION_PATCH = 88
53+
ext.VERSION_PATCH = 89
5454
// =============================================================================
5555
// DERIVED VALUES (auto-calculated, do not edit):
5656
// - versionCode = MAJOR*10000 + MINOR*100 + PATCH

‎memory/todo.md‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,22 @@
11
# CleverKeys Working TODO List
22

33
**Last Updated**: 2026-01-05
4-
**Status**: v1.1.88 RELEASED - Multilanguage support with contraction isolation
4+
**Status**: v1.1.89 - Language isolation fixes for Dict Manager and beam search
55

66
---
77

8-
## Known Issues (Post-Release)
8+
## v1.1.89 Fixes - IN TESTING
99

10-
**English Words in French-Only Mode**:
11-
- Some English words still appear in predictions when Primary=French, Secondary=None
12-
- Dictionary Manager still shows English tab
13-
- Investigation ongoing - likely trie contamination from binary cache
14-
- Contraction isolation fix applied but additional source of contamination exists
10+
**Dictionary Manager Language Fix**:
11+
- [x] `MainDictionarySource` was hardcoded to load `en_enhanced.json`
12+
- [x] Added `languageCode` parameter to load correct language dictionary
13+
- [x] `WordListFragment` now passes language code to `MainDictionarySource`
14+
- [x] Binary dictionary loading added for non-English languages
15+
16+
**Beam Search Trie Defensive Check**:
17+
- [x] `getVocabularyTrie()` now verifies trie matches expected language
18+
- [x] If Primary=non-English but trie is still English, returns null (disables constraining)
19+
- [x] Logs error message to help diagnose initialization issues
1520

1621
---
1722

‎src/main/kotlin/tribixbite/cleverkeys/DictionaryDataSource.kt‎

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,15 @@ interface DictionaryDataSource {
2525
/**
2626
* Main dictionary source - loads from assets dictionary file
2727
* Uses prefix indexing for fast search with 50k vocabulary
28+
*
29+
* v1.1.89: Added language support - loads language-specific dictionary when available
30+
*
31+
* @param languageCode ISO 639-1 language code (e.g., "en", "fr", "es"). Defaults to "en".
2832
*/
2933
class MainDictionarySource(
3034
private val context: Context,
31-
private val disabledSource: DisabledDictionarySource
35+
private val disabledSource: DisabledDictionarySource,
36+
private val languageCode: String = "en"
3237
) : DictionaryDataSource {
3338

3439
// Cache the dictionary after first load
@@ -46,9 +51,25 @@ class MainDictionarySource(
4651
val disabled = disabledSource.getDisabledWords()
4752
val words = mutableListOf<DictionaryWord>()
4853

54+
// v1.1.89: Try language-specific binary dictionary first (for non-English)
55+
if (languageCode != "en") {
56+
try {
57+
val binFilename = "dictionaries/${languageCode}_enhanced.bin"
58+
val loaded = loadBinaryDictionary(binFilename, words, disabled)
59+
if (loaded) {
60+
Log.d(TAG, "Loaded ${words.size} words from binary dictionary: $binFilename")
61+
cachedWords = words.sorted()
62+
buildPrefixIndex(cachedWords!!)
63+
return@withContext cachedWords!!
64+
}
65+
} catch (e: Exception) {
66+
Log.w(TAG, "Binary dictionary not found for $languageCode, trying JSON/TXT fallback")
67+
}
68+
}
69+
4970
// Try JSON format first (50k words with frequencies)
5071
try {
51-
val jsonFilename = "dictionaries/en_enhanced.json"
72+
val jsonFilename = "dictionaries/${languageCode}_enhanced.json"
5273
val jsonString = context.assets.open(jsonFilename).bufferedReader().use { it.readText() }
5374
val jsonDict = org.json.JSONObject(jsonString)
5475
val keys = jsonDict.keys()
@@ -70,10 +91,10 @@ class MainDictionarySource(
7091
}
7192
Log.d(TAG, "Loaded ${words.size} words from JSON dictionary")
7293
} catch (e: Exception) {
73-
Log.w(TAG, "JSON dictionary not found, falling back to text format")
94+
Log.w(TAG, "JSON dictionary not found for $languageCode, falling back to text format")
7495

7596
// Fall back to text format
76-
val filename = "dictionaries/en_enhanced.txt"
97+
val filename = "dictionaries/${languageCode}_enhanced.txt"
7798
context.assets.open(filename).bufferedReader().use { reader ->
7899
reader.lineSequence()
79100
.filter { it.isNotBlank() && !it.startsWith("#") }
@@ -165,6 +186,44 @@ class MainDictionarySource(
165186
throw UnsupportedOperationException("Cannot update words in main dictionary")
166187
}
167188

189+
/**
190+
* Load dictionary from binary format (.bin files for non-English languages).
191+
* Uses NormalizedPrefixIndex to read the binary format.
192+
*/
193+
private fun loadBinaryDictionary(
194+
filename: String,
195+
words: MutableList<DictionaryWord>,
196+
disabled: Set<String>
197+
): Boolean {
198+
return try {
199+
val index = NormalizedPrefixIndex()
200+
val loaded = BinaryDictionaryLoader.loadIntoNormalizedIndex(context, filename, index)
201+
if (loaded) {
202+
// Extract all words from the index
203+
val normalizedWords = index.getAllNormalizedWords()
204+
for (word in normalizedWords) {
205+
// Get canonical form (with accents) if available
206+
val results = index.getWordsWithPrefix(word)
207+
val canonical = results.find { it.normalized == word }?.bestCanonical ?: word
208+
words.add(
209+
DictionaryWord(
210+
word = canonical, // Show accented form
211+
frequency = 100, // Binary format doesn't store frequency for display
212+
source = WordSource.MAIN,
213+
enabled = !disabled.contains(word) && !disabled.contains(canonical)
214+
)
215+
)
216+
}
217+
true
218+
} else {
219+
false
220+
}
221+
} catch (e: Exception) {
222+
Log.e(TAG, "Error loading binary dictionary: $filename", e)
223+
false
224+
}
225+
}
226+
168227
companion object {
169228
private const val TAG = "MainDictionarySource"
170229
private const val PREFIX_INDEX_MAX_LENGTH = 3

‎src/main/kotlin/tribixbite/cleverkeys/OptimizedVocabulary.kt‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -117,15 +117,31 @@ class OptimizedVocabulary(private val context: Context) {
117117
* - Primary=English: Returns English vocabulary trie
118118
* - Primary=French: Returns French normalized words trie
119119
*
120-
* @return The active beam search trie, or null if not loaded
120+
* CRITICAL v1.1.89: If primary is non-English but activeBeamSearchTrie still points
121+
* to vocabularyTrie (English), return null to disable trie constraining rather than
122+
* using the wrong language's vocabulary.
123+
*
124+
* @return The active beam search trie, or null if not loaded or language mismatch
121125
*/
122126
fun getVocabularyTrie(): VocabularyTrie? {
123-
val trie = if (isLoaded) activeBeamSearchTrie else null
124-
val isLanguageTrie = (trie != null && trie !== vocabularyTrie)
125-
val stats = trie?.getStats()
126-
Log.d(TAG, "getVocabularyTrie(): isLoaded=$isLoaded, isLanguageTrie=$isLanguageTrie, " +
127-
"trieWords=${stats?.first ?: 0}, primary=$_primaryLanguageCode, englishFallback=$_englishFallbackEnabled")
128-
return trie
127+
if (!isLoaded) return null
128+
129+
val isLanguageTrie = activeBeamSearchTrie !== vocabularyTrie
130+
val stats = activeBeamSearchTrie.getStats()
131+
132+
// CRITICAL CHECK: If primary is non-English, we MUST have a language-specific trie
133+
// If we're still pointing to vocabularyTrie (English), something went wrong with
134+
// loadPrimaryDictionary() - disable trie constraining to avoid English contamination
135+
if (_primaryLanguageCode != "en" && !isLanguageTrie) {
136+
Log.e(TAG, "🚨 LANGUAGE MISMATCH: primary=$_primaryLanguageCode but using English trie! " +
137+
"Disabling trie constraint to avoid contamination. " +
138+
"Call loadPrimaryDictionary() to fix.")
139+
return null // Return null to disable trie constraining
140+
}
141+
142+
Log.d(TAG, "getVocabularyTrie(): isLanguageTrie=$isLanguageTrie, " +
143+
"trieWords=${stats.first}, primary=$_primaryLanguageCode, englishFallback=$_englishFallbackEnabled")
144+
return activeBeamSearchTrie
129145
}
130146

131147
/**

‎src/main/kotlin/tribixbite/cleverkeys/WordListFragment.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ class WordListFragment : Fragment() {
9595
val disabledSource = DisabledDictionarySource(defaultPrefs, languageCode)
9696

9797
dataSource = when (tabType) {
98-
TabType.ACTIVE -> MainDictionarySource(requireContext(), disabledSource)
98+
// v1.1.89: Pass language code to load correct dictionary (not always English)
99+
TabType.ACTIVE -> MainDictionarySource(requireContext(), disabledSource, languageCode ?: "en")
99100
TabType.DISABLED -> disabledSource
100101
TabType.USER -> UserDictionarySource(requireContext(), requireContext().contentResolver)
101102
TabType.CUSTOM -> {

0 commit comments

Comments
 (0)