Skip to content

Commit b99ef7b

Browse files
snomosclaude
authored and
bbqsrc
committed
Fix grapheme cluster handling in reweighting distance calculations
Fixed incorrect edit distance calculations in the reweighting logic that were counting combining characters as separate characters instead of treating them as parts of grapheme clusters. Changes: - Added grapheme_damerau_levenshtein() function that properly handles Unicode grapheme clusters (base character + combining marks) as single units - Changed reweighting logic to use grapheme clusters instead of chars throughout (input_lower and sugg_lower now Vec<&str> instead of Vec<char>) - Updated string building to use .join("") instead of .iter().collect() - Added logic to correctly classify changes as start/mid/end based on their actual position in the word: * When prefix_len == 0 (no prefix match), the first grapheme difference is a START change, remainder is MID * When suffix_len == 0 (no suffix match), the last grapheme difference is an END change, remainder is MID This ensures that reweighting penalties are calculated correctly based on actual grapheme edit distance and position, essential for languages using combining diacritics like Kildin Sami (а̄, э̄, о̄). Example fixes: - Input "ыббьр" → Suggestion "э̄ббр" Before: rew: 0/15/0 (incorrect - э̄ counted as 2 chars) After: rew: 10/5/0 (correct - э̄ is 1 grapheme at start) - Input "ыббьр" → Suggestion "-аббьр" Before: rew: 0/10/0 (incorrect - first change not classified as start) After: rew: 10/5/0 (correct - first grapheme change is at start) Fixes #54 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent d852368 commit b99ef7b

1 file changed

Lines changed: 104 additions & 21 deletions

File tree

‎src/speller/mod.rs‎

Lines changed: 104 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,65 @@ pub mod suggestion;
2828

2929
mod worker;
3030

31+
/// Calculate Damerau-Levenshtein distance between two strings using grapheme clusters.
32+
///
33+
/// This function properly handles combining characters by treating grapheme clusters
34+
/// (base character + combining marks) as single units, rather than counting them
35+
/// as separate characters. This is essential for languages that use combining
36+
/// diacritics (e.g., Kildin Sami with combining macron).
37+
fn grapheme_damerau_levenshtein(s1: &str, s2: &str) -> usize {
38+
let s1_graphemes: Vec<&str> = Graphemes::new(s1).collect();
39+
let s2_graphemes: Vec<&str> = Graphemes::new(s2).collect();
40+
41+
let len1 = s1_graphemes.len();
42+
let len2 = s2_graphemes.len();
43+
44+
if len1 == 0 {
45+
return len2;
46+
}
47+
if len2 == 0 {
48+
return len1;
49+
}
50+
51+
let mut matrix = vec![vec![0usize; len2 + 1]; len1 + 1];
52+
53+
for i in 0..=len1 {
54+
matrix[i][0] = i;
55+
}
56+
for j in 0..=len2 {
57+
matrix[0][j] = j;
58+
}
59+
60+
for i in 1..=len1 {
61+
for j in 1..=len2 {
62+
let cost = if s1_graphemes[i - 1] == s2_graphemes[j - 1] {
63+
0
64+
} else {
65+
1
66+
};
67+
68+
matrix[i][j] = std::cmp::min(
69+
std::cmp::min(
70+
matrix[i - 1][j] + 1, // deletion
71+
matrix[i][j - 1] + 1, // insertion
72+
),
73+
matrix[i - 1][j - 1] + cost, // substitution
74+
);
75+
76+
// Transposition
77+
if i > 1
78+
&& j > 1
79+
&& s1_graphemes[i - 1] == s2_graphemes[j - 2]
80+
&& s1_graphemes[i - 2] == s2_graphemes[j - 1]
81+
{
82+
matrix[i][j] = std::cmp::min(matrix[i][j], matrix[i - 2][j - 2] + cost);
83+
}
84+
}
85+
}
86+
87+
matrix[len1][len2]
88+
}
89+
3190
/// Temporary struct to store weight details during suggestion generation
3291
#[derive(Clone, Debug)]
3392
struct SuggestionData {
@@ -568,11 +627,13 @@ where
568627

569628
// Calculate distances based on case-insensitive alignment
570629
// by scanning from both ends and splicing in the middle
571-
let input_lower: Vec<char> =
572-
original_input.to_lowercase().chars().collect();
573-
let sugg_lower: Vec<char> = sugg.value().to_lowercase().chars().collect();
630+
// Use grapheme clusters to properly handle combining characters
631+
let input_lower_str = original_input.to_lowercase();
632+
let sugg_lower_str = sugg.value().to_lowercase();
633+
let input_lower: Vec<&str> = Graphemes::new(&input_lower_str).collect();
634+
let sugg_lower: Vec<&str> = Graphemes::new(&sugg_lower_str).collect();
574635

575-
// Special case: both input and suggestion are 2 chars or less - no middle section
636+
// Special case: both input and suggestion are 2 graphemes or less - no middle section
576637
let is_short = input_lower.len() <= 2 && sugg_lower.len() <= 2;
577638

578639
let (start_dist, mid_dist, end_dist): (usize, i32, usize) = if input_lower
@@ -676,10 +737,10 @@ where
676737
} else {
677738
// DL between skipped portions
678739
let inp_start_str: String =
679-
input_lower[0..*start_in_off].iter().collect();
740+
input_lower[0..*start_in_off].join("");
680741
let sug_start_str: String =
681-
sugg_lower[0..*start_su_off].iter().collect();
682-
strsim::damerau_levenshtein(
742+
sugg_lower[0..*start_su_off].join("");
743+
grapheme_damerau_levenshtein(
683744
&inp_start_str,
684745
&sug_start_str,
685746
)
@@ -692,13 +753,11 @@ where
692753
// DL between skipped portions
693754
let inp_end_str: String = input_lower
694755
[input_lower.len().saturating_sub(*end_in_off)..]
695-
.iter()
696-
.collect();
756+
.join("");
697757
let sug_end_str: String = sugg_lower
698758
[sugg_lower.len().saturating_sub(*end_su_off)..]
699-
.iter()
700-
.collect();
701-
strsim::damerau_levenshtein(&inp_end_str, &sug_end_str)
759+
.join("");
760+
grapheme_damerau_levenshtein(&inp_end_str, &sug_end_str)
702761
};
703762

704763
best_score = score;
@@ -761,30 +820,54 @@ where
761820
} else if inp_start_pos < inp_end_pos || sug_start_pos < sug_end_pos {
762821
let inp_mid: String = input_lower[inp_start_pos.min(inp_end_pos)
763822
..inp_end_pos.max(inp_start_pos)]
764-
.iter()
765-
.collect();
823+
.join("");
766824
let sug_mid: String = sugg_lower[sug_start_pos.min(sug_end_pos)
767825
..sug_end_pos.max(sug_start_pos)]
768-
.iter()
769-
.collect();
770-
let d = strsim::damerau_levenshtein(&inp_mid, &sug_mid) as i32;
826+
.join("");
827+
let d = grapheme_damerau_levenshtein(&inp_mid, &sug_mid) as i32;
771828
(d, end_d)
772829
} else {
773830
(0, end_d)
774831
};
775832

776-
(start_d, mid_d, adjusted_end_d)
833+
// Reclassify mid changes based on actual position in the word
834+
// If there's no prefix match (prefix_len == 0), the first grapheme differs, so at least 1 change is at START
835+
// If there's no suffix match (actual_suffix == 0), the last grapheme differs, so at least 1 change is at END
836+
if mid_d > 0 {
837+
if prefix_len == 0 && actual_suffix > 0 {
838+
// No prefix, but there's a suffix - first grapheme change is at START, rest is MID
839+
let start_changes = 1; // First grapheme is different
840+
let remaining_mid =
841+
(mid_d as usize).saturating_sub(start_changes);
842+
(
843+
start_d + start_changes,
844+
remaining_mid as i32,
845+
adjusted_end_d,
846+
)
847+
} else if actual_suffix == 0 && prefix_len > 0 {
848+
// No suffix, but there's a prefix - last grapheme change is at END, rest is MID
849+
let end_changes = 1; // Last grapheme is different
850+
let remaining_mid =
851+
(mid_d as usize).saturating_sub(end_changes);
852+
(start_d, remaining_mid as i32, adjusted_end_d + end_changes)
853+
} else {
854+
// Real middle section or changes span multiple sections
855+
(start_d, mid_d, adjusted_end_d)
856+
}
857+
} else {
858+
(start_d, mid_d, adjusted_end_d)
859+
}
777860
};
778861

779-
// Special case: when input or suggestion has duplicate chars at start/end that match
862+
// Special case: when input or suggestion has duplicate graphemes at start/end that match
780863
// Examples:
781864
// - Insertion: "Anar" → "Aanaar" - both start with 'a', insertion of second 'a' should be middle
782865
// - Deletion: "Aarâhšoddâdem" → "Arâšoddâdem" - both start with 'A', deletion of second 'a' should be middle
783866
let (start_dist, mid_dist, end_dist) =
784867
if !is_short && input_lower.len() > 0 && sugg_lower.len() > 0 {
785868
let adjusted_start =
786869
if start_dist > 0 && input_lower[0] == sugg_lower[0] {
787-
// First char matches - check if either has duplicate at position 1
870+
// First grapheme matches - check if either has duplicate at position 1
788871
let sugg_has_dup =
789872
sugg_lower.len() > 1 && sugg_lower[0] == sugg_lower[1];
790873
let input_has_dup = input_lower.len() > 1
@@ -805,7 +888,7 @@ where
805888
&& input_lower[input_lower.len() - 1]
806889
== sugg_lower[sugg_lower.len() - 1]
807890
{
808-
// Last char matches - check if either has duplicate at position len-2
891+
// Last grapheme matches - check if either has duplicate at position len-2
809892
let sugg_has_dup = sugg_lower.len() > 1
810893
&& sugg_lower[sugg_lower.len() - 1]
811894
== sugg_lower[sugg_lower.len() - 2];

0 commit comments

Comments
 (0)