Skip to content

Commit 2a8a9b6

Browse files
ashvardanianashvardanian
committed
Fix: Dispatch Central European path
1 parent 66cade8 commit 2a8a9b6

1 file changed

Lines changed: 53 additions & 24 deletions

File tree

‎include/stringzilla/utf8_case.h‎

Lines changed: 53 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4110,7 +4110,7 @@ SZ_INTERNAL unsigned sz_utf8_case_rune_safety_profiles_mask_( //
41104110
sz_bool_t at_start = (prev_rune == 0) ? sz_true_k : sz_false_k;
41114111
sz_bool_t at_end = (next_rune == 0) ? sz_true_k : sz_false_k;
41124112

4113-
// Helper: position detection for 's' rule (mid-ß-expansion avoidance)
4113+
// Helper: position detection for 's' rule (mid-ß-expansion avoidance in Western profile)
41144114
// Position 1: prev exists but prev_prev doesn't (prev is at position 0)
41154115
// Position N-2: next exists but next_next doesn't (next is at position N-1)
41164116
sz_bool_t at_pos_1 = (prev_rune != 0 && prev_prev_rune == 0) ? sz_true_k : sz_false_k;
@@ -4183,12 +4183,21 @@ SZ_INTERNAL unsigned sz_utf8_case_rune_safety_profiles_mask_( //
41834183
break;
41844184

41854185
// 'n':
4186-
// - All: Contextual. Can't be first; can't follow 'ʼ' (U+02BC).
4186+
// - ASCII/Cyrillic/Greek: Contextual. Can't be first; can't follow 'ʼ' (U+02BC).
41874187
// Avoids: 'ʼn' (U+0149) → "ʼn".
4188+
// - Armenian: UNSAFE. Armenian kernel cannot handle ʼn (C5 89) → ʼn expansion.
4189+
// The character 'n' can match the 2nd part of the expansion, causing false positives.
4190+
// - Western/Central/Viet: Contextual, same as above.
41884191
// Western profile does NOT detect this in haystack scan.
41894192
case 'n':
4190-
if (at_start == sz_false_k && prev_ascii)
4191-
safety |= strict_ascii_group | central_viet_group | western_group;
4193+
// Exclude Armenian - it cannot handle ʼn → ʼn one-to-many expansion
4194+
if (at_start == sz_false_k && prev_ascii) {
4195+
safety |= (1 << sz_utf8_case_rune_safe_ascii_k) | //
4196+
(1 << sz_utf8_case_rune_safe_cyrillic_k) | //
4197+
(1 << sz_utf8_case_rune_safe_greek_k); //
4198+
// Armenian EXCLUDED: sz_utf8_case_rune_safe_armenian_k
4199+
safety |= central_viet_group | western_group;
4200+
}
41924201
break;
41934202

41944203
// 'i':
@@ -4235,8 +4244,8 @@ SZ_INTERNAL unsigned sz_utf8_case_rune_safety_profiles_mask_( //
42354244

42364245
// 's'
42374246
// - Strict: UNSAFE. 'ſ' (U+017F) folds to 's'.
4238-
// - Central/Viet: Contextual. Can't be first/last; can't be adjacent to 's'/'t'.
4239-
// Avoids: "ss" (Eszett expansion) and "st" (ligature expansion).
4247+
// - Central/Vietnamese: Contextual. Can't be first/last; can't be adjacent to 's'/'t'.
4248+
// Avoids: "ss" (Eszett expansion), "st" (ligature expansion), and ſ→s in danger zones.
42404249
// - Western: Contextual. Can't be at positions 0, 1 (if prev='s'), N-1, or N-2 (if next='s').
42414250
// Avoids mid-ß-expansion matches: ß→"ss" in-place means needle with 's' at these
42424251
// positions could match at byte offset 1 (UTF-8 continuation byte 0x9F).
@@ -4284,8 +4293,9 @@ SZ_INTERNAL unsigned sz_utf8_case_rune_safety_profiles_mask_( //
42844293
safety |= western_group;
42854294
}
42864295
else if (rune == 0x00B5) {
4287-
// 'µ' (Micro Sign) folds to 'μ' (Greek Mu, CE BC) - BANNED from Western Europe
4288-
// to avoid Greek complexity. Needles with µ use serial fallback.
4296+
// 'µ' (Micro Sign) folds to Greek 'μ' (U+03BC, CE BC).
4297+
// Allow only the Greek SIMD path; Latin paths remain unsafe.
4298+
safety |= (1 << sz_utf8_case_rune_safe_greek_k);
42894299
}
42904300
else { safety |= western_group | central_viet_group; }
42914301
}
@@ -4379,8 +4389,9 @@ SZ_INTERNAL unsigned sz_utf8_case_rune_safety_profiles_mask_( //
43794389
case 0x0535: // U+0535 Ech uppercase
43804390
if (at_start || lower_prev_arm == 0x0574 || lower_next_arm == 0x0582) armenian_safe = sz_false_k;
43814391
break;
4382-
case 0x0582: // U+0582 yiwn - can't be last; can't follow U+0565 ech
4383-
if (at_end || lower_prev_arm == 0x0565) armenian_safe = sz_false_k;
4392+
case 0x0582: // U+0582 yiwn - can't be first; can't be last; can't follow U+0565 ech
4393+
// Armenian ligature և (U+0587) → ech + yiwn; needle starting with yiwn matches mid-expansion
4394+
if (at_start || at_end || lower_prev_arm == 0x0565) armenian_safe = sz_false_k;
43844395
break;
43854396
case 0x0574: // U+0574 men - can't be last; can't precede U+0576, U+0565, U+056B, U+056D
43864397
case 0x0544: // U+0544 Men uppercase
@@ -5750,6 +5761,10 @@ SZ_INTERNAL __m512i sz_utf8_case_insensitive_find_ice_greek_fold_zmm_(__m512i te
57505761
// Lead bytes:
57515762
__m512i const x_ce_zmm = _mm512_set1_epi8((char)0xCE);
57525763
__m512i const x_cf_zmm = _mm512_set1_epi8((char)0xCF);
5764+
// Micro sign µ (C2 B5) folds to Greek μ (CE BC) in-place.
5765+
__m512i const x_c2_zmm = _mm512_set1_epi8((char)0xC2);
5766+
__m512i const x_b5_zmm = _mm512_set1_epi8((char)0xB5);
5767+
__m512i const x_bc_zmm = _mm512_set1_epi8((char)0xBC);
57535768

57545769
// Ranges for CE lead byte:
57555770
// CE 86-8F (Accented Upper): 'Ά' (86) ... 'Ώ' (8F)
@@ -5848,6 +5863,12 @@ SZ_INTERNAL __m512i sz_utf8_case_insensitive_find_ice_greek_fold_zmm_(__m512i te
58485863
// Apply Final Sigma
58495864
result_zmm = _mm512_mask_mov_epi8(result_zmm, is_final_sigma, x_83_zmm);
58505865

5866+
// Apply Micro Sign folding: C2 B5 -> CE BC
5867+
__mmask64 is_c2_mask = _mm512_cmpeq_epi8_mask(result_zmm, x_c2_zmm);
5868+
__mmask64 is_micro_second = (is_c2_mask << 1) & _mm512_cmpeq_epi8_mask(result_zmm, x_b5_zmm);
5869+
result_zmm = _mm512_mask_mov_epi8(result_zmm, is_micro_second >> 1, x_ce_zmm); // Lead C2 -> CE
5870+
result_zmm = _mm512_mask_mov_epi8(result_zmm, is_micro_second, x_bc_zmm); // Second B5 -> BC
5871+
58515872
return result_zmm;
58525873
}
58535874

@@ -5899,16 +5920,14 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_greek_( //
58995920
// - CE B0: 'ΰ' -> "ΰ" (3 bytes)
59005921
// - CF 90, 91, 95, 96: Greek Symbols 'ϐ', 'ϑ', 'ϕ', 'ϖ'
59015922
// - CF B0, B1, B5: Greek Symbols 'ϰ', 'ϱ', 'ϵ'
5902-
// - C2 B5: Micro Sign 'µ' -> 'μ' (CE BC)
59035923
// - E2 84 A6: Ohm Sign 'Ω' -> 'ω' (CF 89) (detect E2 84 prefix)
59045924
// - E1 xx xx: Polytonic Greek (excluded by safety profile, but just in case)
5905-
sz_u512_vec_t x_ce_vec, x_cf_vec, x_c2_vec, x_e1_vec, x_e2_vec, x_cd_vec;
5925+
sz_u512_vec_t x_ce_vec, x_cf_vec, x_e1_vec, x_e2_vec, x_cd_vec;
59065926
sz_u512_vec_t x_90_vec, x_b0_vec, x_84_vec;
59075927
sz_u512_vec_t x_91_vec, x_95_vec, x_96_vec, x_b1_vec, x_b5_vec;
59085928

59095929
x_ce_vec.zmm = _mm512_set1_epi8((char)0xCE);
59105930
x_cf_vec.zmm = _mm512_set1_epi8((char)0xCF);
5911-
x_c2_vec.zmm = _mm512_set1_epi8((char)0xC2);
59125931
x_e1_vec.zmm = _mm512_set1_epi8((char)0xE1); // E1 lead byte (Polytonic)
59135932
x_e2_vec.zmm = _mm512_set1_epi8((char)0xE2);
59145933
x_cd_vec.zmm = _mm512_set1_epi8((char)0xCD); // CD lead byte (Combining Diacritics)
@@ -5948,12 +5967,11 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_greek_( //
59485967
// Check for anomalies
59495968
__mmask64 x_ce_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_ce_vec.zmm);
59505969
__mmask64 x_cf_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_cf_vec.zmm);
5951-
__mmask64 x_c2_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_c2_vec.zmm);
59525970
__mmask64 x_e2_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_e2_vec.zmm);
59535971
__mmask64 x_e1_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_e1_vec.zmm);
59545972
__mmask64 x_cd_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_cd_vec.zmm);
59555973

5956-
if (x_ce_mask | x_cf_mask | x_c2_mask | x_e2_mask | x_e1_mask | x_cd_mask) {
5974+
if (x_ce_mask | x_cf_mask | x_e2_mask | x_e1_mask | x_cd_mask) {
59575975
__mmask64 x_90_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_90_vec.zmm);
59585976
__mmask64 x_b0_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_b0_vec.zmm);
59595977
__mmask64 x_be_ce_90_b0 = (x_ce_mask << 1) & (x_90_mask | x_b0_mask); // CE 90 or CE B0
@@ -5969,11 +5987,10 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_greek_( //
59695987

59705988
__mmask64 x_be_cf_sym = (x_cf_mask << 1) & (x_9x_mask | x_bx_mask);
59715989

5972-
__mmask64 x_micro = (x_c2_mask << 1) & x_bx_mask; // C2 B5 (B5 is in bx_mask)
59735990
__mmask64 x_ohm = (x_e2_mask << 1) & _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_84_vec.zmm);
59745991
// E1 is always dangerous in Greek path (Polytonic/Extensions)
59755992
// CD is dangerous (Combining Diacritical Marks)
5976-
__mmask64 danger_mask = x_be_ce_90_b0 | x_be_cf_sym | x_micro | x_ohm | x_e1_mask | x_cd_mask;
5993+
__mmask64 danger_mask = x_be_ce_90_b0 | x_be_cf_sym | x_ohm | x_e1_mask | x_cd_mask;
59775994

59785995
if (danger_mask) {
59795996
sz_cptr_t match = sz_utf8_case_insensitive_find_in_danger_zone_( //
@@ -6176,9 +6193,11 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_vietnamese_( //
61766193
// 1E 9E (ẞ) is in the 96-9F range, so covered.
61776194
// EF AC 80-86: Ligatures (ff-st) that expand when folded.
61786195
// E2 84 AA: Kelvin sign K folds to 'k'.
6196+
// C5 BF: Long S 'ſ' folds to 's'.
61796197

6180-
sz_u512_vec_t x_e1_vec, x_ba_vec, x_96_vec, x_9f_vec;
6181-
sz_u512_vec_t x_c3_vec, x_9f_byte_vec;
6198+
sz_u512_vec_t x_e1_vec, x_ba_vec, x_96_vec;
6199+
sz_u512_vec_t x_c3_vec, x_9f_vec;
6200+
sz_u512_vec_t x_c5_vec, x_bf_vec;
61826201
sz_u512_vec_t x_ef_vec, x_ac_vec;
61836202
sz_u512_vec_t x_e2_vec, x_84_vec;
61846203

@@ -6187,7 +6206,8 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_vietnamese_( //
61876206
x_96_vec.zmm = _mm512_set1_epi8((char)0x96);
61886207
x_9f_vec.zmm = _mm512_set1_epi8((char)0x9F);
61896208
x_c3_vec.zmm = _mm512_set1_epi8((char)0xC3);
6190-
x_9f_byte_vec.zmm = _mm512_set1_epi8((char)0x9F);
6209+
x_c5_vec.zmm = _mm512_set1_epi8((char)0xC5);
6210+
x_bf_vec.zmm = _mm512_set1_epi8((char)0xBF);
61916211
x_ef_vec.zmm = _mm512_set1_epi8((char)0xEF);
61926212
x_ac_vec.zmm = _mm512_set1_epi8((char)0xAC);
61936213
x_e2_vec.zmm = _mm512_set1_epi8((char)0xE2);
@@ -6216,18 +6236,23 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_vietnamese_( //
62166236
// Check for anomalies
62176237
__mmask64 x_e1_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_e1_vec.zmm);
62186238
__mmask64 x_c3_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_c3_vec.zmm);
6239+
__mmask64 x_c5_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_c5_vec.zmm);
62196240
__mmask64 x_ef_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_ef_vec.zmm);
62206241
__mmask64 x_e2_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_e2_vec.zmm);
62216242

6222-
if (x_e1_mask | x_c3_mask | x_ef_mask | x_e2_mask) {
6243+
if (x_e1_mask | x_c3_mask | x_c5_mask | x_ef_mask | x_e2_mask) {
62236244
// Check E1 BA 96-9F
62246245
__mmask64 x_ba_second = (x_e1_mask << 1) & _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_ba_vec.zmm);
62256246
__mmask64 x_bad_third = (x_ba_second << 1) &
62266247
_mm512_mask_cmpge_epu8_mask(load_mask, haystack_vec.zmm, x_96_vec.zmm) &
62276248
_mm512_mask_cmple_epu8_mask(load_mask, haystack_vec.zmm, x_9f_vec.zmm);
62286249

62296250
// Check C3 9F (Sharp S)
6230-
__mmask64 x_sharp_s = (x_c3_mask << 1) & _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_9f_byte_vec.zmm);
6251+
__mmask64 x_sharp_s = (x_c3_mask << 1) & _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_9f_vec.zmm);
6252+
6253+
// Check C5 BF (Long S)
6254+
__mmask64 x_bf_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_bf_vec.zmm);
6255+
__mmask64 x_long_s = (x_c5_mask << 1) & x_bf_mask;
62316256

62326257
// Check EF AC (Ligatures: ff-st, U+FB00-FB06)
62336258
__mmask64 x_ligature = (x_ef_mask << 1) & _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_ac_vec.zmm);
@@ -6244,7 +6269,8 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_vietnamese_( //
62446269
// x_ligature marks 2nd byte. Shift right 1.
62456270
// x_kelvin marks 2nd byte. Shift right 1.
62466271

6247-
__mmask64 danger_mask = (x_bad_third >> 2) | (x_sharp_s >> 1) | (x_ligature >> 1) | (x_kelvin >> 1);
6272+
__mmask64 danger_mask =
6273+
(x_bad_third >> 2) | (x_sharp_s >> 1) | (x_long_s >> 1) | (x_ligature >> 1) | (x_kelvin >> 1);
62486274

62496275
if (danger_mask) {
62506276
sz_cptr_t match = sz_utf8_case_insensitive_find_in_danger_zone_( //
@@ -6332,6 +6358,7 @@ SZ_PUBLIC sz_cptr_t sz_utf8_case_insensitive_find_ice( //
63326358
sz_size_t longest = 0;
63336359
longest = sz_max_of_two(longest, analysis.ascii.length);
63346360
longest = sz_max_of_two(longest, analysis.western_europe.length);
6361+
longest = sz_max_of_two(longest, analysis.central_europe.length);
63356362
longest = sz_max_of_two(longest, analysis.cyrillic.length);
63366363
longest = sz_max_of_two(longest, analysis.greek.length);
63376364
longest = sz_max_of_two(longest, analysis.armenian.length);
@@ -6342,12 +6369,14 @@ SZ_PUBLIC sz_cptr_t sz_utf8_case_insensitive_find_ice( //
63426369
return sz_utf8_case_insensitive_find_serial( //
63436370
haystack, haystack_length, needle, needle_length, needle_metadata, matched_length);
63446371

6345-
// Select the best window (prefer ASCII > Western > others for tie-breaking)
6372+
// Select the best window (prefer ASCII > Western > Central > others for tie-breaking)
63466373
sz_utf8_string_slice_t const *best = 0;
63476374
if (analysis.ascii.length == longest)
63486375
best = &analysis.ascii, needle_metadata->kernel_id = sz_utf8_case_rune_safe_ascii_k;
63496376
else if (analysis.western_europe.length == longest)
63506377
best = &analysis.western_europe, needle_metadata->kernel_id = sz_utf8_case_rune_safe_western_europe_k;
6378+
else if (analysis.central_europe.length == longest)
6379+
best = &analysis.central_europe, needle_metadata->kernel_id = sz_utf8_case_rune_safe_central_europe_k;
63516380
else if (analysis.cyrillic.length == longest)
63526381
best = &analysis.cyrillic, needle_metadata->kernel_id = sz_utf8_case_rune_safe_cyrillic_k;
63536382
else if (analysis.greek.length == longest)

0 commit comments

Comments
 (0)