Skip to content

Commit d327cf8

Browse files
ashvardanianashvardanian
committed
Improve: Detect more danger zones
1 parent 9ad155e commit d327cf8

2 files changed

Lines changed: 136 additions & 21 deletions

File tree

‎include/stringzilla/utf8_case.h‎

Lines changed: 29 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3223,8 +3223,8 @@ SZ_PUBLIC sz_bool_t sz_utf8_case_agnostic_ice(sz_cptr_t str, sz_size_t length) {
32233223
__m512i data = _mm512_maskz_loadu_epi8(load_mask, ptr);
32243224

32253225
// 1. ASCII letter check (zeros beyond string are fine - not letters)
3226-
__mmask64 is_upper = _mm512_cmp_epu8_mask(_mm512_sub_epi8(data, a_upper_vec), z26_vec, _MM_CMPINT_LT);
3227-
__mmask64 is_lower = _mm512_cmp_epu8_mask(_mm512_sub_epi8(data, a_lower_vec), z26_vec, _MM_CMPINT_LT);
3226+
__mmask64 is_upper = _mm512_cmplt_epu8_mask(_mm512_sub_epi8(data, a_upper_vec), z26_vec);
3227+
__mmask64 is_lower = _mm512_cmplt_epu8_mask(_mm512_sub_epi8(data, a_lower_vec), z26_vec);
32283228
if (is_upper | is_lower) return sz_false_k;
32293229

32303230
// 2. Check for non-ASCII in lead positions
@@ -3250,8 +3250,8 @@ SZ_PUBLIC sz_bool_t sz_utf8_case_agnostic_ice(sz_cptr_t str, sz_size_t length) {
32503250
// C3-CF: Latin Extended (umlauts, accents, Eszett)
32513251
// D0-D1: Cyrillic, D4-D6: Armenian (D6 needed for small letters U+0580+)
32523252
if (is_two) {
3253-
__mmask64 is_bicameral = _mm512_cmp_epu8_mask( //
3254-
_mm512_sub_epi8(data, xc3_vec), _mm512_set1_epi8(0x14), _MM_CMPINT_LT);
3253+
__mmask64 is_bicameral = _mm512_cmplt_epu8_mask( //
3254+
_mm512_sub_epi8(data, xc3_vec), _mm512_set1_epi8(0x14));
32553255

32563256
// Special case: C2 B5 = U+00B5 MICRO SIGN folds to Greek mu (U+03BC)
32573257
__mmask64 is_c2 = _mm512_cmpeq_epi8_mask(data, _mm512_set1_epi8((char)0xC2)) & is_two;
@@ -3266,7 +3266,7 @@ SZ_PUBLIC sz_bool_t sz_utf8_case_agnostic_ice(sz_cptr_t str, sz_size_t length) {
32663266
__mmask64 is_ca = _mm512_cmpeq_epi8_mask(data, _mm512_set1_epi8((char)0xCA)) & is_two;
32673267
if (is_ca) {
32683268
__mmask64 ca_sec = is_ca << 1; // positions of second bytes after CA
3269-
__mmask64 is_ge_b0 = _mm512_cmp_epu8_mask(data, _mm512_set1_epi8((char)0xB0), _MM_CMPINT_NLT);
3269+
__mmask64 is_ge_b0 = _mm512_cmpge_epu8_mask(data, _mm512_set1_epi8((char)0xB0));
32703270
// If second byte >= B0, exclude this CA from bicameral
32713271
__mmask64 exclude_ca = ((ca_sec & is_ge_b0) >> 1) & is_ca;
32723272
is_bicameral &= ~exclude_ca;
@@ -3289,19 +3289,19 @@ SZ_PUBLIC sz_bool_t sz_utf8_case_agnostic_ice(sz_cptr_t str, sz_size_t length) {
32893289
__mmask64 is_e2 = _mm512_cmpeq_epi8_mask(data, _mm512_set1_epi8((char)0xE2)) & is_three;
32903290
if (is_e2) {
32913291
__mmask64 e2_sec = is_e2 << 1;
3292-
__mmask64 safe = _mm512_cmp_epu8_mask( //
3293-
_mm512_sub_epi8(data, x80_vec), _mm512_set1_epi8(0x04), _MM_CMPINT_LT);
3292+
__mmask64 safe = _mm512_cmplt_epu8_mask( //
3293+
_mm512_sub_epi8(data, x80_vec), _mm512_set1_epi8(0x04));
32943294
if (e2_sec & ~safe) return sz_false_k;
32953295
}
32963296

32973297
// EA: Bicameral second bytes 99-9F, AD-AE
32983298
__mmask64 is_ea = _mm512_cmpeq_epi8_mask(data, _mm512_set1_epi8((char)0xEA)) & is_three;
32993299
if (is_ea) {
33003300
__mmask64 ea_sec = is_ea << 1;
3301-
__mmask64 is_99 = _mm512_cmp_epu8_mask( //
3302-
_mm512_sub_epi8(data, _mm512_set1_epi8((char)0x99)), _mm512_set1_epi8(0x07), _MM_CMPINT_LT);
3303-
__mmask64 is_ad = _mm512_cmp_epu8_mask( //
3304-
_mm512_sub_epi8(data, _mm512_set1_epi8((char)0xAD)), _mm512_set1_epi8(0x02), _MM_CMPINT_LT);
3301+
__mmask64 is_99 = _mm512_cmplt_epu8_mask( //
3302+
_mm512_sub_epi8(data, _mm512_set1_epi8((char)0x99)), _mm512_set1_epi8(0x07));
3303+
__mmask64 is_ad = _mm512_cmplt_epu8_mask( //
3304+
_mm512_sub_epi8(data, _mm512_set1_epi8((char)0xAD)), _mm512_set1_epi8(0x02));
33053305
if (ea_sec & (is_99 | is_ad)) return sz_false_k;
33063306
}
33073307
}
@@ -4856,7 +4856,7 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_western_europe_( //
48564856
// - E2 84 AA: 'K' (U+212A Kelvin Sign) → 'k' (3 bytes → 1 byte)
48574857
// - EF AC 80-86: 'ff', 'fi', 'fl', 'ffi', 'ffl', 'ſt', 'st' (ligatures) → 2-3 bytes
48584858
// - C5 BF: 'ſ' (U+017F Long S) → 's' (2 bytes → 1 byte)
4859-
sz_u512_vec_t x_e1_vec, x_e2_vec, x_ef_vec, x_c5_vec;
4859+
sz_u512_vec_t x_e1_vec, x_e2_vec, x_ef_vec, x_c5_vec, x_ce_vec;
48604860
sz_u512_vec_t x_ba_vec, x_84_vec, x_ac_vec, x_bf_vec;
48614861
x_e1_vec.zmm = _mm512_set1_epi8((char)0xE1); // for 'ẞ'
48624862
x_e2_vec.zmm = _mm512_set1_epi8((char)0xE2); // for 'K'
@@ -4866,6 +4866,7 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_western_europe_( //
48664866
x_84_vec.zmm = _mm512_set1_epi8((char)0x84); // 2nd byte of 'K'
48674867
x_ac_vec.zmm = _mm512_set1_epi8((char)0xAC); // 2nd byte of ligatures
48684868
x_bf_vec.zmm = _mm512_set1_epi8((char)0xBF); // 2nd byte of 'ſ'
4869+
x_ce_vec.zmm = _mm512_set1_epi8((char)0xCE); // Greek uppercase (CE 9x folds to same target as µ)
48694870

48704871
sz_u512_vec_t haystack_vec;
48714872
sz_cptr_t haystack_ptr = haystack;
@@ -4896,8 +4897,9 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_western_europe_( //
48964897
__mmask64 x_e2_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_e2_vec.zmm);
48974898
__mmask64 x_ef_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_ef_vec.zmm);
48984899
__mmask64 x_c5_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_c5_vec.zmm);
4900+
__mmask64 x_ce_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_ce_vec.zmm);
48994901

4900-
if (x_e1_mask | x_e2_mask | x_ef_mask | x_c5_mask) {
4902+
if (x_e1_mask | x_e2_mask | x_ef_mask | x_c5_mask | x_ce_mask) {
49014903
__mmask64 x_ba_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_ba_vec.zmm);
49024904
__mmask64 x_84_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_84_vec.zmm);
49034905
__mmask64 x_ac_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_ac_vec.zmm);
@@ -4906,7 +4908,8 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_western_europe_( //
49064908
__mmask64 danger_mask = ((x_e1_mask << 1) & x_ba_mask) | // ẞ (E1 BA 9E)
49074909
((x_e2_mask << 1) & x_84_mask) | // K (E2 84 AA)
49084910
((x_ef_mask << 1) & x_ac_mask) | // fi, fl (EF AC xx)
4909-
((x_c5_mask << 1) & x_bf_mask); // ſ (C5 BF)
4911+
((x_c5_mask << 1) & x_bf_mask) | // ſ (C5 BF)
4912+
x_ce_mask; // Greek (CE xx) for µ→μ
49104913

49114914
if (danger_mask & valid_mask) {
49124915
sz_cptr_t match = sz_utf8_case_insensitive_find_in_danger_zone_( //
@@ -5127,7 +5130,7 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_central_europe_( //
51275130
((x_c5_mask << 1) & x_bf_mask) | // ſ
51285131
((x_ef_mask << 1) & x_ac_mask); // Ligatures
51295132

5130-
if (danger_mask & valid_mask) {
5133+
if (danger_mask) {
51315134
sz_cptr_t match = sz_utf8_case_insensitive_find_in_danger_zone_( //
51325135
haystack, haystack_length, needle, needle_length, //
51335136
haystack_ptr, valid_starts, //
@@ -5569,7 +5572,7 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_armenian_( //
55695572

55705573
__mmask64 danger_mask = x_ech_yiwn | x_ef_ac;
55715574

5572-
if ((danger_mask >> 1) & valid_mask) {
5575+
if (danger_mask) {
55735576
sz_cptr_t match = sz_utf8_case_insensitive_find_in_danger_zone_( //
55745577
haystack, haystack_length, needle, needle_length, //
55755578
haystack_ptr, valid_starts, //
@@ -5797,14 +5800,16 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_greek_( //
57975800
// - C2 B5: Micro Sign 'µ' -> 'μ' (CE BC)
57985801
// - E2 84 A6: Ohm Sign 'Ω' -> 'ω' (CF 89) (detect E2 84 prefix)
57995802
// - E1 xx xx: Polytonic Greek (excluded by safety profile, but just in case)
5800-
sz_u512_vec_t x_ce_vec, x_cf_vec, x_c2_vec, x_e2_vec;
5803+
sz_u512_vec_t x_ce_vec, x_cf_vec, x_c2_vec, x_e1_vec, x_e2_vec, x_cd_vec;
58015804
sz_u512_vec_t x_90_vec, x_b0_vec, x_84_vec;
58025805
sz_u512_vec_t x_91_vec, x_95_vec, x_96_vec, x_b1_vec, x_b5_vec;
58035806

58045807
x_ce_vec.zmm = _mm512_set1_epi8((char)0xCE);
58055808
x_cf_vec.zmm = _mm512_set1_epi8((char)0xCF);
58065809
x_c2_vec.zmm = _mm512_set1_epi8((char)0xC2);
5810+
x_e1_vec.zmm = _mm512_set1_epi8((char)0xE1); // E1 lead byte (Polytonic)
58075811
x_e2_vec.zmm = _mm512_set1_epi8((char)0xE2);
5812+
x_cd_vec.zmm = _mm512_set1_epi8((char)0xCD); // CD lead byte (Combining Diacritics)
58085813

58095814
x_90_vec.zmm = _mm512_set1_epi8((char)0x90);
58105815
x_b0_vec.zmm = _mm512_set1_epi8((char)0xB0);
@@ -5845,8 +5850,10 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_greek_( //
58455850
__mmask64 x_cf_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_cf_vec.zmm);
58465851
__mmask64 x_c2_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_c2_vec.zmm);
58475852
__mmask64 x_e2_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_e2_vec.zmm);
5853+
__mmask64 x_e1_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_e1_vec.zmm);
5854+
__mmask64 x_cd_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_cd_vec.zmm);
58485855

5849-
if (x_ce_mask | x_cf_mask | x_c2_mask | x_e2_mask) {
5856+
if (x_ce_mask | x_cf_mask | x_c2_mask | x_e2_mask | x_e1_mask | x_cd_mask) {
58505857
__mmask64 x_90_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_90_vec.zmm);
58515858
__mmask64 x_b0_mask = _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_b0_vec.zmm);
58525859
__mmask64 x_be_ce_90_b0 = (x_ce_mask << 1) & (x_90_mask | x_b0_mask); // CE 90 or CE B0
@@ -5864,10 +5871,11 @@ SZ_INTERNAL sz_cptr_t sz_utf8_case_insensitive_find_ice_greek_( //
58645871

58655872
__mmask64 x_micro = (x_c2_mask << 1) & x_bx_mask; // C2 B5 (B5 is in bx_mask)
58665873
__mmask64 x_ohm = (x_e2_mask << 1) & _mm512_cmpeq_epi8_mask(haystack_vec.zmm, x_84_vec.zmm);
5874+
// E1 is always dangerous in Greek path (Polytonic/Extensions)
5875+
// CD is dangerous (Combining Diacritical Marks)
5876+
__mmask64 danger_mask = x_be_ce_90_b0 | x_be_cf_sym | x_micro | x_ohm | x_e1_mask | x_cd_mask;
58675877

5868-
__mmask64 danger_mask = x_be_ce_90_b0 | x_be_cf_sym | x_micro | x_ohm;
5869-
5870-
if ((danger_mask >> 1) & valid_mask) {
5878+
if (danger_mask) {
58715879
sz_cptr_t match = sz_utf8_case_insensitive_find_in_danger_zone_( //
58725880
haystack, haystack_length, needle, needle_length, //
58735881
haystack_ptr, valid_starts, //

‎scripts/test_stringzilla.cpp‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3277,6 +3277,113 @@ void test_utf8_case() {
32773277
let_assert(auto m = str("HELLO").utf8_case_insensitive_find("hello"), m.offset == 0 && m.length == 5);
32783278
let_assert(auto m = str("Hello").utf8_case_insensitive_find("xyz"), m.offset == str::npos);
32793279
let_assert(auto m = str("Hello").utf8_case_insensitive_find(""), m.offset == 0 && m.length == 0);
3280+
3281+
// ==========================================================================
3282+
// Fuzz-Discovered Regressions (Serial vs SIMD mismatches)
3283+
// These patterns were discovered by test_utf8_ci_find_fuzz() and expose
3284+
// disagreements between serial and SIMD implementations.
3285+
// ==========================================================================
3286+
3287+
// Pattern 1: "st" + Latin-1 char (st ligature expansion issue?)
3288+
// Needle: 73 74 C2 BA = "st" + º (masculine ordinal indicator)
3289+
// Needle: 73 74 C3 B1 = "st" + ñ
3290+
// Needle: 73 74 C3 A5 = "st" + å
3291+
// Needle: 73 74 C3 A9 = "st" + é
3292+
// Needle: 73 74 D5 A2 = "st" + Armenian բ
3293+
// Needle: 73 74 CE B1 = "st" + Greek α
3294+
// These trigger kernel=2 (Central Europe) with safe_window issues
3295+
{
3296+
// "st" followed by º - should this match "st" ligature + º?
3297+
let_assert(auto m = str("test\xEF\xAC\x85\xC2\xBA"
3298+
"end")
3299+
.utf8_case_insensitive_find("st\xC2\xBA"),
3300+
m.offset == 4 && m.length == 5); // st ligature (3) + º (2)
3301+
3302+
// "st" followed by ñ
3303+
let_assert(auto m = str("test\xEF\xAC\x85\xC3\xB1"
3304+
"end")
3305+
.utf8_case_insensitive_find("st\xC3\xB1"),
3306+
m.offset == 4 && m.length == 5); // st ligature (3) + ñ (2)
3307+
3308+
// "st" followed by Greek α
3309+
let_assert(auto m = str("prefix\xEF\xAC\x85\xCE\xB1"
3310+
"suffix")
3311+
.utf8_case_insensitive_find("st\xCE\xB1"),
3312+
m.offset == 6 && m.length == 5); // st ligature (3) + α (2)
3313+
}
3314+
3315+
// Pattern 2: "ss" + Latin-1/Greek (Eszett expansion)
3316+
// Needle: 73 73 CE B1 = "ss" + Greek α
3317+
// Needle: 73 73 C3 A5 = "ss" + å
3318+
{
3319+
// "ss" followed by Greek α - should match ß + α
3320+
let_assert(auto m = str("test\xC3\x9F\xCE\xB1"
3321+
"end")
3322+
.utf8_case_insensitive_find("ss\xCE\xB1"),
3323+
m.offset == 4 && m.length == 4); // ß (2) + α (2)
3324+
3325+
// "ss" followed by å
3326+
let_assert(auto m = str("prefix\xC3\x9F\xC3\xA5"
3327+
"suffix")
3328+
.utf8_case_insensitive_find("ss\xC3\xA5"),
3329+
m.offset == 6 && m.length == 4); // ß (2) + å (2)
3330+
}
3331+
3332+
// Pattern 3: ASCII + combining diacritical + other char
3333+
// Needle: 68 CC B1 D5 A5 = "h" + combining macron below + Armenian ե
3334+
// Needle: 77 CC 8A CE B2 = "w" + combining ring above + Greek β
3335+
// Needle: 6A CC 8C D5 A2 = "j" + combining caron + Armenian բ
3336+
// These test one-to-many expansions (U+1E96 range) mixed with other scripts
3337+
{
3338+
// h + combining macron below should match ẖ (U+1E96)
3339+
let_assert(auto m = str("\xE1\xBA\x96\xD5\xA5").utf8_case_insensitive_find("h\xCC\xB1\xD5\xA5"),
3340+
m.offset == 0 && m.length == 5); // ẖ (3) + ե (2)
3341+
3342+
// w + combining ring above should match ẘ (U+1E98)
3343+
let_assert(auto m = str("\xE1\xBA\x98\xCE\xB2").utf8_case_insensitive_find("w\xCC\x8A\xCE\xB2"),
3344+
m.offset == 0 && m.length == 5); // ẘ (3) + β (2)
3345+
3346+
// j + combining caron should match ǰ (U+01F0)
3347+
let_assert(auto m = str("\xC7\xB0\xD5\xA2").utf8_case_insensitive_find("j\xCC\x8C\xD5\xA2"),
3348+
m.offset == 0 && m.length == 4); // ǰ (2) + բ (2)
3349+
}
3350+
3351+
// Pattern 4: Modifier letters + other chars
3352+
// Needle: CA BC 6E CE BC = modifier apostrophe + "n" + Greek μ
3353+
// Needle: 61 CA BE D5 A5 = "a" + modifier right half ring + Armenian ե
3354+
// These test n-apostrophe (U+0149) and a-right-half-ring (U+1E9A) expansions
3355+
{
3356+
// 'n (U+0149) expands to modifier apostrophe + n
3357+
let_assert(auto m = str("\xC5\x89\xCE\xBC")
3358+
.utf8_case_insensitive_find("\xCA\xBC"
3359+
"n\xCE\xBC"),
3360+
m.offset == 0 && m.length == 4); // ʼn (2) + μ (2)
3361+
3362+
// a + modifier right half ring should match ẚ (U+1E9A)
3363+
let_assert(auto m = str("\xE1\xBA\x9A\xD5\xA5").utf8_case_insensitive_find("a\xCA\xBE\xD5\xA5"),
3364+
m.offset == 0 && m.length == 5); // ẚ (3) + ե (2)
3365+
}
3366+
3367+
// Pattern 5: Armenian + combining chars / ligatures
3368+
// Needle: D5 A5 D6 82 CE B2 = Armenian delays + Greek β
3369+
{
3370+
// Armenian delays character followed by Greek
3371+
let_assert(auto m = str("\xD5\xA5\xD6\x82\xCE\xB2").utf8_case_insensitive_find("\xD5\xA5\xD6\x82\xCE\xB2"),
3372+
m.offset == 0 && m.length == 6);
3373+
}
3374+
3375+
// Pattern 6: Long complex needles crossing multiple scripts
3376+
// These stress test the kernel selection and danger zone handling
3377+
{
3378+
// Armenian barev + Latin ligatures + Vietnamese
3379+
std::string haystack = "\xD5\xA2\xD5\xA1\xD6\x80\xD5\xA5\xD5\xBE" // barev
3380+
"\xEF\xAC\x83" // ffi ligature
3381+
"\xE1\xBB\x87"; // Vietnamese ệ
3382+
std::string needle = "\xD5\xA2\xD5\xA1\xD6\x80\xD5\xA5\xD5\xBE" // barev
3383+
"ffi" // expanded
3384+
"\xE1\xBB\x86"; // Vietnamese Ệ
3385+
let_assert(auto m = str(haystack).utf8_case_insensitive_find(needle), m.offset == 0 && m.length == 16);
3386+
}
32803387
}
32813388

32823389
void test_utf8_words() {

0 commit comments

Comments
 (0)