Skip to content

Commit 91bc50b

Browse files
eshishkieshishki
committed
[UTF-8] Add 3-byte/4-byte switch cases and fix ß expansion in ICE
Serial (generate_uppercase.py): - Add 3-byte switch cases (Greek Extended, Latin Extended-D, etc.) - Add 4-byte switch cases (Vithkuqi) ICE/AVX-512 (generate_uppercase_ice.py): - Exclude ß (C3 9F) from Latin-1 SIMD block - must fall through to serial for SS expansion Tests: - test_case_comprehensive.c: 365 tests covering all offset/parity ranges and switch cases for both upper and fold - test_case_avx512.c: AVX-512 test with Serial vs ICE comparison and performance benchmark (2.2x speedup on Ice Lake)
1 parent 290d49f commit 91bc50b

5 files changed

Lines changed: 895 additions & 7 deletions

File tree

‎include/stringzilla/utf8_case.h‎

Lines changed: 71 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1455,8 +1455,66 @@ SZ_INTERNAL sz_size_t sz_unicode_upper_codepoint_(sz_rune_t rune, sz_rune_t *upp
14551455
if ((sz_u32_t)(rune - 0xA723) <= 0xC && is_odd_3) {
14561456
upper[0] = rune - 1; return 1; }
14571457

1458-
// Typographic ligatures → uppercase expansion
1458+
// 3-byte irregular 1:1 mappings (reversed)
14591459
switch (rune) {
1460+
case 0x1C8A: upper[0] = 0x1C89; return 1;
1461+
case 0x1D79: upper[0] = 0xA77D; return 1;
1462+
case 0x1D7D: upper[0] = 0x2C63; return 1;
1463+
case 0x1D8E: upper[0] = 0xA7C6; return 1;
1464+
case 0x1E61: upper[0] = 0x1E9B; return 1;
1465+
case 0x1F51: upper[0] = 0x1F59; return 1;
1466+
case 0x1F53: upper[0] = 0x1F5B; return 1;
1467+
case 0x1F55: upper[0] = 0x1F5D; return 1;
1468+
case 0x1F57: upper[0] = 0x1F5F; return 1;
1469+
case 0x1F70: upper[0] = 0x1FBA; return 1;
1470+
case 0x1F71: upper[0] = 0x1FBB; return 1;
1471+
case 0x1F76: upper[0] = 0x1FDA; return 1;
1472+
case 0x1F77: upper[0] = 0x1FDB; return 1;
1473+
case 0x1F78: upper[0] = 0x1FF8; return 1;
1474+
case 0x1F79: upper[0] = 0x1FF9; return 1;
1475+
case 0x1F7A: upper[0] = 0x1FEA; return 1;
1476+
case 0x1F7B: upper[0] = 0x1FEB; return 1;
1477+
case 0x1F7C: upper[0] = 0x1FFA; return 1;
1478+
case 0x1F7D: upper[0] = 0x1FFB; return 1;
1479+
case 0x1FB0: upper[0] = 0x1FB8; return 1;
1480+
case 0x1FB1: upper[0] = 0x1FB9; return 1;
1481+
case 0x1FD0: upper[0] = 0x1FD8; return 1;
1482+
case 0x1FD1: upper[0] = 0x1FD9; return 1;
1483+
case 0x1FE0: upper[0] = 0x1FE8; return 1;
1484+
case 0x1FE1: upper[0] = 0x1FE9; return 1;
1485+
case 0x1FE5: upper[0] = 0x1FEC; return 1;
1486+
case 0x214E: upper[0] = 0x2132; return 1;
1487+
case 0x2184: upper[0] = 0x2183; return 1;
1488+
case 0x2C61: upper[0] = 0x2C60; return 1;
1489+
case 0x2C65: upper[0] = 0x023A; return 1;
1490+
case 0x2C66: upper[0] = 0x023E; return 1;
1491+
case 0x2C68: upper[0] = 0x2C67; return 1;
1492+
case 0x2C6A: upper[0] = 0x2C69; return 1;
1493+
case 0x2C6C: upper[0] = 0x2C6B; return 1;
1494+
case 0x2C73: upper[0] = 0x2C72; return 1;
1495+
case 0x2C76: upper[0] = 0x2C75; return 1;
1496+
case 0x2CEC: upper[0] = 0x2CEB; return 1;
1497+
case 0x2CEE: upper[0] = 0x2CED; return 1;
1498+
case 0x2CF3: upper[0] = 0x2CF2; return 1;
1499+
case 0x2D27: upper[0] = 0x10C7; return 1;
1500+
case 0x2D2D: upper[0] = 0x10CD; return 1;
1501+
case 0xA77A: upper[0] = 0xA779; return 1;
1502+
case 0xA77C: upper[0] = 0xA77B; return 1;
1503+
case 0xA78C: upper[0] = 0xA78B; return 1;
1504+
case 0xA794: upper[0] = 0xA7C4; return 1;
1505+
case 0xA7C8: upper[0] = 0xA7C7; return 1;
1506+
case 0xA7CA: upper[0] = 0xA7C9; return 1;
1507+
case 0xA7CD: upper[0] = 0xA7CC; return 1;
1508+
case 0xA7CF: upper[0] = 0xA7CE; return 1;
1509+
case 0xA7D1: upper[0] = 0xA7D0; return 1;
1510+
case 0xA7D3: upper[0] = 0xA7D2; return 1;
1511+
case 0xA7D5: upper[0] = 0xA7D4; return 1;
1512+
case 0xA7D7: upper[0] = 0xA7D6; return 1;
1513+
case 0xA7D9: upper[0] = 0xA7D8; return 1;
1514+
case 0xA7DB: upper[0] = 0xA7DA; return 1;
1515+
case 0xA7F6: upper[0] = 0xA7F5; return 1;
1516+
case 0xAB53: upper[0] = 0xA7B3; return 1;
1517+
// Typographic ligatures → uppercase expansion
14601518
case 0xFB00: upper[0] = 0x0046; upper[1] = 0x0046; return 2; // ff → FF
14611519
case 0xFB01: upper[0] = 0x0046; upper[1] = 0x0049; return 2; // fi → FI
14621520
case 0xFB02: upper[0] = 0x0046; upper[1] = 0x004C; return 2; // fl → FL
@@ -1506,6 +1564,12 @@ SZ_INTERNAL sz_size_t sz_unicode_upper_codepoint_(sz_rune_t rune, sz_rune_t *upp
15061564
if ((sz_u32_t)(rune - 0x1E922) <= 0x21) {
15071565
upper[0] = rune - 0x22; return 1; }
15081566

1567+
// 4-byte irregular 1:1 mappings (reversed)
1568+
switch (rune) {
1569+
case 0x105BB: upper[0] = 0x10594; return 1;
1570+
case 0x105BC: upper[0] = 0x10595; return 1;
1571+
}
1572+
15091573
upper[0] = rune; return 1;
15101574

15111575
// clang-format on
@@ -4072,9 +4136,6 @@ SZ_PUBLIC sz_size_t sz_utf8_case_fold_ice(sz_cptr_t source, sz_size_t source_len
40724136
#undef sz_ice_fold_ascii_in_prefix_
40734137
#undef sz_ice_transform_georgian_
40744138

4075-
// Generated by scripts/generate_uppercase_ice.py
4076-
// Inverse of sz_utf8_case_fold_ice - converts lowercase to uppercase using AVX-512
4077-
40784139
SZ_PUBLIC sz_size_t sz_utf8_case_upper_ice(sz_cptr_t source, sz_size_t source_length, sz_ptr_t target) {
40794140
// This is the inverse of sz_utf8_case_fold_ice - converts lowercase to uppercase.
40804141
// Generated by scripts/generate_uppercase_ice.py
@@ -4173,10 +4234,14 @@ SZ_PUBLIC sz_size_t sz_utf8_case_upper_ice(sz_cptr_t source, sz_size_t source_le
41734234
}
41744235
}
41754236

4176-
// Latin-1 Supplement (C3): à-þ → À-Þ
4237+
// Latin-1 Supplement (C3): à-þ → À-Þ (excluding ß=0x9F which expands to SS)
41774238
__mmask64 is_latin1_lead = _mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0xC3));
41784239
__mmask64 latin1_second_byte_positions = is_latin1_lead << 1;
4240+
// Exclude ß (C3 9F) - it expands to SS and must go through fallback
4241+
__mmask64 is_sharp_s = latin1_second_byte_positions &
4242+
_mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0x9F));
41794243
__mmask64 is_valid_latin1_mix = ~is_non_ascii | is_latin1_lead | latin1_second_byte_positions;
4244+
is_valid_latin1_mix &= ~is_sharp_s; // Stop before ß
41804245
sz_size_t latin1_length = sz_ice_first_invalid_(is_valid_latin1_mix, load_mask, chunk_size);
41814246
latin1_length -= latin1_length && ((is_latin1_lead >> (latin1_length - 1)) & 1);
41824247

@@ -4647,6 +4712,7 @@ SZ_PUBLIC sz_size_t sz_utf8_case_upper_ice(sz_cptr_t source, sz_size_t source_le
46474712
}
46484713

46494714

4715+
46504716
SZ_PUBLIC sz_bool_t sz_utf8_case_invariant_ice(sz_cptr_t str, sz_size_t length) {
46514717
sz_u8_t const *ptr = (sz_u8_t const *)str;
46524718

‎scripts/generate_uppercase.py‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -372,9 +372,29 @@ def generate_uppercase(offset_ranges: List[OffsetRange],
372372
lines.append(f" upper[0] = rune - 1; return 1; }}")
373373
lines.append("")
374374

375+
# 3-byte switch cases (reversed 1:1 mappings)
376+
lines.append(" // 3-byte irregular 1:1 mappings (reversed)")
377+
lines.append(" switch (rune) {")
378+
379+
# Build reverse map for 3-byte codepoints
380+
reverse_map_3byte: Dict[int, int] = {}
381+
for c in switch_cases:
382+
if c.source in non_reversible_sources:
383+
continue
384+
if len(c.dest) != 1:
385+
continue
386+
dest = c.dest[0]
387+
# Only 3-byte destinations (0x0800-0xFFFF)
388+
if dest <= 0x7FF or dest > 0xFFFF:
389+
continue
390+
if dest not in reverse_map_3byte:
391+
reverse_map_3byte[dest] = c.source
392+
393+
for lower, upper in sorted(reverse_map_3byte.items()):
394+
lines.append(f" case 0x{lower:04X}: upper[0] = 0x{upper:04X}; return 1;")
395+
375396
# Typographic ligatures (uppercase-specific)
376397
lines.append(" // Typographic ligatures → uppercase expansion")
377-
lines.append(" switch (rune) {")
378398
lines.append(" case 0xFB00: upper[0] = 0x0046; upper[1] = 0x0046; return 2; // ff → FF")
379399
lines.append(" case 0xFB01: upper[0] = 0x0046; upper[1] = 0x0049; return 2; // fi → FI")
380400
lines.append(" case 0xFB02: upper[0] = 0x0046; upper[1] = 0x004C; return 2; // fl → FL")
@@ -416,6 +436,28 @@ def generate_uppercase(offset_ranges: List[OffsetRange],
416436
if not has_4byte:
417437
lines.append(" // No 4-byte uppercase ranges")
418438

439+
# 4-byte switch cases (reversed 1:1 mappings)
440+
reverse_map_4byte: Dict[int, int] = {}
441+
for c in switch_cases:
442+
if c.source in non_reversible_sources:
443+
continue
444+
if len(c.dest) != 1:
445+
continue
446+
dest = c.dest[0]
447+
# Only 4-byte destinations (> 0xFFFF)
448+
if dest <= 0xFFFF:
449+
continue
450+
if dest not in reverse_map_4byte:
451+
reverse_map_4byte[dest] = c.source
452+
453+
if reverse_map_4byte:
454+
lines.append(" // 4-byte irregular 1:1 mappings (reversed)")
455+
lines.append(" switch (rune) {")
456+
for lower, upper in sorted(reverse_map_4byte.items()):
457+
lines.append(f" case 0x{lower:05X}: upper[0] = 0x{upper:05X}; return 1;")
458+
lines.append(" }")
459+
lines.append("")
460+
419461
lines.append(" upper[0] = rune; return 1;")
420462
lines.append("")
421463
lines.append(" // clang-format on")

‎scripts/generate_uppercase_ice.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,10 +122,14 @@ def generate_ice_upper():
122122
}
123123
}
124124
125-
// Latin-1 Supplement (C3): à-þ → À-Þ
125+
// Latin-1 Supplement (C3): à-þ → À-Þ (excluding ß=0x9F which expands to SS)
126126
__mmask64 is_latin1_lead = _mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0xC3));
127127
__mmask64 latin1_second_byte_positions = is_latin1_lead << 1;
128+
// Exclude ß (C3 9F) - it expands to SS and must go through fallback
129+
__mmask64 is_sharp_s = latin1_second_byte_positions &
130+
_mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0x9F));
128131
__mmask64 is_valid_latin1_mix = ~is_non_ascii | is_latin1_lead | latin1_second_byte_positions;
132+
is_valid_latin1_mix &= ~is_sharp_s; // Stop before ß
129133
sz_size_t latin1_length = sz_ice_first_invalid_(is_valid_latin1_mix, load_mask, chunk_size);
130134
latin1_length -= latin1_length && ((is_latin1_lead >> (latin1_length - 1)) & 1);
131135

0 commit comments

Comments
 (0)