Skip to content

Commit 9513e4d

Browse files
eshishkieshishki
committed
Ice upper: merge four Cyrillic Supplement branches into one
Replaces four separate Ice upper branches (D2, D3, D1-ext, D4) with a single combined branch that detects all four lead bytes at once and applies the unified "-1 (or -0xF) on the second byte" transform in one mask_sub pass. Matches the structure already used in upper_haswell. The old setup cut the chunk at the first non-matching lead byte each time, so interleaved D2/D3/D4 text (the realistic shape of Caucasian Cyrillic Supplement content) advanced 2-4 bytes per iteration and the per-iteration pre-checks dominated. The combined branch processes the whole chunk in one pass. Benchmarked on Intel Xeon Platinum 8375C (32 MiB, best of 7): Mixed Cyrillic Suppl upper (D2/D3/D4 interleaved): before: serial 0.17 → ice 0.21 GB/s (×1.3) after : serial 0.11 → ice 3.23 GB/s (×29.4) Pure D2 (no interleaving): before: ice 3.50 GB/s after : ice 2.69 GB/s (~25% regression — extra lead-mask ops trade off against the huge mixed-pattern win; pure single-lead Suppl content is rare in real text) Verified bit-perfect haswell-vs-serial and ice-vs-serial across all 1.1M codepoints — 0 diffs.
1 parent c6fd3a7 commit 9513e4d

1 file changed

Lines changed: 75 additions & 160 deletions

File tree

‎include/stringzilla/utf8_case.h‎

Lines changed: 75 additions & 160 deletions
Original file line numberDiff line numberDiff line change
@@ -5848,176 +5848,91 @@ SZ_PUBLIC sz_size_t sz_utf8_case_upper_ice(sz_cptr_t source, sz_size_t source_le
58485848
}
58495849
}
58505850

5851-
// Cyrillic Supplement (D2): lowercase → uppercase (parity-based)
5852-
// U+0480-04BF = D2 80-BF. Lowercase letters: D2 81 (ҁ) plus odd second byte in 0x8B-0xBF.
5853-
// Bytes 0x82 (Thousands Sign) and 0x83-0x89 (combining marks) have no case mapping.
5854-
// Examples: ҳ (D2 B3) → Ҳ (D2 B2), ҷ (D2 B7) → Ҷ (D2 B6), ҁ (D2 81) → Ҁ (D2 80).
5851+
// Cyrillic Supplement / Extended (D1 A0-BF, D2, D3, D4): lowercase → uppercase.
5852+
// D1 A0-BF odd → -1 (Cyrillic Supplement extended, U+0460-047F)
5853+
// D2 81 → -1 (irregular ҁ → Ҁ)
5854+
// D2 8B-BF odd → -1 (parity, U+0480-04BF)
5855+
// D3 82-8E even → -1 (parity, range 1)
5856+
// D3 8F → -0xF (irregular ӏ → Ӏ, U+04CF → U+04C0)
5857+
// D3 91-BF odd → -1 (parity, range 2: U+04D1-04FF)
5858+
// D4 81-AF odd → -1 (parity, U+0501-052F)
5859+
// Combined into one branch since all four leads share the D-prefix structure and the
5860+
// transform is uniformly "-1 (or -0xF) on the second byte". Single mask computation
5861+
// processes mixed Cyrillic Supplement text in one chunk rather than four small advances.
58555862
{
5863+
__mmask64 is_d1_mask = _mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0xD1));
58565864
__mmask64 is_d2_mask = _mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0xD2));
5857-
__mmask64 d2_second_byte_positions = is_d2_mask << 1;
5858-
5859-
__mmask64 is_valid_d2_mix_mask = ~is_non_ascii | is_d2_mask | d2_second_byte_positions;
5860-
sz_size_t d2_length = sz_ice_first_invalid_(is_valid_d2_mix_mask, load_mask, chunk_size);
5861-
d2_length -= d2_length && ((is_d2_mask >> (d2_length - 1)) & 1);
5862-
5863-
if (d2_length >= 2) {
5864-
__mmask64 prefix_mask = sz_u64_mask_until_(d2_length);
5865-
__mmask64 is_after_d2_mask = (is_d2_mask << 1) & prefix_mask;
5866-
5867-
// ASCII uppercasing
5868-
__mmask64 is_lower_ascii = _mm512_cmplt_epu8_mask(
5869-
_mm512_sub_epi8(source_vec.zmm, a_lower_vec), subtract26_vec);
5870-
__m512i upper = _mm512_mask_sub_epi8(source_vec.zmm, is_lower_ascii & prefix_mask,
5871-
source_vec.zmm, ascii_case_offset);
5872-
5873-
// Odd second byte in 0x8B..0xBF (skips combining marks 0x83..0x89, even-only symbols)
5874-
__mmask64 is_in_range = _mm512_mask_cmpge_epu8_mask(is_after_d2_mask, source_vec.zmm,
5875-
_mm512_set1_epi8((char)0x8B));
5876-
is_in_range &= _mm512_mask_cmple_epu8_mask(is_after_d2_mask, source_vec.zmm,
5877-
_mm512_set1_epi8((char)0xBF));
5878-
__m512i byte_and_1 = _mm512_and_si512(source_vec.zmm, _mm512_set1_epi8(1));
5879-
__mmask64 is_odd = _mm512_cmpeq_epi8_mask(byte_and_1, _mm512_set1_epi8(1));
5880-
__mmask64 is_d2_lower = is_in_range & is_odd;
5881-
// Plus the irregular ҁ at D2 81
5882-
is_d2_lower |= is_after_d2_mask &
5883-
_mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0x81));
5884-
5885-
upper = _mm512_mask_sub_epi8(upper, is_d2_lower, upper, _mm512_set1_epi8(1));
5886-
5887-
_mm512_mask_storeu_epi8(target, prefix_mask, upper);
5888-
target += d2_length, source += d2_length, source_length -= d2_length;
5889-
continue;
5890-
}
5891-
}
5892-
5893-
// Cyrillic Supplement (D3): lowercase → uppercase
5894-
// U+04C0-04FF = D3 80-BF. Three patterns:
5895-
// - 0x04C2-0x04CE (D3 82-8E): even second byte is lowercase → subtract 1
5896-
// - 0x04CF (D3 8F): irregular — ӏ → Ӏ (U+04C0), subtract 0x0F
5897-
// - 0x04D1-0x04FF (D3 91-BF): odd second byte is lowercase → subtract 1
5898-
// Examples: ӣ (D3 A3) → Ӣ (D3 A2), ӎ (D3 8E) → Ӎ (D3 8D), ӏ (D3 8F) → Ӏ (D3 80)
5899-
{
59005865
__mmask64 is_d3_mask = _mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0xD3));
5901-
__mmask64 d3_second_byte_positions = is_d3_mask << 1;
5902-
5903-
__mmask64 is_valid_d3_mix_mask = ~is_non_ascii | is_d3_mask | d3_second_byte_positions;
5904-
sz_size_t d3_length = sz_ice_first_invalid_(is_valid_d3_mix_mask, load_mask, chunk_size);
5905-
d3_length -= d3_length && ((is_d3_mask >> (d3_length - 1)) & 1);
5906-
5907-
if (d3_length >= 2) {
5908-
__mmask64 prefix_mask = sz_u64_mask_until_(d3_length);
5909-
__mmask64 is_after_d3_mask = (is_d3_mask << 1) & prefix_mask;
5910-
5911-
// ASCII uppercasing
5912-
__mmask64 is_lower_ascii = _mm512_cmplt_epu8_mask(
5913-
_mm512_sub_epi8(source_vec.zmm, a_lower_vec), subtract26_vec);
5914-
__m512i upper = _mm512_mask_sub_epi8(source_vec.zmm, is_lower_ascii & prefix_mask,
5915-
source_vec.zmm, ascii_case_offset);
5916-
5917-
__m512i byte_and_1 = _mm512_and_si512(source_vec.zmm, _mm512_set1_epi8(1));
5918-
__mmask64 is_odd = _mm512_cmpeq_epi8_mask(byte_and_1, _mm512_set1_epi8(1));
5919-
5920-
// Range 1: D3 82-8E (0x04C2-0x04CE) — even is lowercase
5921-
__mmask64 is_range1 = _mm512_mask_cmpge_epu8_mask(is_after_d3_mask, source_vec.zmm,
5922-
_mm512_set1_epi8((char)0x82));
5923-
is_range1 &= _mm512_mask_cmple_epu8_mask(is_after_d3_mask, source_vec.zmm,
5924-
_mm512_set1_epi8((char)0x8E));
5925-
__mmask64 is_d3_lower1 = is_range1 & ~is_odd;
5926-
5927-
// Range 2: D3 91-BF (0x04D1-0x04FF) — odd is lowercase
5928-
__mmask64 is_range2 = _mm512_mask_cmpge_epu8_mask(is_after_d3_mask, source_vec.zmm,
5929-
_mm512_set1_epi8((char)0x91));
5930-
is_range2 &= _mm512_mask_cmple_epu8_mask(is_after_d3_mask, source_vec.zmm,
5931-
_mm512_set1_epi8((char)0xBF));
5932-
__mmask64 is_d3_lower2 = is_range2 & is_odd;
5933-
5934-
upper = _mm512_mask_sub_epi8(upper, is_d3_lower1 | is_d3_lower2, upper, _mm512_set1_epi8(1));
5935-
5936-
// Irregular: D3 8F (U+04CF) → D3 80 (U+04C0), subtract 0x0F from the second byte
5937-
__mmask64 is_d3_8f = is_after_d3_mask &
5938-
_mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0x8F));
5939-
upper = _mm512_mask_sub_epi8(upper, is_d3_8f, upper, _mm512_set1_epi8(0x0F));
5940-
5941-
_mm512_mask_storeu_epi8(target, prefix_mask, upper);
5942-
target += d3_length, source += d3_length, source_length -= d3_length;
5943-
continue;
5944-
}
5945-
}
5946-
5947-
// Cyrillic Extended D1 A0+ (0x0460-0x047F): lowercase → uppercase (parity-based)
5948-
// D1 A0-BF, odd second byte is lowercase → subtract 1
5949-
{
5950-
__mmask64 is_d1_mask = _mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0xD1));
5951-
__mmask64 d1_second_byte_positions = is_d1_mask << 1;
5952-
// Only D1 A0+ (Extended Cyrillic)
5953-
__mmask64 is_d1_extended = d1_second_byte_positions &
5954-
_mm512_cmpge_epu8_mask(source_vec.zmm, _mm512_set1_epi8((char)0xA0));
5955-
5956-
__mmask64 is_valid_d1ext_mix_mask = ~is_non_ascii | is_d1_mask | d1_second_byte_positions;
5957-
// Exclude non-extended D1 (D1 80-9F)
5958-
__mmask64 is_d1_basic = d1_second_byte_positions &
5959-
_mm512_cmplt_epu8_mask(source_vec.zmm, _mm512_set1_epi8((char)0xA0));
5960-
is_valid_d1ext_mix_mask &= ~is_d1_basic;
5961-
5962-
sz_size_t d1ext_length = sz_ice_first_invalid_(is_valid_d1ext_mix_mask, load_mask, chunk_size);
5963-
d1ext_length -= d1ext_length && ((is_d1_mask >> (d1ext_length - 1)) & 1);
5964-
5965-
if (d1ext_length >= 2) {
5966-
__mmask64 prefix_mask = sz_u64_mask_until_(d1ext_length);
5967-
5968-
// ASCII uppercasing
5969-
__mmask64 is_lower_ascii = _mm512_cmplt_epu8_mask(
5970-
_mm512_sub_epi8(source_vec.zmm, a_lower_vec), subtract26_vec);
5971-
__m512i upper = _mm512_mask_sub_epi8(source_vec.zmm, is_lower_ascii & prefix_mask,
5972-
source_vec.zmm, ascii_case_offset);
5973-
5974-
// Parity: odd second byte (A1,A3,...,BF) → subtract 1
5975-
__m512i byte_and_1 = _mm512_and_si512(source_vec.zmm, _mm512_set1_epi8(1));
5976-
__mmask64 is_odd = _mm512_cmpeq_epi8_mask(byte_and_1, _mm512_set1_epi8(1));
5977-
__mmask64 is_d1ext_lower = is_d1_extended & prefix_mask & is_odd;
5978-
5979-
upper = _mm512_mask_sub_epi8(upper, is_d1ext_lower, upper, _mm512_set1_epi8(1));
5980-
5981-
_mm512_mask_storeu_epi8(target, prefix_mask, upper);
5982-
target += d1ext_length, source += d1ext_length, source_length -= d1ext_length;
5983-
continue;
5984-
}
5985-
}
5986-
5987-
// Cyrillic Extended D4 (0x0500-0x052E): lowercase → uppercase (parity-based)
5988-
// D4 80-AE, odd second byte is lowercase → subtract 1
5989-
{
59905866
__mmask64 is_d4_mask = _mm512_cmpeq_epi8_mask(source_vec.zmm, _mm512_set1_epi8((char)0xD4));
5991-
__mmask64 d4_second_byte_positions = is_d4_mask << 1;
5867+
__mmask64 is_sup_lead = is_d1_mask | is_d2_mask | is_d3_mask | is_d4_mask;
5868+
if (is_sup_lead) {
5869+
__mmask64 sup_second = is_sup_lead << 1;
5870+
// Exclude D1 80-9F (basic Cyrillic, handled above) from this branch's valid set.
5871+
__mmask64 is_d1_basic = (is_d1_mask << 1) &
5872+
_mm512_cmplt_epu8_mask(source_vec.zmm, _mm512_set1_epi8((char)0xA0));
5873+
__mmask64 is_valid_sup = (~is_non_ascii | is_sup_lead | sup_second) & ~is_d1_basic;
5874+
sz_size_t sup_length = sz_ice_first_invalid_(is_valid_sup, load_mask, chunk_size);
5875+
sup_length -= sup_length && ((is_sup_lead >> (sup_length - 1)) & 1);
5876+
5877+
if (sup_length >= 2) {
5878+
__mmask64 prefix_mask = sz_u64_mask_until_(sup_length);
5879+
__mmask64 is_after_d1 = (is_d1_mask << 1) & prefix_mask;
5880+
__mmask64 is_after_d2 = (is_d2_mask << 1) & prefix_mask;
5881+
__mmask64 is_after_d3 = (is_d3_mask << 1) & prefix_mask;
5882+
__mmask64 is_after_d4 = (is_d4_mask << 1) & prefix_mask;
59925883

5993-
__mmask64 is_valid_d4_mix_mask = ~is_non_ascii | is_d4_mask | d4_second_byte_positions;
5994-
sz_size_t d4_length = sz_ice_first_invalid_(is_valid_d4_mix_mask, load_mask, chunk_size);
5995-
d4_length -= d4_length && ((is_d4_mask >> (d4_length - 1)) & 1);
5884+
__mmask64 is_lower_ascii = _mm512_cmplt_epu8_mask(
5885+
_mm512_sub_epi8(source_vec.zmm, a_lower_vec), subtract26_vec);
5886+
__m512i upper = _mm512_mask_sub_epi8(source_vec.zmm, is_lower_ascii & prefix_mask,
5887+
source_vec.zmm, ascii_case_offset);
59965888

5997-
if (d4_length >= 2) {
5998-
__mmask64 prefix_mask = sz_u64_mask_until_(d4_length);
5999-
__mmask64 is_after_d4_mask = (is_d4_mask << 1) & prefix_mask;
5889+
__m512i byte_and_1 = _mm512_and_si512(source_vec.zmm, _mm512_set1_epi8(1));
5890+
__mmask64 is_odd = _mm512_cmpeq_epi8_mask(byte_and_1, _mm512_set1_epi8(1));
5891+
__mmask64 is_even = _mm512_cmpeq_epi8_mask(byte_and_1, _mm512_setzero_si512());
60005892

6001-
// ASCII uppercasing
6002-
__mmask64 is_lower_ascii = _mm512_cmplt_epu8_mask(
6003-
_mm512_sub_epi8(source_vec.zmm, a_lower_vec), subtract26_vec);
6004-
__m512i upper = _mm512_mask_sub_epi8(source_vec.zmm, is_lower_ascii & prefix_mask,
6005-
source_vec.zmm, ascii_case_offset);
5893+
// D1 A0-BF odd → -1
5894+
__mmask64 d1_ext_lo = _mm512_mask_cmpge_epu8_mask(is_after_d1, source_vec.zmm,
5895+
_mm512_set1_epi8((char)0xA0));
5896+
d1_ext_lo &= is_odd;
60065897

6007-
// Range D4 81-AF (0x0501-0x052F): odd second byte is lowercase
6008-
__mmask64 is_in_range = _mm512_mask_cmpge_epu8_mask(is_after_d4_mask, source_vec.zmm,
6009-
_mm512_set1_epi8((char)0x81));
6010-
is_in_range &= _mm512_mask_cmple_epu8_mask(is_after_d4_mask, source_vec.zmm,
6011-
_mm512_set1_epi8((char)0xAF));
6012-
__m512i byte_and_1 = _mm512_and_si512(source_vec.zmm, _mm512_set1_epi8(1));
6013-
__mmask64 is_odd = _mm512_cmpeq_epi8_mask(byte_and_1, _mm512_set1_epi8(1));
6014-
__mmask64 is_d4_lower = is_in_range & is_odd;
5898+
// D2 81 irregular + D2 8B-BF odd
5899+
__mmask64 d2_81 = _mm512_mask_cmpeq_epi8_mask(is_after_d2, source_vec.zmm,
5900+
_mm512_set1_epi8((char)0x81));
5901+
__mmask64 d2_par = _mm512_mask_cmpge_epu8_mask(is_after_d2, source_vec.zmm,
5902+
_mm512_set1_epi8((char)0x8B));
5903+
d2_par &= _mm512_mask_cmple_epu8_mask(d2_par, source_vec.zmm,
5904+
_mm512_set1_epi8((char)0xBF));
5905+
d2_par &= is_odd;
5906+
__mmask64 d2_lo = d2_81 | d2_par;
60155907

6016-
upper = _mm512_mask_sub_epi8(upper, is_d4_lower, upper, _mm512_set1_epi8(1));
5908+
// D3 82-8E even (range 1) + D3 91-BF odd (range 2)
5909+
__mmask64 d3_r1 = _mm512_mask_cmpge_epu8_mask(is_after_d3, source_vec.zmm,
5910+
_mm512_set1_epi8((char)0x82));
5911+
d3_r1 &= _mm512_mask_cmple_epu8_mask(d3_r1, source_vec.zmm,
5912+
_mm512_set1_epi8((char)0x8E));
5913+
d3_r1 &= is_even;
5914+
__mmask64 d3_r2 = _mm512_mask_cmpge_epu8_mask(is_after_d3, source_vec.zmm,
5915+
_mm512_set1_epi8((char)0x91));
5916+
d3_r2 &= _mm512_mask_cmple_epu8_mask(d3_r2, source_vec.zmm,
5917+
_mm512_set1_epi8((char)0xBF));
5918+
d3_r2 &= is_odd;
5919+
__mmask64 d3_lo = d3_r1 | d3_r2;
5920+
__mmask64 d3_8f = _mm512_mask_cmpeq_epi8_mask(is_after_d3, source_vec.zmm,
5921+
_mm512_set1_epi8((char)0x8F));
5922+
5923+
// D4 81-AF odd → -1
5924+
__mmask64 d4_lo = _mm512_mask_cmple_epu8_mask(is_after_d4, source_vec.zmm,
5925+
_mm512_set1_epi8((char)0xAF));
5926+
d4_lo &= is_odd;
5927+
5928+
__mmask64 sub1_mask = d1_ext_lo | d2_lo | d3_lo | d4_lo;
5929+
upper = _mm512_mask_sub_epi8(upper, sub1_mask, upper, _mm512_set1_epi8(1));
5930+
upper = _mm512_mask_sub_epi8(upper, d3_8f, upper, _mm512_set1_epi8(0x0F));
60175931

6018-
_mm512_mask_storeu_epi8(target, prefix_mask, upper);
6019-
target += d4_length, source += d4_length, source_length -= d4_length;
6020-
continue;
5932+
_mm512_mask_storeu_epi8(target, prefix_mask, upper);
5933+
target += sup_length, source += sup_length, source_length -= sup_length;
5934+
continue;
5935+
}
60215936
}
60225937
}
60235938

0 commit comments

Comments
 (0)