perf(unicode): index common combining classes

Author: gaborbernat

docs/changelog/799.feature.24.rst

@@ -0,0 +1 @@
+Use direct combining-class lookup for common marks in :func:`~turbohtml.detect.normalize`.

src/turbohtml/_c/data/normalize_table.h

@@ -19,6 +19,18 @@ typedef struct {
 } th_norm_ccc_row;
 
 static const int th_norm_ccc_count = 934;
+static const uint8_t th_norm_ccc_dense[] = {
+    230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 232,
+    220, 220, 220, 220, 232, 216, 220, 220, 220, 220, 220, 202, 202, 220, 220, 220, 220, 202, 202, 220, 220, 220,
+    220, 220, 220, 220, 220, 220, 220, 220, 1, 1, 1, 1, 1, 220, 220, 220, 220, 230, 230, 230, 230, 230, 230, 230,
+    230, 240, 230, 220, 220, 220, 230, 230, 230, 220, 220, 0, 230, 230, 230, 220, 220, 220, 220, 230, 232, 220, 220,
+    230, 233, 234, 234, 233, 234, 234, 233, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
 static const th_norm_ccc_row th_norm_ccc[] = {
     {0x300, 230}, {0x301, 230}, {0x302, 230}, {0x303, 230}, {0x304, 230}, {0x305, 230}, {0x306, 230}, {0x307, 230},
     {0x308, 230}, {0x309, 230}, {0x30A, 230}, {0x30B, 230}, {0x30C, 230}, {0x30D, 230}, {0x30E, 230}, {0x30F, 230},

src/turbohtml/_c/unicode/normalize.c

@@ -35,6 +35,10 @@ static uint8_t ccc_of(Py_UCS4 cp) {
     if (cp < th_norm_ccc[0].code) {
         return 0;
     }
+    Py_UCS4 offset = cp - th_norm_ccc[0].code;
+    if (offset < sizeof(th_norm_ccc_dense)) {
+        return th_norm_ccc_dense[offset];
+    }
     int lo = 0;
     int hi = th_norm_ccc_count;
     while (lo < hi) {

tools/bench/ci.py

@@ -426,6 +426,9 @@ def _book_html() -> str:
     "select-default-many-forms": ("select-default", 3),
     "select-default-tag-control": ("select-default", 4),
     "select-default-explicit-submit": ("select-default", 5),
+    "normalize-marks-large": ("normalize-marks", 2),
+    "normalize-marks-small": ("normalize-marks", 0),
+    "normalize-hebrew-marks": ("normalize", 8),
 }
 
 

tools/bench/competitors/unicodedata.py

@@ -12,4 +12,7 @@ def normalize(text: str) -> None:
     unicodedata.normalize("NFC", text)
 
 
-OPERATIONS = {"normalize": (normalize, "unicodedata")}
+OPERATIONS = {
+    "normalize": (normalize, "unicodedata"),
+    "normalize-marks": (normalize, "unicodedata"),
+}

tools/bench/operations.py

@@ -1471,6 +1471,7 @@ def _normalize_cases() -> tuple[tuple[str, object], ...]:
         ("small ASCII prefix, composable suffix", "abc e\u0301"),
         ("4,096 composable combining pairs", "e\u0301" * 4096),
         ("4,096 Hangul jamo triples", "\u1100\u1161\u11a8" * 4096),
+        ("10,000 Hebrew combining marks", "\u05d0" + "\u05b1" * 5000 + "\u05b0" * 5000),
     )
 
 

tools/generate_normalize.py

@@ -285,6 +285,7 @@ def _emit(
     canon_text = _decomp_block("canon", canonical)
     compat_text = _decomp_block("compat", compatibility)
     max_expansion = max(len(run) for run in (*canonical.values(), *compatibility.values()))
+    ccc_dense: Final = _wrap(str(combining.get(code, 0)) for code in range(ccc_items[0][0], ccc_items[0][0] + 256))
     return (
         "/* Auto-generated by tools/generate_normalize.py - do not edit. */\n"
         f"/* Unicode {UNICODE_VERSION} normalization (UAX #15): the canonical combining classes, the full canonical\n"
@@ -302,6 +303,7 @@ def _emit(
         "    uint8_t ccc;\n"
         "} th_norm_ccc_row;\n\n"
         f"static const int th_norm_ccc_count = {len(ccc_items)};\n"
+        f"static const uint8_t th_norm_ccc_dense[] = {{\n{ccc_dense}\n}};\n\n"
         f"static const th_norm_ccc_row th_norm_ccc[] = {{\n{ccc_rows}\n}};\n\n"
         "/* Full decomposition, sorted by code point: pool[offset, offset+length) is the run. The compatibility table\n"
         "   holds only code points whose NFKD differs from their NFD; the canonical table covers the rest. */\n"