Fix negative token width for glyphs wrapped after a diacritic (#192) (#246)

A word break at a line wrap was suppressed when the preceding glyph was a combining mark, gluing the wrapped glyph onto the previous line's word and giving that token a negative WIDTH. Overriding the suppression on a baseline shift alone proved too broad — accents are drawn raised above their letter, so they shift the baseline on the same line, and ordinary words were split apart ("Álvarez" → "´" + "lvarez"); over the 5,927-PDF GROBID corpus that changed 1,209 files when only 297 had anything to fix. Baseline shift can't separate the two cases (0.24 vs 0.21 median relative to font size), but the horizontal gap can: a combining mark never starts more than 0.79 font sizes left of the word's end, while a wrapped glyph restarts at the left margin and reaches −67. The suppression is now overridden only when both hold — baseline shifted and sp < -fontSize. This PR also merges the duplicated classifyChar tables, which had drifted so that U+00B0 DEGREE SIGN counted as a combining mark in one live call site but not the other, running axis labels together as "0°0", "°20°4" instead of "0°", "20°". Result across the full corpus: negative widths 536 → 37, 394 files changed (vs 1,209), +905 tokens (vs +12,591), no crashes or unparseable output.

Author: lfoppiano

src/XmlAltoOutputDev.cc

@@ -2346,7 +2346,19 @@ void TextPage::beginWord(GfxState *state, double x0, double y0) {
     curWord = new TextRawWord(state, x0, y0, curFont, curFontSize, getIdWORD(), getIdx());
 }
 
-ModifierClass TextPage::classifyChar(Unicode u) {
+// Single source of truth for combining-mark classification.
+//
+// This table used to be duplicated as TextPage::classifyChar and
+// IWord::classifyChar, and the two copies had drifted: the TextPage one also
+// listed U+00B0 DEGREE SIGN as NORDIC_RING. Both were live -- TextPage's from
+// addCharToRawWord, IWord's from TextRawWord::addChar -- so the same character
+// was a combining mark for one word-break decision and not for another.
+//
+// The degree sign is a standalone symbol, not a combining mark; classifying it
+// as one suppressed the word break and glued axis labels such as "40 deg N"
+// into unusable tokens. U+02DA RING ABOVE (730) and U+030A COMBINING RING
+// ABOVE (778) remain, as those genuinely are marks.
+static ModifierClass classifyModifierChar(Unicode u) {
     switch (u) {
         case (Unicode) 776: //COMBINING DIAERESIS
         case (Unicode) 168: //DIAERESIS
@@ -2393,7 +2405,7 @@ ModifierClass TextPage::classifyChar(Unicode u) {
             return TILDE;
 
         case 778: //COMBINING
-        case 176:
+            //case 176:
         case 730:
             return NORDIC_RING;//LOOK AT UNICODE RING BELOW...
 
@@ -2431,6 +2443,8 @@ ModifierClass TextPage::classifyChar(Unicode u) {
 
 }
 
+ModifierClass TextPage::classifyChar(Unicode u) { return classifyModifierChar(u); }
+
 /*
  * Returns the correct base char for composition with icu4c following unicode standard.
  */
@@ -2504,90 +2518,7 @@ Unicode TextPage::getCombiningDiacritic(ModifierClass modifierClass) {
     return diactritic;
 }
 
-ModifierClass IWord::classifyChar(Unicode u) {
-    switch (u) {
-        case (Unicode) 776: //COMBINING DIAERESIS
-        case (Unicode) 168: //DIAERESIS
-            return DIAERESIS;
-
-        case 833:
-        case 779: // COMBINING DOUBLE_ACUTE_ACCENT
-        case 733:
-            return DOUBLE_ACUTE_ACCENT;
-        case 180:
-        case 769: //COMBINING
-        case 714:
-            return ACUTE_ACCENT;
-
-        case 768: //COMBINING
-        case 832:
-        case 715:
-        case 96:
-            return GRAVE_ACCENT;
-
-        case 783: //COMBINING
-            return DOUBLE_GRAVE_ACCENT;
-
-        case 774: //COMBINING
-        case 728:
-            //case '\uA67C':
-            return BREVE_ACCENT;
-
-        case 785: //COMBINING
-        case 1156:
-        case 1159:
-            return INVERTED_BREVE_ACCENT;
-
-
-        case 770: //COMBINING
-        case 94:
-        case 710:
-            return CIRCUMFLEX;
-
-
-        case 771: //COMBINING
-        case 126:
-        case 732:
-            return TILDE;
-
-        case 778: //COMBINING
-            //case 176:
-        case 730:
-            return NORDIC_RING;//LOOK AT UNICODE RING BELOW...
-
-        case 780: //COMBINING
-        case 711:
-            return CZECH_CARON;
-
-        case 807: //COMBINING
-        case 184:
-            return CEDILLA;
-
-        case 775: //COMBINING
-        case 729:
-            return DOT_ABOVE;
-
-        case 777: //COMBINING
-        case 704:
-            return HOOK;
-
-        case 795: //COMBINING
-            return HORN;
-
-        case 808: //COMBINING
-        case 731:
-            //case '\u1AB7':// combining open mark below
-            return OGONEK;
-
-        case 772: //COMBINING
-        case 175:
-        case 713:
-            return MACRON;
-        default:
-            return NOT_A_MODIFIER;
-    }
-
-}
+ModifierClass IWord::classifyChar(Unicode u) { return classifyModifierChar(u); }
 
 Unicode IWord::getCombiningDiacritic(ModifierClass modifierClass) {
 
@@ -2964,12 +2895,29 @@ void TextPage::addCharToRawWord(GfxState *state, double x, double y, double dx,
                 ((TextChar *) curWord->chars->get(curWord->chars->getLength() - 1))->spaceAfter =
                         (char) gTrue;
         }
+        // A combining mark sits on or beside the glyph it modifies: it shares the
+        // baseline and it is never far to the left of the word built so far. A
+        // glyph that breaks both of those -- different baseline *and* starting a
+        // whole font size or more before the end of the current word -- has
+        // wrapped to the next line and is not a mark on this word. Gluing it on
+        // produces a token whose width is negative (issue #192), so the
+        // diacritic break-suppression below must not apply to it.
+        //
+        // Both halves are needed. Baseline alone is far too broad: accents drawn
+        // raised above their base letter shift the baseline on the *same* line,
+        // and splitting there breaks ordinary words apart. Measured over the
+        // GROBID end-to-end corpus, sp/fontSize never goes below -0.79 for those
+        // same-line marks, while wrapped glyphs reach -67, so the two populations
+        // separate cleanly at one font size.
+        GBool sameBaseline = fabs(base - curWord->base) <= 1;
+        GBool wrappedToNextLine = !sameBaseline && sp < -curWord->fontSize;
+
         // take into account rotation angle ??
-        if ( (overlap || 
+        if ( (overlap ||
               fabs(base - curWord->base) > 1 ||
               space ||
-              (sp < -minDupBreakOverlap * curWord->fontSize)) 
-              && modifierClass == NOT_A_MODIFIER) {
+              (sp < -minDupBreakOverlap * curWord->fontSize))
+              && (modifierClass == NOT_A_MODIFIER || wrappedToNextLine)) {
             endWord();
             beginWord(state, x, y);
         }