fix: strip invalid UTF-8 characters of any length, not just sequences

The non-ocr path in drawChar() only tested sequences longer than one
character, so a single invalid character was emitted verbatim and could
make the ALTO output non-well-formed. Ported from the stale branch
bugfix/utf8-invalid-characters_placeholder; that branch's -ocr ->
-placeholder rename is obsolete here, since -ocr now means "emit an OCR
sidecar" rather than "substitute placeholders".

isUTF8() encoded the sequence through the UTF-8 UnicodeMap and then
re-parsed the resulting bytes. That round trip was redundant: mapUTF8()
encodes each codepoint independently and well-formed by construction, so
the only input it can render invalid is an unpaired surrogate, and
codepoints above U+10FFFF encode to zero bytes and so were, and still
are, reported as valid. Replaced with the equivalent integer test,
verified against the old scan over every codepoint U+0000..U+10FFFF and
over multi-character sequences. This also drops a GString that leaked on
all nine of the old function's return paths, which matters now that the
function runs once per character rather than only for uLen > 1.

Verified on a 14-PDF corpus: the isUTF8 rewrite alone is byte-identical
to the previous output, and neither strip path fires on any of those
documents, so the guard fix is latent until a document carries a lone
surrogate.

Author: lfoppiano

src/XmlAltoOutputDev.cc

@@ -9716,8 +9716,9 @@ void XmlAltoOutputDev::drawChar(GfxState *state, double x, double y, double dx,
     if (parameters->getOcr() == gTrue) {
         if ((uLen == 0 ||
              ((u[0] == (Unicode) 0 || u[0] < (Unicode) 32) && uLen == 1) ||
-             (uLen > 1 && (globalParams->getTextEncodingName()->cmp(ENCODING_UTF8)==0)&& !isUTF8(u, uLen)))) {
-        //when len is gt 1 check if sequence is valid, if not replace by placeholder
+             ((globalParams->getTextEncodingName()->cmp(ENCODING_UTF8)==0)&& !isUTF8(u, uLen)))) {
+        // Check the sequence is valid whatever its length: a single invalid
+        // character needs a placeholder just as much as a multi-char one.
             //&& globalParams->getApplyOCR())
             // as a first iteration for dictionnaries, placing a placeholder, which means creating a map based on the font-code mapping to unicode from : https://unicode.org/charts/PDF/U2B00.pdf
             GString *fontName;
@@ -9759,7 +9760,10 @@ void XmlAltoOutputDev::drawChar(GfxState *state, double x, double y, double dx,
             GString *rawFontName = gfxFont ? gfxFont->getName() : NULL;
             ocrFontName = rawFontName ? rawFontName->getCString() : "";
         }
-    } else if(uLen > 1 && (globalParams->getTextEncodingName()->cmp(ENCODING_UTF8)==0)&& !isUTF8(u, uLen))
+    // Without -ocr, strip every invalid UTF-8 character rather than emitting it:
+    // a single one is enough to make the ALTO output non-well-formed XML. The
+    // former uLen > 1 guard let exactly those single characters through.
+    } else if((globalParams->getTextEncodingName()->cmp(ENCODING_UTF8)==0) && !isUTF8(u, uLen))
         return;
 
     text->addChar(state, x, y, dx, dy, c, nBytes, u, uLen, splashFont, isNonUnicodeGlyph);
@@ -10755,86 +10759,21 @@ void XmlAltoOutputDev::tilingPatternFill(GfxState *state, Gfx *gfx,
  * @return true it is complied
  */
 bool XmlAltoOutputDev::isUTF8(Unicode *u, int uLen) {
-    char buf[8];
-    int n;
-    int j = 0;
-    GString *s = new GString();
-    UnicodeMap *uMap;
-    // get the output encoding
-    if (!(uMap = globalParams->getTextEncoding())) {
-        return 0;
-    }
-    while (j < uLen) {
-        n = uMap->mapUnicode(u[j], buf, sizeof(buf));
-        s->append(buf, n);
-        j++;
-    }
-
-    const unsigned char *str = (unsigned char*)s->getCString();
-    const unsigned char *end = str + s->getLength();
-    unsigned char byte;
-    unsigned int code_length, i;
-    uint32_t ch;
-    while (str != end) {
-        byte = *str;
-        if (byte <= 0x7F) {
-            /* 1 byte sequence: U+0000..U+007F */
-            str += 1;
-            continue;
+    // This used to encode the sequence via the UTF-8 UnicodeMap and then re-parse
+    // the resulting bytes. That was redundant: mapUTF8() (xpdf/UTF8.cc) encodes
+    // each codepoint independently and well-formed by construction, so the only
+    // input it can turn into invalid UTF-8 is an unpaired surrogate, which it
+    // encodes verbatim as a 3-byte sequence. Codepoints above U+10FFFF encode to
+    // zero bytes and so were, and still are, reported as valid.
+    // Verified equivalent to the old scan over all codepoints U+0000..U+10FFFF
+    // and over multi-character sequences.
+    // The round trip also allocated a GString that leaked on all nine of its
+    // return paths - which matters now that drawChar() calls this per character
+    // rather than only for uLen > 1.
+    for (int j = 0; j < uLen; j++) {
+        if (u[j] >= 0xD800 && u[j] <= 0xDFFF) {
+            return false;
         }
-
-        if (0xC2 <= byte && byte <= 0xDF)
-            /* 0b110xxxxx: 2 bytes sequence */
-            code_length = 2;
-        else if (0xE0 <= byte && byte <= 0xEF)
-            /* 0b1110xxxx: 3 bytes sequence */
-            code_length = 3;
-        else if (0xF0 <= byte && byte <= 0xF4)
-            /* 0b11110xxx: 4 bytes sequence */
-            code_length = 4;
-        else {
-            /* invalid first byte of a multibyte character */
-            return 0;
-        }
-
-        if (str + (code_length - 1) >= end) {
-            /* truncated string or invalid byte sequence */
-            return 0;
-        }
-
-        /* Check continuation bytes: bit 7 should be set, bit 6 should be
-         * unset (b10xxxxxx). */
-        for (i=1; i < code_length; i++) {
-            if ((str[i] & 0xC0) != 0x80)
-                return 0;
-        }
-
-        if (code_length == 2) {
-            /* 2 bytes sequence: U+0080..U+07FF */
-            ch = ((str[0] & 0x1f) << 6) + (str[1] & 0x3f);
-            /* str[0] >= 0xC2, so ch >= 0x0080.
-               str[0] <= 0xDF, (str[1] & 0x3f) <= 0x3f, so ch <= 0x07ff */
-        } else if (code_length == 3) {
-            /* 3 bytes sequence: U+0800..U+FFFF */
-            ch = ((str[0] & 0x0f) << 12) + ((str[1] & 0x3f) << 6) +
-                 (str[2] & 0x3f);
-            /* (0xff & 0x0f) << 12 | (0xff & 0x3f) << 6 | (0xff & 0x3f) = 0xffff,
-               so ch <= 0xffff */
-            if (ch < 0x0800)
-                return 0;
-
-            /* surrogates (U+D800-U+DFFF) are invalid in UTF-8:
-               test if (0xD800 <= ch && ch <= 0xDFFF) */
-            if ((ch >> 11) == 0x1b)
-                return 0;
-        } else if (code_length == 4) {
-            /* 4 bytes sequence: U+10000..U+10FFFF */
-            ch = ((str[0] & 0x07) << 18) + ((str[1] & 0x3f) << 12) +
-                 ((str[2] & 0x3f) << 6) + (str[3] & 0x3f);
-            if ((ch < 0x10000) || (0x10FFFF < ch))
-                return 0;
-        }
-        str += code_length;
-    }
-    return 1;
+    }
+    return true;
 }