MuPDF: add getBalancedHTML function using story

ffi-cdecl/wrap-mupdf_cdecl.c

@@ -29,6 +29,8 @@ cdecl_func(mupdf_drop_archive)
 /* buffer */
 cdecl_type(fz_buffer)
 cdecl_func(mupdf_new_buffer_from_shared_data)
+cdecl_func(mupdf_new_buffer_from_story_text)
+cdecl_func(mupdf_new_buffer_from_filtered_story_text)
 cdecl_func(mupdf_drop_buffer)
 
 /* context */

ffi/SDL3.lua

@@ -1,5 +1,5 @@
 --[[--
-Module for interfacing SDL 2.0 video/input facilities
+Module for interfacing SDL 3.0 video/input facilities
 
 This module is intended to provide input/output facilities on a
 typical desktop (rather than a dedicated e-ink reader, for which
@@ -600,4 +600,36 @@ function S.getVersion()
     return string.format("%d.%d.%d", getSDLVersion())
 end
 
+-- On Linux, fork() can hang if SDL3's gamepad/joystick subsystem background
+-- thread is mid-malloc when fork() is called: glibc's atfork prepare handler
+-- blocks waiting for all malloc arenas, and on schedulers like CachyOS's
+-- BORE/LAVD the SDL thread may not release the arena for seconds, making the
+-- KOReader window appear completely unresponsive (GNOME "Not Responding").
+-- Fix: stop the gamepad subsystem (and its thread) just before fork(), then
+-- restart it in the parent once fork() has returned.
+if ffi.os == "Linux" then
+    local gamepad_was_init = false
+    util.addRunBeforeForkFunc("sdl_gamepad_quiesce", function()
+        gamepad_was_init = SDL.SDL_WasInit(SDL.SDL_INIT_GAMEPAD) ~= 0
+        if gamepad_was_init then
+            -- Explicitly close the open gamepad handle before shutting down the
+            -- subsystem, so the ffi.gc finalizer doesn't later call SDL_CloseGamepad
+            -- on a stale pointer (after the subsystem restarts in the parent).
+            if S.controller ~= nil then
+                local ctrl = ffi.gc(S.controller, nil)  -- disarm GC finalizer
+                SDL.SDL_CloseGamepad(ctrl)
+                S.controller = nil
+            end
+            SDL.SDL_QuitSubSystem(SDL.SDL_INIT_GAMEPAD)
+        end
+    end)
+    util.addRunAfterForkParentFunc("sdl_gamepad_restore", function()
+        if gamepad_was_init then
+            gamepad_was_init = false
+            SDL.SDL_InitSubSystem(SDL.SDL_INIT_GAMEPAD)
+            openGameController()
+        end
+    end)
+end
+
 return S

ffi/mupdf.lua

@@ -165,6 +165,303 @@ function mupdf.openDocumentFromText(text, magic, html_resource_directory)
     return mupdf_doc
 end
 
+local function is_valid_utf8_bytes(bytes)
+    local index = 1
+    local length = #bytes
+
+    while index <= length do
+        local first = bytes:byte(index)
+        if first < 0x80 then
+            index = index + 1
+        elseif first >= 0xC2 and first <= 0xDF then
+            local second = bytes:byte(index + 1)
+            if not second or second < 0x80 or second > 0xBF then
+                return false
+            end
+            index = index + 2
+        elseif first == 0xE0 then
+            local second = bytes:byte(index + 1)
+            local third = bytes:byte(index + 2)
+            if not second or not third or second < 0xA0 or second > 0xBF or third < 0x80 or third > 0xBF then
+                return false
+            end
+            index = index + 3
+        elseif first >= 0xE1 and first <= 0xEC then
+            local second = bytes:byte(index + 1)
+            local third = bytes:byte(index + 2)
+            if not second or not third or second < 0x80 or second > 0xBF or third < 0x80 or third > 0xBF then
+                return false
+            end
+            index = index + 3
+        elseif first == 0xED then
+            local second = bytes:byte(index + 1)
+            local third = bytes:byte(index + 2)
+            if not second or not third or second < 0x80 or second > 0x9F or third < 0x80 or third > 0xBF then
+                return false
+            end
+            index = index + 3
+        elseif first >= 0xEE and first <= 0xEF then
+            local second = bytes:byte(index + 1)
+            local third = bytes:byte(index + 2)
+            if not second or not third or second < 0x80 or second > 0xBF or third < 0x80 or third > 0xBF then
+                return false
+            end
+            index = index + 3
+        elseif first == 0xF0 then
+            local second = bytes:byte(index + 1)
+            local third = bytes:byte(index + 2)
+            local fourth = bytes:byte(index + 3)
+            if not second or not third or not fourth or second < 0x90 or second > 0xBF or third < 0x80 or third > 0xBF or fourth < 0x80 or fourth > 0xBF then
+                return false
+            end
+            index = index + 4
+        elseif first >= 0xF1 and first <= 0xF3 then
+            local second = bytes:byte(index + 1)
+            local third = bytes:byte(index + 2)
+            local fourth = bytes:byte(index + 3)
+            if not second or not third or not fourth or second < 0x80 or second > 0xBF or third < 0x80 or third > 0xBF or fourth < 0x80 or fourth > 0xBF then
+                return false
+            end
+            index = index + 4
+        elseif first == 0xF4 then
+            local second = bytes:byte(index + 1)
+            local third = bytes:byte(index + 2)
+            local fourth = bytes:byte(index + 3)
+            if not second or not third or not fourth or second < 0x80 or second > 0x8F or third < 0x80 or third > 0xBF or fourth < 0x80 or fourth > 0xBF then
+                return false
+            end
+            index = index + 4
+        else
+            return false
+        end
+    end
+
+    return true
+end
+
+local function utf8_from_codepoint(codepoint)
+    if not codepoint or codepoint < 0 or codepoint > 0x10FFFF or (codepoint >= 0xD800 and codepoint <= 0xDFFF) then
+        return nil
+    end
+
+    if codepoint <= 0x7F then
+        return string.char(codepoint)
+    elseif codepoint <= 0x7FF then
+        return string.char(
+            0xC0 + math.floor(codepoint / 0x40),
+            0x80 + (codepoint % 0x40)
+        )
+    elseif codepoint <= 0xFFFF then
+        return string.char(
+            0xE0 + math.floor(codepoint / 0x1000),
+            0x80 + (math.floor(codepoint / 0x40) % 0x40),
+            0x80 + (codepoint % 0x40)
+        )
+    else
+        return string.char(
+            0xF0 + math.floor(codepoint / 0x40000),
+            0x80 + (math.floor(codepoint / 0x1000) % 0x40),
+            0x80 + (math.floor(codepoint / 0x40) % 0x40),
+            0x80 + (codepoint % 0x40)
+        )
+    end
+end
+
+local function decode_story_hex_entities(text)
+    text = text:gsub("&#x0([9AaDd]);", function(hex)
+        local value = tonumber("0" .. hex, 16)
+        return value and string.char(value) or ""
+    end)
+
+    local parts = {}
+    local index = 1
+
+    while true do
+        local start_pos = text:find("&#x[%x][%x];", index)
+        if not start_pos then
+            parts[#parts + 1] = text:sub(index)
+            break
+        end
+
+        parts[#parts + 1] = text:sub(index, start_pos - 1)
+
+        local cursor = start_pos
+        local bytes = {}
+        while true do
+            local entity_start, entity_end, hex = text:find("&#x([%x][%x]);", cursor)
+            if entity_start ~= cursor then
+                break
+            end
+            local value = tonumber(hex, 16)
+            if not value then
+                break
+            end
+            bytes[#bytes + 1] = string.char(value)
+            cursor = entity_end + 1
+        end
+
+        if #bytes >= 2 then
+            local candidate = table.concat(bytes)
+            if is_valid_utf8_bytes(candidate) then
+                parts[#parts + 1] = candidate
+            else
+                parts[#parts + 1] = text:sub(start_pos, cursor - 1)
+            end
+        elseif cursor > start_pos then
+            parts[#parts + 1] = text:sub(start_pos, cursor - 1)
+        else
+            local entity_start, entity_end, hex = text:find("&#x([%x]+);", start_pos)
+            local decoded = entity_start == start_pos and utf8_from_codepoint(tonumber(hex, 16)) or nil
+            if decoded then
+                parts[#parts + 1] = decoded
+                cursor = entity_end + 1
+            else
+                parts[#parts + 1] = text:sub(start_pos, start_pos)
+                cursor = start_pos + 1
+            end
+        end
+
+        index = cursor
+    end
+
+    return table.concat(parts)
+end
+
+local function normalize_story_xml(text)
+    return decode_story_hex_entities(text):gsub("^%s*<%?xml.-%?>%s*", "")
+end
+
+local function find_first_body_content_tag(text)
+    local tags = {
+        "<main", "<article", "<section", "<div", "<header", "<footer", "<nav", "<aside",
+        "<dialog", "<figure", "<table", "<ul", "<ol", "<p", "<h1", "<h2", "<h3", "<h4", "<h5", "<h6",
+    }
+    local first_index = nil
+
+    for _, tag in ipairs(tags) do
+        local index = text:find(tag, 1, true)
+        if index and (not first_index or index < first_index) then
+            first_index = index
+        end
+    end
+
+    return first_index
+end
+
+local function strip_script_elements(text)
+    text = text:gsub("<script[^>]*/>", "")
+    text = text:gsub("<script[^>]->.-</script>", "")
+    return text
+end
+
+local function escape_bare_ampersands(value)
+    local parts = {}
+    local index = 1
+
+    while true do
+        local ampersand = value:find("&", index, true)
+        if not ampersand then
+            parts[#parts + 1] = value:sub(index)
+            break
+        end
+
+        parts[#parts + 1] = value:sub(index, ampersand - 1)
+
+        local remainder = value:sub(ampersand + 1)
+        local entity = remainder:match("^(#x[%x]+;)")
+            or remainder:match("^(#%d+;)")
+            or remainder:match("^([%a][%w]+;)")
+
+        if entity then
+            parts[#parts + 1] = "&" .. entity
+            index = ampersand + #entity + 1
+        else
+            parts[#parts + 1] = "&amp;"
+            index = ampersand + 1
+        end
+    end
+
+    return table.concat(parts)
+end
+
+local function normalize_selector_input_html(text)
+    local function sanitize_attribute(name, quote)
+        local pattern = "(" .. name .. "%s*=%s*" .. quote .. ")([^" .. quote .. "]*)(" .. quote .. ")"
+        text = text:gsub(pattern, function(prefix, value, suffix)
+            return prefix .. escape_bare_ampersands(value) .. suffix
+        end)
+    end
+
+    sanitize_attribute("class", '"')
+    sanitize_attribute("class", "'")
+    sanitize_attribute("id", '"')
+    sanitize_attribute("id", "'")
+
+    return text
+end
+
+local function normalize_story_input_html(text)
+    return normalize_selector_input_html(strip_script_elements(text))
+end
+
+local function marshal_selector_array(selectors)
+    if type(selectors) ~= "table" or #selectors == 0 then
+        return nil, 0
+    end
+
+    local array = ffi.new("const char *[?]", #selectors)
+    for index, selector in ipairs(selectors) do
+        array[index - 1] = selector
+    end
+
+    return array, #selectors
+end
+
+function mupdf.getBalancedHTML(text, user_css, em)
+    local ctx = context()
+    local normalized_text = normalize_story_input_html(text)
+    local output = W.mupdf_new_buffer_from_story_text(ctx, ffi.cast("const unsigned char*", normalized_text), #normalized_text, user_css, em or 12)
+    if output == nil then
+        return nil, string.format("MuPDF story balancing failed: %s (%d)",
+            ffi.string(W.mupdf_error_message(ctx)),
+            W.mupdf_error_code(ctx))
+    end
+
+    -- local balanced_html = repair_empty_body_html(normalize_story_xml(ffi.string(output.data, output.len)))
+    local balanced_html = (normalize_story_xml(ffi.string(output.data, output.len)))
+    W.mupdf_drop_buffer(ctx, output)
+    return balanced_html
+end
+
+function mupdf.reduceHTML(text, wanted_selectors, unwanted_selectors, user_css, em)
+    local ctx = context()
+    local balanced_html = mupdf.getBalancedHTML(text, user_css, em)
+    local normalized_text = normalize_story_input_html(balanced_html or text)
+    local wanted_array, wanted_count = marshal_selector_array(wanted_selectors)
+    local unwanted_array, unwanted_count = marshal_selector_array(unwanted_selectors)
+    local output = W.mupdf_new_buffer_from_filtered_story_text(
+        ctx,
+        ffi.cast("const unsigned char*", normalized_text),
+        #normalized_text,
+        wanted_array,
+        wanted_count,
+        unwanted_array,
+        unwanted_count,
+        user_css,
+        em or 12
+    )
+
+    if output == nil then
+        return nil, string.format("MuPDF story reduction failed: %s (%d)",
+            ffi.string(W.mupdf_error_message(ctx)),
+            W.mupdf_error_code(ctx))
+    end
+
+    local reduced_html = normalize_story_xml(ffi.string(output.data, output.len))
+    W.mupdf_drop_buffer(ctx, output)
+    return reduced_html
+end
+
 -- Document functions:
 
 --[[

ffi/mupdf_h.lua

@@ -51,6 +51,8 @@ typedef struct {
   int shared;
 } fz_buffer;
 fz_buffer *mupdf_new_buffer_from_shared_data(fz_context *, const unsigned char *, size_t);
+fz_buffer *mupdf_new_buffer_from_story_text(fz_context *, const unsigned char *, size_t, const char *, float);
+fz_buffer *mupdf_new_buffer_from_filtered_story_text(fz_context *, const unsigned char *, size_t, const char **, int, const char **, int, const char *, float);
 void *mupdf_drop_buffer(fz_context *, fz_buffer *);
 typedef struct {
   void *user;