Fix line number recognition for sparse and accumulated numbering

markLineNumber() rejected valid line-number columns in two cases the size-scaled
coverage threshold from #226 does not cover:

1. Sparse numbering (every 5th/10th line, common in bioRxiv preprints): small,
   low-coverage clusters were dropped, so under -noLineNumbers the numbers survived.
2. Accumulated numbering on later pages: line numbers that reach large values (e.g.
   428..444 on page 21) sit in an irregular layout (a figure or column break splits
   the column), so an even-spacing test rejects them even though the values are a
   clean +1 progression.

A low-coverage margin cluster is now accepted as line numbers when its values form a
regular arithmetic progression (dominant constant increment) at the extreme page
margin AND either (a) the smallest value already exceeds the page's line count -
meaning the numbers accumulated from earlier pages and cannot be a per-page list such
as references or figure labels - or (b) the vertical pitch is roughly even (fresh,
small-valued numbering). The margin constraint keeps regular-but-inset labels (figure
panel numbering) intact; reference numbers restart near 1 and fail both paths.

Validated on the GROBID end-to-end corpora: 229336v1 (every 5th line) 242->3, 003657v1
and 018085v1 (every line, values 287..444) fully stripped, removing only digit tokens;
figure panel labels (005124v1) and eLife/PLOS/PMC output unchanged (no regression).

Author: lfoppiano

src/XmlAltoOutputDev.cc

@@ -5217,6 +5217,100 @@ int find_index_relaxed(vector<double> positions, double val, double margin) {
     return index;
 }
 
+// Non-negative integer value of an all-digit word, or -1 if it holds a non-digit.
+static long word_number_value(TextRawWord *word) {
+    long val = 0;
+    for (int i = 0; i < word->len; i++) {
+        Unicode u = ((TextChar *) word->chars->get(i))->c;
+        if (u < '0' || u > '9')
+            return -1;
+        val = val * 10 + (long) (u - '0');
+    }
+    return val;
+}
+
+// A margin cluster of numbers is line numbering even at low page coverage when its values
+// form a regular arithmetic progression (dominant constant increment). Two accept paths:
+//  - "big numbers": the smallest value already exceeds this page's line count, so the numbers
+//    cannot be a per-page list (references / figure labels start near 1) - they are line
+//    numbers accumulated from earlier pages (e.g. 428..444 on a late page). Accept regardless
+//    of vertical spacing, which a figure or column break may disrupt.
+//  - "even pitch": otherwise (fresh, small-valued numbering such as every-5th-line on page 1)
+//    require roughly even vertical spacing.
+// Footnote / bibliographic reference numbers match neither: they restart near 1 and are not
+// evenly spaced down the page.
+static bool is_regular_number_sequence(const vector<int> &cluster,
+                                       const vector<TextRawWord *> &lineNumberWords,
+                                       int totalNumberOfLines) {
+    int n = (int) cluster.size();
+    if (n < 3)
+        return false;
+
+    // collect (yMin, value), ordered top-to-bottom
+    vector<pair<double, long> > seq;
+    for (int i = 0; i < n; i++) {
+        TextRawWord *w = lineNumberWords[cluster[i]];
+        long v = word_number_value(w);
+        if (v < 0)
+            return false;
+        seq.push_back(make_pair(w->yMin, v));
+    }
+    sort(seq.begin(), seq.end());
+
+    // values must strictly increase going down the page
+    vector<long> dv;
+    vector<double> dy;
+    for (int i = 1; i < (int) seq.size(); i++) {
+        long d = seq[i].second - seq[i - 1].second;
+        double gy = seq[i].first - seq[i - 1].first;
+        if (d <= 0 || gy <= 0)
+            return false;
+        dv.push_back(d);
+        dy.push_back(gy);
+    }
+
+    // dominant (median) increment; line numbering steps are small (1, 2, 5, 10, 25...)
+    vector<long> sortedDv = dv;
+    sort(sortedDv.begin(), sortedDv.end());
+    long g = sortedDv[sortedDv.size() / 2];
+    if (g < 1 || g > 50)
+        return false;
+
+    // a strong majority of steps must share that exact increment; a line number missed
+    // by extraction shows up as a multiple of g and is simply not counted here
+    int agree = 0;
+    for (int i = 0; i < (int) dv.size(); i++)
+        if (dv[i] == g)
+            agree++;
+    if (agree < (int) ((dv.size() * 3 + 4) / 5))   // >= 60%
+        return false;
+
+    // "big numbers" path: smallest value exceeds this page's line count -> accumulated line
+    // numbers, accept without requiring even pitch (figure/column breaks disrupt spacing).
+    long minVal = seq[0].second;
+    if (minVal > (long) totalNumberOfLines)
+        return true;
+
+    // otherwise require roughly even vertical pitch per unit increment
+    vector<double> pitch;
+    for (int i = 0; i < (int) dv.size(); i++)
+        if (dv[i] == g)
+            pitch.push_back(dy[i] / (double) dv[i]);
+    if ((int) pitch.size() < 2)
+        return false;
+    double mean = 0.0;
+    for (int i = 0; i < (int) pitch.size(); i++)
+        mean += pitch[i];
+    mean /= pitch.size();
+    if (mean <= 0.0)
+        return false;
+    double var = 0.0;
+    for (int i = 0; i < (int) pitch.size(); i++)
+        var += (pitch[i] - mean) * (pitch[i] - mean);
+    double cv = sqrt(var / pitch.size()) / mean;
+    return cv < 0.30;
+}
+
 bool TextPage::markLineNumber() {
     // Detect the presence of line number column in the page and mark the corresponding TextWord objects for further appropriate handling
 
@@ -5540,8 +5634,20 @@ bool TextPage::markLineNumber() {
         else
             requiredCoverage = 0.5;
 
-        if (coverage < requiredCoverage)
-            continue;
+        // Low coverage can still be genuine sparse line numbering (every Nth line). Accept it
+        // only when the cluster is a regular arithmetic progression at even vertical pitch AND
+        // sits at the extreme page margin. Figure panels / legends with small ascending labels
+        // (e.g. rows numbered 1..6) are regular too but are inset from the margin, so they must
+        // not be stripped; footnote/reference numbers fail the regular-sequence test.
+        if (coverage < requiredCoverage) {
+            double span = (double) (rightMostBoundary - leftMostBoundary);
+            double marginTol = span * 0.06;
+            bool atMargin = (final_vpos - leftMostBoundary <= marginTol) ||
+                            ((double) rightMostBoundary - final_vpos <= marginTol);
+            if (!(atMargin &&
+                  is_regular_number_sequence(clusters[bestClusterIndex[j]], lineNumberWords, totalNumberOfLines)))
+                continue;
+        }
         validClusterIndices.push_back(bestClusterIndex[j]);
     }