Type __index__ int64 for resources above the 32-bit row limit

writeParquet()'s flat / data-frame path typed the __index__ column from each
part's own value range, so part 0 of a large streamed resource narrowed to
int32 and later append parts overflowed on cast once the running index passed
~2.1e9. .writeDataFrameParquet() gains an optional index_max argument: when the
declared total index range exceeds the 32-bit limit, __index__ is typed as a
64-bit integer up front (part 0 included) so every streamed part shares one
type, mirroring the coord-array max_dim typing. Pass index_max = Inf when the
total row count is unknown before streaming (e.g. graph edges). Small resources
are unaffected -- the index still narrows to the smallest integer type. Index
generation is numeric-safe, so offsets past 2^31 no longer overflow to NA.

Adds test-writeParquet-large-index.R (streams a > 2^31 index across two parts,
asserts a consistent on-disk int64 column with exact values, and guards the
small-resource narrowing path). Companion to the DuckDBDataFrame
validateAppendOffset() / buildTableSelectSQL() fix.

Bump version to 0.99.4; update NEWS.

Author: aboyounp-gne

DESCRIPTION

@@ -1,5 +1,5 @@
 Package: BiocDuckDB
-Version: 0.99.3
+Version: 0.99.4
 Date: 2026-07-22
 Title: Bioconductor DuckDB Integration and High-Level I/O
 Description:

NAMESPACE

@@ -123,6 +123,7 @@ importFrom(DuckDBArray,writeCoordArray)
 importFrom(DuckDBDataFrame,DuckDBDataFrame)
 importFrom(DuckDBDataFrame,DuckDBSelfHits)
 importFrom(DuckDBDataFrame,acquireDuckDBConn)
+importFrom(DuckDBDataFrame,arrowIntType)
 importFrom(DuckDBDataFrame,arrowType)
 importFrom(DuckDBDataFrame,clusterSort)
 importFrom(DuckDBDataFrame,configureCloud)

NEWS.md

@@ -1,3 +1,18 @@
+# BiocDuckDB 0.99.4
+
+## Bug fixes
+
+- `writeParquet()` (flat / data-frame path) can now write a resource with more
+  than ~2.1e9 rows without the `__index__` column overflowing 32-bit integers. A
+  new optional `index_max` argument declares the resource's total index range;
+  when it exceeds the 32-bit limit the `__index__` column is typed as a 64-bit
+  integer up front (part 0 included) so every streamed append part shares one
+  type, instead of narrowing part 0 to `int32` and overflowing later parts on
+  cast (mirrors the coord-array `max_dim` typing). Pass `index_max = Inf` when
+  the total row count is unknown before streaming (e.g. graph edges). Small
+  resources are unaffected --- the index still narrows to the smallest integer
+  type. Requires the companion `DuckDBDataFrame` `validateAppendOffset()` fix.
+
 # BiocDuckDB 0.99.3
 
 ## New features

R/writeParquet.R

@@ -732,13 +732,13 @@ function(x, path, indexcol, keycol, dimtbl, name, dimension, layout,
 }
 
 #' @importFrom arrow Array as_arrow_table write_dataset write_parquet
-#' @importFrom DuckDBDataFrame arrowType reconcileParquetSchema setupFlatParquetWrite clusterSort
+#' @importFrom DuckDBDataFrame arrowIntType arrowType reconcileParquetSchema setupFlatParquetWrite clusterSort
 #' @importFrom S4Vectors I
 #' @importFrom stats setNames
 .writeDataFrameParquet <-
 function(x, path, indexcol, keycol, dimtbl, name, dimension, layout,
          refs = NULL, append = FALSE, offset = 0L, part = NULL,
-         part_digits = 0L, cluster_by = NULL, ...)
+         part_digits = 0L, cluster_by = NULL, index_max = NULL, ...)
 {
     prep <- setupFlatParquetWrite(
         path, append = append, offset = offset, part = part,
@@ -813,6 +813,15 @@ function(x, path, indexcol, keycol, dimtbl, name, dimension, layout,
             }
         }
 
+        # Type the __index__ column by the producer-declared total range
+        # (`index_max`), not this block's, so every part of a > 2^31-row resource
+        # shares one wide type instead of narrowing part 0 to int32 and
+        # overflowing later parts (mirrors the coord-array `max_dim` typing).
+        if (!is.null(indexcol) && !is.null(index_max) &&
+            indexcol %in% colnames(x)) {
+            narrow_types[[indexcol]] <- arrowIntType(c(0, index_max))
+        }
+
         # Narrow only the write payload (an Arrow table); `x` stays an R
         # data.frame so the schema builder reports the base "integer" type with
         # the narrowed arrowType recorded per column below.

tests/testthat/test-writeParquet-large-index.R

@@ -0,0 +1,50 @@
+# Regression: append offsets and __index__ values above the 32-bit ceiling (a
+# resource with more than ~2.1e9 rows). `index_max` declares the index range so
+# the __index__ column is typed wide enough (int64) up front -- part 0 included
+# -- and every streamed part shares one type, instead of narrowing part 0 to
+# int32 and overflowing later parts on cast. See NEWS 0.99.3.
+# Run: library(BiocDuckDB); library(testthat); source("test-writeParquet-large-index.R")
+
+library(arrow)
+
+.indexArrowType <- function(f) {
+    ParquetFileReader$create(f)$GetSchema()$GetFieldByName("__index__")$type$ToString()
+}
+
+test_that("index_max streams a > 2^31 index as a consistent int64 column", {
+    dir <- tempfile()
+    # Part 0 at offset 0, then a part whose offset crosses the 32-bit ceiling.
+    writeParquet(data.frame(v = 1:5), dir, indexcol = "__index__", keycol = NULL,
+                 dimension = "sample", layout = "data_frame",
+                 offset = 0, part = 0L, part_digits = 2L, append = FALSE,
+                 index_max = Inf)
+    writeParquet(data.frame(v = 11:15), dir, indexcol = "__index__", keycol = NULL,
+                 dimension = "sample", layout = "data_frame",
+                 offset = 3e9, part = 1L, part_digits = 2L, append = TRUE,
+                 index_max = Inf)
+
+    files <- sort(list.files(dir, pattern = "parquet$", recursive = TRUE,
+                             full.names = TRUE))
+    expect_length(files, 2L)
+    # Both parts, and the unified dataset, are int64 (schema-consistent).
+    expect_true(all(vapply(files, .indexArrowType, character(1L)) == "int64"))
+    ds <- open_dataset(dir)
+    expect_identical(ds$schema$GetFieldByName("__index__")$type$ToString(), "int64")
+
+    # The index values above 2^31 are stored exactly, with no overflow to NA.
+    idx <- sort(as.data.frame(ds)[["__index__"]])
+    expect_false(anyNA(idx))
+    expect_equal(idx[6:10],
+                 c(3000000001, 3000000002, 3000000003, 3000000004, 3000000005))
+    unlink(dir, recursive = TRUE)
+})
+
+test_that("without index_max a small index still narrows (no regression)", {
+    dir <- tempfile()
+    writeParquet(data.frame(v = 1:5), dir, indexcol = "__index__", keycol = NULL,
+                 dimension = "sample", layout = "data_frame")
+    f <- list.files(dir, pattern = "parquet$", recursive = TRUE,
+                    full.names = TRUE)[1L]
+    expect_identical(.indexArrowType(f), "uint8")
+    unlink(dir, recursive = TRUE)
+})