Type coord-array indices past the 32-bit range on the DuckDB fast path

The DuckDB fast write path derived the new_idx column types by inferring from
the remapped indices (arrowType(new_indices)) rather than using the pre-computed,
max_dim-aware idxtypes. With a > 2^31 along-axis offset the remapped indices are
a double, which inferred float64 and degraded to a DuckDB INTEGER temp column
that overflowed; and an append part could narrow differently from part 0.
.buildIndexMappings() now takes idxtypes and types new_idx from idxtypes[[j]]
(falling back to inference only when idxtypes is absent), so the index type
matches the R write path and is consistent across parts.

.validateMaxDim() also accepts a whole-number max_dim beyond the 32-bit range: it
validated integrality with `max_dim == as.integer(max_dim)`, overflowing such a
dimension to NA. It now checks integrality via round() without coercion and keeps
within-32-bit dims as integer, so a coordinate axis larger than ~2.1e9 can be
declared.

Tests: fast path types the index from a pinned max_dim (int32, not the uint8 the
50 present values would infer) and from a > 2^31 max_dim (uint32, no
float64/overflow). Small/within-range writes are unchanged.

Author: aboyounp-gne

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: DuckDBArray
-Version: 0.99.00
-Date: 2026-07-18
+Version: 0.99.1
+Date: 2026-07-22
 Title: DuckDB-Backed DelayedArray Implementation
 Description:
     Provides DuckDB-backed implementations of DelayedArray and DelayedMatrix

NEWS.md

@@ -1,3 +1,20 @@
+# DuckDBArray 0.99.1
+
+## Bug fixes
+
+- The DuckDB fast write path now types the coordinate-index columns from the
+  pre-computed, `max_dim`-aware `idxtypes` (`.buildIndexMappings()`) instead of
+  inferring them from the remapped indices. Previously a `> 2^31` along-axis
+  offset made the remapped indices a double, which inferred `float64` and
+  degraded to a DuckDB `INTEGER` temp column that overflowed; and an append part
+  could narrow differently from part 0. The index type now matches the R write
+  path and is consistent across parts.
+
+- `writeCoordArray()` accepts a whole-number `max_dim` beyond the 32-bit integer
+  range (a coordinate axis larger than ~2.1e9). The validator previously coerced
+  `max_dim` via `as.integer()`, overflowing such a dimension to `NA`; it now
+  validates integrality without coercion and keeps within-32-bit dims as integer.
+
 # DuckDBArray 0.9.20
 
 ## New features

R/writeCoordArray-duckdb.R

@@ -236,14 +236,22 @@ function(x, path, indexcols, datacol, arrowtype, ...)
 #' @importFrom DuckDBDataFrame arrowType
 .buildIndexMappings <-
 function(tbl, indexcols, keycols, grid = NULL, along = NULL, offset = 0L,
-         group_offset = 0L)
+         group_offset = 0L, idxtypes = NULL)
 {
     # For each dimension, create temp table: old_key → new_idx → grid_group
     # Returns list(mapping_tables = c(names), cleanup_sql = c(DROP statements))
 
     # Extract connection from tbl
     conn <- tbl$src$con
 
+    # Per-axis new_idx types resolved up front (max_dim-aware). Using these
+    # instead of inferring from the remapped indices keeps the new_idx column
+    # consistent with the R write path and, on a > 2^31 (append) offset, avoids
+    # the double offset degrading to float64 -> INTEGER and overflowing.
+    if (!is.null(idxtypes)) {
+        idxtypes <- .resolveArrowTypeList(idxtypes)
+    }
+
     mapping_tables <- character(length(indexcols))
     cleanup_sql <- character(length(indexcols))
 
@@ -267,9 +275,17 @@ function(tbl, indexcols, keycols, grid = NULL, along = NULL, offset = 0L,
         temp_name <- sprintf("temp_idxmap_%s_%s", col, temp_suffix)
         mapping_tables[j] <- temp_name
 
-        # Determine optimal integer types using existing helpers
+        # Determine optimal integer types using existing helpers. The new_idx
+        # type is taken from the pre-resolved per-axis idxtypes when available
+        # (max_dim-aware, and correct for a > 2^31 offset that makes the
+        # remapped indices a double); otherwise inferred from the data.
         old_key_type <- arrowType(old_keys)
-        new_idx_type <- arrowType(new_indices)
+        new_idx_type <- if (!is.null(idxtypes) && length(idxtypes) >= j &&
+                            !is.null(idxtypes[[j]])) {
+            idxtypes[[j]]
+        } else {
+            arrowType(new_indices)
+        }
         grid_group_type <- arrowType(grid_groups)
 
         # Convert Arrow types to DuckDB type names
@@ -315,7 +331,8 @@ function(x, path, indexcols, datacol, grid, grid_suffix, idxtypes, arrowtype,
     # The grid_group column is computed using S4Arrays::mapToGrid
     mappings_result <- .buildIndexMappings(tbl, indexcols, keycols, grid = grid,
                                            along = along, offset = offset,
-                                           group_offset = group_offset)
+                                           group_offset = group_offset,
+                                           idxtypes = idxtypes)
     mappings <- mappings_result[["mapping_tables"]]
 
     # Setup cleanup handler for mapping tables

R/writeCoordArray-methods.R

@@ -65,10 +65,12 @@ function(max_dim, x, append, along, offset, ndim)
     }
     if (!is.numeric(max_dim) || length(max_dim) != ndim ||
         anyNA(max_dim) ||
-        !all(max_dim == as.integer(max_dim))) {
+        !all(max_dim == round(max_dim))) {
         stop("'max_dim' must be NULL or an integer vector of length ", ndim)
     }
-    max_dim <- as.integer(max_dim)
+    if (all(max_dim <= .Machine$integer.max)) {
+        max_dim <- as.integer(max_dim)
+    }
     if (any(max_dim < dim(x))) {
         j <- which(max_dim < dim(x))[1L]
         stop("'max_dim[", j, "]' (", max_dim[j],

tests/testthat/test-writeCoordArray-DuckDBArray.R

@@ -78,3 +78,42 @@ test_that("writeCoordArray,DuckDBArray infers schema from DuckDB column types",
                     }, character(1L))
     expect_true(all(types == "double"))
 })
+
+# the DuckDB fast path must type the index columns from the pre-computed
+# idxtypes (max_dim-aware) rather than inferring from the remapped indices, so a
+# pinned-wide index (and, on a > 2^31 offset, a double index) does not narrow /
+# overflow differently from the R write path.
+.f4IndexType <- function(path, column) {
+    f <- list.files(path, pattern = "parquet$", recursive = TRUE,
+                    full.names = TRUE)[1L]
+    arrow::ParquetFileReader$create(f)$GetSchema()$
+        GetFieldByName(column)$type$ToString()
+}
+
+test_that("fast path types the index from max_dim (idxtypes honored, not inferred)", {
+    skip_if_not_installed("arrow")
+    names(dimnames(state.x77)) <- c("index1", "index2")
+    pqmat <- DuckDBMatrix(state_path, datacol = "value",
+                          keycols = lapply(dimnames(state.x77),
+                                           function(x) setNames(seq_along(x), x)))
+    path <- .newCoordPath("f4-maxdim")
+    # Row axis pinned to 70000 (int32) though its 50 values alone infer uint8.
+    writeCoordArray(pqmat, path,
+                    grid = RegularArrayGrid(dim(state.x77), c(10L, 4L)),
+                    max_dim = c(70000L, 8L))
+    expect_identical(.f4IndexType(path, "index1"), "int32")
+})
+
+test_that("fast path types the index from a > 2^31 max_dim (no float64/overflow)", {
+    skip_if_not_installed("arrow")
+    names(dimnames(state.x77)) <- c("index1", "index2")
+    pqmat <- DuckDBMatrix(state_path, datacol = "value",
+                          keycols = lapply(dimnames(state.x77),
+                                           function(x) setNames(seq_along(x), x)))
+    path <- .newCoordPath("f4-bigdim")
+    # Declare axis 2 larger than the 32-bit range.
+    writeCoordArray(pqmat, path,
+                    grid = RegularArrayGrid(dim(state.x77), c(10L, 4L)),
+                    max_dim = c(nrow(state.x77), 3e9 + ncol(state.x77)))
+    expect_identical(.f4IndexType(path, "index2"), "uint32")
+})