Fix DuckDB arrow type columns

Signed-off-by: Andrew Stein <steinlink@gmail.com>

Author: texodus

.github/workflows/build.yaml

@@ -28,7 +28,7 @@ on:
             - docs/
             - examples/
             - rust/perspective-python/README.md
-    pull_request:
+    pull_request_target:
         branches:
             - master
     workflow_dispatch:

Cargo.lock

@@ -87,6 +87,8 @@ export function renderCartesianFrame(
     const facetMode = chart._facetConfig.facet_mode;
     const useGrid = hasSplits && facetMode === "grid";
 
+    chart.computeEffectiveFacetFlags();
+
     // Legend appears only when the user wired a color column with a
     // non-degenerate range. `split_by` alone no longer forces a
     // legend — faceting is the axis of splitting, not coloring.

packages/viewer-charts/src/ts/charts/cartesian/cartesian-render.ts

@@ -68,6 +68,7 @@ protobuf-src = { version = "2.1.1", optional = true }
 arrow-array = { version = "57.3.0", default-features = false }
 arrow-ipc = { version = "57.3.0", default-features = false }
 arrow-schema = { version = "57.3.0", default-features = false }
+arrow-select = { version = "57.3.0", default-features = false }
 async-lock = { version = "2.5.0" }
 futures = { version = "0.3.28" }
 indexmap = { version = "2.2.6", features = ["serde"] }

rust/perspective-client/Cargo.toml

@@ -64,6 +64,21 @@ pub enum VirtualDataCell {
     RowPath(Vec<Scalar>),
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum RowPathStyle {
+    /// Legacy: emit a single `__ROW_PATH__` sidecar (per-row nested
+    /// array in `render_to_rows`, array-of-arrays in
+    /// `render_to_columns_json`). `__ROW_PATH_N__` per-level columns
+    /// are filtered out. Matches the native engine's `to_json` /
+    /// `to_columns` shape.
+    Sidecar,
+
+    /// Native: emit per-level `__ROW_PATH_0__`, `__ROW_PATH_1__`, …
+    /// columns directly. No `__ROW_PATH__` sidecar. Matches the native
+    /// engine's Arrow IPC, CSV, and NDJSON shapes.
+    PerLevel,
+}
+
 /// Trait for types that can be written to a [`ColumnBuilder`] which
 /// enforces sequential construction.
 ///
@@ -593,14 +608,16 @@ impl VirtualDataSlice {
     /// to Perspective-compatible types.
     pub fn from_arrow_ipc(&mut self, ipc: &[u8]) -> Result<(), Box<dyn Error>> {
         let cursor = std::io::Cursor::new(ipc);
-        let batch = if &ipc[0..6] == "ARROW1".as_bytes() {
-            FileReader::try_new(cursor, None)?
-                .next()
-                .ok_or("Arrow IPC stream contained no record batches")??
+        let batches: Vec<RecordBatch> = if &ipc[0..6] == "ARROW1".as_bytes() {
+            FileReader::try_new(cursor, None)?.collect::<Result<Vec<_>, _>>()?
         } else {
-            StreamReader::try_new(cursor, None)?
-                .next()
-                .ok_or("Arrow IPC stream contained no record batches")??
+            StreamReader::try_new(cursor, None)?.collect::<Result<Vec<_>, _>>()?
+        };
+
+        let batch = match batches.len() {
+            0 => return Err("Arrow IPC stream contained no record batches".into()),
+            1 => batches.into_iter().next().unwrap(),
+            _ => arrow_select::concat::concat_batches(&batches[0].schema(), &batches)?,
         };
 
         let has_group_by = !self.config.group_by.is_empty();
@@ -764,17 +781,24 @@ impl VirtualDataSlice {
 
     /// Converts the columnar data to a row-oriented representation for JSON
     /// serialization.
-    pub(crate) fn render_to_rows(&mut self) -> Vec<IndexMap<String, VirtualDataCell>> {
+    ///
+    /// `style` selects between the legacy `__ROW_PATH__` sidecar
+    /// (`Sidecar`, used by `to_json`) and the native per-level
+    /// `__ROW_PATH_N__` columns (`PerLevel`, used by `to_csv` /
+    /// `to_ndjson`). See [`RowPathStyle`] for the deprecation plan.
+    pub(crate) fn render_to_rows(
+        &mut self,
+        style: RowPathStyle,
+    ) -> Vec<IndexMap<String, VirtualDataCell>> {
         let batch = self.freeze().clone();
         let num_rows = batch.num_rows();
         let schema = batch.schema();
 
         (0..num_rows)
             .map(|row_idx| {
                 let mut row = IndexMap::new();
-
-                // Add RowPath column first if present
-                if let Some(ref rp) = self.row_path
+                if style == RowPathStyle::Sidecar
+                    && let Some(ref rp) = self.row_path
                     && row_idx < rp.len()
                 {
                     row.insert(
@@ -783,8 +807,11 @@ impl VirtualDataSlice {
                     );
                 }
 
-                // Add Arrow columns
                 for (col_idx, field) in schema.fields().iter().enumerate() {
+                    if style == RowPathStyle::Sidecar && field.name().starts_with("__ROW_PATH_") {
+                        continue;
+                    }
+
                     let col = batch.column(col_idx);
                     let cell = if col.is_null(row_idx) {
                         match field.data_type() {
@@ -869,17 +896,31 @@ impl VirtualDataSlice {
     }
 
     /// Serializes the data to a column-oriented JSON string.
-    pub fn render_to_columns_json(&mut self) -> Result<String, Box<dyn Error>> {
+    ///
+    /// `style` selects between the legacy `__ROW_PATH__` sidecar
+    /// (`Sidecar`, used by `to_columns`) and the native per-level
+    /// `__ROW_PATH_N__` columns (`PerLevel`, currently unused — reserved
+    /// for the future deprecation of `__ROW_PATH__`). See
+    /// [`RowPathStyle`] for context.
+    pub fn render_to_columns_json(
+        &mut self,
+        style: RowPathStyle,
+    ) -> Result<String, Box<dyn Error>> {
         let batch = self.freeze().clone();
         let schema = batch.schema();
         let mut map = serde_json::Map::new();
 
-        // Add RowPath if present
-        if let Some(ref rp) = self.row_path {
+        if style == RowPathStyle::Sidecar
+            && let Some(ref rp) = self.row_path
+        {
             map.insert("__ROW_PATH__".to_string(), serde_json::to_value(rp)?);
         }
 
         for (col_idx, field) in schema.fields().iter().enumerate() {
+            if style == RowPathStyle::Sidecar && field.name().starts_with("__ROW_PATH_") {
+                continue;
+            }
+
             let col = batch.column(col_idx);
             let num_rows = col.len();
             let values: serde_json::Value = match field.data_type() {

rust/perspective-client/src/rust/virtual_server/data.rs

@@ -22,7 +22,7 @@ mod generic_sql_model;
 mod handler;
 mod server;
 
-pub use data::{SetVirtualDataColumn, VirtualDataCell, VirtualDataSlice};
+pub use data::{RowPathStyle, SetVirtualDataColumn, VirtualDataCell, VirtualDataSlice};
 pub use error::{ResultExt, VirtualServerError};
 pub use features::{AggSpec, Features};
 pub use generic_sql_model::{

rust/perspective-client/src/rust/virtual_server/mod.rs

@@ -16,6 +16,7 @@ use indexmap::IndexMap;
 use prost::Message as ProstMessage;
 use prost::bytes::{Bytes, BytesMut};
 
+use super::data::RowPathStyle;
 use super::error::VirtualServerError;
 use super::handler::VirtualServerHandler;
 use crate::config::{ViewConfig, ViewConfigUpdate};
@@ -322,7 +323,7 @@ impl<T: VirtualServerHandler> VirtualServer<T> {
                     .view_get_data(msg.entity_id.as_str(), config, &schema, &viewport)
                     .await?;
 
-                let rows = cols.render_to_rows();
+                let rows = cols.render_to_rows(RowPathStyle::PerLevel);
                 let mut csv = String::new();
                 if let Some(first_row) = rows.first() {
                     let headers: Vec<&str> = first_row.keys().map(|k| k.as_str()).collect();
@@ -350,7 +351,7 @@ impl<T: VirtualServerHandler> VirtualServer<T> {
                     .view_get_data(msg.entity_id.as_str(), config, &schema, &viewport)
                     .await?;
 
-                let rows = cols.render_to_rows();
+                let rows = cols.render_to_rows(RowPathStyle::PerLevel);
                 let ndjson_string = rows
                     .iter()
                     .map(serde_json::to_string)
@@ -369,7 +370,7 @@ impl<T: VirtualServerHandler> VirtualServer<T> {
                     .view_get_data(msg.entity_id.as_str(), config, &schema, &viewport)
                     .await?;
 
-                let rows = cols.render_to_rows();
+                let rows = cols.render_to_rows(RowPathStyle::Sidecar);
                 let json_string = serde_json::to_string(&rows)
                     .map_err(|e| VirtualServerError::InvalidJSON(std::sync::Arc::new(e)))?;
 
@@ -385,7 +386,7 @@ impl<T: VirtualServerHandler> VirtualServer<T> {
                     .await?;
 
                 let json_string = cols
-                    .render_to_columns_json()
+                    .render_to_columns_json(RowPathStyle::Sidecar)
                     .map_err(|e| VirtualServerError::Other(e.to_string()))?;
 
                 respond!(msg, ViewToColumnsStringResp { json_string })

rust/perspective-client/src/rust/virtual_server/server.rs

@@ -17,7 +17,8 @@ use chrono::{DateTime, TimeZone, Utc};
 use indexmap::IndexMap;
 use perspective_client::proto::{ColumnType, HostedTable};
 use perspective_client::virtual_server::{
-    Features, ResultExt, VirtualDataSlice, VirtualServer, VirtualServerFuture, VirtualServerHandler,
+    Features, ResultExt, RowPathStyle, VirtualDataSlice, VirtualServer, VirtualServerFuture,
+    VirtualServerHandler,
 };
 use pyo3::exceptions::PyValueError;
 use pyo3::types::{
@@ -415,7 +416,7 @@ impl PyVirtualDataSlice {
         self.0
             .lock()
             .unwrap()
-            .render_to_columns_json()
+            .render_to_columns_json(RowPathStyle::Sidecar)
             .map_err(|e| PyValueError::new_err(e.to_string()))
     }
 

rust/perspective-python/src/server/virtual_server_sync.rs

@@ -185,6 +185,9 @@ convert_type(const std::string& src) {
     if (src == "timestamp") {
         return DTYPE_TIME;
     }
+    if (src == "time32" || src == "time64" || src == "time32[s]" ) {
+        return DTYPE_UINT32;
+    }
     if (src == "date32" || src == "date64") {
         return DTYPE_DATE;
     }
@@ -776,6 +779,17 @@ copy_array(
                 dest->set_valid(i, false);
             }
         } break;
+        case arrow::Time32Type::type_id: {
+            auto scol = std::static_pointer_cast<arrow::Time32Array>(src);
+            std::memcpy(
+                dest->get_nth<std::uint64_t>(offset),
+                (void*)scol->raw_values(),
+                len * 8
+            );
+        } break;
+        // case arrow::Type {
+            
+        // } break;
         default: {
             std::stringstream ss;
             std::string arrow_type = src->type()->ToString();

rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp

@@ -85,6 +85,8 @@ const DEVICE_OPTIONS = {
                 // 2× perf hit is under budget for the test suite.
                 "--use-gl=swiftshader",
                 "--use-angle=swiftshader",
+                "--force-color-profile=srgb",
+                "--disable-lcd-text",
             ],
         },
     },