fix: Allow passing arrow tables to ArrowDataContainer

Signed-off-by: Ilya Boyandin <ilyabo@gmail.com>

Author: ilyabo

src/processors/src/file-handler.ts

@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
-import * as arrow from 'apache-arrow';
 import {parseInBatches} from '@loaders.gl/core';
 import {JSONLoader, _JSONPath} from '@loaders.gl/json';
 import {CSVLoader} from '@loaders.gl/csv';
@@ -12,7 +11,8 @@ import {
   isPlainObject,
   generateHashIdFromString,
   getApplicationConfig,
-  getError
+  getError,
+  isArrowTable
 } from '@kepler.gl/utils';
 import {generateHashId} from '@kepler.gl/common-utils';
 import {DATASET_FORMATS} from '@kepler.gl/constants';
@@ -68,14 +68,7 @@ export type ProcessFileDataContent = {
   metadata?: Map<string, string>;
 };
 
-/**
- * check if table is an ArrowTable object
- * @param table - object to check
- * @returns {boolean} - true if table is an ArrowTable object type guarded
- */
-export function isArrowTable(table: any): table is arrow.Table {
-  return Boolean(table instanceof arrow.Table);
-}
+export {isArrowTable};
 
 /**
  * check if data is an ArrowData object, which is an array of RecordBatch

src/table/src/kepler-table.ts

@@ -241,7 +241,7 @@ class KeplerTable<F extends Field = Field> {
    * @param data - new data e.g. the arrow data with new batches loaded
    */
   async update(data: ProtoDataset['data']) {
-    const dataContainerData = data.cols ? data.cols : data.rows;
+    const dataContainerData = data.arrowTable ?? data.cols ?? data.rows;
     this.dataContainer.update?.(dataContainerData);
     this.allIndexes = this.dataContainer.getPlainIndex();
     this.filteredIndex = this.allIndexes;

src/utils/src/arrow-data-container.ts

@@ -1,21 +1,37 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
-import { ALL_FIELD_TYPES } from '@kepler.gl/constants';
-import { ProtoDatasetField } from '@kepler.gl/types';
+import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
+import {ProtoDatasetField} from '@kepler.gl/types';
 import * as arrow from 'apache-arrow';
-import { console as globalConsole } from 'global/window';
-import { DATA_TYPES as AnalyzerDATA_TYPES } from 'type-analyzer';
+import {console as globalConsole} from 'global/window';
+import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer';
 
-import { DataContainerInterface, RangeOptions } from './data-container-interface';
-import { DataRow, SharedRowOptions } from './data-row';
+import {DataContainerInterface, RangeOptions} from './data-container-interface';
+import {DataRow, SharedRowOptions} from './data-row';
 
 type ArrowDataContainerInput = {
   cols: arrow.Vector[];
   fields?: ProtoDatasetField[];
   arrowTable?: arrow.Table;
 };
 
+/**
+ * check if table is an ArrowTable object
+ * @param table - object to check
+ * @returns {boolean} - true if table is an ArrowTable object type guarded
+ */
+export function isArrowTable(data: any): data is arrow.Table {
+  return (
+    typeof data === 'object' &&
+    data !== null &&
+    'schema' in data &&
+    'getChildAt' in data &&
+    typeof data.getChildAt === 'function' &&
+    'batches' in data &&
+    Array.isArray(data.batches)
+  );
+}
 
 /**
  * @param dataContainer
@@ -89,13 +105,24 @@ export class ArrowDataContainer implements DataContainerInterface {
     return this._arrowTable;
   }
 
-  update(updateData: arrow.Vector<any>[]) {
-    this._cols = updateData;
+  update(updateData: arrow.Vector<any>[] | arrow.Table) {
+    const isArrow = isArrowTable(updateData);
+    if (isArrow) {
+      this._cols = Array.from(
+        {length: updateData.numCols},
+        (_, i) => updateData.getChildAt(i) as arrow.Vector
+      ).filter(col => col);
+    } else {
+      this._cols = updateData;
+    }
     this._numColumns = this._cols.length;
     this._numRows = this._cols[0].length;
     this._numChunks = this._cols[0].data.length;
-
-    this._arrowTable = this._createTable();
+    if (isArrow) {
+      this._arrowTable = updateData;
+    } else {
+      this._arrowTable = this._createTable();
+    }
 
     // cache column data to make valueAt() faster
     // this._colData = this._cols.map(c => c.toArray());

src/utils/src/data-container-interface.ts

@@ -3,6 +3,7 @@
 
 import {ProtoDatasetField} from '@kepler.gl/types';
 import {DataRow, SharedRowOptions} from './data-row';
+import * as arrow from 'apache-arrow';
 
 /**
  * Specifies a range of rows of a data container that should be processed.
@@ -16,7 +17,7 @@ export interface DataContainerInterface {
    * Updates the data container with new data.
    * @param updateData updated data, e.g. for arrow data container, it's an array of arrow columns; for row data container, it's an array of rows.
    */
-  update?(updateData: any[]): void;
+  update?(updateData: any[] | arrow.Table): void;
 
   /**
    * Returns the number of rows in the data container.

src/utils/src/index.ts

@@ -127,7 +127,8 @@ export * from './map-utils';
 export {
   ArrowDataContainer,
   arrowDataTypeToAnalyzerDataType,
-  arrowDataTypeToFieldType
+  arrowDataTypeToFieldType,
+  isArrowTable
 } from './arrow-data-container';
 export type {DataContainerInterface} from './data-container-interface';
 export {