|
1 | 1 | // SPDX-License-Identifier: MIT |
2 | 2 | // Copyright contributors to the kepler.gl project |
3 | 3 |
|
4 | | -import { ALL_FIELD_TYPES } from '@kepler.gl/constants'; |
5 | | -import { ProtoDatasetField } from '@kepler.gl/types'; |
| 4 | +import {ALL_FIELD_TYPES} from '@kepler.gl/constants'; |
| 5 | +import {ProtoDatasetField} from '@kepler.gl/types'; |
6 | 6 | import * as arrow from 'apache-arrow'; |
7 | | -import { console as globalConsole } from 'global/window'; |
8 | | -import { DATA_TYPES as AnalyzerDATA_TYPES } from 'type-analyzer'; |
| 7 | +import {console as globalConsole} from 'global/window'; |
| 8 | +import {DATA_TYPES as AnalyzerDATA_TYPES} from 'type-analyzer'; |
9 | 9 |
|
10 | | -import { DataContainerInterface, RangeOptions } from './data-container-interface'; |
11 | | -import { DataRow, SharedRowOptions } from './data-row'; |
| 10 | +import {DataContainerInterface, RangeOptions} from './data-container-interface'; |
| 11 | +import {DataRow, SharedRowOptions} from './data-row'; |
12 | 12 |
|
13 | 13 | type ArrowDataContainerInput = { |
14 | 14 | cols: arrow.Vector[]; |
15 | 15 | fields?: ProtoDatasetField[]; |
16 | 16 | arrowTable?: arrow.Table; |
17 | 17 | }; |
18 | 18 |
|
| 19 | +/** |
| 20 | + * Check if table is an ArrowTable object. |
| 21 | + * |
| 22 | + * We use duck-typing instead of `instanceof arrow.Table` because DuckDB loads its own |
| 23 | + * bundled version of Apache Arrow. When DuckDB creates Arrow tables, they are instances |
| 24 | + * of DuckDB's Arrow.Table class, not the Arrow.Table class from our application's |
| 25 | + * apache-arrow package. This causes `instanceof` checks to fail even though the objects |
| 26 | + * are functionally equivalent Arrow tables. |
| 27 | + * |
| 28 | + * @param data - object to check |
| 29 | + * @returns true if data is an ArrowTable object (type guarded) |
| 30 | + */ |
| 31 | +export function isArrowTable(data: any): data is arrow.Table { |
| 32 | + return ( |
| 33 | + typeof data === 'object' && |
| 34 | + data !== null && |
| 35 | + 'schema' in data && |
| 36 | + 'getChildAt' in data && |
| 37 | + typeof data.getChildAt === 'function' && |
| 38 | + 'batches' in data && |
| 39 | + Array.isArray(data.batches) |
| 40 | + ); |
| 41 | +} |
| 42 | + |
| 43 | +/** |
| 44 | + * Check if data is an ArrowVector object. |
| 45 | + * Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version. |
| 46 | + * |
| 47 | + * @param data - object to check |
| 48 | + * @returns true if data is an ArrowVector object (type guarded) |
| 49 | + */ |
| 50 | +export function isArrowVector(data: any): data is arrow.Vector { |
| 51 | + return ( |
| 52 | + typeof data === 'object' && |
| 53 | + data !== null && |
| 54 | + 'type' in data && |
| 55 | + 'length' in data && |
| 56 | + typeof data.length === 'number' && |
| 57 | + 'get' in data && |
| 58 | + typeof data.get === 'function' && |
| 59 | + 'data' in data && |
| 60 | + Array.isArray(data.data) |
| 61 | + ); |
| 62 | +} |
| 63 | + |
| 64 | +/** |
| 65 | + * Check if data is an Arrow FixedSizeList DataType. |
| 66 | + * Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version. |
| 67 | + * |
| 68 | + * @param data - object to check |
| 69 | + * @returns true if data is an Arrow FixedSizeList DataType (type guarded) |
| 70 | + */ |
| 71 | +export function isArrowFixedSizeList(data: any): data is arrow.FixedSizeList { |
| 72 | + return ( |
| 73 | + typeof data === 'object' && |
| 74 | + data !== null && |
| 75 | + 'typeId' in data && |
| 76 | + 'listSize' in data && |
| 77 | + typeof data.listSize === 'number' && |
| 78 | + 'children' in data && |
| 79 | + Array.isArray(data.children) |
| 80 | + ); |
| 81 | +} |
| 82 | + |
| 83 | +/** |
| 84 | + * Check if data is an Arrow Struct DataType. |
| 85 | + * Uses duck-typing instead of `instanceof` to handle DuckDB's bundled Arrow version. |
| 86 | + * |
| 87 | + * @param data - object to check |
| 88 | + * @returns true if data is an Arrow Struct DataType (type guarded) |
| 89 | + */ |
| 90 | +export function isArrowStruct(data: any): data is arrow.Struct { |
| 91 | + return ( |
| 92 | + typeof data === 'object' && |
| 93 | + data !== null && |
| 94 | + 'typeId' in data && |
| 95 | + 'children' in data && |
| 96 | + Array.isArray(data.children) && |
| 97 | + !('listSize' in data) |
| 98 | + ); |
| 99 | +} |
19 | 100 |
|
20 | 101 | /** |
21 | 102 | * @param dataContainer |
@@ -89,13 +170,20 @@ export class ArrowDataContainer implements DataContainerInterface { |
89 | 170 | return this._arrowTable; |
90 | 171 | } |
91 | 172 |
|
92 | | - update(updateData: arrow.Vector<any>[]) { |
93 | | - this._cols = updateData; |
94 | | - this._numColumns = this._cols.length; |
95 | | - this._numRows = this._cols[0].length; |
96 | | - this._numChunks = this._cols[0].data.length; |
97 | | - |
98 | | - this._arrowTable = this._createTable(); |
| 173 | + update(updateData: arrow.Vector<any>[] | arrow.Table) { |
| 174 | + const isArrow = isArrowTable(updateData); |
| 175 | + if (isArrow) { |
| 176 | + this._cols = Array.from( |
| 177 | + {length: updateData.numCols}, |
| 178 | + (_, i) => updateData.getChildAt(i) as arrow.Vector |
| 179 | + ).filter(col => col); |
| 180 | + } else { |
| 181 | + this._cols = updateData; |
| 182 | + } |
| 183 | + this._numColumns = this._cols?.length ?? 0; |
| 184 | + this._numRows = this._cols?.[0]?.length ?? 0; |
| 185 | + this._numChunks = this._cols?.[0]?.data?.length ?? 0; |
| 186 | + this._arrowTable = isArrow ? updateData : this._createTable(); |
99 | 187 |
|
100 | 188 | // cache column data to make valueAt() faster |
101 | 189 | // this._colData = this._cols.map(c => c.toArray()); |
|
0 commit comments