[Feat] GeoArrow incremental rendering (2) [DRAFT]

examples/demo-app/src/cloud-providers/foursquare/foursquare-icon.js

@@ -1,3 +1,23 @@
+// Copyright (c) 2023 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
 import React, {Component} from 'react';
 import {Icons} from '@kepler.gl/components';
 import PropTypes from 'prop-types';

examples/demo-app/src/cloud-providers/foursquare/foursquare-provider.js

@@ -1,3 +1,23 @@
+// Copyright (c) 2023 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
 import FSQIcon from './foursquare-icon';
 import {Provider, KEPLER_FORMAT} from '@kepler.gl/cloud-providers';
 import {Auth0Client} from '@auth0/auth0-spa-js';

package.json

@@ -114,7 +114,7 @@
     "@hubble.gl/core": "1.2.0-alpha.6",
     "@hubble.gl/react": "1.2.0-alpha.6",
     "@kepler.gl/components": "3.0.0-alpha.1",
-    "@loaders.gl/polyfills": "^4.0.3",
+    "@loaders.gl/polyfills": "^4.1.0-alpha.2",
     "@types/mapbox__geo-viewport": "^0.4.1",
     "@typescript-eslint/parser": "^5.27.0",
     "eslint-config-developit": "^1.2.0",
@@ -142,7 +142,7 @@
     "@babel/traverse": "^7.12.1",
     "@cfaester/enzyme-adapter-react-18": "^0.7.0",
     "@deck.gl/test-utils": "^8.9.27",
-    "@loaders.gl/polyfills": "^4.0.3",
+    "@loaders.gl/polyfills": "^4.1.0-alpha.2",
     "@luma.gl/test-utils": "^8.5.20",
     "@nebula.gl/layers": "1.0.2-alpha.1",
     "@probe.gl/env": "^3.5.0",
@@ -216,12 +216,12 @@
     "webpack-stats-plugin": "^0.2.1"
   },
   "resolutions": {
-    "@loaders.gl/core": "^4.0.3",
-    "@loaders.gl/csv": "^4.0.3",
-    "@loaders.gl/gltf": "^4.0.3",
-    "@loaders.gl/json": "^4.0.3",
-    "@loaders.gl/loader-utils": "^4.0.3",
-    "@loaders.gl/polyfills": "^4.0.3",
+    "@loaders.gl/core": "^4.1.0-alpha.2",
+    "@loaders.gl/csv": "^4.1.0-alpha.2",
+    "@loaders.gl/gltf": "^4.1.0-alpha.2",
+    "@loaders.gl/json": "^4.1.0-alpha.2",
+    "@loaders.gl/loader-utils": "^4.1.0-alpha.2",
+    "@loaders.gl/polyfills": "^4.1.0-alpha.2",
     "@luma.gl/constants": "8.5.20",
     "@luma.gl/core": "8.5.20",
     "@luma.gl/experimental": "8.5.20",

src/actions/src/action-types.ts

@@ -103,6 +103,7 @@ export const ActionTypes = {
   SET_FILTER_PLOT: `${ACTION_PREFIX}SET_FILTER_PLOT`,
   LOAD_FILES: `${ACTION_PREFIX}LOAD_FILES`,
   LOAD_NEXT_FILE: `${ACTION_PREFIX}LOAD_NEXT_FILE`,
+  LOAD_BATCH_DATA_SUCCESS: `${ACTION_PREFIX}LOAD_BATCH_DATA_SUCCESS`,
   LOAD_FILE_STEP_SUCCESS: `${ACTION_PREFIX}LOAD_FILE_STEP_SUCCESS`,
   LOAD_FILES_ERR: `${ACTION_PREFIX}LOAD_FILES_ERR`,
   LOAD_FILES_SUCCESS: `${ACTION_PREFIX}LOAD_FILES_SUCCESS`,
@@ -115,6 +116,7 @@ export const ActionTypes = {
   COPY_TABLE_COLUMN: `${ACTION_PREFIX}COPY_TABLE_COLUMN`,
   SET_COLUMN_DISPLAY_FORMAT: `${ACTION_PREFIX}SET_COLUMN_DISPLAY_FORMAT`,
   NEXT_FILE_BATCH: `${ACTION_PREFIX}NEXT_FILE_BATCH`,
+  PROGRESSIVE_LOAD_COMPLETED: `${ACTION_PREFIX}PROGRESSIVE_LOAD_COMPLETED`,
   PROCESS_FILE_CONTENT: `${ACTION_PREFIX}PROCESS_FILE_CONTENT`,
   UPDATE_TABLE_COLOR: `${ACTION_PREFIX}UPDATE_TABLE_COLOR`,
   ADD_EFFECT: `${ACTION_PREFIX}ADD_EFFECT`,

src/actions/src/vis-state-actions.ts

@@ -19,7 +19,8 @@
 // THE SOFTWARE.
 
 // vis-state-reducer
-import {PickInfo} from '@deck.gl/core/lib/deck';
+import { PickInfo } from '@deck.gl/core/lib/deck';
+import {ParseGeoArrowResult} from '@loaders.gl/arrow';
 import {default as ActionTypes} from './action-types';
 import {FileCacheItem} from '@kepler.gl/processors';
 import {Layer, LayerBaseConfig} from '@kepler.gl/layers';
@@ -37,7 +38,8 @@ import {
   Filter,
   ParsedConfig,
   ParsedLayer,
-  EffectPropsPartial
+  EffectPropsPartial,
+  ProtoDataset
 } from '@kepler.gl/types';
 // TODO - import LoaderObject type from @loaders.gl/core when supported
 // TODO - import LoadOptions type from @loaders.gl/core when supported
@@ -1240,6 +1242,20 @@ export function loadFileStepSuccess({
   };
 }
 
+export function loadBatchDataSuccess({
+  fileName,
+  fileCache
+}: {
+  fileName: string;
+  fileCache: FileCacheItem[];
+}): Merge<LoadFileStepSuccessAction, {type: typeof ActionTypes.LOAD_BATCH_DATA_SUCCESS}> {
+  return {
+    type: ActionTypes.LOAD_BATCH_DATA_SUCCESS,
+    fileName,
+    fileCache
+  };
+}
+
 export type LoadFilesErrUpdaterAction = {
   fileName: string;
   error: any;
@@ -1433,6 +1449,21 @@ export function nextFileBatch(
   };
 }
 
+export type ProgressiveLoadCompletedUpdaterAction = {
+  payload: {
+    parsedData: Record<string, { fieldIndex: number; parsedGeoArrowData: ParseGeoArrowResult}>;
+    inputData: ProtoDataset[];
+  };
+};
+export function progressiveLoadCompleted(
+  payload: ProgressiveLoadCompletedUpdaterAction['payload']
+): Merge<ProgressiveLoadCompletedUpdaterAction, {type: typeof ActionTypes.PROGRESSIVE_LOAD_COMPLETED}> {
+  return {
+    type: ActionTypes.PROGRESSIVE_LOAD_COMPLETED,
+    payload
+  };
+}
+
 export type ProcessFileContentUpdaterAction = {
   payload: {
     content: FileContent;

src/layers/package.json

@@ -42,11 +42,11 @@
     "@kepler.gl/table": "3.0.0-alpha.1",
     "@kepler.gl/types": "3.0.0-alpha.1",
     "@kepler.gl/utils": "3.0.0-alpha.1",
-    "@loaders.gl/arrow": "^4.0.3",
-    "@loaders.gl/core": "^4.0.3",
-    "@loaders.gl/gis": "^4.0.3",
-    "@loaders.gl/gltf": "^4.0.3",
-    "@loaders.gl/wkt": "^4.0.3",
+    "@loaders.gl/arrow": "^4.1.0-alpha.2",
+    "@loaders.gl/core": "^4.1.0-alpha.2",
+    "@loaders.gl/gis": "^4.1.0-alpha.2",
+    "@loaders.gl/gltf": "^4.1.0-alpha.2",
+    "@loaders.gl/wkt": "^4.1.0-alpha.2",
     "@luma.gl/constants": "^8.5.20",
     "@mapbox/geojson-normalize": "0.0.1",
     "@nebula.gl/edit-modes": "1.0.2-alpha.1",

src/layers/src/base-layer.ts

@@ -1104,7 +1104,7 @@ class Layer {
     const dataUpdateTriggers = this.getDataUpdateTriggers(layerDataset);
     const triggerChanged = this.getChangedTriggers(dataUpdateTriggers);
 
-    if (triggerChanged && triggerChanged.getMeta) {
+    if (triggerChanged && (triggerChanged.getMeta || triggerChanged.getData)) {
       this.updateLayerMeta(dataContainer, getPosition);
     }
 

src/layers/src/geojson-layer/geojson-layer.ts

@@ -372,7 +372,7 @@ export default class GeoJsonLayer extends Layer {
       this.filteredIndex[filteredIndex[i]] = 1;
     }
 
-    this.filteredIndexTrigger = filteredIndex;
+    // this.filteredIndexTrigger = filteredIndex;
 
     // for arrow, always return full dataToFeature instead of a filtered one, so there is no need to update attributes in GPU
     // for geojson, this should work as well and more efficient. But we need to update some test cases e.g. #GeojsonLayer -> formatLayerData
@@ -419,20 +419,41 @@ export default class GeoJsonLayer extends Layer {
     const getGeoColumn = geoColumnAccessor(this.config.columns);
     const getGeoField = geoFieldAccessor(this.config.columns);
 
-    if (this.dataToFeature.length === 0) {
-      const updateLayerMetaFunc =
-        dataContainer instanceof ArrowDataContainer
-          ? getGeojsonLayerMetaFromArrow
-          : getGeojsonLayerMeta;
-      const {dataToFeature, bounds, fixedRadius, featureTypes} = updateLayerMetaFunc({
-        dataContainer,
-        getFeature,
-        getGeoColumn,
-        getGeoField
-      });
-
-      this.dataToFeature = dataToFeature;
-      this.updateMeta({bounds, fixedRadius, featureTypes});
+    if (dataContainer instanceof ArrowDataContainer) {
+      const {geojson} = this.config.columns;
+      if (this.dataToFeature.length < dataContainer.numChunks()) {
+        const {binaryGeometries, bounds, featureTypes} =
+          this.dataToFeature.length === 0
+            ? getGeojsonLayerMetaFromArrow({
+                dataContainer,
+                getGeoColumn,
+                getGeoField,
+                chunkIndex: 0
+              })
+            : dataContainer.getBinaryData(geojson.fieldIdx);
+
+        if (this.dataToFeature.length === 0) {
+          dataContainer.updateBinaryData(geojson.fieldIdx, 0, {
+            binaryGeometries,
+            bounds,
+            featureTypes
+          });
+          // not update bounds for every batch, to avoid interrupt user interacts with map while loading the map incrementally
+          // since there is no feature.properties.radius, we set fixedRadius to false
+          const fixedRadius = false;
+          this.updateMeta({bounds, fixedRadius, featureTypes});
+        }
+        this.dataToFeature = binaryGeometries;
+      }
+    } else {
+      if (this.dataToFeature.length === 0) {
+        const {dataToFeature, bounds, fixedRadius, featureTypes} = getGeojsonLayerMeta({
+          dataContainer,
+          getFeature
+        });
+        this.dataToFeature = dataToFeature;
+        this.updateMeta({bounds, fixedRadius, featureTypes});
+      }
     }
   }
 

src/layers/src/layer-utils.ts

@@ -23,7 +23,12 @@ import {Feature, BBox} from 'geojson';
 import {Field, FieldPair} from '@kepler.gl/types';
 import {DataContainerInterface} from '@kepler.gl/utils';
 import {BinaryFeatures} from '@loaders.gl/schema';
-import {getBinaryGeometriesFromArrow, parseGeometryFromArrow} from '@loaders.gl/arrow';
+import {
+  getBinaryGeometriesFromArrow,
+  parseGeometryFromArrow,
+  BinaryGeometriesFromArrowOptions,
+  BinaryDataFromGeoArrow
+} from '@loaders.gl/arrow';
 
 import {DeckGlGeoTypes} from './geojson-layer/geojson-utils';
 
@@ -52,30 +57,34 @@ export type GeojsonLayerMetaProps = {
 export function getGeojsonLayerMetaFromArrow({
   dataContainer,
   getGeoColumn,
-  getGeoField
+  getGeoField,
+  chunkIndex
 }: {
   dataContainer: DataContainerInterface;
   getGeoColumn: (dataContainer: DataContainerInterface) => unknown;
   getGeoField: (dataContainer: DataContainerInterface) => Field | null;
-}): GeojsonLayerMetaProps {
+  chunkIndex?: number;
+}): BinaryDataFromGeoArrow & {meanCenters?: number[][]} {
   const geoColumn = getGeoColumn(dataContainer) as arrow.Vector;
   const arrowField = getGeoField(dataContainer);
 
   const encoding = arrowField?.metadata?.get('ARROW:extension:name');
+  const options: BinaryGeometriesFromArrowOptions = {
+    ...(chunkIndex !== undefined && chunkIndex >= 0 ? {chunkIndex} : {}),
+    triangulate: true,
+    calculateMeanCenters: false
+  };
   // create binary data from arrow data for GeoJsonLayer
   const {binaryGeometries, featureTypes, bounds} = getBinaryGeometriesFromArrow(
     geoColumn,
-    encoding
+    encoding,
+    options
   );
 
-  // since there is no feature.properties.radius, we set fixedRadius to false
-  const fixedRadius = false;
-
   return {
-    dataToFeature: binaryGeometries,
+    binaryGeometries,
     featureTypes,
-    bounds,
-    fixedRadius
+    bounds
   };
 }
 
@@ -105,10 +114,7 @@ export function getHoveredObjectFromArrow(
     const field = fieldAccessor(dataContainer);
     const encoding = field?.metadata?.get('ARROW:extension:name');
 
-    const hoveredFeature = parseGeometryFromArrow({
-      encoding,
-      data: rawGeometry
-    });
+    const hoveredFeature = parseGeometryFromArrow(rawGeometry, encoding);
 
     const properties = dataContainer.rowAsArray(objectInfo.index).reduce((prev, cur, i) => {
       const fieldName = dataContainer?.getField?.(i).name;
@@ -120,7 +126,8 @@ export function getHoveredObjectFromArrow(
 
     return hoveredFeature
       ? {
-          ...hoveredFeature,
+          type: 'Feature',
+          geometry: hoveredFeature,
           properties: {
             ...properties,
             index: objectInfo.index

src/processors/package.json

@@ -35,11 +35,11 @@
     "@kepler.gl/schemas": "3.0.0-alpha.1",
     "@kepler.gl/types": "3.0.0-alpha.1",
     "@kepler.gl/utils": "3.0.0-alpha.1",
-    "@loaders.gl/arrow": "^4.0.3",
-    "@loaders.gl/core": "^4.0.3",
-    "@loaders.gl/csv": "^4.0.3",
-    "@loaders.gl/json": "^4.0.3",
-    "@loaders.gl/loader-utils": "^4.0.3",
+    "@loaders.gl/arrow": "^4.1.0-alpha.2",
+    "@loaders.gl/core": "^4.1.0-alpha.2",
+    "@loaders.gl/csv": "^4.1.0-alpha.2",
+    "@loaders.gl/json": "^4.1.0-alpha.2",
+    "@loaders.gl/loader-utils": "^4.1.0-alpha.2",
     "@mapbox/geojson-normalize": "0.0.1",
     "@nebula.gl/edit-modes": "1.0.2-alpha.1",
     "@turf/helpers": "^6.1.4",