Skip to content

Commit 3b6e904

Browse files
igorDykhtaigorDykhta
authored
[fix] improvements for raster tile layer (#3124)
* fixes Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * fixes Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * fix broken config Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * raster-form improvements Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * increase skirt height Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * switch to enable 3D terrain in top view Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * fix Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * nit Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * throttle requests per raster server Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * nit Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * nit Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> * nit Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com> --------- Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>
1 parent 4237b0a commit 3b6e904

13 files changed

Lines changed: 333 additions & 129 deletions

File tree

‎src/common-utils/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ export * from './data';
55
export * from './data-type';
66
export * from './string';
77
export * from './url';
8+
export * from './promise';
89

910
export {getCentroid, getHexFields, h3IsValid, idToPolygonGeo} from './h3-utils';
1011
export type {Centroid} from './h3-utils';

‎src/common-utils/src/promise.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// SPDX-License-Identifier: MIT
2+
// Copyright contributors to the kepler.gl project
3+
4+
/**
5+
* Utility function to create a promise that resolves after a specified number of milliseconds
6+
* @param ms number of milliseconds to wait
7+
* @returns Promise that resolves after the specified delay
8+
*/
9+
export const sleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));

‎src/components/src/modals/tilesets-modals/tileset-raster-form.tsx‎

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -67,27 +67,36 @@ const parseMetadataAllowCollections = (
6767

6868
const RasterTileForm: React.FC<RasterTileFormProps> = ({setResponse}) => {
6969
const [tileName, setTileName] = useState<string>('');
70-
const [currentUrl, setCurrentUrl] = useState<string>('');
70+
const [tileNameWasModified, setTileNameWasModified] = useState<boolean>(false);
7171
const [metadataUrl, setMetadataUrl] = useState<string>('');
7272
const [rasterTileServerUrls, setRasterTileServerUrls] = useState<string>(
7373
(getApplicationConfig().rasterServerUrls || []).join(',')
7474
);
7575

76+
// Remove trailing slash to prevent issues with raster tile servers
77+
const clearedMetadataUrl = metadataUrl.endsWith('/') ? metadataUrl.slice(0, -1) : metadataUrl;
78+
7679
const onTileNameChange = useCallback(
7780
(event: React.ChangeEvent<HTMLInputElement>) => {
7881
event.preventDefault();
82+
setTileNameWasModified(true);
7983
setTileName(event.target.value);
8084
},
8185
[setTileName]
8286
);
8387

84-
const onMetadataUrlChange = useCallback((event: React.ChangeEvent<HTMLInputElement>) => {
85-
event.preventDefault();
86-
const {value} = event.target;
87-
setMetadataUrl(value);
88-
setTileName(value.split('/').pop() || '');
89-
setCurrentUrl(value);
90-
}, []);
88+
const onMetadataUrlChange = useCallback(
89+
(event: React.ChangeEvent<HTMLInputElement>) => {
90+
event.preventDefault();
91+
const {value} = event.target;
92+
setMetadataUrl(value);
93+
94+
if (!tileNameWasModified) {
95+
setTileName(value.split('/').filter(Boolean).pop() || '');
96+
}
97+
},
98+
[tileNameWasModified]
99+
);
91100

92101
const onRasterTileServerUrlsChange = useCallback(
93102
(event: React.ChangeEvent<HTMLInputElement>) => {
@@ -102,13 +111,13 @@ const RasterTileForm: React.FC<RasterTileFormProps> = ({setResponse}) => {
102111
loading,
103112
error: metaError
104113
} = useFetchJson({
105-
url: currentUrl,
106-
rasterTileType: isPMTilesUrl(currentUrl) ? RasterTileType.PMTILES : RasterTileType.STAC,
114+
url: clearedMetadataUrl,
115+
rasterTileType: isPMTilesUrl(clearedMetadataUrl) ? RasterTileType.PMTILES : RasterTileType.STAC,
107116
process: parseMetadataAllowCollections
108117
});
109118

110119
useEffect(() => {
111-
if (tileName && metadataUrl) {
120+
if (tileName && clearedMetadataUrl) {
112121
const pmtilesType = metadata?.pmtilesType;
113122

114123
if (pmtilesType === PMTilesType.MVT) {
@@ -150,7 +159,7 @@ const RasterTileForm: React.FC<RasterTileFormProps> = ({setResponse}) => {
150159

151160
const dataset = getDatasetAttributesFromRasterTile({
152161
name: tileName,
153-
metadataUrl,
162+
metadataUrl: clearedMetadataUrl,
154163
rasterTileServerUrls: rasterTileServers
155164
});
156165

@@ -172,9 +181,8 @@ const RasterTileForm: React.FC<RasterTileFormProps> = ({setResponse}) => {
172181
metadata,
173182
loading,
174183
metaError,
175-
currentUrl,
176184
tileName,
177-
metadataUrl,
185+
clearedMetadataUrl,
178186
rasterTileServerUrls,
179187
setResponse
180188
]);
@@ -194,7 +202,7 @@ const RasterTileForm: React.FC<RasterTileFormProps> = ({setResponse}) => {
194202
<label htmlFor="tile-metadata">Tileset metadata URL</label>
195203
<InputLight
196204
id="tile-metadata"
197-
placeholder="Tileset metadata"
205+
placeholder="Tileset metadata URL"
198206
value={metadataUrl ?? undefined}
199207
onChange={onMetadataUrlChange}
200208
/>

‎src/components/src/side-panel/layer-panel/raster-tile-layer-configurator.tsx‎

Lines changed: 25 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ import Switch from '../../common/switch';
4141
import InfoHelperFactory from '../../common/info-helper';
4242
import ItemSelector from '../../common/item-selector/item-selector';
4343
import VisConfigSliderFactory from '../../side-panel/layer-panel/vis-config-slider';
44-
import LayerConfigGroupFactory from '../../side-panel/layer-panel/layer-config-group';
44+
import LayerConfigGroupFactory, {
45+
ConfigGroupCollapsibleContent
46+
} from '../../side-panel/layer-panel/layer-config-group';
4547
import VisConfigSwitchFactory from '../../side-panel/layer-panel/vis-config-switch';
4648

4749
type EOBand = StacTypes.Band;
@@ -332,22 +334,34 @@ function RasterTileLayerConfiguratorFactory(
332334
}
333335
}, [visConfiguratorProps, dynamicColor, isDynamicColorsOnly]);
334336

337+
const elevationUI = (
338+
<>
339+
{getApplicationConfig().rasterServerSupportsElevation &&
340+
stac.rasterTileServerUrls?.length && (
341+
<LayerConfigGroup
342+
{...(layer.visConfigSettings.enableTerrain || {label: 'layer.color'})}
343+
{...visConfiguratorProps}
344+
collapsible
345+
>
346+
<ConfigGroupCollapsibleContent>
347+
<VisConfigSwitch
348+
{...visConfiguratorProps}
349+
{...layer.visConfigSettings.enableTerrainTopView}
350+
/>
351+
</ConfigGroupCollapsibleContent>
352+
</LayerConfigGroup>
353+
)}
354+
</>
355+
);
356+
335357
// For PMTiles in raster format, only show opacity and terrain options for now
336358
if (stac.pmtilesType === PMTilesType.RASTER) {
337359
return (
338360
<StyledLayerConfigurator>
339361
<LayerConfigGroup {...visConfiguratorProps} label="Visual Settings" collapsible={false}>
340362
<VisConfigSlider {...layer.visConfigSettings.opacity} {...visConfiguratorProps} />
341363
</LayerConfigGroup>
342-
{getApplicationConfig().rasterServerSupportsElevation &&
343-
stac.rasterTileServerUrls?.length && (
344-
<LayerConfigGroup {...visConfiguratorProps} label="Terrain">
345-
<VisConfigSwitch
346-
{...visConfiguratorProps}
347-
{...layer.visConfigSettings.enableTerrain}
348-
/>
349-
</LayerConfigGroup>
350-
)}
364+
{elevationUI}
351365
</StyledLayerConfigurator>
352366
);
353367
}
@@ -546,15 +560,7 @@ function RasterTileLayerConfiguratorFactory(
546560
</LayerConfigGroup>
547561
)}
548562

549-
{getApplicationConfig().rasterServerSupportsElevation &&
550-
stac.rasterTileServerUrls?.length && (
551-
<LayerConfigGroup {...visConfiguratorProps} label="Terrain">
552-
<VisConfigSwitch
553-
{...visConfiguratorProps}
554-
{...layer.visConfigSettings.enableTerrain}
555-
/>
556-
</LayerConfigGroup>
557-
)}
563+
{elevationUI}
558564
</StyledLayerConfigurator>
559565
);
560566
};

‎src/deckgl-layers/src/raster/webgl/texture/rgba-image.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ export const rgbaImage: ShaderModule = {
4444
inject: {
4545
'fs:DECKGL_CREATE_COLOR': `
4646
image = vec4(texture2D(bitmapTextureRgba, coord));
47+
if (image.a < 0.5) {
48+
discard;
49+
}
4750
`
4851
}
4952
};

‎src/layers/src/raster-tile/config.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -746,6 +746,13 @@ export const rasterVisConfigs = {
746746
label: 'Enable 3D terrain',
747747
group: '',
748748
property: 'enableTerrain',
749-
description: 'Use 3D terrain when available. Only affects Map View in 3D mode.'
749+
description: 'Use terrain when terrain data is available. By default enabled for 3D Map.'
750+
} as VisConfigBoolean,
751+
enableTerrainTopView: {
752+
type: 'boolean',
753+
defaultValue: false,
754+
label: 'Enable in Top view',
755+
group: '',
756+
property: 'enableTerrainTopView'
750757
} as VisConfigBoolean
751758
};

‎src/layers/src/raster-tile/gpu-utils.ts‎

Lines changed: 85 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,16 @@
55
* Functions and constants for handling webgl/luma.gl/deck.gl entities
66
*/
77

8-
import {parse, fetchFile, load} from '@loaders.gl/core';
8+
import {parse, fetchFile, load, FetchError} from '@loaders.gl/core';
99
import {ImageLoader} from '@loaders.gl/images';
1010
import {NPYLoader} from '@loaders.gl/textures';
1111
import GL from '@luma.gl/constants';
1212
import {Texture2DProps} from '@luma.gl/webgl';
1313

14+
import {sleep} from '@kepler.gl/common-utils';
1415
import {getLoaderOptions} from '@kepler.gl/constants';
1516
import {RasterWebGL} from '@kepler.gl/deckgl-layers';
17+
import {getApplicationConfig} from '@kepler.gl/utils';
1618

1719
type ShaderModule = RasterWebGL.ShaderModule;
1820
const {
@@ -51,6 +53,7 @@ import {
5153
NPYLoaderResponse,
5254
RenderSubLayersProps
5355
} from './types';
56+
import {getRequestThrottle} from './request-throttle';
5457

5558
/**
5659
* Describe WebGL2 Texture parameters to use for given input data type
@@ -260,81 +263,100 @@ type LoadingOptions = {
260263
* @return image object to pass to Texture2D constructor
261264
*/
262265
export async function loadNpyArray(
263-
request: {url: string; options: RequestInit},
266+
request: {url: string; rasterServerUrl: string; options: RequestInit},
264267
split: true,
265268
options?: LoadingOptions
266269
): Promise<Texture2DProps[] | null>;
267270
export async function loadNpyArray(
268-
request: {url: string; options: RequestInit},
271+
request: {url: string; rasterServerUrl: string; options: RequestInit},
269272
split: false,
270273
options?: LoadingOptions
271274
): Promise<Texture2DProps | null>;
272275
export async function loadNpyArray(
273-
request: {url: string; options: RequestInit},
276+
request: {url: string; rasterServerUrl: string; options: RequestInit},
274277
split: boolean,
275278
options?: LoadingOptions
276279
): Promise<Texture2DProps | Texture2DProps[] | null> {
277-
try {
278-
const {npy: npyOptions} = getLoaderOptions();
279-
const response: NPYLoaderResponse = await load(request.url, NPYLoader, {
280-
npy: npyOptions,
281-
fetch: options?.fetch
282-
});
283-
284-
if (!response || !response.data || request.options.signal?.aborted) {
285-
return null;
280+
const numAttempts = 1 + getApplicationConfig().rasterServerMaxRetries;
281+
282+
const asset = await getRequestThrottle().throttleRequest(request.rasterServerUrl, async () => {
283+
for (let attempt = 0; attempt < numAttempts; attempt++) {
284+
try {
285+
const {npy: npyOptions} = getLoaderOptions();
286+
const response: NPYLoaderResponse = await load(request.url, NPYLoader, {
287+
npy: npyOptions,
288+
fetch: options?.fetch
289+
});
290+
291+
if (!response || !response.data || request.options.signal?.aborted) {
292+
return null;
293+
}
294+
295+
// Float64 data needs to be coerced to Float32 for the GPU
296+
if (response.data instanceof Float64Array) {
297+
response.data = Float32Array.from(response.data);
298+
}
299+
300+
const {data, header} = response;
301+
const {shape} = header;
302+
const {format, dataFormat, type} = getWebGL2TextureParameters(data);
303+
304+
// TODO: check height-width or width-height
305+
// Regardless, images usually square
306+
// TODO: handle cases of 256x256x1 instead of 1x256x256
307+
const [z, height, width] = shape;
308+
309+
// Since we now use WebGL2 data types for 8-bit textures, we set the following for all textures
310+
const mipmaps = false;
311+
const parameters = DEFAULT_HIGH_BIT_TEXTURE_PARAMETERS;
312+
313+
if (!split) {
314+
return {
315+
data,
316+
width,
317+
height,
318+
format,
319+
dataFormat,
320+
type,
321+
parameters,
322+
mipmaps
323+
};
324+
}
325+
326+
// Split into individual arrays
327+
const channels: Texture2DProps[] = [];
328+
const channelSize = height * width;
329+
for (let i = 0; i < z; i++) {
330+
channels.push({
331+
data: data.subarray(i * channelSize, (i + 1) * channelSize),
332+
width,
333+
height,
334+
format,
335+
dataFormat,
336+
type,
337+
parameters,
338+
mipmaps
339+
});
340+
}
341+
return channels;
342+
} catch (error) {
343+
// Retry if Service Temporarily Unavailable 503 error etc.
344+
if (
345+
attempt < numAttempts &&
346+
error instanceof FetchError &&
347+
getApplicationConfig().rasterServerServerErrorsToRetry?.includes(
348+
error.response?.status as number
349+
)
350+
) {
351+
await sleep(getApplicationConfig().rasterServerRetryDelay);
352+
continue;
353+
}
354+
}
286355
}
287-
288-
// Float64 data needs to be coerced to Float32 for the GPU
289-
if (response.data instanceof Float64Array) {
290-
response.data = Float32Array.from(response.data);
291-
}
292-
293-
const {data, header} = response;
294-
const {shape} = header;
295-
const {format, dataFormat, type} = getWebGL2TextureParameters(data);
296-
297-
// TODO: check height-width or width-height
298-
// Regardless, images usually square
299-
// TODO: handle cases of 256x256x1 instead of 1x256x256
300-
const [z, height, width] = shape;
301-
302-
// Since we now use WebGL2 data types for 8-bit textures, we set the following for all textures
303-
const mipmaps = false;
304-
const parameters = DEFAULT_HIGH_BIT_TEXTURE_PARAMETERS;
305-
306-
if (!split) {
307-
return {
308-
data,
309-
width,
310-
height,
311-
format,
312-
dataFormat,
313-
type,
314-
parameters,
315-
mipmaps
316-
};
317-
}
318-
319-
// Split into individual arrays
320-
const channels: Texture2DProps[] = [];
321-
const channelSize = height * width;
322-
for (let i = 0; i < z; i++) {
323-
channels.push({
324-
data: data.subarray(i * channelSize, (i + 1) * channelSize),
325-
width,
326-
height,
327-
format,
328-
dataFormat,
329-
type,
330-
parameters,
331-
mipmaps
332-
});
333-
}
334-
return channels;
335-
} catch {
336356
return null;
337-
}
357+
});
358+
359+
return asset;
338360
}
339361

340362
/**

0 commit comments

Comments
 (0)