fix(vector): stop a failed restore from deleting multi-layer URL layers

A multi-layer container (a KMZ of KML folders, a GeoPackage of tables)
becomes one store layer per source layer, and every one of them keeps the
container's URL. Three problems compounded from there, reported against
NASA FIRMS fire-footprint KMZ feeds in discussion #1757.

Restoring such a project replays each layer independently, and any layer
whose replay failed was then pruned by the closing store sync, which reads
"the control does not have this layer" as "the user removed it". A feed
that was briefly unreachable therefore deleted the layers from the project
outright, and the next save wrote that loss to disk. Restore now records
which replays failed and keeps those layers, and refresh (the button and
the auto-refresh tick) falls through to a new replay path so the next
successful fetch brings the layer back instead of leaving a dead entry.

Each of those sibling layers also downloaded the container separately: on
desktop a six-layer KMZ pulled the same archive six times over on every
project open and every refresh interval, and unzipped and registered it
into DuckDB six times. Add Vector Layer downloads are now collapsed to one
in-flight request per URL, handing every sibling the identical File so the
control's per-source unzip and registration caches collapse with them.

Those six concurrent downloads then ran into the native fetch command's
tile-sized 8s budget, which is what produced the reporter's "Could not read
response body" failures at exactly 8.4s. fetch_url_bytes now accepts a
caller-supplied timeout, clamped so it can only be raised and never
removed, and the vector loader asks for a download-sized budget.

Finally, a store layer that outlives its map layers no longer has its
style synced, which was filling the Diagnostics panel with "Cannot get
style of non-existing layer" the user could do nothing about.

Ref: #1757

Author: giswqs

apps/geolibre-desktop/src-tauri/src/lib.rs

@@ -136,6 +136,10 @@ const UV_INSTALL_BASE_URL: &str = "https://astral.sh/uv";
 const REMOTE_TILE_TIMEOUT_SECS: u64 = 8;
 const REMOTE_TILE_CONNECT_TIMEOUT_SECS: u64 = 4;
 const URL_RESOLVE_TIMEOUT_SECS: u64 = 15;
+/// Ceiling for a caller-supplied `fetch_url_bytes` budget. The default suits a
+/// tile; callers that download a whole dataset (Add Vector Layer) ask for more,
+/// but not without bound, so a bad value cannot wedge a request indefinitely.
+const MAX_FETCH_TIMEOUT_SECS: u64 = 600;
 
 #[cfg(all(unix, not(feature = "mas")))]
 const SIGTERM: i32 = 15;
@@ -1035,21 +1039,39 @@ fn build_guarded_http_client() -> Result<reqwest::blocking::Client, String> {
         .map_err(|error| format!("Could not create HTTP client: {error}"))
 }
 
+/// Fetches a URL's bytes, bypassing browser CORS.
+///
+/// `timeout_secs` overrides the tile-sized default for callers that download a
+/// whole dataset rather than a tile; it is clamped to
+/// `[REMOTE_TILE_TIMEOUT_SECS, MAX_FETCH_TIMEOUT_SECS]`, so the budget can only
+/// ever be raised and never removed.
 #[tauri::command]
-async fn fetch_url_bytes(url: String) -> Result<Vec<u8>, String> {
-    tauri::async_runtime::spawn_blocking(move || fetch_url_bytes_blocking(url))
+async fn fetch_url_bytes(url: String, timeout_secs: Option<u64>) -> Result<Vec<u8>, String> {
+    tauri::async_runtime::spawn_blocking(move || fetch_url_bytes_blocking(url, timeout_secs))
         .await
         .map_err(|error| format!("Tile fetch task failed: {error}"))?
 }
 
-fn fetch_url_bytes_blocking(url: String) -> Result<Vec<u8>, String> {
+/// Resolves the request budget for a fetch, defaulting to the tile timeout and
+/// clamping a caller-supplied value into `[REMOTE_TILE_TIMEOUT_SECS,
+/// MAX_FETCH_TIMEOUT_SECS]`. A caller can therefore only ever raise the budget,
+/// and never past the ceiling or down to zero (which reqwest reads as "no
+/// timeout" and would let a stalled request hang forever).
+fn resolve_fetch_timeout_secs(timeout_secs: Option<u64>) -> u64 {
+    timeout_secs
+        .unwrap_or(REMOTE_TILE_TIMEOUT_SECS)
+        .clamp(REMOTE_TILE_TIMEOUT_SECS, MAX_FETCH_TIMEOUT_SECS)
+}
+
+fn fetch_url_bytes_blocking(url: String, timeout_secs: Option<u64>) -> Result<Vec<u8>, String> {
     ensure_fetchable_url(&url)?;
 
     let client = guarded_http_client()?;
+    let timeout = resolve_fetch_timeout_secs(timeout_secs);
 
     let response = client
         .get(&url)
-        .timeout(Duration::from_secs(REMOTE_TILE_TIMEOUT_SECS))
+        .timeout(Duration::from_secs(timeout))
         .send()
         .map_err(|error| format!("Request failed: {error}"))?;
     let status = response.status();
@@ -4002,7 +4024,8 @@ mod tests {
     use super::{
         client_cert_is_pkcs12, client_cert_password_without_path, ensure_fetchable_url,
         is_allowed_local_vector_path, is_allowed_project_path, is_disallowed_ip,
-        is_safe_absolute_path, path_is_under, tcp_table_port,
+        is_safe_absolute_path, path_is_under, resolve_fetch_timeout_secs, tcp_table_port,
+        MAX_FETCH_TIMEOUT_SECS, REMOTE_TILE_TIMEOUT_SECS,
     };
     #[cfg(target_os = "linux")]
     use super::{linux_uses_nvidia_renderer, nvidia_is_primary_gpu};
@@ -4661,6 +4684,26 @@ mod tests {
         assert_eq!(tcp_table_port(0x0000_223E), 0x3E22);
     }
 
+    // Add Vector Layer downloads a whole dataset through the same command that
+    // fetches tiles, so it asks for a longer budget. The clamp is what keeps
+    // that override from becoming a way to disable the timeout entirely.
+    #[test]
+    fn clamps_the_fetch_timeout_into_the_allowed_range() {
+        // No override: the tile-sized default.
+        assert_eq!(resolve_fetch_timeout_secs(None), REMOTE_TILE_TIMEOUT_SECS);
+        // A dataset-sized budget is honored as asked.
+        assert_eq!(resolve_fetch_timeout_secs(Some(180)), 180);
+        // Zero would mean "no timeout" to reqwest, so it is raised to the floor
+        // rather than letting a stalled request hang forever.
+        assert_eq!(resolve_fetch_timeout_secs(Some(0)), REMOTE_TILE_TIMEOUT_SECS);
+        // Below the floor is raised; above the ceiling is capped.
+        assert_eq!(resolve_fetch_timeout_secs(Some(1)), REMOTE_TILE_TIMEOUT_SECS);
+        assert_eq!(
+            resolve_fetch_timeout_secs(Some(u64::MAX)),
+            MAX_FETCH_TIMEOUT_SECS
+        );
+    }
+
     // The image-path guard is what keeps the reaper from killing a Jupyter the
     // user installed themselves: only executables under our own runtime
     // directory are ours to terminate.

apps/geolibre-desktop/src/components/panels/LayerPanel.tsx

@@ -55,6 +55,7 @@ import {
   materializeEmbeddableVectorLayers,
   RASTER_SOURCE_KIND,
   reloadVectorControlLayer,
+  replayVectorControlLayerById,
   SKETCHES_SOURCE_KIND,
   TIME_SLIDER_PLUGIN_ID,
   type TimePropertyCandidate,
@@ -1376,10 +1377,16 @@ export function LayerPanel({
           return;
         }
         if (isVectorControlRefreshLayer(layer)) {
-          const info = await reloadVectorControlLayer(layer.id);
+          // A layer whose restore failed stays in the project but never made it
+          // into the control, so reloadLayer cannot find it. Replaying it is
+          // what brings such a layer back once the source is reachable again,
+          // which is why refresh (manual and automatic) tries that second.
+          const info =
+            (await reloadVectorControlLayer(layer.id)) ??
+            (await replayVectorControlLayerById(layer.id));
           if (!info) {
             // The control is unavailable (panel never opened, or torn down
-            // and not yet replayed) or no longer knows this layer id.
+            // and not yet replayed) or the replay above did not succeed.
             // Automatic ticks fire on a timer the user didn't initiate, so
             // skip silently and clear the transient note instead of surfacing
             // an error every interval until the control comes back.

apps/geolibre-desktop/src/hooks/usePlugins.ts

@@ -121,6 +121,7 @@ import { appendDiagnostic } from "../lib/diagnostics";
 import { pickZarrDirectory, zarrDirectoryPickerSupported } from "../lib/zarr-directory-picker";
 import { openExternalLink } from "../lib/open-external";
 import { fetchUrlBytes } from "../lib/native-http";
+import { dedupeVectorUrlFetch, vectorDownloadFileName } from "../lib/vector-url-fetch";
 import { partitionProjectPluginManifestUrls } from "../lib/plugin-trust";
 import { setTimeSliderOpenedByBinding, shouldCloseTimeSliderDock } from "../lib/time-slider-dock";
 import { createWmsTileUrl, normalizeWmsVersion } from "../components/layout/add-data/helpers";
@@ -1033,35 +1034,47 @@ export function createAppAPI(mapControllerRef?: RefObject<MapController | null>)
     // presence to auto-discover shapefile sidecars instead of forcing the user
     // to select every component, and to capture the file's path for restore.
     pickVectorFilesWithSidecars: isTauriRuntime() ? pickVectorFilesWithSidecars : undefined,
-    fetchVectorUrl: async (url: string) => {
-      if (isTauriRuntime()) {
-        try {
-          const bytes = await fetchUrlBytes(url, { context: "Add Vector Layer" });
-          const array = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
-          return new Blob([array as Uint8Array<ArrayBuffer>]);
-        } catch {
-          // The shared native command has a short, tile-oriented timeout.
-          // Preserve the former browser path for large CORS-enabled datasets.
+    // Shared across the sibling layers of one multi-layer container, which all
+    // carry the container's URL: without this a six-layer KMZ downloaded itself
+    // six times over on every project open and every refresh tick.
+    fetchVectorUrl: (url: string) =>
+      dedupeVectorUrlFetch(url, async () => {
+        const name = vectorDownloadFileName(url);
+        if (isTauriRuntime()) {
           try {
-            const response = await fetch(url);
-            if (!response.ok) {
-              throw new Error(`HTTP ${response.status} ${response.statusText}`);
-            }
-            return response.blob();
+            const bytes = await fetchUrlBytes(url, {
+              context: "Add Vector Layer",
+              // The default budget on this command is tile-sized (8s). A vector
+              // dataset is not a tile. A few megabytes from a slow origin
+              // routinely needs longer, and timing out here used to drop the
+              // layer entirely, so ask for a download-sized budget instead.
+              timeoutSecs: VECTOR_DOWNLOAD_TIMEOUT_SECS,
+            });
+            const array = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+            return new File([array as Uint8Array<ArrayBuffer>], name);
           } catch {
-            // GitHub's /raw route rejects browser CORS, so fall through to the
-            // same guarded proxy used by the web build.
+            // Keep the browser path as a fallback for CORS-enabled origins the
+            // native command could not reach.
+            try {
+              const response = await fetch(url);
+              if (!response.ok) {
+                throw new Error(`HTTP ${response.status} ${response.statusText}`);
+              }
+              return new File([await response.blob()], name);
+            } catch {
+              // GitHub's /raw route rejects browser CORS, so fall through to the
+              // same guarded proxy used by the web build.
+            }
           }
         }
-      }
-      const proxyUrl = githubRawVectorProxyUrl(url);
-      if (!proxyUrl) return null;
-      const response = await fetch(proxyUrl);
-      if (!response.ok) {
-        throw new Error(`HTTP ${response.status} ${response.statusText}`);
-      }
-      return response.blob();
-    },
+        const proxyUrl = githubRawVectorProxyUrl(url);
+        if (!proxyUrl) return null;
+        const response = await fetch(proxyUrl);
+        if (!response.ok) {
+          throw new Error(`HTTP ${response.status} ${response.statusText}`);
+        }
+        return new File([await response.blob()], name);
+      }),
     readLocalVectorFile: readVectorFileWithSidecars,
     exportTextFile: (filename: string, content: string, options?: GeoLibreFileDialogOptions) => {
       const description = options?.description ?? "GeoJSON";
@@ -1333,6 +1346,14 @@ function isTauriRuntime(): boolean {
 
 const GITHUB_RAW_VECTOR_PROXY = "https://tiles.geolibre.app/github-raw";
 
+/**
+ * Budget for a native Add Vector Layer download, in seconds. Deliberately far
+ * above `fetch_url_bytes`'s tile-sized default: this command carries whole
+ * datasets, not 256px tiles, and a timeout here is not a slow tile that resolves
+ * next frame but a layer that fails to restore.
+ */
+const VECTOR_DOWNLOAD_TIMEOUT_SECS = 180;
+
 function githubRawVectorProxyUrl(value: string): string | null {
   let url: URL;
   try {

apps/geolibre-desktop/src/lib/native-http.ts

@@ -20,6 +20,13 @@ export type NativeHttpCommand = "fetch_url_bytes" | "resolve_url_redirect";
 interface NativeHttpOptions {
   /** Short feature label (e.g. "WFS GetCapabilities") added to the record. */
   context?: string;
+  /**
+   * Request budget in seconds, overriding the command's tile-sized default.
+   * Set it for calls that carry a whole dataset rather than a tile; the backend
+   * clamps the value, so an out-of-range number degrades to the default rather
+   * than removing the timeout.
+   */
+  timeoutSecs?: number;
 }
 
 function recordSource(command: NativeHttpCommand, context?: string): string {
@@ -83,7 +90,12 @@ async function invokeNativeHttp<T>(
 ): Promise<T> {
   const startedAt = performance.now();
   try {
-    const result = await invoke<T>(command, { url });
+    // `timeoutSecs` is omitted rather than sent as undefined so the Rust side
+    // sees an absent argument and applies its own default.
+    const result = await invoke<T>(command, {
+      url,
+      ...(options?.timeoutSecs === undefined ? {} : { timeoutSecs: options.timeoutSecs }),
+    });
     appendDiagnostic(
       nativeHttpSuccessRecord(
         command,
@@ -112,7 +124,8 @@ async function invokeNativeHttp<T>(
  * subject to browser CORS), recording the request in the diagnostics log.
  *
  * @param url - The absolute HTTP(S) URL to fetch.
- * @param options - Optional context label for the diagnostics record.
+ * @param options - Optional context label for the diagnostics record, and an
+ *   optional request budget for callers downloading more than a tile.
  * @returns The response body bytes (Tauri may hand back a plain number array).
  */
 export function fetchUrlBytes(

apps/geolibre-desktop/src/lib/vector-url-fetch.ts

@@ -0,0 +1,109 @@
+/**
+ * De-duplication for Add Vector Layer's remote downloads.
+ *
+ * A multi-layer container (a KMZ of KML folders, a GeoPackage of tables, a DXF)
+ * becomes one store layer per source layer, but every one of them keeps the
+ * *container's* URL. Restoring such a project replays each layer independently,
+ * and auto-refresh ticks them independently too, so a six-layer KMZ issued six
+ * concurrent downloads of the same archive on every project open and on every
+ * refresh interval (opengeos/GeoLibre discussion #1757). On a slow origin that
+ * is enough to blow the fetch budget and fail layers that would have loaded had
+ * they asked once.
+ *
+ * `dedupeVectorUrlFetch` collapses those into a single in-flight request per
+ * URL. Two properties matter:
+ *
+ * - **In-flight only.** The entry is dropped as soon as the download settles, so
+ *   this is a request collapser and never a cache: a later auto-refresh always
+ *   re-downloads and genuinely refreshes the layer. Sibling layers of one
+ *   container share a download only because they ask within the same window.
+ * - **One identity.** Every caller gets the *same* `File` object, not a copy.
+ *   maplibre-gl-vector keys its per-source caches (the unzipped KML it
+ *   registers with DuckDB, a GeoPackage's bytes) on the source object, so
+ *   handing all six layers one identical `File` collapses the unzip and the
+ *   DuckDB registration too: a 91 MB KML is registered once instead of six
+ *   times. Returning a `Blob` would not: the control wraps a plain `Blob` in a
+ *   fresh `File` per call, and the caches would miss again.
+ */
+
+/**
+ * In-flight downloads keyed by URL. Entries live only until they settle.
+ *
+ * The value is a wrapper rather than the bare promise so the download's own
+ * `finally` can identify (and clear) exactly its own entry while still running
+ * *inside* the promise. A `.finally()` chained onto the outside would resolve a
+ * tick after awaiting callers, leaving a settled entry briefly visible and
+ * letting an immediate retry share an already-finished download.
+ */
+interface InFlightEntry {
+  promise: Promise<File | null>;
+}
+
+const inFlight = new Map<string, InFlightEntry>();
+
+/**
+ * Runs `download` for `url`, sharing the request with any call for the same URL
+ * that is still in flight.
+ *
+ * @param url - The absolute URL being downloaded.
+ * @param download - Performs the actual download. Called at most once per
+ *   in-flight window; returns null when no loader could serve the URL.
+ * @returns The downloaded file (the identical object for every sharer), or null.
+ */
+export function dedupeVectorUrlFetch(
+  url: string,
+  download: () => Promise<File | null>,
+): Promise<File | null> {
+  const existing = inFlight.get(url);
+  if (existing) return existing.promise;
+
+  const entry = {} as InFlightEntry;
+  entry.promise = (async () => {
+    try {
+      return await download();
+    } finally {
+      // Cleared either way: a failure must not be replayed to a later refresh,
+      // and a success must not be served as a stale cache hit. Only our own
+      // entry is cleared, since a retry that started later owns the key.
+      if (inFlight.get(url) === entry) inFlight.delete(url);
+    }
+  })();
+  inFlight.set(url, entry);
+  return entry.promise;
+}
+
+/**
+ * Names a downloaded blob so the vector control's format detection still works.
+ *
+ * The control derives the format from the file name, so the URL's last path
+ * segment is used when it has an extension; otherwise the blob is handed over
+ * under a generic name and the control falls back to sniffing the content type.
+ *
+ * @param url - The URL the bytes came from.
+ * @param fallback - Name to use when the URL carries no usable file name.
+ * @returns A file name for the downloaded bytes.
+ */
+export function vectorDownloadFileName(url: string, fallback = "data"): string {
+  let pathname: string;
+  try {
+    pathname = new URL(url).pathname;
+  } catch {
+    return fallback;
+  }
+  const last = pathname.slice(pathname.lastIndexOf("/") + 1);
+  const decoded = safeDecode(last);
+  return /\.[A-Za-z0-9]+$/.test(decoded) ? decoded : fallback;
+}
+
+function safeDecode(value: string): string {
+  try {
+    return decodeURIComponent(value);
+  } catch {
+    return value;
+  }
+}
+
+/** Clears every in-flight entry. Exported for tests. */
+export function resetVectorUrlFetchDedupe(): void {
+  inFlight.clear();
+}

packages/map/src/layer-sync.ts

@@ -479,6 +479,15 @@ function syncExternalNativeLayer(
     clearExternalNativeExtrusion(map, layer, nativeLayerIds);
 
     for (const nativeLayerId of nativeLayerIds) {
+      // A store layer can legitimately outlive its map layers: a layer whose
+      // restore failed keeps the `nativeLayerIds` it was saved with, and the
+      // owning control has not (yet) recreated them. Styling a layer that is
+      // not on the map raises "Cannot get style of non-existing layer" on the
+      // map's error channel, which fills the Diagnostics panel with noise the
+      // user can do nothing about, so skip those ids until the control brings
+      // them back.
+      const nativeLayer = map.getLayer(nativeLayerId);
+      if (!nativeLayer) continue;
       moveLayer(map, nativeLayerId, beforeId);
       // The owning control mirrors direct per-layer visibility changes from
       // the store, but effective state such as a hidden parent group never
@@ -492,8 +501,7 @@ function syncExternalNativeLayer(
       // hide-unmatched filter: filtering is independent of the paint the
       // control owns. Native layers without a filter (deck.gl / 3D Tiles
       // custom layers) are skipped by the type guard.
-      const nativeLayer = map.getLayer(nativeLayerId);
-      if (nativeLayer && nativeLayerSupportsFilter(nativeLayer.type)) {
+      if (nativeLayerSupportsFilter(nativeLayer.type)) {
         applyExternalNativeFeatureFilters(map, nativeLayerId, layer);
       }
     }

packages/plugins/src/index.ts

@@ -265,6 +265,7 @@ export {
   materializeEmbeddableVectorLayers,
   openVectorLayerPanel,
   reloadVectorControlLayer,
+  replayVectorControlLayerById,
   restoreVectorLayers,
   setKmlFileImportHandler,
   isKmlFileSelection,