Skip to content

Commit 0a31433

Browse files
Cidreeclaude
andcommitted
perf: build ddbs_as_geojson() output via string concatenation.
The previous json_object/json_group_array approach round-tripped every geometry through the JSON type, which was slow and exploded memory on large data (OOM at ~23 GiB for 1e7 points). Build the Feature strings and the FeatureCollection with plain SQL string concatenation and string_agg instead: ~2x faster than the old code, faster than geojsonsf::sf_geojson(), and a fraction of the memory. Also bump the development version and drop the obsolete ddbs_as_geojson() macro test (the macro is per-row and no longer matches the function's FeatureCollection output). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent ad7a112 commit 0a31433

3 files changed

Lines changed: 16 additions & 32 deletions

File tree

DESCRIPTION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
Type: Package
22
Package: duckspatial
33
Title: R Interface to 'DuckDB' Database with Spatial Extension
4-
Version: 1.1.2
4+
Version: 1.1.2.9000
55
Authors@R: c(
66
person(
77
"Adrián", "Cidre González",

R/ddbs_geom_conversion.R

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -154,39 +154,37 @@ ddbs_as_geojson <- function(
154154
desc <- DBI::dbGetQuery(target_conn, glue::glue("DESCRIBE SELECT * FROM {x_list$query_name};"))
155155
prop_cols <- desc$column_name[desc$column_name != x_geom]
156156

157-
## Build a JSON object of all non-geometry columns to use as feature properties
157+
## Per-row properties as compact JSON text. to_json() handles type formatting
158+
## and string escaping; '{}' when there are no non-geometry columns.
158159
if (length(prop_cols)) {
159160
prop_fields <- paste(sprintf('"%s" := "%s"', prop_cols, prop_cols), collapse = ", ")
160-
props_sql <- glue::glue("to_json(struct_pack({prop_fields}))")
161+
props_sql <- sprintf("to_json(struct_pack(%s))::VARCHAR", prop_fields)
161162
} else {
162-
props_sql <- "'{}'::JSON"
163+
props_sql <- "'{}'"
163164
}
164165

165-
## Per-row GeoJSON Feature expression (properties before geometry, matching the
166-
## output of geojsonsf::sf_geojson())
167-
feature_sql <- glue::glue(
168-
"json_object('type', 'Feature', 'properties', {props_sql}, ",
169-
"'geometry', ST_AsGeoJSON(\"{x_geom}\")::JSON)"
166+
## Build each Feature by plain string concatenation. This avoids round-tripping
167+
## the geometry through the JSON type (json_object / json_group_array), which is
168+
## markedly slower and far more memory-hungry on large datasets. Properties
169+
## precede the geometry to match geojsonsf::sf_geojson().
170+
feature_sql <- sprintf(
171+
"'{\"type\":\"Feature\",\"properties\":' || %s || ',\"geometry\":' || coalesce(ST_AsGeoJSON(\"%s\"), 'null') || '}'",
172+
props_sql, x_geom
170173
)
171174

172175
# 3. Build and retrieve the result
173176
if (feature_collection) {
174177
## Single FeatureCollection. coalesce() keeps `features` as [] (not null) when empty.
175-
tmp.query <- glue::glue(
176-
"SELECT json_object(
177-
'type', 'FeatureCollection',
178-
'features', coalesce(json_group_array({feature_sql}), '[]'::JSON)
179-
)::VARCHAR AS geojson
180-
FROM {x_list$query_name};"
178+
tmp.query <- sprintf(
179+
"SELECT '{\"type\":\"FeatureCollection\",\"features\":[' || coalesce(string_agg(%s, ','), '') || ']}' AS geojson FROM %s;",
180+
feature_sql, x_list$query_name
181181
)
182182
out <- DBI::dbGetQuery(target_conn, tmp.query)$geojson
183183
## tag like geojsonsf so downstream tools treat it as raw JSON, not a string
184184
class(out) <- c("geojson", "json")
185185
out
186186
} else {
187-
tmp.query <- glue::glue(
188-
"SELECT {feature_sql}::VARCHAR AS geojson FROM {x_list$query_name};"
189-
)
187+
tmp.query <- sprintf("SELECT %s AS geojson FROM %s;", feature_sql, x_list$query_name)
190188
DBI::dbGetQuery(target_conn, tmp.query)$geojson
191189
}
192190

tests/testthat/test-ddbs_macros.R

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,4 @@
11

2-
3-
countries_ddbs
4-
rivers_ddbs
5-
points_ddbs
6-
72
# 0. Set up --------------------------------------------------------------
83

94
## skip tests on CRAN because they take too much time
@@ -206,15 +201,6 @@ testthat::describe("Format conversion functions", {
206201
expect_equal(normal_result, macro_result$hexwkb)
207202
})
208203

209-
## DDBS_AS_GEOJSON
210-
testthat::it("ddbs_as_geojson() macro works", {
211-
normal_result <- ddbs_as_geojson(countries_ddbs)
212-
macro_result <- countries_ddbs |>
213-
dplyr::mutate(geojson = ddbs_as_geojson(geometry)) |>
214-
ddbs_collect()
215-
expect_equal(normal_result, macro_result$geojson)
216-
})
217-
218204
})
219205

220206

0 commit comments

Comments
 (0)