Skip to content

Commit 1aac158

Browse files
flarcoflarco
committed
feat: add PostGIS geometry replication test and update Postgres type mappings
1 parent 130bc8c commit 1aac158

5 files changed

Lines changed: 197 additions & 0 deletions

File tree

‎cmd/sling/tests/pipelines/postgis/docker-compose.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Self-contained PostGIS environment
2+
# Spins up a PostGIS-enabled Postgres so the pipeline test can run
3+
# without depending on an external POSTGIS connection.
4+
5+
services:
6+
postgis:
7+
image: postgis/postgis:16-3.4
8+
container_name: sling-test-postgis
9+
environment:
10+
POSTGRES_USER: postgres
11+
POSTGRES_PASSWORD: postgres
12+
POSTGRES_DB: postgis_test
13+
ports:
14+
- "55432:5432"
15+
healthcheck:
16+
test: ["CMD-SHELL", "pg_isready -U postgres -d postgis_test"]
17+
interval: 2s
18+
timeout: 5s
19+
retries: 30

‎cmd/sling/tests/pipelines/postgis/p.41.postgis_geometry.yaml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# PostGIS geometry replication regression test
2+
# Verifies that a Postgres geometry source column is replicated to a Postgres
3+
# target as a real geometry column (not VARCHAR(65500)).
4+
# Requires: POSTGIS connection (PostgreSQL with postgis extension installed).
5+
# A self-contained runner is provided at run_docker_test.sh.
6+
7+
steps:
8+
# Cleanup any prior state
9+
- connection: POSTGIS
10+
query: |
11+
DROP TABLE IF EXISTS public.postgis_src;
12+
DROP TABLE IF EXISTS public.postgis_tgt;
13+
14+
# Ensure extension exists (idempotent)
15+
- connection: POSTGIS
16+
query: |
17+
CREATE EXTENSION IF NOT EXISTS postgis;
18+
19+
# Create source table with a geometry column and seed data
20+
- connection: POSTGIS
21+
query: |
22+
CREATE TABLE public.postgis_src (
23+
id BIGINT NOT NULL PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
24+
geom geometry(POINT, 3857)
25+
);
26+
INSERT INTO public.postgis_src (geom)
27+
SELECT ST_POINT(i, i, 3857)
28+
FROM GENERATE_SERIES(0, 50, 10) i;
29+
30+
# Run replication mirroring the configuration from issue #732
31+
- replication:
32+
source: POSTGIS
33+
target: POSTGIS
34+
35+
defaults:
36+
object: '{stream_schema}.{stream_table}'
37+
mode: full-refresh
38+
columns:
39+
geom: geometry
40+
41+
streams:
42+
public.postgis_src:
43+
object: public.postgis_tgt
44+
45+
env:
46+
SLING_THREADS: 1
47+
SLING_RETRIES: 0
48+
49+
# Inspect target column type
50+
- type: query
51+
connection: POSTGIS
52+
query: |
53+
SELECT data_type, udt_name
54+
FROM information_schema.columns
55+
WHERE table_schema = 'public'
56+
AND table_name = 'postgis_tgt'
57+
AND column_name = 'geom'
58+
into: target_col
59+
60+
- type: log
61+
message: "Target geom column data_type={store.target_col[0].data_type}, udt_name={store.target_col[0].udt_name}"
62+
63+
# Verify row count made it through
64+
- type: query
65+
connection: POSTGIS
66+
query: SELECT count(*) AS cnt FROM public.postgis_tgt
67+
into: row_count
68+
69+
- type: log
70+
message: "Target row count: {store.row_count[0].cnt}"
71+
72+
- type: check
73+
check: int_parse(store.row_count[0].cnt) == 6
74+
message: "Expected 6 rows in target, got {store.row_count[0].cnt}"
75+
76+
# Bug regression check: target column must be a real geometry, not VARCHAR.
77+
- type: check
78+
check: store.target_col[0].udt_name == "geometry"
79+
message: "Issue #732 regression: target geom column udt_name='{store.target_col[0].udt_name}' (data_type='{store.target_col[0].data_type}'), expected 'geometry'"
80+
81+
- type: log
82+
message: "PostGIS geometry test PASSED (data_type={store.target_col[0].data_type})"
83+
84+
# Cleanup
85+
- connection: POSTGIS
86+
query: |
87+
DROP TABLE IF EXISTS public.postgis_src;
88+
DROP TABLE IF EXISTS public.postgis_tgt;

‎cmd/sling/tests/pipelines/postgis/run_docker_test.sh‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#!/bin/bash
2+
# Self-contained PostGIS reproduction test.
3+
# Spins up postgis/postgis via docker compose, exposes it as the POSTGIS
4+
# Sling connection, runs the pipeline, then tears the container down.
5+
#
6+
# Usage: bash run_docker_test.sh
7+
# Requires: Docker (with compose v2), a built sling binary at cmd/sling/sling
8+
set -e
9+
10+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
11+
SLING_CLI_DIR="$(cd "$SCRIPT_DIR/../../../../.." && pwd)"
12+
CMD_DIR="$SLING_CLI_DIR/cmd/sling"
13+
SLING_BIN="$CMD_DIR/sling"
14+
15+
if [ ! -x "$SLING_BIN" ]; then
16+
echo "Building sling binary..."
17+
(cd "$CMD_DIR" && go build .)
18+
fi
19+
20+
cleanup() {
21+
echo "=== Tearing down PostGIS container ==="
22+
docker compose -f "$SCRIPT_DIR/docker-compose.yaml" down -v --remove-orphans >/dev/null 2>&1 || true
23+
}
24+
trap cleanup EXIT
25+
26+
echo "=== Starting PostGIS container ==="
27+
docker compose -f "$SCRIPT_DIR/docker-compose.yaml" up -d --wait
28+
29+
# The postgis image bootstraps once on first start: postgres comes up, runs the
30+
# init SQL that loads the extensions, then shuts postgres down and restarts it.
31+
# pg_isready / a single SELECT can succeed during the init phase only to be
32+
# refused seconds later. Wait until the bootstrap log line appears and then
33+
# require N consecutive successful SQL probes from the host.
34+
echo "=== Waiting for PostGIS bootstrap to finish ==="
35+
for i in $(seq 1 60); do
36+
if docker logs sling-test-postgis 2>&1 | grep -q "database system is ready to accept connections" \
37+
&& docker logs sling-test-postgis 2>&1 | grep -q "PostgreSQL init process complete"; then
38+
break
39+
fi
40+
if [ "$i" = "60" ]; then
41+
echo "ERROR: PostGIS init did not finish in time"
42+
docker logs sling-test-postgis | tail -40
43+
exit 1
44+
fi
45+
sleep 1
46+
done
47+
48+
echo "=== Waiting for stable SQL connections from host ==="
49+
stable=0
50+
for i in $(seq 1 60); do
51+
if docker exec sling-test-postgis psql -U postgres -d postgis_test -tAc "SELECT 1" >/dev/null 2>&1; then
52+
stable=$((stable + 1))
53+
if [ "$stable" -ge 3 ]; then
54+
echo "PostGIS stable."
55+
break
56+
fi
57+
else
58+
stable=0
59+
fi
60+
if [ "$i" = "60" ]; then
61+
echo "ERROR: PostGIS did not become stable in time"
62+
docker logs sling-test-postgis | tail -40
63+
exit 1
64+
fi
65+
sleep 1
66+
done
67+
68+
# Expose container as the POSTGIS Sling connection for this run only.
69+
export POSTGIS='postgres://postgres:postgres@127.0.0.1:55432/postgis_test?sslmode=disable'
70+
71+
echo "=== Running PostGIS pipeline test ==="
72+
"$SLING_BIN" run -d -p "$SCRIPT_DIR/p.41.postgis_geometry.yaml"
73+
74+
echo "=== PostGIS reproduction test completed ==="

‎cmd/sling/tests/suite.cli.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2245,3 +2245,16 @@
22452245
run: 'sling run --src-stream file://cmd/sling/tests/files/test9.csv.zip --src-options ''{ delimiter: ";" }'' --stdout'
22462246
output_contains:
22472247
- 'col1,col2,col3'
2248+
2249+
# PostGIS geometry replication preserves geometry type on target (issue #732).
2250+
# Self-contained: cmd/sling/tests/pipelines/postgis/run_docker_test.sh spins up
2251+
# a postgis/postgis container and exports the POSTGIS connection for this run.
2252+
- id: 235
2253+
name: 'PostGIS geometry replication preserves geometry type (issue 732)'
2254+
run: 'sling run -d -p cmd/sling/tests/pipelines/postgis/p.41.postgis_geometry.yaml'
2255+
group: postgis
2256+
output_contains:
2257+
- 'Target geom column data_type=USER-DEFINED, udt_name=geometry'
2258+
- 'Target row count: 6'
2259+
- 'PostGIS geometry test PASSED'
2260+
- 'execution succeeded'

‎core/dbio/templates/postgres.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -722,6 +722,8 @@ native_type_map:
722722
varchar: text
723723
xid: string
724724
xml: string
725+
geometry: geometry
726+
geography: geometry
725727

726728

727729
general_type_map:
@@ -732,6 +734,7 @@ general_type_map:
732734
datetime: timestamp
733735
decimal: numeric
734736
float: "double precision"
737+
geometry: geometry
735738
integer: integer
736739
json: jsonb
737740
smallint: smallint

0 commit comments

Comments
 (0)