feat #2664: Geocoding DB should be completely isolated from vexl db

Author: SamTremko

AGENTS.md

@@ -28,6 +28,7 @@ When changing any UI / UX in the mobile app or in the ui package you must read @
 - packages/cryptography - shared cryptography utilities for both backend and mobile app
 - packages/generic-utils - shared generic utilities for both backend and mobile app - should not depend on any of the other vexl packages
 - packages/localization - shared localization utilities, types, and similar for both backend and mobile app
+- packages/geocoding-db - standalone geocoding database used by location-service: schema, queries, and the OSM ingest pipeline. Lives on its own Postgres instance (`GEOCODING_DB_*` env, never the shared `DB_URL`) so geocoding never shares resources with vexl user data. The dataset refresh (`pnpm refresh:geocoding`) is run manually from an operator's machine — never add server-side cron/updater infrastructure. Details in packages/geocoding-db/README.md.
 - packages/resources-utils - shared utilities for handling offers, chat, and other "resources" operations
 - packages/rest-api - effect-ts based rest api definitions and client for both backend and mobile app
 - tooling/* - shared tooling for the repo (esling, prettier, etc...)

apps/location-service/.env.example

@@ -3,7 +3,7 @@
 
 SIGNATURE_PUBLIC_KEY=
 
-# Postgres with the places dataset (see scripts/ingestPlaces.ts)
-DB_URL=postgresql://localhost:5432/location
-DB_USER=postgres
-DB_PASSWORD=root
+# Standalone geocoding Postgres (see packages/geocoding-db)
+GEOCODING_DB_URL=postgresql://localhost:5433/geocoding
+GEOCODING_DB_USER=postgres
+GEOCODING_DB_PASSWORD=root

apps/location-service/.gitignore

@@ -1,6 +1,2 @@
-
 # Sentry Config File
 .env.sentry-build-plugin
-
-# OSM/Natural Earth datasets for the places DB (fetched by scripts/refresh-places.sh)
-data/

apps/location-service/README.md

@@ -1,22 +1,24 @@
 # location-service
 
 Serves location autocomplete (`GET /suggest`) and reverse geocoding
-(`GET /geocode`) for the mobile app — backed by an **in-house places database**
-built from OpenStreetMap. No third-party geocoding API is involved, so the
-coordinates users pick for offers never leave Vexl infrastructure.
+(`GET /geocode`) for the mobile app — backed by the **standalone geocoding
+database** built from OpenStreetMap (see `packages/geocoding-db`). No
+third-party geocoding API is involved, so the coordinates users pick for
+offers never leave Vexl infrastructure.
+
+The geocoding DB runs on its own Postgres instance, fully separated from the
+vexl service databases — geocoding traffic and the monthly OSM refresh never
+compete with chats/offers/connections for database resources. The service
+connects to it via `GEOCODING_DB_URL` / `GEOCODING_DB_USER` /
+`GEOCODING_DB_PASSWORD` and runs the schema migrations from
+`@vexl-next/geocoding-db` on startup. The only other component that connects
+to that database is the dataset refresh pipeline
+(`packages/geocoding-db/scripts/refresh.sh`).
 
 ## Architecture
 
-- **Data**, three tiers built from OSM, all enriched with the translations the
-  app ships languages for (`name:xx` tags) and a country code stamped via
-  Natural Earth boundaries:
-  - settlements (place=\* nodes: city…city_block) — searchable + reverse
-    geocoding,
-  - POIs (cafés, restaurants, pubs, bars, fast food, parks, gardens,
-    attractions, museums) — searchable only,
-  - streets (named street-like highway ways, deduplicated to one entry per
-    street name per ~10 km grid cell; **no house numbers by design**) —
-    searchable only.
+- **Data + schema + ingest**: `packages/geocoding-db` — see its README for
+  the data tiers, the refresh pipeline, and dev seeding.
 - **Search** (`/suggest`): prefix matching on normalized names cascading from
   cheap to expensive — important places first (partial covering index), then
   all names incl. streets/POIs, then typo-tolerant trigram matching (important
@@ -31,71 +33,19 @@ coordinates users pick for offers never leave Vexl infrastructure.
 - Localization is resolved per request `lang`: place names from the stored
   translations, country names via `Intl.DisplayNames` (CLDR, built into Node).
 
-## Refreshing the dataset
+## Deployment
 
-`scripts/refresh-places.sh` is the whole pipeline, meant to run weekly (cron)
-with no arguments — it fetches raw Geofabrik extracts (needs curl + osmium),
-filters them into the three tiers, and ingests into Postgres:
-
-```sh
-DB_URL=... DB_USER=... DB_PASSWORD=... ./scripts/refresh-places.sh
-```
-
-Raw and filtered files are kept under `data/` (`raw/`, `filtered/`), so stages
-can be re-run independently — e.g. after tweaking the filters, re-run just
-`refresh-places.sh extract ingest` without re-downloading ~85 GB. Regions are
-selectable with `-r` (any Geofabrik path): `-r europe`, `-r europe/slovakia`.
-
-Set `SLACK_ALERT_WEBHOOK_URL` (a Slack incoming-webhook URL) on the cron host
-to get a Slack message whenever any step fails — download, extraction, or
-ingest (including the sanity-gate abort). Leave it unset for dev runs.
-
-The ingest loads into staging tables and swaps them in a single transaction —
-the service keeps serving the old dataset until the new one is complete, and a
-sanity gate refuses to swap in a dataset >30 % smaller than the previous one.
+The service requires `GEOCODING_DB_URL` / `GEOCODING_DB_USER` /
+`GEOCODING_DB_PASSWORD` at startup — there is no fallback to the old `DB_URL`
+vars. Rolling out an image built from this change therefore requires the
+dedicated geocoding Postgres and the new env vars to be provisioned first;
+without them the service crash-loops at startup.
 
 ## Local development
 
-`pnpm dev:backend` seeds the places DB automatically: before the services
-start it runs `scripts/seedDevPlaces.ts`, which creates the `location`
-database and schema if missing and — only when the places table is empty —
-loads data. Because the Postgres volume persists between runs, this is a
-fast no-op on every run except the first one (and after `--fresh-db`).
-
-What an empty table gets seeded with:
-
-- **`--seed-places auto`** (the default): a full OSM ingest of Slovakia +
-  Czechia via `refresh-places.sh` when `osmium` is installed
-  (`brew install osmium-tool`), topped up with a fixture of the European
-  capitals so search and reverse geocoding work across Europe. Downloads
-  (~1 GB) are cached in `~/.cache/vexl/osm`, so re-seeding after
-  `--fresh-db` skips the download and only re-runs the ~minutes-long ingest.
-- **`--seed-places fixture`** (also the automatic fallback when osmium is
-  missing): just the committed fixture — major SK/CZ cities + European
-  capitals (`scripts/devSeedData.ts`). Instant, offline, no tools needed.
-- **`--seed-places off`**: no data — the database and schema are still
-  created if missing (the service crash-loops without them), but nothing is
-  seeded.
-
-The seeder can also run standalone (env from `.env`):
-
-```sh
-pnpm --filter @vexl-next/location-service seed:dev-places
-```
-
-A non-empty table is never touched. To switch an existing DB from fixture to
-real data, truncate first:
-
-```sh
-docker exec vexl-postgres psql -U postgres -d location -c 'TRUNCATE places CASCADE'
-```
-
-For other regions, run the pipeline directly, e.g.:
-
-```sh
-DB_URL=postgresql://localhost:5432/location DB_USER=postgres DB_PASSWORD=root \
-  ./scripts/refresh-places.sh -r europe/austria
-```
+`pnpm dev:backend` starts the dedicated geocoding Postgres container and
+seeds it automatically via `packages/geocoding-db` — see that package's
+README for seeding modes (`--seed-places auto|fixture|off`) and re-seeding.
 
 ## Tests
 

apps/location-service/package.json

@@ -12,26 +12,23 @@
     "format:fix": "prettier --write \"**/*.{js,mjs,cjs,jsx,ts,tsx,md,json}\"",
     "lint": "eslint src",
     "dev": "tsx -r dotenv/config src/index.ts",
-    "start": "node --enable-source-maps dist/index.js",
-    "ingest:places": "tsx -r dotenv/config scripts/ingestPlaces.ts",
-    "seed:dev-places": "tsx -r dotenv/config scripts/seedDevPlaces.ts"
+    "start": "node --enable-source-maps dist/index.js"
   },
   "dependencies": {
     "@effect/platform": "^0.93.6",
     "@effect/platform-node": "^0.103.0",
-    "@effect/sql": "^0.48.6",
-    "@effect/sql-pg": "^0.49.7",
     "@vexl-next/domain": "0.0.0",
+    "@vexl-next/geocoding-db": "0.0.0",
     "@vexl-next/localization": "0.0.0",
     "@vexl-next/rest-api": "0.0.0",
     "@vexl-next/server-utils": "0.0.0",
     "dotenv": "^16.4.5",
-    "effect": "^3.20.0",
-    "pg": "^8.12.0"
+    "effect": "^3.20.0"
   },
   "devDependencies": {
+    "@effect/sql": "^0.48.6",
+    "@effect/sql-pg": "^0.49.7",
     "@types/jest": "^29.4.0",
-    "@types/pg": "^8",
     "@vexl-next/esbuild": "0.0.0",
     "@vexl-next/eslint-config": "0.0.0",
     "@vexl-next/prettier-config": "0.0.0",

apps/location-service/src/__tests__/handlers/geocode.test.ts

@@ -1,11 +1,11 @@
 import {Latitude, Longitude} from '@vexl-next/domain/src/utility/geoCoordinates'
+import {seedPlacesInDb} from '@vexl-next/geocoding-db/src/tests/seedPlaces'
 import {LocationNotFoundError} from '@vexl-next/rest-api/src/services/location/contracts'
 import {expectErrorResponse} from '@vexl-next/server-utils/src/tests/expectErrorResponse'
 import {setDummyAuthHeaders} from '@vexl-next/server-utils/src/tests/nodeTestingApp'
 import {Effect, Schema} from 'effect'
 import {NodeTestingApp} from '../utils/NodeTestingApp'
 import {runPromiseInMockedEnvironment} from '../utils/runPromiseInMockedEnvironment'
-import {seedPlacesInDb} from '../utils/seedPlaces'
 
 const coordinates = (
   latitude: number,

apps/location-service/src/__tests__/handlers/suggest.test.ts

@@ -1,8 +1,8 @@
+import {seedPlacesInDb} from '@vexl-next/geocoding-db/src/tests/seedPlaces'
 import {setDummyAuthHeaders} from '@vexl-next/server-utils/src/tests/nodeTestingApp'
 import {Effect} from 'effect'
 import {NodeTestingApp} from '../utils/NodeTestingApp'
 import {runPromiseInMockedEnvironment} from '../utils/runPromiseInMockedEnvironment'
-import {seedPlacesInDb} from '../utils/seedPlaces'
 
 // The runtime itself is started/disposed globally in jest.afterenv.ts
 beforeAll(async () => {

apps/location-service/src/__tests__/ingest/ingestPipeline.test.ts

@@ -1,5 +1,6 @@
 /**
- * End-to-end test of the places ingest pipeline (scripts/ingestPlaces.ts):
+ * End-to-end test of the places ingest pipeline
+ * (packages/geocoding-db/scripts/ingest.ts):
  * geojsonseq parsing → transform → staging inserts → street merging →
  * importance boosting → index build → sanity gate → atomic table swap — and
  * finally the live API serving the ingested dataset.
@@ -11,18 +12,23 @@
  */
 import {PgClient} from '@effect/sql-pg'
 import {Latitude, Longitude} from '@vexl-next/domain/src/utility/geoCoordinates'
+import {computeImportance} from '@vexl-next/geocoding-db/src/common'
 import {setDummyAuthHeaders} from '@vexl-next/server-utils/src/tests/nodeTestingApp'
 import {Effect, Schema} from 'effect'
 import {execFile} from 'node:child_process'
 import {chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync} from 'node:fs'
 import {tmpdir} from 'node:os'
 import path from 'node:path'
-import {computeImportance} from '../../places/common'
 import {NodeTestingApp} from '../utils/NodeTestingApp'
 import {runPromiseInMockedEnvironment} from '../utils/runPromiseInMockedEnvironment'
 
 const RS = '\u001e'
-const SERVICE_ROOT = path.resolve(__dirname, '../../..')
+// The ingest pipeline lives in the geocoding-db package — resolve it through
+// the workspace symlink so the test does not depend on the repo layout.
+const GEOCODING_DB_ROOT = path.resolve(
+  __dirname,
+  '../../../node_modules/@vexl-next/geocoding-db'
+)
 
 // ---------------------------------------------------------------------------
 // Fixture builders
@@ -256,13 +262,13 @@ const runIngest = async (
       [
         'exec',
         'tsx',
-        'scripts/ingestPlaces.ts',
+        'scripts/ingest.ts',
         '--countries',
         path.join(workDir, 'ne_countries.geojson'),
         ...pbfPaths,
       ],
       {
-        cwd: SERVICE_ROOT,
+        cwd: GEOCODING_DB_ROOT,
         maxBuffer: 32 * 1024 * 1024,
         env: {
           ...process.env,

apps/location-service/src/__tests__/utils/runPromiseInMockedEnvironment.ts

@@ -3,26 +3,26 @@ import * as NodeHttpServer from '@effect/platform-node/NodeHttpServer'
 import {type HttpClient} from '@effect/platform/HttpClient'
 import {HttpApiBuilder} from '@effect/platform/index'
 import {type SqlClient} from '@effect/sql/SqlClient'
+import {GeocodingDbService} from '@vexl-next/geocoding-db/src/GeocodingDbService'
+import {GeocodingDbLayer} from '@vexl-next/geocoding-db/src/layer'
+import {
+  disposeGeocodingTestDatabase,
+  setupGeocodingTestDatabase,
+} from '@vexl-next/geocoding-db/src/tests/testGeocodingDb'
 import {type RateLimitingService} from '@vexl-next/server-utils/src/RateLimiting'
 import {ServerCrypto} from '@vexl-next/server-utils/src/ServerCrypto'
 import {mockedRateLimitingLayer} from '@vexl-next/server-utils/src/tests/mockedRateLimitingLayer'
 import {TestRequestHeaders} from '@vexl-next/server-utils/src/tests/nodeTestingApp'
-import {
-  disposeTestDatabase,
-  setupTestDatabase,
-} from '@vexl-next/server-utils/src/tests/testDb'
 import {Console, Effect, Layer, ManagedRuntime, type Scope} from 'effect'
 import {cryptoConfig} from '../../configs'
-import DbLayer from '../../db/layer'
-import {PlacesDbService} from '../../db/PlacesDbService'
+import {GeocodingService} from '../../geocoding'
 import {LocationApiLive} from '../../httpServer'
-import {PlacesService} from '../../places'
 
 export type MockedContexts =
   | ServerCrypto
   | SqlClient
-  | PlacesDbService
-  | PlacesService
+  | GeocodingDbService
+  | GeocodingService
   | HttpClient
   | TestRequestHeaders
   | RateLimitingService
@@ -36,9 +36,9 @@ const context = Layer.empty.pipe(
   Layer.provideMerge(TestServerLive),
   Layer.provideMerge(TestRequestHeaders.Live),
   Layer.provideMerge(mockedRateLimitingLayer),
-  Layer.provideMerge(PlacesService.Live),
-  Layer.provideMerge(PlacesDbService.Live),
-  Layer.provideMerge(DbLayer),
+  Layer.provideMerge(GeocodingService.Live),
+  Layer.provideMerge(GeocodingDbService.Live),
+  Layer.provideMerge(GeocodingDbLayer),
   Layer.provideMerge(ServerCrypto.layer(cryptoConfig)),
   Layer.provideMerge(NodeContext.layer)
 )
@@ -47,7 +47,7 @@ const runtime = ManagedRuntime.make(context)
 let runtimeReady = false
 
 export const startRuntime = async (): Promise<void> => {
-  await Effect.runPromise(setupTestDatabase)
+  await Effect.runPromise(setupGeocodingTestDatabase)
   await runtime.runPromise(Console.log('Initialized the test environment'))
   runtimeReady = true
 }
@@ -58,7 +58,7 @@ export const disposeRuntime = async (): Promise<void> => {
       Console.log('Disposed test environment')
     )
   )
-  await Effect.runPromise(disposeTestDatabase)
+  await Effect.runPromise(disposeGeocodingTestDatabase)
   runtimeReady = false
 }
 

apps/location-service/src/configs.ts

@@ -1,6 +1,5 @@
 export {
   cryptoConfig,
-  databaseConfig,
   healthServerPortConfig,
   isRunningInTestConfig,
   nodeEnvConfig,