Merge branch 'master' into csv-upload

Author: wadhwamatic

alerting/package.json

@@ -64,7 +64,7 @@
     "jpeg-js": "^0.4.4",
     "lodash": "^4.18.0",
     "node-fetch": "^2.6.7",
-    "nodemailer": "^8.0.9",
+    "nodemailer": "^9.0.1",
     "pg": "^8.5.1",
     "prism-common": "file:../common",
     "puppeteer": "^24.33.0",

alerting/yarn.lock

@@ -3132,10 +3132,10 @@ node-releases@^2.0.27:
   resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.27.tgz#eedca519205cf20f650f61d56b070db111231e4e"
   integrity sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==
 
-nodemailer@^8.0.9:
-  version "8.0.11"
-  resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-8.0.11.tgz#ce46b7c2c8bbf17b0408122fbfb4e47f4bffc688"
-  integrity sha512-nrO/pDAUKl+wXX+lx16tDLbnm0fW6sK/x8mgohaCpg+CdCEl482bD4tCuAZk2DyliruiNTIZxRCoWkDqJEnAiA==
+nodemailer@^9.0.1:
+  version "9.0.1"
+  resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-9.0.1.tgz#871c04d8423fc0c790289f4a8f6f71f1b3563e8c"
+  integrity sha512-Gwv8SQewT616ZM/URn0H54b8PWo/Wum7md3EW2aWy1lO27+WZCX+Xyak3J+NlmHUjDh5ME+uesJUDRbR3Ye8Bw==
 
 normalize-path@^3.0.0:
   version "3.0.0"
@@ -3786,7 +3786,16 @@ string-length@^4.0.2:
     char-regex "^1.0.2"
     strip-ansi "^6.0.0"
 
-"string-width-cjs@npm:string-width@^4.2.0", string-width@4.2.3, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.0.1, string-width@^5.1.2:
+"string-width-cjs@npm:string-width@^4.2.0":
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
+  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.1"
+
+string-width@4.2.3, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.0.1, string-width@^5.1.2:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -3809,7 +3818,14 @@ string_decoder@~1.1.1:
   dependencies:
     safe-buffer "~5.1.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -4190,7 +4206,7 @@ wordwrap@^1.0.0:
   resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb"
   integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -4208,6 +4224,15 @@ wrap-ansi@^6.2.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
+wrap-ansi@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"

api/.env.example

@@ -10,3 +10,10 @@ PRISM_OIDC_CLIENT_SECRET=
 PRISM_SESSION_SECRET=
 # Default one week session time
 PRISM_SESSION_TTL_SECONDS=604800
+
+# Local map export storage (optional — docker-compose.develop.yml sets RustFS defaults).
+# EXPORT_MAP_S3_BUCKET=prism-dev
+# AWS_ACCESS_KEY_ID=rustfsadmin
+# AWS_SECRET_ACCESS_KEY=rustfsadmin
+# AWS_ENDPOINT_URL=http://rustfs:9000          # server-side S3 (Docker network)
+# AWS_PRESIGN_ENDPOINT_URL=http://localhost:9000  # presigned download URLs (browser)

api/README.md

@@ -42,9 +42,33 @@ export KOBO_USERNAME=kobo_user KOBO_PASSWORD=test
 make api
 ```
 
-This starts two containers via `docker-compose.develop.yml`:
+This starts four containers via `docker-compose.develop.yml`:
 - **`db`** — PostGIS (Postgres) on host port **54321**
+- **`rustfs`** — local S3-compatible storage for map export artifacts (API port **9000**, console **9001**)
 - **`api`** — FastAPI (uvicorn with hot reload) on host port **80**
+- **`export_map_worker`** — polls `map_export_jobs`, runs Playwright export, writes artifacts to RustFS
+
+### Local map export storage (RustFS)
+
+Production stores batch and scheduled map exports in **AWS S3** (`EXPORT_MAP_S3_BUCKET`, e.g. `s3://prism-wfp/batch-maps`). Local dev uses **RustFS** so the same code paths run without real AWS credentials for artifacts.
+
+`docker-compose.develop.yml` sets this automatically for **`api`** and **`export_map_worker`**:
+
+| Variable | Local value | Role |
+|---|---|---|
+| `EXPORT_MAP_S3_BUCKET` | `prism-dev` | Bucket created by `rustfs-init` |
+| `AWS_ENDPOINT_URL` | `http://rustfs:9000` | Server-side S3 calls (worker, API inside Docker) |
+| `AWS_PRESIGN_ENDPOINT_URL` | `http://localhost:9000` | Presigned download URLs the browser can reach |
+
+RustFS console: **http://localhost:9001** (login `rustfsadmin` / `rustfsadmin`).
+
+These flows stay aligned with production when `EXPORT_MAP_S3_BUCKET` is set and `EXPORT_MAP_LOCAL_OUTPUT_DIR` is empty (the compose default):
+
+1. **Write** — `export_map_worker`
+2. **Dedupe** — `POST /export-map/jobs` and `scheduled_public_maps` cron
+3. **Download** — `GET /export-map/jobs/{id}` and admin schedule download
+
+The alternative dev mode, `EXPORT_MAP_LOCAL_OUTPUT_DIR`, writes `file://…` URIs to disk instead of S3 and skips the RustFS/prod-aligned path.
 
 ### 3. Run migrations and seed data
 
@@ -55,7 +79,7 @@ make db-migrate
 make db-seed
 ```
 
-`db-migrate` applies Alembic migrations (`upgrade head`). `db-seed` runs migrations first, then inserts sample dev data (Mozambique AA metadata, a `local_dev_user`, and example alert rows).
+`db-migrate` applies Alembic migrations (`upgrade head`). `db-seed` runs migrations first, then inserts sample dev data (Mozambique AA metadata, a `local_dev_user`, example alert rows, and five `[Seed]` map export schedules for cron/download QA).
 
 ### 4. Verify
 

api/docker-compose.develop.yml

@@ -16,13 +16,56 @@ services:
       retries: 12
       start_period: 30s
 
+  # Local S3-compatible storage for map export artifacts (dev only).
+  rustfs:
+    image: rustfs/rustfs:latest
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      RUSTFS_ACCESS_KEY: rustfsadmin
+      RUSTFS_SECRET_KEY: rustfsadmin
+      RUSTFS_CONSOLE_ENABLE: "true"
+      RUSTFS_ADDRESS: 0.0.0.0:9000
+      RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001
+    volumes:
+      - rustfs_data:/data
+    command: ["/data"]
+    healthcheck:
+      test: ["CMD-SHELL", "curl -sf http://127.0.0.1:9000/health || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+      start_period: 40s
+
+  rustfs-init:
+    image: amazon/aws-cli:2.27.50
+    depends_on:
+      rustfs:
+        condition: service_healthy
+    environment:
+      AWS_ACCESS_KEY_ID: rustfsadmin
+      AWS_SECRET_ACCESS_KEY: rustfsadmin
+      AWS_DEFAULT_REGION: us-east-1
+    entrypoint: /bin/sh
+    command:
+      - -c
+      - |
+        until aws --endpoint-url http://rustfs:9000 s3 ls >/dev/null 2>&1; do
+          echo "waiting for rustfs..."
+          sleep 2
+        done
+        aws --endpoint-url http://rustfs:9000 s3 mb s3://prism-dev 2>/dev/null || true
+
   api:
     env_file:
       - path: .env
         required: false
     depends_on:
       db:
         condition: service_healthy
+      rustfs-init:
+        condition: service_completed_successfully
     volumes:
       - ./prism_app:/prism_app
       - ./pyproject.toml:/pyproject.toml
@@ -52,18 +95,27 @@ services:
       - VECTOR_STORE_ACCESS_KEY_ID=${VECTOR_STORE_ACCESS_KEY_ID}
       - VECTOR_STORE_SECRET_ACCESS_KEY=${VECTOR_STORE_SECRET_ACCESS_KEY}
       - VECTOR_STORE_DEFAULT_REGION=${VECTOR_STORE_DEFAULT_REGION}
+      # Map export artifacts → local RustFS
+      - EXPORT_MAP_S3_BUCKET=prism-dev
+      - EXPORT_MAP_LOCAL_OUTPUT_DIR=
+      - AWS_ACCESS_KEY_ID=rustfsadmin
+      - AWS_SECRET_ACCESS_KEY=rustfsadmin
+      - AWS_DEFAULT_REGION=us-east-1
+      - AWS_ENDPOINT_URL=http://rustfs:9000
+      - AWS_PRESIGN_ENDPOINT_URL=http://localhost:9000
     command: uvicorn prism_app.main:app --host 0.0.0.0 --port 80 --reload
     # Infinite loop, to keep it alive, for debugging
     # command: bash -c "while true; do echo 'sleeping...' && sleep 10; done"
-  
+
   export_map_worker:
     shm_size: "2gb"
     depends_on:
       db:
         condition: service_healthy
+      rustfs-init:
+        condition: service_completed_successfully
     volumes:
       - ./prism_app:/prism_app
-      - ./cache:/cache:rw
     build:
       context: ./
       dockerfile: ./Dockerfile.export-map-worker
@@ -74,14 +126,16 @@ services:
       - POSTGRES_DB=postgres
       - POSTGRES_PORT=5432
       - POSTGRES_PASSWORD=!ChangeMe!
-      - EXPORT_MAP_S3_BUCKET=${EXPORT_MAP_S3_BUCKET:-}
-      - EXPORT_MAP_LOCAL_OUTPUT_DIR=${EXPORT_MAP_LOCAL_OUTPUT_DIR:-/cache/map_exports}
-      - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-}
-      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
-      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
+      - EXPORT_MAP_S3_BUCKET=prism-dev
+      - EXPORT_MAP_LOCAL_OUTPUT_DIR=
+      - AWS_ACCESS_KEY_ID=rustfsadmin
+      - AWS_SECRET_ACCESS_KEY=rustfsadmin
+      - AWS_DEFAULT_REGION=us-east-1
+      - AWS_ENDPOINT_URL=http://rustfs:9000
       - BROWSER_POOL_SIZE=1
     working_dir: /prism_app
     command: [ "python", "-m", "prism_app.workers.export_map_worker" ]
 
 volumes:
   db_data:
+  rustfs_data: