Merge pull request #9 from ipnet-mesh/claude/add-data-home-config-01WnK1Q9vczq5atX7U5P3yJz

Add configurable data directory for services

Author: jinglemansweep

.env.example

@@ -11,6 +11,23 @@
 # MESHCORE_IMAGE=ghcr.io/ipnet-mesh/meshcore-hub:v1.0.0
 MESHCORE_IMAGE=
 
+# ===================
+# Data Directory
+# ===================
+
+# Base directory for all service data (collector DB, tags, members, etc.)
+# Default: ./data (relative to docker-compose.yml location)
+# Inside containers this is mapped to /data
+#
+# Structure:
+#   ${DATA_HOME}/
+#   ├── collector/
+#   │   ├── meshcore.db    # SQLite database
+#   │   └── tags.json      # Node tags for import
+#   └── web/
+#       └── members.json   # Network members list
+DATA_HOME=./data
+
 # ===================
 # Common Settings
 # ===================
@@ -72,10 +89,10 @@ NETWORK_RADIO_CONFIG=
 NETWORK_CONTACT_EMAIL=
 NETWORK_CONTACT_DISCORD=
 
-# Path to members JSON file (mounted into container)
-# For production: use ./data/members.json
-# For testing with example data: use ./example/data/members.json
-MEMBERS_FILE_PATH=./example/data/members.json
+# Members file location (optional override)
+# Default: ${DATA_HOME}/web/members.json
+# Only set this if you want to use a different location
+# MEMBERS_FILE=/custom/path/to/members.json
 
 # ===================
 # Webhook Settings

AGENTS.md

@@ -283,9 +283,12 @@ meshcore-hub/
 │       │   └── tags.json     # Example node tags data
 │       └── web/
 │           └── members.json  # Example network members data
-├── data/                     # Runtime data (gitignored)
-│   ├── collector/            # Collector data (tags.json)
-│   └── web/                  # Web data (members.json)
+├── data/                     # Runtime data (gitignored, DATA_HOME default)
+│   ├── collector/            # Collector data
+│   │   ├── meshcore.db       # SQLite database
+│   │   └── tags.json         # Node tags for import
+│   └── web/                  # Web data
+│       └── members.json      # Network members list
 ├── Dockerfile                # Docker build configuration
 ├── docker-compose.yml        # Docker Compose services (gitignored)
 └── docker-compose.yml.example  # Docker Compose template
@@ -433,11 +436,26 @@ meshcore-hub interface --mode receiver --mock
 See [PLAN.md](PLAN.md#configuration-environment-variables) for complete list.
 
 Key variables:
+- `DATA_HOME` - Base directory for all service data (default: `./data`)
 - `MQTT_HOST`, `MQTT_PORT`, `MQTT_PREFIX` - MQTT broker connection
-- `DATABASE_URL` - SQLAlchemy database URL
+- `DATABASE_URL` - SQLAlchemy database URL (default: `sqlite:///{DATA_HOME}/collector/meshcore.db`)
 - `API_READ_KEY`, `API_ADMIN_KEY` - API authentication keys
 - `LOG_LEVEL` - Logging verbosity
 
+### Data Directory Structure
+
+The `DATA_HOME` environment variable controls where all service data is stored:
+```
+${DATA_HOME}/
+├── collector/
+│   ├── meshcore.db    # SQLite database
+│   └── tags.json      # Node tags for import
+└── web/
+    └── members.json   # Network members list
+```
+
+Services automatically create their subdirectories if they don't exist.
+
 ### Webhook Configuration
 
 The collector supports forwarding events to external HTTP endpoints:

PLAN.md

@@ -456,13 +456,25 @@ meshcore/+/command/request_telemetry
 ### Common
 | Variable | Default | Description |
 |----------|---------|-------------|
+| DATA_HOME | ./data | Base directory for service data |
 | LOG_LEVEL | INFO | Logging level |
 | MQTT_HOST | localhost | MQTT broker host |
 | MQTT_PORT | 1883 | MQTT broker port |
 | MQTT_USERNAME | | MQTT username (optional) |
 | MQTT_PASSWORD | | MQTT password (optional) |
 | MQTT_PREFIX | meshcore | Topic prefix |
 
+### Data Directory Structure
+The `DATA_HOME` environment variable controls where all service data is stored:
+```
+${DATA_HOME}/
+├── collector/
+│   ├── meshcore.db    # SQLite database
+│   └── tags.json      # Node tags for import
+└── web/
+    └── members.json   # Network members list
+```
+
 ### Interface
 | Variable | Default | Description |
 |----------|---------|-------------|
@@ -474,7 +486,8 @@ meshcore/+/command/request_telemetry
 ### Collector
 | Variable | Default | Description |
 |----------|---------|-------------|
-| DATABASE_URL | sqlite:///./meshcore.db | SQLAlchemy URL |
+| DATABASE_URL | sqlite:///{DATA_HOME}/collector/meshcore.db | SQLAlchemy URL |
+| TAGS_FILE | {DATA_HOME}/collector/tags.json | Path to tags JSON file |
 
 ### API
 | Variable | Default | Description |
@@ -483,7 +496,7 @@ meshcore/+/command/request_telemetry
 | API_PORT | 8000 | API bind port |
 | API_READ_KEY | | Read-only API key |
 | API_ADMIN_KEY | | Admin API key |
-| DATABASE_URL | sqlite:///./meshcore.db | SQLAlchemy URL |
+| DATABASE_URL | sqlite:///{DATA_HOME}/collector/meshcore.db | SQLAlchemy URL |
 
 ### Web Dashboard
 | Variable | Default | Description |
@@ -500,7 +513,7 @@ meshcore/+/command/request_telemetry
 | NETWORK_RADIO_CONFIG | | Radio config details |
 | NETWORK_CONTACT_EMAIL | | Contact email |
 | NETWORK_CONTACT_DISCORD | | Discord link |
-| MEMBERS_FILE | members.json | Path to members JSON |
+| MEMBERS_FILE | {DATA_HOME}/web/members.json | Path to members JSON |
 
 ---
 

alembic/env.py

@@ -21,10 +21,22 @@
 
 def get_database_url() -> str:
     """Get database URL from environment or config."""
-    # First try environment variable
+    from pathlib import Path
+
+    # First try explicit DATABASE_URL environment variable
     url = os.environ.get("DATABASE_URL")
     if url:
+        # Ensure directory exists for sqlite URLs
+        if url.startswith("sqlite:///"):
+            db_path = Path(url.replace("sqlite:///", ""))
+            db_path.parent.mkdir(parents=True, exist_ok=True)
         return url
+    # Try DATA_HOME environment variable
+    data_home = os.environ.get("DATA_HOME")
+    if data_home:
+        db_path = Path(data_home) / "collector" / "meshcore.db"
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        return f"sqlite:///{db_path}"
     # Fall back to alembic.ini
     return config.get_main_option("sqlalchemy.url", "sqlite:///./meshcore.db")
 

docker-compose.yml.example

@@ -17,9 +17,14 @@
 #   - migrate: Run database migrations
 #   - import-tags: Import node tags from JSON file
 #
-# Data Directory Structure:
-#   ./data/
+# Data Directory (DATA_HOME):
+#   The DATA_HOME environment variable controls where all service data is stored.
+#   Default: ./data (local) or /data (in containers)
+#
+#   Structure:
+#   ${DATA_HOME}/
 #   ├── collector/
+#   │   ├── meshcore.db    # SQLite database
 #   │   └── tags.json      # Node tags for import
 #   └── web/
 #       └── members.json   # Network members list
@@ -175,17 +180,16 @@ services:
       mqtt:
         condition: service_healthy
     volumes:
-      - meshcore_data:/data
-      # Mount collector data directory for tags.json (optional)
-      - ${COLLECTOR_DATA_PATH:-./data/collector}:/app/collector-data:ro
+      # Mount data directory (contains collector/meshcore.db and collector/tags.json)
+      - ${DATA_HOME:-./data}:/data
     environment:
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
       - MQTT_HOST=${MQTT_HOST:-mqtt}
       - MQTT_PORT=${MQTT_PORT:-1883}
       - MQTT_USERNAME=${MQTT_USERNAME:-}
       - MQTT_PASSWORD=${MQTT_PASSWORD:-}
       - MQTT_PREFIX=${MQTT_PREFIX:-meshcore}
-      - DATABASE_URL=sqlite:////data/meshcore.db
+      - DATA_HOME=/data
       # Webhook configuration
       - WEBHOOK_ADVERTISEMENT_URL=${WEBHOOK_ADVERTISEMENT_URL:-}
       - WEBHOOK_ADVERTISEMENT_SECRET=${WEBHOOK_ADVERTISEMENT_SECRET:-}
@@ -227,15 +231,16 @@ services:
     ports:
       - "${API_PORT:-8000}:8000"
     volumes:
-      - meshcore_data:/data
+      # Mount data directory (uses collector/meshcore.db)
+      - ${DATA_HOME:-./data}:/data
     environment:
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
       - MQTT_HOST=${MQTT_HOST:-mqtt}
       - MQTT_PORT=${MQTT_PORT:-1883}
       - MQTT_USERNAME=${MQTT_USERNAME:-}
       - MQTT_PASSWORD=${MQTT_PASSWORD:-}
       - MQTT_PREFIX=${MQTT_PREFIX:-meshcore}
-      - DATABASE_URL=sqlite:////data/meshcore.db
+      - DATA_HOME=/data
       - API_HOST=0.0.0.0
       - API_PORT=8000
       - API_READ_KEY=${API_READ_KEY:-}
@@ -266,23 +271,23 @@ services:
         condition: service_healthy
     ports:
       - "${WEB_PORT:-8080}:8080"
+    volumes:
+      # Mount data directory (uses web/members.json)
+      - ${DATA_HOME:-./data}:/data
     environment:
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
       - API_BASE_URL=http://api:8000
       - API_KEY=${API_READ_KEY:-}
       - WEB_HOST=0.0.0.0
       - WEB_PORT=8080
+      - DATA_HOME=/data
       - NETWORK_NAME=${NETWORK_NAME:-MeshCore Network}
       - NETWORK_CITY=${NETWORK_CITY:-}
       - NETWORK_COUNTRY=${NETWORK_COUNTRY:-}
       - NETWORK_LOCATION=${NETWORK_LOCATION:-}
       - NETWORK_RADIO_CONFIG=${NETWORK_RADIO_CONFIG:-}
       - NETWORK_CONTACT_EMAIL=${NETWORK_CONTACT_EMAIL:-}
       - NETWORK_CONTACT_DISCORD=${NETWORK_CONTACT_DISCORD:-}
-      - MEMBERS_FILE=${MEMBERS_FILE:-/app/web-data/members.json}
-    volumes:
-      # Mount web data directory for members.json (optional)
-      - ${WEB_DATA_PATH:-./data/web}:/app/web-data:ro
     command: ["web"]
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"]
@@ -302,9 +307,10 @@ services:
     profiles:
       - migrate
     volumes:
-      - meshcore_data:/data
+      # Mount data directory (uses collector/meshcore.db)
+      - ${DATA_HOME:-./data}:/data
     environment:
-      - DATABASE_URL=sqlite:////data/meshcore.db
+      - DATA_HOME=/data
     command: ["db", "upgrade"]
 
   # ==========================================================================
@@ -318,12 +324,13 @@ services:
     profiles:
       - import-tags
     volumes:
-      - meshcore_data:/data
-      - ${COLLECTOR_DATA_PATH:-./data/collector}:/app/collector-data:ro
+      # Mount data directory (uses collector/tags.json and collector/meshcore.db)
+      - ${DATA_HOME:-./data}:/data
     environment:
-      - DATABASE_URL=sqlite:////data/meshcore.db
+      - DATA_HOME=/data
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
-    command: ["collector", "import-tags", "/app/collector-data/tags.json"]
+    # Uses default tags file: /data/collector/tags.json
+    command: ["collector", "import-tags"]
 
 # ==========================================================================
 # Volumes
@@ -333,5 +340,3 @@ volumes:
     name: meshcore_mosquitto_data
   mosquitto_log:
     name: meshcore_mosquitto_log
-  meshcore_data:
-    name: meshcore_data

src/meshcore_hub/api/cli.py

@@ -18,12 +18,19 @@
     envvar="API_PORT",
     help="API server port",
 )
+@click.option(
+    "--data-home",
+    type=str,
+    default=None,
+    envvar="DATA_HOME",
+    help="Base data directory (default: ./data)",
+)
 @click.option(
     "--database-url",
     type=str,
-    default="sqlite:///./meshcore.db",
+    default=None,
     envvar="DATABASE_URL",
-    help="Database connection URL",
+    help="Database connection URL (default: sqlite:///{data_home}/collector/meshcore.db)",
 )
 @click.option(
     "--read-key",
@@ -78,7 +85,8 @@ def api(
     ctx: click.Context,
     host: str,
     port: int,
-    database_url: str,
+    data_home: str | None,
+    database_url: str | None,
     read_key: str | None,
     admin_key: str | None,
     mqtt_host: str,
@@ -108,14 +116,27 @@ def api(
     """
     import uvicorn
 
+    from meshcore_hub.common.config import get_api_settings
     from meshcore_hub.api.app import create_app
 
+    # Get settings to compute effective values
+    settings = get_api_settings()
+
+    # Override data_home if provided
+    if data_home:
+        settings = settings.model_copy(update={"data_home": data_home})
+
+    # Use effective database URL if not explicitly provided
+    effective_db_url = database_url if database_url else settings.effective_database_url
+    effective_data_home = data_home or settings.data_home
+
     click.echo("=" * 50)
     click.echo("MeshCore Hub API Server")
     click.echo("=" * 50)
     click.echo(f"Host: {host}")
     click.echo(f"Port: {port}")
-    click.echo(f"Database: {database_url}")
+    click.echo(f"Data home: {effective_data_home}")
+    click.echo(f"Database: {effective_db_url}")
     click.echo(f"MQTT: {mqtt_host}:{mqtt_port} (prefix: {mqtt_prefix})")
     click.echo(f"Read key configured: {read_key is not None}")
     click.echo(f"Admin key configured: {admin_key is not None}")
@@ -144,7 +165,7 @@ def api(
     else:
         # For production, create app directly
         app = create_app(
-            database_url=database_url,
+            database_url=effective_db_url,
             read_key=read_key,
             admin_key=admin_key,
             mqtt_host=mqtt_host,