Merge pull request #1509 from MODSetter/dev

feat(release: 0.0.29): ETL/embedding caches, unified model connections, reverse-proxy support, podcast & indexing improvements

Author: MODSetter

.github/workflows/desktop-release.yml

@@ -95,10 +95,12 @@ jobs:
         run: pnpm build
         working-directory: surfsense_web
         env:
-          NEXT_PUBLIC_FASTAPI_BACKEND_URL: ${{ vars.NEXT_PUBLIC_FASTAPI_BACKEND_URL }}
+          NEXT_PUBLIC_FASTAPI_BACKEND_URL: ${{ vars.HOSTED_BACKEND_URL }}
+          SURFSENSE_BACKEND_INTERNAL_URL: ${{ vars.HOSTED_BACKEND_URL }}
           NEXT_PUBLIC_ZERO_CACHE_URL: ${{ vars.NEXT_PUBLIC_ZERO_CACHE_URL }}
           NEXT_PUBLIC_DEPLOYMENT_MODE: ${{ vars.NEXT_PUBLIC_DEPLOYMENT_MODE }}
-          NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${{ vars.NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE }}
+          NEXT_PUBLIC_AUTH_TYPE: ${{ vars.NEXT_PUBLIC_AUTH_TYPE }}
+          NEXT_PUBLIC_ETL_SERVICE: ${{ vars.NEXT_PUBLIC_ETL_SERVICE }}
           NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }}
 
       - name: Install desktop dependencies
@@ -109,6 +111,7 @@ jobs:
         run: pnpm build
         working-directory: surfsense_desktop
         env:
+          HOSTED_BACKEND_URL: ${{ vars.HOSTED_BACKEND_URL }}
           HOSTED_FRONTEND_URL: ${{ vars.HOSTED_FRONTEND_URL }}
           POSTHOG_KEY: ${{ secrets.POSTHOG_KEY }}
           POSTHOG_HOST: ${{ vars.POSTHOG_HOST }}

.github/workflows/docker-build.yml

@@ -199,11 +199,6 @@ jobs:
           build-args: |
             ${{ matrix.image == 'backend' && format('USE_CUDA={0}', matrix.use_cuda) || '' }}
             ${{ matrix.image == 'backend' && format('CUDA_EXTRA={0}', matrix.cuda_extra) || '' }}
-            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__' || '' }}
-            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__' || '' }}
-            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__' || '' }}
-            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_ZERO_CACHE_URL=__NEXT_PUBLIC_ZERO_CACHE_URL__' || '' }}
-            ${{ matrix.image == 'web' && 'NEXT_PUBLIC_DEPLOYMENT_MODE=__NEXT_PUBLIC_DEPLOYMENT_MODE__' || '' }}
 
       - name: Export digest
         run: |

.github/workflows/e2e-tests.yml

@@ -27,9 +27,10 @@ jobs:
       PLAYWRIGHT_TEST_EMAIL: e2e-test@surfsense.net
       PLAYWRIGHT_TEST_PASSWORD: E2eTestPassword123!
       # Frontend env: Playwright's webServer (surfsense_web/playwright.config.ts)
-      # spawns `pnpm build && pnpm start` in CI; these get baked into the build.
+      # spawns `pnpm build && pnpm start` in CI.
       NEXT_PUBLIC_FASTAPI_BACKEND_URL: http://localhost:8000
-      NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: LOCAL
+      SURFSENSE_BACKEND_INTERNAL_URL: http://localhost:8000
+      AUTH_TYPE: LOCAL
       # Shared secret for the test-only POST /__e2e__/auth/token endpoint.
       # Must match docker-compose.e2e.yml's backend env (x-backend-env).
       E2E_MINT_SECRET: e2e-mint-secret-not-for-production

VERSION

@@ -1 +1 @@
-0.0.28
+0.0.29

docker/.env.example

@@ -30,6 +30,9 @@ SECRET_KEY=replace_me_with_a_random_string
 # Auth type: LOCAL (email/password) or GOOGLE (OAuth)
 AUTH_TYPE=LOCAL
 
+# Deployment mode: self-hosted enables local filesystem connectors; cloud hides them.
+DEPLOYMENT_MODE=self-hosted
+
 # Allow new user registrations (TRUE or FALSE)
 # REGISTRATION_ENABLED=TRUE
 
@@ -43,51 +46,47 @@ ETL_SERVICE=DOCLING
 EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
 
 # ------------------------------------------------------------------------------
-# Ports (change to avoid conflicts with other services on your machine)
+# How You Access SurfSense
 # ------------------------------------------------------------------------------
+# One public URL. Browser traffic stays same-origin and Caddy routes internally.
+SURFSENSE_PUBLIC_URL=http://localhost:3929
 
-# BACKEND_PORT=8929
-# FRONTEND_PORT=3929
-# ZERO_CACHE_PORT=5929
-# SEARXNG_PORT=8888
-# FLOWER_PORT=5555
-
-# ==============================================================================
-# DEV COMPOSE ONLY (docker-compose.dev.yml)
-# You only need them only if you are running `docker-compose.dev.yml`.
-# ==============================================================================
-
-# -- pgAdmin (database GUI) --
-# PGADMIN_PORT=5050
-# PGADMIN_DEFAULT_EMAIL=admin@surfsense.com
-# PGADMIN_DEFAULT_PASSWORD=surfsense
-
-# -- Redis exposed port (dev only; Redis is internal-only in prod) --
-# REDIS_PORT=6379
-
-# -- WhatsApp bridge exposed port (dev/hybrid only; prod keeps it Docker-internal) --
-# WHATSAPP_BRIDGE_PORT=9929
+# ------------------------------------------------------------------------------
+# Public Ports
+# ------------------------------------------------------------------------------
+# Production Docker exposes only Caddy to your machine. Caddy then routes
+# frontend, backend, and zero-cache traffic internally.
+#
+# Local default: LISTEN_HTTP_PORT=3929
+# Domain default: LISTEN_HTTP_PORT=80 and LISTEN_HTTPS_PORT=443
+LISTEN_HTTP_PORT=3929
+LISTEN_HTTPS_PORT=443
 
-# -- Frontend Build Args --
-# In dev, the frontend is built from source and these are passed as build args.
-# In prod, they are automatically derived from AUTH_TYPE, ETL_SERVICE, and the port settings above.
-# NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL
-# NEXT_PUBLIC_ETL_SERVICE=DOCLING
-# NEXT_PUBLIC_DEPLOYMENT_MODE=self-hosted
+# ------------------------------------------------------------------------------
+# Custom Domain / HTTPS
+# ------------------------------------------------------------------------------
+# Leave SURFSENSE_SITE_ADDRESS as :80 for local HTTP.
+# Set it to your domain to enable automatic HTTPS:
+# SURFSENSE_SITE_ADDRESS=surf.example.com
+# CERT_EMAIL=you@example.com
+SURFSENSE_SITE_ADDRESS=:80
+CERT_EMAIL=
 
 # ------------------------------------------------------------------------------
-# Custom Domain / Reverse Proxy
+# Advanced Reverse Proxy Settings
 # ------------------------------------------------------------------------------
-# ONLY set these if you are serving SurfSense on a real domain via a reverse
-# proxy (e.g. Caddy, Nginx, Cloudflare Tunnel).
-# For standard localhost deployments, leave all of these commented out.
-# they are automatically derived from the port settings above.
+# Usually do not change these. They are for custom certificate setup, CDNs/load
+# balancers, trusted proxy IPs, or changing upload limits.
+#
+# CERT_ACME_CA=https://acme-v02.api.letsencrypt.org/directory
+# CERT_ACME_DNS=
+# If a CDN/load balancer sits in front of Caddy, narrow this to that proxy's CIDRs.
+# TRUSTED_PROXIES=0.0.0.0/0
+# SURFSENSE_MAX_BODY_SIZE=5GB
 #
-# NEXT_FRONTEND_URL=https://app.yourdomain.com
-# BACKEND_URL=https://api.yourdomain.com
-# NEXT_PUBLIC_FASTAPI_BACKEND_URL=https://api.yourdomain.com
-# NEXT_PUBLIC_ZERO_CACHE_URL=https://zero.yourdomain.com
-# FASTAPI_BACKEND_INTERNAL_URL=http://backend:8000
+# Browser API and Zero URLs are same-origin relative behind bundled Caddy.
+# Next.js server-side calls use Docker DNS through SURFSENSE_BACKEND_INTERNAL_URL
+# set internally by docker-compose.yml. Usually do not override it.
 
 # ------------------------------------------------------------------------------
 # Zero-cache (real-time sync)
@@ -108,10 +107,9 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
 
 # Sync worker tuning. zero-cache defaults ZERO_NUM_SYNC_WORKERS to the number
 # of CPU cores, which can exceed the connection pool limits on high-core machines.
-# Each sync worker needs at least 1 connection from both the UPSTREAM and CVR
-# pools, so these constraints must hold:
-#   ZERO_UPSTREAM_MAX_CONNS >= ZERO_NUM_SYNC_WORKERS
-#   ZERO_CVR_MAX_CONNS      >= ZERO_NUM_SYNC_WORKERS
+# Each sync worker needs at least 1 connection from both the UPSTREAM and CVR pools.
+# Keep ZERO_UPSTREAM_MAX_CONNS and ZERO_CVR_MAX_CONNS greater than or equal to
+# ZERO_NUM_SYNC_WORKERS.
 # Default of 4 workers is sufficient for self-hosted / personal use.
 # ZERO_NUM_SYNC_WORKERS=4
 # ZERO_UPSTREAM_MAX_CONNS=20
@@ -125,16 +123,16 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
 
 # ZERO_QUERY_URL: where zero-cache forwards query requests for resolution.
 # ZERO_MUTATE_URL: required by zero-cache when auth tokens are used, even though
-#   SurfSense does not use Zero mutators. Setting both URLs tells zero-cache to
-#   skip its own JWT verification and let the app endpoints handle auth instead.
-#   The mutate endpoint is a no-op that returns an empty response.
+# SurfSense does not use Zero mutators. Setting both URLs tells zero-cache to
+# skip its own JWT verification and let the app endpoints handle auth instead.
+# The mutate endpoint is a no-op that returns an empty response.
 # Default: Docker service networking (http://frontend:3000/api/zero/...).
 # Override when running the frontend outside Docker:
-#   ZERO_QUERY_URL=http://host.docker.internal:3000/api/zero/query
-#   ZERO_MUTATE_URL=http://host.docker.internal:3000/api/zero/mutate
-# Override for custom domain:
-#   ZERO_QUERY_URL=https://app.yourdomain.com/api/zero/query
-#   ZERO_MUTATE_URL=https://app.yourdomain.com/api/zero/mutate
+# ZERO_QUERY_URL=http://host.docker.internal:3000/api/zero/query
+# ZERO_MUTATE_URL=http://host.docker.internal:3000/api/zero/mutate
+# Override for custom domain only when zero-cache is not in the bundled Docker network:
+# ZERO_QUERY_URL=https://surf.example.com/api/zero/query
+# ZERO_MUTATE_URL=https://surf.example.com/api/zero/mutate
 # ZERO_QUERY_URL=http://frontend:3000/api/zero/query
 # ZERO_MUTATE_URL=http://frontend:3000/api/zero/mutate
 
@@ -222,73 +220,74 @@ STT_SERVICE=local/base
 # ------------------------------------------------------------------------------
 
 # -- Google Connectors --
-# GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback
-# GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback
-# GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback
+# GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:3929/api/v1/auth/google/calendar/connector/callback
+# GOOGLE_GMAIL_REDIRECT_URI=http://localhost:3929/api/v1/auth/google/gmail/connector/callback
+# GOOGLE_DRIVE_REDIRECT_URI=http://localhost:3929/api/v1/auth/google/drive/connector/callback
 
 # -- Notion --
 # NOTION_CLIENT_ID=
 # NOTION_CLIENT_SECRET=
-# NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback
+# NOTION_REDIRECT_URI=http://localhost:3929/api/v1/auth/notion/connector/callback
 
 # -- Slack --
 # SLACK_CLIENT_ID=
 # SLACK_CLIENT_SECRET=
-# SLACK_REDIRECT_URI=http://localhost:8000/api/v1/auth/slack/connector/callback
+# SLACK_REDIRECT_URI=http://localhost:3929/api/v1/auth/slack/connector/callback
 
 # -- Discord --
 # DISCORD_CLIENT_ID=
 # DISCORD_CLIENT_SECRET=
-# DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback
+# DISCORD_REDIRECT_URI=http://localhost:3929/api/v1/auth/discord/connector/callback
 # DISCORD_BOT_TOKEN=
 
 # -- Atlassian (Jira & Confluence) --
 # ATLASSIAN_CLIENT_ID=
 # ATLASSIAN_CLIENT_SECRET=
-# JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback
-# CONFLUENCE_REDIRECT_URI=http://localhost:8000/api/v1/auth/confluence/connector/callback
+# JIRA_REDIRECT_URI=http://localhost:3929/api/v1/auth/jira/connector/callback
+# CONFLUENCE_REDIRECT_URI=http://localhost:3929/api/v1/auth/confluence/connector/callback
 
 # -- Linear --
 # LINEAR_CLIENT_ID=
 # LINEAR_CLIENT_SECRET=
-# LINEAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/linear/connector/callback
+# LINEAR_REDIRECT_URI=http://localhost:3929/api/v1/auth/linear/connector/callback
 
 # -- ClickUp --
 # CLICKUP_CLIENT_ID=
 # CLICKUP_CLIENT_SECRET=
-# CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback
+# CLICKUP_REDIRECT_URI=http://localhost:3929/api/v1/auth/clickup/connector/callback
 
 # -- Airtable --
 # AIRTABLE_CLIENT_ID=
 # AIRTABLE_CLIENT_SECRET=
-# AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback
+# AIRTABLE_REDIRECT_URI=http://localhost:3929/api/v1/auth/airtable/connector/callback
 
 # -- Microsoft OAuth (Teams & OneDrive) --
 # MICROSOFT_CLIENT_ID=
 # MICROSOFT_CLIENT_SECRET=
-# TEAMS_REDIRECT_URI=http://localhost:8000/api/v1/auth/teams/connector/callback
-# ONEDRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/onedrive/connector/callback
+# TEAMS_REDIRECT_URI=http://localhost:3929/api/v1/auth/teams/connector/callback
+# ONEDRIVE_REDIRECT_URI=http://localhost:3929/api/v1/auth/onedrive/connector/callback
 
 # -- Dropbox --
 # DROPBOX_APP_KEY=
 # DROPBOX_APP_SECRET=
-# DROPBOX_REDIRECT_URI=http://localhost:8000/api/v1/auth/dropbox/connector/callback
+# DROPBOX_REDIRECT_URI=http://localhost:3929/api/v1/auth/dropbox/connector/callback
 
 # -- Composio --
 # COMPOSIO_API_KEY=
 # COMPOSIO_ENABLED=TRUE
-# COMPOSIO_REDIRECT_URI=http://localhost:8000/api/v1/auth/composio/connector/callback
+# COMPOSIO_REDIRECT_URI=http://localhost:3929/api/v1/auth/composio/connector/callback
 
 # ------------------------------------------------------------------------------
 # Messaging Channels (optional)
 # ------------------------------------------------------------------------------
 # Configure only the external chat channels you want to use.
+# GATEWAY_ENABLED=TRUE
 
 # -- Telegram --
 # TELEGRAM_SHARED_BOT_TOKEN=
 # TELEGRAM_SHARED_BOT_USERNAME=
 # TELEGRAM_WEBHOOK_SECRET=
-# GATEWAY_BASE_URL=http://localhost:8929
+# GATEWAY_BASE_URL=http://localhost:3929
 # GATEWAY_TELEGRAM_INTAKE_MODE=webhook
 
 # -- WhatsApp --
@@ -307,20 +306,20 @@ STT_SERVICE=local/base
 #
 # GATEWAY_SLACK_ENABLED=FALSE
 # GATEWAY_SLACK_SIGNING_SECRET=
-# GATEWAY_SLACK_REDIRECT_URI=http://localhost:8929/api/v1/gateway/slack/callback
+# GATEWAY_SLACK_REDIRECT_URI=http://localhost:3929/api/v1/gateway/slack/callback
 
 # -- Discord --
 # Uses DISCORD_CLIENT_ID, DISCORD_CLIENT_SECRET, and DISCORD_BOT_TOKEN from the
 # Discord connector section.
 #
 # GATEWAY_DISCORD_ENABLED=FALSE
-# GATEWAY_DISCORD_REDIRECT_URI=http://localhost:8929/api/v1/gateway/discord/callback
+# GATEWAY_DISCORD_REDIRECT_URI=http://localhost:3929/api/v1/gateway/discord/callback
 
 # ------------------------------------------------------------------------------
 # SearXNG (bundled web search, works out of the box with no config needed)
 # ------------------------------------------------------------------------------
 # SearXNG provides web search to all search spaces automatically.
-# To access the SearXNG UI directly: http://localhost:8888
+# To access the SearXNG UI directly in dev/deps-only compose: http://localhost:8888
 # To disable the service entirely: docker compose up --scale searxng=0
 # To point at your own SearXNG instance instead of the bundled one:
 # SEARXNG_DEFAULT_HOST=http://your-searxng:8080
@@ -457,3 +456,36 @@ NOLOGIN_MODE_ENABLED=FALSE
 # RESIDENTIAL_PROXY_HOSTNAME=
 # RESIDENTIAL_PROXY_LOCATION=
 # RESIDENTIAL_PROXY_TYPE=1
+
+# ==============================================================================
+# DEV / DEPS-ONLY COMPOSE OVERRIDES
+# These are only needed for docker-compose.dev.yml or docker-compose.deps-only.yml.
+# Production Docker exposes Caddy only; raw app ports below do not affect
+# docker-compose.yml.
+# ==============================================================================
+
+# -- pgAdmin (database GUI, dev/deps-only only) --
+# PGADMIN_PORT=5050
+# PGADMIN_DEFAULT_EMAIL=admin@surfsense.com
+# PGADMIN_DEFAULT_PASSWORD=surfsense
+
+# -- Redis exposed port (dev/deps-only only; Redis is internal-only in prod) --
+# REDIS_PORT=6379
+
+# -- SearXNG exposed port (dev/deps-only only; internal-only in prod) --
+# SEARXNG_PORT=8888
+
+# -- WhatsApp bridge exposed port (dev/hybrid only; prod keeps it Docker-internal) --
+# WHATSAPP_BRIDGE_PORT=9929
+
+# -- Raw app ports (dev/deps-only only; prod exposes Caddy instead) --
+# BACKEND_PORT=8000
+# FRONTEND_PORT=3000
+# ZERO_CACHE_PORT=4848
+
+# -- Frontend runtime flags (prod and dev compose) --
+# The frontend reads these at request time in Docker; no NEXT_PUBLIC_* rebuild
+# or startup substitution is required.
+# AUTH_TYPE=LOCAL
+# ETL_SERVICE=DOCLING
+# DEPLOYMENT_MODE=self-hosted

docker/docker-compose.dev.yml

@@ -257,16 +257,15 @@ services:
   frontend:
     build:
       context: ../surfsense_web
-      args:
-        NEXT_PUBLIC_FASTAPI_BACKEND_URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000}
-        NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL}
-        NEXT_PUBLIC_ETL_SERVICE: ${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}
-        NEXT_PUBLIC_ZERO_CACHE_URL: ${NEXT_PUBLIC_ZERO_CACHE_URL:-http://localhost:${ZERO_CACHE_PORT:-4848}}
-        NEXT_PUBLIC_DEPLOYMENT_MODE: ${NEXT_PUBLIC_DEPLOYMENT_MODE:-self-hosted}
     ports:
       - "${FRONTEND_PORT:-3000}:3000"
     env_file:
       - ../surfsense_web/.env
+    environment:
+      AUTH_TYPE: ${AUTH_TYPE:-LOCAL}
+      ETL_SERVICE: ${ETL_SERVICE:-DOCLING}
+      DEPLOYMENT_MODE: ${DEPLOYMENT_MODE:-self-hosted}
+      SURFSENSE_BACKEND_INTERNAL_URL: http://backend:8000
     depends_on:
       backend:
         condition: service_healthy