fix(docker): wire required prod env + fix webhook path for auth/integration cutover (#1325)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: FelixTJDietrich

docker/.env.example

@@ -19,8 +19,13 @@ APP_HOSTNAME=
 # Docker image tag to deploy (e.g., latest, v1.0.0, develop)
 IMAGE_TAG=latest
 
-# Timezone for containers
-TIMEZONE=UTC
+# Tomcat RemoteIpValve internal-proxies REGEX (NOT CIDR) matching ONLY the ingress (Traefik) address.
+# REQUIRED in prod: forward-headers-strategy=native is active and ProxyTrustGuard fails the boot if
+# blank (Boot's default trusts all of RFC-1918, letting a client spoof X-Forwarded-For and defeat the
+# pre-auth IP rate limit). Find Traefik's address with `docker network inspect shared-network` and
+# match its /24, e.g. 172\.18\.0\.\d+. The bridge subnet is assigned dynamically unless pinned, so
+# re-verify after recreation (or pin a subnet / static Traefik IP to keep this value stable).
+HEPHAESTUS_TRUSTED_PROXIES=
 
 # -----------------------------------------------------------------------------
 # AUTHENTICATION (Hephaestus-native — no Keycloak; ADR 0017)
@@ -78,20 +83,6 @@ WEBHOOK_SECRET=
 # requires re-encrypting every connection row. Generate: `openssl rand -base64 24 | cut -c1-32`
 HEPHAESTUS_SECURITY_ENCRYPTION_KEY=
 
-# -----------------------------------------------------------------------------
-# AI MODELS
-# -----------------------------------------------------------------------------
-
-# Model names in format provider:model (e.g., openai:gpt-4o, azure:gpt-5.4-mini)
-MODEL_NAME=openai:gpt-4o
-
-# OpenAI API
-OPENAI_API_KEY=
-
-# Azure OpenAI (alternative to OpenAI)
-AZURE_RESOURCE_NAME=
-AZURE_API_KEY=
-
 # -----------------------------------------------------------------------------
 # NATS - Message Queue
 # -----------------------------------------------------------------------------
@@ -125,21 +116,12 @@ MONITORING_BACKFILL_INTERVAL_SECONDS=60
 # LEADERBOARD NOTIFICATIONS
 # -----------------------------------------------------------------------------
 
-# Enable Slack notifications for leaderboard
+# Global kill-switch for the Slack leaderboard digest. The target channel/team are
+# configured per-workspace on the Slack Connection, not here.
 LEADERBOARD_NOTIFICATION_ENABLED=false
 
-# Team name to show in notifications
-LEADERBOARD_NOTIFICATION_TEAM=
-
-# Slack channel ID for notifications
-LEADERBOARD_NOTIFICATION_CHANNEL_ID=
-
-# Comma-separated workspace slugs allowed to receive notifications.
-# Temporary allowlist until per-workspace Slack routing ships.
-LEADERBOARD_NOTIFICATION_WORKSPACE_SLUGS=
-
-# Schedule for leaderboard notifications
-LEADERBOARD_SCHEDULE_DAY=MONDAY
+# Schedule: day is an integer 1-7 (ISO-8601, Mon=1 … Sun=7); time is HH[:MM].
+LEADERBOARD_SCHEDULE_DAY=2
 LEADERBOARD_SCHEDULE_TIME=09:00
 
 # -----------------------------------------------------------------------------
@@ -209,11 +191,9 @@ GITLAB_WORKSPACE_CREATION=false
 
 # Agent image override (dev only): see docs/admin/agent-image-digests.md.
 
-# Enable the Docker sandbox for running agent containers
-SANDBOX_ENABLED=false
-
-# Practice review requires SANDBOX_ENABLED, AGENT_NATS_ENABLED,
-# GIT_CHECKOUT_ENABLED, and NATS_ENABLED to be enabled together.
+# The sandbox is activated by the worker role (hephaestus.runtime.worker.enabled, default true
+# in the monolith), not a toggle. Practice review additionally needs AGENT_NATS_ENABLED,
+# GIT_CHECKOUT_ENABLED, and NATS_ENABLED enabled together.
 
 # Docker daemon endpoint (default: local socket)
 # SANDBOX_DOCKER_HOST=unix:///var/run/docker.sock
@@ -268,10 +248,9 @@ PRACTICE_REVIEW_FOR_ALL=false
 # Must be ≥32 chars. Rotate by setting a new value and restarting both pods.
 # HEPHAESTUS_WORKER_REGISTRATION_TOKEN=
 
-# RSA-2048 private key (PEM) the hub uses to sign worker JWTs. Generate with:
-#   openssl genrsa -out worker-signing.pem 2048
-# The hub derives the matching public key from CRT params.
-# HEPHAESTUS_WORKER_HUB_SIGNING_KEY=
+# Note: the hub's JWT signing key is a keyring (hephaestus.worker.hub.token.keys[*].private-key +
+# active-kid), not a single env var. With none configured the hub uses an ephemeral key regenerated
+# on restart (fine for a single replica). A stable multi-replica key has no compose passthrough yet.
 
 # LLM endpoint and key the worker passes to sandboxes. Same values you set on
 # the app-server LLM_PROXY_OPENAI_URL but consumed in-process here.

docker/compose.app.yaml

@@ -97,6 +97,10 @@ services:
       APP_VERSION: ${IMAGE_TAG}
       SPRING_PROFILES_ACTIVE: prod
       APPLICATION_HOST_URL: https://${APP_HOSTNAME}
+      # Tomcat RemoteIpValve proxy trust. forward-headers-strategy=native (application-prod.yml) is
+      # active in prod, so ProxyTrustGuard fails the boot unless this is a non-blank internal-proxies
+      # regex pinned to the ingress (Traefik). Empty default surfaces the guard's clear fail-fast.
+      HEPHAESTUS_TRUSTED_PROXIES: ${HEPHAESTUS_TRUSTED_PROXIES:-}
       # Webhook reception runs in a dedicated `webhook-server` container (see ADR 0008
       # + docker/compose.core.yaml). Disable it here so the receiver beans don't load
       # in application-server (saves RAM; prevents accidental localhost traffic).
@@ -108,40 +112,45 @@ services:
       # Auth (Hephaestus-native; ADR 0017). State-cookie key MUST be base64 32-byte AES.
       HEPHAESTUS_AUTH_ISSUER: https://${APP_HOSTNAME}
       HEPHAESTUS_AUTH_STATE_COOKIE_KEY: ${HEPHAESTUS_AUTH_STATE_COOKIE_KEY}
+      # First instance super-admin bootstrap (auth-cutover runbook). The admin UI is admin-gated, so
+      # the first APP_ADMIN must come from operator config — comma-separated <provider>:@user or
+      # <provider>:<subject>. Optional break-glass token enables POST /auth/bootstrap-admin.
+      HEPHAESTUS_AUTH_BOOTSTRAP_ADMINS: ${HEPHAESTUS_AUTH_BOOTSTRAP_ADMINS:-}
+      HEPHAESTUS_AUTH_BOOTSTRAP_TOKEN: ${HEPHAESTUS_AUTH_BOOTSTRAP_TOKEN:-}
       GITHUB_OAUTH_CLIENT_ID: ${GITHUB_OAUTH_CLIENT_ID}
       GITHUB_OAUTH_CLIENT_SECRET: ${GITHUB_OAUTH_CLIENT_SECRET}
       GITLAB_OAUTH_CLIENT_ID: ${GITLAB_OAUTH_CLIENT_ID:-}
       GITLAB_OAUTH_CLIENT_SECRET: ${GITLAB_OAUTH_CLIENT_SECRET:-}
       GITLAB_OAUTH_BASE_URL: ${GITLAB_OAUTH_BASE_URL:-https://gitlab.com}
       GITLAB_OAUTH_DISPLAY_NAME: ${GITLAB_OAUTH_DISPLAY_NAME:-GitLab}
-      GH_APP_ID: ${GH_APP_ID}
+      GH_APP_ID: ${GH_APP_ID:-0}
       GH_APP_PRIVATE_KEY: ${GH_APP_PRIVATE_KEY}
       GH_APP_PRIVATE_KEY_LOCATION: ${GH_APP_PRIVATE_KEY_LOCATION:-}
       GH_APP_INSTALLATION_URL: ${GH_APP_INSTALLATION_URL:-}
       GH_AUTH_TOKEN: ${GH_AUTH_TOKEN}
-      NATS_ENABLED: ${NATS_ENABLED}
-      NATS_DURABLE_CONSUMER_NAME: ${NATS_DURABLE_CONSUMER_NAME}
-      MONITORING_TIMEFRAME: ${MONITORING_TIMEFRAME}
-      MONITORING_RUN_ON_STARTUP: ${MONITORING_RUN_ON_STARTUP}
-      MONITORING_SYNC_CRON: ${MONITORING_SYNC_CRON}
-      MONITORING_SYNC_COOLDOWN_IN_MINUTES: ${MONITORING_SYNC_COOLDOWN_IN_MINUTES}
+      # Default on: the integration event flow needs NATS in prod (the code-level fallback is off for local dev).
+      NATS_ENABLED: ${NATS_ENABLED:-true}
+      NATS_DURABLE_CONSUMER_NAME: ${NATS_DURABLE_CONSUMER_NAME:-}
+      MONITORING_TIMEFRAME: ${MONITORING_TIMEFRAME:-7}
+      MONITORING_RUN_ON_STARTUP: ${MONITORING_RUN_ON_STARTUP:-true}
+      MONITORING_SYNC_CRON: ${MONITORING_SYNC_CRON:-0 0 * * * *}
+      MONITORING_SYNC_COOLDOWN_IN_MINUTES: ${MONITORING_SYNC_COOLDOWN_IN_MINUTES:-60}
       MONITORING_BACKFILL_ENABLED: ${MONITORING_BACKFILL_ENABLED:-true}
       MONITORING_BACKFILL_BATCH_SIZE: ${MONITORING_BACKFILL_BATCH_SIZE:-50}
       MONITORING_BACKFILL_RATE_LIMIT_THRESHOLD: ${MONITORING_BACKFILL_RATE_LIMIT_THRESHOLD:-500}
       MONITORING_BACKFILL_INTERVAL_SECONDS: ${MONITORING_BACKFILL_INTERVAL_SECONDS:-60}
-      LEADERBOARD_NOTIFICATION_ENABLED: ${LEADERBOARD_NOTIFICATION_ENABLED}
-      LEADERBOARD_NOTIFICATION_TEAM: ${LEADERBOARD_NOTIFICATION_TEAM}
-      LEADERBOARD_NOTIFICATION_CHANNEL_ID: ${LEADERBOARD_NOTIFICATION_CHANNEL_ID}
-      LEADERBOARD_NOTIFICATION_WORKSPACE_SLUGS: ${LEADERBOARD_NOTIFICATION_WORKSPACE_SLUGS}
-      LEADERBOARD_SCHEDULE_DAY: ${LEADERBOARD_SCHEDULE_DAY}
-      LEADERBOARD_SCHEDULE_TIME: ${LEADERBOARD_SCHEDULE_TIME}
+      # Per-workspace channel/team now live on the Slack Connection config; this is the global kill-switch.
+      LEADERBOARD_NOTIFICATION_ENABLED: ${LEADERBOARD_NOTIFICATION_ENABLED:-true}
+      LEADERBOARD_SCHEDULE_DAY: ${LEADERBOARD_SCHEDULE_DAY:-2}
+      LEADERBOARD_SCHEDULE_TIME: ${LEADERBOARD_SCHEDULE_TIME:-9}
       SENTRY_DSN: ${SENTRY_DSN}
       POSTHOG_ENABLED: ${POSTHOG_ENABLED:-false}
       POSTHOG_API_HOST: ${POSTHOG_API_HOST:-https://app.posthog.com}
       POSTHOG_PROJECT_ID: ${POSTHOG_PROJECT_ID:-}
       POSTHOG_PERSONAL_API_KEY: ${POSTHOG_PERSONAL_API_KEY:-}
       WEBHOOK_SECRET: ${WEBHOOK_SECRET}
-      WEBHOOK_EXTERNAL_URL: https://${APP_HOSTNAME}/webhooks
+      # WEBHOOK_EXTERNAL_URL intentionally unset: GitLab auto-registration defaults it to
+      # APPLICATION_HOST_URL (same origin here), then appends /webhooks/gitlab itself.
       # AES-256-GCM key for credentials encrypted at rest in the `connection` table. Exactly 32
       # chars. Required in prod (fail-fast); the Liquibase backfill re-encrypts on boot, so this
       # MUST be set on application-server (where Liquibase runs). Keep it stable.
@@ -154,9 +163,8 @@ services:
       # GitLab integration
       GITLAB_ENABLED: ${GITLAB_ENABLED:-false}
       GITLAB_DEFAULT_SERVER_URL: ${GITLAB_DEFAULT_SERVER_URL:-https://gitlab.lrz.de}
-      # Agent sandbox (practice review + Pi mentor chat — both gated per-workspace via
-      # WorkspaceFeatures.{practicesEnabled,mentorEnabled})
-      SANDBOX_ENABLED: ${SANDBOX_ENABLED:-false}
+      # Agent sandbox (practice review + Pi mentor chat). Activated by the worker role
+      # (hephaestus.runtime.worker.enabled), then gated per-workspace via WorkspaceFeatures.
       SANDBOX_DOCKER_HOST: ${SANDBOX_DOCKER_HOST:-unix:///var/run/docker.sock}
       SANDBOX_CONTAINER_RUNTIME: ${SANDBOX_CONTAINER_RUNTIME:-}
       SANDBOX_MAX_CONCURRENT: ${SANDBOX_MAX_CONCURRENT:-5}
@@ -186,8 +194,6 @@ services:
       LLM_PROXY_AZURE_OPENAI_USE_BEARER: ${LLM_PROXY_AZURE_OPENAI_USE_BEARER:-false}
       # Worker control channel: same env var feeds the worker pod's registration-token claim.
       HEPHAESTUS_WORKER_HUB_TOKEN_REGISTRATION_TOKEN: ${HEPHAESTUS_WORKER_REGISTRATION_TOKEN:-}
-      # PEM PKCS#8 RSA private key. Unset = ephemeral keypair regenerated on restart (dev only).
-      HEPHAESTUS_WORKER_HUB_TOKEN_SIGNING_KEY: ${HEPHAESTUS_WORKER_HUB_SIGNING_KEY:-}
     depends_on:
       postgres:
         condition: service_started
@@ -256,11 +262,11 @@ services:
     environment:
       APP_VERSION: ${IMAGE_TAG}
       SPRING_PROFILES_ACTIVE: prod,worker
+      HEPHAESTUS_TRUSTED_PROXIES: ${HEPHAESTUS_TRUSTED_PROXIES:-}
       HEPHAESTUS_HUB_URL: ws://application-server:8080/api/workers/connect
       HEPHAESTUS_WORKER_REGISTRATION_TOKEN: ${HEPHAESTUS_WORKER_REGISTRATION_TOKEN:-}
       NATS_SERVER: nats://nats-server:4222
       AGENT_NATS_ENABLED: ${AGENT_NATS_ENABLED:-false}
-      SANDBOX_ENABLED: ${SANDBOX_ENABLED:-false}
       SANDBOX_DOCKER_HOST: ${SANDBOX_DOCKER_HOST:-unix:///var/run/docker.sock}
       SANDBOX_CONTAINER_RUNTIME: ${SANDBOX_CONTAINER_RUNTIME:-}
       SANDBOX_MAX_CONCURRENT: ${SANDBOX_MAX_CONCURRENT:-5}

docker/compose.core.yaml

@@ -30,6 +30,8 @@ services:
       DATABASE_USERNAME: root
       DATABASE_PASSWORD: root
       SENTRY_DSN: ${SENTRY_DSN}
+      # forward-headers=native (application-prod.yml) → ProxyTrustGuard fails the boot unless pinned.
+      HEPHAESTUS_TRUSTED_PROXIES: ${HEPHAESTUS_TRUSTED_PROXIES:-}
     depends_on:
       nats-server:
         condition: service_healthy
@@ -42,13 +44,14 @@ services:
     labels:
       - "traefik.enable=true"
       - "traefik.http.middlewares.gzip.compress=true"
-      - "traefik.http.middlewares.https-webhook-server-stripprefix.stripprefix.prefixes=/webhooks"
       - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
       - "traefik.http.routers.http-webhook-server.entryPoints=http"
       - "traefik.http.routers.http-webhook-server.middlewares=redirect-to-https"
       - "traefik.http.routers.http-webhook-server.rule=Host(`${APP_HOSTNAME}`) && PathPrefix(`/webhooks`)"
       - "traefik.http.routers.https-webhook-server.entryPoints=https"
-      - "traefik.http.routers.https-webhook-server.middlewares=https-webhook-server-stripprefix,gzip"
+      # No stripprefix: the receiver owns the full "/webhooks/{kind}" path (controller + payload-size
+      # filter + security chain all bind to /webhooks/**). Stripping it would 404 every delivery.
+      - "traefik.http.routers.https-webhook-server.middlewares=gzip"
       - "traefik.http.routers.https-webhook-server.rule=Host(`${APP_HOSTNAME}`) && PathPrefix(`/webhooks`)"
       - "traefik.http.routers.https-webhook-server.tls.certresolver=letsencrypt"
       - "traefik.http.routers.https-webhook-server.tls=true"

docker/preview/.env.example

@@ -53,16 +53,6 @@ GH_AUTH_TOKEN=
 # Install URL shown in the workspace creation wizard
 GH_APP_INSTALLATION_URL=
 
-# AI Model configuration (provider:model format)
-MODEL_NAME=openai:gpt-4o
-
-# OpenAI
-OPENAI_API_KEY=
-
-# Azure OpenAI (alternative to OpenAI)
-AZURE_RESOURCE_NAME=
-AZURE_API_KEY=
-
 # -----------------------------------------------------------------------------
 # OPTIONAL
 # -----------------------------------------------------------------------------

docker/preview/compose.app.yaml

@@ -191,6 +191,9 @@ services:
     restart: unless-stopped
     environment:
       SPRING_PROFILES_ACTIVE: prod
+      # prod runs forward-headers=native, so ProxyTrustGuard aborts the boot unless this is set.
+      # Coolify fronts previews with its own proxy — set its ingress regex, not staging's range.
+      HEPHAESTUS_TRUSTED_PROXIES: ${HEPHAESTUS_TRUSTED_PROXIES:?Required}
       # URL - Coolify provides SERVICE_FQDN_WEBAPP
       APPLICATION_HOST_URL: https://${SERVICE_FQDN_WEBAPP}
       DATABASE_URL: postgresql://postgres:5432/hephaestus
@@ -225,9 +228,11 @@ services:
       LEADERBOARD_NOTIFICATION_ENABLED: "false"
       # Observability (optional)
       SENTRY_DSN: ${SENTRY_DSN:-}
-      # Webhook auto-registration
+      # Webhook auto-registration. Override (not the APPLICATION_HOST_URL default) because previews
+      # route webhooks via PREVIEW_DOMAIN, not the webapp's SERVICE_FQDN_WEBAPP. Bare origin:
+      # GitLabWebhookService appends /webhooks/gitlab itself.
       WEBHOOK_SECRET: ${WEBHOOK_SECRET:-}
-      WEBHOOK_EXTERNAL_URL: https://${PREVIEW_DOMAIN:?Required}/webhooks
+      WEBHOOK_EXTERNAL_URL: https://${PREVIEW_DOMAIN:?Required}
     depends_on:
       postgres:
         condition: service_healthy

docker/preview/compose.shared-infra.yaml

@@ -77,6 +77,8 @@ services:
       HEPHAESTUS_SYNC_NATS_ENABLED: "true"
       HEPHAESTUS_SYNC_NATS_SERVER: "nats://nats-server:4222"
       WEBHOOK_SECRET: ${WEBHOOK_SECRET:?Required}
+      # prod runs forward-headers=native → ProxyTrustGuard aborts the boot unless set (Coolify ingress regex).
+      HEPHAESTUS_TRUSTED_PROXIES: ${HEPHAESTUS_TRUSTED_PROXIES:?Required}
     depends_on:
       nats-server:
         condition: service_healthy

server/src/main/resources/application.yml

@@ -449,8 +449,10 @@ hephaestus:
     #     publish/stream/http blocks for the inbound receiver.
     # `secret` is shared by design — same value registered with the provider and used
     # to verify incoming bodies. The receiver runs in the `webhook-server` container.
+    # external-url defaults to the app's own origin (APPLICATION_HOST_URL); set WEBHOOK_EXTERNAL_URL
+    # only when the receiver is fronted at a different origin than the app (e.g. preview).
     webhook:
-        external-url: ${WEBHOOK_EXTERNAL_URL:}
+        external-url: ${WEBHOOK_EXTERNAL_URL:${APPLICATION_HOST_URL:}}
         secret: ${WEBHOOK_SECRET:}
         token-rotation:
             threshold-days: ${WEBHOOK_TOKEN_ROTATION_THRESHOLD_DAYS:7}