feat(security): harden edge and runtime envelopes (#1661)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: FelixTJDietrich

.changeset/harden-runtime-envelope.md

@@ -0,0 +1,7 @@
+---
+"hephaestus": minor
+---
+
+Hardens the reference deployment with HTTPS security headers, a TLS floor, a request-size ceiling, shared rate limits for costly operations, authenticated internal messaging, and container resource limits.
+
+**Operators:** Set the new required `NATS_USERNAME` and `NATS_PASSWORD` variables. Optional `*_CPUS` and `*_PIDS_LIMIT` variables tune container ceilings. Remote databases require TLS unless `HEPHAESTUS_DATABASE_ALLOW_INSECURE_REMOTE=true` explicitly accepts plaintext transport. Each server role's database pool now defaults to 20 connections instead of 30; the optional `HIKARI_MAXIMUM_POOL_SIZE` variable tunes it.

.github/workflows/ci-compose-validate.yml

@@ -48,6 +48,8 @@ jobs:
             -e 's|^HEPHAESTUS_SECURITY_ENCRYPTION_KEY=$|HEPHAESTUS_SECURITY_ENCRYPTION_KEY=0123456789abcdef0123456789abcdef|' \
             -e 's|^HEPHAESTUS_AUTH_STATE_COOKIE_KEY=$|HEPHAESTUS_AUTH_STATE_COOKIE_KEY=Y2ktbm90LWEtcmVhbC1zdGF0ZS1jb29raWUta2V5|' \
             -e 's|^WEBHOOK_SECRET=$|WEBHOOK_SECRET=ci000000000000000000000000000000000|' \
+            -e 's|^NATS_USERNAME=$|NATS_USERNAME=ci|' \
+            -e 's|^NATS_PASSWORD=$|NATS_PASSWORD=ci-not-a-real-password|' \
             .env
           while IFS= read -r image; do
             name="HEPHAESTUS_IMAGE_${image^^}"
@@ -88,6 +90,28 @@ jobs:
           # publish the reference's dashboard port and keep the maintainers' ACME
           # email. Assert the merged result rather than trust the runner's version.
           rendered=$(docker compose config)
+          for required in \
+            'entrypoints.https.http.middlewares=security-headers@docker' \
+            'request-body-limit@docker' \
+            'providers.file.filename=/etc/traefik/dynamic.yml' \
+            'contentsecuritypolicy' \
+            'default-src' \
+            'stsseconds' \
+            '2592000' \
+            'maxrequestbodybytes' \
+            '26214400' \
+            'minversion' \
+            'versiontls12'; do
+            grep -Fiq "$required" <<< "$rendered" || {
+              echo "::error::runtime edge envelope is missing '$required'"; exit 1; }
+          done
+          edge_csp=$(sed -n 's/.*contentSecurityPolicy=\(.*\)"/\1/p' ../compose.proxy.yaml)
+          selfhost_csp=$(sed -n 's/.*contentSecurityPolicy=\(.*\)"/\1/p' compose.single-host.yaml)
+          nginx_csp=$(sed -n 's/add_header Content-Security-Policy "\(.*\)" always;/\1/p' ../../webapp/docker/security-headers.conf)
+          [ "$edge_csp" = "$nginx_csp" ] || {
+            echo "::error::Traefik and nginx Content-Security-Policy values differ"; exit 1; }
+          [ "$edge_csp" = "$selfhost_csp" ] || {
+            echo "::error::reference and single-host Content-Security-Policy values differ"; exit 1; }
           grep -q "admin@tum.de" <<< "$rendered" && {
             echo "::error::the maintainers' ACME email survived the override — Compose is too old to honour !override"; exit 1; } || true
           published=$(docker compose config --format json \

.github/workflows/deploy-locked-compose.yml

@@ -73,6 +73,8 @@ jobs:
           SECRET_HEPHAESTUS_INTEGRATION_SLACK_SIGNING_SECRET: ${{ secrets.HEPHAESTUS_INTEGRATION_SLACK_SIGNING_SECRET }}
           SECRET_HEPHAESTUS_SECURITY_ENCRYPTION_KEY: ${{ secrets.HEPHAESTUS_SECURITY_ENCRYPTION_KEY }}
           SECRET_HEPHAESTUS_WORKER_REGISTRATION_TOKEN: ${{ secrets.HEPHAESTUS_WORKER_REGISTRATION_TOKEN }}
+          SECRET_NATS_USERNAME: ${{ secrets.NATS_USERNAME }}
+          SECRET_NATS_PASSWORD: ${{ secrets.NATS_PASSWORD }}
           SECRET_OUTLINE_OAUTH_CLIENT_SECRET: ${{ secrets.OUTLINE_OAUTH_CLIENT_SECRET }}
           SECRET_POSTHOG_PERSONAL_API_KEY: ${{ secrets.POSTHOG_PERSONAL_API_KEY }}
           SECRET_POSTHOG_PROJECT_API_KEY: ${{ secrets.POSTHOG_PROJECT_API_KEY }}

.github/workflows/verify-changesets.yml

@@ -69,6 +69,11 @@ jobs:
             node scripts/verify-changesets.ts /tmp/changeset-status.json "${changesets_changed[@]}"
           fi
           if [ "${#migration_changed[@]}" -gt 0 ]; then
+            # No --since here: a fragment may pair with a pending changeset merged by an earlier
+            # PR, and --since filters those out of the status output. Without --since the
+            # changesets CLI resolves its configured baseBranch by the exact ref name `main`,
+            # which a detached PR merge-ref checkout does not have — give it one.
+            git rev-parse --verify --quiet main >/dev/null || git branch main origin/main
             pnpm changeset status --output /tmp/pending-changeset-status.json
             node scripts/verify-changesets.ts /tmp/pending-changeset-status.json --migration "${migration_changed[@]}"
           fi

.migration/harden-runtime-envelope.md

@@ -0,0 +1,21 @@
+#### 🔴 The runtime envelope is hardened: NATS requires credentials, remote databases require TLS
+
+**Affected**: every reference and self-hosted deployment. Deployments using a remote (non-Compose)
+PostgreSQL host are additionally affected by the TLS requirement.
+
+**Before**: the bundled NATS broker accepted unauthenticated connections (mitigated only by its
+loopback bind), a remote `DATABASE_URL` with `sslmode=disable` connected silently, and each server
+role opened up to 30 database connections.
+
+**After**: the broker, publisher, and consumer all require the same credentials and the stack
+refuses to start without them; in production a remote PostgreSQL host without `sslmode=require`
+(or stronger) aborts startup; each server role's connection pool defaults to 20.
+
+**Migration**: before deploying, set `NATS_USERNAME` and `NATS_PASSWORD` in your `.env` to freshly
+generated random values (do not reuse an application key); deployments driven by the deploy
+workflow need the same pair as environment secrets. If your database host is remote, add
+`sslmode=require`, `verify-ca`, or `verify-full` to `DATABASE_URL` — only set
+`HEPHAESTUS_DATABASE_ALLOW_INSECURE_REMOTE=true` after explicitly accepting plaintext transport.
+Optional tuning: `HIKARI_MAXIMUM_POOL_SIZE` restores a larger pool, and the new
+`APPLICATION_SERVER_CPUS`/`APPLICATION_SERVER_PIDS_LIMIT` (plus the worker and webhook variants)
+adjust the container ceilings.

docker/.env.example

@@ -97,8 +97,19 @@ HEPHAESTUS_SECURITY_CREDENTIAL_ENCRYPTION_KEY=
 # -----------------------------------------------------------------------------
 
 NATS_ENABLED=true
+# Required. Generate independent random values; do not reuse an application key.
+NATS_USERNAME=
+NATS_PASSWORD=
 NATS_DURABLE_CONSUMER_NAME=hephaestus-consumer
 
+# Container resource ceilings
+#APPLICATION_SERVER_CPUS=4.0
+#APPLICATION_SERVER_PIDS_LIMIT=512
+#APPLICATION_WORKER_CPUS=4.0
+#APPLICATION_WORKER_PIDS_LIMIT=512
+#WEBHOOK_SERVER_CPUS=2.0
+#WEBHOOK_SERVER_PIDS_LIMIT=256
+
 # How long a durable consumer may go with nothing CONNECTED to it before JetStream deletes it — not
 # how long it goes without traffic. Set 0s on a deployment that may be offline longer than this and
 # must resume exactly where it left off; between 0s and 1h is rejected at startup.

docker/compose.app.yaml

@@ -66,6 +66,10 @@ services:
       - "traefik.http.routers.http-webapp.rule=Host(`${APP_HOSTNAME}`) && PathPrefix(`/`)"
       - "traefik.http.routers.http-webapp.service=http-webapp"
       - "traefik.http.routers.https-webapp.entryPoints=https"
+      # Request-body cap (defined in compose.proxy.yaml). Router-scoped rather than entrypoint-wide
+      # because the buffering middleware also buffers responses, which would stall the /api SSE
+      # streams; the static SPA and the webhook receiver have no streaming responses.
+      - "traefik.http.routers.https-webapp.middlewares=request-body-limit@docker"
       - "traefik.http.routers.https-webapp.rule=Host(`${APP_HOSTNAME}`) && PathPrefix(`/`)"
       - "traefik.http.routers.https-webapp.service=https-webapp"
       - "traefik.http.routers.https-webapp.tls.certresolver=letsencrypt"
@@ -112,6 +116,8 @@ services:
     # Bounds heap sizing to this container; without it the JVM targets host RAM and co-located
     # services oversubscribe and swap. Hosts the NATS sync consumers, so it takes the largest limit.
     mem_limit: ${APPLICATION_SERVER_MEM_LIMIT:-5g}
+    cpus: ${APPLICATION_SERVER_CPUS:-4.0}
+    pids_limit: ${APPLICATION_SERVER_PIDS_LIMIT:-512}
     ports:
       - "8080"
     environment:
@@ -136,6 +142,8 @@ services:
       DATABASE_USERNAME: root
       DATABASE_PASSWORD: root
       NATS_SERVER: nats://nats-server:4222
+      NATS_USERNAME: ${NATS_USERNAME:?Set NATS_USERNAME}
+      NATS_PASSWORD: ${NATS_PASSWORD:?Set NATS_PASSWORD}
       # Auth (Hephaestus-native; ADR 0017). State-cookie key MUST be base64 32-byte AES.
       HEPHAESTUS_AUTH_ISSUER: https://${APP_HOSTNAME}
       HEPHAESTUS_AUTH_STATE_COOKIE_KEY: ${HEPHAESTUS_AUTH_STATE_COOKIE_KEY}
@@ -256,7 +264,7 @@ services:
       THC_PATH: /actuator/health/liveness
     depends_on:
       postgres:
-        condition: service_started
+        condition: service_healthy
       volume-init:
         condition: service_completed_successfully
     restart: unless-stopped
@@ -276,19 +284,25 @@ services:
       # the sandbox network IP and route requests to an unreachable internal network.
       - "traefik.docker.network=shared-network"
       - "traefik.http.middlewares.https-application-server-stripprefix.stripprefix.prefixes=/api"
+      - "traefik.http.middlewares.edge-api-rate-limit.ratelimit.average=100"
+      - "traefik.http.middlewares.edge-api-rate-limit.ratelimit.period=1s"
+      - "traefik.http.middlewares.edge-api-rate-limit.ratelimit.burst=200"
       - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
       - "traefik.http.routers.http-application-server.entryPoints=http"
       - "traefik.http.routers.http-application-server.middlewares=redirect-to-https"
       - "traefik.http.routers.http-application-server.rule=Host(`${APP_HOSTNAME}`) && PathPrefix(`/api`)"
       - "traefik.http.routers.http-application-server.service=http-application-server"
       - "traefik.http.routers.https-application-server.entryPoints=https"
-      - "traefik.http.routers.https-application-server.middlewares=https-application-server-stripprefix"
+      - "traefik.http.routers.https-application-server.middlewares=edge-api-rate-limit,https-application-server-stripprefix"
       - "traefik.http.routers.https-application-server.rule=Host(`${APP_HOSTNAME}`) && PathPrefix(`/api`)"
       - "traefik.http.routers.https-application-server.tls.certresolver=letsencrypt"
       - "traefik.http.routers.https-application-server.tls=true"
       - "traefik.http.routers.https-application-server.service=https-application-server"
       - "traefik.http.services.http-application-server.loadbalancer.server.port=8080"
       - "traefik.http.services.https-application-server.loadbalancer.server.port=8080"
+      # The transport is defined by the proxy's file provider (compose.proxy.yaml): serversTransports
+      # cannot be defined via docker labels, and referencing an undefined one disables this router.
+      - "traefik.http.services.https-application-server.loadbalancer.serverstransport=application@file"
       - "traefik.http.routers.http-application-server.priority=5"
       - "traefik.http.routers.https-application-server.priority=10"
       # Replica affinity for workspace traffic (mentor lives here) — see docs/contributor/unified-pi-runtime.mdx.
@@ -325,6 +339,8 @@ services:
     image: "${HEPHAESTUS_IMAGE_APPLICATION_SERVER:?verified release lock required}"
     # Bounds heap sizing to this container (see application-server).
     mem_limit: ${APPLICATION_WORKER_MEM_LIMIT:-3g}
+    cpus: ${APPLICATION_WORKER_CPUS:-4.0}
+    pids_limit: ${APPLICATION_WORKER_PIDS_LIMIT:-512}
     environment:
       # See application-server: the buildpack's 10M direct-memory default is too small.
       JAVA_TOOL_OPTIONS: "-XX:MaxDirectMemorySize=${APP_MAX_DIRECT_MEMORY:-128M}"
@@ -395,7 +411,9 @@ services:
       THC_PATH: /actuator/health/liveness
     depends_on:
       application-server:
-        condition: service_started
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
       volume-init:
         condition: service_completed_successfully
     restart: unless-stopped

docker/compose.core.yaml

@@ -12,6 +12,8 @@ services:
     # Must fit the Spring classpath-scan startup spike (well above the small steady footprint),
     # or the JVM OOMs during boot.
     mem_limit: ${WEBHOOK_SERVER_MEM_LIMIT:-2g}
+    cpus: ${WEBHOOK_SERVER_CPUS:-2.0}
+    pids_limit: ${WEBHOOK_SERVER_PIDS_LIMIT:-256}
     expose:
       - "8080"
     environment:
@@ -24,6 +26,8 @@ services:
       # NATS — reuses the existing `natsConnection` bean to publish inbound events.
       HEPHAESTUS_SYNC_NATS_ENABLED: "true"
       HEPHAESTUS_SYNC_NATS_SERVER: "nats://nats-server:4222"
+      NATS_USERNAME: ${NATS_USERNAME:?Set NATS_USERNAME}
+      NATS_PASSWORD: ${NATS_PASSWORD:?Set NATS_PASSWORD}
       # Stream bounds belong on THIS container. The beans that read hephaestus.webhook.stream.* are
       # gated on the webhook role, so the same variables on application-server (which runs with the
       # role off) would configure nothing at all. `check:env` fails the build if they drift back.
@@ -95,6 +99,8 @@ services:
     depends_on:
       nats-server:
         condition: service_healthy
+      postgres:
+        condition: service_healthy
     restart: unless-stopped
     # Spring drains HTTP (timeout-per-shutdown-phase 20s) THEN WebhookGracefulShutdown drains
     # NATS (up to 15s). Docker must allow ≥35s before SIGKILL, or the JVM is cut mid-drain.
@@ -111,7 +117,10 @@ services:
       - "traefik.http.routers.https-webhook-server.entryPoints=https"
       # No stripprefix: the receiver owns the full "/webhooks/{kind}" path (controller + payload-size
       # filter + security chain all bind to /webhooks/**). Stripping it would 404 every delivery.
-      - "traefik.http.routers.https-webhook-server.middlewares=gzip"
+      # request-body-limit (defined in compose.proxy.yaml) caps webhook deliveries at the same
+      # 25 MiB the receiver admits; router-scoped because entrypoint-wide buffering would stall
+      # the /api SSE streams (see compose.proxy.yaml).
+      - "traefik.http.routers.https-webhook-server.middlewares=request-body-limit@docker,gzip"
       - "traefik.http.routers.https-webhook-server.rule=Host(`${APP_HOSTNAME}`) && PathPrefix(`/webhooks`)"
       - "traefik.http.routers.https-webhook-server.tls.certresolver=letsencrypt"
       - "traefik.http.routers.https-webhook-server.tls=true"
@@ -139,10 +148,12 @@ services:
     image: ${HEPHAESTUS_IMAGE_NATS:?verified release lock required}
     restart: unless-stopped
     ports:
-      # Loopback-only by default. Set NATS_BIND_HOST=0.0.0.0 (or a specific interface) to let other
-      # hosts reach the bus — only on a trusted or firewalled network, as NATS runs unauthenticated.
+      # Publish on loopback by default.
       - "${NATS_BIND_HOST:-127.0.0.1}:4222:4222"
     command: ["--config", "/etc/nats/nats-server.conf"]
+    environment:
+      NATS_USERNAME: ${NATS_USERNAME:?Set NATS_USERNAME}
+      NATS_PASSWORD: ${NATS_PASSWORD:?Set NATS_PASSWORD}
     volumes:
       - nats-data:/data
     configs:
@@ -176,6 +187,11 @@ configs:
     content: |
       listen: "0.0.0.0:4222" 
 
+      authorization {
+        user: $$NATS_USERNAME
+        password: $$NATS_PASSWORD
+      }
+
       http_port: 8222
 
       # A message the receiver accepts but the broker refuses is verified, admitted and then lost.

docker/compose.proxy.yaml

@@ -9,17 +9,20 @@ services:
     ports:
       - "80:80"
       - "443:443"
-      - "8080:8080"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - ./letsencrypt:/letsencrypt
     command:
       - "--ping=true"
       - "--ping.entrypoint=http"
-      - "--api.dashboard=true"
-      - "--api.insecure=false"
+      - "--metrics.prometheus=true"
       - "--entrypoints.http.address=:80"
       - "--entrypoints.https.address=:443"
+      # Entrypoint-wide response headers only. The request-body-limit buffering middleware must NOT
+      # sit on the entrypoint: Traefik's buffering middleware buffers whole responses, which would
+      # hold back every SSE stream (mentor chat, sync live-push) until it closes. It is attached
+      # per-router to the non-streaming surfaces (webapp, webhook-server) instead.
+      - "--entrypoints.https.http.middlewares=security-headers@docker"
       - "--entrypoints.http.http.encodequerysemicolons=true"
       - "--entryPoints.http.http2.maxConcurrentStreams=50"
       - "--entrypoints.https.http.encodequerysemicolons=true"
@@ -37,11 +40,24 @@ services:
       - "--certificatesresolvers.letsencrypt.acme.email=admin@tum.de"
       - "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=http"
       - "--providers.docker=true"
+      # TLS options and serversTransports are NOT definable through Docker labels (the docker
+      # provider silently drops them, and a dangling serversTransport reference disables the whole
+      # router), so they live in this file-provider fragment.
+      - "--providers.file.filename=/etc/traefik/dynamic.yml"
+    configs:
+      - source: traefik-dynamic
+        target: /etc/traefik/dynamic.yml
     labels:
       - traefik.enable=true
-      - traefik.http.routers.traefik.entrypoints=http
-      - traefik.http.routers.traefik.service=api@internal
-      - traefik.http.services.traefik.loadbalancer.server.port=8080
+      - traefik.http.middlewares.security-headers.headers.stsSeconds=2592000
+      - traefik.http.middlewares.security-headers.headers.stsIncludeSubdomains=true
+      - traefik.http.middlewares.security-headers.headers.contentTypeNosniff=true
+      - traefik.http.middlewares.security-headers.headers.frameDeny=true
+      - traefik.http.middlewares.security-headers.headers.referrerPolicy=strict-origin-when-cross-origin
+      - traefik.http.middlewares.security-headers.headers.permissionsPolicy=geolocation=(), camera=(), microphone=(), payment=()
+      - "traefik.http.middlewares.security-headers.headers.contentSecurityPolicy=default-src 'self'; base-uri 'self'; object-src 'none'; frame-ancestors 'none'; form-action 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https:; font-src 'self' data:; connect-src 'self' https: wss:; worker-src 'self' blob:; manifest-src 'self'; upgrade-insecure-requests"
+      - traefik.http.middlewares.request-body-limit.buffering.maxRequestBodyBytes=26214400
+      - traefik.http.middlewares.request-body-limit.buffering.memRequestBodyBytes=1048576
       - traefik.http.middlewares.errorpage.errors.status=501,502,503,504
       - traefik.http.middlewares.errorpage.errors.service=maintenance
       - traefik.http.middlewares.errorpage.errors.query=/index.html
@@ -103,6 +119,22 @@ networks:
     driver: bridge
 
 configs:
+  traefik-dynamic:
+    content: |
+      # File-provider fragment for dynamic configuration the docker provider cannot express as
+      # labels. Traefik silently ignores tls.options and serversTransports labels, so putting
+      # these there would leave TLS 1.0/1.1 accepted and (for a referenced transport) disable
+      # the router that names it.
+      tls:
+        options:
+          default:
+            minVersion: VersionTLS12
+      http:
+        serversTransports:
+          application:
+            forwardingTimeouts:
+              dialTimeout: 10s
+              responseHeaderTimeout: 30s
   nginx-default-config:
     content: |
       server {

docker/preview/compose.app.yaml

@@ -251,6 +251,8 @@ services:
       # The integration consumer reads staging's JetStream, so a preview sees the events a shared
       # GitHub App delivers there rather than an empty broker.
       NATS_SERVER: ${NATS_SERVER:-nats://nats-server:4222}
+      NATS_USERNAME: ${NATS_USERNAME}
+      NATS_PASSWORD: ${NATS_PASSWORD}
       NATS_ENABLED: "true"
       # Per-deploy durable name, so previews get their own JetStream consumers instead of competing
       # for one and stealing each other's messages.