fix(docker): unbreak prod deploy — Traefik 3.7, distroless healthchecks, agent-pin override (#1328)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: FelixTJDietrich

docker/compose.app.yaml

@@ -18,7 +18,9 @@ services:
     ports:
       - "80"
     depends_on:
-      - application-server
+      # Gate on app-server readiness so the SPA isn't served before the API answers.
+      application-server:
+        condition: service_healthy
     restart: unless-stopped
     networks:
       - shared-network
@@ -167,6 +169,10 @@ services:
       GITLAB_DEFAULT_SERVER_URL: ${GITLAB_DEFAULT_SERVER_URL:-https://gitlab.lrz.de}
       # Agent sandbox (practice review + Pi mentor chat). Activated by the worker role
       # (hephaestus.runtime.worker.enabled), then gated per-workspace via WorkspaceFeatures.
+      # AgentImagePinGuard requires a digest-pinned agent image in prod; the release-pin-fetcher
+      # supplies it for release tags. Set false for non-release (e.g. main/latest) deploys that skip
+      # the pin, otherwise the boot fails on agent-pi:<tag>. Keep true for release deploys.
+      HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST: ${HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST:-true}
       SANDBOX_DOCKER_HOST: ${SANDBOX_DOCKER_HOST:-unix:///var/run/docker.sock}
       SANDBOX_CONTAINER_RUNTIME: ${SANDBOX_CONTAINER_RUNTIME:-}
       SANDBOX_MAX_CONCURRENT: ${SANDBOX_MAX_CONCURRENT:-5}
@@ -196,6 +202,9 @@ services:
       LLM_PROXY_AZURE_OPENAI_USE_BEARER: ${LLM_PROXY_AZURE_OPENAI_USE_BEARER:-false}
       # Worker control channel: same env var feeds the worker pod's registration-token claim.
       HEPHAESTUS_WORKER_HUB_TOKEN_REGISTRATION_TOKEN: ${HEPHAESTUS_WORKER_REGISTRATION_TOKEN:-}
+      # thc healthcheck target: actuator liveness on the management port.
+      THC_PORT: "8080"
+      THC_PATH: /actuator/health/liveness
     depends_on:
       postgres:
         condition: service_started
@@ -243,12 +252,14 @@ services:
       - "traefik.http.services.https-application-server.loadbalancer.healthcheck.path=/actuator/health/liveness"
       - "traefik.http.services.https-application-server.loadbalancer.healthcheck.interval=10s"
       - "traefik.http.services.https-application-server.loadbalancer.healthcheck.timeout=3s"
+    # The distroless image has no shell for a wget probe; the probe is the static thc binary the
+    # health-checker buildpack adds at /workspace/health-check (server/pom.xml).
     healthcheck:
-      test: "wget -qO- http://localhost:8080/actuator/health || exit 1"
-      interval: 5s
-      timeout: 10s
-      retries: 10
-      start_period: 10s
+      test: ["CMD", "/workspace/health-check"]
+      interval: 15s
+      timeout: 5s
+      retries: 3
+      start_period: 90s
     # Matches application.yml's SHUTDOWN_TIMEOUT default (20s) plus headroom so SIGTERM has time
     # to drain in-flight requests before Docker SIGKILLs the launcher.
     stop_grace_period: 30s
@@ -265,6 +276,7 @@ services:
       APP_VERSION: ${IMAGE_TAG}
       SPRING_PROFILES_ACTIVE: prod,worker
       HEPHAESTUS_TRUSTED_PROXIES: ${HEPHAESTUS_TRUSTED_PROXIES:-}
+      HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST: ${HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST:-true}
       HEPHAESTUS_HUB_URL: ws://application-server:8080/api/workers/connect
       HEPHAESTUS_WORKER_REGISTRATION_TOKEN: ${HEPHAESTUS_WORKER_REGISTRATION_TOKEN:-}
       NATS_SERVER: nats://nats-server:4222
@@ -282,6 +294,9 @@ services:
       HEPHAESTUS_WORKER_CAPACITY_MENTOR_MAX: ${HEPHAESTUS_WORKER_MENTOR_MAX:-auto}
       HEPHAESTUS_WORKER_DRAIN_TIMEOUT: ${HEPHAESTUS_WORKER_DRAIN_TIMEOUT:-5m}
       SENTRY_DSN: ${SENTRY_DSN}
+      # thc healthcheck target: actuator liveness on the management port (still :8080 though server.port=-1).
+      THC_PORT: "8080"
+      THC_PATH: /actuator/health/liveness
     depends_on:
       application-server:
         condition: service_started
@@ -296,12 +311,14 @@ services:
       - shared-network
     # 5m drain budget + 1m headroom; matches spring.lifecycle.timeout-per-shutdown-phase.
     stop_grace_period: 6m
+    # thc healthcheck (see application-server). The worker fronts no Traefik router, so this is its
+    # only health signal.
     healthcheck:
-      test: "wget -qO- http://localhost:8080/actuator/health/liveness || exit 1"
-      interval: 10s
+      test: ["CMD", "/workspace/health-check"]
+      interval: 15s
       timeout: 5s
-      retries: 6
-      start_period: 30s
+      retries: 3
+      start_period: 90s
     logging:
       driver: "json-file"
       options:

docker/compose.core.yaml

@@ -32,6 +32,11 @@ services:
       SENTRY_DSN: ${SENTRY_DSN}
       # forward-headers=native (application-prod.yml) → ProxyTrustGuard fails the boot unless pinned.
       HEPHAESTUS_TRUSTED_PROXIES: ${HEPHAESTUS_TRUSTED_PROXIES:-}
+      # AgentImagePinGuard loads in any prod pod; set false for non-release deploys that skip the pin.
+      HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST: ${HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST:-true}
+      # thc healthcheck target: readiness (reports unhealthy while NATS/JetStream is unavailable).
+      THC_PORT: "8080"
+      THC_PATH: /actuator/health/readiness
     depends_on:
       nats-server:
         condition: service_healthy
@@ -58,12 +63,17 @@ services:
       - "traefik.http.routers.http-webhook-server.priority=100"
       - "traefik.http.routers.https-webhook-server.priority=100"
       - "traefik.http.services.https-webhook-server.loadbalancer.server.port=8080"
+      # Routing-layer probe: drops a bad backend from the LB pool.
+      - "traefik.http.services.https-webhook-server.loadbalancer.healthcheck.path=/actuator/health/readiness"
+      - "traefik.http.services.https-webhook-server.loadbalancer.healthcheck.interval=15s"
+      - "traefik.http.services.https-webhook-server.loadbalancer.healthcheck.timeout=5s"
+    # thc healthcheck (see docker/compose.app.yaml).
     healthcheck:
-      test: ["CMD", "wget", "-qO-", "http://localhost:8080/actuator/health/readiness"]
+      test: ["CMD", "/workspace/health-check"]
       interval: 15s
       timeout: 5s
-      retries: 5
-      start_period: 30s
+      retries: 3
+      start_period: 90s
     logging:
       driver: "json-file"
       options:

docker/compose.proxy.yaml

@@ -1,6 +1,8 @@
 services:
   reverse-proxy:
-    image: traefik:v3.4
+    # Keep at >=3.3: below it the app-server sticky.cookie.path label (workspace replica affinity)
+    # fails to parse and Traefik silently drops the whole router. Pin a current 3.x.
+    image: traefik:v3.7.4
     restart: unless-stopped
     networks:
       - shared-network

docker/preview/compose.app.yaml

@@ -194,6 +194,8 @@ services:
       # prod runs forward-headers=native, so ProxyTrustGuard aborts the boot unless this is set.
       # Coolify fronts previews with its own proxy — set its ingress regex, not staging's range.
       HEPHAESTUS_TRUSTED_PROXIES: ${HEPHAESTUS_TRUSTED_PROXIES:?Required}
+      # Previews run non-release images with no release-pin, so don't require a digest-pinned agent image.
+      HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST: ${HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST:-false}
       # URL - Coolify provides SERVICE_FQDN_WEBAPP
       APPLICATION_HOST_URL: https://${SERVICE_FQDN_WEBAPP}
       DATABASE_URL: postgresql://postgres:5432/hephaestus
@@ -234,23 +236,20 @@ services:
       # GitLabWebhookService appends /webhooks/gitlab itself.
       WEBHOOK_SECRET: ${WEBHOOK_SECRET:-}
       WEBHOOK_EXTERNAL_URL: https://${PREVIEW_DOMAIN:?Required}
+      # thc healthcheck target: actuator liveness on the management port.
+      THC_PORT: "8080"
+      THC_PATH: /actuator/health/liveness
     depends_on:
       postgres:
         condition: service_healthy
       seed-loader:
         condition: service_completed_successfully
+    # thc healthcheck (see docker/compose.app.yaml).
     healthcheck:
-      test:
-        [
-          "CMD",
-          "wget",
-          "--spider",
-          "--quiet",
-          "http://127.0.0.1:8080/actuator/health",
-        ]
-      interval: 10s
-      timeout: 10s
-      retries: 20
+      test: ["CMD", "/workspace/health-check"]
+      interval: 15s
+      timeout: 5s
+      retries: 3
       start_period: 90s
     logging:
       driver: "json-file"
@@ -286,6 +285,7 @@ services:
       POSTHOG_ENABLED: "false"
     depends_on:
       application-server:
+        # Gate on app-server readiness so the SPA isn't served before the API answers.
         condition: service_healthy
     healthcheck:
       test: ["CMD", "curl", "-f", "http://127.0.0.1:80/"]

docker/preview/compose.shared-infra.yaml

@@ -79,18 +79,24 @@ services:
       WEBHOOK_SECRET: ${WEBHOOK_SECRET:?Required}
       # prod runs forward-headers=native → ProxyTrustGuard aborts the boot unless set (Coolify ingress regex).
       HEPHAESTUS_TRUSTED_PROXIES: ${HEPHAESTUS_TRUSTED_PROXIES:?Required}
+      # Previews run non-release images with no release-pin, so don't require a digest-pinned agent image.
+      HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST: ${HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST:-false}
+      # thc healthcheck target: actuator readiness on the management port.
+      THC_PORT: "8080"
+      THC_PATH: /actuator/health/readiness
     depends_on:
       nats-server:
         condition: service_healthy
     # Spring drains HTTP (20s) THEN WebhookGracefulShutdown drains NATS (up to 15s).
     # Docker must allow ≥35s before SIGKILL, or the JVM is cut mid-drain.
     stop_grace_period: 40s
+    # thc healthcheck (see docker/compose.app.yaml).
     healthcheck:
-      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/actuator/health/readiness"]
+      test: ["CMD", "/workspace/health-check"]
       interval: 15s
       timeout: 5s
-      retries: 5
-      start_period: 30s
+      retries: 3
+      start_period: 90s
     logging:
       driver: "json-file"
       options:

docs/admin/buildpacks-cds-decision.md

@@ -27,15 +27,29 @@ AOT processing evaluates `@Conditional` at **build time**, baking the build-time
 
 ## Builder pinning
 
-`pom.xml` pins `builder-noble-java-tiny` + `ubuntu-noble-run-tiny` by sha256 digest. Refresh:
+`pom.xml` pins `builder-noble-java-tiny` + `ubuntu-noble-run-tiny` + the `health-checker` buildpack by
+sha256 digest. Refresh:
 
 ```
 docker buildx imagetools inspect paketobuildpacks/builder-noble-java-tiny:latest --format '{{.Manifest.Digest}}'
 docker buildx imagetools inspect paketobuildpacks/ubuntu-noble-run-tiny:latest    --format '{{.Manifest.Digest}}'
+docker buildx imagetools inspect paketobuildpacks/health-checker:latest           --format '{{.Manifest.Digest}}'
 ```
 
 Bump as part of release cycles; the digest is the source of truth.
 
+## Container healthcheck on the distroless run image
+
+On Docker Compose the container `HEALTHCHECK` is the only container-level health signal —
+`service_healthy` gating and the `docker compose ps` column both depend on it. `run-tiny` has no
+shell/wget and `builder-noble-java-tiny` bundles no probe, so `pom.xml` adds an explicit `<buildpacks>`
+order. Specifying `<buildpacks>` **replaces** the builder's default order, so the `java` composite must
+be re-listed (`urn:cnb:builder:paketo-buildpacks/java`) before appending
+`docker://paketobuildpacks/health-checker`; `BP_HEALTH_CHECKER_ENABLED=true` opts it in. It contributes
+the static, shell-free `thc` binary at `/workspace/health-check`, which the compose services invoke as an
+exec-form `HEALTHCHECK` (`THC_PORT`/`THC_PATH` → actuator liveness/readiness). No `health-check` process
+type is added, so the JVM-spawn-per-probe issue (health-checker#87) does not apply.
+
 ## git CLI in the runtime image
 
 `GitDiffOperations` previously shelled out to `git`; it was ported to JGit in the prerequisite commit, eliminating the runtime `git` dependency. The Paketo run image is used unmodified.

server/pom.xml

@@ -922,9 +922,18 @@
                     <image>
                         <builder>paketobuildpacks/builder-noble-java-tiny@sha256:c8b5f936e6d9af492af4a493b4a03eaf496fe51f206c5d53ccddab7871ed4ef5</builder>
                         <runImage>paketobuildpacks/ubuntu-noble-run-tiny@sha256:b5a167cf9a8021e289e5701a82f7c6b430ba85bb8360e9ddc767340e3f8824c3</runImage>
+                        <!-- Explicit order: the builder's `java` composite, then the health-checker buildpack
+                             (not bundled in builder-noble-java-tiny) so the distroless run image gets the static,
+                             shell-free `thc` binary at /workspace/health-check for a real Docker HEALTHCHECK.
+                             See docs/admin/buildpacks-cds-decision.md and docker/compose.app.yaml. -->
+                        <buildpacks>
+                            <buildpack>urn:cnb:builder:paketo-buildpacks/java</buildpack>
+                            <buildpack>docker://paketobuildpacks/health-checker@sha256:c95183658c32d47d8902bc59b9950f5bfa57cd4b7f655d2d5ba4d902051a0964</buildpack>
+                        </buildpacks>
                         <env>
                             <BP_JVM_VERSION>21</BP_JVM_VERSION>
                             <BP_JVM_CDS_ENABLED>true</BP_JVM_CDS_ENABLED>
+                            <BP_HEALTH_CHECKER_ENABLED>true</BP_HEALTH_CHECKER_ENABLED>
                             <!-- Activates application-cds-training.yml during the build-time training run.
                                  Coolify's runtime SPRING_PROFILES_ACTIVE=prod overrides this. -->
                             <SPRING_PROFILES_ACTIVE>cds-training</SPRING_PROFILES_ACTIVE>