fix(server): run the agent image built from the same commit as the server (#1500)

Derives the agent sandbox image reference from the deployment's own IMAGE_TAG instead of defaulting to the agent-pi:latest release channel, so the sandbox and the application server are built from one commit by construction. A reference naming a moving tag is refused at startup. Also fixes the release workflow asserting node --version against an image that no longer ships Node.

Author: FelixTJDietrich

.changeset/agent-image-matches-the-server.md

@@ -0,0 +1,17 @@
+---
+"hephaestus": minor
+---
+
+Practice reviews and mentor conversations now run in the agent sandbox image built from the same
+commit as the application server. A deployment that tracked `main` previously fell back to the newest
+released sandbox image, which pairs a server with a sandbox nobody built it against — reviews and
+mentor turns then failed inside the container with nothing explaining why. The server now reports at
+startup when the sandbox image cannot run it, naming both versions.
+
+**Operators:** the sandbox image now follows `IMAGE_TAG`, so a tag that moves between builds refuses
+to start — `IMAGE_TAG=latest`, which earlier example configuration shipped, and equally a partial
+version such as `0.73`, which every patch release moves. The same goes for setting
+`HEPHAESTUS_AGENT_IMAGE_REFERENCE` to one. Set `IMAGE_TAG` to a full release version or a commit SHA,
+and remove or digest-pin the reference override, before upgrading. This applies even with the agent
+disabled. Release deployments that changed neither, and take the signed digest pin, are unaffected.
+See MIGRATION.md.

.github/workflows/cd-staging.yml

@@ -35,21 +35,42 @@ jobs:
     outputs:
       image-exists: ${{ steps.check.outputs.exists }}
     steps:
-      # Every main push builds :<sha>, except when skip-duplicate-actions dedups an
-      # identical tree. Skip the deploy (instead of failing red) when that happens.
-      - name: Look up application-server image for this commit
+      # The server and the agent image are one runtime contract, so this asks about the PAIR: the
+      # server stages TypeScript runners and execs `bun` inside agent-pi, and deploying a commit
+      # whose agent image never published leaves the app resolving an unmatched one (ADR 0031).
+      #
+      # A missing image is normal, not a failure — skip rather than deploy half a pair. Both known
+      # causes are benign: skip-duplicate-actions dedups an identical tree, and a CI/CD run
+      # cancelled by a later push (merging several PRs in quick succession) leaves the earlier
+      # commit's images unpublished.
+      - name: Look up the image pair for this commit
         id: check
         env:
-          IMAGE: ghcr.io/ls1intum/hephaestus/application-server
           SHA: ${{ github.event.workflow_run.head_sha }}
           TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
+          set -euo pipefail
           echo "$TOKEN" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-          if docker manifest inspect "${IMAGE}:${SHA}" >/dev/null 2>&1; then
-            echo "exists=true" >> "$GITHUB_OUTPUT"
-          else
+          missing=""
+          for img in application-server agent-pi; do
+            ref="ghcr.io/ls1intum/hephaestus/${img}:${SHA}"
+            # Presence is a bare probe on its own exit status, as in release.yml. Reading the digest
+            # in the condition instead would test `jq`'s status, and `jq` exits 0 on empty input — an
+            # absent image would then look present and the pair would go unchecked.
+            if ! docker buildx imagetools inspect "$ref" > /dev/null 2>&1; then
+              missing="${missing} ${img}"
+              continue
+            fi
+            digest=$(docker buildx imagetools inspect "$ref" --format '{{json .Manifest}}' | jq -r '.digest')
+            [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || {
+              echo "::error::${img} digest at ${SHA} is malformed: ${digest:-<empty>}"; exit 1; }
+            echo "${img}: ${digest}"
+          done
+          if [ -n "$missing" ]; then
             echo "exists=false" >> "$GITHUB_OUTPUT"
-            echo "No \`application-server:${SHA}\` image (build deduplicated) — nothing to deploy." >> "$GITHUB_STEP_SUMMARY"
+            echo "No \`${missing# }\` image at \`${SHA}\` (build deduplicated or its CI run cancelled) — nothing to deploy." >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "exists=true" >> "$GITHUB_OUTPUT"
           fi
 
   deploy-staging:

.github/workflows/ci-docker-build.yml

@@ -109,9 +109,13 @@ jobs:
       registry: "ghcr.io"
       # amd64-only on PRs (main/release build both arches). See reusable-docker-build.yml.
       single-arch: ${{ github.event_name == 'pull_request' }}
+      # The head-SHA tag matters here for the same reason it does on application-server: a preview
+      # runs at SOURCE_COMMIT and derives its agent image from that tag, so without it no preview can
+      # ever resolve a matched pair, however the pull request touched the agent tree (ADR 0031).
       tags: |
         ${{ github.ref_name }}
         ${{ github.sha }}
+        ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}
         ci-${{ github.run_number }}
         ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.number) || '' }}
       labels: |

.github/workflows/ci-quality-gates.yml

@@ -370,6 +370,22 @@ jobs:
             echo "::error::Legacy agent image is removed in issue #1076; relocate any new references to MIGRATION.md."
             exit 1
           fi
+      # The defect ADR 0031 removed: a reference that floats on the release channel resolves to
+      # whatever released most recently, which is a pair no release ever built. The reference is
+      # derived from the deployment's own image tag now, so nothing may name a channel again.
+      # Needle built at runtime so this guard file doesn't self-match.
+      - name: Assert no release-channel agent image references
+        run: |
+          set -euo pipefail
+          NEEDLE="agent-pi:$(printf 'lat\x65st')"
+          if grep -rEnI "$NEEDLE" \
+                --exclude-dir=node_modules --exclude-dir=target --exclude-dir=.git \
+                . \
+              | grep -vE '^\./(MIGRATION\.md|docs/admin/agent-image-digests\.md|docs/decisions/0031-.*\.md|server/src/test/java/de/tum/cit/aet/hephaestus/agent/sandbox/AgentImageReferenceGuardTest\.java):' ; then
+            echo "::error::A release-channel agent image tag resolves to another release's image (ADR 0031). Leave the reference unset so it follows this deployment's image tag, or pin a digest."
+            exit 1
+          fi
+
       - name: Assert no legacy agent-image-pin.env references
         run: |
           set -euo pipefail

.github/workflows/release.yml

@@ -159,7 +159,10 @@ jobs:
         id: retag
         env:
           VERSION: ${{ needs.release.outputs.version }}
-          CHANNEL: ${{ needs.release.outputs.major }}.${{ needs.release.outputs.minor }}
+          # The `major.minor` series tag: it moves onto every patch release in the line, exactly as
+          # `latest` moves onto every release. Both are channel tags; only the spelling differs, and
+          # naming this one `CHANNEL` made them look like different things. ADR 0031.
+          SERIES: ${{ needs.release.outputs.major }}.${{ needs.release.outputs.minor }}
           SHA: ${{ needs.release.outputs.sha }}
         run: |
           set -euo pipefail
@@ -195,11 +198,11 @@ jobs:
 
             docker buildx imagetools create \
               -t "$FULL_IMAGE:$VERSION" \
-              -t "$FULL_IMAGE:$CHANNEL" \
+              -t "$FULL_IMAGE:$SERIES" \
               -t "$FULL_IMAGE:latest" \
               "$FULL_IMAGE:$SHA"
 
-            for tag in "$VERSION" "$CHANNEL" "latest"; do
+            for tag in "$VERSION" "$SERIES" "latest"; do
               DST_DIGEST=$(manifest_digest "$FULL_IMAGE:$tag")
               if [[ "$SRC_DIGEST" != "$DST_DIGEST" ]]; then
                 echo "::error::Retag changed the digest for $img:$tag — refusing to publish."
@@ -214,13 +217,33 @@ jobs:
             echo "::endgroup::"
           done
 
+      - name: Check out the released tree
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          ref: ${{ needs.release.outputs.sha }}
+          fetch-depth: 1
+
+      # The pin asset below tells every production deploy to run THIS agent image, so this is the
+      # last point at which an image the released server cannot drive is still catchable. The
+      # sandbox is Bun-only (ADR 0030), so `node` must be absent, not present.
       - name: Smoke-test agent-pi
         env:
           IMAGE: ghcr.io/ls1intum/hephaestus/agent-pi@${{ steps.retag.outputs.agent-pi-digest }}
+          LAYOUT: server/src/main/java/de/tum/cit/aet/hephaestus/agent/runtime/SandboxLayout.java
         run: |
           set -euo pipefail
           docker pull "$IMAGE"
-          docker run --rm --entrypoint /bin/sh "$IMAGE" -c 'bun --version && node --version'
+          docker run --rm --entrypoint /bin/sh "$IMAGE" -c \
+            'bun --version && ! command -v node >/dev/null 2>&1'
+
+          expected=$(grep -oE 'RUNTIME_CONTRACT_VERSION = [0-9]+' "$LAYOUT" | grep -oE '[0-9]+$')
+          [ -n "$expected" ] || { echo "::error::Could not read RUNTIME_CONTRACT_VERSION from $LAYOUT"; exit 1; }
+          declared=$(docker inspect --format '{{index .Config.Labels "hephaestus.agent.runtime-contract"}}' "$IMAGE")
+          if [ "$declared" != "$expected" ]; then
+            echo "::error::agent-pi implements runtime contract '${declared:-<none>}' but the released server stages for v${expected} — refusing to publish a pin for an unmatched pair."
+            exit 1
+          fi
+          echo "agent-pi implements runtime contract v${expected}"
 
       - name: Write release pin asset
         id: pin

.gitignore

@@ -6,6 +6,7 @@
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 application-live-local.yml
 application-local.yml
+application-local.yaml
 **/atlassian-ide-plugin.xml
 # auto-import.
 # Beads (AI agent issue tracker) - local-only files

MIGRATION.md

@@ -65,6 +65,64 @@ Entries exist only for releases that need operator action. Everything else is in
 
 ### Next release
 
+#### 🔴 An agent image reference naming a channel tag is now refused
+
+**Affected**, and either one is enough:
+
+- `HEPHAESTUS_AGENT_IMAGE_REFERENCE` set to a channel tag — `:latest`, `:stable`, `:edge`, `:main`,
+  or a partial version such as `:0.73`, which we retag onto every patch release in that line — or to
+  a reference with no tag at all.
+- **`IMAGE_TAG=latest`** — which earlier versions of `docker/.env.example` shipped as the default —
+  or **`IMAGE_TAG=0.73`**. The reference now derives from `IMAGE_TAG`, so such a deployment resolves
+  `agent-pi:latest` or `agent-pi:0.73` without ever naming it, and the refusal applies just the same.
+
+Check both with `grep -E 'IMAGE_TAG|HEPHAESTUS_AGENT_IMAGE_REFERENCE'` over your deployment
+configuration before you upgrade. `AGENT_ENABLED=false` does **not** exempt you: the check runs at
+startup, whether or not the sandbox is ever used. A release deploy that leaves both alone takes the
+signed digest pin and is unaffected.
+
+The boot fails with one of:
+
+```
+hephaestus.agent.image.reference must not be a channel tag
+hephaestus.agent.image.reference names a version series rather than one release
+```
+
+**Before**: the agent sandbox image fell back to `ghcr.io/ls1intum/hephaestus/agent-pi:latest` when
+nothing else supplied a reference. `latest` tracks the newest **release**, so a deployment tracking
+`main` ran its application server against an agent image built from a different commit. Nothing
+reported it: practice reviews and mentor sessions simply failed inside the container.
+
+**After**: the reference follows your deployment's own `IMAGE_TAG`, so the sandbox image is the one
+built from the same commit as the application server. A channel tag is refused at startup with a
+message naming the fix, because it can only ever name a pairing no release produced.
+
+**Action**: set `IMAGE_TAG` to a full release version (`0.74.0`) or to a full commit SHA — never
+`latest`, and never the `0.74` series. Then, if you also set the reference override, remove it or
+replace it with a digest:
+
+```bash
+# either: remove the HEPHAESTUS_AGENT_IMAGE_REFERENCE line entirely (recommended) — do not
+# leave it present and empty, which binds an empty reference and fails the boot for a second reason
+#
+# or: pin the exact image you mean
+HEPHAESTUS_AGENT_IMAGE_REFERENCE=ghcr.io/ls1intum/hephaestus/agent-pi@sha256:<digest>
+```
+
+A deployment tracking `main` keeps `HEPHAESTUS_RELEASE_PIN_SKIP=true` and
+`HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST=false`; the derived reference is a matched tag, not a digest.
+See [Agent image digests](https://ls1intum.github.io/Hephaestus/admin/agent-image-digests).
+
+#### 🟡 Preview deployments name their own agent image
+
+**Affected**: preview stacks (`docker/preview/`) that run practice reviews or the mentor from a pull
+request which does not touch `docker/agents/**`.
+
+A preview now derives its agent image from its own commit, and CI publishes one at that commit only
+when the pull request changed the agent tree or a workflow. Previously such a preview silently used
+the last release's image. Set `HEPHAESTUS_AGENT_IMAGE_REFERENCE` in the preview's `.env` to the agent
+image you want it to exercise — `docker/preview/.env.example` shows the line.
+
 #### 🟡 Reviewer-side practices keep the old wording until you update them
 
 **Affected**: workspaces created before this release that use the shipped practices

docker/.env.example

@@ -16,8 +16,12 @@
 # Hostname for the application (e.g., hephaestus.cit.tum.de)
 APP_HOSTNAME=
 
-# Docker image tag to deploy (e.g., latest, v1.0.0, develop)
-IMAGE_TAG=latest
+# Release to deploy, or the full commit SHA for a deploy that tracks main. Every image in the stack
+# is pulled at this tag, and the agent sandbox image is resolved from it too, so the whole
+# deployment is one matched build. Never a tag that moves between builds — `latest`, or a partial
+# version like `0.74` that every patch release retags — because it pairs a server with an agent image
+# nobody built it against, and the boot is refused. See ADR 0031.
+IMAGE_TAG=0.74.0
 
 # Tomcat RemoteIpValve internal-proxies REGEX (NOT CIDR) matching ONLY the ingress (Traefik) address.
 # REQUIRED in prod: forward-headers-strategy=native is active and ProxyTrustGuard fails the boot if
@@ -231,7 +235,17 @@ GITLAB_WORKSPACE_CREATION=false
 # AGENT SANDBOX (Practice Review)
 # -----------------------------------------------------------------------------
 
-# Agent image override (dev only): see docs/admin/agent-image-digests.md.
+# Agent sandbox image. Left unset it follows IMAGE_TAG above on a deploy tracking main, and the
+# release's cosign-verified digest on a release deploy — so setting it is an override of both, for
+# when neither names the image you mean. A digest is immutable and therefore reproducible; never a
+# release channel, which names whatever released most recently. See docs/admin/agent-image-digests.md.
+# HEPHAESTUS_AGENT_IMAGE_REFERENCE=ghcr.io/ls1intum/hephaestus/agent-pi@sha256:<digest>
+
+# A deploy that tracks main rather than a release sets BOTH of these: there is no signed pin asset
+# for a main commit, so the fetcher must be told to skip it and the digest requirement must come off.
+# The derived reference is still a matched pair, just a tag rather than a digest.
+# HEPHAESTUS_RELEASE_PIN_SKIP=true
+# HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST=false
 
 # The sandbox is activated by the worker role (hephaestus.runtime.worker.enabled, default true
 # in the monolith), not a toggle. Practice review additionally needs AGENT_ENABLED and

docker/agents/pi/Dockerfile

@@ -105,5 +105,14 @@ RUN set -eu; \
     fi; \
     echo "OK: no node/npm in image; runtime is $(bun --version) at $(command -v bun)"
 
+# The build above proves the runtime contract — Bun resolves and imports the SDK exactly as
+# PiRuntimeFactory arranges it, and no Node survives. These labels are the only part of that proof a
+# server can read before it commits a job to a container: it compares the contract version against
+# SandboxLayout.RUNTIME_CONTRACT_VERSION, which AgentImageContractSyncTest pins to the line below.
+# Bump both together when an older image could no longer run the staged runners. ADR 0031.
+LABEL hephaestus.agent.runtime-contract=1
+LABEL hephaestus.agent.bun-version=${BUN_VERSION}
+LABEL hephaestus.agent.pi-version=${PI_VERSION}
+
 USER 1000:1000
 WORKDIR /workspace

docker/compose.app.yaml

@@ -258,10 +258,19 @@ services:
       GITLAB_DEFAULT_SERVER_URL: ${GITLAB_DEFAULT_SERVER_URL:-${GITLAB_OAUTH_BASE_URL:-https://gitlab.com}}
       # Agent sandbox (practice review + Pi mentor chat). Activated by the worker role
       # (hephaestus.runtime.worker.enabled), then gated per-workspace via WorkspaceFeatures.
-      # AgentImagePinGuard requires a digest-pinned agent image in prod; the release-pin-fetcher
-      # supplies it for release tags. Set false for non-release (e.g. main/latest) deploys that skip
-      # the pin, otherwise the boot fails on agent-pi:<tag>. Keep true for release deploys.
+      # AgentImagePinGuard requires a digest-pinned agent image in prod, and the release-pin-fetcher
+      # supplies that digest for release tags. A non-release deploy has no signed pin, so the server
+      # falls back to agent-pi:${IMAGE_TAG} — the agent image CI built from the very commit this
+      # server image came from. That is a matched pair, but a tag rather than a digest, so a deploy
+      # tracking main sets this false. Keep true for release deploys, where the pin is authoritative
+      # and stronger. Never point this at a release channel instead — see ADR 0031.
       HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST: ${HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST:-true}
+      # The operator's override, and the only lever when neither the signed pin nor the derivation
+      # names the right image. Valueless on purpose: Compose then sets it only when the deployment
+      # actually supplies one, from the stack `.env` or the host. `${VAR:-}` would set it to the
+      # empty string instead, and Spring reads present-but-empty as a value rather than as absent —
+      # the reference would bind to "" and every deployment that does not override would fail to boot.
+      HEPHAESTUS_AGENT_IMAGE_REFERENCE:
       SANDBOX_DOCKER_HOST: ${SANDBOX_DOCKER_HOST:-unix:///var/run/docker.sock}
       SANDBOX_CONTAINER_RUNTIME: ${SANDBOX_CONTAINER_RUNTIME:-}
       SANDBOX_MAX_CONCURRENT: ${SANDBOX_MAX_CONCURRENT:-5}
@@ -376,6 +385,8 @@ services:
       <<: [*practice-review-env, *scm-access-env]
       HEPHAESTUS_TRUSTED_PROXIES: ${HEPHAESTUS_TRUSTED_PROXIES:-}
       HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST: ${HEPHAESTUS_AGENT_IMAGE_REQUIRE_DIGEST:-true}
+      # Valueless, as on application-server above; this is the pod that actually opens sandboxes.
+      HEPHAESTUS_AGENT_IMAGE_REFERENCE:
       HEPHAESTUS_HUB_URL: ws://application-server:8080/api/workers/connect
       HEPHAESTUS_WORKER_REGISTRATION_TOKEN: ${HEPHAESTUS_WORKER_REGISTRATION_TOKEN:-}
       # Agent job queue (practice review). The queue runs on PostgreSQL — this pod needs no NATS at