feat(release): enforce supply-chain evidence

Author: FelixTJDietrich

.changeset/secure-release-evidence.md

@@ -0,0 +1,5 @@
+---
+"hephaestus": patch
+---
+
+Releases now publish verifiable SBOM, license, provenance, signature, and vulnerability evidence for every supported production image and platform. Production NATS, Traefik, nginx, and Alpine images are pinned to reviewed digests and covered by the same release evidence and recurring scans as Hephaestus images.

.github/actions/setup-release-security-tools/action.yml

@@ -0,0 +1,34 @@
+name: Setup release security tools
+description: Install checksum-verified Syft and Trivy binaries used by release evidence workflows
+inputs:
+  install-syft:
+    description: Install Syft in addition to Trivy
+    default: "true"
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        INSTALL_SYFT: ${{ inputs.install-syft }}
+        # renovate: datasource=github-releases depName=anchore/syft
+        SYFT_VERSION: 1.51.1
+        # renovate: datasource=github-releases depName=aquasecurity/trivy
+        TRIVY_VERSION: 0.74.0
+      run: |
+        set -euo pipefail
+        if [ "$INSTALL_SYFT" = true ]; then
+          syft_archive="syft_${SYFT_VERSION}_linux_amd64.tar.gz"
+          syft_base="https://github.com/anchore/syft/releases/download/v${SYFT_VERSION}"
+          curl -fsSL --proto =https -o "/tmp/$syft_archive" "$syft_base/$syft_archive"
+          curl -fsSL --proto =https -o /tmp/syft-checksums "$syft_base/syft_${SYFT_VERSION}_checksums.txt"
+          (cd /tmp && grep " $syft_archive$" syft-checksums | sha256sum -c -)
+          tar -xzf "/tmp/$syft_archive" -C /usr/local/bin syft
+          syft version
+        fi
+        trivy_archive="trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
+        trivy_base="https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}"
+        curl -fsSL --proto =https -o "/tmp/$trivy_archive" "$trivy_base/$trivy_archive"
+        curl -fsSL --proto =https -o /tmp/trivy-checksums "$trivy_base/trivy_${TRIVY_VERSION}_checksums.txt"
+        (cd /tmp && grep " $trivy_archive$" trivy-checksums | sha256sum -c -)
+        tar -xzf "/tmp/$trivy_archive" -C /usr/local/bin trivy
+        trivy --version

.github/workflows/release.yml

@@ -0,0 +1,92 @@
+name: Rescan supported release images
+on:
+  schedule:
+    - cron: "17 4 * * 1"
+  workflow_dispatch:
+permissions:
+  contents: read
+  packages: read
+  issues: write
+concurrency:
+  group: rescan-supported-release
+  cancel-in-progress: false
+jobs:
+  rescan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/setup-bun
+      - uses: ./.github/actions/setup-release-security-tools
+        with:
+          install-syft: "false"
+      - name: Download and verify latest supported release evidence
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          mkdir release-evidence
+          gh release download --repo "${{ github.repository }}" --dir release-evidence
+          (cd release-evidence && sha256sum -c SHA256SUMS)
+          for file in manifest.json release-images.json tool-versions.json trivy-db.json vulnerability-policy.json; do
+            test -s "release-evidence/$file"
+          done
+          jq -e --slurpfile inventory release-evidence/release-images.json \
+            '.schemaVersion == 1 and
+             ([.subjects[].image] | unique == (($inventory[0].images + [$inventory[0].upstream[].name]) | sort)) and
+             (group_by(.image) | all(.[]; [.[].platform] == ["linux/amd64", "linux/arm64"])) and
+             all(.subjects[]; (.digest | test("^sha256:[a-f0-9]{64}$")) and (.indexDigest | test("^sha256:[a-f0-9]{64}$")))' \
+            release-evidence/manifest.json >/dev/null
+          jq -er '.subjects[] | [.image, .platform, .indexDigest, .digest, .repository] | @tsv' release-evidence/manifest.json |
+          while IFS=$'\t' read -r image platform index_digest digest repository; do
+            suffix=${platform//\//-}
+            bun scripts/check-release-sbom.ts \
+              "release-evidence/$image-$suffix.syft.json" \
+              "release-evidence/$image-$suffix.spdx.json" \
+              "release-evidence/$image-$suffix.cdx.json" \
+              "$digest" "$platform" /tmp/sbom-summary.json
+            cmp "release-evidence/$image-$suffix.sbom-summary.json" /tmp/sbom-summary.json
+            for kind in trivy policy; do
+              test -s "release-evidence/$image-$suffix.$kind.json"
+            done
+            architecture=${platform#*/}
+            docker buildx imagetools inspect "$repository@$index_digest" --raw |
+              jq -e --arg architecture "$architecture" --arg digest "$digest" \
+                'any(.manifests[]; .platform.os == "linux" and .platform.architecture == $architecture and .digest == $digest)' >/dev/null
+          done
+      - name: Rescan immutable subjects
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          mkdir reports
+          trivy image --download-db-only
+          db_metadata="${TRIVY_CACHE_DIR:-$HOME/.cache/trivy}/db/metadata.json"
+          jq -e '(.UpdatedAt | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) as $updated |
+            ((now - $updated) >= 0) and ((now - $updated) <= 86400)' \
+            "$db_metadata" >/dev/null
+          cp "$db_metadata" reports/trivy-db.json
+          jq -er '.subjects[] | [.image, .platform, .digest, .repository] | @tsv' release-evidence/manifest.json |
+          while IFS=$'\t' read -r image platform digest repository; do
+            suffix=${platform//\//-}
+            trivy image --scanners vuln --format json --output "reports/$image-$suffix.json" "$repository@$digest"
+            bun scripts/check-release-vulnerabilities.ts "$image" "reports/$image-$suffix.json" security/vulnerability-policy.json "reports/$image-$suffix.policy.json"
+          done
+      - name: Upload diagnostic reports
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: supported-release-rescan-${{ github.run_id }}
+          path: reports
+          if-no-files-found: warn
+          retention-days: 30
+      - name: Notify vulnerability response tracking
+        if: failure()
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh issue comment 1369 --repo "${{ github.repository }}" --body \
+            "The supported-release rescan is **unknown or failing policy**. Treat this as actionable until triaged: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/rescan-release-images.yml

@@ -71,7 +71,6 @@ jobs:
     permissions:
       contents: read
       packages: write
-      security-events: write
       id-token: write
       attestations: write
     steps:
@@ -228,89 +227,6 @@ jobs:
           echo "digest=$DIGEST" >> "$GITHUB_OUTPUT"
           echo "Per-arch digest: $DIGEST"
 
-      # SBOMs are emitted by Paketo into /layers/sbom/ during the build; pack downloads them via
-      # the OCI label io.buildpacks.lifecycle.metadata.
-      - name: Install pack CLI (Buildpacks SBOM extraction)
-        if: inputs.use-buildpacks
-        env:
-          PACK_VERSION: v0.40.6
-        run: |
-          set -euo pipefail
-          ARCH_SUFFIX=$([ "${{ runner.arch }}" = "ARM64" ] && echo "-arm64" || echo "")
-          BASE="https://github.com/buildpacks/pack/releases/download/${PACK_VERSION}"
-          curl -fsSL --proto =https -o /tmp/pack.tgz       "${BASE}/pack-${PACK_VERSION}-linux${ARCH_SUFFIX}.tgz"
-          curl -fsSL --proto =https -o /tmp/pack.sha256    "${BASE}/pack-${PACK_VERSION}-linux${ARCH_SUFFIX}.tgz.sha256"
-          ( cd /tmp && awk '{print $1"  pack.tgz"}' pack.sha256 | sha256sum -c - )
-          tar -xzf /tmp/pack.tgz -C /usr/local/bin pack
-          pack version
-
-      - name: Extract SBOM from buildpack image
-        if: inputs.use-buildpacks
-        continue-on-error: true
-        env:
-          PER_ARCH_TAG: ${{ inputs.registry }}/${{ inputs.image-name }}:ci-${{ github.run_id }}-${{ steps.prep.outputs.platform_pair }}
-        run: |
-          set -euo pipefail
-          mkdir -p "${{ runner.temp }}/sbom"
-          # --remote reads from the registry rather than the local daemon — the image was just
-          # pushed by spring-boot:build-image. Auth is inherited from the docker/login-action
-          # earlier in this job via ~/.docker/config.json.
-          pack sbom download "$PER_ARCH_TAG" --remote --output-dir "${{ runner.temp }}/sbom"
-          ls -la "${{ runner.temp }}/sbom/layers/sbom/"
-
-      - name: Upload SBOM
-        if: inputs.use-buildpacks
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: sbom-${{ steps.prep.outputs.image_name }}-${{ steps.prep.outputs.platform_pair }}
-          path: ${{ runner.temp }}/sbom/
-          if-no-files-found: error
-          retention-days: 90
-
-      # Direct trivy binary install — aquasecurity/trivy-action's transitive setup-trivy dep
-      # breaks consumers of this reusable workflow when it can't resolve. Report-only baseline;
-      # flip --exit-code 1 once HIGH+ is at zero on main.
-      - name: Install Trivy
-        id: install-trivy
-        if: inputs.use-buildpacks
-        continue-on-error: true
-        env:
-          TRIVY_VERSION: 0.70.0
-        run: |
-          set -euo pipefail
-          ARCH=$([ "${{ runner.arch }}" = "ARM64" ] && echo "ARM64" || echo "64bit")
-          BASE="https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}"
-          ARCHIVE="trivy_${TRIVY_VERSION}_Linux-${ARCH}.tar.gz"
-          curl -fsSL --proto =https -o "/tmp/${ARCHIVE}"          "${BASE}/${ARCHIVE}"
-          curl -fsSL --proto =https -o /tmp/trivy_checksums.txt   "${BASE}/trivy_${TRIVY_VERSION}_checksums.txt"
-          ( cd /tmp && grep " ${ARCHIVE}$" trivy_checksums.txt | sha256sum -c - )
-          tar -xzf "/tmp/${ARCHIVE}" -C /usr/local/bin trivy
-          trivy --version
-
-      - name: Run Trivy (CVE scan, report-only)
-        if: inputs.use-buildpacks && steps.install-trivy.outcome == 'success'
-        continue-on-error: true
-        env:
-          # Trivy queries the registry directly (not the local daemon) and does not read
-          # ~/.docker/config.json — explicit credentials are required for ghcr.io.
-          TRIVY_USERNAME: ${{ github.actor }}
-          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
-          PER_ARCH_TAG: ${{ inputs.registry }}/${{ inputs.image-name }}:ci-${{ github.run_id }}-${{ steps.prep.outputs.platform_pair }}
-        run: |
-          set -euo pipefail
-          trivy image --severity CRITICAL,HIGH --ignore-unfixed --exit-code 0 \
-            --format sarif --output "${{ runner.temp }}/trivy.sarif" "$PER_ARCH_TAG"
-
-      # Surface findings in the GitHub Security tab + PR annotations rather than a 30-day artifact
-      # nobody opens. continue-on-error keeps the workflow green while the baseline is non-zero.
-      - name: Upload Trivy SARIF to GitHub code scanning
-        if: inputs.use-buildpacks && steps.install-trivy.outcome == 'success'
-        continue-on-error: true
-        uses: github/codeql-action/upload-sarif@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.30.9
-        with:
-          sarif_file: ${{ runner.temp }}/trivy.sarif
-          category: trivy-image-${{ steps.prep.outputs.platform_pair }}
-
       - name: Export digest
         env:
           DOCKERFILE_DIGEST: ${{ steps.build.outputs.digest }}

.github/workflows/reusable-docker-build.yml

@@ -131,7 +131,7 @@ services:
         max-file: "3"
 
   nats-server:
-    image: nats:alpine
+    image: nats:2.14.6-alpine@sha256:ad7a43eb7e3337c3c38ce5d784d1461791f95f730f252d2b25eee699752a0ca3
     restart: unless-stopped
     ports:
       # Loopback-only by default. Set NATS_BIND_HOST=0.0.0.0 (or a specific interface) to let other

docker/compose.core.yaml

@@ -2,7 +2,7 @@ services:
   reverse-proxy:
     # Keep at >=3.3: below it the app-server sticky.cookie.path label (workspace replica affinity)
     # fails to parse and Traefik silently drops the whole router. Pin a current 3.x.
-    image: traefik:v3.7.4
+    image: traefik:v3.7.4@sha256:fcdef599e6259359833dd2e1d49f9e964f66825d69bd3dd468f51102ce013d03
     restart: unless-stopped
     networks:
       - shared-network
@@ -60,7 +60,7 @@ services:
         max-file: "3"
 
   maintenance:
-    image: nginx:alpine
+    image: nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913
     restart: unless-stopped
     configs:
       - source: maintenance-page
@@ -202,4 +202,4 @@ configs:
               <p>&mdash; Your Hephaestus Administrators</p>
           </div>
       </article>
-      </html>
+      </html>

docker/compose.proxy.yaml

@@ -64,8 +64,6 @@ Revert `.github/workflows/ci-docker-build.yml` (the `use-buildpacks: true` line)
 
 - **Coolify graceful shutdown** — `application.yml` sets `SHUTDOWN_TIMEOUT:20s`. Coolify's default container stop-grace is 10s; bump it to ≥25s in the deploy substrate so SIGTERM has time to drain in-flight requests. The Paketo launcher `exec`s the JVM; signal forwarding is native, no `tini`.
 - **JVM memory** — do NOT set `MaxRAMPercentage`, `-Xmx`, or `-Xss` in Coolify env. Paketo's memory calculator handles them. Override only `BPL_JVM_HEAD_ROOM` if needed.
-- **SBOM** — Paketo emits Syft + SPDX + CycloneDX at `/layers/sbom/`. CI extracts via `pack sbom download` and uploads as a 90-day artifact.
-- **CVE scan** — Trivy runs on every PR and uploads SARIF to GitHub Security. Until the baseline is clean, results are non-blocking; flip `--exit-code 1` in the workflow once HIGH+ is at zero.
 - **CI build time** — expect +60–120s per build vs the prior Dockerfile baseline (CDS training run dominates).
 
 ## Sources

docs/admin/buildpacks-cds-decision.md

@@ -32,7 +32,11 @@ sequenceDiagram
     M->>VP: changesets action folds pending changesets into the accumulating Version PR
     Note over VP: CHANGELOG.md + version bump preview
     VP->>M: Maintainer merges when the release is ready
-    M->>R: Tag vX.Y.Z, GitHub Release with curated notes,<br/>docker images X.Y.Z / X.Y / latest
+    M->>R: Create draft vX.Y.Z release
+    R->>R: Generate and verify digest-bound evidence
+    R->>R: Promote X.Y.Z / X.Y / latest image tags
+    R->>R: Publish release and deploy staging
+    Note over R: Production still requires approval
 ```
 
 1. **Every user-facing PR carries a changeset.** `bun changeset` asks for the bump type and an
@@ -50,10 +54,73 @@ sequenceDiagram
    the full suite and a release is only cut if that run succeeds. Maintainers bypass the branch
    ruleset, so the absent required checks don't block the merge. Versioning also moves any
    `### Next release` section in `MIGRATION.md` under the version being cut.
-3. **Merging the Version PR cuts the release.** The release workflow tags `vX.Y.Z` at the merge
-   commit, creates the GitHub Release from the new changelog section, retags the CI-built Docker
-   images as `X.Y.Z`, `X.Y`, and `latest`, publishes the signed release-pin asset, and starts the
-   deploy chain (staging automatically, production after approval).
+3. **Merging the Version PR cuts the release.** The workflow creates a draft release at the merge
+   commit. After its evidence gate passes, it promotes the CI-built images to `X.Y.Z`, `X.Y`, and
+   `latest`, publishes the release, deploys staging, and requests production approval.
+
+## Supply-chain evidence
+
+A release remains a draft until every first-party and upstream production image in
+`security/release-images.json` has passed the evidence
+gate. The gate resolves each image index and its `linux/amd64` and `linux/arm64` manifests to immutable
+digests. It generates a lossless Syft inventory plus SPDX and CycloneDX SBOMs for each deployed platform.
+The gate proves that the Syft source metadata identifies the exact digest and architecture, checks that
+every discovered package survives both standard-format conversions, and records packages whose license
+could not be detected. It also scans each platform manifest with Trivy, signs its SPDX predicate as an OCI
+attestation, and verifies the image signature and build provenance. The SBOMs, license summaries, scan
+reports, applied policy, checksums, and `manifest.json` are GitHub Release assets, so they outlive Actions
+artifact retention. Publication requires every artifact to be present, well formed, and bound to its
+recorded digest.
+
+Upstream NATS, Traefik, nginx, and Alpine images are versioned and digest-pinned in the production
+Compose topology. They receive the same per-platform SBOM, license, vulnerability, and index-membership
+evidence as first-party images. Hephaestus requires its own signatures and build provenance only for
+images it builds; it does not misrepresent observed upstream images as Hephaestus-built artifacts.
+
+The version-controlled policy rejects new fixable HIGH or CRITICAL findings. Baselines use exact
+image/CVE/package/installed-version fingerprints. Exceptions use the same scope and require an owner,
+justification, and future expiry.
+
+The latest release is rescanned weekly. A policy violation or scan failure is reported on
+[the vulnerability response issue](https://github.com/ls1intum/Hephaestus/issues/1369); scanner failure
+is never treated as no findings.
+
+### Operator verification
+
+Install `gh`, `jq`, `cosign`, and GNU `sha256sum`, then use a clean directory and replace `vX.Y.Z`
+below. Verification uses the digests in the release manifest, never mutable tags:
+
+```bash
+gh release download vX.Y.Z --repo ls1intum/Hephaestus
+sha256sum -c SHA256SUMS
+
+jq -r '.subjects[] | [.image, .platform, .digest] | @tsv' manifest.json |
+while IFS=$'\t' read -r image platform digest; do
+  ref="ghcr.io/ls1intum/hephaestus/$image@$digest"
+  suffix=${platform//\//-}
+  cosign verify-attestation --type spdxjson \
+    --certificate-identity 'https://github.com/ls1intum/Hephaestus/.github/workflows/release.yml@refs/heads/main' \
+    --certificate-oidc-issuer https://token.actions.githubusercontent.com "$ref" \
+    > verified-attestations.json
+  jq -e --slurpfile sbom "$image-$suffix.spdx.json" \
+    'any(.[]; (.payload | @base64d | fromjson | .predicate) == $sbom[0])' \
+    verified-attestations.json >/dev/null
+done
+
+jq -r '[.subjects[] | [.image, .indexDigest]] | unique[] | @tsv' manifest.json |
+while IFS=$'\t' read -r image digest; do
+  ref="ghcr.io/ls1intum/hephaestus/$image@$digest"
+  cosign verify "$ref" \
+    --certificate-identity 'https://github.com/ls1intum/Hephaestus/.github/workflows/reusable-docker-build.yml@refs/heads/main' \
+    --certificate-oidc-issuer https://token.actions.githubusercontent.com
+  gh attestation verify "oci://$ref" --owner ls1intum \
+    --signer-workflow ls1intum/Hephaestus/.github/workflows/reusable-docker-build.yml
+done
+```
+
+The downloaded `*.spdx.json` and `*.cdx.json` files are the machine-readable package and license
+inventories. `*.trivy.json` is a point-in-time scan, not a promise that vulnerability knowledge remains
+unchanged.
 
 ## Writing changesets