Skip to content

Release

Release #774

Workflow file for this run

name: Release
# Versioning contract: docs/admin/compatibility-policy.mdx
on:
workflow_run: # zizmor: ignore[dangerous-triggers] The guard authenticates the completed run before privileged jobs start.
workflows: ["CI/CD"]
types: [completed]
branches: [main]
# Serialize promotion of the moving series and latest tags.
concurrency:
group: release
cancel-in-progress: false
permissions: {}
jobs:
release:
# Only real pushes to our own main — a fork PR's CI/CD run reports
# head_branch of the fork; without these guards a fork branch named "main"
# could drive the privileged release job.
if: >-
${{ github.event.workflow_run.conclusion == 'success'
&& github.event.workflow_run.event == 'push'
&& github.event.workflow_run.head_repository.full_name == github.repository }}
timeout-minutes: 15
runs-on: ubuntu-latest
permissions:
# No release is created here — tag-images creates the draft once the evidence gate has
# passed. Draft releases are invisible to tokens without push access, and the plan turns on
# whether this version's release is a draft, so listing them still needs write.
contents: write
outputs:
released: ${{ steps.cut.outputs.released }}
version: ${{ steps.cut.outputs.version }}
major: ${{ steps.cut.outputs.major }}
minor: ${{ steps.cut.outputs.minor }}
tag_name: ${{ steps.cut.outputs.tag_name }}
previous_version: ${{ steps.cut.outputs.previous_version }}
sha: ${{ steps.cut.outputs.sha }}
migrations: ${{ steps.cut.outputs.migrations }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.sha }}
fetch-depth: 0
persist-credentials: false
- uses: ./.github/actions/setup-toolchain
with:
install: "none"
# Decides whether this commit cuts a release, and creates nothing. The rule — a version with
# no published release is cut, from whichever commit carries it — and the four cases it
# decides between are documented in scripts/plan-release.ts. The draft is created in
# tag-images, after the evidence gate, so a gate failure leaves nothing behind: release images
# are promoted by digest and never rebuilt, so a draft cut at a commit the gate rejected can
# never pass. The same version then re-cuts from the commit that carries the fix.
- name: Plan the release for this commit
id: cut
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SHA: ${{ github.event.workflow_run.head_sha }}
run: |
set -euo pipefail
# The commit CI/CD built has to be one of ours; everything downstream checks it out.
git merge-base --is-ancestor "$SHA" origin/main || {
echo "::error::Commit $SHA is not on main"; exit 1; }
node scripts/plan-release.ts "$SHA"
- name: Summary
if: steps.cut.outputs.released == 'true'
run: |
echo "## Preparing ${{ steps.cut.outputs.tag_name }}" >> "$GITHUB_STEP_SUMMARY"
echo "Evidence verification is in progress; the draft is created once it passes." >> "$GITHUB_STEP_SUMMARY"
tag-images:
needs: release
if: needs.release.outputs.released == 'true'
timeout-minutes: 30
runs-on: ubuntu-latest
permissions:
packages: write
id-token: write
attestations: write
contents: write # gh release create/upload, once the evidence gate has passed
outputs:
agent-pi-digest: ${{ steps.retag.outputs.agent-pi-digest }}
application-server-digest: ${{ steps.retag.outputs.application-server-digest }}
postgres-digest: ${{ steps.retag.outputs.postgres-digest }}
steps:
- name: Check out the released tree
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.release.outputs.sha }}
fetch-depth: 1
persist-credentials: false
- name: Log in to GitHub Container Registry
uses: ./.github/actions/ghcr-login
with:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: ./.github/actions/setup-toolchain
with:
install: "none"
# The images this release promotes, resolved through the same script the pre-merge evidence
# preflight resolves them with, so the two can never disagree about which artefact is subject.
- name: Capture release image digests
id: retag
env:
SOURCE_TAG: ${{ github.event.workflow_run.head_sha }}
run: node scripts/resolve-release-images.ts "$SOURCE_TAG" release-images.tsv
- name: Verify Node-only agent-pi sandbox
env:
IMAGE: ghcr.io/hephaestus-build/agent-pi@${{ steps.retag.outputs.agent-pi-digest }}
LAYOUT: server/application/src/main/java/de/tum/cit/aet/hephaestus/agent/runtime/SandboxLayout.java
run: |
set -euo pipefail
docker pull "$IMAGE"
docker run --rm --entrypoint /bin/sh "$IMAGE" -c \
'node --version && for runtime in npm npx corepack yarn yarnpkg pnpm pnpx; do ! command -v "$runtime" || exit 1; done'
expected=$(grep -oE 'RUNTIME_CONTRACT_VERSION = [0-9]+' "$LAYOUT" | grep -oE '[0-9]+$')
[ -n "$expected" ] || { echo "::error::Could not read RUNTIME_CONTRACT_VERSION from $LAYOUT"; exit 1; }
declared=$(docker inspect --format '{{index .Config.Labels "hephaestus.agent.runtime-contract"}}' "$IMAGE")
if [ "$declared" != "$expected" ]; then
echo "::error::agent-pi implements runtime contract '${declared:-<none>}' but the released server stages for v${expected} — refusing to publish a pin for an unmatched pair."
exit 1
fi
echo "agent-pi implements runtime contract v${expected}"
- uses: ./.github/actions/setup-release-security-tools
# The database the whole evidence bundle is scanned against: recorded into the bundle, and
# refused if it is over a day old, so a stale mirror cannot be signed as a fresh scan.
- uses: ./.github/actions/download-trivy-db
with:
max-age-hours: "24"
metadata-path: evidence/trivy-db.json
- name: Generate and enforce release evidence
env:
TRIVY_USERNAME: ${{ github.actor }}
TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
RELEASE: ${{ needs.release.outputs.tag_name }}
COMMIT: ${{ needs.release.outputs.sha }}
run: |
set -euo pipefail
node scripts/generate-release-evidence.ts evidence \
--release "$RELEASE" --commit "$COMMIT" --digests release-images.tsv
node scripts/verify-release-evidence.ts evidence --write-validation
jq -er '.subjects[] | select(.provenance == "first-party") | [.repository, .digest, .image, .platform] | @tsv' evidence/manifest.json |
while IFS=$'\t' read -r repository digest image platform; do
suffix=${platform//\//-}
cosign attest --yes --type spdxjson --predicate "evidence/$image-$suffix.spdx.json" "$repository@$digest"
done
(cd evidence && sha256sum -- * > SHA256SUMS)
- name: Verify evidence from registry subjects
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: node scripts/verify-release-evidence.ts evidence --verify-signatures
- name: Write release image lock
id: pin
env:
VERSION: ${{ needs.release.outputs.version }}
run: |
set -euo pipefail
ASSET="release-v${VERSION}.json"
jq '{schemaVersion, release, commit,
images: [.subjects[]]
| group_by(.image)
| map({image: .[0].image, repository: .[0].repository, provenance: .[0].provenance,
indexDigest: .[0].indexDigest,
platforms: (map({key: .platform, value: .digest}) | from_entries)})}' \
evidence/manifest.json > "$ASSET"
node scripts/release-image-lock.ts "$ASSET" evidence/manifest.json \
"v${VERSION}" /tmp/release-lock.env
echo "asset-path=$ASSET" >> "$GITHUB_OUTPUT"
jq . "$ASSET"
- name: Generate subject checksums
id: subjects
env:
ASSET: ${{ steps.pin.outputs.asset-path }}
run: |
set -euo pipefail
while IFS=$'\t' read -r image _ digest; do
printf '%s %s\n' "${digest#sha256:}" "$image"
done < release-images.tsv > subjects.sha256
sha256sum "$ASSET" >> subjects.sha256
cat subjects.sha256
# purl namespace/name are canonically lowercase; derive them from the run
# context so the attestation follows a repository transfer (issue #1599).
echo "purl-repository=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
- name: Attest release image lock
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
with:
subject-checksums: subjects.sha256
predicate-type: https://in-toto.io/attestation/release/v0.1
predicate: |
{ "purl": "pkg:github/${{ steps.subjects.outputs.purl-repository }}@${{ needs.release.outputs.tag_name }}" }
- name: Sign release image lock
env:
ASSET: ${{ steps.pin.outputs.asset-path }}
run: |
set -euo pipefail
cosign sign-blob --yes --bundle "${ASSET}.sigstore.json" "$ASSET"
- name: Verify signature with the deploy-side identity (fail fast)
env:
ASSET: ${{ steps.pin.outputs.asset-path }}
run: |
set -euo pipefail
cosign verify-blob \
--bundle "${ASSET}.sigstore.json" \
--certificate-identity '${{ github.server_url }}/${{ github.repository }}/.github/workflows/release.yml@refs/heads/main' \
--certificate-oidc-issuer 'https://token.actions.githubusercontent.com' \
"$ASSET"
# First and only write to the release surface: every step above fails without creating one,
# so a rejected release leaves no draft to delete and no tag — a draft materialises none —
# and the version re-cuts unchanged from a commit that carries the fix.
- name: Create the draft release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG_NAME: ${{ needs.release.outputs.tag_name }}
VERSION: ${{ needs.release.outputs.version }}
SHA: ${{ needs.release.outputs.sha }}
MIGRATIONS: ${{ needs.release.outputs.migrations }}
run: |
set -euo pipefail
# The release job has already proved that any existing release is a draft at this commit.
if gh release view "$TAG_NAME" --repo "${{ github.repository }}" >/dev/null 2>&1; then
echo "Resuming the existing draft $TAG_NAME."
exit 0
fi
NOTES=$(mktemp)
if [ "$MIGRATIONS" = true ]; then
{
echo "> [!WARNING]"
echo "> This release contains **schema migrations**. They run automatically on startup — back up your database before upgrading. See the [migration guide](${{ github.server_url }}/${{ github.repository }}/blob/main/MIGRATION.md)."
echo ""
} >> "$NOTES"
fi
# CHANGELOG.md as checked out at the released commit; awk exits at the next section.
awk -v ver="## $VERSION" '
$0 == ver { on=1; next }
on && /^## / { exit }
on { print }
' CHANGELOG.md >> "$NOTES"
echo "----- release notes -----"; cat "$NOTES"; echo "-------------------------"
gh release create "$TAG_NAME" --draft \
--title "$TAG_NAME" \
--notes-file "$NOTES" \
--target "$SHA" \
--repo "${{ github.repository }}"
- name: Upload release image lock to GitHub Release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG_NAME: ${{ needs.release.outputs.tag_name }}
ASSET: ${{ steps.pin.outputs.asset-path }}
run: |
set -euo pipefail
# --clobber so a re-run over a resumed draft replaces its assets instead of failing on
# every name that already exists.
gh release upload "$TAG_NAME" \
"$ASSET" "${ASSET}.sigstore.json" evidence/* \
--clobber \
--repo "${{ github.repository }}"
upgrade-test:
needs: [release, tag-images]
if: needs.release.outputs.released == 'true'
uses: ./.github/workflows/release-upgrade.yml
permissions:
contents: read
packages: read
with:
# The previous release keeps the namespace it was published under (GHCR packages
# do not transfer between organizations — issue #1599), so the reusable workflow
# resolves its repository per version from security/release-identities.json.
previous-version: ${{ needs.release.outputs.previous_version }}
candidate-application-image: ghcr.io/hephaestus-build/application-server@${{ needs.tag-images.outputs.application-server-digest }}
postgres-image: ghcr.io/hephaestus-build/postgres@${{ needs.tag-images.outputs.postgres-digest }}
candidate-source-sha: ${{ needs.release.outputs.sha }}
supported-host-smoke:
needs: [release, tag-images]
if: needs.release.outputs.released == 'true'
name: Host smoke (${{ matrix.architecture }})
strategy:
fail-fast: false
matrix:
include:
- architecture: amd64
runner: ubuntu-24.04
- architecture: arm64
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
timeout-minutes: 20
permissions:
# The release is still a draft here, so its assets require authenticated push access.
contents: write
packages: read
steps:
- name: Check out the released tree
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.release.outputs.sha }}
persist-credentials: false
- uses: ./.github/actions/setup-toolchain
with:
install: "none"
- name: Install Cosign
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Log in to GitHub Container Registry
uses: ./.github/actions/ghcr-login
with:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Smoke-test the blessed install
env:
ARCHITECTURE: ${{ matrix.architecture }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
RELEASE_SHA: ${{ needs.release.outputs.sha }}
TAG_NAME: ${{ needs.release.outputs.tag_name }}
run: |
set -euo pipefail
cd docker/self-host
cleanup() {
status=$?
if [ "$status" -ne 0 ]; then
docker compose --env-file .env --env-file release-lock.env logs --no-color || true
fi
docker compose --env-file .env --env-file release-lock.env down -v --remove-orphans || true
exit "$status"
}
trap cleanup EXIT
actual_arch=$(docker version --format '{{.Server.Arch}}')
[ "$actual_arch" = "$ARCHITECTURE" ] || { echo "::error::Expected native $ARCHITECTURE, got $actual_arch"; exit 1; }
. /etc/os-release
[ "$ID" = ubuntu ] && [ "$VERSION_ID" = 24.04 ] || { echo "::error::Ubuntu 24.04 is required"; exit 1; }
docker_major=$(docker version --format '{{.Server.Version}}' | cut -d. -f1)
[ "$docker_major" -ge 28 ] || { echo "::error::Docker Engine 28 or newer is required"; exit 1; }
compose_version=$(docker compose version --short)
[ "$(printf '%s\n' 2.24.4 "$compose_version" | sort -V | head -n1)" = 2.24.4 ] || {
echo "::error::Docker Compose 2.24.4 or newer is required"; exit 1; }
node ../../scripts/prepare-host-smoke-env.ts
node ../../scripts/prepare-release-lock.ts "$TAG_NAME"
docker compose --env-file .env --env-file release-lock.env config --quiet
docker compose --env-file .env --env-file release-lock.env up -d --wait --wait-timeout 600
# Traefik routes by the hostname the installer answered with: SMOKE_HOSTNAME in
# scripts/prepare-host-smoke-env.ts, which release-deployment-policy.test.ts holds to this.
curl --fail --insecure --silent --show-error --max-time 10 --noproxy '*' \
--resolve hephaestus-smoke.invalid:443:127.0.0.1 \
https://hephaestus-smoke.invalid/ >/dev/null
jq -n \
--arg schemaVersion "1" \
--arg release "$TAG_NAME" \
--arg commit "$RELEASE_SHA" \
--arg testedAt "$(date -u +%FT%TZ)" \
--arg os "$(. /etc/os-release; printf '%s %s' "$NAME" "$VERSION_ID")" \
--arg architecture "$actual_arch" \
--arg dockerEngine "$(docker version --format '{{.Server.Version}}')" \
--arg dockerCompose "$compose_version" \
--argjson services "$(docker compose --env-file .env --env-file release-lock.env ps --all --format json | jq -s 'map({service: .Service, state: .State, health: .Health, exitCode: .ExitCode}) | sort_by(.service)')" \
'{schemaVersion: ($schemaVersion | tonumber), release: $release, commit: $commit,
testedAt: $testedAt, host: {os: $os, architecture: $architecture,
dockerEngine: $dockerEngine, dockerCompose: $dockerCompose}, services: $services}' \
> "../../host-smoke-$ARCHITECTURE.json"
- name: Upload smoke-test record
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: host-smoke-${{ matrix.architecture }}
path: host-smoke-${{ matrix.architecture }}.json
if-no-files-found: error
publish-release:
needs: [release, tag-images, upgrade-test, supported-host-smoke]
if: needs.release.outputs.released == 'true'
timeout-minutes: 20
runs-on: ubuntu-latest
permissions:
contents: write
packages: write
attestations: read
steps:
- name: Checkout release verifier
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.release.outputs.sha }}
fetch-depth: 1
persist-credentials: false
- uses: ./.github/actions/setup-toolchain
with:
install: "none"
- name: Install Cosign
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Log in to GitHub Container Registry
uses: ./.github/actions/ghcr-login
with:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download supported-host smoke records
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: host-smoke-*
path: host-smoke
merge-multiple: true
- name: Verify supported-host smoke records
env:
RELEASE_SHA: ${{ needs.release.outputs.sha }}
TAG_NAME: ${{ needs.release.outputs.tag_name }}
run: |
set -euo pipefail
jq -s -e --arg release "$TAG_NAME" --arg commit "$RELEASE_SHA" '
length == 2 and
([.[].host.architecture] | sort) == ["amd64", "arm64"] and
all(.[];
.schemaVersion == 1 and .release == $release and .commit == $commit and
(.testedAt | fromdateiso8601) and
.host.os == "Ubuntu 24.04" and
(.host.dockerEngine | type) == "string" and
(.host.dockerCompose | type) == "string" and
(.services | length) > 0 and
([.services[].service] | length) == ([.services[].service] | unique | length) and
all(.services[];
if .state == "exited" then .exitCode == 0
else .state == "running" and .health == "healthy" end))' host-smoke/*.json >/dev/null
- name: Verify durable assets from a clean environment
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG_NAME: ${{ needs.release.outputs.tag_name }}
run: |
set -euo pipefail
mkdir evidence
gh release download "$TAG_NAME" --repo "${{ github.repository }}" --dir evidence
(cd evidence && sha256sum -c SHA256SUMS)
lock="release-${TAG_NAME}.json"
cosign verify-blob \
--bundle "evidence/$lock.sigstore.json" \
--certificate-identity '${{ github.server_url }}/${{ github.repository }}/.github/workflows/release.yml@refs/heads/main' \
--certificate-oidc-issuer 'https://token.actions.githubusercontent.com' \
"evidence/$lock"
node scripts/release-image-lock.ts "evidence/$lock" evidence/manifest.json \
"$TAG_NAME" /tmp/release-lock.env
node scripts/verify-release-evidence.ts evidence --verify-signatures
asset="evidence/release-${TAG_NAME}.json"
cosign verify-blob --bundle "${asset}.sigstore.json" \
--certificate-identity '${{ github.server_url }}/${{ github.repository }}/.github/workflows/release.yml@refs/heads/main' \
--certificate-oidc-issuer 'https://token.actions.githubusercontent.com' "$asset" >/dev/null
- name: Publish immutable release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG_NAME: ${{ needs.release.outputs.tag_name }}
run: |
set -euo pipefail
gh release upload "$TAG_NAME" host-smoke/*.json --clobber --repo "${{ github.repository }}"
gh release edit "$TAG_NAME" --repo "${{ github.repository }}" --draft=false
[ "$(gh release view "$TAG_NAME" --repo "${{ github.repository }}" --json isImmutable --jq .isImmutable)" = true ] || {
echo "::error::The published release is not immutable; refusing to promote image aliases"; exit 1; }
- name: Promote verified image digests
env:
VERSION: ${{ needs.release.outputs.version }}
SERIES: ${{ needs.release.outputs.major }}.${{ needs.release.outputs.minor }}
run: |
set -euo pipefail
jq -er '[.subjects[] | select(.provenance == "first-party") | [.repository, .indexDigest]] | unique[] | @tsv' \
evidence/manifest.json > "$RUNNER_TEMP/promotions.tsv"
# Resolve every immutable source before changing aliases. OCI registries update one tag
# at a time, so retries and final verification recover any interrupted promotion.
while IFS=$'\t' read -r ref digest; do
source=$(docker buildx imagetools inspect "$ref@$digest" --format '{{json .Manifest}}' | jq -r .digest)
[ "$source" = "$digest" ] || { echo "::error::Cannot resolve promotion source $ref@$digest"; exit 1; }
done < "$RUNNER_TEMP/promotions.tsv"
while IFS=$'\t' read -r ref digest; do
docker buildx imagetools create \
-t "$ref:$VERSION" -t "$ref:$SERIES" -t "$ref:latest" "$ref@$digest"
done < "$RUNNER_TEMP/promotions.tsv"
while IFS=$'\t' read -r ref digest; do
for tag in "$VERSION" "$SERIES" latest; do
promoted=$(docker buildx imagetools inspect "$ref:$tag" --format '{{json .Manifest}}' | jq -r .digest)
[ "$promoted" = "$digest" ] || { echo "::error::Promotion changed $ref:$tag digest"; exit 1; }
done
done < "$RUNNER_TEMP/promotions.tsv"
# Staging is not promoted here — it follows the default branch, and a release re-tags that commit's
# images rather than rebuilding them, so the digests production is offered are the digests staging
# runs. But following is not the same as having arrived: this release and that promotion start
# from the same finished build, so production waits until staging is actually on this commit.
awaiting-staging:
needs: [release, tag-images, publish-release]
if: needs.release.outputs.released == 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
steps:
- name: Wait for staging to be running this commit
env:
GH_TOKEN: ${{ github.token }}
COMMIT: ${{ needs.release.outputs.sha }}
run: |
set -euo pipefail
deadline=$(( SECONDS + 1500 ))
while [ "$SECONDS" -lt "$deadline" ]; do
channel=$(gh api "repos/$GITHUB_REPOSITORY/contents/channels/staging.json?ref=deploy-state" \
--jq '.content' 2>/dev/null | base64 -d 2>/dev/null || echo '{}')
[ "$(jq -r '.commit // empty' <<< "$channel")" = "$COMMIT" ] && {
echo "staging is on $COMMIT"; exit 0; }
[ "$(jq -r '.freeze // false' <<< "$channel")" = true ] && {
echo "::error::staging is frozen, so this release has not been rehearsed there"; exit 1; }
sleep 20
done
echo "::error::staging did not reach ${COMMIT} — production is not offered an unrehearsed release"
exit 1
deploy-production:
needs: [release, awaiting-staging]
if: needs.release.outputs.released == 'true'
uses: ./.github/workflows/deploy-prod.yml
permissions:
contents: read
with:
image-tag: ${{ needs.release.outputs.tag_name }}
# Deploy credentials include organization secrets, which reach nested reusable
# workflows only through inheritance — environment secrets alone do not cover them.
secrets: inherit # zizmor: ignore[secrets-inherit]