Skip to content

Commit 33da9a6

Browse files
jonn-smithjonn-smith
committed
ci(docker): honor per-image PLATFORMS for build-per-arch matrix + merge
The build-per-arch matrix was a hardcoded cross-product of every changed image with {amd64, arm64}, and merge-and-scan hardcoded both `${VERSION}-amd64` and `${VERSION}-arm64` source refs into `imagetools create`. That forced amd64-only images (foldseek-gpu, whose CUDA Debian repo has no arm64 path) to schedule an arm64 build that fails at apt-get install, and would have failed merge even if the arm64 build were skipped. Each image's Makefile already declares `PLATFORMS = ...` (AGENTS.md section "Per-image Makefile required variables"). ci-detect-changed- images.sh now reads PLATFORMS from each changed image's Makefile and emits a new `build_matrix` output: a flat list of `{image, arch, runner, platform}` records, default both arches when PLATFORMS is absent. build-per-arch uses `matrix.include` from that record list, so each (image, arch) job is scheduled iff the image declares that arch. merge-and-scan reads the same PLATFORMS line to build the source-ref list dynamically — single-arch images produce a manifest list with one entry, multi-arch images preserve current behavior bit-for-bit. Other consumers unchanged: - bump-versions iterates image names only - pr-build-scan / rescan-latest iterate image names; both pull/build amd64 only and resolve the multi-arch tag on an amd64 runner - cd.yml release notes iterate Makefile VERSION only (no arch refs) - scripts/mirror-ghcr-to-gar.sh copies the merged manifest list, single-arch lists transit identically foldseek-gpu/Makefile already has `PLATFORMS = linux/amd64` (the four other images already have both arches). No Makefile edits needed.
1 parent c3bc0d9 commit 33da9a6

2 files changed

Lines changed: 112 additions & 25 deletions

File tree

‎.github/workflows/docker.yml‎

Lines changed: 52 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,13 @@ jobs:
6262
outputs:
6363
images: ${{ steps.detect.outputs.images }}
6464
any: ${{ steps.detect.outputs.any }}
65+
# Per-(image, arch) build records derived from each image's Makefile
66+
# PLATFORMS line; default both arches when PLATFORMS is absent. Drives
67+
# the build-per-arch matrix below — replaces the old unconditional
68+
# `{image} x {amd64, arm64}` cross-product so amd64-only images
69+
# (e.g. foldseek-gpu, whose CUDA repo has no arm64 path) don't spawn
70+
# an arm64 build that's guaranteed to fail.
71+
build_matrix: ${{ steps.detect.outputs.build_matrix }}
6572
steps:
6673
- name: Checkout
6774
uses: actions/checkout@v4
@@ -129,18 +136,18 @@ jobs:
129136
&& github.event_name != 'schedule'
130137
&& github.event_name != 'workflow_dispatch'
131138
&& github.event_name != 'pull_request'
132-
runs-on: ${{ matrix.arch.runner }}
139+
runs-on: ${{ matrix.runner }}
133140
permissions:
134141
contents: read
135142
packages: write
136143
actions: read
137144
strategy:
138145
fail-fast: false
146+
# `include`-only matrix: each entry is one (image, arch) job. Driven by
147+
# detect-changes which honors each image's Makefile PLATFORMS line, so
148+
# amd64-only images never schedule an arm64 build (and vice versa).
139149
matrix:
140-
image: ${{ fromJSON(needs.detect-changes.outputs.images) }}
141-
arch:
142-
- { name: amd64, runner: ubuntu-latest, platform: linux/amd64 }
143-
- { name: arm64, runner: ubuntu-24.04-arm, platform: linux/arm64 }
150+
include: ${{ fromJSON(needs.detect-changes.outputs.build_matrix) }}
144151
steps:
145152
- name: Checkout (pick up auto-bump if any)
146153
uses: actions/checkout@v4
@@ -188,18 +195,18 @@ jobs:
188195
id: refs
189196
run: |
190197
owner_repo_lc="$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')"
191-
echo "intermediate=${{ env.GHCR_REGISTRY }}/${owner_repo_lc}/${{ matrix.image }}:${{ steps.ver.outputs.version }}-${{ matrix.arch.name }}" >> "$GITHUB_OUTPUT"
198+
echo "intermediate=${{ env.GHCR_REGISTRY }}/${owner_repo_lc}/${{ matrix.image }}:${{ steps.ver.outputs.version }}-${{ matrix.arch }}" >> "$GITHUB_OUTPUT"
192199
193200
- name: Build and push intermediate per-arch tag
194201
uses: docker/build-push-action@v6
195202
with:
196203
context: docker/${{ matrix.image }}
197-
platforms: ${{ matrix.arch.platform }}
204+
platforms: ${{ matrix.platform }}
198205
tags: ${{ steps.refs.outputs.intermediate }}
199206
# Cache scoped per image AND arch so amd64 and arm64 don't clobber
200207
# each other in the GHA cache backend.
201-
cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.arch.name }}
202-
cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.arch.name }}
208+
cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.arch }}
209+
cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.arch }}
203210
# Force Docker v2.2 mediatypes on the per-arch manifests so the merge
204211
# step in 3b produces a Docker manifest list (not an OCI index).
205212
provenance: false
@@ -243,6 +250,17 @@ jobs:
243250
fi
244251
echo "version=$v" >> "$GITHUB_OUTPUT"
245252
253+
- name: Read PLATFORMS from Makefile
254+
id: plats
255+
run: |
256+
# Same source of truth as build-per-arch (ci-detect-changed-images.sh
257+
# reads this too). Default both arches when PLATFORMS line is absent
258+
# so existing images keep current multi-arch behavior.
259+
p=$(awk -F'=' '/^[[:space:]]*PLATFORMS[[:space:]]*=/ {gsub(/[[:space:]]/,"",$2); print $2; exit}' \
260+
docker/${{ matrix.image }}/Makefile)
261+
[ -z "$p" ] && p="linux/amd64,linux/arm64"
262+
echo "list=$p" >> "$GITHUB_OUTPUT"
263+
246264
- name: Detect GCR mirror availability
247265
id: gcr
248266
env:
@@ -351,11 +369,12 @@ jobs:
351369
352370
- name: Merge per-arch intermediates into multi-arch manifest list
353371
env:
354-
GHCR_TAGS: ${{ steps.meta_ghcr.outputs.tags }}
355-
GCR_TAGS: ${{ steps.meta_gcr.outputs.tags }}
356-
GAR_TAGS: ${{ steps.meta_gar.outputs.tags }}
357-
GHCR_IMAGE: ${{ steps.refs.outputs.ghcr_image }}
358-
VERSION: ${{ steps.ver.outputs.version }}
372+
GHCR_TAGS: ${{ steps.meta_ghcr.outputs.tags }}
373+
GCR_TAGS: ${{ steps.meta_gcr.outputs.tags }}
374+
GAR_TAGS: ${{ steps.meta_gar.outputs.tags }}
375+
GHCR_IMAGE: ${{ steps.refs.outputs.ghcr_image }}
376+
VERSION: ${{ steps.ver.outputs.version }}
377+
PLATFORMS_CSV: ${{ steps.plats.outputs.list }}
359378
run: |
360379
set -euo pipefail
361380
tag_args=()
@@ -372,14 +391,25 @@ jobs:
372391
[ -n "$t" ] && tag_args+=("-t" "$t")
373392
done <<< "$GAR_TAGS"
374393
fi
375-
# `imagetools create` preserves the source mediatype: since both
376-
# per-arch sources were pushed with oci-mediatypes=false (Docker v2),
377-
# the resulting manifest list is also Docker v2.2 — as required by
378-
# downstream consumers that reject OCI mediatypes.
379-
docker buildx imagetools create \
380-
"${tag_args[@]}" \
381-
"${GHCR_IMAGE}:${VERSION}-amd64" \
382-
"${GHCR_IMAGE}:${VERSION}-arm64"
394+
# Build the source-ref list from PLATFORMS so amd64-only images
395+
# (e.g. foldseek-gpu) produce a single-arch manifest list and don't
396+
# try to merge a non-existent arm64 intermediate. Multi-arch images
397+
# pick up both refs — behavior identical to the old hardcoded form.
398+
sources=()
399+
IFS=',' read -ra _plats <<< "$PLATFORMS_CSV"
400+
for p in "${_plats[@]}"; do
401+
case "$p" in
402+
linux/amd64) sources+=("${GHCR_IMAGE}:${VERSION}-amd64") ;;
403+
linux/arm64) sources+=("${GHCR_IMAGE}:${VERSION}-arm64") ;;
404+
*) echo "::error::unknown platform '$p' in docker/${{ matrix.image }}/Makefile"; exit 1 ;;
405+
esac
406+
done
407+
# `imagetools create` preserves the source mediatype: per-arch sources
408+
# were pushed with oci-mediatypes=false (Docker v2), so the resulting
409+
# manifest list is also Docker v2.2 — as required by downstream
410+
# consumers that reject OCI mediatypes. Works for single-arch too:
411+
# the result is a manifest list containing one platform entry.
412+
docker buildx imagetools create "${tag_args[@]}" "${sources[@]}"
383413
384414
- name: Pull merged image for Trivy scan
385415
run: docker pull ${{ steps.refs.outputs.ghcr_image }}:${{ steps.ver.outputs.version }}

‎scripts/ci-detect-changed-images.sh‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
#!/usr/bin/env bash
22
# Detect which docker/<image>/ directories changed between a base ref and HEAD.
33
# Emits to $GITHUB_OUTPUT:
4-
# images=<json-array> e.g. ["hvp-monolith","foo"]
4+
# images=<json-array> e.g. ["hvp-monolith","foo"]
55
# any=true|false
6+
# build_matrix=<json-array> per-(image,arch) build records, e.g.
7+
# [{"image":"foo","arch":"amd64","runner":"ubuntu-latest","platform":"linux/amd64"},
8+
# {"image":"foo","arch":"arm64","runner":"ubuntu-24.04-arm","platform":"linux/arm64"}]
9+
# Drives the build-per-arch matrix in docker.yml. Per-image platform list
10+
# is read from `PLATFORMS = ...` in each image's Makefile; default both
11+
# arches if absent or unparseable.
612
#
713
# Driven by env vars set by the workflow:
814
# GITHUB_EVENT_NAME : push | pull_request | schedule | workflow_dispatch
@@ -52,17 +58,66 @@ resolve_base() {
5258
esac
5359
}
5460

61+
# Map a `linux/<arch>` platform string to (arch-name, GH-Actions runner).
62+
# Stdout: "<arch> <runner>"; empty if unknown.
63+
platform_runner() {
64+
case "$1" in
65+
linux/amd64) echo "amd64 ubuntu-latest" ;;
66+
linux/arm64) echo "arm64 ubuntu-24.04-arm" ;;
67+
*) echo "" ;;
68+
esac
69+
}
70+
71+
# Build the per-(image, arch) matrix JSON from each image's Makefile
72+
# PLATFORMS line. Default both arches when the line is absent / unparseable
73+
# so existing images keep current behavior with no Makefile edit.
74+
build_matrix_json() {
75+
local _images_ref="$1"
76+
eval "local arr=(\"\${${_images_ref}[@]}\")"
77+
local json="["
78+
local first=1
79+
local img mk plats p map arch runner
80+
for img in "${arr[@]}"; do
81+
[[ -n "$img" ]] || continue
82+
mk="docker/$img/Makefile"
83+
if [[ -f "$mk" ]]; then
84+
plats=$(awk -F'=' '/^[[:space:]]*PLATFORMS[[:space:]]*=/ {gsub(/[[:space:]]/,"",$2); print $2; exit}' "$mk")
85+
else
86+
plats=""
87+
fi
88+
[[ -z "$plats" ]] && plats="linux/amd64,linux/arm64"
89+
IFS=',' read -ra pa <<<"$plats"
90+
for p in "${pa[@]}"; do
91+
map="$(platform_runner "$p")"
92+
if [[ -z "$map" ]]; then
93+
echo "warning: $img PLATFORMS entry '$p' is not linux/amd64 or linux/arm64; skipping" >&2
94+
continue
95+
fi
96+
arch="${map% *}"
97+
runner="${map#* }"
98+
(( first )) || json+=","
99+
json+="{\"image\":\"$img\",\"arch\":\"$arch\",\"runner\":\"$runner\",\"platform\":\"$p\"}"
100+
first=0
101+
done
102+
done
103+
json+="]"
104+
printf '%s' "$json"
105+
}
106+
55107
emit() {
56108
local any="$1"
57109
local images_json="$2"
110+
local matrix_json="$3"
58111
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
59112
{
60113
echo "any=$any"
61114
echo "images=$images_json"
115+
echo "build_matrix=$matrix_json"
62116
} >> "$GITHUB_OUTPUT"
63117
fi
64118
echo "any=$any"
65119
echo "images=$images_json"
120+
echo "build_matrix=$matrix_json"
66121
}
67122

68123
# Schedule / dispatch / initial-commit push: scan every image dir.
@@ -84,7 +139,7 @@ else
84139
fi
85140

86141
if (( ${#images[@]} == 0 )); then
87-
emit "false" "[]"
142+
emit "false" "[]" "[]"
88143
exit 0
89144
fi
90145

@@ -96,4 +151,6 @@ for i in "${!images[@]}"; do
96151
done
97152
json+="]"
98153

99-
emit "true" "$json"
154+
matrix="$(build_matrix_json images)"
155+
156+
emit "true" "$json" "$matrix"

0 commit comments

Comments
 (0)