Smoke-boot published image #330
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # Layer B — smoke-boot the published Margine image in QEMU and verify | |
| # it actually reaches multi-user.target before we trust :stable. | |
| # | |
| # Layer A (in build.yml) checks image internals statically: required | |
| # files, paths, labels, modules. Useful but insufficient — every bug | |
| # we hit during the 2026-05-28/29 smoke test was a RUNTIME bug that | |
| # Layer A could not see. Layer B closes that gap. | |
| # | |
| # Triggers: workflow_dispatch (manual rebuild any time) + | |
| # workflow_run on every "Build Margine image" success on main (auto- | |
| # gate before :stable promotion). Promotion to :stable happens only | |
| # if QEMU reaches multi-user.target — see "Promote candidate → | |
| # stable (only if boot passed)" at the bottom of this file. | |
| # | |
| # What it does: | |
| # 1. Install bootc-image-builder + qemu + ovmf | |
| # 2. Pull the latest published Margine OCI image | |
| # 3. Build a qcow2 from it (no LUKS — automated boot can't enter | |
| # a passphrase; LUKS boot is exercised manually in the VM lab) | |
| # 4. Boot the qcow2 in QEMU with serial console | |
| # 5. Wait up to 30 min (loop below) for a usable-boot marker | |
| # 6. If reached → pass; if not → upload serial log as artifact, fail | |
| # | |
| # Notes: | |
| # - GHA ubuntu-24.04 runners DO have KVM acceleration since 2024. | |
| # Boot is fast (typically ~30s to multi-user). | |
| # - bootc-image-builder runs as root, in a privileged container. | |
| # - The qcow2 stays local to the runner; we don't publish it. The | |
| # publication target is the OCI image (already pushed by build.yml). | |
| name: Smoke-boot published image | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| image_ref: | |
| description: 'OCI ref to smoke-boot (default: :candidate)' | |
| required: false | |
| default: 'ghcr.io/daniel-g-carrasco/margine:candidate' | |
| # Auto-trigger after every successful Build Margine image on main. | |
| # The guard `if: github.event.workflow_run.conclusion == 'success'` | |
| # on the job ensures we don't waste a runner on cancelled / failed | |
| # build outputs (which never get pushed to :stable anyway). | |
| workflow_run: | |
| workflows: ["Build Margine image"] | |
| types: [completed] | |
| branches: [main] | |
| # This is the only workflow that mutates :stable. Two builds finishing | |
| # close together must not race their promotions — queue, never cancel | |
| # (a cancelled run mid-`skopeo copy` could leave the three stable tags | |
| # pointing at two different digests). | |
| # Least privilege (Scorecard Token-Permissions): nothing at the workflow | |
| # level; the smoke_boot job declares its own contents:read+packages:write. | |
| permissions: {} | |
| concurrency: | |
| group: smoke-boot | |
| cancel-in-progress: false | |
| jobs: | |
| smoke_boot: | |
| name: Build qcow2 + boot in QEMU | |
| # Skip when triggered by workflow_run if the upstream build wasn't | |
| # a clean success. workflow_dispatch always passes this check. | |
| if: >- | |
| github.event_name != 'workflow_run' | |
| || github.event.workflow_run.conclusion == 'success' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 # bib ~5min + first-boot ~12-15min + slack for a slow apt mirror | |
| # (40 -> 60 on 2026-06-17: a 37-min transient apt hang in "Install QEMU" | |
| # ate the whole 40-min budget before the boot ran — run 27719837482.) | |
| permissions: | |
| contents: read | |
| packages: write # needed for the candidate→stable promotion step | |
| steps: | |
| # The Layer C injection + boot-watch logic lives in versioned, | |
| # shellcheck-gated scripts (extracted from inline heredocs | |
| # 2026-06-12 — the ordering-cycle bug that silently skipped the | |
| # first probe run was exactly the kind of thing nothing could | |
| # lint while the payloads were YAML strings). | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Maximize build space | |
| # ubuntu-24.04 ships with ~14 GiB free, but BIB needs to | |
| # extract the 14 GB Margine bootc image into /sysroot/ostree/ | |
| # repo and then build a qcow2 from it — easily 25 GB peak. | |
| # Without this step we hit "no space left on device" mid- | |
| # extraction (build #26919583711, 2026-06-04). Same action | |
| # that build-disk.yml uses for the same reason. | |
| uses: ublue-os/remove-unwanted-software@cc0becac701cf642c8f0a6613bbdaf5dc36b259e # v9 | |
| with: | |
| remove-codeql: true | |
| - name: Install QEMU + OVMF + tools | |
| run: | | |
| # Retry + per-attempt timeout so a transient slow/hung apt mirror | |
| # can't silently eat the job budget. A 37-min apt-get install hang | |
| # (run 27719837482, 2026-06-17) cancelled the smoke at the 40-min | |
| # timeout before the boot even started. timeout kills a hung attempt | |
| # at 10min; the retry usually lands on a recovered/different mirror. | |
| apt_retry() { | |
| for i in 1 2 3; do | |
| sudo timeout 600 apt-get "$@" && return 0 | |
| echo "::warning::apt-get $* failed/timed out (attempt $i); retrying in $((i * 15))s" | |
| sleep $((i * 15)) | |
| done | |
| return 1 | |
| } | |
| apt_retry update -qq | |
| # qemu-utils = qemu-nbd (NOT pulled by qemu-system-x86), parted = | |
| # partprobe, util-linux = partx — all needed by the Layer C | |
| # inject-gui-probe nbd partition exposure (2026-06-13 hardening). | |
| apt_retry install -y -qq qemu-system-x86 qemu-utils ovmf swtpm parted util-linux | |
| # Verify KVM | |
| ls -la /dev/kvm || true | |
| kvm-ok 2>&1 || true | |
| # Resolve the ref under test to an immutable digest ONCE, and use | |
| # that digest for both the boot test and the promotion. Until | |
| # 2026-06-12 the boot step fell back to :stable while the promote | |
| # step fell back to :candidate — on every workflow_run trigger | |
| # (inputs empty) the gate booted the PREVIOUS stable and then | |
| # promoted the never-tested candidate. Found by the code-quality | |
| # review (finding A1); the tag was also re-resolved once per | |
| # promoted tag, a race window when a build finishes mid-smoke (A2). | |
| - name: Resolve image ref to digest | |
| id: ref | |
| run: | | |
| set -euo pipefail | |
| REF="${{ github.event.inputs.image_ref || 'ghcr.io/daniel-g-carrasco/margine:candidate' }}" | |
| if [[ "$REF" == *"@sha256:"* ]]; then | |
| # Dispatch input was already a digest ref. | |
| BASE="${REF%@*}" | |
| DIGEST="${REF#*@}" | |
| else | |
| BASE="${REF%:*}" | |
| DIGEST="$(sudo skopeo inspect --no-tags --format '{{.Digest}}' "docker://$REF")" | |
| fi | |
| echo "Testing $REF @ $DIGEST" | |
| { | |
| echo "pinned=${BASE}@${DIGEST}" | |
| echo "registry_image=${BASE}" | |
| echo "digest=${DIGEST}" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Build qcow2 with bootc-image-builder | |
| run: | | |
| set -euo pipefail | |
| mkdir -p output | |
| IMG="${{ steps.ref.outputs.pinned }}" | |
| echo "Building qcow2 from: $IMG" | |
| # Pull the image first so bootc-image-builder doesn't have to | |
| # re-download under root. | |
| sudo podman pull "$IMG" | |
| # bootc-image-builder needs --privileged + a few specific | |
| # mounts. Defaults produce qcow2 with no LUKS. | |
| # BIB pinned to the same digest build-disk.yml uses — this | |
| # workflow floated on :latest --pull=newer until 2026-06-12. | |
| sudo podman run --rm \ | |
| --privileged \ | |
| --security-opt label=type:unconfined_t \ | |
| -v "$(pwd)/output:/output" \ | |
| -v /var/lib/containers/storage:/var/lib/containers/storage \ | |
| quay.io/centos-bootc/bootc-image-builder@sha256:7ae88b8d6f2cabfa971d7836b96d6cac19cd1384e658031bd154f9687e929905 \ | |
| --type qcow2 \ | |
| --rootfs btrfs \ | |
| "$IMG" | |
| ls -la output/qcow2/ | |
| sudo chown "$(id -u):$(id -g)" -R output/ | |
| # Layer C (2026-06-12): offline-inject a GUI smoke probe into the | |
| # qcow2 — GDM autologin user + a oneshot that, once the session | |
| # has settled, verifies gnome-shell is alive, the distro | |
| # extensions actually loaded, and nothing dumped core, then | |
| # prints MARGINE-GUI-SMOKE: PASS/FAIL to the serial console and | |
| # powers off. Motivation: a crashing gnome-shell extension passed | |
| # Layer B (multi-user is reached regardless) and shipped. | |
| # WARN-ONLY for now — flip to gating after two green runs. | |
| # Injection failures must never block the Layer B gate: every | |
| # step is guarded and ends with a ::warning:: at worst. | |
| - name: Inject GUI smoke probe (Layer C) | |
| id: inject | |
| continue-on-error: true | |
| # Injection failures must never block the Layer B gate: the | |
| # script exits 1 with a ::warning:: at worst. Probe payloads | |
| # live in .github/smoke/, the nbd/mount dance in the script. | |
| run: | | |
| set -euo pipefail | |
| QCOW=$(ls output/qcow2/*.qcow2 | head -1) | |
| sudo .github/scripts/inject-gui-probe.sh "$QCOW" | |
| - name: Boot qcow2 in QEMU and wait for a usable state | |
| id: boot | |
| # Shared boot-watcher (same script gates the live ISO in | |
| # build-disk.yml). Emits passed= and gui= to GITHUB_OUTPUT; | |
| # Layer C verdict stays WARN-ONLY until two green runs. | |
| run: | | |
| set -euo pipefail | |
| QCOW=$(ls output/qcow2/*.qcow2 | head -1) | |
| echo "Booting: $QCOW" | |
| sudo -E .github/scripts/qemu-boot-wait.sh \ | |
| --disk "$QCOW" \ | |
| --log serial.log \ | |
| --timeout 1800 \ | |
| --gui-watch | |
| # User-smoke SOFT gate (2026-06-14): summarize the warn-only | |
| # MARGINE-USER-SMOKE identity markers the injected probe wrote to | |
| # serial.log. ANNOTATE ONLY — never fails the job, never gates | |
| # promotion (which keys solely on steps.boot.outputs.passed). Three | |
| # guards keep it non-blocking: if:always() + continue-on-error + | |
| # the trailing || true. No actions used → nothing to SHA-pin. | |
| - name: User-smoke soft gate (warn-only summary) | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| { | |
| echo "### User-smoke (soft gate — annotation only, promotion NOT blocked)" | |
| if [[ ! -f serial.log ]]; then | |
| echo "- no serial.log (boot step did not run)"; exit 0 | |
| fi | |
| if ! grep -q 'MARGINE-USER-SMOKE:' serial.log; then | |
| echo "- :grey_question: no MARGINE-USER-SMOKE markers — probe injection skipped or unit did not run (see inject step + serial-log artifact)" | |
| echo "::warning::User-smoke soft gate produced no verdict (probe not injected or did not run) — see serial log." | |
| exit 0 | |
| fi | |
| echo "" | |
| echo "| Check | Result | Detail |" | |
| echo "|-------|--------|--------|" | |
| grep -E 'MARGINE-USER-SMOKE: [A-Z_]+ (PASS|WARN)' serial.log \ | |
| | sed -E 's/.*MARGINE-USER-SMOKE: ([A-Z_]+) (PASS|WARN) ?(.*)/\1|\2|\3/' \ | |
| | awk -F'|' '{ icon=($2=="PASS")?":white_check_mark:":":warning:"; gsub(/\|/,"\\|",$3); print "| `"$1"` | "icon" "$2" | "$3" |" }' | |
| SUMMARY="$(grep -oE 'MARGINE-USER-SMOKE: SUMMARY .*' serial.log | tail -1)" | |
| VERDICT="$(grep -oE 'MARGINE-USER-SMOKE: VERDICT .*' serial.log | tail -1)" | |
| echo "" | |
| echo "- ${SUMMARY:-SUMMARY: n/a}" | |
| echo "- ${VERDICT:-VERDICT: n/a}" | |
| if grep -qE 'MARGINE-USER-SMOKE: [A-Z_]+ WARN' serial.log; then | |
| echo "::warning::User-smoke soft gate: one or more identity checks WARNed (kernel/o-tiling/keybindings/search-light/gaming-ujust/gschema/gdm). Soft gate — promotion NOT blocked. See the step summary + serial-log artifact." | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" || true | |
| - name: Upload serial log | |
| # Always: the Layer C warn-only verdict is judged from this log. | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: smoke-boot-serial-log | |
| path: serial.log | |
| retention-days: 7 | |
| - name: Notify ntfy (boot test outcome) | |
| if: always() && steps.boot.outcome != 'skipped' | |
| env: | |
| NTFY_URL: ${{ secrets.NTFY_TOPIC_URL }} | |
| run: | | |
| if [[ -z "${NTFY_URL:-}" ]]; then | |
| echo "NTFY_TOPIC_URL secret not set, skipping notification." | |
| exit 0 | |
| fi | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| if [[ "${{ steps.boot.outputs.passed }}" == "true" ]]; then | |
| TITLE="QEMU boot OK → promoting :stable" | |
| PRIO="default" | |
| TAGS="rocket,white_check_mark" | |
| GUI="${{ steps.boot.outputs.gui }}" | |
| case "$GUI" in | |
| pass) MSG="Candidate healthy. GUI probe: PASS. New :stable available." ;; | |
| fail) MSG="Boot OK but GUI probe FAILED (warn-only) — check the serial log before trusting this stable." ; PRIO="high"; TAGS="warning" ;; | |
| *) MSG="Candidate healthy. GUI probe: no verdict (${GUI:-n/a})." ;; | |
| esac | |
| GAMING="${{ steps.boot.outputs.gaming }}" | |
| case "$GAMING" in | |
| pass) MSG="${MSG} Gaming-native: resolves." ;; | |
| fail) MSG="${MSG} Gaming-native: WON'T resolve (warn-only) — ujust margine-gaming-native would break." ; PRIO="high"; TAGS="warning" ;; | |
| skip) : ;; | |
| *) MSG="${MSG} Gaming-native: no verdict." ;; | |
| esac | |
| else | |
| TITLE="QEMU boot FAILED — :stable NOT advanced" | |
| PRIO="high" | |
| TAGS="x,boom" | |
| MSG="Boot never reached multi-user.target. Investigate before any bootc upgrade." | |
| fi | |
| curl --silent --show-error --fail \ | |
| -H "Title: ${TITLE}" \ | |
| -H "Priority: ${PRIO}" \ | |
| -H "Tags: ${TAGS}" \ | |
| -H "Click: ${RUN_URL}" \ | |
| -d "$MSG" \ | |
| "$NTFY_URL" || echo "ntfy notify failed (non-fatal)" | |
| # Promotion step: only runs when the boot actually reached | |
| # multi-user.target (steps.boot.outputs.passed == 'true'). | |
| # Copies the EXACT digest resolved in the `ref` step — the same | |
| # bytes the QEMU boot exercised. No tag re-resolution: a build | |
| # finishing mid-smoke can no longer slip an untested digest into | |
| # :stable (review findings A1/A2, fixed 2026-06-12). | |
| - name: Promote candidate → stable (only if boot passed) | |
| if: success() && steps.boot.outputs.passed == 'true' | |
| env: | |
| GHCR_USER: ${{ github.actor }} | |
| GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PINNED: ${{ steps.ref.outputs.pinned }} | |
| REGISTRY_IMAGE: ${{ steps.ref.outputs.registry_image }} | |
| run: | | |
| set -euo pipefail | |
| DATE_TAG="$(date -u +%Y%m%d)" | |
| echo "$GHCR_TOKEN" | sudo skopeo login --username "$GHCR_USER" --password-stdin ghcr.io | |
| for promo_tag in stable "stable.${DATE_TAG}" "${DATE_TAG}"; do | |
| echo "Promoting ${PINNED} → ${REGISTRY_IMAGE}:${promo_tag}" | |
| sudo skopeo copy --retry-times 3 --preserve-digests \ | |
| "docker://${PINNED}" \ | |
| "docker://${REGISTRY_IMAGE}:${promo_tag}" | |
| done | |
| echo "Promoted ${PINNED} to :stable, :stable.${DATE_TAG}, :${DATE_TAG}" | |
| { | |
| echo "### Promotion" | |
| echo "- digest: \`${PINNED}\`" | |
| echo "- tags: \`stable\`, \`stable.${DATE_TAG}\`, \`${DATE_TAG}\`" | |
| echo "- Layer C GUI probe: \`${{ steps.boot.outputs.gui || 'n/a' }}\`" | |
| echo "- Gaming-native dry-run: \`${{ steps.boot.outputs.gaming || 'n/a' }}\`" | |
| } >> "$GITHUB_STEP_SUMMARY" |