kingpanther13
diff --git a/‎.github/workflows/build-haos-test-image.yml‎
Lines changed: 93 additions & 3 deletions b/‎.github/workflows/build-haos-test-image.yml‎
Lines changed: 93 additions & 3 deletions
diff --git a/‎.github/workflows/haos-e2e-inaddon-tests.yml‎
Lines changed: 34 additions & 10 deletions b/‎.github/workflows/haos-e2e-inaddon-tests.yml‎
Lines changed: 34 additions & 10 deletions
diff --git a/‎.github/workflows/haos-e2e-tests.yml‎
Lines changed: 27 additions & 0 deletions b/‎.github/workflows/haos-e2e-tests.yml‎
Lines changed: 27 additions & 0 deletions
diff --git a/‎tests/haos_image_build/build_image.py‎
Lines changed: 5 additions & 5 deletions b/‎tests/haos_image_build/build_image.py‎
Lines changed: 5 additions & 5 deletions
@@ -21,6 +21,23 @@ name: Build HAOS Test Image (Publish to GHCR)
 
 on:
   workflow_dispatch:
+    inputs:
+      tag_suffix:
+        description: >-
+          Moving-tag suffix (default: ``latest``). Override to publish
+          to ``:HAOS_VERSION-<suffix>`` instead of ``:HAOS_VERSION-latest``
+          — used by perf-iteration branches that need a non-latest tag
+          they can point the e2e test workflows at without touching the
+          master-served image. Pointing the e2e lanes at the override is
+          a SEPARATE manual edit on the branch: change the ``tag=`` line
+          in the ``Try pulling from GHCR`` step of both
+          ``haos-e2e-tests.yml`` and ``haos-e2e-inaddon-tests.yml``.
+          Allowed character set is ``[A-Za-z0-9._-]+`` (OCI tag rules);
+          the ``Push image to GHCR`` step validates and fails fast on
+          anything else.
+        type: string
+        required: false
+        default: 'latest'
   push:
     branches: [master]
     paths:
@@ -146,9 +163,68 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
+      - name: Compress qcow2 in-format (qemu-img convert -c)
+        # Re-pack the qcow2 with qcow2-native zlib *cluster* compression
+        # before upload. The post-bake file is ~12 GB *apparent* size on
+        # disk — sparse, with ~5 GB actually allocated; the rest is
+        # un-allocated holes from ``qemu-img resize 32G`` in
+        # ``fetch_haos_qcow2``. ORAS pushes/pulls
+        # ``application/octet-stream`` as a raw byte stream, so the
+        # sparse holes get serialised over the wire as actual zero bytes
+        # — the *apparent* size, not the allocated size, is what GHCR
+        # pull pays for.
+        #
+        # ``qemu-img convert -c -O qcow2`` re-encodes the data densely
+        # at the cluster level (default 64 KiB). The output stays a
+        # standard qcow2 (same OCI media-type, same consumer code path);
+        # QEMU decompresses clusters on-demand during VM I/O when the
+        # e2e workflow boots, so no upfront decompress step on the pull
+        # side. Trade-off worth knowing about: qcow2 cluster compression
+        # is read-only — the first write to any compressed cluster makes
+        # QEMU re-allocate it uncompressed in place. The e2e workflow's
+        # boot is read-heavy (HA starts, recorder DB rebuilds, addon
+        # containers come up), so write-side re-allocation cost is small
+        # relative to the pull-time saved, but it's why this convert
+        # pass lives at publish time and not at every consumer.
+        #
+        # Measured at publish run 26361263298 step 11:
+        #   12 GB → 5.1 GB (2.3x ratio), convert pass took 6m 15s.
+        # First e2e run that consumed the result (runs 26361656780 +
+        # 26361656754) saw GHCR pull drop from ~6 min (uncompressed
+        # baseline) to 47–58 s.
+        #
+        # Cost: the convert pass adds ~6 min to the publish runner —
+        # paid only on master push / weekly cron / manual dispatch,
+        # not on PR runs.
+        run: |
+          orig_bytes=$(stat -c%s haos-test-image.qcow2)
+          qemu-img convert -c -O qcow2 \
+            haos-test-image.qcow2 haos-test-image.qcow2.compressed
+          mv haos-test-image.qcow2.compressed haos-test-image.qcow2
+          # Integrity gate before consumers see the file: catch torn
+          # writes (e.g. ``qemu-img convert`` killed mid-run after the
+          # output file already exists, exit-0 from the shell wrapper
+          # but file is truncated) and broken refcount tables.
+          # ``qemu-img check`` reads metadata + L1/L2 tables only — it
+          # does not stream the full image — so this is fast.
+          qemu-img check haos-test-image.qcow2
+          new_bytes=$(stat -c%s haos-test-image.qcow2)
+          # Sanity floor: the baked HAOS image is gigabytes; anything
+          # under 1 GiB after compress is corruption ``qemu-img check``
+          # didn't catch. Fail before the moving tag points at bad bytes.
+          if [ "$new_bytes" -lt 1073741824 ]; then
+            echo "::error::compressed qcow2 suspiciously small: ${new_bytes} bytes"
+            exit 1
+          fi
+          ratio=$(awk -v o="$orig_bytes" -v n="$new_bytes" 'BEGIN{printf "%.1f", o/n}')
+          echo "qcow2 size before: $(numfmt --to=iec "$orig_bytes")"
+          echo "qcow2 size after : $(numfmt --to=iec "$new_bytes")"
+          echo "compression ratio: ${ratio}x"
+
       - name: Upload built image as workflow artifact
         # Always upload — even on PRs that don't push to GHCR — so reviewers
-        # can pull the artifact and verify locally.
+        # can pull the artifact and verify locally. (Now also smaller thanks
+        # to the in-format compression step above.)
         uses: actions/upload-artifact@v7
         with:
           name: haos-test-image-${{ steps.haos_version.outputs.version }}-${{ github.sha }}
@@ -166,13 +242,27 @@ jobs:
         env:
           HAOS_VERSION: ${{ steps.haos_version.outputs.version }}
           SHORT_SHA: ${{ github.sha }}
+          # See the ``tag_suffix`` workflow_dispatch input at the top of
+          # this file for the override semantics and required edits on
+          # the consumer side. The ``|| 'latest'`` fallback covers the
+          # ``push:`` / ``schedule:`` triggers where
+          # ``github.event.inputs`` is null.
+          TAG_SUFFIX: ${{ github.event.inputs.tag_suffix || 'latest' }}
         run: |
+          # Validate the suffix against the OCI tag character set before
+          # building any tag strings — a typo like ``foo/bar``,
+          # ``:evil``, or a trailing space would otherwise surface as a
+          # confusing ``oras push`` error ten lines later.
+          if [[ ! "$TAG_SUFFIX" =~ ^[A-Za-z0-9._-]+$ ]]; then
+            echo "::error::TAG_SUFFIX must match [A-Za-z0-9._-]+ (OCI tag chars), got: '$TAG_SUFFIX'"
+            exit 1
+          fi
           short_sha="${SHORT_SHA:0:7}"
           versioned="${IMAGE_REPO}:${HAOS_VERSION}-${short_sha}"
-          moving="${IMAGE_REPO}:${HAOS_VERSION}-latest"
+          moving="${IMAGE_REPO}:${HAOS_VERSION}-${TAG_SUFFIX}"
           oras push "$versioned" \
             --artifact-type application/vnd.homeassistant-ai.haos-test-image.v1 \
             haos-test-image.qcow2:application/octet-stream
-          oras tag "$versioned" "${HAOS_VERSION}-latest"
+          oras tag "$versioned" "${HAOS_VERSION}-${TAG_SUFFIX}"
           echo "Pushed $versioned"
           echo "Pushed $moving"
@@ -142,6 +142,20 @@ jobs:
           (cd /tmp/haos-pull && /tmp/oras pull "$tag")
           mv /tmp/haos-pull/haos-test-image.qcow2 /tmp/haos-test-image.qcow2
 
+      - name: Free disk space for local build (cache miss + GHCR miss only)
+        # See ``haos-e2e-tests.yml`` for the full rationale — same prune
+        # set, same condition. PRs touching ``tests/haos_image_build/``
+        # force the local-build path on both lanes and would otherwise
+        # trip "No space left on device" mid-bake on the 14 GB SSD.
+        if: steps.restore-cache.outputs.cache-hit != 'true' && steps.ghcr-pull.outcome != 'success'
+        run: |
+          df -h /
+          sudo rm -rf /usr/share/dotnet /usr/share/swift /opt/ghc \
+            /usr/local/lib/android /usr/local/.ghcup
+          sudo apt-get clean
+          docker system prune -af --volumes || true
+          df -h /
+
       - name: Install build-script Python deps (cache miss + GHCR miss)
         if: steps.restore-cache.outputs.cache-hit != 'true' && steps.ghcr-pull.outcome != 'success'
         run: pip install -r tests/haos_image_build/requirements.txt
@@ -152,16 +166,26 @@ jobs:
           python3 tests/haos_image_build/build_image.py --verbose \
             --output /tmp/haos-test-image.qcow2
 
-      - name: Save image to cache (cache miss only)
-        # Save whenever the runtime cache missed — covers both the
-        # local-build path AND a successful GHCR pull. Without saving on
-        # the GHCR-served branch, the actions/cache entry stays empty
-        # forever and every future run pays the GHCR pull cost again.
-        if: steps.restore-cache.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v5
-        with:
-          path: /tmp/haos-test-image.qcow2
-          key: ${{ steps.key.outputs.cache-key }}
+      # No cache-save step here on purpose. The external lane
+      # (haos-e2e-tests.yml) computes the same cache key (#1407) and
+      # always tries to save the same qcow2; both lanes start in
+      # parallel on the same PR push, so whichever lost the save race
+      # was burning ~25s on a tar+zstd-write of a 12 GB qcow2 that
+      # the actions/cache service then refused with
+      # "another job may be creating this cache." Letting only the
+      # external lane save eliminates the wasted work without
+      # changing what gets cached — both lanes still restore from the
+      # shared key on the next run.
+      #
+      # The shared-key bet is safe because both lanes consume the SAME
+      # byte-identical qcow2: the cache key hashes the bake-input
+      # paths only, the qcow2 is built once at publish time and
+      # pulled identically by both lanes, and PR-level addon/source
+      # overrides happen *in-VM at test time* (see
+      # ``refresh_dev_addon_source_in_qcow2`` /
+      # ``refresh_recorder_in_qcow2`` in conftest.py) on each
+      # worker's per-worker overlay — never on the base qcow2 that
+      # gets cached.
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
 
@@ -157,6 +157,29 @@ jobs:
           # files into a subdirectory of pwd doesn't silently no-op the mv.
           mv /tmp/haos-pull/haos-test-image.qcow2 /tmp/haos-test-image.qcow2
 
+      - name: Free disk space for local build (cache miss + GHCR miss only)
+        # GitHub-hosted Linux runners ship with ~30 GB of pre-installed
+        # tooling (dotnet, swift, ghc, android sdk) that the HAOS bake
+        # doesn't use, and only ~14 GB of usable SSD. The local-build
+        # path's qcow2 download (~1.5 GB xz) + decompress (~7 GB qcow2)
+        # + boot+addon-install (qcow2 grows to ~12 GB apparent / ~5 GB
+        # allocated) + ``cp`` to ``/tmp/haos-test-image.qcow2`` (another
+        # ~5 GB if reflink fails) peaks at the edge of available disk.
+        # Without this prune, PRs that touch ``tests/haos_image_build/``
+        # (which forces the local-build path via the ``Detect PR-modified
+        # bake inputs`` gate above) trip "No space left on device"
+        # mid-bake. The directories below are well-known unused on
+        # standard runners; we deliberately keep ``/opt/hostedtoolcache``
+        # because Python from there is still on PATH for build_image.py.
+        if: steps.restore-cache.outputs.cache-hit != 'true' && steps.ghcr-pull.outcome != 'success'
+        run: |
+          df -h /
+          sudo rm -rf /usr/share/dotnet /usr/share/swift /opt/ghc \
+            /usr/local/lib/android /usr/local/.ghcup
+          sudo apt-get clean
+          docker system prune -af --volumes || true
+          df -h /
+
       - name: Install build-script Python deps (cache miss + GHCR miss)
         if: steps.restore-cache.outputs.cache-hit != 'true' && steps.ghcr-pull.outcome != 'success'
         run: pip install -r tests/haos_image_build/requirements.txt
@@ -172,6 +195,10 @@ jobs:
         # local-build path AND a successful GHCR pull. Without saving on
         # the GHCR-served branch, the actions/cache entry stays empty
         # forever and every future run pays the GHCR pull cost again.
+        # This is the sole writer for the shared cache key (#1407 +
+        # #1428) — the inaddon lane intentionally doesn't save, see the
+        # block comment at the same position in
+        # ``haos-e2e-inaddon-tests.yml``.
         if: steps.restore-cache.outputs.cache-hit != 'true'
         uses: actions/cache/save@v5
         with:
 
@@ -1397,11 +1397,11 @@ def build(work_dir: Path, output: Path) -> None:
     # HAOS is shut down — safe to open the qcow2 with libguestfs and bake
     # the testcontainer's seed state into /config/ for the e2e suite.
     bake_test_state(qcow2)
-    # Skip post-build compression for now: empirically qemu-img convert -c
-    # only shrinks ~7 GB → ~7 GB (Docker layer contents don't compress well
-    # with zlib) but adds 9 min, and xz -9 -T0 adds >25 min. Just sparse-copy
-    # the raw qcow2. Image-size optimization (zstd? strip unused docker
-    # layers? slim addons?) is tracked separately as a follow-up.
+    # Output uncompressed: nothing downstream of this script on the
+    # developer iteration path benefits from a smaller file, and the
+    # convert pass adds ~6 min. GHCR-served image is compressed at
+    # publish time by build-haos-test-image.yml's ``Compress qcow2
+    # in-format`` step (#1428, measured 12 GB → 5.1 GB / 2.3x).
     LOG.info("Copying qcow2 to %s (uncompressed)", output)
     output.parent.mkdir(parents=True, exist_ok=True)
     _run(["cp", "--reflink=auto", str(qcow2), str(output)])