docs(overview): enumerate dismissed_repair_count in fields= description + static drift test #297
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: HAOS E2E Tests | |
| # Runs the HAOS-tier E2E suite for #1281 in parallel with the existing | |
| # testcontainer e2e-tests.yml. | |
| # | |
| # Image resolution strategy (per run): | |
| # 1. Try GitHub Actions cache (keyed on HAOS version + build inputs) | |
| # 2. Fall back to GHCR (published by build-haos-test-image.yml on master) | |
| # 3. Fall back to local build (build_image.py) | |
| # | |
| # Single-job design: the qcow2 stays on the runner's local disk through | |
| # build → test, so there's no cross-job upload+download of the ~7 GB image | |
| # (previously cost ~5 min per PR). | |
| on: | |
| pull_request: | |
| paths: | |
| - 'src/**' | |
| - 'tests/**' | |
| - 'pyproject.toml' | |
| - 'homeassistant-addon/**' | |
| - '.github/workflows/haos-e2e-tests.yml' | |
| # Trigger on inaddon workflow changes too — both lanes share the | |
| # qcow2 cache key, so a change to either workflow's cache logic | |
| # needs the other lane to re-run for parity verification. | |
| - '.github/workflows/haos-e2e-inaddon-tests.yml' | |
| workflow_dispatch: | |
| inputs: | |
| pytest_args: | |
| description: 'Extra pytest args (e.g. "-k test_simple_connection") for targeted iteration' | |
| type: string | |
| required: false | |
| default: '' | |
| permissions: | |
| contents: read | |
| packages: read | |
| env: | |
| PYTHON_VERSION: "3.13" | |
| UV_CACHE_DIR: /tmp/.uv-cache | |
| IMAGE_REPO: ghcr.io/${{ github.repository_owner }}/haos-test-image | |
| # libguestfs defaults to libvirt which isn't running on GitHub-hosted | |
| # runners (per-user libvirtd socket missing for the unprivileged | |
| # runner UID). Direct backend bypasses libvirt and uses QEMU directly. | |
| LIBGUESTFS_BACKEND: direct | |
| jobs: | |
| haos-e2e: | |
| name: HAOS E2E Tests | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| # Required for tests/src/e2e/tools/test_skills_resources.py — the | |
| # skills bundle lives at src/ha_mcp/resources/skills-vendor. | |
| submodules: true | |
| # Full history needed for the bake-changes detection step below | |
| # (``git diff origin/<base>...HEAD``). Default depth=1 doesn't | |
| # have the base ref locally. | |
| fetch-depth: 0 | |
| - name: Compute image cache key | |
| id: key | |
| # Cache invalidates when ANY input that gets baked into the qcow2 | |
| # changes — build script (addon list / HAOS pin), initial_test_state | |
| # (seed config + recorder DB + .storage), and the custom components | |
| # the bake stages into /config. git ls-tree uses object IDs so the | |
| # hash is reproducible across runs (no mtime noise like `find` | |
| # would have). | |
| run: | | |
| hash=$(git ls-tree -r HEAD \ | |
| tests/haos_image_build \ | |
| tests/initial_test_state \ | |
| custom_components/ha_mcp_tools \ | |
| homeassistant-addon-webhook-proxy/mcp_proxy \ | |
| | sha256sum | cut -d' ' -f1 | head -c16) | |
| echo "cache-key=haos-image-$hash" >> "$GITHUB_OUTPUT" | |
| - name: Restore image from cache | |
| id: restore-cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: /tmp/haos-test-image.qcow2 | |
| key: ${{ steps.key.outputs.cache-key }} | |
| - name: Install QEMU + OVMF + libguestfs | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| qemu-system-x86 qemu-utils ovmf xz-utils curl libguestfs-tools sshpass | |
| # Ubuntu installs the kernel image as 0600 root:root, but | |
| # libguestfs's supermin appliance builder needs to read it as | |
| # the invoking (unprivileged) user. Documented at | |
| # https://libguestfs.org/guestfs-faq.1.html#kernel-permissions | |
| # (search "kernel image is not readable") — workaround is to | |
| # relax perms on /boot/vmlinuz-*. | |
| sudo chmod +r /boot/vmlinuz-* | |
| - name: Enable KVM group perms | |
| run: | | |
| echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ | |
| | sudo tee /etc/udev/rules.d/99-kvm4all.rules | |
| sudo udevadm control --reload-rules | |
| sudo udevadm trigger --name-match=kvm | |
| - name: Detect PR-modified bake inputs | |
| # GHCR's ``:HAOS_VERSION-latest`` tag is published from master, so | |
| # any PR that modifies the qcow2's bake inputs (custom_components, | |
| # initial_test_state, the build script itself, or the webhook-proxy | |
| # source the bake stages) would otherwise see a stale image with | |
| # master's content. The cache key already invalidates on these | |
| # paths (see ``Compute image cache key`` above), but the GHCR | |
| # fallback below would still hand back master's image. Detect the | |
| # condition here and skip GHCR — fall straight through to the | |
| # local-build step instead. Keep this list in sync with the cache | |
| # key's ``git ls-tree`` paths. | |
| id: bake-changes | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| if git diff --name-only \ | |
| "origin/${{ github.base_ref }}...HEAD" \ | |
| | grep -qE '^(tests/haos_image_build/|tests/initial_test_state/|custom_components/ha_mcp_tools/|homeassistant-addon-webhook-proxy/mcp_proxy/)'; then | |
| echo "bake_inputs_changed=true" >> "$GITHUB_OUTPUT" | |
| echo "PR modifies bake inputs; skipping GHCR fallback (would return stale master image)." | |
| fi | |
| - name: Try pulling from GHCR | |
| if: steps.restore-cache.outputs.cache-hit != 'true' && steps.bake-changes.outputs.bake_inputs_changed != 'true' | |
| id: ghcr-pull | |
| continue-on-error: true | |
| run: | | |
| version=$(python3 -c "from tests.haos_image_build.build_image import HAOS_VERSION; print(HAOS_VERSION)") | |
| tag="${IMAGE_REPO}:${version}-latest" | |
| echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io \ | |
| -u ${{ github.actor }} --password-stdin | |
| curl -fsSL https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz \ | |
| | tar -xz -C /tmp oras | |
| mkdir -p /tmp/haos-pull | |
| (cd /tmp/haos-pull && /tmp/oras pull "$tag") | |
| # Explicit source path so a future oras layout change that drops | |
| # files into a subdirectory of pwd doesn't silently no-op the mv. | |
| mv /tmp/haos-pull/haos-test-image.qcow2 /tmp/haos-test-image.qcow2 | |
| - name: Install build-script Python deps (cache miss + GHCR miss) | |
| if: steps.restore-cache.outputs.cache-hit != 'true' && steps.ghcr-pull.outcome != 'success' | |
| run: pip install -r tests/haos_image_build/requirements.txt | |
| - name: Build image locally (cache miss + GHCR miss) | |
| if: steps.restore-cache.outputs.cache-hit != 'true' && steps.ghcr-pull.outcome != 'success' | |
| run: | | |
| python3 tests/haos_image_build/build_image.py --verbose \ | |
| --output /tmp/haos-test-image.qcow2 | |
| - name: Save image to cache (cache miss only) | |
| # Save whenever the runtime cache missed — covers both the | |
| # local-build path AND a successful GHCR pull. Without saving on | |
| # the GHCR-served branch, the actions/cache entry stays empty | |
| # forever and every future run pays the GHCR pull cost again. | |
| if: steps.restore-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: /tmp/haos-test-image.qcow2 | |
| key: ${{ steps.key.outputs.cache-key }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| version: "latest" | |
| - name: Set up Python | |
| run: uv python install ${{ env.PYTHON_VERSION }} | |
| - name: Install test dependencies | |
| run: uv sync --all-extras --dev | |
| - name: Confirm image present | |
| run: ls -lh /tmp/haos-test-image.qcow2 | |
| - name: Run full E2E suite against HAOS backend | |
| # HAOS_TEST_IMAGE_PATH switches the e2e/conftest.py dispatch from | |
| # testcontainer → HAOS-QEMU. Same fixture chain, same test code. | |
| # haos_only tests under tests/src/e2e/haos_only/ run here; tests | |
| # that need testcontainer-specific behavior (file mounts, etc.) | |
| # are marked container_only and skip automatically. | |
| # | |
| # -n2 --dist loadscope: pytest-xdist parallelizes across 2 workers, | |
| # each booting its own QEMU (per-worker port offset + qcow2 overlay | |
| # — see _haos_worker_setup in tests/src/e2e/conftest.py). loadscope | |
| # keeps all tests in the same module on the same worker so each | |
| # worker amortizes its boot cost. 4-vCPU/16-GB runner fits 2x | |
| # QEMU at -smp 2 -m 4096 each (#1350). | |
| # | |
| # --maxfail=0 overrides pytest.ini's --maxfail=3 so we see the full | |
| # failure surface for triage — early-terminating loses signal. | |
| run: | | |
| cd tests | |
| uv run pytest src/e2e/ -n2 --dist loadscope -v --tb=short --maxfail=0 ${{ github.event.inputs.pytest_args }} | |
| env: | |
| HAMCP_ENV_FILE: "tests/.env.test" | |
| HAOS_TEST_IMAGE_PATH: /tmp/haos-test-image.qcow2 | |
| - name: Extract HA diagnostics from booted qcow2 (always) | |
| # First lists what's actually present in /supervisor/homeassistant | |
| # (so missing files in artifact uploads are diagnosable), then | |
| # tar-out the whole .storage + log files to a local tarball. | |
| # Reads-only mount so the qcow2 isn't mutated. | |
| if: always() | |
| run: | | |
| sudo chmod +r /boot/vmlinuz-* || true | |
| mkdir -p /tmp/haos-diagnostics | |
| # Probe: full directory listing of HA's config dir. Captures | |
| # whatever HA actually writes (log filenames vary by config). | |
| guestfish --ro -a /tmp/haos-test-image.qcow2 run \ | |
| : mount /dev/sda8 / \ | |
| : ll /supervisor/homeassistant \ | |
| > /tmp/haos-diagnostics/config-dir-listing.txt 2>&1 \ | |
| || echo "directory listing failed" | |
| # Pull everything log-shaped + the .storage tree out as a tar. | |
| # tar-out includes the full subtree so anything-named-log gets | |
| # captured; .storage gives us config_entries + other state. | |
| guestfish --ro -a /tmp/haos-test-image.qcow2 run \ | |
| : mount /dev/sda8 / \ | |
| : tar-out /supervisor/homeassistant/.storage /tmp/haos-diagnostics/storage.tar \ | |
| || echo ".storage tar-out failed" | |
| # Same for any log files in the config root. | |
| guestfish --ro -a /tmp/haos-test-image.qcow2 run \ | |
| : mount /dev/sda8 / \ | |
| : glob copy-out '/supervisor/homeassistant/*.log*' /tmp/haos-diagnostics/ \ | |
| || echo "log glob copy failed" | |
| ls -la /tmp/haos-diagnostics/ || true | |
| echo '--- config dir listing ---' | |
| cat /tmp/haos-diagnostics/config-dir-listing.txt || true | |
| - name: Upload HAOS diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: haos-diagnostics | |
| path: | | |
| /tmp/haos-diagnostics/ | |
| /tmp/haos-e2e-serial.log | |
| if-no-files-found: warn | |
| retention-days: 7 |