Skip to content

Commit 3352b2c

Browse files
claudeclaude
committed
test(e2e): add HAOS stdio visibility coverage
1 parent 25b33d4 commit 3352b2c

11 files changed

Lines changed: 617 additions & 42 deletions

‎.github/workflows/haos-e2e-stdio-tests.yml‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
name: HAOS E2E Tests (stdio)
2+
3+
# Runs the complete E2E suite against real HAOS while the server-under-test is
4+
# the installed ha-mcp command connected through stdio JSON-RPC. This is a
5+
# separate parallel lane so transport coverage does not extend the existing
6+
# external HAOS lane's wall time.
7+
8+
on:
9+
pull_request:
10+
workflow_dispatch:
11+
inputs:
12+
pytest_args:
13+
description: 'Extra pytest args (e.g. "-k test_simple_connection") for targeted iteration'
14+
type: string
15+
required: false
16+
default: ''
17+
force_local_build:
18+
description: 'Build the qcow2 locally even on a cache hit'
19+
type: boolean
20+
required: false
21+
default: false
22+
23+
permissions:
24+
contents: read
25+
26+
env:
27+
PYTHON_VERSION: "3.13"
28+
UV_CACHE_DIR: /tmp/.uv-cache
29+
LIBGUESTFS_BACKEND: direct
30+
31+
jobs:
32+
changes:
33+
name: Detect relevant changes
34+
runs-on: ubuntu-latest
35+
timeout-minutes: 5
36+
outputs:
37+
run: ${{ steps.filter.outputs.run }}
38+
steps:
39+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
40+
with:
41+
persist-credentials: false
42+
- name: Classify changed files
43+
id: filter
44+
env:
45+
BASE_REF: ${{ github.base_ref }}
46+
run: |
47+
# Fail closed: only a successfully resolved, non-empty, docs-only PR
48+
# may skip a required heavy job.
49+
run=true
50+
if [ "${{ github.event_name }}" = "pull_request" ] \
51+
&& git fetch --depth=1 origin "$BASE_REF" \
52+
&& base_sha=$(git rev-parse "origin/$BASE_REF") \
53+
&& changed=$(git diff --name-only --diff-filter=ACMRD "$base_sha" HEAD) \
54+
&& [ -n "$changed" ]; then
55+
echo "Changed files:"; printf '%s\n' "$changed" | sed 's/^/ /'
56+
run=false
57+
while IFS= read -r f; do
58+
[ -z "$f" ] && continue
59+
case "$f" in
60+
homeassistant-addon/*.md|homeassistant-addon-dev/*.md|homeassistant-addon-webhook-proxy/*.md|custom_components/ha_mcp_tools/*.md)
61+
continue ;;
62+
homeassistant-addon/*|homeassistant-addon-dev/*|homeassistant-addon-webhook-proxy/*|custom_components/ha_mcp_tools/*|tests/haos_image_build/*|tests/initial_test_state/*)
63+
run=true; break ;;
64+
*.md|*.mdx|docs/*|site/*)
65+
continue ;;
66+
*)
67+
run=true; break ;;
68+
esac
69+
done <<< "$changed"
70+
else
71+
echo "Non-PR event or unresolved diff — running suite (fail-closed)."
72+
fi
73+
echo "run=$run"
74+
echo "run=$run" >> "$GITHUB_OUTPUT"
75+
76+
haos-e2e-stdio:
77+
name: HAOS E2E Tests (stdio)
78+
needs: changes
79+
if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.run != 'false') }}
80+
runs-on: ubuntu-22.04
81+
timeout-minutes: 45
82+
83+
steps:
84+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
85+
with:
86+
persist-credentials: false
87+
submodules: true
88+
fetch-depth: 0
89+
90+
- name: Compute image cache key
91+
id: key
92+
run: |
93+
hash=$(git ls-tree -r HEAD \
94+
tests/haos_image_build \
95+
tests/initial_test_state \
96+
custom_components/ha_mcp_tools \
97+
homeassistant-addon-webhook-proxy \
98+
| sha256sum | cut -d' ' -f1 | head -c16)
99+
echo "cache-key=haos-image-$hash" >> "$GITHUB_OUTPUT"
100+
101+
- name: Restore image from cache
102+
id: restore-cache
103+
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
104+
with:
105+
path: /tmp/haos-test-image.qcow2
106+
key: ${{ steps.key.outputs.cache-key }}
107+
108+
- name: Report image-acquisition path
109+
run: |
110+
echo "::notice title=HAOS image cache::cache-hit=${{ steps.restore-cache.outputs.cache-hit }} key=${{ steps.key.outputs.cache-key }}"
111+
112+
- name: Install QEMU + OVMF + libguestfs
113+
run: |
114+
sudo apt-get update
115+
sudo apt-get install -y --no-install-recommends \
116+
qemu-system-x86 qemu-utils ovmf xz-utils curl libguestfs-tools sshpass
117+
sudo chmod +r /boot/vmlinuz-*
118+
119+
- name: Enable KVM group perms
120+
run: |
121+
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
122+
| sudo tee /etc/udev/rules.d/99-kvm4all.rules
123+
sudo udevadm control --reload-rules
124+
sudo udevadm trigger --name-match=kvm
125+
126+
- name: Free disk space for the local build
127+
if: steps.restore-cache.outputs.cache-hit != 'true' || github.event.inputs.force_local_build == 'true'
128+
run: |
129+
df -h /
130+
sudo rm -rf /usr/share/dotnet /usr/share/swift /opt/ghc \
131+
/usr/local/lib/android /usr/local/.ghcup
132+
sudo apt-get clean
133+
docker system prune -af --volumes || true
134+
df -h /
135+
136+
- name: Install build-script Python deps
137+
if: steps.restore-cache.outputs.cache-hit != 'true' || github.event.inputs.force_local_build == 'true'
138+
run: pip install -r tests/haos_image_build/requirements.txt
139+
140+
- name: Build image locally
141+
if: steps.restore-cache.outputs.cache-hit != 'true' || github.event.inputs.force_local_build == 'true'
142+
run: |
143+
python3 tests/haos_image_build/build_image.py --verbose \
144+
--output /tmp/haos-test-image.qcow2
145+
146+
# The external HAOS lane is the sole writer for this shared key. A
147+
# simultaneous cache miss can build here, but this parallel lane must not
148+
# race it with a second cache save.
149+
150+
- name: Install uv
151+
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
152+
with:
153+
version: "latest"
154+
155+
- name: Set up Python
156+
run: uv python install ${{ env.PYTHON_VERSION }}
157+
158+
- name: Install test dependencies
159+
run: uv sync --all-extras --dev
160+
161+
- name: Confirm image present
162+
run: ls -lh /tmp/haos-test-image.qcow2
163+
164+
- name: Run full E2E suite through stdio against HAOS
165+
run: |
166+
cd tests
167+
uv run pytest src/e2e/ -n2 --dist loadscope -v --tb=short --maxfail=0 ${{ github.event.inputs.pytest_args }}
168+
env:
169+
HAMCP_ENV_FILE: "tests/.env.test"
170+
HAOS_TEST_IMAGE_PATH: /tmp/haos-test-image.qcow2
171+
HAOS_TEST_MODE: stdio
172+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
173+
174+
- name: Extract HA diagnostics from booted qcow2
175+
if: always()
176+
run: |
177+
sudo chmod +r /boot/vmlinuz-* || true
178+
mkdir -p /tmp/haos-diagnostics
179+
cp /tmp/haos-build/haos-serial.log /tmp/haos-diagnostics/bake-serial.log 2>/dev/null || true
180+
QCOW2=/tmp/haos-test-image.qcow2
181+
[ -f "$QCOW2" ] || QCOW2=/tmp/haos-build/haos-test-image.qcow2
182+
guestfish --ro -a "$QCOW2" run \
183+
: mount /dev/sda8 / \
184+
: ll /supervisor/homeassistant \
185+
> /tmp/haos-diagnostics/config-dir-listing.txt 2>&1 \
186+
|| echo "directory listing failed"
187+
guestfish --ro -a "$QCOW2" run \
188+
: mount /dev/sda8 / \
189+
: tar-out /supervisor/homeassistant/.storage /tmp/haos-diagnostics/storage.tar \
190+
|| echo ".storage tar-out failed"
191+
guestfish --ro -a "$QCOW2" run \
192+
: mount /dev/sda8 / \
193+
: glob copy-out '/supervisor/homeassistant/*.log*' /tmp/haos-diagnostics/ \
194+
|| echo "log glob copy failed"
195+
ls -la /tmp/haos-diagnostics/ || true
196+
cat /tmp/haos-diagnostics/config-dir-listing.txt || true
197+
python3 scripts/redact_diagnostics_secrets.py /tmp/haos-diagnostics
198+
199+
- name: Upload HAOS diagnostics
200+
if: always()
201+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
202+
with:
203+
name: haos-stdio-diagnostics
204+
path: |
205+
/tmp/haos-diagnostics/
206+
/tmp/haos-e2e-serial.log
207+
if-no-files-found: warn
208+
retention-days: 7

‎tests/AGENTS.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@ before adding a gate:
2020
|---|---|
2121
| `haos_only` | HAOS backends only (`HAOS_TEST_IMAGE_PATH` set). **Auto-applied** to everything under `src/e2e/haos_only/` — no marker needed there |
2222
| `container_only` | the testcontainer backend only (includes the container-embedded lane) |
23-
| `external_only` | anywhere the server-under-test runs IN the pytest process: plain testcontainer and HAOS external. Skips inaddon, container-embedded and HAOS-embedded, which cannot be reconfigured via test-process env / monkeypatch or reach an in-process mock. The name is historical — it does NOT mean "HAOS external only" |
23+
| `external_only` | anywhere the server-under-test runs IN the pytest process: plain testcontainer and HAOS external. Skips stdio, inaddon, container-embedded and HAOS-embedded, which cannot be reconfigured via test-process env / monkeypatch or reach an in-process mock. The name is historical — it does NOT mean "HAOS external only" |
2424
| `inaddon_only` | HAOS inaddon mode only (`HAOS_TEST_MODE=inaddon`), where `is_running_in_addon()` paths are live |
25+
| `haos_stdio_only` | HAOS stdio mode only (`HAOS_TEST_MODE=stdio`), where the installed `ha-mcp` command is exercised through a real subprocess transport |
2526
| `not_on_embedded` / `not_on_haos_embedded` | everywhere except that lane, for tests the lane's own session backend already covers |
2627

2728
Pick the marker by what the test *needs*, not by where it happens to pass:

‎tests/pytest.ini‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ markers =
6363
container_only: skip when HAOS backend is active (testcontainer-specific behavior; #1281)
6464
external_only: in-process-server tiers only — skip where the server runs in another process (inaddon HAOS addon + embedded backend); #1349, #1527
6565
inaddon_only: HAOS inaddon mode only — exercises is_running_in_addon()=True paths; #1349
66+
haos_stdio_only: HAOS stdio mode only — exercises the installed ha-mcp command over a real stdio subprocess transport
6667
not_on_embedded: skip on the embedded backend (E2E_BACKEND=embedded) — provably redundant with the lane's own in-process ha_mcp_server session backend; #1527
6768
not_on_haos_embedded: skip on the haos_embedded backend (HAOS_TEST_MODE=embedded) — provably redundant with the lane's own session backend, which enables the entry once and drives the in-process server for the whole suite; #1527
6869

‎tests/src/e2e/basic/test_backend_dispatch_smoke.py‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
"""Multi-layer smoke tests for backend dispatch correctness.
22
3-
The three e2e CI lanes set env vars that ``conftest.ha_container_with_fresh_config``
3+
The e2e CI lanes set env vars that ``conftest.ha_container_with_fresh_config``
44
reads to choose a backend:
55
66
| Lane | HAOS_TEST_IMAGE_PATH | HAOS_TEST_MODE | expected backend |
77
| ----------------------------- | -------------------- | -------------- | ---------------- |
88
| e2e-tests.yml (testcontainer) | unset | unset | ``container`` |
99
| haos-e2e-tests.yml (external) | set | unset | ``haos`` |
10+
| haos-e2e-stdio-tests.yml | set | ``stdio`` | ``haos_stdio`` |
1011
| haos-e2e-inaddon-tests.yml | set | ``inaddon`` | ``haos_inaddon`` |
1112
| haos-e2e-embedded-tests.yml | set | ``embedded`` | ``haos_embedded``|
1213
@@ -84,9 +85,14 @@
8485
# gain 10; container is unchanged because the tests run there.
8586
# Entries below are CI-observed item counts, bumped only for intentional
8687
# marker-gated additions rather than runtime skips.
87-
"container": 73, # was 72; +1 inaddon startup-nudge e2e (haos_only + inaddon_only)
88-
"haos": 48, # was 46; +1 embedded HACS-nudge skip e2e (container_only), +1 inaddon startup-nudge e2e (inaddon_only)
89-
"haos_inaddon": 76, # was 75; +1 embedded HACS-nudge skip e2e (container_only)
88+
"container": 74, # +1 HAOS stdio visibility e2e (haos_only)
89+
"haos": 49, # +1 HAOS stdio visibility e2e (haos_stdio_only)
90+
# HAOS stdio is the external HAOS set plus ``external_only`` tests, whose
91+
# test-process monkeypatches cannot reach the subprocess server. Start with
92+
# a conservative static ceiling; replace it with the CI-observed count once
93+
# the first lane run reports its collection summary.
94+
"haos_stdio": 112,
95+
"haos_inaddon": 77, # +1 HAOS stdio visibility e2e (haos_stdio_only)
9096
# Embedded backend (#1527, E2E_BACKEND=embedded). Skips exactly the container
9197
# lane's marker-skips PLUS two embedded-specific additions:
9298
# - haos_only + inaddon_only tests skip on embedded just like on container
@@ -100,7 +106,7 @@
100106
# 1) + not_on_embedded 2 = 101. Parametrize inflates that to the CI-observed
101107
# count the entry below is pinned to — 133 on this PR's run, 132 before the
102108
# self-restart e2e. Read the count off a run rather than deriving it.
103-
"embedded": 133, # was 132; +1 embedded self-restart e2e (haos_only)
109+
"embedded": 134, # +1 HAOS stdio visibility e2e (haos_only)
104110
# HAOS embedded backend (#1527, HAOS_TEST_MODE=embedded). A HAOS lane, so it
105111
# skips the SAME set as the external HAOS lane (container_only + inaddon_only)
106112
# PLUS two haos_embedded-specific additions:
@@ -117,7 +123,7 @@
117123
# container_only). Parametrize inflates that to the CI-observed count the
118124
# entry below is pinned to — 107 on this PR's run, 106 before the
119125
# self-restart e2e. Read the count off a run rather than deriving it.
120-
"haos_embedded": 107, # was 106; +1 embedded self-restart e2e (not_on_haos_embedded)
126+
"haos_embedded": 108, # +1 HAOS stdio visibility e2e (haos_stdio_only)
121127
}
122128

123129

@@ -154,6 +160,17 @@ def test_backend_dispatch_matches_workflow_env(
154160
assert ha_container_with_fresh_config["container"] is None
155161
assert ha_container_with_fresh_config["port"] is None
156162
assert ha_container_with_fresh_config["config_path"] is None
163+
elif image_path and mode == "stdio":
164+
assert backend == "haos_stdio", (
165+
"Workflow set HAOS_TEST_IMAGE_PATH + HAOS_TEST_MODE=stdio "
166+
f"but dispatch picked backend={backend!r}. The installed stdio "
167+
"server is NOT being exercised by this run."
168+
)
169+
assert ha_container_with_fresh_config["container"] is None
170+
assert ha_container_with_fresh_config["port"] is None
171+
assert ha_container_with_fresh_config["config_path"] is None
172+
assert ha_container_with_fresh_config.get("addon_mcp_url") is None
173+
assert ha_container_with_fresh_config.get("embedded_webhook_url") is None
157174
elif image_path and mode == "embedded":
158175
# haos_embedded (#1527): a HAOS backend whose server-under-test is the
159176
# baked in-process MCP server, driven over its ingress webhook on the
@@ -250,7 +267,7 @@ async def test_supervisor_addon_tool_behavior_matches_backend(
250267
# HA Core's supervisor/api WS proxy (it runs standalone with HA_MCP_EMBEDDED,
251268
# so it does not use SUPERVISOR_TOKEN directly, but the proxy path still works
252269
# because HA Core itself is supervised).
253-
if backend in ("haos", "haos_inaddon", "haos_embedded"):
270+
if backend in ("haos", "haos_stdio", "haos_inaddon", "haos_embedded"):
254271
assert result.get("success") is True, (
255272
f"ha_get_app failed on {backend} backend; Supervisor must "
256273
f"be running. Result: {result!r}"

0 commit comments

Comments
 (0)